diff --git a/mercurial/wireprotoframing.py b/mercurial/wireprotoframing.py --- a/mercurial/wireprotoframing.py +++ b/mercurial/wireprotoframing.py @@ -21,6 +21,7 @@ from . import ( encoding, error, + pycompat, util, wireprototypes, ) @@ -429,6 +430,26 @@ flags=FLAG_COMMAND_RESPONSE_EOS, payload=b'') +def createalternatelocationresponseframe(stream, requestid, location): + data = { + b'status': b'redirect', + b'location': { + b'url': location.url, + b'mediatype': location.mediatype, + } + } + + for a in (r'size', r'fullhashes', r'fullhashseed', r'serverdercerts', + r'servercadercerts'): + value = getattr(location, a) + if value is not None: + data[b'location'][pycompat.bytestr(a)] = value + + return stream.makeframe(requestid=requestid, + typeid=FRAME_TYPE_COMMAND_RESPONSE, + flags=FLAG_COMMAND_RESPONSE_CONTINUATION, + payload=b''.join(cborutil.streamencode(data))) + def createcommanderrorresponse(stream, requestid, message, args=None): # TODO should this be using a list of {'msg': ..., 'args': {}} so atom # formatting works consistently? @@ -813,6 +834,7 @@ def sendframes(): emitted = False + alternatelocationsent = False emitter = bufferingcommandresponseemitter(stream, requestid) while True: try: @@ -841,6 +863,25 @@ break try: + # Alternate location responses can only be the first and + # only object in the output stream. + if isinstance(o, wireprototypes.alternatelocationresponse): + if emitted: + raise error.ProgrammingError( + 'alternatelocationresponse seen after initial ' + 'output object') + + yield createalternatelocationresponseframe( + stream, requestid, o) + + alternatelocationsent = True + emitted = True + continue + + if alternatelocationsent: + raise error.ProgrammingError( + 'object follows alternatelocationresponse') + if not emitted: yield createcommandresponseokframe(stream, requestid) emitted = True @@ -977,6 +1018,7 @@ 'requestid': requestid, 'command': request[b'name'], 'args': request[b'args'], + 'redirect': request.get(b'redirect'), 'data': entry['data'].getvalue() if entry['data'] else None, } diff --git a/mercurial/wireprototypes.py b/mercurial/wireprototypes.py --- a/mercurial/wireprototypes.py +++ b/mercurial/wireprototypes.py @@ -368,3 +368,20 @@ and the content from this object is used instead. """ data = attr.ib() + +@attr.s +class alternatelocationresponse(object): + """Represents a response available at an alternate location. + + Instances are sent in place of actual response objects when the server + is sending a "content redirect" response. + + Only compatible with wire protocol version 2. + """ + url = attr.ib() + mediatype = attr.ib() + size = attr.ib(default=None) + fullhashes = attr.ib(default=None) + fullhashseed = attr.ib(default=None) + serverdercerts = attr.ib(default=None) + servercadercerts = attr.ib(default=None) diff --git a/mercurial/wireprotov2server.py b/mercurial/wireprotov2server.py --- a/mercurial/wireprotov2server.py +++ b/mercurial/wireprotov2server.py @@ -312,7 +312,7 @@ res.headers[b'Content-Type'] = FRAMINGTYPE try: - objs = dispatch(repo, proto, command['command']) + objs = dispatch(repo, proto, command['command'], command['redirect']) action, meta = reactor.oncommandresponsereadyobjects( outstream, command['requestid'], objs) @@ -339,7 +339,7 @@ def getdispatchrepo(repo, proto, command): return repo.filtered('served') -def dispatch(repo, proto, command): +def dispatch(repo, proto, command, redirect): """Run a wire protocol command. Returns an iterable of objects that will be sent to the client. @@ -364,8 +364,17 @@ yield o return + if redirect: + redirecttargets = redirect[b'targets'] + redirecthashes = redirect[b'hashes'] + else: + redirecttargets = [] + redirecthashes = [] + cacher = makeresponsecacher(repo, proto, command, args, - cborutil.streamencode) + cborutil.streamencode, + redirecttargets=redirecttargets, + redirecthashes=redirecthashes) # But we have no cacher. Do default handling. if not cacher: @@ -751,7 +760,8 @@ return cachekeyfn -def makeresponsecacher(repo, proto, command, args, objencoderfn): +def makeresponsecacher(repo, proto, command, args, objencoderfn, + redirecttargets, redirecthashes): """Construct a cacher for a cacheable command. Returns an ``iwireprotocolcommandcacher`` instance. diff --git a/tests/test-http-api-httpv2.t b/tests/test-http-api-httpv2.t --- a/tests/test-http-api-httpv2.t +++ b/tests/test-http-api-httpv2.t @@ -430,10 +430,10 @@ s> Server: testing stub value\r\n s> Date: $HTTP_DATE$\r\n s> Content-Type: text/plain\r\n - s> Content-Length: 205\r\n + s> Content-Length: 223\r\n s> \r\n s> received: 1 1 1 \xa2Dargs\xa2Dbar1CvalCfooDval1DnameHcommand1\n - s> ["runcommand", {"args": {"bar1": "val", "foo": "val1"}, "command": "command1", "data": null, "requestid": 1}]\n + s> ["runcommand", {"args": {"bar1": "val", "foo": "val1"}, "command": "command1", "data": null, "redirect": null, "requestid": 1}]\n s> received: \n s> {"action": "noop"} diff --git a/tests/test-wireproto-content-redirects.t b/tests/test-wireproto-content-redirects.t --- a/tests/test-wireproto-content-redirects.t +++ b/tests/test-wireproto-content-redirects.t @@ -1,11 +1,20 @@ $ . $TESTDIR/wireprotohelpers.sh + $ cat >> $HGRCPATH << EOF + > [extensions] + > blackbox = + > [blackbox] + > track = simplecache + > EOF + $ hg init server $ enablehttpv2 server $ cd server $ cat >> .hg/hgrc << EOF > [extensions] > simplecache = $TESTDIR/wireprotosimplecache.py + > [simplecache] + > cacheapi = true > EOF $ echo a0 > a @@ -1183,5 +1192,178 @@ } ] +Set up the server to issue content redirects to its built-in API server. + + $ cat > redirects.py << EOF + > [ + > { + > b'name': b'local', + > b'protocol': b'http', + > b'uris': [b'http://example.com/'], + > }, + > ] + > EOF + +Request to eventual cache URL should return 404 (validating the cache server works) + + $ sendhttpraw << EOF + > httprequest GET api/simplecache/missingkey + > user-agent: test + > EOF + using raw connection to peer + s> GET /api/simplecache/missingkey HTTP/1.1\r\n + s> Accept-Encoding: identity\r\n + s> user-agent: test\r\n + s> host: $LOCALIP:$HGPORT\r\n (glob) + s> \r\n + s> makefile('rb', None) + s> HTTP/1.1 404 Not Found\r\n + s> Server: testing stub value\r\n + s> Date: $HTTP_DATE$\r\n + s> Content-Type: text/plain\r\n + s> Content-Length: 22\r\n + s> \r\n + s> key not found in cache + +Send a cacheable request + + $ sendhttpv2peer << EOF + > command manifestdata + > nodes eval:[b'\x99\x2f\x47\x79\x02\x9a\x3d\xf8\xd0\x66\x6d\x00\xbb\x92\x4f\x69\x63\x4e\x26\x41'] + > tree eval:b'' + > fields eval:[b'parents'] + > EOF + creating http peer for wire protocol version 2 + sending manifestdata command + s> POST /api/exp-http-v2-0002/ro/manifestdata HTTP/1.1\r\n + s> Accept-Encoding: identity\r\n + s> accept: application/mercurial-exp-framing-0005\r\n + s> content-type: application/mercurial-exp-framing-0005\r\n + s> content-length: 128\r\n + s> host: $LOCALIP:$HGPORT\r\n (glob) + s> user-agent: Mercurial debugwireproto\r\n + s> \r\n + s> x\x00\x00\x01\x00\x01\x01\x11\xa3Dargs\xa3Ffields\x81GparentsEnodes\x81T\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&ADtree@DnameLmanifestdataHredirect\xa2Fhashes\x82Fsha256Dsha1Gtargets\x81Elocal + s> makefile('rb', None) + s> HTTP/1.1 200 OK\r\n + s> Server: testing stub value\r\n + s> Date: $HTTP_DATE$\r\n + s> Content-Type: application/mercurial-exp-framing-0005\r\n + s> Transfer-Encoding: chunked\r\n + s> \r\n + s> 13\r\n + s> \x0b\x00\x00\x01\x00\x02\x011 + s> \xa1FstatusBok + s> \r\n + received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) + s> 63\r\n + s> [\x00\x00\x01\x00\x02\x001 + s> \xa1Jtotalitems\x01\xa2DnodeT\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&AGparents\x82T\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 + s> \r\n + received frame(size=91; request=1; stream=2; streamflags=; type=command-response; flags=continuation) + s> 8\r\n + s> \x00\x00\x00\x01\x00\x02\x002 + s> \r\n + s> 0\r\n + s> \r\n + received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) + response: gen[ + { + b'totalitems': 1 + }, + { + b'node': b'\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&A', + b'parents': [ + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + ] + } + ] + +Cached entry should be available on server + + $ sendhttpraw << EOF + > httprequest GET api/simplecache/c045a581599d58608efd3d93d8129841f2af04a0 + > user-agent: test + > EOF + using raw connection to peer + s> GET /api/simplecache/c045a581599d58608efd3d93d8129841f2af04a0 HTTP/1.1\r\n + s> Accept-Encoding: identity\r\n + s> user-agent: test\r\n + s> host: $LOCALIP:$HGPORT\r\n (glob) + s> \r\n + s> makefile('rb', None) + s> HTTP/1.1 200 OK\r\n + s> Server: testing stub value\r\n + s> Date: $HTTP_DATE$\r\n + s> Content-Type: application/mercurial-cbor\r\n + s> Content-Length: 91\r\n + s> \r\n + s> \xa1Jtotalitems\x01\xa2DnodeT\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&AGparents\x82T\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 + cbor> [ + { + b'totalitems': 1 + }, + { + b'node': b'\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&A', + b'parents': [ + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + ] + } + ] + +2nd request should result in content redirect response + + $ sendhttpv2peer << EOF + > command manifestdata + > nodes eval:[b'\x99\x2f\x47\x79\x02\x9a\x3d\xf8\xd0\x66\x6d\x00\xbb\x92\x4f\x69\x63\x4e\x26\x41'] + > tree eval:b'' + > fields eval:[b'parents'] + > EOF + creating http peer for wire protocol version 2 + sending manifestdata command + s> POST /api/exp-http-v2-0002/ro/manifestdata HTTP/1.1\r\n + s> Accept-Encoding: identity\r\n + s> accept: application/mercurial-exp-framing-0005\r\n + s> content-type: application/mercurial-exp-framing-0005\r\n + s> content-length: 128\r\n + s> host: $LOCALIP:$HGPORT\r\n (glob) + s> user-agent: Mercurial debugwireproto\r\n + s> \r\n + s> x\x00\x00\x01\x00\x01\x01\x11\xa3Dargs\xa3Ffields\x81GparentsEnodes\x81T\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&ADtree@DnameLmanifestdataHredirect\xa2Fhashes\x82Fsha256Dsha1Gtargets\x81Elocal + s> makefile('rb', None) + s> HTTP/1.1 200 OK\r\n + s> Server: testing stub value\r\n + s> Date: $HTTP_DATE$\r\n + s> Content-Type: application/mercurial-exp-framing-0005\r\n + s> Transfer-Encoding: chunked\r\n + s> \r\n + s> *\r\n (glob) + s> \x*\x00\x00\x01\x00\x02\x011 (glob) + s> \xa2Hlocation\xa2ImediatypeX\x1aapplication/mercurial-cborCurl*http://*:$HGPORT/api/simplecache/c045a581599d58608efd3d93d8129841f2af04a0FstatusHredirect (glob) + s> \r\n + received frame(size=*; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) (glob) + s> 8\r\n + s> \x00\x00\x00\x01\x00\x02\x001 + s> \r\n + s> 8\r\n + received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=continuation) + s> \x00\x00\x00\x01\x00\x02\x002 + s> \r\n + s> 0\r\n + s> \r\n + received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) + abort: redirect responses not yet supported + [255] + $ cat error.log $ killdaemons.py + + $ cat .hg/blackbox.log + *> cacher constructed for manifestdata (glob) + *> cache miss for c045a581599d58608efd3d93d8129841f2af04a0 (glob) + *> storing cache entry for c045a581599d58608efd3d93d8129841f2af04a0 (glob) + *> cacher constructed for manifestdata (glob) + *> cache hit for c045a581599d58608efd3d93d8129841f2af04a0 (glob) + *> sending content redirect for c045a581599d58608efd3d93d8129841f2af04a0 to http://*:$HGPORT/api/simplecache/c045a581599d58608efd3d93d8129841f2af04a0 (glob) diff --git a/tests/test-wireproto-serverreactor.py b/tests/test-wireproto-serverreactor.py --- a/tests/test-wireproto-serverreactor.py +++ b/tests/test-wireproto-serverreactor.py @@ -69,6 +69,7 @@ b'requestid': 1, b'command': b'mycommand', b'args': {}, + b'redirect': None, b'data': None, }) @@ -86,6 +87,7 @@ b'requestid': 41, b'command': b'mycommand', b'args': {b'foo': b'bar'}, + b'redirect': None, b'data': None, }) @@ -100,6 +102,7 @@ b'requestid': 1, b'command': b'mycommand', b'args': {b'foo': b'bar', b'biz': b'baz'}, + b'redirect': None, b'data': None, }) @@ -115,6 +118,7 @@ b'requestid': 1, b'command': b'mycommand', b'args': {}, + b'redirect': None, b'data': b'data!', }) @@ -137,6 +141,7 @@ b'requestid': 1, b'command': b'mycommand', b'args': {}, + b'redirect': None, b'data': b'data1data2data3', }) @@ -160,6 +165,7 @@ b'key': b'val', b'foo': b'bar', }, + b'redirect': None, b'data': b'value1value2', }) @@ -235,6 +241,7 @@ b'requestid': 1, b'command': b'command', b'args': {}, + b'redirect': None, b'data': None, }) @@ -291,12 +298,14 @@ b'requestid': 3, b'command': b'command3', b'args': {b'biz': b'baz', b'key': b'val'}, + b'redirect': None, b'data': None, }) self.assertEqual(results[5][1], { b'requestid': 1, b'command': b'command1', b'args': {b'foo': b'bar', b'key1': b'val'}, + b'redirect': None, b'data': None, }) diff --git a/tests/wireprotosimplecache.py b/tests/wireprotosimplecache.py --- a/tests/wireprotosimplecache.py +++ b/tests/wireprotosimplecache.py @@ -12,6 +12,7 @@ registrar, repository, util, + wireprotoserver, wireprototypes, wireprotov2server, ) @@ -25,18 +26,59 @@ configtable = {} configitem = registrar.configitem(configtable) +configitem('simplecache', 'cacheapi', + default=False) configitem('simplecache', 'cacheobjects', default=False) configitem('simplecache', 'redirectsfile', default=None) +# API handler that makes cached keys available. +def handlecacherequest(rctx, req, res, checkperm, urlparts): + if rctx.repo.ui.configbool('simplecache', 'cacheobjects'): + res.status = b'500 Internal Server Error' + res.setbodybytes(b'cacheobjects not supported for api server') + return + + if not urlparts: + res.status = b'200 OK' + res.headers[b'Content-Type'] = b'text/plain' + res.setbodybytes(b'simple cache server') + return + + key = b'/'.join(urlparts) + + if key not in CACHE: + res.status = b'404 Not Found' + res.headers[b'Content-Type'] = b'text/plain' + res.setbodybytes(b'key not found in cache') + return + + res.status = b'200 OK' + res.headers[b'Content-Type'] = b'application/mercurial-cbor' + res.setbodybytes(CACHE[key]) + +def cachedescriptor(req, repo): + return {} + +wireprotoserver.API_HANDLERS[b'simplecache'] = { + 'config': (b'simplecache', b'cacheapi'), + 'handler': handlecacherequest, + 'apidescriptor': cachedescriptor, +} + @interfaceutil.implementer(repository.iwireprotocolcommandcacher) class memorycacher(object): - def __init__(self, ui, command, encodefn): + def __init__(self, ui, command, encodefn, redirecttargets, redirecthashes, + req): self.ui = ui self.encodefn = encodefn + self.redirecttargets = redirecttargets + self.redirecthashes = redirecthashes + self.req = req self.key = None self.cacheobjects = ui.configbool('simplecache', 'cacheobjects') + self.cacheapi = ui.configbool('simplecache', 'cacheapi') self.buffered = [] ui.log('simplecache', 'cacher constructed for %s\n', command) @@ -65,6 +107,37 @@ entry = CACHE[self.key] self.ui.log('simplecache', 'cache hit for %s\n', self.key) + redirectable = True + + if not self.cacheapi: + redirectable = False + elif not self.redirecttargets: + redirectable = False + else: + clienttargets = set(self.redirecttargets) + ourtargets = set(t[b'name'] for t in loadredirecttargets(self.ui)) + + # We only ever redirect to a single target (for now). So we don't + # need to store which target matched. + if not clienttargets & ourtargets: + redirectable = False + + if redirectable: + paths = self.req.dispatchparts[:-3] + paths.append(b'simplecache') + paths.append(self.key) + + url = b'%s/%s' % (self.req.advertisedbaseurl, b'/'.join(paths)) + + #url = b'http://example.com/%s' % self.key + self.ui.log('simplecache', 'sending content redirect for %s to ' + '%s\n', self.key, url) + response = wireprototypes.alternatelocationresponse( + url=url, + mediatype=b'application/mercurial-cbor') + + return {'objs': [response]} + if self.cacheobjects: return { 'objs': entry, @@ -91,8 +164,10 @@ return [] -def makeresponsecacher(orig, repo, proto, command, args, objencoderfn): - return memorycacher(repo.ui, command, objencoderfn) +def makeresponsecacher(orig, repo, proto, command, args, objencoderfn, + redirecttargets, redirecthashes): + return memorycacher(repo.ui, command, objencoderfn, redirecttargets, + redirecthashes, proto._req) def loadredirecttargets(ui): path = ui.config('simplecache', 'redirectsfile')