diff --git a/hgext3rd/fastannotate/protocol.py b/hgext3rd/fastannotate/protocol.py --- a/hgext3rd/fastannotate/protocol.py +++ b/hgext3rd/fastannotate/protocol.py @@ -10,6 +10,7 @@ extensions, hg, localrepo, + peer as peermod, scmutil, wireproto, ) @@ -160,6 +161,16 @@ else: conn.__exit__(None, None, None) +class localbatch(peermod.batcher): + """performs the queued calls directly (backported from D320)""" + def __init__(self, local): + super(localbatch, self).__init__(self) + self.local = local + + def submit(self): + for name, args, opts, resref in self.calls: + resref.set(getattr(self.local, name)(*args, **opts)) + def clientfetch(repo, paths, lastnodemap=None, peer=None): """download annotate cache from the server for paths""" if not paths: @@ -173,7 +184,7 @@ lastnodemap = {} ui = repo.ui - batcher = peer.batch() + batcher = localbatch(peer) ui.debug('fastannotate: requesting %d files\n' % len(paths)) results = [batcher.getannotate(p, lastnodemap.get(p)) for p in paths] # Note: This is the only place that fastannotate sends a request via SSH. diff --git a/remotefilelog/__init__.py b/remotefilelog/__init__.py --- a/remotefilelog/__init__.py +++ b/remotefilelog/__init__.py @@ -152,7 +152,8 @@ # Replace remote.stream_out with a version that sends file # patterns. def stream_out_shallow(orig): - if shallowrepo.requirement in remote._capabilities(): + caps = shallowutil.peercapabilities(remote) + if shallowrepo.requirement in caps: opts = {} if repo.includepattern: opts['includepattern'] = '\0'.join(repo.includepattern) diff --git a/remotefilelog/connectionpool.py b/remotefilelog/connectionpool.py --- a/remotefilelog/connectionpool.py +++ b/remotefilelog/connectionpool.py @@ -5,6 +5,8 @@ # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. +from __future__ import absolute_import + from mercurial import ( extensions, hg, @@ -12,6 +14,10 @@ util, ) +from . import ( + shallowutil, +) + class connectionpool(object): def __init__(self, repo): self._repo = repo @@ -29,8 +35,10 @@ conn = pathpool.pop() peer = conn.peer # If the connection has died, drop it + proc = shallowutil.trygetattr(peer, + ('_subprocess', 'subprocess')) if (isinstance(peer, sshpeer.sshpeer) and - peer.subprocess.poll() is not None): + proc.poll() is not None): conn = None except IndexError: pass diff --git a/remotefilelog/fileserverclient.py b/remotefilelog/fileserverclient.py --- a/remotefilelog/fileserverclient.py +++ b/remotefilelog/fileserverclient.py @@ -5,11 +5,22 @@ # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. +from __future__ import absolute_import + +import hashlib, os, time, io, struct +import itertools + from mercurial.i18n import _ from mercurial.node import hex, bin, nullid -from mercurial import util, sshpeer, error, util, wireproto, httppeer -import hashlib, os, time, io, struct -import itertools +from mercurial import ( + error, + httppeer, + peer as peermod, + sshpeer, + util, + util, + wireproto, +) from . import ( connectionpool, @@ -19,7 +30,7 @@ ) from .contentstore import unioncontentstore from .metadatastore import unionmetadatastore -from lz4wrapper import lz4decompress +from .lz4wrapper import lz4decompress # Statistics for debugging fetchcost = 0 @@ -64,7 +75,7 @@ def _updatecallstreamopts(self, command, opts): if command != 'getbundle': return - if 'remotefilelog' not in self._capabilities(): + if 'remotefilelog' not in shallowutil.peercapabilities(self): return if not util.safehasattr(self, '_localrepo'): return @@ -166,6 +177,16 @@ return result +class localbatch(peermod.batcher): + """performs the queued calls directly (backported from D320)""" + def __init__(self, local): + super(localbatch, self).__init__(self) + self.local = local + + def submit(self): + for name, args, opts, resref in self.calls: + resref.set(getattr(self.local, name)(*args, **opts)) + def _getfilesbatch( remote, receivemissing, progresstick, missed, idmap, batchsize): # Over http(s), iterbatch is a streamy method and we can start @@ -194,7 +215,7 @@ return while missed: chunk, missed = missed[:batchsize], missed[batchsize:] - b = remote.batch() + b = localbatch(remote) futures = {} for m in chunk: file_ = idmap[m] @@ -212,6 +233,8 @@ remote, receivemissing, progresstick, missed, idmap, step): remote._callstream("getfiles") i = 0 + pipeo = shallowutil.trygetattr(remote, ('_pipeo', 'pipeo')) + pipei = shallowutil.trygetattr(remote, ('_pipei', 'pipei')) while i < len(missed): # issue a batch of requests start = i @@ -222,19 +245,19 @@ versionid = missingid[-40:] file = idmap[missingid] sshrequest = "%s%s\n" % (versionid, file) - remote.pipeo.write(sshrequest) - remote.pipeo.flush() + pipeo.write(sshrequest) + pipeo.flush() # receive batch results for missingid in missed[start:end]: versionid = missingid[-40:] file = idmap[missingid] - receivemissing(remote.pipei, file, versionid) + receivemissing(pipei, file, versionid) progresstick() # End the command - remote.pipeo.write('\n') - remote.pipeo.flush() + pipeo.write('\n') + pipeo.flush() class fileserverclient(object): """A client for requesting files from the remote file server. @@ -428,8 +451,9 @@ packpath = shallowutil.getcachepackpath( self.repo, constants.FILEPACK_CATEGORY) + pipei = shallowutil.trygetattr(remote, ('_pipei', 'pipei')) receiveddata, receivedhistory = wirepack.receivepack( - self.repo.ui, remote.pipei, packpath) + self.repo.ui, pipei, packpath) rcvd = len(receiveddata) self.ui.log("remotefilefetchlog", @@ -453,16 +477,17 @@ grouped.setdefault(filename, set()).add(node) # Issue request + pipeo = shallowutil.trygetattr(remote, ('_pipeo', 'pipeo')) for filename, nodes in grouped.iteritems(): filenamelen = struct.pack(constants.FILENAMESTRUCT, len(filename)) countlen = struct.pack(constants.PACKREQUESTCOUNTSTRUCT, len(nodes)) rawnodes = ''.join(bin(n) for n in nodes) - remote.pipeo.write('%s%s%s%s' % (filenamelen, filename, countlen, - rawnodes)) - remote.pipeo.flush() - remote.pipeo.write(struct.pack(constants.FILENAMESTRUCT, 0)) - remote.pipeo.flush() + pipeo.write('%s%s%s%s' % (filenamelen, filename, countlen, + rawnodes)) + pipeo.flush() + pipeo.write(struct.pack(constants.FILENAMESTRUCT, 0)) + pipeo.flush() def connect(self): if self.cacheprocess: diff --git a/remotefilelog/remotefilelogserver.py b/remotefilelog/remotefilelogserver.py --- a/remotefilelog/remotefilelogserver.py +++ b/remotefilelog/remotefilelogserver.py @@ -193,7 +193,10 @@ caps.append('getflogheads') caps.append('getfile') return caps - wrapfunction(wireproto, '_capabilities', _capabilities) + if util.safehasattr(wireproto, '_capabilities'): + wrapfunction(wireproto, '_capabilities', _capabilities) + else: + wrapfunction(wireproto, 'capabilities', _capabilities) def _adjustlinkrev(orig, self, *args, **kwargs): # When generating file blobs, taking the real path is too slow on large diff --git a/remotefilelog/shallowutil.py b/remotefilelog/shallowutil.py --- a/remotefilelog/shallowutil.py +++ b/remotefilelog/shallowutil.py @@ -463,3 +463,17 @@ setstickygroupdir(path, gid, ui.warn) finally: os.umask(oldumask) + +def trygetattr(obj, names): + """try different attribute names, return the first matched attribute, + or raise if no names are matched. + """ + for name in names: + result = getattr(obj, name, None) + if result is not None: + return result + raise AttributeError + +def peercapabilities(peer): + """return capabilities of a peer""" + return trygetattr(peer, ('_capabilities', 'capabilities'))() diff --git a/tests/test-remotefilelog-http.t b/tests/test-remotefilelog-http.t --- a/tests/test-remotefilelog-http.t +++ b/tests/test-remotefilelog-http.t @@ -23,10 +23,10 @@ $ hgcloneshallow http://localhost:$HGPORT/ shallow -q 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob) - $ grep batch access.log | grep getfile - * "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=getfile+*node%3D1406e74118627694268417491f018a4a883152f0* (glob) - $ grep batch access.log | grep getfile - * "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=getfile+*file%3Dx* (glob) + $ grep getfile access.log + * "GET /?cmd=getfile HTTP/1.1" 200 * (glob) + $ grep getfile access.log + * "GET /?cmd=getfile HTTP/1.1" 200 * (glob) Clear filenode cache so we can test fetching with a modified batch size $ rm -r $TESTTMP/hgcache