diff --git a/hgext3rd/extutil.py b/hgext3rd/extutil.py --- a/hgext3rd/extutil.py +++ b/hgext3rd/extutil.py @@ -5,6 +5,8 @@ # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. +from __future__ import absolute_import + import errno import os import platform diff --git a/hgext3rd/fastannotate/protocol.py b/hgext3rd/fastannotate/protocol.py --- a/hgext3rd/fastannotate/protocol.py +++ b/hgext3rd/fastannotate/protocol.py @@ -10,6 +10,7 @@ extensions, hg, localrepo, + peer as peermod, scmutil, wireproto, ) @@ -173,18 +174,18 @@ lastnodemap = {} ui = repo.ui - batcher = peer.batch() + batcher = peer.iterbatch() ui.debug('fastannotate: requesting %d files\n' % len(paths)) - results = [batcher.getannotate(p, lastnodemap.get(p)) for p in paths] + for p in paths: + batcher.getannotate(p, lastnodemap.get(p)) # Note: This is the only place that fastannotate sends a request via SSH. # The SSH stream should not be in the remotefilelog "getfiles" loop. batcher.submit() + results = list(batcher.results()) ui.debug('fastannotate: server returned\n') for result in results: - if result.value is None: - continue - for path, content in result.value.iteritems(): + for path, content in result.iteritems(): # ignore malicious paths if not path.startswith('fastannotate/') or '/../' in (path + '/'): ui.debug('fastannotate: ignored malicious path %s\n' % path) diff --git a/remotefilelog/__init__.py b/remotefilelog/__init__.py --- a/remotefilelog/__init__.py +++ b/remotefilelog/__init__.py @@ -152,7 +152,8 @@ # Replace remote.stream_out with a version that sends file # patterns. def stream_out_shallow(orig): - if shallowrepo.requirement in remote._capabilities(): + caps = shallowutil.peercapabilities(remote) + if shallowrepo.requirement in caps: opts = {} if repo.includepattern: opts['includepattern'] = '\0'.join(repo.includepattern) diff --git a/remotefilelog/connectionpool.py b/remotefilelog/connectionpool.py --- a/remotefilelog/connectionpool.py +++ b/remotefilelog/connectionpool.py @@ -5,6 +5,8 @@ # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. +from __future__ import absolute_import + from mercurial import ( extensions, hg, @@ -12,6 +14,10 @@ util, ) +from . import ( + shallowutil, +) + class connectionpool(object): def __init__(self, repo): self._repo = repo @@ -29,9 +35,11 @@ conn = pathpool.pop() peer = conn.peer # If the connection has died, drop it - if (isinstance(peer, sshpeer.sshpeer) and - peer.subprocess.poll() is not None): - conn = None + if isinstance(peer, sshpeer.sshpeer): + proc = shallowutil.trygetattr( + peer, ('_subprocess', 'subprocess')) + if proc.poll() is not None: + conn = None except IndexError: pass diff --git a/remotefilelog/fileserverclient.py b/remotefilelog/fileserverclient.py --- a/remotefilelog/fileserverclient.py +++ b/remotefilelog/fileserverclient.py @@ -5,11 +5,22 @@ # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. +from __future__ import absolute_import + +import hashlib, os, time, io, struct +import itertools + from mercurial.i18n import _ from mercurial.node import hex, bin, nullid -from mercurial import util, sshpeer, error, util, wireproto, httppeer -import hashlib, os, time, io, struct -import itertools +from mercurial import ( + error, + httppeer, + peer as peermod, + sshpeer, + util, + util, + wireproto, +) from . import ( connectionpool, @@ -19,7 +30,7 @@ ) from .contentstore import unioncontentstore from .metadatastore import unionmetadatastore -from lz4wrapper import lz4decompress +from .lz4wrapper import lz4decompress # Statistics for debugging fetchcost = 0 @@ -64,7 +75,7 @@ def _updatecallstreamopts(self, command, opts): if command != 'getbundle': return - if 'remotefilelog' not in self._capabilities(): + if 'remotefilelog' not in shallowutil.peercapabilities(self): return if not util.safehasattr(self, '_localrepo'): return @@ -194,15 +205,13 @@ return while missed: chunk, missed = missed[:batchsize], missed[batchsize:] - b = remote.batch() - futures = {} + b = remote.iterbatch() for m in chunk: file_ = idmap[m] node = m[-40:] - futures[m] = b.getfile(file_, node) + b.getfile(file_, node) b.submit() - for m in chunk: - v = futures[m].value + for m, v in zip(chunk, b.results()): file_ = idmap[m] node = m[-40:] receivemissing(io.BytesIO('%d\n%s' % (len(v), v)), file_, node) @@ -212,6 +221,8 @@ remote, receivemissing, progresstick, missed, idmap, step): remote._callstream("getfiles") i = 0 + pipeo = shallowutil.trygetattr(remote, ('_pipeo', 'pipeo')) + pipei = shallowutil.trygetattr(remote, ('_pipei', 'pipei')) while i < len(missed): # issue a batch of requests start = i @@ -222,19 +233,19 @@ versionid = missingid[-40:] file = idmap[missingid] sshrequest = "%s%s\n" % (versionid, file) - remote.pipeo.write(sshrequest) - remote.pipeo.flush() + pipeo.write(sshrequest) + pipeo.flush() # receive batch results for missingid in missed[start:end]: versionid = missingid[-40:] file = idmap[missingid] - receivemissing(remote.pipei, file, versionid) + receivemissing(pipei, file, versionid) progresstick() # End the command - remote.pipeo.write('\n') - remote.pipeo.flush() + pipeo.write('\n') + pipeo.flush() class fileserverclient(object): """A client for requesting files from the remote file server. @@ -428,8 +439,9 @@ packpath = shallowutil.getcachepackpath( self.repo, constants.FILEPACK_CATEGORY) + pipei = shallowutil.trygetattr(remote, ('_pipei', 'pipei')) receiveddata, receivedhistory = wirepack.receivepack( - self.repo.ui, remote.pipei, packpath) + self.repo.ui, pipei, packpath) rcvd = len(receiveddata) self.ui.log("remotefilefetchlog", @@ -453,16 +465,17 @@ grouped.setdefault(filename, set()).add(node) # Issue request + pipeo = shallowutil.trygetattr(remote, ('_pipeo', 'pipeo')) for filename, nodes in grouped.iteritems(): filenamelen = struct.pack(constants.FILENAMESTRUCT, len(filename)) countlen = struct.pack(constants.PACKREQUESTCOUNTSTRUCT, len(nodes)) rawnodes = ''.join(bin(n) for n in nodes) - remote.pipeo.write('%s%s%s%s' % (filenamelen, filename, countlen, - rawnodes)) - remote.pipeo.flush() - remote.pipeo.write(struct.pack(constants.FILENAMESTRUCT, 0)) - remote.pipeo.flush() + pipeo.write('%s%s%s%s' % (filenamelen, filename, countlen, + rawnodes)) + pipeo.flush() + pipeo.write(struct.pack(constants.FILENAMESTRUCT, 0)) + pipeo.flush() def connect(self): if self.cacheprocess: diff --git a/remotefilelog/remotefilelogserver.py b/remotefilelog/remotefilelogserver.py --- a/remotefilelog/remotefilelogserver.py +++ b/remotefilelog/remotefilelogserver.py @@ -193,7 +193,10 @@ caps.append('getflogheads') caps.append('getfile') return caps - wrapfunction(wireproto, '_capabilities', _capabilities) + if util.safehasattr(wireproto, '_capabilities'): + wrapfunction(wireproto, '_capabilities', _capabilities) + else: + wrapfunction(wireproto, 'capabilities', _capabilities) def _adjustlinkrev(orig, self, *args, **kwargs): # When generating file blobs, taking the real path is too slow on large diff --git a/remotefilelog/shallowutil.py b/remotefilelog/shallowutil.py --- a/remotefilelog/shallowutil.py +++ b/remotefilelog/shallowutil.py @@ -463,3 +463,17 @@ setstickygroupdir(path, gid, ui.warn) finally: os.umask(oldumask) + +def trygetattr(obj, names): + """try different attribute names, return the first matched attribute, + or raise if no names are matched. + """ + for name in names: + result = getattr(obj, name, None) + if result is not None: + return result + raise AttributeError + +def peercapabilities(peer): + """return capabilities of a peer""" + return trygetattr(peer, ('_capabilities', 'capabilities'))() diff --git a/tests/test-remotefilelog-http.t b/tests/test-remotefilelog-http.t --- a/tests/test-remotefilelog-http.t +++ b/tests/test-remotefilelog-http.t @@ -25,8 +25,6 @@ $ grep batch access.log | grep getfile * "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=getfile+*node%3D1406e74118627694268417491f018a4a883152f0* (glob) - $ grep batch access.log | grep getfile - * "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=getfile+*file%3Dx* (glob) Clear filenode cache so we can test fetching with a modified batch size $ rm -r $TESTTMP/hgcache