diff --git a/remotefilelog/remotefilectx.py b/remotefilelog/remotefilectx.py --- a/remotefilelog/remotefilectx.py +++ b/remotefilelog/remotefilectx.py @@ -8,9 +8,31 @@ import collections from mercurial.i18n import _ from mercurial.node import bin, hex, nullid, nullrev -from mercurial import context, util, error, ancestor, phases +from mercurial import context, util, error, ancestor, phases, extensions propertycache = util.propertycache +conduit = None +FASTLOG_TIMEOUT_IN_SECS = 0.5 + +def createconduit(ui): + try: + conduit = extensions.find("fbconduit") + except KeyError: + try: + from hgext3rd import fbconduit as conduit + except ImportError: + ui.log('fastloglinkrevfixup', + _('unable to find fbconduit extension\n')) + return False + if not util.safehasattr(conduit, 'conduit_config'): + ui.log('fastloglinkrevfixup', + _('incompatible conduit module; disabling fastlog\n')) + return False + if not conduit.conduit_config(ui): + ui.log('fastloglinkrevfixup', + _('no conduit host specified in config; disabling fastlog\n')) + return False + return conduit class remotefilectx(context.filectx): def __init__(self, repo, path, changeid=None, fileid=None, @@ -27,6 +49,17 @@ return self._filelog.size(self._filenode) @propertycache + def _conduit(self): + global conduit + if conduit is None: + conduit = createconduit(self._repo.ui) + # If createconduit fails, conduit will be set to False. We use this to + # avoid calling createconduit multiple times + if conduit is False: + return None + return conduit + + @propertycache def _changeid(self): if '_changeid' in self.__dict__: return self._changeid @@ -156,6 +189,16 @@ return results + def _nodefromancrev(self, ancrev, cl, mfl, path, fnode): + """returns the node for in if content matches """ + ancctx = cl.read(ancrev) # This avoids object creation. + manifestnode, files = ancctx[0], ancctx[3] + # If the file was touched in this ancestor, and the content is similar + # to the one we are searching for. + if path in files and fnode == mfl[manifestnode].readfast().get(path): + return cl.node(ancrev) + return None + def _adjustlinknode(self, path, filelog, fnode, srcrev, inclusive=False): """return the first ancestor of introducing @@ -202,13 +245,9 @@ iteranc = cl.ancestors(revs, inclusive=inclusive) for ancrev in iteranc: # First, check locally-available history. - ancctx = cl.read(ancrev) # This avoids object creation. - manifestnode, files = ancctx[0], ancctx[3] - if path in files: - # The file was touched in this ancestor, so check if the - # content is similar to the one we are searching for. - if fnode == mfl[manifestnode].readfast().get(path): - return cl.node(ancrev) + lnode = self._nodefromancrev(ancrev, cl, mfl, path, fnode) + if lnode is not None: + return lnode # This next part is super non-obvious, so big comment block time! # @@ -251,6 +290,16 @@ # the slow path is used too much. One promising possibility is using # obsolescence markers to find a more-likely-correct linkrev. if not seenpublic and pc.phase(repo, ancrev) == phases.public: + # If the commit is public and fastlog is enabled for this repo + # then we will can try to fetch the right linknode via fastlog + # since fastlog already has the right linkrev for all public + # commits + if repo.ui.configbool('fastlog', 'enabled'): + lnode = self._linknodeviafastlog(repo, path, ancrev, fnode, + cl, mfl) + if lnode: + repo.ui.log('fastloglinkrevfixup', _('success\n')) + return lnode seenpublic = True try: repo.fileservice.prefetch([(path, hex(fnode))], force=True) @@ -272,6 +321,36 @@ return linknode + def _linknodeviafastlog(self, repo, path, srcrev, fnode, cl, mfl): + reponame = repo.ui.config('fbconduit', 'reponame') + if self._conduit is None: + return None + try: + srchex = repo[srcrev].hex() + results = self._conduit.call_conduit( + 'scmquery.log_v2', + timeout=FASTLOG_TIMEOUT_IN_SECS, + repo=reponame, + scm_type='hg', + rev=srchex, + file_paths=[path], + skip=0, + ) + if results is None: + repo.ui.log('fastloglinkrevfixup', + _('fastlog returned 0 results\n')) + return None + for anc in results: + ancrev = repo[str(anc['hash'])].rev() + lnode = self._nodefromancrev(ancrev, cl, mfl, path, fnode) + if lnode is not None: + return lnode + return None + except Exception as e: + repo.ui.log('fastloglinkrevfixup', + _('call to fastlog failed (%s)\n') % e) + return None + def _verifylinknode(self, revs, linknode): """ Check if a linknode is correct one for the current history. diff --git a/tests/test-remotefilelog-linknodes.t b/tests/test-remotefilelog-linknodes.t --- a/tests/test-remotefilelog-linknodes.t +++ b/tests/test-remotefilelog-linknodes.t @@ -194,3 +194,124 @@ d4a3ed9310e5 => aee31534993a 000000000000 32e6611f6149 aee31534993a => 1406e7411862 000000000000 0632994590a8 1406e7411862 => 000000000000 000000000000 b292c1e3311f + +Test the same scenario as above but with fastlog enabled + + $ cd .. + $ clearcache + + $ rm -rf master + $ rm -rf shallow + $ hginit master + $ cd master + $ cat >> .hg/hgrc < [remotefilelog] + > server=True + > serverexpiration=-1 + > EOF + $ echo x > x + $ hg commit -qAm x + $ echo x >> x + $ hg commit -Aqm xx + $ cd .. + + $ hgcloneshallow ssh://user@dummy/master shallow -q + 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) + $ cd shallow + $ echo x >> x + $ hg commit -Aqm xx2 + $ cd ../master + $ echo y >> y + $ hg commit -Aqm yy2 + $ echo x >> x + $ hg commit -Aqm xx2-fake-rebased + $ echo y >> y + $ hg commit -Aqm yy3 + $ cd ../shallow + $ hg pull -q + $ hg update tip -q + 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob) + $ echo x > x + $ hg commit -qAm xx3 + +Verfiy correct linkrev despite fastlog failures + +Case 1: fastlog service calls fails or times out + + $ cat >> .hg/hgrc < [extensions] + > fbconduit=$TESTTMP/bad_conduit.py + > [fastlog] + > enabled=True + > EOF + $ cat > $TESTTMP/bad_conduit.py < def call_conduit(*args, **kwargs): + > raise Exception('error') + > def conduit_config(*args, **kwargs): + > return True + > EOF + $ hg log -f x -T '{node|short} {desc} {phase} {files}\n' + a5957b6bf0bd xx3 draft x + 32e6611f6149 xx2-fake-rebased public x + 0632994590a8 xx public x + b292c1e3311f x public x + 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob) + +Case 2: fastlog returns empty results + + $ cat > $TESTTMP/bad_conduit.py < def call_conduit(*args, **kwargs): + > return [] + > def conduit_config(*args, **kwargs): + > return True + > EOF + $ rm $TESTTMP/bad_conduit.pyc + $ hg log -f x -T '{node|short} {desc} {phase} {files}\n' + a5957b6bf0bd xx3 draft x + 32e6611f6149 xx2-fake-rebased public x + 0632994590a8 xx public x + b292c1e3311f x public x + +Case 3: fastlog returns a bad hash + + $ cat > $TESTTMP/bad_conduit.py < def call_conduit(*args, **kwargs): + > return [{'hash': '123456'}] + > def conduit_config(*args, **kwargs): + > return True + > EOF + $ rm $TESTTMP/bad_conduit.pyc + $ hg log -f x -T '{node|short} {desc} {phase} {files}\n' + a5957b6bf0bd xx3 draft x + 32e6611f6149 xx2-fake-rebased public x + 0632994590a8 xx public x + b292c1e3311f x public x + +Fastlog succeeds and returns the correct results + + $ cat > $TESTTMP/bad_conduit.py < def call_conduit(*args, **kwargs): + > return [{'hash': '32e6611f6149e85f58def77ee0c22549bb6953a2'}] + > def conduit_config(*args, **kwargs): + > return True + > EOF + $ rm $TESTTMP/bad_conduit.pyc + $ hg log -f x -T '{node|short} {desc} {phase} {files}\n' + a5957b6bf0bd xx3 draft x + 32e6611f6149 xx2-fake-rebased public x + 0632994590a8 xx public x + b292c1e3311f x public x + +Fastlog should never get called on draft commits + + $ cat > $TESTTMP/bad_conduit.py < import sys + > def call_conduit(*args, **kwargs): + > if kwargs['rev'].startswith('a5957b6bf0bd'): + > sys.exit(80) + > return [{'hash': '32e6611f6149e85f58def77ee0c22549bb6953a2'}] + > def conduit_config(*args, **kwargs): + > return True + > EOF + $ rm $TESTTMP/bad_conduit.pyc + $ hg log -f x > /dev/null