This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5190

shallowutil: introduce a helper function isenabled()
ClosedPublic

Authored by pulkit on Oct 24 2018, 10:55 AM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHG6f0b6905ef6f: shallowutil: introduce a helper function isenabled()
Summary

This patch introduces a inenabled() function which will check whether
remotefilelog is enabled or not. The function is then also used at all the
places where check whether remotefilelog is enabled or not. The new function
makes code easy to read without need to understand what is the constant involved
and why we are checking repo.requirements.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

pulkit created this revision.Oct 24 2018, 10:55 AM
durin42 accepted this revision.Nov 5 2018, 11:01 AM
This revision is now accepted and ready to land.Nov 5 2018, 11:01 AM
Closed by commit rHG6f0b6905ef6f: shallowutil: introduce a helper function isenabled() (authored by pulkit). · Explain WhyNov 5 2018, 2:56 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 12337

hgext/remotefilelog/__init__.py

# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
# be specifying the version(s) of Mercurial they are tested with, or # be specifying the version(s) of Mercurial they are tested with, or
# leave the attribute unspecified. # leave the attribute unspecified.
testedwith = 'ships-with-hg-core' testedwith = 'ships-with-hg-core'
repoclass = localrepo.localrepository repoclass = localrepo.localrepository
repoclass._basesupported.add(constants.SHALLOWREPO_REQUIREMENT) repoclass._basesupported.add(constants.SHALLOWREPO_REQUIREMENT)
isenabled = shallowutil.isenabled
def uisetup(ui): def uisetup(ui):
"""Wraps user facing Mercurial commands to swap them out with shallow """Wraps user facing Mercurial commands to swap them out with shallow
versions. versions.
""" """
hg.wirepeersetupfuncs.append(fileserverclient.peersetup) hg.wirepeersetupfuncs.append(fileserverclient.peersetup)
entry = extensions.wrapcommand(commands.table, 'clone', cloneshallow) entry = extensions.wrapcommand(commands.table, 'clone', cloneshallow)
entry[1].append(('', 'shallow', None, entry[1].append(('', 'shallow', None,
_("create a shallow clone which uses remote file " _("create a shallow clone which uses remote file "
"history"))) "history")))
extensions.wrapcommand(commands.table, 'debugindex', extensions.wrapcommand(commands.table, 'debugindex',
debugcommands.debugindex) debugcommands.debugindex)
extensions.wrapcommand(commands.table, 'debugindexdot', extensions.wrapcommand(commands.table, 'debugindexdot',
debugcommands.debugindexdot) debugcommands.debugindexdot)
extensions.wrapcommand(commands.table, 'log', log) extensions.wrapcommand(commands.table, 'log', log)
extensions.wrapcommand(commands.table, 'pull', pull) extensions.wrapcommand(commands.table, 'pull', pull)
# Prevent 'hg manifest --all' # Prevent 'hg manifest --all'
def _manifest(orig, ui, repo, *args, **opts): def _manifest(orig, ui, repo, *args, **opts):
if (constants.SHALLOWREPO_REQUIREMENT in repo.requirements if (isenabled(repo) and opts.get('all')):
and opts.get('all')):
raise error.Abort(_("--all is not supported in a shallow repo")) raise error.Abort(_("--all is not supported in a shallow repo"))
return orig(ui, repo, *args, **opts) return orig(ui, repo, *args, **opts)
extensions.wrapcommand(commands.table, "manifest", _manifest) extensions.wrapcommand(commands.table, "manifest", _manifest)
# Wrap remotefilelog with lfs code # Wrap remotefilelog with lfs code
def _lfsloaded(loaded=False): def _lfsloaded(loaded=False):
lfsmod = None lfsmod = None
try: try:
lfsmod = extensions.find('lfs') lfsmod = extensions.find('lfs')
except KeyError: except KeyError:
pass pass
if lfsmod: if lfsmod:
lfsmod.wrapfilelog(remotefilelog.remotefilelog) lfsmod.wrapfilelog(remotefilelog.remotefilelog)
fileserverclient._lfsmod = lfsmod fileserverclient._lfsmod = lfsmod
extensions.afterloaded('lfs', _lfsloaded) extensions.afterloaded('lfs', _lfsloaded)
# debugdata needs remotefilelog.len to work # debugdata needs remotefilelog.len to work
extensions.wrapcommand(commands.table, 'debugdata', debugdatashallow) extensions.wrapcommand(commands.table, 'debugdata', debugdatashallow)
def cloneshallow(orig, ui, repo, *args, **opts): def cloneshallow(orig, ui, repo, *args, **opts):
if opts.get('shallow'): if opts.get('shallow'):
repos = [] repos = []
def pull_shallow(orig, self, *args, **kwargs): def pull_shallow(orig, self, *args, **kwargs):
if constants.SHALLOWREPO_REQUIREMENT not in self.requirements: if not isenabled(self):
repos.append(self.unfiltered()) repos.append(self.unfiltered())
# set up the client hooks so the post-clone update works # set up the client hooks so the post-clone update works
setupclient(self.ui, self.unfiltered()) setupclient(self.ui, self.unfiltered())
# setupclient fixed the class on the repo itself # setupclient fixed the class on the repo itself
# but we also need to fix it on the repoview # but we also need to fix it on the repoview
if isinstance(self, repoview.repoview): if isinstance(self, repoview.repoview):
self.__class__.__bases__ = (self.__class__.__bases__[0], self.__class__.__bases__ = (self.__class__.__bases__[0],
if not isinstance(repo, localrepo.localrepository): if not isinstance(repo, localrepo.localrepository):
return return
# put here intentionally bc doesnt work in uisetup # put here intentionally bc doesnt work in uisetup
ui.setconfig('hooks', 'update.prefetch', wcpprefetch) ui.setconfig('hooks', 'update.prefetch', wcpprefetch)
ui.setconfig('hooks', 'commit.prefetch', wcpprefetch) ui.setconfig('hooks', 'commit.prefetch', wcpprefetch)
isserverenabled = ui.configbool('remotefilelog', 'server') isserverenabled = ui.configbool('remotefilelog', 'server')
isshallowclient = constants.SHALLOWREPO_REQUIREMENT in repo.requirements isshallowclient = isenabled(repo)
if isserverenabled and isshallowclient: if isserverenabled and isshallowclient:
raise RuntimeError("Cannot be both a server and shallow client.") raise RuntimeError("Cannot be both a server and shallow client.")
if isshallowclient: if isshallowclient:
setupclient(ui, repo) setupclient(ui, repo)
if isserverenabled: if isserverenabled:
return s return s
extensions.wrapfunction(localrepo, 'makestore', storewrapper) extensions.wrapfunction(localrepo, 'makestore', storewrapper)
extensions.wrapfunction(exchange, 'pull', exchangepull) extensions.wrapfunction(exchange, 'pull', exchangepull)
# prefetch files before update # prefetch files before update
def applyupdates(orig, repo, actions, wctx, mctx, overwrite, labels=None): def applyupdates(orig, repo, actions, wctx, mctx, overwrite, labels=None):
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if isenabled(repo):
manifest = mctx.manifest() manifest = mctx.manifest()
files = [] files = []
for f, args, msg in actions['g']: for f, args, msg in actions['g']:
files.append((f, hex(manifest[f]))) files.append((f, hex(manifest[f])))
# batch fetch the needed files from the server # batch fetch the needed files from the server
repo.fileservice.prefetch(files) repo.fileservice.prefetch(files)
return orig(repo, actions, wctx, mctx, overwrite, labels=labels) return orig(repo, actions, wctx, mctx, overwrite, labels=labels)
extensions.wrapfunction(merge, 'applyupdates', applyupdates) extensions.wrapfunction(merge, 'applyupdates', applyupdates)
# Prefetch merge checkunknownfiles # Prefetch merge checkunknownfiles
def checkunknownfiles(orig, repo, wctx, mctx, force, actions, def checkunknownfiles(orig, repo, wctx, mctx, force, actions,
*args, **kwargs): *args, **kwargs):
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if isenabled(repo):
files = [] files = []
sparsematch = repo.maybesparsematch(mctx.rev()) sparsematch = repo.maybesparsematch(mctx.rev())
for f, (m, actionargs, msg) in actions.iteritems(): for f, (m, actionargs, msg) in actions.iteritems():
if sparsematch and not sparsematch(f): if sparsematch and not sparsematch(f):
continue continue
if m in ('c', 'dc', 'cm'): if m in ('c', 'dc', 'cm'):
files.append((f, hex(mctx.filenode(f)))) files.append((f, hex(mctx.filenode(f))))
elif m == 'dg': elif m == 'dg':
f2 = actionargs[0] f2 = actionargs[0]
files.append((f2, hex(mctx.filenode(f2)))) files.append((f2, hex(mctx.filenode(f2))))
# batch fetch the needed files from the server # batch fetch the needed files from the server
repo.fileservice.prefetch(files) repo.fileservice.prefetch(files)
return orig(repo, wctx, mctx, force, actions, *args, **kwargs) return orig(repo, wctx, mctx, force, actions, *args, **kwargs)
extensions.wrapfunction(merge, '_checkunknownfiles', checkunknownfiles) extensions.wrapfunction(merge, '_checkunknownfiles', checkunknownfiles)
# Prefetch files before status attempts to look at their size and contents # Prefetch files before status attempts to look at their size and contents
def checklookup(orig, self, files): def checklookup(orig, self, files):
repo = self._repo repo = self._repo
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if isenabled(repo):
prefetchfiles = [] prefetchfiles = []
for parent in self._parents: for parent in self._parents:
for f in files: for f in files:
if f in parent: if f in parent:
prefetchfiles.append((f, hex(parent.filenode(f)))) prefetchfiles.append((f, hex(parent.filenode(f))))
# batch fetch the needed files from the server # batch fetch the needed files from the server
repo.fileservice.prefetch(prefetchfiles) repo.fileservice.prefetch(prefetchfiles)
return orig(self, files) return orig(self, files)
extensions.wrapfunction(context.workingctx, '_checklookup', checklookup) extensions.wrapfunction(context.workingctx, '_checklookup', checklookup)
# Prefetch the logic that compares added and removed files for renames # Prefetch the logic that compares added and removed files for renames
def findrenames(orig, repo, matcher, added, removed, *args, **kwargs): def findrenames(orig, repo, matcher, added, removed, *args, **kwargs):
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if isenabled(repo):
files = [] files = []
parentctx = repo['.'] parentctx = repo['.']
for f in removed: for f in removed:
files.append((f, hex(parentctx.filenode(f)))) files.append((f, hex(parentctx.filenode(f))))
# batch fetch the needed files from the server # batch fetch the needed files from the server
repo.fileservice.prefetch(files) repo.fileservice.prefetch(files)
return orig(repo, matcher, added, removed, *args, **kwargs) return orig(repo, matcher, added, removed, *args, **kwargs)
extensions.wrapfunction(scmutil, '_findrenames', findrenames) extensions.wrapfunction(scmutil, '_findrenames', findrenames)
# prefetch files before mergecopies check # prefetch files before mergecopies check
def computenonoverlap(orig, repo, c1, c2, *args, **kwargs): def computenonoverlap(orig, repo, c1, c2, *args, **kwargs):
u1, u2 = orig(repo, c1, c2, *args, **kwargs) u1, u2 = orig(repo, c1, c2, *args, **kwargs)
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if isenabled(repo):
m1 = c1.manifest() m1 = c1.manifest()
m2 = c2.manifest() m2 = c2.manifest()
files = [] files = []
sparsematch1 = repo.maybesparsematch(c1.rev()) sparsematch1 = repo.maybesparsematch(c1.rev())
if sparsematch1: if sparsematch1:
sparseu1 = [] sparseu1 = []
for f in u1: for f in u1:
repo.fileservice.prefetch(files) repo.fileservice.prefetch(files)
return u1, u2 return u1, u2
extensions.wrapfunction(copies, '_computenonoverlap', computenonoverlap) extensions.wrapfunction(copies, '_computenonoverlap', computenonoverlap)
# prefetch files before pathcopies check # prefetch files before pathcopies check
def computeforwardmissing(orig, a, b, match=None): def computeforwardmissing(orig, a, b, match=None):
missing = list(orig(a, b, match=match)) missing = list(orig(a, b, match=match))
repo = a._repo repo = a._repo
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if isenabled(repo):
mb = b.manifest() mb = b.manifest()
files = [] files = []
sparsematch = repo.maybesparsematch(b.rev()) sparsematch = repo.maybesparsematch(b.rev())
if sparsematch: if sparsematch:
sparsemissing = [] sparsemissing = []
for f in missing: for f in missing:
if sparsematch(f): if sparsematch(f):
# close cache miss server connection after the command has finished # close cache miss server connection after the command has finished
def runcommand(orig, lui, repo, *args, **kwargs): def runcommand(orig, lui, repo, *args, **kwargs):
try: try:
return orig(lui, repo, *args, **kwargs) return orig(lui, repo, *args, **kwargs)
finally: finally:
# repo can be None when running in chg: # repo can be None when running in chg:
# - at startup, reposetup was called because serve is not norepo # - at startup, reposetup was called because serve is not norepo
# - a norepo command like "help" is called # - a norepo command like "help" is called
if repo and constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if repo and isenabled(repo):
repo.fileservice.close() repo.fileservice.close()
extensions.wrapfunction(dispatch, 'runcommand', runcommand) extensions.wrapfunction(dispatch, 'runcommand', runcommand)
# disappointing hacks below # disappointing hacks below
templatekw.getrenamedfn = getrenamedfn templatekw.getrenamedfn = getrenamedfn
extensions.wrapfunction(revset, 'filelog', filelogrevset) extensions.wrapfunction(revset, 'filelog', filelogrevset)
revset.symbols['filelog'] = revset.filelog revset.symbols['filelog'] = revset.filelog
extensions.wrapfunction(cmdutil, 'walkfilerevs', walkfilerevs) extensions.wrapfunction(cmdutil, 'walkfilerevs', walkfilerevs)
# prevent strip from stripping remotefilelogs # prevent strip from stripping remotefilelogs
def _collectbrokencsets(orig, repo, files, striprev): def _collectbrokencsets(orig, repo, files, striprev):
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if isenabled(repo):
files = list([f for f in files if not repo.shallowmatch(f)]) files = list([f for f in files if not repo.shallowmatch(f)])
return orig(repo, files, striprev) return orig(repo, files, striprev)
extensions.wrapfunction(repair, '_collectbrokencsets', _collectbrokencsets) extensions.wrapfunction(repair, '_collectbrokencsets', _collectbrokencsets)
# Don't commit filelogs until we know the commit hash, since the hash # Don't commit filelogs until we know the commit hash, since the hash
# is present in the filelog blob. # is present in the filelog blob.
# This violates Mercurial's filelog->manifest->changelog write order, # This violates Mercurial's filelog->manifest->changelog write order,
# but is generally fine for client repos. # but is generally fine for client repos.
del pendingfilecommits[:] del pendingfilecommits[:]
return node return node
extensions.wrapfunction(changelog.changelog, 'add', changelogadd) extensions.wrapfunction(changelog.changelog, 'add', changelogadd)
# changectx wrappers # changectx wrappers
def filectx(orig, self, path, fileid=None, filelog=None): def filectx(orig, self, path, fileid=None, filelog=None):
if fileid is None: if fileid is None:
fileid = self.filenode(path) fileid = self.filenode(path)
if (constants.SHALLOWREPO_REQUIREMENT in self._repo.requirements and if (isenabled(self._repo) and self._repo.shallowmatch(path)):
self._repo.shallowmatch(path)):
return remotefilectx.remotefilectx(self._repo, path, return remotefilectx.remotefilectx(self._repo, path,
fileid=fileid, changectx=self, filelog=filelog) fileid=fileid, changectx=self, filelog=filelog)
return orig(self, path, fileid=fileid, filelog=filelog) return orig(self, path, fileid=fileid, filelog=filelog)
extensions.wrapfunction(context.changectx, 'filectx', filectx) extensions.wrapfunction(context.changectx, 'filectx', filectx)
def workingfilectx(orig, self, path, filelog=None): def workingfilectx(orig, self, path, filelog=None):
if (constants.SHALLOWREPO_REQUIREMENT in self._repo.requirements and if (isenabled(self._repo) and self._repo.shallowmatch(path)):
self._repo.shallowmatch(path)):
return remotefilectx.remoteworkingfilectx(self._repo, return remotefilectx.remoteworkingfilectx(self._repo,
path, workingctx=self, filelog=filelog) path, workingctx=self, filelog=filelog)
return orig(self, path, filelog=filelog) return orig(self, path, filelog=filelog)
extensions.wrapfunction(context.workingctx, 'filectx', workingfilectx) extensions.wrapfunction(context.workingctx, 'filectx', workingfilectx)
# prefetch required revisions before a diff # prefetch required revisions before a diff
def trydiff(orig, repo, revs, ctx1, ctx2, modified, added, removed, def trydiff(orig, repo, revs, ctx1, ctx2, modified, added, removed,
copy, getfilectx, *args, **kwargs): copy, getfilectx, *args, **kwargs):
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if isenabled(repo):
prefetch = [] prefetch = []
mf1 = ctx1.manifest() mf1 = ctx1.manifest()
for fname in modified + added + removed: for fname in modified + added + removed:
if fname in mf1: if fname in mf1:
fnode = getfilectx(fname, ctx1).filenode() fnode = getfilectx(fname, ctx1).filenode()
# fnode can be None if it's a edited working ctx file # fnode can be None if it's a edited working ctx file
if fnode: if fnode:
prefetch.append((fname, hex(fnode))) prefetch.append((fname, hex(fnode)))
return fctx.renamed() return fctx.renamed()
except error.LookupError: except error.LookupError:
return None return None
return getrenamed return getrenamed
def walkfilerevs(orig, repo, match, follow, revs, fncache): def walkfilerevs(orig, repo, match, follow, revs, fncache):
if not constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if not enabled(repo):
return orig(repo, match, follow, revs, fncache) return orig(repo, match, follow, revs, fncache)
# remotefilelog's can't be walked in rev order, so throw. # remotefilelog's can't be walked in rev order, so throw.
# The caller will see the exception and walk the commit tree instead. # The caller will see the exception and walk the commit tree instead.
if not follow: if not follow:
raise cmdutil.FileWalkError("Cannot walk via filelog") raise cmdutil.FileWalkError("Cannot walk via filelog")
wanted = set() wanted = set()
"""``filelog(pattern)`` """``filelog(pattern)``
Changesets connected to the specified filelog. Changesets connected to the specified filelog.
For performance reasons, ``filelog()`` does not show every changeset For performance reasons, ``filelog()`` does not show every changeset
that affects the requested file(s). See :hg:`help log` for details. For that affects the requested file(s). See :hg:`help log` for details. For
a slower, more accurate result, use ``file()``. a slower, more accurate result, use ``file()``.
""" """
if not constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if not isenabled(repo):
return orig(repo, subset, x) return orig(repo, subset, x)
# i18n: "filelog" is a keyword # i18n: "filelog" is a keyword
pat = revset.getstring(x, _("filelog requires a pattern")) pat = revset.getstring(x, _("filelog requires a pattern"))
m = match.match(repo.root, repo.getcwd(), [pat], default='relpath', m = match.match(repo.root, repo.getcwd(), [pat], default='relpath',
ctx=repo[None]) ctx=repo[None])
s = set() s = set()
except error.RepoError: except error.RepoError:
continue continue
validrepos.append(path) validrepos.append(path)
# Protect against any repo or config changes that have happened since # Protect against any repo or config changes that have happened since
# this repo was added to the repos file. We'd rather this loop succeed # this repo was added to the repos file. We'd rather this loop succeed
# and too much be deleted, than the loop fail and nothing gets deleted. # and too much be deleted, than the loop fail and nothing gets deleted.
if constants.SHALLOWREPO_REQUIREMENT not in repo.requirements: if not isenabled(repo):
continue continue
if not util.safehasattr(repo, 'name'): if not util.safehasattr(repo, 'name'):
ui.warn(_("repo %s is a misconfigured remotefilelog repo\n") % path) ui.warn(_("repo %s is a misconfigured remotefilelog repo\n") % path)
continue continue
# If garbage collection on repack and repack on hg gc are enabled # If garbage collection on repack and repack on hg gc are enabled
# then loose files are repacked and garbage collected. # then loose files are repacked and garbage collected.
# prune cache # prune cache
if sharedcache is not None: if sharedcache is not None:
sharedcache.gc(keepkeys) sharedcache.gc(keepkeys)
elif not filesrepacked: elif not filesrepacked:
ui.warn(_("warning: no valid repos in repofile\n")) ui.warn(_("warning: no valid repos in repofile\n"))
def log(orig, ui, repo, *pats, **opts): def log(orig, ui, repo, *pats, **opts):
if constants.SHALLOWREPO_REQUIREMENT not in repo.requirements: if not isenabled(repo):
return orig(ui, repo, *pats, **opts) return orig(ui, repo, *pats, **opts)
follow = opts.get('follow') follow = opts.get('follow')
revs = opts.get('rev') revs = opts.get('rev')
if pats: if pats:
# Force slowpath for non-follow patterns and follows that start from # Force slowpath for non-follow patterns and follows that start from
# non-working-copy-parent revs. # non-working-copy-parent revs.
if not follow or revs: if not follow or revs:
ready = True ready = True
return ready return ready
def wcpprefetch(ui, repo, **kwargs): def wcpprefetch(ui, repo, **kwargs):
"""Prefetches in background revisions specified by bgprefetchrevs revset. """Prefetches in background revisions specified by bgprefetchrevs revset.
Does background repack if backgroundrepack flag is set in config. Does background repack if backgroundrepack flag is set in config.
""" """
shallow = constants.SHALLOWREPO_REQUIREMENT in repo.requirements shallow = isenabled(repo)
bgprefetchrevs = ui.config('remotefilelog', 'bgprefetchrevs') bgprefetchrevs = ui.config('remotefilelog', 'bgprefetchrevs')
isready = readytofetch(repo) isready = readytofetch(repo)
if not (shallow and bgprefetchrevs and isready): if not (shallow and bgprefetchrevs and isready):
return return
bgrepack = repo.ui.configbool('remotefilelog', 'backgroundrepack') bgrepack = repo.ui.configbool('remotefilelog', 'backgroundrepack')
# update a revset with a date limit # update a revset with a date limit
bgprefetchrevs = revdatelimit(ui, bgprefetchrevs) bgprefetchrevs = revdatelimit(ui, bgprefetchrevs)
def anon(): def anon():
if util.safehasattr(repo, 'ranprefetch') and repo.ranprefetch: if util.safehasattr(repo, 'ranprefetch') and repo.ranprefetch:
return return
repo.ranprefetch = True repo.ranprefetch = True
repo.backgroundprefetch(bgprefetchrevs, repack=bgrepack) repo.backgroundprefetch(bgprefetchrevs, repack=bgrepack)
repo._afterlock(anon) repo._afterlock(anon)
def pull(orig, ui, repo, *pats, **opts): def pull(orig, ui, repo, *pats, **opts):
result = orig(ui, repo, *pats, **opts) result = orig(ui, repo, *pats, **opts)
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if isenabled(repo):
# prefetch if it's configured # prefetch if it's configured
prefetchrevset = ui.config('remotefilelog', 'pullprefetch') prefetchrevset = ui.config('remotefilelog', 'pullprefetch')
bgrepack = repo.ui.configbool('remotefilelog', 'backgroundrepack') bgrepack = repo.ui.configbool('remotefilelog', 'backgroundrepack')
bgprefetch = repo.ui.configbool('remotefilelog', 'backgroundprefetch') bgprefetch = repo.ui.configbool('remotefilelog', 'backgroundprefetch')
if prefetchrevset: if prefetchrevset:
ui.status(_("prefetching file contents\n")) ui.status(_("prefetching file contents\n"))
revs = scmutil.revrange(repo, [prefetchrevset]) revs = scmutil.revrange(repo, [prefetchrevset])
if util.safehasattr(remote, '_callstream'): if util.safehasattr(remote, '_callstream'):
remote._localrepo = repo remote._localrepo = repo
elif util.safehasattr(remote, 'getbundle'): elif util.safehasattr(remote, 'getbundle'):
extensions.wrapfunction(remote, 'getbundle', localgetbundle) extensions.wrapfunction(remote, 'getbundle', localgetbundle)
return orig(repo, remote, *args, **kwargs) return orig(repo, remote, *args, **kwargs)
def _fileprefetchhook(repo, revs, match): def _fileprefetchhook(repo, revs, match):
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if isenabled(repo):
allfiles = [] allfiles = []
for rev in revs: for rev in revs:
if rev == nodemod.wdirrev or rev is None: if rev == nodemod.wdirrev or rev is None:
continue continue
ctx = repo[rev] ctx = repo[rev]
mf = ctx.manifest() mf = ctx.manifest()
sparsematch = repo.maybesparsematch(ctx.rev()) sparsematch = repo.maybesparsematch(ctx.rev())
for path in ctx.walk(match): for path in ctx.walk(match):
Prefetchs file revisions for the specified revs and stores them in the Prefetchs file revisions for the specified revs and stores them in the
local remotefilelog cache. If no rev is specified, the default rev is local remotefilelog cache. If no rev is specified, the default rev is
used which is the union of dot, draft, pullprefetch and bgprefetchrev. used which is the union of dot, draft, pullprefetch and bgprefetchrev.
File names or patterns can be used to limit which files are downloaded. File names or patterns can be used to limit which files are downloaded.
Return 0 on success. Return 0 on success.
""" """
if not constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if not isenabled(repo):
raise error.Abort(_("repo is not shallow")) raise error.Abort(_("repo is not shallow"))
opts = resolveprefetchopts(ui, opts) opts = resolveprefetchopts(ui, opts)
revs = scmutil.revrange(repo, opts.get('rev')) revs = scmutil.revrange(repo, opts.get('rev'))
repo.prefetch(revs, opts.get('base'), pats, opts) repo.prefetch(revs, opts.get('base'), pats, opts)
# Run repack in background # Run repack in background
if opts.get('repack'): if opts.get('repack'):

hgext/remotefilelog/debugcommands.py

return r return r
def debugindex(orig, ui, repo, file_=None, **opts): def debugindex(orig, ui, repo, file_=None, **opts):
"""dump the contents of an index file""" """dump the contents of an index file"""
if (opts.get('changelog') or if (opts.get('changelog') or
opts.get('manifest') or opts.get('manifest') or
opts.get('dir') or opts.get('dir') or
not constants.SHALLOWREPO_REQUIREMENT in repo.requirements or not shallowutil.isenabled(repo) or
not repo.shallowmatch(file_)): not repo.shallowmatch(file_)):
return orig(ui, repo, file_, **opts) return orig(ui, repo, file_, **opts)
r = buildtemprevlog(repo, file_) r = buildtemprevlog(repo, file_)
# debugindex like normal # debugindex like normal
format = opts.get('format', 0) format = opts.get('format', 0)
if format not in (0, 1): if format not in (0, 1):
elif format == 1: elif format == 1:
pr = r.parentrevs(i) pr = r.parentrevs(i)
ui.write("% 6d %04x % 8d % 8d % 8d % 6d % 6d % 6d % 6d %s\n" % ( ui.write("% 6d %04x % 8d % 8d % 8d % 6d % 6d % 6d % 6d %s\n" % (
i, r.flags(i), r.start(i), r.length(i), r.rawsize(i), i, r.flags(i), r.start(i), r.length(i), r.rawsize(i),
base, r.linkrev(i), pr[0], pr[1], short(node))) base, r.linkrev(i), pr[0], pr[1], short(node)))
def debugindexdot(orig, ui, repo, file_): def debugindexdot(orig, ui, repo, file_):
"""dump an index DAG as a graphviz dot file""" """dump an index DAG as a graphviz dot file"""
if not constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if not shallowutil.isenabled(repo):
return orig(ui, repo, file_) return orig(ui, repo, file_)
r = buildtemprevlog(repo, os.path.basename(file_)[:-2]) r = buildtemprevlog(repo, os.path.basename(file_)[:-2])
ui.write(("digraph G {\n")) ui.write(("digraph G {\n"))
for i in r: for i in r:
node = r.node(i) node = r.node(i)
pp = r.parents(node) pp = r.parents(node)

hgext/remotefilelog/remotefilelogserver.py

state.noflatmf = oldnoflatmf state.noflatmf = oldnoflatmf
wireprotov1server.commands['stream_out_shallow'] = (stream_out_shallow, '*') wireprotov1server.commands['stream_out_shallow'] = (stream_out_shallow, '*')
# don't clone filelogs to shallow clients # don't clone filelogs to shallow clients
def _walkstreamfiles(orig, repo): def _walkstreamfiles(orig, repo):
if state.shallowremote: if state.shallowremote:
# if we are shallow ourselves, stream our local commits # if we are shallow ourselves, stream our local commits
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if shallowutil.isenabled(repo):
striplen = len(repo.store.path) + 1 striplen = len(repo.store.path) + 1
readdir = repo.store.rawvfs.readdir readdir = repo.store.rawvfs.readdir
visit = [os.path.join(repo.store.path, 'data')] visit = [os.path.join(repo.store.path, 'data')]
while visit: while visit:
p = visit.pop() p = visit.pop()
for f, kind, st in readdir(p, stat=True): for f, kind, st in readdir(p, stat=True):
fp = p + '/' + f fp = p + '/' + f
if kind == stat.S_IFREG: if kind == stat.S_IFREG:
if not state.match(f): if not state.match(f):
yield (u, e, s) yield (u, e, s)
for x in repo.store.topfiles(): for x in repo.store.topfiles():
if state.noflatmf and x[0][:11] == '00manifest.': if state.noflatmf and x[0][:11] == '00manifest.':
continue continue
yield x yield x
elif constants.SHALLOWREPO_REQUIREMENT in repo.requirements: elif shallowutil.isenabled(repo):
# don't allow cloning from a shallow repo to a full repo # don't allow cloning from a shallow repo to a full repo
# since it would require fetching every version of every # since it would require fetching every version of every
# file in order to create the revlogs. # file in order to create the revlogs.
raise error.Abort(_("Cannot clone from a shallow repo " raise error.Abort(_("Cannot clone from a shallow repo "
"to a full repo.")) "to a full repo."))
else: else:
for x in orig(repo): for x in orig(repo):
yield x yield x
return wireprotov1server.commands["getbundle"][0](repo, proto, others) return wireprotov1server.commands["getbundle"][0](repo, proto, others)
wireprotov1server.commands["getbundle_shallow"] = (getbundleshallow, '*') wireprotov1server.commands["getbundle_shallow"] = (getbundleshallow, '*')
# expose remotefilelog capabilities # expose remotefilelog capabilities
def _capabilities(orig, repo, proto): def _capabilities(orig, repo, proto):
caps = orig(repo, proto) caps = orig(repo, proto)
if ((constants.SHALLOWREPO_REQUIREMENT in repo.requirements or if (shallowutil.isenabled(repo) or ui.configbool('remotefilelog',
ui.configbool('remotefilelog', 'server'))): 'server')):
if isinstance(proto, _sshv1server): if isinstance(proto, _sshv1server):
# legacy getfiles method which only works over ssh # legacy getfiles method which only works over ssh
caps.append(constants.NETWORK_CAP_LEGACY_SSH_GETFILES) caps.append(constants.NETWORK_CAP_LEGACY_SSH_GETFILES)
caps.append('x_rfl_getflogheads') caps.append('x_rfl_getflogheads')
caps.append('x_rfl_getfile') caps.append('x_rfl_getfile')
return caps return caps
extensions.wrapfunction(wireprotov1server, '_capabilities', _capabilities) extensions.wrapfunction(wireprotov1server, '_capabilities', _capabilities)
"""A server api for requesting a particular version of a file. Can be used """A server api for requesting a particular version of a file. Can be used
in batches to request many files at once. The return protocol is: in batches to request many files at once. The return protocol is:
<errorcode>\0<data/errormsg> where <errorcode> is 0 for success or <errorcode>\0<data/errormsg> where <errorcode> is 0 for success or
non-zero for an error. non-zero for an error.
data is a compressed blob with revlog flag and ancestors information. See data is a compressed blob with revlog flag and ancestors information. See
createfileblob for its content. createfileblob for its content.
""" """
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if shallowutil.isenabled(repo):
return '1\0' + _('cannot fetch remote files from shallow repo') return '1\0' + _('cannot fetch remote files from shallow repo')
cachepath = repo.ui.config("remotefilelog", "servercachepath") cachepath = repo.ui.config("remotefilelog", "servercachepath")
if not cachepath: if not cachepath:
cachepath = os.path.join(repo.path, "remotefilelogcache") cachepath = os.path.join(repo.path, "remotefilelogcache")
node = bin(node.strip()) node = bin(node.strip())
if node == nullid: if node == nullid:
return '0\0' return '0\0'
return '0\0' + _loadfileblob(repo, cachepath, file, node) return '0\0' + _loadfileblob(repo, cachepath, file, node)
def getfiles(repo, proto): def getfiles(repo, proto):
"""A server api for requesting particular versions of particular files. """A server api for requesting particular versions of particular files.
""" """
if constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if shallowutil.isenabled(repo):
raise error.Abort(_('cannot fetch remote files from shallow repo')) raise error.Abort(_('cannot fetch remote files from shallow repo'))
if not isinstance(proto, _sshv1server): if not isinstance(proto, _sshv1server):
raise error.Abort(_('cannot fetch remote files over non-ssh protocol')) raise error.Abort(_('cannot fetch remote files over non-ssh protocol'))
def streamer(): def streamer():
fin = proto._fin fin = proto._fin
cachepath = repo.ui.config("remotefilelog", "servercachepath") cachepath = repo.ui.config("remotefilelog", "servercachepath")

hgext/remotefilelog/shallowbundle.py

linknode = lookup(curr) linknode = lookup(curr)
for c in self.nodechunk(rlog, curr, prev, linknode): for c in self.nodechunk(rlog, curr, prev, linknode):
yield c yield c
yield self.close() yield self.close()
class shallowcg1packer(changegroup.cgpacker): class shallowcg1packer(changegroup.cgpacker):
def generate(self, commonrevs, clnodes, fastpathlinkrev, source): def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
if constants.SHALLOWREPO_REQUIREMENT in self._repo.requirements: if shallowutil.isenabled(self._repo):
fastpathlinkrev = False fastpathlinkrev = False
return super(shallowcg1packer, self).generate(commonrevs, clnodes, return super(shallowcg1packer, self).generate(commonrevs, clnodes,
fastpathlinkrev, source) fastpathlinkrev, source)
def group(self, nodelist, rlog, lookup, units=None, reorder=None): def group(self, nodelist, rlog, lookup, units=None, reorder=None):
return shallowgroup(shallowcg1packer, self, nodelist, rlog, lookup, return shallowgroup(shallowcg1packer, self, nodelist, rlog, lookup,
units=units) units=units)
def generatefiles(self, changedfiles, *args): def generatefiles(self, changedfiles, *args):
try: try:
linknodes, commonrevs, source = args linknodes, commonrevs, source = args
except ValueError: except ValueError:
commonrevs, source, mfdicts, fastpathlinkrev, fnodes, clrevs = args commonrevs, source, mfdicts, fastpathlinkrev, fnodes, clrevs = args
if constants.SHALLOWREPO_REQUIREMENT in self._repo.requirements: if shallowutil.isenabled(self._repo):
repo = self._repo repo = self._repo
if isinstance(repo, bundlerepo.bundlerepository): if isinstance(repo, bundlerepo.bundlerepository):
# If the bundle contains filelogs, we can't pull from it, since # If the bundle contains filelogs, we can't pull from it, since
# bundlerepo is heavily tied to revlogs. Instead require that # bundlerepo is heavily tied to revlogs. Instead require that
# the user use unbundle instead. # the user use unbundle instead.
# Force load the filelog data. # Force load the filelog data.
bundlerepo.bundlerepository.file(repo, 'foo') bundlerepo.bundlerepository.file(repo, 'foo')
if repo._cgfilespos: if repo._cgfilespos:
raise error.Abort("cannot pull from full bundles", raise error.Abort("cannot pull from full bundles",
hint="use `hg unbundle` instead") hint="use `hg unbundle` instead")
return [] return []
filestosend = self.shouldaddfilegroups(source) filestosend = self.shouldaddfilegroups(source)
if filestosend == NoFiles: if filestosend == NoFiles:
changedfiles = list([f for f in changedfiles changedfiles = list([f for f in changedfiles
if not repo.shallowmatch(f)]) if not repo.shallowmatch(f)])
return super(shallowcg1packer, self).generatefiles( return super(shallowcg1packer, self).generatefiles(
changedfiles, *args) changedfiles, *args)
def shouldaddfilegroups(self, source): def shouldaddfilegroups(self, source):
repo = self._repo repo = self._repo
if not constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if not shallowutil.isenabled(repo):
return AllFiles return AllFiles
if source == "push" or source == "bundle": if source == "push" or source == "bundle":
return AllFiles return AllFiles
caps = self._bundlecaps or [] caps = self._bundlecaps or []
if source == "serve" or source == "pull": if source == "serve" or source == "pull":
if constants.BUNDLE2_CAPABLITY in caps: if constants.BUNDLE2_CAPABLITY in caps:
meta = self.builddeltaheader(node, p1, p2, prevnode, linknode, flags) meta = self.builddeltaheader(node, p1, p2, prevnode, linknode, flags)
meta += prefix meta += prefix
l = len(meta) + len(delta) l = len(meta) + len(delta)
yield changegroup.chunkheader(l) yield changegroup.chunkheader(l)
yield meta yield meta
yield delta yield delta
def makechangegroup(orig, repo, outgoing, version, source, *args, **kwargs): def makechangegroup(orig, repo, outgoing, version, source, *args, **kwargs):
if not constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if not shallowutil.isenabled(repo):
return orig(repo, outgoing, version, source, *args, **kwargs) return orig(repo, outgoing, version, source, *args, **kwargs)
original = repo.shallowmatch original = repo.shallowmatch
try: try:
# if serving, only send files the clients has patterns for # if serving, only send files the clients has patterns for
if source == 'serve': if source == 'serve':
bundlecaps = kwargs.get('bundlecaps') bundlecaps = kwargs.get('bundlecaps')
includepattern = None includepattern = None
includepattern, excludepattern) includepattern, excludepattern)
else: else:
repo.shallowmatch = match.always(repo.root, '') repo.shallowmatch = match.always(repo.root, '')
return orig(repo, outgoing, version, source, *args, **kwargs) return orig(repo, outgoing, version, source, *args, **kwargs)
finally: finally:
repo.shallowmatch = original repo.shallowmatch = original
def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args): def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args):
if not constants.SHALLOWREPO_REQUIREMENT in repo.requirements: if not shallowutil.isenabled(repo):
return orig(repo, source, revmap, trp, expectedfiles, *args) return orig(repo, source, revmap, trp, expectedfiles, *args)
files = 0 files = 0
newfiles = 0 newfiles = 0
visited = set() visited = set()
revisiondatas = {} revisiondatas = {}
queue = [] queue = []

hgext/remotefilelog/shallowutil.py

storageutil, storageutil,
stringutil, stringutil,
) )
from . import constants from . import constants
if not pycompat.iswindows: if not pycompat.iswindows:
import grp import grp
def isenabled(repo):
"""returns whether the repository is remotefilelog enabled or not"""
return constants.SHALLOWREPO_REQUIREMENT in repo.requirements
def getcachekey(reponame, file, id): def getcachekey(reponame, file, id):
pathhash = hashlib.sha1(file).hexdigest() pathhash = hashlib.sha1(file).hexdigest()
return os.path.join(reponame, pathhash[:2], pathhash[2:], id) return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
def getlocalkey(file, id): def getlocalkey(file, id):
pathhash = hashlib.sha1(file).hexdigest() pathhash = hashlib.sha1(file).hexdigest()
return os.path.join(pathhash, id) return os.path.join(pathhash, id)