This is an archive of the discontinued Mercurial Phabricator instance.

Differential D442

remotefilelog: efficient keepset computation
ClosedPublic

Authored by ms2316 on Aug 18 2017, 6:53 AM.
Tags
None
Subscribers
None

Details

Reviewers
ryanmce
simonfar
durham
Group Reviewers
Restricted Project
Commits
rFBHGX6dd1f0c039cb: remotefilelog: efficient keepset computation
Summary

New keepset computation algorithm yields 50% time improvement on a small set of
commits and about 90% time improvement on a large set of commits. Currently it
takes about 1m10s to compute a keepset of 10 commits with treemanifest
usecunionstore flag unset and 3 seconds with cunionstore config set, provided
all trees are prefetched.

Test Plan
  • Ensure that unit tests pass, in particular *-gc.t and *-gcrepack.t

Diff Detail

Repository
rFBHGX Facebook Mercurial Extensions
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

ms2316 created this revision.Aug 18 2017, 6:53 AM
Herald added a reviewer: Restricted Project. · View Herald TranscriptAug 18 2017, 6:53 AM
ms2316 added inline comments.Aug 18 2017, 7:03 AM
remotefilelog/repack.py
296

We can make keyfn not optional, but instead make user always specify the function.

ryanmce accepted this revision.Aug 21 2017, 11:45 AM
ryanmce added inline comments.
remotefilelog/repack.py
296

I'd somewhat prefer that myself.

This revision is now accepted and ready to land.Aug 21 2017, 11:45 AM
ms2316 updated this revision to Diff 1126.Aug 21 2017, 12:06 PM

Made keyfn function compulsory

Closed by commit rFBHGX6dd1f0c039cb: remotefilelog: efficient keepset computation (authored by ms2316). · Explain WhyAug 21 2017, 8:17 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mremotefilelog/__init__.py (35 lines)
Mremotefilelog/repack.py (79 lines)

Diff 1138

remotefilelog/__init__.py

# continue doing garbage collection of loose files w/o repack # continue doing garbage collection of loose files w/o repack
pass pass
reponame = repo.name reponame = repo.name
if not sharedcache: if not sharedcache:
sharedcache = repo.sharedstore sharedcache = repo.sharedstore
# Compute a keepset which is not garbage collected # Compute a keepset which is not garbage collected
# We want to keep: def keyfn(fname, fnode):
# 1. Working copy parent return fileserverclient.getcachekey(reponame, fname, hex(fnode))
# 2. Draft commits keepkeys = repackmod.keepset(repo, keyfn=keyfn, lastkeepkeys=keepkeys)
# 3. All parents of draft commits
# 4. Recent heads in the repo
# 5. Pullprefetch and bgprefetchrevs revsets if specified
revs = ['.', 'draft()', 'parents(draft())', '(heads(all()) & date(-7))']
prefetchrevs = repo.ui.config('remotefilelog',
'pullprefetch', None)
if prefetchrevs:
revs.append('(%s)' % prefetchrevs)
prefetchrevs = repo.ui.config('remotefilelog',
'bgprefetchrevs', None)
if prefetchrevs:
revs.append('(%s)' % prefetchrevs)
keep = scmutil.revrange(repo, ['+'.join(revs)])
# TODO: Compute keepkeys more efficiently
for r in keep:
m = repo[r].manifest()
for filename, filenode in m.iteritems():
key = fileserverclient.getcachekey(reponame, filename,
hex(filenode))
keepkeys.add(key)
ui.progress(_analyzing, None) ui.progress(_analyzing, None)
# write list of valid repos back # write list of valid repos back
oldumask = os.umask(0o002) oldumask = os.umask(0o002)
try: try:
reposfile = open(repospath, 'w') reposfile = open(repospath, 'w')
reposfile.writelines([("%s\n" % r) for r in validrepos]) reposfile.writelines([("%s\n" % r) for r in validrepos])
def debugdatapack(ui, path, **opts): def debugdatapack(ui, path, **opts):
return debugcommands.debugdatapack(ui, path, **opts) return debugcommands.debugdatapack(ui, path, **opts)
@command('debughistorypack', [ @command('debughistorypack', [
], _('hg debughistorypack <path>'), norepo=True) ], _('hg debughistorypack <path>'), norepo=True)
def debughistorypack(ui, path, **opts): def debughistorypack(ui, path, **opts):
return debugcommands.debughistorypack(ui, path) return debugcommands.debughistorypack(ui, path)
@command('debugkeepset', [
], _('hg debugkeepset'))
def debugkeepset(ui, repo, **opts):
# The command is used to measure keepset computation time
repackmod.keepset(repo)
return
@command('debugwaitonrepack', [ @command('debugwaitonrepack', [
], _('hg debugwaitonrepack')) ], _('hg debugwaitonrepack'))
def debugwaitonrepack(ui, repo, **opts): def debugwaitonrepack(ui, repo, **opts):
return debugcommands.debugwaitonrepack(repo) return debugcommands.debugwaitonrepack(repo)
@command('debugwaitonprefetch', [ @command('debugwaitonprefetch', [
], _('hg debugwaitonprefetch')) ], _('hg debugwaitonprefetch'))
def debugwaitonprefetch(ui, repo, **opts): def debugwaitonprefetch(ui, repo, **opts):

remotefilelog/repack.py

with datapack.mutabledatapack(repo.ui, packpath, version=dv) as dpack: with datapack.mutabledatapack(repo.ui, packpath, version=dv) as dpack:
with historypack.mutablehistorypack(repo.ui, packpath) as hpack: with historypack.mutablehistorypack(repo.ui, packpath) as hpack:
try: try:
packer.run(dpack, hpack) packer.run(dpack, hpack)
except error.LockHeld: except error.LockHeld:
raise error.Abort(_("skipping repack - another repack is " raise error.Abort(_("skipping repack - another repack is "
"already running")) "already running"))
def keepset(repo, keyfn, lastkeepkeys=None):
"""Computes a keepset which is not garbage collected.
'keyfn' is a function that maps filename, node to a unique key.
'lastkeepkeys' is an optional argument and if provided the keepset
ms2316AuthorUnsubmitted
Not Done

We can make keyfn not optional, but instead make user always specify the function.

ms2316: We can make keyfn not optional, but instead make user always specify the function.
ryanmceUnsubmitted
Not Done

I'd somewhat prefer that myself.

ryanmce: I'd somewhat prefer that myself.
function updates lastkeepkeys with more keys and returns the result.
"""
if not lastkeepkeys:
keepkeys = set()
else:
keepkeys = lastkeepkeys
# We want to keep:
# 1. Working copy parent
# 2. Draft commits
# 3. Parents of draft commits
# 4. Pullprefetch and bgprefetchrevs revsets if specified
revs = ['.', 'draft()', 'parents(draft())']
prefetchrevs = repo.ui.config('remotefilelog', 'pullprefetch', None)
if prefetchrevs:
revs.append('(%s)' % prefetchrevs)
prefetchrevs = repo.ui.config('remotefilelog', 'bgprefetchrevs', None)
if prefetchrevs:
revs.append('(%s)' % prefetchrevs)
revs = '+'.join(revs)
revs = ['sort((%s), "topo")' % revs]
keep = scmutil.revrange(repo, revs)
processed = set()
lastmanifest = None
# process the commits in toposorted order starting from the oldest
for r in reversed(keep._list):
if repo[r].p1().rev() in processed:
# if the direct parent has already been processed
# then we only need to process the delta
m = repo[r].manifestctx().readdelta()
else:
# otherwise take the manifest and diff it
# with the previous manifest if one exists
if lastmanifest:
m = repo[r].manifest().diff(lastmanifest)
else:
m = repo[r].manifest()
lastmanifest = repo[r].manifest()
processed.add(r)
# populate keepkeys with keys from the current manifest
for filename, filenode in m.iteritems():
keepkeys.add(keyfn(filename, filenode))
return keepkeys
class repacker(object): class repacker(object):
"""Class for orchestrating the repack of data and history information into a """Class for orchestrating the repack of data and history information into a
new format. new format.
""" """
def __init__(self, repo, data, history, category, gc=False, isold=None): def __init__(self, repo, data, history, category, gc=False, isold=None):
self.repo = repo self.repo = repo
self.data = data self.data = data
self.history = history self.history = history
self.unit = constants.getunits(category) self.unit = constants.getunits(category)
self.garbagecollect = gc self.garbagecollect = gc
if self.garbagecollect: if self.garbagecollect:
if not isold: if not isold:
raise ValueError("Function 'isold' is not properly specified") raise ValueError("Function 'isold' is not properly specified")
self.keepkeys = self._gckeepset() # use (filename, node) tuple as a keepset key
self.keepkeys = keepset(repo, lambda f, n : (f, n))
self.isold = isold self.isold = isold
def _gckeepset(self):
"""Computes a keepset which is not garbage collected.
"""
repo = self.repo
revs = ['.', 'draft()', 'parents(draft())', '(heads(all()) & date(-7))']
# If pullprefetch and bgprefetchrevs are specified include them as well
# since we don't want to prefetch and immediately garbage collect them
prefetchrevs = repo.ui.config('remotefilelog', 'pullprefetch', None)
if prefetchrevs:
revs.append('(%s)' % prefetchrevs)
prefetchrevs = repo.ui.config('remotefilelog', 'bgprefetchrevs', None)
if prefetchrevs:
revs.append('(%s)' % prefetchrevs)
keep = scmutil.revrange(repo, ['+'.join(revs)])
keepkeys = set()
for r in keep:
m = repo[r].manifest()
keepkeys.update(m.iteritems())
return keepkeys
def run(self, targetdata, targethistory): def run(self, targetdata, targethistory):
ledger = repackledger() ledger = repackledger()
with self.repo._lock(self.repo.svfs, "repacklock", False, None, with self.repo._lock(self.repo.svfs, "repacklock", False, None,
None, _('repacking %s') % self.repo.origroot): None, _('repacking %s') % self.repo.origroot):
self.repo.hook('prerepack') self.repo.hook('prerepack')
# Populate ledger from source # Populate ledger from source