This is an archive of the discontinued Mercurial Phabricator instance.

Differential D11265

debugrebuildfncache: add a cheaper option to rebuild the fncache
ClosedPublic

Authored by valentin.gatienbaron on Aug 6 2021, 4:44 PM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG0fb328bb2459: debugrebuildfncache: add a cheaper option to rebuild the fncache
rHG8e4659b5cf41: debugrebuildfncache: add a cheaper option to rebuild the fncache
Summary

On my repository, debugrebuildfncache takes 5-10min with the lock.

With the flag added in this commit, it takes 10s. The tradeoff is that
it only recovers from certain kinds of corruptions. It is intended to
to recover faster from fncaches broken by a revlog split during a
transaction that ends up being rolled back.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

Diff 30002

mercurial/debugcommands.py

manifestonly = manifestfiles - dirstatefiles manifestonly = manifestfiles - dirstatefiles
dsonly = dirstatefiles - manifestfiles dsonly = dirstatefiles - manifestfiles
dsnotadded = {f for f in dsonly if dirstate[f] != b'a'} dsnotadded = {f for f in dsonly if dirstate[f] != b'a'}
changedfiles = manifestonly | dsnotadded changedfiles = manifestonly | dsnotadded
dirstate.rebuild(ctx.node(), ctx.manifest(), changedfiles) dirstate.rebuild(ctx.node(), ctx.manifest(), changedfiles)
@command(b'debugrebuildfncache', [], b'') @command(
def debugrebuildfncache(ui, repo): b'debugrebuildfncache',
[
(
b'',
b'only-data',
False,
_(b'only look for wrong .d files (much faster)'),
)
],
b'',
)
def debugrebuildfncache(ui, repo, **opts):
"""rebuild the fncache file""" """rebuild the fncache file"""
repair.rebuildfncache(ui, repo) opts = pycompat.byteskwargs(opts)
repair.rebuildfncache(ui, repo, opts.get(b"only_data"))
@command( @command(
b'debugrename', b'debugrename',
[(b'r', b'rev', b'', _(b'revision to debug'), _(b'REV'))], [(b'r', b'rev', b'', _(b'revision to debug'), _(b'REV'))],
_(b'[-r REV] [FILE]...'), _(b'[-r REV] [FILE]...'),
) )
def debugrename(ui, repo, *pats, **opts): def debugrename(ui, repo, *pats, **opts):

mercurial/repair.py

for t, unencoded, encoded, size in repo.store.datafiles(): for t, unencoded, encoded, size in repo.store.datafiles():
if unencoded.startswith(b'meta/') and unencoded.endswith( if unencoded.startswith(b'meta/') and unencoded.endswith(
b'00manifest.i' b'00manifest.i'
): ):
dir = unencoded[5:-12] dir = unencoded[5:-12]
yield repo.manifestlog.getstorage(dir) yield repo.manifestlog.getstorage(dir)
def rebuildfncache(ui, repo): def rebuildfncache(ui, repo, only_data=False):
"""Rebuilds the fncache file from repo history. """Rebuilds the fncache file from repo history.
Missing entries will be added. Extra entries will be removed. Missing entries will be added. Extra entries will be removed.
""" """
repo = repo.unfiltered() repo = repo.unfiltered()
if requirements.FNCACHE_REQUIREMENT not in repo.requirements: if requirements.FNCACHE_REQUIREMENT not in repo.requirements:
ui.warn( ui.warn(
_( _(
b'(not rebuilding fncache because repository does not ' b'(not rebuilding fncache because repository does not '
b'support fncache)\n' b'support fncache)\n'
) )
) )
return return
with repo.lock(): with repo.lock():
fnc = repo.store.fncache fnc = repo.store.fncache
fnc.ensureloaded(warn=ui.warn) fnc.ensureloaded(warn=ui.warn)
oldentries = set(fnc.entries) oldentries = set(fnc.entries)
newentries = set() newentries = set()
seenfiles = set() seenfiles = set()
if only_data:
# Trust the listing of .i from the fncache, but not the .d. This is
# much faster, because we only need to stat every possible .d files,
# instead of reading the full changelog
for f in fnc:
if f[:5] == b'data/' and f[-2:] == b'.i':
seenfiles.add(f[5:-2])
newentries.add(f)
dataf = f[:-2] + b'.d'
if repo.store._exists(dataf):
newentries.add(dataf)
else:
progress = ui.makeprogress( progress = ui.makeprogress(
_(b'rebuilding'), unit=_(b'changesets'), total=len(repo) _(b'rebuilding'), unit=_(b'changesets'), total=len(repo)
) )
for rev in repo: for rev in repo:
progress.update(rev) progress.update(rev)
ctx = repo[rev] ctx = repo[rev]
for f in ctx.files(): for f in ctx.files():
# This is to minimize I/O. # This is to minimize I/O.
if f in seenfiles: if f in seenfiles:
continue continue
seenfiles.add(f) seenfiles.add(f)
i = b'data/%s.i' % f i = b'data/%s.i' % f
d = b'data/%s.d' % f d = b'data/%s.d' % f
if repo.store._exists(i): if repo.store._exists(i):
newentries.add(i) newentries.add(i)
if repo.store._exists(d): if repo.store._exists(d):
newentries.add(d) newentries.add(d)
progress.complete() progress.complete()
if requirements.TREEMANIFEST_REQUIREMENT in repo.requirements: if requirements.TREEMANIFEST_REQUIREMENT in repo.requirements:
# This logic is safe if treemanifest isn't enabled, but also # This logic is safe if treemanifest isn't enabled, but also
# pointless, so we skip it if treemanifest isn't enabled. # pointless, so we skip it if treemanifest isn't enabled.
for dir in pathutil.dirs(seenfiles): for dir in pathutil.dirs(seenfiles):
i = b'meta/%s/00manifest.i' % dir i = b'meta/%s/00manifest.i' % dir
d = b'meta/%s/00manifest.d' % dir d = b'meta/%s/00manifest.d' % dir

tests/test-completion.t

debugp2copies: rev debugp2copies: rev
debugpathcomplete: full, normal, added, removed debugpathcomplete: full, normal, added, removed
debugpathcopies: include, exclude debugpathcopies: include, exclude
debugpeer: debugpeer:
debugpickmergetool: rev, changedelete, include, exclude, tool debugpickmergetool: rev, changedelete, include, exclude, tool
debugpushkey: debugpushkey:
debugpvec: debugpvec:
debugrebuilddirstate: rev, minimal debugrebuilddirstate: rev, minimal
debugrebuildfncache: debugrebuildfncache: only-data
debugrename: rev debugrename: rev
debugrequires: debugrequires:
debugrevlog: changelog, manifest, dir, dump debugrevlog: changelog, manifest, dir, dump
debugrevlogindex: changelog, manifest, dir, format debugrevlogindex: changelog, manifest, dir, format
debugrevspec: optimize, show-revs, show-set, show-stage, no-optimized, verify-optimized debugrevspec: optimize, show-revs, show-set, show-stage, no-optimized, verify-optimized
debugserve: sshstdio, logiofd, logiofile debugserve: sshstdio, logiofd, logiofile
debugsetparents: debugsetparents:
debugshell: debugshell:

tests/test-transaction-rollback-on-revlog-split.t

user: test user: test
date: Thu Jan 01 00:00:00 1970 +0000 date: Thu Jan 01 00:00:00 1970 +0000
summary: _ summary: _
$ hg verify -q $ hg verify -q
warning: revlog 'data/file.d' not in fncache! warning: revlog 'data/file.d' not in fncache!
1 warnings encountered! 1 warnings encountered!
hint: run "hg debugrebuildfncache" to recover from corrupt fncache hint: run "hg debugrebuildfncache" to recover from corrupt fncache
$ hg debugrebuildfncache --only-data
adding data/file.d
1 items added, 0 removed from fncache
$ hg verify -q
$ cd .. $ cd ..
Now retry the procedure but intercept the rename of the index and check that Now retry the procedure but intercept the rename of the index and check that
the journal does not contain the new index size. This demonstrates the edge case the journal does not contain the new index size. This demonstrates the edge case
where the data file is left as garbage. where the data file is left as garbage.
$ hg clone -r 1 troffset-computation troffset-computation-copy2 --config format.revlog-compression=none -q $ hg clone -r 1 troffset-computation troffset-computation-copy2 --config format.revlog-compression=none -q