( )⚙ D6722 fncache: make debugrebuildfncache not fail on broken fncache

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6722

fncache: make debugrebuildfncache not fail on broken fncache
ClosedPublic

Authored by valentin.gatienbaron on Aug 12 2019, 2:13 PM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHGf59f8a5e9096: fncache: make debugrebuildfncache not fail on broken fncache
Summary

The code reading the fncache changed in 5.0, to complain if the file
is not \n terminated. This makes apparent the fact that the fncache
gets corrupted.

Make it possible to recover, instead of having `hg
debugrebuildfncache` failing by saying (run hg debugrebuildfncache).

The corruption itself is most likely due to hg not using fsync in
general, and so various bad things can happen. Here, the reported
problems happened when running out of disk space. So I suspect that
because the fncache is much bigger than the average commit/pull, when
running out of disk space, the bulk of the pull may succeed, but the
new fncache may get half-written and still renamed into place.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

pulkit accepted this revision.Aug 12 2019, 4:45 PM
pulkit added a subscriber: pulkit.

Queued these for stable branch. Many thanks!

This revision is now accepted and ready to land.Aug 12 2019, 4:45 PM

Revision Contents
Changeset List

PathPackages
Mmercurial/repair.py (4 lines)
Mmercurial/store.py (29 lines)
Mtests/test-fncache.t (12 lines)

Diff 16177

mercurial/repair.py

if 'fncache' not in repo.requirements: if 'fncache' not in repo.requirements:
ui.warn(_('(not rebuilding fncache because repository does not ' ui.warn(_('(not rebuilding fncache because repository does not '
'support fncache)\n')) 'support fncache)\n'))
return return
with repo.lock(): with repo.lock():
fnc = repo.store.fncache fnc = repo.store.fncache
# Trigger load of fncache. fnc.ensureloaded(warn=ui.warn)
if 'irrelevant' in fnc:
pass
oldentries = set(fnc.entries) oldentries = set(fnc.entries)
newentries = set() newentries = set()
seenfiles = set() seenfiles = set()
progress = ui.makeprogress(_('rebuilding'), unit=_('changesets'), progress = ui.makeprogress(_('rebuilding'), unit=_('changesets'),
total=len(repo)) total=len(repo))
for rev in repo: for rev in repo:

mercurial/store.py

# hence the encodedir/decodedir dance # hence the encodedir/decodedir dance
def __init__(self, vfs): def __init__(self, vfs):
self.vfs = vfs self.vfs = vfs
self.entries = None self.entries = None
self._dirty = False self._dirty = False
# set of new additions to fncache # set of new additions to fncache
self.addls = set() self.addls = set()
def _load(self): def ensureloaded(self, warn=None):
'''read the fncache file if not already read.
If the file on disk is corrupted, raise. If warn is provided,
warn and keep going instead.'''
if self.entries is None:
self._load(warn)
def _load(self, warn=None):
'''fill the entries from the fncache file''' '''fill the entries from the fncache file'''
self._dirty = False self._dirty = False
try: try:
fp = self.vfs('fncache', mode='rb') fp = self.vfs('fncache', mode='rb')
except IOError: except IOError:
# skip nonexistent file # skip nonexistent file
self.entries = set() self.entries = set()
return return
self.entries = set() self.entries = set()
chunk = b'' chunk = b''
for c in iter(functools.partial(fp.read, fncache_chunksize), b''): for c in iter(functools.partial(fp.read, fncache_chunksize), b''):
chunk += c chunk += c
try: try:
p = chunk.rindex(b'\n') p = chunk.rindex(b'\n')
self.entries.update(decodedir(chunk[:p + 1]).splitlines()) self.entries.update(decodedir(chunk[:p + 1]).splitlines())
chunk = chunk[p + 1:] chunk = chunk[p + 1:]
except ValueError: except ValueError:
# substring '\n' not found, maybe the entry is bigger than the # substring '\n' not found, maybe the entry is bigger than the
# chunksize, so let's keep iterating # chunksize, so let's keep iterating
pass pass
if chunk: if chunk:
raise error.Abort(_("fncache does not ends with a newline"), msg = _("fncache does not ends with a newline")
hint=_("use 'hg debugrebuildfncache' to rebuild" if warn:
" the fncache")) warn(msg + '\n')
self._checkentries(fp) else:
raise error.Abort(msg,
hint=_("use 'hg debugrebuildfncache' to "
"rebuild the fncache"))
self._checkentries(fp, warn)
fp.close() fp.close()
def _checkentries(self, fp): def _checkentries(self, fp, warn):
""" make sure there is no empty string in entries """ """ make sure there is no empty string in entries """
if '' in self.entries: if '' in self.entries:
fp.seek(0) fp.seek(0)
for n, line in enumerate(util.iterfile(fp)): for n, line in enumerate(util.iterfile(fp)):
if not line.rstrip('\n'): if not line.rstrip('\n'):
t = _('invalid entry in fncache, line %d') % (n + 1) t = _('invalid entry in fncache, line %d') % (n + 1)
if warn:
warn(t + '\n')
else:
raise error.Abort(t) raise error.Abort(t)
def write(self, tr): def write(self, tr):
if self._dirty: if self._dirty:
assert self.entries is not None assert self.entries is not None
self.entries = self.entries | self.addls self.entries = self.entries | self.addls
self.addls = set() self.addls = set()
tr.addbackup('fncache') tr.addbackup('fncache')
fp = self.vfs('fncache', mode='wb', atomictemp=True) fp = self.vfs('fncache', mode='wb', atomictemp=True)

tests/test-fncache.t

removing data/bad-entry.i removing data/bad-entry.i
adding data/.bar.i adding data/.bar.i
1 items added, 1 removed from fncache 1 items added, 1 removed from fncache
$ cat .hg/store/fncache | sort $ cat .hg/store/fncache | sort
data/.bar.i data/.bar.i
data/foo.i data/foo.i
debugrebuildfncache fails to recover from truncated line in fncache debugrebuildfncache recovers from truncated line in fncache
$ printf a > .hg/store/fncache $ printf a > .hg/store/fncache
$ hg debugrebuildfncache $ hg debugrebuildfncache
abort: fncache does not ends with a newline fncache does not ends with a newline
(use 'hg debugrebuildfncache' to rebuild the fncache) adding data/.bar.i
[255] adding data/foo.i
2 items added, 0 removed from fncache
$ cat .hg/store/fncache | sort $ cat .hg/store/fncache | sort
a data/.bar.i
data/foo.i
$ cd .. $ cd ..
Try a simple variation without dotencode to ensure fncache is ignorant of encoding Try a simple variation without dotencode to ensure fncache is ignorant of encoding
$ hg --config format.dotencode=false init nodotencode $ hg --config format.dotencode=false init nodotencode
$ cd nodotencode $ cd nodotencode
$ echo initial > foo $ echo initial > foo