This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7888

nodemap: track the maximum revision tracked in the nodemap
ClosedPublic

Authored by marmoute on Jan 15 2020, 9:55 AM.

Details

Reviewers
indygreg
Group Reviewers
hg-reviewers
Commits
rHGe41a164db7a9: nodemap: track the maximum revision tracked in the nodemap
Summary

We need a simple way to detect when the on disk data contains less revision
than the index we read from disk. The docket file is meant for this, we just had
to start tracking that data.

We should also try to detect strip operation, but we will deal with this in
later changesets. Right now we are focusing on defining the API for index
supporting persistent nodemap.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.Jan 15 2020, 9:55 AM
marmoute updated this revision to Diff 19433.Jan 17 2020, 1:07 PM
marmoute updated this revision to Diff 19769.Jan 31 2020, 10:29 AM
marmoute updated this revision to Diff 19796.Jan 31 2020, 6:00 PM
marmoute updated this revision to Diff 19840.Feb 2 2020, 3:19 AM

small doc update on .#s[1]

marmoute updated this revision to Diff 19900.Feb 4 2020, 7:25 PM

rebase to latest default

marmoute added a commit: rHGe41a164db7a9: nodemap: track the maximum revision tracked in the nodemap.Feb 10 2020, 8:26 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHGe41a164db7a9: nodemap: track the maximum revision tracked in the nodemap (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 20121

mercurial/debugcommands.py

return nodemap.check_data(ui, cl.index, data) return nodemap.check_data(ui, cl.index, data)
elif opts['metadata']: elif opts['metadata']:
unfi = repo.unfiltered() unfi = repo.unfiltered()
cl = unfi.changelog cl = unfi.changelog
nm_data = nodemap.persisted_data(cl) nm_data = nodemap.persisted_data(cl)
if nm_data is not None: if nm_data is not None:
docket, data = nm_data docket, data = nm_data
ui.write((b"uid: %s\n") % docket.uid) ui.write((b"uid: %s\n") % docket.uid)
ui.write((b"tip-rev: %d\n") % docket.tip_rev)
@command( @command(
b'debugobsolete', b'debugobsolete',
[ [
(b'', b'flags', 0, _(b'markers flag')), (b'', b'flags', 0, _(b'markers flag')),
( (
b'', b'',

mercurial/pure/parsers.py

if self._nm_root is None: if self._nm_root is None:
return None return None
data = nodemaputil.update_persistent_data( data = nodemaputil.update_persistent_data(
self, self._nm_root, self._nm_max_idx, self._nm_rev self, self._nm_root, self._nm_max_idx, self._nm_rev
) )
self._nm_root = self._nm_max_idx = self._nm_rev = None self._nm_root = self._nm_max_idx = self._nm_rev = None
return data return data
def update_nodemap_data(self, nm_data): def update_nodemap_data(self, docket, nm_data):
"""provide full blokc of persisted binary data for a nodemap """provide full block of persisted binary data for a nodemap
The data are expected to come from disk. See `nodemap_data_all` for a The data are expected to come from disk. See `nodemap_data_all` for a
produceur of such data.""" produceur of such data."""
if nm_data is not None: if nm_data is not None:
self._nm_root, self._nm_max_idx = nodemaputil.parse_data(nm_data) self._nm_root, self._nm_max_idx = nodemaputil.parse_data(nm_data)
if self._nm_root: if self._nm_root:
self._nm_rev = len(self) - 1 self._nm_rev = docket.tip_rev
else: else:
self._nm_root = self._nm_max_idx = self._nm_rev = None self._nm_root = self._nm_max_idx = self._nm_rev = None
class InlinedIndexObject(BaseIndexObject): class InlinedIndexObject(BaseIndexObject):
def __init__(self, data, inline=0): def __init__(self, data, inline=0):
self._data = data self._data = data
self._lgt = self._inline_scan(None) self._lgt = self._inline_scan(None)

mercurial/revlog.py

use_nodemap = ( use_nodemap = (
not self._inline not self._inline
and self.nodemap_file is not None and self.nodemap_file is not None
and util.safehasattr(index, 'update_nodemap_data') and util.safehasattr(index, 'update_nodemap_data')
) )
if use_nodemap: if use_nodemap:
nodemap_data = nodemaputil.persisted_data(self) nodemap_data = nodemaputil.persisted_data(self)
if nodemap_data is not None: if nodemap_data is not None:
index.update_nodemap_data(nodemap_data[1]) index.update_nodemap_data(*nodemap_data)
except (ValueError, IndexError): except (ValueError, IndexError):
raise error.RevlogError( raise error.RevlogError(
_(b"index %s is corrupted") % self.indexfile _(b"index %s is corrupted") % self.indexfile
) )
self.index, self._chunkcache = d self.index, self._chunkcache = d
if not self._chunkcache: if not self._chunkcache:
self._chunkclear() self._chunkclear()
# revnum -> (chain-length, sum-delta-length) # revnum -> (chain-length, sum-delta-length)

mercurial/revlogutils/nodemap.py

pdata = revlog.opener.tryread(revlog.nodemap_file) pdata = revlog.opener.tryread(revlog.nodemap_file)
if not pdata: if not pdata:
return None return None
offset = 0 offset = 0
(version,) = S_VERSION.unpack(pdata[offset : offset + S_VERSION.size]) (version,) = S_VERSION.unpack(pdata[offset : offset + S_VERSION.size])
if version != ONDISK_VERSION: if version != ONDISK_VERSION:
return None return None
offset += S_VERSION.size offset += S_VERSION.size
(uid_size,) = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size]) headers = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size])
uid_size, tip_rev = headers
offset += S_HEADER.size offset += S_HEADER.size
docket = NodeMapDocket(pdata[offset : offset + uid_size]) docket = NodeMapDocket(pdata[offset : offset + uid_size])
docket.tip_rev = tip_rev
filename = _rawdata_filepath(revlog, docket) filename = _rawdata_filepath(revlog, docket)
return docket, revlog.opener.tryread(filename) return docket, revlog.opener.tryread(filename)
def setup_persistent_nodemap(tr, revlog): def setup_persistent_nodemap(tr, revlog):
"""Install whatever is needed transaction side to persist a nodemap on disk """Install whatever is needed transaction side to persist a nodemap on disk
if util.safehasattr(revlog.index, "nodemap_data_all"): if util.safehasattr(revlog.index, "nodemap_data_all"):
data = revlog.index.nodemap_data_all() data = revlog.index.nodemap_data_all()
else: else:
data = persistent_data(revlog.index) data = persistent_data(revlog.index)
# EXP-TODO: if this is a cache, this should use a cache vfs, not a # EXP-TODO: if this is a cache, this should use a cache vfs, not a
# store vfs # store vfs
with revlog.opener(datafile, b'w') as fd: with revlog.opener(datafile, b'w') as fd:
fd.write(data) fd.write(data)
target_docket.tip_rev = revlog.tiprev()
# EXP-TODO: if this is a cache, this should use a cache vfs, not a # EXP-TODO: if this is a cache, this should use a cache vfs, not a
# store vfs # store vfs
with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp: with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
fp.write(target_docket.serialize()) fp.write(target_docket.serialize())
revlog._nodemap_docket = target_docket revlog._nodemap_docket = target_docket
# EXP-TODO: if the transaction abort, we should remove the new data and # EXP-TODO: if the transaction abort, we should remove the new data and
# reinstall the old one. # reinstall the old one.
# The docket file contains information to find, qualify and validate the raw # The docket file contains information to find, qualify and validate the raw
# data. Its content is currently very light, but it will expand as the on disk # data. Its content is currently very light, but it will expand as the on disk
# nodemap gains the necessary features to be used in production. # nodemap gains the necessary features to be used in production.
# version 0 is experimental, no BC garantee, do no use outside of tests. # version 0 is experimental, no BC garantee, do no use outside of tests.
ONDISK_VERSION = 0 ONDISK_VERSION = 0
S_VERSION = struct.Struct(">B") S_VERSION = struct.Struct(">B")
S_HEADER = struct.Struct(">B") S_HEADER = struct.Struct(">BQ")
ID_SIZE = 8 ID_SIZE = 8
def _make_uid(): def _make_uid():
"""return a new unique identifier. """return a new unique identifier.
The identifier is random and composed of ascii characters.""" The identifier is random and composed of ascii characters."""
return nodemod.hex(os.urandom(ID_SIZE)) return nodemod.hex(os.urandom(ID_SIZE))
class NodeMapDocket(object): class NodeMapDocket(object):
"""metadata associated with persistent nodemap data """metadata associated with persistent nodemap data
The persistent data may come from disk or be on their way to disk. The persistent data may come from disk or be on their way to disk.
""" """
def __init__(self, uid=None): def __init__(self, uid=None):
if uid is None: if uid is None:
uid = _make_uid() uid = _make_uid()
self.uid = uid self.uid = uid
self.tip_rev = None
def copy(self): def copy(self):
return NodeMapDocket(uid=self.uid) new = NodeMapDocket(uid=self.uid)
new.tip_rev = self.tip_rev
return new
def serialize(self): def serialize(self):
"""return serialized bytes for a docket using the passed uid""" """return serialized bytes for a docket using the passed uid"""
data = [] data = []
data.append(S_VERSION.pack(ONDISK_VERSION)) data.append(S_VERSION.pack(ONDISK_VERSION))
data.append(S_HEADER.pack(len(self.uid))) headers = (len(self.uid), self.tip_rev)
data.append(S_HEADER.pack(*headers))
data.append(self.uid) data.append(self.uid)
return b''.join(data) return b''.join(data)
def _rawdata_filepath(revlog, docket): def _rawdata_filepath(revlog, docket):
"""The (vfs relative) nodemap's rawdata file for a given uid""" """The (vfs relative) nodemap's rawdata file for a given uid"""
prefix = revlog.nodemap_file[:-2] prefix = revlog.nodemap_file[:-2]
return b"%s-%s.nd" % (prefix, docket.uid) return b"%s-%s.nd" % (prefix, docket.uid)

tests/test-persistent-nodemap.t

=================================== ===================================
Test the persistent on-disk nodemap Test the persistent on-disk nodemap
=================================== ===================================
$ hg init test-repo $ hg init test-repo
$ cd test-repo $ cd test-repo
$ cat << EOF >> .hg/hgrc $ cat << EOF >> .hg/hgrc
> [experimental] > [experimental]
> exp-persistent-nodemap=yes > exp-persistent-nodemap=yes
> [devel] > [devel]
> persistent-nodemap=yes > persistent-nodemap=yes
> EOF > EOF
$ hg debugbuilddag .+5000 $ hg debugbuilddag .+5000
$ hg debugnodemap --metadata $ hg debugnodemap --metadata
uid: ???????????????? (glob) uid: ???????????????? (glob)
tip-rev: 5000
$ f --size .hg/store/00changelog.n $ f --size .hg/store/00changelog.n
.hg/store/00changelog.n: size=18 .hg/store/00changelog.n: size=26
$ f --sha256 .hg/store/00changelog-*.nd $ f --sha256 .hg/store/00changelog-*.nd
.hg/store/00changelog-????????????????.nd: sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7 (glob) .hg/store/00changelog-????????????????.nd: sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7 (glob)
$ hg debugnodemap --dump-new | f --sha256 --size $ hg debugnodemap --dump-new | f --sha256 --size
size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7 size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
$ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size $ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size
size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7 size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| 0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff |................| 0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff |................|
$ hg up $ hg up
0 files updated, 0 files merged, 0 files removed, 0 files unresolved 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ echo foo > foo $ echo foo > foo
$ hg add foo $ hg add foo
$ hg ci -m 'foo' $ hg ci -m 'foo'
$ hg debugnodemap --metadata $ hg debugnodemap --metadata
uid: ???????????????? (glob) uid: ???????????????? (glob)
tip-rev: 5001
$ f --size .hg/store/00changelog.n $ f --size .hg/store/00changelog.n
.hg/store/00changelog.n: size=18 .hg/store/00changelog.n: size=26
(The pure code use the debug code that perform incremental update, the C code reencode from scratch) (The pure code use the debug code that perform incremental update, the C code reencode from scratch)
#if pure #if pure
$ f --sha256 .hg/store/00changelog-*.nd --size $ f --sha256 .hg/store/00changelog-*.nd --size
.hg/store/00changelog-????????????????.nd: size=123072, sha256=136472751566c8198ff09e306a7d2f9bd18bd32298d614752b73da4d6df23340 (glob) .hg/store/00changelog-????????????????.nd: size=123072, sha256=136472751566c8198ff09e306a7d2f9bd18bd32298d614752b73da4d6df23340 (glob)
#else #else
$ f --sha256 .hg/store/00changelog-*.nd --size $ f --sha256 .hg/store/00changelog-*.nd --size
.hg/store/00changelog-????????????????.nd: size=122880, sha256=bfafebd751c4f6d116a76a37a1dee2a251747affe7efbcc4f4842ccc746d4db9 (glob) .hg/store/00changelog-????????????????.nd: size=122880, sha256=bfafebd751c4f6d116a76a37a1dee2a251747affe7efbcc4f4842ccc746d4db9 (glob)
#endif #endif
$ hg debugnodemap --check $ hg debugnodemap --check
revision in index: 5002 revision in index: 5002
revision in nodemap: 5002 revision in nodemap: 5002