This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7840

nodemap: add a (python) index class for persistent nodemap testing
ClosedPublic

Authored by marmoute on Jan 13 2020, 10:06 AM.

Details

Reviewers
indygreg
Group Reviewers
hg-reviewers
Commits
rHG6f9e8e142cea: nodemap: add a (python) index class for persistent nodemap testing
Summary

Using the persistent nodemap require a compeling performance boost and an
existing implementation. The benefit of the persistent nodemap for pure python
code is unclear and we don't have a C implementation for it. Yet we would like
to actually start testing it in more details and define an API for using that
persistent nodemap.

We introduce a new devel config option to use an index class dedicated to
Nodemap Testing. This feature is "pure" only because having using a pure-python
index with the cext policy proved more difficult than I would like.

There is nothing going on in that class for now, but the coming changeset will
change that.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.Jan 13 2020, 10:06 AM
marmoute updated this revision to Diff 19186.Jan 13 2020, 1:05 PM
marmoute updated this revision to Diff 19428.Jan 17 2020, 1:06 PM
marmoute updated this revision to Diff 19756.Jan 31 2020, 10:28 AM
marmoute updated this revision to Diff 19888.Feb 4 2020, 7:24 PM

rebase to latest default

marmoute updated this revision to Diff 20017.Feb 7 2020, 5:28 PM
marmoute added a commit: rHG6f9e8e142cea: nodemap: add a (python) index class for persistent nodemap testing.Feb 10 2020, 8:24 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG6f9e8e142cea: nodemap: add a (python) index class for persistent nodemap testing (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 20108

mercurial/configitems.py

) )
coreconfigitem( coreconfigitem(
b'devel', b'warn-empty-changegroup', default=False, b'devel', b'warn-empty-changegroup', default=False,
) )
coreconfigitem( coreconfigitem(
b'devel', b'legacy.exchange', default=list, b'devel', b'legacy.exchange', default=list,
) )
coreconfigitem( coreconfigitem(
b'devel', b'persistent-nodemap', default=False,
)
coreconfigitem(
b'devel', b'servercafile', default=b'', b'devel', b'servercafile', default=b'',
) )
coreconfigitem( coreconfigitem(
b'devel', b'serverexactprotocol', default=b'', b'devel', b'serverexactprotocol', default=b'',
) )
coreconfigitem( coreconfigitem(
b'devel', b'serverrequirecert', default=False, b'devel', b'serverrequirecert', default=False,
) )

mercurial/localrepo.py

if repository.NARROW_REQUIREMENT in requirements: if repository.NARROW_REQUIREMENT in requirements:
options[b'enableellipsis'] = True options[b'enableellipsis'] = True
if ui.configbool(b'experimental', b'rust.index'): if ui.configbool(b'experimental', b'rust.index'):
options[b'rust.index'] = True options[b'rust.index'] = True
if ui.configbool(b'experimental', b'exp-persistent-nodemap'): if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
options[b'exp-persistent-nodemap'] = True options[b'exp-persistent-nodemap'] = True
if ui.configbool(b'devel', b'persistent-nodemap'):
options[b'devel-force-nodemap'] = True
return options return options
def makemain(**kwargs): def makemain(**kwargs):
"""Produce a type conforming to ``ilocalrepositorymain``.""" """Produce a type conforming to ``ilocalrepositorymain``."""
return localrepository return localrepository

mercurial/pure/parsers.py

if i < self._lgt: if i < self._lgt:
self._data = self._data[: i * indexsize] self._data = self._data[: i * indexsize]
self._lgt = i self._lgt = i
self._extra = [] self._extra = []
else: else:
self._extra = self._extra[: i - self._lgt] self._extra = self._extra[: i - self._lgt]
class PersistentNodeMapIndexObject(IndexObject):
"""a Debug oriented class to test persistent nodemap
We need a simple python object to test API and higher level behavior. See
the Rust implementation for more serious usage. This should be used only
through the dedicated `devel.persistent-nodemap` config.
"""
class InlinedIndexObject(BaseIndexObject): class InlinedIndexObject(BaseIndexObject):
def __init__(self, data, inline=0): def __init__(self, data, inline=0):
self._data = data self._data = data
self._lgt = self._inline_scan(None) self._lgt = self._inline_scan(None)
self._inline_scan(self._lgt) self._inline_scan(self._lgt)
self._extra = [] self._extra = []
def _inline_scan(self, lgt): def _inline_scan(self, lgt):
def parse_index2(data, inline): def parse_index2(data, inline):
if not inline: if not inline:
return IndexObject(data), None return IndexObject(data), None
return InlinedIndexObject(data, inline), (0, data) return InlinedIndexObject(data, inline), (0, data)
def parse_index_devel_nodemap(data, inline):
"""like parse_index2, but alway return a PersistentNodeMapIndexObject
"""
return PersistentNodeMapIndexObject(data), None
def parse_dirstate(dmap, copymap, st): def parse_dirstate(dmap, copymap, st):
parents = [st[:20], st[20:40]] parents = [st[:20], st[20:40]]
# dereference fields so they will be local in loop # dereference fields so they will be local in loop
format = b">cllll" format = b">cllll"
e_size = struct.calcsize(format) e_size = struct.calcsize(format)
pos1 = 40 pos1 = 40
l = len(st) l = len(st)

mercurial/revlog.py

def packentry(self, entry, node, version, rev): def packentry(self, entry, node, version, rev):
p = indexformatng_pack(*entry) p = indexformatng_pack(*entry)
if rev == 0: if rev == 0:
p = versionformat_pack(version) + p[4:] p = versionformat_pack(version) + p[4:]
return p return p
NodemapRevlogIO = None
if util.safehasattr(parsers, 'parse_index_devel_nodemap'):
class NodemapRevlogIO(revlogio):
"""A debug oriented IO class that return a PersistentNodeMapIndexObject
The PersistentNodeMapIndexObject object is meant to test the persistent nodemap feature.
"""
def parseindex(self, data, inline):
index, cache = parsers.parse_index_devel_nodemap(data, inline)
return index, cache
class rustrevlogio(revlogio): class rustrevlogio(revlogio):
def parseindex(self, data, inline): def parseindex(self, data, inline):
index, cache = super(rustrevlogio, self).parseindex(data, inline) index, cache = super(rustrevlogio, self).parseindex(data, inline)
return rustrevlog.MixedIndex(index), cache return rustrevlog.MixedIndex(index), cache
class revlog(object): class revlog(object):
""" """
_(b'unknown version (%d) in revlog %s') % (fmt, self.indexfile) _(b'unknown version (%d) in revlog %s') % (fmt, self.indexfile)
) )
# sparse-revlog can't be on without general-delta (issue6056) # sparse-revlog can't be on without general-delta (issue6056)
if not self._generaldelta: if not self._generaldelta:
self._sparserevlog = False self._sparserevlog = False
self._storedeltachains = True self._storedeltachains = True
devel_nodemap = (
self.nodemap_file
and opts.get(b'devel-force-nodemap', False)
and NodemapRevlogIO is not None
)
self._io = revlogio() self._io = revlogio()
if self.version == REVLOGV0: if self.version == REVLOGV0:
self._io = revlogoldio() self._io = revlogoldio()
elif devel_nodemap:
self._io = NodemapRevlogIO()
elif rustrevlog is not None and self.opener.options.get(b'rust.index'): elif rustrevlog is not None and self.opener.options.get(b'rust.index'):
self._io = rustrevlogio() self._io = rustrevlogio()
try: try:
d = self._io.parseindex(indexdata, self._inline) d = self._io.parseindex(indexdata, self._inline)
except (ValueError, IndexError): except (ValueError, IndexError):
raise error.RevlogError( raise error.RevlogError(
_(b"index %s is corrupted") % self.indexfile _(b"index %s is corrupted") % self.indexfile
) )

tests/test-persistent-nodemap.t

=================================== ===================================
Test the persistent on-disk nodemap Test the persistent on-disk nodemap
=================================== ===================================
$ hg init test-repo $ hg init test-repo
$ cd test-repo $ cd test-repo
$ cat << EOF >> .hg/hgrc $ cat << EOF >> .hg/hgrc
> [experimental] > [experimental]
> exp-persistent-nodemap=yes > exp-persistent-nodemap=yes
> [devel]
> persistent-nodemap=yes
> EOF > EOF
$ hg debugbuilddag .+5000 $ hg debugbuilddag .+5000
$ f --size .hg/store/00changelog.n $ f --size .hg/store/00changelog.n
.hg/store/00changelog.n: size=18 .hg/store/00changelog.n: size=18
$ f --sha256 .hg/store/00changelog-*.nd $ f --sha256 .hg/store/00changelog-*.nd
.hg/store/00changelog-????????????????.nd: sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7 (glob) .hg/store/00changelog-????????????????.nd: sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7 (glob)
$ hg debugnodemap --dump-new | f --sha256 --size $ hg debugnodemap --dump-new | f --sha256 --size
size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7 size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7