This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7838

nodemap: use an intermediate "docket" file to carry small metadata
ClosedPublic

Authored by marmoute on Jan 11 2020, 12:03 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG2b72c4ff8ed1: nodemap: use an intermediate "docket" file to carry small metadata
Summary

This intermediate file will make mmapping, transaction and content validation
easier. (Most of this usefulness will arrive gradually in later changeset). In
particular it will become very useful to append new data are the end of raw file
instead of rewriting on the file on each transaction.

See in code comments for details.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

marmoute created this revision.Jan 11 2020, 12:03 PM
marmoute edited the summary of this revision. (Show Details)Jan 13 2020, 10:06 AM
marmoute updated this revision to Diff 19175.
marmoute updated this revision to Diff 19754.Jan 31 2020, 10:28 AM
marmoute updated this revision to Diff 19784.Jan 31 2020, 5:58 PM
marmoute updated this revision to Diff 19828.Feb 2 2020, 3:17 AM

small doc update on .#s[1]

marmoute updated this revision to Diff 19886.Feb 4 2020, 7:24 PM

rebase to latest default

marmoute added a commit: rHG2b72c4ff8ed1: nodemap: use an intermediate "docket" file to carry small metadata.Feb 10 2020, 8:23 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG2b72c4ff8ed1: nodemap: use an intermediate "docket" file to carry small metadata (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 19886

mercurial/revlogutils/nodemap.py

# nodemap.py - nodemap related code and utilities # nodemap.py - nodemap related code and utilities
# #
# Copyright 2019 Pierre-Yves David <pierre-yves.david@octobus.net> # Copyright 2019 Pierre-Yves David <pierre-yves.david@octobus.net>
# Copyright 2019 George Racinet <georges.racinet@octobus.net> # Copyright 2019 George Racinet <georges.racinet@octobus.net>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import os
import struct import struct
from .. import ( from .. import (
error, error,
node as nodemod, node as nodemod,
pycompat, pycompat,
) )
class NodeMap(dict): class NodeMap(dict):
def __missing__(self, x): def __missing__(self, x):
raise error.RevlogError(b'unknown node: %s' % x) raise error.RevlogError(b'unknown node: %s' % x)
def persisted_data(revlog): def persisted_data(revlog):
"""read the nodemap for a revlog from disk""" """read the nodemap for a revlog from disk"""
if revlog.nodemap_file is None: if revlog.nodemap_file is None:
return None return None
return revlog.opener.tryread(revlog.nodemap_file) pdata = revlog.opener.tryread(revlog.nodemap_file)
if not pdata:
return None
offset = 0
(version,) = S_VERSION.unpack(pdata[offset : offset + S_VERSION.size])
if version != ONDISK_VERSION:
return None
offset += S_VERSION.size
(uuid_size,) = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size])
offset += S_HEADER.size
uid = pdata[offset : offset + uuid_size]
filename = _rawdata_filepath(revlog, uid)
return revlog.opener.tryread(filename)
def setup_persistent_nodemap(tr, revlog): def setup_persistent_nodemap(tr, revlog):
"""Install whatever is needed transaction side to persist a nodemap on disk """Install whatever is needed transaction side to persist a nodemap on disk
(only actually persist the nodemap if this is relevant for this revlog) (only actually persist the nodemap if this is relevant for this revlog)
""" """
if revlog._inline: if revlog._inline:
if getattr(revlog, 'filteredrevs', ()): if getattr(revlog, 'filteredrevs', ()):
raise error.ProgrammingError( raise error.ProgrammingError(
"cannot persist nodemap of a filtered changelog" "cannot persist nodemap of a filtered changelog"
) )
if revlog.nodemap_file is None: if revlog.nodemap_file is None:
msg = "calling persist nodemap on a revlog without the feature enableb" msg = "calling persist nodemap on a revlog without the feature enableb"
raise error.ProgrammingError(msg) raise error.ProgrammingError(msg)
data = persistent_data(revlog.index) data = persistent_data(revlog.index)
uid = _make_uid()
datafile = _rawdata_filepath(revlog, uid)
# EXP-TODO: if this is a cache, this should use a cache vfs, not a # EXP-TODO: if this is a cache, this should use a cache vfs, not a
# store vfs # store vfs
with revlog.opener(revlog.nodemap_file, b'w') as f: with revlog.opener(datafile, b'w') as fd:
f.write(data) fd.write(data)
# EXP-TODO: if this is a cache, this should use a cache vfs, not a
# store vfs
with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
fp.write(_serialize_docket(uid))
# EXP-TODO: if the transaction abort, we should remove the new data and # EXP-TODO: if the transaction abort, we should remove the new data and
# reinstall the old one. (This will be simpler when the file format get a # reinstall the old one.
# bit more advanced)
### Nodemap docket file
#
# The nodemap data are stored on disk using 2 files:
#
# * a raw data files containing a persistent nodemap
# (see `Nodemap Trie` section)
#
# * a small "docket" file containing medatadata
#
# While the nodemap data can be multiple tens of megabytes, the "docket" is
# small, it is easy to update it automatically or to duplicated its content
# during a transaction.
#
# Multiple raw data can exist at the same time (The currently valid one and a
# new one beind used by an in progress transaction). To accomodate this, the
# filename hosting the raw data has a variable parts. The exact filename is
# specified inside the "docket" file.
#
# The docket file contains information to find, qualify and validate the raw
# data. Its content is currently very light, but it will expand as the on disk
# nodemap gains the necessary features to be used in production.
# version 0 is experimental, no BC garantee, do no use outside of tests.
ONDISK_VERSION = 0
S_VERSION = struct.Struct(">B")
S_HEADER = struct.Struct(">B")
ID_SIZE = 8
def _make_uid():
"""return a new unique identifier.
The identifier is random and composed of ascii characters."""
return nodemod.hex(os.urandom(ID_SIZE))
def _serialize_docket(uid):
"""return serialized bytes for a docket using the passed uid"""
data = []
data.append(S_VERSION.pack(ONDISK_VERSION))
data.append(S_HEADER.pack(len(uid)))
data.append(uid)
return b''.join(data)
def _rawdata_filepath(revlog, uid):
"""The (vfs relative) nodemap's rawdata file for a given uid"""
prefix = revlog.nodemap_file[:-2]
return b"%s-%s.nd" % (prefix, uid)
### Nodemap Trie ### Nodemap Trie
# #
# This is a simple reference implementation to compute and persist a nodemap # This is a simple reference implementation to compute and persist a nodemap
# trie. This reference implementation is write only. The python version of this # trie. This reference implementation is write only. The python version of this
# is not expected to be actually used, since it wont provide performance # is not expected to be actually used, since it wont provide performance
# improvement over existing non-persistent C implementation. # improvement over existing non-persistent C implementation.

tests/test-persistent-nodemap.t

=================================== ===================================
Test the persistent on-disk nodemap Test the persistent on-disk nodemap
=================================== ===================================
$ hg init test-repo $ hg init test-repo
$ cd test-repo $ cd test-repo
$ cat << EOF >> .hg/hgrc $ cat << EOF >> .hg/hgrc
> [experimental] > [experimental]
> exp-persistent-nodemap=yes > exp-persistent-nodemap=yes
> EOF > EOF
$ hg debugbuilddag .+5000 $ hg debugbuilddag .+5000
$ f --size .hg/store/00changelog.n
.hg/store/00changelog.n: size=18
$ hg debugnodemap --dump-new | f --sha256 --size $ hg debugnodemap --dump-new | f --sha256 --size
size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7 size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
$ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size $ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size
size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7 size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| 0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff |................| 0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff |................|
0020: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| 0020: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
0030: ff ff ff ff ff ff ed b3 ff ff ff ff ff ff ff ff |................| 0030: ff ff ff ff ff ff ed b3 ff ff ff ff ff ff ff ff |................|