This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10304

revlog: move the details of revlog "v1" index inside revlog.utils.constants
ClosedPublic

Authored by marmoute on Apr 5 2021, 11:49 AM.

Details

Reviewers
indygreg
Alphare
pulkit
Group Reviewers
hg-reviewers
Commits
rHGcc65cea90edb: revlog: move the details of revlog "v1" index inside revlog.utils.constants
Summary

The revlog module is quite large and this kind of format information would handy
for other module. So let us start to gather this information about the format in
a more appropriate place.

We update various reference to this information to use the new "source of truth"
in the process.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

Diff 26694

mercurial/pure/parsers.py

from ..node import nullid, nullrev from ..node import nullid, nullrev
from .. import ( from .. import (
pycompat, pycompat,
util, util,
) )
from ..revlogutils import nodemap as nodemaputil from ..revlogutils import nodemap as nodemaputil
from ..revlogutils import constants as revlog_constants
stringio = pycompat.bytesio stringio = pycompat.bytesio
_pack = struct.pack _pack = struct.pack
_unpack = struct.unpack _unpack = struct.unpack
_compress = zlib.compress _compress = zlib.compress
_decompress = zlib.decompress _decompress = zlib.decompress
def offset_type(offset, type): def offset_type(offset, type):
return int(int(offset) << 16 | type) return int(int(offset) << 16 | type)
class BaseIndexObject(object): class BaseIndexObject(object):
# Format of an index entry according to Python's `struct` language # Format of an index entry according to Python's `struct` language
index_format = b">Qiiiiii20s12x" index_format = revlog_constants.INDEX_ENTRY_V1.format
# Size of a C unsigned long long int, platform independent # Size of a C unsigned long long int, platform independent
big_int_size = struct.calcsize(b'>Q') big_int_size = struct.calcsize(b'>Q')
# Size of a C long int, platform independent # Size of a C long int, platform independent
int_size = struct.calcsize(b'>i') int_size = struct.calcsize(b'>i')
# Size of the entire index format # Size of the entire index format
index_size = struct.calcsize(index_format) index_size = revlog_constants.INDEX_ENTRY_V1.size
# An empty index entry, used as a default value to be overridden, or nullrev # An empty index entry, used as a default value to be overridden, or nullrev
null_item = (0, 0, 0, -1, -1, -1, -1, nullid) null_item = (0, 0, 0, -1, -1, -1, -1, nullid)
@property @property
def nodemap(self): def nodemap(self):
msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]" msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]"
util.nouideprecwarn(msg, b'5.3', stacklevel=2) util.nouideprecwarn(msg, b'5.3', stacklevel=2)
return self._nodemap return self._nodemap

mercurial/revlog.py

wdirrev, wdirrev,
) )
from .i18n import _ from .i18n import _
from .pycompat import getattr from .pycompat import getattr
from .revlogutils.constants import ( from .revlogutils.constants import (
FLAG_GENERALDELTA, FLAG_GENERALDELTA,
FLAG_INLINE_DATA, FLAG_INLINE_DATA,
INDEX_ENTRY_V0, INDEX_ENTRY_V0,
INDEX_ENTRY_V1,
REVLOGV0, REVLOGV0,
REVLOGV1, REVLOGV1,
REVLOGV1_FLAGS, REVLOGV1_FLAGS,
REVLOGV2, REVLOGV2,
REVLOGV2_FLAGS, REVLOGV2_FLAGS,
REVLOG_DEFAULT_FLAGS, REVLOG_DEFAULT_FLAGS,
REVLOG_DEFAULT_FORMAT, REVLOG_DEFAULT_FORMAT,
REVLOG_DEFAULT_VERSION, REVLOG_DEFAULT_VERSION,
entry[4], entry[4],
node(entry[5]), node(entry[5]),
node(entry[6]), node(entry[6]),
entry[7], entry[7],
) )
return INDEX_ENTRY_V0.pack(*e2) return INDEX_ENTRY_V0.pack(*e2)
# index ng:
# 6 bytes: offset
# 2 bytes: flags
# 4 bytes: compressed length
# 4 bytes: uncompressed length
# 4 bytes: base rev
# 4 bytes: link rev
# 4 bytes: parent 1 rev
# 4 bytes: parent 2 rev
# 32 bytes: nodeid
indexformatng = struct.Struct(b">Qiiiiii20s12x")
indexformatng_pack = indexformatng.pack
versionformat = struct.Struct(b">I") versionformat = struct.Struct(b">I")
versionformat_pack = versionformat.pack versionformat_pack = versionformat.pack
versionformat_unpack = versionformat.unpack versionformat_unpack = versionformat.unpack
# corresponds to uncompressed length of indexformatng (2 gigs, 4-byte # corresponds to uncompressed length of indexformatng (2 gigs, 4-byte
# signed integer) # signed integer)
_maxentrysize = 0x7FFFFFFF _maxentrysize = 0x7FFFFFFF
class revlogio(object): class revlogio(object):
def __init__(self): def __init__(self):
self.size = indexformatng.size self.size = INDEX_ENTRY_V1.size
def parseindex(self, data, inline): def parseindex(self, data, inline):
# call the C implementation to parse the index data # call the C implementation to parse the index data
index, cache = parsers.parse_index2(data, inline) index, cache = parsers.parse_index2(data, inline)
return index, cache return index, cache
def packentry(self, entry, node, version, rev): def packentry(self, entry, node, version, rev):
p = indexformatng_pack(*entry) p = INDEX_ENTRY_V1.pack(*entry)
if rev == 0: if rev == 0:
p = versionformat_pack(version) + p[4:] p = versionformat_pack(version) + p[4:]
return p return p
indexformatv2 = struct.Struct(pureparsers.Index2Mixin.index_format) indexformatv2 = struct.Struct(pureparsers.Index2Mixin.index_format)
indexformatv2_pack = indexformatv2.pack indexformatv2_pack = indexformatv2.pack

mercurial/revlogutils/constants.py

# 4 bytes: compressed length # 4 bytes: compressed length
# 4 bytes: base rev # 4 bytes: base rev
# 4 bytes: link rev # 4 bytes: link rev
# 20 bytes: parent 1 nodeid # 20 bytes: parent 1 nodeid
# 20 bytes: parent 2 nodeid # 20 bytes: parent 2 nodeid
# 20 bytes: nodeid # 20 bytes: nodeid
INDEX_ENTRY_V0 = struct.Struct(b">4l20s20s20s") INDEX_ENTRY_V0 = struct.Struct(b">4l20s20s20s")
## index v1
# 6 bytes: offset
# 2 bytes: flags
# 4 bytes: compressed length
# 4 bytes: uncompressed length
# 4 bytes: base rev
# 4 bytes: link rev
# 4 bytes: parent 1 rev
# 4 bytes: parent 2 rev
# 32 bytes: nodeid
INDEX_ENTRY_V1 = struct.Struct(b">Qiiiiii20s12x")
assert INDEX_ENTRY_V1.size == 32 * 2
# revlog index flags # revlog index flags
# For historical reasons, revlog's internal flags were exposed via the # For historical reasons, revlog's internal flags were exposed via the
# wire protocol and are even exposed in parts of the storage APIs. # wire protocol and are even exposed in parts of the storage APIs.
# revision has censor metadata, must be verified # revision has censor metadata, must be verified
REVIDX_ISCENSORED = repository.REVISION_FLAG_CENSORED REVIDX_ISCENSORED = repository.REVISION_FLAG_CENSORED
# revision hash does not match data (narrowhg) # revision hash does not match data (narrowhg)