( )⚙ D10772 revlogv2: use a unique filename for data

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10772

revlogv2: use a unique filename for data
ClosedPublic

Authored by marmoute on May 27 2021, 3:05 AM.

Details

Reviewers
indygreg
Group Reviewers
hg-reviewers
Commits
rHG0a3fa41fa719: revlogv2: use a unique filename for data
rHG5035d31e764b: revlogv2: use a unique filename for data
Summary

Having a unique data will allow for ambiguity less rewriting of revlog content,
something useful to clarify handling of some operation like censoring or
stripping.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

marmoute created this revision.May 27 2021, 3:05 AM
marmoute added a commit: rHG5035d31e764b: revlogv2: use a unique filename for data.May 27 2021, 5:36 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG5035d31e764b: revlogv2: use a unique filename for data (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 28209

mercurial/revlog.py

self._inline = False self._inline = False
# generaldelta implied by version 2 revlogs. # generaldelta implied by version 2 revlogs.
self._generaldelta = True self._generaldelta = True
# the logic for persistent nodemap will be dealt with within the # the logic for persistent nodemap will be dealt with within the
# main docket, so disable it for now. # main docket, so disable it for now.
self._nodemap_file = None self._nodemap_file = None
if self.postfix is None: if self._docket is not None:
self._datafile = self._docket.data_filepath()
elif self.postfix is None:
self._datafile = b'%s.d' % self.radix self._datafile = b'%s.d' % self.radix
else: else:
self._datafile = b'%s.d.%s' % (self.radix, self.postfix) self._datafile = b'%s.d.%s' % (self.radix, self.postfix)
self.nodeconstants = sha1nodeconstants self.nodeconstants = sha1nodeconstants
self.nullid = self.nodeconstants.nullid self.nullid = self.nodeconstants.nullid
# sparse-revlog can't be on without general-delta (issue6056) # sparse-revlog can't be on without general-delta (issue6056)

mercurial/revlogutils/docket.py

# Docket format # Docket format
# #
# * 4 bytes: revlog version # * 4 bytes: revlog version
# | This is mandatory as docket must be compatible with the previous # | This is mandatory as docket must be compatible with the previous
# | revlog index header. # | revlog index header.
# * 1 bytes: size of index uuid # * 1 bytes: size of index uuid
# * 1 bytes: size of data uuid
# * 8 bytes: size of index-data # * 8 bytes: size of index-data
# * 8 bytes: pending size of index-data # * 8 bytes: pending size of index-data
# * 8 bytes: size of data # * 8 bytes: size of data
# * 8 bytes: pending size of data # * 8 bytes: pending size of data
# * 1 bytes: default compression header # * 1 bytes: default compression header
S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'BLLLLc') S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'BBLLLLc')
class RevlogDocket(object): class RevlogDocket(object):
"""metadata associated with revlog""" """metadata associated with revlog"""
def __init__( def __init__(
self, self,
revlog, revlog,
use_pending=False, use_pending=False,
version_header=None, version_header=None,
index_uuid=None, index_uuid=None,
data_uuid=None,
index_end=0, index_end=0,
pending_index_end=0, pending_index_end=0,
data_end=0, data_end=0,
pending_data_end=0, pending_data_end=0,
default_compression_header=None, default_compression_header=None,
): ):
self._version_header = version_header self._version_header = version_header
self._read_only = bool(use_pending) self._read_only = bool(use_pending)
self._dirty = False self._dirty = False
self._radix = revlog.radix self._radix = revlog.radix
self._path = revlog._docket_file self._path = revlog._docket_file
self._opener = revlog.opener self._opener = revlog.opener
self._index_uuid = index_uuid self._index_uuid = index_uuid
self._data_uuid = data_uuid
# thes asserts should be True as long as we have a single index filename # thes asserts should be True as long as we have a single index filename
assert index_end <= pending_index_end assert index_end <= pending_index_end
assert data_end <= pending_data_end assert data_end <= pending_data_end
self._initial_index_end = index_end self._initial_index_end = index_end
self._pending_index_end = pending_index_end self._pending_index_end = pending_index_end
self._initial_data_end = data_end self._initial_data_end = data_end
self._pending_data_end = pending_data_end self._pending_data_end = pending_data_end
if use_pending: if use_pending:
self._index_end = self._pending_index_end self._index_end = self._pending_index_end
self._data_end = self._pending_data_end self._data_end = self._pending_data_end
else: else:
self._index_end = self._initial_index_end self._index_end = self._initial_index_end
self._data_end = self._initial_data_end self._data_end = self._initial_data_end
self.default_compression_header = default_compression_header self.default_compression_header = default_compression_header
def index_filepath(self): def index_filepath(self):
"""file path to the current index file associated to this docket""" """file path to the current index file associated to this docket"""
# very simplistic version at first # very simplistic version at first
if self._index_uuid is None: if self._index_uuid is None:
self._index_uuid = make_uid() self._index_uuid = make_uid()
return b"%s-%s.idx" % (self._radix, self._index_uuid) return b"%s-%s.idx" % (self._radix, self._index_uuid)
def data_filepath(self):
"""file path to the current index file associated to this docket"""
# very simplistic version at first
if self._data_uuid is None:
self._data_uuid = make_uid()
return b"%s-%s.dat" % (self._radix, self._data_uuid)
@property @property
def index_end(self): def index_end(self):
return self._index_end return self._index_end
@index_end.setter @index_end.setter
def index_end(self, new_size): def index_end(self, new_size):
if new_size != self._index_end: if new_size != self._index_end:
self._index_end = new_size self._index_end = new_size
official_index_end = self._index_end official_index_end = self._index_end
official_data_end = self._data_end official_data_end = self._data_end
# this assert should be True as long as we have a single index filename # this assert should be True as long as we have a single index filename
assert official_data_end <= self._data_end assert official_data_end <= self._data_end
data = ( data = (
self._version_header, self._version_header,
len(self._index_uuid), len(self._index_uuid),
len(self._data_uuid),
official_index_end, official_index_end,
self._index_end, self._index_end,
official_data_end, official_data_end,
self._data_end, self._data_end,
self.default_compression_header, self.default_compression_header,
) )
s = [] s = []
s.append(S_HEADER.pack(*data)) s.append(S_HEADER.pack(*data))
s.append(self._index_uuid) s.append(self._index_uuid)
s.append(self._data_uuid)
return b''.join(s) return b''.join(s)
def default_docket(revlog, version_header): def default_docket(revlog, version_header):
"""given a revlog version a new docket object for the given revlog""" """given a revlog version a new docket object for the given revlog"""
rl_version = version_header & 0xFFFF rl_version = version_header & 0xFFFF
if rl_version not in (constants.REVLOGV2, constants.CHANGELOGV2): if rl_version not in (constants.REVLOGV2, constants.CHANGELOGV2):
return None return None
def parse_docket(revlog, data, use_pending=False): def parse_docket(revlog, data, use_pending=False):
"""given some docket data return a docket object for the given revlog""" """given some docket data return a docket object for the given revlog"""
header = S_HEADER.unpack(data[: S_HEADER.size]) header = S_HEADER.unpack(data[: S_HEADER.size])
offset = S_HEADER.size offset = S_HEADER.size
version_header = header[0] version_header = header[0]
index_uuid_size = header[1] index_uuid_size = header[1]
index_uuid = data[offset : offset + index_uuid_size] index_uuid = data[offset : offset + index_uuid_size]
offset += index_uuid_size offset += index_uuid_size
index_size = header[2] data_uuid_size = header[2]
pending_index_size = header[3] data_uuid = data[offset : offset + data_uuid_size]
data_size = header[4] offset += data_uuid_size
pending_data_size = header[5] index_size = header[3]
default_compression_header = header[6] pending_index_size = header[4]
data_size = header[5]
pending_data_size = header[6]
default_compression_header = header[7]
docket = RevlogDocket( docket = RevlogDocket(
revlog, revlog,
use_pending=use_pending, use_pending=use_pending,
version_header=version_header, version_header=version_header,
index_uuid=index_uuid, index_uuid=index_uuid,
data_uuid=data_uuid,
index_end=index_size, index_end=index_size,
pending_index_end=pending_index_size, pending_index_end=pending_index_size,
data_end=data_size, data_end=data_size,
pending_data_end=pending_data_size, pending_data_end=pending_data_size,
default_compression_header=default_compression_header, default_compression_header=default_compression_header,
) )
return docket return docket

mercurial/store.py

b'00changelog.d', b'00changelog.d',
b'00changelog.i', b'00changelog.i',
b'phaseroots', b'phaseroots',
b'obsstore', b'obsstore',
b'requires', b'requires',
] ]
REVLOG_FILES_MAIN_EXT = (b'.i', b'i.tmpcensored') REVLOG_FILES_MAIN_EXT = (b'.i', b'i.tmpcensored')
REVLOG_FILES_OTHER_EXT = (b'.idx', b'.d', b'.n', b'.nd', b'd.tmpcensored') REVLOG_FILES_OTHER_EXT = (
b'.idx',
b'.d',
b'.dat',
b'.n',
b'.nd',
b'd.tmpcensored',
)
# files that are "volatile" and might change between listing and streaming # files that are "volatile" and might change between listing and streaming
# #
# note: the ".nd" file are nodemap data and won't "change" but they might be # note: the ".nd" file are nodemap data and won't "change" but they might be
# deleted. # deleted.
REVLOG_FILES_VOLATILE_EXT = (b'.n', b'.nd') REVLOG_FILES_VOLATILE_EXT = (b'.n', b'.nd')
# some exception to the above matching # some exception to the above matching
EXCLUDED = re.compile(b'.*undo\.[^/]+\.(nd?|i)$') EXCLUDED = re.compile(b'.*undo\.[^/]+\.(nd?|i)$')
def is_revlog(f, kind, st): def is_revlog(f, kind, st):
if kind != stat.S_IFREG: if kind != stat.S_IFREG:
return None return None
return revlog_type(f) return revlog_type(f)
def revlog_type(f): def revlog_type(f):
if f.endswith(REVLOG_FILES_MAIN_EXT) and EXCLUDED.match(f) is None: if f.endswith(REVLOG_FILES_MAIN_EXT) and EXCLUDED.match(f) is None:
return FILEFLAGS_REVLOG_MAIN return FILEFLAGS_REVLOG_MAIN
elif f.endswith(REVLOG_FILES_OTHER_EXT) and EXCLUDED.match(f) is None: elif f.endswith(REVLOG_FILES_OTHER_EXT) and EXCLUDED.match(f) is None:
t = FILETYPE_FILELOG_OTHER t = FILETYPE_FILELOG_OTHER
if f.endswith(REVLOG_FILES_VOLATILE_EXT): if f.endswith(REVLOG_FILES_VOLATILE_EXT):
t |= FILEFLAGS_VOLATILE t |= FILEFLAGS_VOLATILE
return t return t
return None
# the file is part of changelog data # the file is part of changelog data
FILEFLAGS_CHANGELOG = 1 << 13 FILEFLAGS_CHANGELOG = 1 << 13
# the file is part of manifest data # the file is part of manifest data
FILEFLAGS_MANIFESTLOG = 1 << 12 FILEFLAGS_MANIFESTLOG = 1 << 12
# the file is part of filelog data # the file is part of filelog data
FILEFLAGS_FILELOG = 1 << 11 FILEFLAGS_FILELOG = 1 << 11
def datafiles(self, matcher=None): def datafiles(self, matcher=None):
for f in sorted(self.fncache): for f in sorted(self.fncache):
if not _matchtrackedpath(f, matcher): if not _matchtrackedpath(f, matcher):
continue continue
ef = self.encode(f) ef = self.encode(f)
try: try:
t = revlog_type(f) t = revlog_type(f)
assert t is not None, f
t |= FILEFLAGS_FILELOG t |= FILEFLAGS_FILELOG
yield t, f, ef, self.getsize(ef) yield t, f, ef, self.getsize(ef)
except OSError as err: except OSError as err:
if err.errno != errno.ENOENT: if err.errno != errno.ENOENT:
raise raise
def copylist(self): def copylist(self):
d = ( d = (

tests/test-revlog-v2.t

$ hg log $ hg log
changeset: 0:96ee1d7354c4 changeset: 0:96ee1d7354c4
tag: tip tag: tip
user: test user: test
date: Thu Jan 01 00:00:00 1970 +0000 date: Thu Jan 01 00:00:00 1970 +0000
summary: initial summary: initial
Header written as expected Header written as expected
$ f --hexdump --bytes 4 .hg/store/00changelog.i $ f --hexdump --bytes 4 .hg/store/00changelog.i
.hg/store/00changelog.i: .hg/store/00changelog.i:
0000: 00 00 de ad |....| 0000: 00 00 de ad |....|
$ f --hexdump --bytes 4 .hg/store/data/foo.i $ f --hexdump --bytes 4 .hg/store/data/foo.i
.hg/store/data/foo.i: .hg/store/data/foo.i:
0000: 00 00 de ad |....| 0000: 00 00 de ad |....|
The expected files are generated The expected files are generated
-------------------------------- --------------------------------
We should have have: We should have have:
- a docket - a docket
- a index file with a unique name - a index file with a unique name
- a data file - a data file
$ ls .hg/store/00changelog* .hg/store/00manifest* $ ls .hg/store/00changelog* .hg/store/00manifest*
.hg/store/00changelog-b870a51b.idx .hg/store/00changelog-6b8ab34b.dat
.hg/store/00changelog.d .hg/store/00changelog-88698448.idx
.hg/store/00changelog.i .hg/store/00changelog.i
.hg/store/00manifest-88698448.idx .hg/store/00manifest-1335303a.dat
.hg/store/00manifest.d .hg/store/00manifest-b875dfc5.idx
.hg/store/00manifest.i .hg/store/00manifest.i