This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10866

revlog: add a way to keep track of older uids in the docket
ClosedPublic

Authored by marmoute on Jun 11 2021, 6:55 AM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHGc252631500e4: revlog: add a way to keep track of older uids in the docket
Summary

When the revlog content is rewritten, we will use new files, to avoid truncating
the previous ones. We need some way to keep track of the older files before we
clean them up.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.Jun 11 2021, 6:55 AM
baymax updated this revision to Diff 28566.Jun 15 2021, 7:39 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare accepted this revision.Jun 16 2021, 10:56 AM
Alphare added a subscriber: Alphare.
Alphare added inline comments.
mercurial/revlogutils/docket.py
146

Aren't those asserts kind of overkill? I don't mind keeping this, but I'm wondering if there is a better chance of this happening than I imagine

This revision is now accepted and ready to land.Jun 16 2021, 10:56 AM
marmoute added inline comments.Jun 18 2021, 6:17 AM
mercurial/revlogutils/docket.py
146

They are no reason for them to arrive except when the serialization/deserialization code is wrong. In that case, it is nice to have a clear error about the issue. We could drop them, but they are cheap and might help while developing future patch.

marmoute updated this revision to Diff 28613.Jun 20 2021, 5:22 PM

Revision Contents
Changeset List

PathPackages
Mmercurial/revlogutils/docket.py (60 lines)

Diff 28661

mercurial/revlogutils/docket.py

# Docket format # Docket format
# #
# * 4 bytes: revlog version # * 4 bytes: revlog version
# | This is mandatory as docket must be compatible with the previous # | This is mandatory as docket must be compatible with the previous
# | revlog index header. # | revlog index header.
# * 1 bytes: size of index uuid # * 1 bytes: size of index uuid
# * 1 bytes: number of outdated index uuid
# * 1 bytes: size of data uuid # * 1 bytes: size of data uuid
# * 1 bytes: number of outdated data uuid
# * 1 bytes: size of sizedata uuid # * 1 bytes: size of sizedata uuid
# * 1 bytes: number of outdated data uuid
# * 8 bytes: size of index-data # * 8 bytes: size of index-data
# * 8 bytes: pending size of index-data # * 8 bytes: pending size of index-data
# * 8 bytes: size of data # * 8 bytes: size of data
# * 8 bytes: size of sidedata # * 8 bytes: size of sidedata
# * 8 bytes: pending size of data # * 8 bytes: pending size of data
# * 8 bytes: pending size of sidedata # * 8 bytes: pending size of sidedata
# * 1 bytes: default compression header # * 1 bytes: default compression header
S_HEADER = struct.Struct(constants.INDEX_HEADER_FMT + b'BBBLLLLLLc') S_HEADER = struct.Struct(constants.INDEX_HEADER_FMT + b'BBBBBBLLLLLLc')
# * 1 bytes: size of index uuid
# * 8 bytes: size of file
S_OLD_UID = struct.Struct('>BL')
class RevlogDocket(object): class RevlogDocket(object):
"""metadata associated with revlog""" """metadata associated with revlog"""
def __init__( def __init__(
self, self,
revlog, revlog,
use_pending=False, use_pending=False,
version_header=None, version_header=None,
index_uuid=None, index_uuid=None,
older_index_uuids=(),
data_uuid=None, data_uuid=None,
older_data_uuids=(),
sidedata_uuid=None, sidedata_uuid=None,
older_sidedata_uuids=(),
index_end=0, index_end=0,
pending_index_end=0, pending_index_end=0,
data_end=0, data_end=0,
pending_data_end=0, pending_data_end=0,
sidedata_end=0, sidedata_end=0,
pending_sidedata_end=0, pending_sidedata_end=0,
default_compression_header=None, default_compression_header=None,
): ):
self._version_header = version_header self._version_header = version_header
self._read_only = bool(use_pending) self._read_only = bool(use_pending)
self._dirty = False self._dirty = False
self._radix = revlog.radix self._radix = revlog.radix
self._path = revlog._docket_file self._path = revlog._docket_file
self._opener = revlog.opener self._opener = revlog.opener
self._index_uuid = index_uuid self._index_uuid = index_uuid
self._older_index_uuids = older_index_uuids
self._data_uuid = data_uuid self._data_uuid = data_uuid
self._older_data_uuids = older_data_uuids
self._sidedata_uuid = sidedata_uuid self._sidedata_uuid = sidedata_uuid
self._older_sidedata_uuids = older_sidedata_uuids
assert not set(older_index_uuids) & set(older_data_uuids)
AlphareUnsubmitted
Not Done

Aren't those asserts kind of overkill? I don't mind keeping this, but I'm wondering if there is a better chance of this happening than I imagine

Alphare: Aren't those asserts kind of overkill? I don't mind keeping this, but I'm wondering if there is…
marmouteAuthorUnsubmitted
Done

They are no reason for them to arrive except when the serialization/deserialization code is wrong. In that case, it is nice to have a clear error about the issue. We could drop them, but they are cheap and might help while developing future patch.

marmoute: They are no reason for them to arrive except when the serialization/deserialization code is…
assert not set(older_data_uuids) & set(older_sidedata_uuids)
assert not set(older_index_uuids) & set(older_sidedata_uuids)
# thes asserts should be True as long as we have a single index filename # thes asserts should be True as long as we have a single index filename
assert index_end <= pending_index_end assert index_end <= pending_index_end
assert data_end <= pending_data_end assert data_end <= pending_data_end
assert sidedata_end <= pending_sidedata_end assert sidedata_end <= pending_sidedata_end
self._initial_index_end = index_end self._initial_index_end = index_end
self._pending_index_end = pending_index_end self._pending_index_end = pending_index_end
self._initial_data_end = data_end self._initial_data_end = data_end
self._pending_data_end = pending_data_end self._pending_data_end = pending_data_end
official_sidedata_end = self._sidedata_end official_sidedata_end = self._sidedata_end
# this assert should be True as long as we have a single index filename # this assert should be True as long as we have a single index filename
assert official_data_end <= self._data_end assert official_data_end <= self._data_end
assert official_sidedata_end <= self._sidedata_end assert official_sidedata_end <= self._sidedata_end
data = ( data = (
self._version_header, self._version_header,
len(self._index_uuid), len(self._index_uuid),
len(self._older_index_uuids),
len(self._data_uuid), len(self._data_uuid),
len(self._older_data_uuids),
len(self._sidedata_uuid), len(self._sidedata_uuid),
len(self._older_sidedata_uuids),
official_index_end, official_index_end,
self._index_end, self._index_end,
official_data_end, official_data_end,
self._data_end, self._data_end,
official_sidedata_end, official_sidedata_end,
self._sidedata_end, self._sidedata_end,
self.default_compression_header, self.default_compression_header,
) )
s = [] s = []
s.append(S_HEADER.pack(*data)) s.append(S_HEADER.pack(*data))
s.append(self._index_uuid) s.append(self._index_uuid)
for u, size in self._older_index_uuids:
s.append(S_OLD_UID.pack(len(u), size))
for u, size in self._older_index_uuids:
s.append(u)
s.append(self._data_uuid) s.append(self._data_uuid)
for u, size in self._older_data_uuids:
s.append(S_OLD_UID.pack(len(u), size))
for u, size in self._older_data_uuids:
s.append(u)
s.append(self._sidedata_uuid) s.append(self._sidedata_uuid)
for u, size in self._older_sidedata_uuids:
s.append(S_OLD_UID.pack(len(u), size))
for u, size in self._older_sidedata_uuids:
s.append(u)
return b''.join(s) return b''.join(s)
def default_docket(revlog, version_header): def default_docket(revlog, version_header):
"""given a revlog version a new docket object for the given revlog""" """given a revlog version a new docket object for the given revlog"""
rl_version = version_header & 0xFFFF rl_version = version_header & 0xFFFF
if rl_version not in (constants.REVLOGV2, constants.CHANGELOGV2): if rl_version not in (constants.REVLOGV2, constants.CHANGELOGV2):
return None return None
comp = util.compengines[revlog._compengine].revlogheader() comp = util.compengines[revlog._compengine].revlogheader()
docket = RevlogDocket( docket = RevlogDocket(
revlog, revlog,
version_header=version_header, version_header=version_header,
default_compression_header=comp, default_compression_header=comp,
) )
docket._dirty = True docket._dirty = True
return docket return docket
def _parse_old_uids(get_data, count):
all_sizes = []
all_uids = []
for i in range(0, count):
raw = get_data(S_OLD_UID.size)
all_sizes.append(S_OLD_UID.unpack(raw))
for uid_size, file_size in all_sizes:
uid = get_data(uid_size)
all_uids.append((uid, file_size))
return all_uids
def parse_docket(revlog, data, use_pending=False): def parse_docket(revlog, data, use_pending=False):
"""given some docket data return a docket object for the given revlog""" """given some docket data return a docket object for the given revlog"""
header = S_HEADER.unpack(data[: S_HEADER.size]) header = S_HEADER.unpack(data[: S_HEADER.size])
# this is a mutable closure capture used in `get_data` # this is a mutable closure capture used in `get_data`
offset = [S_HEADER.size] offset = [S_HEADER.size]
def get_data(size): def get_data(size):
iheader = iter(header) iheader = iter(header)
version_header = next(iheader) version_header = next(iheader)
index_uuid_size = next(iheader) index_uuid_size = next(iheader)
index_uuid = get_data(index_uuid_size) index_uuid = get_data(index_uuid_size)
older_index_uuid_count = next(iheader)
older_index_uuids = _parse_old_uids(get_data, older_index_uuid_count)
data_uuid_size = next(iheader) data_uuid_size = next(iheader)
data_uuid = get_data(data_uuid_size) data_uuid = get_data(data_uuid_size)
older_data_uuid_count = next(iheader)
older_data_uuids = _parse_old_uids(get_data, older_data_uuid_count)
sidedata_uuid_size = next(iheader) sidedata_uuid_size = next(iheader)
sidedata_uuid = get_data(sidedata_uuid_size) sidedata_uuid = get_data(sidedata_uuid_size)
older_sidedata_uuid_count = next(iheader)
older_sidedata_uuids = _parse_old_uids(get_data, older_sidedata_uuid_count)
index_size = next(iheader) index_size = next(iheader)
pending_index_size = next(iheader) pending_index_size = next(iheader)
data_size = next(iheader) data_size = next(iheader)
pending_data_size = next(iheader) pending_data_size = next(iheader)
sidedata_size = next(iheader) sidedata_size = next(iheader)
pending_sidedata_size = next(iheader) pending_sidedata_size = next(iheader)
default_compression_header = next(iheader) default_compression_header = next(iheader)
docket = RevlogDocket( docket = RevlogDocket(
revlog, revlog,
use_pending=use_pending, use_pending=use_pending,
version_header=version_header, version_header=version_header,
index_uuid=index_uuid, index_uuid=index_uuid,
older_index_uuids=older_index_uuids,
data_uuid=data_uuid, data_uuid=data_uuid,
older_data_uuids=older_data_uuids,
sidedata_uuid=sidedata_uuid, sidedata_uuid=sidedata_uuid,
older_sidedata_uuids=older_sidedata_uuids,
index_end=index_size, index_end=index_size,
pending_index_end=pending_index_size, pending_index_end=pending_index_size,
data_end=data_size, data_end=data_size,
pending_data_end=pending_data_size, pending_data_end=pending_data_size,
sidedata_end=sidedata_size, sidedata_end=sidedata_size,
pending_sidedata_end=pending_sidedata_size, pending_sidedata_end=pending_sidedata_size,
default_compression_header=default_compression_header, default_compression_header=default_compression_header,
) )
return docket return docket