( )⚙ D10565 revlog: split the `version` attribute into its two components

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10565

revlog: split the `version` attribute into its two components
ClosedPublic

Authored by marmoute on May 3 2021, 7:47 AM.

Details

Reviewers
indygreg
Alphare
Group Reviewers
hg-reviewers
Commits
rHGf58a13c52726: revlog: split the `version` attribute into its two components
Summary

The revlog.version attribute contained an integer coding 2 different informations:

  • the revlog version number
  • a bit field defining some specific feature of the revlog

We now explicitly store the two components independently. This avoid exposing
the implementation details all around the code and prepare for future revlog
version that would encode the information in a different way.

In the process we drop the version attribute from the interface. It was
flagged for removal when that interface was created.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

marmoute created this revision.May 3 2021, 7:47 AM
baymax updated this revision to Diff 27480.May 3 2021, 9:43 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare accepted this revision.May 7 2021, 8:55 AM
This revision is now accepted and ready to land.May 7 2021, 8:55 AM

Revision Contents
Changeset List

Diff 27342

mercurial/changelog.py

indexfile=indexfile, indexfile=indexfile,
datafile=datafile, datafile=datafile,
checkambig=True, checkambig=True,
mmaplargeindex=True, mmaplargeindex=True,
persistentnodemap=opener.options.get(b'persistent-nodemap', False), persistentnodemap=opener.options.get(b'persistent-nodemap', False),
concurrencychecker=concurrencychecker, concurrencychecker=concurrencychecker,
) )
if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1): if self._initempty and (self._format_version == revlog.REVLOGV1):
# changelogs don't benefit from generaldelta. # changelogs don't benefit from generaldelta.
self.version &= ~revlog.FLAG_GENERALDELTA self._format_flags &= ~revlog.FLAG_GENERALDELTA
self._generaldelta = False self._generaldelta = False
# Delta chains for changelogs tend to be very small because entries # Delta chains for changelogs tend to be very small because entries
# tend to be small and don't delta well with each. So disable delta # tend to be small and don't delta well with each. So disable delta
# chains. # chains.
self._storedeltachains = False self._storedeltachains = False
self._realopener = opener self._realopener = opener

mercurial/debugcommands.py

ts, ts,
compression, compression,
len(heads), len(heads),
clen, clen,
) )
) )
return 0 return 0
v = r.version format = r._format_version
format = v & 0xFFFF v = r._format_flags
flags = [] flags = []
gdelta = False gdelta = False
if v & revlog.FLAG_INLINE_DATA: if v & revlog.FLAG_INLINE_DATA:
flags.append(b'inline') flags.append(b'inline')
if v & revlog.FLAG_GENERALDELTA: if v & revlog.FLAG_GENERALDELTA:
gdelta = True gdelta = True
flags.append(b'generaldelta') flags.append(b'generaldelta')
if not flags: if not flags:

mercurial/interfaces/repository.py

opener = interfaceutil.Attribute( opener = interfaceutil.Attribute(
"""VFS opener to use to access underlying files used for storage. """VFS opener to use to access underlying files used for storage.
TODO this is revlog specific and should not be exposed. TODO this is revlog specific and should not be exposed.
""" """
) )
version = interfaceutil.Attribute(
"""Revlog version number.
TODO this is revlog specific and should not be exposed.
"""
)
_generaldelta = interfaceutil.Attribute( _generaldelta = interfaceutil.Attribute(
"""Whether generaldelta storage is being used. """Whether generaldelta storage is being used.
TODO this is revlog specific and should not be exposed. TODO this is revlog specific and should not be exposed.
""" """
) )
fulltextcache = interfaceutil.Attribute( fulltextcache = interfaceutil.Attribute(

mercurial/manifest.py

# only root indexfile is cached # only root indexfile is cached
checkambig=not bool(tree), checkambig=not bool(tree),
mmaplargeindex=True, mmaplargeindex=True,
upperboundcomp=MAXCOMPRESSION, upperboundcomp=MAXCOMPRESSION,
persistentnodemap=opener.options.get(b'persistent-nodemap', False), persistentnodemap=opener.options.get(b'persistent-nodemap', False),
) )
self.index = self._revlog.index self.index = self._revlog.index
self.version = self._revlog.version
self._generaldelta = self._revlog._generaldelta self._generaldelta = self._revlog._generaldelta
def _setupmanifestcachehooks(self, repo): def _setupmanifestcachehooks(self, repo):
"""Persist the manifestfulltextcache on lock release""" """Persist the manifestfulltextcache on lock release"""
if not util.safehasattr(repo, b'_wlockref'): if not util.safehasattr(repo, b'_wlockref'):
return return
self._fulltextcache._opener = repo.wcachevfs self._fulltextcache._opener = repo.wcachevfs

mercurial/revlog.py

else: else:
versionflags = newversionflags versionflags = newversionflags
except IOError as inst: except IOError as inst:
if inst.errno != errno.ENOENT: if inst.errno != errno.ENOENT:
raise raise
versionflags = newversionflags versionflags = newversionflags
self.version = versionflags flags = self._format_flags = versionflags & ~0xFFFF
fmt = self._format_version = versionflags & 0xFFFF
flags = versionflags & ~0xFFFF
fmt = versionflags & 0xFFFF
if fmt == REVLOGV0: if fmt == REVLOGV0:
if flags: if flags:
raise error.RevlogError( raise error.RevlogError(
_(b'unknown flags (%#04x) in version %d revlog %s') _(b'unknown flags (%#04x) in version %d revlog %s')
% (flags >> 16, fmt, self.indexfile) % (flags >> 16, fmt, self.indexfile)
) )
use_rust_index = False use_rust_index = False
if rustrevlog is not None: if rustrevlog is not None:
if self.nodemap_file is not None: if self.nodemap_file is not None:
use_rust_index = True use_rust_index = True
else: else:
use_rust_index = self.opener.options.get(b'rust.index') use_rust_index = self.opener.options.get(b'rust.index')
self._parse_index = parse_index_v1 self._parse_index = parse_index_v1
if self.version == REVLOGV0: if self._format_version == REVLOGV0:
self._parse_index = revlogv0.parse_index_v0 self._parse_index = revlogv0.parse_index_v0
elif fmt == REVLOGV2: elif fmt == REVLOGV2:
self._parse_index = parse_index_v2 self._parse_index = parse_index_v2
elif devel_nodemap: elif devel_nodemap:
self._parse_index = parse_index_v1_nodemap self._parse_index = parse_index_v1_nodemap
elif use_rust_index: elif use_rust_index:
self._parse_index = parse_index_v1_mixed self._parse_index = parse_index_v1_mixed
try: try:
with self._indexfp(b'r') as ifh, self._datafp(b'w') as dfh: with self._indexfp(b'r') as ifh, self._datafp(b'w') as dfh:
for r in self: for r in self:
dfh.write(self._getsegmentforrevs(r, r, df=ifh)[1]) dfh.write(self._getsegmentforrevs(r, r, df=ifh)[1])
if troffset <= self.start(r): if troffset <= self.start(r):
trindex = r trindex = r
with self._indexfp(b'w') as fp: with self._indexfp(b'w') as fp:
self.version &= ~FLAG_INLINE_DATA self._format_flags &= ~FLAG_INLINE_DATA
self._inline = False self._inline = False
for i in self: for i in self:
e = self.index.entry_binary(i) e = self.index.entry_binary(i)
if i == 0: if i == 0:
header = self.index.pack_header(self.version) header = self._format_flags | self._format_version
header = self.index.pack_header(header)
e = header + e e = header + e
fp.write(e) fp.write(e)
# the temp file replace the real index when we exit the context # the temp file replace the real index when we exit the context
# manager # manager
tr.replace(self.indexfile, trindex * self.index.entry_size) tr.replace(self.indexfile, trindex * self.index.entry_size)
nodemaputil.setup_persistent_nodemap(tr, self) nodemaputil.setup_persistent_nodemap(tr, self)
link, link,
p1r, p1r,
p2r, p2r,
node, node,
sidedata_offset, sidedata_offset,
len(serialized_sidedata), len(serialized_sidedata),
) )
if self.version & 0xFFFF != REVLOGV2: if self._format_version != REVLOGV2:
e = e[:8] e = e[:8]
self.index.append(e) self.index.append(e)
entry = self.index.entry_binary(curr) entry = self.index.entry_binary(curr)
if curr == 0: if curr == 0:
header = self.index.pack_header(self.version) header = self._format_flags | self._format_version
header = self.index.pack_header(header)
entry = header + entry entry = header + entry
self._writeentry( self._writeentry(
transaction, transaction,
ifh, ifh,
dfh, dfh,
entry, entry,
deltainfo.data, deltainfo.data,
link, link,
"""Returns the current offset in the (in-transaction) data file. """Returns the current offset in the (in-transaction) data file.
Versions < 2 of the revlog can get this 0(1), revlog v2 needs a docket Versions < 2 of the revlog can get this 0(1), revlog v2 needs a docket
file to store that information: since sidedata can be rewritten to the file to store that information: since sidedata can be rewritten to the
end of the data file within a transaction, you can have cases where, for end of the data file within a transaction, you can have cases where, for
example, rev `n` does not have sidedata while rev `n - 1` does, leading example, rev `n` does not have sidedata while rev `n - 1` does, leading
to `n - 1`'s sidedata being written after `n`'s data. to `n - 1`'s sidedata being written after `n`'s data.
TODO cache this in a docket file before getting out of experimental.""" TODO cache this in a docket file before getting out of experimental."""
if self.version & 0xFFFF != REVLOGV2: if self._format_version != REVLOGV2:
return self.end(prev) return self.end(prev)
offset = 0 offset = 0
for rev, entry in enumerate(self.index): for rev, entry in enumerate(self.index):
sidedata_end = entry[8] + entry[9] sidedata_end = entry[8] + entry[9]
# Sidedata for a previous rev has potentially been written after # Sidedata for a previous rev has potentially been written after
# this rev's end, so take the max. # this rev's end, so take the max.
offset = max(self.end(rev), offset, sidedata_end) offset = max(self.end(rev), offset, sidedata_end)
if dfh: if dfh:
dfh.close() dfh.close()
ifh.close() ifh.close()
if addrevisioncb: if addrevisioncb:
addrevisioncb(self, rev, node) addrevisioncb(self, rev, node)
def censorrevision(self, tr, censornode, tombstone=b''): def censorrevision(self, tr, censornode, tombstone=b''):
if (self.version & 0xFFFF) == REVLOGV0: if self._format_version == REVLOGV0:
raise error.RevlogError( raise error.RevlogError(
_(b'cannot censor with version %d revlogs') % self.version _(b'cannot censor with version %d revlogs')
% self._format_version
) )
censorrev = self.rev(censornode) censorrev = self.rev(censornode)
tombstone = storageutil.packmeta({b'censored': tombstone}, b'') tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
if len(tombstone) > self.rawsize(censorrev): if len(tombstone) > self.rawsize(censorrev):
raise error.Abort( raise error.Abort(
_(b'censor tombstone must be no longer than censored data') _(b'censor tombstone must be no longer than censored data')
# This is a bit dangerous. We could easily have a mismatch of state. # This is a bit dangerous. We could easily have a mismatch of state.
newrl = revlog( newrl = revlog(
self.opener, self.opener,
target=self.target, target=self.target,
indexfile=newindexfile, indexfile=newindexfile,
datafile=newdatafile, datafile=newdatafile,
censorable=True, censorable=True,
) )
newrl.version = self.version newrl._format_version = self._format_version
newrl._format_flags = self._format_flags
newrl._generaldelta = self._generaldelta newrl._generaldelta = self._generaldelta
newrl._parse_index = self._parse_index newrl._parse_index = self._parse_index
for rev in self.revs(): for rev in self.revs():
node = self.node(rev) node = self.node(rev)
p1, p2 = self.parents(node) p1, p2 = self.parents(node)
if rev == censorrev: if rev == censorrev:
found. found.
""" """
dd, di = self.checksize() dd, di = self.checksize()
if dd: if dd:
yield revlogproblem(error=_(b'data length off by %d bytes') % dd) yield revlogproblem(error=_(b'data length off by %d bytes') % dd)
if di: if di:
yield revlogproblem(error=_(b'index contains %d extra bytes') % di) yield revlogproblem(error=_(b'index contains %d extra bytes') % di)
version = self.version & 0xFFFF version = self._format_version
# The verifier tells us what version revlog we should be. # The verifier tells us what version revlog we should be.
if version != state[b'expectedversion']: if version != state[b'expectedversion']:
yield revlogproblem( yield revlogproblem(
warning=_(b"warning: '%s' uses revlog format %d; expected %d") warning=_(b"warning: '%s' uses revlog format %d; expected %d")
% (self.indexfile, version, state[b'expectedversion']) % (self.indexfile, version, state[b'expectedversion'])
) )
# rewrite the new index entries # rewrite the new index entries
with self._indexfp(b'r+') as fp: with self._indexfp(b'r+') as fp:
fp.seek(startrev * self.index.entry_size) fp.seek(startrev * self.index.entry_size)
for i, e in enumerate(new_entries): for i, e in enumerate(new_entries):
rev = startrev + i rev = startrev + i
self.index.replace_sidedata_info(rev, e[8], e[9], e[0]) self.index.replace_sidedata_info(rev, e[8], e[9], e[0])
packed = self.index.entry_binary(rev) packed = self.index.entry_binary(rev)
if rev == 0: if rev == 0:
header = self.index.pack_header(self.version) header = self._format_flags | self._format_version
header = self.index.pack_header(header)
packed = header + packed packed = header + packed
fp.write(packed) fp.write(packed)

mercurial/verify.py

if level is None: if level is None:
level = VERIFY_DEFAULT level = VERIFY_DEFAULT
self._level = level self._level = level
self.badrevs = set() self.badrevs = set()
self.errors = 0 self.errors = 0
self.warnings = 0 self.warnings = 0
self.havecl = len(repo.changelog) > 0 self.havecl = len(repo.changelog) > 0
self.havemf = len(repo.manifestlog.getstorage(b'')) > 0 self.havemf = len(repo.manifestlog.getstorage(b'')) > 0
self.revlogv1 = repo.changelog.version != revlog.REVLOGV0 self.revlogv1 = repo.changelog._format_version != revlog.REVLOGV0
self.lrugetctx = util.lrucachefunc(repo.unfiltered().__getitem__) self.lrugetctx = util.lrucachefunc(repo.unfiltered().__getitem__)
self.refersmf = False self.refersmf = False
self.fncachewarned = False self.fncachewarned = False
# developer config: verify.skipflags # developer config: verify.skipflags
self.skipflags = repo.ui.configint(b'verify', b'skipflags') self.skipflags = repo.ui.configint(b'verify', b'skipflags')
self.warnorphanstorefiles = True self.warnorphanstorefiles = True
def _warn(self, msg): def _warn(self, msg):
return return
d = obj.checksize() d = obj.checksize()
if d[0]: if d[0]:
self._err(None, _(b"data length off by %d bytes") % d[0], name) self._err(None, _(b"data length off by %d bytes") % d[0], name)
if d[1]: if d[1]:
self._err(None, _(b"index contains %d extra bytes") % d[1], name) self._err(None, _(b"index contains %d extra bytes") % d[1], name)
if obj.version != revlog.REVLOGV0: if obj._format_version != revlog.REVLOGV0:
if not self.revlogv1: if not self.revlogv1:
self._warn(_(b"warning: `%s' uses revlog format 1") % name) self._warn(_(b"warning: `%s' uses revlog format 1") % name)
elif self.revlogv1: elif self.revlogv1:
self._warn(_(b"warning: `%s' uses revlog format 0") % name) self._warn(_(b"warning: `%s' uses revlog format 0") % name)
def _checkentry(self, obj, i, node, seen, linkrevs, f): def _checkentry(self, obj, i, node, seen, linkrevs, f):
"""verify a single revlog entry """verify a single revlog entry
for rl_type, f, f2, size in repo.store.datafiles(): for rl_type, f, f2, size in repo.store.datafiles():
if not f: if not f:
self._err(None, _(b"cannot decode filename '%s'") % f2) self._err(None, _(b"cannot decode filename '%s'") % f2)
elif (size > 0 or not revlogv1) and f.startswith(b'data/'): elif (size > 0 or not revlogv1) and f.startswith(b'data/'):
storefiles.add(_normpath(f)) storefiles.add(_normpath(f))
state = { state = {
# TODO this assumes revlog storage for changelog. # TODO this assumes revlog storage for changelog.
b'expectedversion': self.repo.changelog.version & 0xFFFF, b'expectedversion': self.repo.changelog._format_version,
b'skipflags': self.skipflags, b'skipflags': self.skipflags,
# experimental config: censor.policy # experimental config: censor.policy
b'erroroncensored': ui.config(b'censor', b'policy') == b'abort', b'erroroncensored': ui.config(b'censor', b'policy') == b'abort',
} }
files = sorted(set(filenodes) | set(filelinkrevs)) files = sorted(set(filenodes) | set(filelinkrevs))
revisions = 0 revisions = 0
progress = ui.makeprogress( progress = ui.makeprogress(