( )⚙ D10352 revlog: introduce an explicit tracking of what the revlog is about

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10352

revlog: introduce an explicit tracking of what the revlog is about
ClosedPublic

Authored by Alphare on Apr 9 2021, 8:30 AM.

Details

Reviewers
indygreg
marmoute
Group Reviewers
hg-reviewers
Commits
rHG4c041c71ec01: revlog: introduce an explicit tracking of what the revlog is about
Summary

Since the dawn of time, people have been forced to rely to lossy introspection
of the index filename to determine what the purpose and role of the revlog they
encounter is. This is hacky, error prone, inflexible, abstraction-leaky,
<insert-your-own-complaints-here>.

In f63299ee7e4d Raphaël introduced a new attribute to track this information:
revlog_kind. However it is initialized in an odd place and various instances
end up not having it set. In addition is only tracking some of the information
we end up having to introspect in various pieces of code.

So we add a new attribute that holds more data and is more strictly enforced.
This work is done in collaboration with Raphaël.

The revlog_kind one will be removed/adapted in the next changeset. We expect
to be able to clean up various existing piece of code and to simplify coming
work around the newer revlog format.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

Alphare created this revision.Apr 9 2021, 8:30 AM
marmoute accepted this revision.Apr 19 2021, 8:43 AM
marmoute added a subscriber: marmoute.

I co-authored this revision with Alphare. So I am accepting it on the behalf of both of use since it already got two pairs on eyes on it.

baymax updated this revision to Diff 27092.Apr 20 2021, 5:55 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 27094.Apr 20 2021, 9:33 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 27223.Apr 30 2021, 6:52 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 27295.May 3 2021, 4:14 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 27464.May 3 2021, 9:02 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

marmoute added a commit: rHG4c041c71ec01: revlog: introduce an explicit tracking of what the revlog is about.May 5 2021, 6:36 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG4c041c71ec01: revlog: introduce an explicit tracking of what the revlog is about (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 27598

contrib/dumprevlog

#!/usr/bin/env python3 #!/usr/bin/env python3
# Dump revlogs as raw data stream # Dump revlogs as raw data stream
# $ find .hg/store/ -name "*.i" | xargs dumprevlog > repo.dump # $ find .hg/store/ -name "*.i" | xargs dumprevlog > repo.dump
from __future__ import absolute_import, print_function from __future__ import absolute_import, print_function
import sys import sys
from mercurial.node import hex from mercurial.node import hex
from mercurial import ( from mercurial import (
encoding, encoding,
pycompat, pycompat,
revlog, revlog,
) )
from mercurial.utils import procutil from mercurial.utils import procutil
from mercurial.revlogutils import (
constants as revlog_constants,
)
for fp in (sys.stdin, sys.stdout, sys.stderr): for fp in (sys.stdin, sys.stdout, sys.stderr):
procutil.setbinary(fp) procutil.setbinary(fp)
def binopen(path, mode=b'rb'): def binopen(path, mode=b'rb'):
if b'b' not in mode: if b'b' not in mode:
mode = mode + b'b' mode = mode + b'b'
return open(path, pycompat.sysstr(mode)) return open(path, pycompat.sysstr(mode))
binopen.options = {} binopen.options = {}
def printb(data, end=b'\n'): def printb(data, end=b'\n'):
sys.stdout.flush() sys.stdout.flush()
procutil.stdout.write(data + end) procutil.stdout.write(data + end)
for f in sys.argv[1:]: for f in sys.argv[1:]:
r = revlog.revlog(binopen, encoding.strtolocal(f)) r = revlog.revlog(
binopen,
target=(revlog_constants.KIND_OTHER, b'dump-revlog'),
indexfile=encoding.strtolocal(f),
)
print("file:", f) print("file:", f)
for i in r: for i in r:
n = r.node(i) n = r.node(i)
p = r.parents(n) p = r.parents(n)
d = r.revision(n) d = r.revision(n)
printb(b"node: %s" % hex(n)) printb(b"node: %s" % hex(n))
printb(b"linkrev: %d" % r.linkrev(i)) printb(b"linkrev: %d" % r.linkrev(i))
printb(b"parents: %s %s" % (hex(p[0]), hex(p[1]))) printb(b"parents: %s %s" % (hex(p[0]), hex(p[1])))
printb(b"length: %d" % len(d)) printb(b"length: %d" % len(d))
printb(b"-start-") printb(b"-start-")
printb(d) printb(d)
printb(b"-end-") printb(b"-end-")

contrib/perf.py

import os import os
import random import random
import shutil import shutil
import struct import struct
import sys import sys
import tempfile import tempfile
import threading import threading
import time import time
import mercurial.revlog
from mercurial import ( from mercurial import (
changegroup, changegroup,
cmdutil, cmdutil,
commands, commands,
copies, copies,
error, error,
extensions, extensions,
hg, hg,
mdiff, mdiff,
merge, merge,
revlog,
util, util,
) )
# for "historical portability": # for "historical portability":
# try to import modules separately (in dict order), and ignore # try to import modules separately (in dict order), and ignore
# failure, because these aren't available with early Mercurial # failure, because these aren't available with early Mercurial
try: try:
from mercurial import branchmap # since 2.5 (or bcee63733aad) from mercurial import branchmap # since 2.5 (or bcee63733aad)
except ImportError: except ImportError:
pass pass
try: try:
from mercurial import profiling from mercurial import profiling
except ImportError: except ImportError:
profiling = None profiling = None
try:
from mercurial.revlogutils import constants as revlog_constants
perf_rl_kind = (revlog_constants.KIND_OTHER, b'created-by-perf')
def revlog(opener, *args, **kwargs):
return mercurial.revlog.revlog(opener, perf_rl_kind, *args, **kwargs)
except (ImportError, AttributeError):
perf_rl_kind = None
def revlog(opener, *args, **kwargs):
return mercurial.revlog.revlog(opener, *args, **kwargs)
def identity(a): def identity(a):
return a return a
try: try:
from mercurial import pycompat from mercurial import pycompat
@command(b'perf::nodelookup|perfnodelookup', formatteropts) @command(b'perf::nodelookup|perfnodelookup', formatteropts)
def perfnodelookup(ui, repo, rev, **opts): def perfnodelookup(ui, repo, rev, **opts):
opts = _byteskwargs(opts) opts = _byteskwargs(opts)
timer, fm = gettimer(ui, opts) timer, fm = gettimer(ui, opts)
import mercurial.revlog import mercurial.revlog
mercurial.revlog._prereadsize = 2 ** 24 # disable lazy parser in old hg mercurial.revlog._prereadsize = 2 ** 24 # disable lazy parser in old hg
n = scmutil.revsingle(repo, rev).node() n = scmutil.revsingle(repo, rev).node()
cl = mercurial.revlog.revlog(getsvfs(repo), b"00changelog.i")
cl = revlog(getsvfs(repo), indexfile=b"00changelog.i")
def d(): def d():
cl.rev(n) cl.rev(n)
clearcaches(cl) clearcaches(cl)
timer(d) timer(d)
fm.end() fm.end()
header = struct.unpack(b'>I', data[0:4])[0] header = struct.unpack(b'>I', data[0:4])[0]
version = header & 0xFFFF version = header & 0xFFFF
if version == 1: if version == 1:
inline = header & (1 << 16) inline = header & (1 << 16)
else: else:
raise error.Abort(b'unsupported revlog version: %d' % version) raise error.Abort(b'unsupported revlog version: %d' % version)
parse_index_v1 = getattr(revlog, 'parse_index_v1', None) parse_index_v1 = getattr(mercurial.revlog, 'parse_index_v1', None)
if parse_index_v1 is None: if parse_index_v1 is None:
parse_index_v1 = revlog.revlogio().parseindex parse_index_v1 = mercurial.revlog.revlogio().parseindex
rllen = len(rl) rllen = len(rl)
node0 = rl.node(0) node0 = rl.node(0)
node25 = rl.node(rllen // 4) node25 = rl.node(rllen // 4)
node50 = rl.node(rllen // 2) node50 = rl.node(rllen // 2)
node75 = rl.node(rllen // 4 * 3) node75 = rl.node(rllen // 4 * 3)
node100 = rl.node(rllen - 1) node100 = rl.node(rllen - 1)
allrevs = range(rllen) allrevs = range(rllen)
allrevsrev = list(reversed(allrevs)) allrevsrev = list(reversed(allrevs))
allnodes = [rl.node(rev) for rev in range(rllen)] allnodes = [rl.node(rev) for rev in range(rllen)]
allnodesrev = list(reversed(allnodes)) allnodesrev = list(reversed(allnodes))
def constructor(): def constructor():
revlog.revlog(opener, indexfile) revlog(opener, indexfile=indexfile)
def read(): def read():
with opener(indexfile) as fh: with opener(indexfile) as fh:
fh.read() fh.read()
def parseindex(): def parseindex():
parse_index_v1(data, inline) parse_index_v1(data, inline)
data.seek(0) data.seek(0)
data.truncate(orig.start(truncaterev)) data.truncate(orig.start(truncaterev))
# instantiate a new revlog from the temporary copy # instantiate a new revlog from the temporary copy
ui.debug('truncating adding to be rewritten\n') ui.debug('truncating adding to be rewritten\n')
vfs = vfsmod.vfs(tmpdir) vfs = vfsmod.vfs(tmpdir)
vfs.options = getattr(orig.opener, 'options', None) vfs.options = getattr(orig.opener, 'options', None)
dest = revlog.revlog( dest = revlog(
vfs, indexfile=indexname, datafile=dataname, **revlogkwargs vfs, indexfile=indexname, datafile=dataname, **revlogkwargs
) )
if dest._inline: if dest._inline:
raise error.Abort('not supporting inline revlog (yet)') raise error.Abort('not supporting inline revlog (yet)')
# make sure internals are initialized # make sure internals are initialized
dest.revision(len(dest) - 1) dest.revision(len(dest) - 1)
yield dest yield dest
del dest, vfs del dest, vfs

contrib/undumprevlog

from mercurial import ( from mercurial import (
encoding, encoding,
revlog, revlog,
transaction, transaction,
vfs as vfsmod, vfs as vfsmod,
) )
from mercurial.utils import procutil from mercurial.utils import procutil
from mercurial.revlogutils import (
constants as revlog_constants,
)
for fp in (sys.stdin, sys.stdout, sys.stderr): for fp in (sys.stdin, sys.stdout, sys.stderr):
procutil.setbinary(fp) procutil.setbinary(fp)
opener = vfsmod.vfs(b'.', False) opener = vfsmod.vfs(b'.', False)
tr = transaction.transaction( tr = transaction.transaction(
sys.stderr.write, opener, {b'store': opener}, b"undump.journal" sys.stderr.write, opener, {b'store': opener}, b"undump.journal"
) )
while True: while True:
l = sys.stdin.readline() l = sys.stdin.readline()
if not l: if not l:
break break
if l.startswith("file:"): if l.startswith("file:"):
f = encoding.strtolocal(l[6:-1]) f = encoding.strtolocal(l[6:-1])
r = revlog.revlog(opener, f) r = revlog.revlog(
opener,
target=(revlog_constants.KIND_OTHER, b'undump-revlog'),
indexfile=f,
)
procutil.stdout.write(b'%s\n' % f) procutil.stdout.write(b'%s\n' % f)
elif l.startswith("node:"): elif l.startswith("node:"):
n = bin(l[6:-1]) n = bin(l[6:-1])
elif l.startswith("linkrev:"): elif l.startswith("linkrev:"):
lr = int(l[9:-1]) lr = int(l[9:-1])
elif l.startswith("parents:"): elif l.startswith("parents:"):
p = l[9:-1].split() p = l[9:-1].split()
p1 = bin(p[0]) p1 = bin(p[0])

mercurial/bundlerepo.py

revlog, revlog,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
from .utils import ( from .utils import (
urlutil, urlutil,
) )
from .revlogutils import (
constants as revlog_constants,
)
class bundlerevlog(revlog.revlog): class bundlerevlog(revlog.revlog):
def __init__(self, opener, indexfile, cgunpacker, linkmapper): def __init__(self, opener, target, indexfile, cgunpacker, linkmapper):
# How it works: # How it works:
# To retrieve a revision, we need to know the offset of the revision in # To retrieve a revision, we need to know the offset of the revision in
# the bundle (an unbundle object). We store this offset in the index # the bundle (an unbundle object). We store this offset in the index
# (start). The base of the delta is stored in the base field. # (start). The base of the delta is stored in the base field.
# #
# To differentiate a rev in the bundle from a rev in the revlog, we # To differentiate a rev in the bundle from a rev in the revlog, we
# check revision against repotiprev. # check revision against repotiprev.
opener = vfsmod.readonlyvfs(opener) opener = vfsmod.readonlyvfs(opener)
revlog.revlog.__init__(self, opener, indexfile) revlog.revlog.__init__(self, opener, target=target, indexfile=indexfile)
self.bundle = cgunpacker self.bundle = cgunpacker
n = len(self) n = len(self)
self.repotiprev = n - 1 self.repotiprev = n - 1
self.bundlerevs = set() # used by 'bundle()' revset expression self.bundlerevs = set() # used by 'bundle()' revset expression
for deltadata in cgunpacker.deltaiter(): for deltadata in cgunpacker.deltaiter():
node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
size = len(delta) size = len(delta)
raise NotImplementedError raise NotImplementedError
class bundlechangelog(bundlerevlog, changelog.changelog): class bundlechangelog(bundlerevlog, changelog.changelog):
def __init__(self, opener, cgunpacker): def __init__(self, opener, cgunpacker):
changelog.changelog.__init__(self, opener) changelog.changelog.__init__(self, opener)
linkmapper = lambda x: x linkmapper = lambda x: x
bundlerevlog.__init__( bundlerevlog.__init__(
self, opener, self.indexfile, cgunpacker, linkmapper self,
opener,
(revlog_constants.KIND_CHANGELOG, None),
self.indexfile,
cgunpacker,
linkmapper,
) )
class bundlemanifest(bundlerevlog, manifest.manifestrevlog): class bundlemanifest(bundlerevlog, manifest.manifestrevlog):
def __init__( def __init__(
self, self,
nodeconstants, nodeconstants,
opener, opener,
cgunpacker, cgunpacker,
linkmapper, linkmapper,
dirlogstarts=None, dirlogstarts=None,
dir=b'', dir=b'',
): ):
manifest.manifestrevlog.__init__(self, nodeconstants, opener, tree=dir) manifest.manifestrevlog.__init__(self, nodeconstants, opener, tree=dir)
bundlerevlog.__init__( bundlerevlog.__init__(
self, opener, self.indexfile, cgunpacker, linkmapper self,
opener,
(revlog_constants.KIND_MANIFESTLOG, dir),
self.indexfile,
cgunpacker,
linkmapper,
) )
if dirlogstarts is None: if dirlogstarts is None:
dirlogstarts = {} dirlogstarts = {}
if self.bundle.version == b"03": if self.bundle.version == b"03":
dirlogstarts = _getfilestarts(self.bundle) dirlogstarts = _getfilestarts(self.bundle)
self._dirlogstarts = dirlogstarts self._dirlogstarts = dirlogstarts
self._linkmapper = linkmapper self._linkmapper = linkmapper
) )
return super(bundlemanifest, self).dirlog(d) return super(bundlemanifest, self).dirlog(d)
class bundlefilelog(filelog.filelog): class bundlefilelog(filelog.filelog):
def __init__(self, opener, path, cgunpacker, linkmapper): def __init__(self, opener, path, cgunpacker, linkmapper):
filelog.filelog.__init__(self, opener, path) filelog.filelog.__init__(self, opener, path)
self._revlog = bundlerevlog( self._revlog = bundlerevlog(
opener, self.indexfile, cgunpacker, linkmapper opener,
# XXX should use the unencoded path
target=(revlog_constants.KIND_FILELOG, path),
indexfile=self.indexfile,
cgunpacker=cgunpacker,
linkmapper=linkmapper,
) )
class bundlepeer(localrepo.localpeer): class bundlepeer(localrepo.localpeer):
def canpush(self): def canpush(self):
return False return False

mercurial/changelog.py

metadata, metadata,
pycompat, pycompat,
revlog, revlog,
) )
from .utils import ( from .utils import (
dateutil, dateutil,
stringutil, stringutil,
) )
from .revlogutils import flagutil from .revlogutils import (
constants as revlog_constants,
flagutil,
)
_defaultextra = {b'branch': b'default'} _defaultextra = {b'branch': b'default'}
def _string_escape(text): def _string_escape(text):
""" """
>>> from .pycompat import bytechr as chr >>> from .pycompat import bytechr as chr
>>> d = {b'nl': chr(10), b'bs': chr(92), b'cr': chr(13), b'nul': chr(0)} >>> d = {b'nl': chr(10), b'bs': chr(92), b'cr': chr(13), b'nul': chr(0)}
indexfile = b'00changelog.i.a' indexfile = b'00changelog.i.a'
else: else:
indexfile = b'00changelog.i' indexfile = b'00changelog.i'
datafile = b'00changelog.d' datafile = b'00changelog.d'
revlog.revlog.__init__( revlog.revlog.__init__(
self, self,
opener, opener,
indexfile, target=(revlog_constants.KIND_CHANGELOG, None),
indexfile=indexfile,
datafile=datafile, datafile=datafile,
checkambig=True, checkambig=True,
mmaplargeindex=True, mmaplargeindex=True,
persistentnodemap=opener.options.get(b'persistent-nodemap', False), persistentnodemap=opener.options.get(b'persistent-nodemap', False),
concurrencychecker=concurrencychecker, concurrencychecker=concurrencychecker,
) )
if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1): if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):

mercurial/cmdutil.py

vfs as vfsmod, vfs as vfsmod,
) )
from .utils import ( from .utils import (
dateutil, dateutil,
stringutil, stringutil,
) )
from .revlogutils import (
constants as revlog_constants,
)
if pycompat.TYPE_CHECKING: if pycompat.TYPE_CHECKING:
from typing import ( from typing import (
Any, Any,
Dict, Dict,
) )
for t in (Any, Dict): for t in (Any, Dict):
assert t assert t
if not r: if not r:
if not returnrevlog: if not returnrevlog:
raise error.InputError(_(b'cannot give path to non-revlog')) raise error.InputError(_(b'cannot give path to non-revlog'))
if not file_: if not file_:
raise error.CommandError(cmd, _(b'invalid arguments')) raise error.CommandError(cmd, _(b'invalid arguments'))
if not os.path.isfile(file_): if not os.path.isfile(file_):
raise error.InputError(_(b"revlog '%s' not found") % file_) raise error.InputError(_(b"revlog '%s' not found") % file_)
target = (revlog_constants.KIND_OTHER, b'free-form:%s' % file_)
r = revlog.revlog( r = revlog.revlog(
vfsmod.vfs(encoding.getcwd(), audit=False), file_[:-2] + b".i" vfsmod.vfs(encoding.getcwd(), audit=False),
target=target,
indexfile=file_[:-2] + b".i",
) )
return r return r
def openrevlog(repo, cmd, file_, opts): def openrevlog(repo, cmd, file_, opts):
"""Obtain a revlog backing storage of an item. """Obtain a revlog backing storage of an item.
This is similar to ``openstorage()`` except it always returns a revlog. This is similar to ``openstorage()`` except it always returns a revlog.

mercurial/filelog.py

error, error,
revlog, revlog,
) )
from .interfaces import ( from .interfaces import (
repository, repository,
util as interfaceutil, util as interfaceutil,
) )
from .utils import storageutil from .utils import storageutil
from .revlogutils import (
constants as revlog_constants,
)
@interfaceutil.implementer(repository.ifilestorage) @interfaceutil.implementer(repository.ifilestorage)
class filelog(object): class filelog(object):
def __init__(self, opener, path): def __init__(self, opener, path):
self._revlog = revlog.revlog( self._revlog = revlog.revlog(
opener, b'/'.join((b'data', path + b'.i')), censorable=True opener,
# XXX should use the unencoded path
target=(revlog_constants.KIND_FILELOG, path),
indexfile=b'/'.join((b'data', path + b'.i')),
censorable=True,
) )
# Full name of the user visible file, relative to the repository root. # Full name of the user visible file, relative to the repository root.
# Used by LFS. # Used by LFS.
self._revlog.filename = path self._revlog.filename = path
self._revlog.revlog_kind = b'filelog' self._revlog.revlog_kind = b'filelog'
self.nullid = self._revlog.nullid self.nullid = self._revlog.nullid
def __len__(self): def __len__(self):

mercurial/manifest.py

pycompat, pycompat,
revlog, revlog,
util, util,
) )
from .interfaces import ( from .interfaces import (
repository, repository,
util as interfaceutil, util as interfaceutil,
) )
from .revlogutils import (
constants as revlog_constants,
)
parsers = policy.importmod('parsers') parsers = policy.importmod('parsers')
propertycache = util.propertycache propertycache = util.propertycache
# Allow tests to more easily test the alternate path in manifestdict.fastdelta() # Allow tests to more easily test the alternate path in manifestdict.fastdelta()
FASTDELTA_TEXTDIFF_THRESHOLD = 1000 FASTDELTA_TEXTDIFF_THRESHOLD = 1000
# The dirlogcache is kept on the root manifest log # The dirlogcache is kept on the root manifest log
if tree: if tree:
self._dirlogcache = dirlogcache self._dirlogcache = dirlogcache
else: else:
self._dirlogcache = {b'': self} self._dirlogcache = {b'': self}
self._revlog = revlog.revlog( self._revlog = revlog.revlog(
opener, opener,
indexfile, target=(revlog_constants.KIND_MANIFESTLOG, self.tree),
indexfile=indexfile,
# only root indexfile is cached # only root indexfile is cached
checkambig=not bool(tree), checkambig=not bool(tree),
mmaplargeindex=True, mmaplargeindex=True,
upperboundcomp=MAXCOMPRESSION, upperboundcomp=MAXCOMPRESSION,
persistentnodemap=opener.options.get(b'persistent-nodemap', False), persistentnodemap=opener.options.get(b'persistent-nodemap', False),
) )
self.index = self._revlog.index self.index = self._revlog.index

mercurial/revlog.py

nullrev, nullrev,
sha1nodeconstants, sha1nodeconstants,
short, short,
wdirrev, wdirrev,
) )
from .i18n import _ from .i18n import _
from .pycompat import getattr from .pycompat import getattr
from .revlogutils.constants import ( from .revlogutils.constants import (
ALL_KINDS,
FLAG_GENERALDELTA, FLAG_GENERALDELTA,
FLAG_INLINE_DATA, FLAG_INLINE_DATA,
INDEX_HEADER, INDEX_HEADER,
REVLOGV0, REVLOGV0,
REVLOGV1, REVLOGV1,
REVLOGV1_FLAGS, REVLOGV1_FLAGS,
REVLOGV2, REVLOGV2,
REVLOGV2_FLAGS, REVLOGV2_FLAGS,
raising). raising).
""" """
_flagserrorclass = error.RevlogError _flagserrorclass = error.RevlogError
def __init__( def __init__(
self, self,
opener, opener,
indexfile, target,
indexfile=None,
datafile=None, datafile=None,
checkambig=False, checkambig=False,
mmaplargeindex=False, mmaplargeindex=False,
censorable=False, censorable=False,
upperboundcomp=None, upperboundcomp=None,
persistentnodemap=False, persistentnodemap=False,
concurrencychecker=None, concurrencychecker=None,
): ):
""" """
create a revlog object create a revlog object
opener is a function that abstracts the file opening operation opener is a function that abstracts the file opening operation
and can be used to implement COW semantics or the like. and can be used to implement COW semantics or the like.
`target`: a (KIND, ID) tuple that identify the content stored in
this revlog. It help the rest of the code to understand what the revlog
is about without having to resort to heuristic and index filename
analysis. Note: that this must be reliably be set by normal code, but
that test, debug, or performance measurement code might not set this to
accurate value.
""" """
self.upperboundcomp = upperboundcomp self.upperboundcomp = upperboundcomp
self.indexfile = indexfile self.indexfile = indexfile
self.datafile = datafile or (indexfile[:-2] + b".d") self.datafile = datafile or (indexfile[:-2] + b".d")
self.nodemap_file = None self.nodemap_file = None
if persistentnodemap: if persistentnodemap:
self.nodemap_file = nodemaputil.get_nodemap_file( self.nodemap_file = nodemaputil.get_nodemap_file(
opener, self.indexfile opener, self.indexfile
) )
self.opener = opener self.opener = opener
assert target[0] in ALL_KINDS
assert len(target) == 2
self.target = target
# When True, indexfile is opened with checkambig=True at writing, to # When True, indexfile is opened with checkambig=True at writing, to
# avoid file stat ambiguity. # avoid file stat ambiguity.
self._checkambig = checkambig self._checkambig = checkambig
self._mmaplargeindex = mmaplargeindex self._mmaplargeindex = mmaplargeindex
self._censorable = censorable self._censorable = censorable
# 3-tuple of (node, rev, text) for a raw revision. # 3-tuple of (node, rev, text) for a raw revision.
self._revisioncache = None self._revisioncache = None
# Maps rev to chain base rev. # Maps rev to chain base rev.
# Rewriting the revlog in place is hard. Our strategy for censoring is # Rewriting the revlog in place is hard. Our strategy for censoring is
# to create a new revlog, copy all revisions to it, then replace the # to create a new revlog, copy all revisions to it, then replace the
# revlogs on transaction close. # revlogs on transaction close.
newindexfile = self.indexfile + b'.tmpcensored' newindexfile = self.indexfile + b'.tmpcensored'
newdatafile = self.datafile + b'.tmpcensored' newdatafile = self.datafile + b'.tmpcensored'
# This is a bit dangerous. We could easily have a mismatch of state. # This is a bit dangerous. We could easily have a mismatch of state.
newrl = revlog(self.opener, newindexfile, newdatafile, censorable=True) newrl = revlog(
self.opener,
target=self.target,
indexfile=newindexfile,
datafile=newdatafile,
censorable=True,
)
newrl.version = self.version newrl.version = self.version
newrl._generaldelta = self._generaldelta newrl._generaldelta = self._generaldelta
newrl._parse_index = self._parse_index newrl._parse_index = self._parse_index
for rev in self.revs(): for rev in self.revs():
node = self.node(rev) node = self.node(rev)
p1, p2 = self.parents(node) p1, p2 = self.parents(node)

mercurial/revlogutils/constants.py

# revlogdeltas.py - constant used for revlog logic # revlogdeltas.py - constant used for revlog logic
# #
# Copyright 2005-2007 Olivia Mackall <olivia@selenic.com> # Copyright 2005-2007 Olivia Mackall <olivia@selenic.com>
# Copyright 2018 Octobus <contact@octobus.net> # Copyright 2018 Octobus <contact@octobus.net>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
"""Helper class to compute deltas stored inside revlogs""" """Helper class to compute deltas stored inside revlogs"""
from __future__ import absolute_import from __future__ import absolute_import
import struct import struct
from ..interfaces import repository from ..interfaces import repository
### Internal utily constants
KIND_CHANGELOG = 1001 # over 256 to not be comparable with a bytes
KIND_MANIFESTLOG = 1002
KIND_FILELOG = 1003
KIND_OTHER = 1004
ALL_KINDS = {
KIND_CHANGELOG,
KIND_MANIFESTLOG,
KIND_FILELOG,
KIND_OTHER,
}
### main revlog header ### main revlog header
INDEX_HEADER = struct.Struct(b">I") INDEX_HEADER = struct.Struct(b">I")
## revlog version ## revlog version
REVLOGV0 = 0 REVLOGV0 = 0
REVLOGV1 = 1 REVLOGV1 = 1
# Dummy value until file format is finalized. # Dummy value until file format is finalized.

mercurial/unionrepo.py

def __init__(self, opener, indexfile, revlog2, linkmapper): def __init__(self, opener, indexfile, revlog2, linkmapper):
# How it works: # How it works:
# To retrieve a revision, we just need to know the node id so we can # To retrieve a revision, we just need to know the node id so we can
# look it up in revlog2. # look it up in revlog2.
# #
# To differentiate a rev in the second revlog from a rev in the revlog, # To differentiate a rev in the second revlog from a rev in the revlog,
# we check revision against repotiprev. # we check revision against repotiprev.
opener = vfsmod.readonlyvfs(opener) opener = vfsmod.readonlyvfs(opener)
revlog.revlog.__init__(self, opener, indexfile) target = getattr(revlog2, 'target', None)
if target is None:
# a revlog wrapper, eg: the manifestlog that is not an actual revlog
target = revlog2._revlog.target
revlog.revlog.__init__(self, opener, target=target, indexfile=indexfile)
self.revlog2 = revlog2 self.revlog2 = revlog2
n = len(self) n = len(self)
self.repotiprev = n - 1 self.repotiprev = n - 1
self.bundlerevs = set() # used by 'bundle()' revset expression self.bundlerevs = set() # used by 'bundle()' revset expression
for rev2 in self.revlog2: for rev2 in self.revlog2:
rev = self.revlog2.index[rev2] rev = self.revlog2.index[rev2]
# rev numbers - in revlog2, very different from self.rev # rev numbers - in revlog2, very different from self.rev

tests/test-revlog-raw.py

# test revlog interaction about raw data (flagprocessor) # test revlog interaction about raw data (flagprocessor)
from __future__ import absolute_import, print_function from __future__ import absolute_import, print_function
import collections import collections
import hashlib import hashlib
import sys import sys
from mercurial import ( from mercurial import (
encoding, encoding,
revlog, revlog,
transaction, transaction,
vfs, vfs,
) )
from mercurial.revlogutils import ( from mercurial.revlogutils import (
constants,
deltas, deltas,
flagutil, flagutil,
) )
# TESTTMP is optional. This makes it convenient to run without run-tests.py # TESTTMP is optional. This makes it convenient to run without run-tests.py
tvfs = vfs.vfs(encoding.environ.get(b'TESTTMP', b'/tmp')) tvfs = vfs.vfs(encoding.environ.get(b'TESTTMP', b'/tmp'))
# Enable generaldelta otherwise revlog won't use delta as expected by the test # Enable generaldelta otherwise revlog won't use delta as expected by the test
# A transaction is required to write revlogs # A transaction is required to write revlogs
report = lambda msg: None report = lambda msg: None
return transaction.transaction(report, tvfs, {'plain': tvfs}, b'journal') return transaction.transaction(report, tvfs, {'plain': tvfs}, b'journal')
def newrevlog(name=b'_testrevlog.i', recreate=False): def newrevlog(name=b'_testrevlog.i', recreate=False):
if recreate: if recreate:
tvfs.tryunlink(name) tvfs.tryunlink(name)
rlog = revlog.revlog(tvfs, name) rlog = revlog.revlog(
tvfs, target=(constants.KIND_OTHER, b'test'), indexfile=name
)
return rlog return rlog
def appendrev(rlog, text, tr, isext=False, isdelta=True): def appendrev(rlog, text, tr, isext=False, isdelta=True):
"""Append a revision. If isext is True, set the EXTSTORED flag so flag """Append a revision. If isext is True, set the EXTSTORED flag so flag
processor will be used (and rawtext is different from text). If isdelta is processor will be used (and rawtext is different from text). If isdelta is
True, force the revision to be a delta, otherwise it's full text. True, force the revision to be a delta, otherwise it's full text.
""" """

tests/test-revlog.t

... Joa3dYtcYYYBAQ8Qr4OqZAYRICPTSr5WKd/42rV36d+8/VmrNpv7NP1jQAXrQE4BqQUARngwVA==""", ... Joa3dYtcYYYBAQ8Qr4OqZAYRICPTSr5WKd/42rV36d+8/VmrNpv7NP1jQAXrQE4BqQUARngwVA==""",
... "base64"), "zlib")) and None ... "base64"), "zlib")) and None
$ hg debugrevlogindex a.i $ hg debugrevlogindex a.i
rev linkrev nodeid p1 p2 rev linkrev nodeid p1 p2
0 2 99e0332bd498 000000000000 000000000000 0 2 99e0332bd498 000000000000 000000000000
1 3 6674f57a23d8 99e0332bd498 000000000000 1 3 6674f57a23d8 99e0332bd498 000000000000
>>> from mercurial.revlogutils.constants import KIND_OTHER
>>> from mercurial import revlog, vfs >>> from mercurial import revlog, vfs
>>> tvfs = vfs.vfs(b'.') >>> tvfs = vfs.vfs(b'.')
>>> tvfs.options = {b'revlogv1': True} >>> tvfs.options = {b'revlogv1': True}
>>> rl = revlog.revlog(tvfs, b'a.i') >>> rl = revlog.revlog(tvfs, target=(KIND_OTHER, b'test'), indexfile=b'a.i')
>>> rl.revision(1) >>> rl.revision(1)
mpatchError(*'patch cannot be decoded'*) (glob) mpatchError(*'patch cannot be decoded'*) (glob)