( )⚙ D10576 revlog: use a "radix" to address revlog

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10576

revlog: use a "radix" to address revlog
ClosedPublic

Authored by marmoute on May 3 2021, 7:48 AM.

Details

Reviewers
indygreg
Alphare
Group Reviewers
hg-reviewers
Commits
rHG8d3c2f9d4af7: revlog: use a "radix" to address revlog
Summary

Instead of pointing to the index directly and to derive the other file from
that, we directly provide the radix and let the revlog determine the associated
file path internally. This is more robust and will give us more flexibility for
picking this file name in the future.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.May 3 2021, 7:48 AM
baymax updated this revision to Diff 27491.May 3 2021, 9:43 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare added a subscriber: Alphare.May 11 2021, 9:51 AM

I will fix the typo in-flight.

contrib/dumprevlog
42

s/invalida/invalid/

Alphare accepted this revision.May 11 2021, 9:51 AM
This revision is now accepted and ready to land.May 11 2021, 9:51 AM
marmoute added a commit: rHG8d3c2f9d4af7: revlog: use a "radix" to address revlog.May 11 2021, 12:50 PM
Closed by commit rHG8d3c2f9d4af7: revlog: use a "radix" to address revlog (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 27822

contrib/dumprevlog

def printb(data, end=b'\n'): def printb(data, end=b'\n'):
sys.stdout.flush() sys.stdout.flush()
procutil.stdout.write(data + end) procutil.stdout.write(data + end)
for f in sys.argv[1:]: for f in sys.argv[1:]:
localf = encoding.strtolocal(f)
if not localf.endswith(b'.i'):
print("file:", f, file=sys.stderr)
print(" invalida filename", file=sys.stderr)
AlphareUnsubmitted
Not Done

s/invalida/invalid/

Alphare: s/invalida/invalid/
r = revlog.revlog( r = revlog.revlog(
binopen, binopen,
target=(revlog_constants.KIND_OTHER, b'dump-revlog'), target=(revlog_constants.KIND_OTHER, b'dump-revlog'),
indexfile=encoding.strtolocal(f), radix=localf[:-2],
) )
print("file:", f) print("file:", f)
for i in r: for i in r:
n = r.node(i) n = r.node(i)
p = r.parents(n) p = r.parents(n)
d = r.revision(n) d = r.revision(n)
printb(b"node: %s" % hex(n)) printb(b"node: %s" % hex(n))
printb(b"linkrev: %d" % r.linkrev(i)) printb(b"linkrev: %d" % r.linkrev(i))
printb(b"parents: %s %s" % (hex(p[0]), hex(p[1]))) printb(b"parents: %s %s" % (hex(p[0]), hex(p[1])))
printb(b"length: %d" % len(d)) printb(b"length: %d" % len(d))
printb(b"-start-") printb(b"-start-")
printb(d) printb(d)
printb(b"-end-") printb(b"-end-")

contrib/perf.py

def perfnodelookup(ui, repo, rev, **opts): def perfnodelookup(ui, repo, rev, **opts):
opts = _byteskwargs(opts) opts = _byteskwargs(opts)
timer, fm = gettimer(ui, opts) timer, fm = gettimer(ui, opts)
import mercurial.revlog import mercurial.revlog
mercurial.revlog._prereadsize = 2 ** 24 # disable lazy parser in old hg mercurial.revlog._prereadsize = 2 ** 24 # disable lazy parser in old hg
n = scmutil.revsingle(repo, rev).node() n = scmutil.revsingle(repo, rev).node()
try:
cl = revlog(getsvfs(repo), radix=b"00changelog")
except TypeError:
cl = revlog(getsvfs(repo), indexfile=b"00changelog.i") cl = revlog(getsvfs(repo), indexfile=b"00changelog.i")
def d(): def d():
cl.rev(n) cl.rev(n)
clearcaches(cl) clearcaches(cl)
timer(d) timer(d)
fm.end() fm.end()
""" """
opts = _byteskwargs(opts) opts = _byteskwargs(opts)
rl = cmdutil.openrevlog(repo, b'perfrevlogindex', file_, opts) rl = cmdutil.openrevlog(repo, b'perfrevlogindex', file_, opts)
opener = getattr(rl, 'opener') # trick linter opener = getattr(rl, 'opener') # trick linter
# compat with hg <= 5.8 # compat with hg <= 5.8
radix = getattr(rl, 'radix', None)
indexfile = getattr(rl, '_indexfile', None) indexfile = getattr(rl, '_indexfile', None)
if indexfile is None: if indexfile is None:
# compatibility with <= hg-5.8 # compatibility with <= hg-5.8
indexfile = getattr(rl, 'indexfile') indexfile = getattr(rl, 'indexfile')
data = opener.read(indexfile) data = opener.read(indexfile)
header = struct.unpack(b'>I', data[0:4])[0] header = struct.unpack(b'>I', data[0:4])[0]
version = header & 0xFFFF version = header & 0xFFFF
node100 = rl.node(rllen - 1) node100 = rl.node(rllen - 1)
allrevs = range(rllen) allrevs = range(rllen)
allrevsrev = list(reversed(allrevs)) allrevsrev = list(reversed(allrevs))
allnodes = [rl.node(rev) for rev in range(rllen)] allnodes = [rl.node(rev) for rev in range(rllen)]
allnodesrev = list(reversed(allnodes)) allnodesrev = list(reversed(allnodes))
def constructor(): def constructor():
if radix is not None:
revlog(opener, radix=radix)
else:
# hg <= 5.8
revlog(opener, indexfile=indexfile) revlog(opener, indexfile=indexfile)
def read(): def read():
with opener(indexfile) as fh: with opener(indexfile) as fh:
fh.read() fh.read()
def parseindex(): def parseindex():
parse_index_v1(data, inline) parse_index_v1(data, inline)
indexfile = getattr(orig, '_indexfile', None) indexfile = getattr(orig, '_indexfile', None)
if indexfile is None: if indexfile is None:
# compatibility with <= hg-5.8 # compatibility with <= hg-5.8
indexfile = getattr(orig, 'indexfile') indexfile = getattr(orig, 'indexfile')
origindexpath = orig.opener.join(indexfile) origindexpath = orig.opener.join(indexfile)
datafile = getattr(orig, '_datafile', getattr(orig, 'datafile')) datafile = getattr(orig, '_datafile', getattr(orig, 'datafile'))
origdatapath = orig.opener.join(datafile) origdatapath = orig.opener.join(datafile)
indexname = 'revlog.i' radix = b'revlog'
dataname = 'revlog.d' indexname = b'revlog.i'
dataname = b'revlog.d'
tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-') tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-')
try: try:
# copy the data file in a temporary directory # copy the data file in a temporary directory
ui.debug('copying data in %s\n' % tmpdir) ui.debug('copying data in %s\n' % tmpdir)
destindexpath = os.path.join(tmpdir, 'revlog.i') destindexpath = os.path.join(tmpdir, 'revlog.i')
destdatapath = os.path.join(tmpdir, 'revlog.d') destdatapath = os.path.join(tmpdir, 'revlog.d')
shutil.copyfile(origindexpath, destindexpath) shutil.copyfile(origindexpath, destindexpath)
shutil.copyfile(origdatapath, destdatapath) shutil.copyfile(origdatapath, destdatapath)
# remove the data we want to add again # remove the data we want to add again
ui.debug('truncating data to be rewritten\n') ui.debug('truncating data to be rewritten\n')
with open(destindexpath, 'ab') as index: with open(destindexpath, 'ab') as index:
index.seek(0) index.seek(0)
index.truncate(truncaterev * orig._io.size) index.truncate(truncaterev * orig._io.size)
with open(destdatapath, 'ab') as data: with open(destdatapath, 'ab') as data:
data.seek(0) data.seek(0)
data.truncate(orig.start(truncaterev)) data.truncate(orig.start(truncaterev))
# instantiate a new revlog from the temporary copy # instantiate a new revlog from the temporary copy
ui.debug('truncating adding to be rewritten\n') ui.debug('truncating adding to be rewritten\n')
vfs = vfsmod.vfs(tmpdir) vfs = vfsmod.vfs(tmpdir)
vfs.options = getattr(orig.opener, 'options', None) vfs.options = getattr(orig.opener, 'options', None)
try:
dest = revlog(vfs, radix=radix, **revlogkwargs)
except TypeError:
dest = revlog( dest = revlog(
vfs, indexfile=indexname, datafile=dataname, **revlogkwargs vfs, indexfile=indexname, datafile=dataname, **revlogkwargs
) )
if dest._inline: if dest._inline:
raise error.Abort('not supporting inline revlog (yet)') raise error.Abort('not supporting inline revlog (yet)')
# make sure internals are initialized # make sure internals are initialized
dest.revision(len(dest) - 1) dest.revision(len(dest) - 1)
yield dest yield dest
del dest, vfs del dest, vfs
finally: finally:
shutil.rmtree(tmpdir, True) shutil.rmtree(tmpdir, True)

contrib/undumprevlog

sys.stderr.write, opener, {b'store': opener}, b"undump.journal" sys.stderr.write, opener, {b'store': opener}, b"undump.journal"
) )
while True: while True:
l = sys.stdin.readline() l = sys.stdin.readline()
if not l: if not l:
break break
if l.startswith("file:"): if l.startswith("file:"):
f = encoding.strtolocal(l[6:-1]) f = encoding.strtolocal(l[6:-1])
assert f.endswith(b'.i')
r = revlog.revlog( r = revlog.revlog(
opener, opener,
target=(revlog_constants.KIND_OTHER, b'undump-revlog'), target=(revlog_constants.KIND_OTHER, b'undump-revlog'),
indexfile=f, radix=f[:-2],
) )
procutil.stdout.write(b'%s\n' % f) procutil.stdout.write(b'%s\n' % f)
elif l.startswith("node:"): elif l.startswith("node:"):
n = bin(l[6:-1]) n = bin(l[6:-1])
elif l.startswith("linkrev:"): elif l.startswith("linkrev:"):
lr = int(l[9:-1]) lr = int(l[9:-1])
elif l.startswith("parents:"): elif l.startswith("parents:"):
p = l[9:-1].split() p = l[9:-1].split()

hgext/remotefilelog/contentstore.py

pass pass
class manifestrevlogstore(object): class manifestrevlogstore(object):
def __init__(self, repo): def __init__(self, repo):
self._store = repo.store self._store = repo.store
self._svfs = repo.svfs self._svfs = repo.svfs
self._revlogs = dict() self._revlogs = dict()
self._cl = revlog.revlog(self._svfs, indexfile=b'00changelog.i') self._cl = revlog.revlog(self._svfs, radix=b'00changelog.i')
self._repackstartlinkrev = 0 self._repackstartlinkrev = 0
def get(self, name, node): def get(self, name, node):
return self._revlog(name).rawdata(node) return self._revlog(name).rawdata(node)
def getdelta(self, name, node): def getdelta(self, name, node):
revision = self.get(name, node) revision = self.get(name, node)
return revision, name, self._cl.nullid, self.getmeta(name, node) return revision, name, self._cl.nullid, self.getmeta(name, node)
return (parents[0], parents[1], cl.node(linkrev), None) return (parents[0], parents[1], cl.node(linkrev), None)
def add(self, *args): def add(self, *args):
raise RuntimeError(b"cannot add to a revlog store") raise RuntimeError(b"cannot add to a revlog store")
def _revlog(self, name): def _revlog(self, name):
rl = self._revlogs.get(name) rl = self._revlogs.get(name)
if rl is None: if rl is None:
revlogname = b'00manifesttree.i' revlogname = b'00manifesttree'
if name != b'': if name != b'':
revlogname = b'meta/%s/00manifest.i' % name revlogname = b'meta/%s/00manifest' % name
rl = revlog.revlog(self._svfs, indexfile=revlogname) rl = revlog.revlog(self._svfs, radix=revlogname)
self._revlogs[name] = rl self._revlogs[name] = rl
return rl return rl
def getmissing(self, keys): def getmissing(self, keys):
missing = [] missing = []
for name, node in keys: for name, node in keys:
mfrevlog = self._revlog(name) mfrevlog = self._revlog(name)
if node not in mfrevlog.nodemap: if node not in mfrevlog.nodemap:
missing.append((name, node)) missing.append((name, node))
return missing return missing
def setrepacklinkrevrange(self, startrev, endrev): def setrepacklinkrevrange(self, startrev, endrev):
self._repackstartlinkrev = startrev self._repackstartlinkrev = startrev
self._repackendlinkrev = endrev self._repackendlinkrev = endrev
def markledger(self, ledger, options=None): def markledger(self, ledger, options=None):
if options and options.get(constants.OPTION_PACKSONLY): if options and options.get(constants.OPTION_PACKSONLY):
return return
treename = b'' treename = b''
rl = revlog.revlog(self._svfs, indexfile=b'00manifesttree.i') rl = revlog.revlog(self._svfs, radix=b'00manifesttree')
startlinkrev = self._repackstartlinkrev startlinkrev = self._repackstartlinkrev
endlinkrev = self._repackendlinkrev endlinkrev = self._repackendlinkrev
for rev in pycompat.xrange(len(rl) - 1, -1, -1): for rev in pycompat.xrange(len(rl) - 1, -1, -1):
linkrev = rl.linkrev(rev) linkrev = rl.linkrev(rev)
if linkrev < startlinkrev: if linkrev < startlinkrev:
break break
if linkrev > endlinkrev: if linkrev > endlinkrev:
continue continue
node = rl.node(rev) node = rl.node(rev)
ledger.markdataentry(self, treename, node) ledger.markdataentry(self, treename, node)
ledger.markhistoryentry(self, treename, node) ledger.markhistoryentry(self, treename, node)
for t, path, encoded, size in self._store.datafiles(): for t, path, encoded, size in self._store.datafiles():
if path[:5] != b'meta/' or path[-2:] != b'.i': if path[:5] != b'meta/' or path[-2:] != b'.i':
continue continue
treename = path[5 : -len(b'/00manifest.i')] treename = path[5 : -len(b'/00manifest')]
rl = revlog.revlog(self._svfs, indexfile=path) rl = revlog.revlog(self._svfs, indexfile=path[:-2])
for rev in pycompat.xrange(len(rl) - 1, -1, -1): for rev in pycompat.xrange(len(rl) - 1, -1, -1):
linkrev = rl.linkrev(rev) linkrev = rl.linkrev(rev)
if linkrev < startlinkrev: if linkrev < startlinkrev:
break break
if linkrev > endlinkrev: if linkrev > endlinkrev:
continue continue
node = rl.node(rev) node = rl.node(rev)
ledger.markdataentry(self, treename, node) ledger.markdataentry(self, treename, node)
ledger.markhistoryentry(self, treename, node) ledger.markhistoryentry(self, treename, node)
def cleanup(self, ledger): def cleanup(self, ledger):
pass pass

mercurial/bundlerepo.py

) )
from .revlogutils import ( from .revlogutils import (
constants as revlog_constants, constants as revlog_constants,
) )
class bundlerevlog(revlog.revlog): class bundlerevlog(revlog.revlog):
def __init__(self, opener, target, indexfile, cgunpacker, linkmapper): def __init__(self, opener, target, radix, cgunpacker, linkmapper):
# How it works: # How it works:
# To retrieve a revision, we need to know the offset of the revision in # To retrieve a revision, we need to know the offset of the revision in
# the bundle (an unbundle object). We store this offset in the index # the bundle (an unbundle object). We store this offset in the index
# (start). The base of the delta is stored in the base field. # (start). The base of the delta is stored in the base field.
# #
# To differentiate a rev in the bundle from a rev in the revlog, we # To differentiate a rev in the bundle from a rev in the revlog, we
# check revision against repotiprev. # check revision against repotiprev.
opener = vfsmod.readonlyvfs(opener) opener = vfsmod.readonlyvfs(opener)
revlog.revlog.__init__(self, opener, target=target, indexfile=indexfile) revlog.revlog.__init__(self, opener, target=target, radix=radix)
self.bundle = cgunpacker self.bundle = cgunpacker
n = len(self) n = len(self)
self.repotiprev = n - 1 self.repotiprev = n - 1
self.bundlerevs = set() # used by 'bundle()' revset expression self.bundlerevs = set() # used by 'bundle()' revset expression
for deltadata in cgunpacker.deltaiter(): for deltadata in cgunpacker.deltaiter():
node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
size = len(delta) size = len(delta)
class bundlechangelog(bundlerevlog, changelog.changelog): class bundlechangelog(bundlerevlog, changelog.changelog):
def __init__(self, opener, cgunpacker): def __init__(self, opener, cgunpacker):
changelog.changelog.__init__(self, opener) changelog.changelog.__init__(self, opener)
linkmapper = lambda x: x linkmapper = lambda x: x
bundlerevlog.__init__( bundlerevlog.__init__(
self, self,
opener, opener,
(revlog_constants.KIND_CHANGELOG, None), (revlog_constants.KIND_CHANGELOG, None),
self._indexfile, self.radix,
cgunpacker, cgunpacker,
linkmapper, linkmapper,
) )
class bundlemanifest(bundlerevlog, manifest.manifestrevlog): class bundlemanifest(bundlerevlog, manifest.manifestrevlog):
def __init__( def __init__(
self, self,
nodeconstants, nodeconstants,
opener, opener,
cgunpacker, cgunpacker,
linkmapper, linkmapper,
dirlogstarts=None, dirlogstarts=None,
dir=b'', dir=b'',
): ):
manifest.manifestrevlog.__init__(self, nodeconstants, opener, tree=dir) manifest.manifestrevlog.__init__(self, nodeconstants, opener, tree=dir)
bundlerevlog.__init__( bundlerevlog.__init__(
self, self,
opener, opener,
(revlog_constants.KIND_MANIFESTLOG, dir), (revlog_constants.KIND_MANIFESTLOG, dir),
self._revlog._indexfile, self._revlog.radix,
cgunpacker, cgunpacker,
linkmapper, linkmapper,
) )
if dirlogstarts is None: if dirlogstarts is None:
dirlogstarts = {} dirlogstarts = {}
if self.bundle.version == b"03": if self.bundle.version == b"03":
dirlogstarts = _getfilestarts(self.bundle) dirlogstarts = _getfilestarts(self.bundle)
self._dirlogstarts = dirlogstarts self._dirlogstarts = dirlogstarts
class bundlefilelog(filelog.filelog): class bundlefilelog(filelog.filelog):
def __init__(self, opener, path, cgunpacker, linkmapper): def __init__(self, opener, path, cgunpacker, linkmapper):
filelog.filelog.__init__(self, opener, path) filelog.filelog.__init__(self, opener, path)
self._revlog = bundlerevlog( self._revlog = bundlerevlog(
opener, opener,
# XXX should use the unencoded path # XXX should use the unencoded path
target=(revlog_constants.KIND_FILELOG, path), target=(revlog_constants.KIND_FILELOG, path),
indexfile=self._revlog._indexfile, radix=self._revlog.radix,
cgunpacker=cgunpacker, cgunpacker=cgunpacker,
linkmapper=linkmapper, linkmapper=linkmapper,
) )
class bundlepeer(localrepo.localpeer): class bundlepeer(localrepo.localpeer):
def canpush(self): def canpush(self):
return False return False

mercurial/changelog.py

revision) data for a transaction that hasn't been finalized yet. revision) data for a transaction that hasn't been finalized yet.
It exists in a separate file to facilitate readers (such as It exists in a separate file to facilitate readers (such as
hooks processes) accessing data before a transaction is finalized. hooks processes) accessing data before a transaction is finalized.
``concurrencychecker`` will be passed to the revlog init function, see ``concurrencychecker`` will be passed to the revlog init function, see
the documentation there. the documentation there.
""" """
indexfile = b'00changelog.i'
if trypending and opener.exists(b'00changelog.i.a'): if trypending and opener.exists(b'00changelog.i.a'):
postfix = b'a' postfix = b'a'
else: else:
postfix = None postfix = None
datafile = b'00changelog.d'
revlog.revlog.__init__( revlog.revlog.__init__(
self, self,
opener, opener,
target=(revlog_constants.KIND_CHANGELOG, None), target=(revlog_constants.KIND_CHANGELOG, None),
radix=b'00changelog',
postfix=postfix, postfix=postfix,
indexfile=indexfile,
datafile=datafile,
checkambig=True, checkambig=True,
mmaplargeindex=True, mmaplargeindex=True,
persistentnodemap=opener.options.get(b'persistent-nodemap', False), persistentnodemap=opener.options.get(b'persistent-nodemap', False),
concurrencychecker=concurrencychecker, concurrencychecker=concurrencychecker,
) )
if self._initempty and (self._format_version == revlog.REVLOGV1): if self._initempty and (self._format_version == revlog.REVLOGV1):
# changelogs don't benefit from generaldelta. # changelogs don't benefit from generaldelta.

mercurial/cmdutil.py

raise error.CommandError(cmd, _(b'invalid arguments')) raise error.CommandError(cmd, _(b'invalid arguments'))
if not os.path.isfile(file_): if not os.path.isfile(file_):
raise error.InputError(_(b"revlog '%s' not found") % file_) raise error.InputError(_(b"revlog '%s' not found") % file_)
target = (revlog_constants.KIND_OTHER, b'free-form:%s' % file_) target = (revlog_constants.KIND_OTHER, b'free-form:%s' % file_)
r = revlog.revlog( r = revlog.revlog(
vfsmod.vfs(encoding.getcwd(), audit=False), vfsmod.vfs(encoding.getcwd(), audit=False),
target=target, target=target,
indexfile=file_[:-2] + b".i", radix=file_[:-2],
) )
return r return r
def openrevlog(repo, cmd, file_, opts): def openrevlog(repo, cmd, file_, opts):
"""Obtain a revlog backing storage of an item. """Obtain a revlog backing storage of an item.
This is similar to ``openstorage()`` except it always returns a revlog. This is similar to ``openstorage()`` except it always returns a revlog.

mercurial/filelog.py

@interfaceutil.implementer(repository.ifilestorage) @interfaceutil.implementer(repository.ifilestorage)
class filelog(object): class filelog(object):
def __init__(self, opener, path): def __init__(self, opener, path):
self._revlog = revlog.revlog( self._revlog = revlog.revlog(
opener, opener,
# XXX should use the unencoded path # XXX should use the unencoded path
target=(revlog_constants.KIND_FILELOG, path), target=(revlog_constants.KIND_FILELOG, path),
indexfile=b'/'.join((b'data', path + b'.i')), radix=b'/'.join((b'data', path)),
censorable=True, censorable=True,
) )
# Full name of the user visible file, relative to the repository root. # Full name of the user visible file, relative to the repository root.
# Used by LFS. # Used by LFS.
self._revlog.filename = path self._revlog.filename = path
self.nullid = self._revlog.nullid self.nullid = self._revlog.nullid
def __len__(self): def __len__(self):

mercurial/manifest.py

""" """
def __init__( def __init__(
self, self,
nodeconstants, nodeconstants,
opener, opener,
tree=b'', tree=b'',
dirlogcache=None, dirlogcache=None,
indexfile=None,
treemanifest=False, treemanifest=False,
): ):
"""Constructs a new manifest revlog """Constructs a new manifest revlog
`indexfile` - used by extensions to have two manifests at once, like `indexfile` - used by extensions to have two manifests at once, like
when transitioning between flatmanifeset and treemanifests. when transitioning between flatmanifeset and treemanifests.
`treemanifest` - used to indicate this is a tree manifest revlog. Opener `treemanifest` - used to indicate this is a tree manifest revlog. Opener
self._treeondisk = optiontreemanifest or treemanifest self._treeondisk = optiontreemanifest or treemanifest
self._fulltextcache = manifestfulltextcache(cachesize) self._fulltextcache = manifestfulltextcache(cachesize)
if tree: if tree:
assert self._treeondisk, b'opts is %r' % opts assert self._treeondisk, b'opts is %r' % opts
if indexfile is None: radix = b'00manifest'
indexfile = b'00manifest.i'
if tree: if tree:
indexfile = b"meta/" + tree + indexfile radix = b"meta/" + tree + radix
self.tree = tree self.tree = tree
# The dirlogcache is kept on the root manifest log # The dirlogcache is kept on the root manifest log
if tree: if tree:
self._dirlogcache = dirlogcache self._dirlogcache = dirlogcache
else: else:
self._dirlogcache = {b'': self} self._dirlogcache = {b'': self}
self._revlog = revlog.revlog( self._revlog = revlog.revlog(
opener, opener,
target=(revlog_constants.KIND_MANIFESTLOG, self.tree), target=(revlog_constants.KIND_MANIFESTLOG, self.tree),
indexfile=indexfile, radix=radix,
# only root indexfile is cached # only root indexfile is cached
checkambig=not bool(tree), checkambig=not bool(tree),
mmaplargeindex=True, mmaplargeindex=True,
upperboundcomp=MAXCOMPRESSION, upperboundcomp=MAXCOMPRESSION,
persistentnodemap=opener.options.get(b'persistent-nodemap', False), persistentnodemap=opener.options.get(b'persistent-nodemap', False),
) )
self.index = self._revlog.index self.index = self._revlog.index

mercurial/revlog.py

""" """
_flagserrorclass = error.RevlogError _flagserrorclass = error.RevlogError
def __init__( def __init__(
self, self,
opener, opener,
target, target,
radix,
postfix=None, postfix=None,
indexfile=None,
datafile=None,
checkambig=False, checkambig=False,
mmaplargeindex=False, mmaplargeindex=False,
censorable=False, censorable=False,
upperboundcomp=None, upperboundcomp=None,
persistentnodemap=False, persistentnodemap=False,
concurrencychecker=None, concurrencychecker=None,
): ):
""" """
create a revlog object create a revlog object
opener is a function that abstracts the file opening operation opener is a function that abstracts the file opening operation
and can be used to implement COW semantics or the like. and can be used to implement COW semantics or the like.
`target`: a (KIND, ID) tuple that identify the content stored in `target`: a (KIND, ID) tuple that identify the content stored in
this revlog. It help the rest of the code to understand what the revlog this revlog. It help the rest of the code to understand what the revlog
is about without having to resort to heuristic and index filename is about without having to resort to heuristic and index filename
analysis. Note: that this must be reliably be set by normal code, but analysis. Note: that this must be reliably be set by normal code, but
that test, debug, or performance measurement code might not set this to that test, debug, or performance measurement code might not set this to
accurate value. accurate value.
""" """
self.upperboundcomp = upperboundcomp self.upperboundcomp = upperboundcomp
if not indexfile.endswith(b'.i'):
raise error.ProgrammingError( self.radix = radix
b"revlog's indexfile should end with `.i`"
) if postfix is None:
if datafile is None: indexfile = b'%s.i' % self.radix
datafile = indexfile[:-2] + b".d" datafile = b'%s.d' % self.radix
if postfix is not None: elif postfix == b'a':
datafile = b'%s.%s' % (datafile, postfix) indexfile = b'%s.i.a' % self.radix
if postfix is not None: datafile = b'%s.d' % self.radix
indexfile = b'%s.%s' % (indexfile, postfix) else:
indexfile = b'%s.i.%s' % (self.radix, postfix)
datafile = b'%s.d.%s' % (self.radix, postfix)
self._indexfile = indexfile self._indexfile = indexfile
self._datafile = datafile self._datafile = datafile
self.nodemap_file = None self.nodemap_file = None
self.postfix = postfix self.postfix = postfix
if persistentnodemap: if persistentnodemap:
self.nodemap_file = nodemaputil.get_nodemap_file( self.nodemap_file = nodemaputil.get_nodemap_file(
opener, self._indexfile opener, self._indexfile
) )
# Rewriting the revlog in place is hard. Our strategy for censoring is # Rewriting the revlog in place is hard. Our strategy for censoring is
# to create a new revlog, copy all revisions to it, then replace the # to create a new revlog, copy all revisions to it, then replace the
# revlogs on transaction close. # revlogs on transaction close.
# #
# This is a bit dangerous. We could easily have a mismatch of state. # This is a bit dangerous. We could easily have a mismatch of state.
newrl = revlog( newrl = revlog(
self.opener, self.opener,
target=self.target, target=self.target,
radix=self.radix,
postfix=b'tmpcensored', postfix=b'tmpcensored',
indexfile=self._indexfile,
censorable=True, censorable=True,
) )
newrl._format_version = self._format_version newrl._format_version = self._format_version
newrl._format_flags = self._format_flags newrl._format_flags = self._format_flags
newrl._generaldelta = self._generaldelta newrl._generaldelta = self._generaldelta
newrl._parse_index = self._parse_index newrl._parse_index = self._parse_index
for rev in self.revs(): for rev in self.revs():

mercurial/unionrepo.py

pathutil, pathutil,
revlog, revlog,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
class unionrevlog(revlog.revlog): class unionrevlog(revlog.revlog):
def __init__(self, opener, indexfile, revlog2, linkmapper): def __init__(self, opener, radix, revlog2, linkmapper):
# How it works: # How it works:
# To retrieve a revision, we just need to know the node id so we can # To retrieve a revision, we just need to know the node id so we can
# look it up in revlog2. # look it up in revlog2.
# #
# To differentiate a rev in the second revlog from a rev in the revlog, # To differentiate a rev in the second revlog from a rev in the revlog,
# we check revision against repotiprev. # we check revision against repotiprev.
opener = vfsmod.readonlyvfs(opener) opener = vfsmod.readonlyvfs(opener)
target = getattr(revlog2, 'target', None) target = getattr(revlog2, 'target', None)
if target is None: if target is None:
# a revlog wrapper, eg: the manifestlog that is not an actual revlog # a revlog wrapper, eg: the manifestlog that is not an actual revlog
target = revlog2._revlog.target target = revlog2._revlog.target
revlog.revlog.__init__(self, opener, target=target, indexfile=indexfile) revlog.revlog.__init__(self, opener, target=target, radix=radix)
self.revlog2 = revlog2 self.revlog2 = revlog2
n = len(self) n = len(self)
self.repotiprev = n - 1 self.repotiprev = n - 1
self.bundlerevs = set() # used by 'bundle()' revset expression self.bundlerevs = set() # used by 'bundle()' revset expression
for rev2 in self.revlog2: for rev2 in self.revlog2:
rev = self.revlog2.index[rev2] rev = self.revlog2.index[rev2]
# rev numbers - in revlog2, very different from self.rev # rev numbers - in revlog2, very different from self.rev
raise NotImplementedError raise NotImplementedError
class unionchangelog(unionrevlog, changelog.changelog): class unionchangelog(unionrevlog, changelog.changelog):
def __init__(self, opener, opener2): def __init__(self, opener, opener2):
changelog.changelog.__init__(self, opener) changelog.changelog.__init__(self, opener)
linkmapper = None linkmapper = None
changelog2 = changelog.changelog(opener2) changelog2 = changelog.changelog(opener2)
unionrevlog.__init__( unionrevlog.__init__(self, opener, self.radix, changelog2, linkmapper)
self, opener, self._indexfile, changelog2, linkmapper
)
class unionmanifest(unionrevlog, manifest.manifestrevlog): class unionmanifest(unionrevlog, manifest.manifestrevlog):
def __init__(self, nodeconstants, opener, opener2, linkmapper): def __init__(self, nodeconstants, opener, opener2, linkmapper):
manifest.manifestrevlog.__init__(self, nodeconstants, opener) manifest.manifestrevlog.__init__(self, nodeconstants, opener)
manifest2 = manifest.manifestrevlog(nodeconstants, opener2) manifest2 = manifest.manifestrevlog(nodeconstants, opener2)
unionrevlog.__init__( unionrevlog.__init__(
self, opener, self._revlog._indexfile, manifest2, linkmapper self, opener, self._revlog.radix, manifest2, linkmapper
) )
class unionfilelog(filelog.filelog): class unionfilelog(filelog.filelog):
def __init__(self, opener, path, opener2, linkmapper, repo): def __init__(self, opener, path, opener2, linkmapper, repo):
filelog.filelog.__init__(self, opener, path) filelog.filelog.__init__(self, opener, path)
filelog2 = filelog.filelog(opener2, path) filelog2 = filelog.filelog(opener2, path)
self._revlog = unionrevlog( self._revlog = unionrevlog(
opener, self._revlog._indexfile, filelog2._revlog, linkmapper opener, self._revlog.radix, filelog2._revlog, linkmapper
) )
self._repo = repo self._repo = repo
self.repotiprev = self._revlog.repotiprev self.repotiprev = self._revlog.repotiprev
self.revlog2 = self._revlog.revlog2 self.revlog2 = self._revlog.revlog2
def iscensored(self, rev): def iscensored(self, rev):
"""Check if a revision is censored.""" """Check if a revision is censored."""
if rev <= self.repotiprev: if rev <= self.repotiprev:

tests/test-revlog-raw.py

def newtransaction(): def newtransaction():
# A transaction is required to write revlogs # A transaction is required to write revlogs
report = lambda msg: None report = lambda msg: None
return transaction.transaction(report, tvfs, {'plain': tvfs}, b'journal') return transaction.transaction(report, tvfs, {'plain': tvfs}, b'journal')
def newrevlog(name=b'_testrevlog.i', recreate=False): def newrevlog(name=b'_testrevlog', recreate=False):
if recreate: if recreate:
tvfs.tryunlink(name) tvfs.tryunlink(name + b'.i')
rlog = revlog.revlog( target = (constants.KIND_OTHER, b'test')
tvfs, target=(constants.KIND_OTHER, b'test'), indexfile=name rlog = revlog.revlog(tvfs, target=target, radix=name)
)
return rlog return rlog
def appendrev(rlog, text, tr, isext=False, isdelta=True): def appendrev(rlog, text, tr, isext=False, isdelta=True):
"""Append a revision. If isext is True, set the EXTSTORED flag so flag """Append a revision. If isext is True, set the EXTSTORED flag so flag
processor will be used (and rawtext is different from text). If isdelta is processor will be used (and rawtext is different from text). If isdelta is
True, force the revision to be a delta, otherwise it's full text. True, force the revision to be a delta, otherwise it's full text.
""" """
return nextrev return nextrev
except Exception as ex: except Exception as ex:
abort('rev %d: failed to append: %s' % (nextrev, ex)) abort('rev %d: failed to append: %s' % (nextrev, ex))
finally: finally:
# Restore storedeltachains. It is always True, see revlog.__init__ # Restore storedeltachains. It is always True, see revlog.__init__
rlog._storedeltachains = True rlog._storedeltachains = True
def addgroupcopy(rlog, tr, destname=b'_destrevlog.i', optimaldelta=True): def addgroupcopy(rlog, tr, destname=b'_destrevlog', optimaldelta=True):
"""Copy revlog to destname using revlog.addgroup. Return the copied revlog. """Copy revlog to destname using revlog.addgroup. Return the copied revlog.
This emulates push or pull. They use changegroup. Changegroup requires This emulates push or pull. They use changegroup. Changegroup requires
repo to work. We don't have a repo, so a dummy changegroup is used. repo to work. We don't have a repo, so a dummy changegroup is used.
If optimaldelta is True, use optimized delta parent, so the destination If optimaldelta is True, use optimized delta parent, so the destination
revlog could probably reuse it. Otherwise it builds sub-optimal delta, and revlog could probably reuse it. Otherwise it builds sub-optimal delta, and
the destination revlog needs more work to use it. the destination revlog needs more work to use it.
return rlog.rev(lnode) return rlog.rev(lnode)
dlog = newrevlog(destname, recreate=True) dlog = newrevlog(destname, recreate=True)
dummydeltas = dummychangegroup().deltaiter() dummydeltas = dummychangegroup().deltaiter()
dlog.addgroup(dummydeltas, linkmap, tr) dlog.addgroup(dummydeltas, linkmap, tr)
return dlog return dlog
def lowlevelcopy(rlog, tr, destname=b'_destrevlog.i'): def lowlevelcopy(rlog, tr, destname=b'_destrevlog'):
"""Like addgroupcopy, but use the low level revlog._addrevision directly. """Like addgroupcopy, but use the low level revlog._addrevision directly.
It exercises some code paths that are hard to reach easily otherwise. It exercises some code paths that are hard to reach easily otherwise.
""" """
dlog = newrevlog(destname, recreate=True) dlog = newrevlog(destname, recreate=True)
for r in rlog: for r in rlog:
p1 = rlog.node(r - 1) p1 = rlog.node(r - 1)
p2 = rlog.nullid p2 = rlog.nullid
_maketext((0, 120), (60, 60), (567, 30), (629, 40)), _maketext((0, 120), (60, 60), (567, 30), (629, 40)),
_maketext((0, 120), (60, 60), (569, 30), (745, 40)), _maketext((0, 120), (60, 60), (569, 30), (745, 40)),
_maketext((0, 120), (60, 60), (777, 30), (700, 40)), _maketext((0, 120), (60, 60), (777, 30), (700, 40)),
_maketext((0, 120), (60, 60), (618, 30), (398, 40), (158, 10)), _maketext((0, 120), (60, 60), (618, 30), (398, 40), (158, 10)),
] ]
def makesnapshot(tr): def makesnapshot(tr):
rl = newrevlog(name=b'_snaprevlog3.i', recreate=True) rl = newrevlog(name=b'_snaprevlog3', recreate=True)
for i in data: for i in data:
appendrev(rl, i, tr) appendrev(rl, i, tr)
return rl return rl
snapshots = [-1, 0, 6, 8, 11, 17, 19, 21, 25, 30] snapshots = [-1, 0, 6, 8, 11, 17, 19, 21, 25, 30]
print('local test passed') print('local test passed')
# Copy via revlog.addgroup # Copy via revlog.addgroup
rl1 = addgroupcopy(rl, tr) rl1 = addgroupcopy(rl, tr)
checkrevlog(rl1, expected) checkrevlog(rl1, expected)
rl2 = addgroupcopy(rl, tr, optimaldelta=False) rl2 = addgroupcopy(rl, tr, optimaldelta=False)
checkrevlog(rl2, expected) checkrevlog(rl2, expected)
print('addgroupcopy test passed') print('addgroupcopy test passed')
# Copy via revlog.clone # Copy via revlog.clone
rl3 = newrevlog(name=b'_destrevlog3.i', recreate=True) rl3 = newrevlog(name=b'_destrevlog3', recreate=True)
rl.clone(tr, rl3) rl.clone(tr, rl3)
checkrevlog(rl3, expected) checkrevlog(rl3, expected)
print('clone test passed') print('clone test passed')
# Copy via low-level revlog._addrevision # Copy via low-level revlog._addrevision
rl4 = lowlevelcopy(rl, tr) rl4 = lowlevelcopy(rl, tr)
checkrevlog(rl4, expected) checkrevlog(rl4, expected)
print('lowlevelcopy test passed') print('lowlevelcopy test passed')
slicingtest(rl) slicingtest(rl)

tests/test-revlog.t

rev linkrev nodeid p1 p2 rev linkrev nodeid p1 p2
0 2 99e0332bd498 000000000000 000000000000 0 2 99e0332bd498 000000000000 000000000000
1 3 6674f57a23d8 99e0332bd498 000000000000 1 3 6674f57a23d8 99e0332bd498 000000000000
>>> from mercurial.revlogutils.constants import KIND_OTHER >>> from mercurial.revlogutils.constants import KIND_OTHER
>>> from mercurial import revlog, vfs >>> from mercurial import revlog, vfs
>>> tvfs = vfs.vfs(b'.') >>> tvfs = vfs.vfs(b'.')
>>> tvfs.options = {b'revlogv1': True} >>> tvfs.options = {b'revlogv1': True}
>>> rl = revlog.revlog(tvfs, target=(KIND_OTHER, b'test'), indexfile=b'a.i') >>> rl = revlog.revlog(tvfs, target=(KIND_OTHER, b'test'), radix=b'a')
>>> rl.revision(1) >>> rl.revision(1)
mpatchError(*'patch cannot be decoded'*) (glob) mpatchError(*'patch cannot be decoded'*) (glob)