This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4277

manifest: add getstorage() to manifestlog and use it globally
ClosedPublic

Authored by indygreg on Aug 13 2018, 12:55 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG73cf21b2e8a6: manifest: add getstorage() to manifestlog and use it globally
Summary

It is a common pattern to obtain a directory manifest storage instance
(a manifestrevlog) by going through manifestlog._revlog.dirlog().

Why access to storage and caching of other manifests is done through
manifestrevlog instead of manifestlog, I don't know.

This commit establishes a getstorage(tree) API on manifestlog and
imanifestlog that provides a public API for accessing manifest storage.

All consumers previously using private attributes have been updated
to use this new method.

.. api:: manifestlog now has a getstorage(tree) method

It should be used for obtaining an object representing the
manifest's storage implementation. Accessing
manifestlog._revlog should be avoided.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

Diff 10524

hgext/convert/hg.py

if ctx.phase() < phases.draft: if ctx.phase() < phases.draft:
phases.registernew(self.repo, tr, phases.draft, phases.registernew(self.repo, tr, phases.draft,
[ctx.node()]) [ctx.node()])
text = "(octopus merge fixup)\n" text = "(octopus merge fixup)\n"
p2 = node p2 = node
if self.filemapmode and nparents == 1: if self.filemapmode and nparents == 1:
man = self.repo.manifestlog._revlog man = self.repo.manifestlog.getstorage(b'')
mnode = self.repo.changelog.read(nodemod.bin(p2))[0] mnode = self.repo.changelog.read(nodemod.bin(p2))[0]
closed = 'close' in commit.extra closed = 'close' in commit.extra
if not closed and not man.cmp(m1node, man.revision(mnode)): if not closed and not man.cmp(m1node, man.revision(mnode)):
self.ui.status(_("filtering out empty revision\n")) self.ui.status(_("filtering out empty revision\n"))
self.repo.rollback(force=True) self.repo.rollback(force=True)
return parent return parent
return p2 return p2

mercurial/changegroup.py

def _unpackmanifests(self, repo, revmap, trp, prog): def _unpackmanifests(self, repo, revmap, trp, prog):
self.callback = prog.increment self.callback = prog.increment
# no need to check for empty manifest group here: # no need to check for empty manifest group here:
# if the result of the merge of 1 and 2 is the same in 3 and 4, # if the result of the merge of 1 and 2 is the same in 3 and 4,
# no new manifest will be created and the manifest group will # no new manifest will be created and the manifest group will
# be empty during the pull # be empty during the pull
self.manifestheader() self.manifestheader()
deltas = self.deltaiter() deltas = self.deltaiter()
# TODO this violates storage abstraction. repo.manifestlog.getstorage(b'').addgroup(deltas, revmap, trp)
repo.manifestlog._revlog.addgroup(deltas, revmap, trp)
prog.complete() prog.complete()
self.callback = None self.callback = None
def apply(self, repo, tr, srctype, url, targetphase=phases.draft, def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
expectedtotal=None): expectedtotal=None):
"""Add the changegroup returned by source.read() to this repo. """Add the changegroup returned by source.read() to this repo.
srctype is a string like 'push', 'pull', or 'unbundle'. url is srctype is a string like 'push', 'pull', or 'unbundle'. url is
the URL of the repo where this changegroup is coming from. the URL of the repo where this changegroup is coming from.
return node, p1, p2, deltabase, cs, flags return node, p1, p2, deltabase, cs, flags
def _unpackmanifests(self, repo, revmap, trp, prog): def _unpackmanifests(self, repo, revmap, trp, prog):
super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog) super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
for chunkdata in iter(self.filelogheader, {}): for chunkdata in iter(self.filelogheader, {}):
# If we get here, there are directory manifests in the changegroup # If we get here, there are directory manifests in the changegroup
d = chunkdata["filename"] d = chunkdata["filename"]
repo.ui.debug("adding %s revisions\n" % d) repo.ui.debug("adding %s revisions\n" % d)
dirlog = repo.manifestlog._revlog.dirlog(d)
deltas = self.deltaiter() deltas = self.deltaiter()
if not dirlog.addgroup(deltas, revmap, trp): if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp):
raise error.Abort(_("received dir revlog group is empty")) raise error.Abort(_("received dir revlog group is empty"))
class headerlessfixup(object): class headerlessfixup(object):
def __init__(self, fh, h): def __init__(self, fh, h):
self._h = h self._h = h
self._fh = fh self._fh = fh
def read(self, n): def read(self, n):
if self._h: if self._h:
manifests, fnodes, source, clrevtolocalrev): manifests, fnodes, source, clrevtolocalrev):
"""Returns an iterator of changegroup chunks containing manifests. """Returns an iterator of changegroup chunks containing manifests.
`source` is unused here, but is used by extensions like remotefilelog to `source` is unused here, but is used by extensions like remotefilelog to
change what is sent based in pulls vs pushes, etc. change what is sent based in pulls vs pushes, etc.
""" """
repo = self._repo repo = self._repo
mfl = repo.manifestlog mfl = repo.manifestlog
dirlog = mfl._revlog.dirlog
tmfnodes = {'': manifests} tmfnodes = {'': manifests}
# Callback for the manifest, used to collect linkrevs for filelog # Callback for the manifest, used to collect linkrevs for filelog
# revisions. # revisions.
# Returns the linkrev node (collected in lookupcl). # Returns the linkrev node (collected in lookupcl).
def makelookupmflinknode(tree, nodes): def makelookupmflinknode(tree, nodes):
if fastpathlinkrev: if fastpathlinkrev:
assert not tree assert not tree
fclnode = fclnodes.setdefault(n, clnode) fclnode = fclnodes.setdefault(n, clnode)
if clrevorder[clnode] < clrevorder[fclnode]: if clrevorder[clnode] < clrevorder[fclnode]:
fclnodes[n] = clnode fclnodes[n] = clnode
return clnode return clnode
return lookupmflinknode return lookupmflinknode
while tmfnodes: while tmfnodes:
tree, nodes = tmfnodes.popitem() tree, nodes = tmfnodes.popitem()
store = dirlog(tree) store = mfl.getstorage(tree)
if not self._filematcher.visitdir(store._tree[:-1] or '.'): if not self._filematcher.visitdir(store._tree[:-1] or '.'):
prunednodes = [] prunednodes = []
else: else:
frev, flr = store.rev, store.linkrev frev, flr = store.rev, store.linkrev
prunednodes = [n for n in nodes prunednodes = [n for n in nodes
if flr(frev(n)) not in commonrevs] if flr(frev(n)) not in commonrevs]

mercurial/cmdutil.py

if cl: if cl:
r = repo.unfiltered().changelog r = repo.unfiltered().changelog
elif dir: elif dir:
if 'treemanifest' not in repo.requirements: if 'treemanifest' not in repo.requirements:
raise error.Abort(_("--dir can only be used on repos with " raise error.Abort(_("--dir can only be used on repos with "
"treemanifest enabled")) "treemanifest enabled"))
if not dir.endswith('/'): if not dir.endswith('/'):
dir = dir + '/' dir = dir + '/'
dirlog = repo.manifestlog._revlog.dirlog(dir) dirlog = repo.manifestlog.getstorage(dir)
if len(dirlog): if len(dirlog):
r = dirlog r = dirlog
elif mf: elif mf:
r = repo.manifestlog._revlog r = repo.manifestlog.getstorage(b'')
elif file_: elif file_:
filelog = repo.file(file_) filelog = repo.file(file_)
if len(filelog): if len(filelog):
r = filelog r = filelog
if not r: if not r:
if not file_: if not file_:
raise error.CommandError(cmd, _('invalid arguments')) raise error.CommandError(cmd, _('invalid arguments'))
if not os.path.isfile(file_): if not os.path.isfile(file_):

mercurial/debugcommands.py

@command('debugmanifestfulltextcache', [ @command('debugmanifestfulltextcache', [
('', 'clear', False, _('clear the cache')), ('', 'clear', False, _('clear the cache')),
('a', 'add', '', _('add the given manifest node to the cache'), ('a', 'add', '', _('add the given manifest node to the cache'),
_('NODE')) _('NODE'))
], '') ], '')
def debugmanifestfulltextcache(ui, repo, add=None, **opts): def debugmanifestfulltextcache(ui, repo, add=None, **opts):
"""show, clear or amend the contents of the manifest fulltext cache""" """show, clear or amend the contents of the manifest fulltext cache"""
with repo.lock(): with repo.lock():
r = repo.manifestlog._revlog r = repo.manifestlog.getstorage(b'')
try: try:
cache = r._fulltextcache cache = r._fulltextcache
except AttributeError: except AttributeError:
ui.warn(_( ui.warn(_(
"Current revlog implementation doesn't appear to have a " "Current revlog implementation doesn't appear to have a "
'manifest fulltext cache\n')) 'manifest fulltext cache\n'))
return return

mercurial/manifest.py

return self._dirmancache[tree][node] return self._dirmancache[tree][node]
if not self._narrowmatch.always(): if not self._narrowmatch.always():
if not self._narrowmatch.visitdir(tree[:-1] or '.'): if not self._narrowmatch.visitdir(tree[:-1] or '.'):
return excludeddirmanifestctx(tree, node) return excludeddirmanifestctx(tree, node)
if tree: if tree:
if self._revlog._treeondisk: if self._revlog._treeondisk:
if verify: if verify:
dirlog = self._revlog.dirlog(tree) dirlog = self.getstorage(tree)
if node not in dirlog.nodemap: if node not in dirlog.nodemap:
raise LookupError(node, dirlog.indexfile, raise LookupError(node, dirlog.indexfile,
_('no node')) _('no node'))
m = treemanifestctx(self, tree, node) m = treemanifestctx(self, tree, node)
else: else:
raise error.Abort( raise error.Abort(
_("cannot ask for manifest directory '%s' in a flat " _("cannot ask for manifest directory '%s' in a flat "
"manifest") % tree) "manifest") % tree)
if node != revlog.nullid: if node != revlog.nullid:
mancache = self._dirmancache.get(tree) mancache = self._dirmancache.get(tree)
if not mancache: if not mancache:
mancache = util.lrucachedict(self._cachesize) mancache = util.lrucachedict(self._cachesize)
self._dirmancache[tree] = mancache self._dirmancache[tree] = mancache
mancache[node] = m mancache[node] = m
return m return m
def getstorage(self, tree):
return self._revlog.dirlog(tree)
def clearcaches(self, clear_persisted_data=False): def clearcaches(self, clear_persisted_data=False):
self._dirmancache.clear() self._dirmancache.clear()
self._revlog.clearcaches(clear_persisted_data=clear_persisted_data) self._revlog.clearcaches(clear_persisted_data=clear_persisted_data)
def rev(self, node): def rev(self, node):
return self._revlog.rev(node) return self._revlog.rev(node)
@interfaceutil.implementer(repository.imanifestrevisionwritable) @interfaceutil.implementer(repository.imanifestrevisionwritable)
#rev = revlog.rev(node) #rev = revlog.rev(node)
#self.linkrev = revlog.linkrev(rev) #self.linkrev = revlog.linkrev(rev)
def _revlog(self): def _revlog(self):
narrowmatch = self._manifestlog._narrowmatch narrowmatch = self._manifestlog._narrowmatch
if not narrowmatch.always(): if not narrowmatch.always():
if not narrowmatch.visitdir(self._dir[:-1] or '.'): if not narrowmatch.visitdir(self._dir[:-1] or '.'):
return excludedmanifestrevlog(self._dir) return excludedmanifestrevlog(self._dir)
return self._manifestlog._revlog.dirlog(self._dir) return self._manifestlog.getstorage(self._dir)
def read(self): def read(self):
if self._data is None: if self._data is None:
rl = self._revlog() rl = self._revlog()
if self._node == revlog.nullid: if self._node == revlog.nullid:
self._data = treemanifest() self._data = treemanifest()
elif rl._treeondisk: elif rl._treeondisk:
m = treemanifest(dir=self._dir) m = treemanifest(dir=self._dir)

mercurial/repair.py

return sorted(files) return sorted(files)
def _collectrevlog(revlog, striprev): def _collectrevlog(revlog, striprev):
_, brokenset = revlog.getstrippoint(striprev) _, brokenset = revlog.getstrippoint(striprev)
return [revlog.linkrev(r) for r in brokenset] return [revlog.linkrev(r) for r in brokenset]
def _collectmanifest(repo, striprev): def _collectmanifest(repo, striprev):
return _collectrevlog(repo.manifestlog._revlog, striprev) return _collectrevlog(repo.manifestlog.getstorage(b''), striprev)
def _collectbrokencsets(repo, files, striprev): def _collectbrokencsets(repo, files, striprev):
"""return the changesets which will be broken by the truncation""" """return the changesets which will be broken by the truncation"""
s = set() s = set()
s.update(_collectmanifest(repo, striprev)) s.update(_collectmanifest(repo, striprev))
for fname in files: for fname in files:
s.update(_collectrevlog(repo.file(fname), striprev)) s.update(_collectrevlog(repo.file(fname), striprev))
if callback is None: if callback is None:
callback = stripcallback(ui, repo, backup=backup, topic=topic) callback = stripcallback(ui, repo, backup=backup, topic=topic)
tr.addpostclose('\xffstrip', callback) tr.addpostclose('\xffstrip', callback)
if topic: if topic:
callback.topic = topic callback.topic = topic
callback.addnodes(nodelist) callback.addnodes(nodelist)
def stripmanifest(repo, striprev, tr, files): def stripmanifest(repo, striprev, tr, files):
revlog = repo.manifestlog._revlog revlog = repo.manifestlog.getstorage(b'')
revlog.strip(striprev, tr) revlog.strip(striprev, tr)
striptrees(repo, tr, striprev, files) striptrees(repo, tr, striprev, files)
def striptrees(repo, tr, striprev, files): def striptrees(repo, tr, striprev, files):
if 'treemanifest' in repo.requirements: # safe but unnecessary if 'treemanifest' in repo.requirements: # safe but unnecessary
# otherwise # otherwise
for unencoded, encoded, size in repo.store.datafiles(): for unencoded, encoded, size in repo.store.datafiles():
if (unencoded.startswith('meta/') and if (unencoded.startswith('meta/') and
unencoded.endswith('00manifest.i')): unencoded.endswith('00manifest.i')):
dir = unencoded[5:-12] dir = unencoded[5:-12]
repo.manifestlog._revlog.dirlog(dir).strip(striprev, tr) repo.manifestlog.getstorage(dir).strip(striprev, tr)
def rebuildfncache(ui, repo): def rebuildfncache(ui, repo):
"""Rebuilds the fncache file from repo history. """Rebuilds the fncache file from repo history.
Missing entries will be added. Extra entries will be removed. Missing entries will be added. Extra entries will be removed.
""" """
repo = repo.unfiltered() repo = repo.unfiltered()

mercurial/repository.py

If ``verify`` is True, ``LookupError`` is raised if the node is not If ``verify`` is True, ``LookupError`` is raised if the node is not
known. known.
The returned object conforms to the ``imanifestrevisionstored`` The returned object conforms to the ``imanifestrevisionstored``
interface. interface.
""" """
def getstorage(tree):
"""Retrieve an interface to storage for a particular tree.
If ``tree`` is the empty bytestring, storage for the root manifest will
be returned. Otherwise storage for a tree manifest is returned.
TODO formalize interface for returned object.
"""
def clearcaches(): def clearcaches():
"""Clear caches associated with this collection.""" """Clear caches associated with this collection."""
def rev(node): def rev(node):
"""Obtain the revision number for a binary node. """Obtain the revision number for a binary node.
Raises ``error.LookupError`` if the node is not known. Raises ``error.LookupError`` if the node is not known.
""" """

mercurial/verify.py

def __init__(self, repo, match=None): def __init__(self, repo, match=None):
self.repo = repo.unfiltered() self.repo = repo.unfiltered()
self.ui = repo.ui self.ui = repo.ui
self.match = match or scmutil.matchall(repo) self.match = match or scmutil.matchall(repo)
self.badrevs = set() self.badrevs = set()
self.errors = 0 self.errors = 0
self.warnings = 0 self.warnings = 0
self.havecl = len(repo.changelog) > 0 self.havecl = len(repo.changelog) > 0
self.havemf = len(repo.manifestlog._revlog) > 0 self.havemf = len(repo.manifestlog.getstorage(b'')) > 0
self.revlogv1 = repo.changelog.version != revlog.REVLOGV0 self.revlogv1 = repo.changelog.version != revlog.REVLOGV0
self.lrugetctx = util.lrucachefunc(repo.__getitem__) self.lrugetctx = util.lrucachefunc(repo.__getitem__)
self.refersmf = False self.refersmf = False
self.fncachewarned = False self.fncachewarned = False
# developer config: verify.skipflags # developer config: verify.skipflags
self.skipflags = repo.ui.configint('verify', 'skipflags') self.skipflags = repo.ui.configint('verify', 'skipflags')
self.warnorphanstorefiles = True self.warnorphanstorefiles = True
return mflinkrevs, filelinkrevs return mflinkrevs, filelinkrevs
def _verifymanifest(self, mflinkrevs, dir="", storefiles=None, def _verifymanifest(self, mflinkrevs, dir="", storefiles=None,
subdirprogress=None): subdirprogress=None):
repo = self.repo repo = self.repo
ui = self.ui ui = self.ui
match = self.match match = self.match
mfl = self.repo.manifestlog mfl = self.repo.manifestlog
mf = mfl._revlog.dirlog(dir) mf = mfl.getstorage(dir)
if not dir: if not dir:
self.ui.status(_("checking manifests\n")) self.ui.status(_("checking manifests\n"))
filenodes = {} filenodes = {}
subdirnodes = {} subdirnodes = {}
seen = {} seen = {}
label = "manifest" label = "manifest"