This is an archive of the discontinued Mercurial Phabricator instance.

Differential D9553

treemanifest: stop storing full path for each item in manifest._lazydirs
ClosedPublic

Authored by spectral on Dec 9 2020, 4:41 PM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHG93e09d370003: treemanifest: stop storing full path for each item in manifest._lazydirs
Summary

This information is obtainable, if needed, based on the lazydirs key (which is
the entry name) and the manifest's dir() method.

Performance

This is actually both a memory and a performance improvement, but it's likely to
be a very small one in most situations. In the pathological repo I've been using
for testing other performance work I've done recently, this reduced the time for
a rebase operation (rebasing two commits across a public-phase change that
touches a sibling of one of my tracked directories where the common parent is
massive (>>10k entries)):

Before
Time (mean ± σ):      4.059 s ±  0.121 s    [User: 0.9 ms, System: 0.6 ms]
Range (min … max):    3.941 s …  4.352 s    10 runs
After
Time (mean ± σ):      3.707 s ±  0.060 s    [User: 0.8 ms, System: 0.8 ms]
Range (min … max):    3.648 s …  3.818 s    10 runs

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

spectral created this revision.Dec 9 2020, 4:41 PM
pulkit accepted this revision.Dec 10 2020, 3:43 AM
This revision is now accepted and ready to land.Dec 10 2020, 3:43 AM

Revision Contents
Changeset List

PathPackages
Mmercurial/manifest.py (26 lines)

Diff 24166

mercurial/manifest.py

self.parse(text, readsubtree) self.parse(text, readsubtree)
self._dirty = True # Mark flat manifest dirty after parsing self._dirty = True # Mark flat manifest dirty after parsing
def _subpath(self, path): def _subpath(self, path):
return self._dir + path return self._dir + path
def _loadalllazy(self): def _loadalllazy(self):
selfdirs = self._dirs selfdirs = self._dirs
for d, (path, node, readsubtree, docopy) in pycompat.iteritems( subpath = self._subpath
for d, (node, readsubtree, docopy) in pycompat.iteritems(
self._lazydirs self._lazydirs
): ):
if docopy: if docopy:
selfdirs[d] = readsubtree(path, node).copy() selfdirs[d] = readsubtree(subpath(d), node).copy()
else: else:
selfdirs[d] = readsubtree(path, node) selfdirs[d] = readsubtree(subpath(d), node)
self._lazydirs = {} self._lazydirs = {}
def _loadlazy(self, d): def _loadlazy(self, d):
v = self._lazydirs.get(d) v = self._lazydirs.get(d)
if v: if v:
path, node, readsubtree, docopy = v node, readsubtree, docopy = v
if docopy: if docopy:
self._dirs[d] = readsubtree(path, node).copy() self._dirs[d] = readsubtree(self._subpath(d), node).copy()
else: else:
self._dirs[d] = readsubtree(path, node) self._dirs[d] = readsubtree(self._subpath(d), node)
del self._lazydirs[d] del self._lazydirs[d]
def _loadchildrensetlazy(self, visit): def _loadchildrensetlazy(self, visit):
if not visit: if not visit:
return None return None
if visit == b'all' or visit == b'this': if visit == b'all' or visit == b'this':
self._loadalllazy() self._loadalllazy()
return None return None
- if it's not present in _lazydirs in either t1 or t2, load it in the - if it's not present in _lazydirs in either t1 or t2, load it in the
other (it may already be loaded or it may not exist, doesn't matter) other (it may already be loaded or it may not exist, doesn't matter)
- if it's present in _lazydirs in both, compare the nodeid; if it - if it's present in _lazydirs in both, compare the nodeid; if it
differs, load it in both differs, load it in both
""" """
toloadlazy = [] toloadlazy = []
for d, v1 in pycompat.iteritems(t1._lazydirs): for d, v1 in pycompat.iteritems(t1._lazydirs):
v2 = t2._lazydirs.get(d) v2 = t2._lazydirs.get(d)
if not v2 or v2[1] != v1[1]: if not v2 or v2[0] != v1[0]:
toloadlazy.append(d) toloadlazy.append(d)
for d, v1 in pycompat.iteritems(t2._lazydirs): for d, v1 in pycompat.iteritems(t2._lazydirs):
if d not in t1._lazydirs: if d not in t1._lazydirs:
toloadlazy.append(d) toloadlazy.append(d)
for d in toloadlazy: for d in toloadlazy:
t1._loadlazy(d) t1._loadlazy(d)
t2._loadlazy(d) t2._loadlazy(d)
copy = treemanifest(self._dir) copy = treemanifest(self._dir)
copy._node = self._node copy._node = self._node
copy._dirty = self._dirty copy._dirty = self._dirty
if self._copyfunc is _noop: if self._copyfunc is _noop:
def _copyfunc(s): def _copyfunc(s):
self._load() self._load()
s._lazydirs = { s._lazydirs = {
d: (p, n, r, True) d: (n, r, True)
for d, (p, n, r, c) in pycompat.iteritems(self._lazydirs) for d, (n, r, c) in pycompat.iteritems(self._lazydirs)
} }
sdirs = s._dirs sdirs = s._dirs
for d, v in pycompat.iteritems(self._dirs): for d, v in pycompat.iteritems(self._dirs):
sdirs[d] = v.copy() sdirs[d] = v.copy()
s._files = dict.copy(self._files) s._files = dict.copy(self._files)
s._flags = dict.copy(self._flags) s._flags = dict.copy(self._flags)
if self._loadfunc is _noop: if self._loadfunc is _noop:
_iterativediff(t1, t2, stackls) _iterativediff(t1, t2, stackls)
return result return result
def unmodifiedsince(self, m2): def unmodifiedsince(self, m2):
return not self._dirty and not m2._dirty and self._node == m2._node return not self._dirty and not m2._dirty and self._node == m2._node
def parse(self, text, readsubtree): def parse(self, text, readsubtree):
selflazy = self._lazydirs selflazy = self._lazydirs
subpath = self._subpath
for f, n, fl in _parse(text): for f, n, fl in _parse(text):
if fl == b't': if fl == b't':
f = f + b'/' f = f + b'/'
# False below means "doesn't need to be copied" and can use the # False below means "doesn't need to be copied" and can use the
# cached value from readsubtree directly. # cached value from readsubtree directly.
selflazy[f] = (subpath(f), n, readsubtree, False) selflazy[f] = (n, readsubtree, False)
elif b'/' in f: elif b'/' in f:
# This is a flat manifest, so use __setitem__ and setflag rather # This is a flat manifest, so use __setitem__ and setflag rather
# than assigning directly to _files and _flags, so we can # than assigning directly to _files and _flags, so we can
# assign a path in a subdirectory, and to mark dirty (compared # assign a path in a subdirectory, and to mark dirty (compared
# to nullid). # to nullid).
self[f] = n self[f] = n
if fl: if fl:
self.setflag(f, fl) self.setflag(f, fl)
def dirtext(self): def dirtext(self):
"""Get the full data of this directory as a bytestring. Make sure that """Get the full data of this directory as a bytestring. Make sure that
any submanifests have been written first, so their nodeids are correct. any submanifests have been written first, so their nodeids are correct.
""" """
self._load() self._load()
flags = self.flags flags = self.flags
lazydirs = [ lazydirs = [
(d[:-1], v[1], b't') for d, v in pycompat.iteritems(self._lazydirs) (d[:-1], v[0], b't') for d, v in pycompat.iteritems(self._lazydirs)
] ]
dirs = [(d[:-1], self._dirs[d]._node, b't') for d in self._dirs] dirs = [(d[:-1], self._dirs[d]._node, b't') for d in self._dirs]
files = [(f, self._files[f], flags(f)) for f in self._files] files = [(f, self._files[f], flags(f)) for f in self._files]
return _text(sorted(dirs + files + lazydirs)) return _text(sorted(dirs + files + lazydirs))
def read(self, gettext, readsubtree): def read(self, gettext, readsubtree):
def _load_for_read(s): def _load_for_read(s):
s.parse(gettext(), readsubtree) s.parse(gettext(), readsubtree)
s._dirty = False s._dirty = False
self._loadfunc = _load_for_read self._loadfunc = _load_for_read
def writesubtrees(self, m1, m2, writesubtree, match): def writesubtrees(self, m1, m2, writesubtree, match):
self._load() # for consistency; should never have any effect here self._load() # for consistency; should never have any effect here
m1._load() m1._load()
m2._load() m2._load()
emptytree = treemanifest() emptytree = treemanifest()
def getnode(m, d): def getnode(m, d):
ld = m._lazydirs.get(d) ld = m._lazydirs.get(d)
if ld: if ld:
return ld[1] return ld[0]
return m._dirs.get(d, emptytree)._node return m._dirs.get(d, emptytree)._node
# let's skip investigating things that `match` says we do not need. # let's skip investigating things that `match` says we do not need.
visit = match.visitchildrenset(self._dir[:-1]) visit = match.visitchildrenset(self._dir[:-1])
visit = self._loadchildrensetlazy(visit) visit = self._loadchildrensetlazy(visit)
if visit == b'this' or visit == b'all': if visit == b'this' or visit == b'all':
visit = None visit = None
for d, subm in pycompat.iteritems(self._dirs): for d, subm in pycompat.iteritems(self._dirs):