( )⚙ D4089 changegroup: pass ellipsis roots into cgpacker constructor

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4089

changegroup: pass ellipsis roots into cgpacker constructor
ClosedPublic

Authored by indygreg on Aug 3 2018, 5:04 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGad4c4cc9a5ac: changegroup: pass ellipsis roots into cgpacker constructor
Summary

And rename the internal variable to conform with naming conventions.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

indygreg created this revision.Aug 3 2018, 5:04 PM
Closed by commit rHGad4c4cc9a5ac: changegroup: pass ellipsis roots into cgpacker constructor (authored by indygreg). · Explain WhyAug 6 2018, 2:57 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/changegroup.py (44 lines)

Diff 9851

mercurial/changegroup.py

# 2 bytes of flags to apply to revision data. # 2 bytes of flags to apply to revision data.
flags = attr.ib() flags = attr.ib()
# Iterable of chunks holding raw delta data. # Iterable of chunks holding raw delta data.
deltachunks = attr.ib() deltachunks = attr.ib()
class cgpacker(object): class cgpacker(object):
def __init__(self, repo, filematcher, version, allowreorder, def __init__(self, repo, filematcher, version, allowreorder,
useprevdelta, builddeltaheader, manifestsend, useprevdelta, builddeltaheader, manifestsend,
sendtreemanifests, bundlecaps=None, shallow=False): sendtreemanifests, bundlecaps=None, shallow=False,
ellipsisroots=None):
"""Given a source repo, construct a bundler. """Given a source repo, construct a bundler.
filematcher is a matcher that matches on files to include in the filematcher is a matcher that matches on files to include in the
changegroup. Used to facilitate sparse changegroups. changegroup. Used to facilitate sparse changegroups.
allowreorder controls whether reordering of revisions is allowed. allowreorder controls whether reordering of revisions is allowed.
This value is used when ``bundle.reorder`` is ``auto`` or isn't This value is used when ``bundle.reorder`` is ``auto`` or isn't
set. set.
self._sendtreemanifests = sendtreemanifests self._sendtreemanifests = sendtreemanifests
# Set of capabilities we can use to build the bundle. # Set of capabilities we can use to build the bundle.
if bundlecaps is None: if bundlecaps is None:
bundlecaps = set() bundlecaps = set()
self._bundlecaps = bundlecaps self._bundlecaps = bundlecaps
self._isshallow = shallow self._isshallow = shallow
# Maps ellipsis revs to their roots at the changelog level.
self._precomputedellipsis = ellipsisroots
# experimental config: bundle.reorder # experimental config: bundle.reorder
reorder = repo.ui.config('bundle', 'reorder') reorder = repo.ui.config('bundle', 'reorder')
if reorder == 'auto': if reorder == 'auto':
self._reorder = allowreorder self._reorder = allowreorder
else: else:
self._reorder = stringutil.parsebool(reorder) self._reorder = stringutil.parsebool(reorder)
self._repo = repo self._repo = repo
clrevorder[x] = len(clrevorder) clrevorder[x] = len(clrevorder)
if ellipsesmode: if ellipsesmode:
# Only update mfs if x is going to be sent. Otherwise we # Only update mfs if x is going to be sent. Otherwise we
# end up with bogus linkrevs specified for manifests and # end up with bogus linkrevs specified for manifests and
# we skip some manifest nodes that we should otherwise # we skip some manifest nodes that we should otherwise
# have sent. # have sent.
if (x in self._full_nodes if (x in self._full_nodes
or cl.rev(x) in self._precomputed_ellipsis): or cl.rev(x) in self._precomputedellipsis):
n = c[0] n = c[0]
# Record the first changeset introducing this manifest # Record the first changeset introducing this manifest
# version. # version.
mfs.setdefault(n, x) mfs.setdefault(n, x)
# Set this narrow-specific dict so we have the lowest # Set this narrow-specific dict so we have the lowest
# manifest revnum to look up for this cl revnum. (Part of # manifest revnum to look up for this cl revnum. (Part of
# mapping changelog ellipsis parents to manifest ellipsis # mapping changelog ellipsis parents to manifest ellipsis
# parents) # parents)
# This is a node to send in full, because the changeset it # This is a node to send in full, because the changeset it
# corresponds to was a full changeset. # corresponds to was a full changeset.
if linknode in self._full_nodes: if linknode in self._full_nodes:
return self._revisiondeltanormal(store, rev, prev, linknode) return self._revisiondeltanormal(store, rev, prev, linknode)
# At this point, a node can either be one we should skip or an # At this point, a node can either be one we should skip or an
# ellipsis. If it's not an ellipsis, bail immediately. # ellipsis. If it's not an ellipsis, bail immediately.
if linkrev not in self._precomputed_ellipsis: if linkrev not in self._precomputedellipsis:
return return
linkparents = self._precomputed_ellipsis[linkrev] linkparents = self._precomputedellipsis[linkrev]
def local(clrev): def local(clrev):
"""Turn a changelog revnum into a local revnum. """Turn a changelog revnum into a local revnum.
The ellipsis dag is stored as revnums on the changelog, The ellipsis dag is stored as revnums on the changelog,
but when we're producing ellipsis entries for but when we're producing ellipsis entries for
non-changelog revlogs, we need to turn those numbers into non-changelog revlogs, we need to turn those numbers into
something local. This does that for us, and during the something local. This does that for us, and during the
changelog sending phase will also expand the stored changelog sending phase will also expand the stored
while walk: while walk:
p = walk[0] p = walk[0]
walk = walk[1:] walk = walk[1:]
if p in self._clrevtolocalrev: if p in self._clrevtolocalrev:
return self._clrevtolocalrev[p] return self._clrevtolocalrev[p]
elif p in self._full_nodes: elif p in self._full_nodes:
walk.extend([pp for pp in self._repo.changelog.parentrevs(p) walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
if pp != nullrev]) if pp != nullrev])
elif p in self._precomputed_ellipsis: elif p in self._precomputedellipsis:
walk.extend([pp for pp in self._precomputed_ellipsis[p] walk.extend([pp for pp in self._precomputedellipsis[p]
if pp != nullrev]) if pp != nullrev])
else: else:
# In this case, we've got an ellipsis with parents # In this case, we've got an ellipsis with parents
# outside the current bundle (likely an # outside the current bundle (likely an
# incremental pull). We "know" that we can use the # incremental pull). We "know" that we can use the
# value of this same revlog at whatever revision # value of this same revlog at whatever revision
# is pointed to by linknode. "Know" is in scare # is pointed to by linknode. "Know" is in scare
# quotes because I haven't done enough examination # quotes because I haven't done enough examination
p1node=p1n, p1node=p1n,
p2node=p2n, p2node=p2n,
basenode=nullid, basenode=nullid,
linknode=linknode, linknode=linknode,
flags=flags, flags=flags,
deltachunks=(diffheader, data), deltachunks=(diffheader, data),
) )
def _makecg1packer(repo, filematcher, bundlecaps, shallow=False): def _makecg1packer(repo, filematcher, bundlecaps, shallow=False,
ellipsisroots=None):
builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.linknode) d.node, d.p1node, d.p2node, d.linknode)
return cgpacker(repo, filematcher, b'01', return cgpacker(repo, filematcher, b'01',
useprevdelta=True, useprevdelta=True,
allowreorder=None, allowreorder=None,
builddeltaheader=builddeltaheader, builddeltaheader=builddeltaheader,
manifestsend=b'', manifestsend=b'',
sendtreemanifests=False, sendtreemanifests=False,
bundlecaps=bundlecaps, bundlecaps=bundlecaps,
shallow=shallow) shallow=shallow,
ellipsisroots=ellipsisroots)
def _makecg2packer(repo, filematcher, bundlecaps, shallow=False): def _makecg2packer(repo, filematcher, bundlecaps, shallow=False,
ellipsisroots=None):
builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode) d.node, d.p1node, d.p2node, d.basenode, d.linknode)
# Since generaldelta is directly supported by cg2, reordering # Since generaldelta is directly supported by cg2, reordering
# generally doesn't help, so we disable it by default (treating # generally doesn't help, so we disable it by default (treating
# bundle.reorder=auto just like bundle.reorder=False). # bundle.reorder=auto just like bundle.reorder=False).
return cgpacker(repo, filematcher, b'02', return cgpacker(repo, filematcher, b'02',
useprevdelta=False, useprevdelta=False,
allowreorder=False, allowreorder=False,
builddeltaheader=builddeltaheader, builddeltaheader=builddeltaheader,
manifestsend=b'', manifestsend=b'',
sendtreemanifests=False, sendtreemanifests=False,
bundlecaps=bundlecaps, bundlecaps=bundlecaps,
shallow=shallow) shallow=shallow,
ellipsisroots=ellipsisroots)
def _makecg3packer(repo, filematcher, bundlecaps, shallow=False): def _makecg3packer(repo, filematcher, bundlecaps, shallow=False,
ellipsisroots=None):
builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
return cgpacker(repo, filematcher, b'03', return cgpacker(repo, filematcher, b'03',
useprevdelta=False, useprevdelta=False,
allowreorder=False, allowreorder=False,
builddeltaheader=builddeltaheader, builddeltaheader=builddeltaheader,
manifestsend=closechunk(), manifestsend=closechunk(),
sendtreemanifests=True, sendtreemanifests=True,
bundlecaps=bundlecaps, bundlecaps=bundlecaps,
shallow=shallow) shallow=shallow,
ellipsisroots=ellipsisroots)
_packermap = {'01': (_makecg1packer, cg1unpacker), _packermap = {'01': (_makecg1packer, cg1unpacker),
# cg2 adds support for exchanging generaldelta # cg2 adds support for exchanging generaldelta
'02': (_makecg2packer, cg2unpacker), '02': (_makecg2packer, cg2unpacker),
# cg3 adds support for exchanging revlog flags and treemanifests # cg3 adds support for exchanging revlog flags and treemanifests
'03': (_makecg3packer, cg3unpacker), '03': (_makecg3packer, cg3unpacker),
} }
# support changegroup 02. # support changegroup 02.
versions = supportedoutgoingversions(repo) versions = supportedoutgoingversions(repo)
if 'generaldelta' in repo.requirements: if 'generaldelta' in repo.requirements:
versions.discard('01') versions.discard('01')
assert versions assert versions
return min(versions) return min(versions)
def getbundler(version, repo, bundlecaps=None, filematcher=None, def getbundler(version, repo, bundlecaps=None, filematcher=None,
shallow=False): shallow=False, ellipsisroots=None):
assert version in supportedoutgoingversions(repo) assert version in supportedoutgoingversions(repo)
if filematcher is None: if filematcher is None:
filematcher = matchmod.alwaysmatcher(repo.root, '') filematcher = matchmod.alwaysmatcher(repo.root, '')
if version == '01' and not filematcher.always(): if version == '01' and not filematcher.always():
raise error.ProgrammingError('version 01 changegroups do not support ' raise error.ProgrammingError('version 01 changegroups do not support '
'sparse file matchers') 'sparse file matchers')
# Requested files could include files not in the local store. So # Requested files could include files not in the local store. So
# filter those out. # filter those out.
filematcher = matchmod.intersectmatchers(repo.narrowmatch(), filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
filematcher) filematcher)
fn = _packermap[version][0] fn = _packermap[version][0]
return fn(repo, filematcher, bundlecaps, shallow=shallow) return fn(repo, filematcher, bundlecaps, shallow=shallow,
ellipsisroots=ellipsisroots)
def getunbundler(version, fh, alg, extras=None): def getunbundler(version, fh, alg, extras=None):
return _packermap[version][1](fh, alg, extras=extras) return _packermap[version][1](fh, alg, extras=extras)
def _changegroupinfo(repo, nodes, source): def _changegroupinfo(repo, nodes, source):
if repo.ui.verbose or source == 'bundle': if repo.ui.verbose or source == 'bundle':
repo.ui.status(_("%d changesets found\n") % len(nodes)) repo.ui.status(_("%d changesets found\n") % len(nodes))
if repo.ui.debugflag: if repo.ui.debugflag:
'ellipsis nodes require at least cg3 on client and server, ' 'ellipsis nodes require at least cg3 on client and server, '
'but negotiated version %s' % version) 'but negotiated version %s' % version)
# We wrap cg1packer.revchunk, using a side channel to pass # We wrap cg1packer.revchunk, using a side channel to pass
# relevant_nodes into that area. Then if linknode isn't in the # relevant_nodes into that area. Then if linknode isn't in the
# set, we know we have an ellipsis node and we should defer # set, we know we have an ellipsis node and we should defer
# sending that node's data. We override close() to detect # sending that node's data. We override close() to detect
# pending ellipsis nodes and flush them. # pending ellipsis nodes and flush them.
packer = getbundler(version, repo, filematcher=match, packer = getbundler(version, repo, filematcher=match,
shallow=depth is not None) shallow=depth is not None,
ellipsisroots=ellipsisroots)
# Give the packer the list of nodes which should not be # Give the packer the list of nodes which should not be
# ellipsis nodes. We store this rather than the set of nodes # ellipsis nodes. We store this rather than the set of nodes
# that should be an ellipsis because for very large histories # that should be an ellipsis because for very large histories
# we expect this to be significantly smaller. # we expect this to be significantly smaller.
packer._full_nodes = relevant_nodes packer._full_nodes = relevant_nodes
# Maps ellipsis revs to their roots at the changelog level.
packer._precomputed_ellipsis = ellipsisroots
return packer.generate(common, visitnodes, False, source) return packer.generate(common, visitnodes, False, source)