This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4081

changegroup: consolidate tree manifests sending into cg1packer
ClosedPublic

Authored by indygreg on Aug 3 2018, 5:04 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG7a154778fb46: changegroup: consolidate tree manifests sending into cg1packer
Summary

Previously, we overrode a method to control how manifests were
serialized. This method was redefined on cg3packer to send tree
manifests.

This commit moves the tree manifests sending variation to cg1packer
and teaches the cgpacker constructor to control which version to
use.

After these changes, cg3packer was empty. So it has been removed.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

indygreg created this revision.Aug 3 2018, 5:04 PM
Closed by commit rHG7a154778fb46: changegroup: consolidate tree manifests sending into cg1packer (authored by indygreg). · Explain WhyAug 6 2018, 2:57 AM
This revision was automatically updated to reflect the committed changes.
martinvonz added a subscriber: martinvonz.Aug 6 2018, 3:28 AM
martinvonz added inline comments.
mercurial/changegroup.py
674

Note that this also works for flat manifests (IIRC). You should be able always use this version, as long as you remove the assertion. You can perhaps move the assertion 3 lines down into the if dir: block.

Revision Contents
Changeset List

PathPackages
Mmercurial/changegroup.py (48 lines)

Diff 9843

mercurial/changegroup.py

linknode = attr.ib() linknode = attr.ib()
# 2 bytes of flags to apply to revision data. # 2 bytes of flags to apply to revision data.
flags = attr.ib() flags = attr.ib()
# Iterable of chunks holding raw delta data. # Iterable of chunks holding raw delta data.
deltachunks = attr.ib() deltachunks = attr.ib()
class cg1packer(object): class cg1packer(object):
def __init__(self, repo, filematcher, version, builddeltaheader, def __init__(self, repo, filematcher, version, builddeltaheader,
manifestsend, manifestsend, sendtreemanifests,
bundlecaps=None): bundlecaps=None):
"""Given a source repo, construct a bundler. """Given a source repo, construct a bundler.
filematcher is a matcher that matches on files to include in the filematcher is a matcher that matches on files to include in the
changegroup. Used to facilitate sparse changegroups. changegroup. Used to facilitate sparse changegroups.
builddeltaheader is a callable that constructs the header for a group builddeltaheader is a callable that constructs the header for a group
delta. delta.
manifestsend is a chunk to send after manifests have been fully emitted. manifestsend is a chunk to send after manifests have been fully emitted.
sendtreemanifests indicates whether tree manifests should be emitted.
bundlecaps is optional and can be used to specify the set of bundlecaps is optional and can be used to specify the set of
capabilities which can be used to build the bundle. While bundlecaps is capabilities which can be used to build the bundle. While bundlecaps is
unused in core Mercurial, extensions rely on this feature to communicate unused in core Mercurial, extensions rely on this feature to communicate
capabilities to customize the changegroup packer. capabilities to customize the changegroup packer.
""" """
assert filematcher assert filematcher
self._filematcher = filematcher self._filematcher = filematcher
self.version = version self.version = version
self._builddeltaheader = builddeltaheader self._builddeltaheader = builddeltaheader
self._manifestsend = manifestsend self._manifestsend = manifestsend
self._sendtreemanifests = sendtreemanifests
# Set of capabilities we can use to build the bundle. # Set of capabilities we can use to build the bundle.
if bundlecaps is None: if bundlecaps is None:
bundlecaps = set() bundlecaps = set()
self._bundlecaps = bundlecaps self._bundlecaps = bundlecaps
# experimental config: bundle.reorder # experimental config: bundle.reorder
reorder = repo.ui.config('bundle', 'reorder') reorder = repo.ui.config('bundle', 'reorder')
if reorder == 'auto': if reorder == 'auto':
def _packmanifests(self, dir, mfnodes, lookuplinknode): def _packmanifests(self, dir, mfnodes, lookuplinknode):
"""Pack flat manifests into a changegroup stream.""" """Pack flat manifests into a changegroup stream."""
assert not dir assert not dir
for chunk in self.group(mfnodes, self._repo.manifestlog._revlog, for chunk in self.group(mfnodes, self._repo.manifestlog._revlog,
lookuplinknode, units=_('manifests')): lookuplinknode, units=_('manifests')):
yield chunk yield chunk
def _packtreemanifests(self, dir, mfnodes, lookuplinknode):
martinvonzUnsubmitted
Not Done

Note that this also works for flat manifests (IIRC). You should be able always use this version, as long as you remove the assertion. You can perhaps move the assertion 3 lines down into the if dir: block.

martinvonz: Note that this also works for flat manifests (IIRC). You should be able always use this version…
"""Version of _packmanifests that operates on directory manifests.
Encodes the directory name in the output so multiple manifests
can be sent.
"""
assert self.version == b'03'
if dir:
yield self.fileheader(dir)
# TODO violates storage abstractions by assuming revlogs.
dirlog = self._repo.manifestlog._revlog.dirlog(dir)
for chunk in self.group(mfnodes, dirlog, lookuplinknode,
units=_('manifests')):
yield chunk
def generate(self, commonrevs, clnodes, fastpathlinkrev, source): def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
'''yield a sequence of changegroup chunks (strings)''' '''yield a sequence of changegroup chunks (strings)'''
repo = self._repo repo = self._repo
cl = repo.changelog cl = repo.changelog
clrevorder = {} clrevorder = {}
mfs = {} # needed manifests mfs = {} # needed manifests
fnodes = {} # needed file nodes fnodes = {} # needed file nodes
f = dir + p f = dir + p
fclnodes = fnodes.setdefault(f, {}) fclnodes = fnodes.setdefault(f, {})
fclnode = fclnodes.setdefault(n, clnode) fclnode = fclnodes.setdefault(n, clnode)
if clrevorder[clnode] < clrevorder[fclnode]: if clrevorder[clnode] < clrevorder[fclnode]:
fclnodes[n] = clnode fclnodes[n] = clnode
return clnode return clnode
return lookupmflinknode return lookupmflinknode
fn = (self._packtreemanifests if self._sendtreemanifests
else self._packmanifests)
size = 0 size = 0
while tmfnodes: while tmfnodes:
dir, nodes = tmfnodes.popitem() dir, nodes = tmfnodes.popitem()
prunednodes = self.prune(dirlog(dir), nodes, commonrevs) prunednodes = self.prune(dirlog(dir), nodes, commonrevs)
if not dir or prunednodes: if not dir or prunednodes:
for x in self._packmanifests(dir, prunednodes, for x in fn(dir, prunednodes, makelookupmflinknode(dir, nodes)):
makelookupmflinknode(dir, nodes)):
size += len(x) size += len(x)
yield x yield x
self._verbosenote(_('%8.i (manifests)\n') % size) self._verbosenote(_('%8.i (manifests)\n') % size)
yield self._manifestsend yield self._manifestsend
# The 'source' parameter is useful for extensions # The 'source' parameter is useful for extensions
def generatefiles(self, changedfiles, linknodes, commonrevs, source): def generatefiles(self, changedfiles, linknodes, commonrevs, source):
changedfiles = list(filter(self._filematcher, changedfiles)) changedfiles = list(filter(self._filematcher, changedfiles))
basenode=nullid, basenode=nullid,
linknode=linknode, linknode=linknode,
flags=flags, flags=flags,
deltachunks=(diffheader, data), deltachunks=(diffheader, data),
) )
class cg2packer(cg1packer): class cg2packer(cg1packer):
def __init__(self, repo, filematcher, version, builddeltaheader, def __init__(self, repo, filematcher, version, builddeltaheader,
manifestsend, bundlecaps=None): manifestsend, sendtreemanifests, bundlecaps=None):
super(cg2packer, self).__init__(repo, filematcher, version, super(cg2packer, self).__init__(repo, filematcher, version,
builddeltaheader, manifestsend, builddeltaheader, manifestsend,
sendtreemanifests,
bundlecaps=bundlecaps) bundlecaps=bundlecaps)
if self._reorder is None: if self._reorder is None:
# Since generaldelta is directly supported by cg2, reordering # Since generaldelta is directly supported by cg2, reordering
# generally doesn't help, so we disable it by default (treating # generally doesn't help, so we disable it by default (treating
# bundle.reorder=auto just like bundle.reorder=False). # bundle.reorder=auto just like bundle.reorder=False).
self._reorder = False self._reorder = False
# Pick prev when we can't be sure remote has the base revision. # Pick prev when we can't be sure remote has the base revision.
return prev return prev
else: else:
base = dp base = dp
if base != nullrev and not store.candelta(base, rev): if base != nullrev and not store.candelta(base, rev):
base = nullrev base = nullrev
return base return base
class cg3packer(cg2packer):
def _packmanifests(self, dir, mfnodes, lookuplinknode):
if dir:
yield self.fileheader(dir)
dirlog = self._repo.manifestlog._revlog.dirlog(dir)
for chunk in self.group(mfnodes, dirlog, lookuplinknode,
units=_('manifests')):
yield chunk
def _makecg1packer(repo, filematcher, bundlecaps): def _makecg1packer(repo, filematcher, bundlecaps):
builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.linknode) d.node, d.p1node, d.p2node, d.linknode)
return cg1packer(repo, filematcher, b'01', builddeltaheader, return cg1packer(repo, filematcher, b'01', builddeltaheader,
manifestsend=b'', manifestsend=b'', sendtreemanifests=False,
bundlecaps=bundlecaps) bundlecaps=bundlecaps)
def _makecg2packer(repo, filematcher, bundlecaps): def _makecg2packer(repo, filematcher, bundlecaps):
builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode) d.node, d.p1node, d.p2node, d.basenode, d.linknode)
return cg2packer(repo, filematcher, b'02', builddeltaheader, return cg2packer(repo, filematcher, b'02', builddeltaheader,
manifestsend=b'', manifestsend=b'', sendtreemanifests=False,
bundlecaps=bundlecaps) bundlecaps=bundlecaps)
def _makecg3packer(repo, filematcher, bundlecaps): def _makecg3packer(repo, filematcher, bundlecaps):
builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
return cg3packer(repo, filematcher, b'03', builddeltaheader, return cg2packer(repo, filematcher, b'03', builddeltaheader,
manifestsend=closechunk(), manifestsend=closechunk(), sendtreemanifests=True,
bundlecaps=bundlecaps) bundlecaps=bundlecaps)
_packermap = {'01': (_makecg1packer, cg1unpacker), _packermap = {'01': (_makecg1packer, cg1unpacker),
# cg2 adds support for exchanging generaldelta # cg2 adds support for exchanging generaldelta
'02': (_makecg2packer, cg2unpacker), '02': (_makecg2packer, cg2unpacker),
# cg3 adds support for exchanging revlog flags and treemanifests # cg3 adds support for exchanging revlog flags and treemanifests
'03': (_makecg3packer, cg3unpacker), '03': (_makecg3packer, cg3unpacker),
} }