| hg-reviewers |
Currently, code throughout changegroup relies on the presence
of self._full_nodes to enable ellipses mode. This is a very tenuous
check. And the check may be wrong once we move _full_nodes into
cgpacker.
Let's capture the enabling of ellipses mode explicitly as a constructor
argument and as an instance variable.
We could probably derive ellipses mode by presence of other
variables. But for now, this explicit approach seems simplest
since it is most compatible with existing code.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/changegroup.py (52 lines) |
| Status | Author | Revision | |
|---|---|---|---|
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg |
| # 2 bytes of flags to apply to revision data. | # 2 bytes of flags to apply to revision data. | ||||
| flags = attr.ib() | flags = attr.ib() | ||||
| # Iterable of chunks holding raw delta data. | # Iterable of chunks holding raw delta data. | ||||
| deltachunks = attr.ib() | deltachunks = attr.ib() | ||||
| class cgpacker(object): | class cgpacker(object): | ||||
| def __init__(self, repo, filematcher, version, allowreorder, | def __init__(self, repo, filematcher, version, allowreorder, | ||||
| useprevdelta, builddeltaheader, manifestsend, | useprevdelta, builddeltaheader, manifestsend, | ||||
| sendtreemanifests, bundlecaps=None, shallow=False, | sendtreemanifests, bundlecaps=None, ellipses=False, | ||||
| ellipsisroots=None): | shallow=False, ellipsisroots=None): | ||||
| """Given a source repo, construct a bundler. | """Given a source repo, construct a bundler. | ||||
| filematcher is a matcher that matches on files to include in the | filematcher is a matcher that matches on files to include in the | ||||
| changegroup. Used to facilitate sparse changegroups. | changegroup. Used to facilitate sparse changegroups. | ||||
| allowreorder controls whether reordering of revisions is allowed. | allowreorder controls whether reordering of revisions is allowed. | ||||
| This value is used when ``bundle.reorder`` is ``auto`` or isn't | This value is used when ``bundle.reorder`` is ``auto`` or isn't | ||||
| set. | set. | ||||
| useprevdelta controls whether revisions should always delta against | useprevdelta controls whether revisions should always delta against | ||||
| the previous revision in the changegroup. | the previous revision in the changegroup. | ||||
| builddeltaheader is a callable that constructs the header for a group | builddeltaheader is a callable that constructs the header for a group | ||||
| delta. | delta. | ||||
| manifestsend is a chunk to send after manifests have been fully emitted. | manifestsend is a chunk to send after manifests have been fully emitted. | ||||
| sendtreemanifests indicates whether tree manifests should be emitted. | sendtreemanifests indicates whether tree manifests should be emitted. | ||||
| ellipses indicates whether ellipsis serving mode is enabled. | |||||
| bundlecaps is optional and can be used to specify the set of | bundlecaps is optional and can be used to specify the set of | ||||
| capabilities which can be used to build the bundle. While bundlecaps is | capabilities which can be used to build the bundle. While bundlecaps is | ||||
| unused in core Mercurial, extensions rely on this feature to communicate | unused in core Mercurial, extensions rely on this feature to communicate | ||||
| capabilities to customize the changegroup packer. | capabilities to customize the changegroup packer. | ||||
| shallow indicates whether shallow data might be sent. The packer may | shallow indicates whether shallow data might be sent. The packer may | ||||
| need to pack file contents not introduced by the changes being packed. | need to pack file contents not introduced by the changes being packed. | ||||
| """ | """ | ||||
| assert filematcher | assert filematcher | ||||
| self._filematcher = filematcher | self._filematcher = filematcher | ||||
| self.version = version | self.version = version | ||||
| self._useprevdelta = useprevdelta | self._useprevdelta = useprevdelta | ||||
| self._builddeltaheader = builddeltaheader | self._builddeltaheader = builddeltaheader | ||||
| self._manifestsend = manifestsend | self._manifestsend = manifestsend | ||||
| self._sendtreemanifests = sendtreemanifests | self._sendtreemanifests = sendtreemanifests | ||||
| self._ellipses = ellipses | |||||
| # Set of capabilities we can use to build the bundle. | # Set of capabilities we can use to build the bundle. | ||||
| if bundlecaps is None: | if bundlecaps is None: | ||||
| bundlecaps = set() | bundlecaps = set() | ||||
| self._bundlecaps = bundlecaps | self._bundlecaps = bundlecaps | ||||
| self._isshallow = shallow | self._isshallow = shallow | ||||
| # Maps ellipsis revs to their roots at the changelog level. | # Maps ellipsis revs to their roots at the changelog level. | ||||
| # a case like 882681bc3166 and its ancestors (back about 10 | # a case like 882681bc3166 and its ancestors (back about 10 | ||||
| # revisions or so) in the main hg repo. | # revisions or so) in the main hg repo. | ||||
| # | # | ||||
| # The one invariant we *know* holds is that the new (potentially | # The one invariant we *know* holds is that the new (potentially | ||||
| # bogus) DAG shape will be valid if we order the nodes in the | # bogus) DAG shape will be valid if we order the nodes in the | ||||
| # order that they're introduced in dramatis personae by the | # order that they're introduced in dramatis personae by the | ||||
| # changelog, so what we do is we sort the non-changelog histories | # changelog, so what we do is we sort the non-changelog histories | ||||
| # by the order in which they are used by the changelog. | # by the order in which they are used by the changelog. | ||||
| if util.safehasattr(self, '_full_nodes') and self._clnodetorev: | if self._ellipses and self._clnodetorev: | ||||
| key = lambda n: self._clnodetorev[lookup(n)] | key = lambda n: self._clnodetorev[lookup(n)] | ||||
| return [store.rev(n) for n in sorted(nodelist, key=key)] | return [store.rev(n) for n in sorted(nodelist, key=key)] | ||||
| # for generaldelta revlogs, we linearize the revs; this will both be | # for generaldelta revlogs, we linearize the revs; this will both be | ||||
| # much quicker and generate a much smaller bundle | # much quicker and generate a much smaller bundle | ||||
| if (store._generaldelta and self._reorder is None) or self._reorder: | if (store._generaldelta and self._reorder is None) or self._reorder: | ||||
| dag = dagutil.revlogdag(store) | dag = dagutil.revlogdag(store) | ||||
| return dag.linearize(set(store.rev(n) for n in nodelist)) | return dag.linearize(set(store.rev(n) for n in nodelist)) | ||||
| clrevorder = {} | clrevorder = {} | ||||
| mfs = {} # needed manifests | mfs = {} # needed manifests | ||||
| fnodes = {} # needed file nodes | fnodes = {} # needed file nodes | ||||
| mfl = repo.manifestlog | mfl = repo.manifestlog | ||||
| # TODO violates storage abstraction. | # TODO violates storage abstraction. | ||||
| mfrevlog = mfl._revlog | mfrevlog = mfl._revlog | ||||
| changedfiles = set() | changedfiles = set() | ||||
| ellipsesmode = util.safehasattr(self, '_full_nodes') | |||||
| # Callback for the changelog, used to collect changed files and | # Callback for the changelog, used to collect changed files and | ||||
| # manifest nodes. | # manifest nodes. | ||||
| # Returns the linkrev node (identity in the changelog case). | # Returns the linkrev node (identity in the changelog case). | ||||
| def lookupcl(x): | def lookupcl(x): | ||||
| c = cl.read(x) | c = cl.read(x) | ||||
| clrevorder[x] = len(clrevorder) | clrevorder[x] = len(clrevorder) | ||||
| if ellipsesmode: | if self._ellipses: | ||||
| # Only update mfs if x is going to be sent. Otherwise we | # Only update mfs if x is going to be sent. Otherwise we | ||||
| # end up with bogus linkrevs specified for manifests and | # end up with bogus linkrevs specified for manifests and | ||||
| # we skip some manifest nodes that we should otherwise | # we skip some manifest nodes that we should otherwise | ||||
| # have sent. | # have sent. | ||||
| if (x in self._full_nodes | if (x in self._full_nodes | ||||
| or cl.rev(x) in self._precomputedellipsis): | or cl.rev(x) in self._precomputedellipsis): | ||||
| n = c[0] | n = c[0] | ||||
| # Record the first changeset introducing this manifest | # Record the first changeset introducing this manifest | ||||
| # send along with files. This could probably be fixed. | # send along with files. This could probably be fixed. | ||||
| fastpathlinkrev = fastpathlinkrev and ( | fastpathlinkrev = fastpathlinkrev and ( | ||||
| 'treemanifest' not in repo.requirements) | 'treemanifest' not in repo.requirements) | ||||
| for chunk in self.generatemanifests(commonrevs, clrevorder, | for chunk in self.generatemanifests(commonrevs, clrevorder, | ||||
| fastpathlinkrev, mfs, fnodes, source): | fastpathlinkrev, mfs, fnodes, source): | ||||
| yield chunk | yield chunk | ||||
| if ellipsesmode: | if self._ellipses: | ||||
| mfdicts = None | mfdicts = None | ||||
| if self._isshallow: | if self._isshallow: | ||||
| mfdicts = [(self._repo.manifestlog[n].read(), lr) | mfdicts = [(self._repo.manifestlog[n].read(), lr) | ||||
| for (n, lr) in mfs.iteritems()] | for (n, lr) in mfs.iteritems()] | ||||
| mfs.clear() | mfs.clear() | ||||
| clrevs = set(cl.rev(x) for x in clnodes) | clrevs = set(cl.rev(x) for x in clnodes) | ||||
| if not fastpathlinkrev: | if not fastpathlinkrev: | ||||
| def linknodes(unused, fname): | def linknodes(unused, fname): | ||||
| return fnodes.get(fname, {}) | return fnodes.get(fname, {}) | ||||
| else: | else: | ||||
| cln = cl.node | cln = cl.node | ||||
| def linknodes(filerevlog, fname): | def linknodes(filerevlog, fname): | ||||
| llr = filerevlog.linkrev | llr = filerevlog.linkrev | ||||
| fln = filerevlog.node | fln = filerevlog.node | ||||
| revs = ((r, llr(r)) for r in filerevlog) | revs = ((r, llr(r)) for r in filerevlog) | ||||
| return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs) | return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs) | ||||
| if ellipsesmode: | if self._ellipses: | ||||
| # We need to pass the mfdicts variable down into | # We need to pass the mfdicts variable down into | ||||
| # generatefiles(), but more than one command might have | # generatefiles(), but more than one command might have | ||||
| # wrapped generatefiles so we can't modify the function | # wrapped generatefiles so we can't modify the function | ||||
| # signature. Instead, we pass the data to ourselves using an | # signature. Instead, we pass the data to ourselves using an | ||||
| # instance attribute. I'm sorry. | # instance attribute. I'm sorry. | ||||
| self._mfdicts = mfdicts | self._mfdicts = mfdicts | ||||
| for chunk in self.generatefiles(changedfiles, linknodes, commonrevs, | for chunk in self.generatefiles(changedfiles, linknodes, commonrevs, | ||||
| def _deltaparent(self, store, rev, p1, p2, prev): | def _deltaparent(self, store, rev, p1, p2, prev): | ||||
| if self._useprevdelta: | if self._useprevdelta: | ||||
| if not store.candelta(prev, rev): | if not store.candelta(prev, rev): | ||||
| raise error.ProgrammingError( | raise error.ProgrammingError( | ||||
| 'cg1 should not be used in this case') | 'cg1 should not be used in this case') | ||||
| return prev | return prev | ||||
| # Narrow ellipses mode. | # Narrow ellipses mode. | ||||
| if util.safehasattr(self, '_full_nodes'): | if self._ellipses: | ||||
| # TODO: send better deltas when in narrow mode. | # TODO: send better deltas when in narrow mode. | ||||
| # | # | ||||
| # changegroup.group() loops over revisions to send, | # changegroup.group() loops over revisions to send, | ||||
| # including revisions we'll skip. What this means is that | # including revisions we'll skip. What this means is that | ||||
| # `prev` will be a potentially useless delta base for all | # `prev` will be a potentially useless delta base for all | ||||
| # ellipsis nodes, as the client likely won't have it. In | # ellipsis nodes, as the client likely won't have it. In | ||||
| # the future we should do bookkeeping about which nodes | # the future we should do bookkeeping about which nodes | ||||
| # have been sent to the client, and try to be | # have been sent to the client, and try to be | ||||
| base = dp | base = dp | ||||
| if base != nullrev and not store.candelta(base, rev): | if base != nullrev and not store.candelta(base, rev): | ||||
| base = nullrev | base = nullrev | ||||
| return base | return base | ||||
| def _revchunk(self, store, rev, prev, linknode): | def _revchunk(self, store, rev, prev, linknode): | ||||
| if util.safehasattr(self, '_full_nodes'): | if self._ellipses: | ||||
| fn = self._revisiondeltanarrow | fn = self._revisiondeltanarrow | ||||
| else: | else: | ||||
| fn = self._revisiondeltanormal | fn = self._revisiondeltanormal | ||||
| delta = fn(store, rev, prev, linknode) | delta = fn(store, rev, prev, linknode) | ||||
| if not delta: | if not delta: | ||||
| return | return | ||||
| p1node=p1n, | p1node=p1n, | ||||
| p2node=p2n, | p2node=p2n, | ||||
| basenode=nullid, | basenode=nullid, | ||||
| linknode=linknode, | linknode=linknode, | ||||
| flags=flags, | flags=flags, | ||||
| deltachunks=(diffheader, data), | deltachunks=(diffheader, data), | ||||
| ) | ) | ||||
| def _makecg1packer(repo, filematcher, bundlecaps, shallow=False, | def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False, | ||||
| ellipsisroots=None): | shallow=False, ellipsisroots=None): | ||||
| builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( | builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( | ||||
| d.node, d.p1node, d.p2node, d.linknode) | d.node, d.p1node, d.p2node, d.linknode) | ||||
| return cgpacker(repo, filematcher, b'01', | return cgpacker(repo, filematcher, b'01', | ||||
| useprevdelta=True, | useprevdelta=True, | ||||
| allowreorder=None, | allowreorder=None, | ||||
| builddeltaheader=builddeltaheader, | builddeltaheader=builddeltaheader, | ||||
| manifestsend=b'', | manifestsend=b'', | ||||
| sendtreemanifests=False, | sendtreemanifests=False, | ||||
| bundlecaps=bundlecaps, | bundlecaps=bundlecaps, | ||||
| ellipses=ellipses, | |||||
| shallow=shallow, | shallow=shallow, | ||||
| ellipsisroots=ellipsisroots) | ellipsisroots=ellipsisroots) | ||||
| def _makecg2packer(repo, filematcher, bundlecaps, shallow=False, | def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False, | ||||
| ellipsisroots=None): | shallow=False, ellipsisroots=None): | ||||
| builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( | builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( | ||||
| d.node, d.p1node, d.p2node, d.basenode, d.linknode) | d.node, d.p1node, d.p2node, d.basenode, d.linknode) | ||||
| # Since generaldelta is directly supported by cg2, reordering | # Since generaldelta is directly supported by cg2, reordering | ||||
| # generally doesn't help, so we disable it by default (treating | # generally doesn't help, so we disable it by default (treating | ||||
| # bundle.reorder=auto just like bundle.reorder=False). | # bundle.reorder=auto just like bundle.reorder=False). | ||||
| return cgpacker(repo, filematcher, b'02', | return cgpacker(repo, filematcher, b'02', | ||||
| useprevdelta=False, | useprevdelta=False, | ||||
| allowreorder=False, | allowreorder=False, | ||||
| builddeltaheader=builddeltaheader, | builddeltaheader=builddeltaheader, | ||||
| manifestsend=b'', | manifestsend=b'', | ||||
| sendtreemanifests=False, | sendtreemanifests=False, | ||||
| bundlecaps=bundlecaps, | bundlecaps=bundlecaps, | ||||
| ellipses=ellipses, | |||||
| shallow=shallow, | shallow=shallow, | ||||
| ellipsisroots=ellipsisroots) | ellipsisroots=ellipsisroots) | ||||
| def _makecg3packer(repo, filematcher, bundlecaps, shallow=False, | def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False, | ||||
| ellipsisroots=None): | shallow=False, ellipsisroots=None): | ||||
| builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( | builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( | ||||
| d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) | d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) | ||||
| return cgpacker(repo, filematcher, b'03', | return cgpacker(repo, filematcher, b'03', | ||||
| useprevdelta=False, | useprevdelta=False, | ||||
| allowreorder=False, | allowreorder=False, | ||||
| builddeltaheader=builddeltaheader, | builddeltaheader=builddeltaheader, | ||||
| manifestsend=closechunk(), | manifestsend=closechunk(), | ||||
| sendtreemanifests=True, | sendtreemanifests=True, | ||||
| bundlecaps=bundlecaps, | bundlecaps=bundlecaps, | ||||
| ellipses=ellipses, | |||||
| shallow=shallow, | shallow=shallow, | ||||
| ellipsisroots=ellipsisroots) | ellipsisroots=ellipsisroots) | ||||
| _packermap = {'01': (_makecg1packer, cg1unpacker), | _packermap = {'01': (_makecg1packer, cg1unpacker), | ||||
| # cg2 adds support for exchanging generaldelta | # cg2 adds support for exchanging generaldelta | ||||
| '02': (_makecg2packer, cg2unpacker), | '02': (_makecg2packer, cg2unpacker), | ||||
| # cg3 adds support for exchanging revlog flags and treemanifests | # cg3 adds support for exchanging revlog flags and treemanifests | ||||
| '03': (_makecg3packer, cg3unpacker), | '03': (_makecg3packer, cg3unpacker), | ||||
| # support changegroup 02. | # support changegroup 02. | ||||
| versions = supportedoutgoingversions(repo) | versions = supportedoutgoingversions(repo) | ||||
| if 'generaldelta' in repo.requirements: | if 'generaldelta' in repo.requirements: | ||||
| versions.discard('01') | versions.discard('01') | ||||
| assert versions | assert versions | ||||
| return min(versions) | return min(versions) | ||||
| def getbundler(version, repo, bundlecaps=None, filematcher=None, | def getbundler(version, repo, bundlecaps=None, filematcher=None, | ||||
| shallow=False, ellipsisroots=None): | ellipses=False, shallow=False, ellipsisroots=None): | ||||
| assert version in supportedoutgoingversions(repo) | assert version in supportedoutgoingversions(repo) | ||||
| if filematcher is None: | if filematcher is None: | ||||
| filematcher = matchmod.alwaysmatcher(repo.root, '') | filematcher = matchmod.alwaysmatcher(repo.root, '') | ||||
| if version == '01' and not filematcher.always(): | if version == '01' and not filematcher.always(): | ||||
| raise error.ProgrammingError('version 01 changegroups do not support ' | raise error.ProgrammingError('version 01 changegroups do not support ' | ||||
| 'sparse file matchers') | 'sparse file matchers') | ||||
| if ellipses and version in (b'01', b'02'): | |||||
| raise error.Abort( | |||||
| _('ellipsis nodes require at least cg3 on client and server, ' | |||||
| 'but negotiated version %s') % version) | |||||
| # Requested files could include files not in the local store. So | # Requested files could include files not in the local store. So | ||||
| # filter those out. | # filter those out. | ||||
| filematcher = matchmod.intersectmatchers(repo.narrowmatch(), | filematcher = matchmod.intersectmatchers(repo.narrowmatch(), | ||||
| filematcher) | filematcher) | ||||
| fn = _packermap[version][0] | fn = _packermap[version][0] | ||||
| return fn(repo, filematcher, bundlecaps, shallow=shallow, | return fn(repo, filematcher, bundlecaps, ellipses=ellipses, | ||||
| ellipsisroots=ellipsisroots) | shallow=shallow, ellipsisroots=ellipsisroots) | ||||
| def getunbundler(version, fh, alg, extras=None): | def getunbundler(version, fh, alg, extras=None): | ||||
| return _packermap[version][1](fh, alg, extras=extras) | return _packermap[version][1](fh, alg, extras=extras) | ||||
| def _changegroupinfo(repo, nodes, source): | def _changegroupinfo(repo, nodes, source): | ||||
| if repo.ui.verbose or source == 'bundle': | if repo.ui.verbose or source == 'bundle': | ||||
| repo.ui.status(_("%d changesets found\n") % len(nodes)) | repo.ui.status(_("%d changesets found\n") % len(nodes)) | ||||
| if repo.ui.debugflag: | if repo.ui.debugflag: | ||||
| raise error.Abort( | raise error.Abort( | ||||
| _('missing file data for %s:%s - run hg verify') % | _('missing file data for %s:%s - run hg verify') % | ||||
| (f, hex(n))) | (f, hex(n))) | ||||
| return revisions, files | return revisions, files | ||||
| def _packellipsischangegroup(repo, common, match, relevant_nodes, | def _packellipsischangegroup(repo, common, match, relevant_nodes, | ||||
| ellipsisroots, visitnodes, depth, source, version): | ellipsisroots, visitnodes, depth, source, version): | ||||
| if version in ('01', '02'): | |||||
| raise error.Abort( | |||||
| 'ellipsis nodes require at least cg3 on client and server, ' | |||||
| 'but negotiated version %s' % version) | |||||
| # We wrap cg1packer.revchunk, using a side channel to pass | # We wrap cg1packer.revchunk, using a side channel to pass | ||||
| # relevant_nodes into that area. Then if linknode isn't in the | # relevant_nodes into that area. Then if linknode isn't in the | ||||
| # set, we know we have an ellipsis node and we should defer | # set, we know we have an ellipsis node and we should defer | ||||
| # sending that node's data. We override close() to detect | # sending that node's data. We override close() to detect | ||||
| # pending ellipsis nodes and flush them. | # pending ellipsis nodes and flush them. | ||||
| packer = getbundler(version, repo, filematcher=match, | packer = getbundler(version, repo, filematcher=match, | ||||
| ellipses=True, | |||||
| shallow=depth is not None, | shallow=depth is not None, | ||||
| ellipsisroots=ellipsisroots) | ellipsisroots=ellipsisroots) | ||||
| # Give the packer the list of nodes which should not be | # Give the packer the list of nodes which should not be | ||||
| # ellipsis nodes. We store this rather than the set of nodes | # ellipsis nodes. We store this rather than the set of nodes | ||||
| # that should be an ellipsis because for very large histories | # that should be an ellipsis because for very large histories | ||||
| # we expect this to be significantly smaller. | # we expect this to be significantly smaller. | ||||
| packer._full_nodes = relevant_nodes | packer._full_nodes = relevant_nodes | ||||
| return packer.generate(common, visitnodes, False, source) | return packer.generate(common, visitnodes, False, source) | ||||