| hg-reviewers |
"revlog" shadows the module import. But more importantly, changegroup
generation should be storage agnostic and not assume the existence
of revlogs. Let's rename the thing providing revision storage to
"store" to reflect this ideal property.
| Lint Skipped |
| Unit Tests Skipped |
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/changegroup.py (86 lines) |
| Commit | Parents | Author | Summary | Date |
|---|---|---|---|---|
| Gregory Szorc | Aug 2 2018, 8:05 PM |
| Status | Author | Revision | |
|---|---|---|---|
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg |
| self.changelog_done = True | self.changelog_done = True | ||||
| return closechunk() | return closechunk() | ||||
| def fileheader(self, fname): | def fileheader(self, fname): | ||||
| return chunkheader(len(fname)) + fname | return chunkheader(len(fname)) + fname | ||||
| # Extracted both for clarity and for overriding in extensions. | # Extracted both for clarity and for overriding in extensions. | ||||
| def _sortgroup(self, revlog, nodelist, lookup): | def _sortgroup(self, store, nodelist, lookup): | ||||
| """Sort nodes for change group and turn them into revnums.""" | """Sort nodes for change group and turn them into revnums.""" | ||||
| # Ellipses serving mode. | # Ellipses serving mode. | ||||
| # | # | ||||
| # In a perfect world, we'd generate better ellipsis-ified graphs | # In a perfect world, we'd generate better ellipsis-ified graphs | ||||
| # for non-changelog revlogs. In practice, we haven't started doing | # for non-changelog revlogs. In practice, we haven't started doing | ||||
| # that yet, so the resulting DAGs for the manifestlog and filelogs | # that yet, so the resulting DAGs for the manifestlog and filelogs | ||||
| # are actually full of bogus parentage on all the ellipsis | # are actually full of bogus parentage on all the ellipsis | ||||
| # nodes. This has the side effect that, while the contents are | # nodes. This has the side effect that, while the contents are | ||||
| # correct, the individual DAGs might be completely out of whack in | # correct, the individual DAGs might be completely out of whack in | ||||
| # a case like 882681bc3166 and its ancestors (back about 10 | # a case like 882681bc3166 and its ancestors (back about 10 | ||||
| # revisions or so) in the main hg repo. | # revisions or so) in the main hg repo. | ||||
| # | # | ||||
| # The one invariant we *know* holds is that the new (potentially | # The one invariant we *know* holds is that the new (potentially | ||||
| # bogus) DAG shape will be valid if we order the nodes in the | # bogus) DAG shape will be valid if we order the nodes in the | ||||
| # order that they're introduced in dramatis personae by the | # order that they're introduced in dramatis personae by the | ||||
| # changelog, so what we do is we sort the non-changelog histories | # changelog, so what we do is we sort the non-changelog histories | ||||
| # by the order in which they are used by the changelog. | # by the order in which they are used by the changelog. | ||||
| if util.safehasattr(self, 'full_nodes') and self.clnode_to_rev: | if util.safehasattr(self, 'full_nodes') and self.clnode_to_rev: | ||||
| key = lambda n: self.clnode_to_rev[lookup(n)] | key = lambda n: self.clnode_to_rev[lookup(n)] | ||||
| return [revlog.rev(n) for n in sorted(nodelist, key=key)] | return [store.rev(n) for n in sorted(nodelist, key=key)] | ||||
| # for generaldelta revlogs, we linearize the revs; this will both be | # for generaldelta revlogs, we linearize the revs; this will both be | ||||
| # much quicker and generate a much smaller bundle | # much quicker and generate a much smaller bundle | ||||
| if (revlog._generaldelta and self._reorder is None) or self._reorder: | if (store._generaldelta and self._reorder is None) or self._reorder: | ||||
| dag = dagutil.revlogdag(revlog) | dag = dagutil.revlogdag(store) | ||||
| return dag.linearize(set(revlog.rev(n) for n in nodelist)) | return dag.linearize(set(store.rev(n) for n in nodelist)) | ||||
| else: | else: | ||||
| return sorted([revlog.rev(n) for n in nodelist]) | return sorted([store.rev(n) for n in nodelist]) | ||||
| def group(self, nodelist, revlog, lookup, units=None): | def group(self, nodelist, store, lookup, units=None): | ||||
| """Calculate a delta group, yielding a sequence of changegroup chunks | """Calculate a delta group, yielding a sequence of changegroup chunks | ||||
| (strings). | (strings). | ||||
| Given a list of changeset revs, return a set of deltas and | Given a list of changeset revs, return a set of deltas and | ||||
| metadata corresponding to nodes. The first delta is | metadata corresponding to nodes. The first delta is | ||||
| first parent(nodelist[0]) -> nodelist[0], the receiver is | first parent(nodelist[0]) -> nodelist[0], the receiver is | ||||
| guaranteed to have this parent as it has all history before | guaranteed to have this parent as it has all history before | ||||
| these changesets. In the case firstparent is nullrev the | these changesets. In the case firstparent is nullrev the | ||||
| changegroup starts with a full revision. | changegroup starts with a full revision. | ||||
| If units is not None, progress detail will be generated, units specifies | If units is not None, progress detail will be generated, units specifies | ||||
| the type of revlog that is touched (changelog, manifest, etc.). | the type of revlog that is touched (changelog, manifest, etc.). | ||||
| """ | """ | ||||
| # if we don't have any revisions touched by these changesets, bail | # if we don't have any revisions touched by these changesets, bail | ||||
| if len(nodelist) == 0: | if len(nodelist) == 0: | ||||
| yield self.close() | yield self.close() | ||||
| return | return | ||||
| revs = self._sortgroup(revlog, nodelist, lookup) | revs = self._sortgroup(store, nodelist, lookup) | ||||
| # add the parent of the first rev | # add the parent of the first rev | ||||
| p = revlog.parentrevs(revs[0])[0] | p = store.parentrevs(revs[0])[0] | ||||
| revs.insert(0, p) | revs.insert(0, p) | ||||
| # build deltas | # build deltas | ||||
| progress = None | progress = None | ||||
| if units is not None: | if units is not None: | ||||
| progress = self._repo.ui.makeprogress(_('bundling'), unit=units, | progress = self._repo.ui.makeprogress(_('bundling'), unit=units, | ||||
| total=(len(revs) - 1)) | total=(len(revs) - 1)) | ||||
| for r in pycompat.xrange(len(revs) - 1): | for r in pycompat.xrange(len(revs) - 1): | ||||
| if progress: | if progress: | ||||
| progress.update(r + 1) | progress.update(r + 1) | ||||
| prev, curr = revs[r], revs[r + 1] | prev, curr = revs[r], revs[r + 1] | ||||
| linknode = lookup(revlog.node(curr)) | linknode = lookup(store.node(curr)) | ||||
| for c in self.revchunk(revlog, curr, prev, linknode): | for c in self.revchunk(store, curr, prev, linknode): | ||||
| yield c | yield c | ||||
| if progress: | if progress: | ||||
| progress.complete() | progress.complete() | ||||
| yield self.close() | yield self.close() | ||||
| # filter any nodes that claim to be part of the known set | # filter any nodes that claim to be part of the known set | ||||
| def prune(self, revlog, missing, commonrevs): | def prune(self, store, missing, commonrevs): | ||||
| # TODO this violates storage abstraction for manifests. | # TODO this violates storage abstraction for manifests. | ||||
| if isinstance(revlog, manifest.manifestrevlog): | if isinstance(store, manifest.manifestrevlog): | ||||
| if not self._filematcher.visitdir(revlog._dir[:-1] or '.'): | if not self._filematcher.visitdir(store._dir[:-1] or '.'): | ||||
| return [] | return [] | ||||
| rr, rl = revlog.rev, revlog.linkrev | rr, rl = store.rev, store.linkrev | ||||
| return [n for n in missing if rl(rr(n)) not in commonrevs] | return [n for n in missing if rl(rr(n)) not in commonrevs] | ||||
| def _packmanifests(self, dir, mfnodes, lookuplinknode): | def _packmanifests(self, dir, mfnodes, lookuplinknode): | ||||
| """Pack flat manifests into a changegroup stream.""" | """Pack flat manifests into a changegroup stream.""" | ||||
| assert not dir | assert not dir | ||||
| for chunk in self.group(mfnodes, self._repo.manifestlog._revlog, | for chunk in self.group(mfnodes, self._repo.manifestlog._revlog, | ||||
| lookuplinknode, units=_('manifests')): | lookuplinknode, units=_('manifests')): | ||||
| yield chunk | yield chunk | ||||
| size = len(h) | size = len(h) | ||||
| yield h | yield h | ||||
| for chunk in self.group(filenodes, filerevlog, lookupfilelog): | for chunk in self.group(filenodes, filerevlog, lookupfilelog): | ||||
| size += len(chunk) | size += len(chunk) | ||||
| yield chunk | yield chunk | ||||
| self._verbosenote(_('%8.i %s\n') % (size, fname)) | self._verbosenote(_('%8.i %s\n') % (size, fname)) | ||||
| progress.complete() | progress.complete() | ||||
| def deltaparent(self, revlog, rev, p1, p2, prev): | def deltaparent(self, store, rev, p1, p2, prev): | ||||
| if not revlog.candelta(prev, rev): | if not store.candelta(prev, rev): | ||||
| raise error.ProgrammingError('cg1 should not be used in this case') | raise error.ProgrammingError('cg1 should not be used in this case') | ||||
| return prev | return prev | ||||
| def revchunk(self, revlog, rev, prev, linknode): | def revchunk(self, store, rev, prev, linknode): | ||||
| if util.safehasattr(self, 'full_nodes'): | if util.safehasattr(self, 'full_nodes'): | ||||
| fn = self._revchunknarrow | fn = self._revchunknarrow | ||||
| else: | else: | ||||
| fn = self._revchunknormal | fn = self._revchunknormal | ||||
| return fn(revlog, rev, prev, linknode) | return fn(store, rev, prev, linknode) | ||||
| def _revchunknormal(self, revlog, rev, prev, linknode): | def _revchunknormal(self, store, rev, prev, linknode): | ||||
| node = revlog.node(rev) | node = store.node(rev) | ||||
| p1, p2 = revlog.parentrevs(rev) | p1, p2 = store.parentrevs(rev) | ||||
| base = self.deltaparent(revlog, rev, p1, p2, prev) | base = self.deltaparent(store, rev, p1, p2, prev) | ||||
| prefix = '' | prefix = '' | ||||
| if revlog.iscensored(base) or revlog.iscensored(rev): | if store.iscensored(base) or store.iscensored(rev): | ||||
| try: | try: | ||||
| delta = revlog.revision(node, raw=True) | delta = store.revision(node, raw=True) | ||||
| except error.CensoredNodeError as e: | except error.CensoredNodeError as e: | ||||
| delta = e.tombstone | delta = e.tombstone | ||||
| if base == nullrev: | if base == nullrev: | ||||
| prefix = mdiff.trivialdiffheader(len(delta)) | prefix = mdiff.trivialdiffheader(len(delta)) | ||||
| else: | else: | ||||
| baselen = revlog.rawsize(base) | baselen = store.rawsize(base) | ||||
| prefix = mdiff.replacediffheader(baselen, len(delta)) | prefix = mdiff.replacediffheader(baselen, len(delta)) | ||||
| elif base == nullrev: | elif base == nullrev: | ||||
| delta = revlog.revision(node, raw=True) | delta = store.revision(node, raw=True) | ||||
| prefix = mdiff.trivialdiffheader(len(delta)) | prefix = mdiff.trivialdiffheader(len(delta)) | ||||
| else: | else: | ||||
| delta = revlog.revdiff(base, rev) | delta = store.revdiff(base, rev) | ||||
| p1n, p2n = revlog.parents(node) | p1n, p2n = store.parents(node) | ||||
| basenode = revlog.node(base) | basenode = store.node(base) | ||||
| flags = revlog.flags(rev) | flags = store.flags(rev) | ||||
| meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags) | meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags) | ||||
| meta += prefix | meta += prefix | ||||
| l = len(meta) + len(delta) | l = len(meta) + len(delta) | ||||
| yield chunkheader(l) | yield chunkheader(l) | ||||
| yield meta | yield meta | ||||
| yield delta | yield delta | ||||
| def _revchunknarrow(self, revlog, rev, prev, linknode): | def _revchunknarrow(self, store, rev, prev, linknode): | ||||
| # build up some mapping information that's useful later. See | # build up some mapping information that's useful later. See | ||||
| # the local() nested function below. | # the local() nested function below. | ||||
| if not self.changelog_done: | if not self.changelog_done: | ||||
| self.clnode_to_rev[linknode] = rev | self.clnode_to_rev[linknode] = rev | ||||
| linkrev = rev | linkrev = rev | ||||
| self.clrev_to_localrev[linkrev] = rev | self.clrev_to_localrev[linkrev] = rev | ||||
| else: | else: | ||||
| linkrev = self.clnode_to_rev[linknode] | linkrev = self.clnode_to_rev[linknode] | ||||
| self.clrev_to_localrev[linkrev] = rev | self.clrev_to_localrev[linkrev] = rev | ||||
| # This is a node to send in full, because the changeset it | # This is a node to send in full, because the changeset it | ||||
| # corresponds to was a full changeset. | # corresponds to was a full changeset. | ||||
| if linknode in self.full_nodes: | if linknode in self.full_nodes: | ||||
| for x in self._revchunknormal(revlog, rev, prev, linknode): | for x in self._revchunknormal(store, rev, prev, linknode): | ||||
| yield x | yield x | ||||
| return | return | ||||
| # At this point, a node can either be one we should skip or an | # At this point, a node can either be one we should skip or an | ||||
| # ellipsis. If it's not an ellipsis, bail immediately. | # ellipsis. If it's not an ellipsis, bail immediately. | ||||
| if linkrev not in self.precomputed_ellipsis: | if linkrev not in self.precomputed_ellipsis: | ||||
| return | return | ||||
| if not self.changelog_done: | if not self.changelog_done: | ||||
| # If we're doing the changelog, it's possible that we | # If we're doing the changelog, it's possible that we | ||||
| # have a parent that is already on the client, and we | # have a parent that is already on the client, and we | ||||
| # need to store some extra mapping information so that | # need to store some extra mapping information so that | ||||
| # our contained ellipsis nodes will be able to resolve | # our contained ellipsis nodes will be able to resolve | ||||
| # their parents. | # their parents. | ||||
| if clrev not in self.clrev_to_localrev: | if clrev not in self.clrev_to_localrev: | ||||
| clnode = revlog.node(clrev) | clnode = store.node(clrev) | ||||
| self.clnode_to_rev[clnode] = clrev | self.clnode_to_rev[clnode] = clrev | ||||
| return clrev | return clrev | ||||
| # Walk the ellipsis-ized changelog breadth-first looking for a | # Walk the ellipsis-ized changelog breadth-first looking for a | ||||
| # change that has been linked from the current revlog. | # change that has been linked from the current revlog. | ||||
| # | # | ||||
| # For a flat manifest revlog only a single step should be necessary | # For a flat manifest revlog only a single step should be necessary | ||||
| # as all relevant changelog entries are relevant to the flat | # as all relevant changelog entries are relevant to the flat | ||||
| # quotes because I haven't done enough examination | # quotes because I haven't done enough examination | ||||
| # of edge cases to convince myself this is really | # of edge cases to convince myself this is really | ||||
| # a fact - it works for all the (admittedly | # a fact - it works for all the (admittedly | ||||
| # thorough) cases in our testsuite, but I would be | # thorough) cases in our testsuite, but I would be | ||||
| # somewhat unsurprised to find a case in the wild | # somewhat unsurprised to find a case in the wild | ||||
| # where this breaks down a bit. That said, I don't | # where this breaks down a bit. That said, I don't | ||||
| # know if it would hurt anything. | # know if it would hurt anything. | ||||
| for i in pycompat.xrange(rev, 0, -1): | for i in pycompat.xrange(rev, 0, -1): | ||||
| if revlog.linkrev(i) == clrev: | if store.linkrev(i) == clrev: | ||||
| return i | return i | ||||
| # We failed to resolve a parent for this node, so | # We failed to resolve a parent for this node, so | ||||
| # we crash the changegroup construction. | # we crash the changegroup construction. | ||||
| raise error.Abort( | raise error.Abort( | ||||
| 'unable to resolve parent while packing %r %r' | 'unable to resolve parent while packing %r %r' | ||||
| ' for changeset %r' % (revlog.indexfile, rev, clrev)) | ' for changeset %r' % (store.indexfile, rev, clrev)) | ||||
| return nullrev | return nullrev | ||||
| if not linkparents or ( | if not linkparents or ( | ||||
| revlog.parentrevs(rev) == (nullrev, nullrev)): | store.parentrevs(rev) == (nullrev, nullrev)): | ||||
| p1, p2 = nullrev, nullrev | p1, p2 = nullrev, nullrev | ||||
| elif len(linkparents) == 1: | elif len(linkparents) == 1: | ||||
| p1, = sorted(local(p) for p in linkparents) | p1, = sorted(local(p) for p in linkparents) | ||||
| p2 = nullrev | p2 = nullrev | ||||
| else: | else: | ||||
| p1, p2 = sorted(local(p) for p in linkparents) | p1, p2 = sorted(local(p) for p in linkparents) | ||||
| n = revlog.node(rev) | n = store.node(rev) | ||||
| yield ellipsisdata( | yield ellipsisdata( | ||||
| self, rev, revlog, p1, p2, revlog.revision(n), linknode) | self, rev, store, p1, p2, store.revision(n), linknode) | ||||
| def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): | def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): | ||||
| # do nothing with basenode, it is implicitly the previous one in HG10 | # do nothing with basenode, it is implicitly the previous one in HG10 | ||||
| # do nothing with flags, it is implicitly 0 for cg1 and cg2 | # do nothing with flags, it is implicitly 0 for cg1 and cg2 | ||||
| return struct.pack(self.deltaheader, node, p1n, p2n, linknode) | return struct.pack(self.deltaheader, node, p1n, p2n, linknode) | ||||
| class cg2packer(cg1packer): | class cg2packer(cg1packer): | ||||
| version = '02' | version = '02' | ||||
| deltaheader = _CHANGEGROUPV2_DELTA_HEADER | deltaheader = _CHANGEGROUPV2_DELTA_HEADER | ||||
| def __init__(self, repo, filematcher, bundlecaps=None): | def __init__(self, repo, filematcher, bundlecaps=None): | ||||
| super(cg2packer, self).__init__(repo, filematcher, | super(cg2packer, self).__init__(repo, filematcher, | ||||
| bundlecaps=bundlecaps) | bundlecaps=bundlecaps) | ||||
| if self._reorder is None: | if self._reorder is None: | ||||
| # Since generaldelta is directly supported by cg2, reordering | # Since generaldelta is directly supported by cg2, reordering | ||||
| # generally doesn't help, so we disable it by default (treating | # generally doesn't help, so we disable it by default (treating | ||||
| # bundle.reorder=auto just like bundle.reorder=False). | # bundle.reorder=auto just like bundle.reorder=False). | ||||
| self._reorder = False | self._reorder = False | ||||
| def deltaparent(self, revlog, rev, p1, p2, prev): | def deltaparent(self, store, rev, p1, p2, prev): | ||||
| # Narrow ellipses mode. | # Narrow ellipses mode. | ||||
| if util.safehasattr(self, 'full_nodes'): | if util.safehasattr(self, 'full_nodes'): | ||||
| # TODO: send better deltas when in narrow mode. | # TODO: send better deltas when in narrow mode. | ||||
| # | # | ||||
| # changegroup.group() loops over revisions to send, | # changegroup.group() loops over revisions to send, | ||||
| # including revisions we'll skip. What this means is that | # including revisions we'll skip. What this means is that | ||||
| # `prev` will be a potentially useless delta base for all | # `prev` will be a potentially useless delta base for all | ||||
| # ellipsis nodes, as the client likely won't have it. In | # ellipsis nodes, as the client likely won't have it. In | ||||
| # the future we should do bookkeeping about which nodes | # the future we should do bookkeeping about which nodes | ||||
| # have been sent to the client, and try to be | # have been sent to the client, and try to be | ||||
| # significantly smarter about delta bases. This is | # significantly smarter about delta bases. This is | ||||
| # slightly tricky because this same code has to work for | # slightly tricky because this same code has to work for | ||||
| # all revlogs, and we don't have the linkrev/linknode here. | # all revlogs, and we don't have the linkrev/linknode here. | ||||
| return p1 | return p1 | ||||
| dp = revlog.deltaparent(rev) | dp = store.deltaparent(rev) | ||||
| if dp == nullrev and revlog.storedeltachains: | if dp == nullrev and store.storedeltachains: | ||||
| # Avoid sending full revisions when delta parent is null. Pick prev | # Avoid sending full revisions when delta parent is null. Pick prev | ||||
| # in that case. It's tempting to pick p1 in this case, as p1 will | # in that case. It's tempting to pick p1 in this case, as p1 will | ||||
| # be smaller in the common case. However, computing a delta against | # be smaller in the common case. However, computing a delta against | ||||
| # p1 may require resolving the raw text of p1, which could be | # p1 may require resolving the raw text of p1, which could be | ||||
| # expensive. The revlog caches should have prev cached, meaning | # expensive. The revlog caches should have prev cached, meaning | ||||
| # less CPU for changegroup generation. There is likely room to add | # less CPU for changegroup generation. There is likely room to add | ||||
| # a flag and/or config option to control this behavior. | # a flag and/or config option to control this behavior. | ||||
| base = prev | base = prev | ||||
| elif dp == nullrev: | elif dp == nullrev: | ||||
| # revlog is configured to use full snapshot for a reason, | # revlog is configured to use full snapshot for a reason, | ||||
| # stick to full snapshot. | # stick to full snapshot. | ||||
| base = nullrev | base = nullrev | ||||
| elif dp not in (p1, p2, prev): | elif dp not in (p1, p2, prev): | ||||
| # Pick prev when we can't be sure remote has the base revision. | # Pick prev when we can't be sure remote has the base revision. | ||||
| return prev | return prev | ||||
| else: | else: | ||||
| base = dp | base = dp | ||||
| if base != nullrev and not revlog.candelta(base, rev): | if base != nullrev and not store.candelta(base, rev): | ||||
| base = nullrev | base = nullrev | ||||
| return base | return base | ||||
| def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): | def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): | ||||
| # Do nothing with flags, it is implicitly 0 in cg1 and cg2 | # Do nothing with flags, it is implicitly 0 in cg1 and cg2 | ||||
| return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode) | return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode) | ||||
| class cg3packer(cg2packer): | class cg3packer(cg2packer): | ||||