| hg-reviewers |
cgpacker._clnodetorev is a glorified cache/index of changelog
nodes to revision numbers.
I'm not sure why it exists. Maybe performance? But its presence
is making refactoring of this code more complicated than it needs
to be.
This commit removes the cache and replaces it with direct lookups
against the changelog.
If this cache was for performance reasons, we should be able to
restore it easily enough... after the changegroup refactor is
complete.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/changegroup.py (30 lines) |
| Status | Author | Revision | |
|---|---|---|---|
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg | ||
| Closed | indygreg |
| # for generaldelta revlogs, we linearize the revs; this will both be | # for generaldelta revlogs, we linearize the revs; this will both be | ||||
| # much quicker and generate a much smaller bundle | # much quicker and generate a much smaller bundle | ||||
| if (store._generaldelta and reorder is None) or reorder: | if (store._generaldelta and reorder is None) or reorder: | ||||
| dag = dagutil.revlogdag(store) | dag = dagutil.revlogdag(store) | ||||
| return dag.linearize(set(store.rev(n) for n in nodes)) | return dag.linearize(set(store.rev(n) for n in nodes)) | ||||
| else: | else: | ||||
| return sorted([store.rev(n) for n in nodes]) | return sorted([store.rev(n) for n in nodes]) | ||||
| def _sortnodesellipsis(store, nodes, clnodetorev, lookup): | def _sortnodesellipsis(store, nodes, cl, lookup): | ||||
| """Sort nodes for changegroup generation and turn into revnums.""" | """Sort nodes for changegroup generation and turn into revnums.""" | ||||
| # Ellipses serving mode. | # Ellipses serving mode. | ||||
| # | # | ||||
| # In a perfect world, we'd generate better ellipsis-ified graphs | # In a perfect world, we'd generate better ellipsis-ified graphs | ||||
| # for non-changelog revlogs. In practice, we haven't started doing | # for non-changelog revlogs. In practice, we haven't started doing | ||||
| # that yet, so the resulting DAGs for the manifestlog and filelogs | # that yet, so the resulting DAGs for the manifestlog and filelogs | ||||
| # are actually full of bogus parentage on all the ellipsis | # are actually full of bogus parentage on all the ellipsis | ||||
| # nodes. This has the side effect that, while the contents are | # nodes. This has the side effect that, while the contents are | ||||
| # correct, the individual DAGs might be completely out of whack in | # correct, the individual DAGs might be completely out of whack in | ||||
| # a case like 882681bc3166 and its ancestors (back about 10 | # a case like 882681bc3166 and its ancestors (back about 10 | ||||
| # revisions or so) in the main hg repo. | # revisions or so) in the main hg repo. | ||||
| # | # | ||||
| # The one invariant we *know* holds is that the new (potentially | # The one invariant we *know* holds is that the new (potentially | ||||
| # bogus) DAG shape will be valid if we order the nodes in the | # bogus) DAG shape will be valid if we order the nodes in the | ||||
| # order that they're introduced in dramatis personae by the | # order that they're introduced in dramatis personae by the | ||||
| # changelog, so what we do is we sort the non-changelog histories | # changelog, so what we do is we sort the non-changelog histories | ||||
| # by the order in which they are used by the changelog. | # by the order in which they are used by the changelog. | ||||
| key = lambda n: clnodetorev[lookup(n)] | key = lambda n: cl.rev(lookup(n)) | ||||
| return [store.rev(n) for n in sorted(nodes, key=key)] | return [store.rev(n) for n in sorted(nodes, key=key)] | ||||
| def _revisiondeltanormal(store, rev, prev, linknode, deltaparentfn): | def _revisiondeltanormal(store, rev, prev, linknode, deltaparentfn): | ||||
| """Construct a revision delta for non-ellipses changegroup generation.""" | """Construct a revision delta for non-ellipses changegroup generation.""" | ||||
| node = store.node(rev) | node = store.node(rev) | ||||
| p1, p2 = store.parentrevs(rev) | p1, p2 = store.parentrevs(rev) | ||||
| base = deltaparentfn(store, rev, p1, p2, prev) | base = deltaparentfn(store, rev, p1, p2, prev) | ||||
| else: | else: | ||||
| self._verbosenote = lambda s: None | self._verbosenote = lambda s: None | ||||
| # Maps CL revs to per-revlog revisions. Cleared in close() at | # Maps CL revs to per-revlog revisions. Cleared in close() at | ||||
| # the end of each group. | # the end of each group. | ||||
| self._clrevtolocalrev = {} | self._clrevtolocalrev = {} | ||||
| self._nextclrevtolocalrev = {} | self._nextclrevtolocalrev = {} | ||||
| # Maps changelog nodes to changelog revs. Filled in once | |||||
| # during changelog stage and then left unmodified. | |||||
| self._clnodetorev = {} | |||||
| def _close(self): | def _close(self): | ||||
| # Ellipses serving mode. | # Ellipses serving mode. | ||||
| self._clrevtolocalrev.clear() | self._clrevtolocalrev.clear() | ||||
| if self._nextclrevtolocalrev is not None: | if self._nextclrevtolocalrev is not None: | ||||
| self._clrevtolocalrev = self._nextclrevtolocalrev | self._clrevtolocalrev = self._nextclrevtolocalrev | ||||
| self._nextclrevtolocalrev = None | self._nextclrevtolocalrev = None | ||||
| return closechunk() | return closechunk() | ||||
| If units is not None, progress detail will be generated, units specifies | If units is not None, progress detail will be generated, units specifies | ||||
| the type of revlog that is touched (changelog, manifest, etc.). | the type of revlog that is touched (changelog, manifest, etc.). | ||||
| """ | """ | ||||
| # if we don't have any revisions touched by these changesets, bail | # if we don't have any revisions touched by these changesets, bail | ||||
| if len(revs) == 0: | if len(revs) == 0: | ||||
| yield self._close() | yield self._close() | ||||
| return | return | ||||
| cl = self._repo.changelog | |||||
| # add the parent of the first rev | # add the parent of the first rev | ||||
| p = store.parentrevs(revs[0])[0] | p = store.parentrevs(revs[0])[0] | ||||
| revs.insert(0, p) | revs.insert(0, p) | ||||
| # build deltas | # build deltas | ||||
| progress = None | progress = None | ||||
| if units is not None: | if units is not None: | ||||
| progress = self._repo.ui.makeprogress(_('bundling'), unit=units, | progress = self._repo.ui.makeprogress(_('bundling'), unit=units, | ||||
| total=(len(revs) - 1)) | total=(len(revs) - 1)) | ||||
| for r in pycompat.xrange(len(revs) - 1): | for r in pycompat.xrange(len(revs) - 1): | ||||
| if progress: | if progress: | ||||
| progress.update(r + 1) | progress.update(r + 1) | ||||
| prev, curr = revs[r], revs[r + 1] | prev, curr = revs[r], revs[r + 1] | ||||
| linknode = lookup(store.node(curr)) | linknode = lookup(store.node(curr)) | ||||
| if self._ellipses: | if self._ellipses: | ||||
| linkrev = self._clnodetorev[linknode] | linkrev = cl.rev(linknode) | ||||
| self._clrevtolocalrev[linkrev] = curr | self._clrevtolocalrev[linkrev] = curr | ||||
| # This is a node to send in full, because the changeset it | # This is a node to send in full, because the changeset it | ||||
| # corresponds to was a full changeset. | # corresponds to was a full changeset. | ||||
| if linknode in self._fullclnodes: | if linknode in self._fullclnodes: | ||||
| delta = _revisiondeltanormal(store, curr, prev, linknode, | delta = _revisiondeltanormal(store, curr, prev, linknode, | ||||
| self._deltaparentfn) | self._deltaparentfn) | ||||
| elif linkrev not in self._precomputedellipsis: | elif linkrev not in self._precomputedellipsis: | ||||
| # Callback for the changelog, used to collect changed files and | # Callback for the changelog, used to collect changed files and | ||||
| # manifest nodes. | # manifest nodes. | ||||
| # Returns the linkrev node (identity in the changelog case). | # Returns the linkrev node (identity in the changelog case). | ||||
| def lookupcl(x): | def lookupcl(x): | ||||
| c = cl.read(x) | c = cl.read(x) | ||||
| clrevorder[x] = len(clrevorder) | clrevorder[x] = len(clrevorder) | ||||
| if self._ellipses: | if self._ellipses: | ||||
| self._clnodetorev[x] = cl.rev(x) | |||||
| # Only update mfs if x is going to be sent. Otherwise we | # Only update mfs if x is going to be sent. Otherwise we | ||||
| # end up with bogus linkrevs specified for manifests and | # end up with bogus linkrevs specified for manifests and | ||||
| # we skip some manifest nodes that we should otherwise | # we skip some manifest nodes that we should otherwise | ||||
| # have sent. | # have sent. | ||||
| if (x in self._fullclnodes | if (x in self._fullclnodes | ||||
| or cl.rev(x) in self._precomputedellipsis): | or cl.rev(x) in self._precomputedellipsis): | ||||
| n = c[0] | n = c[0] | ||||
| # Record the first changeset introducing this manifest | # Record the first changeset introducing this manifest | ||||
| def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs, | def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs, | ||||
| fnodes, source): | fnodes, source): | ||||
| """Returns an iterator of changegroup chunks containing manifests. | """Returns an iterator of changegroup chunks containing manifests. | ||||
| `source` is unused here, but is used by extensions like remotefilelog to | `source` is unused here, but is used by extensions like remotefilelog to | ||||
| change what is sent based in pulls vs pushes, etc. | change what is sent based in pulls vs pushes, etc. | ||||
| """ | """ | ||||
| repo = self._repo | repo = self._repo | ||||
| cl = repo.changelog | |||||
| mfl = repo.manifestlog | mfl = repo.manifestlog | ||||
| dirlog = mfl._revlog.dirlog | dirlog = mfl._revlog.dirlog | ||||
| tmfnodes = {'': mfs} | tmfnodes = {'': mfs} | ||||
| # Callback for the manifest, used to collect linkrevs for filelog | # Callback for the manifest, used to collect linkrevs for filelog | ||||
| # revisions. | # revisions. | ||||
| # Returns the linkrev node (collected in lookupcl). | # Returns the linkrev node (collected in lookupcl). | ||||
| def makelookupmflinknode(dir, nodes): | def makelookupmflinknode(dir, nodes): | ||||
| while tmfnodes: | while tmfnodes: | ||||
| dir, nodes = tmfnodes.popitem() | dir, nodes = tmfnodes.popitem() | ||||
| store = dirlog(dir) | store = dirlog(dir) | ||||
| prunednodes = self._prune(store, nodes, commonrevs) | prunednodes = self._prune(store, nodes, commonrevs) | ||||
| if not dir or prunednodes: | if not dir or prunednodes: | ||||
| lookupfn = makelookupmflinknode(dir, nodes) | lookupfn = makelookupmflinknode(dir, nodes) | ||||
| if self._ellipses: | if self._ellipses: | ||||
| revs = _sortnodesellipsis(store, prunednodes, | revs = _sortnodesellipsis(store, prunednodes, cl, | ||||
| self._clnodetorev, lookupfn) | lookupfn) | ||||
| else: | else: | ||||
| revs = _sortnodesnormal(store, prunednodes, | revs = _sortnodesnormal(store, prunednodes, | ||||
| self._reorder) | self._reorder) | ||||
| for x in self._packmanifests(dir, store, revs, lookupfn): | for x in self._packmanifests(dir, store, revs, lookupfn): | ||||
| size += len(x) | size += len(x) | ||||
| yield x | yield x | ||||
| self._verbosenote(_('%8.i (manifests)\n') % size) | self._verbosenote(_('%8.i (manifests)\n') % size) | ||||
| elif fnode: | elif fnode: | ||||
| links[fnode] = lr | links[fnode] = lr | ||||
| return links | return links | ||||
| return self._generatefiles(changedfiles, linknodes, commonrevs, source) | return self._generatefiles(changedfiles, linknodes, commonrevs, source) | ||||
| def _generatefiles(self, changedfiles, linknodes, commonrevs, source): | def _generatefiles(self, changedfiles, linknodes, commonrevs, source): | ||||
| repo = self._repo | repo = self._repo | ||||
| cl = repo.changelog | |||||
| progress = repo.ui.makeprogress(_('bundling'), unit=_('files'), | progress = repo.ui.makeprogress(_('bundling'), unit=_('files'), | ||||
| total=len(changedfiles)) | total=len(changedfiles)) | ||||
| for i, fname in enumerate(sorted(changedfiles)): | for i, fname in enumerate(sorted(changedfiles)): | ||||
| filerevlog = repo.file(fname) | filerevlog = repo.file(fname) | ||||
| if not filerevlog: | if not filerevlog: | ||||
| raise error.Abort(_("empty or missing file data for %s") % | raise error.Abort(_("empty or missing file data for %s") % | ||||
| fname) | fname) | ||||
| linkrevnodes = linknodes(filerevlog, fname) | linkrevnodes = linknodes(filerevlog, fname) | ||||
| # Lookup for filenodes, we collected the linkrev nodes above in the | # Lookup for filenodes, we collected the linkrev nodes above in the | ||||
| # fastpath case and with lookupmf in the slowpath case. | # fastpath case and with lookupmf in the slowpath case. | ||||
| def lookupfilelog(x): | def lookupfilelog(x): | ||||
| return linkrevnodes[x] | return linkrevnodes[x] | ||||
| filenodes = self._prune(filerevlog, linkrevnodes, commonrevs) | filenodes = self._prune(filerevlog, linkrevnodes, commonrevs) | ||||
| if filenodes: | if filenodes: | ||||
| if self._ellipses: | if self._ellipses: | ||||
| revs = _sortnodesellipsis(filerevlog, filenodes, | revs = _sortnodesellipsis(filerevlog, filenodes, | ||||
| self._clnodetorev, lookupfilelog) | cl, lookupfilelog) | ||||
| else: | else: | ||||
| revs = _sortnodesnormal(filerevlog, filenodes, | revs = _sortnodesnormal(filerevlog, filenodes, | ||||
| self._reorder) | self._reorder) | ||||
| progress.update(i + 1, item=fname) | progress.update(i + 1, item=fname) | ||||
| h = _fileheader(fname) | h = _fileheader(fname) | ||||
| size = len(h) | size = len(h) | ||||
| yield h | yield h | ||||
| something local. This does that for us, and during the | something local. This does that for us, and during the | ||||
| changelog sending phase will also expand the stored | changelog sending phase will also expand the stored | ||||
| mappings as needed. | mappings as needed. | ||||
| """ | """ | ||||
| if clrev == nullrev: | if clrev == nullrev: | ||||
| return nullrev | return nullrev | ||||
| if ischangelog: | if ischangelog: | ||||
| # If we're doing the changelog, it's possible that we | |||||
| # have a parent that is already on the client, and we | |||||
| # need to store some extra mapping information so that | |||||
| # our contained ellipsis nodes will be able to resolve | |||||
| # their parents. | |||||
| if clrev not in self._clrevtolocalrev: | |||||
| clnode = store.node(clrev) | |||||
| self._clnodetorev[clnode] = clrev | |||||
| return clrev | return clrev | ||||
| # Walk the ellipsis-ized changelog breadth-first looking for a | # Walk the ellipsis-ized changelog breadth-first looking for a | ||||
| # change that has been linked from the current revlog. | # change that has been linked from the current revlog. | ||||
| # | # | ||||
| # For a flat manifest revlog only a single step should be necessary | # For a flat manifest revlog only a single step should be necessary | ||||
| # as all relevant changelog entries are relevant to the flat | # as all relevant changelog entries are relevant to the flat | ||||
| # manifest. | # manifest. | ||||