( )⚙ D4139 changegroup: pass sorted revisions into group() (API)

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4139

changegroup: pass sorted revisions into group() (API)
ClosedPublic

Authored by indygreg on Aug 6 2018, 3:51 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG037debbf869c: changegroup: pass sorted revisions into group() (API)
Summary

Currently, group() receives a list of nodes and calls _sortgroup()
to sort them and turn them into revs. Since the sorting behavior
varies depending on the type of data being transferred, I think it
makes sense to perform the sorting before group() is invoked.

This commit extracts _sortgroup() to a pair of standalone functions.
It then moves the calling of these functions to the 3 call sites of
group(). group() now receives an iterable of revs instead of nodes.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

indygreg created this revision.Aug 6 2018, 3:51 PM
Closed by commit rHG037debbf869c: changegroup: pass sorted revisions into group() (API) (authored by indygreg). · Explain WhyAug 9 2018, 2:15 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/changegroup.py (106 lines)

Diff 10010

mercurial/changegroup.py

basenode = attr.ib() basenode = attr.ib()
# 20 byte node of changeset revision this delta is associated with. # 20 byte node of changeset revision this delta is associated with.
linknode = attr.ib() linknode = attr.ib()
# 2 bytes of flags to apply to revision data. # 2 bytes of flags to apply to revision data.
flags = attr.ib() flags = attr.ib()
# Iterable of chunks holding raw delta data. # Iterable of chunks holding raw delta data.
deltachunks = attr.ib() deltachunks = attr.ib()
def _sortnodesnormal(store, nodes, reorder):
"""Sort nodes for changegroup generation and turn into revnums."""
# for generaldelta revlogs, we linearize the revs; this will both be
# much quicker and generate a much smaller bundle
if (store._generaldelta and reorder is None) or reorder:
dag = dagutil.revlogdag(store)
return dag.linearize(set(store.rev(n) for n in nodes))
else:
return sorted([store.rev(n) for n in nodes])
def _sortnodesellipsis(store, nodes, clnodetorev, lookup):
"""Sort nodes for changegroup generation and turn into revnums."""
# Ellipses serving mode.
#
# In a perfect world, we'd generate better ellipsis-ified graphs
# for non-changelog revlogs. In practice, we haven't started doing
# that yet, so the resulting DAGs for the manifestlog and filelogs
# are actually full of bogus parentage on all the ellipsis
# nodes. This has the side effect that, while the contents are
# correct, the individual DAGs might be completely out of whack in
# a case like 882681bc3166 and its ancestors (back about 10
# revisions or so) in the main hg repo.
#
# The one invariant we *know* holds is that the new (potentially
# bogus) DAG shape will be valid if we order the nodes in the
# order that they're introduced in dramatis personae by the
# changelog, so what we do is we sort the non-changelog histories
# by the order in which they are used by the changelog.
key = lambda n: clnodetorev[lookup(n)]
return [store.rev(n) for n in sorted(nodes, key=key)]
class cgpacker(object): class cgpacker(object):
def __init__(self, repo, filematcher, version, allowreorder, def __init__(self, repo, filematcher, version, allowreorder,
deltaparentfn, builddeltaheader, manifestsend, deltaparentfn, builddeltaheader, manifestsend,
bundlecaps=None, ellipses=False, bundlecaps=None, ellipses=False,
shallow=False, ellipsisroots=None, fullnodes=None): shallow=False, ellipsisroots=None, fullnodes=None):
"""Given a source repo, construct a bundler. """Given a source repo, construct a bundler.
filematcher is a matcher that matches on files to include in the filematcher is a matcher that matches on files to include in the
# Ellipses serving mode. # Ellipses serving mode.
self._clrevtolocalrev.clear() self._clrevtolocalrev.clear()
if self._nextclrevtolocalrev: if self._nextclrevtolocalrev:
self.clrevtolocalrev = self._nextclrevtolocalrev self.clrevtolocalrev = self._nextclrevtolocalrev
self._nextclrevtolocalrev.clear() self._nextclrevtolocalrev.clear()
return closechunk() return closechunk()
# Extracted both for clarity and for overriding in extensions. def group(self, revs, store, ischangelog, lookup, units=None):
def _sortgroup(self, store, ischangelog, nodelist, lookup):
"""Sort nodes for change group and turn them into revnums."""
# Ellipses serving mode.
#
# In a perfect world, we'd generate better ellipsis-ified graphs
# for non-changelog revlogs. In practice, we haven't started doing
# that yet, so the resulting DAGs for the manifestlog and filelogs
# are actually full of bogus parentage on all the ellipsis
# nodes. This has the side effect that, while the contents are
# correct, the individual DAGs might be completely out of whack in
# a case like 882681bc3166 and its ancestors (back about 10
# revisions or so) in the main hg repo.
#
# The one invariant we *know* holds is that the new (potentially
# bogus) DAG shape will be valid if we order the nodes in the
# order that they're introduced in dramatis personae by the
# changelog, so what we do is we sort the non-changelog histories
# by the order in which they are used by the changelog.
if self._ellipses and not ischangelog:
key = lambda n: self._clnodetorev[lookup(n)]
return [store.rev(n) for n in sorted(nodelist, key=key)]
# for generaldelta revlogs, we linearize the revs; this will both be
# much quicker and generate a much smaller bundle
if (store._generaldelta and self._reorder is None) or self._reorder:
dag = dagutil.revlogdag(store)
return dag.linearize(set(store.rev(n) for n in nodelist))
else:
return sorted([store.rev(n) for n in nodelist])
def group(self, nodelist, store, ischangelog, lookup, units=None):
"""Calculate a delta group, yielding a sequence of changegroup chunks """Calculate a delta group, yielding a sequence of changegroup chunks
(strings). (strings).
Given a list of changeset revs, return a set of deltas and Given a list of changeset revs, return a set of deltas and
metadata corresponding to nodes. The first delta is metadata corresponding to nodes. The first delta is
first parent(nodelist[0]) -> nodelist[0], the receiver is first parent(nodelist[0]) -> nodelist[0], the receiver is
guaranteed to have this parent as it has all history before guaranteed to have this parent as it has all history before
these changesets. In the case firstparent is nullrev the these changesets. In the case firstparent is nullrev the
changegroup starts with a full revision. changegroup starts with a full revision.
If units is not None, progress detail will be generated, units specifies If units is not None, progress detail will be generated, units specifies
the type of revlog that is touched (changelog, manifest, etc.). the type of revlog that is touched (changelog, manifest, etc.).
""" """
# if we don't have any revisions touched by these changesets, bail # if we don't have any revisions touched by these changesets, bail
if len(nodelist) == 0: if len(revs) == 0:
yield self._close() yield self._close()
return return
revs = self._sortgroup(store, ischangelog, nodelist, lookup)
# add the parent of the first rev # add the parent of the first rev
p = store.parentrevs(revs[0])[0] p = store.parentrevs(revs[0])[0]
revs.insert(0, p) revs.insert(0, p)
# build deltas # build deltas
progress = None progress = None
if units is not None: if units is not None:
progress = self._repo.ui.makeprogress(_('bundling'), unit=units, progress = self._repo.ui.makeprogress(_('bundling'), unit=units,
# TODO this violates storage abstraction for manifests. # TODO this violates storage abstraction for manifests.
if isinstance(store, manifest.manifestrevlog): if isinstance(store, manifest.manifestrevlog):
if not self._filematcher.visitdir(store._dir[:-1] or '.'): if not self._filematcher.visitdir(store._dir[:-1] or '.'):
return [] return []
rr, rl = store.rev, store.linkrev rr, rl = store.rev, store.linkrev
return [n for n in missing if rl(rr(n)) not in commonrevs] return [n for n in missing if rl(rr(n)) not in commonrevs]
def _packmanifests(self, dir, mfnodes, lookuplinknode): def _packmanifests(self, dir, dirlog, revs, lookuplinknode):
"""Pack manifests into a changegroup stream. """Pack manifests into a changegroup stream.
Encodes the directory name in the output so multiple manifests Encodes the directory name in the output so multiple manifests
can be sent. Multiple manifests is not supported by cg1 and cg2. can be sent. Multiple manifests is not supported by cg1 and cg2.
""" """
if dir: if dir:
assert self.version == b'03' assert self.version == b'03'
yield _fileheader(dir) yield _fileheader(dir)
# TODO violates storage abstractions by assuming revlogs. for chunk in self.group(revs, dirlog, False, lookuplinknode,
dirlog = self._repo.manifestlog._revlog.dirlog(dir)
for chunk in self.group(mfnodes, dirlog, False, lookuplinknode,
units=_('manifests')): units=_('manifests')):
yield chunk yield chunk
def generate(self, commonrevs, clnodes, fastpathlinkrev, source): def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
"""Yield a sequence of changegroup byte chunks.""" """Yield a sequence of changegroup byte chunks."""
repo = self._repo repo = self._repo
cl = repo.changelog cl = repo.changelog
# record the first changeset introducing this manifest version # record the first changeset introducing this manifest version
mfs.setdefault(n, x) mfs.setdefault(n, x)
# Record a complete list of potentially-changed files in # Record a complete list of potentially-changed files in
# this manifest. # this manifest.
changedfiles.update(c[3]) changedfiles.update(c[3])
return x return x
# Changelog doesn't benefit from reordering revisions. So send out
# revisions in store order.
revs = sorted(cl.rev(n) for n in nodes)
state = { state = {
'clrevorder': clrevorder, 'clrevorder': clrevorder,
'mfs': mfs, 'mfs': mfs,
'changedfiles': changedfiles, 'changedfiles': changedfiles,
} }
gen = self.group(nodes, cl, True, lookupcl, units=_('changesets')) gen = self.group(revs, cl, True, lookupcl, units=_('changesets'))
return state, gen return state, gen
def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs, def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
fnodes, source): fnodes, source):
"""Returns an iterator of changegroup chunks containing manifests. """Returns an iterator of changegroup chunks containing manifests.
`source` is unused here, but is used by extensions like remotefilelog to `source` is unused here, but is used by extensions like remotefilelog to
if clrevorder[clnode] < clrevorder[fclnode]: if clrevorder[clnode] < clrevorder[fclnode]:
fclnodes[n] = clnode fclnodes[n] = clnode
return clnode return clnode
return lookupmflinknode return lookupmflinknode
size = 0 size = 0
while tmfnodes: while tmfnodes:
dir, nodes = tmfnodes.popitem() dir, nodes = tmfnodes.popitem()
prunednodes = self._prune(dirlog(dir), nodes, commonrevs) store = dirlog(dir)
prunednodes = self._prune(store, nodes, commonrevs)
if not dir or prunednodes: if not dir or prunednodes:
for x in self._packmanifests(dir, prunednodes, lookupfn = makelookupmflinknode(dir, nodes)
makelookupmflinknode(dir, nodes)):
if self._ellipses:
revs = _sortnodesellipsis(store, prunednodes,
self._clnodetorev, lookupfn)
else:
revs = _sortnodesnormal(store, prunednodes,
self._reorder)
for x in self._packmanifests(dir, store, revs, lookupfn):
size += len(x) size += len(x)
yield x yield x
self._verbosenote(_('%8.i (manifests)\n') % size) self._verbosenote(_('%8.i (manifests)\n') % size)
yield self._manifestsend yield self._manifestsend
# The 'source' parameter is useful for extensions # The 'source' parameter is useful for extensions
def generatefiles(self, changedfiles, linknodes, commonrevs, source): def generatefiles(self, changedfiles, linknodes, commonrevs, source):
changedfiles = list(filter(self._filematcher, changedfiles)) changedfiles = list(filter(self._filematcher, changedfiles))
linkrevnodes = linknodes(filerevlog, fname) linkrevnodes = linknodes(filerevlog, fname)
# Lookup for filenodes, we collected the linkrev nodes above in the # Lookup for filenodes, we collected the linkrev nodes above in the
# fastpath case and with lookupmf in the slowpath case. # fastpath case and with lookupmf in the slowpath case.
def lookupfilelog(x): def lookupfilelog(x):
return linkrevnodes[x] return linkrevnodes[x]
filenodes = self._prune(filerevlog, linkrevnodes, commonrevs) filenodes = self._prune(filerevlog, linkrevnodes, commonrevs)
if filenodes: if filenodes:
if self._ellipses:
revs = _sortnodesellipsis(filerevlog, filenodes,
self._clnodetorev, lookupfilelog)
else:
revs = _sortnodesnormal(filerevlog, filenodes,
self._reorder)
progress.update(i + 1, item=fname) progress.update(i + 1, item=fname)
h = _fileheader(fname) h = _fileheader(fname)
size = len(h) size = len(h)
yield h yield h
for chunk in self.group(filenodes, filerevlog, False, for chunk in self.group(revs, filerevlog, False, lookupfilelog):
lookupfilelog):
size += len(chunk) size += len(chunk)
yield chunk yield chunk
self._verbosenote(_('%8.i %s\n') % (size, fname)) self._verbosenote(_('%8.i %s\n') % (size, fname))
progress.complete() progress.complete()
def _revchunk(self, store, ischangelog, rev, prev, linknode): def _revchunk(self, store, ischangelog, rev, prev, linknode):
if self._ellipses: if self._ellipses:
fn = self._revisiondeltanarrow fn = self._revisiondeltanarrow