This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4211

changegroup: emit revisiondelta instances from deltagroup()
ClosedPublic

Authored by indygreg on Aug 9 2018, 4:52 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGd662959dc881: changegroup: emit revisiondelta instances from deltagroup()
Summary

By abstracting the concept of a delta group away from its
serialization (the changegroup format), we make it easier
to establish alternate serialization formats. We also make
it possible to move aspects of delta group generation into
the storage layer. This will allow storage to make decisions
about e.g. delta parent choices without the changegroup code
needing storage APIs to determine delta parents. We're still
a bit of a way from there. Future commits will work towards
that world.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Aug 9 2018, 4:52 PM
Closed by commit rHGd662959dc881: changegroup: emit revisiondelta instances from deltagroup() (authored by indygreg). · Explain WhyAug 9 2018, 10:24 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/changegroup.py (75 lines)

Diff 10235

mercurial/changegroup.py

basenode = attr.ib() basenode = attr.ib()
# 20 byte node of changeset revision this delta is associated with. # 20 byte node of changeset revision this delta is associated with.
linknode = attr.ib() linknode = attr.ib()
# 2 bytes of flags to apply to revision data. # 2 bytes of flags to apply to revision data.
flags = attr.ib() flags = attr.ib()
# Iterable of chunks holding raw delta data. # Iterable of chunks holding raw delta data.
deltachunks = attr.ib() deltachunks = attr.ib()
def _revisiondeltatochunks(delta, headerfn):
"""Serialize a revisiondelta to changegroup chunks."""
meta = headerfn(delta)
l = len(meta) + sum(len(x) for x in delta.deltachunks)
yield chunkheader(l)
yield meta
for x in delta.deltachunks:
yield x
def _sortnodesnormal(store, nodes, reorder): def _sortnodesnormal(store, nodes, reorder):
"""Sort nodes for changegroup generation and turn into revnums.""" """Sort nodes for changegroup generation and turn into revnums."""
# for generaldelta revlogs, we linearize the revs; this will both be # for generaldelta revlogs, we linearize the revs; this will both be
# much quicker and generate a much smaller bundle # much quicker and generate a much smaller bundle
if (store._generaldelta and reorder is None) or reorder: if (store._generaldelta and reorder is None) or reorder:
dag = dagutil.revlogdag(store) dag = dagutil.revlogdag(store)
return dag.linearize(set(store.rev(n) for n in nodes)) return dag.linearize(set(store.rev(n) for n in nodes))
else: else:
p2node=p2n, p2node=p2n,
basenode=nullid, basenode=nullid,
linknode=linknode, linknode=linknode,
flags=flags, flags=flags,
deltachunks=(diffheader, data), deltachunks=(diffheader, data),
) )
def deltagroup(repo, revs, store, ischangelog, lookup, deltaparentfn, def deltagroup(repo, revs, store, ischangelog, lookup, deltaparentfn,
deltaheaderfn, units=None, units=None,
ellipses=False, clrevtolocalrev=None, fullclnodes=None, ellipses=False, clrevtolocalrev=None, fullclnodes=None,
precomputedellipsis=None): precomputedellipsis=None):
"""Calculate a delta group, yielding a sequence of changegroup chunks """Calculate deltas for a set of revisions.
(strings).
Given a list of changeset revs, return a set of deltas and Is a generator of ``revisiondelta`` instances.
metadata corresponding to nodes. The first delta is
first parent(nodelist[0]) -> nodelist[0], the receiver is
guaranteed to have this parent as it has all history before
these changesets. In the case firstparent is nullrev the
changegroup starts with a full revision.
If units is not None, progress detail will be generated, units specifies If units is not None, progress detail will be generated, units specifies
the type of revlog that is touched (changelog, manifest, etc.). the type of revlog that is touched (changelog, manifest, etc.).
""" """
# if we don't have any revisions touched by these changesets, bail # if we don't have any revisions touched by these changesets, bail
if len(revs) == 0: if len(revs) == 0:
return return
delta = _revisiondeltanarrow( delta = _revisiondeltanarrow(
cl, store, ischangelog, curr, linkrev, linknode, cl, store, ischangelog, curr, linkrev, linknode,
clrevtolocalrev, fullclnodes, clrevtolocalrev, fullclnodes,
precomputedellipsis) precomputedellipsis)
else: else:
delta = _revisiondeltanormal(store, curr, prev, linknode, delta = _revisiondeltanormal(store, curr, prev, linknode,
deltaparentfn) deltaparentfn)
if not delta: if delta:
continue yield delta
meta = deltaheaderfn(delta)
l = len(meta) + sum(len(x) for x in delta.deltachunks)
yield chunkheader(l)
yield meta
for x in delta.deltachunks:
yield x
if progress: if progress:
progress.complete() progress.complete()
class cgpacker(object): class cgpacker(object):
def __init__(self, repo, filematcher, version, allowreorder, def __init__(self, repo, filematcher, version, allowreorder,
deltaparentfn, builddeltaheader, manifestsend, deltaparentfn, builddeltaheader, manifestsend,
bundlecaps=None, ellipses=False, bundlecaps=None, ellipses=False,
"""Yield a sequence of changegroup byte chunks.""" """Yield a sequence of changegroup byte chunks."""
repo = self._repo repo = self._repo
cl = repo.changelog cl = repo.changelog
self._verbosenote(_('uncompressed size of bundle content:\n')) self._verbosenote(_('uncompressed size of bundle content:\n'))
size = 0 size = 0
clstate, chunks = self._generatechangelog(cl, clnodes) clstate, deltas = self._generatechangelog(cl, clnodes)
for chunk in chunks: for delta in deltas:
for chunk in _revisiondeltatochunks(delta, self._builddeltaheader):
size += len(chunk) size += len(chunk)
yield chunk yield chunk
close = closechunk() close = closechunk()
size += len(close) size += len(close)
yield closechunk() yield closechunk()
self._verbosenote(_('%8.i (changelog)\n') % size) self._verbosenote(_('%8.i (changelog)\n') % size)
clrevorder = clstate['clrevorder'] clrevorder = clstate['clrevorder']
fnodes = {} # needed file nodes fnodes = {} # needed file nodes
size = 0 size = 0
it = self.generatemanifests( it = self.generatemanifests(
commonrevs, clrevorder, fastpathlinkrev, mfs, fnodes, source, commonrevs, clrevorder, fastpathlinkrev, mfs, fnodes, source,
clstate['clrevtomanifestrev']) clstate['clrevtomanifestrev'])
for dir, chunks in it: for dir, deltas in it:
if dir: if dir:
assert self.version == b'03' assert self.version == b'03'
chunk = _fileheader(dir) chunk = _fileheader(dir)
size += len(chunk) size += len(chunk)
yield chunk yield chunk
for delta in deltas:
chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
for chunk in chunks: for chunk in chunks:
size += len(chunk) size += len(chunk)
yield chunk yield chunk
close = closechunk() close = closechunk()
size += len(close) size += len(close)
yield close yield close
self._verbosenote(_('%8.i (manifests)\n') % size) self._verbosenote(_('%8.i (manifests)\n') % size)
yield self._manifestsend yield self._manifestsend
mfdicts = None mfdicts = None
if self._ellipses and self._isshallow: if self._ellipses and self._isshallow:
mfdicts = [(self._repo.manifestlog[n].read(), lr) mfdicts = [(self._repo.manifestlog[n].read(), lr)
for (n, lr) in mfs.iteritems()] for (n, lr) in mfs.iteritems()]
mfs.clear() mfs.clear()
clrevs = set(cl.rev(x) for x in clnodes) clrevs = set(cl.rev(x) for x in clnodes)
it = self.generatefiles(changedfiles, commonrevs, it = self.generatefiles(changedfiles, commonrevs,
source, mfdicts, fastpathlinkrev, source, mfdicts, fastpathlinkrev,
fnodes, clrevs) fnodes, clrevs)
for path, chunks in it: for path, deltas in it:
h = _fileheader(path) h = _fileheader(path)
size = len(h) size = len(h)
yield h yield h
for delta in deltas:
chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
for chunk in chunks: for chunk in chunks:
size += len(chunk) size += len(chunk)
yield chunk yield chunk
close = closechunk() close = closechunk()
size += len(close) size += len(close)
yield close yield close
self._verbosenote(_('%8.i %s\n') % (size, path)) self._verbosenote(_('%8.i %s\n') % (size, path))
yield closechunk() yield closechunk()
'clrevorder': clrevorder, 'clrevorder': clrevorder,
'mfs': mfs, 'mfs': mfs,
'changedfiles': changedfiles, 'changedfiles': changedfiles,
'clrevtomanifestrev': clrevtomanifestrev, 'clrevtomanifestrev': clrevtomanifestrev,
} }
gen = deltagroup( gen = deltagroup(
self._repo, revs, cl, True, lookupcl, self._repo, revs, cl, True, lookupcl,
self._deltaparentfn, self._builddeltaheader, self._deltaparentfn,
ellipses=self._ellipses, ellipses=self._ellipses,
units=_('changesets'), units=_('changesets'),
clrevtolocalrev={}, clrevtolocalrev={},
fullclnodes=self._fullclnodes, fullclnodes=self._fullclnodes,
precomputedellipsis=self._precomputedellipsis) precomputedellipsis=self._precomputedellipsis)
return state, gen return state, gen
if self._ellipses: if self._ellipses:
revs = _sortnodesellipsis(store, prunednodes, cl, revs = _sortnodesellipsis(store, prunednodes, cl,
lookupfn) lookupfn)
else: else:
revs = _sortnodesnormal(store, prunednodes, revs = _sortnodesnormal(store, prunednodes,
self._reorder) self._reorder)
it = deltagroup( deltas = deltagroup(
self._repo, revs, store, False, lookupfn, self._repo, revs, store, False, lookupfn,
self._deltaparentfn, self._builddeltaheader, self._deltaparentfn,
ellipses=self._ellipses, ellipses=self._ellipses,
units=_('manifests'), units=_('manifests'),
clrevtolocalrev=clrevtolocalrev, clrevtolocalrev=clrevtolocalrev,
fullclnodes=self._fullclnodes, fullclnodes=self._fullclnodes,
precomputedellipsis=self._precomputedellipsis) precomputedellipsis=self._precomputedellipsis)
yield dir, it yield dir, deltas
# The 'source' parameter is useful for extensions # The 'source' parameter is useful for extensions
def generatefiles(self, changedfiles, commonrevs, source, def generatefiles(self, changedfiles, commonrevs, source,
mfdicts, fastpathlinkrev, fnodes, clrevs): mfdicts, fastpathlinkrev, fnodes, clrevs):
changedfiles = list(filter(self._filematcher, changedfiles)) changedfiles = list(filter(self._filematcher, changedfiles))
if not fastpathlinkrev: if not fastpathlinkrev:
def normallinknodes(unused, fname): def normallinknodes(unused, fname):
revs = _sortnodesellipsis(filerevlog, filenodes, revs = _sortnodesellipsis(filerevlog, filenodes,
cl, lookupfilelog) cl, lookupfilelog)
else: else:
revs = _sortnodesnormal(filerevlog, filenodes, revs = _sortnodesnormal(filerevlog, filenodes,
self._reorder) self._reorder)
progress.update(i + 1, item=fname) progress.update(i + 1, item=fname)
it = deltagroup( deltas = deltagroup(
self._repo, revs, filerevlog, False, lookupfilelog, self._repo, revs, filerevlog, False, lookupfilelog,
self._deltaparentfn, self._builddeltaheader, self._deltaparentfn,
ellipses=self._ellipses, ellipses=self._ellipses,
clrevtolocalrev=clrevtolocalrev, clrevtolocalrev=clrevtolocalrev,
fullclnodes=self._fullclnodes, fullclnodes=self._fullclnodes,
precomputedellipsis=self._precomputedellipsis) precomputedellipsis=self._precomputedellipsis)
yield fname, it yield fname, deltas
progress.complete() progress.complete()
def _deltaparentprev(store, rev, p1, p2, prev): def _deltaparentprev(store, rev, p1, p2, prev):
"""Resolve a delta parent to the previous revision. """Resolve a delta parent to the previous revision.
Used for version 1 changegroups, which don't support generaldelta. Used for version 1 changegroups, which don't support generaldelta.
""" """