This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4075

changegroup: capture revision delta in a data structure
ClosedPublic

Authored by indygreg on Aug 3 2018, 5:04 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG23d582caae30: changegroup: capture revision delta in a data structure
Summary

The current changegroup generation code is tightly coupled to
the revlog API. This tight coupling makes it difficult to implement
alternate storage backends without requiring a large surface area
of the revlog API to be exposed. This is not desirable.

In order to support changegroup generation with non-revlog storage,
we'll need to abstract the concept of delta generation.

This commit is the first step down that road. We introduce a
data structure for representing a delta in a changegroup.

The API still leaves a lot to be desired. But at least we now
have separation between data and actions performed on it.

As part of this, we tweak behavior slightly: we no longer
concatenate the delta prefix with the metadata header. Instead,
we track and emit the prefix as a separate chunk. This shouldn't
have any meaningful impact since all the chunks just get sent to
the wire, the compressor, etc.

Because we're introducing a new object, this does add some
overhead to changegroup execution. hg perfchangegroupchangelog
on my clone of the Mercurial repo (~40,000 visible revisions in
the changelog) slows down a bit:

! wall 1.268600 comb 1.270000 user 1.270000 sys 0.000000 (best of 8)
! wall 1.419479 comb 1.410000 user 1.410000 sys 0.000000 (best of 8)

With for hg bundle -t none-v2 -a /dev/null:

before: real 6.610 secs (user 6.460+0.000 sys 0.140+0.000)
after: real 7.210 secs (user 7.060+0.000 sys 0.140+0.000)

I plan to claw back this regression in future commits. And I may
even do away with this data structure once the refactor is complete.
For now, it makes things easier to comprehend.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

PathPackages
Mmercurial/changegroup.py (86 lines)

Diff 9951

mercurial/changegroup.py

from .i18n import _ from .i18n import _
from .node import ( from .node import (
hex, hex,
nullid, nullid,
nullrev, nullrev,
short, short,
) )
from .thirdparty import (
attr,
)
from . import ( from . import (
dagutil, dagutil,
error, error,
manifest, manifest,
match as matchmod, match as matchmod,
mdiff, mdiff,
phases, phases,
pycompat, pycompat,
def read(self, n): def read(self, n):
if self._h: if self._h:
d, self._h = self._h[:n], self._h[n:] d, self._h = self._h[:n], self._h[n:]
if len(d) < n: if len(d) < n:
d += readexactly(self._fh, n - len(d)) d += readexactly(self._fh, n - len(d))
return d return d
return readexactly(self._fh, n) return readexactly(self._fh, n)
@attr.s(slots=True, frozen=True)
class revisiondelta(object):
"""Describes a delta entry in a changegroup.
Captured data is sufficient to serialize the delta into multiple
formats.
"""
# 20 byte node of this revision.
node = attr.ib()
# 20 byte nodes of parent revisions.
p1node = attr.ib()
p2node = attr.ib()
# 20 byte node of node this delta is against.
basenode = attr.ib()
# 20 byte node of changeset revision this delta is associated with.
linknode = attr.ib()
# 2 bytes of flags to apply to revision data.
flags = attr.ib()
# Iterable of chunks holding raw delta data.
deltachunks = attr.ib()
class cg1packer(object): class cg1packer(object):
deltaheader = _CHANGEGROUPV1_DELTA_HEADER deltaheader = _CHANGEGROUPV1_DELTA_HEADER
version = '01' version = '01'
def __init__(self, repo, filematcher, bundlecaps=None): def __init__(self, repo, filematcher, bundlecaps=None):
"""Given a source repo, construct a bundler. """Given a source repo, construct a bundler.
filematcher is a matcher that matches on files to include in the filematcher is a matcher that matches on files to include in the
def deltaparent(self, store, rev, p1, p2, prev): def deltaparent(self, store, rev, p1, p2, prev):
if not store.candelta(prev, rev): if not store.candelta(prev, rev):
raise error.ProgrammingError('cg1 should not be used in this case') raise error.ProgrammingError('cg1 should not be used in this case')
return prev return prev
def revchunk(self, store, rev, prev, linknode): def revchunk(self, store, rev, prev, linknode):
if util.safehasattr(self, 'full_nodes'): if util.safehasattr(self, 'full_nodes'):
fn = self._revchunknarrow fn = self._revisiondeltanarrow
else: else:
fn = self._revchunknormal fn = self._revisiondeltanormal
delta = fn(store, rev, prev, linknode)
if not delta:
return
meta = self.builddeltaheader(delta.node, delta.p1node, delta.p2node,
delta.basenode, delta.linknode,
delta.flags)
l = len(meta) + sum(len(x) for x in delta.deltachunks)
return fn(store, rev, prev, linknode) yield chunkheader(l)
yield meta
for x in delta.deltachunks:
yield x
def _revchunknormal(self, store, rev, prev, linknode): def _revisiondeltanormal(self, store, rev, prev, linknode):
node = store.node(rev) node = store.node(rev)
p1, p2 = store.parentrevs(rev) p1, p2 = store.parentrevs(rev)
base = self.deltaparent(store, rev, p1, p2, prev) base = self.deltaparent(store, rev, p1, p2, prev)
prefix = '' prefix = ''
if store.iscensored(base) or store.iscensored(rev): if store.iscensored(base) or store.iscensored(rev):
try: try:
delta = store.revision(node, raw=True) delta = store.revision(node, raw=True)
except error.CensoredNodeError as e: except error.CensoredNodeError as e:
delta = e.tombstone delta = e.tombstone
if base == nullrev: if base == nullrev:
prefix = mdiff.trivialdiffheader(len(delta)) prefix = mdiff.trivialdiffheader(len(delta))
else: else:
baselen = store.rawsize(base) baselen = store.rawsize(base)
prefix = mdiff.replacediffheader(baselen, len(delta)) prefix = mdiff.replacediffheader(baselen, len(delta))
elif base == nullrev: elif base == nullrev:
delta = store.revision(node, raw=True) delta = store.revision(node, raw=True)
prefix = mdiff.trivialdiffheader(len(delta)) prefix = mdiff.trivialdiffheader(len(delta))
else: else:
delta = store.revdiff(base, rev) delta = store.revdiff(base, rev)
p1n, p2n = store.parents(node) p1n, p2n = store.parents(node)
basenode = store.node(base)
flags = store.flags(rev)
meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags)
meta += prefix
l = len(meta) + len(delta)
yield chunkheader(l)
yield meta
yield delta
def _revchunknarrow(self, store, rev, prev, linknode): return revisiondelta(
node=node,
p1node=p1n,
p2node=p2n,
basenode=store.node(base),
linknode=linknode,
flags=store.flags(rev),
deltachunks=(prefix, delta),
)
def _revisiondeltanarrow(self, store, rev, prev, linknode):
# build up some mapping information that's useful later. See # build up some mapping information that's useful later. See
# the local() nested function below. # the local() nested function below.
if not self.changelog_done: if not self.changelog_done:
self.clnode_to_rev[linknode] = rev self.clnode_to_rev[linknode] = rev
linkrev = rev linkrev = rev
self.clrev_to_localrev[linkrev] = rev self.clrev_to_localrev[linkrev] = rev
else: else:
linkrev = self.clnode_to_rev[linknode] linkrev = self.clnode_to_rev[linknode]
self.clrev_to_localrev[linkrev] = rev self.clrev_to_localrev[linkrev] = rev
# This is a node to send in full, because the changeset it # This is a node to send in full, because the changeset it
# corresponds to was a full changeset. # corresponds to was a full changeset.
if linknode in self.full_nodes: if linknode in self.full_nodes:
for x in self._revchunknormal(store, rev, prev, linknode): return self._revisiondeltanormal(store, rev, prev, linknode)
yield x
return
# At this point, a node can either be one we should skip or an # At this point, a node can either be one we should skip or an
# ellipsis. If it's not an ellipsis, bail immediately. # ellipsis. If it's not an ellipsis, bail immediately.
if linkrev not in self.precomputed_ellipsis: if linkrev not in self.precomputed_ellipsis:
return return
linkparents = self.precomputed_ellipsis[linkrev] linkparents = self.precomputed_ellipsis[linkrev]
def local(clrev): def local(clrev):
p2 = nullrev p2 = nullrev
else: else:
p1, p2 = sorted(local(p) for p in linkparents) p1, p2 = sorted(local(p) for p in linkparents)
n = store.node(rev) n = store.node(rev)
p1n, p2n = store.node(p1), store.node(p2) p1n, p2n = store.node(p1), store.node(p2)
flags = store.flags(rev) flags = store.flags(rev)
flags |= revlog.REVIDX_ELLIPSIS flags |= revlog.REVIDX_ELLIPSIS
meta = self.builddeltaheader(
n, p1n, p2n, nullid, linknode, flags)
# TODO: try and actually send deltas for ellipsis data blocks # TODO: try and actually send deltas for ellipsis data blocks
data = store.revision(n) data = store.revision(n)
diffheader = mdiff.trivialdiffheader(len(data)) diffheader = mdiff.trivialdiffheader(len(data))
l = len(meta) + len(diffheader) + len(data)
yield ''.join((chunkheader(l), return revisiondelta(
meta, node=n,
diffheader, p1node=p1n,
data)) p2node=p2n,
basenode=nullid,
linknode=linknode,
flags=flags,
deltachunks=(diffheader, data),
)
def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
# do nothing with basenode, it is implicitly the previous one in HG10 # do nothing with basenode, it is implicitly the previous one in HG10
# do nothing with flags, it is implicitly 0 for cg1 and cg2 # do nothing with flags, it is implicitly 0 for cg1 and cg2
return struct.pack(self.deltaheader, node, p1n, p2n, linknode) return struct.pack(self.deltaheader, node, p1n, p2n, linknode)
class cg2packer(cg1packer): class cg2packer(cg1packer):
version = '02' version = '02'