This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4132

changegroup: pass function to resolve delta parents into constructor
ClosedPublic

Authored by indygreg on Aug 6 2018, 3:51 PM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHG87b737b78bd0: changegroup: pass function to resolve delta parents into constructor
Summary

Previously, _deltaparent() encapsulated the logic for all 3
delta parent modes of operation. The choice of delta parent
is static for the lifetime of the packer and can be passed into
the packer as a callable. So do that.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Aug 6 2018, 3:51 PM
durin42 accepted this revision.Aug 9 2018, 2:03 PM
durin42 added a subscriber: durin42.

Nice cleanup. Big improvement.

This revision is now accepted and ready to land.Aug 9 2018, 2:03 PM
Closed by commit rHG87b737b78bd0: changegroup: pass function to resolve delta parents into constructor (authored by indygreg). · Explain WhyAug 9 2018, 2:15 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/changegroup.py (115 lines)

Diff 10139

mercurial/changegroup.py

linknode = attr.ib() linknode = attr.ib()
# 2 bytes of flags to apply to revision data. # 2 bytes of flags to apply to revision data.
flags = attr.ib() flags = attr.ib()
# Iterable of chunks holding raw delta data. # Iterable of chunks holding raw delta data.
deltachunks = attr.ib() deltachunks = attr.ib()
class cgpacker(object): class cgpacker(object):
def __init__(self, repo, filematcher, version, allowreorder, def __init__(self, repo, filematcher, version, allowreorder,
useprevdelta, builddeltaheader, manifestsend, deltaparentfn, builddeltaheader, manifestsend,
bundlecaps=None, ellipses=False, bundlecaps=None, ellipses=False,
shallow=False, ellipsisroots=None, fullnodes=None): shallow=False, ellipsisroots=None, fullnodes=None):
"""Given a source repo, construct a bundler. """Given a source repo, construct a bundler.
filematcher is a matcher that matches on files to include in the filematcher is a matcher that matches on files to include in the
changegroup. Used to facilitate sparse changegroups. changegroup. Used to facilitate sparse changegroups.
allowreorder controls whether reordering of revisions is allowed. allowreorder controls whether reordering of revisions is allowed.
This value is used when ``bundle.reorder`` is ``auto`` or isn't This value is used when ``bundle.reorder`` is ``auto`` or isn't
set. set.
useprevdelta controls whether revisions should always delta against deltaparentfn is a callable that resolves the delta parent for
the previous revision in the changegroup. a specific revision.
builddeltaheader is a callable that constructs the header for a group builddeltaheader is a callable that constructs the header for a group
delta. delta.
manifestsend is a chunk to send after manifests have been fully emitted. manifestsend is a chunk to send after manifests have been fully emitted.
ellipses indicates whether ellipsis serving mode is enabled. ellipses indicates whether ellipsis serving mode is enabled.
bundlecaps is optional and can be used to specify the set of bundlecaps is optional and can be used to specify the set of
capabilities which can be used to build the bundle. While bundlecaps is capabilities which can be used to build the bundle. While bundlecaps is
unused in core Mercurial, extensions rely on this feature to communicate unused in core Mercurial, extensions rely on this feature to communicate
capabilities to customize the changegroup packer. capabilities to customize the changegroup packer.
shallow indicates whether shallow data might be sent. The packer may shallow indicates whether shallow data might be sent. The packer may
need to pack file contents not introduced by the changes being packed. need to pack file contents not introduced by the changes being packed.
fullnodes is the list of nodes which should not be ellipsis nodes. We fullnodes is the list of nodes which should not be ellipsis nodes. We
store this rather than the set of nodes that should be ellipsis because store this rather than the set of nodes that should be ellipsis because
for very large histories we expect this to be significantly smaller. for very large histories we expect this to be significantly smaller.
""" """
assert filematcher assert filematcher
self._filematcher = filematcher self._filematcher = filematcher
self.version = version self.version = version
self._useprevdelta = useprevdelta self._deltaparentfn = deltaparentfn
self._builddeltaheader = builddeltaheader self._builddeltaheader = builddeltaheader
self._manifestsend = manifestsend self._manifestsend = manifestsend
self._ellipses = ellipses self._ellipses = ellipses
# Set of capabilities we can use to build the bundle. # Set of capabilities we can use to build the bundle.
if bundlecaps is None: if bundlecaps is None:
bundlecaps = set() bundlecaps = set()
self._bundlecaps = bundlecaps self._bundlecaps = bundlecaps
size = len(h) size = len(h)
yield h yield h
for chunk in self.group(filenodes, filerevlog, lookupfilelog): for chunk in self.group(filenodes, filerevlog, lookupfilelog):
size += len(chunk) size += len(chunk)
yield chunk yield chunk
self._verbosenote(_('%8.i %s\n') % (size, fname)) self._verbosenote(_('%8.i %s\n') % (size, fname))
progress.complete() progress.complete()
def _deltaparent(self, store, rev, p1, p2, prev):
if self._useprevdelta:
if not store.candelta(prev, rev):
raise error.ProgrammingError(
'cg1 should not be used in this case')
return prev
# Narrow ellipses mode.
if self._ellipses:
# TODO: send better deltas when in narrow mode.
#
# changegroup.group() loops over revisions to send,
# including revisions we'll skip. What this means is that
# `prev` will be a potentially useless delta base for all
# ellipsis nodes, as the client likely won't have it. In
# the future we should do bookkeeping about which nodes
# have been sent to the client, and try to be
# significantly smarter about delta bases. This is
# slightly tricky because this same code has to work for
# all revlogs, and we don't have the linkrev/linknode here.
return p1
dp = store.deltaparent(rev)
if dp == nullrev and store.storedeltachains:
# Avoid sending full revisions when delta parent is null. Pick prev
# in that case. It's tempting to pick p1 in this case, as p1 will
# be smaller in the common case. However, computing a delta against
# p1 may require resolving the raw text of p1, which could be
# expensive. The revlog caches should have prev cached, meaning
# less CPU for changegroup generation. There is likely room to add
# a flag and/or config option to control this behavior.
base = prev
elif dp == nullrev:
# revlog is configured to use full snapshot for a reason,
# stick to full snapshot.
base = nullrev
elif dp not in (p1, p2, prev):
# Pick prev when we can't be sure remote has the base revision.
return prev
else:
base = dp
if base != nullrev and not store.candelta(base, rev):
base = nullrev
return base
def _revchunk(self, store, rev, prev, linknode): def _revchunk(self, store, rev, prev, linknode):
if self._ellipses: if self._ellipses:
fn = self._revisiondeltanarrow fn = self._revisiondeltanarrow
else: else:
fn = self._revisiondeltanormal fn = self._revisiondeltanormal
delta = fn(store, rev, prev, linknode) delta = fn(store, rev, prev, linknode)
if not delta: if not delta:
return return
meta = self._builddeltaheader(delta) meta = self._builddeltaheader(delta)
l = len(meta) + sum(len(x) for x in delta.deltachunks) l = len(meta) + sum(len(x) for x in delta.deltachunks)
yield chunkheader(l) yield chunkheader(l)
yield meta yield meta
for x in delta.deltachunks: for x in delta.deltachunks:
yield x yield x
def _revisiondeltanormal(self, store, rev, prev, linknode): def _revisiondeltanormal(self, store, rev, prev, linknode):
node = store.node(rev) node = store.node(rev)
p1, p2 = store.parentrevs(rev) p1, p2 = store.parentrevs(rev)
base = self._deltaparent(store, rev, p1, p2, prev) base = self._deltaparentfn(store, rev, p1, p2, prev)
prefix = '' prefix = ''
if store.iscensored(base) or store.iscensored(rev): if store.iscensored(base) or store.iscensored(rev):
try: try:
delta = store.revision(node, raw=True) delta = store.revision(node, raw=True)
except error.CensoredNodeError as e: except error.CensoredNodeError as e:
delta = e.tombstone delta = e.tombstone
if base == nullrev: if base == nullrev:
p1node=p1n, p1node=p1n,
p2node=p2n, p2node=p2n,
basenode=nullid, basenode=nullid,
linknode=linknode, linknode=linknode,
flags=flags, flags=flags,
deltachunks=(diffheader, data), deltachunks=(diffheader, data),
) )
def _deltaparentprev(store, rev, p1, p2, prev):
"""Resolve a delta parent to the previous revision.
Used for version 1 changegroups, which don't support generaldelta.
"""
return prev
def _deltaparentgeneraldelta(store, rev, p1, p2, prev):
"""Resolve a delta parent when general deltas are supported."""
dp = store.deltaparent(rev)
if dp == nullrev and store.storedeltachains:
# Avoid sending full revisions when delta parent is null. Pick prev
# in that case. It's tempting to pick p1 in this case, as p1 will
# be smaller in the common case. However, computing a delta against
# p1 may require resolving the raw text of p1, which could be
# expensive. The revlog caches should have prev cached, meaning
# less CPU for changegroup generation. There is likely room to add
# a flag and/or config option to control this behavior.
base = prev
elif dp == nullrev:
# revlog is configured to use full snapshot for a reason,
# stick to full snapshot.
base = nullrev
elif dp not in (p1, p2, prev):
# Pick prev when we can't be sure remote has the base revision.
return prev
else:
base = dp
if base != nullrev and not store.candelta(base, rev):
base = nullrev
return base
def _deltaparentellipses(store, rev, p1, p2, prev):
"""Resolve a delta parent when in ellipses mode."""
# TODO: send better deltas when in narrow mode.
#
# changegroup.group() loops over revisions to send,
# including revisions we'll skip. What this means is that
# `prev` will be a potentially useless delta base for all
# ellipsis nodes, as the client likely won't have it. In
# the future we should do bookkeeping about which nodes
# have been sent to the client, and try to be
# significantly smarter about delta bases. This is
# slightly tricky because this same code has to work for
# all revlogs, and we don't have the linkrev/linknode here.
return p1
def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False, def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
shallow=False, ellipsisroots=None, fullnodes=None): shallow=False, ellipsisroots=None, fullnodes=None):
builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.linknode) d.node, d.p1node, d.p2node, d.linknode)
return cgpacker(repo, filematcher, b'01', return cgpacker(repo, filematcher, b'01',
useprevdelta=True, deltaparentfn=_deltaparentprev,
allowreorder=None, allowreorder=None,
builddeltaheader=builddeltaheader, builddeltaheader=builddeltaheader,
manifestsend=b'', manifestsend=b'',
bundlecaps=bundlecaps, bundlecaps=bundlecaps,
ellipses=ellipses, ellipses=ellipses,
shallow=shallow, shallow=shallow,
ellipsisroots=ellipsisroots, ellipsisroots=ellipsisroots,
fullnodes=fullnodes) fullnodes=fullnodes)
def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False, def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False,
shallow=False, ellipsisroots=None, fullnodes=None): shallow=False, ellipsisroots=None, fullnodes=None):
builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode) d.node, d.p1node, d.p2node, d.basenode, d.linknode)
# Since generaldelta is directly supported by cg2, reordering # Since generaldelta is directly supported by cg2, reordering
# generally doesn't help, so we disable it by default (treating # generally doesn't help, so we disable it by default (treating
# bundle.reorder=auto just like bundle.reorder=False). # bundle.reorder=auto just like bundle.reorder=False).
return cgpacker(repo, filematcher, b'02', return cgpacker(repo, filematcher, b'02',
useprevdelta=False, deltaparentfn=_deltaparentgeneraldelta,
allowreorder=False, allowreorder=False,
builddeltaheader=builddeltaheader, builddeltaheader=builddeltaheader,
manifestsend=b'', manifestsend=b'',
bundlecaps=bundlecaps, bundlecaps=bundlecaps,
ellipses=ellipses, ellipses=ellipses,
shallow=shallow, shallow=shallow,
ellipsisroots=ellipsisroots, ellipsisroots=ellipsisroots,
fullnodes=fullnodes) fullnodes=fullnodes)
def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False, def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False,
shallow=False, ellipsisroots=None, fullnodes=None): shallow=False, ellipsisroots=None, fullnodes=None):
builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
deltaparentfn = (_deltaparentellipses if ellipses
else _deltaparentgeneraldelta)
return cgpacker(repo, filematcher, b'03', return cgpacker(repo, filematcher, b'03',
useprevdelta=False, deltaparentfn=deltaparentfn,
allowreorder=False, allowreorder=False,
builddeltaheader=builddeltaheader, builddeltaheader=builddeltaheader,
manifestsend=closechunk(), manifestsend=closechunk(),
bundlecaps=bundlecaps, bundlecaps=bundlecaps,
ellipses=ellipses, ellipses=ellipses,
shallow=shallow, shallow=shallow,
ellipsisroots=ellipsisroots, ellipsisroots=ellipsisroots,
fullnodes=fullnodes) fullnodes=fullnodes)