( )⚙ D4193 changegroup: pass clrevtolocalrev to each group

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4193

changegroup: pass clrevtolocalrev to each group
ClosedPublic

Authored by indygreg on Aug 9 2018, 2:51 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGa6e1ff40e335: changegroup: pass clrevtolocalrev to each group
Summary

clrevtolocalrev is a per-changegroup group mapping revisions to
aid with shallow clone.

Back when this functionality was implemented in an extension, this
dict was added to the packer instance so monkeypatched functions
could reference it there. Now that this code is part of core, we
can pass the dict to each consumer properly so it doesn't have to
live on the cgpacker instance. This commit does that.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Aug 9 2018, 2:51 PM
Closed by commit rHGa6e1ff40e335: changegroup: pass clrevtolocalrev to each group (authored by indygreg). · Explain WhyAug 9 2018, 4:27 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/changegroup.py (51 lines)

Diff 10182

mercurial/changegroup.py

self._repo = repo self._repo = repo
if self._repo.ui.verbose and not self._repo.ui.debugflag: if self._repo.ui.verbose and not self._repo.ui.debugflag:
self._verbosenote = self._repo.ui.note self._verbosenote = self._repo.ui.note
else: else:
self._verbosenote = lambda s: None self._verbosenote = lambda s: None
# Maps CL revs to per-revlog revisions. Cleared in close() at
# the end of each group.
self._clrevtolocalrev = {}
def _close(self): def _close(self):
# Ellipses serving mode.
self._clrevtolocalrev.clear()
return closechunk() return closechunk()
def group(self, revs, store, ischangelog, lookup, units=None): def group(self, revs, store, ischangelog, lookup, units=None,
clrevtolocalrev=None):
"""Calculate a delta group, yielding a sequence of changegroup chunks """Calculate a delta group, yielding a sequence of changegroup chunks
(strings). (strings).
Given a list of changeset revs, return a set of deltas and Given a list of changeset revs, return a set of deltas and
metadata corresponding to nodes. The first delta is metadata corresponding to nodes. The first delta is
first parent(nodelist[0]) -> nodelist[0], the receiver is first parent(nodelist[0]) -> nodelist[0], the receiver is
guaranteed to have this parent as it has all history before guaranteed to have this parent as it has all history before
these changesets. In the case firstparent is nullrev the these changesets. In the case firstparent is nullrev the
for r in pycompat.xrange(len(revs) - 1): for r in pycompat.xrange(len(revs) - 1):
if progress: if progress:
progress.update(r + 1) progress.update(r + 1)
prev, curr = revs[r], revs[r + 1] prev, curr = revs[r], revs[r + 1]
linknode = lookup(store.node(curr)) linknode = lookup(store.node(curr))
if self._ellipses: if self._ellipses:
linkrev = cl.rev(linknode) linkrev = cl.rev(linknode)
self._clrevtolocalrev[linkrev] = curr clrevtolocalrev[linkrev] = curr
# This is a node to send in full, because the changeset it # This is a node to send in full, because the changeset it
# corresponds to was a full changeset. # corresponds to was a full changeset.
if linknode in self._fullclnodes: if linknode in self._fullclnodes:
delta = _revisiondeltanormal(store, curr, prev, linknode, delta = _revisiondeltanormal(store, curr, prev, linknode,
self._deltaparentfn) self._deltaparentfn)
elif linkrev not in self._precomputedellipsis: elif linkrev not in self._precomputedellipsis:
delta = None delta = None
else: else:
delta = self._revisiondeltanarrow(store, ischangelog, delta = self._revisiondeltanarrow(store, ischangelog,
curr, linkrev, linknode) curr, linkrev, linknode,
clrevtolocalrev)
else: else:
delta = _revisiondeltanormal(store, curr, prev, linknode, delta = _revisiondeltanormal(store, curr, prev, linknode,
self._deltaparentfn) self._deltaparentfn)
if not delta: if not delta:
continue continue
meta = self._builddeltaheader(delta) meta = self._builddeltaheader(delta)
# TODO this violates storage abstraction for manifests. # TODO this violates storage abstraction for manifests.
if isinstance(store, manifest.manifestrevlog): if isinstance(store, manifest.manifestrevlog):
if not self._filematcher.visitdir(store._dir[:-1] or '.'): if not self._filematcher.visitdir(store._dir[:-1] or '.'):
return [] return []
rr, rl = store.rev, store.linkrev rr, rl = store.rev, store.linkrev
return [n for n in missing if rl(rr(n)) not in commonrevs] return [n for n in missing if rl(rr(n)) not in commonrevs]
def _packmanifests(self, dir, dirlog, revs, lookuplinknode): def _packmanifests(self, dir, dirlog, revs, lookuplinknode,
clrevtolocalrev):
"""Pack manifests into a changegroup stream. """Pack manifests into a changegroup stream.
Encodes the directory name in the output so multiple manifests Encodes the directory name in the output so multiple manifests
can be sent. Multiple manifests is not supported by cg1 and cg2. can be sent. Multiple manifests is not supported by cg1 and cg2.
""" """
if dir: if dir:
assert self.version == b'03' assert self.version == b'03'
yield _fileheader(dir) yield _fileheader(dir)
for chunk in self.group(revs, dirlog, False, lookuplinknode, for chunk in self.group(revs, dirlog, False, lookuplinknode,
units=_('manifests')): units=_('manifests'),
clrevtolocalrev=clrevtolocalrev):
yield chunk yield chunk
def generate(self, commonrevs, clnodes, fastpathlinkrev, source): def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
"""Yield a sequence of changegroup byte chunks.""" """Yield a sequence of changegroup byte chunks."""
repo = self._repo repo = self._repo
cl = repo.changelog cl = repo.changelog
self._verbosenote(_('uncompressed size of bundle content:\n')) self._verbosenote(_('uncompressed size of bundle content:\n'))
size = 0 size = 0
clstate, chunks = self._generatechangelog(cl, clnodes) clstate, chunks = self._generatechangelog(cl, clnodes)
for chunk in chunks: for chunk in chunks:
size += len(chunk) size += len(chunk)
yield chunk yield chunk
self._verbosenote(_('%8.i (changelog)\n') % size) self._verbosenote(_('%8.i (changelog)\n') % size)
clrevorder = clstate['clrevorder'] clrevorder = clstate['clrevorder']
mfs = clstate['mfs'] mfs = clstate['mfs']
changedfiles = clstate['changedfiles'] changedfiles = clstate['changedfiles']
if self._ellipses:
self._clrevtolocalrev = clstate['clrevtomanifestrev']
# We need to make sure that the linkrev in the changegroup refers to # We need to make sure that the linkrev in the changegroup refers to
# the first changeset that introduced the manifest or file revision. # the first changeset that introduced the manifest or file revision.
# The fastpath is usually safer than the slowpath, because the filelogs # The fastpath is usually safer than the slowpath, because the filelogs
# are walked in revlog order. # are walked in revlog order.
# #
# When taking the slowpath with reorder=None and the manifest revlog # When taking the slowpath with reorder=None and the manifest revlog
# uses generaldelta, the manifest may be walked in the "wrong" order. # uses generaldelta, the manifest may be walked in the "wrong" order.
# Without 'clrevorder', we would get an incorrect linkrev (see fix in # Without 'clrevorder', we would get an incorrect linkrev (see fix in
# either, because we don't discover which directory nodes to # either, because we don't discover which directory nodes to
# send along with files. This could probably be fixed. # send along with files. This could probably be fixed.
fastpathlinkrev = fastpathlinkrev and ( fastpathlinkrev = fastpathlinkrev and (
'treemanifest' not in repo.requirements) 'treemanifest' not in repo.requirements)
fnodes = {} # needed file nodes fnodes = {} # needed file nodes
for chunk in self.generatemanifests(commonrevs, clrevorder, for chunk in self.generatemanifests(commonrevs, clrevorder,
fastpathlinkrev, mfs, fnodes, source): fastpathlinkrev, mfs, fnodes, source,
clstate['clrevtomanifestrev']):
yield chunk yield chunk
mfdicts = None mfdicts = None
if self._ellipses and self._isshallow: if self._ellipses and self._isshallow:
mfdicts = [(self._repo.manifestlog[n].read(), lr) mfdicts = [(self._repo.manifestlog[n].read(), lr)
for (n, lr) in mfs.iteritems()] for (n, lr) in mfs.iteritems()]
mfs.clear() mfs.clear()
state = { state = {
'clrevorder': clrevorder, 'clrevorder': clrevorder,
'mfs': mfs, 'mfs': mfs,
'changedfiles': changedfiles, 'changedfiles': changedfiles,
'clrevtomanifestrev': clrevtomanifestrev, 'clrevtomanifestrev': clrevtomanifestrev,
} }
gen = self.group(revs, cl, True, lookupcl, units=_('changesets')) gen = self.group(revs, cl, True, lookupcl, units=_('changesets'),
clrevtolocalrev={})
return state, gen return state, gen
def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs, def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
fnodes, source): fnodes, source, clrevtolocalrev):
"""Returns an iterator of changegroup chunks containing manifests. """Returns an iterator of changegroup chunks containing manifests.
`source` is unused here, but is used by extensions like remotefilelog to `source` is unused here, but is used by extensions like remotefilelog to
change what is sent based in pulls vs pushes, etc. change what is sent based in pulls vs pushes, etc.
""" """
repo = self._repo repo = self._repo
cl = repo.changelog cl = repo.changelog
mfl = repo.manifestlog mfl = repo.manifestlog
if self._ellipses: if self._ellipses:
revs = _sortnodesellipsis(store, prunednodes, cl, revs = _sortnodesellipsis(store, prunednodes, cl,
lookupfn) lookupfn)
else: else:
revs = _sortnodesnormal(store, prunednodes, revs = _sortnodesnormal(store, prunednodes,
self._reorder) self._reorder)
for x in self._packmanifests(dir, store, revs, lookupfn): for x in self._packmanifests(dir, store, revs, lookupfn,
clrevtolocalrev):
size += len(x) size += len(x)
yield x yield x
self._verbosenote(_('%8.i (manifests)\n') % size) self._verbosenote(_('%8.i (manifests)\n') % size)
yield self._manifestsend yield self._manifestsend
# The 'source' parameter is useful for extensions # The 'source' parameter is useful for extensions
def generatefiles(self, changedfiles, commonrevs, source, def generatefiles(self, changedfiles, commonrevs, source,
mfdicts, fastpathlinkrev, fnodes, clrevs): mfdicts, fastpathlinkrev, fnodes, clrevs):
changedfiles = list(filter(self._filematcher, changedfiles)) changedfiles = list(filter(self._filematcher, changedfiles))
if not fastpathlinkrev: if not fastpathlinkrev:
def normallinknodes(unused, fname): def normallinknodes(unused, fname):
return fnodes.get(fname, {}) return fnodes.get(fname, {})
else: else:
cln = self._repo.changelog.node cln = self._repo.changelog.node
def normallinknodes(store, fname): def normallinknodes(store, fname):
flinkrev = store.linkrev flinkrev = store.linkrev
fnode = store.node fnode = store.node
revs = ((r, flinkrev(r)) for r in store) revs = ((r, flinkrev(r)) for r in store)
return dict((fnode(r), cln(lr)) return dict((fnode(r), cln(lr))
for r, lr in revs if lr in clrevs) for r, lr in revs if lr in clrevs)
clrevtolocalrev = {}
if self._isshallow: if self._isshallow:
# In a shallow clone, the linknodes callback needs to also include # In a shallow clone, the linknodes callback needs to also include
# those file nodes that are in the manifests we sent but weren't # those file nodes that are in the manifests we sent but weren't
# introduced by those manifests. # introduced by those manifests.
commonctxs = [self._repo[c] for c in commonrevs] commonctxs = [self._repo[c] for c in commonrevs]
clrev = self._repo.changelog.rev clrev = self._repo.changelog.rev
# Defining this function has a side-effect of overriding the # Defining this function has a side-effect of overriding the
# function of the same name that was passed in as an argument. # function of the same name that was passed in as an argument.
# TODO have caller pass in appropriate function. # TODO have caller pass in appropriate function.
def linknodes(flog, fname): def linknodes(flog, fname):
for c in commonctxs: for c in commonctxs:
try: try:
fnode = c.filenode(fname) fnode = c.filenode(fname)
self._clrevtolocalrev[c.rev()] = flog.rev(fnode) clrevtolocalrev[c.rev()] = flog.rev(fnode)
except error.ManifestLookupError: except error.ManifestLookupError:
pass pass
links = normallinknodes(flog, fname) links = normallinknodes(flog, fname)
if len(links) != len(mfdicts): if len(links) != len(mfdicts):
for mf, lr in mfdicts: for mf, lr in mfdicts:
fnode = mf.get(fname, None) fnode = mf.get(fname, None)
if fnode in links: if fnode in links:
links[fnode] = min(links[fnode], lr, key=clrev) links[fnode] = min(links[fnode], lr, key=clrev)
elif fnode: elif fnode:
links[fnode] = lr links[fnode] = lr
return links return links
else: else:
linknodes = normallinknodes linknodes = normallinknodes
repo = self._repo repo = self._repo
cl = repo.changelog cl = repo.changelog
progress = repo.ui.makeprogress(_('bundling'), unit=_('files'), progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
total=len(changedfiles)) total=len(changedfiles))
for i, fname in enumerate(sorted(changedfiles)): for i, fname in enumerate(sorted(changedfiles)):
filerevlog = repo.file(fname) filerevlog = repo.file(fname)
if not filerevlog: if not filerevlog:
raise error.Abort(_("empty or missing file data for %s") % raise error.Abort(_("empty or missing file data for %s") %
fname) fname)
clrevtolocalrev.clear()
linkrevnodes = linknodes(filerevlog, fname) linkrevnodes = linknodes(filerevlog, fname)
# Lookup for filenodes, we collected the linkrev nodes above in the # Lookup for filenodes, we collected the linkrev nodes above in the
# fastpath case and with lookupmf in the slowpath case. # fastpath case and with lookupmf in the slowpath case.
def lookupfilelog(x): def lookupfilelog(x):
return linkrevnodes[x] return linkrevnodes[x]
filenodes = self._prune(filerevlog, linkrevnodes, commonrevs) filenodes = self._prune(filerevlog, linkrevnodes, commonrevs)
if filenodes: if filenodes:
if self._ellipses: if self._ellipses:
revs = _sortnodesellipsis(filerevlog, filenodes, revs = _sortnodesellipsis(filerevlog, filenodes,
cl, lookupfilelog) cl, lookupfilelog)
else: else:
revs = _sortnodesnormal(filerevlog, filenodes, revs = _sortnodesnormal(filerevlog, filenodes,
self._reorder) self._reorder)
progress.update(i + 1, item=fname) progress.update(i + 1, item=fname)
h = _fileheader(fname) h = _fileheader(fname)
size = len(h) size = len(h)
yield h yield h
for chunk in self.group(revs, filerevlog, False, lookupfilelog): for chunk in self.group(revs, filerevlog, False, lookupfilelog,
clrevtolocalrev=clrevtolocalrev):
size += len(chunk) size += len(chunk)
yield chunk yield chunk
self._verbosenote(_('%8.i %s\n') % (size, fname)) self._verbosenote(_('%8.i %s\n') % (size, fname))
progress.complete() progress.complete()
def _revisiondeltanarrow(self, store, ischangelog, rev, linkrev, linknode): def _revisiondeltanarrow(self, store, ischangelog, rev, linkrev, linknode,
clrevtolocalrev):
linkparents = self._precomputedellipsis[linkrev] linkparents = self._precomputedellipsis[linkrev]
def local(clrev): def local(clrev):
"""Turn a changelog revnum into a local revnum. """Turn a changelog revnum into a local revnum.
The ellipsis dag is stored as revnums on the changelog, The ellipsis dag is stored as revnums on the changelog,
but when we're producing ellipsis entries for but when we're producing ellipsis entries for
non-changelog revlogs, we need to turn those numbers into non-changelog revlogs, we need to turn those numbers into
something local. This does that for us, and during the something local. This does that for us, and during the
# #
# For a filelog or tree manifest dirlog however not every changelog # For a filelog or tree manifest dirlog however not every changelog
# entry will have been relevant, so we need to skip some changelog # entry will have been relevant, so we need to skip some changelog
# nodes even after ellipsis-izing. # nodes even after ellipsis-izing.
walk = [clrev] walk = [clrev]
while walk: while walk:
p = walk[0] p = walk[0]
walk = walk[1:] walk = walk[1:]
if p in self._clrevtolocalrev: if p in clrevtolocalrev:
return self._clrevtolocalrev[p] return clrevtolocalrev[p]
elif p in self._fullclnodes: elif p in self._fullclnodes:
walk.extend([pp for pp in self._repo.changelog.parentrevs(p) walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
if pp != nullrev]) if pp != nullrev])
elif p in self._precomputedellipsis: elif p in self._precomputedellipsis:
walk.extend([pp for pp in self._precomputedellipsis[p] walk.extend([pp for pp in self._precomputedellipsis[p]
if pp != nullrev]) if pp != nullrev])
else: else:
# In this case, we've got an ellipsis with parents # In this case, we've got an ellipsis with parents