( )⚙ D4067 changegroup: move generate() modifications from narrow

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4067

changegroup: move generate() modifications from narrow
ClosedPublic

Authored by indygreg on Aug 2 2018, 5:15 PM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHGd706c77449f9: changegroup: move generate() modifications from narrow
Summary

Narrow had a custom version of generate() that was essentially a copy
of generate() with inline additions to facilitate ellipses serving.

This commit inlines those modifications into generate().

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

Diff 9948

hgext/narrow/__init__.py

localrepo, localrepo,
registrar, registrar,
repository, repository,
verify as verifymod, verify as verifymod,
) )
from . import ( from . import (
narrowbundle2, narrowbundle2,
narrowchangegroup,
narrowcommands, narrowcommands,
narrowcopies, narrowcopies,
narrowpatch, narrowpatch,
narrowrepo, narrowrepo,
narrowrevlog, narrowrevlog,
narrowtemplates, narrowtemplates,
narrowwirepeer, narrowwirepeer,
) )
features.add(repository.NARROW_REQUIREMENT) features.add(repository.NARROW_REQUIREMENT)
def uisetup(ui): def uisetup(ui):
"""Wraps user-facing mercurial commands with narrow-aware versions.""" """Wraps user-facing mercurial commands with narrow-aware versions."""
localrepo.featuresetupfuncs.add(featuresetup) localrepo.featuresetupfuncs.add(featuresetup)
narrowrevlog.setup() narrowrevlog.setup()
narrowbundle2.setup() narrowbundle2.setup()
narrowcommands.setup() narrowcommands.setup()
narrowchangegroup.setup()
narrowwirepeer.uisetup() narrowwirepeer.uisetup()
def reposetup(ui, repo): def reposetup(ui, repo):
"""Wraps local repositories with narrow repo support.""" """Wraps local repositories with narrow repo support."""
if not repo.local(): if not repo.local():
return return
if repository.NARROW_REQUIREMENT in repo.requirements: if repository.NARROW_REQUIREMENT in repo.requirements:

hgext/narrow/narrowchangegroup.py

  • This file was deleted.
# narrowchangegroup.py - narrow clone changegroup creation and consumption
#
# Copyright 2017 Google, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
from mercurial.i18n import _
from mercurial import (
changegroup,
extensions,
node,
util,
)
def setup():
def generate(orig, self, commonrevs, clnodes, fastpathlinkrev, source):
'''yield a sequence of changegroup chunks (strings)'''
# Note: other than delegating to orig, the only deviation in
# logic from normal hg's generate is marked with BEGIN/END
# NARROW HACK.
if not util.safehasattr(self, 'full_nodes'):
# not sending a narrow bundle
for x in orig(self, commonrevs, clnodes, fastpathlinkrev, source):
yield x
return
repo = self._repo
cl = repo.changelog
mfl = repo.manifestlog
mfrevlog = mfl._revlog
clrevorder = {}
mfs = {} # needed manifests
fnodes = {} # needed file nodes
changedfiles = set()
# Callback for the changelog, used to collect changed files and manifest
# nodes.
# Returns the linkrev node (identity in the changelog case).
def lookupcl(x):
c = cl.read(x)
clrevorder[x] = len(clrevorder)
# BEGIN NARROW HACK
#
# Only update mfs if x is going to be sent. Otherwise we
# end up with bogus linkrevs specified for manifests and
# we skip some manifest nodes that we should otherwise
# have sent.
if x in self.full_nodes or cl.rev(x) in self.precomputed_ellipsis:
n = c[0]
# record the first changeset introducing this manifest version
mfs.setdefault(n, x)
# Set this narrow-specific dict so we have the lowest manifest
# revnum to look up for this cl revnum. (Part of mapping
# changelog ellipsis parents to manifest ellipsis parents)
self.next_clrev_to_localrev.setdefault(cl.rev(x),
mfrevlog.rev(n))
# We can't trust the changed files list in the changeset if the
# client requested a shallow clone.
if self.is_shallow:
changedfiles.update(mfl[c[0]].read().keys())
else:
changedfiles.update(c[3])
# END NARROW HACK
# Record a complete list of potentially-changed files in
# this manifest.
return x
self._verbosenote(_('uncompressed size of bundle content:\n'))
size = 0
for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
size += len(chunk)
yield chunk
self._verbosenote(_('%8.i (changelog)\n') % size)
# We need to make sure that the linkrev in the changegroup refers to
# the first changeset that introduced the manifest or file revision.
# The fastpath is usually safer than the slowpath, because the filelogs
# are walked in revlog order.
#
# When taking the slowpath with reorder=None and the manifest revlog
# uses generaldelta, the manifest may be walked in the "wrong" order.
# Without 'clrevorder', we would get an incorrect linkrev (see fix in
# cc0ff93d0c0c).
#
# When taking the fastpath, we are only vulnerable to reordering
# of the changelog itself. The changelog never uses generaldelta, so
# it is only reordered when reorder=True. To handle this case, we
# simply take the slowpath, which already has the 'clrevorder' logic.
# This was also fixed in cc0ff93d0c0c.
fastpathlinkrev = fastpathlinkrev and not self._reorder
# Treemanifests don't work correctly with fastpathlinkrev
# either, because we don't discover which directory nodes to
# send along with files. This could probably be fixed.
fastpathlinkrev = fastpathlinkrev and (
'treemanifest' not in repo.requirements)
# Shallow clones also don't work correctly with fastpathlinkrev
# because file nodes may need to be sent for a manifest even if they
# weren't introduced by that manifest.
fastpathlinkrev = fastpathlinkrev and not self.is_shallow
for chunk in self.generatemanifests(commonrevs, clrevorder,
fastpathlinkrev, mfs, fnodes, source):
yield chunk
# BEGIN NARROW HACK
mfdicts = None
if self.is_shallow:
mfdicts = [(self._repo.manifestlog[n].read(), lr)
for (n, lr) in mfs.iteritems()]
# END NARROW HACK
mfs.clear()
clrevs = set(cl.rev(x) for x in clnodes)
if not fastpathlinkrev:
def linknodes(unused, fname):
return fnodes.get(fname, {})
else:
cln = cl.node
def linknodes(filerevlog, fname):
llr = filerevlog.linkrev
fln = filerevlog.node
revs = ((r, llr(r)) for r in filerevlog)
return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
# BEGIN NARROW HACK
#
# We need to pass the mfdicts variable down into
# generatefiles(), but more than one command might have
# wrapped generatefiles so we can't modify the function
# signature. Instead, we pass the data to ourselves using an
# instance attribute. I'm sorry.
self._mfdicts = mfdicts
# END NARROW HACK
for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
source):
yield chunk
yield self.close()
if clnodes:
repo.hook('outgoing', node=node.hex(clnodes[0]), source=source)
extensions.wrapfunction(changegroup.cg1packer, 'generate', generate)

mercurial/changegroup.py

def generate(self, commonrevs, clnodes, fastpathlinkrev, source): def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
'''yield a sequence of changegroup chunks (strings)''' '''yield a sequence of changegroup chunks (strings)'''
repo = self._repo repo = self._repo
cl = repo.changelog cl = repo.changelog
clrevorder = {} clrevorder = {}
mfs = {} # needed manifests mfs = {} # needed manifests
fnodes = {} # needed file nodes fnodes = {} # needed file nodes
mfl = repo.manifestlog
# TODO violates storage abstraction.
mfrevlog = mfl._revlog
changedfiles = set() changedfiles = set()
# Callback for the changelog, used to collect changed files and manifest ellipsesmode = util.safehasattr(self, 'full_nodes')
# nodes.
# Callback for the changelog, used to collect changed files and
# manifest nodes.
# Returns the linkrev node (identity in the changelog case). # Returns the linkrev node (identity in the changelog case).
def lookupcl(x): def lookupcl(x):
c = cl.read(x) c = cl.read(x)
clrevorder[x] = len(clrevorder) clrevorder[x] = len(clrevorder)
if ellipsesmode:
# Only update mfs if x is going to be sent. Otherwise we
# end up with bogus linkrevs specified for manifests and
# we skip some manifest nodes that we should otherwise
# have sent.
if (x in self.full_nodes
or cl.rev(x) in self.precomputed_ellipsis):
n = c[0]
# Record the first changeset introducing this manifest
# version.
mfs.setdefault(n, x)
# Set this narrow-specific dict so we have the lowest
# manifest revnum to look up for this cl revnum. (Part of
# mapping changelog ellipsis parents to manifest ellipsis
# parents)
self.next_clrev_to_localrev.setdefault(cl.rev(x),
mfrevlog.rev(n))
# We can't trust the changed files list in the changeset if the
# client requested a shallow clone.
if self.is_shallow:
changedfiles.update(mfl[c[0]].read().keys())
else:
changedfiles.update(c[3])
else:
n = c[0] n = c[0]
# record the first changeset introducing this manifest version # record the first changeset introducing this manifest version
mfs.setdefault(n, x) mfs.setdefault(n, x)
# Record a complete list of potentially-changed files in # Record a complete list of potentially-changed files in
# this manifest. # this manifest.
changedfiles.update(c[3]) changedfiles.update(c[3])
return x return x
self._verbosenote(_('uncompressed size of bundle content:\n')) self._verbosenote(_('uncompressed size of bundle content:\n'))
size = 0 size = 0
for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')): for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
size += len(chunk) size += len(chunk)
yield chunk yield chunk
self._verbosenote(_('%8.i (changelog)\n') % size) self._verbosenote(_('%8.i (changelog)\n') % size)
# either, because we don't discover which directory nodes to # either, because we don't discover which directory nodes to
# send along with files. This could probably be fixed. # send along with files. This could probably be fixed.
fastpathlinkrev = fastpathlinkrev and ( fastpathlinkrev = fastpathlinkrev and (
'treemanifest' not in repo.requirements) 'treemanifest' not in repo.requirements)
for chunk in self.generatemanifests(commonrevs, clrevorder, for chunk in self.generatemanifests(commonrevs, clrevorder,
fastpathlinkrev, mfs, fnodes, source): fastpathlinkrev, mfs, fnodes, source):
yield chunk yield chunk
if ellipsesmode:
mfdicts = None
if self.is_shallow:
mfdicts = [(self._repo.manifestlog[n].read(), lr)
for (n, lr) in mfs.iteritems()]
mfs.clear() mfs.clear()
clrevs = set(cl.rev(x) for x in clnodes) clrevs = set(cl.rev(x) for x in clnodes)
if not fastpathlinkrev: if not fastpathlinkrev:
def linknodes(unused, fname): def linknodes(unused, fname):
return fnodes.get(fname, {}) return fnodes.get(fname, {})
else: else:
cln = cl.node cln = cl.node
def linknodes(filerevlog, fname): def linknodes(filerevlog, fname):
llr = filerevlog.linkrev llr = filerevlog.linkrev
fln = filerevlog.node fln = filerevlog.node
revs = ((r, llr(r)) for r in filerevlog) revs = ((r, llr(r)) for r in filerevlog)
return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs) return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
if ellipsesmode:
# We need to pass the mfdicts variable down into
# generatefiles(), but more than one command might have
# wrapped generatefiles so we can't modify the function
# signature. Instead, we pass the data to ourselves using an
# instance attribute. I'm sorry.
self._mfdicts = mfdicts
for chunk in self.generatefiles(changedfiles, linknodes, commonrevs, for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
source): source):
yield chunk yield chunk
yield self.close() yield self.close()
if clnodes: if clnodes:
repo.hook('outgoing', node=hex(clnodes[0]), source=source) repo.hook('outgoing', node=hex(clnodes[0]), source=source)
hgext/narrow/__init__.py