This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4011

changegroup: move file matcher from narrow extension
ClosedPublic

Authored by indygreg on Aug 1 2018, 1:03 PM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHG1d01cf0416a5: changegroup: move file matcher from narrow extension
rHGa08a00df21db: changegroup: move file matcher from narrow extension
rHG9c057acba656: changegroup: move file matcher from narrow extension
Summary

Sparse changegroup generation requires the use of a matcher to
filter which files are relevant.

This commit moves the file matcher from the narrow extension to core
and updates the narrow extension to use it.

I'm not sure why the narrow extension was storing the matcher as
a callable that resolved to a matcher. So I changed it to be a
simple matcher instance.

In addition, code from narrow to intersect the matcher with the local
narrow spec is now performed automatically when the changegroup
packer is created.

If a matcher is not passed into getbundler() an alwaysmatcher() is
assumed. This ensures that a matcher is always defined for all
operations.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Aug 1 2018, 1:03 PM
durin42 added a comment.Aug 1 2018, 4:52 PM

This one needs rebased

indygreg updated this revision to Diff 9746.Aug 1 2018, 5:04 PM
indygreg added a comment.Aug 1 2018, 5:05 PM
In D4011#62541, @durin42 wrote:

This one needs rebased

I ran hg rebase against @ and the 3-way merge "just worked." Limitations of our hg import based workflow, I suppose.

durin42 accepted this revision.Aug 1 2018, 7:27 PM
This revision is now accepted and ready to land.Aug 1 2018, 7:27 PM
Closed by commit rHG9c057acba656: changegroup: move file matcher from narrow extension (authored by indygreg). · Explain WhyAug 1 2018, 7:30 PM
This revision was automatically updated to reflect the committed changes.
martinvonz added a subscriber: martinvonz.Aug 1 2018, 11:44 PM

Looks good, thanks for cleaning up! (I know it's already been queued.)

Revision Contents
Changeset List

Diff 9770

hgext/narrow/narrowbundle2.py

raise error.Abort( raise error.Abort(
'ellipsis nodes require at least cg3 on client and server, ' 'ellipsis nodes require at least cg3 on client and server, '
'but negotiated version %s' % version) 'but negotiated version %s' % version)
# We wrap cg1packer.revchunk, using a side channel to pass # We wrap cg1packer.revchunk, using a side channel to pass
# relevant_nodes into that area. Then if linknode isn't in the # relevant_nodes into that area. Then if linknode isn't in the
# set, we know we have an ellipsis node and we should defer # set, we know we have an ellipsis node and we should defer
# sending that node's data. We override close() to detect # sending that node's data. We override close() to detect
# pending ellipsis nodes and flush them. # pending ellipsis nodes and flush them.
packer = changegroup.getbundler(version, repo) packer = changegroup.getbundler(version, repo,
# Let the packer have access to the narrow matcher so it can filematcher=match)
# omit filelogs and dirlogs as needed
packer._narrow_matcher = lambda : match
# Give the packer the list of nodes which should not be # Give the packer the list of nodes which should not be
# ellipsis nodes. We store this rather than the set of nodes # ellipsis nodes. We store this rather than the set of nodes
# that should be an ellipsis because for very large histories # that should be an ellipsis because for very large histories
# we expect this to be significantly smaller. # we expect this to be significantly smaller.
packer.full_nodes = relevant_nodes packer.full_nodes = relevant_nodes
# Maps ellipsis revs to their roots at the changelog level. # Maps ellipsis revs to their roots at the changelog level.
packer.precomputed_ellipsis = ellipsisroots packer.precomputed_ellipsis = ellipsisroots
# Maps CL revs to per-revlog revisions. Cleared in close() at # Maps CL revs to per-revlog revisions. Cleared in close() at
include = sorted(filter(bool, kwargs.get(r'includepats', []))) include = sorted(filter(bool, kwargs.get(r'includepats', [])))
exclude = sorted(filter(bool, kwargs.get(r'excludepats', []))) exclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
newmatch = narrowspec.match(repo.root, include=include, exclude=exclude) newmatch = narrowspec.match(repo.root, include=include, exclude=exclude)
if not repo.ui.configbool("experimental", "narrowservebrokenellipses"): if not repo.ui.configbool("experimental", "narrowservebrokenellipses"):
outgoing = exchange._computeoutgoing(repo, heads, common) outgoing = exchange._computeoutgoing(repo, heads, common)
if not outgoing.missing: if not outgoing.missing:
return return
def wrappedgetbundler(orig, *args, **kwargs):
bundler = orig(*args, **kwargs) cg = changegroup.makestream(repo, outgoing, version, source,
bundler._narrow_matcher = lambda : newmatch filematcher=newmatch)
return bundler
with extensions.wrappedfunction(changegroup, 'getbundler',
wrappedgetbundler):
cg = changegroup.makestream(repo, outgoing, version, source)
part = bundler.newpart('changegroup', data=cg) part = bundler.newpart('changegroup', data=cg)
part.addparam('version', version) part.addparam('version', version)
if 'treemanifest' in repo.requirements: if 'treemanifest' in repo.requirements:
part.addparam('treemanifest', '1') part.addparam('treemanifest', '1')
if include or exclude: if include or exclude:
narrowspecpart = bundler.newpart(_SPECPART) narrowspecpart = bundler.newpart(_SPECPART)
if include: if include:

hgext/narrow/narrowchangegroup.py

# narrowchangegroup.py - narrow clone changegroup creation and consumption # narrowchangegroup.py - narrow clone changegroup creation and consumption
# #
# Copyright 2017 Google, Inc. # Copyright 2017 Google, Inc.
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial import ( from mercurial import (
changegroup, changegroup,
error, error,
extensions, extensions,
manifest, manifest,
match as matchmod,
mdiff, mdiff,
node, node,
pycompat, pycompat,
revlog, revlog,
util, util,
) )
def setup(): def setup():
def _cgmatcher(cgpacker):
localmatcher = cgpacker._repo.narrowmatch()
remotematcher = getattr(cgpacker, '_narrow_matcher', lambda: None)()
if remotematcher:
return matchmod.intersectmatchers(localmatcher, remotematcher)
else:
return localmatcher
def prune(orig, self, revlog, missing, commonrevs): def prune(orig, self, revlog, missing, commonrevs):
if isinstance(revlog, manifest.manifestrevlog): if isinstance(revlog, manifest.manifestrevlog):
matcher = _cgmatcher(self) if not self._filematcher.visitdir(revlog._dir[:-1] or '.'):
if (matcher and
not matcher.visitdir(revlog._dir[:-1] or '.')):
return [] return []
return orig(self, revlog, missing, commonrevs) return orig(self, revlog, missing, commonrevs)
extensions.wrapfunction(changegroup.cg1packer, 'prune', prune) extensions.wrapfunction(changegroup.cg1packer, 'prune', prune)
def generatefiles(orig, self, changedfiles, linknodes, commonrevs, def generatefiles(orig, self, changedfiles, linknodes, commonrevs,
source): source):
matcher = _cgmatcher(self) changedfiles = list(filter(self._filematcher, changedfiles))
if matcher:
changedfiles = list(filter(matcher, changedfiles))
if getattr(self, 'is_shallow', False): if getattr(self, 'is_shallow', False):
# See comment in generate() for why this sadness is a thing. # See comment in generate() for why this sadness is a thing.
mfdicts = self._mfdicts mfdicts = self._mfdicts
del self._mfdicts del self._mfdicts
# In a shallow clone, the linknodes callback needs to also include # In a shallow clone, the linknodes callback needs to also include
# those file nodes that are in the manifests we sent but weren't # those file nodes that are in the manifests we sent but weren't
# introduced by those manifests. # introduced by those manifests.
commonctxs = [self._repo[c] for c in commonrevs] commonctxs = [self._repo[c] for c in commonrevs]

mercurial/changegroup.py

hex, hex,
nullrev, nullrev,
short, short,
) )
from . import ( from . import (
dagutil, dagutil,
error, error,
match as matchmod,
mdiff, mdiff,
phases, phases,
pycompat, pycompat,
util, util,
) )
from .utils import ( from .utils import (
stringutil, stringutil,
if len(d) < n: if len(d) < n:
d += readexactly(self._fh, n - len(d)) d += readexactly(self._fh, n - len(d))
return d return d
return readexactly(self._fh, n) return readexactly(self._fh, n)
class cg1packer(object): class cg1packer(object):
deltaheader = _CHANGEGROUPV1_DELTA_HEADER deltaheader = _CHANGEGROUPV1_DELTA_HEADER
version = '01' version = '01'
def __init__(self, repo, bundlecaps=None): def __init__(self, repo, filematcher, bundlecaps=None):
"""Given a source repo, construct a bundler. """Given a source repo, construct a bundler.
filematcher is a matcher that matches on files to include in the
changegroup. Used to facilitate sparse changegroups.
bundlecaps is optional and can be used to specify the set of bundlecaps is optional and can be used to specify the set of
capabilities which can be used to build the bundle. While bundlecaps is capabilities which can be used to build the bundle. While bundlecaps is
unused in core Mercurial, extensions rely on this feature to communicate unused in core Mercurial, extensions rely on this feature to communicate
capabilities to customize the changegroup packer. capabilities to customize the changegroup packer.
""" """
assert filematcher
self._filematcher = filematcher
# Set of capabilities we can use to build the bundle. # Set of capabilities we can use to build the bundle.
if bundlecaps is None: if bundlecaps is None:
bundlecaps = set() bundlecaps = set()
self._bundlecaps = bundlecaps self._bundlecaps = bundlecaps
# experimental config: bundle.reorder # experimental config: bundle.reorder
reorder = repo.ui.config('bundle', 'reorder') reorder = repo.ui.config('bundle', 'reorder')
if reorder == 'auto': if reorder == 'auto':
reorder = None reorder = None
# do nothing with basenode, it is implicitly the previous one in HG10 # do nothing with basenode, it is implicitly the previous one in HG10
# do nothing with flags, it is implicitly 0 for cg1 and cg2 # do nothing with flags, it is implicitly 0 for cg1 and cg2
return struct.pack(self.deltaheader, node, p1n, p2n, linknode) return struct.pack(self.deltaheader, node, p1n, p2n, linknode)
class cg2packer(cg1packer): class cg2packer(cg1packer):
version = '02' version = '02'
deltaheader = _CHANGEGROUPV2_DELTA_HEADER deltaheader = _CHANGEGROUPV2_DELTA_HEADER
def __init__(self, repo, bundlecaps=None): def __init__(self, repo, filematcher, bundlecaps=None):
super(cg2packer, self).__init__(repo, bundlecaps) super(cg2packer, self).__init__(repo, filematcher,
bundlecaps=bundlecaps)
if self._reorder is None: if self._reorder is None:
# Since generaldelta is directly supported by cg2, reordering # Since generaldelta is directly supported by cg2, reordering
# generally doesn't help, so we disable it by default (treating # generally doesn't help, so we disable it by default (treating
# bundle.reorder=auto just like bundle.reorder=False). # bundle.reorder=auto just like bundle.reorder=False).
self._reorder = False self._reorder = False
def deltaparent(self, revlog, rev, p1, p2, prev): def deltaparent(self, revlog, rev, p1, p2, prev):
dp = revlog.deltaparent(rev) dp = revlog.deltaparent(rev)
# will support. For example, all hg versions that support generaldelta also # will support. For example, all hg versions that support generaldelta also
# support changegroup 02. # support changegroup 02.
versions = supportedoutgoingversions(repo) versions = supportedoutgoingversions(repo)
if 'generaldelta' in repo.requirements: if 'generaldelta' in repo.requirements:
versions.discard('01') versions.discard('01')
assert versions assert versions
return min(versions) return min(versions)
def getbundler(version, repo, bundlecaps=None): def getbundler(version, repo, bundlecaps=None, filematcher=None):
assert version in supportedoutgoingversions(repo) assert version in supportedoutgoingversions(repo)
return _packermap[version][0](repo, bundlecaps)
if filematcher is None:
filematcher = matchmod.alwaysmatcher(repo.root, '')
if version == '01' and not filematcher.always():
raise error.ProgrammingError('version 01 changegroups do not support '
'sparse file matchers')
# Requested files could include files not in the local store. So
# filter those out.
filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
filematcher)
return _packermap[version][0](repo, filematcher=filematcher,
bundlecaps=bundlecaps)
def getunbundler(version, fh, alg, extras=None): def getunbundler(version, fh, alg, extras=None):
return _packermap[version][1](fh, alg, extras=extras) return _packermap[version][1](fh, alg, extras=extras)
def _changegroupinfo(repo, nodes, source): def _changegroupinfo(repo, nodes, source):
if repo.ui.verbose or source == 'bundle': if repo.ui.verbose or source == 'bundle':
repo.ui.status(_("%d changesets found\n") % len(nodes)) repo.ui.status(_("%d changesets found\n") % len(nodes))
if repo.ui.debugflag: if repo.ui.debugflag:
repo.ui.debug("list of changesets:\n") repo.ui.debug("list of changesets:\n")
for node in nodes: for node in nodes:
repo.ui.debug("%s\n" % hex(node)) repo.ui.debug("%s\n" % hex(node))
def makechangegroup(repo, outgoing, version, source, fastpath=False, def makechangegroup(repo, outgoing, version, source, fastpath=False,
bundlecaps=None): bundlecaps=None):
cgstream = makestream(repo, outgoing, version, source, cgstream = makestream(repo, outgoing, version, source,
fastpath=fastpath, bundlecaps=bundlecaps) fastpath=fastpath, bundlecaps=bundlecaps)
return getunbundler(version, util.chunkbuffer(cgstream), None, return getunbundler(version, util.chunkbuffer(cgstream), None,
{'clcount': len(outgoing.missing) }) {'clcount': len(outgoing.missing) })
def makestream(repo, outgoing, version, source, fastpath=False, def makestream(repo, outgoing, version, source, fastpath=False,
bundlecaps=None): bundlecaps=None, filematcher=None):
bundler = getbundler(version, repo, bundlecaps=bundlecaps) bundler = getbundler(version, repo, bundlecaps=bundlecaps,
filematcher=filematcher)
repo = repo.unfiltered() repo = repo.unfiltered()
commonrevs = outgoing.common commonrevs = outgoing.common
csets = outgoing.missing csets = outgoing.missing
heads = outgoing.missingheads heads = outgoing.missingheads
# We go through the fast path if we get told to, or if all (unfiltered # We go through the fast path if we get told to, or if all (unfiltered
# heads have been requested (since we then know there all linkrevs will # heads have been requested (since we then know there all linkrevs will
# be pulled by the client). # be pulled by the client).