diff --git a/hgext/sqlitestore.py b/hgext/sqlitestore.py --- a/hgext/sqlitestore.py +++ b/hgext/sqlitestore.py @@ -587,6 +587,7 @@ revisiondata=False, assumehaveparentrevisions=False, deltamode=repository.CG_DELTAMODE_STD, + sidedata_helpers=None, ): if nodesorder not in (b'nodes', b'storage', b'linear', None): raise error.ProgrammingError( @@ -625,6 +626,7 @@ revisiondata=revisiondata, assumehaveparentrevisions=assumehaveparentrevisions, deltamode=deltamode, + sidedata_helpers=sidedata_helpers, ): yield delta diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py --- a/mercurial/changegroup.py +++ b/mercurial/changegroup.py @@ -273,6 +273,7 @@ url, targetphase=phases.draft, expectedtotal=None, + sidedata_categories=None, ): """Add the changegroup returned by source.read() to this repo. srctype is a string like 'push', 'pull', or 'unbundle'. url is @@ -286,6 +287,17 @@ """ repo = repo.unfiltered() + # Only useful if we're adding sidedata categories. If both peers have + # the same categories, then we simply don't do anything. + if self.version == b'04' and srctype == b'pull': + sidedata_helpers = get_sidedata_helpers( + repo, + sidedata_categories or set(), + pull=True, + ) + else: + sidedata_helpers = None + def csmap(x): repo.ui.debug(b"add changeset %s\n" % short(x)) return len(cl) @@ -754,6 +766,7 @@ clrevtolocalrev=None, fullclnodes=None, precomputedellipsis=None, + sidedata_helpers=None, ): """Calculate deltas for a set of revisions. @@ -859,6 +872,7 @@ revisiondata=True, assumehaveparentrevisions=not ellipses, deltamode=deltamode, + sidedata_helpers=sidedata_helpers, ) for i, revision in enumerate(revisions): @@ -977,8 +991,19 @@ self._verbosenote(_(b'uncompressed size of bundle content:\n')) size = 0 + sidedata_helpers = None + if self.version == b'04': + remote_sidedata = self._remote_sidedata + if source == b'strip': + # We're our own remote when stripping, get the no-op helpers + remote_sidedata = repo._wanted_sidedata + sidedata_helpers = get_sidedata_helpers(repo, remote_sidedata) + clstate, deltas = self._generatechangelog( - cl, clnodes, generate=changelog + cl, + clnodes, + generate=changelog, + sidedata_helpers=sidedata_helpers, ) for delta in deltas: for chunk in _revisiondeltatochunks(delta, self._builddeltaheader): @@ -1026,6 +1051,7 @@ fnodes, source, clstate[b'clrevtomanifestrev'], + sidedata_helpers=sidedata_helpers, ) for tree, deltas in it: @@ -1066,6 +1092,7 @@ fastpathlinkrev, fnodes, clrevs, + sidedata_helpers=sidedata_helpers, ) for path, deltas in it: @@ -1090,7 +1117,9 @@ if clnodes: repo.hook(b'outgoing', node=hex(clnodes[0]), source=source) - def _generatechangelog(self, cl, nodes, generate=True): + def _generatechangelog( + self, cl, nodes, generate=True, sidedata_helpers=None + ): """Generate data for changelog chunks. Returns a 2-tuple of a dict containing state and an iterable of @@ -1182,6 +1211,7 @@ clrevtolocalrev={}, fullclnodes=self._fullclnodes, precomputedellipsis=self._precomputedellipsis, + sidedata_helpers=sidedata_helpers, ) return state, gen @@ -1195,6 +1225,7 @@ fnodes, source, clrevtolocalrev, + sidedata_helpers=None, ): """Returns an iterator of changegroup chunks containing manifests. @@ -1288,6 +1319,7 @@ clrevtolocalrev=clrevtolocalrev, fullclnodes=self._fullclnodes, precomputedellipsis=self._precomputedellipsis, + sidedata_helpers=sidedata_helpers, ) if not self._oldmatcher.visitdir(store.tree[:-1]): @@ -1326,6 +1358,7 @@ fastpathlinkrev, fnodes, clrevs, + sidedata_helpers=None, ): changedfiles = [ f @@ -1420,6 +1453,7 @@ clrevtolocalrev=clrevtolocalrev, fullclnodes=self._fullclnodes, precomputedellipsis=self._precomputedellipsis, + sidedata_helpers=sidedata_helpers, ) yield fname, deltas @@ -1793,3 +1827,25 @@ ) return revisions, files + + +def get_sidedata_helpers(repo, remote_sd_categories, pull=False): + # Computers for computing sidedata on-the-fly + sd_computers = collections.defaultdict(list) + # Computers for categories to remove from sidedata + sd_removers = collections.defaultdict(list) + + to_generate = remote_sd_categories - repo._wanted_sidedata + to_remove = repo._wanted_sidedata - remote_sd_categories + if pull: + to_generate, to_remove = to_remove, to_generate + + for revlog_kind, computers in repo._sidedata_computers.items(): + for category, computer in computers.items(): + if category in to_generate: + sd_computers[revlog_kind].append(computer) + if category in to_remove: + sd_removers[revlog_kind].append(computer) + + sidedata_helpers = (repo, sd_computers, sd_removers) + return sidedata_helpers diff --git a/mercurial/filelog.py b/mercurial/filelog.py --- a/mercurial/filelog.py +++ b/mercurial/filelog.py @@ -102,6 +102,7 @@ revisiondata=False, assumehaveparentrevisions=False, deltamode=repository.CG_DELTAMODE_STD, + sidedata_helpers=None, ): return self._revlog.emitrevisions( nodes, @@ -109,6 +110,7 @@ revisiondata=revisiondata, assumehaveparentrevisions=assumehaveparentrevisions, deltamode=deltamode, + sidedata_helpers=sidedata_helpers, ) def addrevision( diff --git a/mercurial/manifest.py b/mercurial/manifest.py --- a/mercurial/manifest.py +++ b/mercurial/manifest.py @@ -1825,6 +1825,7 @@ revisiondata=False, assumehaveparentrevisions=False, deltamode=repository.CG_DELTAMODE_STD, + sidedata_helpers=None, ): return self._revlog.emitrevisions( nodes, @@ -1832,6 +1833,7 @@ revisiondata=revisiondata, assumehaveparentrevisions=assumehaveparentrevisions, deltamode=deltamode, + sidedata_helpers=sidedata_helpers, ) def addgroup( diff --git a/mercurial/revlog.py b/mercurial/revlog.py --- a/mercurial/revlog.py +++ b/mercurial/revlog.py @@ -2742,6 +2742,7 @@ revisiondata=False, assumehaveparentrevisions=False, deltamode=repository.CG_DELTAMODE_STD, + sidedata_helpers=None, ): if nodesorder not in (b'nodes', b'storage', b'linear', None): raise error.ProgrammingError( @@ -2770,6 +2771,7 @@ deltamode=deltamode, revisiondata=revisiondata, assumehaveparentrevisions=assumehaveparentrevisions, + sidedata_helpers=sidedata_helpers, ) DELTAREUSEALWAYS = b'always' diff --git a/mercurial/utils/storageutil.py b/mercurial/utils/storageutil.py --- a/mercurial/utils/storageutil.py +++ b/mercurial/utils/storageutil.py @@ -23,6 +23,7 @@ pycompat, ) from ..interfaces import repository +from ..revlogutils import sidedata as sidedatamod from ..utils import hashutil _nullhash = hashutil.sha1(nullid) @@ -294,6 +295,7 @@ deltamode=repository.CG_DELTAMODE_STD, revisiondata=False, assumehaveparentrevisions=False, + sidedata_helpers=None, ): """Generic implementation of ifiledata.emitrevisions(). @@ -356,6 +358,21 @@ ``nodesorder`` ``revisiondata`` ``assumehaveparentrevisions`` + ``sidedata_helpers`` (optional) + If not None, means that sidedata should be included. + A dictionary of revlog type to tuples of `(repo, computers, removers)`: + * `repo` is used as an argument for computers + * `computers` is a list of `(category, (keys, computer)` that + compute the missing sidedata categories that were asked: + * `category` is the sidedata category + * `keys` are the sidedata keys to be affected + * `computer` is the function `(repo, store, rev, sidedata)` that + returns a new sidedata dict. + * `removers` will remove the keys corresponding to the categories + that are present, but not needed. + If both `computers` and `removers` are empty, sidedata are simply not + transformed. + Revlog types are `changelog`, `manifest` or `filelog`. """ fnode = store.node @@ -469,6 +486,18 @@ available.add(rev) + sidedata = None + if sidedata_helpers: + sidedata = store.sidedata(rev) + sidedata = run_sidedata_helpers( + store=store, + sidedata_helpers=sidedata_helpers, + sidedata=sidedata, + rev=rev, + kind=None, + ) + sidedata = sidedatamod.serialize_sidedata(sidedata) + yield resultcls( node=node, p1node=fnode(p1rev), @@ -484,6 +513,21 @@ prevrev = rev +def run_sidedata_helpers(store, sidedata_helpers, sidedata, rev, kind): + repo, sd_computers, sd_removers = sidedata_helpers + if kind: + kinds = [kind] + else: + kinds = [b'changelog', b'manifest', b'filelog'] + for kind in kinds: + for _keys, sd_computer in sd_computers.get(kind, []): + sidedata = sd_computer(repo, store, rev, sidedata) + for keys, _computer in sd_removers.get(kind, []): + for key in keys: + sidedata.pop(key, None) + return sidedata + + def deltaiscensored(delta, baserev, baselenfn): """Determine if a delta represents censored revision data. diff --git a/tests/simplestorerepo.py b/tests/simplestorerepo.py --- a/tests/simplestorerepo.py +++ b/tests/simplestorerepo.py @@ -446,6 +446,7 @@ revisiondata=False, assumehaveparentrevisions=False, deltamode=repository.CG_DELTAMODE_STD, + sidedata_helpers=None, ): # TODO this will probably break on some ordering options. nodes = [n for n in nodes if n != nullid] @@ -459,6 +460,7 @@ revisiondata=revisiondata, assumehaveparentrevisions=assumehaveparentrevisions, deltamode=deltamode, + sidedata_helpers=sidedata_helpers, ): yield delta