This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10026

changegroup: add v4 changegroup for revlog v2 exchange
ClosedPublic

Authored by Alphare on Feb 19 2021, 6:16 AM.

Details

Reviewers
indygreg
marmoute
Group Reviewers
hg-reviewers
Commits
rHGa41565bef69f: changegroup: add v4 changegroup for revlog v2 exchange
Summary

This change only adds the required infrastructure for the new changegroup format
and does not do any actual exchange. This will be done in the next patches.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

Alphare created this revision.Feb 19 2021, 6:16 AM
baymax updated this revision to Diff 25993.Mar 1 2021, 11:52 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

marmoute added a subscriber: marmoute.Mar 2 2021, 1:55 PM

So in this changeset you add most of the infrastrucure to excahgne sidedata, but don't make any decision to exchange any yet, right? So we could exchange them, but still don't. Can you clarify this in the commit message ?

marmoute requested changes to this revision.Mar 2 2021, 1:56 PM
This revision now requires changes to proceed.Mar 2 2021, 1:56 PM
Alphare edited the summary of this revision. (Show Details)Mar 4 2021, 10:18 AM
Alphare updated this revision to Diff 26089.
marmoute requested changes to this revision.Mar 9 2021, 11:21 AM
marmoute added inline comments.
mercurial/changegroup.py
578–580

Can you explain the 4 here ?

1519–1521

What is going on here ?

This revision now requires changes to proceed.Mar 9 2021, 11:21 AM
Alphare added inline comments.Mar 10 2021, 1:41 PM
mercurial/changegroup.py
578–580

I've found the getchunks function that encapsulates the logic

1519–1521

I've left comments (pending refresh) to be more explicit.

Alphare updated this revision to Diff 26220.Mar 10 2021, 2:11 PM
marmoute accepted this revision.Mar 11 2021, 1:09 PM

Thanks for the update

mercurial/changegroup.py
578–580

A yep, better ☺

Alphare updated this revision to Diff 26254.Mar 12 2021, 6:33 AM
Alphare updated this revision to Diff 26332.Mar 15 2021, 6:24 AM
Alphare added a commit: rHGa41565bef69f: changegroup: add v4 changegroup for revlog v2 exchange.Mar 15 2021, 2:32 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHGa41565bef69f: changegroup: add v4 changegroup for revlog v2 exchange (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 26377

hgext/remotefilelog/remotefilelog.py

def emitrevisions( def emitrevisions(
self, self,
nodes, nodes,
nodesorder=None, nodesorder=None,
revisiondata=False, revisiondata=False,
assumehaveparentrevisions=False, assumehaveparentrevisions=False,
deltaprevious=False, deltaprevious=False,
deltamode=None, deltamode=None,
sidedata_helpers=None,
): ):
# we don't use any of these parameters here # we don't use any of these parameters here
del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious
del deltamode del deltamode
prevnode = None prevnode = None
for node in nodes: for node in nodes:
p1, p2 = self.parents(node) p1, p2 = self.parents(node)
if prevnode is None: if prevnode is None:
p1node=p1, p1node=p1,
p2node=p2, p2node=p2,
linknode=self.linknode(node), linknode=self.linknode(node),
basenode=basenode, basenode=basenode,
flags=self.flags(node), flags=self.flags(node),
baserevisionsize=None, baserevisionsize=None,
revision=revision, revision=revision,
delta=delta, delta=delta,
# Sidedata is not supported yet
sidedata=None,
) )
def revdiff(self, node1, node2): def revdiff(self, node1, node2):
return mdiff.textdiff(self.rawdata(node1), self.rawdata(node2)) return mdiff.textdiff(self.rawdata(node1), self.rawdata(node2))
def lookup(self, node): def lookup(self, node):
if len(node) == 40: if len(node) == 40:
node = bin(node) node = bin(node)

hgext/remotefilelog/shallowbundle.py

commonrevs, clnodes, fastpathlinkrev, source, **kwargs commonrevs, clnodes, fastpathlinkrev, source, **kwargs
) )
def group(self, nodelist, rlog, lookup, units=None, reorder=None): def group(self, nodelist, rlog, lookup, units=None, reorder=None):
return shallowgroup( return shallowgroup(
shallowcg1packer, self, nodelist, rlog, lookup, units=units shallowcg1packer, self, nodelist, rlog, lookup, units=units
) )
def generatefiles(self, changedfiles, *args): def generatefiles(self, changedfiles, *args, **kwargs):
try: try:
linknodes, commonrevs, source = args linknodes, commonrevs, source = args
except ValueError: except ValueError:
commonrevs, source, mfdicts, fastpathlinkrev, fnodes, clrevs = args commonrevs, source, mfdicts, fastpathlinkrev, fnodes, clrevs = args
if shallowutil.isenabled(self._repo): if shallowutil.isenabled(self._repo):
repo = self._repo repo = self._repo
if isinstance(repo, bundlerepo.bundlerepository): if isinstance(repo, bundlerepo.bundlerepository):
# If the bundle contains filelogs, we can't pull from it, since # If the bundle contains filelogs, we can't pull from it, since
# bundlerepo is heavily tied to revlogs. Instead require that # bundlerepo is heavily tied to revlogs. Instead require that
# the user use unbundle instead. # the user use unbundle instead.
# Force load the filelog data. # Force load the filelog data.
bundlerepo.bundlerepository.file(repo, b'foo') bundlerepo.bundlerepository.file(repo, b'foo')
if repo._cgfilespos: if repo._cgfilespos:
raise error.Abort( raise error.Abort(
b"cannot pull from full bundles", b"cannot pull from full bundles",
hint=b"use `hg unbundle` instead", hint=b"use `hg unbundle` instead",
) )
return [] return []
filestosend = self.shouldaddfilegroups(source) filestosend = self.shouldaddfilegroups(source)
if filestosend == NoFiles: if filestosend == NoFiles:
changedfiles = list( changedfiles = list(
[f for f in changedfiles if not repo.shallowmatch(f)] [f for f in changedfiles if not repo.shallowmatch(f)]
) )
return super(shallowcg1packer, self).generatefiles(changedfiles, *args) return super(shallowcg1packer, self).generatefiles(
changedfiles, *args, **kwargs
)
def shouldaddfilegroups(self, source): def shouldaddfilegroups(self, source):
repo = self._repo repo = self._repo
if not shallowutil.isenabled(repo): if not shallowutil.isenabled(repo):
return AllFiles return AllFiles
if source == b"push" or source == b"bundle": if source == b"push" or source == b"bundle":
return AllFiles return AllFiles
) )
else: else:
repo.shallowmatch = match.always() repo.shallowmatch = match.always()
return orig(repo, outgoing, version, source, *args, **kwargs) return orig(repo, outgoing, version, source, *args, **kwargs)
finally: finally:
repo.shallowmatch = original repo.shallowmatch = original
def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args): def addchangegroupfiles(
orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs
):
if not shallowutil.isenabled(repo): if not shallowutil.isenabled(repo):
return orig(repo, source, revmap, trp, expectedfiles, *args) return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs)
newfiles = 0 newfiles = 0
visited = set() visited = set()
revisiondatas = {} revisiondatas = {}
queue = [] queue = []
# Normal Mercurial processes each file one at a time, adding all # Normal Mercurial processes each file one at a time, adding all
# the new revisions for that file at once. In remotefilelog a file # the new revisions for that file at once. In remotefilelog a file
skipcount += 1 skipcount += 1
if skipcount > len(queue) + 1: if skipcount > len(queue) + 1:
raise error.Abort(_(b"circular node dependency")) raise error.Abort(_(b"circular node dependency"))
fl = repo.file(f) fl = repo.file(f)
revisiondata = revisiondatas[(f, node)] revisiondata = revisiondatas[(f, node)]
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags) # revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
node, p1, p2, linknode, deltabase, delta, flags = revisiondata node, p1, p2, linknode, deltabase, delta, flags, sidedata = revisiondata
if not available(f, node, f, deltabase): if not available(f, node, f, deltabase):
continue continue
base = fl.rawdata(deltabase) base = fl.rawdata(deltabase)
text = mdiff.patch(base, delta) text = mdiff.patch(base, delta)
if not isinstance(text, bytes): if not isinstance(text, bytes):
text = bytes(text) text = bytes(text)

hgext/sqlitestore.py

linkmapper, linkmapper,
transaction, transaction,
addrevisioncb=None, addrevisioncb=None,
duplicaterevisioncb=None, duplicaterevisioncb=None,
maybemissingparents=False, maybemissingparents=False,
): ):
empty = True empty = True
for node, p1, p2, linknode, deltabase, delta, wireflags in deltas: for (
node,
p1,
p2,
linknode,
deltabase,
delta,
wireflags,
sidedata,
) in deltas:
storeflags = 0 storeflags = 0
if wireflags & repository.REVISION_FLAG_CENSORED: if wireflags & repository.REVISION_FLAG_CENSORED:
storeflags |= FLAG_CENSORED storeflags |= FLAG_CENSORED
if wireflags & ~repository.REVISION_FLAG_CENSORED: if wireflags & ~repository.REVISION_FLAG_CENSORED:
raise SQLiteStoreError(b'unhandled revision flag') raise SQLiteStoreError(b'unhandled revision flag')

mercurial/bundlerepo.py

# check revision against repotiprev. # check revision against repotiprev.
opener = vfsmod.readonlyvfs(opener) opener = vfsmod.readonlyvfs(opener)
revlog.revlog.__init__(self, opener, indexfile) revlog.revlog.__init__(self, opener, indexfile)
self.bundle = cgunpacker self.bundle = cgunpacker
n = len(self) n = len(self)
self.repotiprev = n - 1 self.repotiprev = n - 1
self.bundlerevs = set() # used by 'bundle()' revset expression self.bundlerevs = set() # used by 'bundle()' revset expression
for deltadata in cgunpacker.deltaiter(): for deltadata in cgunpacker.deltaiter():
node, p1, p2, cs, deltabase, delta, flags = deltadata node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
size = len(delta) size = len(delta)
start = cgunpacker.tell() - size start = cgunpacker.tell() - size
if self.index.has_node(node): if self.index.has_node(node):
# this can happen if two branches make the same change # this can happen if two branches make the same change
self.bundlerevs.add(self.index.rev(node)) self.bundlerevs.add(self.index.rev(node))
continue continue

mercurial/changegroup.py

phases, phases,
pycompat, pycompat,
requirements, requirements,
scmutil, scmutil,
util, util,
) )
from .interfaces import repository from .interfaces import repository
from .revlogutils import sidedata as sidedatamod
_CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s") _CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s")
_CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s") _CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s")
_CHANGEGROUPV3_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sH") _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sH")
LFS_REQUIREMENT = b'lfs' LFS_REQUIREMENT = b'lfs'
readexactly = util.readexactly readexactly = util.readexactly
def deltachunk(self, prevnode): def deltachunk(self, prevnode):
l = self._chunklength() l = self._chunklength()
if not l: if not l:
return {} return {}
headerdata = readexactly(self._stream, self.deltaheadersize) headerdata = readexactly(self._stream, self.deltaheadersize)
header = self.deltaheader.unpack(headerdata) header = self.deltaheader.unpack(headerdata)
delta = readexactly(self._stream, l - self.deltaheadersize) delta = readexactly(self._stream, l - self.deltaheadersize)
node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode) node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
return (node, p1, p2, cs, deltabase, delta, flags) # cg4 forward-compat
sidedata = {}
return (node, p1, p2, cs, deltabase, delta, flags, sidedata)
def getchunks(self): def getchunks(self):
"""returns all the chunks contains in the bundle """returns all the chunks contains in the bundle
Used when you need to forward the binary stream to a file or another Used when you need to forward the binary stream to a file or another
network API. To do so, it parse the changegroup data, otherwise it will network API. To do so, it parse the changegroup data, otherwise it will
block in case of sshrepo because it don't know the end of the stream. block in case of sshrepo because it don't know the end of the stream.
""" """
# If we get here, there are directory manifests in the changegroup # If we get here, there are directory manifests in the changegroup
d = chunkdata[b"filename"] d = chunkdata[b"filename"]
repo.ui.debug(b"adding %s revisions\n" % d) repo.ui.debug(b"adding %s revisions\n" % d)
deltas = self.deltaiter() deltas = self.deltaiter()
if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp): if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp):
raise error.Abort(_(b"received dir revlog group is empty")) raise error.Abort(_(b"received dir revlog group is empty"))
class cg4unpacker(cg3unpacker):
"""Unpacker for cg4 streams.
cg4 streams add support for exchanging sidedata.
"""
version = b'04'
def deltachunk(self, prevnode):
res = super(cg4unpacker, self).deltachunk(prevnode)
if not res:
return res
(node, p1, p2, cs, deltabase, delta, flags, _sidedata) = res
sidedata_raw = getchunk(self._stream)
sidedata = {}
if len(sidedata_raw) > 0:
sidedata = sidedatamod.deserialize_sidedata(sidedata_raw)
return node, p1, p2, cs, deltabase, delta, flags, sidedata
marmouteUnsubmitted
Not Done

Can you explain the 4 here ?

marmoute: Can you explain the `4` here ?
AlphareAuthorUnsubmitted
Done

I've found the getchunks function that encapsulates the logic

Alphare: I've found the `getchunks` function that encapsulates the logic
marmouteUnsubmitted
Not Done

A yep, better ☺

marmoute: A yep, better ☺
class headerlessfixup(object): class headerlessfixup(object):
def __init__(self, fh, h): def __init__(self, fh, h):
self._h = h self._h = h
self._fh = fh self._fh = fh
def read(self, n): def read(self, n):
if self._h: if self._h:
d, self._h = self._h[:n], self._h[n:] d, self._h = self._h[:n], self._h[n:]
builddeltaheader, builddeltaheader,
manifestsend, manifestsend,
forcedeltaparentprev=False, forcedeltaparentprev=False,
bundlecaps=None, bundlecaps=None,
ellipses=False, ellipses=False,
shallow=False, shallow=False,
ellipsisroots=None, ellipsisroots=None,
fullnodes=None, fullnodes=None,
remote_sidedata=None,
): ):
"""Given a source repo, construct a bundler. """Given a source repo, construct a bundler.
oldmatcher is a matcher that matches on files the client already has. oldmatcher is a matcher that matches on files the client already has.
These will not be included in the changegroup. These will not be included in the changegroup.
matcher is a matcher that matches on files to include in the matcher is a matcher that matches on files to include in the
changegroup. Used to facilitate sparse changegroups. changegroup. Used to facilitate sparse changegroups.
shallow indicates whether shallow data might be sent. The packer may shallow indicates whether shallow data might be sent. The packer may
need to pack file contents not introduced by the changes being packed. need to pack file contents not introduced by the changes being packed.
fullnodes is the set of changelog nodes which should not be ellipsis fullnodes is the set of changelog nodes which should not be ellipsis
nodes. We store this rather than the set of nodes that should be nodes. We store this rather than the set of nodes that should be
ellipsis because for very large histories we expect this to be ellipsis because for very large histories we expect this to be
significantly smaller. significantly smaller.
remote_sidedata is the set of sidedata categories wanted by the remote.
""" """
assert oldmatcher assert oldmatcher
assert matcher assert matcher
self._oldmatcher = oldmatcher self._oldmatcher = oldmatcher
self._matcher = matcher self._matcher = matcher
self.version = version self.version = version
self._forcedeltaparentprev = forcedeltaparentprev self._forcedeltaparentprev = forcedeltaparentprev
manifests, manifests,
fnodes, fnodes,
source, source,
clstate[b'clrevtomanifestrev'], clstate[b'clrevtomanifestrev'],
) )
for tree, deltas in it: for tree, deltas in it:
if tree: if tree:
assert self.version == b'03' assert self.version in (b'03', b'04')
chunk = _fileheader(tree) chunk = _fileheader(tree)
size += len(chunk) size += len(chunk)
yield chunk yield chunk
for delta in deltas: for delta in deltas:
chunks = _revisiondeltatochunks(delta, self._builddeltaheader) chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
for chunk in chunks: for chunk in chunks:
size += len(chunk) size += len(chunk)
repo, repo,
oldmatcher, oldmatcher,
matcher, matcher,
bundlecaps, bundlecaps,
ellipses=False, ellipses=False,
shallow=False, shallow=False,
ellipsisroots=None, ellipsisroots=None,
fullnodes=None, fullnodes=None,
remote_sidedata=None,
): ):
builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.linknode d.node, d.p1node, d.p2node, d.linknode
) )
return cgpacker( return cgpacker(
repo, repo,
oldmatcher, oldmatcher,
repo, repo,
oldmatcher, oldmatcher,
matcher, matcher,
bundlecaps, bundlecaps,
ellipses=False, ellipses=False,
shallow=False, shallow=False,
ellipsisroots=None, ellipsisroots=None,
fullnodes=None, fullnodes=None,
remote_sidedata=None,
): ):
builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode d.node, d.p1node, d.p2node, d.basenode, d.linknode
) )
return cgpacker( return cgpacker(
repo, repo,
oldmatcher, oldmatcher,
repo, repo,
oldmatcher, oldmatcher,
matcher, matcher,
bundlecaps, bundlecaps,
ellipses=False, ellipses=False,
shallow=False, shallow=False,
ellipsisroots=None, ellipsisroots=None,
fullnodes=None, fullnodes=None,
remote_sidedata=None,
): ):
builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
) )
return cgpacker( return cgpacker(
repo, repo,
oldmatcher, oldmatcher,
matcher, matcher,
b'03', b'03',
builddeltaheader=builddeltaheader, builddeltaheader=builddeltaheader,
manifestsend=closechunk(), manifestsend=closechunk(),
bundlecaps=bundlecaps, bundlecaps=bundlecaps,
ellipses=ellipses, ellipses=ellipses,
shallow=shallow, shallow=shallow,
ellipsisroots=ellipsisroots, ellipsisroots=ellipsisroots,
fullnodes=fullnodes, fullnodes=fullnodes,
) )
def _makecg4packer(
repo,
oldmatcher,
matcher,
bundlecaps,
ellipses=False,
shallow=False,
ellipsisroots=None,
fullnodes=None,
remote_sidedata=None,
):
# Same header func as cg3. Sidedata is in a separate chunk from the delta to
# differenciate "raw delta" and sidedata.
builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
marmouteUnsubmitted
Not Done

What is going on here ?

marmoute: What is going on here ?
AlphareAuthorUnsubmitted
Done

I've left comments (pending refresh) to be more explicit.

Alphare: I've left comments (pending refresh) to be more explicit.
d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
)
return cgpacker(
repo,
oldmatcher,
matcher,
b'04',
builddeltaheader=builddeltaheader,
manifestsend=closechunk(),
bundlecaps=bundlecaps,
ellipses=ellipses,
shallow=shallow,
ellipsisroots=ellipsisroots,
fullnodes=fullnodes,
remote_sidedata=remote_sidedata,
)
_packermap = { _packermap = {
b'01': (_makecg1packer, cg1unpacker), b'01': (_makecg1packer, cg1unpacker),
# cg2 adds support for exchanging generaldelta # cg2 adds support for exchanging generaldelta
b'02': (_makecg2packer, cg2unpacker), b'02': (_makecg2packer, cg2unpacker),
# cg3 adds support for exchanging revlog flags and treemanifests # cg3 adds support for exchanging revlog flags and treemanifests
b'03': (_makecg3packer, cg3unpacker), b'03': (_makecg3packer, cg3unpacker),
# ch4 adds support for exchanging sidedata
b'04': (_makecg4packer, cg4unpacker),
} }
def allsupportedversions(repo): def allsupportedversions(repo):
versions = set(_packermap.keys()) versions = set(_packermap.keys())
needv03 = False needv03 = False
if ( if (
repo.ui.configbool(b'experimental', b'changegroup3') repo.ui.configbool(b'experimental', b'changegroup3')
or repo.ui.configbool(b'experimental', b'treemanifest') or repo.ui.configbool(b'experimental', b'treemanifest')
or scmutil.istreemanifest(repo) or scmutil.istreemanifest(repo)
): ):
# we keep version 03 because we need to to exchange treemanifest data # we keep version 03 because we need to to exchange treemanifest data
# #
# we also keep vresion 01 and 02, because it is possible for repo to # we also keep vresion 01 and 02, because it is possible for repo to
# contains both normal and tree manifest at the same time. so using # contains both normal and tree manifest at the same time. so using
# older version to pull data is viable # older version to pull data is viable
# #
# (or even to push subset of history) # (or even to push subset of history)
needv03 = True needv03 = True
if b'exp-sidedata-flag' in repo.requirements: has_revlogv2 = requirements.REVLOGV2_REQUIREMENT in repo.requirements
needv03 = True if not has_revlogv2:
# don't attempt to use 01/02 until we do sidedata cleaning versions.discard(b'04')
versions.discard(b'01')
versions.discard(b'02')
if not needv03: if not needv03:
versions.discard(b'03') versions.discard(b'03')
return versions return versions
# Changegroup versions that can be applied to the repo # Changegroup versions that can be applied to the repo
def supportedincomingversions(repo): def supportedincomingversions(repo):
return allsupportedversions(repo) return allsupportedversions(repo)
repo, repo,
bundlecaps=None, bundlecaps=None,
oldmatcher=None, oldmatcher=None,
matcher=None, matcher=None,
ellipses=False, ellipses=False,
shallow=False, shallow=False,
ellipsisroots=None, ellipsisroots=None,
fullnodes=None, fullnodes=None,
remote_sidedata=None,
): ):
assert version in supportedoutgoingversions(repo) assert version in supportedoutgoingversions(repo)
if matcher is None: if matcher is None:
matcher = matchmod.always() matcher = matchmod.always()
if oldmatcher is None: if oldmatcher is None:
oldmatcher = matchmod.never() oldmatcher = matchmod.never()
repo, repo,
oldmatcher, oldmatcher,
matcher, matcher,
bundlecaps, bundlecaps,
ellipses=ellipses, ellipses=ellipses,
shallow=shallow, shallow=shallow,
ellipsisroots=ellipsisroots, ellipsisroots=ellipsisroots,
fullnodes=fullnodes, fullnodes=fullnodes,
remote_sidedata=remote_sidedata,
) )
def getunbundler(version, fh, alg, extras=None): def getunbundler(version, fh, alg, extras=None):
return _packermap[version][1](fh, alg, extras=extras) return _packermap[version][1](fh, alg, extras=extras)
def _changegroupinfo(repo, nodes, source): def _changegroupinfo(repo, nodes, source):
def makestream( def makestream(
repo, repo,
outgoing, outgoing,
version, version,
source, source,
fastpath=False, fastpath=False,
bundlecaps=None, bundlecaps=None,
matcher=None, matcher=None,
remote_sidedata=None,
): ):
bundler = getbundler(version, repo, bundlecaps=bundlecaps, matcher=matcher) bundler = getbundler(
version,
repo,
bundlecaps=bundlecaps,
matcher=matcher,
remote_sidedata=remote_sidedata,
)
repo = repo.unfiltered() repo = repo.unfiltered()
commonrevs = outgoing.common commonrevs = outgoing.common
csets = outgoing.missing csets = outgoing.missing
heads = outgoing.ancestorsof heads = outgoing.ancestorsof
# We go through the fast path if we get told to, or if all (unfiltered # We go through the fast path if we get told to, or if all (unfiltered
# heads have been requested (since we then know there all linkrevs will # heads have been requested (since we then know there all linkrevs will
# be pulled by the client). # be pulled by the client).

mercurial/debugcommands.py

ui.writenoi18n( ui.writenoi18n(
b"%sformat: id, p1, p2, cset, delta base, len(delta)\n" b"%sformat: id, p1, p2, cset, delta base, len(delta)\n"
% indent_string % indent_string
) )
def showchunks(named): def showchunks(named):
ui.write(b"\n%s%s\n" % (indent_string, named)) ui.write(b"\n%s%s\n" % (indent_string, named))
for deltadata in gen.deltaiter(): for deltadata in gen.deltaiter():
node, p1, p2, cs, deltabase, delta, flags = deltadata node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
ui.write( ui.write(
b"%s%s %s %s %s %s %d\n" b"%s%s %s %s %s %s %d\n"
% ( % (
indent_string, indent_string,
hex(node), hex(node),
hex(p1), hex(p1),
hex(p2), hex(p2),
hex(cs), hex(cs),
for chunkdata in iter(gen.filelogheader, {}): for chunkdata in iter(gen.filelogheader, {}):
fname = chunkdata[b'filename'] fname = chunkdata[b'filename']
showchunks(fname) showchunks(fname)
else: else:
if isinstance(gen, bundle2.unbundle20): if isinstance(gen, bundle2.unbundle20):
raise error.Abort(_(b'use debugbundle2 for this file')) raise error.Abort(_(b'use debugbundle2 for this file'))
gen.changelogheader() gen.changelogheader()
for deltadata in gen.deltaiter(): for deltadata in gen.deltaiter():
node, p1, p2, cs, deltabase, delta, flags = deltadata node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
ui.write(b"%s%s\n" % (indent_string, hex(node))) ui.write(b"%s%s\n" % (indent_string, hex(node)))
def _debugobsmarkers(ui, part, indent=0, **opts): def _debugobsmarkers(ui, part, indent=0, **opts):
"""display version and markers contained in 'data'""" """display version and markers contained in 'data'"""
opts = pycompat.byteskwargs(opts) opts = pycompat.byteskwargs(opts)
data = part.read() data = part.read()
indent_string = b' ' * indent indent_string = b' ' * indent

mercurial/exchange.py

def bundle2requested(bundlecaps): def bundle2requested(bundlecaps):
if bundlecaps is not None: if bundlecaps is not None:
return any(cap.startswith(b'HG2') for cap in bundlecaps) return any(cap.startswith(b'HG2') for cap in bundlecaps)
return False return False
def getbundlechunks( def getbundlechunks(
repo, source, heads=None, common=None, bundlecaps=None, **kwargs repo,
source,
heads=None,
common=None,
bundlecaps=None,
remote_sidedata=None,
**kwargs
): ):
"""Return chunks constituting a bundle's raw data. """Return chunks constituting a bundle's raw data.
Could be a bundle HG10 or a bundle HG20 depending on bundlecaps Could be a bundle HG10 or a bundle HG20 depending on bundlecaps
passed. passed.
Returns a 2-tuple of a dict with metadata about the generated bundle Returns a 2-tuple of a dict with metadata about the generated bundle
and an iterator over raw chunks (of varying sizes). and an iterator over raw chunks (of varying sizes).
_(b'unsupported getbundle arguments: %s') _(b'unsupported getbundle arguments: %s')
% b', '.join(sorted(kwargs.keys())) % b', '.join(sorted(kwargs.keys()))
) )
outgoing = _computeoutgoing(repo, heads, common) outgoing = _computeoutgoing(repo, heads, common)
info[b'bundleversion'] = 1 info[b'bundleversion'] = 1
return ( return (
info, info,
changegroup.makestream( changegroup.makestream(
repo, outgoing, b'01', source, bundlecaps=bundlecaps repo,
outgoing,
b'01',
source,
bundlecaps=bundlecaps,
remote_sidedata=remote_sidedata,
), ),
) )
# bundle20 case # bundle20 case
info[b'bundleversion'] = 2 info[b'bundleversion'] = 2
b2caps = {} b2caps = {}
for bcaps in bundlecaps: for bcaps in bundlecaps:
if bcaps.startswith(b'bundle2='): if bcaps.startswith(b'bundle2='):
blob = urlreq.unquote(bcaps[len(b'bundle2=') :]) blob = urlreq.unquote(bcaps[len(b'bundle2=') :])
b2caps.update(bundle2.decodecaps(blob)) b2caps.update(bundle2.decodecaps(blob))
bundler = bundle2.bundle20(repo.ui, b2caps) bundler = bundle2.bundle20(repo.ui, b2caps)
kwargs[b'heads'] = heads kwargs[b'heads'] = heads
kwargs[b'common'] = common kwargs[b'common'] = common
for name in getbundle2partsorder: for name in getbundle2partsorder:
func = getbundle2partsmapping[name] func = getbundle2partsmapping[name]
func( func(
bundler, bundler,
repo, repo,
source, source,
bundlecaps=bundlecaps, bundlecaps=bundlecaps,
b2caps=b2caps, b2caps=b2caps,
remote_sidedata=remote_sidedata,
**pycompat.strkwargs(kwargs) **pycompat.strkwargs(kwargs)
) )
info[b'prefercompressed'] = bundler.prefercompressed info[b'prefercompressed'] = bundler.prefercompressed
return info, bundler.getchunks() return info, bundler.getchunks()
@getbundle2partsgenerator(b'stream2') @getbundle2partsgenerator(b'stream2')
def _getbundlestream2(bundler, repo, *args, **kwargs): def _getbundlestream2(bundler, repo, *args, **kwargs):
return bundle2.addpartbundlestream2(bundler, repo, **kwargs) return bundle2.addpartbundlestream2(bundler, repo, **kwargs)
@getbundle2partsgenerator(b'changegroup') @getbundle2partsgenerator(b'changegroup')
def _getbundlechangegrouppart( def _getbundlechangegrouppart(
bundler, bundler,
repo, repo,
source, source,
bundlecaps=None, bundlecaps=None,
b2caps=None, b2caps=None,
heads=None, heads=None,
common=None, common=None,
remote_sidedata=None,
**kwargs **kwargs
): ):
"""add a changegroup part to the requested bundle""" """add a changegroup part to the requested bundle"""
if not kwargs.get('cg', True) or not b2caps: if not kwargs.get('cg', True) or not b2caps:
return return
version = b'01' version = b'01'
cgversions = b2caps.get(b'changegroup') cgversions = b2caps.get(b'changegroup')
if kwargs.get('narrow', False): if kwargs.get('narrow', False):
include = sorted(filter(bool, kwargs.get('includepats', []))) include = sorted(filter(bool, kwargs.get('includepats', [])))
exclude = sorted(filter(bool, kwargs.get('excludepats', []))) exclude = sorted(filter(bool, kwargs.get('excludepats', [])))
matcher = narrowspec.match(repo.root, include=include, exclude=exclude) matcher = narrowspec.match(repo.root, include=include, exclude=exclude)
else: else:
matcher = None matcher = None
cgstream = changegroup.makestream( cgstream = changegroup.makestream(
repo, outgoing, version, source, bundlecaps=bundlecaps, matcher=matcher repo,
outgoing,
version,
source,
bundlecaps=bundlecaps,
matcher=matcher,
remote_sidedata=remote_sidedata,
) )
part = bundler.newpart(b'changegroup', data=cgstream) part = bundler.newpart(b'changegroup', data=cgstream)
if cgversions: if cgversions:
part.addparam(b'version', version) part.addparam(b'version', version)
part.addparam(b'nbchanges', b'%d' % len(outgoing.missing), mandatory=False) part.addparam(b'nbchanges', b'%d' % len(outgoing.missing), mandatory=False)

mercurial/exchangev2.py

cset[b'parents'][1], cset[b'parents'][1],
# Linknode is always itself for changesets. # Linknode is always itself for changesets.
cset[b'node'], cset[b'node'],
# We always send full revisions. So delta base is not set. # We always send full revisions. So delta base is not set.
nullid, nullid,
mdiff.trivialdiffheader(len(data)) + data, mdiff.trivialdiffheader(len(data)) + data,
# Flags not yet supported. # Flags not yet supported.
0, 0,
# Sidedata not yet supported
{},
) )
cl.addgroup( cl.addgroup(
iterrevisions(), iterrevisions(),
linkrev, linkrev,
weakref.proxy(tr), weakref.proxy(tr),
alwayscache=True, alwayscache=True,
addrevisioncb=onchangeset, addrevisioncb=onchangeset,
# changelog revision number. So we just pass in the # changelog revision number. So we just pass in the
# manifest node here and use linkrevs.__getitem__ as the # manifest node here and use linkrevs.__getitem__ as the
# resolution function. # resolution function.
node, node,
basenode, basenode,
delta, delta,
# Flags not yet supported. # Flags not yet supported.
0, 0,
# Sidedata not yet supported.
{},
) )
progress.increment() progress.increment()
progress = repo.ui.makeprogress( progress = repo.ui.makeprogress(
_(b'manifests'), unit=_(b'chunks'), total=len(fetchnodes) _(b'manifests'), unit=_(b'chunks'), total=len(fetchnodes)
) )
node, node,
filerevision[b'parents'][0], filerevision[b'parents'][0],
filerevision[b'parents'][1], filerevision[b'parents'][1],
node, node,
basenode, basenode,
delta, delta,
# Flags not yet supported. # Flags not yet supported.
0, 0,
# Sidedata not yet supported.
{},
) )
progress.increment() progress.increment()
progress = repo.ui.makeprogress( progress = repo.ui.makeprogress(
_(b'files'), _(b'files'),
unit=_(b'chunks'), unit=_(b'chunks'),
total=sum(len(v) for v in pycompat.itervalues(fnodes)), total=sum(len(v) for v in pycompat.itervalues(fnodes)),
node, node,
filerevision[b'parents'][0], filerevision[b'parents'][0],
filerevision[b'parents'][1], filerevision[b'parents'][1],
linknode, linknode,
basenode, basenode,
delta, delta,
# Flags not yet supported. # Flags not yet supported.
0, 0,
# Sidedata not yet supported.
{},
) )
progress.increment() progress.increment()
remaining -= 1 remaining -= 1
progress = repo.ui.makeprogress( progress = repo.ui.makeprogress(
_(b'files'), _(b'files'),
unit=_(b'chunks'), unit=_(b'chunks'),

mercurial/localrepo.py

one, one,
two, two,
pycompat.bytestr(three), pycompat.bytestr(three),
pycompat.bytestr(four), pycompat.bytestr(four),
pycompat.bytestr(five), pycompat.bytestr(five),
) )
def getbundle( def getbundle(
self, source, heads=None, common=None, bundlecaps=None, **kwargs self,
source,
heads=None,
common=None,
bundlecaps=None,
remote_sidedata=None,
**kwargs
): ):
chunks = exchange.getbundlechunks( chunks = exchange.getbundlechunks(
self._repo, self._repo,
source, source,
heads=heads, heads=heads,
common=common, common=common,
bundlecaps=bundlecaps, bundlecaps=bundlecaps,
remote_sidedata=remote_sidedata,
**kwargs **kwargs
)[1] )[1]
cb = util.chunkbuffer(chunks) cb = util.chunkbuffer(chunks)
if exchange.bundle2requested(bundlecaps): if exchange.bundle2requested(bundlecaps):
# When requesting a bundle2, getbundle returns a stream to make the # When requesting a bundle2, getbundle returns a stream to make the
# wire level function happier. We need to build a proper object # wire level function happier. We need to build a proper object
# from it in local peer. # from it in local peer.

mercurial/revlog.py

self._writinghandles = (ifh, dfh) self._writinghandles = (ifh, dfh)
empty = True empty = True
try: try:
deltacomputer = deltautil.deltacomputer(self) deltacomputer = deltautil.deltacomputer(self)
# loop through our set of deltas # loop through our set of deltas
for data in deltas: for data in deltas:
node, p1, p2, linknode, deltabase, delta, flags = data node, p1, p2, linknode, deltabase, delta, flags, sidedata = data
link = linkmapper(linknode) link = linkmapper(linknode)
flags = flags or REVIDX_DEFAULT_FLAGS flags = flags or REVIDX_DEFAULT_FLAGS
rev = self.index.get_rev(node) rev = self.index.get_rev(node)
if rev is not None: if rev is not None:
# this can happen if two branches make the same change # this can happen if two branches make the same change
self._nodeduplicatecallback(transaction, rev) self._nodeduplicatecallback(transaction, rev)
if duplicaterevisioncb: if duplicaterevisioncb:

tests/test-check-interfaces.py

node=b'', node=b'',
p1node=b'', p1node=b'',
p2node=b'', p2node=b'',
basenode=b'', basenode=b'',
linknode=b'', linknode=b'',
flags=b'', flags=b'',
baserevisionsize=None, baserevisionsize=None,
revision=b'', revision=b'',
sidedata=b'',
delta=None, delta=None,
) )
checkzobject(rd) checkzobject(rd)
ziverify.verifyClass(repository.iverifyproblem, revlog.revlogproblem) ziverify.verifyClass(repository.iverifyproblem, revlog.revlogproblem)
checkzobject(revlog.revlogproblem()) checkzobject(revlog.revlogproblem())
main() main()

tests/test-revlog-raw.py

return { return {
b'node': rlog.node(r), b'node': rlog.node(r),
b'p1': pnode, b'p1': pnode,
b'p2': nullid, b'p2': nullid,
b'cs': rlog.node(rlog.linkrev(r)), b'cs': rlog.node(rlog.linkrev(r)),
b'flags': rlog.flags(r), b'flags': rlog.flags(r),
b'deltabase': rlog.node(deltaparent), b'deltabase': rlog.node(deltaparent),
b'delta': rlog.revdiff(deltaparent, r), b'delta': rlog.revdiff(deltaparent, r),
b'sidedata': rlog.sidedata(r),
} }
def deltaiter(self): def deltaiter(self):
chain = None chain = None
for chunkdata in iter(lambda: self.deltachunk(chain), {}): for chunkdata in iter(lambda: self.deltachunk(chain), {}):
node = chunkdata[b'node'] node = chunkdata[b'node']
p1 = chunkdata[b'p1'] p1 = chunkdata[b'p1']
p2 = chunkdata[b'p2'] p2 = chunkdata[b'p2']
cs = chunkdata[b'cs'] cs = chunkdata[b'cs']
deltabase = chunkdata[b'deltabase'] deltabase = chunkdata[b'deltabase']
delta = chunkdata[b'delta'] delta = chunkdata[b'delta']
flags = chunkdata[b'flags'] flags = chunkdata[b'flags']
sidedata = chunkdata[b'sidedata']
chain = node chain = node
yield (node, p1, p2, cs, deltabase, delta, flags) yield (node, p1, p2, cs, deltabase, delta, flags, sidedata)
def linkmap(lnode): def linkmap(lnode):
return rlog.rev(lnode) return rlog.rev(lnode)
dlog = newrevlog(destname, recreate=True) dlog = newrevlog(destname, recreate=True)
dummydeltas = dummychangegroup().deltaiter() dummydeltas = dummychangegroup().deltaiter()
dlog.addgroup(dummydeltas, linkmap, tr) dlog.addgroup(dummydeltas, linkmap, tr)
return dlog return dlog