This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4078

changegroup: make delta header struct formatters actual structs
ClosedPublic

Authored by indygreg on Aug 3 2018, 5:04 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG271854adc3a6: changegroup: make delta header struct formatters actual structs
Summary

Why we weren't using compiled Struct instances, I don't know. They
make code simpler. In theory they are faster. Although I don't
believe I was able to measure any meaningful change. That could be
because this code is often dominated by compression, deltafication,
and function call overhead.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

PathPackages
Mmercurial/changegroup.py (21 lines)

Diff 9954

mercurial/changegroup.py

revlog, revlog,
util, util,
) )
from .utils import ( from .utils import (
stringutil, stringutil,
) )
_CHANGEGROUPV1_DELTA_HEADER = "20s20s20s20s" _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
_CHANGEGROUPV2_DELTA_HEADER = "20s20s20s20s20s" _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
_CHANGEGROUPV3_DELTA_HEADER = ">20s20s20s20s20sH" _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
LFS_REQUIREMENT = 'lfs' LFS_REQUIREMENT = 'lfs'
readexactly = util.readexactly readexactly = util.readexactly
def getchunk(stream): def getchunk(stream):
"""return the next chunk from stream as a string""" """return the next chunk from stream as a string"""
d = readexactly(stream, 4) d = readexactly(stream, 4)
deltachunk() is useful only if you're applying delta data. Most deltachunk() is useful only if you're applying delta data. Most
consumers should prefer apply() instead. consumers should prefer apply() instead.
A few other public methods exist. Those are used only for A few other public methods exist. Those are used only for
bundlerepo and some debug commands - their use is discouraged. bundlerepo and some debug commands - their use is discouraged.
""" """
deltaheader = _CHANGEGROUPV1_DELTA_HEADER deltaheader = _CHANGEGROUPV1_DELTA_HEADER
deltaheadersize = struct.calcsize(deltaheader) deltaheadersize = deltaheader.size
version = '01' version = '01'
_grouplistcount = 1 # One list of files after the manifests _grouplistcount = 1 # One list of files after the manifests
def __init__(self, fh, alg, extras=None): def __init__(self, fh, alg, extras=None):
if alg is None: if alg is None:
alg = 'UN' alg = 'UN'
if alg not in util.compengines.supportedbundletypes: if alg not in util.compengines.supportedbundletypes:
raise error.Abort(_('unknown stream compression type: %s') raise error.Abort(_('unknown stream compression type: %s')
flags = 0 flags = 0
return node, p1, p2, deltabase, cs, flags return node, p1, p2, deltabase, cs, flags
def deltachunk(self, prevnode): def deltachunk(self, prevnode):
l = self._chunklength() l = self._chunklength()
if not l: if not l:
return {} return {}
headerdata = readexactly(self._stream, self.deltaheadersize) headerdata = readexactly(self._stream, self.deltaheadersize)
header = struct.unpack(self.deltaheader, headerdata) header = self.deltaheader.unpack(headerdata)
delta = readexactly(self._stream, l - self.deltaheadersize) delta = readexactly(self._stream, l - self.deltaheadersize)
node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode) node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
return (node, p1, p2, cs, deltabase, delta, flags) return (node, p1, p2, cs, deltabase, delta, flags)
def getchunks(self): def getchunks(self):
"""returns all the chunks contains in the bundle """returns all the chunks contains in the bundle
Used when you need to forward the binary stream to a file or another Used when you need to forward the binary stream to a file or another
class cg2unpacker(cg1unpacker): class cg2unpacker(cg1unpacker):
"""Unpacker for cg2 streams. """Unpacker for cg2 streams.
cg2 streams add support for generaldelta, so the delta header cg2 streams add support for generaldelta, so the delta header
format is slightly different. All other features about the data format is slightly different. All other features about the data
remain the same. remain the same.
""" """
deltaheader = _CHANGEGROUPV2_DELTA_HEADER deltaheader = _CHANGEGROUPV2_DELTA_HEADER
deltaheadersize = struct.calcsize(deltaheader) deltaheadersize = deltaheader.size
version = '02' version = '02'
def _deltaheader(self, headertuple, prevnode): def _deltaheader(self, headertuple, prevnode):
node, p1, p2, deltabase, cs = headertuple node, p1, p2, deltabase, cs = headertuple
flags = 0 flags = 0
return node, p1, p2, deltabase, cs, flags return node, p1, p2, deltabase, cs, flags
class cg3unpacker(cg2unpacker): class cg3unpacker(cg2unpacker):
"""Unpacker for cg3 streams. """Unpacker for cg3 streams.
cg3 streams add support for exchanging treemanifests and revlog cg3 streams add support for exchanging treemanifests and revlog
flags. It adds the revlog flags to the delta header and an empty chunk flags. It adds the revlog flags to the delta header and an empty chunk
separating manifests and files. separating manifests and files.
""" """
deltaheader = _CHANGEGROUPV3_DELTA_HEADER deltaheader = _CHANGEGROUPV3_DELTA_HEADER
deltaheadersize = struct.calcsize(deltaheader) deltaheadersize = deltaheader.size
version = '03' version = '03'
_grouplistcount = 2 # One list of manifests and one list of files _grouplistcount = 2 # One list of manifests and one list of files
def _deltaheader(self, headertuple, prevnode): def _deltaheader(self, headertuple, prevnode):
node, p1, p2, deltabase, cs, flags = headertuple node, p1, p2, deltabase, cs, flags = headertuple
return node, p1, p2, deltabase, cs, flags return node, p1, p2, deltabase, cs, flags
def _unpackmanifests(self, repo, revmap, trp, prog): def _unpackmanifests(self, repo, revmap, trp, prog):
linknode=linknode, linknode=linknode,
flags=flags, flags=flags,
deltachunks=(diffheader, data), deltachunks=(diffheader, data),
) )
def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
# do nothing with basenode, it is implicitly the previous one in HG10 # do nothing with basenode, it is implicitly the previous one in HG10
# do nothing with flags, it is implicitly 0 for cg1 and cg2 # do nothing with flags, it is implicitly 0 for cg1 and cg2
return struct.pack(self.deltaheader, node, p1n, p2n, linknode) return self.deltaheader.pack(node, p1n, p2n, linknode)
class cg2packer(cg1packer): class cg2packer(cg1packer):
deltaheader = _CHANGEGROUPV2_DELTA_HEADER deltaheader = _CHANGEGROUPV2_DELTA_HEADER
def __init__(self, repo, filematcher, version, bundlecaps=None): def __init__(self, repo, filematcher, version, bundlecaps=None):
super(cg2packer, self).__init__(repo, filematcher, version, super(cg2packer, self).__init__(repo, filematcher, version,
bundlecaps=bundlecaps) bundlecaps=bundlecaps)
else: else:
base = dp base = dp
if base != nullrev and not store.candelta(base, rev): if base != nullrev and not store.candelta(base, rev):
base = nullrev base = nullrev
return base return base
def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
# Do nothing with flags, it is implicitly 0 in cg1 and cg2 # Do nothing with flags, it is implicitly 0 in cg1 and cg2
return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode) return self.deltaheader.pack(node, p1n, p2n, basenode, linknode)
class cg3packer(cg2packer): class cg3packer(cg2packer):
deltaheader = _CHANGEGROUPV3_DELTA_HEADER deltaheader = _CHANGEGROUPV3_DELTA_HEADER
def _packmanifests(self, dir, mfnodes, lookuplinknode): def _packmanifests(self, dir, mfnodes, lookuplinknode):
if dir: if dir:
yield self.fileheader(dir) yield self.fileheader(dir)
dirlog = self._repo.manifestlog._revlog.dirlog(dir) dirlog = self._repo.manifestlog._revlog.dirlog(dir)
for chunk in self.group(mfnodes, dirlog, lookuplinknode, for chunk in self.group(mfnodes, dirlog, lookuplinknode,
units=_('manifests')): units=_('manifests')):
yield chunk yield chunk
def _manifestsdone(self): def _manifestsdone(self):
return self.close() return self.close()
def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
return struct.pack( return self.deltaheader.pack(node, p1n, p2n, basenode, linknode, flags)
self.deltaheader, node, p1n, p2n, basenode, linknode, flags)
def _makecg1packer(repo, filematcher, bundlecaps): def _makecg1packer(repo, filematcher, bundlecaps):
return cg1packer(repo, filematcher, b'01', bundlecaps=bundlecaps) return cg1packer(repo, filematcher, b'01', bundlecaps=bundlecaps)
def _makecg2packer(repo, filematcher, bundlecaps): def _makecg2packer(repo, filematcher, bundlecaps):
return cg2packer(repo, filematcher, b'02', bundlecaps=bundlecaps) return cg2packer(repo, filematcher, b'02', bundlecaps=bundlecaps)
def _makecg3packer(repo, filematcher, bundlecaps): def _makecg3packer(repo, filematcher, bundlecaps):