This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6942

upgrade: allow upgrade to repository using sidedata
ClosedPublic

Authored by marmoute on Oct 1 2019, 12:07 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG75ad8af9c95e: upgrade: allow upgrade to repository using sidedata
Summary

Repository can now be migrated to support sidedata. More requirements and
migration will be needed to actual side-data usage. This is a step in that
direction.

To test the feature, we leverage the test extension. It make sure the update
part of the side-data migration actually works.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

marmoute created this revision.Oct 1 2019, 12:07 PM
marmoute updated this revision to Diff 16773.Oct 2 2019, 12:08 AM
marmoute updated this revision to Diff 16946.Oct 7 2019, 8:36 PM
marmoute added a commit: rHG75ad8af9c95e: upgrade: allow upgrade to repository using sidedata.Oct 9 2019, 12:46 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG75ad8af9c95e: upgrade: allow upgrade to repository using sidedata (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 16773

mercurial/upgrade.py

from .utils import ( from .utils import (
compression, compression,
) )
# list of requirements that request a clone of all revlog if added/removed # list of requirements that request a clone of all revlog if added/removed
RECLONES_REQUIREMENTS = { RECLONES_REQUIREMENTS = {
'generaldelta', 'generaldelta',
localrepo.SPARSEREVLOG_REQUIREMENT, localrepo.SPARSEREVLOG_REQUIREMENT,
localrepo.SIDEDATA_REQUIREMENT,
} }
def requiredsourcerequirements(repo): def requiredsourcerequirements(repo):
"""Obtain requirements required to be present to upgrade a repo. """Obtain requirements required to be present to upgrade a repo.
An upgrade will not be allowed if the repository doesn't have the An upgrade will not be allowed if the repository doesn't have the
requirements returned by this function. requirements returned by this function.
""" """
""" """
supported = { supported = {
'dotencode', 'dotencode',
'fncache', 'fncache',
'generaldelta', 'generaldelta',
'revlogv1', 'revlogv1',
'store', 'store',
localrepo.SPARSEREVLOG_REQUIREMENT, localrepo.SPARSEREVLOG_REQUIREMENT,
localrepo.SIDEDATA_REQUIREMENT,
} }
for name in compression.compengines: for name in compression.compengines:
engine = compression.compengines[name] engine = compression.compengines[name]
if engine.available() and engine.revlogheader(): if engine.available() and engine.revlogheader():
supported.add(b'exp-compression-%s' % name) supported.add(b'exp-compression-%s' % name)
if engine.name() == 'zstd': if engine.name() == 'zstd':
supported.add(b'revlog-compression-zstd') supported.add(b'revlog-compression-zstd')
return supported return supported
def allowednewrequirements(repo): def allowednewrequirements(repo):
"""Obtain requirements that can be added to a repository during upgrade. """Obtain requirements that can be added to a repository during upgrade.
This is used to disallow proposed requirements from being added when This is used to disallow proposed requirements from being added when
they weren't present before. they weren't present before.
We use a list of allowed requirement additions instead of a list of known We use a list of allowed requirement additions instead of a list of known
bad additions because the whitelist approach is safer and will prevent bad additions because the whitelist approach is safer and will prevent
future, unknown requirements from accidentally being added. future, unknown requirements from accidentally being added.
""" """
supported = { supported = {
'dotencode', 'dotencode',
'fncache', 'fncache',
'generaldelta', 'generaldelta',
localrepo.SPARSEREVLOG_REQUIREMENT, localrepo.SPARSEREVLOG_REQUIREMENT,
localrepo.SIDEDATA_REQUIREMENT,
} }
for name in compression.compengines: for name in compression.compengines:
engine = compression.compengines[name] engine = compression.compengines[name]
if engine.available() and engine.revlogheader(): if engine.available() and engine.revlogheader():
supported.add(b'exp-compression-%s' % name) supported.add(b'exp-compression-%s' % name)
if engine.name() == 'zstd': if engine.name() == 'zstd':
supported.add(b'revlog-compression-zstd') supported.add(b'revlog-compression-zstd')
return supported return supported
UPGRADE_CHANGELOG = object() UPGRADE_CHANGELOG = object()
UPGRADE_MANIFEST = object() UPGRADE_MANIFEST = object()
UPGRADE_FILELOG = object() UPGRADE_FILELOG = object()
UPGRADE_ALL_REVLOGS = frozenset([UPGRADE_CHANGELOG, UPGRADE_ALL_REVLOGS = frozenset([UPGRADE_CHANGELOG,
UPGRADE_MANIFEST, UPGRADE_MANIFEST,
UPGRADE_FILELOG]) UPGRADE_FILELOG])
def getsidedatacompanion(srcrepo, destrepo):
return None
def matchrevlog(revlogfilter, entry): def matchrevlog(revlogfilter, entry):
"""check is a revlog is selected for cloning """check is a revlog is selected for cloning
The store entry is checked against the passed filter""" The store entry is checked against the passed filter"""
if entry.endswith('00changelog.i'): if entry.endswith('00changelog.i'):
return UPGRADE_CHANGELOG in revlogfilter return UPGRADE_CHANGELOG in revlogfilter
elif entry.endswith('00manifest.i'): elif entry.endswith('00manifest.i'):
return UPGRADE_MANIFEST in revlogfilter return UPGRADE_MANIFEST in revlogfilter
ui.write(_('migrating %s in store; %s tracked data\n') % ( ui.write(_('migrating %s in store; %s tracked data\n') % (
(util.bytecount(srcsize), util.bytecount(srcrawsize)))) (util.bytecount(srcsize), util.bytecount(srcrawsize))))
# Used to keep track of progress. # Used to keep track of progress.
progress = None progress = None
def oncopiedrevision(rl, rev, node): def oncopiedrevision(rl, rev, node):
progress.increment() progress.increment()
sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo)
# Do the actual copying. # Do the actual copying.
# FUTURE this operation can be farmed off to worker processes. # FUTURE this operation can be farmed off to worker processes.
seen = set() seen = set()
for unencoded, encoded, size in alldatafiles: for unencoded, encoded, size in alldatafiles:
if unencoded.endswith('.d'): if unencoded.endswith('.d'):
continue continue
oldrl = _revlogfrompath(srcrepo, unencoded) oldrl = _revlogfrompath(srcrepo, unencoded)
total=frevcount) total=frevcount)
if matchrevlog(revlogs, unencoded): if matchrevlog(revlogs, unencoded):
ui.note(_('cloning %d revisions from %s\n') ui.note(_('cloning %d revisions from %s\n')
% (len(oldrl), unencoded)) % (len(oldrl), unencoded))
newrl = _revlogfrompath(dstrepo, unencoded) newrl = _revlogfrompath(dstrepo, unencoded)
oldrl.clone(tr, newrl, addrevisioncb=oncopiedrevision, oldrl.clone(tr, newrl, addrevisioncb=oncopiedrevision,
deltareuse=deltareuse, deltareuse=deltareuse,
forcedeltabothparents=forcedeltabothparents) forcedeltabothparents=forcedeltabothparents,
sidedatacompanion=sidedatacompanion)
else: else:
msg = _('blindly copying %s containing %i revisions\n') msg = _('blindly copying %s containing %i revisions\n')
ui.note(msg % (unencoded, len(oldrl))) ui.note(msg % (unencoded, len(oldrl)))
_copyrevlog(tr, dstrepo, oldrl, unencoded) _copyrevlog(tr, dstrepo, oldrl, unencoded)
newrl = _revlogfrompath(dstrepo, unencoded) newrl = _revlogfrompath(dstrepo, unencoded)
info = newrl.storageinfo(storedsize=True) info = newrl.storageinfo(storedsize=True)

tests/test-sidedata.t

entry-0001 size 4 entry-0001 size 4
entry-0002 size 32 entry-0002 size 32
Check upgrade behavior Check upgrade behavior
====================== ======================
Right now, sidedata has not upgrade support Right now, sidedata has not upgrade support
Check that we cannot upgrade to sidedata Check that we can upgrade to sidedata
---------------------------------------- -------------------------------------
$ hg init up-no-side-data --config format.use-side-data=no $ hg init up-no-side-data --config format.use-side-data=no
$ hg debugformat -v -R up-no-side-data $ hg debugformat -v -R up-no-side-data
format-variant repo config default format-variant repo config default
fncache: yes yes yes fncache: yes yes yes
dotencode: yes yes yes dotencode: yes yes yes
generaldelta: yes yes yes generaldelta: yes yes yes
sparserevlog: yes yes yes sparserevlog: yes yes yes
sidedata: no no no sidedata: no no no
plain-cl-delta: yes yes yes plain-cl-delta: yes yes yes
compression: zlib zlib zlib compression: zlib zlib zlib
compression-level: default default default compression-level: default default default
$ hg debugformat -v -R up-no-side-data --config format.use-side-data=yes $ hg debugformat -v -R up-no-side-data --config format.use-side-data=yes
format-variant repo config default format-variant repo config default
fncache: yes yes yes fncache: yes yes yes
dotencode: yes yes yes dotencode: yes yes yes
generaldelta: yes yes yes generaldelta: yes yes yes
sparserevlog: yes yes yes sparserevlog: yes yes yes
sidedata: no yes no sidedata: no yes no
plain-cl-delta: yes yes yes plain-cl-delta: yes yes yes
compression: zlib zlib zlib compression: zlib zlib zlib
compression-level: default default default compression-level: default default default
$ hg debugupgraderepo -R up-no-side-data --config format.use-side-data=yes $ hg debugupgraderepo -R up-no-side-data --config format.use-side-data=yes > /dev/null
abort: cannot upgrade repository; do not support adding requirement: exp-sidedata-flag
[255]
Check that we cannot upgrade to sidedata Check that we cannot upgrade to sidedata
---------------------------------------- ----------------------------------------
$ hg init up-side-data --config format.use-side-data=yes $ hg init up-side-data --config format.use-side-data=yes
$ hg debugformat -v -R up-side-data $ hg debugformat -v -R up-side-data
format-variant repo config default format-variant repo config default
fncache: yes yes yes fncache: yes yes yes

tests/test-upgrade-repo.t

dotencode dotencode
fncache fncache
generaldelta generaldelta
revlog-compression-zstd revlog-compression-zstd
revlogv1 revlogv1
sparserevlog sparserevlog
store store
$ cd ..
#endif #endif
Check upgrading to a side-data revlog
-------------------------------------
upgrade
$ hg --config format.use-side-data=yes debugupgraderepo --run --no-backup --config "extensions.sidedata=$TESTDIR/testlib/ext-sidedata.py" >/dev/null
$ hg debugformat -v
format-variant repo config default
fncache: yes yes yes
dotencode: yes yes yes
generaldelta: yes yes yes
sparserevlog: yes yes yes
sidedata: yes no no
plain-cl-delta: yes yes yes
compression: zstd zstd zlib
compression-level: default default default
$ cat .hg/requires
dotencode
exp-sidedata-flag
fncache
generaldelta
revlog-compression-zstd
revlogv1
sparserevlog
store
$ hg debugsidedata -c 0
2 sidedata entries
entry-0001 size 4
entry-0002 size 32

tests/testlib/ext-sidedata.py

# ext-sidedata.py - small extension to test the sidedata logic # ext-sidedata.py - small extension to test the sidedata logic
# #
# Copyright 2019 Pierre-Yves David <pierre-yves.david@octobus.net) # Copyright 2019 Pierre-Yves David <pierre-yves.david@octobus.net)
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import hashlib import hashlib
import struct import struct
from mercurial import ( from mercurial import (
extensions, extensions,
localrepo,
node, node,
revlog, revlog,
upgrade,
) )
from mercurial.revlogutils import ( from mercurial.revlogutils import (
sidedata, sidedata,
) )
def wrapaddrevision(orig, self, text, transaction, link, p1, p2, *args, def wrapaddrevision(orig, self, text, transaction, link, p1, p2, *args,
**kwargs): **kwargs):
if kwargs.get('sidedata') is None: if kwargs.get('sidedata') is None:
kwargs['sidedata'] = {} kwargs['sidedata'] = {}
sd = kwargs['sidedata'] sd = kwargs['sidedata']
## let's store some arbitrary data just for testing ## let's store some arbitrary data just for testing
# text length # text length
sd[sidedata.SD_TEST1] = struct.pack('>I', len(text)) sd[sidedata.SD_TEST1] = struct.pack('>I', len(text))
# and sha2 hashes # and sha2 hashes
sha256 = hashlib.sha256(text).digest() sha256 = hashlib.sha256(text).digest()
sd[sidedata.SD_TEST2] = struct.pack('>32s', sha256) sd[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
return orig(self, text, transaction, link, p1, p2, *args, **kwargs) return orig(self, text, transaction, link, p1, p2, *args, **kwargs)
def wraprevision(orig, self, nodeorrev, *args, **kwargs): def wraprevision(orig, self, nodeorrev, *args, **kwargs):
text = orig(self, nodeorrev, *args, **kwargs) text = orig(self, nodeorrev, *args, **kwargs)
if getattr(self, 'sidedatanocheck', False):
return text
if nodeorrev != node.nullrev and nodeorrev != node.nullid: if nodeorrev != node.nullrev and nodeorrev != node.nullid:
sd = self.sidedata(nodeorrev) sd = self.sidedata(nodeorrev)
if len(text) != struct.unpack('>I', sd[sidedata.SD_TEST1])[0]: if len(text) != struct.unpack('>I', sd[sidedata.SD_TEST1])[0]:
raise RuntimeError('text size mismatch') raise RuntimeError('text size mismatch')
expected = sd[sidedata.SD_TEST2] expected = sd[sidedata.SD_TEST2]
got = hashlib.sha256(text).digest() got = hashlib.sha256(text).digest()
if got != expected: if got != expected:
raise RuntimeError('sha256 mismatch') raise RuntimeError('sha256 mismatch')
return text return text
def wrapgetsidedatacompanion(orig, srcrepo, dstrepo):
sidedatacompanion = orig(srcrepo, dstrepo)
addedreqs = dstrepo.requirements - srcrepo.requirements
if (localrepo.SIDEDATA_REQUIREMENT in addedreqs):
assert sidedatacompanion is None # deal with composition later
def sidedatacompanion(revlog, rev):
update = {}
revlog.sidedatanocheck = True
try:
text = revlog.revision(rev)
finally:
del revlog.sidedatanocheck
## let's store some arbitrary data just for testing
# text length
update[sidedata.SD_TEST1] = struct.pack('>I', len(text))
# and sha2 hashes
sha256 = hashlib.sha256(text).digest()
update[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
return False, (), update
return sidedatacompanion
def extsetup(ui): def extsetup(ui):
extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision) extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision)
extensions.wrapfunction(revlog.revlog, 'revision', wraprevision) extensions.wrapfunction(revlog.revlog, 'revision', wraprevision)
extensions.wrapfunction(upgrade, 'getsidedatacompanion',
wrapgetsidedatacompanion)