( )⚙ D10359 sidedata: replace sidedata upgrade mechanism with the new one

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10359

sidedata: replace sidedata upgrade mechanism with the new one
ClosedPublic

Authored by Alphare on Apr 10 2021, 3:15 PM.

Details

Reviewers
indygreg
marmoute
Group Reviewers
hg-reviewers
Commits
rHG27f1191b1305: sidedata: replace sidedata upgrade mechanism with the new one
Summary

Note: this is split into a separate change (like some other patches in this
series) because it's not easy to have all patches work 100% and this seemed
easier for reviewers.

When cloning or upgrading a repo, we may need to compute (or remove) sidedata.
This is the same mechanism that is used in exchange, so we re-use the new
system to simplify the code and fix the remaining issues (correctly dropping
flags and handling partial removal, etc.).

This also highlighted an issue with test-copies-in-changeset.t that kept
sidedata categories that are not relevant anymore. They should probably be
dropped entirely, but that would be for another patch.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

Alphare created this revision.Apr 10 2021, 3:15 PM
Alphare updated this revision to Diff 27036.Apr 19 2021, 6:10 AM
Alphare updated this revision to Diff 27102.Apr 20 2021, 9:39 AM
marmoute requested changes to this revision.Apr 22 2021, 1:52 PM
marmoute added a subscriber: marmoute.
marmoute added inline comments.
mercurial/revlog.py
2121–2122

This is a gratuitous harmful change. If someone passed something else than the default None value. We should keep it.

This revision now requires changes to proceed.Apr 22 2021, 1:52 PM
Alphare marked an inline comment as done.Apr 30 2021, 6:00 AM
baymax updated this revision to Diff 27304.May 3 2021, 6:06 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

marmoute accepted this revision.May 3 2021, 7:46 AM
baymax updated this revision to Diff 27454.May 3 2021, 9:01 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare added a commit: rHG27f1191b1305: sidedata: replace sidedata upgrade mechanism with the new one.May 5 2021, 6:43 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG27f1191b1305: sidedata: replace sidedata upgrade mechanism with the new one (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 26781

mercurial/metadata.py

import struct import struct
from .node import ( from .node import (
nullid, nullid,
nullrev, nullrev,
) )
from . import ( from . import (
error, error,
pycompat,
requirements as requirementsmod, requirements as requirementsmod,
util, util,
) )
from .revlogutils import ( from .revlogutils import (
flagutil as sidedataflag, flagutil as sidedataflag,
sidedata as sidedatamod, sidedata as sidedatamod,
) )
b"changelog", b"changelog",
sidedatamod.SD_FILES, sidedatamod.SD_FILES,
(sidedatamod.SD_FILES,), (sidedatamod.SD_FILES,),
copies_sidedata_computer, copies_sidedata_computer,
sidedataflag.REVIDX_HASCOPIESINFO, sidedataflag.REVIDX_HASCOPIESINFO,
) )
def getsidedataadder(srcrepo, destrepo):
use_w = srcrepo.ui.configbool(b'experimental', b'worker.repository-upgrade')
if pycompat.iswindows or not use_w:
return _get_simple_sidedata_adder(srcrepo, destrepo)
else:
return _get_worker_sidedata_adder(srcrepo, destrepo)
def _sidedata_worker(srcrepo, revs_queue, sidedata_queue, tokens): def _sidedata_worker(srcrepo, revs_queue, sidedata_queue, tokens):
"""The function used by worker precomputing sidedata """The function used by worker precomputing sidedata
It read an input queue containing revision numbers It read an input queue containing revision numbers
It write in an output queue containing (rev, <sidedata-map>) It write in an output queue containing (rev, <sidedata-map>)
The `None` input value is used as a stop signal. The `None` input value is used as a stop signal.
allworkers.append(w) allworkers.append(w)
w.start() w.start()
# dictionnary to store results for revision higher than we one we are # dictionnary to store results for revision higher than we one we are
# looking for. For example, if we need the sidedatamap for 42, and 43 is # looking for. For example, if we need the sidedatamap for 42, and 43 is
# received, when shelve 43 for later use. # received, when shelve 43 for later use.
staging = {} staging = {}
def sidedata_companion(revlog, rev): def sidedata_companion(repo, revlog, rev, old_sidedata):
data = {}, False
if util.safehasattr(revlog, b'filteredrevs'): # this is a changelog
# Is the data previously shelved ? # Is the data previously shelved ?
data = staging.pop(rev, None) data = staging.pop(rev, None)
if data is None: if data is None:
# look at the queued result until we find the one we are lookig # look at the queued result until we find the one we are lookig
# for (shelve the other ones) # for (shelve the other ones)
r, data = sidedataq.get() r, data = sidedataq.get()
while r != rev: while r != rev:
staging[r] = data staging[r] = data
r, data = sidedataq.get() r, data = sidedataq.get()
tokens.release() tokens.release()
sidedata, has_copies_info = data sidedata, has_copies_info = data
new_flag = 0 new_flag = 0
if has_copies_info: if has_copies_info:
new_flag = sidedataflag.REVIDX_HASCOPIESINFO new_flag = sidedataflag.REVIDX_HASCOPIESINFO
return False, (), sidedata, new_flag, 0 return sidedata, (new_flag, 0)
return sidedata_companion return sidedata_companion
def _get_simple_sidedata_adder(srcrepo, destrepo):
"""The simple version of the sidedata computation
It just compute it in the same thread on request"""
def sidedatacompanion(revlog, rev):
sidedata, has_copies_info = {}, False
if util.safehasattr(revlog, 'filteredrevs'): # this is a changelog
sidedata, has_copies_info = _getsidedata(srcrepo, rev)
new_flag = 0
if has_copies_info:
new_flag = sidedataflag.REVIDX_HASCOPIESINFO
return False, (), sidedata, new_flag, 0
return sidedatacompanion
def getsidedataremover(srcrepo, destrepo):
def sidedatacompanion(revlog, rev):
f = ()
if util.safehasattr(revlog, 'filteredrevs'): # this is a changelog
if revlog.flags(rev) & sidedataflag.REVIDX_SIDEDATA:
f = (
sidedatamod.SD_P1COPIES,
sidedatamod.SD_P2COPIES,
sidedatamod.SD_FILESADDED,
sidedatamod.SD_FILESREMOVED,
)
return False, f, {}, 0, sidedataflag.REVIDX_HASCOPIESINFO
return sidedatacompanion

mercurial/revlog.py

) )
from .utils import ( from .utils import (
storageutil, storageutil,
stringutil, stringutil,
) )
# blanked usage of all the name to prevent pyflakes constraints # blanked usage of all the name to prevent pyflakes constraints
# We need these name available in the module for extensions. # We need these name available in the module for extensions.
REVLOGV0 REVLOGV0
REVLOGV1 REVLOGV1
REVLOGV2 REVLOGV2
FLAG_INLINE_DATA FLAG_INLINE_DATA
FLAG_GENERALDELTA FLAG_GENERALDELTA
REVLOG_DEFAULT_FLAGS REVLOG_DEFAULT_FLAGS
REVLOG_DEFAULT_FORMAT REVLOG_DEFAULT_FORMAT
REVLOG_DEFAULT_VERSION REVLOG_DEFAULT_VERSION
deltacomputer - an optional deltacomputer instance shared between deltacomputer - an optional deltacomputer instance shared between
multiple calls multiple calls
""" """
if link == nullrev: if link == nullrev:
raise error.RevlogError( raise error.RevlogError(
_(b"attempted to add linkrev -1 to %s") % self.indexfile _(b"attempted to add linkrev -1 to %s") % self.indexfile
) )
if sidedata is None: if not sidedata:
sidedata = {} sidedata = {}
marmouteUnsubmitted
Done

This is a gratuitous harmful change. If someone passed something else than the default None value. We should keep it.

marmoute: This is a gratuitous harmful change. If someone passed something else than the default `None`…
elif not self.hassidedata: elif not self.hassidedata:
raise error.ProgrammingError( raise error.ProgrammingError(
_(b"trying to add sidedata to a revlog who don't support them") _(b"trying to add sidedata to a revlog who don't support them")
) )
if flags: if flags:
node = node or self.hash(text, p1, p2) node = node or self.hash(text, p1, p2)
def clone( def clone(
self, self,
tr, tr,
destrevlog, destrevlog,
addrevisioncb=None, addrevisioncb=None,
deltareuse=DELTAREUSESAMEREVS, deltareuse=DELTAREUSESAMEREVS,
forcedeltabothparents=None, forcedeltabothparents=None,
sidedatacompanion=None, sidedata_helpers=None,
): ):
"""Copy this revlog to another, possibly with format changes. """Copy this revlog to another, possibly with format changes.
The destination revlog will contain the same revisions and nodes. The destination revlog will contain the same revisions and nodes.
However, it may not be bit-for-bit identical due to e.g. delta encoding However, it may not be bit-for-bit identical due to e.g. delta encoding
differences. differences.
The ``deltareuse`` argument control how deltas from the existing revlog The ``deltareuse`` argument control how deltas from the existing revlog
are converting a non-generaldelta revlog to a generaldelta revlog, are converting a non-generaldelta revlog to a generaldelta revlog,
deltas will be recomputed if the delta's parent isn't a parent of the deltas will be recomputed if the delta's parent isn't a parent of the
revision. revision.
In addition to the delta policy, the ``forcedeltabothparents`` In addition to the delta policy, the ``forcedeltabothparents``
argument controls whether to force compute deltas against both parents argument controls whether to force compute deltas against both parents
for merges. By default, the current default is used. for merges. By default, the current default is used.
If not None, the `sidedatacompanion` is callable that accept two See `storageutil.emitrevisions` for the doc on `sidedata_helpers`.
arguments:
(srcrevlog, rev)
and return a quintet that control changes to sidedata content from the
old revision to the new clone result:
(dropall, filterout, update, new_flags, dropped_flags)
* if `dropall` is True, all sidedata should be dropped
* `filterout` is a set of sidedata keys that should be dropped
* `update` is a mapping of additionnal/new key -> value
* new_flags is a bitfields of new flags that the revision should get
* dropped_flags is a bitfields of new flags that the revision shoudl not longer have
""" """
if deltareuse not in self.DELTAREUSEALL: if deltareuse not in self.DELTAREUSEALL:
raise ValueError( raise ValueError(
_(b'value for deltareuse invalid: %s') % deltareuse _(b'value for deltareuse invalid: %s') % deltareuse
) )
if len(destrevlog): if len(destrevlog):
raise ValueError(_(b'destination revlog is not empty')) raise ValueError(_(b'destination revlog is not empty'))
destrevlog._deltabothparents = forcedeltabothparents or oldamd destrevlog._deltabothparents = forcedeltabothparents or oldamd
self._clone( self._clone(
tr, tr,
destrevlog, destrevlog,
addrevisioncb, addrevisioncb,
deltareuse, deltareuse,
forcedeltabothparents, forcedeltabothparents,
sidedatacompanion, sidedata_helpers,
) )
finally: finally:
destrevlog._lazydelta = oldlazydelta destrevlog._lazydelta = oldlazydelta
destrevlog._lazydeltabase = oldlazydeltabase destrevlog._lazydeltabase = oldlazydeltabase
destrevlog._deltabothparents = oldamd destrevlog._deltabothparents = oldamd
def _clone( def _clone(
self, self,
tr, tr,
destrevlog, destrevlog,
addrevisioncb, addrevisioncb,
deltareuse, deltareuse,
forcedeltabothparents, forcedeltabothparents,
sidedatacompanion, sidedata_helpers,
): ):
"""perform the core duty of `revlog.clone` after parameter processing""" """perform the core duty of `revlog.clone` after parameter processing"""
deltacomputer = deltautil.deltacomputer(destrevlog) deltacomputer = deltautil.deltacomputer(destrevlog)
index = self.index index = self.index
for rev in self: for rev in self:
entry = index[rev] entry = index[rev]
# Some classes override linkrev to take filtered revs into # Some classes override linkrev to take filtered revs into
# account. Use raw entry from index. # account. Use raw entry from index.
flags = entry[0] & 0xFFFF flags = entry[0] & 0xFFFF
linkrev = entry[4] linkrev = entry[4]
p1 = index[entry[5]][7] p1 = index[entry[5]][7]
p2 = index[entry[6]][7] p2 = index[entry[6]][7]
node = entry[7] node = entry[7]
sidedataactions = (False, [], {}, 0, 0)
if sidedatacompanion is not None:
sidedataactions = sidedatacompanion(self, rev)
# (Possibly) reuse the delta from the revlog if allowed and # (Possibly) reuse the delta from the revlog if allowed and
# the revlog chunk is a delta. # the revlog chunk is a delta.
cachedelta = None cachedelta = None
rawtext = None rawtext = None
if any(sidedataactions) or deltareuse == self.DELTAREUSEFULLADD: if deltareuse == self.DELTAREUSEFULLADD:
dropall = sidedataactions[0]
filterout = sidedataactions[1]
update = sidedataactions[2]
new_flags = sidedataactions[3]
dropped_flags = sidedataactions[4]
text, sidedata = self._revisiondata(rev) text, sidedata = self._revisiondata(rev)
if dropall:
sidedata = {}
for key in filterout:
sidedata.pop(key, None)
sidedata.update(update)
if not sidedata:
sidedata = None
flags |= new_flags if sidedata_helpers is not None:
flags &= ~dropped_flags (sidedata, new_flags) = storageutil.run_sidedata_helpers(
self, sidedata_helpers, sidedata, rev
)
flags = flags | new_flags[0] & ~new_flags[1]
destrevlog.addrevision( destrevlog.addrevision(
text, text,
tr, tr,
linkrev, linkrev,
p1, p1,
p2, p2,
cachedelta=cachedelta, cachedelta=cachedelta,
node=node, node=node,
flags=flags, flags=flags,
deltacomputer=deltacomputer, deltacomputer=deltacomputer,
sidedata=sidedata, sidedata=sidedata,
) )
else: else:
if destrevlog._lazydelta: if destrevlog._lazydelta:
dp = self.deltaparent(rev) dp = self.deltaparent(rev)
if dp != nullrev: if dp != nullrev:
cachedelta = (dp, bytes(self._chunk(rev))) cachedelta = (dp, bytes(self._chunk(rev)))
sidedata = None
if not cachedelta: if not cachedelta:
rawtext = self.rawdata(rev) rawtext, sidedata = self._revisiondata(rev)
if sidedata is None:
sidedata = self.sidedata(rev)
if sidedata_helpers is not None:
(sidedata, new_flags) = storageutil.run_sidedata_helpers(
self, sidedata_helpers, sidedata, rev
)
flags = flags | new_flags[0] & ~new_flags[1]
ifh = destrevlog.opener( ifh = destrevlog.opener(
destrevlog.indexfile, b'a+', checkambig=False destrevlog.indexfile, b'a+', checkambig=False
) )
dfh = None dfh = None
if not destrevlog._inline: if not destrevlog._inline:
dfh = destrevlog.opener(destrevlog.datafile, b'a+') dfh = destrevlog.opener(destrevlog.datafile, b'a+')
try: try:
destrevlog._addrevision( destrevlog._addrevision(
node, node,
rawtext, rawtext,
tr, tr,
linkrev, linkrev,
p1, p1,
p2, p2,
flags, flags,
cachedelta, cachedelta,
ifh, ifh,
dfh, dfh,
deltacomputer=deltacomputer, deltacomputer=deltacomputer,
sidedata=sidedata,
) )
finally: finally:
if dfh: if dfh:
dfh.close() dfh.close()
ifh.close() ifh.close()
if addrevisioncb: if addrevisioncb:
addrevisioncb(self, rev, node) addrevisioncb(self, rev, node)

mercurial/upgrade_utils/engine.py

# upgrade.py - functions for in place upgrade of Mercurial repository # upgrade.py - functions for in place upgrade of Mercurial repository
# #
# Copyright (c) 2016-present, Gregory Szorc # Copyright (c) 2016-present, Gregory Szorc
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import stat import stat
from ..i18n import _ from ..i18n import _
from ..pycompat import getattr from ..pycompat import getattr
from .. import ( from .. import (
changegroup,
changelog, changelog,
error, error,
filelog, filelog,
manifest, manifest,
metadata, metadata,
pycompat, pycompat,
requirements, requirements,
revlog,
scmutil, scmutil,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
from ..revlogutils import nodemap from ..revlogutils import (
flagutil,
nodemap,
sidedata as sidedatamod,
)
def get_sidedata_helpers(srcrepo, dstrepo):
use_w = srcrepo.ui.configbool(b'experimental', b'worker.repository-upgrade')
sequential = pycompat.iswindows or not use_w
if not sequential:
srcrepo.register_sidedata_computer(
b"changelog",
sidedatamod.SD_FILES,
(sidedatamod.SD_FILES,),
metadata._get_worker_sidedata_adder(srcrepo, dstrepo),
flagutil.REVIDX_HASCOPIESINFO,
replace=True,
)
return changegroup.get_sidedata_helpers(srcrepo, dstrepo._wanted_sidedata)
def _revlogfrompath(repo, path): def _revlogfrompath(repo, path):
"""Obtain a revlog from a repo path. """Obtain a revlog from a repo path.
An instance of the appropriate class is returned. An instance of the appropriate class is returned.
""" """
if path == b'00changelog.i': if path == b'00changelog.i':
UPGRADE_MANIFEST = b"manifest" UPGRADE_MANIFEST = b"manifest"
UPGRADE_FILELOGS = b"all-filelogs" UPGRADE_FILELOGS = b"all-filelogs"
UPGRADE_ALL_REVLOGS = frozenset( UPGRADE_ALL_REVLOGS = frozenset(
[UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOGS] [UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOGS]
) )
def getsidedatacompanion(srcrepo, dstrepo):
sidedatacompanion = None
removedreqs = srcrepo.requirements - dstrepo.requirements
addedreqs = dstrepo.requirements - srcrepo.requirements
if requirements.SIDEDATA_REQUIREMENT in removedreqs:
def sidedatacompanion(rl, rev):
rl = getattr(rl, '_revlog', rl)
if rl.flags(rev) & revlog.REVIDX_SIDEDATA:
return True, (), {}, 0, 0
return False, (), {}, 0, 0
elif requirements.COPIESSDC_REQUIREMENT in addedreqs:
sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo)
elif requirements.COPIESSDC_REQUIREMENT in removedreqs:
sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo)
return sidedatacompanion
def matchrevlog(revlogfilter, entry): def matchrevlog(revlogfilter, entry):
"""check if a revlog is selected for cloning. """check if a revlog is selected for cloning.
In other words, are there any updates which need to be done on revlog In other words, are there any updates which need to be done on revlog
or it can be blindly copied. or it can be blindly copied.
The store entry is checked against the passed filter""" The store entry is checked against the passed filter"""
if entry.endswith(b'00changelog.i'): if entry.endswith(b'00changelog.i'):
return UPGRADE_CHANGELOG in revlogfilter return UPGRADE_CHANGELOG in revlogfilter
elif entry.endswith(b'00manifest.i'): elif entry.endswith(b'00manifest.i'):
return UPGRADE_MANIFEST in revlogfilter return UPGRADE_MANIFEST in revlogfilter
return UPGRADE_FILELOGS in revlogfilter return UPGRADE_FILELOGS in revlogfilter
def _perform_clone( def _perform_clone(
ui, ui,
dstrepo, dstrepo,
tr, tr,
old_revlog, old_revlog,
unencoded, unencoded,
upgrade_op, upgrade_op,
sidedatacompanion, sidedata_helpers,
oncopiedrevision, oncopiedrevision,
): ):
""" returns the new revlog object created""" """ returns the new revlog object created"""
newrl = None newrl = None
if matchrevlog(upgrade_op.revlogs_to_process, unencoded): if matchrevlog(upgrade_op.revlogs_to_process, unencoded):
ui.note( ui.note(
_(b'cloning %d revisions from %s\n') % (len(old_revlog), unencoded) _(b'cloning %d revisions from %s\n') % (len(old_revlog), unencoded)
) )
newrl = _revlogfrompath(dstrepo, unencoded) newrl = _revlogfrompath(dstrepo, unencoded)
old_revlog.clone( old_revlog.clone(
tr, tr,
newrl, newrl,
addrevisioncb=oncopiedrevision, addrevisioncb=oncopiedrevision,
deltareuse=upgrade_op.delta_reuse_mode, deltareuse=upgrade_op.delta_reuse_mode,
forcedeltabothparents=upgrade_op.force_re_delta_both_parents, forcedeltabothparents=upgrade_op.force_re_delta_both_parents,
sidedatacompanion=sidedatacompanion, sidedata_helpers=sidedata_helpers,
) )
else: else:
msg = _(b'blindly copying %s containing %i revisions\n') msg = _(b'blindly copying %s containing %i revisions\n')
ui.note(msg % (unencoded, len(old_revlog))) ui.note(msg % (unencoded, len(old_revlog)))
_copyrevlog(tr, dstrepo, old_revlog, unencoded) _copyrevlog(tr, dstrepo, old_revlog, unencoded)
newrl = _revlogfrompath(dstrepo, unencoded) newrl = _revlogfrompath(dstrepo, unencoded)
return newrl return newrl
) )
# Used to keep track of progress. # Used to keep track of progress.
progress = None progress = None
def oncopiedrevision(rl, rev, node): def oncopiedrevision(rl, rev, node):
progress.increment() progress.increment()
sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo) sidedata_helpers = get_sidedata_helpers(srcrepo, dstrepo)
# Migrating filelogs # Migrating filelogs
ui.status( ui.status(
_( _(
b'migrating %d filelogs containing %d revisions ' b'migrating %d filelogs containing %d revisions '
b'(%s in store; %s tracked data)\n' b'(%s in store; %s tracked data)\n'
) )
% ( % (
fcount, fcount,
frevcount, frevcount,
util.bytecount(fsrcsize), util.bytecount(fsrcsize),
util.bytecount(frawsize), util.bytecount(frawsize),
) )
) )
progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount) progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount)
for unencoded, oldrl in sorted(filelogs.items()): for unencoded, oldrl in sorted(filelogs.items()):
newrl = _perform_clone( newrl = _perform_clone(
ui, ui,
dstrepo, dstrepo,
tr, tr,
oldrl, oldrl,
unencoded, unencoded,
upgrade_op, upgrade_op,
sidedatacompanion, sidedata_helpers,
oncopiedrevision, oncopiedrevision,
) )
info = newrl.storageinfo(storedsize=True) info = newrl.storageinfo(storedsize=True)
fdstsize += info[b'storedsize'] or 0 fdstsize += info[b'storedsize'] or 0
ui.status( ui.status(
_( _(
b'finished migrating %d filelog revisions across %d ' b'finished migrating %d filelog revisions across %d '
b'filelogs; change in size: %s\n' b'filelogs; change in size: %s\n'
for unencoded, oldrl in sorted(manifests.items()): for unencoded, oldrl in sorted(manifests.items()):
newrl = _perform_clone( newrl = _perform_clone(
ui, ui,
dstrepo, dstrepo,
tr, tr,
oldrl, oldrl,
unencoded, unencoded,
upgrade_op, upgrade_op,
sidedatacompanion, sidedata_helpers,
oncopiedrevision, oncopiedrevision,
) )
info = newrl.storageinfo(storedsize=True) info = newrl.storageinfo(storedsize=True)
mdstsize += info[b'storedsize'] or 0 mdstsize += info[b'storedsize'] or 0
ui.status( ui.status(
_( _(
b'finished migrating %d manifest revisions across %d ' b'finished migrating %d manifest revisions across %d '
b'manifests; change in size: %s\n' b'manifests; change in size: %s\n'
for unencoded, oldrl in sorted(changelogs.items()): for unencoded, oldrl in sorted(changelogs.items()):
newrl = _perform_clone( newrl = _perform_clone(
ui, ui,
dstrepo, dstrepo,
tr, tr,
oldrl, oldrl,
unencoded, unencoded,
upgrade_op, upgrade_op,
sidedatacompanion, sidedata_helpers,
oncopiedrevision, oncopiedrevision,
) )
info = newrl.storageinfo(storedsize=True) info = newrl.storageinfo(storedsize=True)
cdstsize += info[b'storedsize'] or 0 cdstsize += info[b'storedsize'] or 0
progress.complete() progress.complete()
ui.status( ui.status(
_( _(
b'finished migrating %d changelog revisions; change in size: ' b'finished migrating %d changelog revisions; change in size: '

tests/test-copies-in-changeset.t

$ hg mv a b $ hg mv a b
$ hg ci -m 'remove a' a $ hg ci -m 'remove a' a
#if sidedata #if sidedata
Test upgrading/downgrading to sidedata storage Test upgrading/downgrading to sidedata storage
============================================== ==============================================
downgrading (keeping some sidedata) downgrading
$ hg debugformat -v $ hg debugformat -v
format-variant repo config default format-variant repo config default
fncache: yes yes yes fncache: yes yes yes
dotencode: yes yes yes dotencode: yes yes yes
generaldelta: yes yes yes generaldelta: yes yes yes
share-safe: no no no share-safe: no no no
sparserevlog: yes yes yes sparserevlog: yes yes yes
persistent-nodemap: no no no persistent-nodemap: no no no
copies-sdc: no no no copies-sdc: no no no
revlog-v2: yes yes no revlog-v2: yes yes no
plain-cl-delta: yes yes yes plain-cl-delta: yes yes yes
compression: zlib zlib zlib (no-zstd !) compression: zlib zlib zlib (no-zstd !)
compression: zstd zstd zstd (zstd !) compression: zstd zstd zstd (zstd !)
compression-level: default default default compression-level: default default default
$ hg debugsidedata -c -- 0 $ hg debugsidedata -c -- 0
1 sidedata entries
entry-0014 size 14
$ hg debugsidedata -c -- 1 $ hg debugsidedata -c -- 1
1 sidedata entries
entry-0014 size 14
$ hg debugsidedata -m -- 0 $ hg debugsidedata -m -- 0
upgrading upgrading
$ cat << EOF > .hg/hgrc $ cat << EOF > .hg/hgrc
> [format] > [format]
> exp-use-copies-side-data-changeset = yes > exp-use-copies-side-data-changeset = yes
> EOF > EOF

tests/testlib/ext-sidedata.py

import hashlib import hashlib
import struct import struct
from mercurial.node import ( from mercurial.node import (
nullid, nullid,
nullrev, nullrev,
) )
from mercurial import ( from mercurial import (
changegroup,
extensions, extensions,
requirements, requirements,
revlog, revlog,
) )
from mercurial.upgrade_utils import engine as upgrade_engine from mercurial.upgrade_utils import engine as upgrade_engine
from mercurial.revlogutils import sidedata from mercurial.revlogutils import sidedata
raise RuntimeError('text size mismatch') raise RuntimeError('text size mismatch')
expected = sd.get(sidedata.SD_TEST2) expected = sd.get(sidedata.SD_TEST2)
got = hashlib.sha256(text).digest() got = hashlib.sha256(text).digest()
if expected is not None and got != expected: if expected is not None and got != expected:
raise RuntimeError('sha256 mismatch') raise RuntimeError('sha256 mismatch')
return text, sd return text, sd
def wrapgetsidedatacompanion(orig, srcrepo, dstrepo): def wrapget_sidedata_helpers(orig, srcrepo, dstrepo):
sidedatacompanion = orig(srcrepo, dstrepo) repo, computers, removers = orig(srcrepo, dstrepo)
assert not computers and not removers # deal with composition later
addedreqs = dstrepo.requirements - srcrepo.requirements addedreqs = dstrepo.requirements - srcrepo.requirements
if requirements.SIDEDATA_REQUIREMENT in addedreqs: if requirements.SIDEDATA_REQUIREMENT in addedreqs:
assert sidedatacompanion is None # deal with composition later
def sidedatacompanion(revlog, rev): def computer(repo, revlog, rev, old_sidedata):
assert not old_sidedata # not supported yet
update = {} update = {}
revlog.sidedatanocheck = True revlog.sidedatanocheck = True
try: try:
text = revlog.revision(rev) text = revlog.revision(rev)
finally: finally:
del revlog.sidedatanocheck del revlog.sidedatanocheck
## let's store some arbitrary data just for testing ## let's store some arbitrary data just for testing
# text length # text length
update[sidedata.SD_TEST1] = struct.pack('>I', len(text)) update[sidedata.SD_TEST1] = struct.pack('>I', len(text))
# and sha2 hashes # and sha2 hashes
sha256 = hashlib.sha256(text).digest() sha256 = hashlib.sha256(text).digest()
update[sidedata.SD_TEST2] = struct.pack('>32s', sha256) update[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
return False, (), update, 0, 0 return update, (0, 0)
srcrepo.register_sidedata_computer(
b"changelog",
b"whatever",
(sidedata.SD_TEST1, sidedata.SD_TEST2),
computer,
0,
)
dstrepo.register_wanted_sidedata(b"whatever")
return sidedatacompanion return changegroup.get_sidedata_helpers(srcrepo, dstrepo._wanted_sidedata)
def extsetup(ui): def extsetup(ui):
extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision) extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision)
extensions.wrapfunction(revlog.revlog, '_revisiondata', wrap_revisiondata) extensions.wrapfunction(revlog.revlog, '_revisiondata', wrap_revisiondata)
extensions.wrapfunction( extensions.wrapfunction(
upgrade_engine, 'getsidedatacompanion', wrapgetsidedatacompanion upgrade_engine, 'get_sidedata_helpers', wrapget_sidedata_helpers
) )
def reposetup(ui, repo): def reposetup(ui, repo):
# We don't register sidedata computers because we don't care within these # We don't register sidedata computers because we don't care within these
# tests # tests
repo.register_wanted_sidedata(sidedata.SD_TEST1) repo.register_wanted_sidedata(sidedata.SD_TEST1)
repo.register_wanted_sidedata(sidedata.SD_TEST2) repo.register_wanted_sidedata(sidedata.SD_TEST2)