This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6955

sidedatacopies: deal with upgrading and downgrading to that format
ClosedPublic

Authored by marmoute on Oct 3 2019, 1:55 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG843da18386d5: sidedatacopies: deal with upgrading and downgrading to that format
Summary

This is quite useful to test this on real life data.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

marmoute created this revision.Oct 3 2019, 1:55 AM
marmoute updated this revision to Diff 16956.Oct 7 2019, 8:38 PM
martinvonz added a subscriber: martinvonz.Oct 9 2019, 2:15 PM
martinvonz added inline comments.
mercurial/copies.py
989

nit: drop the b'' here and below

marmoute updated this revision to Diff 17010.Oct 9 2019, 5:31 PM
marmoute added a commit: rHG843da18386d5: sidedatacopies: deal with upgrading and downgrading to that format.Oct 9 2019, 7:47 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG843da18386d5: sidedatacopies: deal with upgrading and downgrading to that format (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/copies.py (52 lines)
Mmercurial/upgrade.py (10 lines)
Mtests/test-copies-in-changeset.t (87 lines)

Diff 17010

mercurial/copies.py

# copies.py - copy detection for Mercurial # copies.py - copy detection for Mercurial
# #
# Copyright 2008 Matt Mackall <mpm@selenic.com> # Copyright 2008 Matt Mackall <mpm@selenic.com>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import collections import collections
import heapq import heapq
import os import os
from .i18n import _ from .i18n import _
from .revlogutils.flagutil import REVIDX_SIDEDATA
from . import ( from . import (
error, error,
match as matchmod, match as matchmod,
node, node,
pathutil, pathutil,
pycompat, pycompat,
util, util,
) )
from .revlogutils import sidedata as sidedatamod
from .utils import stringutil from .utils import stringutil
def _findlimit(repo, ctxa, ctxb): def _findlimit(repo, ctxa, ctxb):
""" """
Find the last revision that needs to be checked to ensure that a full Find the last revision that needs to be checked to ensure that a full
transitive closure for file copies can be properly calculated. transitive closure for file copies can be properly calculated.
Generally, this means finding the earliest revision number that's an Generally, this means finding the earliest revision number that's an
if i < 0 or i >= len(files): if i < 0 or i >= len(files):
return None return None
subset.append(files[i]) subset.append(files[i])
return subset return subset
except (ValueError, IndexError): except (ValueError, IndexError):
# Perhaps someone had chosen the same key name (e.g. "added") and # Perhaps someone had chosen the same key name (e.g. "added") and
# used different syntax for the value. # used different syntax for the value.
return None return None
def _getsidedata(srcrepo, rev):
ctx = srcrepo[rev]
filescopies = computechangesetcopies(ctx)
filesadded = computechangesetfilesadded(ctx)
filesremoved = computechangesetfilesremoved(ctx)
sidedata = {}
if any([filescopies, filesadded, filesremoved]):
sortedfiles = sorted(ctx.files())
p1copies, p2copies = filescopies
p1copies = encodecopies(sortedfiles, p1copies)
p2copies = encodecopies(sortedfiles, p2copies)
filesadded = encodefileindices(sortedfiles, filesadded)
filesremoved = encodefileindices(sortedfiles, filesremoved)
sidedata[sidedatamod.SD_P1COPIES] = p1copies
sidedata[sidedatamod.SD_P2COPIES] = p2copies
sidedata[sidedatamod.SD_FILESADDED] = filesadded
sidedata[sidedatamod.SD_FILESREMOVED] = filesremoved
return sidedata
def getsidedataadder(srcrepo, destrepo):
def sidedatacompanion(revlog, rev):
sidedata = {}
if util.safehasattr(revlog, 'filteredrevs'): # this is a changelog
martinvonzUnsubmitted
Not Done

nit: drop the b'' here and below

martinvonz: nit: drop the `b''` here and below
sidedata = _getsidedata(srcrepo, rev)
return False, (), sidedata
return sidedatacompanion
def getsidedataremover(srcrepo, destrepo):
def sidedatacompanion(revlog, rev):
f = ()
if util.safehasattr(revlog, 'filteredrevs'): # this is a changelog
if revlog.flags(rev) & REVIDX_SIDEDATA:
f = (
sidedatamod.SD_P1COPIES,
sidedatamod.SD_P2COPIES,
sidedatamod.SD_FILESADDED,
sidedatamod.SD_FILESREMOVED,
)
return False, f, {}
return sidedatacompanion

mercurial/upgrade.py

# upgrade.py - functions for in place upgrade of Mercurial repository # upgrade.py - functions for in place upgrade of Mercurial repository
# #
# Copyright (c) 2016-present, Gregory Szorc # Copyright (c) 2016-present, Gregory Szorc
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import stat import stat
from .i18n import _ from .i18n import _
from .pycompat import getattr from .pycompat import getattr
from . import ( from . import (
changelog, changelog,
copies,
error, error,
filelog, filelog,
hg, hg,
localrepo, localrepo,
manifest, manifest,
pycompat, pycompat,
revlog, revlog,
scmutil, scmutil,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
from .utils import compression from .utils import compression
# list of requirements that request a clone of all revlog if added/removed # list of requirements that request a clone of all revlog if added/removed
RECLONES_REQUIREMENTS = { RECLONES_REQUIREMENTS = {
b'generaldelta', b'generaldelta',
localrepo.SPARSEREVLOG_REQUIREMENT, localrepo.SPARSEREVLOG_REQUIREMENT,
localrepo.SIDEDATA_REQUIREMENT,
} }
def requiredsourcerequirements(repo): def requiredsourcerequirements(repo):
"""Obtain requirements required to be present to upgrade a repo. """Obtain requirements required to be present to upgrade a repo.
An upgrade will not be allowed if the repository doesn't have the An upgrade will not be allowed if the repository doesn't have the
requirements returned by this function. requirements returned by this function.
If an upgrade were to create a repository that dropped a requirement, If an upgrade were to create a repository that dropped a requirement,
the dropped requirement must appear in the returned set for the upgrade the dropped requirement must appear in the returned set for the upgrade
to be allowed. to be allowed.
""" """
supported = { supported = {
localrepo.SPARSEREVLOG_REQUIREMENT, localrepo.SPARSEREVLOG_REQUIREMENT,
localrepo.SIDEDATA_REQUIREMENT, localrepo.SIDEDATA_REQUIREMENT,
localrepo.COPIESSDC_REQUIREMENT,
} }
for name in compression.compengines: for name in compression.compengines:
engine = compression.compengines[name] engine = compression.compengines[name]
if engine.available() and engine.revlogheader(): if engine.available() and engine.revlogheader():
supported.add(b'exp-compression-%s' % name) supported.add(b'exp-compression-%s' % name)
if engine.name() == b'zstd': if engine.name() == b'zstd':
supported.add(b'revlog-compression-zstd') supported.add(b'revlog-compression-zstd')
return supported return supported
supported = { supported = {
b'dotencode', b'dotencode',
b'fncache', b'fncache',
b'generaldelta', b'generaldelta',
b'revlogv1', b'revlogv1',
b'store', b'store',
localrepo.SPARSEREVLOG_REQUIREMENT, localrepo.SPARSEREVLOG_REQUIREMENT,
localrepo.SIDEDATA_REQUIREMENT, localrepo.SIDEDATA_REQUIREMENT,
localrepo.COPIESSDC_REQUIREMENT,
} }
for name in compression.compengines: for name in compression.compengines:
engine = compression.compengines[name] engine = compression.compengines[name]
if engine.available() and engine.revlogheader(): if engine.available() and engine.revlogheader():
supported.add(b'exp-compression-%s' % name) supported.add(b'exp-compression-%s' % name)
if engine.name() == b'zstd': if engine.name() == b'zstd':
supported.add(b'revlog-compression-zstd') supported.add(b'revlog-compression-zstd')
return supported return supported
future, unknown requirements from accidentally being added. future, unknown requirements from accidentally being added.
""" """
supported = { supported = {
b'dotencode', b'dotencode',
b'fncache', b'fncache',
b'generaldelta', b'generaldelta',
localrepo.SPARSEREVLOG_REQUIREMENT, localrepo.SPARSEREVLOG_REQUIREMENT,
localrepo.SIDEDATA_REQUIREMENT, localrepo.SIDEDATA_REQUIREMENT,
localrepo.COPIESSDC_REQUIREMENT,
} }
for name in compression.compengines: for name in compression.compengines:
engine = compression.compengines[name] engine = compression.compengines[name]
if engine.available() and engine.revlogheader(): if engine.available() and engine.revlogheader():
supported.add(b'exp-compression-%s' % name) supported.add(b'exp-compression-%s' % name)
if engine.name() == b'zstd': if engine.name() == b'zstd':
supported.add(b'revlog-compression-zstd') supported.add(b'revlog-compression-zstd')
return supported return supported
UPGRADE_ALL_REVLOGS = frozenset( UPGRADE_ALL_REVLOGS = frozenset(
[UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOG] [UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOG]
) )
def getsidedatacompanion(srcrepo, dstrepo): def getsidedatacompanion(srcrepo, dstrepo):
sidedatacompanion = None sidedatacompanion = None
removedreqs = srcrepo.requirements - dstrepo.requirements removedreqs = srcrepo.requirements - dstrepo.requirements
addedreqs = dstrepo.requirements - srcrepo.requirements
if localrepo.SIDEDATA_REQUIREMENT in removedreqs: if localrepo.SIDEDATA_REQUIREMENT in removedreqs:
def sidedatacompanion(rl, rev): def sidedatacompanion(rl, rev):
rl = getattr(rl, '_revlog', rl) rl = getattr(rl, '_revlog', rl)
if rl.flags(rev) & revlog.REVIDX_SIDEDATA: if rl.flags(rev) & revlog.REVIDX_SIDEDATA:
return True, (), {} return True, (), {}
return False, (), {} return False, (), {}
elif localrepo.COPIESSDC_REQUIREMENT in addedreqs:
sidedatacompanion = copies.getsidedataadder(srcrepo, dstrepo)
elif localrepo.COPIESSDC_REQUIREMENT in removedreqs:
sidedatacompanion = copies.getsidedataremover(srcrepo, dstrepo)
return sidedatacompanion return sidedatacompanion
def matchrevlog(revlogfilter, entry): def matchrevlog(revlogfilter, entry):
"""check is a revlog is selected for cloning """check is a revlog is selected for cloning
The store entry is checked against the passed filter""" The store entry is checked against the passed filter"""
if entry.endswith(b'00changelog.i'): if entry.endswith(b'00changelog.i'):

tests/test-copies-in-changeset.t

Test committing half a rename Test committing half a rename
$ hg init partial $ hg init partial
$ cd partial $ cd partial
$ echo a > a $ echo a > a
$ hg ci -Aqm 'add a' $ hg ci -Aqm 'add a'
$ hg mv a b $ hg mv a b
$ hg ci -m 'remove a' a $ hg ci -m 'remove a' a
#if sidedata
Test upgrading/downgrading to sidedata storage
==============================================
downgrading (keeping some sidedata)
$ hg debugformat -v
format-variant repo config default
fncache: yes yes yes
dotencode: yes yes yes
generaldelta: yes yes yes
sparserevlog: yes yes yes
sidedata: yes yes no
copies-sdc: yes yes no
plain-cl-delta: yes yes yes
compression: zlib zlib zlib
compression-level: default default default
$ hg debugsidedata -c -- 0
4 sidedata entries
entry-0010 size 0
entry-0011 size 0
entry-0012 size 1
entry-0013 size 0
$ hg debugsidedata -c -- 1
4 sidedata entries
entry-0010 size 0
entry-0011 size 0
entry-0012 size 0
entry-0013 size 1
$ hg debugsidedata -m -- 0
$ cat << EOF > .hg/hgrc
> [format]
> use-side-data = yes
> exp-use-copies-side-data-changeset = no
> EOF
$ hg debugupgraderepo --run --quiet --no-backup > /dev/null
$ hg debugformat -v
format-variant repo config default
fncache: yes yes yes
dotencode: yes yes yes
generaldelta: yes yes yes
sparserevlog: yes yes yes
sidedata: yes yes no
copies-sdc: no no no
plain-cl-delta: yes yes yes
compression: zlib zlib zlib
compression-level: default default default
$ hg debugsidedata -c -- 0
$ hg debugsidedata -c -- 1
$ hg debugsidedata -m -- 0
upgrading
$ cat << EOF > .hg/hgrc
> [format]
> exp-use-copies-side-data-changeset = yes
> EOF
$ hg debugupgraderepo --run --quiet --no-backup > /dev/null
$ hg debugformat -v
format-variant repo config default
fncache: yes yes yes
dotencode: yes yes yes
generaldelta: yes yes yes
sparserevlog: yes yes yes
sidedata: yes yes no
copies-sdc: yes yes no
plain-cl-delta: yes yes yes
compression: zlib zlib zlib
compression-level: default default default
$ hg debugsidedata -c -- 0
4 sidedata entries
entry-0010 size 0
entry-0011 size 0
entry-0012 size 1
entry-0013 size 0
$ hg debugsidedata -c -- 1
4 sidedata entries
entry-0010 size 0
entry-0011 size 0
entry-0012 size 0
entry-0013 size 1
$ hg debugsidedata -m -- 0
#endif
$ cd .. $ cd ..