This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6950

sidedatacopies: write copies information in sidedata when applicable
ClosedPublic

Authored by marmoute on Oct 3 2019, 1:54 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGbeed7ce61681: sidedatacopies: write copies information in sidedata when applicable
rHGd37361888720: sidedatacopies: write copies information in sidedata when applicable
Summary

If the format of the repository indicate it stores copies information into
changeset's sidedata, then we actually write that information into sidedata at
commit time. It will be put to use in later changesets.

Currently, we store all field unconditionally, but that is likely to change in
the future for the sake of efficiency.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Revision Contents
Changeset List

Diff 16782

mercurial/changelog.py

revlog, revlog,
util, util,
) )
from .utils import ( from .utils import (
dateutil, dateutil,
stringutil, stringutil,
) )
from .revlogutils import (
sidedata as sidedatamod,
)
_defaultextra = {'branch': 'default'} _defaultextra = {'branch': 'default'}
def _string_escape(text): def _string_escape(text):
""" """
>>> from .pycompat import bytechr as chr >>> from .pycompat import bytechr as chr
>>> d = {b'nl': chr(10), b'bs': chr(92), b'cr': chr(13), b'nul': chr(0)} >>> d = {b'nl': chr(10), b'bs': chr(92), b'cr': chr(13), b'nul': chr(0)}
>>> s = b"ab%(nl)scd%(bs)s%(bs)sn%(nul)s12ab%(cr)scd%(bs)s%(nl)s" % d >>> s = b"ab%(nl)scd%(bs)s%(bs)sn%(nul)s12ab%(cr)scd%(bs)s%(nl)s" % d
>>> s >>> s
if extra: if extra:
branch = extra.get("branch") branch = extra.get("branch")
if branch in ("default", ""): if branch in ("default", ""):
del extra["branch"] del extra["branch"]
elif branch in (".", "null", "tip"): elif branch in (".", "null", "tip"):
raise error.StorageError(_('the name \'%s\' is reserved') raise error.StorageError(_('the name \'%s\' is reserved')
% branch) % branch)
sortedfiles = sorted(files) sortedfiles = sorted(files)
sidedata = None
if extra is not None: if extra is not None:
for name in ('p1copies', 'p2copies', 'filesadded', 'filesremoved'): for name in ('p1copies', 'p2copies', 'filesadded', 'filesremoved'):
extra.pop(name, None) extra.pop(name, None)
if p1copies is not None: if p1copies is not None:
p1copies = encodecopies(sortedfiles, p1copies) p1copies = encodecopies(sortedfiles, p1copies)
if p2copies is not None: if p2copies is not None:
p2copies = encodecopies(sortedfiles, p2copies) p2copies = encodecopies(sortedfiles, p2copies)
if filesadded is not None: if filesadded is not None:
filesadded = encodefileindices(sortedfiles, filesadded) filesadded = encodefileindices(sortedfiles, filesadded)
if filesremoved is not None: if filesremoved is not None:
filesremoved = encodefileindices(sortedfiles, filesremoved) filesremoved = encodefileindices(sortedfiles, filesremoved)
if self._copiesstorage == 'extra': if self._copiesstorage == 'extra':
extrasentries = p1copies, p2copies, filesadded, filesremoved extrasentries = p1copies, p2copies, filesadded, filesremoved
if extra is None and any(x is not None for x in extrasentries): if extra is None and any(x is not None for x in extrasentries):
extra = {} extra = {}
if p1copies is not None: if p1copies is not None:
extra['p1copies'] = p1copies extra['p1copies'] = p1copies
if p2copies is not None: if p2copies is not None:
extra['p2copies'] = p2copies extra['p2copies'] = p2copies
if filesadded is not None: if filesadded is not None:
extra['filesadded'] = filesadded extra['filesadded'] = filesadded
if filesremoved is not None: if filesremoved is not None:
extra['filesremoved'] = filesremoved extra['filesremoved'] = filesremoved
elif self._copiesstorage == 'changeset-sidedata':
sidedata = {}
if p1copies is not None:
sidedata[sidedatamod.SD_P1COPIES] = p1copies
if p2copies is not None:
sidedata[sidedatamod.SD_P2COPIES] = p2copies
if filesadded is not None:
sidedata[sidedatamod.SD_FILESADDED] = filesadded
if filesremoved is not None:
sidedata[sidedatamod.SD_FILESREMOVED] = filesremoved
if extra: if extra:
extra = encodeextra(extra) extra = encodeextra(extra)
parseddate = "%s %s" % (parseddate, extra) parseddate = "%s %s" % (parseddate, extra)
l = [hex(manifest), user, parseddate] + sortedfiles + ["", desc] l = [hex(manifest), user, parseddate] + sortedfiles + ["", desc]
text = "\n".join(l) text = "\n".join(l)
return self.addrevision(text, transaction, len(self), p1, p2) return self.addrevision(text, transaction, len(self), p1, p2,
sidedata=sidedata)
def branchinfo(self, rev): def branchinfo(self, rev):
"""return the branch name and open/close state of a revision """return the branch name and open/close state of a revision
This function exists because creating a changectx object This function exists because creating a changectx object
just to access this is costly.""" just to access this is costly."""
extra = self.read(rev)[5] extra = self.read(rev)[5]
return encoding.tolocal(extra.get("branch")), 'close' in extra return encoding.tolocal(extra.get("branch")), 'close' in extra
def _nodeduplicatecallback(self, transaction, node): def _nodeduplicatecallback(self, transaction, node):
# keep track of revisions that got "re-added", eg: unbunde of know rev. # keep track of revisions that got "re-added", eg: unbunde of know rev.
# #
# We track them in a list to preserve their order from the source bundle # We track them in a list to preserve their order from the source bundle
duplicates = transaction.changes.setdefault('revduplicates', []) duplicates = transaction.changes.setdefault('revduplicates', [])
duplicates.append(self.rev(node)) duplicates.append(self.rev(node))

mercurial/interfaces/repository.py

""") """)
filteredrevcache = interfaceutil.Attribute( filteredrevcache = interfaceutil.Attribute(
"""Holds sets of revisions to be filtered.""") """Holds sets of revisions to be filtered.""")
names = interfaceutil.Attribute( names = interfaceutil.Attribute(
"""A ``namespaces`` instance.""") """A ``namespaces`` instance.""")
filecopiesmode = interfaceutil.Attribute(
"""The way files copies should be dealt with in this repo.""")
def close(): def close():
"""Close the handle on this repository.""" """Close the handle on this repository."""
def peer(): def peer():
"""Obtain an object conforming to the ``peer`` interface.""" """Obtain an object conforming to the ``peer`` interface."""
def unfiltered(): def unfiltered():
"""Obtain an unfiltered/raw view of this repo.""" """Obtain an unfiltered/raw view of this repo."""

mercurial/localrepo.py

# This revlog format is super old and we don't bother trying to parse # This revlog format is super old and we don't bother trying to parse
# opener options for it because those options wouldn't do anything # opener options for it because those options wouldn't do anything
# meaningful on such old repos. # meaningful on such old repos.
if b'revlogv1' in requirements or REVLOGV2_REQUIREMENT in requirements: if b'revlogv1' in requirements or REVLOGV2_REQUIREMENT in requirements:
options.update(resolverevlogstorevfsoptions(ui, requirements, features)) options.update(resolverevlogstorevfsoptions(ui, requirements, features))
else: # explicitly mark repo as using revlogv0 else: # explicitly mark repo as using revlogv0
options['revlogv0'] = True options['revlogv0'] = True
if COPIESSDC_REQUIREMENT in requirements:
options['copies-storage'] = 'changeset-sidedata'
else:
writecopiesto = ui.config('experimental', 'copies.write-to') writecopiesto = ui.config('experimental', 'copies.write-to')
copiesextramode = ('changeset-only', 'compatibility') copiesextramode = ('changeset-only', 'compatibility')
if (writecopiesto in copiesextramode): if (writecopiesto in copiesextramode):
options['copies-storage'] = 'extra' options['copies-storage'] = 'extra'
return options return options
def resolverevlogstorevfsoptions(ui, requirements, features): def resolverevlogstorevfsoptions(ui, requirements, features):
"""Resolve opener options specific to revlogs.""" """Resolve opener options specific to revlogs."""
options = {} options = {}
options[b'flagprocessors'] = {} options[b'flagprocessors'] = {}
# Key to signature value. # Key to signature value.
self._sparsesignaturecache = {} self._sparsesignaturecache = {}
# Signature to cached matcher instance. # Signature to cached matcher instance.
self._sparsematchercache = {} self._sparsematchercache = {}
self._extrafilterid = repoview.extrafilter(ui) self._extrafilterid = repoview.extrafilter(ui)
self.filecopiesmode = None
if COPIESSDC_REQUIREMENT in self.requirements:
self.filecopiesmode = 'changeset-sidedata'
def _getvfsward(self, origfunc): def _getvfsward(self, origfunc):
"""build a ward for self.vfs""" """build a ward for self.vfs"""
rref = weakref.ref(self) rref = weakref.ref(self)
def checkvfs(path, mode=None): def checkvfs(path, mode=None):
ret = origfunc(path, mode=mode) ret = origfunc(path, mode=mode)
repo = rref() repo = rref()
if (repo is None if (repo is None
or not util.safehasattr(repo, '_wlockref') or not util.safehasattr(repo, '_wlockref')
convert to be the identity, it can pass an origctx and this convert to be the identity, it can pass an origctx and this
function will use the same files list when it makes sense to function will use the same files list when it makes sense to
do so. do so.
""" """
p1, p2 = ctx.p1(), ctx.p2() p1, p2 = ctx.p1(), ctx.p2()
user = ctx.user() user = ctx.user()
if self.filecopiesmode == 'changeset-sidedata':
writechangesetcopy = True
writefilecopymeta = True
writecopiesto = None
else:
writecopiesto = self.ui.config('experimental', 'copies.write-to') writecopiesto = self.ui.config('experimental', 'copies.write-to')
writefilecopymeta = writecopiesto != 'changeset-only' writefilecopymeta = writecopiesto != 'changeset-only'
writechangesetcopy = (writecopiesto in writechangesetcopy = (writecopiesto in
('changeset-only', 'compatibility')) ('changeset-only', 'compatibility'))
p1copies, p2copies = None, None p1copies, p2copies = None, None
if writechangesetcopy: if writechangesetcopy:
p1copies = ctx.p1copies() p1copies = ctx.p1copies()
p2copies = ctx.p2copies() p2copies = ctx.p2copies()
filesadded, filesremoved = None, None filesadded, filesremoved = None, None
with self.lock(), self.transaction("commit") as tr: with self.lock(), self.transaction("commit") as tr:
trp = weakref.proxy(tr) trp = weakref.proxy(tr)

mercurial/revlogutils/sidedata.py

SD_TEST1 = 1 SD_TEST1 = 1
SD_TEST2 = 2 SD_TEST2 = 2
SD_TEST3 = 3 SD_TEST3 = 3
SD_TEST4 = 4 SD_TEST4 = 4
SD_TEST5 = 5 SD_TEST5 = 5
SD_TEST6 = 6 SD_TEST6 = 6
SD_TEST7 = 7 SD_TEST7 = 7
# key to store copies related information
SD_P1COPIES = 8
SD_P2COPIES = 9
SD_FILESADDED = 10
SD_FILESREMOVED = 11
# internal format constant # internal format constant
SIDEDATA_HEADER = struct.Struct('>H') SIDEDATA_HEADER = struct.Struct('>H')
SIDEDATA_ENTRY = struct.Struct('>HL20s') SIDEDATA_ENTRY = struct.Struct('>HL20s')
def sidedatawriteprocessor(rl, text, sidedata): def sidedatawriteprocessor(rl, text, sidedata):
sidedata = list(sidedata.items()) sidedata = list(sidedata.items())
sidedata.sort() sidedata.sort()
rawtext = [SIDEDATA_HEADER.pack(len(sidedata))] rawtext = [SIDEDATA_HEADER.pack(len(sidedata))]

tests/test-copies-in-changeset.t

files: b c d files: b c d
filesadded: 0 filesadded: 0
1 1
2 2
p1copies: 0\x00a (esc) p1copies: 0\x00a (esc)
1\x00a (esc) 1\x00a (esc)
2\x00a (esc) 2\x00a (esc)
#else
$ hg debugsidedata -c -v -- -1
4 sidedata entries
entry-0010 size 11
'0\x00a\n1\x00a\n2\x00a'
entry-0011 size 0
''
entry-0012 size 5
'0\n1\n2'
entry-0013 size 0
''
#endif #endif
$ hg showcopies $ hg showcopies
a -> b a -> b
a -> c a -> c
a -> d a -> d
#if extra #if extra
$ hg changesetcopies $ hg changesetcopies
files: b b2 files: b b2
filesadded: 1 filesadded: 1
filesremoved: 0 filesremoved: 0
p1copies: 1\x00b (esc) p1copies: 1\x00b (esc)
#else
$ hg debugsidedata -c -v -- -1
4 sidedata entries
entry-0010 size 3
'1\x00b'
entry-0011 size 0
''
entry-0012 size 1
'1'
entry-0013 size 1
'0'
#endif #endif
$ hg showcopies $ hg showcopies
b -> b2 b -> b2
Rename onto existing file. This should get recorded in the changeset files list and in the extras, Rename onto existing file. This should get recorded in the changeset files list and in the extras,
even though there is no filelog entry. even though there is no filelog entry.
#if extra #if extra
$ hg changesetcopies $ hg changesetcopies
files: c files: c
p1copies: 0\x00b2 (esc) p1copies: 0\x00b2 (esc)
#else
$ hg debugsidedata -c -v -- -1
4 sidedata entries
entry-0010 size 4
'0\x00b2'
entry-0011 size 0
''
entry-0012 size 0
''
entry-0013 size 0
''
#endif #endif
$ hg showcopies $ hg showcopies
b2 -> c b2 -> c
#if extra #if extra
$ hg debugindex c $ hg debugindex c
filesadded: 0 filesadded: 0
1 1
2 2
p1copies: 0\x00a (esc) p1copies: 0\x00a (esc)
2\x00f (esc) 2\x00f (esc)
p2copies: 1\x00d (esc) p2copies: 1\x00d (esc)
#else
$ hg debugsidedata -c -v -- -1
4 sidedata entries
entry-0010 size 7
'0\x00a\n2\x00f'
entry-0011 size 3
'1\x00d'
entry-0012 size 5
'0\n1\n2'
entry-0013 size 0
''
#endif #endif
$ hg showcopies $ hg showcopies
a -> g a -> g
d -> h d -> h
f -> i f -> i
Test writing to both changeset and filelog Test writing to both changeset and filelog
$ hg cp a j $ hg cp a j
#if extra #if extra
$ hg ci -m 'copy a to j' --config experimental.copies.write-to=compatibility $ hg ci -m 'copy a to j' --config experimental.copies.write-to=compatibility
$ hg changesetcopies $ hg changesetcopies
files: j files: j
filesadded: 0 filesadded: 0
filesremoved: filesremoved:
p1copies: 0\x00a (esc) p1copies: 0\x00a (esc)
p2copies: p2copies:
#else #else
$ hg ci -m 'copy a to j' $ hg ci -m 'copy a to j'
$ hg debugsidedata -c -v -- -1
4 sidedata entries
entry-0010 size 3
'0\x00a'
entry-0011 size 0
''
entry-0012 size 1
'0'
entry-0013 size 0
''
#endif #endif
$ hg debugdata j 0 $ hg debugdata j 0
\x01 (esc) \x01 (esc)
copy: a copy: a
copyrev: b789fdd96dc2f3bd229c1dd8eedf0fc60e2b68e3 copyrev: b789fdd96dc2f3bd229c1dd8eedf0fc60e2b68e3
\x01 (esc) \x01 (esc)
a a
$ hg showcopies $ hg showcopies
> --config experimental.copies.write-to=filelog-only > --config experimental.copies.write-to=filelog-only
saved backup bundle to $TESTTMP/repo/.hg/strip-backup/*-*-amend.hg (glob) saved backup bundle to $TESTTMP/repo/.hg/strip-backup/*-*-amend.hg (glob)
$ hg changesetcopies $ hg changesetcopies
files: j files: j
#else #else
$ hg ci --amend -m 'copy a to j, v2' $ hg ci --amend -m 'copy a to j, v2'
saved backup bundle to $TESTTMP/repo/.hg/strip-backup/*-*-amend.hg (glob) saved backup bundle to $TESTTMP/repo/.hg/strip-backup/*-*-amend.hg (glob)
$ hg debugsidedata -c -v -- -1
4 sidedata entries
entry-0010 size 3
'0\x00a'
entry-0011 size 0
''
entry-0012 size 1
'0'
entry-0013 size 0
''
#endif #endif
$ hg showcopies --config experimental.copies.read-from=filelog-only $ hg showcopies --config experimental.copies.read-from=filelog-only
a -> j a -> j
The entries should be written to extras even if they're empty (so the client The entries should be written to extras even if they're empty (so the client
won't have to fall back to reading from filelogs) won't have to fall back to reading from filelogs)
$ echo x >> j $ echo x >> j
#if extra #if extra
$ hg ci -m 'modify j' --config experimental.copies.write-to=compatibility $ hg ci -m 'modify j' --config experimental.copies.write-to=compatibility
$ hg changesetcopies $ hg changesetcopies
files: j files: j
filesadded: filesadded:
filesremoved: filesremoved:
p1copies: p1copies:
p2copies: p2copies:
#else #else
$ hg ci -m 'modify j' $ hg ci -m 'modify j'
$ hg debugsidedata -c -v -- -1
4 sidedata entries
entry-0010 size 0
''
entry-0011 size 0
''
entry-0012 size 0
''
entry-0013 size 0
''
#endif #endif
Test writing only to filelog Test writing only to filelog
$ hg cp a k $ hg cp a k
#if extra #if extra
$ hg ci -m 'copy a to k' --config experimental.copies.write-to=filelog-only $ hg ci -m 'copy a to k' --config experimental.copies.write-to=filelog-only
$ hg changesetcopies $ hg changesetcopies
files: k files: k
#else #else
$ hg ci -m 'copy a to k' $ hg ci -m 'copy a to k'
$ hg debugsidedata -c -v -- -1
4 sidedata entries
entry-0010 size 3
'0\x00a'
entry-0011 size 0
''
entry-0012 size 1
'0'
entry-0013 size 0
''
#endif #endif
$ hg debugdata k 0 $ hg debugdata k 0
\x01 (esc) \x01 (esc)
copy: a copy: a
copyrev: b789fdd96dc2f3bd229c1dd8eedf0fc60e2b68e3 copyrev: b789fdd96dc2f3bd229c1dd8eedf0fc60e2b68e3
\x01 (esc) \x01 (esc)
a a