( )⚙ D10360 sidedata: move sidedata-related utils to the dedicated module

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10360

sidedata: move sidedata-related utils to the dedicated module
ClosedPublic

Authored by Alphare on Apr 10 2021, 3:15 PM.

Details

Reviewers
indygreg
marmoute
Group Reviewers
hg-reviewers
Commits
rHG3aab2330b7d3: sidedata: move sidedata-related utils to the dedicated module

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

Alphare created this revision.Apr 10 2021, 3:15 PM
Alphare updated this revision to Diff 27037.Apr 19 2021, 6:10 AM
Alphare updated this revision to Diff 27103.Apr 20 2021, 9:39 AM
marmoute accepted this revision.Apr 22 2021, 1:53 PM
baymax updated this revision to Diff 27305.May 3 2021, 6:06 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 27455.May 3 2021, 9:01 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare added a commit: rHG3aab2330b7d3: sidedata: move sidedata-related utils to the dedicated module.May 5 2021, 6:43 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG3aab2330b7d3: sidedata: move sidedata-related utils to the dedicated module (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 27037

mercurial/changegroup.py

# changegroup.py - Mercurial changegroup manipulation functions # changegroup.py - Mercurial changegroup manipulation functions
# #
# Copyright 2006 Olivia Mackall <olivia@selenic.com> # Copyright 2006 Olivia Mackall <olivia@selenic.com>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import collections
import os import os
import struct import struct
import weakref import weakref
from .i18n import _ from .i18n import _
from .node import ( from .node import (
hex, hex,
nullid, nullid,
# Only useful if we're adding sidedata categories. If both peers have # Only useful if we're adding sidedata categories. If both peers have
# the same categories, then we simply don't do anything. # the same categories, then we simply don't do anything.
adding_sidedata = ( adding_sidedata = (
requirements.REVLOGV2_REQUIREMENT in repo.requirements requirements.REVLOGV2_REQUIREMENT in repo.requirements
and self.version == b'04' and self.version == b'04'
and srctype == b'pull' and srctype == b'pull'
) )
if adding_sidedata: if adding_sidedata:
sidedata_helpers = get_sidedata_helpers( sidedata_helpers = sidedatamod.get_sidedata_helpers(
repo, repo,
sidedata_categories or set(), sidedata_categories or set(),
pull=True, pull=True,
) )
else: else:
sidedata_helpers = None sidedata_helpers = None
def csmap(x): def csmap(x):
sidedata_helpers = None sidedata_helpers = None
if self.version == b'04': if self.version == b'04':
remote_sidedata = self._remote_sidedata remote_sidedata = self._remote_sidedata
if source == b'strip': if source == b'strip':
# We're our own remote when stripping, get the no-op helpers # We're our own remote when stripping, get the no-op helpers
# TODO a better approach would be for the strip bundle to # TODO a better approach would be for the strip bundle to
# correctly advertise its sidedata categories directly. # correctly advertise its sidedata categories directly.
remote_sidedata = repo._wanted_sidedata remote_sidedata = repo._wanted_sidedata
sidedata_helpers = get_sidedata_helpers(repo, remote_sidedata) sidedata_helpers = sidedatamod.get_sidedata_helpers(
repo, remote_sidedata
)
clstate, deltas = self._generatechangelog( clstate, deltas = self._generatechangelog(
cl, cl,
clnodes, clnodes,
generate=changelog, generate=changelog,
sidedata_helpers=sidedata_helpers, sidedata_helpers=sidedata_helpers,
) )
for delta in deltas: for delta in deltas:
fl.rev(n) fl.rev(n)
except error.LookupError: except error.LookupError:
raise error.Abort( raise error.Abort(
_(b'missing file data for %s:%s - run hg verify') _(b'missing file data for %s:%s - run hg verify')
% (f, hex(n)) % (f, hex(n))
) )
return revisions, files return revisions, files
def get_sidedata_helpers(repo, remote_sd_categories, pull=False):
# Computers for computing sidedata on-the-fly
sd_computers = collections.defaultdict(list)
# Computers for categories to remove from sidedata
sd_removers = collections.defaultdict(list)
to_generate = remote_sd_categories - repo._wanted_sidedata
to_remove = repo._wanted_sidedata - remote_sd_categories
if pull:
to_generate, to_remove = to_remove, to_generate
for revlog_kind, computers in repo._sidedata_computers.items():
for category, computer in computers.items():
if category in to_generate:
sd_computers[revlog_kind].append(computer)
if category in to_remove:
sd_removers[revlog_kind].append(computer)
sidedata_helpers = (repo, sd_computers, sd_removers)
return sidedata_helpers

mercurial/localrepo.py

exchange, exchange,
extensions, extensions,
filelog, filelog,
hook, hook,
lock as lockmod, lock as lockmod,
match as matchmod, match as matchmod,
mergestate as mergestatemod, mergestate as mergestatemod,
mergeutil, mergeutil,
metadata as metadatamod,
namespaces, namespaces,
narrowspec, narrowspec,
obsolete, obsolete,
pathutil, pathutil,
phases, phases,
pushkey, pushkey,
pycompat, pycompat,
rcutil, rcutil,
procutil, procutil,
stringutil, stringutil,
urlutil, urlutil,
) )
from .revlogutils import ( from .revlogutils import (
concurrency_checker as revlogchecker, concurrency_checker as revlogchecker,
constants as revlogconst, constants as revlogconst,
sidedata as sidedatamod,
) )
release = lockmod.release release = lockmod.release
urlerr = util.urlerr urlerr = util.urlerr
urlreq = util.urlreq urlreq = util.urlreq
# set of (path, vfs-location) tuples. vfs-location is: # set of (path, vfs-location) tuples. vfs-location is:
# - 'plain for vfs relative paths # - 'plain for vfs relative paths
self._extrafilterid = repoview.extrafilter(ui) self._extrafilterid = repoview.extrafilter(ui)
self.filecopiesmode = None self.filecopiesmode = None
if requirementsmod.COPIESSDC_REQUIREMENT in self.requirements: if requirementsmod.COPIESSDC_REQUIREMENT in self.requirements:
self.filecopiesmode = b'changeset-sidedata' self.filecopiesmode = b'changeset-sidedata'
self._wanted_sidedata = set() self._wanted_sidedata = set()
self._sidedata_computers = {} self._sidedata_computers = {}
metadatamod.set_sidedata_spec_for_repo(self) sidedatamod.set_sidedata_spec_for_repo(self)
def _getvfsward(self, origfunc): def _getvfsward(self, origfunc):
"""build a ward for self.vfs""" """build a ward for self.vfs"""
rref = weakref.ref(self) rref = weakref.ref(self)
def checkvfs(path, mode=None): def checkvfs(path, mode=None):
ret = origfunc(path, mode=mode) ret = origfunc(path, mode=mode)
repo = rref() repo = rref()

mercurial/metadata.py

import struct import struct
from .node import ( from .node import (
nullid, nullid,
nullrev, nullrev,
) )
from . import ( from . import (
error, error,
requirements as requirementsmod,
util, util,
) )
from .revlogutils import ( from .revlogutils import (
flagutil as sidedataflag, flagutil as sidedataflag,
sidedata as sidedatamod, sidedata as sidedatamod,
) )
def copies_sidedata_computer(repo, revlog, rev, existing_sidedata): def copies_sidedata_computer(repo, revlog, rev, existing_sidedata):
sidedata, has_copies_info = _getsidedata(repo, rev) sidedata, has_copies_info = _getsidedata(repo, rev)
flags_to_add = sidedataflag.REVIDX_HASCOPIESINFO if has_copies_info else 0 flags_to_add = sidedataflag.REVIDX_HASCOPIESINFO if has_copies_info else 0
return sidedata, (flags_to_add, 0) return sidedata, (flags_to_add, 0)
def set_sidedata_spec_for_repo(repo):
if requirementsmod.COPIESSDC_REQUIREMENT in repo.requirements:
repo.register_wanted_sidedata(sidedatamod.SD_FILES)
repo.register_sidedata_computer(
b"changelog",
sidedatamod.SD_FILES,
(sidedatamod.SD_FILES,),
copies_sidedata_computer,
sidedataflag.REVIDX_HASCOPIESINFO,
)
def _sidedata_worker(srcrepo, revs_queue, sidedata_queue, tokens): def _sidedata_worker(srcrepo, revs_queue, sidedata_queue, tokens):
"""The function used by worker precomputing sidedata """The function used by worker precomputing sidedata
It read an input queue containing revision numbers It read an input queue containing revision numbers
It write in an output queue containing (rev, <sidedata-map>) It write in an output queue containing (rev, <sidedata-map>)
The `None` input value is used as a stop signal. The `None` input value is used as a stop signal.

mercurial/revlog.py

# (Possibly) reuse the delta from the revlog if allowed and # (Possibly) reuse the delta from the revlog if allowed and
# the revlog chunk is a delta. # the revlog chunk is a delta.
cachedelta = None cachedelta = None
rawtext = None rawtext = None
if deltareuse == self.DELTAREUSEFULLADD: if deltareuse == self.DELTAREUSEFULLADD:
text, sidedata = self._revisiondata(rev) text, sidedata = self._revisiondata(rev)
if sidedata_helpers is not None: if sidedata_helpers is not None:
(sidedata, new_flags) = storageutil.run_sidedata_helpers( (sidedata, new_flags) = sidedatautil.run_sidedata_helpers(
self, sidedata_helpers, sidedata, rev self, sidedata_helpers, sidedata, rev
) )
flags = flags | new_flags[0] & ~new_flags[1] flags = flags | new_flags[0] & ~new_flags[1]
destrevlog.addrevision( destrevlog.addrevision(
text, text,
tr, tr,
linkrev, linkrev,
sidedata = None sidedata = None
if not cachedelta: if not cachedelta:
rawtext, sidedata = self._revisiondata(rev) rawtext, sidedata = self._revisiondata(rev)
if sidedata is None: if sidedata is None:
sidedata = self.sidedata(rev) sidedata = self.sidedata(rev)
if sidedata_helpers is not None: if sidedata_helpers is not None:
(sidedata, new_flags) = storageutil.run_sidedata_helpers( (sidedata, new_flags) = sidedatautil.run_sidedata_helpers(
self, sidedata_helpers, sidedata, rev self, sidedata_helpers, sidedata, rev
) )
flags = flags | new_flags[0] & ~new_flags[1] flags = flags | new_flags[0] & ~new_flags[1]
ifh = destrevlog.opener( ifh = destrevlog.opener(
destrevlog.indexfile, b'a+', checkambig=False destrevlog.indexfile, b'a+', checkambig=False
) )
dfh = None dfh = None
new_entries = [] new_entries = []
# append the new sidedata # append the new sidedata
with self._datafp(b'a+') as fp: with self._datafp(b'a+') as fp:
# Maybe this bug still exists, see revlog._writeentry # Maybe this bug still exists, see revlog._writeentry
fp.seek(0, os.SEEK_END) fp.seek(0, os.SEEK_END)
current_offset = fp.tell() current_offset = fp.tell()
for rev in range(startrev, endrev + 1): for rev in range(startrev, endrev + 1):
entry = self.index[rev] entry = self.index[rev]
new_sidedata, flags = storageutil.run_sidedata_helpers( new_sidedata, flags = sidedatautil.run_sidedata_helpers(
store=self, store=self,
sidedata_helpers=helpers, sidedata_helpers=helpers,
sidedata={}, sidedata={},
rev=rev, rev=rev,
) )
serialized_sidedata = sidedatautil.serialize_sidedata( serialized_sidedata = sidedatautil.serialize_sidedata(
new_sidedata new_sidedata

mercurial/revlogutils/sidedata.py

<all bytes remaining in the rawtext> <all bytes remaining in the rawtext>
This is a simple and effective format. It should be enough to experiment with This is a simple and effective format. It should be enough to experiment with
the concept. the concept.
""" """
from __future__ import absolute_import from __future__ import absolute_import
import collections
import struct import struct
from .. import error from .. import error, requirements as requirementsmod
from ..revlogutils import flagutil
from ..utils import hashutil from ..utils import hashutil
## sidedata type constant ## sidedata type constant
# reserve a block for testing purposes. # reserve a block for testing purposes.
SD_TEST1 = 1 SD_TEST1 = 1
SD_TEST2 = 2 SD_TEST2 = 2
SD_TEST3 = 3 SD_TEST3 = 3
SD_TEST4 = 4 SD_TEST4 = 4
nextdataoffset = dataoffset + size nextdataoffset = dataoffset + size
entrytext = bytes(blob[dataoffset:nextdataoffset]) entrytext = bytes(blob[dataoffset:nextdataoffset])
readdigest = hashutil.sha1(entrytext).digest() readdigest = hashutil.sha1(entrytext).digest()
if storeddigest != readdigest: if storeddigest != readdigest:
raise error.SidedataHashError(key, storeddigest, readdigest) raise error.SidedataHashError(key, storeddigest, readdigest)
sidedata[key] = entrytext sidedata[key] = entrytext
dataoffset = nextdataoffset dataoffset = nextdataoffset
return sidedata return sidedata
def get_sidedata_helpers(repo, remote_sd_categories, pull=False):
# Computers for computing sidedata on-the-fly
sd_computers = collections.defaultdict(list)
# Computers for categories to remove from sidedata
sd_removers = collections.defaultdict(list)
to_generate = remote_sd_categories - repo._wanted_sidedata
to_remove = repo._wanted_sidedata - remote_sd_categories
if pull:
to_generate, to_remove = to_remove, to_generate
for revlog_kind, computers in repo._sidedata_computers.items():
for category, computer in computers.items():
if category in to_generate:
sd_computers[revlog_kind].append(computer)
if category in to_remove:
sd_removers[revlog_kind].append(computer)
sidedata_helpers = (repo, sd_computers, sd_removers)
return sidedata_helpers
def run_sidedata_helpers(store, sidedata_helpers, sidedata, rev):
"""Returns the sidedata for the given revision after running through
the given helpers.
- `store`: the revlog this applies to (changelog, manifest, or filelog
instance)
- `sidedata_helpers`: see `storageutil.emitrevisions`
- `sidedata`: previous sidedata at the given rev, if any
- `rev`: affected rev of `store`
"""
repo, sd_computers, sd_removers = sidedata_helpers
kind = store.revlog_kind
flags_to_add = 0
flags_to_remove = 0
for _keys, sd_computer, _flags in sd_computers.get(kind, []):
sidedata, flags = sd_computer(repo, store, rev, sidedata)
flags_to_add |= flags[0]
flags_to_remove |= flags[1]
for keys, _computer, flags in sd_removers.get(kind, []):
for key in keys:
sidedata.pop(key, None)
flags_to_remove |= flags
return sidedata, (flags_to_add, flags_to_remove)
def set_sidedata_spec_for_repo(repo):
# prevent cycle metadata -> revlogutils.sidedata -> metadata
from .. import metadata
if requirementsmod.COPIESSDC_REQUIREMENT in repo.requirements:
repo.register_wanted_sidedata(SD_FILES)
repo.register_sidedata_computer(
b"changelog",
SD_FILES,
(SD_FILES,),
metadata.copies_sidedata_computer,
flagutil.REVIDX_HASCOPIESINFO,
)

mercurial/upgrade_utils/engine.py

# upgrade.py - functions for in place upgrade of Mercurial repository # upgrade.py - functions for in place upgrade of Mercurial repository
# #
# Copyright (c) 2016-present, Gregory Szorc # Copyright (c) 2016-present, Gregory Szorc
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import stat import stat
from ..i18n import _ from ..i18n import _
from ..pycompat import getattr from ..pycompat import getattr
from .. import ( from .. import (
changegroup,
changelog, changelog,
error, error,
filelog, filelog,
manifest, manifest,
metadata, metadata,
pycompat, pycompat,
requirements, requirements,
scmutil, scmutil,
srcrepo.register_sidedata_computer( srcrepo.register_sidedata_computer(
b"changelog", b"changelog",
sidedatamod.SD_FILES, sidedatamod.SD_FILES,
(sidedatamod.SD_FILES,), (sidedatamod.SD_FILES,),
metadata._get_worker_sidedata_adder(srcrepo, dstrepo), metadata._get_worker_sidedata_adder(srcrepo, dstrepo),
flagutil.REVIDX_HASCOPIESINFO, flagutil.REVIDX_HASCOPIESINFO,
replace=True, replace=True,
) )
return changegroup.get_sidedata_helpers(srcrepo, dstrepo._wanted_sidedata) return sidedatamod.get_sidedata_helpers(srcrepo, dstrepo._wanted_sidedata)
def _revlogfrompath(repo, rl_type, path): def _revlogfrompath(repo, rl_type, path):
"""Obtain a revlog from a repo path. """Obtain a revlog from a repo path.
An instance of the appropriate class is returned. An instance of the appropriate class is returned.
""" """
if rl_type & store.FILEFLAGS_CHANGELOG: if rl_type & store.FILEFLAGS_CHANGELOG:

mercurial/utils/storageutil.py

) )
available.add(rev) available.add(rev)
serialized_sidedata = None serialized_sidedata = None
sidedata_flags = (0, 0) sidedata_flags = (0, 0)
if sidedata_helpers: if sidedata_helpers:
old_sidedata = store.sidedata(rev) old_sidedata = store.sidedata(rev)
sidedata, sidedata_flags = run_sidedata_helpers( sidedata, sidedata_flags = sidedatamod.run_sidedata_helpers(
store=store, store=store,
sidedata_helpers=sidedata_helpers, sidedata_helpers=sidedata_helpers,
sidedata=old_sidedata, sidedata=old_sidedata,
rev=rev, rev=rev,
) )
if sidedata: if sidedata:
serialized_sidedata = sidedatamod.serialize_sidedata(sidedata) serialized_sidedata = sidedatamod.serialize_sidedata(sidedata)
delta=delta, delta=delta,
sidedata=serialized_sidedata, sidedata=serialized_sidedata,
protocol_flags=protocol_flags, protocol_flags=protocol_flags,
) )
prevrev = rev prevrev = rev
def run_sidedata_helpers(store, sidedata_helpers, sidedata, rev):
"""Returns the sidedata for the given revision after running through
the given helpers.
- `store`: the revlog this applies to (changelog, manifest, or filelog
instance)
- `sidedata_helpers`: see `storageutil.emitrevisions`
- `sidedata`: previous sidedata at the given rev, if any
- `rev`: affected rev of `store`
"""
repo, sd_computers, sd_removers = sidedata_helpers
kind = store.revlog_kind
flags_to_add = 0
flags_to_remove = 0
for _keys, sd_computer, _flags in sd_computers.get(kind, []):
sidedata, flags = sd_computer(repo, store, rev, sidedata)
flags_to_add |= flags[0]
flags_to_remove |= flags[1]
for keys, _computer, flags in sd_removers.get(kind, []):
for key in keys:
sidedata.pop(key, None)
flags_to_remove |= flags
return sidedata, (flags_to_add, flags_to_remove)
def deltaiscensored(delta, baserev, baselenfn): def deltaiscensored(delta, baserev, baselenfn):
"""Determine if a delta represents censored revision data. """Determine if a delta represents censored revision data.
``baserev`` is the base revision this delta is encoded against. ``baserev`` is the base revision this delta is encoded against.
``baselenfn`` is a callable receiving a revision number that resolves the ``baselenfn`` is a callable receiving a revision number that resolves the
length of the revision fulltext. length of the revision fulltext.
Returns a bool indicating if the result of the delta represents a censored Returns a bool indicating if the result of the delta represents a censored

tests/testlib/ext-sidedata.py

import hashlib import hashlib
import struct import struct
from mercurial.node import ( from mercurial.node import (
nullid, nullid,
nullrev, nullrev,
) )
from mercurial import ( from mercurial import (
changegroup,
extensions, extensions,
requirements, requirements,
revlog, revlog,
) )
from mercurial.upgrade_utils import engine as upgrade_engine from mercurial.upgrade_utils import engine as upgrade_engine
from mercurial.revlogutils import sidedata from mercurial.revlogutils import sidedata
b"changelog", b"changelog",
b"whatever", b"whatever",
(sidedata.SD_TEST1, sidedata.SD_TEST2), (sidedata.SD_TEST1, sidedata.SD_TEST2),
computer, computer,
0, 0,
) )
dstrepo.register_wanted_sidedata(b"whatever") dstrepo.register_wanted_sidedata(b"whatever")
return changegroup.get_sidedata_helpers(srcrepo, dstrepo._wanted_sidedata) return sidedata.get_sidedata_helpers(srcrepo, dstrepo._wanted_sidedata)
def extsetup(ui): def extsetup(ui):
extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision) extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision)
extensions.wrapfunction(revlog.revlog, '_revisiondata', wrap_revisiondata) extensions.wrapfunction(revlog.revlog, '_revisiondata', wrap_revisiondata)
extensions.wrapfunction( extensions.wrapfunction(
upgrade_engine, 'get_sidedata_helpers', wrapget_sidedata_helpers upgrade_engine, 'get_sidedata_helpers', wrapget_sidedata_helpers
) )
def reposetup(ui, repo): def reposetup(ui, repo):
# We don't register sidedata computers because we don't care within these # We don't register sidedata computers because we don't care within these
# tests # tests
repo.register_wanted_sidedata(sidedata.SD_TEST1) repo.register_wanted_sidedata(sidedata.SD_TEST1)
repo.register_wanted_sidedata(sidedata.SD_TEST2) repo.register_wanted_sidedata(sidedata.SD_TEST2)