This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8916

repository: introduce constant for treemanifest requirement and use it
ClosedPublic

Authored by pulkit on Aug 9 2020, 5:25 AM.

Details

Reviewers
durin42
martinvonz
indygreg
Group Reviewers
hg-reviewers
Commits
rHGe58e234096de: repository: introduce constant for treemanifest requirement and use it
Summary

In future we will like to much cleaner logic around which requirement is for
working copy and which can go in store. To start with that, we first need to
de-clutter the requirement values spread around and replace them with constants.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Revision Contents
Changeset List

Diff 22371

hgext/narrow/narrowbundle2.py

shallow=depth is not None, shallow=depth is not None,
ellipsisroots=ellipsisroots, ellipsisroots=ellipsisroots,
fullnodes=relevant_nodes, fullnodes=relevant_nodes,
) )
cgdata = packer.generate(common, visitnodes, False, b'narrow_widen') cgdata = packer.generate(common, visitnodes, False, b'narrow_widen')
part = bundler.newpart(b'changegroup', data=cgdata) part = bundler.newpart(b'changegroup', data=cgdata)
part.addparam(b'version', version) part.addparam(b'version', version)
if b'treemanifest' in repo.requirements: if repository.TREEMANIFEST_REQUIREMENT in repo.requirements:
part.addparam(b'treemanifest', b'1') part.addparam(b'treemanifest', b'1')
def generate_ellipses_bundle2_for_widening( def generate_ellipses_bundle2_for_widening(
bundler, repo, oldmatch, newmatch, version, common, known, bundler, repo, oldmatch, newmatch, version, common, known,
): ):
common = set(common or [nullid]) common = set(common or [nullid])
# Steps: # Steps:
shallow=False, shallow=False,
ellipsisroots=newellipsis, ellipsisroots=newellipsis,
fullnodes=newfull, fullnodes=newfull,
) )
cgdata = packer.generate(common, newvisit, False, b'narrow_widen') cgdata = packer.generate(common, newvisit, False, b'narrow_widen')
part = bundler.newpart(b'changegroup', data=cgdata) part = bundler.newpart(b'changegroup', data=cgdata)
part.addparam(b'version', version) part.addparam(b'version', version)
if b'treemanifest' in repo.requirements: if repository.TREEMANIFEST_REQUIREMENT in repo.requirements:
part.addparam(b'treemanifest', b'1') part.addparam(b'treemanifest', b'1')
@bundle2.parthandler(_SPECPART, (_SPECPART_INCLUDE, _SPECPART_EXCLUDE)) @bundle2.parthandler(_SPECPART, (_SPECPART_INCLUDE, _SPECPART_EXCLUDE))
def _handlechangespec_2(op, inpart): def _handlechangespec_2(op, inpart):
# XXX: This bundle2 handling is buggy and should be removed after hg5.2 is # XXX: This bundle2 handling is buggy and should be removed after hg5.2 is
# released. New servers will send a mandatory bundle2 part named # released. New servers will send a mandatory bundle2 part named
# 'Narrowspec' and will send specs as data instead of params. # 'Narrowspec' and will send specs as data instead of params.

hgext/remotefilelog/remotefilelogserver.py

pycompat, pycompat,
store, store,
streamclone, streamclone,
util, util,
wireprotoserver, wireprotoserver,
wireprototypes, wireprototypes,
wireprotov1server, wireprotov1server,
) )
from mercurial.interfaces import repository
from . import ( from . import (
constants, constants,
shallowutil, shallowutil,
) )
_sshv1server = wireprotoserver.sshv1protocolhandler _sshv1server = wireprotoserver.sshv1protocolhandler
if not fp.endswith(b'.i') and not fp.endswith( if not fp.endswith(b'.i') and not fp.endswith(
b'.d' b'.d'
): ):
n = util.pconvert(fp[striplen:]) n = util.pconvert(fp[striplen:])
yield (store.decodedir(n), n, st.st_size) yield (store.decodedir(n), n, st.st_size)
if kind == stat.S_IFDIR: if kind == stat.S_IFDIR:
visit.append(fp) visit.append(fp)
if b'treemanifest' in repo.requirements: if repository.TREEMANIFEST_REQUIREMENT in repo.requirements:
for (u, e, s) in repo.store.datafiles(): for (u, e, s) in repo.store.datafiles():
if u.startswith(b'meta/') and ( if u.startswith(b'meta/') and (
u.endswith(b'.i') or u.endswith(b'.d') u.endswith(b'.i') or u.endswith(b'.d')
): ):
yield (u, e, s) yield (u, e, s)
# Return .d and .i files that do not match the shallow pattern # Return .d and .i files that do not match the shallow pattern
match = state.match match = state.match

mercurial/bundle2.py

pushkey, pushkey,
pycompat, pycompat,
scmutil, scmutil,
streamclone, streamclone,
tags, tags,
url, url,
util, util,
) )
from .interfaces import repository
from .utils import stringutil from .utils import stringutil
urlerr = util.urlerr urlerr = util.urlerr
urlreq = util.urlreq urlreq = util.urlreq
_pack = struct.pack _pack = struct.pack
_unpack = struct.unpack _unpack = struct.unpack
cg = changegroup.getunbundler(unpackerversion, inpart, None) cg = changegroup.getunbundler(unpackerversion, inpart, None)
# the source and url passed here are overwritten by the one contained in # the source and url passed here are overwritten by the one contained in
# the transaction.hookargs argument. So 'bundle2' is a placeholder # the transaction.hookargs argument. So 'bundle2' is a placeholder
nbchangesets = None nbchangesets = None
if b'nbchanges' in inpart.params: if b'nbchanges' in inpart.params:
nbchangesets = int(inpart.params.get(b'nbchanges')) nbchangesets = int(inpart.params.get(b'nbchanges'))
if ( if (
b'treemanifest' in inpart.params b'treemanifest' in inpart.params
and b'treemanifest' not in op.repo.requirements and repository.TREEMANIFEST_REQUIREMENT not in op.repo.requirements
): ):
if len(op.repo.changelog) != 0: if len(op.repo.changelog) != 0:
raise error.Abort( raise error.Abort(
_( _(
b"bundle contains tree manifests, but local repo is " b"bundle contains tree manifests, but local repo is "
b"non-empty and does not use tree manifests" b"non-empty and does not use tree manifests"
) )
) )
op.repo.requirements.add(b'treemanifest') op.repo.requirements.add(repository.TREEMANIFEST_REQUIREMENT)
op.repo.svfs.options = localrepo.resolvestorevfsoptions( op.repo.svfs.options = localrepo.resolvestorevfsoptions(
op.repo.ui, op.repo.requirements, op.repo.features op.repo.ui, op.repo.requirements, op.repo.features
) )
scmutil.writereporequirements(op.repo) scmutil.writereporequirements(op.repo)
bundlesidedata = bool(b'exp-sidedata' in inpart.params) bundlesidedata = bool(b'exp-sidedata' in inpart.params)
reposidedata = bool(b'exp-sidedata-flag' in op.repo.requirements) reposidedata = bool(b'exp-sidedata-flag' in op.repo.requirements)
if reposidedata and not bundlesidedata: if reposidedata and not bundlesidedata:
list(commonnodes), list(commonnodes),
False, False,
b'narrow_widen', b'narrow_widen',
changelog=False, changelog=False,
) )
part = bundler.newpart(b'changegroup', data=cgdata) part = bundler.newpart(b'changegroup', data=cgdata)
part.addparam(b'version', cgversion) part.addparam(b'version', cgversion)
if b'treemanifest' in repo.requirements: if repository.TREEMANIFEST_REQUIREMENT in repo.requirements:
part.addparam(b'treemanifest', b'1') part.addparam(b'treemanifest', b'1')
if b'exp-sidedata-flag' in repo.requirements: if b'exp-sidedata-flag' in repo.requirements:
part.addparam(b'exp-sidedata', b'1') part.addparam(b'exp-sidedata', b'1')
return bundler return bundler

mercurial/changegroup.py

# never reordered. To handle this case, we simply take the slowpath, # never reordered. To handle this case, we simply take the slowpath,
# which already has the 'clrevorder' logic. This was also fixed in # which already has the 'clrevorder' logic. This was also fixed in
# cc0ff93d0c0c. # cc0ff93d0c0c.
# Treemanifests don't work correctly with fastpathlinkrev # Treemanifests don't work correctly with fastpathlinkrev
# either, because we don't discover which directory nodes to # either, because we don't discover which directory nodes to
# send along with files. This could probably be fixed. # send along with files. This could probably be fixed.
fastpathlinkrev = fastpathlinkrev and ( fastpathlinkrev = fastpathlinkrev and (
b'treemanifest' not in repo.requirements repository.TREEMANIFEST_REQUIREMENT not in repo.requirements
) )
fnodes = {} # needed file nodes fnodes = {} # needed file nodes
size = 0 size = 0
it = self.generatemanifests( it = self.generatemanifests(
commonrevs, commonrevs,
clrevorder, clrevorder,
def allsupportedversions(repo): def allsupportedversions(repo):
versions = set(_packermap.keys()) versions = set(_packermap.keys())
needv03 = False needv03 = False
if ( if (
repo.ui.configbool(b'experimental', b'changegroup3') repo.ui.configbool(b'experimental', b'changegroup3')
or repo.ui.configbool(b'experimental', b'treemanifest') or repo.ui.configbool(b'experimental', b'treemanifest')
or b'treemanifest' in repo.requirements or repository.TREEMANIFEST_REQUIREMENT in repo.requirements
): ):
# we keep version 03 because we need to to exchange treemanifest data # we keep version 03 because we need to to exchange treemanifest data
# #
# we also keep vresion 01 and 02, because it is possible for repo to # we also keep vresion 01 and 02, because it is possible for repo to
# contains both normal and tree manifest at the same time. so using # contains both normal and tree manifest at the same time. so using
# older version to pull data is viable # older version to pull data is viable
# #
# (or even to push subset of history) # (or even to push subset of history)
# Changegroup versions that can be applied to the repo # Changegroup versions that can be applied to the repo
def supportedincomingversions(repo): def supportedincomingversions(repo):
return allsupportedversions(repo) return allsupportedversions(repo)
# Changegroup versions that can be created from the repo # Changegroup versions that can be created from the repo
def supportedoutgoingversions(repo): def supportedoutgoingversions(repo):
versions = allsupportedversions(repo) versions = allsupportedversions(repo)
if b'treemanifest' in repo.requirements: if repository.TREEMANIFEST_REQUIREMENT in repo.requirements:
# Versions 01 and 02 support only flat manifests and it's just too # Versions 01 and 02 support only flat manifests and it's just too
# expensive to convert between the flat manifest and tree manifest on # expensive to convert between the flat manifest and tree manifest on
# the fly. Since tree manifests are hashed differently, all of history # the fly. Since tree manifests are hashed differently, all of history
# would have to be converted. Instead, we simply don't even pretend to # would have to be converted. Instead, we simply don't even pretend to
# support versions 01 and 02. # support versions 01 and 02.
versions.discard(b'01') versions.discard(b'01')
versions.discard(b'02') versions.discard(b'02')
if repository.NARROW_REQUIREMENT in repo.requirements: if repository.NARROW_REQUIREMENT in repo.requirements:

mercurial/cmdutil.py

state as statemod, state as statemod,
subrepoutil, subrepoutil,
templatekw, templatekw,
templater, templater,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
from .interfaces import repository
from .utils import ( from .utils import (
dateutil, dateutil,
stringutil, stringutil,
) )
if pycompat.TYPE_CHECKING: if pycompat.TYPE_CHECKING:
from typing import ( from typing import (
Any, Any,
if msg: if msg:
raise error.Abort(msg) raise error.Abort(msg)
r = None r = None
if repo: if repo:
if cl: if cl:
r = repo.unfiltered().changelog r = repo.unfiltered().changelog
elif dir: elif dir:
if b'treemanifest' not in repo.requirements: if repository.TREEMANIFEST_REQUIREMENT not in repo.requirements:
raise error.Abort( raise error.Abort(
_( _(
b"--dir can only be used on repos with " b"--dir can only be used on repos with "
b"treemanifest enabled" b"treemanifest enabled"
) )
) )
if not dir.endswith(b'/'): if not dir.endswith(b'/'):
dir = dir + b'/' dir = dir + b'/'

mercurial/exchange.py

raise error.Abort(_(b'no common changegroup version')) raise error.Abort(_(b'no common changegroup version'))
version = max(cgversions) version = max(cgversions)
cgstream = changegroup.makestream( cgstream = changegroup.makestream(
pushop.repo, pushop.outgoing, version, b'push' pushop.repo, pushop.outgoing, version, b'push'
) )
cgpart = bundler.newpart(b'changegroup', data=cgstream) cgpart = bundler.newpart(b'changegroup', data=cgstream)
if cgversions: if cgversions:
cgpart.addparam(b'version', version) cgpart.addparam(b'version', version)
if b'treemanifest' in pushop.repo.requirements: if repository.TREEMANIFEST_REQUIREMENT in pushop.repo.requirements:
cgpart.addparam(b'treemanifest', b'1') cgpart.addparam(b'treemanifest', b'1')
if b'exp-sidedata-flag' in pushop.repo.requirements: if b'exp-sidedata-flag' in pushop.repo.requirements:
cgpart.addparam(b'exp-sidedata', b'1') cgpart.addparam(b'exp-sidedata', b'1')
def handlereply(op): def handlereply(op):
"""extract addchangegroup returns from server reply""" """extract addchangegroup returns from server reply"""
cgreplies = op.records.getreplies(cgpart.id) cgreplies = op.records.getreplies(cgpart.id)
assert len(cgreplies[b'changegroup']) == 1 assert len(cgreplies[b'changegroup']) == 1
) )
part = bundler.newpart(b'changegroup', data=cgstream) part = bundler.newpart(b'changegroup', data=cgstream)
if cgversions: if cgversions:
part.addparam(b'version', version) part.addparam(b'version', version)
part.addparam(b'nbchanges', b'%d' % len(outgoing.missing), mandatory=False) part.addparam(b'nbchanges', b'%d' % len(outgoing.missing), mandatory=False)
if b'treemanifest' in repo.requirements: if repository.TREEMANIFEST_REQUIREMENT in repo.requirements:
part.addparam(b'treemanifest', b'1') part.addparam(b'treemanifest', b'1')
if b'exp-sidedata-flag' in repo.requirements: if b'exp-sidedata-flag' in repo.requirements:
part.addparam(b'exp-sidedata', b'1') part.addparam(b'exp-sidedata', b'1')
if ( if (
kwargs.get('narrow', False) kwargs.get('narrow', False)
and kwargs.get('narrow_acl', False) and kwargs.get('narrow_acl', False)

mercurial/interfaces/repository.py

# Enables sparse working directory usage # Enables sparse working directory usage
SPARSE_REQUIREMENT = b'exp-sparse' SPARSE_REQUIREMENT = b'exp-sparse'
# Enables the internal phase which is used to hide changesets instead # Enables the internal phase which is used to hide changesets instead
# of stripping them # of stripping them
INTERNAL_PHASE_REQUIREMENT = b'internal-phase' INTERNAL_PHASE_REQUIREMENT = b'internal-phase'
# Stores manifest in Tree structure
TREEMANIFEST_REQUIREMENT = b'treemanifest'
# Local repository feature string. # Local repository feature string.
# Revlogs are being used for file storage. # Revlogs are being used for file storage.
REPO_FEATURE_REVLOG_FILE_STORAGE = b'revlogfilestorage' REPO_FEATURE_REVLOG_FILE_STORAGE = b'revlogfilestorage'
# The storage part of the repository is shared from an external source. # The storage part of the repository is shared from an external source.
REPO_FEATURE_SHARED_STORAGE = b'sharedstore' REPO_FEATURE_SHARED_STORAGE = b'sharedstore'
# LFS supported for backing file storage. # LFS supported for backing file storage.
REPO_FEATURE_LFS = b'lfs' REPO_FEATURE_LFS = b'lfs'

mercurial/localrepo.py

def resolvestorevfsoptions(ui, requirements, features): def resolvestorevfsoptions(ui, requirements, features):
"""Resolve the options to pass to the store vfs opener. """Resolve the options to pass to the store vfs opener.
The returned dict is used to influence behavior of the storage layer. The returned dict is used to influence behavior of the storage layer.
""" """
options = {} options = {}
if b'treemanifest' in requirements: if repository.TREEMANIFEST_REQUIREMENT in requirements:
options[b'treemanifest'] = True options[b'treemanifest'] = True
# experimental config: format.manifestcachesize # experimental config: format.manifestcachesize
manifestcachesize = ui.configint(b'format', b'manifestcachesize') manifestcachesize = ui.configint(b'format', b'manifestcachesize')
if manifestcachesize is not None: if manifestcachesize is not None:
options[b'manifestcachesize'] = manifestcachesize options[b'manifestcachesize'] = manifestcachesize
# In the absence of another requirement superseding a revlog-related # In the absence of another requirement superseding a revlog-related
# obsolete experimental requirements: # obsolete experimental requirements:
# - manifestv2: An experimental new manifest format that allowed # - manifestv2: An experimental new manifest format that allowed
# for stem compression of long paths. Experiment ended up not # for stem compression of long paths. Experiment ended up not
# being successful (repository sizes went up due to worse delta # being successful (repository sizes went up due to worse delta
# chains), and the code was deleted in 4.6. # chains), and the code was deleted in 4.6.
supportedformats = { supportedformats = {
b'revlogv1', b'revlogv1',
b'generaldelta', b'generaldelta',
b'treemanifest', repository.TREEMANIFEST_REQUIREMENT,
COPIESSDC_REQUIREMENT, COPIESSDC_REQUIREMENT,
REVLOGV2_REQUIREMENT, REVLOGV2_REQUIREMENT,
SIDEDATA_REQUIREMENT, SIDEDATA_REQUIREMENT,
SPARSEREVLOG_REQUIREMENT, SPARSEREVLOG_REQUIREMENT,
NODEMAP_REQUIREMENT, NODEMAP_REQUIREMENT,
bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT, bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT,
} }
_basesupported = supportedformats | { _basesupported = supportedformats | {
# experimental config: format.exp-use-side-data # experimental config: format.exp-use-side-data
if ui.configbool(b'format', b'exp-use-side-data'): if ui.configbool(b'format', b'exp-use-side-data'):
requirements.add(SIDEDATA_REQUIREMENT) requirements.add(SIDEDATA_REQUIREMENT)
# experimental config: format.exp-use-copies-side-data-changeset # experimental config: format.exp-use-copies-side-data-changeset
if ui.configbool(b'format', b'exp-use-copies-side-data-changeset'): if ui.configbool(b'format', b'exp-use-copies-side-data-changeset'):
requirements.add(SIDEDATA_REQUIREMENT) requirements.add(SIDEDATA_REQUIREMENT)
requirements.add(COPIESSDC_REQUIREMENT) requirements.add(COPIESSDC_REQUIREMENT)
if ui.configbool(b'experimental', b'treemanifest'): if ui.configbool(b'experimental', b'treemanifest'):
requirements.add(b'treemanifest') requirements.add(repository.TREEMANIFEST_REQUIREMENT)
revlogv2 = ui.config(b'experimental', b'revlogv2') revlogv2 = ui.config(b'experimental', b'revlogv2')
if revlogv2 == b'enable-unstable-format-and-corrupt-my-data': if revlogv2 == b'enable-unstable-format-and-corrupt-my-data':
requirements.remove(b'revlogv1') requirements.remove(b'revlogv1')
# generaldelta is implied by revlogv2. # generaldelta is implied by revlogv2.
requirements.discard(b'generaldelta') requirements.discard(b'generaldelta')
requirements.add(REVLOGV2_REQUIREMENT) requirements.add(REVLOGV2_REQUIREMENT)
# experimental config: format.internal-phase # experimental config: format.internal-phase

mercurial/repair.py

exchange, exchange,
obsolete, obsolete,
obsutil, obsutil,
pathutil, pathutil,
phases, phases,
pycompat, pycompat,
util, util,
) )
from .interfaces import repository
from .utils import ( from .utils import (
hashutil, hashutil,
stringutil, stringutil,
) )
def backupbundle( def backupbundle(
repo, bases, heads, node, suffix, compress=True, obsolescence=True repo, bases, heads, node, suffix, compress=True, obsolescence=True
def stripmanifest(repo, striprev, tr, files): def stripmanifest(repo, striprev, tr, files):
for revlog in manifestrevlogs(repo): for revlog in manifestrevlogs(repo):
revlog.strip(striprev, tr) revlog.strip(striprev, tr)
def manifestrevlogs(repo): def manifestrevlogs(repo):
yield repo.manifestlog.getstorage(b'') yield repo.manifestlog.getstorage(b'')
if b'treemanifest' in repo.requirements: if repository.TREEMANIFEST_REQUIREMENT in repo.requirements:
# This logic is safe if treemanifest isn't enabled, but also # This logic is safe if treemanifest isn't enabled, but also
# pointless, so we skip it if treemanifest isn't enabled. # pointless, so we skip it if treemanifest isn't enabled.
for unencoded, encoded, size in repo.store.datafiles(): for unencoded, encoded, size in repo.store.datafiles():
if unencoded.startswith(b'meta/') and unencoded.endswith( if unencoded.startswith(b'meta/') and unencoded.endswith(
b'00manifest.i' b'00manifest.i'
): ):
dir = unencoded[5:-12] dir = unencoded[5:-12]
yield repo.manifestlog.getstorage(dir) yield repo.manifestlog.getstorage(dir)
if repo.store._exists(i): if repo.store._exists(i):
newentries.add(i) newentries.add(i)
if repo.store._exists(d): if repo.store._exists(d):
newentries.add(d) newentries.add(d)
progress.complete() progress.complete()
if b'treemanifest' in repo.requirements: if repository.TREEMANIFEST_REQUIREMENT in repo.requirements:
# This logic is safe if treemanifest isn't enabled, but also # This logic is safe if treemanifest isn't enabled, but also
# pointless, so we skip it if treemanifest isn't enabled. # pointless, so we skip it if treemanifest isn't enabled.
for dir in pathutil.dirs(seenfiles): for dir in pathutil.dirs(seenfiles):
i = b'meta/%s/00manifest.i' % dir i = b'meta/%s/00manifest.i' % dir
d = b'meta/%s/00manifest.d' % dir d = b'meta/%s/00manifest.d' % dir
if repo.store._exists(i): if repo.store._exists(i):
newentries.add(i) newentries.add(i)

mercurial/upgrade.py

metadata, metadata,
pycompat, pycompat,
revlog, revlog,
scmutil, scmutil,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
from .interfaces import repository
from .utils import compression from .utils import compression
# list of requirements that request a clone of all revlog if added/removed # list of requirements that request a clone of all revlog if added/removed
RECLONES_REQUIREMENTS = { RECLONES_REQUIREMENTS = {
b'generaldelta', b'generaldelta',
localrepo.SPARSEREVLOG_REQUIREMENT, localrepo.SPARSEREVLOG_REQUIREMENT,
} }
"""Obtain requirements that will prevent an upgrade from occurring. """Obtain requirements that will prevent an upgrade from occurring.
An upgrade cannot be performed if the source repository contains a An upgrade cannot be performed if the source repository contains a
requirements in the returned set. requirements in the returned set.
""" """
return { return {
# The upgrade code does not yet support these experimental features. # The upgrade code does not yet support these experimental features.
# This is an artificial limitation. # This is an artificial limitation.
b'treemanifest', repository.TREEMANIFEST_REQUIREMENT,
# This was a precursor to generaldelta and was never enabled by default. # This was a precursor to generaldelta and was never enabled by default.
# It should (hopefully) not exist in the wild. # It should (hopefully) not exist in the wild.
b'parentdelta', b'parentdelta',
# Upgrade should operate on the actual store, not the shared link. # Upgrade should operate on the actual store, not the shared link.
b'shared', b'shared',
} }