This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5166

localrepo: support marking repos as having shallow file storage
ClosedPublic

Authored by indygreg on Oct 19 2018, 1:00 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG7e3b6c4f01a2: localrepo: support marking repos as having shallow file storage
Summary

Various operations against repositories need to know if repository
storage is full or partial. For example, a checkout (including possibly
a widening of a sparse checkout), needs to know if it can assume all file
revisions are available or whether to look for missing revisions first.

This commit lays the plumbing for doing that.

We define a repo creation option that indicates that shallow file storage
is desired.

The SQLite store uses this creation option to add an extra repo requirement
indicating file storage is shallow.

A new repository feature has been added to indicate that file storage is
shallow. The SQLite store adds this feature when the shallow file store
requirement is present.

Code can now look at repo.features to determine if repo file storage may
be shallow and take additional actions if so.

While we're here, we also teach the SQLite store to handle the narrow repo
requirement, which gets added when making narrow clones.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

PathPackages
Mhgext/sqlitestore.py (14 lines)
Mmercurial/hg.py (3 lines)
Mmercurial/localrepo.py (4 lines)
Mmercurial/repository.py (2 lines)

Diff 12302

hgext/sqlitestore.py

# be specifying the version(s) of Mercurial they are tested with, or # be specifying the version(s) of Mercurial they are tested with, or
# leave the attribute unspecified. # leave the attribute unspecified.
testedwith = 'ships-with-hg-core' testedwith = 'ships-with-hg-core'
REQUIREMENT = b'exp-sqlite-001' REQUIREMENT = b'exp-sqlite-001'
REQUIREMENT_ZSTD = b'exp-sqlite-comp-001=zstd' REQUIREMENT_ZSTD = b'exp-sqlite-comp-001=zstd'
REQUIREMENT_ZLIB = b'exp-sqlite-comp-001=zlib' REQUIREMENT_ZLIB = b'exp-sqlite-comp-001=zlib'
REQUIREMENT_NONE = b'exp-sqlite-comp-001=none' REQUIREMENT_NONE = b'exp-sqlite-comp-001=none'
REQUIREMENT_SHALLOW_FILES = b'exp-sqlite-shallow-files'
CURRENT_SCHEMA_VERSION = 1 CURRENT_SCHEMA_VERSION = 1
COMPRESSION_NONE = 1 COMPRESSION_NONE = 1
COMPRESSION_ZSTD = 2 COMPRESSION_ZSTD = 2
COMPRESSION_ZLIB = 3 COMPRESSION_ZLIB = 3
FLAG_CENSORED = 1 FLAG_CENSORED = 1
def featuresetup(ui, supported): def featuresetup(ui, supported):
supported.add(REQUIREMENT) supported.add(REQUIREMENT)
if zstd: if zstd:
supported.add(REQUIREMENT_ZSTD) supported.add(REQUIREMENT_ZSTD)
supported.add(REQUIREMENT_ZLIB) supported.add(REQUIREMENT_ZLIB)
supported.add(REQUIREMENT_NONE) supported.add(REQUIREMENT_NONE)
supported.add(REQUIREMENT_SHALLOW_FILES)
supported.add(repository.NARROW_REQUIREMENT)
def newreporequirements(orig, ui, createopts): def newreporequirements(orig, ui, createopts):
if createopts['backend'] != 'sqlite': if createopts['backend'] != 'sqlite':
return orig(ui, createopts) return orig(ui, createopts)
# This restriction can be lifted once we have more confidence. # This restriction can be lifted once we have more confidence.
if 'sharedrepo' in createopts: if 'sharedrepo' in createopts:
raise error.Abort(_('shared repositories not supported with SQLite ' raise error.Abort(_('shared repositories not supported with SQLite '
'store')) 'store'))
# This filtering is out of an abundance of caution: we want to ensure # This filtering is out of an abundance of caution: we want to ensure
# we honor creation options and we do that by annotating exactly the # we honor creation options and we do that by annotating exactly the
# creation options we recognize. # creation options we recognize.
known = { known = {
'narrowfiles', 'narrowfiles',
'backend', 'backend',
'shallowfilestore',
} }
unsupported = set(createopts) - known unsupported = set(createopts) - known
if unsupported: if unsupported:
raise error.Abort(_('SQLite store does not support repo creation ' raise error.Abort(_('SQLite store does not support repo creation '
'option: %s') % ', '.join(sorted(unsupported))) 'option: %s') % ', '.join(sorted(unsupported)))
# Since we're a hybrid store that still relies on revlogs, we fall back # Since we're a hybrid store that still relies on revlogs, we fall back
elif compression == 'zlib': elif compression == 'zlib':
requirements.add(REQUIREMENT_ZLIB) requirements.add(REQUIREMENT_ZLIB)
elif compression == 'none': elif compression == 'none':
requirements.add(REQUIREMENT_NONE) requirements.add(REQUIREMENT_NONE)
else: else:
raise error.Abort(_('unknown compression engine defined in ' raise error.Abort(_('unknown compression engine defined in '
'storage.sqlite.compression: %s') % compression) 'storage.sqlite.compression: %s') % compression)
if createopts.get('shallowfilestore'):
requirements.add(REQUIREMENT_SHALLOW_FILES)
return requirements return requirements
@interfaceutil.implementer(repository.ilocalrepositoryfilestorage) @interfaceutil.implementer(repository.ilocalrepositoryfilestorage)
class sqlitefilestorage(object): class sqlitefilestorage(object):
"""Repository file storage backed by SQLite.""" """Repository file storage backed by SQLite."""
def file(self, path): def file(self, path):
if path[0] == b'/': if path[0] == b'/':
path = path[1:] path = path[1:]
if REQUIREMENT_ZSTD in self.requirements: if REQUIREMENT_ZSTD in self.requirements:
compression = 'zstd' compression = 'zstd'
elif REQUIREMENT_ZLIB in self.requirements: elif REQUIREMENT_ZLIB in self.requirements:
compression = 'zlib' compression = 'zlib'
elif REQUIREMENT_NONE in self.requirements: elif REQUIREMENT_NONE in self.requirements:
compression = 'none' compression = 'none'
else: else:
raise error.Abort(_('unable to determine what compression engine ' raise error.Abort(_('unable to determine what compression engine '
'to use for SQLite storage')) 'to use for SQLite storage'))
return sqlitefilestore(self._dbconn, path, compression) return sqlitefilestore(self._dbconn, path, compression)
def makefilestorage(orig, requirements, **kwargs): def makefilestorage(orig, requirements, features, **kwargs):
"""Produce a type conforming to ``ilocalrepositoryfilestorage``.""" """Produce a type conforming to ``ilocalrepositoryfilestorage``."""
if REQUIREMENT in requirements: if REQUIREMENT in requirements:
if REQUIREMENT_SHALLOW_FILES in requirements:
features.add(repository.REPO_FEATURE_SHALLOW_FILE_STORAGE)
return sqlitefilestorage return sqlitefilestorage
else: else:
return orig(requirements=requirements, **kwargs) return orig(requirements=requirements, features=features, **kwargs)
def makemain(orig, ui, requirements, **kwargs): def makemain(orig, ui, requirements, **kwargs):
if REQUIREMENT in requirements: if REQUIREMENT in requirements:
if REQUIREMENT_ZSTD in requirements and not zstd: if REQUIREMENT_ZSTD in requirements and not zstd:
raise error.Abort(_('repository uses zstandard compression, which ' raise error.Abort(_('repository uses zstandard compression, which '
'is not available to this Mercurial install')) 'is not available to this Mercurial install'))
return sqliterepository return sqliterepository

mercurial/hg.py

if narrow: if narrow:
# Include everything by default if only exclusion patterns defined. # Include everything by default if only exclusion patterns defined.
if storeexcludepats and not storeincludepats: if storeexcludepats and not storeincludepats:
storeincludepats = {'path:.'} storeincludepats = {'path:.'}
createopts['narrowfiles'] = True createopts['narrowfiles'] = True
if depth:
createopts['shallowfilestore'] = True
if srcpeer.capable(b'lfs-serve'): if srcpeer.capable(b'lfs-serve'):
# Repository creation honors the config if it disabled the extension, so # Repository creation honors the config if it disabled the extension, so
# we can't just announce that lfs will be enabled. This check avoids # we can't just announce that lfs will be enabled. This check avoids
# saying that lfs will be enabled, and then saying it's an unknown # saying that lfs will be enabled, and then saying it's an unknown
# feature. The lfs creation option is set in either case so that a # feature. The lfs creation option is set in either case so that a
# requirement is added. If the extension is explicitly disabled but the # requirement is added. If the extension is explicitly disabled but the
# requirement is set, the clone aborts early, before transferring any # requirement is set, the clone aborts early, before transferring any
# data. # data.

mercurial/localrepo.py

""" """
known = { known = {
'backend', 'backend',
'lfs', 'lfs',
'narrowfiles', 'narrowfiles',
'sharedrepo', 'sharedrepo',
'sharedrelative', 'sharedrelative',
'shareditems', 'shareditems',
'shallowfilestore',
} }
return {k: v for k, v in createopts.items() if k not in known} return {k: v for k, v in createopts.items() if k not in known}
def createrepository(ui, path, createopts=None): def createrepository(ui, path, createopts=None):
"""Create a new repository in a vfs. """Create a new repository in a vfs.
``path`` path to the new repo's working directory. ``path`` path to the new repo's working directory.
sharedrepo sharedrepo
Repository object from which storage should be shared. Repository object from which storage should be shared.
sharedrelative sharedrelative
Boolean indicating if the path to the shared repo should be Boolean indicating if the path to the shared repo should be
stored as relative. By default, the pointer to the "parent" repo stored as relative. By default, the pointer to the "parent" repo
is stored as an absolute path. is stored as an absolute path.
shareditems shareditems
Set of items to share to the new repository (in addition to storage). Set of items to share to the new repository (in addition to storage).
shallowfilestore
Indicates that storage for files should be shallow (not all ancestor
revisions are known).
""" """
createopts = defaultcreateopts(ui, createopts=createopts) createopts = defaultcreateopts(ui, createopts=createopts)
unknownopts = filterknowncreateopts(ui, createopts) unknownopts = filterknowncreateopts(ui, createopts)
if not isinstance(unknownopts, dict): if not isinstance(unknownopts, dict):
raise error.ProgrammingError('filterknowncreateopts() did not return ' raise error.ProgrammingError('filterknowncreateopts() did not return '
'a dict') 'a dict')

mercurial/repository.py

# Revlogs are being used for file storage. # Revlogs are being used for file storage.
REPO_FEATURE_REVLOG_FILE_STORAGE = b'revlogfilestorage' REPO_FEATURE_REVLOG_FILE_STORAGE = b'revlogfilestorage'
# The storage part of the repository is shared from an external source. # The storage part of the repository is shared from an external source.
REPO_FEATURE_SHARED_STORAGE = b'sharedstore' REPO_FEATURE_SHARED_STORAGE = b'sharedstore'
# LFS supported for backing file storage. # LFS supported for backing file storage.
REPO_FEATURE_LFS = b'lfs' REPO_FEATURE_LFS = b'lfs'
# Repository supports being stream cloned. # Repository supports being stream cloned.
REPO_FEATURE_STREAM_CLONE = b'streamclone' REPO_FEATURE_STREAM_CLONE = b'streamclone'
# Files storage may lack data for all ancestors.
REPO_FEATURE_SHALLOW_FILE_STORAGE = b'shallowfilestorage'
REVISION_FLAG_CENSORED = 1 << 15 REVISION_FLAG_CENSORED = 1 << 15
REVISION_FLAG_ELLIPSIS = 1 << 14 REVISION_FLAG_ELLIPSIS = 1 << 14
REVISION_FLAG_EXTSTORED = 1 << 13 REVISION_FLAG_EXTSTORED = 1 << 13
REVISION_FLAGS_KNOWN = ( REVISION_FLAGS_KNOWN = (
REVISION_FLAG_CENSORED | REVISION_FLAG_ELLIPSIS | REVISION_FLAG_EXTSTORED) REVISION_FLAG_CENSORED | REVISION_FLAG_ELLIPSIS | REVISION_FLAG_EXTSTORED)