This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10316

upgrade: take advantage of the new information returned by `store.walk`
ClosedPublic

Authored by marmoute on Apr 6 2021, 4:39 AM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHGcf49e54ef965: upgrade: take advantage of the new information returned by `store.walk`
Summary

Before this change the upgrade code had to analyse filename to process them directly. Lets keep that logic private to the store and more to a more robust explicit approach.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.Apr 6 2021, 4:39 AM
pulkit accepted this revision.Apr 8 2021, 3:05 AM
This revision is now accepted and ready to land.Apr 8 2021, 3:05 AM
baymax updated this revision to Diff 26702.Apr 8 2021, 7:02 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 26705.Apr 8 2021, 10:09 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

pulkit accepted this revision.Apr 8 2021, 12:53 PM
baymax updated this revision to Diff 26734.Apr 8 2021, 7:54 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

pulkit accepted this revision.Apr 11 2021, 5:25 PM

Revision Contents
Changeset List

PathPackages
Mmercurial/upgrade_utils/engine.py (68 lines)

Diff 26808

mercurial/upgrade_utils/engine.py

error, error,
filelog, filelog,
manifest, manifest,
metadata, metadata,
pycompat, pycompat,
requirements, requirements,
revlog, revlog,
scmutil, scmutil,
store,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
from ..revlogutils import nodemap from ..revlogutils import nodemap
def _revlogfrompath(repo, path): def _revlogfrompath(repo, rl_type, path):
"""Obtain a revlog from a repo path. """Obtain a revlog from a repo path.
An instance of the appropriate class is returned. An instance of the appropriate class is returned.
""" """
if path == b'00changelog.i': if rl_type & store.FILEFLAGS_CHANGELOG:
return changelog.changelog(repo.svfs) return changelog.changelog(repo.svfs)
elif path.endswith(b'00manifest.i'): elif rl_type & store.FILEFLAGS_MANIFESTLOG:
mandir = path[: -len(b'00manifest.i')] mandir = b''
if b'/' in path:
mandir = path.rsplit(b'/', 1)[0]
return manifest.manifestrevlog( return manifest.manifestrevlog(
repo.nodeconstants, repo.svfs, tree=mandir repo.nodeconstants, repo.svfs, tree=mandir
) )
else: else:
# reverse of "/".join(("data", path + ".i")) # drop the extension and the `data/` prefix
return filelog.filelog(repo.svfs, path[5:-2]) path = path.rsplit(b'.', 1)[0].split(b'/', 1)[1]
return filelog.filelog(repo.svfs, path)
def _copyrevlog(tr, destrepo, oldrl, unencodedname): def _copyrevlog(tr, destrepo, oldrl, rl_type, unencodedname):
"""copy all relevant files for `oldrl` into `destrepo` store """copy all relevant files for `oldrl` into `destrepo` store
Files are copied "as is" without any transformation. The copy is performed Files are copied "as is" without any transformation. The copy is performed
without extra checks. Callers are responsible for making sure the copied without extra checks. Callers are responsible for making sure the copied
content is compatible with format of the destination repository. content is compatible with format of the destination repository.
""" """
oldrl = getattr(oldrl, '_revlog', oldrl) oldrl = getattr(oldrl, '_revlog', oldrl)
newrl = _revlogfrompath(destrepo, unencodedname) newrl = _revlogfrompath(destrepo, rl_type, unencodedname)
newrl = getattr(newrl, '_revlog', newrl) newrl = getattr(newrl, '_revlog', newrl)
oldvfs = oldrl.opener oldvfs = oldrl.opener
newvfs = newrl.opener newvfs = newrl.opener
oldindex = oldvfs.join(oldrl.indexfile) oldindex = oldvfs.join(oldrl.indexfile)
newindex = newvfs.join(newrl.indexfile) newindex = newvfs.join(newrl.indexfile)
olddata = oldvfs.join(oldrl.datafile) olddata = oldvfs.join(oldrl.datafile)
newdata = newvfs.join(newrl.datafile) newdata = newvfs.join(newrl.datafile)
with newvfs(newrl.indexfile, b'w'): with newvfs(newrl.indexfile, b'w'):
pass # create all the directories pass # create all the directories
util.copyfile(oldindex, newindex) util.copyfile(oldindex, newindex)
copydata = oldrl.opener.exists(oldrl.datafile) copydata = oldrl.opener.exists(oldrl.datafile)
if copydata: if copydata:
util.copyfile(olddata, newdata) util.copyfile(olddata, newdata)
if not ( if rl_type & store.FILEFLAGS_FILELOG:
unencodedname.endswith(b'00changelog.i')
or unencodedname.endswith(b'00manifest.i')
):
destrepo.svfs.fncache.add(unencodedname) destrepo.svfs.fncache.add(unencodedname)
if copydata: if copydata:
destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d') destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d')
UPGRADE_CHANGELOG = b"changelog" UPGRADE_CHANGELOG = b"changelog"
UPGRADE_MANIFEST = b"manifest" UPGRADE_MANIFEST = b"manifest"
UPGRADE_FILELOGS = b"all-filelogs" UPGRADE_FILELOGS = b"all-filelogs"
elif requirements.COPIESSDC_REQUIREMENT in addedreqs: elif requirements.COPIESSDC_REQUIREMENT in addedreqs:
sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo) sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo)
elif requirements.COPIESSDC_REQUIREMENT in removedreqs: elif requirements.COPIESSDC_REQUIREMENT in removedreqs:
sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo) sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo)
return sidedatacompanion return sidedatacompanion
def matchrevlog(revlogfilter, entry): def matchrevlog(revlogfilter, rl_type):
"""check if a revlog is selected for cloning. """check if a revlog is selected for cloning.
In other words, are there any updates which need to be done on revlog In other words, are there any updates which need to be done on revlog
or it can be blindly copied. or it can be blindly copied.
The store entry is checked against the passed filter""" The store entry is checked against the passed filter"""
if entry.endswith(b'00changelog.i'): if rl_type & store.FILEFLAGS_CHANGELOG:
return UPGRADE_CHANGELOG in revlogfilter return UPGRADE_CHANGELOG in revlogfilter
elif entry.endswith(b'00manifest.i'): elif rl_type & store.FILEFLAGS_MANIFESTLOG:
return UPGRADE_MANIFEST in revlogfilter return UPGRADE_MANIFEST in revlogfilter
assert rl_type & store.FILEFLAGS_FILELOG
return UPGRADE_FILELOGS in revlogfilter return UPGRADE_FILELOGS in revlogfilter
def _perform_clone( def _perform_clone(
ui, ui,
dstrepo, dstrepo,
tr, tr,
old_revlog, old_revlog,
rl_type,
unencoded, unencoded,
upgrade_op, upgrade_op,
sidedatacompanion, sidedatacompanion,
oncopiedrevision, oncopiedrevision,
): ):
""" returns the new revlog object created""" """ returns the new revlog object created"""
newrl = None newrl = None
if matchrevlog(upgrade_op.revlogs_to_process, unencoded): if matchrevlog(upgrade_op.revlogs_to_process, rl_type):
ui.note( ui.note(
_(b'cloning %d revisions from %s\n') % (len(old_revlog), unencoded) _(b'cloning %d revisions from %s\n') % (len(old_revlog), unencoded)
) )
newrl = _revlogfrompath(dstrepo, unencoded) newrl = _revlogfrompath(dstrepo, rl_type, unencoded)
old_revlog.clone( old_revlog.clone(
tr, tr,
newrl, newrl,
addrevisioncb=oncopiedrevision, addrevisioncb=oncopiedrevision,
deltareuse=upgrade_op.delta_reuse_mode, deltareuse=upgrade_op.delta_reuse_mode,
forcedeltabothparents=upgrade_op.force_re_delta_both_parents, forcedeltabothparents=upgrade_op.force_re_delta_both_parents,
sidedatacompanion=sidedatacompanion, sidedatacompanion=sidedatacompanion,
) )
else: else:
msg = _(b'blindly copying %s containing %i revisions\n') msg = _(b'blindly copying %s containing %i revisions\n')
ui.note(msg % (unencoded, len(old_revlog))) ui.note(msg % (unencoded, len(old_revlog)))
_copyrevlog(tr, dstrepo, old_revlog, unencoded) _copyrevlog(tr, dstrepo, old_revlog, rl_type, unencoded)
newrl = _revlogfrompath(dstrepo, unencoded) newrl = _revlogfrompath(dstrepo, rl_type, unencoded)
return newrl return newrl
def _clonerevlogs( def _clonerevlogs(
ui, ui,
srcrepo, srcrepo,
dstrepo, dstrepo,
tr, tr,
# key is unencoded filename # key is unencoded filename
# value is revlog_object_from_srcrepo # value is revlog_object_from_srcrepo
manifests = {} manifests = {}
changelogs = {} changelogs = {}
filelogs = {} filelogs = {}
# Perform a pass to collect metadata. This validates we can open all # Perform a pass to collect metadata. This validates we can open all
# source files and allows a unified progress bar to be displayed. # source files and allows a unified progress bar to be displayed.
for revlog_type, unencoded, encoded, size in alldatafiles: for rl_type, unencoded, encoded, size in alldatafiles:
if not unencoded.endswith(b'.i'): if not rl_type & store.FILEFLAGS_REVLOG_MAIN:
continue continue
rl = _revlogfrompath(srcrepo, unencoded) rl = _revlogfrompath(srcrepo, rl_type, unencoded)
info = rl.storageinfo( info = rl.storageinfo(
exclusivefiles=True, exclusivefiles=True,
revisionscount=True, revisionscount=True,
trackedsize=True, trackedsize=True,
storedsize=True, storedsize=True,
) )
revcount += info[b'revisionscount'] or 0 revcount += info[b'revisionscount'] or 0
datasize = info[b'storedsize'] or 0 datasize = info[b'storedsize'] or 0
rawsize = info[b'trackedsize'] or 0 rawsize = info[b'trackedsize'] or 0
srcsize += datasize srcsize += datasize
srcrawsize += rawsize srcrawsize += rawsize
# This is for the separate progress bars. # This is for the separate progress bars.
if isinstance(rl, changelog.changelog): if rl_type & store.FILEFLAGS_CHANGELOG:
changelogs[unencoded] = rl changelogs[unencoded] = (rl_type, rl)
crevcount += len(rl) crevcount += len(rl)
csrcsize += datasize csrcsize += datasize
crawsize += rawsize crawsize += rawsize
elif isinstance(rl, manifest.manifestrevlog): elif rl_type & store.FILEFLAGS_MANIFESTLOG:
manifests[unencoded] = rl manifests[unencoded] = (rl_type, rl)
mcount += 1 mcount += 1
mrevcount += len(rl) mrevcount += len(rl)
msrcsize += datasize msrcsize += datasize
mrawsize += rawsize mrawsize += rawsize
elif isinstance(rl, filelog.filelog): elif rl_type & store.FILEFLAGS_FILELOG:
filelogs[unencoded] = rl filelogs[unencoded] = (rl_type, rl)
fcount += 1 fcount += 1
frevcount += len(rl) frevcount += len(rl)
fsrcsize += datasize fsrcsize += datasize
frawsize += rawsize frawsize += rawsize
else: else:
error.ProgrammingError(b'unknown revlog type') error.ProgrammingError(b'unknown revlog type')
if not revcount: if not revcount:
% ( % (
fcount, fcount,
frevcount, frevcount,
util.bytecount(fsrcsize), util.bytecount(fsrcsize),
util.bytecount(frawsize), util.bytecount(frawsize),
) )
) )
progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount) progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount)
for unencoded, oldrl in sorted(filelogs.items()): for unencoded, (rl_type, oldrl) in sorted(filelogs.items()):
newrl = _perform_clone( newrl = _perform_clone(
ui, ui,
dstrepo, dstrepo,
tr, tr,
oldrl, oldrl,
rl_type,
unencoded, unencoded,
upgrade_op, upgrade_op,
sidedatacompanion, sidedatacompanion,
oncopiedrevision, oncopiedrevision,
) )
info = newrl.storageinfo(storedsize=True) info = newrl.storageinfo(storedsize=True)
fdstsize += info[b'storedsize'] or 0 fdstsize += info[b'storedsize'] or 0
ui.status( ui.status(
util.bytecount(mrawsize), util.bytecount(mrawsize),
) )
) )
if progress: if progress:
progress.complete() progress.complete()
progress = srcrepo.ui.makeprogress( progress = srcrepo.ui.makeprogress(
_(b'manifest revisions'), total=mrevcount _(b'manifest revisions'), total=mrevcount
) )
for unencoded, oldrl in sorted(manifests.items()): for unencoded, (rl_type, oldrl) in sorted(manifests.items()):
newrl = _perform_clone( newrl = _perform_clone(
ui, ui,
dstrepo, dstrepo,
tr, tr,
oldrl, oldrl,
rl_type,
unencoded, unencoded,
upgrade_op, upgrade_op,
sidedatacompanion, sidedatacompanion,
oncopiedrevision, oncopiedrevision,
) )
info = newrl.storageinfo(storedsize=True) info = newrl.storageinfo(storedsize=True)
mdstsize += info[b'storedsize'] or 0 mdstsize += info[b'storedsize'] or 0
ui.status( ui.status(
util.bytecount(crawsize), util.bytecount(crawsize),
) )
) )
if progress: if progress:
progress.complete() progress.complete()
progress = srcrepo.ui.makeprogress( progress = srcrepo.ui.makeprogress(
_(b'changelog revisions'), total=crevcount _(b'changelog revisions'), total=crevcount
) )
for unencoded, oldrl in sorted(changelogs.items()): for unencoded, (rl_type, oldrl) in sorted(changelogs.items()):
newrl = _perform_clone( newrl = _perform_clone(
ui, ui,
dstrepo, dstrepo,
tr, tr,
oldrl, oldrl,
rl_type,
unencoded, unencoded,
upgrade_op, upgrade_op,
sidedatacompanion, sidedatacompanion,
oncopiedrevision, oncopiedrevision,
) )
info = newrl.storageinfo(storedsize=True) info = newrl.storageinfo(storedsize=True)
cdstsize += info[b'storedsize'] or 0 cdstsize += info[b'storedsize'] or 0
progress.complete() progress.complete()