This is an archive of the discontinued Mercurial Phabricator instance.

Differential D9580

engine: unwrap a hard to understand for loop
ClosedPublic

Authored by pulkit on Dec 14 2020, 4:55 AM.

Details

Reviewers
marmoute
mharbison72
Group Reviewers
hg-reviewers
Commits
rHG4d1cec4e5e1f: engine: unwrap a hard to understand for loop
Summary

The loop was iterating over all the datafiles and maintaining a set to check
whether filelogs have been processed, manifests have been processed or not.

The code was hard to understand and it assumed that alldatafiles are ordered
in a certain way.

This refactors the for loop in separate parts for each manifests, changelog and
filelogs. This will also help in future work where we will like more better
handling on whether we want to upgrade filelogs or not.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

pulkit created this revision.Dec 14 2020, 4:55 AM
marmoute accepted this revision.Dec 22 2020, 3:39 AM
marmoute added a subscriber: marmoute.

looks like a good change.

pulkit updated this revision to Diff 24540.Dec 30 2020, 5:31 AM
mharbison72 accepted this revision.Jan 5 2021, 10:34 AM
This revision is now accepted and ready to land.Jan 5 2021, 10:34 AM
pulkit added a commit: rHG4d1cec4e5e1f: engine: unwrap a hard to understand for loop.Jan 5 2021, 10:50 AM
Closed by commit rHG4d1cec4e5e1f: engine: unwrap a hard to understand for loop (authored by pulkit). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/upgrade_utils/engine.py (196 lines)

Diff 24608

mercurial/upgrade_utils/engine.py

mrawsize = 0 mrawsize = 0
mdstsize = 0 mdstsize = 0
crevcount = 0 crevcount = 0
csrcsize = 0 csrcsize = 0
crawsize = 0 crawsize = 0
cdstsize = 0 cdstsize = 0
alldatafiles = list(srcrepo.store.walk()) alldatafiles = list(srcrepo.store.walk())
# mapping of data files which needs to be cloned
# key is unencoded filename
# value is revlog_object_from_srcrepo
manifests = {}
changelogs = {}
filelogs = {}
# Perform a pass to collect metadata. This validates we can open all # Perform a pass to collect metadata. This validates we can open all
# source files and allows a unified progress bar to be displayed. # source files and allows a unified progress bar to be displayed.
for unencoded, encoded, size in alldatafiles: for unencoded, encoded, size in alldatafiles:
if unencoded.endswith(b'.d'): if unencoded.endswith(b'.d'):
continue continue
rl = _revlogfrompath(srcrepo, unencoded) rl = _revlogfrompath(srcrepo, unencoded)
datasize = info[b'storedsize'] or 0 datasize = info[b'storedsize'] or 0
rawsize = info[b'trackedsize'] or 0 rawsize = info[b'trackedsize'] or 0
srcsize += datasize srcsize += datasize
srcrawsize += rawsize srcrawsize += rawsize
# This is for the separate progress bars. # This is for the separate progress bars.
if isinstance(rl, changelog.changelog): if isinstance(rl, changelog.changelog):
changelogs[unencoded] = rl
crevcount += len(rl) crevcount += len(rl)
csrcsize += datasize csrcsize += datasize
crawsize += rawsize crawsize += rawsize
elif isinstance(rl, manifest.manifestrevlog): elif isinstance(rl, manifest.manifestrevlog):
manifests[unencoded] = rl
mcount += 1 mcount += 1
mrevcount += len(rl) mrevcount += len(rl)
msrcsize += datasize msrcsize += datasize
mrawsize += rawsize mrawsize += rawsize
elif isinstance(rl, filelog.filelog): elif isinstance(rl, filelog.filelog):
filelogs[unencoded] = rl
fcount += 1 fcount += 1
frevcount += len(rl) frevcount += len(rl)
fsrcsize += datasize fsrcsize += datasize
frawsize += rawsize frawsize += rawsize
else: else:
error.ProgrammingError(b'unknown revlog type') error.ProgrammingError(b'unknown revlog type')
if not revcount: if not revcount:
# Used to keep track of progress. # Used to keep track of progress.
progress = None progress = None
def oncopiedrevision(rl, rev, node): def oncopiedrevision(rl, rev, node):
progress.increment() progress.increment()
sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo) sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo)
# Do the actual copying. # Migrating filelogs
# FUTURE this operation can be farmed off to worker processes.
seen = set()
for unencoded, encoded, size in alldatafiles:
if unencoded.endswith(b'.d'):
continue
oldrl = _revlogfrompath(srcrepo, unencoded)
if isinstance(oldrl, changelog.changelog) and b'c' not in seen:
ui.status( ui.status(
_( _(
b'finished migrating %d manifest revisions across %d ' b'migrating %d filelogs containing %d revisions '
b'manifests; change in size: %s\n'
)
% (mrevcount, mcount, util.bytecount(mdstsize - msrcsize))
)
ui.status(
_(
b'migrating changelog containing %d revisions '
b'(%s in store; %s tracked data)\n' b'(%s in store; %s tracked data)\n'
) )
% ( % (
crevcount, fcount,
util.bytecount(csrcsize), frevcount,
util.bytecount(crawsize), util.bytecount(fsrcsize),
util.bytecount(frawsize),
) )
) )
seen.add(b'c') progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount)
progress = srcrepo.ui.makeprogress( for unencoded, oldrl in sorted(filelogs.items()):
_(b'changelog revisions'), total=crevcount newrl = _perform_clone(
ui,
dstrepo,
tr,
oldrl,
unencoded,
deltareuse,
forcedeltabothparents,
revlogs,
sidedatacompanion,
oncopiedrevision,
) )
elif isinstance(oldrl, manifest.manifestrevlog) and b'm' not in seen: info = newrl.storageinfo(storedsize=True)
datasize = info[b'storedsize'] or 0
dstsize += datasize
fdstsize += datasize
ui.status( ui.status(
_( _(
b'finished migrating %d filelog revisions across %d ' b'finished migrating %d filelog revisions across %d '
b'filelogs; change in size: %s\n' b'filelogs; change in size: %s\n'
) )
% (frevcount, fcount, util.bytecount(fdstsize - fsrcsize)) % (frevcount, fcount, util.bytecount(fdstsize - fsrcsize))
) )
# Migrating manifests
ui.status( ui.status(
_( _(
b'migrating %d manifests containing %d revisions ' b'migrating %d manifests containing %d revisions '
b'(%s in store; %s tracked data)\n' b'(%s in store; %s tracked data)\n'
) )
% ( % (
mcount, mcount,
mrevcount, mrevcount,
util.bytecount(msrcsize), util.bytecount(msrcsize),
util.bytecount(mrawsize), util.bytecount(mrawsize),
) )
) )
seen.add(b'm')
if progress: if progress:
progress.complete() progress.complete()
progress = srcrepo.ui.makeprogress( progress = srcrepo.ui.makeprogress(
_(b'manifest revisions'), total=mrevcount _(b'manifest revisions'), total=mrevcount
) )
elif b'f' not in seen: for unencoded, oldrl in sorted(manifests.items()):
newrl = _perform_clone(
ui,
dstrepo,
tr,
oldrl,
unencoded,
deltareuse,
forcedeltabothparents,
revlogs,
sidedatacompanion,
oncopiedrevision,
)
info = newrl.storageinfo(storedsize=True)
datasize = info[b'storedsize'] or 0
dstsize += datasize
mdstsize += datasize
ui.status( ui.status(
_( _(
b'migrating %d filelogs containing %d revisions ' b'finished migrating %d manifest revisions across %d '
b'manifests; change in size: %s\n'
)
% (mrevcount, mcount, util.bytecount(mdstsize - msrcsize))
)
# Migrating changelog
ui.status(
_(
b'migrating changelog containing %d revisions '
b'(%s in store; %s tracked data)\n' b'(%s in store; %s tracked data)\n'
) )
% ( % (
fcount, crevcount,
frevcount, util.bytecount(csrcsize),
util.bytecount(fsrcsize), util.bytecount(crawsize),
util.bytecount(frawsize),
) )
) )
seen.add(b'f')
if progress: if progress:
progress.complete() progress.complete()
progress = srcrepo.ui.makeprogress( progress = srcrepo.ui.makeprogress(
_(b'file revisions'), total=frevcount _(b'changelog revisions'), total=crevcount
) )
for unencoded, oldrl in sorted(changelogs.items()):
newrl = _perform_clone( newrl = _perform_clone(
ui, ui,
dstrepo, dstrepo,
tr, tr,
oldrl, oldrl,
unencoded, unencoded,
deltareuse, deltareuse,
forcedeltabothparents, forcedeltabothparents,
revlogs, revlogs,
sidedatacompanion, sidedatacompanion,
oncopiedrevision, oncopiedrevision,
) )
info = newrl.storageinfo(storedsize=True) info = newrl.storageinfo(storedsize=True)
datasize = info[b'storedsize'] or 0 datasize = info[b'storedsize'] or 0
dstsize += datasize dstsize += datasize
if isinstance(newrl, changelog.changelog):
cdstsize += datasize cdstsize += datasize
elif isinstance(newrl, manifest.manifestrevlog):
mdstsize += datasize
else:
fdstsize += datasize
progress.complete() progress.complete()
ui.status( ui.status(
_( _(
b'finished migrating %d changelog revisions; change in size: ' b'finished migrating %d changelog revisions; change in size: '
b'%s\n' b'%s\n'
) )
% (crevcount, util.bytecount(cdstsize - csrcsize)) % (crevcount, util.bytecount(cdstsize - csrcsize))
) )