This is an archive of the discontinued Mercurial Phabricator instance.

Differential D3805

relink: use progress helper
ClosedPublic

Authored by martinvonz on Jun 18 2018, 8:45 PM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHG398716063c2d: relink: use progress helper
Summary

This doesn't use progress.increment() because progress output is
skipped for some positions (so we may end up calling "update(0),
update(2), update(7)", or similar).

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

martinvonz created this revision.Jun 18 2018, 8:45 PM
pulkit accepted this revision.Jun 19 2018, 7:01 AM
Closed by commit rHG398716063c2d: relink: use progress helper (authored by martinvonz). · Explain WhyJun 19 2018, 7:18 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mhgext/relink.py (19 lines)

Diff 9221

hgext/relink.py

# Your average repository has some files which were deleted before # Your average repository has some files which were deleted before
# the tip revision. We account for that by assuming that there are # the tip revision. We account for that by assuming that there are
# 3 tracked files for every 2 live files as of the tip version of # 3 tracked files for every 2 live files as of the tip version of
# the repository. # the repository.
# #
# mozilla-central as of 2010-06-10 had a ratio of just over 7:5. # mozilla-central as of 2010-06-10 had a ratio of just over 7:5.
total = live * 3 // 2 total = live * 3 // 2
src = src.store.path src = src.store.path
progress = ui.makeprogress(_('collecting'), unit=_('files'), total=total)
pos = 0 pos = 0
ui.status(_("tip has %d files, estimated total number of files: %d\n") ui.status(_("tip has %d files, estimated total number of files: %d\n")
% (live, total)) % (live, total))
for dirpath, dirnames, filenames in os.walk(src): for dirpath, dirnames, filenames in os.walk(src):
dirnames.sort() dirnames.sort()
relpath = dirpath[len(src) + seplen:] relpath = dirpath[len(src) + seplen:]
for filename in sorted(filenames): for filename in sorted(filenames):
if filename[-2:] not in ('.d', '.i'): if filename[-2:] not in ('.d', '.i'):
continue continue
st = os.stat(os.path.join(dirpath, filename)) st = os.stat(os.path.join(dirpath, filename))
if not stat.S_ISREG(st.st_mode): if not stat.S_ISREG(st.st_mode):
continue continue
pos += 1 pos += 1
candidates.append((os.path.join(relpath, filename), st)) candidates.append((os.path.join(relpath, filename), st))
ui.progress(_('collecting'), pos, filename, _('files'), total) progress.update(pos, item=filename)
ui.progress(_('collecting'), None) progress.complete()
ui.status(_('collected %d candidate storage files\n') % len(candidates)) ui.status(_('collected %d candidate storage files\n') % len(candidates))
return candidates return candidates
def prune(candidates, src, dst, ui): def prune(candidates, src, dst, ui):
def linkfilter(src, dst, st): def linkfilter(src, dst, st):
try: try:
ts = os.stat(dst) ts = os.stat(dst)
except OSError: except OSError:
# Destination doesn't have this file? # Destination doesn't have this file?
return False return False
if util.samefile(src, dst): if util.samefile(src, dst):
return False return False
if not util.samedevice(src, dst): if not util.samedevice(src, dst):
# No point in continuing # No point in continuing
raise error.Abort( raise error.Abort(
_('source and destination are on different devices')) _('source and destination are on different devices'))
if st.st_size != ts.st_size: if st.st_size != ts.st_size:
return False return False
return st return st
targets = [] targets = []
total = len(candidates) progress = ui.makeprogress(_('pruning'), unit=_('files'),
total=len(candidates))
pos = 0 pos = 0
for fn, st in candidates: for fn, st in candidates:
pos += 1 pos += 1
srcpath = os.path.join(src, fn) srcpath = os.path.join(src, fn)
tgt = os.path.join(dst, fn) tgt = os.path.join(dst, fn)
ts = linkfilter(srcpath, tgt, st) ts = linkfilter(srcpath, tgt, st)
if not ts: if not ts:
ui.debug('not linkable: %s\n' % fn) ui.debug('not linkable: %s\n' % fn)
continue continue
targets.append((fn, ts.st_size)) targets.append((fn, ts.st_size))
ui.progress(_('pruning'), pos, fn, _('files'), total) progress.update(pos, item=fn)
ui.progress(_('pruning'), None) progress.complete()
ui.status(_('pruned down to %d probably relinkable files\n') % len(targets)) ui.status(_('pruned down to %d probably relinkable files\n') % len(targets))
return targets return targets
def do_relink(src, dst, files, ui): def do_relink(src, dst, files, ui):
def relinkfile(src, dst): def relinkfile(src, dst):
bak = dst + '.bak' bak = dst + '.bak'
os.rename(dst, bak) os.rename(dst, bak)
try: try:
util.oslink(src, dst) util.oslink(src, dst)
except OSError: except OSError:
os.rename(bak, dst) os.rename(bak, dst)
raise raise
os.remove(bak) os.remove(bak)
CHUNKLEN = 65536 CHUNKLEN = 65536
relinked = 0 relinked = 0
savedbytes = 0 savedbytes = 0
progress = ui.makeprogress(_('relinking'), unit=_('files'),
total=len(files))
pos = 0 pos = 0
total = len(files)
for f, sz in files: for f, sz in files:
pos += 1 pos += 1
source = os.path.join(src, f) source = os.path.join(src, f)
tgt = os.path.join(dst, f) tgt = os.path.join(dst, f)
# Binary mode, so that read() works correctly, especially on Windows # Binary mode, so that read() works correctly, especially on Windows
sfp = open(source, 'rb') sfp = open(source, 'rb')
dfp = open(tgt, 'rb') dfp = open(tgt, 'rb')
sin = sfp.read(CHUNKLEN) sin = sfp.read(CHUNKLEN)
while sin: while sin:
din = dfp.read(CHUNKLEN) din = dfp.read(CHUNKLEN)
if sin != din: if sin != din:
break break
sin = sfp.read(CHUNKLEN) sin = sfp.read(CHUNKLEN)
sfp.close() sfp.close()
dfp.close() dfp.close()
if sin: if sin:
ui.debug('not linkable: %s\n' % f) ui.debug('not linkable: %s\n' % f)
continue continue
try: try:
relinkfile(source, tgt) relinkfile(source, tgt)
ui.progress(_('relinking'), pos, f, _('files'), total) progress.update(pos, item=f)
relinked += 1 relinked += 1
savedbytes += sz savedbytes += sz
except OSError as inst: except OSError as inst:
ui.warn('%s: %s\n' % (tgt, stringutil.forcebytestr(inst))) ui.warn('%s: %s\n' % (tgt, stringutil.forcebytestr(inst)))
ui.progress(_('relinking'), None) progress.complete()
ui.status(_('relinked %d files (%s reclaimed)\n') % ui.status(_('relinked %d files (%s reclaimed)\n') %
(relinked, util.bytecount(savedbytes))) (relinked, util.bytecount(savedbytes)))