This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8722

fix: obtain base paths before starting workers
ClosedPublic

Authored by rdamazio on Jul 10 2020, 2:45 AM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHG54009f8c3e25: fix: obtain base paths before starting workers
rHGf204ba74004b: fix: obtain base paths before starting workers
Summary

This moves calculation of base paths to before work is dispatched.
While this does mean that copy tracing will be serialized instead of
parallel, it is necessary to be able to prefetch the base contents
in a batch, which will likely be more efficient.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

PathPackages
Mhgext/fix.py (29 lines)

Diff 21831

hgext/fix.py

) )
with repo.wlock(), repo.lock(), repo.transaction(b'fix'): with repo.wlock(), repo.lock(), repo.transaction(b'fix'):
revstofix = getrevstofix(ui, repo, opts) revstofix = getrevstofix(ui, repo, opts)
basectxs = getbasectxs(repo, opts, revstofix) basectxs = getbasectxs(repo, opts, revstofix)
workqueue, numitems = getworkqueue( workqueue, numitems = getworkqueue(
ui, repo, pats, opts, revstofix, basectxs ui, repo, pats, opts, revstofix, basectxs
) )
basepaths = getbasepaths(repo, opts, workqueue, basectxs)
fixers = getfixers(ui) fixers = getfixers(ui)
# There are no data dependencies between the workers fixing each file # There are no data dependencies between the workers fixing each file
# revision, so we can use all available parallelism. # revision, so we can use all available parallelism.
def getfixes(items): def getfixes(items):
for rev, path in items: for rev, path in items:
ctx = repo[rev] ctx = repo[rev]
olddata = ctx[path].data() olddata = ctx[path].data()
metadata, newdata = fixfile( metadata, newdata = fixfile(
ui, repo, opts, fixers, ctx, path, basectxs[rev] ui, repo, opts, fixers, ctx, path, basepaths, basectxs[rev]
) )
# Don't waste memory/time passing unchanged content back, but # Don't waste memory/time passing unchanged content back, but
# produce one result per item either way. # produce one result per item either way.
yield ( yield (
rev, rev,
path, path,
metadata, metadata,
newdata if newdata != olddata else None, newdata if newdata != olddata else None,
itertools.chain( itertools.chain(
stat.added, stat.modified, stat.clean, stat.unknown stat.added, stat.modified, stat.clean, stat.unknown
) )
) )
) )
return files return files
def lineranges(opts, path, basectxs, fixctx, content2):
def lineranges(opts, path, basepaths, basectxs, fixctx, content2):
"""Returns the set of line ranges that should be fixed in a file """Returns the set of line ranges that should be fixed in a file
Of the form [(10, 20), (30, 40)]. Of the form [(10, 20), (30, 40)].
This depends on the given base contexts; we must consider lines that have This depends on the given base contexts; we must consider lines that have
changed versus any of the base contexts, and whether the file has been changed versus any of the base contexts, and whether the file has been
renamed versus any of them. renamed versus any of them.
Another way to understand this is that we exclude line ranges that are Another way to understand this is that we exclude line ranges that are
common to the file in all base contexts. common to the file in all base contexts.
""" """
if opts.get(b'whole'): if opts.get(b'whole'):
# Return a range containing all lines. Rely on the diff implementation's # Return a range containing all lines. Rely on the diff implementation's
# idea of how many lines are in the file, instead of reimplementing it. # idea of how many lines are in the file, instead of reimplementing it.
return difflineranges(b'', content2) return difflineranges(b'', content2)
rangeslist = [] rangeslist = []
for basectx in basectxs: for basectx in basectxs:
basepath = copies.pathcopies(basectx, fixctx).get(path, path) basepath = basepaths.get((basectx.rev(), fixctx.rev(), path), path)
if basepath in basectx: if basepath in basectx:
content1 = basectx[basepath].data() content1 = basectx[basepath].data()
else: else:
content1 = b'' content1 = b''
rangeslist.extend(difflineranges(content1, content2)) rangeslist.extend(difflineranges(content1, content2))
return unionranges(rangeslist) return unionranges(rangeslist)
def getbasepaths(repo, opts, workqueue, basectxs):
if opts.get(b'whole'):
# Base paths will never be fetched for line range determination.
return {}
basepaths = {}
for rev, path in workqueue:
fixctx = repo[rev]
for basectx in basectxs[rev]:
basepath = copies.pathcopies(basectx, fixctx).get(path, path)
if basepath in basectx:
basepaths[(basectx.rev(), fixctx.rev(), path)] = basepath
return basepaths
def unionranges(rangeslist): def unionranges(rangeslist):
"""Return the union of some closed intervals """Return the union of some closed intervals
>>> unionranges([]) >>> unionranges([])
[] []
>>> unionranges([(1, 100)]) >>> unionranges([(1, 100)])
[(1, 100)] [(1, 100)]
>>> unionranges([(1, 100), (1, 100)]) >>> unionranges([(1, 100), (1, 100)])
for pctx in ctx.parents(): for pctx in ctx.parents():
if pctx.rev() in basectxs: if pctx.rev() in basectxs:
basectxs[rev].update(basectxs[pctx.rev()]) basectxs[rev].update(basectxs[pctx.rev()])
else: else:
basectxs[rev].add(pctx) basectxs[rev].add(pctx)
return basectxs return basectxs
def fixfile(ui, repo, opts, fixers, fixctx, path, basectxs): def fixfile(ui, repo, opts, fixers, fixctx, path, basepaths, basectxs):
"""Run any configured fixers that should affect the file in this context """Run any configured fixers that should affect the file in this context
Returns the file content that results from applying the fixers in some order Returns the file content that results from applying the fixers in some order
starting with the file's content in the fixctx. Fixers that support line starting with the file's content in the fixctx. Fixers that support line
ranges will affect lines that have changed relative to any of the basectxs ranges will affect lines that have changed relative to any of the basectxs
(i.e. they will only avoid lines that are common to all basectxs). (i.e. they will only avoid lines that are common to all basectxs).
A fixer tool's stdout will become the file's new content if and only if it A fixer tool's stdout will become the file's new content if and only if it
exits with code zero. The fixer tool's working directory is the repository's exits with code zero. The fixer tool's working directory is the repository's
root. root.
""" """
metadata = {} metadata = {}
newdata = fixctx[path].data() newdata = fixctx[path].data()
for fixername, fixer in pycompat.iteritems(fixers): for fixername, fixer in pycompat.iteritems(fixers):
if fixer.affects(opts, fixctx, path): if fixer.affects(opts, fixctx, path):
ranges = lineranges(opts, path, basectxs, fixctx, newdata) ranges = lineranges(
opts, path, basepaths, basectxs, fixctx, newdata)
command = fixer.command(ui, path, ranges) command = fixer.command(ui, path, ranges)
if command is None: if command is None:
continue continue
ui.debug(b'subprocess: %s\n' % (command,)) ui.debug(b'subprocess: %s\n' % (command,))
proc = subprocess.Popen( proc = subprocess.Popen(
procutil.tonativestr(command), procutil.tonativestr(command),
shell=True, shell=True,
cwd=procutil.tonativestr(repo.root), cwd=procutil.tonativestr(repo.root),