This is an archive of the discontinued Mercurial Phabricator instance.

Differential D1624

py3: handle keyword arguments correctly in hgext/largefiles/
ClosedPublic

Authored by pulkit on Dec 9 2017, 8:08 PM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHG576ba8194fa8: py3: handle keyword arguments correctly in hgext/largefiles/
Summary

Keys of keyword arguments must be str(unicode) on Python 3. The transformer
which we use on Python 3, appends b'' in front of each string literal, so this
may lead in KeyError or None return even when the key is present by we are using
bytes value and it's stored in unicodes. This patch and all the similar patches
handle this by either converting the keys of kwargs to bytes using
'pycompat.byteskwargs()' or adding r'' so that the transformer won't append
b''.

This next 23 patches follows the above mentioned way to handle keyword
arguments.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

pulkit created this revision.Dec 9 2017, 8:08 PM
durin42 accepted this revision.Dec 9 2017, 9:00 PM
This revision is now accepted and ready to land.Dec 9 2017, 9:00 PM
Closed by commit rHG576ba8194fa8: py3: handle keyword arguments correctly in hgext/largefiles/ (authored by pulkit). · Explain WhyDec 9 2017, 9:06 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 4304

hgext/largefiles/lfcommands.py

from mercurial import ( from mercurial import (
cmdutil, cmdutil,
context, context,
error, error,
hg, hg,
lock, lock,
match as matchmod, match as matchmod,
node, node,
pycompat,
registrar, registrar,
scmutil, scmutil,
util, util,
) )
from ..convert import ( from ..convert import (
convcmd, convcmd,
filemap, filemap,
After running this command you will need to make sure that After running this command you will need to make sure that
largefiles is enabled anywhere you intend to push the new largefiles is enabled anywhere you intend to push the new
repository. repository.
Use --to-normal to convert largefiles back to normal files; after Use --to-normal to convert largefiles back to normal files; after
this, the DEST repository can be used without largefiles at all.''' this, the DEST repository can be used without largefiles at all.'''
opts = pycompat.byteskwargs(opts)
if opts['to_normal']: if opts['to_normal']:
tolfile = False tolfile = False
else: else:
tolfile = True tolfile = True
size = lfutil.getminsize(ui, True, opts.get('size'), default=None) size = lfutil.getminsize(ui, True, opts.get('size'), default=None)
if not hg.islocal(src): if not hg.islocal(src):
raise error.Abort(_('%s is not a local Mercurial repo') % src) raise error.Abort(_('%s is not a local Mercurial repo') % src)
hg lfpull -r "head() and not closed()" hg lfpull -r "head() and not closed()"
- pull largefiles on the default branch:: - pull largefiles on the default branch::
hg lfpull -r "branch(default)" hg lfpull -r "branch(default)"
""" """
repo.lfpullsource = source repo.lfpullsource = source
revs = opts.get('rev', []) revs = opts.get(r'rev', [])
if not revs: if not revs:
raise error.Abort(_('no revisions specified')) raise error.Abort(_('no revisions specified'))
revs = scmutil.revrange(repo, revs) revs = scmutil.revrange(repo, revs)
numcached = 0 numcached = 0
for rev in revs: for rev in revs:
ui.note(_('pulling largefiles for revision %s\n') % rev) ui.note(_('pulling largefiles for revision %s\n') % rev)
(cached, missing) = cachelfiles(ui, repo, rev) (cached, missing) = cachelfiles(ui, repo, rev)
numcached += len(cached) numcached += len(cached)
ui.status(_("%d largefiles cached\n") % numcached) ui.status(_("%d largefiles cached\n") % numcached)

hgext/largefiles/overrides.py

from mercurial import ( from mercurial import (
archival, archival,
cmdutil, cmdutil,
error, error,
hg, hg,
match as matchmod, match as matchmod,
pathutil, pathutil,
pycompat,
registrar, registrar,
scmutil, scmutil,
smartset, smartset,
util, util,
) )
from . import ( from . import (
lfcommands, lfcommands,
if ui.verbose or not exact: if ui.verbose or not exact:
ui.status(_('adding %s as a largefile\n') % name) ui.status(_('adding %s as a largefile\n') % name)
bad = [] bad = []
# Need to lock, otherwise there could be a race condition between # Need to lock, otherwise there could be a race condition between
# when standins are created and added to the repo. # when standins are created and added to the repo.
with repo.wlock(): with repo.wlock():
if not opts.get('dry_run'): if not opts.get(r'dry_run'):
standins = [] standins = []
lfdirstate = lfutil.openlfdirstate(ui, repo) lfdirstate = lfutil.openlfdirstate(ui, repo)
for f in lfnames: for f in lfnames:
standinname = lfutil.standin(f) standinname = lfutil.standin(f)
lfutil.writestandin(repo, standinname, hash='', lfutil.writestandin(repo, standinname, hash='',
executable=lfutil.getexecutable(repo.wjoin(f))) executable=lfutil.getexecutable(repo.wjoin(f)))
standins.append(standinname) standins.append(standinname)
if lfdirstate[f] == 'r': if lfdirstate[f] == 'r':
lfdirstate.normallookup(f) lfdirstate.normallookup(f)
else: else:
lfdirstate.add(f) lfdirstate.add(f)
lfdirstate.write() lfdirstate.write()
bad += [lfutil.splitstandin(f) bad += [lfutil.splitstandin(f)
for f in repo[None].add(standins) for f in repo[None].add(standins)
if f in m.files()] if f in m.files()]
added = [f for f in lfnames if f not in bad] added = [f for f in lfnames if f not in bad]
return added, bad return added, bad
def removelargefiles(ui, repo, isaddremove, matcher, **opts): def removelargefiles(ui, repo, isaddremove, matcher, **opts):
after = opts.get('after') after = opts.get(r'after')
m = composelargefilematcher(matcher, repo[None].manifest()) m = composelargefilematcher(matcher, repo[None].manifest())
try: try:
repo.lfstatus = True repo.lfstatus = True
s = repo.status(match=m, clean=not isaddremove) s = repo.status(match=m, clean=not isaddremove)
finally: finally:
repo.lfstatus = False repo.lfstatus = False
manifest = repo[None].manifest() manifest = repo[None].manifest()
modified, added, deleted, clean = [[f for f in list modified, added, deleted, clean = [[f for f in list
if ui.verbose or not m.exact(f): if ui.verbose or not m.exact(f):
# addremove in core gets fancy with the name, remove doesn't # addremove in core gets fancy with the name, remove doesn't
if isaddremove: if isaddremove:
name = m.uipath(f) name = m.uipath(f)
else: else:
name = m.rel(f) name = m.rel(f)
ui.status(_('removing %s\n') % name) ui.status(_('removing %s\n') % name)
if not opts.get('dry_run'): if not opts.get(r'dry_run'):
if not after: if not after:
repo.wvfs.unlinkpath(f, ignoremissing=True) repo.wvfs.unlinkpath(f, ignoremissing=True)
if opts.get('dry_run'): if opts.get(r'dry_run'):
return result return result
remove = [lfutil.standin(f) for f in remove] remove = [lfutil.standin(f) for f in remove]
# If this is being called by addremove, let the original addremove # If this is being called by addremove, let the original addremove
# function handle this. # function handle this.
if not isaddremove: if not isaddremove:
for f in remove: for f in remove:
repo.wvfs.unlinkpath(f, ignoremissing=True) repo.wvfs.unlinkpath(f, ignoremissing=True)
# For overriding mercurial.hgweb.webcommands so that largefiles will # For overriding mercurial.hgweb.webcommands so that largefiles will
# appear at their right place in the manifests. # appear at their right place in the manifests.
def decodepath(orig, path): def decodepath(orig, path):
return lfutil.splitstandin(path) or path return lfutil.splitstandin(path) or path
# -- Wrappers: modify existing commands -------------------------------- # -- Wrappers: modify existing commands --------------------------------
def overrideadd(orig, ui, repo, *pats, **opts): def overrideadd(orig, ui, repo, *pats, **opts):
if opts.get('normal') and opts.get('large'): if opts.get(r'normal') and opts.get(r'large'):
raise error.Abort(_('--normal cannot be used with --large')) raise error.Abort(_('--normal cannot be used with --large'))
return orig(ui, repo, *pats, **opts) return orig(ui, repo, *pats, **opts)
def cmdutiladd(orig, ui, repo, matcher, prefix, explicitonly, **opts): def cmdutiladd(orig, ui, repo, matcher, prefix, explicitonly, **opts):
# The --normal flag short circuits this override # The --normal flag short circuits this override
if opts.get(r'normal'): if opts.get(r'normal'):
return orig(ui, repo, matcher, prefix, explicitonly, **opts) return orig(ui, repo, matcher, prefix, explicitonly, **opts)
try: try:
return orig(ui, repo, *pats, **opts) return orig(ui, repo, *pats, **opts)
finally: finally:
restorematchandpatsfn() restorematchandpatsfn()
setattr(cmdutil, '_makenofollowlogfilematcher', oldmakelogfilematcher) setattr(cmdutil, '_makenofollowlogfilematcher', oldmakelogfilematcher)
def overrideverify(orig, ui, repo, *pats, **opts): def overrideverify(orig, ui, repo, *pats, **opts):
large = opts.pop('large', False) large = opts.pop(r'large', False)
all = opts.pop('lfa', False) all = opts.pop(r'lfa', False)
contents = opts.pop('lfc', False) contents = opts.pop(r'lfc', False)
result = orig(ui, repo, *pats, **opts) result = orig(ui, repo, *pats, **opts)
if large or all or contents: if large or all or contents:
result = result or lfcommands.verifylfiles(ui, repo, all, contents) result = result or lfcommands.verifylfiles(ui, repo, all, contents)
return result return result
def overridedebugstate(orig, ui, repo, *pats, **opts): def overridedebugstate(orig, ui, repo, *pats, **opts):
large = opts.pop('large', False) large = opts.pop(r'large', False)
if large: if large:
class fakerepo(object): class fakerepo(object):
dirstate = lfutil.openlfdirstate(ui, repo) dirstate = lfutil.openlfdirstate(ui, repo)
orig(ui, fakerepo, *pats, **opts) orig(ui, fakerepo, *pats, **opts)
else: else:
orig(ui, repo, *pats, **opts) orig(ui, repo, *pats, **opts)
# Before starting the manifest merge, merge.updates will call # Before starting the manifest merge, merge.updates will call
# largefiles updated remotely # largefiles updated remotely
def overridepull(orig, ui, repo, source=None, **opts): def overridepull(orig, ui, repo, source=None, **opts):
revsprepull = len(repo) revsprepull = len(repo)
if not source: if not source:
source = 'default' source = 'default'
repo.lfpullsource = source repo.lfpullsource = source
result = orig(ui, repo, source, **opts) result = orig(ui, repo, source, **opts)
revspostpull = len(repo) revspostpull = len(repo)
lfrevs = opts.get('lfrev', []) lfrevs = opts.get(r'lfrev', [])
if opts.get('all_largefiles'): if opts.get(r'all_largefiles'):
lfrevs.append('pulled()') lfrevs.append('pulled()')
if lfrevs and revspostpull > revsprepull: if lfrevs and revspostpull > revsprepull:
numcached = 0 numcached = 0
repo.firstpulled = revsprepull # for pulled() revset expression repo.firstpulled = revsprepull # for pulled() revset expression
try: try:
for rev in scmutil.revrange(repo, lfrevs): for rev in scmutil.revrange(repo, lfrevs):
ui.note(_('pulling largefiles for revision %s\n') % rev) ui.note(_('pulling largefiles for revision %s\n') % rev)
(cached, missing) = lfcommands.cachelfiles(ui, repo, rev) (cached, missing) = lfcommands.cachelfiles(ui, repo, rev)
numcached += len(cached) numcached += len(cached)
finally: finally:
del repo.firstpulled del repo.firstpulled
ui.status(_("%d largefiles cached\n") % numcached) ui.status(_("%d largefiles cached\n") % numcached)
return result return result
def overridepush(orig, ui, repo, *args, **kwargs): def overridepush(orig, ui, repo, *args, **kwargs):
"""Override push command and store --lfrev parameters in opargs""" """Override push command and store --lfrev parameters in opargs"""
lfrevs = kwargs.pop('lfrev', None) lfrevs = kwargs.pop(r'lfrev', None)
if lfrevs: if lfrevs:
opargs = kwargs.setdefault('opargs', {}) opargs = kwargs.setdefault('opargs', {})
opargs['lfrevs'] = scmutil.revrange(repo, lfrevs) opargs['lfrevs'] = scmutil.revrange(repo, lfrevs)
return orig(ui, repo, *args, **kwargs) return orig(ui, repo, *args, **kwargs)
def exchangepushoperation(orig, *args, **kwargs): def exchangepushoperation(orig, *args, **kwargs):
"""Override pushoperation constructor and store lfrevs parameter""" """Override pushoperation constructor and store lfrevs parameter"""
lfrevs = kwargs.pop('lfrevs', None) lfrevs = kwargs.pop(r'lfrevs', None)
pushop = orig(*args, **kwargs) pushop = orig(*args, **kwargs)
pushop.lfrevs = lfrevs pushop.lfrevs = lfrevs
return pushop return pushop
revsetpredicate = registrar.revsetpredicate() revsetpredicate = registrar.revsetpredicate()
@revsetpredicate('pulled()') @revsetpredicate('pulled()')
def pulledrevsetsymbol(repo, subset, x): def pulledrevsetsymbol(repo, subset, x):
except AttributeError: except AttributeError:
raise error.Abort(_("pulled() only available in --lfrev")) raise error.Abort(_("pulled() only available in --lfrev"))
return smartset.baseset([r for r in subset if r >= firstpulled]) return smartset.baseset([r for r in subset if r >= firstpulled])
def overrideclone(orig, ui, source, dest=None, **opts): def overrideclone(orig, ui, source, dest=None, **opts):
d = dest d = dest
if d is None: if d is None:
d = hg.defaultdest(source) d = hg.defaultdest(source)
if opts.get('all_largefiles') and not hg.islocal(d): if opts.get(r'all_largefiles') and not hg.islocal(d):
raise error.Abort(_( raise error.Abort(_(
'--all-largefiles is incompatible with non-local destination %s') % '--all-largefiles is incompatible with non-local destination %s') %
d) d)
return orig(ui, source, dest, **opts) return orig(ui, source, dest, **opts)
def hgclone(orig, ui, opts, *args, **kwargs): def hgclone(orig, ui, opts, *args, **kwargs):
result = orig(ui, opts, *args, **kwargs) result = orig(ui, opts, *args, **kwargs)
# If largefiles is required for this repo, permanently enable it locally # If largefiles is required for this repo, permanently enable it locally
if 'largefiles' in repo.requirements: if 'largefiles' in repo.requirements:
with repo.vfs('hgrc', 'a', text=True) as fp: with repo.vfs('hgrc', 'a', text=True) as fp:
fp.write('\n[extensions]\nlargefiles=\n') fp.write('\n[extensions]\nlargefiles=\n')
# Caching is implicitly limited to 'rev' option, since the dest repo was # Caching is implicitly limited to 'rev' option, since the dest repo was
# truncated at that point. The user may expect a download count with # truncated at that point. The user may expect a download count with
# this option, so attempt whether or not this is a largefile repo. # this option, so attempt whether or not this is a largefile repo.
if opts.get('all_largefiles'): if opts.get(r'all_largefiles'):
success, missing = lfcommands.downloadlfiles(ui, repo, None) success, missing = lfcommands.downloadlfiles(ui, repo, None)
if missing != 0: if missing != 0:
return None return None
return result return result
def hgpostshare(orig, sourcerepo, destrepo, bookmarks=True, defaultpath=None): def hgpostshare(orig, sourcerepo, destrepo, bookmarks=True, defaultpath=None):
orig(sourcerepo, destrepo, bookmarks, defaultpath) orig(sourcerepo, destrepo, bookmarks, defaultpath)
# If largefiles is required for this repo, permanently enable it locally # If largefiles is required for this repo, permanently enable it locally
if 'largefiles' in destrepo.requirements: if 'largefiles' in destrepo.requirements:
with destrepo.vfs('hgrc', 'a+', text=True) as fp: with destrepo.vfs('hgrc', 'a+', text=True) as fp:
fp.write('\n[extensions]\nlargefiles=\n') fp.write('\n[extensions]\nlargefiles=\n')
def overriderebase(orig, ui, repo, **opts): def overriderebase(orig, ui, repo, **opts):
if not util.safehasattr(repo, '_largefilesenabled'): if not util.safehasattr(repo, '_largefilesenabled'):
return orig(ui, repo, **opts) return orig(ui, repo, **opts)
resuming = opts.get('continue') resuming = opts.get(r'continue')
repo._lfcommithooks.append(lfutil.automatedcommithook(resuming)) repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
repo._lfstatuswriters.append(lambda *msg, **opts: None) repo._lfstatuswriters.append(lambda *msg, **opts: None)
try: try:
return orig(ui, repo, **opts) return orig(ui, repo, **opts)
finally: finally:
repo._lfstatuswriters.pop() repo._lfstatuswriters.pop()
repo._lfcommithooks.pop() repo._lfcommithooks.pop()
lfdirstate = lfutil.openlfdirstate(ui, repo) lfdirstate = lfutil.openlfdirstate(ui, repo)
unknown = [f for f in r.unknown if lfdirstate[f] == '?'] unknown = [f for f in r.unknown if lfdirstate[f] == '?']
ignored = [f for f in r.ignored if lfdirstate[f] == '?'] ignored = [f for f in r.ignored if lfdirstate[f] == '?']
return scmutil.status(r.modified, r.added, r.removed, r.deleted, return scmutil.status(r.modified, r.added, r.removed, r.deleted,
unknown, ignored, r.clean) unknown, ignored, r.clean)
repo.status = overridestatus repo.status = overridestatus
orig(ui, repo, *dirs, **opts) orig(ui, repo, *dirs, **opts)
repo.status = oldstatus repo.status = oldstatus
def overriderollback(orig, ui, repo, **opts): def overriderollback(orig, ui, repo, **opts):
with repo.wlock(): with repo.wlock():
before = repo.dirstate.parents() before = repo.dirstate.parents()
orphans = set(f for f in repo.dirstate orphans = set(f for f in repo.dirstate
if lfutil.isstandin(f) and repo.dirstate[f] != 'r') if lfutil.isstandin(f) and repo.dirstate[f] != 'r')
result = orig(ui, repo, **opts) result = orig(ui, repo, **opts)
after = repo.dirstate.parents() after = repo.dirstate.parents()
if before == after: if before == after:
lfutil.synclfdirstate(repo, lfdirstate, file, True) lfutil.synclfdirstate(repo, lfdirstate, file, True)
orphans.discard(file) orphans.discard(file)
for lfile in orphans: for lfile in orphans:
lfdirstate.drop(lfile) lfdirstate.drop(lfile)
lfdirstate.write() lfdirstate.write()
return result return result
def overridetransplant(orig, ui, repo, *revs, **opts): def overridetransplant(orig, ui, repo, *revs, **opts):
resuming = opts.get('continue') resuming = opts.get(r'continue')
repo._lfcommithooks.append(lfutil.automatedcommithook(resuming)) repo._lfcommithooks.append(lfutil.automatedcommithook(resuming))
repo._lfstatuswriters.append(lambda *msg, **opts: None) repo._lfstatuswriters.append(lambda *msg, **opts: None)
try: try:
result = orig(ui, repo, *revs, **opts) result = orig(ui, repo, *revs, **opts)
finally: finally:
repo._lfstatuswriters.pop() repo._lfstatuswriters.pop()
repo._lfcommithooks.pop() repo._lfcommithooks.pop()
return result return result
def overridecat(orig, ui, repo, file1, *pats, **opts): def overridecat(orig, ui, repo, file1, *pats, **opts):
opts = pycompat.byteskwargs(opts)
ctx = scmutil.revsingle(repo, opts.get('rev')) ctx = scmutil.revsingle(repo, opts.get('rev'))
err = 1 err = 1
notbad = set() notbad = set()
m = scmutil.match(ctx, (file1,) + pats, opts) m = scmutil.match(ctx, (file1,) + pats, opts)
origmatchfn = m.matchfn origmatchfn = m.matchfn
def lfmatchfn(f): def lfmatchfn(f):
if origmatchfn(f): if origmatchfn(f):
return True return True
with open(path, "rb") as fpin: with open(path, "rb") as fpin:
for chunk in util.filechunkiter(fpin): for chunk in util.filechunkiter(fpin):
fp.write(chunk) fp.write(chunk)
err = 0 err = 0
return err return err
def mergeupdate(orig, repo, node, branchmerge, force, def mergeupdate(orig, repo, node, branchmerge, force,
*args, **kwargs): *args, **kwargs):
matcher = kwargs.get('matcher', None) matcher = kwargs.get(r'matcher', None)
# note if this is a partial update # note if this is a partial update
partial = matcher and not matcher.always() partial = matcher and not matcher.always()
with repo.wlock(): with repo.wlock():
# branch | | | # branch | | |
# merge | force | partial | action # merge | force | partial | action
# -------+-------+---------+-------------- # -------+-------+---------+--------------
# x | x | x | linear-merge # x | x | x | linear-merge
# o | x | x | branch-merge # o | x | x | branch-merge
for lfile in oldclean: for lfile in oldclean:
lfdirstate.normallookup(lfile) lfdirstate.normallookup(lfile)
lfdirstate.write() lfdirstate.write()
oldstandins = lfutil.getstandinsstate(repo) oldstandins = lfutil.getstandinsstate(repo)
# Make sure the merge runs on disk, not in-memory. largefiles is not a # Make sure the merge runs on disk, not in-memory. largefiles is not a
# good candidate for in-memory merge (large files, custom dirstate, # good candidate for in-memory merge (large files, custom dirstate,
# matcher usage). # matcher usage).
kwargs['wc'] = repo[None] kwargs[r'wc'] = repo[None]
result = orig(repo, node, branchmerge, force, *args, **kwargs) result = orig(repo, node, branchmerge, force, *args, **kwargs)
newstandins = lfutil.getstandinsstate(repo) newstandins = lfutil.getstandinsstate(repo)
filelist = lfutil.getlfilestoupdate(oldstandins, newstandins) filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
# to avoid leaving all largefiles as dirty and thus rehash them, mark # to avoid leaving all largefiles as dirty and thus rehash them, mark
# all the ones that didn't change as clean # all the ones that didn't change as clean
for lfile in oldclean.difference(filelist): for lfile in oldclean.difference(filelist):

hgext/largefiles/proto.py

if lfutil.islfilesrepo(repo): if lfutil.islfilesrepo(repo):
return wireproto.ooberror(LARGEFILES_REQUIRED_MSG) return wireproto.ooberror(LARGEFILES_REQUIRED_MSG)
return wireproto.heads(repo, proto) return wireproto.heads(repo, proto)
def sshrepocallstream(self, cmd, **args): def sshrepocallstream(self, cmd, **args):
if cmd == 'heads' and self.capable('largefiles'): if cmd == 'heads' and self.capable('largefiles'):
cmd = 'lheads' cmd = 'lheads'
if cmd == 'batch' and self.capable('largefiles'): if cmd == 'batch' and self.capable('largefiles'):
args['cmds'] = args['cmds'].replace('heads ', 'lheads ') args[r'cmds'] = args[r'cmds'].replace('heads ', 'lheads ')
return ssholdcallstream(self, cmd, **args) return ssholdcallstream(self, cmd, **args)
headsre = re.compile(r'(^|;)heads\b') headsre = re.compile(r'(^|;)heads\b')
def httprepocallstream(self, cmd, **args): def httprepocallstream(self, cmd, **args):
if cmd == 'heads' and self.capable('largefiles'): if cmd == 'heads' and self.capable('largefiles'):
cmd = 'lheads' cmd = 'lheads'
if cmd == 'batch' and self.capable('largefiles'): if cmd == 'batch' and self.capable('largefiles'):
args['cmds'] = headsre.sub('lheads', args['cmds']) args[r'cmds'] = headsre.sub('lheads', args[r'cmds'])
return httpoldcallstream(self, cmd, **args) return httpoldcallstream(self, cmd, **args)