This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7012

py3: stop normalizing 2nd argument of *attr() to unicode
ClosedPublic

Authored by indygreg on Oct 7 2019, 4:04 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGc95b2f40db7c: py3: stop normalizing 2nd argument of *attr() to unicode
Summary

Now that we don't byteify strings, we can stop normalizing the 2nd
string argument to getattr() and remove explicit overrides we were
using in the code base.

We no longer use some helper functions in the source transformer,
so we remove those as well.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

indygreg created this revision.Oct 7 2019, 4:04 PM
indygreg added a commit: rHGc95b2f40db7c: py3: stop normalizing 2nd argument of *attr() to unicode.Oct 8 2019, 2:17 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHGc95b2f40db7c: py3: stop normalizing 2nd argument of *attr() to unicode (authored by indygreg). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 16935

mercurial/__init__.py

# the for i, t enumerate(tokens) loop below # the for i, t enumerate(tokens) loop below
def _isop(j, *o): def _isop(j, *o):
"""Assert that tokens[j] is an OP with one of the given values""" """Assert that tokens[j] is an OP with one of the given values"""
try: try:
return tokens[j].type == token.OP and tokens[j].string in o return tokens[j].type == token.OP and tokens[j].string in o
except IndexError: except IndexError:
return False return False
def _findargnofcall(n):
"""Find arg n of a call expression (start at 0)
Returns index of the first token of that argument, or None if
there is not that many arguments.
Assumes that token[i + 1] is '('.
"""
nested = 0
for j in range(i + 2, len(tokens)):
if _isop(j, ')', ']', '}'):
# end of call, tuple, subscription or dict / set
nested -= 1
if nested < 0:
return None
elif n == 0:
# this is the starting position of arg
return j
elif _isop(j, '(', '[', '{'):
nested += 1
elif _isop(j, ',') and nested == 0:
n -= 1
return None
def _ensureunicode(j):
"""Make sure the token at j is a unicode string
This rewrites a string token to include the unicode literal prefix
so the string transformer won't add the byte prefix.
Ignores tokens that are not strings. Assumes bounds checking has
already been done.
"""
st = tokens[j]
if st.type == token.STRING and st.string.startswith(("'", '"')):
tokens[j] = st._replace(string='u%s' % st.string)
for i, t in enumerate(tokens): for i, t in enumerate(tokens):
# This looks like a function call. # This looks like a function call.
if t.type == token.NAME and _isop(i + 1, '('): if t.type == token.NAME and _isop(i + 1, '('):
fn = t.string fn = t.string
# *attr() builtins don't accept byte strings to 2nd argument.
if fn in (
'getattr',
'setattr',
'hasattr',
'safehasattr',
) and not _isop(i - 1, '.'):
arg1idx = _findargnofcall(1)
if arg1idx is not None:
_ensureunicode(arg1idx)
# It changes iteritems/values to items/values as they are not # It changes iteritems/values to items/values as they are not
# present in Python 3 world. # present in Python 3 world.
elif fn in ('iteritems', 'itervalues') and not ( if fn in ('iteritems', 'itervalues') and not (
tokens[i - 1].type == token.NAME tokens[i - 1].type == token.NAME
and tokens[i - 1].string == 'def' and tokens[i - 1].string == 'def'
): ):
yield t._replace(string=fn[4:]) yield t._replace(string=fn[4:])
continue continue
# Emit unmodified token. # Emit unmodified token.
yield t yield t
# Header to add to bytecode files. This MUST be changed when # Header to add to bytecode files. This MUST be changed when
# ``replacetoken`` or any mechanism that changes semantics of module # ``replacetoken`` or any mechanism that changes semantics of module
# loading is changed. Otherwise cached bytecode may get loaded without # loading is changed. Otherwise cached bytecode may get loaded without
# the new transformation mechanisms applied. # the new transformation mechanisms applied.
BYTECODEHEADER = b'HG\x00\x13' BYTECODEHEADER = b'HG\x00\x14'
class hgloader(importlib.machinery.SourceFileLoader): class hgloader(importlib.machinery.SourceFileLoader):
"""Custom module loader that transforms source code. """Custom module loader that transforms source code.
When the source code is converted to a code object, we transform When the source code is converted to a code object, we transform
certain patterns to be Python 3 compatible. This allows us to write code certain patterns to be Python 3 compatible. This allows us to write code
that is natively Python 2 and compatible with Python 3 without that is natively Python 2 and compatible with Python 3 without
making the code excessively ugly. making the code excessively ugly.

mercurial/keepalive.py

_(b'bad HTTP status line: %s') % pycompat.sysbytes(err.line) _(b'bad HTTP status line: %s') % pycompat.sysbytes(err.line)
) )
except (socket.error, httplib.HTTPException) as err: except (socket.error, httplib.HTTPException) as err:
raise urlerr.urlerror(err) raise urlerr.urlerror(err)
# If not a persistent connection, don't try to reuse it. Look # If not a persistent connection, don't try to reuse it. Look
# for this using getattr() since vcr doesn't define this # for this using getattr() since vcr doesn't define this
# attribute, and in that case always close the connection. # attribute, and in that case always close the connection.
if getattr(r, r'will_close', True): if getattr(r, 'will_close', True):
self._cm.remove(h) self._cm.remove(h)
if DEBUG: if DEBUG:
DEBUG.info(b"STATUS: %s, %s", r.status, r.reason) DEBUG.info(b"STATUS: %s, %s", r.status, r.reason)
r._handler = self r._handler = self
r._host = host r._host = host
r._url = req.get_full_url() r._url = req.get_full_url()
r._connection = h r._connection = h

mercurial/policy.py

# from .<pkgname> import <modname> (where . is looked through this module) # from .<pkgname> import <modname> (where . is looked through this module)
fakelocals = {} fakelocals = {}
pkg = __import__(pkgname, globals(), fakelocals, [modname], level=1) pkg = __import__(pkgname, globals(), fakelocals, [modname], level=1)
try: try:
fakelocals[modname] = mod = getattr(pkg, modname) fakelocals[modname] = mod = getattr(pkg, modname)
except AttributeError: except AttributeError:
raise ImportError(r'cannot import name %s' % modname) raise ImportError(r'cannot import name %s' % modname)
# force import; fakelocals[modname] may be replaced with the real module # force import; fakelocals[modname] may be replaced with the real module
getattr(mod, r'__doc__', None) getattr(mod, '__doc__', None)
return fakelocals[modname] return fakelocals[modname]
# keep in sync with "version" in C modules # keep in sync with "version" in C modules
_cextversions = { _cextversions = {
(r'cext', r'base85'): 1, (r'cext', r'base85'): 1,
(r'cext', r'bdiff'): 3, (r'cext', r'bdiff'): 3,
(r'cext', r'mpatch'): 1, (r'cext', r'mpatch'): 1,
(r'cext', r'osutil'): 4, (r'cext', r'osutil'): 4,
(r'cext', r'parsers'): 13, (r'cext', r'parsers'): 13,
} }
# map import request to other package or module # map import request to other package or module
_modredirects = { _modredirects = {
(r'cext', r'charencode'): (r'cext', r'parsers'), (r'cext', r'charencode'): (r'cext', r'parsers'),
(r'cffi', r'base85'): (r'pure', r'base85'), (r'cffi', r'base85'): (r'pure', r'base85'),
(r'cffi', r'charencode'): (r'pure', r'charencode'), (r'cffi', r'charencode'): (r'pure', r'charencode'),
(r'cffi', r'parsers'): (r'pure', r'parsers'), (r'cffi', r'parsers'): (r'pure', r'parsers'),
} }
def _checkmod(pkgname, modname, mod): def _checkmod(pkgname, modname, mod):
expected = _cextversions.get((pkgname, modname)) expected = _cextversions.get((pkgname, modname))
actual = getattr(mod, r'version', None) actual = getattr(mod, 'version', None)
if actual != expected: if actual != expected:
raise ImportError( raise ImportError(
r'cannot import module %s.%s ' r'cannot import module %s.%s '
r'(expected version: %d, actual: %r)' r'(expected version: %d, actual: %r)'
% (pkgname, modname, expected, actual) % (pkgname, modname, expected, actual)
) )

mercurial/pycompat.py

def raisewithtb(exc, tb): def raisewithtb(exc, tb):
"""Raise exception with the given traceback""" """Raise exception with the given traceback"""
raise exc.with_traceback(tb) raise exc.with_traceback(tb)
def getdoc(obj): def getdoc(obj):
"""Get docstring as bytes; may be None so gettext() won't confuse it """Get docstring as bytes; may be None so gettext() won't confuse it
with _('')""" with _('')"""
doc = getattr(obj, u'__doc__', None) doc = getattr(obj, '__doc__', None)
if doc is None: if doc is None:
return doc return doc
return sysbytes(doc) return sysbytes(doc)
def _wrapattrfunc(f): def _wrapattrfunc(f):
@functools.wraps(f) @functools.wraps(f)
def w(object, name, *args): def w(object, name, *args):
return f(object, sysstr(name), *args) return f(object, sysstr(name), *args)

mercurial/testing/storage.py

error, error,
mdiff, mdiff,
) )
from ..interfaces import repository from ..interfaces import repository
from ..utils import storageutil from ..utils import storageutil
class basetestcase(unittest.TestCase): class basetestcase(unittest.TestCase):
if not getattr(unittest.TestCase, r'assertRaisesRegex', False): if not getattr(unittest.TestCase, 'assertRaisesRegex', False):
assertRaisesRegex = ( # camelcase-required assertRaisesRegex = ( # camelcase-required
unittest.TestCase.assertRaisesRegexp unittest.TestCase.assertRaisesRegexp
) )
class ifileindextests(basetestcase): class ifileindextests(basetestcase):
"""Generic tests for the ifileindex interface. """Generic tests for the ifileindex interface.

mercurial/util.py

This variation of ``bufferedinputpipe`` can notify observers about This variation of ``bufferedinputpipe`` can notify observers about
``os.read()`` events. It also re-publishes other events, such as ``os.read()`` events. It also re-publishes other events, such as
``read()`` and ``readline()``. ``read()`` and ``readline()``.
""" """
def _fillbuffer(self): def _fillbuffer(self):
res = super(observedbufferedinputpipe, self)._fillbuffer() res = super(observedbufferedinputpipe, self)._fillbuffer()
fn = getattr(self._input._observer, r'osread', None) fn = getattr(self._input._observer, 'osread', None)
if fn: if fn:
fn(res, _chunksize) fn(res, _chunksize)
return res return res
# We use different observer methods because the operation isn't # We use different observer methods because the operation isn't
# performed on the actual file object but on us. # performed on the actual file object but on us.
def read(self, size): def read(self, size):
res = super(observedbufferedinputpipe, self).read(size) res = super(observedbufferedinputpipe, self).read(size)
fn = getattr(self._input._observer, r'bufferedread', None) fn = getattr(self._input._observer, 'bufferedread', None)
if fn: if fn:
fn(res, size) fn(res, size)
return res return res
def readline(self, *args, **kwargs): def readline(self, *args, **kwargs):
res = super(observedbufferedinputpipe, self).readline(*args, **kwargs) res = super(observedbufferedinputpipe, self).readline(*args, **kwargs)
fn = getattr(self._input._observer, r'bufferedreadline', None) fn = getattr(self._input._observer, 'bufferedreadline', None)
if fn: if fn:
fn(res) fn(res)
return res return res
PROXIED_SOCKET_METHODS = { PROXIED_SOCKET_METHODS = {
r'makefile', r'makefile',