This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7275

encoding: add comment-based type hints for pytype
ClosedPublic

Authored by durin42 on Nov 6 2019, 5:57 PM.

Details

Reviewers
indygreg
Group Reviewers
hg-reviewers
Commits
rHG2ade00f3b03b: encoding: add comment-based type hints for pytype

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durin42 created this revision.Nov 6 2019, 5:57 PM
durin42 updated this revision to Diff 17666.Nov 6 2019, 6:22 PM
durin42 updated this revision to Diff 17674.Nov 6 2019, 6:39 PM
indygreg added a subscriber: indygreg.Nov 7 2019, 3:14 AM
indygreg added inline comments.
mercurial/encoding.py
24

Am I missing something here? Won't globals() always return a dict with elements? And isn't typing only available on Python 3?

durin42 added inline comments.Nov 7 2019, 11:08 AM
mercurial/encoding.py
24

This is a moderately bonkers way to write if False that pyflakes won't complain about and pytype won't be able to elide, so we can have pytype-only imports in the file to make the type annotation comments work.

indygreg accepted this revision.Nov 8 2019, 11:39 AM
This revision is now accepted and ready to land.Nov 8 2019, 11:39 AM
durin42 added a commit: rHG2ade00f3b03b: encoding: add comment-based type hints for pytype.Nov 8 2019, 11:48 AM
Closed by commit rHG2ade00f3b03b: encoding: add comment-based type hints for pytype (authored by durin42). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/encoding.py (35 lines)
Mtests/test-check-pyflakes.t (1 line)

Diff 17786

mercurial/encoding.py

from . import ( from . import (
error, error,
policy, policy,
pycompat, pycompat,
) )
from .pure import charencode as charencodepure from .pure import charencode as charencodepure
if not globals(): # hide this from non-pytype users
from typing import (
indygregUnsubmitted
Not Done

Am I missing something here? Won't globals() always return a dict with elements? And isn't typing only available on Python 3?

indygreg: Am I missing something here? Won't `globals()` always return a dict with elements? And isn't…
durin42AuthorUnsubmitted
Done

This is a moderately bonkers way to write if False that pyflakes won't complain about and pytype won't be able to elide, so we can have pytype-only imports in the file to make the type annotation comments work.

durin42: This is a moderately bonkers way to write `if False` that pyflakes won't complain about and…
Any,
Callable,
List,
Text,
Type,
TypeVar,
Union,
)
# keep pyflakes happy
for t in (Any, Callable, List, Text, Type, Union):
assert t
_Tlocalstr = TypeVar('_Tlocalstr', bound=localstr)
charencode = policy.importmod(r'charencode') charencode = policy.importmod(r'charencode')
isasciistr = charencode.isasciistr isasciistr = charencode.isasciistr
asciilower = charencode.asciilower asciilower = charencode.asciilower
asciiupper = charencode.asciiupper asciiupper = charencode.asciiupper
_jsonescapeu8fast = charencode.jsonescapeu8fast _jsonescapeu8fast = charencode.jsonescapeu8fast
_sysstr = pycompat.sysstr _sysstr = pycompat.sysstr
for x in b"200c 200d 200e 200f 202a 202b 202c 202d 202e " for x in b"200c 200d 200e 200f 202a 202b 202c 202d 202e "
b"206a 206b 206c 206d 206e 206f feff".split() b"206a 206b 206c 206d 206e 206f feff".split()
] ]
# verify the next function will work # verify the next function will work
assert all(i.startswith((b"\xe2", b"\xef")) for i in _ignore) assert all(i.startswith((b"\xe2", b"\xef")) for i in _ignore)
def hfsignoreclean(s): def hfsignoreclean(s):
# type: (bytes) -> bytes
"""Remove codepoints ignored by HFS+ from s. """Remove codepoints ignored by HFS+ from s.
>>> hfsignoreclean(u'.h\u200cg'.encode('utf-8')) >>> hfsignoreclean(u'.h\u200cg'.encode('utf-8'))
'.hg' '.hg'
>>> hfsignoreclean(u'.h\ufeffg'.encode('utf-8')) >>> hfsignoreclean(u'.h\ufeffg'.encode('utf-8'))
'.hg' '.hg'
""" """
if b"\xe2" in s or b"\xef" in s: if b"\xe2" in s or b"\xef" in s:
fallbackencoding = b'ISO-8859-1' fallbackencoding = b'ISO-8859-1'
class localstr(bytes): class localstr(bytes):
'''This class allows strings that are unmodified to be '''This class allows strings that are unmodified to be
round-tripped to the local encoding and back''' round-tripped to the local encoding and back'''
def __new__(cls, u, l): def __new__(cls, u, l):
# type: (Type[_Tlocalstr], Text, bytes) -> _Tlocalstr
s = bytes.__new__(cls, l) s = bytes.__new__(cls, l)
s._utf8 = u s._utf8 = u
return s return s
def __hash__(self): def __hash__(self):
return hash(self._utf8) # avoid collisions in local string space return hash(self._utf8) # avoid collisions in local string space
class safelocalstr(bytes): class safelocalstr(bytes):
"""Tagged string denoting it was previously an internal UTF-8 string, """Tagged string denoting it was previously an internal UTF-8 string,
and can be converted back to UTF-8 losslessly and can be converted back to UTF-8 losslessly
>>> assert safelocalstr(b'\\xc3') == b'\\xc3' >>> assert safelocalstr(b'\\xc3') == b'\\xc3'
>>> assert b'\\xc3' == safelocalstr(b'\\xc3') >>> assert b'\\xc3' == safelocalstr(b'\\xc3')
>>> assert b'\\xc3' in {safelocalstr(b'\\xc3'): 0} >>> assert b'\\xc3' in {safelocalstr(b'\\xc3'): 0}
>>> assert safelocalstr(b'\\xc3') in {b'\\xc3': 0} >>> assert safelocalstr(b'\\xc3') in {b'\\xc3': 0}
""" """
def tolocal(s): def tolocal(s):
# type: (Text) -> bytes
""" """
Convert a string from internal UTF-8 to local encoding Convert a string from internal UTF-8 to local encoding
All internal strings should be UTF-8 but some repos before the All internal strings should be UTF-8 but some repos before the
implementation of locale support may contain latin1 or possibly implementation of locale support may contain latin1 or possibly
other character sets. We attempt to decode everything strictly other character sets. We attempt to decode everything strictly
using UTF-8, then Latin-1, and failing that, we use UTF-8 and using UTF-8, then Latin-1, and failing that, we use UTF-8 and
replace unknown characters. replace unknown characters.
u = s.decode("utf-8", "replace") # last ditch u = s.decode("utf-8", "replace") # last ditch
# can't round-trip # can't round-trip
return u.encode(_sysstr(encoding), r"replace") return u.encode(_sysstr(encoding), r"replace")
except LookupError as k: except LookupError as k:
raise error.Abort(k, hint=b"please check your locale settings") raise error.Abort(k, hint=b"please check your locale settings")
def fromlocal(s): def fromlocal(s):
# type: (bytes) -> Text
""" """
Convert a string from the local character encoding to UTF-8 Convert a string from the local character encoding to UTF-8
We attempt to decode strings using the encoding mode set by We attempt to decode strings using the encoding mode set by
HGENCODINGMODE, which defaults to 'strict'. In this mode, unknown HGENCODINGMODE, which defaults to 'strict'. In this mode, unknown
characters will cause an error message. Other modes include characters will cause an error message. Other modes include
'replace', which replaces unknown characters with a special 'replace', which replaces unknown characters with a special
Unicode character, and 'ignore', which drops the character. Unicode character, and 'ignore', which drops the character.
raise error.Abort( raise error.Abort(
b"decoding near '%s': %s!" % (sub, pycompat.bytestr(inst)) b"decoding near '%s': %s!" % (sub, pycompat.bytestr(inst))
) )
except LookupError as k: except LookupError as k:
raise error.Abort(k, hint=b"please check your locale settings") raise error.Abort(k, hint=b"please check your locale settings")
def unitolocal(u): def unitolocal(u):
# type: (Text) -> bytes
"""Convert a unicode string to a byte string of local encoding""" """Convert a unicode string to a byte string of local encoding"""
return tolocal(u.encode('utf-8')) return tolocal(u.encode('utf-8'))
def unifromlocal(s): def unifromlocal(s):
# type: (bytes) -> Text
"""Convert a byte string of local encoding to a unicode string""" """Convert a byte string of local encoding to a unicode string"""
return fromlocal(s).decode('utf-8') return fromlocal(s).decode('utf-8')
def unimethod(bytesfunc): def unimethod(bytesfunc):
# type: (Callable[[Any], bytes]) -> Callable[[Any], Text]
"""Create a proxy method that forwards __unicode__() and __str__() of """Create a proxy method that forwards __unicode__() and __str__() of
Python 3 to __bytes__()""" Python 3 to __bytes__()"""
def unifunc(obj): def unifunc(obj):
return unifromlocal(bytesfunc(obj)) return unifromlocal(bytesfunc(obj))
return unifunc return unifunc
_wide = _sysstr( _wide = _sysstr(
environ.get(b"HGENCODINGAMBIGUOUS", b"narrow") == b"wide" environ.get(b"HGENCODINGAMBIGUOUS", b"narrow") == b"wide"
and b"WFA" and b"WFA"
or b"WF" or b"WF"
) )
def colwidth(s): def colwidth(s):
# type: (bytes) -> int
b"Find the column width of a string for display in the local encoding" b"Find the column width of a string for display in the local encoding"
return ucolwidth(s.decode(_sysstr(encoding), r'replace')) return ucolwidth(s.decode(_sysstr(encoding), r'replace'))
def ucolwidth(d): def ucolwidth(d):
# type: (Text) -> int
b"Find the column width of a Unicode string for display" b"Find the column width of a Unicode string for display"
eaw = getattr(unicodedata, 'east_asian_width', None) eaw = getattr(unicodedata, 'east_asian_width', None)
if eaw is not None: if eaw is not None:
return sum([eaw(c) in _wide and 2 or 1 for c in d]) return sum([eaw(c) in _wide and 2 or 1 for c in d])
return len(d) return len(d)
def getcols(s, start, c): def getcols(s, start, c):
# type: (bytes, int, int) -> bytes
'''Use colwidth to find a c-column substring of s starting at byte '''Use colwidth to find a c-column substring of s starting at byte
index start''' index start'''
for x in pycompat.xrange(start + c, len(s)): for x in pycompat.xrange(start + c, len(s)):
t = s[start:x] t = s[start:x]
if colwidth(t) == c: if colwidth(t) == c:
return t return t
def trim(s, width, ellipsis=b'', leftside=False): def trim(s, width, ellipsis=b'', leftside=False):
# type: (bytes, int, bytes, bool) -> bytes
"""Trim string 's' to at most 'width' columns (including 'ellipsis'). """Trim string 's' to at most 'width' columns (including 'ellipsis').
If 'leftside' is True, left side of string 's' is trimmed. If 'leftside' is True, left side of string 's' is trimmed.
'ellipsis' is always placed at trimmed side. 'ellipsis' is always placed at trimmed side.
>>> from .node import bin >>> from .node import bin
>>> def bprint(s): >>> def bprint(s):
... print(pycompat.sysstr(s)) ... print(pycompat.sysstr(s))
for i in pycompat.xrange(1, len(u)): for i in pycompat.xrange(1, len(u)):
usub = uslice(i) usub = uslice(i)
if ucolwidth(usub) <= width: if ucolwidth(usub) <= width:
return concat(usub.encode(_sysstr(encoding))) return concat(usub.encode(_sysstr(encoding)))
return ellipsis # no enough room for multi-column characters return ellipsis # no enough room for multi-column characters
def lower(s): def lower(s):
# type: (bytes) -> bytes
b"best-effort encoding-aware case-folding of local string s" b"best-effort encoding-aware case-folding of local string s"
try: try:
return asciilower(s) return asciilower(s)
except UnicodeDecodeError: except UnicodeDecodeError:
pass pass
try: try:
if isinstance(s, localstr): if isinstance(s, localstr):
u = s._utf8.decode("utf-8") u = s._utf8.decode("utf-8")
else: else:
u = s.decode(_sysstr(encoding), _sysstr(encodingmode)) u = s.decode(_sysstr(encoding), _sysstr(encodingmode))
lu = u.lower() lu = u.lower()
if u == lu: if u == lu:
return s # preserve localstring return s # preserve localstring
return lu.encode(_sysstr(encoding)) return lu.encode(_sysstr(encoding))
except UnicodeError: except UnicodeError:
return s.lower() # we don't know how to fold this except in ASCII return s.lower() # we don't know how to fold this except in ASCII
except LookupError as k: except LookupError as k:
raise error.Abort(k, hint=b"please check your locale settings") raise error.Abort(k, hint=b"please check your locale settings")
def upper(s): def upper(s):
# type: (bytes) -> bytes
b"best-effort encoding-aware case-folding of local string s" b"best-effort encoding-aware case-folding of local string s"
try: try:
return asciiupper(s) return asciiupper(s)
except UnicodeDecodeError: except UnicodeDecodeError:
return upperfallback(s) return upperfallback(s)
def upperfallback(s): def upperfallback(s):
# type: (Any) -> Any
try: try:
if isinstance(s, localstr): if isinstance(s, localstr):
u = s._utf8.decode("utf-8") u = s._utf8.decode("utf-8")
else: else:
u = s.decode(_sysstr(encoding), _sysstr(encodingmode)) u = s.decode(_sysstr(encoding), _sysstr(encodingmode))
uu = u.upper() uu = u.upper()
if u == uu: if u == uu:
This should be kept in sync with normcase_spec in util.h.''' This should be kept in sync with normcase_spec in util.h.'''
lower = -1 lower = -1
upper = 1 upper = 1
other = 0 other = 0
def jsonescape(s, paranoid=False): def jsonescape(s, paranoid=False):
# type: (Any, Any) -> Any
'''returns a string suitable for JSON '''returns a string suitable for JSON
JSON is problematic for us because it doesn't support non-Unicode JSON is problematic for us because it doesn't support non-Unicode
bytes. To deal with this, we take the following approach: bytes. To deal with this, we take the following approach:
- localstr/safelocalstr objects are converted back to UTF-8 - localstr/safelocalstr objects are converted back to UTF-8
- valid UTF-8/ASCII strings are passed as-is - valid UTF-8/ASCII strings are passed as-is
- other strings are converted to UTF-8b surrogate encoding - other strings are converted to UTF-8b surrogate encoding
_utf8strict = r'surrogatepass' _utf8strict = r'surrogatepass'
else: else:
_utf8strict = r'strict' _utf8strict = r'strict'
_utf8len = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4] _utf8len = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4]
def getutf8char(s, pos): def getutf8char(s, pos):
# type: (Any, Any) -> Any
'''get the next full utf-8 character in the given string, starting at pos '''get the next full utf-8 character in the given string, starting at pos
Raises a UnicodeError if the given location does not start a valid Raises a UnicodeError if the given location does not start a valid
utf-8 character. utf-8 character.
''' '''
# find how many bytes to attempt decoding from first nibble # find how many bytes to attempt decoding from first nibble
l = _utf8len[ord(s[pos : pos + 1]) >> 4] l = _utf8len[ord(s[pos : pos + 1]) >> 4]
if not l: # ascii if not l: # ascii
return s[pos : pos + 1] return s[pos : pos + 1]
c = s[pos : pos + l] c = s[pos : pos + l]
# validate with attempted decode # validate with attempted decode
c.decode("utf-8", _utf8strict) c.decode("utf-8", _utf8strict)
return c return c
def toutf8b(s): def toutf8b(s):
# type: (Any) -> Any
'''convert a local, possibly-binary string into UTF-8b '''convert a local, possibly-binary string into UTF-8b
This is intended as a generic method to preserve data when working This is intended as a generic method to preserve data when working
with schemes like JSON and XML that have no provision for with schemes like JSON and XML that have no provision for
arbitrary byte strings. As Mercurial often doesn't know arbitrary byte strings. As Mercurial often doesn't know
what encoding data is in, we use so-called UTF-8b. what encoding data is in, we use so-called UTF-8b.
If a string is already valid UTF-8 (or ASCII), it passes unmodified. If a string is already valid UTF-8 (or ASCII), it passes unmodified.
except UnicodeDecodeError: except UnicodeDecodeError:
c = unichr(0xDC00 + ord(s[pos])).encode('utf-8', _utf8strict) c = unichr(0xDC00 + ord(s[pos])).encode('utf-8', _utf8strict)
pos += 1 pos += 1
r += c r += c
return r return r
def fromutf8b(s): def fromutf8b(s):
# type: (Text) -> bytes
'''Given a UTF-8b string, return a local, possibly-binary string. '''Given a UTF-8b string, return a local, possibly-binary string.
return the original binary string. This return the original binary string. This
is a round-trip process for strings like filenames, but metadata is a round-trip process for strings like filenames, but metadata
that's was passed through tolocal will remain in UTF-8. that's was passed through tolocal will remain in UTF-8.
>>> roundtrip = lambda x: fromutf8b(toutf8b(x)) == x >>> roundtrip = lambda x: fromutf8b(toutf8b(x)) == x
>>> m = b"\\xc3\\xa9\\x99abcd" >>> m = b"\\xc3\\xa9\\x99abcd"

tests/test-check-pyflakes.t

> -X mercurial/thirdparty/cbor \ > -X mercurial/thirdparty/cbor \
> -X mercurial/thirdparty/concurrent \ > -X mercurial/thirdparty/concurrent \
> -X mercurial/thirdparty/zope \ > -X mercurial/thirdparty/zope \
> 2>/dev/null \ > 2>/dev/null \
> | xargs pyflakes 2>/dev/null | "$TESTDIR/filterpyflakes.py" > | xargs pyflakes 2>/dev/null | "$TESTDIR/filterpyflakes.py"
contrib/perf.py:*: undefined name 'xrange' (glob) (?) contrib/perf.py:*: undefined name 'xrange' (glob) (?)
mercurial/hgweb/server.py:*: undefined name 'reload' (glob) (?) mercurial/hgweb/server.py:*: undefined name 'reload' (glob) (?)
mercurial/util.py:*: undefined name 'file' (glob) (?) mercurial/util.py:*: undefined name 'file' (glob) (?)
mercurial/encoding.py:*: undefined name 'localstr' (glob) (?)