| hg-reviewers |
This prevents the internals of Python from coercing a unicode to str
on Python 2 and makes tests run with HGUNICODEPEDANTRY=1 a lot
happier.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| _nativeenviron = (not pycompat.ispy3 or os.supports_bytes_environ) | _nativeenviron = (not pycompat.ispy3 or os.supports_bytes_environ) | ||||
| if not pycompat.ispy3: | if not pycompat.ispy3: | ||||
| environ = os.environ # re-exports | environ = os.environ # re-exports | ||||
| elif _nativeenviron: | elif _nativeenviron: | ||||
| environ = os.environb # re-exports | environ = os.environb # re-exports | ||||
| else: | else: | ||||
| # preferred encoding isn't known yet; use utf-8 to avoid unicode error | # preferred encoding isn't known yet; use utf-8 to avoid unicode error | ||||
| # and recreate it once encoding is settled | # and recreate it once encoding is settled | ||||
| environ = dict((k.encode(u'utf-8'), v.encode(u'utf-8')) | environ = dict((k.encode(r'utf-8'), v.encode(r'utf-8')) | ||||
| for k, v in os.environ.items()) # re-exports | for k, v in os.environ.items()) # re-exports | ||||
| _encodingrewrites = { | _encodingrewrites = { | ||||
| '646': 'ascii', | '646': 'ascii', | ||||
| 'ANSI_X3.4-1968': 'ascii', | 'ANSI_X3.4-1968': 'ascii', | ||||
| } | } | ||||
| # cp65001 is a Windows variant of utf-8, which isn't supported on Python 2. | # cp65001 is a Windows variant of utf-8, which isn't supported on Python 2. | ||||
| # No idea if it should be rewritten to the canonical name 'utf-8' on Python 3. | # No idea if it should be rewritten to the canonical name 'utf-8' on Python 3. | ||||
| try: | try: | ||||
| try: | try: | ||||
| # make sure string is actually stored in UTF-8 | # make sure string is actually stored in UTF-8 | ||||
| u = s.decode('UTF-8') | u = s.decode('UTF-8') | ||||
| if encoding == 'UTF-8': | if encoding == 'UTF-8': | ||||
| # fast path | # fast path | ||||
| return s | return s | ||||
| r = u.encode(_sysstr(encoding), u"replace") | r = u.encode(_sysstr(encoding), r"replace") | ||||
| if u == r.decode(_sysstr(encoding)): | if u == r.decode(_sysstr(encoding)): | ||||
| # r is a safe, non-lossy encoding of s | # r is a safe, non-lossy encoding of s | ||||
| return safelocalstr(r) | return safelocalstr(r) | ||||
| return localstr(s, r) | return localstr(s, r) | ||||
| except UnicodeDecodeError: | except UnicodeDecodeError: | ||||
| # we should only get here if we're looking at an ancient changeset | # we should only get here if we're looking at an ancient changeset | ||||
| try: | try: | ||||
| u = s.decode(_sysstr(fallbackencoding)) | u = s.decode(_sysstr(fallbackencoding)) | ||||
| r = u.encode(_sysstr(encoding), u"replace") | r = u.encode(_sysstr(encoding), r"replace") | ||||
| if u == r.decode(_sysstr(encoding)): | if u == r.decode(_sysstr(encoding)): | ||||
| # r is a safe, non-lossy encoding of s | # r is a safe, non-lossy encoding of s | ||||
| return safelocalstr(r) | return safelocalstr(r) | ||||
| return localstr(u.encode('UTF-8'), r) | return localstr(u.encode('UTF-8'), r) | ||||
| except UnicodeDecodeError: | except UnicodeDecodeError: | ||||
| u = s.decode("utf-8", "replace") # last ditch | u = s.decode("utf-8", "replace") # last ditch | ||||
| # can't round-trip | # can't round-trip | ||||
| return u.encode(_sysstr(encoding), u"replace") | return u.encode(_sysstr(encoding), r"replace") | ||||
| except LookupError as k: | except LookupError as k: | ||||
| raise error.Abort(k, hint="please check your locale settings") | raise error.Abort(k, hint="please check your locale settings") | ||||
| def fromlocal(s): | def fromlocal(s): | ||||
| """ | """ | ||||
| Convert a string from the local character encoding to UTF-8 | Convert a string from the local character encoding to UTF-8 | ||||
| We attempt to decode strings using the encoding mode set by | We attempt to decode strings using the encoding mode set by | ||||
| else: | else: | ||||
| strtolocal = pycompat.identity | strtolocal = pycompat.identity | ||||
| strfromlocal = pycompat.identity | strfromlocal = pycompat.identity | ||||
| strmethod = pycompat.identity | strmethod = pycompat.identity | ||||
| if not _nativeenviron: | if not _nativeenviron: | ||||
| # now encoding and helper functions are available, recreate the environ | # now encoding and helper functions are available, recreate the environ | ||||
| # dict to be exported to other modules | # dict to be exported to other modules | ||||
| environ = dict((tolocal(k.encode(u'utf-8')), tolocal(v.encode(u'utf-8'))) | environ = dict((tolocal(k.encode(r'utf-8')), tolocal(v.encode(r'utf-8'))) | ||||
| for k, v in os.environ.items()) # re-exports | for k, v in os.environ.items()) # re-exports | ||||
| if pycompat.ispy3: | if pycompat.ispy3: | ||||
| # os.getcwd() on Python 3 returns string, but it has os.getcwdb() which | # os.getcwd() on Python 3 returns string, but it has os.getcwdb() which | ||||
| # returns bytes. | # returns bytes. | ||||
| if pycompat.iswindows: | if pycompat.iswindows: | ||||
| # Python 3 on Windows issues a DeprecationWarning about using the bytes | # Python 3 on Windows issues a DeprecationWarning about using the bytes | ||||
| # API when os.getcwdb() is called. | # API when os.getcwdb() is called. | ||||
| getcwd = lambda: strtolocal(os.getcwd()) # re-exports | getcwd = lambda: strtolocal(os.getcwd()) # re-exports | ||||
| else: | else: | ||||
| getcwd = os.getcwdb # re-exports | getcwd = os.getcwdb # re-exports | ||||
| else: | else: | ||||
| getcwd = os.getcwd # re-exports | getcwd = os.getcwd # re-exports | ||||
| # How to treat ambiguous-width characters. Set to 'wide' to treat as wide. | # How to treat ambiguous-width characters. Set to 'wide' to treat as wide. | ||||
| _wide = _sysstr(environ.get("HGENCODINGAMBIGUOUS", "narrow") == "wide" | _wide = _sysstr(environ.get("HGENCODINGAMBIGUOUS", "narrow") == "wide" | ||||
| and "WFA" or "WF") | and "WFA" or "WF") | ||||
| def colwidth(s): | def colwidth(s): | ||||
| "Find the column width of a string for display in the local encoding" | "Find the column width of a string for display in the local encoding" | ||||
| return ucolwidth(s.decode(_sysstr(encoding), u'replace')) | return ucolwidth(s.decode(_sysstr(encoding), r'replace')) | ||||
| def ucolwidth(d): | def ucolwidth(d): | ||||
| "Find the column width of a Unicode string for display" | "Find the column width of a Unicode string for display" | ||||
| eaw = getattr(unicodedata, 'east_asian_width', None) | eaw = getattr(unicodedata, 'east_asian_width', None) | ||||
| if eaw is not None: | if eaw is not None: | ||||
| return sum([eaw(c) in _wide and 2 or 1 for c in d]) | return sum([eaw(c) in _wide and 2 or 1 for c in d]) | ||||
| return len(d) | return len(d) | ||||