This is an archive of the discontinued Mercurial Phabricator instance.

Differential D958

i18n: clean msgcache when encoding changes
AbandonedPublic

Authored by quark on Oct 5 2017, 4:47 PM.

Details

Reviewers
lothiraldan
Group Reviewers
hg-reviewers
Summary

In rare cases (ex. chg), encoding may change on the fly and existing i18n
msgcache needs to be cleared.

With demandimport disabled, test-commit-interactive.t now fails consistently
with and without chg.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

quark created this revision.Oct 5 2017, 4:47 PM
quark edited the summary of this revision. (Show Details)Oct 5 2017, 6:41 PM
quark updated this revision to Diff 2490.
lothiraldan added a subscriber: lothiraldan.Oct 6 2017, 2:35 AM
lothiraldan added inline comments.
mercurial/encoding.py
596

Should we catch exceptions here if a callback fails?

ryanmce added a subscriber: ryanmce.Oct 6 2017, 4:59 AM

Is there a bugzilla bug where I can see what issue this addresses?

quark added a comment.Oct 6 2017, 1:27 PM
In D958#16241, @ryanmce wrote:

Is there a bugzilla bug where I can see what issue this addresses?

There isn't. This series is to make test-commit-interactive.t pass with chg.

mercurial/encoding.py
596

I think it's the callback's responsibility to catch exceptions if they need to. Besides, we don't do that for hg.wirepeersetupfuncs.

lothiraldan accepted this revision.Oct 9 2017, 5:46 AM
lothiraldan added inline comments.
mercurial/encoding.py
596

Ok, thank you for the explanation.

yuja added a subscriber: yuja.Oct 13 2017, 8:58 AM

I've sent an alternative idea as D1053. Can you take a look?

quark abandoned this revision.Oct 13 2017, 1:05 PM

D1053 is a cleaner fix.

Revision Contents
Changeset List

Diff 2473

mercurial/dispatch.py

raise error.Abort(_("option --config may not be abbreviated!")) raise error.Abort(_("option --config may not be abbreviated!"))
if options["cwd"]: if options["cwd"]:
raise error.Abort(_("option --cwd may not be abbreviated!")) raise error.Abort(_("option --cwd may not be abbreviated!"))
if options["repository"]: if options["repository"]:
raise error.Abort(_( raise error.Abort(_(
"option -R has to be separated from other options (e.g. not " "option -R has to be separated from other options (e.g. not "
"-qR) and --repository may only be abbreviated as --repo!")) "-qR) and --repository may only be abbreviated as --repo!"))
if options["encoding"]: encoding.setencoding(options["encoding"], options["encodingmode"])
encoding.encoding = options["encoding"]
if options["encodingmode"]:
encoding.encodingmode = options["encodingmode"]
if options["time"]: if options["time"]:
def get_times(): def get_times():
t = os.times() t = os.times()
if t[4] == 0.0: if t[4] == 0.0:
# Windows leaves this as zero, so use time.clock() # Windows leaves this as zero, so use time.clock()
t = (t[0], t[1], t[2], t[3], time.clock()) t = (t[0], t[1], t[2], t[3], time.clock())
return t return t
s = get_times() s = get_times()

mercurial/encoding.py

c = getutf8char(s, pos) c = getutf8char(s, pos)
pos += len(c) pos += len(c)
# unescape U+DCxx characters # unescape U+DCxx characters
if "\xed\xb0\x80" <= c <= "\xed\xb3\xbf": if "\xed\xb0\x80" <= c <= "\xed\xb3\xbf":
c = pycompat.bytechr(ord(c.decode("utf-8", _utf8strict)) & 0xff) c = pycompat.bytechr(ord(c.decode("utf-8", _utf8strict)) & 0xff)
r += c r += c
return r return r
# used by i18n to clean its message cache after encoding change
setencodingcallbacks = []
def setencoding(newencoding=None, newencodingmode=None):
global encoding, encodingmode
oldvalues = (encoding, encodingmode)
if newencoding:
encoding = newencoding
if newencodingmode:
encodingmode = newencodingmode
if oldvalues != (encoding, encodingmode):
for func in setencodingcallbacks:
func()
lothiraldanUnsubmitted
Not Done

Should we catch exceptions here if a callback fails?

lothiraldan: Should we catch exceptions here if a callback fails?
quarkAuthorUnsubmitted
Not Done

I think it's the callback's responsibility to catch exceptions if they need to. Besides, we don't do that for hg.wirepeersetupfuncs.

quark: I think it's the callback's responsibility to catch exceptions if they need to. Besides, we…
lothiraldanUnsubmitted
Not Done

Ok, thank you for the explanation.

lothiraldan: Ok, thank you for the explanation.
if pycompat.ispy3: if pycompat.ispy3:
class strio(io.TextIOWrapper): class strio(io.TextIOWrapper):
"""Wrapper around TextIOWrapper that respects hg's encoding assumptions. """Wrapper around TextIOWrapper that respects hg's encoding assumptions.
Also works around Python closing streams. Also works around Python closing streams.
""" """
def __init__(self, buffer): def __init__(self, buffer):
super(strio, self).__init__(buffer, encoding=_sysstr(encoding)) super(strio, self).__init__(buffer, encoding=_sysstr(encoding))
def __del__(self): def __del__(self):
"""Override __del__ so it doesn't close the underlying stream.""" """Override __del__ so it doesn't close the underlying stream."""
else: else:
strio = pycompat.identity strio = pycompat.identity

mercurial/hgweb/hgweb_mod.py

with profiling.profile(repo.ui, enabled=profile): with profiling.profile(repo.ui, enabled=profile):
for r in self._runwsgi(req, repo): for r in self._runwsgi(req, repo):
yield r yield r
def _runwsgi(self, req, repo): def _runwsgi(self, req, repo):
rctx = requestcontext(self, repo) rctx = requestcontext(self, repo)
# This state is global across all threads. # This state is global across all threads.
encoding.encoding = rctx.config('web', 'encoding') encoding.setencoding(rctx.config('web', 'encoding'))
rctx.repo.ui.environ = req.env rctx.repo.ui.environ = req.env
if rctx.csp: if rctx.csp:
# hgwebdir may have added CSP header. Since we generate our own, # hgwebdir may have added CSP header. Since we generate our own,
# replace it. # replace it.
req.headers = [h for h in req.headers req.headers = [h for h in req.headers
if h[0] != 'Content-Security-Policy'] if h[0] != 'Content-Security-Policy']
req.headers.append(('Content-Security-Policy', rctx.csp)) req.headers.append(('Content-Security-Policy', rctx.csp))

mercurial/hgweb/hgwebdir_mod.py

repo = os.path.normpath(path) repo = os.path.normpath(path)
name = util.pconvert(repo) name = util.pconvert(repo)
if name.startswith(prefix): if name.startswith(prefix):
name = name[len(prefix):] name = name[len(prefix):]
repos.append((name.lstrip('/'), repo)) repos.append((name.lstrip('/'), repo))
self.repos = repos self.repos = repos
self.ui = u self.ui = u
encoding.encoding = self.ui.config('web', 'encoding') encoding.setencoding(self.ui.config('web', 'encoding'))
self.style = self.ui.config('web', 'style') self.style = self.ui.config('web', 'style')
self.templatepath = self.ui.config('web', 'templates', untrusted=False) self.templatepath = self.ui.config('web', 'templates', untrusted=False)
self.stripecount = self.ui.config('web', 'stripes') self.stripecount = self.ui.config('web', 'stripes')
if self.stripecount: if self.stripecount:
self.stripecount = int(self.stripecount) self.stripecount = int(self.stripecount)
self._baseurl = self.ui.config('web', 'baseurl') self._baseurl = self.ui.config('web', 'baseurl')
prefix = self.ui.config('web', 'prefix') prefix = self.ui.config('web', 'prefix')
if prefix.startswith('/'): if prefix.startswith('/'):

mercurial/i18n.py

global _ugettext global _ugettext
try: try:
_ugettext = t.ugettext _ugettext = t.ugettext
except AttributeError: except AttributeError:
_ugettext = t.gettext _ugettext = t.gettext
_msgcache = {} _msgcache = {}
encoding.setencodingcallbacks.append(_msgcache.clear)
def gettext(message): def gettext(message):
"""Translate message. """Translate message.
The message is looked up in the catalog to get a Unicode string, The message is looked up in the catalog to get a Unicode string,
which is encoded in the local encoding before being returned. which is encoded in the local encoding before being returned.
Important: message is restricted to characters in the encoding Important: message is restricted to characters in the encoding
given by sys.getdefaultencoding() which is most likely 'ascii'. given by sys.getdefaultencoding() which is most likely 'ascii'.

tests/test-commit-interactive.t

> for l in sys.stdin: > for l in sys.stdin:
> sys.stdout.write(''.join(escape(c) for c in l)) > sys.stdout.write(''.join(escape(c) for c in l))
> EOF > EOF
$ hg commit -i --encoding cp932 2>&1 <<EOF | $PYTHON $TESTTMP/escape.py | grep '^y - ' $ hg commit -i --encoding cp932 2>&1 <<EOF | $PYTHON $TESTTMP/escape.py | grep '^y - '
> ? > ?
> q > q
> EOF > EOF
y - \x82\xb1\x82\xcc\x95\xcf\x8dX\x82\xf0\x8bL\x98^(yes) y - ???????(yes)
$ LANGUAGE= $ LANGUAGE=
#endif #endif
Skip Skip
$ hg commit -i <<EOF $ hg commit -i <<EOF
> s > s

tests/test-i18n.t

Check Mercurial specific translation problems in each *.po files, and Check Mercurial specific translation problems in each *.po files, and
tool itself by doctest tool itself by doctest
$ cd "$TESTDIR"/../i18n $ cd "$TESTDIR"/../i18n
$ $PYTHON check-translation.py *.po $ $PYTHON check-translation.py *.po
$ $PYTHON check-translation.py --doctest $ $PYTHON check-translation.py --doctest
$ cd $TESTTMP $ cd $TESTTMP
Check i18n cache gets invalidated correctly with encoding change:
$ cat > $TESTTMP/encodingchange.py << EOF
> from mercurial import encoding, registrar
> from mercurial.i18n import _
> cmdtable = {}
> command = registrar.command(cmdtable)
> @command('encodingchange', norepo=True)
> def encodingchange(ui):
> for encode in ('ascii', 'UTF-8', 'ascii', 'UTF-8'):
> encoding.setencoding(encode)
> ui.write('%s\n' % encoding.jsonescape(_('(EXPERIMENTAL)')))
> EOF
$ LANGUAGE=ja hg --config extensions.encodingchange=$TESTTMP/encodingchange.py encodingchange
(?????)
(\xe5\xae\x9f\xe9\xa8\x93\xe7\x9a\x84\xe5\xae\x9f\xe8\xa3\x85) (esc)
(?????)
(\xe5\xae\x9f\xe9\xa8\x93\xe7\x9a\x84\xe5\xae\x9f\xe8\xa3\x85) (esc)