This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6832

highlight: fix encoding issues to enable Py3 compatibility
ClosedPublic

Authored by sheehan on Sep 9 2019, 3:22 PM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHGa7abc6081bc5: highlight: fix encoding issues to enable Py3 compatibility
Summary

This commit fixes various encoding issues with the highlight extension
to enable compatibility with Python 3. Python .encode() and .decode()
requires the target encoding to be passed as a str, so the value of
mercurial.encoding.encoding must be converted before passing to the
function. Pygments also assumes the str type for values it works with,
so we must perform conversions before and after receiving values from its
APIs.

After applying this patch, test-highlight.t passes under Python 3. We
add it to python3-whitelist as well.

Tested with Pygments 2.4.2.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

sheehan created this revision.Sep 9 2019, 3:22 PM
pulkit accepted this revision.Sep 11 2019, 2:28 AM
This revision is now accepted and ready to land.Sep 11 2019, 2:28 AM

Revision Contents
Changeset List

Diff 16514

contrib/python3-whitelist

test-hgweb-non-interactive.t test-hgweb-non-interactive.t
test-hgweb-raw.t test-hgweb-raw.t
test-hgweb-removed.t test-hgweb-removed.t
test-hgweb-symrev.t test-hgweb-symrev.t
test-hgweb.t test-hgweb.t
test-hgwebdir-paths.py test-hgwebdir-paths.py
test-hgwebdir.t test-hgwebdir.t
test-hgwebdirsym.t test-hgwebdirsym.t
test-highlight.t
test-histedit-arguments.t test-histedit-arguments.t
test-histedit-base.t test-histedit-base.t
test-histedit-bookmark-motion.t test-histedit-bookmark-motion.t
test-histedit-commute.t test-histedit-commute.t
test-histedit-drop.t test-histedit-drop.t
test-histedit-edit.t test-histedit-edit.t
test-histedit-fold-non-commute.t test-histedit-fold-non-commute.t
test-histedit-fold.t test-histedit-fold.t

hgext/highlight/__init__.py

from . import highlight from . import highlight
from mercurial.hgweb import ( from mercurial.hgweb import (
webcommands, webcommands,
webutil, webutil,
) )
from mercurial import ( from mercurial import (
extensions, extensions,
pycompat,
) )
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
# be specifying the version(s) of Mercurial they are tested with, or # be specifying the version(s) of Mercurial they are tested with, or
# leave the attribute unspecified. # leave the attribute unspecified.
testedwith = 'ships-with-hg-core' testedwith = 'ships-with-hg-core'
if 'html' in mt: if 'html' in mt:
fctx = webutil.filectx(web.repo, web.req) fctx = webutil.filectx(web.repo, web.req)
pygmentize(web, 'annotateline', fctx, web.tmpl) pygmentize(web, 'annotateline', fctx, web.tmpl)
return orig(web) return orig(web)
def generate_css(web): def generate_css(web):
pg_style = web.config('web', 'pygments_style', 'colorful') pg_style = web.config('web', 'pygments_style', 'colorful')
fmter = highlight.HtmlFormatter(style=pg_style) fmter = highlight.HtmlFormatter(style=pycompat.sysstr(pg_style))
web.res.headers['Content-Type'] = 'text/css' web.res.headers['Content-Type'] = 'text/css'
style_defs = fmter.get_style_defs(pycompat.sysstr(''))
web.res.setbodybytes(''.join([ web.res.setbodybytes(''.join([
'/* pygments_style = %s */\n\n' % pg_style, '/* pygments_style = %s */\n\n' % pg_style,
fmter.get_style_defs(''), pycompat.bytestr(style_defs),
])) ]))
return web.res.sendresponse() return web.res.sendresponse()
def extsetup(ui): def extsetup(ui):
# monkeypatch in the new version # monkeypatch in the new version
extensions.wrapfunction(webcommands, '_filerevision', extensions.wrapfunction(webcommands, '_filerevision',
filerevision_highlight) filerevision_highlight)
extensions.wrapfunction(webcommands, 'annotate', annotate_highlight) extensions.wrapfunction(webcommands, 'annotate', annotate_highlight)
webcommands.highlightcss = generate_css webcommands.highlightcss = generate_css
webcommands.__all__.append('highlightcss') webcommands.__all__.append('highlightcss')

hgext/highlight/highlight.py

from __future__ import absolute_import from __future__ import absolute_import
from mercurial import demandimport from mercurial import demandimport
demandimport.IGNORES.update(['pkgutil', 'pkg_resources', '__main__']) demandimport.IGNORES.update(['pkgutil', 'pkg_resources', '__main__'])
from mercurial import ( from mercurial import (
encoding, encoding,
pycompat,
) )
from mercurial.utils import ( from mercurial.utils import (
stringutil, stringutil,
) )
with demandimport.deactivated(): with demandimport.deactivated():
import pygments import pygments
# str.splitlines() != unicode.splitlines() because "reasons" # str.splitlines() != unicode.splitlines() because "reasons"
for c in "\x0c\x1c\x1d\x1e": for c in "\x0c\x1c\x1d\x1e":
if c in text: if c in text:
text = text.replace(c, '') text = text.replace(c, '')
# Pygments is best used with Unicode strings: # Pygments is best used with Unicode strings:
# <http://pygments.org/docs/unicode/> # <http://pygments.org/docs/unicode/>
text = text.decode(encoding.encoding, 'replace') text = text.decode(pycompat.sysstr(encoding.encoding), 'replace')
# To get multi-line strings right, we can't format line-by-line # To get multi-line strings right, we can't format line-by-line
try: try:
lexer = guess_lexer_for_filename(fctx.path(), text[:1024], path = pycompat.sysstr(fctx.path())
lexer = guess_lexer_for_filename(path, text[:1024],
stripnl=False) stripnl=False)
except (ClassNotFound, ValueError): except (ClassNotFound, ValueError):
# guess_lexer will return a lexer if *any* lexer matches. There is # guess_lexer will return a lexer if *any* lexer matches. There is
# no way to specify a minimum match score. This can give a high rate of # no way to specify a minimum match score. This can give a high rate of
# false positives on files with an unknown filename pattern. # false positives on files with an unknown filename pattern.
if guessfilenameonly: if guessfilenameonly:
return return
try: try:
lexer = guess_lexer(text[:1024], stripnl=False) lexer = guess_lexer(text[:1024], stripnl=False)
except (ClassNotFound, ValueError): except (ClassNotFound, ValueError):
# Don't highlight unknown files # Don't highlight unknown files
return return
# Don't highlight text files # Don't highlight text files
if isinstance(lexer, TextLexer): if isinstance(lexer, TextLexer):
return return
formatter = HtmlFormatter(nowrap=True, style=style) formatter = HtmlFormatter(nowrap=True, style=pycompat.sysstr(style))
colorized = highlight(text, lexer, formatter) colorized = highlight(text, lexer, formatter)
coloriter = (s.encode(encoding.encoding, 'replace') coloriter = (s.encode(pycompat.sysstr(encoding.encoding), 'replace')
for s in colorized.splitlines()) for s in colorized.splitlines())
tmpl._filters['colorize'] = lambda x: next(coloriter) tmpl._filters['colorize'] = lambda x: next(coloriter)
oldl = tmpl.cache[field] oldl = tmpl.cache[field]
newl = oldl.replace('line|escape', 'line|colorize') newl = oldl.replace('line|escape', 'line|colorize')
tmpl.cache[field] = newl tmpl.cache[field] = newl