This is an archive of the discontinued Mercurial Phabricator instance.

Differential D2730

hgweb: ensure all wsgi environment values are str
ClosedPublic

Authored by indygreg on Mar 8 2018, 8:06 PM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHG7fc80c982656: hgweb: ensure all wsgi environment values are str
Summary

Previously, we had a few entries that were bytes on Python 3.
PEP-0333 states that all entries must be the native str type
(bytes on Python 2, str on Python 3).

This required a number of changes to hgweb_mod to unbreak
things on Python 3. I suspect there still may be some regressions.

I'm going to introduce a data structure that represents a parsed
WSGI request in upcoming commits. This will hold bytes and will
allow us to stop using raw literals throughout the WSGI code.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Mar 8 2018, 8:06 PM
durin42 accepted this revision.Mar 9 2018, 1:55 PM
This revision is now accepted and ready to land.Mar 9 2018, 1:55 PM
Closed by commit rHG7fc80c982656: hgweb: ensure all wsgi environment values are str (authored by indygreg). · Explain WhyMar 9 2018, 2:29 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 6774

mercurial/hgweb/hgweb_mod.py

proto = 'http' proto = 'http'
default_port = '80' default_port = '80'
port = req.env[r'SERVER_PORT'] port = req.env[r'SERVER_PORT']
port = port != default_port and (r':' + port) or r'' port = port != default_port and (r':' + port) or r''
urlbase = r'%s://%s%s' % (proto, req.env[r'SERVER_NAME'], port) urlbase = r'%s://%s%s' % (proto, req.env[r'SERVER_NAME'], port)
logourl = self.config('web', 'logourl') logourl = self.config('web', 'logourl')
logoimg = self.config('web', 'logoimg') logoimg = self.config('web', 'logoimg')
staticurl = self.config('web', 'staticurl') or req.url + 'static/' staticurl = (self.config('web', 'staticurl')
or pycompat.sysbytes(req.url) + 'static/')
if not staticurl.endswith('/'): if not staticurl.endswith('/'):
staticurl += '/' staticurl += '/'
# some functions for the templater # some functions for the templater
def motd(**map): def motd(**map):
yield self.config('web', 'motd') yield self.config('web', 'motd')
# figure out which style to use # figure out which style to use
vars = {} vars = {}
styles, (style, mapfile) = getstyle(req, self.config, styles, (style, mapfile) = getstyle(req, self.config,
self.templatepath) self.templatepath)
if style == styles[0]: if style == styles[0]:
vars['style'] = style vars['style'] = style
start = '&' if req.url[-1] == r'?' else '?' start = '&' if req.url[-1] == r'?' else '?'
sessionvars = webutil.sessionvars(vars, start) sessionvars = webutil.sessionvars(vars, start)
if not self.reponame: if not self.reponame:
self.reponame = (self.config('web', 'name', '') self.reponame = (self.config('web', 'name', '')
or req.env.get('REPO_NAME') or req.env.get('REPO_NAME')
or req.url.strip('/') or self.repo.root) or req.url.strip(r'/') or self.repo.root)
def websubfilter(text): def websubfilter(text):
return templatefilters.websub(text, self.websubtable) return templatefilters.websub(text, self.websubtable)
# create the templater # create the templater
# TODO: export all keywords: defaults = templatekw.keywords.copy() # TODO: export all keywords: defaults = templatekw.keywords.copy()
defaults = { defaults = {
'url': req.url, 'url': pycompat.sysbytes(req.url),
'logourl': logourl, 'logourl': logourl,
'logoimg': logoimg, 'logoimg': logoimg,
'staticurl': staticurl, 'staticurl': staticurl,
'urlbase': urlbase, 'urlbase': urlbase,
'repo': self.reponame, 'repo': self.reponame,
'encoding': encoding.encoding, 'encoding': encoding.encoding,
'motd': motd, 'motd': motd,
'sessionvars': sessionvars, 'sessionvars': sessionvars,
'pathdef': makebreadcrumb(req.url), 'pathdef': makebreadcrumb(pycompat.sysbytes(req.url)),
'style': style, 'style': style,
'nonce': self.nonce, 'nonce': self.nonce,
} }
tres = formatter.templateresources(self.repo.ui, self.repo) tres = formatter.templateresources(self.repo.ui, self.repo)
tmpl = templater.templater.frommapfile(mapfile, tmpl = templater.templater.frommapfile(mapfile,
filters={'websub': websubfilter}, filters={'websub': websubfilter},
defaults=defaults, defaults=defaults,
resources=tres) resources=tres)
req.headers = [h for h in req.headers req.headers = [h for h in req.headers
if h[0] != 'Content-Security-Policy'] if h[0] != 'Content-Security-Policy']
req.headers.append(('Content-Security-Policy', rctx.csp)) req.headers.append(('Content-Security-Policy', rctx.csp))
# work with CGI variables to create coherent structure # work with CGI variables to create coherent structure
# use SCRIPT_NAME, PATH_INFO and QUERY_STRING as well as our REPO_NAME # use SCRIPT_NAME, PATH_INFO and QUERY_STRING as well as our REPO_NAME
req.url = req.env[r'SCRIPT_NAME'] req.url = req.env[r'SCRIPT_NAME']
if not req.url.endswith('/'): if not req.url.endswith(r'/'):
req.url += '/' req.url += r'/'
if req.env.get('REPO_NAME'): if req.env.get('REPO_NAME'):
req.url += req.env[r'REPO_NAME'] + r'/' req.url += req.env[r'REPO_NAME'] + r'/'
if r'PATH_INFO' in req.env: if r'PATH_INFO' in req.env:
parts = req.env[r'PATH_INFO'].strip('/').split('/') parts = req.env[r'PATH_INFO'].strip(r'/').split(r'/')
repo_parts = req.env.get(r'REPO_NAME', r'').split(r'/') repo_parts = req.env.get(r'REPO_NAME', r'').split(r'/')
if parts[:len(repo_parts)] == repo_parts: if parts[:len(repo_parts)] == repo_parts:
parts = parts[len(repo_parts):] parts = parts[len(repo_parts):]
query = '/'.join(parts) query = r'/'.join(parts)
else: else:
query = req.env[r'QUERY_STRING'].partition(r'&')[0] query = req.env[r'QUERY_STRING'].partition(r'&')[0]
query = query.partition(r';')[0] query = query.partition(r';')[0]
# Route it to a wire protocol handler if it looks like a wire protocol # Route it to a wire protocol handler if it looks like a wire protocol
# request. # request.
protohandler = wireprotoserver.parsehttprequest(rctx, req, query, protohandler = wireprotoserver.parsehttprequest(rctx, req, query,
self.check_perm) self.check_perm)
if protohandler: if protohandler:
try: try:
if query: if query:
raise ErrorResponse(HTTP_NOT_FOUND) raise ErrorResponse(HTTP_NOT_FOUND)
return protohandler['dispatch']() return protohandler['dispatch']()
except ErrorResponse as inst: except ErrorResponse as inst:
return protohandler['handleerror'](inst) return protohandler['handleerror'](inst)
# translate user-visible url structure to internal structure # translate user-visible url structure to internal structure
args = query.split('/', 2) args = query.split(r'/', 2)
if 'cmd' not in req.form and args and args[0]: if 'cmd' not in req.form and args and args[0]:
cmd = args.pop(0) cmd = args.pop(0)
style = cmd.rfind('-') style = cmd.rfind('-')
if style != -1: if style != -1:
req.form['style'] = [cmd[:style]] req.form['style'] = [cmd[:style]]
cmd = cmd[style + 1:] cmd = cmd[style + 1:]
# avoid accepting e.g. style parameter as command # avoid accepting e.g. style parameter as command

mercurial/hgweb/server.py

path, query = _splitURI(self.path) path, query = _splitURI(self.path)
env = {} env = {}
env[r'GATEWAY_INTERFACE'] = r'CGI/1.1' env[r'GATEWAY_INTERFACE'] = r'CGI/1.1'
env[r'REQUEST_METHOD'] = self.command env[r'REQUEST_METHOD'] = self.command
env[r'SERVER_NAME'] = self.server.server_name env[r'SERVER_NAME'] = self.server.server_name
env[r'SERVER_PORT'] = str(self.server.server_port) env[r'SERVER_PORT'] = str(self.server.server_port)
env[r'REQUEST_URI'] = self.path env[r'REQUEST_URI'] = self.path
env[r'SCRIPT_NAME'] = self.server.prefix env[r'SCRIPT_NAME'] = pycompat.sysstr(self.server.prefix)
env[r'PATH_INFO'] = path[len(self.server.prefix):] env[r'PATH_INFO'] = pycompat.sysstr(path[len(self.server.prefix):])
env[r'REMOTE_HOST'] = self.client_address[0] env[r'REMOTE_HOST'] = self.client_address[0]
env[r'REMOTE_ADDR'] = self.client_address[0] env[r'REMOTE_ADDR'] = self.client_address[0]
if query: if query:
env[r'QUERY_STRING'] = query env[r'QUERY_STRING'] = query
if pycompat.ispy3: if pycompat.ispy3:
if self.headers.get_content_type() is None: if self.headers.get_content_type() is None:
env[r'CONTENT_TYPE'] = self.headers.get_default_type() env[r'CONTENT_TYPE'] = self.headers.get_default_type()
if h not in ('content-type', 'content-length')]: if h not in ('content-type', 'content-length')]:
hkey = r'HTTP_' + header.replace(r'-', r'_').upper() hkey = r'HTTP_' + header.replace(r'-', r'_').upper()
hval = self.headers.get(header) hval = self.headers.get(header)
hval = hval.replace(r'\n', r'').strip() hval = hval.replace(r'\n', r'').strip()
if hval: if hval:
env[hkey] = hval env[hkey] = hval
env[r'SERVER_PROTOCOL'] = self.request_version env[r'SERVER_PROTOCOL'] = self.request_version
env[r'wsgi.version'] = (1, 0) env[r'wsgi.version'] = (1, 0)
env[r'wsgi.url_scheme'] = self.url_scheme env[r'wsgi.url_scheme'] = pycompat.sysstr(self.url_scheme)
if env.get(r'HTTP_EXPECT', '').lower() == '100-continue': if env.get(r'HTTP_EXPECT', '').lower() == '100-continue':
self.rfile = common.continuereader(self.rfile, self.wfile.write) self.rfile = common.continuereader(self.rfile, self.wfile.write)
env[r'wsgi.input'] = self.rfile env[r'wsgi.input'] = self.rfile
env[r'wsgi.errors'] = _error_logger(self) env[r'wsgi.errors'] = _error_logger(self)
env[r'wsgi.multithread'] = isinstance(self.server, env[r'wsgi.multithread'] = isinstance(self.server,
socketserver.ThreadingMixIn) socketserver.ThreadingMixIn)
env[r'wsgi.multiprocess'] = isinstance(self.server, env[r'wsgi.multiprocess'] = isinstance(self.server,