This is an archive of the discontinued Mercurial Phabricator instance.

Differential D2772

hgweb: parse and store POST form data
AbandonedPublic

Authored by indygreg on Mar 9 2018, 8:23 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Summary

This is essentially a port of what wsgirequest.init was doing
inline via cgi.parse().

Our version is better because - unlike cgi.parse() - we keep the
query string and POST data separate. We do still expose a unified
dict for convenience, however. But consumers can now be explicit
about whether a parameter should be fetched from the query string
or form data.

Because we can only read from POST data once, we had to support
passing in previously resolved values to support hgwebdir's ugly
hack of having to construct a new parsedrequest type.

The only consumer we update to use the new API is wsgirequest.

Also, the normalization of values is explicitly not performed in
the new code. This is because the intent of our request object is
to represent the raw request with minimal transformation.

With this commit, all meaningful request components are now
reflected on our parsedrequest type!

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Revision Contents
Changeset List

Diff 6816

mercurial/hgweb/hgwebdir_mod.py

for virtualrepo in _virtualdirs(): for virtualrepo in _virtualdirs():
real = repos.get(virtualrepo) real = repos.get(virtualrepo)
if real: if real:
wsgireq.env['REPO_NAME'] = virtualrepo wsgireq.env['REPO_NAME'] = virtualrepo
# We have to re-parse because of updated environment # We have to re-parse because of updated environment
# variable. # variable.
# TODO this is kind of hacky and we should have a better # TODO this is kind of hacky and we should have a better
# way of doing this than with REPO_NAME side-effects. # way of doing this than with REPO_NAME side-effects.
previousdata = (
wsgireq.req.postformlist,
wsgireq.req.postformdict,
wsgireq.req.params,
)
wsgireq.req = requestmod.parserequestfromenv( wsgireq.req = requestmod.parserequestfromenv(
wsgireq.env, wsgireq.req.bodyfh) wsgireq.env, wsgireq.req.bodyfh,
previousformdata=previousdata)
try: try:
# ensure caller gets private copy of ui # ensure caller gets private copy of ui
repo = hg.repository(self.ui.copy(), real) repo = hg.repository(self.ui.copy(), real)
return hgweb_mod.hgweb(repo).run_wsgi(wsgireq) return hgweb_mod.hgweb(repo).run_wsgi(wsgireq)
except IOError as inst: except IOError as inst:
msg = encoding.strtolocal(inst.strerror) msg = encoding.strtolocal(inst.strerror)
raise ErrorResponse(HTTP_SERVER_ERROR, msg) raise ErrorResponse(HTTP_SERVER_ERROR, msg)
except error.RepoError as inst: except error.RepoError as inst:

mercurial/hgweb/request.py

querystringlist = attr.ib() querystringlist = attr.ib()
# Dict of query string arguments. Values are lists with at least 1 item. # Dict of query string arguments. Values are lists with at least 1 item.
querystringdict = attr.ib() querystringdict = attr.ib()
# wsgiref.headers.Headers instance. Operates like a dict with case # wsgiref.headers.Headers instance. Operates like a dict with case
# insensitive keys. # insensitive keys.
headers = attr.ib() headers = attr.ib()
# Request body input stream. # Request body input stream.
bodyfh = attr.ib() bodyfh = attr.ib()
# Like ``querystringlist`` and ``querystringdict`` but for form data
# submitted on POST requests decoded from well-known content types.
postformlist = attr.ib()
postformdict = attr.ib()
def parserequestfromenv(env, bodyfh): # All "form" parameters. A combination of query string and POST form data.
params = attr.ib()
def parserequestfromenv(env, bodyfh, previousformdata=None):
"""Parse URL components from environment variables. """Parse URL components from environment variables.
WSGI defines request attributes via environment variables. This function WSGI defines request attributes via environment variables. This function
parses the environment variables into a data structure. parses the environment variables into a data structure.
""" """
# PEP-0333 defines the WSGI spec and is a useful reference for this code. # PEP-0333 defines the WSGI spec and is a useful reference for this code.
# We first validate that the incoming object conforms with the WSGI spec. # We first validate that the incoming object conforms with the WSGI spec.
headers['Content-Length'] = env['CONTENT_LENGTH'] headers['Content-Length'] = env['CONTENT_LENGTH']
# TODO do this once we remove wsgirequest.inp, otherwise we could have # TODO do this once we remove wsgirequest.inp, otherwise we could have
# multiple readers from the underlying input stream. # multiple readers from the underlying input stream.
#bodyfh = env['wsgi.input'] #bodyfh = env['wsgi.input']
#if 'Content-Length' in headers: #if 'Content-Length' in headers:
# bodyfh = util.cappedreader(bodyfh, int(headers['Content-Length'])) # bodyfh = util.cappedreader(bodyfh, int(headers['Content-Length']))
# Form data is kinda wonky. It can come from request bodies, which we can
# only read once. Since hgwebdir may construct a new parsedrequest, we
# allow existing form data to be passed in to this function. That's kinda
# hacky.
if previousformdata is None:
if env['REQUEST_METHOD'] == 'POST':
# This is based on cgi.parse(), but without the hacky parts (like
# merging QUERY_STRING and setting QUERY_STRING as a side-effect).
ct, params = cgi.parse_header(env.get('CONTENT_TYPE', ''))
if ct == 'multipart/form-data':
# We don't have a way to preserve order. So we normalize to a
# list for consistency with x-www-form-urlencoded.
postformlist = []
for k, l in cgi.parse_multipart(bodyfh, params).iteritems():
for v in l:
postformlist.append((k, v))
elif ct == 'application/x-www-form-urlencoded':
cl = int(headers['Content-Length'])
postformlist = util.urlreq.parseqsl(bodyfh.read(cl),
keep_blank_values=True)
else:
postformlist = []
postformdict = {}
for k, v in postformdict:
if k in postformdict:
postformdict[k].append(v)
else:
postformdict[k] = [v]
else:
postformlist = []
postformdict = {}
# Now that we have the raw post data. Merge in query string data to
# provide a unified interface.
formdict = {k: list(v) for k, v in postformdict.iteritems()}
for k, l in querystringdict.iteritems():
if k not in formdict:
formdict[k] = []
formdict[k].extend(l)
else:
postformlist, postformdict, formdict = previousformdata
return parsedrequest(method=env['REQUEST_METHOD'], return parsedrequest(method=env['REQUEST_METHOD'],
url=fullurl, baseurl=baseurl, url=fullurl, baseurl=baseurl,
advertisedurl=advertisedfullurl, advertisedurl=advertisedfullurl,
advertisedbaseurl=advertisedbaseurl, advertisedbaseurl=advertisedbaseurl,
apppath=apppath, apppath=apppath,
dispatchparts=dispatchparts, dispatchpath=dispatchpath, dispatchparts=dispatchparts, dispatchpath=dispatchpath,
havepathinfo='PATH_INFO' in env, havepathinfo='PATH_INFO' in env,
querystring=querystring, querystring=querystring,
querystringlist=querystringlist, querystringlist=querystringlist,
querystringdict=querystringdict, querystringdict=querystringdict,
headers=headers, headers=headers,
bodyfh=bodyfh) bodyfh=bodyfh,
postformlist=postformlist, postformdict=postformdict,
params=formdict)
class wsgirequest(object): class wsgirequest(object):
"""Higher-level API for a WSGI request. """Higher-level API for a WSGI request.
WSGI applications are invoked with 2 arguments. They are used to WSGI applications are invoked with 2 arguments. They are used to
instantiate instances of this class, which provides higher-level APIs instantiate instances of this class, which provides higher-level APIs
for obtaining request parameters, writing HTTP output, etc. for obtaining request parameters, writing HTTP output, etc.
""" """
elif r'CONTENT_LENGTH' in wsgienv: elif r'CONTENT_LENGTH' in wsgienv:
inp = util.cappedreader(inp, int(wsgienv[r'CONTENT_LENGTH'])) inp = util.cappedreader(inp, int(wsgienv[r'CONTENT_LENGTH']))
self.err = wsgienv[r'wsgi.errors'] self.err = wsgienv[r'wsgi.errors']
self.threaded = wsgienv[r'wsgi.multithread'] self.threaded = wsgienv[r'wsgi.multithread']
self.multiprocess = wsgienv[r'wsgi.multiprocess'] self.multiprocess = wsgienv[r'wsgi.multiprocess']
self.run_once = wsgienv[r'wsgi.run_once'] self.run_once = wsgienv[r'wsgi.run_once']
self.env = wsgienv self.env = wsgienv
self.form = normalize(cgi.parse(inp, self.req = parserequestfromenv(wsgienv, inp)
self.env, self.form = normalize(self.req.params)
keep_blank_values=1))
self._start_response = start_response self._start_response = start_response
self.server_write = None self.server_write = None
self.headers = [] self.headers = []
self.req = parserequestfromenv(wsgienv, inp)
def respond(self, status, type, filename=None, body=None): def respond(self, status, type, filename=None, body=None):
if not isinstance(type, str): if not isinstance(type, str):
type = pycompat.sysstr(type) type = pycompat.sysstr(type)
if self._start_response is not None: if self._start_response is not None:
self.headers.append((r'Content-Type', type)) self.headers.append((r'Content-Type', type))
if filename: if filename:
filename = (filename.rpartition('/')[-1] filename = (filename.rpartition('/')[-1]
.replace('\\', '\\\\').replace('"', '\\"')) .replace('\\', '\\\\').replace('"', '\\"'))