This is an archive of the discontinued Mercurial Phabricator instance.

Differential D2771

hgweb: expose input stream on parsed WSGI request object
ClosedPublic

Authored by indygreg on Mar 9 2018, 8:23 PM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHGda4e2f87167d: hgweb: expose input stream on parsed WSGI request object
Summary

Our next step towards moving away from wsgirequest to our newer,
friendlier parsedrequest type is input stream access.

This commit exposes the input stream on the instance. Consumers
in the HTTP protocol server switch to it.

Because there were very few consumers of the input stream, we stopped
storing a reference to the input stream on wsgirequest directly. All
access now goes through parsedrequest. However, wsgirequest still
may read from this stream as part of cgi.parse(). So we still need to
create the stream from wsgirequest.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Mar 9 2018, 8:23 PM
indygreg updated this revision to Diff 6833.Mar 10 2018, 3:03 PM
durin42 accepted this revision.Mar 12 2018, 4:58 PM
This revision is now accepted and ready to land.Mar 12 2018, 4:58 PM
Closed by commit rHGda4e2f87167d: hgweb: expose input stream on parsed WSGI request object (authored by indygreg). · Explain WhyMar 12 2018, 5:32 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 6909

mercurial/hgweb/hgwebdir_mod.py

for virtualrepo in _virtualdirs(): for virtualrepo in _virtualdirs():
real = repos.get(virtualrepo) real = repos.get(virtualrepo)
if real: if real:
wsgireq.env['REPO_NAME'] = virtualrepo wsgireq.env['REPO_NAME'] = virtualrepo
# We have to re-parse because of updated environment # We have to re-parse because of updated environment
# variable. # variable.
# TODO this is kind of hacky and we should have a better # TODO this is kind of hacky and we should have a better
# way of doing this than with REPO_NAME side-effects. # way of doing this than with REPO_NAME side-effects.
wsgireq.req = requestmod.parserequestfromenv(wsgireq.env) wsgireq.req = requestmod.parserequestfromenv(
wsgireq.env, wsgireq.req.bodyfh)
try: try:
# ensure caller gets private copy of ui # ensure caller gets private copy of ui
repo = hg.repository(self.ui.copy(), real) repo = hg.repository(self.ui.copy(), real)
return hgweb_mod.hgweb(repo).run_wsgi(wsgireq) return hgweb_mod.hgweb(repo).run_wsgi(wsgireq)
except IOError as inst: except IOError as inst:
msg = encoding.strtolocal(inst.strerror) msg = encoding.strtolocal(inst.strerror)
raise ErrorResponse(HTTP_SERVER_ERROR, msg) raise ErrorResponse(HTTP_SERVER_ERROR, msg)
except error.RepoError as inst: except error.RepoError as inst:

mercurial/hgweb/request.py

bytesform = {} bytesform = {}
for k, v in form.iteritems(): for k, v in form.iteritems():
bytesform[pycompat.bytesurl(k)] = [ bytesform[pycompat.bytesurl(k)] = [
pycompat.bytesurl(i.strip()) for i in v] pycompat.bytesurl(i.strip()) for i in v]
return bytesform return bytesform
@attr.s(frozen=True) @attr.s(frozen=True)
class parsedrequest(object): class parsedrequest(object):
"""Represents a parsed WSGI request / static HTTP request parameters.""" """Represents a parsed WSGI request.
Contains both parsed parameters as well as a handle on the input stream.
"""
# Request method. # Request method.
method = attr.ib() method = attr.ib()
# Full URL for this request. # Full URL for this request.
url = attr.ib() url = attr.ib()
# URL without any path components. Just <proto>://<host><port>. # URL without any path components. Just <proto>://<host><port>.
baseurl = attr.ib() baseurl = attr.ib()
# Advertised URL. Like ``url`` and ``baseurl`` but uses SERVER_NAME instead # Advertised URL. Like ``url`` and ``baseurl`` but uses SERVER_NAME instead
querystring = attr.ib() querystring = attr.ib()
# List of 2-tuples of query string arguments. # List of 2-tuples of query string arguments.
querystringlist = attr.ib() querystringlist = attr.ib()
# Dict of query string arguments. Values are lists with at least 1 item. # Dict of query string arguments. Values are lists with at least 1 item.
querystringdict = attr.ib() querystringdict = attr.ib()
# wsgiref.headers.Headers instance. Operates like a dict with case # wsgiref.headers.Headers instance. Operates like a dict with case
# insensitive keys. # insensitive keys.
headers = attr.ib() headers = attr.ib()
# Request body input stream.
bodyfh = attr.ib()
def parserequestfromenv(env): def parserequestfromenv(env, bodyfh):
"""Parse URL components from environment variables. """Parse URL components from environment variables.
WSGI defines request attributes via environment variables. This function WSGI defines request attributes via environment variables. This function
parses the environment variables into a data structure. parses the environment variables into a data structure.
""" """
# PEP-0333 defines the WSGI spec and is a useful reference for this code. # PEP-0333 defines the WSGI spec and is a useful reference for this code.
# We first validate that the incoming object conforms with the WSGI spec. # We first validate that the incoming object conforms with the WSGI spec.
# This is kind of a lie because the HTTP header wasn't explicitly # This is kind of a lie because the HTTP header wasn't explicitly
# sent. But for all intents and purposes it should be OK to lie about # sent. But for all intents and purposes it should be OK to lie about
# this, since a consumer will either either value to determine how many # this, since a consumer will either either value to determine how many
# bytes are available to read. # bytes are available to read.
if 'CONTENT_LENGTH' in env and 'HTTP_CONTENT_LENGTH' not in env: if 'CONTENT_LENGTH' in env and 'HTTP_CONTENT_LENGTH' not in env:
headers['Content-Length'] = env['CONTENT_LENGTH'] headers['Content-Length'] = env['CONTENT_LENGTH']
# TODO do this once we remove wsgirequest.inp, otherwise we could have
# multiple readers from the underlying input stream.
#bodyfh = env['wsgi.input']
#if 'Content-Length' in headers:
# bodyfh = util.cappedreader(bodyfh, int(headers['Content-Length']))
return parsedrequest(method=env['REQUEST_METHOD'], return parsedrequest(method=env['REQUEST_METHOD'],
url=fullurl, baseurl=baseurl, url=fullurl, baseurl=baseurl,
advertisedurl=advertisedfullurl, advertisedurl=advertisedfullurl,
advertisedbaseurl=advertisedbaseurl, advertisedbaseurl=advertisedbaseurl,
apppath=apppath, apppath=apppath,
dispatchparts=dispatchparts, dispatchpath=dispatchpath, dispatchparts=dispatchparts, dispatchpath=dispatchpath,
havepathinfo='PATH_INFO' in env, havepathinfo='PATH_INFO' in env,
querystring=querystring, querystring=querystring,
querystringlist=querystringlist, querystringlist=querystringlist,
querystringdict=querystringdict, querystringdict=querystringdict,
headers=headers) headers=headers,
bodyfh=bodyfh)
class wsgirequest(object): class wsgirequest(object):
"""Higher-level API for a WSGI request. """Higher-level API for a WSGI request.
WSGI applications are invoked with 2 arguments. They are used to WSGI applications are invoked with 2 arguments. They are used to
instantiate instances of this class, which provides higher-level APIs instantiate instances of this class, which provides higher-level APIs
for obtaining request parameters, writing HTTP output, etc. for obtaining request parameters, writing HTTP output, etc.
""" """
def __init__(self, wsgienv, start_response): def __init__(self, wsgienv, start_response):
version = wsgienv[r'wsgi.version'] version = wsgienv[r'wsgi.version']
if (version < (1, 0)) or (version >= (2, 0)): if (version < (1, 0)) or (version >= (2, 0)):
raise RuntimeError("Unknown and unsupported WSGI version %d.%d" raise RuntimeError("Unknown and unsupported WSGI version %d.%d"
% version) % version)
self.inp = wsgienv[r'wsgi.input']
inp = wsgienv[r'wsgi.input']
if r'HTTP_CONTENT_LENGTH' in wsgienv: if r'HTTP_CONTENT_LENGTH' in wsgienv:
self.inp = util.cappedreader(self.inp, inp = util.cappedreader(inp, int(wsgienv[r'HTTP_CONTENT_LENGTH']))
int(wsgienv[r'HTTP_CONTENT_LENGTH']))
elif r'CONTENT_LENGTH' in wsgienv: elif r'CONTENT_LENGTH' in wsgienv:
self.inp = util.cappedreader(self.inp, inp = util.cappedreader(inp, int(wsgienv[r'CONTENT_LENGTH']))
int(wsgienv[r'CONTENT_LENGTH']))
self.err = wsgienv[r'wsgi.errors'] self.err = wsgienv[r'wsgi.errors']
self.threaded = wsgienv[r'wsgi.multithread'] self.threaded = wsgienv[r'wsgi.multithread']
self.multiprocess = wsgienv[r'wsgi.multiprocess'] self.multiprocess = wsgienv[r'wsgi.multiprocess']
self.run_once = wsgienv[r'wsgi.run_once'] self.run_once = wsgienv[r'wsgi.run_once']
self.env = wsgienv self.env = wsgienv
self.form = normalize(cgi.parse(self.inp, self.form = normalize(cgi.parse(inp,
self.env, self.env,
keep_blank_values=1)) keep_blank_values=1))
self._start_response = start_response self._start_response = start_response
self.server_write = None self.server_write = None
self.headers = [] self.headers = []
self.req = parserequestfromenv(wsgienv) self.req = parserequestfromenv(wsgienv, inp)
def respond(self, status, type, filename=None, body=None): def respond(self, status, type, filename=None, body=None):
if not isinstance(type, str): if not isinstance(type, str):
type = pycompat.sysstr(type) type = pycompat.sysstr(type)
if self._start_response is not None: if self._start_response is not None:
self.headers.append((r'Content-Type', type)) self.headers.append((r'Content-Type', type))
if filename: if filename:
filename = (filename.rpartition('/')[-1] filename = (filename.rpartition('/')[-1]
pass pass
else: else:
# If we don't know how much data to read, there's no guarantee # If we don't know how much data to read, there's no guarantee
# that we can drain the request responsibly. The WSGI # that we can drain the request responsibly. The WSGI
# specification only says that servers *should* ensure the # specification only says that servers *should* ensure the
# input stream doesn't overrun the actual request. So there's # input stream doesn't overrun the actual request. So there's
# no guarantee that reading until EOF won't corrupt the stream # no guarantee that reading until EOF won't corrupt the stream
# state. # state.
if not isinstance(self.inp, util.cappedreader): if not isinstance(self.req.bodyfh, util.cappedreader):
close = True close = True
else: else:
# We /could/ only drain certain HTTP response codes. But 200 # We /could/ only drain certain HTTP response codes. But 200
# and non-200 wire protocol responses both require draining. # and non-200 wire protocol responses both require draining.
# Since we have a capped reader in place for all situations # Since we have a capped reader in place for all situations
# where we drain, it is safe to read from that stream. We'll # where we drain, it is safe to read from that stream. We'll
# either do a drain or no-op if we're already at EOF. # either do a drain or no-op if we're already at EOF.
drain = True drain = True
if close: if close:
self.headers.append((r'Connection', r'Close')) self.headers.append((r'Connection', r'Close'))
if drain: if drain:
assert isinstance(self.inp, util.cappedreader) assert isinstance(self.req.bodyfh, util.cappedreader)
while True: while True:
chunk = self.inp.read(32768) chunk = self.req.bodyfh.read(32768)
if not chunk: if not chunk:
break break
self.server_write = self._start_response( self.server_write = self._start_response(
pycompat.sysstr(status), self.headers) pycompat.sysstr(status), self.headers)
self._start_response = None self._start_response = None
self.headers = [] self.headers = []
if body is not None: if body is not None:

mercurial/wireprotoserver.py

data[k] = knownargs[k][0] data[k] = knownargs[k][0]
return [data[k] for k in keys] return [data[k] for k in keys]
def _args(self): def _args(self):
args = util.rapply(pycompat.bytesurl, self._wsgireq.form.copy()) args = util.rapply(pycompat.bytesurl, self._wsgireq.form.copy())
postlen = int(self._req.headers.get(b'X-HgArgs-Post', 0)) postlen = int(self._req.headers.get(b'X-HgArgs-Post', 0))
if postlen: if postlen:
args.update(urlreq.parseqs( args.update(urlreq.parseqs(
self._wsgireq.inp.read(postlen), keep_blank_values=True)) self._req.bodyfh.read(postlen), keep_blank_values=True))
return args return args
argvalue = decodevaluefromheaders(self._req, b'X-HgArg') argvalue = decodevaluefromheaders(self._req, b'X-HgArg')
args.update(urlreq.parseqs(argvalue, keep_blank_values=True)) args.update(urlreq.parseqs(argvalue, keep_blank_values=True))
return args return args
def forwardpayload(self, fp): def forwardpayload(self, fp):
# Existing clients *always* send Content-Length. # Existing clients *always* send Content-Length.
length = int(self._req.headers[b'Content-Length']) length = int(self._req.headers[b'Content-Length'])
# If httppostargs is used, we need to read Content-Length # If httppostargs is used, we need to read Content-Length
# minus the amount that was consumed by args. # minus the amount that was consumed by args.
length -= int(self._req.headers.get(b'X-HgArgs-Post', 0)) length -= int(self._req.headers.get(b'X-HgArgs-Post', 0))
for s in util.filechunkiter(self._wsgireq.inp, limit=length): for s in util.filechunkiter(self._req.bodyfh, limit=length):
fp.write(s) fp.write(s)
@contextlib.contextmanager @contextlib.contextmanager
def mayberedirectstdio(self): def mayberedirectstdio(self):
oldout = self._ui.fout oldout = self._ui.fout
olderr = self._ui.ferr olderr = self._ui.ferr
out = util.stringio() out = util.stringio()