This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4857

keepalive: track number of bytes received from an HTTP response
ClosedPublic

Authored by indygreg on Oct 3 2018, 12:51 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG5e5b06087ec5: keepalive: track number of bytes received from an HTTP response
Summary

We also bubble the byte count up to the HTTPConnection instance and its
parent opener at read time. Unlike sending, there isn't a clear
"end of response" signal we can intercept to defer updating the
accounting.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Oct 3 2018, 12:51 PM
Closed by commit rHG5e5b06087ec5: keepalive: track number of bytes received from an HTTP response (authored by indygreg). · Explain WhyOct 3 2018, 8:07 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/keepalive.py (28 lines)

Diff 11664

mercurial/keepalive.py

extrakw = {} extrakw = {}
if not pycompat.ispy3: if not pycompat.ispy3:
extrakw[r'strict'] = True extrakw[r'strict'] = True
extrakw[r'buffering'] = True extrakw[r'buffering'] = True
httplib.HTTPResponse.__init__(self, sock, debuglevel=debuglevel, httplib.HTTPResponse.__init__(self, sock, debuglevel=debuglevel,
method=method, **extrakw) method=method, **extrakw)
self.fileno = sock.fileno self.fileno = sock.fileno
self.code = None self.code = None
self.receivedbytescount = 0
self._rbuf = '' self._rbuf = ''
self._rbufsize = 8096 self._rbufsize = 8096
self._handler = None # inserted by the handler later self._handler = None # inserted by the handler later
self._host = None # (same) self._host = None # (same)
self._url = None # (same) self._url = None # (same)
self._connection = None # (same) self._connection = None # (same)
_raw_read = httplib.HTTPResponse.read _raw_read = httplib.HTTPResponse.read
s = self._rbuf[:amt] s = self._rbuf[:amt]
self._rbuf = self._rbuf[amt:] self._rbuf = self._rbuf[amt:]
return s return s
# Careful! http.client.HTTPResponse.read() on Python 3 is # Careful! http.client.HTTPResponse.read() on Python 3 is
# implemented using readinto(), which can duplicate self._rbuf # implemented using readinto(), which can duplicate self._rbuf
# if it's not empty. # if it's not empty.
s = self._rbuf s = self._rbuf
self._rbuf = '' self._rbuf = ''
s += self._raw_read(amt) data = self._raw_read(amt)
self.receivedbytescount += len(data)
self._connection.receivedbytescount += len(data)
try:
self._handler.parent.receivedbytescount += len(data)
except AttributeError:
pass
s += data
return s return s
# stolen from Python SVN #68532 to fix issue1088 # stolen from Python SVN #68532 to fix issue1088
def _read_chunked(self, amt): def _read_chunked(self, amt):
chunk_left = self.chunk_left chunk_left = self.chunk_left
parts = [] parts = []
while True: while True:
chunks = [self._rbuf] chunks = [self._rbuf]
i = -1 i = -1
readsize = self._rbufsize readsize = self._rbufsize
while True: while True:
new = self._raw_read(readsize) new = self._raw_read(readsize)
if not new: if not new:
break break
self.receivedbytescount += len(new)
self._connection.receivedbytescount += len(new)
try:
self._handler.parent.receivedbytescount += len(new)
except AttributeError:
pass
chunks.append(new) chunks.append(new)
i = new.find('\n') i = new.find('\n')
if i >= 0: if i >= 0:
break break
# We either have exhausted the stream or have a newline in chunks[-1]. # We either have exhausted the stream or have a newline in chunks[-1].
# EOF # EOF
total = len(dest) total = len(dest)
have = len(self._rbuf) have = len(self._rbuf)
if have >= total: if have >= total:
dest[0:total] = self._rbuf[:total] dest[0:total] = self._rbuf[:total]
self._rbuf = self._rbuf[total:] self._rbuf = self._rbuf[total:]
return total return total
mv = memoryview(dest) mv = memoryview(dest)
got = self._raw_readinto(mv[have:total]) got = self._raw_readinto(mv[have:total])
self.receivedbytescount += got
self._connection.receivedbytescount += got
try:
self._handler.receivedbytescount += got
except AttributeError:
pass
dest[0:have] = self._rbuf dest[0:have] = self._rbuf
got += len(self._rbuf) got += len(self._rbuf)
self._rbuf = '' self._rbuf = ''
return got return got
def safesend(self, str): def safesend(self, str):
"""Send `str' to the server. """Send `str' to the server.
# use the modified response class # use the modified response class
response_class = HTTPResponse response_class = HTTPResponse
send = safesend send = safesend
getresponse = wrapgetresponse(httplib.HTTPConnection) getresponse = wrapgetresponse(httplib.HTTPConnection)
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
httplib.HTTPConnection.__init__(self, *args, **kwargs) httplib.HTTPConnection.__init__(self, *args, **kwargs)
self.sentbytescount = 0 self.sentbytescount = 0
self.receivedbytescount = 0
######################################################################### #########################################################################
##### TEST FUNCTIONS ##### TEST FUNCTIONS
######################################################################### #########################################################################
def continuity(url): def continuity(url):
md5 = hashlib.md5 md5 = hashlib.md5