This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7882

lfs: avoid quadratic performance in processing server responses
ClosedPublic

Authored by mharbison72 on Jan 14 2020, 11:09 PM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHGffac09da7a19: lfs: avoid quadratic performance in processing server responses
Summary

This is also adapted from the Facebook repo[1]. Unlike there, we were already
reading the download stream in chunks and immediately writing it to disk, so we
basically avoided the problem on download. There shouldn't be a lot of data to
read on upload, but it's better to get rid of this pattern.

[1] https://github.com/facebookexperimental/eden/commit/82df66ffe97e21f3ee73dfec093c87500fc1f6a7

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

mharbison72 created this revision.Jan 14 2020, 11:09 PM
pulkit accepted this revision.Jan 15 2020, 7:15 AM
This revision is now accepted and ready to land.Jan 15 2020, 7:15 AM

Revision Contents
Changeset List

PathPackages
Mhgext/lfs/blobstore.py (6 lines)

Diff 19292

hgext/lfs/blobstore.py

request.data = filewithprogress(localstore.open(oid), None) request.data = filewithprogress(localstore.open(oid), None)
request.get_method = lambda: r'PUT' request.get_method = lambda: r'PUT'
request.add_header('Content-Type', 'application/octet-stream') request.add_header('Content-Type', 'application/octet-stream')
request.add_header('Content-Length', len(request.data)) request.add_header('Content-Length', len(request.data))
for k, v in headers: for k, v in headers:
request.add_header(pycompat.strurl(k), pycompat.strurl(v)) request.add_header(pycompat.strurl(k), pycompat.strurl(v))
response = b''
try: try:
with contextlib.closing(self.urlopener.open(request)) as res: with contextlib.closing(self.urlopener.open(request)) as res:
contentlength = res.info().get(b"content-length") contentlength = res.info().get(b"content-length")
ui = self.ui # Shorten debug lines ui = self.ui # Shorten debug lines
if self.ui.debugflag: if self.ui.debugflag:
ui.debug(b'Status: %d\n' % res.status) ui.debug(b'Status: %d\n' % res.status)
# lfs-test-server and hg serve return headers in different # lfs-test-server and hg serve return headers in different
# order # order
headers = pycompat.bytestr(res.info()).strip() headers = pycompat.bytestr(res.info()).strip()
ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines())))
if action == b'download': if action == b'download':
# If downloading blobs, store downloaded data to local # If downloading blobs, store downloaded data to local
# blobstore # blobstore
localstore.download(oid, res, contentlength) localstore.download(oid, res, contentlength)
else: else:
blocks = []
while True: while True:
data = res.read(1048576) data = res.read(1048576)
if not data: if not data:
break break
response += data blocks.append(data)
response = b"".join(blocks)
if response: if response:
ui.debug(b'lfs %s response: %s' % (action, response)) ui.debug(b'lfs %s response: %s' % (action, response))
except util.urlerr.httperror as ex: except util.urlerr.httperror as ex:
if self.ui.debugflag: if self.ui.debugflag:
self.ui.debug( self.ui.debug(
b'%s: %s\n' % (oid, ex.read()) b'%s: %s\n' % (oid, ex.read())
) # XXX: also bytes? ) # XXX: also bytes?
raise LfsRemoteError( raise LfsRemoteError(