This is an archive of the discontinued Mercurial Phabricator instance.

Differential D2377

util: add a file object proxy that can read at most N bytes
ClosedPublic

Authored by indygreg on Feb 21 2018, 5:07 PM.

Details

Reviewers
lothiraldan
yuja
Group Reviewers
hg-reviewers
Commits
rHG01e29e885600: util: add a file object proxy that can read at most N bytes
Summary

Sometimes we have data of a known size within a stream. For
performance reasons, we don't want to pre-read this data (we want
to allow consumers to read on demand). For simplicitly reasons,
we don't want callers to necessarily know their data is coming
from within an outer stream and there is a limit to how much
they should read.

The class introduced by this commit provides a very simple proxy
around an underlying file object that allows the consumer to
.read() up to N bytes from the file object. Attempts to read
past this many bytes results in a simulated EOF.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Feb 21 2018, 5:07 PM
lothiraldan accepted this revision.Feb 22 2018, 3:14 AM
lothiraldan added a subscriber: lothiraldan.

LGTM

yuja accepted this revision.Feb 22 2018, 9:05 AM
This revision is now accepted and ready to land.Feb 22 2018, 9:05 AM
Closed by commit rHG01e29e885600: util: add a file object proxy that can read at most N bytes (authored by indygreg). · Explain WhyFeb 22 2018, 9:18 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/util.py (29 lines)
AMtests/test-cappedreader.py (91 lines)

Diff 5995

mercurial/util.py

nbytes = min(limit, size) nbytes = min(limit, size)
s = nbytes and f.read(nbytes) s = nbytes and f.read(nbytes)
if not s: if not s:
break break
if limit: if limit:
limit -= len(s) limit -= len(s)
yield s yield s
class cappedreader(object):
"""A file object proxy that allows reading up to N bytes.
Given a source file object, instances of this type allow reading up to
N bytes from that source file object. Attempts to read past the allowed
limit are treated as EOF.
It is assumed that I/O is not performed on the original file object
in addition to I/O that is performed by this instance. If there is,
state tracking will get out of sync and unexpected results will ensue.
"""
def __init__(self, fh, limit):
"""Allow reading up to <limit> bytes from <fh>."""
self._fh = fh
self._left = limit
def read(self, n=-1):
if not self._left:
return b''
if n < 0:
n = self._left
data = self._fh.read(min(n, self._left))
self._left -= len(data)
assert self._left >= 0
return data
def makedate(timestamp=None): def makedate(timestamp=None):
'''Return a unix timestamp (or the current time) as a (unixtime, '''Return a unix timestamp (or the current time) as a (unixtime,
offset) tuple based off the local timezone.''' offset) tuple based off the local timezone.'''
if timestamp is None: if timestamp is None:
timestamp = time.time() timestamp = time.time()
if timestamp < 0: if timestamp < 0:
hint = _("check your clock") hint = _("check your clock")
raise Abort(_("negative timestamp: %d") % timestamp, hint=hint) raise Abort(_("negative timestamp: %d") % timestamp, hint=hint)

tests/test-cappedreader.py

  • This file was added.
from __future__ import absolute_import, print_function
import io
import unittest
from mercurial import (
util,
)
class CappedReaderTests(unittest.TestCase):
def testreadfull(self):
source = io.BytesIO(b'x' * 100)
reader = util.cappedreader(source, 10)
res = reader.read(10)
self.assertEqual(res, b'x' * 10)
self.assertEqual(source.tell(), 10)
source.seek(0)
reader = util.cappedreader(source, 15)
res = reader.read(16)
self.assertEqual(res, b'x' * 15)
self.assertEqual(source.tell(), 15)
source.seek(0)
reader = util.cappedreader(source, 100)
res = reader.read(100)
self.assertEqual(res, b'x' * 100)
self.assertEqual(source.tell(), 100)
source.seek(0)
reader = util.cappedreader(source, 50)
res = reader.read()
self.assertEqual(res, b'x' * 50)
self.assertEqual(source.tell(), 50)
source.seek(0)
def testreadnegative(self):
source = io.BytesIO(b'x' * 100)
reader = util.cappedreader(source, 20)
res = reader.read(-1)
self.assertEqual(res, b'x' * 20)
self.assertEqual(source.tell(), 20)
source.seek(0)
reader = util.cappedreader(source, 100)
res = reader.read(-1)
self.assertEqual(res, b'x' * 100)
self.assertEqual(source.tell(), 100)
source.seek(0)
def testreadmultiple(self):
source = io.BytesIO(b'x' * 100)
reader = util.cappedreader(source, 10)
for i in range(10):
res = reader.read(1)
self.assertEqual(res, b'x')
self.assertEqual(source.tell(), i + 1)
self.assertEqual(source.tell(), 10)
res = reader.read(1)
self.assertEqual(res, b'')
self.assertEqual(source.tell(), 10)
source.seek(0)
reader = util.cappedreader(source, 45)
for i in range(4):
res = reader.read(10)
self.assertEqual(res, b'x' * 10)
self.assertEqual(source.tell(), (i + 1) * 10)
res = reader.read(10)
self.assertEqual(res, b'x' * 5)
self.assertEqual(source.tell(), 45)
def readlimitpasteof(self):
source = io.BytesIO(b'x' * 100)
reader = util.cappedreader(source, 1024)
res = reader.read(1000)
self.assertEqual(res, b'x' * 100)
self.assertEqual(source.tell(), 100)
res = reader.read(1000)
self.assertEqual(res, b'')
self.assertEqual(source.tell(), 100)
if __name__ == '__main__':
import silenttestrunner
silenttestrunner.main(__name__)