This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4434

cborutil: add a buffering decoder
ClosedPublic

Authored by indygreg on Aug 31 2018, 6:57 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGbabad5ebaf0a: cborutil: add a buffering decoder
Summary

The sansiodecoder leaves it up to the callers to feed in data that
wasn't fully consumed last time.

This commit implements a decoder that performs buffering of
leftover chunks from the previous invocation. It otherwise
behaves identically to sansiodecoder.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Aug 31 2018, 6:57 PM
yuja added a subscriber: yuja.Sep 3 2018, 9:38 AM

Queued up to this patch, thanks. The decoder looks promising.

+ def decode(self, b):
+ """Attempt to decode bytes to CBOR values.
+
+ Returns a tuple with the following fields:
+
+ * Bool indicating whether new values are available for retrieval.
+ * Integer number of bytes decoded from the new input.
+ * Integer number of bytes wanted to decode the next value.
+ """
+
+ if self._leftover:
+ oldlen = len(self._leftover)
+ b = self._leftover + b
+ self._leftover = None
+ else:
+ b = b
+ oldlen = 0
+
+ available, readcount, wanted = self._decoder.decode(b)
+
+ if readcount < len(b):
+ self._leftover = b[readcount:]
+
+ return available, readcount - oldlen, wanted

I think wanted should be wanted - len(leftover) since leftover is
internally buffered by the decoder. I have no idea about the readcount.

+ def getavailable(self):
+ return self._decoder.getavailable()

Nit: pop might be a better word to denote that this function consumes
the available values.

Closed by commit rHGbabad5ebaf0a: cborutil: add a buffering decoder (authored by indygreg). · Explain WhySep 3 2018, 9:38 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/utils/cborutil.py (42 lines)
Mtests/test-cbor.py (24 lines)

Diff 10730

mercurial/utils/cborutil.py

Once values are retrieved, they won't be available on the next call. Once values are retrieved, they won't be available on the next call.
""" """
l = list(self._decodedvalues) l = list(self._decodedvalues)
self._decodedvalues = [] self._decodedvalues = []
return l return l
class bufferingdecoder(object):
"""A CBOR decoder that buffers undecoded input.
This is a glorified wrapper around ``sansiodecoder`` that adds a buffering
layer. All input that isn't consumed by ``sansiodecoder`` will be buffered
and concatenated with any new input that arrives later.
TODO consider adding limits as to the maximum amount of data that can
be buffered.
"""
def __init__(self):
self._decoder = sansiodecoder()
self._leftover = None
def decode(self, b):
"""Attempt to decode bytes to CBOR values.
Returns a tuple with the following fields:
* Bool indicating whether new values are available for retrieval.
* Integer number of bytes decoded from the new input.
* Integer number of bytes wanted to decode the next value.
"""
if self._leftover:
oldlen = len(self._leftover)
b = self._leftover + b
self._leftover = None
else:
b = b
oldlen = 0
available, readcount, wanted = self._decoder.decode(b)
if readcount < len(b):
self._leftover = b[readcount:]
return available, readcount - oldlen, wanted
def getavailable(self):
return self._decoder.getavailable()
def decodeall(b): def decodeall(b):
"""Decode all CBOR items present in an iterable of bytes. """Decode all CBOR items present in an iterable of bytes.
In addition to regular decode errors, raises CBORDecodeError if the In addition to regular decode errors, raises CBORDecodeError if the
entirety of the passed buffer does not fully decode to complete CBOR entirety of the passed buffer does not fully decode to complete CBOR
values. This includes failure to decode any value, incomplete collection values. This includes failure to decode any value, incomplete collection
types, incomplete indefinite length items, and extra data at the end of types, incomplete indefinite length items, and extra data at the end of
the buffer. the buffer.

tests/test-cbor.py

'special type \d+ not allowed'): 'special type \d+ not allowed'):
cborutil.decodeitem(encoded) cborutil.decodeitem(encoded)
class SansIODecoderTests(TestCase): class SansIODecoderTests(TestCase):
def testemptyinput(self): def testemptyinput(self):
decoder = cborutil.sansiodecoder() decoder = cborutil.sansiodecoder()
self.assertEqual(decoder.decode(b''), (False, 0, 0)) self.assertEqual(decoder.decode(b''), (False, 0, 0))
class BufferingDecoderTests(TestCase):
def testsimple(self):
source = [
b'foobar',
b'x' * 128,
{b'foo': b'bar'},
True,
False,
None,
[None for i in range(128)],
]
encoded = b''.join(cborutil.streamencode(source))
for step in range(1, 32):
decoder = cborutil.bufferingdecoder()
start = 0
while start < len(encoded):
decoder.decode(encoded[start:start + step])
start += step
self.assertEqual(decoder.getavailable(), [source])
class DecodeallTests(TestCase): class DecodeallTests(TestCase):
def testemptyinput(self): def testemptyinput(self):
self.assertEqual(cborutil.decodeall(b''), []) self.assertEqual(cborutil.decodeall(b''), [])
def testpartialinput(self): def testpartialinput(self):
encoded = b''.join([ encoded = b''.join([
b'\x82', # array of 2 elements b'\x82', # array of 2 elements
b'\x01', # integer 1 b'\x01', # integer 1