Mercurial > public > mercurial-scm > hg-stable
diff mercurial/utils/cborutil.py @ 43076:2372284d9457
formatting: blacken the codebase
This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.
Done with:
hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S
# skip-blame mass-reformatting only
# no-check-commit reformats foo_bar functions
Differential Revision: https://phab.mercurial-scm.org/D6971
author | Augie Fackler <augie@google.com> |
---|---|
date | Sun, 06 Oct 2019 09:45:02 -0400 |
parents | b6387a65851d |
children | 687b865b95ad |
line wrap: on
line diff
--- a/mercurial/utils/cborutil.py Sat Oct 05 10:29:34 2019 -0400 +++ b/mercurial/utils/cborutil.py Sun Oct 06 09:45:02 2019 -0400 @@ -46,11 +46,14 @@ # Indefinite types begin with their major type ORd with information value 31. BEGIN_INDEFINITE_BYTESTRING = struct.pack( - r'>B', MAJOR_TYPE_BYTESTRING << 5 | SUBTYPE_INDEFINITE) + r'>B', MAJOR_TYPE_BYTESTRING << 5 | SUBTYPE_INDEFINITE +) BEGIN_INDEFINITE_ARRAY = struct.pack( - r'>B', MAJOR_TYPE_ARRAY << 5 | SUBTYPE_INDEFINITE) + r'>B', MAJOR_TYPE_ARRAY << 5 | SUBTYPE_INDEFINITE +) BEGIN_INDEFINITE_MAP = struct.pack( - r'>B', MAJOR_TYPE_MAP << 5 | SUBTYPE_INDEFINITE) + r'>B', MAJOR_TYPE_MAP << 5 | SUBTYPE_INDEFINITE +) ENCODED_LENGTH_1 = struct.Struct(r'>B') ENCODED_LENGTH_2 = struct.Struct(r'>BB') @@ -62,6 +65,7 @@ BREAK = b'\xff' BREAK_INT = 255 + def encodelength(majortype, length): """Obtain a value encoding the major type and its length.""" if length < 24: @@ -75,10 +79,12 @@ else: return ENCODED_LENGTH_5.pack(majortype << 5 | 27, length) + def streamencodebytestring(v): yield encodelength(MAJOR_TYPE_BYTESTRING, len(v)) yield v + def streamencodebytestringfromiter(it): """Convert an iterator of chunks to an indefinite bytestring. @@ -93,6 +99,7 @@ yield BREAK + def streamencodeindefinitebytestring(source, chunksize=65536): """Given a large source buffer, emit as an indefinite length bytestring. @@ -104,7 +111,7 @@ l = len(source) while True: - chunk = source[i:i + chunksize] + chunk = source[i : i + chunksize] i += len(chunk) yield encodelength(MAJOR_TYPE_BYTESTRING, len(chunk)) @@ -115,6 +122,7 @@ yield BREAK + def streamencodeint(v): if v >= 18446744073709551616 or v < -18446744073709551616: raise ValueError('big integers not supported') @@ -124,6 +132,7 @@ else: yield encodelength(MAJOR_TYPE_NEGINT, abs(v) - 1) + def streamencodearray(l): """Encode a known size iterable to an array.""" @@ -133,6 +142,7 @@ for chunk in streamencode(i): yield chunk + def streamencodearrayfromiter(it): """Encode an iterator of items to an indefinite length array.""" @@ -144,9 +154,11 @@ yield BREAK + def _mixedtypesortkey(v): return type(v).__name__, v + def streamencodeset(s): # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml defines # semantic tag 258 for finite sets. @@ -155,6 +167,7 @@ for chunk in streamencodearray(sorted(s, key=_mixedtypesortkey)): yield chunk + def streamencodemap(d): """Encode dictionary to a generator. @@ -162,13 +175,15 @@ """ yield encodelength(MAJOR_TYPE_MAP, len(d)) - for key, value in sorted(d.iteritems(), - key=lambda x: _mixedtypesortkey(x[0])): + for key, value in sorted( + d.iteritems(), key=lambda x: _mixedtypesortkey(x[0]) + ): for chunk in streamencode(key): yield chunk for chunk in streamencode(value): yield chunk + def streamencodemapfromiter(it): """Given an iterable of (key, value), encode to an indefinite length map.""" yield BEGIN_INDEFINITE_MAP @@ -181,14 +196,17 @@ yield BREAK + def streamencodebool(b): # major type 7, simple value 20 and 21. yield b'\xf5' if b else b'\xf4' + def streamencodenone(v): # major type 7, simple value 22. yield b'\xf6' + STREAM_ENCODERS = { bytes: streamencodebytestring, int: streamencodeint, @@ -201,6 +219,7 @@ type(None): streamencodenone, } + def streamencode(v): """Encode a value in a streaming manner. @@ -226,16 +245,23 @@ return fn(v) + class CBORDecodeError(Exception): """Represents an error decoding CBOR.""" + if sys.version_info.major >= 3: + def _elementtointeger(b, i): return b[i] + + else: + def _elementtointeger(b, i): return ord(b[i]) + STRUCT_BIG_UBYTE = struct.Struct(r'>B') STRUCT_BIG_USHORT = struct.Struct('>H') STRUCT_BIG_ULONG = struct.Struct('>L') @@ -248,6 +274,7 @@ SPECIAL_START_SET = 4 SPECIAL_INDEFINITE_BREAK = 5 + def decodeitem(b, offset=0): """Decode a new CBOR value from a buffer at offset. @@ -301,8 +328,9 @@ elif majortype == MAJOR_TYPE_BYTESTRING: # Beginning of bytestrings are treated as uints in order to # decode their length, which may be indefinite. - complete, size, readcount = decodeuint(subtype, b, offset, - allowindefinite=True) + complete, size, readcount = decodeuint( + subtype, b, offset, allowindefinite=True + ) # We don't know the size of the bytestring. It must be a definitive # length since the indefinite subtype would be encoded in the initial @@ -314,7 +342,7 @@ if size is not None: # And the data is available in the buffer. if offset + readcount + size <= len(b): - value = b[offset + readcount:offset + readcount + size] + value = b[offset + readcount : offset + readcount + size] return True, value, readcount + size + 1, SPECIAL_NONE # And we need more data in order to return the bytestring. @@ -367,15 +395,17 @@ if offset + readcount >= len(b): return False, None, -1, SPECIAL_NONE - complete, size, readcount2, special = decodeitem(b, - offset + readcount) + complete, size, readcount2, special = decodeitem( + b, offset + readcount + ) if not complete: return False, None, readcount2, SPECIAL_NONE if special != SPECIAL_START_ARRAY: - raise CBORDecodeError('expected array after finite set ' - 'semantic tag') + raise CBORDecodeError( + 'expected array after finite set ' 'semantic tag' + ) return True, size, readcount + readcount2 + 1, SPECIAL_START_SET @@ -398,6 +428,7 @@ else: assert False + def decodeuint(subtype, b, offset=0, allowindefinite=False): """Decode an unsigned integer. @@ -428,8 +459,9 @@ else: raise CBORDecodeError('indefinite length uint not allowed here') elif subtype >= 28: - raise CBORDecodeError('unsupported subtype on integer type: %d' % - subtype) + raise CBORDecodeError( + 'unsupported subtype on integer type: %d' % subtype + ) if subtype == 24: s = STRUCT_BIG_UBYTE @@ -447,6 +479,7 @@ else: return False, None, len(b) - offset - s.size + class bytestringchunk(bytes): """Represents a chunk/segment in an indefinite length bytestring. @@ -462,6 +495,7 @@ return self + class sansiodecoder(object): """A CBOR decoder that doesn't perform its own I/O. @@ -606,32 +640,30 @@ self._decodedvalues.append(value) elif special == SPECIAL_START_ARRAY: - self._collectionstack.append({ - 'remaining': value, - 'v': [], - }) + self._collectionstack.append( + {'remaining': value, 'v': [],} + ) self._state = self._STATE_WANT_ARRAY_VALUE elif special == SPECIAL_START_MAP: - self._collectionstack.append({ - 'remaining': value, - 'v': {}, - }) + self._collectionstack.append( + {'remaining': value, 'v': {},} + ) self._state = self._STATE_WANT_MAP_KEY elif special == SPECIAL_START_SET: - self._collectionstack.append({ - 'remaining': value, - 'v': set(), - }) + self._collectionstack.append( + {'remaining': value, 'v': set(),} + ) self._state = self._STATE_WANT_SET_VALUE elif special == SPECIAL_START_INDEFINITE_BYTESTRING: self._state = self._STATE_WANT_BYTESTRING_CHUNK_FIRST else: - raise CBORDecodeError('unhandled special state: %d' % - special) + raise CBORDecodeError( + 'unhandled special state: %d' % special + ) # This value becomes an element of the current array. elif self._state == self._STATE_WANT_ARRAY_VALUE: @@ -651,10 +683,9 @@ lastc['v'].append(newvalue) lastc['remaining'] -= 1 - self._collectionstack.append({ - 'remaining': value, - 'v': newvalue, - }) + self._collectionstack.append( + {'remaining': value, 'v': newvalue,} + ) # self._state doesn't need changed. @@ -666,10 +697,9 @@ lastc['v'].append(newvalue) lastc['remaining'] -= 1 - self._collectionstack.append({ - 'remaining': value, - 'v': newvalue - }) + self._collectionstack.append( + {'remaining': value, 'v': newvalue} + ) self._state = self._STATE_WANT_MAP_KEY @@ -680,20 +710,23 @@ lastc['v'].append(newvalue) lastc['remaining'] -= 1 - self._collectionstack.append({ - 'remaining': value, - 'v': newvalue, - }) + self._collectionstack.append( + {'remaining': value, 'v': newvalue,} + ) self._state = self._STATE_WANT_SET_VALUE elif special == SPECIAL_START_INDEFINITE_BYTESTRING: - raise CBORDecodeError('indefinite length bytestrings ' - 'not allowed as array values') + raise CBORDecodeError( + 'indefinite length bytestrings ' + 'not allowed as array values' + ) else: - raise CBORDecodeError('unhandled special item when ' - 'expecting array value: %d' % special) + raise CBORDecodeError( + 'unhandled special item when ' + 'expecting array value: %d' % special + ) # This value becomes the key of the current map instance. elif self._state == self._STATE_WANT_MAP_KEY: @@ -702,18 +735,26 @@ self._state = self._STATE_WANT_MAP_VALUE elif special == SPECIAL_START_INDEFINITE_BYTESTRING: - raise CBORDecodeError('indefinite length bytestrings ' - 'not allowed as map keys') + raise CBORDecodeError( + 'indefinite length bytestrings ' + 'not allowed as map keys' + ) - elif special in (SPECIAL_START_ARRAY, SPECIAL_START_MAP, - SPECIAL_START_SET): - raise CBORDecodeError('collections not supported as map ' - 'keys') + elif special in ( + SPECIAL_START_ARRAY, + SPECIAL_START_MAP, + SPECIAL_START_SET, + ): + raise CBORDecodeError( + 'collections not supported as map ' 'keys' + ) # We do not allow special values to be used as map keys. else: - raise CBORDecodeError('unhandled special item when ' - 'expecting map key: %d' % special) + raise CBORDecodeError( + 'unhandled special item when ' + 'expecting map key: %d' % special + ) # This value becomes the value of the current map key. elif self._state == self._STATE_WANT_MAP_VALUE: @@ -733,10 +774,9 @@ lastc['v'][self._currentmapkey] = newvalue lastc['remaining'] -= 1 - self._collectionstack.append({ - 'remaining': value, - 'v': newvalue, - }) + self._collectionstack.append( + {'remaining': value, 'v': newvalue,} + ) self._state = self._STATE_WANT_ARRAY_VALUE @@ -748,10 +788,9 @@ lastc['v'][self._currentmapkey] = newvalue lastc['remaining'] -= 1 - self._collectionstack.append({ - 'remaining': value, - 'v': newvalue, - }) + self._collectionstack.append( + {'remaining': value, 'v': newvalue,} + ) self._state = self._STATE_WANT_MAP_KEY @@ -763,20 +802,23 @@ lastc['v'][self._currentmapkey] = newvalue lastc['remaining'] -= 1 - self._collectionstack.append({ - 'remaining': value, - 'v': newvalue, - }) + self._collectionstack.append( + {'remaining': value, 'v': newvalue,} + ) self._state = self._STATE_WANT_SET_VALUE elif special == SPECIAL_START_INDEFINITE_BYTESTRING: - raise CBORDecodeError('indefinite length bytestrings not ' - 'allowed as map values') + raise CBORDecodeError( + 'indefinite length bytestrings not ' + 'allowed as map values' + ) else: - raise CBORDecodeError('unhandled special item when ' - 'expecting map value: %d' % special) + raise CBORDecodeError( + 'unhandled special item when ' + 'expecting map value: %d' % special + ) self._currentmapkey = None @@ -788,27 +830,35 @@ lastc['remaining'] -= 1 elif special == SPECIAL_START_INDEFINITE_BYTESTRING: - raise CBORDecodeError('indefinite length bytestrings not ' - 'allowed as set values') + raise CBORDecodeError( + 'indefinite length bytestrings not ' + 'allowed as set values' + ) - elif special in (SPECIAL_START_ARRAY, - SPECIAL_START_MAP, - SPECIAL_START_SET): - raise CBORDecodeError('collections not allowed as set ' - 'values') + elif special in ( + SPECIAL_START_ARRAY, + SPECIAL_START_MAP, + SPECIAL_START_SET, + ): + raise CBORDecodeError( + 'collections not allowed as set ' 'values' + ) # We don't allow non-trivial types to exist as set values. else: - raise CBORDecodeError('unhandled special item when ' - 'expecting set value: %d' % special) + raise CBORDecodeError( + 'unhandled special item when ' + 'expecting set value: %d' % special + ) # This value represents the first chunk in an indefinite length # bytestring. elif self._state == self._STATE_WANT_BYTESTRING_CHUNK_FIRST: # We received a full chunk. if special == SPECIAL_NONE: - self._decodedvalues.append(bytestringchunk(value, - first=True)) + self._decodedvalues.append( + bytestringchunk(value, first=True) + ) self._state = self._STATE_WANT_BYTESTRING_CHUNK_SUBSEQUENT @@ -818,9 +868,9 @@ # We /could/ convert this to a b''. But we want to preserve # the nature of the underlying data so consumers expecting # an indefinite length bytestring get one. - self._decodedvalues.append(bytestringchunk(b'', - first=True, - last=True)) + self._decodedvalues.append( + bytestringchunk(b'', first=True, last=True) + ) # Since indefinite length bytestrings can't be used in # collections, we must be at the root level. @@ -828,9 +878,10 @@ self._state = self._STATE_NONE else: - raise CBORDecodeError('unexpected special value when ' - 'expecting bytestring chunk: %d' % - special) + raise CBORDecodeError( + 'unexpected special value when ' + 'expecting bytestring chunk: %d' % special + ) # This value represents the non-initial chunk in an indefinite # length bytestring. @@ -849,21 +900,25 @@ self._state = self._STATE_NONE else: - raise CBORDecodeError('unexpected special value when ' - 'expecting bytestring chunk: %d' % - special) + raise CBORDecodeError( + 'unexpected special value when ' + 'expecting bytestring chunk: %d' % special + ) else: - raise CBORDecodeError('unhandled decoder state: %d' % - self._state) + raise CBORDecodeError( + 'unhandled decoder state: %d' % self._state + ) # We could have just added the final value in a collection. End # all complete collections at the top of the stack. while True: # Bail if we're not waiting on a new collection item. - if self._state not in (self._STATE_WANT_ARRAY_VALUE, - self._STATE_WANT_MAP_KEY, - self._STATE_WANT_SET_VALUE): + if self._state not in ( + self._STATE_WANT_ARRAY_VALUE, + self._STATE_WANT_MAP_KEY, + self._STATE_WANT_SET_VALUE, + ): break # Or we are expecting more items for this collection. @@ -909,6 +964,7 @@ self._decodedvalues = [] return l + class bufferingdecoder(object): """A CBOR decoder that buffers undecoded input. @@ -919,6 +975,7 @@ TODO consider adding limits as to the maximum amount of data that can be buffered. """ + def __init__(self): self._decoder = sansiodecoder() self._chunks = [] @@ -978,6 +1035,7 @@ def getavailable(self): return self._decoder.getavailable() + def decodeall(b): """Decode all CBOR items present in an iterable of bytes.