From accb9895ac33748cd48946e1fd114c329194ebba Mon Sep 17 00:00:00 2001 From: Vaughn Kottler Date: Sat, 26 Aug 2023 16:06:57 -0500 Subject: [PATCH] Serializable dev --- runtimepy/primitives/__init__.py | 4 + runtimepy/primitives/array/__init__.py | 2 +- runtimepy/primitives/int.py | 1 + runtimepy/primitives/serializable/__init__.py | 9 ++ .../{serializable.py => serializable/base.py} | 53 +++++------ runtimepy/primitives/serializable/fixed.py | 43 +++++++++ runtimepy/primitives/serializable/prefixed.py | 94 +++++++++++++++++++ runtimepy/primitives/type/__init__.py | 27 +----- runtimepy/primitives/type/base.py | 3 +- runtimepy/primitives/type/bool.py | 2 +- runtimepy/primitives/type/int.py | 8 +- tests/primitives/serializable/__init__.py | 0 .../primitives/serializable/test_prefixed.py | 44 +++++++++ 13 files changed, 230 insertions(+), 60 deletions(-) create mode 100644 runtimepy/primitives/serializable/__init__.py rename runtimepy/primitives/{serializable.py => serializable/base.py} (72%) create mode 100644 runtimepy/primitives/serializable/fixed.py create mode 100644 runtimepy/primitives/serializable/prefixed.py create mode 100644 tests/primitives/serializable/__init__.py create mode 100644 tests/primitives/serializable/test_prefixed.py diff --git a/runtimepy/primitives/__init__.py b/runtimepy/primitives/__init__.py index a6dfcd86..dd4a12dd 100644 --- a/runtimepy/primitives/__init__.py +++ b/runtimepy/primitives/__init__.py @@ -16,10 +16,12 @@ Int16, Int32, Int64, + SignedInt, Uint8, Uint16, Uint32, Uint64, + UnsignedInt, ) __all__ = [ @@ -40,6 +42,8 @@ "Primitivelike", "normalize", "create", + "SignedInt", + "UnsignedInt", ] AnyPrimitive = _Union[ diff --git a/runtimepy/primitives/array/__init__.py b/runtimepy/primitives/array/__init__.py index 92b9630d..c60a496f 100644 --- a/runtimepy/primitives/array/__init__.py +++ b/runtimepy/primitives/array/__init__.py @@ -57,7 +57,7 @@ def __init__( for item in primitives: self.add(item) - super().__init__(chain=next_array) + super().__init__(byte_order=self.byte_order, chain=next_array) self._fragments: _List["PrimitiveArray"] = [] self._fragment_specs: _List[ArrayFragmentSpec] = [] diff --git a/runtimepy/primitives/int.py b/runtimepy/primitives/int.py index 4f2baccc..e07761b3 100644 --- a/runtimepy/primitives/int.py +++ b/runtimepy/primitives/int.py @@ -119,4 +119,5 @@ def __init__(self, value: int = 0) -> None: Uint64 = Uint64Primitive +SignedInt = _Union[Int8, Int16, Int32, Int64] UnsignedInt = _Union[Uint8, Uint16, Uint32, Uint64] diff --git a/runtimepy/primitives/serializable/__init__.py b/runtimepy/primitives/serializable/__init__.py new file mode 100644 index 00000000..803ffd15 --- /dev/null +++ b/runtimepy/primitives/serializable/__init__.py @@ -0,0 +1,9 @@ +""" +A module defining an interface for serializable objects. +""" + +# internal +from runtimepy.primitives.serializable.base import Serializable +from runtimepy.primitives.serializable.fixed import FixedChunk + +__all__ = ["Serializable", "FixedChunk"] diff --git a/runtimepy/primitives/serializable.py b/runtimepy/primitives/serializable/base.py similarity index 72% rename from runtimepy/primitives/serializable.py rename to runtimepy/primitives/serializable/base.py index 6359df0a..03eec399 100644 --- a/runtimepy/primitives/serializable.py +++ b/runtimepy/primitives/serializable/base.py @@ -1,5 +1,5 @@ """ -A module defining an interface for serializable objects. +A module defining a base interface fore serializable objects. """ # built-in @@ -8,6 +8,12 @@ from typing import BinaryIO as _BinaryIO from typing import TypeVar +# internal +from runtimepy.primitives.byte_order import ( + DEFAULT_BYTE_ORDER as _DEFAULT_BYTE_ORDER, +) +from runtimepy.primitives.byte_order import ByteOrder as _ByteOrder + T = TypeVar("T", bound="Serializable") @@ -16,11 +22,17 @@ class Serializable(ABC): size: int - def __init__(self, chain: T = None) -> None: + def __init__( + self, + byte_order: _ByteOrder = _DEFAULT_BYTE_ORDER, + chain: T = None, + ) -> None: """Initialize this instance.""" if not hasattr(self, "size"): self.size = 0 + + self.byte_order = byte_order self.chain = chain def length(self) -> int: @@ -51,6 +63,7 @@ def __copy__(self: T) -> T: if self.chain is not None: result.assign(self.chain.copy()) + result.byte_order = self.byte_order return result @@ -80,14 +93,19 @@ def to_stream(self, stream: _BinaryIO) -> int: return result - @abstractmethod def update(self, data: bytes) -> int: """Update this serializable from a bytes instance.""" + raise NotImplementedError + + def _from_stream(self, stream: _BinaryIO) -> int: + """Update just this instance from a stream.""" + + return self.update(stream.read(self.size)) def from_stream(self, stream: _BinaryIO) -> int: """Update this serializable from a stream.""" - result = self.update(stream.read(self.size)) + result = self._from_stream(stream) if self.chain is not None: result += self.chain.from_stream(stream) @@ -104,30 +122,3 @@ def add_to_end(self, chain: T) -> None: """Add a new serializable to the end of this chain.""" self.end.assign(chain) - - -class FixedChunk(Serializable): - """A simple fixed-size serializable chunk.""" - - def __init__(self, data: bytes, chain: Serializable = None) -> None: - """Initialize this instance.""" - - super().__init__(chain=chain) - assert data - self.data = data - self.size = len(self.data) - - def _copy_impl(self) -> "FixedChunk": - """Make a copy of this instance.""" - return FixedChunk(self.data) - - def __bytes__(self) -> bytes: - """Get this serializable as a bytes instance.""" - return self.data - - def update(self, data: bytes) -> int: - """Update this serializable from a bytes instance.""" - - assert len(data) == self.size - self.data = data - return self.size diff --git a/runtimepy/primitives/serializable/fixed.py b/runtimepy/primitives/serializable/fixed.py new file mode 100644 index 00000000..89231c49 --- /dev/null +++ b/runtimepy/primitives/serializable/fixed.py @@ -0,0 +1,43 @@ +""" +A module implementing a fixed-size bytes serializable. +""" + +# built-in +from copy import copy as _copy + +# internal +from runtimepy.primitives.serializable.base import Serializable + + +class FixedChunk(Serializable): + """A simple fixed-size serializable chunk.""" + + def __init__(self, data: bytes, chain: Serializable = None) -> None: + """Initialize this instance.""" + + super().__init__(chain=chain) + self.data = data + self.size = len(self.data) + + def __str__(self) -> str: + """Get this chunk as a string.""" + return self.data.decode() + + def _copy_impl(self) -> "FixedChunk": + """Make a copy of this instance.""" + return FixedChunk(_copy(self.data)) + + def __bytes__(self) -> bytes: + """Get this serializable as a bytes instance.""" + return self.data + + def update(self, data: bytes) -> int: + """Update this serializable from a bytes instance.""" + + self.data = data + self.size = len(self.data) + return self.size + + def update_str(self, data: str) -> int: + """Update this chunk from a string.""" + return self.update(data.encode()) diff --git a/runtimepy/primitives/serializable/prefixed.py b/runtimepy/primitives/serializable/prefixed.py new file mode 100644 index 00000000..a7b89caf --- /dev/null +++ b/runtimepy/primitives/serializable/prefixed.py @@ -0,0 +1,94 @@ +""" +A module implementing a variable-size bytes serializable, using an integer +primitive prefix to determine the size of the chunk portion. +""" + +# built-in +from typing import BinaryIO as _BinaryIO +from typing import Type, TypeVar + +# internal +from runtimepy.primitives import Primitivelike, UnsignedInt, create +from runtimepy.primitives.byte_order import ( + DEFAULT_BYTE_ORDER as _DEFAULT_BYTE_ORDER, +) +from runtimepy.primitives.byte_order import ByteOrder as _ByteOrder +from runtimepy.primitives.serializable.base import Serializable +from runtimepy.primitives.serializable.fixed import FixedChunk + +T = TypeVar("T", bound="PrefixedChunk") + + +class PrefixedChunk(Serializable): + """A simple integer-prefixed chunk serializable.""" + + def __init__( + self, + prefix: UnsignedInt, + byte_order: _ByteOrder = _DEFAULT_BYTE_ORDER, + chain: Serializable = None, + ) -> None: + """Initialize this instance.""" + + super().__init__(byte_order=byte_order, chain=chain) + + # Validate prefix. + assert prefix.kind.is_integer, prefix + assert not prefix.kind.signed, prefix + + self.prefix = prefix + self.chunk = FixedChunk(bytes(self.prefix.value)) + self._update_size() + + def __str__(self) -> str: + """Get this chunk as a string.""" + return str(self.chunk) + + def update_str(self, data: str) -> int: + """Update this chunk from a string.""" + + size = self.chunk.update_str(data) + self.prefix.value = size + return self._update_size() + + def _update_size(self) -> int: + """Update this instance's size.""" + + assert self.prefix.value == self.chunk.size + self.size = self.prefix.kind.size + self.prefix.value + return self.size + + def _copy_impl(self) -> "PrefixedChunk": + """Make a copy of this instance.""" + + result = PrefixedChunk(self.prefix.copy()) # type: ignore + result.chunk = self.chunk.copy() + + return result + + def __bytes__(self) -> bytes: + """Get this serializable as a bytes instance.""" + + return self.prefix.binary(byte_order=self.byte_order) + bytes( + self.chunk + ) + + def _from_stream(self, stream: _BinaryIO) -> int: + """Update just this instance from a stream.""" + + self.chunk.update( + stream.read( + self.prefix.from_stream(stream, byte_order=self.byte_order) + ) + ) + return self._update_size() + + @classmethod + def create( + cls: Type[T], + prefix: Primitivelike = "uint16", + chain: Serializable = None, + ) -> T: + """Create a prefixed chunk.""" + + return cls(create(prefix), chain=chain) # type: ignore diff --git a/runtimepy/primitives/type/__init__.py b/runtimepy/primitives/type/__init__.py index 3a730f62..4aa4ccbc 100644 --- a/runtimepy/primitives/type/__init__.py +++ b/runtimepy/primitives/type/__init__.py @@ -35,30 +35,13 @@ Uint64Type, ) -AnyIntegerType = _Union[ - Int8Type, - Int16Type, - Int32Type, - Int64Type, - Uint8Type, - Uint16Type, - Uint32Type, - Uint64Type, -] +SignedIntegerType = _Union[Int8Type, Int16Type, Int32Type, Int64Type] +UnsignedIntegerType = _Union[Uint8Type, Uint16Type, Uint32Type, Uint64Type] + +AnyIntegerType = _Union[SignedIntegerType, UnsignedIntegerType] AnyPrimitiveType = _Union[ - Int8Type, - Int16Type, - Int32Type, - Int64Type, - Uint8Type, - Uint16Type, - Uint32Type, - Uint64Type, - HalfType, - FloatType, - DoubleType, - BooleanType, + AnyIntegerType, HalfType, FloatType, DoubleType, BooleanType ] PrimitiveTypes: _Dict[str, AnyPrimitiveType] = { diff --git a/runtimepy/primitives/type/base.py b/runtimepy/primitives/type/base.py index 86b2b7e6..670098d4 100644 --- a/runtimepy/primitives/type/base.py +++ b/runtimepy/primitives/type/base.py @@ -68,10 +68,11 @@ class PrimitiveType(_Generic[T]): name: str c_type: _Type[T] - def __init__(self, struct_format: str) -> None: + def __init__(self, struct_format: str, signed: bool = True) -> None: """Initialize this primitive type.""" self.format = struct_format + self.signed = signed # Make sure that the struct size and ctype size match. There's # unfortunately no obvious (or via public interfaces) way to just diff --git a/runtimepy/primitives/type/bool.py b/runtimepy/primitives/type/bool.py index 99d2a10b..1d7b5913 100644 --- a/runtimepy/primitives/type/bool.py +++ b/runtimepy/primitives/type/bool.py @@ -15,7 +15,7 @@ class BooleanType(_PrimitiveType[_BoolCtype]): def __init__(self) -> None: """Initialize this type.""" - super().__init__("?") + super().__init__("?", signed=False) assert self.is_boolean diff --git a/runtimepy/primitives/type/int.py b/runtimepy/primitives/type/int.py index 915c5a4e..cf94461a 100644 --- a/runtimepy/primitives/type/int.py +++ b/runtimepy/primitives/type/int.py @@ -82,7 +82,7 @@ class Uint8Type(_PrimitiveType[_Uint8Ctype]): def __init__(self) -> None: """Initialize this type.""" - super().__init__("B") + super().__init__("B", signed=False) assert self.is_integer @@ -97,7 +97,7 @@ class Uint16Type(_PrimitiveType[_Uint16Ctype]): def __init__(self) -> None: """Initialize this type.""" - super().__init__("H") + super().__init__("H", signed=False) assert self.is_integer @@ -112,7 +112,7 @@ class Uint32Type(_PrimitiveType[_Uint32Ctype]): def __init__(self) -> None: """Initialize this type.""" - super().__init__("I") + super().__init__("I", signed=False) assert self.is_integer @@ -127,7 +127,7 @@ class Uint64Type(_PrimitiveType[_Uint64Ctype]): def __init__(self) -> None: """Initialize this type.""" - super().__init__("Q") + super().__init__("Q", signed=False) assert self.is_integer diff --git a/tests/primitives/serializable/__init__.py b/tests/primitives/serializable/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/primitives/serializable/test_prefixed.py b/tests/primitives/serializable/test_prefixed.py new file mode 100644 index 00000000..7f4724cc --- /dev/null +++ b/tests/primitives/serializable/test_prefixed.py @@ -0,0 +1,44 @@ +""" +Test the 'primitives.serializable.prefixed' module. +""" + +# built-in +from io import BytesIO +from typing import cast + +# module under test +from runtimepy.primitives.serializable.prefixed import PrefixedChunk + + +def test_prefixed_chunk_basic(): + """Test basic interactions with a prefixed chunk.""" + + end = PrefixedChunk.create("uint8") + chunk = PrefixedChunk.create(chain=end) + + assert chunk.update_str("Hello") == 7 + assert end.update_str(", world!") == 9 + + assert len(bytes(chunk)) == 7 + assert len(bytes(end)) == 9 + + assert str(chunk) + str(end) == "Hello, world!" + assert chunk.length() == 16 + + chunk_copy = chunk.copy() + end_copy: PrefixedChunk = cast(PrefixedChunk, chunk_copy.end) + + assert chunk.update_str("a") == 3 + assert end.update_str("b") == 2 + + assert str(chunk_copy) + str(end_copy) == "Hello, world!" + + with BytesIO() as stream: + assert chunk.to_stream(stream) == 5 + + stream.seek(0) + + assert chunk_copy.from_stream(stream) == 5 + + assert str(chunk_copy) + str(end_copy) == "ab" + assert chunk_copy.length() == 5