5.11.0 - Enabling arbitrarily nested protocols

Refactor Protocol as PrimitiveArray extension

Improve array handling

Author: vkottler

.github/workflows/python-package.yml

@@ -77,7 +77,7 @@ jobs:
 
       - run: |
           mk python-release owner=vkottler \
-                            repo=runtimepy version=5.10.2
+                            repo=runtimepy version=5.11.0
         if: |
           matrix.python-version == '3.12'
           && matrix.system == 'ubuntu-latest'

README.md

@@ -2,11 +2,11 @@
     =====================================
     generator=datazen
     version=3.1.4
-    hash=887650b5b88ff596b3ab067e01de6fbd
+    hash=12eb2a9d91dea3ae83e38a864f6cab4b
     =====================================
 -->
 
-# runtimepy ([5.10.2](https://pypi.org/project/runtimepy/))
+# runtimepy ([5.11.0](https://pypi.org/project/runtimepy/))
 
 [![python](https://img.shields.io/pypi/pyversions/runtimepy.svg)](https://pypi.org/project/runtimepy/)
 ![Build Status](https://github.com/vkottler/runtimepy/workflows/Python%20Package/badge.svg)

local/variables/package.yaml

@@ -1,5 +1,5 @@
 ---
 major: 5
-minor: 10
-patch: 2
+minor: 11
+patch: 0
 entry: runtimepy

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__"
 
 [project]
 name = "runtimepy"
-version = "5.10.2"
+version = "5.11.0"
 description = "A framework for implementing Python services."
 readme = "README.md"
 requires-python = ">=3.12"

runtimepy/__init__.py

@@ -1,7 +1,7 @@
 # =====================================
 # generator=datazen
 # version=3.1.4
-# hash=9114f245a523f60ffbbfdd040548fc7c
+# hash=c4e165f8c6309dd6819689f0262cf92d
 # =====================================
 
 """
@@ -10,7 +10,7 @@
 
 DESCRIPTION = "A framework for implementing Python services."
 PKG_NAME = "runtimepy"
-VERSION = "5.10.2"
+VERSION = "5.11.0"
 
 # runtimepy-specific content.
 METRICS_NAME = "metrics"

runtimepy/channel/environment/telemetry.py

@@ -99,7 +99,7 @@ def _parse_channel_event(self, name: str) -> Optional[ParsedEvent]:
                 cast(int, channels.event_header["timestamp"]),
                 kind.decode(
                     data,
-                    byte_order=channels.event_header.array.byte_order,
+                    byte_order=channels.event_header.byte_order,
                 ),
             )
 
@@ -120,7 +120,7 @@ def _parse_field_event(self, name: str) -> Optional[ParsedEvent]:
                 cast(int, self.channels.event_header["timestamp"]),
                 kind.decode(
                     data,
-                    byte_order=self.channels.event_header.array.byte_order,
+                    byte_order=self.channels.event_header.byte_order,
                 ),
             )
 
@@ -146,7 +146,7 @@ def parse_event_stream(self, stream: BinaryIO) -> Iterator[ParsedEvent]:
                 read_size = channels.event_header.size
                 data = channels.event_fifo.pop(read_size)
                 if data is not None:
-                    channels.event_header.array.update(data)
+                    channels.event_header.update(data)
 
                     # Update local variables.
                     ident = cast(int, channels.event_header["identifier"])

runtimepy/channel/event/__init__.py

@@ -90,7 +90,7 @@ def _poll(
             self.header["timestamp"] = curr_ns
 
             # Write header then value.
-            array = self.header.array
+            array = self.header
             written += array.to_stream(stream)
             written += raw.to_stream(stream, byte_order=array.byte_order)
             if flush:

runtimepy/codec/protocol/__init__.py

@@ -4,7 +4,6 @@
 
 # built-in
 from abc import ABC, abstractmethod
-from copy import copy
 from typing import Optional
 
 # internal
@@ -43,7 +42,7 @@ def instance(cls) -> Protocol:
             cls.initialize(cls.protocol)
             cls.initialized = True
 
-        return copy(cls.protocol)
+        return cls.protocol.copy()
 
     @classmethod
     def singleton(cls) -> Protocol:

runtimepy/codec/protocol/base.py

@@ -3,7 +3,7 @@
 """
 
 # built-in
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from copy import copy as _copy
 from typing import Iterator as _Iterator
 from typing import NamedTuple
@@ -57,17 +57,18 @@ def asdict(self) -> _JsonObject:
 
 
 T = _TypeVar("T", bound="ProtocolBase")
+ProtocolBuild = list[_Union[int, FieldSpec, tuple[str, int]]]
 
 
-class ProtocolBase:
+class ProtocolBase(PrimitiveArray):
     """A class for defining runtime communication protocols."""
 
     def __init__(
         self,
         enum_registry: _EnumRegistry,
         names: _NameRegistry = None,
         fields: BitFieldsManager = None,
-        build: list[_Union[int, FieldSpec, str]] = None,
+        build: ProtocolBuild = None,
         identifier: int = 1,
         byte_order: _Union[_ByteOrder, _RegistryKey] = _DEFAULT_BYTE_ORDER,
         serializables: SerializableMap = None,
@@ -86,7 +87,8 @@ def __init__(
             byte_order = _ByteOrder(
                 self._enum_registry["ByteOrder"].get_int(byte_order)
             )
-        self.array = PrimitiveArray(byte_order=byte_order)
+
+        super().__init__(byte_order=byte_order)
 
         if names is None:
             names = _NameRegistry()
@@ -96,11 +98,11 @@ def __init__(
             fields = BitFieldsManager(self.names, self._enum_registry)
         self._fields = fields
 
-        self._regular_fields: dict[str, _AnyPrimitive] = {}
+        self._regular_fields: dict[str, list[_AnyPrimitive]] = {}
         self._enum_fields: dict[str, _RuntimeEnum] = {}
 
         # Keep track of the order that the protocol was created.
-        self._build: list[_Union[int, FieldSpec, str]] = []
+        self._build: ProtocolBuild = []
 
         # Keep track of named serializables.
         self.serializables: SerializableMap = {}
@@ -110,34 +112,38 @@ def __init__(
             build = []
         for item in build:
             if isinstance(item, int):
-                self._add_bit_fields(self._fields.fields[item], track=False)
-            elif isinstance(item, str):
-                assert serializables, (item, serializables)
-                self.add_field(item, serializable=serializables[item])
-                del serializables[item]
-            else:
+                self._add_bit_fields(self._fields.fields[item])
+            elif isinstance(item, FieldSpec):
                 self.add_field(
                     item.name,
                     item.kind,
                     enum=item.enum,
-                    track=False,
                     array_length=item.array_length,
                 )
+            else:
+                assert serializables, (item, serializables)
+                name = item[0]
+                self.add_serializable(
+                    name,
+                    serializables[name][0],
+                    array_length=None if item[1] == 1 else item[1],
+                )
+                del serializables[name]
 
         # Ensure all serializables were handled via build.
         assert not serializables, serializables
 
-    def __copy__(self: T) -> T:
+    def _copy_impl(self: T) -> T:
         """Create another protocol instance from this one."""
 
         return self.__class__(
             self._enum_registry,
             names=self.names,
             fields=_copy(self._fields),
             build=self._build,
-            byte_order=self.array.byte_order,
+            byte_order=self.byte_order,
             serializables={
-                key: val.copy_without_chain()
+                key: [val[0].copy_without_chain()]
                 for key, val in self.serializables.items()
             },
         )
@@ -151,31 +157,32 @@ def register_name(self, name: str) -> int:
 
     def add_serializable(
         self, name: str, serializable: Serializable, array_length: int = None
-    ) -> int:
+    ) -> None:
         """Add a serializable instance."""
 
         self.register_name(name)
-        self.serializables[name] = serializable
-        self._build.append(name)
-        return self.array.add_to_end(serializable, array_length=array_length)
+
+        instances = self.add_to_end(serializable, array_length=array_length)
+        self._build.append((name, len(instances)))
+        self.serializables[name] = instances
 
     def add_field(
         self,
         name: str,
         kind: _Primitivelike = None,
         enum: _RegistryKey = None,
         serializable: Serializable = None,
-        track: bool = True,
         array_length: int = None,
-    ) -> int:
+    ) -> None:
         """Add a new field to the protocol."""
 
         # Add the serializable to the end of this protocol.
         if serializable is not None:
             assert kind is None and enum is None
-            return self.add_serializable(
+            self.add_serializable(
                 name, serializable, array_length=array_length
             )
+            return
 
         self.register_name(name)
 
@@ -189,25 +196,20 @@ def add_field(
                 kind = runtime_enum.primitive
 
         assert kind is not None
-        new = _create(kind)
-
-        result = self.array.add(new, array_length=array_length)
-        self._regular_fields[name] = new
 
-        if track:
-            self._build.append(
-                FieldSpec(name, kind, enum, array_length=array_length)
-            )
+        self._regular_fields[name] = self.add(
+            _create(kind), array_length=array_length
+        )
 
-        return result
+        self._build.append(
+            FieldSpec(name, kind, enum, array_length=array_length)
+        )
 
-    def _add_bit_fields(self, fields: _BitFields, track: bool = True) -> None:
+    def _add_bit_fields(self, fields: _BitFields) -> None:
         """Add a bit-fields instance."""
 
-        idx = self._fields.add(fields)
-        self.array.add(fields.raw)
-        if track:
-            self._build.append(idx)
+        self._build.append(self._fields.add(fields))
+        self.add(fields.raw)
 
     @contextmanager
     def add_bit_fields(
@@ -219,55 +221,61 @@ def add_bit_fields(
         yield new
         self._add_bit_fields(new)
 
-    def value(self, name: str, resolve_enum: bool = True) -> ProtocolPrimitive:
+    def value(
+        self, name: str, resolve_enum: bool = True, index: int = 0
+    ) -> ProtocolPrimitive:
         """Get the value of a field belonging to the protocol."""
 
         val: ProtocolPrimitive = 0
 
         if name in self._regular_fields:
-            val = self._regular_fields[name].value
+            val = self._regular_fields[name][index].value
 
             # Resolve the enum value.
             if resolve_enum and name in self._enum_fields:
-                val = self._enum_fields[name].get_str(val)  # type: ignore
+                with suppress(KeyError):
+                    val = self._enum_fields[name].get_str(val)  # type: ignore
 
             return val
 
         return self._fields.get(name, resolve_enum=resolve_enum)
 
-    @property
-    def size(self) -> int:
-        """Get this protocol's size in bytes."""
-        return self.array.length()
-
     def trace_size(self, logger: LoggerType) -> None:
         """Log a size trace."""
-        logger.info("%s: %s", self, self.array.length_trace())
+        logger.info("%s: %s", self, self.length_trace())
 
     def __str__(self) -> str:
         """Get this instance as a string."""
 
-        return f"({self.size}) " + " ".join(
-            f"{name}={self[name]}" for name in self.names.registered_order
+        return (
+            self.length_trace()
+            + f" | ({self.length()}) "
+            + " ".join(
+                f"{name}={self[name]}" for name in self.names.registered_order
+            )
         )
 
-    def __getitem__(self, name: str) -> ProtocolPrimitive:
+    def __getitem__(self, name: str) -> ProtocolPrimitive:  # type: ignore
         """Get the value of a protocol field."""
 
         if name in self.serializables:
-            return str(self.serializables[name])
+            return str(self.serializables[name][0])
 
         return self.value(name)
 
-    def __setitem__(self, name: str, val: ProtocolPrimitive) -> None:
+    def set(self, name: str, val: ProtocolPrimitive, index: int = 0) -> None:
         """Set a value of a field belonging to the protocol."""
 
         if name in self._regular_fields:
             # Resolve an enum value.
             if isinstance(val, str):
                 val = self._enum_fields[name].get_int(val)
-            self._regular_fields[name].value = val
+            self._regular_fields[name][index].value = val
         elif name in self.serializables and isinstance(val, str):
-            self.serializables[name].update_str(val)
+            self.serializables[name][index].update_str(val)
         else:
             self._fields.set(name, val)  # type: ignore
+
+    def __setitem__(self, name: str, val: ProtocolPrimitive) -> None:
+        """Set a value of a field belonging to the protocol."""
+        self.set(name, val)