New type: prefixed arrays and strings

+ updated documentation
+ some examples were updated
+ corrected the name of BItField

caterpillar/_common.py

@@ -16,7 +16,7 @@
 
 from typing import List, Any, Union, Iterable
 
-from caterpillar.abc import _GreedyType, _ContextLike, isgreedy, _StreamType
+from caterpillar.abc import _GreedyType, _ContextLike, isgreedy, _StreamType, isprefixed
 from caterpillar.context import (
     Context,
     CTX_PATH,
@@ -26,9 +26,26 @@
     CTX_OBJECT,
     CTX_STREAM,
 )
+from caterpillar.options import F_SEQUENTIAL
 from caterpillar.exception import Stop, StructException, InvalidValueError
 
 
+class WithoutFlag:
+    def __init__(self, context: _ContextLike, flag) -> None:
+        self.context = context
+        self.field = context[CTX_FIELD]
+        self.flag = flag
+
+    def __enter__(self) -> None:
+        self.field ^= self.flag
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.field |= self.flag
+        # We have to apply the right field as instance of the Field class
+        # might set their own value into the context.
+        self.context[CTX_FIELD] = self.field
+
+
 def unpack_seq(context: _ContextLike, unpack_one) -> List[Any]:
     """Generic function to unpack sequenced elements.
 
@@ -49,10 +66,28 @@ def unpack_seq(context: _ContextLike, unpack_one) -> List[Any]:
     # the new context. The '_pos' attribute will be adjusted automatically.
     values = []  # always list (maybe add factory)
     seq_context = Context(
-        _parent=context, _io=stream, _length=length, _lst=values, _field=field
+        _parent=context,
+        _io=stream,
+        _length=length,
+        _lst=values,
+        _field=field,
+        _obj=context.get(CTX_OBJECT),
     )
     greedy = isgreedy(length)
+    prefixed = isprefixed(length)
     seq_context[CTX_POS] = stream.tell()
+    if prefixed:
+        # We have to temporarily remove the array status from the parsing field
+        with WithoutFlag(context, F_SEQUENTIAL):
+            field.amount = 1
+            new_length = length.start.__unpack__(context)
+            field.amount, length = length, new_length
+
+        if not isinstance(length, int):
+            raise InvalidValueError(
+                f"Prefix struct returned non-integer: {length!r}", context
+            )
+
     for i in range(length) if not greedy else itertools.count():
         try:
             seq_context[CTX_PATH] = ".".join([base_path, str(i)])
@@ -93,10 +128,24 @@ def pack_seq(seq: List[Any], context: _ContextLike, pack_one) -> None:
 
     # REVISIT: when to use field.length(context)
     count = len(seq)
+    length = field.amount
+    if isprefixed(length):
+        struct = length.start
+        # We have to temporatily alter the field's values,
+        with WithoutFlag(context, F_SEQUENTIAL):
+            field.amount = 1
+            struct.__pack__(count, context)
+            field.amount = length
 
     # Special elements '_index' and '_length' can be referenced within
     # the new context. The '_pos' attribute will be adjusted automatically.
-    seq_context = Context(_parent=context, _io=stream, _length=count, _field=field)
+    seq_context = Context(
+        _parent=context,
+        _io=stream,
+        _length=count,
+        _field=field,
+        _obj=context.get(CTX_OBJECT),
+    )
     seq_context[CTX_POS] = stream.tell()
     for i, elem in enumerate(seq):
         # The path will contain an additional hint on what element is processed

caterpillar/abc.py

@@ -24,6 +24,7 @@
 _StreamFactory = Callable[[], _StreamType]
 
 _GreedyType = type(...)
+_PrefixedType = slice
 
 
 class _ContextLike(dict):
@@ -177,3 +178,6 @@ def typeof(struct: Union[_StructLike, _ContainsStruct]) -> type:
 
 def isgreedy(obj) -> bool:
     return isinstance(obj, _GreedyType)
+
+def isprefixed(obj) -> bool:
+    return isinstance(obj, _PrefixedType)

caterpillar/fields/__init__.py

@@ -32,6 +32,7 @@
     Computed,
     Pass,
     CString,
+    Prefixed,
     padding,
     char,
     boolean,

caterpillar/fields/_base.py

@@ -24,6 +24,7 @@
     _StreamType,
     _ContextLike,
     _GreedyType,
+    _PrefixedType,
     hasstruct,
     getstruct,
     typeof,
@@ -100,7 +101,7 @@ class Field(_StructLike):
         An automatic flag that indicates this field stores a sequential struct.
     """
 
-    amount: Union[_ContextLambda, int, _GreedyType]
+    amount: Union[_ContextLambda, int, _GreedyType, _PrefixedType]
     """
     A constant or dynamic value to represent the amount of structs. Zero indicates
     there are no sequence types associated with this field.
@@ -140,7 +141,7 @@ def __init__(
         order: ByteOrder = SysNative,
         offset: Union[_ContextLambda, int] = -1,
         flags: Set[Flag] = None,
-        amount: Union[_ContextLambda, int] = 0,
+        amount: Union[_ContextLambda, int, _PrefixedType] = 0,
         options: Union[_Switch, Dict[Any, _StructLike], None] = None,
         condition: Union[_ContextLambda, bool] = True,
         arch: Arch = None,
@@ -195,7 +196,7 @@ def __matmul__(self, offset: Union[_ContextLambda, int]) -> Self:
         return self
 
     def __getitem__(self, dim: Union[_ContextLambda, int, _GreedyType]) -> Self:
-        self._verify_context_value(dim, (_GreedyType, int))
+        self._verify_context_value(dim, (_GreedyType, int, _PrefixedType))
         self.amount = dim
         if self.amount != 0:
             self.flags.add(F_SEQUENTIAL)
@@ -218,6 +219,17 @@ def __rsub__(self, bits: Union[_ContextLambda, int]) -> Self:
         self.bits = bits
         return self
 
+    def __set_byteorder__(self, order: ByteOrder) -> Self:
+        self.order = order
+        return self
+
+    __ixor__ = __xor__
+    __ior__ = __or__
+    __ifloordiv__ = __floordiv__
+    __irshift__ = __rshift__
+    __imatmul__ = __matmul__
+    __isub__ = __rsub__
+
     def is_seq(self) -> bool:
         """Returns whether this field is sequential.
 
@@ -256,7 +268,7 @@ def length(self, context: _ContextLike) -> Union[int, _GreedyType]:
         :rtype: Union[int, _GreedyType]
         """
         try:
-            if isinstance(self.amount, (int, _GreedyType)):
+            if isinstance(self.amount, (int, _GreedyType, _PrefixedType)):
                 return self.amount
 
             return self.amount(context)

caterpillar/fields/common.py

@@ -26,8 +26,14 @@
     _EnumLike,
     isgreedy,
 )
-from caterpillar.exception import ValidationError, StructException, InvalidValueError
+from caterpillar.exception import (
+    ValidationError,
+    StructException,
+    InvalidValueError,
+    DynamicSizeError,
+)
 from caterpillar.context import CTX_FIELD, CTX_STREAM
+from caterpillar.options import F_SEQUENTIAL
 from ._base import Field, FieldStruct
 
 
@@ -531,6 +537,7 @@ def unpack_single(self, context: _ContextLike) -> Any:
     def __class_getitem__(cls, dim) -> Field:
         return CString(...)[dim]
 
+
 class ConstString(Const):
     """
     A specialized constant field for handling string values.
@@ -612,3 +619,56 @@ def pack_single(self, obj: Any, context: _ContextLike) -> None:
     def unpack_single(self, context: _ContextLike) -> None:
         # No need for an implementation
         pass
+
+
+class Prefixed(FieldStruct):
+    def __init__(self, prefix: Optional[_StructLike] = None, encoding: Optional[str] = None):
+        self.encoding = encoding
+        self.prefix = prefix or uint32
+
+    def __type__(self) -> type:
+        return bytes if not self.encoding else str
+
+    def __size__(self, context: _ContextLike) -> int:
+        """
+        Calculate the size of the Prefixed field.
+
+        :param context: The current context.
+        :return: The size of the Bytes field.
+        """
+        raise DynamicSizeError("Prefixed does not store a size", context)
+
+    def pack_single(self, obj: bytes, context: _ContextLike) -> None:
+        """
+        Pack a single bytes object into the stream.
+
+        :param obj: The bytes object to pack.
+        :param context: The current context.
+        """
+        self.prefix.__pack__(len(obj), context)
+        if self.encoding:
+            obj = obj.encode(self.encoding)
+        context[CTX_STREAM].write(obj)
+
+    def unpack_single(self, context: _ContextLike) -> Any:
+        """
+        Unpack a single bytes object from the stream.
+
+        :param context: The current context.
+        :return: The unpacked bytes object.
+        """
+        field: Field = context[CTX_FIELD]
+        is_seq = field.is_seq()
+        if is_seq:
+            # We have to remove the sequence status temporarily
+            field ^= F_SEQUENTIAL
+
+        size = self.prefix.unpack_single(context)
+        data = context[CTX_STREAM].read(size)
+        if self.encoding:
+            data = data.decode(self.encoding)
+
+        # The status has to be added again
+        if is_seq:
+            field |= F_SEQUENTIAL
+        return data

caterpillar/model/__init__.py

@@ -24,6 +24,7 @@
     pack_file,
 )
 from ._bitfield import (
-    Bitfield,
-    bitfield
+    BitField,
+    bitfield,
+    BitFieldGroup
 )

caterpillar/model/_base.py

@@ -45,6 +45,8 @@
 
 @dataclass(init=False)
 class Sequence(_StructLike, FieldMixin):
+    """Default implementation for a sequence of fields."""
+
     model: Any
     """
     Specifies the target class/dictionary used as the base model.

caterpillar/model/_bitfield.py

@@ -67,7 +67,7 @@ class BitFieldGroup:
     fields: Dict[BitTuple, Field] = dcfield(default_factory=dict)
 
 
-class Bitfield(Struct):
+class BitField(Struct):
     groups: List[BitFieldGroup]
 
     def __init__(
@@ -101,7 +101,7 @@ def __init__(
         del self._current_group
 
     def __add__(self, other: "BitField") -> Self:
-        if not isinstance(other, Bitfield):
+        if not isinstance(other, BitField):
             raise ValidationError(
                 f"Attempted to add a non-bitfield struct to a bitfield! (type={type(other)})"
             )
@@ -319,7 +319,7 @@ def _make_bitfield(
     arch: Optional[Arch] = None,
     field_options: Iterable[Flag] = None,
 ) -> type:
-    _ = Bitfield(
+    _ = BitField(
         cls, order=order, arch=arch, options=options, field_options=field_options
     )
     return cls

docs/source/development/changelog.rst

@@ -0,0 +1,5 @@
+.. _changelog:
+
+*********
+Changelog
+*********

docs/source/development/contribution.rst

@@ -0,0 +1,24 @@
+.. _contribution:
+
+***********************
+Contribution Guidelines
+***********************
+
+
+Submit a new Feature
+--------------------
+
+*TODO*
+
+Submit an Issue
+---------------
+
+*TODO*
+
+Running Tests
+-------------
+
+*TODO*
+
+Having a general question?
+--------------------------

docs/source/development/index.rst

@@ -0,0 +1,5 @@
+.. _development-index:
+
+***********
+Development
+***********

docs/source/index.rst

@@ -29,8 +29,7 @@ to write complex structures in a compact and readable manner.
       magic: b"Foo"                       # constant values
       name: CString(...)                  # C-String without a fixed length
       value: le + uint16                  # little endian encoding
-      num_entries: be + uint32            # simple field definition + big endian encoding
-      entries: CString[this.num_entries]  # arrays just like that
+      entries: be + CString[uint32::]     # arrays with big-endian prefixed length
 
 .. admonition:: Hold up, wait a minute!
 
@@ -40,16 +39,16 @@ to write complex structures in a compact and readable manner.
 Working with defined classes is as straightforward as working with normal classes. *All
 constant values are created automatically!*
 
->>> obj = Format(name="Hello, World!", value=1, num_entries=1, entries=["Bar"])
+>>> obj = Format(name="Hello, World!", value=10, entries=["Bar", "Baz"])
 >>> print(obj)
-Format(magic=b'Foo', name='Hello, World!', value=1, num_entries=1, entries=['Bar'])
+Format(magic=b'Foo', name='Hello, World!', value=10, entries=['Bar', 'Baz'])
 
 Packing and unpacking have never been easier:
 
 >>> pack(obj)
-b'FooHello, World!\x00\x01\x00\x00\x00\x00\x00\x00\x01Bar\x00'
+b'FooHello, World!\x00\n\x00\x00\x00\x00\x02Bar\x00Baz\x00'
 >>> unpack(Format, _)
-Format(magic=b'Foo', name='Hello, World!', value=1, num_entries=1, entries=['Bar'])
+Format(magic=b'Foo', name='Hello, World!', value=10, entries=['Bar', 'Baz'])
 
 .. admonition:: What about documentation?
 
@@ -70,9 +69,10 @@ what configuration options can be used. Alternatively you can follow the :ref:`t
    :caption: Contents:
 
    installing/index.rst
-   reference/index.rst
    tutorial/index.rst
+   reference/index.rst
    library/index.rst
+   development/index.rst