Updated documentation

+ New Struct: NotRequired
+ BitField now uses struct to unpack integers

caterpillar/_common.py

@@ -105,8 +105,8 @@ def unpack_seq(context: _ContextLike, unpack_one) -> List[Any]:
         except Stop:
             break
         except Exception as exc:
-            # if greedy:
-            #     break
+            if greedy:
+                break
             raise StructException(str(exc), context) from exc
     return values
 

caterpillar/fields/common.py

@@ -34,7 +34,6 @@
 from caterpillar.options import F_SEQUENTIAL
 from caterpillar.byteorder import LittleEndian
 from ._base import Field, FieldStruct, INVALID_DEFAULT, singleton
-from .._common import WithoutContextVar
 
 
 class FormatField(FieldStruct):
@@ -54,6 +53,9 @@ def __init__(self, ch: str, type_: type) -> None:
         self.__bits__ = calcsize(self.text) * 8
         self._padding_ = self.text == "x"
 
+    def __fmt__(self) -> str:
+        return self.text
+
     def __repr__(self) -> str:
         """
         String representation of the FormatField.
@@ -114,7 +116,7 @@ def pack_seq(self, seq: Sequence, context: _ContextLike) -> None:
         :param seq: The sequence of values.
         :param context: The current context.
         """
-        if context[CTX_FIELD].length(context) is not Ellipsis:
+        if context[CTX_FIELD].amount is not Ellipsis:
             self.pack_single(seq, context)
         else:
             super().pack_seq(seq, context)
@@ -193,6 +195,7 @@ def is_padding(self) -> bool:
 ssize_t = FormatField("n", int)
 size_t = FormatField("N", int)
 
+float16 = FormatField("e", float)
 float32 = FormatField("f", float)
 float64 = FormatField("d", float)
 double = float64
@@ -216,6 +219,9 @@ def __init__(self, struct: _StructLike) -> None:
         self.struct = struct
         self.__bits__ = getattr(self.struct, "__bits__", None)
 
+    def __fmt__(self) -> str:
+        return self.struct.__fmt__()
+
     def __type__(self) -> type:
         """
         Get the type of the data encoded/decoded by the transformer.
@@ -520,7 +526,7 @@ def unpack_single(self, context: _ContextLike) -> Any:
         :param context: The current context.
         :return: The unpacked string.
         """
-        raw_pad = self.pad.to_bytes(1, byteorder='big')
+        raw_pad = self.pad.to_bytes(1, byteorder="big")
         if self.length is Ellipsis:
             # Parse actual C-String
             stream: _StreamType = context[CTX_STREAM]
@@ -718,3 +724,24 @@ def unpack_single(self, context: _ContextLike) -> memoryview:
 class UInt(Int):
     def __init__(self, bits: int) -> None:
         super().__init__(bits, signed=False)
+
+
+# still experimental
+class NotRequired(Transformer):
+    def __type__(self) -> type:
+        return Optional[super().__type__()]
+
+    def __pack__(self, obj: Any, context: _ContextLike) -> None:
+        if obj is None:
+            return
+        super().__pack__(obj, context)
+
+    def __unpack__(self, context: _ContextLike) -> Optional[Any]:
+        try:
+            return super().__unpack__(context)
+        except StructException:
+            return None
+
+
+def optional(struct) -> Transformer:
+    return NotRequired(struct)

caterpillar/model/_base.py

@@ -92,7 +92,7 @@ def __init__(
     ) -> None:
         self.model = model
         self.arch = arch
-        self.order = order
+        self.order = order or SysNative
         self.options = set(options or [])
         self.field_options = set(field_options or [])
 

caterpillar/model/_bitfield.py

@@ -12,6 +12,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import struct as libstruct
+
 from typing import Optional, Any, Dict
 from typing import Iterable, Tuple
 from typing import Self, List
@@ -61,6 +63,11 @@ def issigned(obj: Any) -> bool:
     return bool(getattr(obj, SIGNED_ATTR, None))
 
 
+def getformat(obj: Any) -> str:
+    attr = getattr(obj, "__fmt__")
+    return attr() if callable(attr) else attr
+
+
 @dataclass(init=False)
 class BitFieldGroup:
     size: int
@@ -72,6 +79,7 @@ def __init__(self, size: int, pos: int, fields: Dict = None) -> None:
         self.size = size
         self.pos = pos
         self.fields = fields or {}
+        # this has to get refactored
         if 8 < size <= 16:
             self.fmt = "H"
         elif 16 < size <= 32:
@@ -110,6 +118,7 @@ def __init__(
         self.options.difference_update(GLOBAL_STRUCT_OPTIONS, GLOBAL_UNION_OPTIONS)
         self.options.update(GLOBAL_BITFIELD_FLAGS)
         self.__bits__ = sum(map(lambda x: x.size, self.groups))
+        self.__fmt__ = "".join(map(lambda x: x.fmt, self.groups))
 
         del self._bit_pos
         del self._abs_bit_pos
@@ -120,6 +129,7 @@ def __add__(self, other: "BitField") -> Self:
             raise ValidationError(
                 f"Attempted to add a non-bitfield struct to a bitfield! (type={type(other)})"
             )
+        # REVISIT: undefined bahaviour when parsing
         return super(Struct, self).__add__(other)
 
     def _process_field(
@@ -230,7 +240,10 @@ def _process_field(
 
         type_ = typeof(field.struct)
         bit_pos = max(group.size - 1 - self._bit_pos, 0)
-        group.fields[(bit_pos, width, type_)] = field
+        # NOTE: I know, we're calling this method twice now, but it saves some
+        # iterations later on.
+        if self._included(name, default, annotation):
+            group.fields[(bit_pos, width, type_)] = field
         self._bit_pos += width
         self._abs_bit_pos += width
         return field
@@ -263,12 +276,15 @@ def unpack_one(self, context: _ContextLike) -> Optional[Any]:
         context[CTX_OBJECT] = Context(_parent=context)
         base_path = context[CTX_PATH]
         data = memoryview(context[CTX_STREAM].read(self.__bits__ // 8))
+
         for i, group in enumerate(self.groups):
             # each group specifies the fields we are about to unpack. But first, we have
             # to read the bits from the stream
             start = group.pos // 8
-            value = data[start : start + group.size // 8].cast(group.fmt)[0]
-            for bit_info, field in reversed(group.fields.items()):
+            value = libstruct.unpack(
+                f"{self.order.ch}{group.fmt}", data[start : start + group.size // 8]
+            )[0]
+            for bit_info, field in group.fields.items():
                 name: str = field.__name__
                 # The field should be ignored if it is not within the
                 # member map (this usually means we have a padding field)
@@ -300,15 +316,15 @@ def pack_one(self, obj: Any, context: _ContextLike) -> None:
             # The same applies here, but we convert all values to int instead of reading
             # them from the stream
             value = 0
-            for bit_info, field in reversed(group.fields.items()):
+            for bit_info, field in group.fields.items():
                 # Setup the field's context
                 name: str = field.__name__
-                context[CTX_PATH] = f"{base_path}.({i}).{name}"
-                bit_pos, width, _ = bit_info
                 # Padding is translated into zeros
                 if name not in self._member_map_:
                     continue
 
+                context[CTX_PATH] = f"{base_path}.({i}).{name}"
+                bit_pos, width, _ = bit_info
                 field_value = getattr(obj, name, 0) or 0
                 shift = bit_pos + 1 - width
                 # Here's the tricky part: we have to convert all values to int

docs/source/development/index.rst

@@ -2,4 +2,13 @@
 
 ***********
 Development
-***********
+***********
+
+*TODO*
+
+.. toctree::
+    :numbered:
+    :maxdepth: 2
+
+    contribution.rst
+    changelog.rst

docs/source/index.rst

@@ -9,7 +9,6 @@ Python class definitions and annotations.
    Before using this library, be aware that the current status is "beta/testing". Therefore,
    expect bugs and undocumented issues. Additionally, the documentation is a WIP.
 
-
 Whetting Your Appetite
 ----------------------
 
@@ -74,7 +73,10 @@ what configuration options can be used. Alternatively you can follow the :ref:`t
    library/index.rst
    development/index.rst
 
-
+.. seealso::
+   * `Github Source <https://github.com/MatrixEditor/caterpillar>`_
+   * `Github Issues <https://github.com/MatrixEditor/caterpillar/issues>`_
+   * `Discussions <https://github.com/MatrixEditor/caterpillar/discussions>`_
 
 Indices and tables
 ==================

docs/source/reference/snippets/comparison_1_caterpillar.py

@@ -4,8 +4,8 @@
 
 @bitfield(order=LittleEndian)
 class Flags:
-    bool1: 1
-    num4: 3
+    bool1 : 1
+    num4  : 3
     # padding is generated automatically
 
 

docs/source/tutorial/advanced.rst

@@ -4,4 +4,78 @@
 Advanced Concepts
 *****************
 
-*TODO*
+*TODO*
+
+Now that you've acquired a general understanding of how this library works, let's
+start the more advanced concepts. This document will bring the tutorial to a close,
+leaving you well-equipped to create your own struct classes in Python.
+
+.. attention::
+    Most of the structs and techniques showcased here are subject to change, notably
+    :class:`BitField` and :code:`@union`. Their current usage is not as user-friendly
+    as expected.
+
+
+Operators
+---------
+
+Switch
+^^^^^^
+
+*TODO*
+
+Offset
+^^^^^^
+
+*TODO*
+
+BitFields
+---------
+
+*TODO* description
+
+.. code-block:: python
+    :caption: Implementing the `chunk-naming <https://www.w3.org/TR/png/#5Chunk-naming-conventions>`_ convention
+
+    @bitfield(options={S_DISCARD_UNNAMED})
+    class ChunkOptions:
+        _            : 2        # <-- first two bits are not used
+        ancillary    : 1        # f0
+        _1           : 0
+        _2           : 2
+        private      : 1        # <-- the 5-th bit (from right to left)
+        _3           : 0
+        _4           : 2
+        reserved     : 1        # f2
+        _5           : 0        # <-- padding until the end of this byte
+        _6           : 2
+        safe_to_copy : 1        # f3
+
+    # byte     :     0        1       2        3
+    # bit      : 76543210 76543210 76543210 76543210
+    # ----------------------------------------------
+    # breakdown: 00100000 00100000 00100000 00100000
+    #            \/|\___/ \/|\___/ \/|\___/ \/|\___/
+    #            u f0 a   u f1 a   u f2  a  u f3 a
+    # Where u='unnamed', a='added' and 'f..'=corresponding fields
+
+
+Unions
+------
+
+*TODO* description
+
+.. code-block:: python
+    :caption: Combining the name with its naming convention
+
+    @union
+    class ChunkName:
+        text: Bytes(4)
+        options: ChunkOptions
+
+
+The End!
+--------
+
+*TODO: implement final PNG struct*
+

docs/source/tutorial/basics.rst

@@ -7,7 +7,7 @@ Basic Concepts
 In this section, we'll explore some common techniques used in binary file formats, setting
 the stage for more advanced topics in the next chapter.
 
-.. attention::
+.. note::
     Some examples using the interpreter prompts make use of a shortcut to define :class:`Field`
     objects:
 
@@ -299,7 +299,9 @@ Specials
 Computed
 ~~~~~~~~
 
-A runtime computed variable that does not pack any data.
+A runtime computed variable that does not pack any data. It is rarely recommended to use this
+struct, because you can simply define a :code:`@property` or method for what this structs
+represents, **unless** you need the value later on while packing or unpacking.
 
 >>> struct = Computed(this.foobar) # context lambda or constant value
 
@@ -325,13 +327,12 @@ A runtime computed variable that does not pack any data.
 Pass
 ~~~~
 
-In case nothing should be done, just use :class:`Pass`.
+In case nothing should be done, just use :class:`Pass`. This struct won't affect the stream in any way.
 
 .. raw:: html
 
     <hr>
 
-
 .. important::
     Congratulations! You have successfully mastered the basics of *caterpillar*! Are you
     ready for the next level? Brace yourself for some breathtaking action!

docs/source/tutorial/first_steps.rst

@@ -61,8 +61,7 @@ Packing data
 ^^^^^^^^^^^^
 
 This library's packing and unpacking is similar to Python's `struct <https://docs.python.org/3/library/struct.html>`_
-module. In order to pack data, we need a struct and an input object. When packing data, a struct and an input
-object are needed.
+module. When packing data, a struct and an input object are needed.
 
 Thanks to the RGB class encapsulating its struct instance, explicitly stating the struct to use
 becomes unnecessary.

examples/comparison/comparison_1_caterpillar.py

@@ -4,8 +4,8 @@
 
 @bitfield(order=LittleEndian)
 class Flags:
-    bool1: 1
-    num4: 3
+    bool1 : 1
+    num4  : 3
     # padding is generated automatically