New feature: global type handlers for capi structs

---

+ Fixed refcount issue in _CpPack_EvalLength and _CpUnpack_EvalLength
+ Added Py_TPFLAGS_BASETYPE to all C types
+ Struct model classes can now be used in pack() and unpack() calls
+ Added missing operations to CpBinaryExpr and CpUnaryExpr

docs/sphinx/source/index.rst

@@ -39,10 +39,11 @@ to write complex structures in a compact and readable manner.
 
          @struct
          class Format:
-            magic: const(b"Foo", octetstring(3))  # constant values
+            magic: b"Foo"                         # constant values
             name: cstring(...)                    # C-String without a fixed length
             value: u16                            # little endian encoding
             entries: cstring(...)[be + u32::]     # arrays with big-endian prefixed length
+            value2: bool                          # python type mapping configurable
 
 .. admonition:: Hold up, wait a minute!
 
@@ -52,27 +53,39 @@ to write complex structures in a compact and readable manner.
 Working with defined classes is as straightforward as working with normal classes. *All
 constant values are created automatically!*
 
+
 >>> obj = Format(name="Hello, World!", value=10, entries=["Bar", "Baz"])
 >>> print(obj)
 Format(magic=b'Foo', name='Hello, World!', value=10, entries=['Bar', 'Baz'])
 
 Packing and unpacking have never been easier:
 
->>> pack(obj)
-b'FooHello, World!\x00\n\x00\x00\x00\x00\x02Bar\x00Baz\x00'
->>> unpack(Format, _)
-Format(magic=b'Foo', name='Hello, World!', value=10, entries=['Bar', 'Baz'])
 
-- What about documentation?
+.. tab-set::
+
+   .. tab-item:: Python
 
-   There are specialized options created only for documentation purposes, so you don't
-   have to worry about documenting fields. Just apply the documentation as usual.
+      >>> pack(obj)
+      b'FooHello, World!\x00\n\x00\x00\x00\x00\x02Bar\x00Baz\x00'
+      >>> unpack(Format, _)
+      Format(magic=b'Foo', name='Hello, World!', value=10, entries=['Bar', 'Baz'])
+
+   .. tab-item:: Caterpillar C
+
+      >>> pack(obj, Format)
+      b'FooHello, World!\x00\n\x00\x00\x00\x00\x02Bar\x00Baz\x00\x01'
+      >>> unpack(_, Format)
+      <Format object at 0x...>
+
+- What about documentation?
+  There are specialized options created only for documentation purposes, so you don't
+  have to worry about documenting fields. Just apply the documentation as usual.
 
 - You want to optimize memory space?
+  No problem! It is possible to shrink the memory space occupied by unpacked objects up
+  to 4 times. More information are provided when discussing available configuration
+  :ref:`options`.
 
-   No problem! It is possible to shrink the memory space occupied by unpacked objects up
-   to 4 times. More information are provided when discussing available configuration
-   :ref:`options`.
 
 Where to start?
 ---------------

docs/sphinx/source/reference/capi/objects/atoms/index.rst

@@ -7,9 +7,8 @@ Atom Objects
 .. toctree::
     :maxdepth: 1
 
-    fieldatom.rst
-    intatom.rst
-    charatom.rst
-    floatatom.rst
-    boolatom.rst
-    paddingatom.rst
+    int
+    char
+    float
+    bool
+    padding

docs/sphinx/source/reference/capi/objects/objimpl.rst

@@ -14,6 +14,5 @@ Concrete Objects Layer
     contextpath
     state
     layer
-    fieldobj
     struct
     atoms/index.rst

src/capi.dat

@@ -114,6 +114,8 @@ func:54:CpUnaryExpr_New:CpUnaryExprObject*:+1
 func:55:CpBinaryExpr_New:CpBinaryExprObject*:+1
 func:56:CpContextPath_New:CpContextPathObject*:+1
 func:57:CpContextPath_FromString:CpContextPathObject*:+1
+func:58:CpTypeMap_Lookup:PyObject*:+1
+func:59:CpTypeMap_Register:int:null
 # func:58:CpField_New:CpFieldObject*:null
 # func:59:CpField_HasCondition:int:null
 # func:60:CpField_IsEnabled:int:null

src/caterpillar/_C.pyi

@@ -194,7 +194,9 @@ class fieldinfo:
     def __init__(self, field: atom, excluded: bool = ...) -> None: ...
 
 class lengthinfo:
-    def __init__(self, *args, **kwargs) -> None: ...
+    length: int
+    greedy: bool
+    def __init__(self, length: int = ..., greedy: bool = ...) -> None: ...
 
 class Struct(builtinatom):
     members: dict[str, fieldinfo]
@@ -279,10 +281,6 @@ class atoffset(builtinatom):
     def get_offset(self, layer: layer) -> int: ...
     def __set_byteorder__(self, byteorder: Endian) -> atoffset: ...
 
-class objlayer(layer):
-    obj: Context
-    def __init__(self, *args, **kwargs) -> None: ...
-    def __context_getattr__(self, *args, **kwargs): ...
 
 class repeated(builtinatom):
     atom: Any

src/caterpillar/include/caterpillar/atoms/const.h

@@ -23,11 +23,18 @@ struct _constatomobj
 {
   CpBuiltinAtom_HEAD
 
-  PyObject *m_value;
-  PyObject *m_atom;
+    PyObject* m_value;
+  PyObject* m_atom;
 };
 
 #define CpConstAtom_CheckExact(op) Py_IS_TYPE((op), &CpConstAtom_Type)
 #define CpConstAtom_Check(op) PyObject_TypeCheck((op), &CpConstAtom_Type)
 
+static inline CpConstAtomObject*
+CpConstAtom_New(PyObject* value, PyObject* atom)
+{
+  return (CpConstAtomObject*)CpObject_Create(
+    &CpConstAtom_Type, "OO", atom, value);
+}
+
 #endif

src/caterpillar/include/caterpillar/caterpillarapi.h

@@ -191,6 +191,8 @@ CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value);
 CpBinaryExprObject* CpBinaryExpr_New(int op, PyObject* left, PyObject* right);
 CpContextPathObject* CpContextPath_New(PyObject* path);
 CpContextPathObject* CpContextPath_FromString(const char* path);
+PyObject* CpTypeMap_Lookup(PyObject* annotation, _modulestate* mod);
+int CpTypeMap_Register(PyObject* annotation, PyObject* handler, _modulestate* mod);
 PyObject* CpTypeOf(PyObject* op);
 PyObject* CpTypeOf_Common(PyObject* op);
 int CpPack(PyObject* op, PyObject* atom, PyObject* io, PyObject* globals);
@@ -332,6 +334,8 @@ caterpillar_api.py
 #define CpBinaryExpr_New (*((CpBinaryExprObject* (*)(int op, PyObject* left, PyObject* right)))Cp_API[55])
 #define CpContextPath_New (*((CpContextPathObject* (*)(PyObject* path)))Cp_API[56])
 #define CpContextPath_FromString (*((CpContextPathObject* (*)(const char* path)))Cp_API[57])
+#define CpTypeMap_Lookup (*((PyObject* (*)(PyObject* annotation, _modulestate* mod)))Cp_API[58])
+#define CpTypeMap_Register (*((int (*)(PyObject* annotation, PyObject* handler, _modulestate* mod)))Cp_API[59])
 #define CpTypeOf (*((PyObject* (*)(PyObject* op)))Cp_API[64])
 #define CpTypeOf_Common (*((PyObject* (*)(PyObject* op)))Cp_API[66])
 #define CpPack (*((int (*)(PyObject* op, PyObject* atom, PyObject* io, PyObject* globals)))Cp_API[67])

src/caterpillar/include/caterpillar/module.h

@@ -99,6 +99,9 @@ struct _modulestate
   // cached objects
   PyObject* cp_bytes__true;
   PyObject* cp_bytes__false;
+
+  // type handler map
+  PyObject* cp_typehandler_map;
 };
 
 /**

src/ccaterpillar/atomimpl/builtins/repeated.c

@@ -56,6 +56,8 @@ cp_repeatedatomobj_new(PyTypeObject* type, PyObject* args, PyObject* kw)
   CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL;
   CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_repeatedatomobj_size;
   CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_repeatedatomobj_type;
+  self->m_length = NULL;
+  self->m_atom = NULL;
   return (PyObject*)self;
 }
 
@@ -229,6 +231,9 @@ CpRepeatedAtom_Unpack(CpRepeatedAtomObject* self, CpLayerObject* layer)
   _modulestate* mod = layer->m_state->mod;
   CpLengthInfoObject* lengthinfo =
     (CpLengthInfoObject*)CpObject_CreateNoArgs(&CpLengthInfo_Type);
+  if (!lengthinfo) {
+    return NULL;
+  }
 
   bool unpack_many_attr =
     PyObject_HasAttr(self->m_atom, mod->str___unpack_many__);

src/ccaterpillar/atomimpl/float.c

@@ -199,7 +199,7 @@ PyTypeObject CpFloatAtom_Type = {
   PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpFloatAtom_NAME),
   .tp_basicsize = sizeof(CpFloatAtomObject),
   .tp_dealloc = (destructor)cp_floatatom_dealloc,
-  .tp_flags = Py_TPFLAGS_DEFAULT,
+  .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
   .tp_doc = NULL,
   .tp_members = CpFloatAtom_Members,
   .tp_methods = CpFloatAtom_Methods,

src/ccaterpillar/atomimpl/int.c

@@ -230,7 +230,7 @@ static PyMethodDef CpIntAtom_Methods[] = {
   PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpIntAtom_NAME),
   .tp_basicsize = sizeof(CpIntAtomObject),
   .tp_dealloc = (destructor)cp_intatom_dealloc,
-  .tp_flags = Py_TPFLAGS_DEFAULT,
+  .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
   .tp_doc = cp_intatom__doc,
   .tp_members = CpIntAtom_Members,
   .tp_new = (newfunc)cp_intatom_new,

src/ccaterpillar/atomimpl/lazy.c

@@ -153,7 +153,7 @@ PyTypeObject CpLazyAtom_Type = {
   PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpLazyAtom_NAME),
   .tp_basicsize = sizeof(CpLazyAtomObject),
   .tp_dealloc = (destructor)cp_lazyatom_dealloc,
-  .tp_flags = Py_TPFLAGS_DEFAULT,
+  .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
   .tp_doc = NULL,
   .tp_init = (initproc)cp_lazyatom_init,
   .tp_new = (newfunc)cp_lazyatom_new,

src/ccaterpillar/atomimpl/pad.c

@@ -183,7 +183,7 @@ PyTypeObject CpPaddingAtom_Type = {
   .tp_basicsize = sizeof(CpPaddingAtomObject),
   .tp_dealloc = (destructor)cp_paddingatom_dealloc,
   .tp_repr = (reprfunc)cp_paddingatom_repr,
-  .tp_flags = Py_TPFLAGS_DEFAULT,
+  .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
   .tp_doc = NULL,
   .tp_init = (initproc)cp_paddingatom_init,
   .tp_new = (newfunc)cp_paddingatom_new,