From 883dbd77b762edd85e218239aeb76fdc6cdf13b5 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sat, 7 Sep 2024 13:12:47 +0200 Subject: [PATCH 01/29] Updated name schema of C atoms --- docs/sphinx/source/development/index.rst | 1 + docs/sphinx/source/development/roadmap.rst | 32 +++++++++++++ docs/sphinx/source/extensions/refcounts.dat | 6 ++- .../capi/objects/atoms/floatatom.rst | 7 ++- .../reference/capi/objects/atoms/intatom.rst | 29 ++++++++++-- src/atomimpl/boolatomobj.c | 10 ++-- src/atomimpl/charatomobj.c | 9 ++-- src/atomimpl/floatatomobj.c | 9 ++-- src/atomimpl/intatomobj.c | 35 ++++++-------- src/atomimpl/padatomobj.c | 11 +++-- src/atomimpl/stringatomobj.c | 7 ++- src/caterpillar/_C.pyi | 40 ++++++++-------- .../include/caterpillar/atoms/float.h | 9 +--- .../include/caterpillar/atoms/int.h | 8 +--- .../include/caterpillar/atoms/primitive.h | 46 ++----------------- .../include/caterpillar/atoms/string.h | 8 +--- .../include/caterpillar/caterpillarapi.h | 28 +++++++++++ src/caterpillar/include/caterpillar/macros.h | 2 + src/caterpillarapi.c | 29 +++++++++++- src/code_gen/caterpillar_api.py | 16 ++++++- src/module.c | 17 +++---- test/_C/atoms/test_padding.py | 12 ++--- test/_Py/test_flag.py | 4 ++ 23 files changed, 224 insertions(+), 151 deletions(-) create mode 100644 docs/sphinx/source/development/roadmap.rst create mode 100644 test/_Py/test_flag.py diff --git a/docs/sphinx/source/development/index.rst b/docs/sphinx/source/development/index.rst index 5e0c89d..8a5a340 100644 --- a/docs/sphinx/source/development/index.rst +++ b/docs/sphinx/source/development/index.rst @@ -10,5 +10,6 @@ Development :numbered: :maxdepth: 2 + roadmap.rst contribution.rst changelog.rst \ No newline at end of file diff --git a/docs/sphinx/source/development/roadmap.rst b/docs/sphinx/source/development/roadmap.rst new file mode 100644 index 0000000..0ee99d5 --- /dev/null +++ b/docs/sphinx/source/development/roadmap.rst @@ -0,0 +1,32 @@ +.. _dev-roadmap: + +******** +Roadmap +******** + +.. |check_| raw:: html + + + +.. |uncheck_| raw:: html + + + +C API +----- + +|check_| Implementation of parsing process (unpack, pack) + +|check_| Struct class (:c:type:`CpStructObject`) + +|uncheck_| Struct wrapper function + +Atom Objects: +^^^^^^^^^^^^^ + +- |check_| Integer (uint8-128 and int8-128) (C type: :c:type:`CpIntAtomObject`, Py type: :class:`int_t`) +- |check_| Float, Double (C type: :c:type:`CpFloatAtomObject`, Py type: :class:`float_t`) +- |check_| Boolean (C type: :c:type:`CpBoolAtomObject`, Py type: :class:`bool_t`), global instance: :code:`boolean` +- |check_| Char (C type: :c:type:`CpCharAtomObject`, Py type: :class:`char_t`), global instance: :code:`char` +- |check_| Padding (C type: :c:type:`CpPaddingAtomObject`, Py type: :class:`padding_t`), global instance: :code:`padding` +- |check_| String (C type: :c:type:`CpStringAtomObject`, Py type: :class:`string`) \ No newline at end of file diff --git a/docs/sphinx/source/extensions/refcounts.dat b/docs/sphinx/source/extensions/refcounts.dat index 36aa46f..a95b590 100644 --- a/docs/sphinx/source/extensions/refcounts.dat +++ b/docs/sphinx/source/extensions/refcounts.dat @@ -39,4 +39,8 @@ CpState_Read:PyObject*:+1 CpState_ReadFully:PyObject*:+1 CpState_Write:PyObject*:+1 -CpLayer_New:CpLayerObject*:+1 \ No newline at end of file +CpLayer_New:CpLayerObject*:+1 + +CpIntAtom_Unpack:PyObject*:+1 + +CpFloatAtom_Unpack:PyObject*:+1 \ No newline at end of file diff --git a/docs/sphinx/source/reference/capi/objects/atoms/floatatom.rst b/docs/sphinx/source/reference/capi/objects/atoms/floatatom.rst index 02cb855..31bcfaa 100644 --- a/docs/sphinx/source/reference/capi/objects/atoms/floatatom.rst +++ b/docs/sphinx/source/reference/capi/objects/atoms/floatatom.rst @@ -8,7 +8,7 @@ Float Atoms .. c:var:: PyTypeObject CpFloatAtom_Type - The type object for the :c:type:`floatatom` class. + The type object for the :c:type:`CpFloatAtomObject` class. .. c:type:: CpFloatAtomObject @@ -20,6 +20,11 @@ Float Atoms .. c:function:: PyObject* CpFloatAtom_Unpack(CpFloatAtomObject* self, CpLayerObject* layer) +.. c:function:: int CpFloatAtom_Check(PyObject *op) + + +.. c:function:: int CpFloatAtom_CheckExact(PyObject *op) + Runtime Performance ------------------- diff --git a/docs/sphinx/source/reference/capi/objects/atoms/intatom.rst b/docs/sphinx/source/reference/capi/objects/atoms/intatom.rst index 5398106..b1e0043 100644 --- a/docs/sphinx/source/reference/capi/objects/atoms/intatom.rst +++ b/docs/sphinx/source/reference/capi/objects/atoms/intatom.rst @@ -11,7 +11,7 @@ counterpart (:class:`FormatField`), this C-based class focuses solely on integer .. c:var:: PyTypeObject CpIntAtom_Type - The type object for the :c:type:`intatom` class. + The type object for the :c:type:`int_t` class. This implementation utilizes :code:`int.from_bytes` and :code:`int.to_bytes`. Direct C calls are optimized, reducing runtime overhead compared to Python. @@ -34,24 +34,43 @@ calls are optimized, reducing runtime overhead compared to Python. Indicates the endianness of the integer. A value of ``1`` signifies little-endian, while ``0`` signifies big-endian. +.. c:function:: int CpIntAtom_Pack(CpIntAtomObject* self, PyObject* value, CpLayerObject* layer) + + Packs the given value into the given layer. Returns ``-1`` if an error occurs. + + +.. c:function:: PyObject* CpIntAtom_Unpack(CpIntAtomObject* self, CpLayerObject* layer) + + Unpacks the value from the given layer. Returns *NULL* if an error occurs. + + +.. c:function:: int CpIntAtom_Check(PyObject *op) + + Checks if the given object is an :c:type:`CpIntAtomObject` + + +.. c:function:: int CpIntAtom_CheckExact(PyObject *op) + + Checks if the given object is an instance of an :c:type:`CpIntAtomObject` + Recommendations --------------- -The following examples illustrate how to effectively utilize the :c:type:`intatom` class +The following examples illustrate how to effectively utilize the :c:type:`int_t` class and the associated methods: .. code-block:: python :linenos: from caterpillar._C import LITTLE_ENDIAN as le - from caterpillar._C import intatom, unpack, i16 + from caterpillar._C import int_t, unpack, i16 # Define a global 16-bit little-endian signed integer atom I16_LE = le + i16 - _I16_LE = intatom(16, signed=True, little_endian=True) + _I16_LE = int_t(16, signed=True, little_endian=True) unpack(b"\x01\x02", _I16_LE) - unpack(b"\x01\x02", intatom(16, signed=True, little_endian=True)) + unpack(b"\x01\x02", int_t(16, signed=True, little_endian=True)) unpack(b"\x01\x02", I16_LE) unpack(b"\x01\x02", le + i16) diff --git a/src/atomimpl/boolatomobj.c b/src/atomimpl/boolatomobj.c index bc5a058..5332a7a 100644 --- a/src/atomimpl/boolatomobj.c +++ b/src/atomimpl/boolatomobj.c @@ -1,7 +1,6 @@ /* boolatom C implementation */ -#include "caterpillar/atoms/primitive.h" -#include "caterpillar/module.h" -#include "caterpillar/state.h" +#include "caterpillar/caterpillar.h" + #include static PyObject* @@ -45,7 +44,7 @@ cp_boolatom_init(CpBoolAtomObject* self, PyObject* args, PyObject* kwds) } /* Public API */ - +/*CpAPI*/ int CpBoolAtom_Pack(CpBoolAtomObject* self, PyObject* value, CpLayerObject* layer) { @@ -63,6 +62,7 @@ CpBoolAtom_Pack(CpBoolAtomObject* self, PyObject* value, CpLayerObject* layer) return 0; } +/*CpAPI*/ PyObject* CpBoolAtom_Unpack(CpBoolAtomObject* self, CpLayerObject* layer) { @@ -84,7 +84,7 @@ CpBoolAtom_Unpack(CpBoolAtomObject* self, CpLayerObject* layer) /* type setup */ PyTypeObject CpBoolAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(boolatom), + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpBoolAtom_NAME), .tp_basicsize = sizeof(CpBoolAtomObject), .tp_dealloc = (destructor)cp_boolatom_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, diff --git a/src/atomimpl/charatomobj.c b/src/atomimpl/charatomobj.c index 7a05d88..3804f36 100644 --- a/src/atomimpl/charatomobj.c +++ b/src/atomimpl/charatomobj.c @@ -1,6 +1,6 @@ /* charatom C implementation */ -#include "caterpillar/atoms/primitive.h" -#include "caterpillar/state.h" +#include "caterpillar/caterpillar.h" + #include static PyObject* @@ -44,7 +44,7 @@ cp_charatom_init(CpCharAtomObject* self, PyObject* args, PyObject* kwds) } /* Public API */ - +/*CpAPI*/ int CpCharAtom_Pack(CpCharAtomObject* self, PyObject* value, CpLayerObject* layer) { @@ -66,6 +66,7 @@ CpCharAtom_Pack(CpCharAtomObject* self, PyObject* value, CpLayerObject* layer) return 0; } +/*CpAPI*/ PyObject* CpCharAtom_Unpack(CpCharAtomObject* self, CpLayerObject* layer) { @@ -78,7 +79,7 @@ CpCharAtom_Unpack(CpCharAtomObject* self, CpLayerObject* layer) /* type setup */ PyTypeObject CpCharAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(charatom), + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpCharAtom_NAME), .tp_basicsize = sizeof(CpCharAtomObject), .tp_dealloc = (destructor)cp_charatom_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, diff --git a/src/atomimpl/floatatomobj.c b/src/atomimpl/floatatomobj.c index bdf277d..342d543 100644 --- a/src/atomimpl/floatatomobj.c +++ b/src/atomimpl/floatatomobj.c @@ -1,9 +1,5 @@ /* floatatom C implementation */ #include "caterpillar/caterpillar.h" -#include "caterpillar/arch.h" -#include "caterpillar/atoms/float.h" -#include "caterpillar/state.h" -#include "caterpillar/module.h" #include static PyObject* @@ -74,7 +70,7 @@ cp_floatatom_init(CpFloatAtomObject* self, PyObject* args, PyObject* kwds) } /* Public API */ - +/*CpAPI*/ int CpFloatAtom_Pack(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer) { @@ -135,6 +131,7 @@ CpFloatAtom_Pack(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer) return 0; } +/*CpAPI*/ PyObject* CpFloatAtom_Unpack(CpFloatAtomObject* self, CpLayerObject* layer) { @@ -195,7 +192,7 @@ static PyMemberDef CpFloatAtom_Members[] = { }; PyTypeObject CpFloatAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(floatatom), + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpFloatAtom_NAME), .tp_basicsize = sizeof(CpFloatAtomObject), .tp_dealloc = (destructor)cp_floatatom_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, diff --git a/src/atomimpl/intatomobj.c b/src/atomimpl/intatomobj.c index 323ef49..352654c 100644 --- a/src/atomimpl/intatomobj.c +++ b/src/atomimpl/intatomobj.c @@ -2,10 +2,7 @@ #include #include "caterpillar/caterpillar.h" -#include "caterpillar/atoms/int.h" -#include "caterpillar/arch.h" -#include "caterpillar/state.h" -#include "caterpillar/module.h" + #include static PyObject* @@ -14,8 +11,8 @@ cp_intatom__type__(CpIntAtomObject* self) return Py_XNewRef(&PyLong_Type); } -static PyObject * -cp_intatom__size__(CpIntAtomObject* self, PyObject *ctx) +static PyObject* +cp_intatom__size__(CpIntAtomObject* self, PyObject* ctx) { return Py_XNewRef(self->m_byte_count); } @@ -78,9 +75,8 @@ cp_intatom_init(CpIntAtomObject* self, PyObject* args, PyObject* kwds) return 0; } - - /* Public API */ +/*CpAPI*/ int CpIntAtom_Pack(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer) { @@ -96,12 +92,12 @@ CpIntAtom_Pack(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer) int little_endian = self->_m_little_endian; if (layer->m_field) { - _modulestate *mod = layer->m_state->mod; - PyObject *endian = ((CpFieldObject *)layer->m_field)->m_endian; + _modulestate* mod = layer->m_state->mod; + PyObject* endian = ((CpFieldObject*)layer->m_field)->m_endian; if (CpEndian_Check(endian)) - /* If the field has an endian specified, use that. */ - little_endian = CpEndian_IsLittleEndian((CpEndianObject *)endian, mod); + /* If the field has an endian specified, use that. */ + little_endian = CpEndian_IsLittleEndian((CpEndianObject*)endian, mod); } int res = _PyLong_AsByteArray((PyLongObject*)op, @@ -121,6 +117,7 @@ CpIntAtom_Pack(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer) return 0; } +/*CpAPI*/ PyObject* CpIntAtom_Unpack(CpIntAtomObject* self, CpLayerObject* layer) { @@ -131,12 +128,12 @@ CpIntAtom_Unpack(CpIntAtomObject* self, CpLayerObject* layer) int little_endian = self->_m_little_endian; if (layer->m_field) { - _modulestate *mod = layer->m_state->mod; - PyObject *endian = ((CpFieldObject *)layer->m_field)->m_endian; + _modulestate* mod = layer->m_state->mod; + PyObject* endian = ((CpFieldObject*)layer->m_field)->m_endian; if (CpEndian_Check(endian)) - /* If the field has an endian specified, use that. */ - little_endian = CpEndian_IsLittleEndian((CpEndianObject *)endian, mod); + /* If the field has an endian specified, use that. */ + little_endian = CpEndian_IsLittleEndian((CpEndianObject*)endian, mod); } PyObject* obj = @@ -159,14 +156,12 @@ static PyMemberDef CpIntAtom_Members[] = { NULL }, { "nbits", T_PYSSIZET, offsetof(CpIntAtomObject, _m_bits), READONLY, NULL }, { "signed", T_BOOL, offsetof(CpIntAtomObject, _m_signed), READONLY, NULL }, - { "little_endian", - T_BOOL, - offsetof(CpIntAtomObject, _m_little_endian)}, + { "little_endian", T_BOOL, offsetof(CpIntAtomObject, _m_little_endian) }, { NULL } /* Sentinel */ }; PyTypeObject CpIntAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(intatom), + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpIntAtom_NAME), .tp_basicsize = sizeof(CpIntAtomObject), .tp_dealloc = (destructor)cp_intatom_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, diff --git a/src/atomimpl/padatomobj.c b/src/atomimpl/padatomobj.c index c411c85..c24108a 100644 --- a/src/atomimpl/padatomobj.c +++ b/src/atomimpl/padatomobj.c @@ -1,9 +1,7 @@ /* padding atom implementation */ #include "caterpillar/caterpillar.h" -#include "caterpillar/atoms/primitive.h" -#include "caterpillar/parsing.h" -#include "caterpillar/module.h" + #include static PyObject* @@ -62,7 +60,7 @@ cp_paddingatom_init(CpPaddingAtomObject* self, PyObject* args, PyObject* kwds) } /* Public API */ - +/*CpAPI*/ int CpPaddingAtom_Pack(CpPaddingAtomObject* self, PyObject* value, @@ -88,6 +86,7 @@ CpPaddingAtom_Pack(CpPaddingAtomObject* self, return 0; } +/*CpAPI*/ int CpPaddingAtom_PackMany(CpPaddingAtomObject* self, PyObject* value, @@ -148,6 +147,7 @@ CpPaddingAtom_PackMany(CpPaddingAtomObject* self, return 0; } +/*CpAPI*/ PyObject* CpPaddingAtom_Unpack(CpPaddingAtomObject* self, CpLayerObject* layer) { @@ -159,6 +159,7 @@ CpPaddingAtom_Unpack(CpPaddingAtomObject* self, CpLayerObject* layer) Py_RETURN_NONE; } +/*CpAPI*/ PyObject* CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, CpLayerObject* layer) { @@ -194,7 +195,7 @@ cp_paddingatom__repr__(CpPaddingAtomObject* self) /* type setup */ PyTypeObject CpPaddingAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(paddingatom), + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpPaddingAtom_NAME), .tp_basicsize = sizeof(CpPaddingAtomObject), .tp_dealloc = (destructor)cp_paddingatom_dealloc, .tp_repr = (reprfunc)cp_paddingatom__repr__, diff --git a/src/atomimpl/stringatomobj.c b/src/atomimpl/stringatomobj.c index b480d03..4eaae28 100644 --- a/src/atomimpl/stringatomobj.c +++ b/src/atomimpl/stringatomobj.c @@ -1,9 +1,6 @@ /* string atom implementation */ #include "caterpillar/caterpillar.h" -#include "caterpillar/atoms/string.h" -#include "caterpillar/parsing.h" -#include "caterpillar/module.h" #include @@ -80,6 +77,7 @@ cp_stringatom_init(CpStringAtomObject* self, PyObject* args, PyObject* kwds) /* Public API */ +/*CpAPI*/ int CpStringAtom_Pack(CpStringAtomObject* self, PyObject* value, @@ -99,6 +97,7 @@ CpStringAtom_Pack(CpStringAtomObject* self, return 0; } +/*CpAPI*/ PyObject* CpStringAtom_Unpack(CpStringAtomObject* self, CpLayerObject* layer) { @@ -134,7 +133,7 @@ static PyMemberDef CpStringAtom_Members[] = { }; PyTypeObject CpStringAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(string), + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpStringAtom_NAME), .tp_basicsize = sizeof(CpStringAtomObject), .tp_dealloc = (destructor)cp_stringatom_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, diff --git a/src/caterpillar/_C.pyi b/src/caterpillar/_C.pyi index 0244f9e..f8ad62c 100644 --- a/src/caterpillar/_C.pyi +++ b/src/caterpillar/_C.pyi @@ -252,41 +252,43 @@ def pack(__obj: Any, __struct: atom, **globals) -> bytes: ... def sizeof(obj: atom, globals: Optional[dict | Context] = ...): ... def unpack(__io: Any, __struct: atom, **globals) -> Any: ... -class intatom(fieldcatom): +class int_t(fieldcatom): little_endian: bool nbits: int nbytes: int signed: bool def __init__(self, nbits: int, signed: bool = ..., little_endian: bool = ...) -> None: ... -i16: intatom -i24: intatom -i32: intatom -i64: intatom -i8: intatom -u16: intatom -u24: intatom -u32: intatom -u64: intatom -u8: intatom - -class floatatom(fieldcatom): +i16: int_t +i24: int_t +i32: int_t +i64: int_t +i8: int_t +u16: int_t +u24: int_t +u32: int_t +u64: int_t +u8: int_t + +class float_t(fieldcatom): little_endian: bool nbits: int nbytes: int def __init__(self, nbits: int, little_endian: bool = ...) -> None: ... -f16: floatatom -f32: floatatom -f64: floatatom +f16: float_t +f32: float_t +f64: float_t -class paddingatom(fieldcatom): +class padding_t(fieldcatom): def __init__(self, pad: int) -> None: ... -padding: paddingatom +padding: padding_t -class stringatom(fieldcatom): +class string(fieldcatom): encoding: str errors: str length: _Length def __init__(self, length: _Length, encoding: str, errors: str = ...) -> None: ... + + diff --git a/src/caterpillar/include/caterpillar/atoms/float.h b/src/caterpillar/include/caterpillar/atoms/float.h index 5aac58a..767c7e2 100644 --- a/src/caterpillar/include/caterpillar/atoms/float.h +++ b/src/caterpillar/include/caterpillar/atoms/float.h @@ -35,18 +35,11 @@ struct _floatatomobj int _m_little_endian; }; -// PyAPI_DATA(PyTypeObject) CpFloatAtom_Type; +#define CpFloatAtom_NAME "float_t" /** @brief Checks if the given object is an integer atom object */ #define CpFloatAtom_CheckExact(op) Py_IS_TYPE((op), &CpFloatAtom_Type) /** @brief Checks if the given object is an integer atom object */ #define CpFloatAtom_Check(op) (PyObject_IsInstance((op), &CpFloatAtom_Type)) -PyAPI_FUNC(int) CpFloatAtom_Pack(CpFloatAtomObject* self, - PyObject* value, - CpLayerObject* layer); - -PyAPI_FUNC(PyObject*) - CpFloatAtom_Unpack(CpFloatAtomObject* self, CpLayerObject* layer); - #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/int.h b/src/caterpillar/include/caterpillar/atoms/int.h index b5a79f9..8a41599 100644 --- a/src/caterpillar/include/caterpillar/atoms/int.h +++ b/src/caterpillar/include/caterpillar/atoms/int.h @@ -40,17 +40,11 @@ struct _intatomobj int _m_little_endian; }; -/// Integer atom object type -// PyAPI_DATA(PyTypeObject) CpIntAtom_Type; +#define CpIntAtom_NAME "int_t" /** @brief Checks if the given object is an integer atom object */ #define CpIntAtom_CheckExact(op) Py_IS_TYPE((op), &CpIntAtom_Type) /** @brief Checks if the given object is an integer atom object */ #define CpIntAtom_Check(op) (PyObject_IsInstance((op), &CpIntAtom_Type)) -PyAPI_FUNC(int) - CpIntAtom_Pack(CpIntAtomObject* self, PyObject* value, CpLayerObject* layer); - -PyAPI_FUNC(PyObject*) - CpIntAtom_Unpack(CpIntAtomObject* self, CpLayerObject* layer); #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/primitive.h b/src/caterpillar/include/caterpillar/atoms/primitive.h index f5c5668..a7ad399 100644 --- a/src/caterpillar/include/caterpillar/atoms/primitive.h +++ b/src/caterpillar/include/caterpillar/atoms/primitive.h @@ -35,6 +35,7 @@ struct _boolatomobj /// This variable defines the type object for bool atom objects, allowing /// them to be used across the library. // PyAPI_DATA(PyTypeObject) CpBoolAtom_Type; +#define CpBoolAtom_NAME "bool_t" /** * @brief Checks if the given object is a bool atom object. @@ -60,28 +61,6 @@ struct _boolatomobj */ #define CpBoolAtom_Check(op) (PyObject_IsInstance((op), &CpBoolAtom_Type)) -/** - * @brief Packs a value into the underlying stream. - * - * @param self The bool atom object - * @param value The boolean value to be packed. - * @param layer The layer object providing the context for packing. - * @return Integer status code (typically 0 for success, non-zero for error). - */ -PyAPI_FUNC(int) CpBoolAtom_Pack(CpBoolAtomObject* self, - PyObject* value, - CpLayerObject* layer); - -/** - * @brief Unpacks a value from the underlying stream. - * - * @param self The bool atom object instance. - * @param layer The layer object providing the context for unpacking. - * @return The unpacked boolean value as a PyObject. - */ -PyAPI_FUNC(PyObject*) - CpBoolAtom_Unpack(CpBoolAtomObject* self, CpLayerObject* layer); - //------------------------------------------------------------------------------ // Char Atom struct _charatomobj @@ -91,19 +70,13 @@ struct _charatomobj /// Char atom object type // PyAPI_DATA(PyTypeObject) CpCharAtom_Type; +#define CpCharAtom_NAME "char_t" /** @brief Checks if the given object is a char atom object */ #define CpCharAtom_CheckExact(op) Py_IS_TYPE((op), &CpCharAtom_Type) /** @brief Checks if the given object is a char atom object */ #define CpCharAtom_Check(op) (PyObject_IsInstance((op), &CpCharAtom_Type)) -PyAPI_FUNC(int) CpCharAtom_Pack(CpCharAtomObject* self, - PyObject* value, - CpLayerObject* layer); - -PyAPI_FUNC(PyObject*) - CpCharAtom_Unpack(CpCharAtomObject* self, CpLayerObject* layer); - //------------------------------------------------------------------------------ // Padding struct _paddingatomobj @@ -115,24 +88,11 @@ struct _paddingatomobj /// Padding atom object type // PyAPI_DATA(PyTypeObject) CpPaddingAtom_Type; +#define CpPaddingAtom_NAME "padding_t" /** @brief Checks if the given object is a padding atom object */ #define CpPaddingAtom_CheckExact(op) Py_IS_TYPE((op), &CpPaddingAtom_Type) /** @brief Checks if the given object is a padding atom object */ #define CpPaddingAtom_Check(op) (PyObject_IsInstance((op), &CpPaddingAtom_Type)) -PyAPI_FUNC(int) CpPaddingAtom_Pack(CpPaddingAtomObject* self, - PyObject* _, - CpLayerObject* layer); - -PyAPI_FUNC(int) CpPaddingAtom_PackMany(CpPaddingAtomObject* self, - PyObject* _, - CpLayerObject* layer); - -PyAPI_FUNC(PyObject*) - CpPaddingAtom_Unpack(CpPaddingAtomObject* self, CpLayerObject* layer); - -PyAPI_FUNC(PyObject*) - CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, CpLayerObject* layer); - #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/string.h b/src/caterpillar/include/caterpillar/atoms/string.h index 6b9ea98..ce6708a 100644 --- a/src/caterpillar/include/caterpillar/atoms/string.h +++ b/src/caterpillar/include/caterpillar/atoms/string.h @@ -30,15 +30,9 @@ struct _stringatomobj }; // PyAPI_DATA(PyTypeObject) CpStringAtom_Type; +#define CpStringAtom_NAME "string" #define CpStringAtom_CheckExact(op) Py_IS_TYPE((op), &CpStringAtom_Type) #define CpStringAtom_Check(op) PyObject_TypeCheck((op), &CpStringAtom_Type) -PyAPI_FUNC(int) CpStringAtom_Pack(CpStringAtomObject* self, - PyObject* value, - CpLayerObject* layer); - -PyAPI_FUNC(PyObject*) - CpStringAtom_Unpack(CpStringAtomObject* self, CpLayerObject* layer); - #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index 06d440d..58baade 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -162,6 +162,20 @@ int CpStruct_ReplaceType(CpStructObject* o, PyObject* name, PyObject* type); int CpStruct_HasOption(CpStructObject* o, PyObject* option); int CpStructModel_Check(PyObject* model, _modulestate* state); PyObject* CpStructModel_GetStruct(PyObject* model, _modulestate* state); +int CpIntAtom_Pack(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer); +PyObject* CpIntAtom_Unpack(CpIntAtomObject* self, CpLayerObject* layer); +int CpFloatAtom_Pack(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer); +PyObject* CpFloatAtom_Unpack(CpFloatAtomObject* self, CpLayerObject* layer); +int CpBoolAtom_Pack(CpBoolAtomObject* self, PyObject* value, CpLayerObject* layer); +PyObject* CpBoolAtom_Unpack(CpBoolAtomObject* self, CpLayerObject* layer); +int CpCharAtom_Pack(CpCharAtomObject* self, PyObject* value, CpLayerObject* layer); +PyObject* CpCharAtom_Unpack(CpCharAtomObject* self, CpLayerObject* layer); +int CpPaddingAtom_Pack(CpPaddingAtomObject* self,PyObject* value,CpLayerObject* layer); +int CpPaddingAtom_PackMany(CpPaddingAtomObject* self,PyObject* value,CpLayerObject* layer); +PyObject* CpPaddingAtom_Unpack(CpPaddingAtomObject* self, CpLayerObject* layer); +PyObject* CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, CpLayerObject* layer); +int CpStringAtom_Pack(CpStringAtomObject* self,PyObject* value,CpLayerObject* layer); +PyObject* CpStringAtom_Unpack(CpStringAtomObject* self, CpLayerObject* layer); #else @@ -253,6 +267,20 @@ caterpillar_api.py #define CpStruct_HasOption (*((int (*)(CpStructObject* o, PyObject* option)))Cp_API[104]) #define CpStructModel_Check (*((int (*)(PyObject* model, _modulestate* state)))Cp_API[105]) #define CpStructModel_GetStruct (*((PyObject* (*)(PyObject* model, _modulestate* state)))Cp_API[106]) +#define CpIntAtom_Pack (*((int (*)(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer)))Cp_API[120]) +#define CpIntAtom_Unpack (*((PyObject* (*)(CpIntAtomObject* self, CpLayerObject* layer)))Cp_API[121]) +#define CpFloatAtom_Pack (*((int (*)(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[122]) +#define CpFloatAtom_Unpack (*((PyObject* (*)(CpFloatAtomObject* self, CpLayerObject* layer)))Cp_API[123]) +#define CpBoolAtom_Pack (*((int (*)(CpBoolAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[124]) +#define CpBoolAtom_Unpack (*((PyObject* (*)(CpBoolAtomObject* self, CpLayerObject* layer)))Cp_API[125]) +#define CpCharAtom_Pack (*((int (*)(CpCharAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[126]) +#define CpCharAtom_Unpack (*((PyObject* (*)(CpCharAtomObject* self, CpLayerObject* layer)))Cp_API[127]) +#define CpPaddingAtom_Pack (*((int (*)(CpPaddingAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[128]) +#define CpPaddingAtom_PackMany (*((int (*)(CpPaddingAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[129]) +#define CpPaddingAtom_Unpack (*((PyObject* (*)(CpPaddingAtomObject* self, CpLayerObject* layer)))Cp_API[130]) +#define CpPaddingAtom_UnpackMany (*((PyObject* (*)(CpPaddingAtomObject* self, CpLayerObject* layer)))Cp_API[131]) +#define CpStringAtom_Pack (*((int (*)(CpStringAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[132]) +#define CpStringAtom_Unpack (*((PyObject* (*)(CpStringAtomObject* self, CpLayerObject* layer)))Cp_API[133]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillar/include/caterpillar/macros.h b/src/caterpillar/include/caterpillar/macros.h index 11cb4ca..c3fb6bd 100644 --- a/src/caterpillar/include/caterpillar/macros.h +++ b/src/caterpillar/include/caterpillar/macros.h @@ -23,6 +23,8 @@ #include +#define _Cp_NameStr(x) ("caterpillar._C." x) + /** * @brief Appends the name of the C module to the given name. */ diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index 5af36dd..2445e76 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -109,6 +109,33 @@ void *Cp_API[] = { (void *) &CpStruct_ReplaceType, (void *) &CpStruct_HasOption, (void *) &CpStructModel_Check, - (void *) &CpStructModel_GetStruct + (void *) &CpStructModel_GetStruct, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + (void *) &CpIntAtom_Pack, + (void *) &CpIntAtom_Unpack, + (void *) &CpFloatAtom_Pack, + (void *) &CpFloatAtom_Unpack, + (void *) &CpBoolAtom_Pack, + (void *) &CpBoolAtom_Unpack, + (void *) &CpCharAtom_Pack, + (void *) &CpCharAtom_Unpack, + (void *) &CpPaddingAtom_Pack, + (void *) &CpPaddingAtom_PackMany, + (void *) &CpPaddingAtom_Unpack, + (void *) &CpPaddingAtom_UnpackMany, + (void *) &CpStringAtom_Pack, + (void *) &CpStringAtom_Unpack }; diff --git a/src/code_gen/caterpillar_api.py b/src/code_gen/caterpillar_api.py index e335fe5..9af18dd 100644 --- a/src/code_gen/caterpillar_api.py +++ b/src/code_gen/caterpillar_api.py @@ -133,7 +133,21 @@ "CpStructModel_Check": 105, "CpStructModel_GetStruct": 106, - + # atom api + "CpIntAtom_Pack": 120, + "CpIntAtom_Unpack": 121, + "CpFloatAtom_Pack": 122, + "CpFloatAtom_Unpack": 123, + "CpBoolAtom_Pack": 124, + "CpBoolAtom_Unpack": 125, + "CpCharAtom_Pack": 126, + "CpCharAtom_Unpack": 127, + "CpPaddingAtom_Pack": 128, + "CpPaddingAtom_PackMany": 129, + "CpPaddingAtom_Unpack": 130, + "CpPaddingAtom_UnpackMany": 131, + "CpStringAtom_Pack": 132, + "CpStringAtom_Unpack": 133, } API_SRC = [ diff --git a/src/module.c b/src/module.c index 5502121..579ac2a 100644 --- a/src/module.c +++ b/src/module.c @@ -3,8 +3,7 @@ #define _CPMODULE #endif -#include "caterpillar/module.h" /* invalid default object and types through caterpillar.h*/ -#include "caterpillar/parsing.h" /* Pack and unpack functions */ +#include "caterpillar/caterpillar.h" #include "caterpillarapi.c" @@ -432,12 +431,12 @@ PyInit__C(void) CpModule_AddObject("fieldinfo", &CpStructFieldInfo_Type); CpModule_AddObject("Struct", &CpStruct_Type); - CpModule_AddObject("intatom", &CpIntAtom_Type); - CpModule_AddObject("floatatom", &CpFloatAtom_Type); - CpModule_AddObject("boolatom", &CpBoolAtom_Type); - CpModule_AddObject("charatom", &CpCharAtom_Type); - CpModule_AddObject("paddingatom", &CpPaddingAtom_Type); - CpModule_AddObject("string", &CpStringAtom_Type); + CpModule_AddObject(CpIntAtom_NAME, &CpIntAtom_Type); + CpModule_AddObject(CpFloatAtom_NAME, &CpFloatAtom_Type); + CpModule_AddObject(CpBoolAtom_NAME, &CpBoolAtom_Type); + CpModule_AddObject(CpCharAtom_NAME, &CpCharAtom_Type); + CpModule_AddObject(CpPaddingAtom_NAME, &CpPaddingAtom_Type); + CpModule_AddObject(CpStringAtom_NAME, &CpStringAtom_Type); /* setup custom intatoms */ #define CpModule_DefAtom(name, ...) \ @@ -463,6 +462,8 @@ PyInit__C(void) CpModule_DefIntAtom("u32", 32, false); CpModule_DefIntAtom("i64", 64, true); CpModule_DefIntAtom("u64", 64, false); + CpModule_DefIntAtom("i128", 128, true); + CpModule_DefIntAtom("u128", 128, false); #define CpModule_DefFloatAtom(name, bits) \ CpModule_DefAtom(name, CpObject_Create(&CpFloatAtom_Type, "I", bits)); diff --git a/test/_C/atoms/test_padding.py b/test/_C/atoms/test_padding.py index 955b710..7b3038c 100644 --- a/test/_C/atoms/test_padding.py +++ b/test/_C/atoms/test_padding.py @@ -5,7 +5,7 @@ if caterpillar.native_support(): - from caterpillar._C import unpack, pack, padding, paddingatom + from caterpillar._C import unpack, pack, padding, padding_t def test_padding_pack(): # The padding atom is special, because it does not need @@ -14,13 +14,13 @@ def test_padding_pack(): # In addition, we can define custom padding values, # which will affect only packing data. - assert pack(None, paddingatom(0x20)) == b"\x20" + assert pack(None, padding_t(0x20)) == b"\x20" def test_padding_pack_many(): # The padding atom also implements __pack_many__ and # __unpack_many__. (Time measurements are work in progress). assert pack(None, padding[10]) == b"\x00" * 10 - assert pack(None, paddingatom(0x20)[10]) == b"\x20" * 10 + assert pack(None, padding_t(0x20)[10]) == b"\x20" * 10 # REVISIT: this should return an exception # with pytest.raises(ValueError): @@ -30,12 +30,12 @@ def test_padding_unpack(): # NOTE: unpack using the padding atom always returns None # and DOES NOT validate the parsed value. assert unpack(b"\x00", padding) is None - assert unpack(b"\x20", paddingatom(0x20)) is None + assert unpack(b"\x20", padding_t(0x20)) is None # Therefore, the following code IS valid if the underlying # stream does not throw an exception. - assert unpack(b"\x00" * 10, paddingatom(0x02)[10]) is None + assert unpack(b"\x00" * 10, padding_t(0x02)[10]) is None def test_padding_unpack_many(): assert unpack(b"\x00" * 10, padding[10]) is None - assert unpack(b"\x20" * 10, paddingatom(0x20)[10]) is None + assert unpack(b"\x20" * 10, padding_t(0x20)[10]) is None diff --git a/test/_Py/test_flag.py b/test/_Py/test_flag.py new file mode 100644 index 0000000..ed2839d --- /dev/null +++ b/test/_Py/test_flag.py @@ -0,0 +1,4 @@ + +import pytest + +from caterpillar._Py import Flag \ No newline at end of file From fc8f68baa262b0aaca5eb7f33c690d50e2779703 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sat, 7 Sep 2024 13:51:31 +0200 Subject: [PATCH 02/29] Renamed _Py to py and added c importer module --- README.md | 2 +- docs/sphinx/source/development/roadmap.rst | 42 ++++++++++++++++------ examples/bitfield_example.py | 2 +- examples/greedy_example.py | 2 +- examples/inheritance_example.py | 2 +- examples/inner_struct.py | 2 +- examples/operator_example.py | 2 +- examples/pointer_example.py | 2 +- examples/template_example.py | 2 +- examples/union_example.py | 2 +- examples/varint_example.py | 2 +- src/caterpillar/c.py | 23 ++++++++++++ src/caterpillar/{_Py.py => py.py} | 0 test/_C/test_struct.py | 10 +++++- test/_Py/test_flag.py | 11 +++++- 15 files changed, 83 insertions(+), 23 deletions(-) create mode 100644 src/caterpillar/c.py rename src/caterpillar/{_Py.py => py.py} (100%) diff --git a/README.md b/README.md index dc1eb65..53dc400 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ options will be added in the future. Documentation is [here >](https://matrixedi ## What does it look like? ```python -from caterpillar._Py import * +from caterpillar.py import * @struct(order=LittleEndian) class Format: diff --git a/docs/sphinx/source/development/roadmap.rst b/docs/sphinx/source/development/roadmap.rst index 0ee99d5..465bc83 100644 --- a/docs/sphinx/source/development/roadmap.rst +++ b/docs/sphinx/source/development/roadmap.rst @@ -12,21 +12,41 @@ Roadmap -C API ------ +.. role:: text-danger + +Python API +---------- -|check_| Implementation of parsing process (unpack, pack) +- |check_| Implementation of parsing process (unpack, pack) +- |check_| Struct class (:class:`Struct`) with wrapper function (:code:`@struct`) -|check_| Struct class (:c:type:`CpStructObject`) -|uncheck_| Struct wrapper function +C API +----- + +- |check_| Implementation of parsing process (unpack, pack) +- |check_| Struct class (:c:type:`CpStructObject`) +- |uncheck_| Struct wrapper function +- |uncheck_| Python docs Atom Objects: ^^^^^^^^^^^^^ -- |check_| Integer (uint8-128 and int8-128) (C type: :c:type:`CpIntAtomObject`, Py type: :class:`int_t`) -- |check_| Float, Double (C type: :c:type:`CpFloatAtomObject`, Py type: :class:`float_t`) -- |check_| Boolean (C type: :c:type:`CpBoolAtomObject`, Py type: :class:`bool_t`), global instance: :code:`boolean` -- |check_| Char (C type: :c:type:`CpCharAtomObject`, Py type: :class:`char_t`), global instance: :code:`char` -- |check_| Padding (C type: :c:type:`CpPaddingAtomObject`, Py type: :class:`padding_t`), global instance: :code:`padding` -- |check_| String (C type: :c:type:`CpStringAtomObject`, Py type: :class:`string`) \ No newline at end of file +- |check_| Integer (uint8-128 and int8-128) (C type: :c:type:`CpIntAtomObject`, Py type: :class:`int_t`) [:text-danger:`missing docs`] +- |check_| Float, Double (C type: :c:type:`CpFloatAtomObject`, Py type: :class:`float_t`) [:text-danger:`missing docs`] +- |check_| Boolean (C type: :c:type:`CpBoolAtomObject`, Py type: :class:`bool_t`), global instance: :code:`boolean` [:text-danger:`missing docs`] +- |check_| Char (C type: :c:type:`CpCharAtomObject`, Py type: :class:`char_t`), global instance: :code:`char` [:text-danger:`missing docs`] +- |check_| Padding (C type: :c:type:`CpPaddingAtomObject`, Py type: :class:`padding_t`), global instance: :code:`padding` [:text-danger:`missing docs`] +- |check_| String (C type: :c:type:`CpStringAtomObject`, Py type: :class:`string`) [:text-danger:`missing docs`] +- |uncheck_| Const +- |uncheck_| CString +- |uncheck_| Bytes +- |uncheck_| Enum +- |uncheck_| Computed +- |uncheck_| PString +- |uncheck_| Prefixed +- |uncheck_| Lazy +- |uncheck_| uuid +- |uncheck_| Conditional: If, Else, ElseIf +- |uncheck_| VarInt +- |uncheck_| While \ No newline at end of file diff --git a/examples/bitfield_example.py b/examples/bitfield_example.py index d9fc6b7..039feb0 100644 --- a/examples/bitfield_example.py +++ b/examples/bitfield_example.py @@ -1,5 +1,5 @@ # type: ignore -from caterpillar._Py import bitfield, char, int8, unpack, pack +from caterpillar.py import bitfield, char, int8, unpack, pack try: from rich import print diff --git a/examples/greedy_example.py b/examples/greedy_example.py index e194406..4b381cb 100644 --- a/examples/greedy_example.py +++ b/examples/greedy_example.py @@ -1,5 +1,5 @@ # type: ignore -from caterpillar._Py import struct, BigEndian, this, unpack, uint8 +from caterpillar.py import struct, BigEndian, this, unpack, uint8 try: from rich import print diff --git a/examples/inheritance_example.py b/examples/inheritance_example.py index 2715b59..0b6f2b2 100644 --- a/examples/inheritance_example.py +++ b/examples/inheritance_example.py @@ -1,5 +1,5 @@ # type: ignore -from caterpillar._Py import struct, pack, unpack, this, uint8, int32, Bytes +from caterpillar.py import struct, pack, unpack, this, uint8, int32, Bytes try: from rich import print diff --git a/examples/inner_struct.py b/examples/inner_struct.py index 9bd2993..270fe3d 100644 --- a/examples/inner_struct.py +++ b/examples/inner_struct.py @@ -1,5 +1,5 @@ # type: ignore -from caterpillar._Py import struct, pack, unpack, uint8 +from caterpillar.py import struct, pack, unpack, uint8 try: from rich import print diff --git a/examples/operator_example.py b/examples/operator_example.py index a031452..c999d8d 100644 --- a/examples/operator_example.py +++ b/examples/operator_example.py @@ -1,5 +1,5 @@ # type: ignore -from caterpillar._Py import struct, uint16, Operator, S_REPLACE_TYPES +from caterpillar.py import struct, uint16, Operator, S_REPLACE_TYPES # Here, we define a custom operator named 'M' that will multiply # the second argument by 2. diff --git a/examples/pointer_example.py b/examples/pointer_example.py index eef81df..f9d3237 100644 --- a/examples/pointer_example.py +++ b/examples/pointer_example.py @@ -1,5 +1,5 @@ # type: ignore -from caterpillar._Py import ( +from caterpillar.py import ( set_struct_flags, uintptr, CString, diff --git a/examples/template_example.py b/examples/template_example.py index 52e0c17..9fb0b49 100644 --- a/examples/template_example.py +++ b/examples/template_example.py @@ -1,5 +1,5 @@ # type: ignore -from caterpillar._Py import ( +from caterpillar.py import ( struct, set_struct_flags, S_REPLACE_TYPES, diff --git a/examples/union_example.py b/examples/union_example.py index 12ad58d..aa56c55 100644 --- a/examples/union_example.py +++ b/examples/union_example.py @@ -1,5 +1,5 @@ # type: ignore -from caterpillar._Py import uint16, uint32, boolean, union +from caterpillar.py import uint16, uint32, boolean, union try: from rich import print diff --git a/examples/varint_example.py b/examples/varint_example.py index 78b6796..d8ad9e8 100644 --- a/examples/varint_example.py +++ b/examples/varint_example.py @@ -1,5 +1,5 @@ # type: ignore -from caterpillar._Py import ( +from caterpillar.py import ( vint, VARINT_LSB, pack, diff --git a/src/caterpillar/c.py b/src/caterpillar/c.py new file mode 100644 index 0000000..8d79d74 --- /dev/null +++ b/src/caterpillar/c.py @@ -0,0 +1,23 @@ +from caterpillar._C import * # noqa + + +def struct(cls: type = None, /, **kwargs) -> type: + """Decorator for new struct classes using the C API. + + >>> @struct + ... class MyStruct: + ... pass + + :param cls: The target class used as the base model. + :param options: Additional options specifying what to include in the final class. + :param endian: Optional configuration value for the byte order of a field. + :param arch: Global architecture definition (will be inferred on all fields). + :param field_options: Additional field options. + """ + def wrap(cls) -> type: + return Struct(cls, alter_model=True, **kwargs).model + + if cls is not None: + return Struct(cls, alter_model=True, **kwargs).model + + return wrap diff --git a/src/caterpillar/_Py.py b/src/caterpillar/py.py similarity index 100% rename from src/caterpillar/_Py.py rename to src/caterpillar/py.py diff --git a/test/_C/test_struct.py b/test/_C/test_struct.py index f699e30..0800e9f 100644 --- a/test/_C/test_struct.py +++ b/test/_C/test_struct.py @@ -4,7 +4,7 @@ if caterpillar.native_support(): - from caterpillar._C import atom, Struct + from caterpillar.c import atom, Struct, struct def test_struct_init(): @@ -51,3 +51,11 @@ class Foo: # value for 'a'. assert Foo().a == 1 assert Foo(a=2).a == 2 + + def test_struct_decorator(): + @struct + class Foo: + a: atom() = 1 + + assert len(Foo.__struct__.members) == 1 + assert Foo().a == 1 diff --git a/test/_Py/test_flag.py b/test/_Py/test_flag.py index ed2839d..29c38df 100644 --- a/test/_Py/test_flag.py +++ b/test/_Py/test_flag.py @@ -1,4 +1,13 @@ import pytest -from caterpillar._Py import Flag \ No newline at end of file +from caterpillar.py import Flag + +def test_flag_value(): + # Each flag stores a value which can be used + # to configure fields and structs. + flag = Flag("test1", None) + assert flag.value is None + + flag.value = 1 + assert flag.value == 1 From fac0cd30aa7a5d13eb8b5e3bf8bd52a45e08ec39 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sat, 7 Sep 2024 17:06:15 +0200 Subject: [PATCH 03/29] ConsAtom (C) --- CMakeLists.txt | 1 + README.md | 2 +- src/atomimpl/constatomobj.c | 98 +++++++++++++++++++ src/atomimpl/intatomobj.c | 5 +- .../include/caterpillar/atoms/const.h | 35 +++++++ .../include/caterpillar/caterpillar.h | 1 + .../include/caterpillar/caterpillarapi.h | 8 ++ src/caterpillarapi.c | 6 +- src/code_gen/caterpillar_api.py | 5 + src/module.c | 4 + src/parsing_pack.c | 3 - 11 files changed, 160 insertions(+), 8 deletions(-) create mode 100644 src/atomimpl/constatomobj.c create mode 100644 src/caterpillar/include/caterpillar/atoms/const.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a80ac3..3189ab3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ python_add_library( src/atomimpl/charatomobj.c src/atomimpl/padatomobj.c src/atomimpl/stringatomobj.c + src/atomimpl/constatomobj.c WITH_SOABI ) diff --git a/README.md b/README.md index 53dc400..bf68ee5 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ options will be added in the future. Documentation is [here >](https://matrixedi * it helps you to create cleaner and more compact code. * You can even extend Caterpillar and write your parsing logic in C or C++!! -## What does it look like? +## Give me some code! ```python from caterpillar.py import * diff --git a/src/atomimpl/constatomobj.c b/src/atomimpl/constatomobj.c new file mode 100644 index 0000000..26c3a9c --- /dev/null +++ b/src/atomimpl/constatomobj.c @@ -0,0 +1,98 @@ +/* constatom C implementation */ +#include "caterpillar/caterpillar.h" + +#include + +static PyObject* +cp_constatom__type__(CpConstAtomObject* self) +{ + return CpTypeOf(self->m_atom); +} + +static PyObject* +cp_constatom__size__(CpConstAtomObject* self, CpLayerObject* layer) +{ + return _Cp_SizeOf(self->m_atom, layer); +} + +static PyObject* +cp_constatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) +{ + CpConstAtomObject* self = (CpConstAtomObject*)type->tp_alloc(type, 0); + if (self != NULL) { + CpFieldCAtom_CATOM(self).ob_pack = (packfunc)CpConstAtom_Pack; + CpFieldCAtom_CATOM(self).ob_unpack = (unpackfunc)CpConstAtom_Unpack; + CpFieldCAtom_CATOM(self).ob_pack_many = NULL; + CpFieldCAtom_CATOM(self).ob_unpack_many = NULL; + CpFieldCAtom_CATOM(self).ob_size = (sizefunc)cp_constatom__size__; + CpFieldCAtom_CATOM(self).ob_type = (typefunc)cp_constatom__type__; + CpFieldCAtom_CATOM(self).ob_bits = NULL; + + self->m_atom = NULL; + self->m_value = NULL; + } + return (PyObject*)self; +} + +static void +cp_constatom_dealloc(CpConstAtomObject* self) +{ + Py_XDECREF(self->m_atom); + Py_XDECREF(self->m_value); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_constatom_init(CpConstAtomObject* self, PyObject* args, PyObject* kwds) +{ + static char* kwlist[] = { "atom", "value", NULL }; + PyObject* atom = NULL; + PyObject* value = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &atom, &value)) { + return -1; + } + _Cp_SetObj(self->m_atom, atom); + _Cp_SetObj(self->m_value, value); + return 0; +} + +/* Public API */ + +/*CpAPI*/ +int +CpConstAtom_Pack(CpConstAtomObject* self, PyObject* value, CpLayerObject* layer) +{ + return _Cp_Pack(self->m_value, self->m_atom, layer); +} + +/*CpAPI*/ +PyObject* +CpConstAtom_Unpack(CpConstAtomObject* self, CpLayerObject* layer) +{ + PyObject* res = _Cp_Unpack(self->m_atom, layer); + if (!res) { + return NULL; + } + + if (PyObject_RichCompareBool(res, self->m_value, Py_NE)) { + PyErr_Format(PyExc_ValueError, + "Value is not equal to constant (parsed=%R, constant=%R)", + res, + self->m_value); + Py_DECREF(res); + return NULL; + } + // REVISIT: Should we free the 'res' object here? + return res; +} + +/* type setup */ + +PyTypeObject CpConstAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpConstAtom_NAME), + .tp_basicsize = sizeof(CpConstAtomObject), + .tp_dealloc = (destructor)cp_constatom_dealloc, + .tp_init = (initproc)cp_constatom_init, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = (newfunc)cp_constatom_new, +}; \ No newline at end of file diff --git a/src/atomimpl/intatomobj.c b/src/atomimpl/intatomobj.c index 352654c..8a12e14 100644 --- a/src/atomimpl/intatomobj.c +++ b/src/atomimpl/intatomobj.c @@ -50,7 +50,7 @@ cp_intatom_init(CpIntAtomObject* self, PyObject* args, PyObject* kwds) { static char* kwlist[] = { "nbits", "signed", "little_endian", NULL }; int _signed = true, little_endian = true; - size_t bits = 0; + unsigned int bits = 0; if (!PyArg_ParseTupleAndKeywords( args, kwds, "I|pp", kwlist, &bits, &_signed, &little_endian)) { return -1; @@ -101,7 +101,7 @@ CpIntAtom_Pack(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer) } int res = _PyLong_AsByteArray((PyLongObject*)op, - (unsigned char*)PyBytes_AS_STRING(op), + (unsigned char*)PyBytes_AS_STRING(bytes), self->_m_byte_count, little_endian, self->_m_signed); @@ -168,4 +168,5 @@ PyTypeObject CpIntAtom_Type = { .tp_doc = NULL, .tp_members = CpIntAtom_Members, .tp_new = (newfunc)cp_intatom_new, + .tp_init = (initproc)cp_intatom_init, }; \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/const.h b/src/caterpillar/include/caterpillar/atoms/const.h new file mode 100644 index 0000000..35c3243 --- /dev/null +++ b/src/caterpillar/include/caterpillar/atoms/const.h @@ -0,0 +1,35 @@ +/** + * Copyright (C) MatrixEditor 2024 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef CONSTATOMOBJ_H +#define CONSTATOMOBJ_H + +#include "caterpillar/caterpillarapi.h" +#include "caterpillar/field.h" + +struct _constatomobj +{ + CpFieldCAtom_HEAD + + PyObject *m_value; + PyObject *m_atom; +}; + +#define CpConstAtom_NAME "const_t" +#define CpConstAtom_CheckExact(op) Py_IS_TYPE((op), &CpConstAtom_Type) +#define CpConstAtom_Check(op) PyObject_TypeCheck((op), &CpConstAtom_Type) + +#endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillar.h b/src/caterpillar/include/caterpillar/caterpillar.h index c8923c2..0a2b35d 100644 --- a/src/caterpillar/include/caterpillar/caterpillar.h +++ b/src/caterpillar/include/caterpillar/caterpillar.h @@ -19,5 +19,6 @@ #include "caterpillar/atoms/float.h" #include "caterpillar/atoms/primitive.h" #include "caterpillar/atoms/string.h" +#include "caterpillar/atoms/const.h" #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index 58baade..5da33d4 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -72,6 +72,8 @@ struct _paddingatomobj; typedef struct _paddingatomobj CpPaddingAtomObject; struct _stringatomobj; typedef struct _stringatomobj CpStringAtomObject; +struct _constatomobj; +typedef struct _constatomobj CpConstAtomObject; #ifdef _CPMODULE @@ -107,6 +109,7 @@ extern PyTypeObject CpBoolAtom_Type; extern PyTypeObject CpCharAtom_Type; extern PyTypeObject CpPaddingAtom_Type; extern PyTypeObject CpStringAtom_Type; +extern PyTypeObject CpConstAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -176,6 +179,8 @@ PyObject* CpPaddingAtom_Unpack(CpPaddingAtomObject* self, CpLayerObject* layer); PyObject* CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, CpLayerObject* layer); int CpStringAtom_Pack(CpStringAtomObject* self,PyObject* value,CpLayerObject* layer); PyObject* CpStringAtom_Unpack(CpStringAtomObject* self, CpLayerObject* layer); +int CpConstAtom_Pack(CpConstAtomObject* self, PyObject* value, CpLayerObject* layer); +PyObject* CpConstAtom_Unpack(CpConstAtomObject* self, CpLayerObject* layer); #else @@ -212,6 +217,7 @@ caterpillar_api.py #define CpCharAtom_Type (*(PyTypeObject *)Cp_API[24]) #define CpPaddingAtom_Type (*(PyTypeObject *)Cp_API[25]) #define CpStringAtom_Type (*(PyTypeObject *)Cp_API[26]) +#define CpConstAtom_Type (*(PyTypeObject *)Cp_API[27]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -281,6 +287,8 @@ caterpillar_api.py #define CpPaddingAtom_UnpackMany (*((PyObject* (*)(CpPaddingAtomObject* self, CpLayerObject* layer)))Cp_API[131]) #define CpStringAtom_Pack (*((int (*)(CpStringAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[132]) #define CpStringAtom_Unpack (*((PyObject* (*)(CpStringAtomObject* self, CpLayerObject* layer)))Cp_API[133]) +#define CpConstAtom_Pack (*((int (*)(CpConstAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[134]) +#define CpConstAtom_Unpack (*((PyObject* (*)(CpConstAtomObject* self, CpLayerObject* layer)))Cp_API[135]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index 2445e76..15bcbab 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -30,7 +30,7 @@ void *Cp_API[] = { (void *) &CpCharAtom_Type, (void *) &CpPaddingAtom_Type, (void *) &CpStringAtom_Type, - NULL, + (void *) &CpConstAtom_Type, NULL, NULL, NULL, @@ -136,6 +136,8 @@ void *Cp_API[] = { (void *) &CpPaddingAtom_Unpack, (void *) &CpPaddingAtom_UnpackMany, (void *) &CpStringAtom_Pack, - (void *) &CpStringAtom_Unpack + (void *) &CpStringAtom_Unpack, + (void *) &CpConstAtom_Pack, + (void *) &CpConstAtom_Unpack }; diff --git a/src/code_gen/caterpillar_api.py b/src/code_gen/caterpillar_api.py index 9af18dd..db0f3b0 100644 --- a/src/code_gen/caterpillar_api.py +++ b/src/code_gen/caterpillar_api.py @@ -41,6 +41,7 @@ "_charatomobj": "CpCharAtomObject", "_paddingatomobj": "CpPaddingAtomObject", "_stringatomobj": "CpStringAtomObject", + "_constatomobj": "CpConstAtomObject", } cp_type_api = { @@ -71,6 +72,7 @@ "CpCharAtom_Type": (24,), "CpPaddingAtom_Type": (25,), "CpStringAtom_Type": (26,), + "CpConstAtom_Type": (27,), } cp_func_api = { @@ -148,6 +150,8 @@ "CpPaddingAtom_UnpackMany": 131, "CpStringAtom_Pack": 132, "CpStringAtom_Unpack": 133, + "CpConstAtom_Pack": 134, + "CpConstAtom_Unpack": 135, } API_SRC = [ @@ -168,6 +172,7 @@ "atomimpl/intatomobj.c", "atomimpl/padatomobj.c", "atomimpl/stringatomobj.c", + "atomimpl/constatomobj.c", ] diff --git a/src/module.c b/src/module.c index 579ac2a..9631c46 100644 --- a/src/module.c +++ b/src/module.c @@ -404,6 +404,9 @@ PyInit__C(void) CpStringAtom_Type.tp_base = &CpFieldCAtom_Type; CpModule_SetupType(&CpStringAtom_Type); + CpConstAtom_Type.tp_base = &CpFieldCAtom_Type; + CpModule_SetupType(&CpConstAtom_Type); + // module setup m = PyModule_Create(&CpModule); if (!m) { @@ -437,6 +440,7 @@ PyInit__C(void) CpModule_AddObject(CpCharAtom_NAME, &CpCharAtom_Type); CpModule_AddObject(CpPaddingAtom_NAME, &CpPaddingAtom_Type); CpModule_AddObject(CpStringAtom_NAME, &CpStringAtom_Type); + CpModule_AddObject(CpConstAtom_NAME, &CpConstAtom_Type); /* setup custom intatoms */ #define CpModule_DefAtom(name, ...) \ diff --git a/src/parsing_pack.c b/src/parsing_pack.c index cb6a68a..289ca9f 100644 --- a/src/parsing_pack.c +++ b/src/parsing_pack.c @@ -439,9 +439,6 @@ _Cp_Pack(PyObject* op, PyObject* atom, CpLayerObject* layer) success = CpPack_Struct(op, (CpStructObject*)atom, layer); } else if (CpCAtom_Check(atom)) { return CpPack_CAtom(op, (CpCAtomObject*)atom, layer); - // else CASE(&CpCAtom_Type, atom) { - // success = cp_pack_catom - // } } else { success = CpPack_Common(op, atom, layer); } From b261135758e31aa84a8432274bb8c7f499437cd1 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 8 Sep 2024 08:14:36 +0200 Subject: [PATCH 04/29] Changed CpLayer parsing logic --- CMakeLists.txt | 4 + docs/sphinx/source/extensions/refcounts.dat | 5 +- src/atomimpl/builtins/builtinatomobj.c | 63 +++ src/atomimpl/builtins/repeatedatomobj.c | 309 ++++++++++++++ src/atomimpl/floatatomobj.c | 31 +- src/atomimpl/intatomobj.c | 60 +-- src/atomimpl/padatomobj.c | 7 +- src/atomobj.c | 6 + src/caterpillar/include/caterpillar/arch.h | 47 +++ .../include/caterpillar/atoms/builtins.h | 62 +++ .../include/caterpillar/atoms/int.h | 5 +- .../include/caterpillar/caterpillar.h | 2 + .../include/caterpillar/caterpillarapi.h | 34 +- src/caterpillar/include/caterpillar/parsing.h | 50 +++ src/caterpillar/include/caterpillar/state.h | 137 +++++-- src/caterpillarapi.c | 25 +- src/code_gen/caterpillar_api.py | 21 +- src/layer.c | 380 ++++++++++++++++++ src/module.c | 18 +- src/parsing_pack.c | 323 +++++++-------- src/parsing_sizeof.c | 2 +- src/parsing_unpack.c | 38 +- src/state.c | 242 ----------- 23 files changed, 1333 insertions(+), 538 deletions(-) create mode 100644 src/atomimpl/builtins/builtinatomobj.c create mode 100644 src/atomimpl/builtins/repeatedatomobj.c create mode 100644 src/caterpillar/include/caterpillar/atoms/builtins.h create mode 100644 src/caterpillar/include/caterpillar/parsing.h create mode 100644 src/layer.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 3189ab3..157f2a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ python_add_library( src/context.c src/state.c src/struct.c + src/layer.c src/parsing_typeof.c src/parsing_sizeof.c @@ -37,6 +38,9 @@ python_add_library( src/atomimpl/stringatomobj.c src/atomimpl/constatomobj.c + src/atomimpl/builtins/builtinatomobj.c + src/atomimpl/builtins/repeatedatomobj.c + WITH_SOABI ) diff --git a/docs/sphinx/source/extensions/refcounts.dat b/docs/sphinx/source/extensions/refcounts.dat index a95b590..d4bb573 100644 --- a/docs/sphinx/source/extensions/refcounts.dat +++ b/docs/sphinx/source/extensions/refcounts.dat @@ -43,4 +43,7 @@ CpLayer_New:CpLayerObject*:+1 CpIntAtom_Unpack:PyObject*:+1 -CpFloatAtom_Unpack:PyObject*:+1 \ No newline at end of file +CpFloatAtom_Unpack:PyObject*:+1 + +CpRepeatedAtom_GetLength:PyObject*:+1 +CpRepeatedAtom_Unpack:PyObject*:+1 \ No newline at end of file diff --git a/src/atomimpl/builtins/builtinatomobj.c b/src/atomimpl/builtins/builtinatomobj.c new file mode 100644 index 0000000..f45cc2a --- /dev/null +++ b/src/atomimpl/builtins/builtinatomobj.c @@ -0,0 +1,63 @@ +/* builtinatom C implementation */ + +#include "caterpillar/caterpillar.h" + +#include + +/* impl */ +static PyObject* +cp_builtinatomobj_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + CpBuiltinAtomObject* self; + self = (CpBuiltinAtomObject*)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + self->ob_base.ob_bits = NULL; + self->ob_base.ob_pack = NULL; + self->ob_base.ob_pack_many = NULL; + self->ob_base.ob_unpack = NULL; + self->ob_base.ob_unpack_many = NULL; + self->ob_base.ob_size = NULL; + self->ob_base.ob_type = NULL; + return (PyObject*)self; +} + +static void +cp_builtinatomobj_dealloc(CpBuiltinAtomObject* self) +{ + self->ob_base.ob_bits = NULL; + self->ob_base.ob_pack = NULL; + self->ob_base.ob_pack_many = NULL; + self->ob_base.ob_unpack = NULL; + self->ob_base.ob_unpack_many = NULL; + self->ob_base.ob_size = NULL; + self->ob_base.ob_type = NULL; + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_builtinatomobj_init(CpBuiltinAtomObject* self, PyObject* args, PyObject* kw) +{ + _Cp_InitNoArgs(CpBuiltinAtomObject, args, kw); +} + +// TODO member methods +static PyObject* +cp_builtinatomobj_as_mapping_getitem(CpBuiltinAtomObject* self, + PyObject* length) +{ + return (PyObject*)CpRepeatedAtom_New((PyObject*)self, length); +} + +static PyMappingMethods CpFieldAtom_MappingMethods = { + .mp_subscript = (binaryfunc)cp_builtinatomobj_as_mapping_getitem, +}; + +PyTypeObject CpBuiltinAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) "CpBuiltinAtom", // tp_name + .tp_basicsize = sizeof(CpBuiltinAtomObject), + .tp_dealloc = (destructor)cp_builtinatomobj_dealloc, + .tp_init = (initproc)cp_builtinatomobj_init, + .tp_new = (newfunc)cp_builtinatomobj_new, + .tp_as_mapping = &CpFieldAtom_MappingMethods, +}; diff --git a/src/atomimpl/builtins/repeatedatomobj.c b/src/atomimpl/builtins/repeatedatomobj.c new file mode 100644 index 0000000..abcb283 --- /dev/null +++ b/src/atomimpl/builtins/repeatedatomobj.c @@ -0,0 +1,309 @@ +/* repeated atom C implementation */ + +#include "caterpillar/caterpillar.h" + +#include + +/* impl */ + +static PyObject* +cp_repeatedatomobj__type__(CpRepeatedAtomObject* self) +{ + PyObject* atom_type = CpTypeOf(self->m_atom); + if (!atom_type) + return NULL; + + _modulestate* state = get_global_module_state(); + Py_XSETREF(atom_type, PyObject_GetItem(state->List_Type, atom_type)); + return atom_type; +} + +static PyObject* +cp_repeatedatomobj__size__(CpRepeatedAtomObject* self, CpLayerObject* layer) +{ + return _Cp_SizeOf(self->m_atom, layer); +} + +static PyObject* +cp_repeatedatomobj__repr__(CpRepeatedAtomObject* self) +{ + return PyUnicode_FromFormat("", self->m_atom); +} + +static PyObject* +cp_repeatedatomobj_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + CpRepeatedAtomObject* self; + self = (CpRepeatedAtomObject*)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpRepeatedAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpRepeatedAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_repeatedatomobj__size__; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_repeatedatomobj__type__; + return (PyObject*)self; +} + +static void +cp_repeatedatomobj_dealloc(CpRepeatedAtomObject* self) +{ + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = NULL; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = NULL; + CpBuiltinAtom_CATOM(self).ob_type = NULL; + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_repeatedatomobj_init(CpRepeatedAtomObject* self, + PyObject* args, + PyObject* kw) +{ + static char* kwlist[] = { "atom", "length", NULL }; + PyObject *atom = NULL, *length = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kw, "OO", kwlist, &atom, &length)) { + return -1; + } + _Cp_SetObj(self->m_atom, atom); + _Cp_SetObj(self->m_length, length); + return 0; +} + +static PyObject* +cp_repeatedatomobj_set_byteorder(CpRepeatedAtomObject* self, + PyObject* args, + PyObject* kw) +{ + static char* kwlist[] = { "byteorder", NULL }; + PyObject* byteorder = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &byteorder)) { + return NULL; + } + if (!CpEndian_Check(byteorder)) { + PyErr_SetString(PyExc_TypeError, "byteorder must be an Endian object"); + return NULL; + } + + return CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder) ? NULL + : Py_None; +} + +/* Public API */ + +/*CpAPI*/ +PyObject* +CpRepeatedAtom_GetLength(CpRepeatedAtomObject* self, PyObject* context) +{ + if (PyCallable_Check(self->m_length)) { + PyObject* result = PyObject_CallOneArg(self->m_length, context); + if (!result) + return NULL; + return result; + } + return Py_NewRef(self->m_length); +} + +/*CpAPI*/ +int +CpRepeatedAtom_Pack(CpRepeatedAtomObject* self, + PyObject* op, + CpLayerObject* layer) +{ + CpStateObject* state = layer->m_state; + if (PyObject_HasAttr(self->m_atom, state->mod->str___pack_many__)) { + // class explicitly defines __pack_many__ -> use it + PyObject* res = CpAtom_Pack( + self->m_atom, state->mod->str___pack_many__, op, (PyObject*)layer); + PyObject* exc = NULL; + if ((exc = PyErr_GetRaisedException(), + exc && PyErr_GivenExceptionMatches(exc, PyExc_NotImplementedError))) { + // Make sure this method continues to pack the given object + Py_XDECREF(exc); + PyErr_Clear(); + Py_XDECREF(res); + } else { + int success = res ? 0 : -1; + Py_XDECREF(res); + Py_XDECREF(exc); + return success; + } + } + + if (!PySequence_Check(op)) { + PyErr_Format(PyExc_TypeError, "input object (%R) is not a sequence", op); + return -1; + } + + CpSeqLayerObject* seq_layer = CpSeqLayer_New(state, layer); + if (!seq_layer) { + return -1; + } + + Py_ssize_t size = PySequence_Size(op); + bool greedy = false; + Py_ssize_t layer_length = 0; + PyObject* length = CpRepeatedAtom_GetLength(self, (PyObject*)layer); + if (!length) { + goto failure; + } + if (_CpPack_EvalLength(layer, length, size, &greedy, &layer_length) < 0) { + Py_XDECREF(length); + goto failure; + } + Py_XDECREF(length); + + if (layer_length <= 0) { + // continue packing, here's nothing to store + Py_XDECREF(seq_layer); + return 0; + } + + CpSeqLayer_SetSequence(seq_layer, op, layer_length, greedy); + + PyObject* obj = NULL; + int success = 0; + for (seq_layer->m_index = 0; seq_layer->m_index < seq_layer->m_length; + seq_layer->m_index++) { + obj = PySequence_GetItem(op, seq_layer->m_index); + if (!obj) { + goto failure; + } + seq_layer->ob_base.m_path = PyUnicode_FromFormat( + "%s.%d", _PyUnicode_AsString(layer->m_path), seq_layer->m_index); + if (!seq_layer->ob_base.m_path) { + Py_XDECREF(obj); + goto failure; + } + success = _Cp_Pack(obj, self->m_atom, (CpLayerObject*)seq_layer); + Py_XSETREF(obj, NULL); + if (success < 0) { + goto failure; + } + } + CpLayer_Invalidate((CpLayerObject*)seq_layer); + return 0; +failure: + CpLayer_Invalidate((CpLayerObject*)seq_layer); + return -1; +} + +/*CpAPI*/ +PyObject* +CpRepeatedAtom_Unpack(CpRepeatedAtomObject* self, CpLayerObject* layer) +{ + _modulestate* mod = layer->m_state->mod; + if (PyObject_HasAttr(self->m_atom, mod->str___unpack_many__)) { + PyObject* res = PyObject_CallMethodOneArg( + self->m_atom, mod->str___unpack_many__, (PyObject*)layer); + PyObject* exc = NULL; + if ((exc = PyErr_GetRaisedException(), + exc && PyErr_GivenExceptionMatches(exc, PyExc_NotImplementedError))) { + // Make sure this method continues to unpack + Py_XDECREF(exc); + PyErr_Clear(); + Py_XDECREF(res); + } else { + Py_XDECREF(exc); + return res; + } + } + + PyObject *obj = NULL, *length = NULL; + // First, get the amount of elements we have to parse + bool seq_greedy = false; + Py_ssize_t seq_length = 0; + length = CpRepeatedAtom_GetLength(self, (PyObject*)layer); + if (!length) { + goto fail; + } + if (_CpUnpack_EvalLength(layer, length, &seq_greedy, &seq_length) < 0) { + goto fail; + } + CpSeqLayerObject* seq_layer = CpSeqLayer_New(layer->m_state, layer); + if (!layer) { + goto fail; + } + + // REVISIT: add sequence factory here + PyObject* seq = PyList_New(0); + if (!seq) { + goto fail; + } + + CpSeqLayer_SetSequence(seq_layer, seq, seq_length, seq_greedy); + while (seq_layer->s_greedy || (seq_layer->m_index < seq_layer->m_length)) { + seq_layer->ob_base.m_path = PyUnicode_FromFormat( + "%s.%d", _PyUnicode_AsString(layer->m_path), seq_layer->m_index); + if (!seq_layer->ob_base.m_path) { + goto fail; + } + + Py_XSETREF(obj, _Cp_Unpack(self->m_atom, (CpLayerObject*)seq_layer)); + if (!obj) { + + if (seq_layer->s_greedy) { + PyErr_Clear(); + break; + } + goto fail; + } + + if (PyList_Append(seq_layer->m_sequence, Py_NewRef(obj)) < 0) { + goto fail; + } + seq_layer->m_index++; + } + +success: + Py_XDECREF(obj); + Py_XDECREF(length); + CpLayer_Invalidate((CpLayerObject*)seq_layer); + seq_layer = NULL; + return Py_NewRef(seq); + +fail: + Py_XDECREF(obj); + Py_XDECREF(length); + if (seq_layer) { + CpLayer_Invalidate((CpLayerObject*)seq_layer); + } + return NULL; +} + +/* type setup */ +static PyMemberDef CpRepeatedAtom_Members[] = { + { "length", T_OBJECT, offsetof(CpRepeatedAtomObject, m_length), READONLY }, + { "atom", T_OBJECT, offsetof(CpRepeatedAtomObject, m_atom), READONLY }, + { NULL } +}; + +static PyMethodDef CpRepeatedAtom_Methods[] = { + { "__set_byteorder__", + (PyCFunction)cp_repeatedatomobj_set_byteorder, + METH_VARARGS, + NULL }, + { + NULL, + } +}; + +PyTypeObject CpRepeatedAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpRepeatedAtom_NAME), + .tp_basicsize = sizeof(CpRepeatedAtomObject), + .tp_dealloc = (destructor)cp_repeatedatomobj_dealloc, + .tp_repr = (reprfunc)cp_repeatedatomobj__repr__, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = NULL, + .tp_new = (newfunc)cp_repeatedatomobj_new, + .tp_init = (initproc)cp_repeatedatomobj_init, + .tp_members = CpRepeatedAtom_Members, + .tp_methods = CpRepeatedAtom_Methods, +}; \ No newline at end of file diff --git a/src/atomimpl/floatatomobj.c b/src/atomimpl/floatatomobj.c index 342d543..a858c55 100644 --- a/src/atomimpl/floatatomobj.c +++ b/src/atomimpl/floatatomobj.c @@ -69,6 +69,8 @@ cp_floatatom_init(CpFloatAtomObject* self, PyObject* args, PyObject* kwds) return 0; } +_CpEndian_ImplSetByteorder(floatatom, self->_m_little_endian); + /* Public API */ /*CpAPI*/ int @@ -85,15 +87,6 @@ CpFloatAtom_Pack(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer) } int little_endian = self->_m_little_endian; - if (layer->m_field) { - _modulestate* mod = layer->m_state->mod; - PyObject* endian = ((CpFieldObject*)layer->m_field)->m_endian; - - if (CpEndian_Check(endian)) - /* If the field has an endian specified, use that. */ - little_endian = CpEndian_IsLittleEndian((CpEndianObject*)endian, mod); - } - int res = 0; switch (self->_m_byte_count) { case 2: { @@ -141,16 +134,6 @@ CpFloatAtom_Unpack(CpFloatAtomObject* self, CpLayerObject* layer) } int little_endian = self->_m_little_endian; - if (layer->m_field) { - _modulestate* mod = layer->m_state->mod; - PyObject* endian = ((CpFieldObject*)layer->m_field)->m_endian; - - if (CpEndian_Check(endian)) { - /* If the field has an endian specified, use that. */ - little_endian = CpEndian_IsLittleEndian((CpEndianObject*)endian, mod); - } - } - double res; switch (self->_m_byte_count) { case 2: { @@ -185,9 +168,12 @@ static PyMemberDef CpFloatAtom_Members[] = { READONLY, NULL }, { "bits", T_PYSSIZET, offsetof(CpFloatAtomObject, _m_bits), READONLY, NULL }, - { "little_endian", - T_BOOL, - offsetof(CpFloatAtomObject, _m_little_endian)}, + { "little_endian", T_BOOL, offsetof(CpFloatAtomObject, _m_little_endian) }, + { NULL } /* Sentinel */ +}; + +static PyMethodDef CpFloatAtom_Methods[] = { + _CpEndian_ImplSetByteorder_MethDef(floatatom, NULL), { NULL } /* Sentinel */ }; @@ -198,6 +184,7 @@ PyTypeObject CpFloatAtom_Type = { .tp_flags = Py_TPFLAGS_DEFAULT, .tp_doc = NULL, .tp_members = CpFloatAtom_Members, + .tp_methods = CpFloatAtom_Methods, .tp_init = (initproc)cp_floatatom_init, .tp_new = (newfunc)cp_floatatom_new, }; \ No newline at end of file diff --git a/src/atomimpl/intatomobj.c b/src/atomimpl/intatomobj.c index 8a12e14..4932a2b 100644 --- a/src/atomimpl/intatomobj.c +++ b/src/atomimpl/intatomobj.c @@ -75,6 +75,22 @@ cp_intatom_init(CpIntAtomObject* self, PyObject* args, PyObject* kwds) return 0; } +static PyObject* +cp_intatom_repr(CpIntAtomObject* self) +{ + char sign = 'i'; + if (!self->_m_signed) { + sign = 'u'; + } + char endian = 'l'; + if (!self->_m_little_endian) { + endian = 'b'; + } + return PyUnicode_FromFormat("<%ce %cint%d>", endian, sign, self->_m_bits); +} + +_CpEndian_ImplSetByteorder(intatom, self->_m_little_endian); + /* Public API */ /*CpAPI*/ int @@ -91,15 +107,6 @@ CpIntAtom_Pack(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer) } int little_endian = self->_m_little_endian; - if (layer->m_field) { - _modulestate* mod = layer->m_state->mod; - PyObject* endian = ((CpFieldObject*)layer->m_field)->m_endian; - - if (CpEndian_Check(endian)) - /* If the field has an endian specified, use that. */ - little_endian = CpEndian_IsLittleEndian((CpEndianObject*)endian, mod); - } - int res = _PyLong_AsByteArray((PyLongObject*)op, (unsigned char*)PyBytes_AS_STRING(bytes), self->_m_byte_count, @@ -127,15 +134,6 @@ CpIntAtom_Unpack(CpIntAtomObject* self, CpLayerObject* layer) } int little_endian = self->_m_little_endian; - if (layer->m_field) { - _modulestate* mod = layer->m_state->mod; - PyObject* endian = ((CpFieldObject*)layer->m_field)->m_endian; - - if (CpEndian_Check(endian)) - /* If the field has an endian specified, use that. */ - little_endian = CpEndian_IsLittleEndian((CpEndianObject*)endian, mod); - } - PyObject* obj = _PyLong_FromByteArray((unsigned char*)PyBytes_AS_STRING(bytes), self->_m_byte_count, @@ -160,13 +158,19 @@ static PyMemberDef CpIntAtom_Members[] = { { NULL } /* Sentinel */ }; -PyTypeObject CpIntAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpIntAtom_NAME), - .tp_basicsize = sizeof(CpIntAtomObject), - .tp_dealloc = (destructor)cp_intatom_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = NULL, - .tp_members = CpIntAtom_Members, - .tp_new = (newfunc)cp_intatom_new, - .tp_init = (initproc)cp_intatom_init, -}; \ No newline at end of file +static PyMethodDef CpIntAtom_Methods[] = { + _CpEndian_ImplSetByteorder_MethDef(intatom, NULL), + { NULL } /* Sentinel */ +}; + +PyTypeObject CpIntAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) + _Cp_NameStr(CpIntAtom_NAME), + .tp_basicsize = sizeof(CpIntAtomObject), + .tp_dealloc = (destructor)cp_intatom_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = NULL, + .tp_members = CpIntAtom_Members, + .tp_new = (newfunc)cp_intatom_new, + .tp_init = (initproc)cp_intatom_init, + .tp_repr = (reprfunc)cp_intatom_repr, + .tp_methods = CpIntAtom_Methods }; \ No newline at end of file diff --git a/src/atomimpl/padatomobj.c b/src/atomimpl/padatomobj.c index c24108a..fc615c1 100644 --- a/src/atomimpl/padatomobj.c +++ b/src/atomimpl/padatomobj.c @@ -92,6 +92,7 @@ CpPaddingAtom_PackMany(CpPaddingAtomObject* self, PyObject* value, CpLayerObject* layer) { +#if 0 /* value will be ignored here */ PyObject *res = NULL, *bytes = NULL, *objSize = NULL, *objLengh = NULL; bool greedy = false; @@ -145,6 +146,8 @@ CpPaddingAtom_PackMany(CpPaddingAtomObject* self, } Py_XDECREF(res); return 0; +#endif + return 0; } /*CpAPI*/ @@ -156,13 +159,14 @@ CpPaddingAtom_Unpack(CpPaddingAtomObject* self, CpLayerObject* layer) return NULL; } Py_XDECREF(res); - Py_RETURN_NONE; + return 0; } /*CpAPI*/ PyObject* CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, CpLayerObject* layer) { +#if 0 PyObject *res = NULL, *bytes = NULL, *objLengh = NULL; bool greedy = false; Py_ssize_t length = 0, parsedLength = 0; @@ -184,6 +188,7 @@ CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, CpLayerObject* layer) } Py_XDECREF(res); +#endif Py_RETURN_NONE; } diff --git a/src/atomobj.c b/src/atomobj.c index a75686f..476dfcc 100644 --- a/src/atomobj.c +++ b/src/atomobj.c @@ -264,6 +264,12 @@ cp_catom_size(CpCAtomObject* self, PyObject* args, PyObject* kw) if (PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &context) < 0) { return NULL; } + + if (context == Py_None || context == NULL) { + PyErr_SetString(PyExc_ValueError, "context must be set!"); + return NULL; + } + return self->ob_size((PyObject*)self, context); } diff --git a/src/caterpillar/include/caterpillar/arch.h b/src/caterpillar/include/caterpillar/arch.h index 09fa1bb..f420e39 100644 --- a/src/caterpillar/include/caterpillar/arch.h +++ b/src/caterpillar/include/caterpillar/arch.h @@ -88,4 +88,51 @@ struct _endianobj */ #define CpEndian_Check(op) PyObject_TypeCheck((PyObject*)(op), &CpEndian_Type) +/** + * @brief Set the byteorder of the given object + * + * @param op the object to set + * @param endian the new byteorder + * @return 0 on success, -1 on error + */ +inline int +CpEndian_SetEndian(PyObject* op, CpEndianObject* endian) +{ + PyObject* attr = PyObject_GetAttrString(op, "__set_byteorder__"); + if (!attr) { + return -1; + } + PyObject* ret = PyObject_CallOneArg(attr, (PyObject*)endian); + if (!ret) { + return -1; + } + Py_DECREF(ret); + Py_DECREF(attr); + return 0; +} + +#define _CpEndian_ImplSetByteorder(name, field) \ + static PyObject* cp_##name##_set_byteorder( \ + CpFloatAtomObject* self, PyObject* args, PyObject* kw) \ + { \ + static char* kwlist[] = { "byteorder", NULL }; \ + PyObject* byteorder = NULL; \ + if (!PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &byteorder)) { \ + return NULL; \ + } \ + if (!CpEndian_Check(byteorder)) { \ + PyErr_SetString(PyExc_TypeError, "byteorder must be an Endian object"); \ + return NULL; \ + } \ + field = CpEndian_IsLittleEndian((CpEndianObject*)byteorder, \ + get_global_module_state()); \ + Py_RETURN_NONE; \ + } + +#define _CpEndian_ImplSetByteorder_MethDef(name, docs) \ + { \ + "__set_byteorder__", (PyCFunction)cp_##name##_set_byteorder, METH_VARARGS, \ + (docs) \ + } + #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/builtins.h b/src/caterpillar/include/caterpillar/atoms/builtins.h new file mode 100644 index 0000000..6136d70 --- /dev/null +++ b/src/caterpillar/include/caterpillar/atoms/builtins.h @@ -0,0 +1,62 @@ +/** + * Copyright (C) MatrixEditor 2024 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef BUILTINS_H +#define BUILTINS_H + +#include "caterpillar/caterpillarapi.h" +#include "caterpillar/atomobj.h" + +// PROPOSAL: Builtin atoms are similar to the Python class FieldMixin +// but implement all parts of a field separately. For instance, the +// builtin atom for a sequence only implements the sequence part. +struct _builtinatomobj +{ + CpCAtom_HEAD +}; + +#define CpBuiltinAtom_NAME "builtinatom" +#define CpBuiltinAtom_CheckExact(op) Py_IS_TYPE((op), &CpBuiltinAtom_Type) +#define CpBuiltinAtom_Check(op) PyObject_IsType((op), &CpBuiltinAtom_Type) +#define CpBuiltinAtom_HEAD CpBuiltinAtomObject ob_base; +#define CpBuiltinAtom_CATOM(self) (self)->ob_base.ob_base + +//------------------------------------------------------------------------------ +// Repeated +struct _repeatedatomobj +{ + CpBuiltinAtom_HEAD + + /// Stores a reference to the actual parsing struct that will be used + /// to parse or build our data. This attribute is never null. + PyObject *m_atom; + + /// A constant or dynamic value to represent the amount of structs. Zero + /// indicates there are no sequence types associated with this field. + PyObject* m_length; +}; + +#define CpRepeatedAtom_NAME "repeated" +#define CpRepeatedAtom_CheckExact(op) Py_IS_TYPE((op), &CpRepeatedAtom_Type) +#define CpRepeatedAtom_Check(op) PyObject_IsType((op), &CpRepeatedAtom_Type) + +inline CpRepeatedAtomObject * +CpRepeatedAtom_New(PyObject* atom, PyObject* length) +{ + return (CpRepeatedAtomObject*)CpObject_Create(&CpRepeatedAtom_Type, "OO", atom, length); +} + +#endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/int.h b/src/caterpillar/include/caterpillar/atoms/int.h index 8a41599..02f741d 100644 --- a/src/caterpillar/include/caterpillar/atoms/int.h +++ b/src/caterpillar/include/caterpillar/atoms/int.h @@ -17,12 +17,11 @@ #ifndef INTATOMOBJ_H #define INTATOMOBJ_H -#include "caterpillar/caterpillarapi.h" -#include "caterpillar/field.h" +#include "caterpillar/atoms/builtins.h" struct _intatomobj { - CpFieldCAtom_HEAD + CpBuiltinAtom_HEAD /// Stores the amount of bits this integer atom /// has in total diff --git a/src/caterpillar/include/caterpillar/caterpillar.h b/src/caterpillar/include/caterpillar/caterpillar.h index 0a2b35d..b965070 100644 --- a/src/caterpillar/include/caterpillar/caterpillar.h +++ b/src/caterpillar/include/caterpillar/caterpillar.h @@ -13,8 +13,10 @@ #include "caterpillar/atomobj.h" #include "caterpillar/field.h" #include "caterpillar/state.h" +#include "caterpillar/parsing.h" // Atom Objects +#include "caterpillar/atoms/builtins.h" #include "caterpillar/atoms/int.h" #include "caterpillar/atoms/float.h" #include "caterpillar/atoms/primitive.h" diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index 5da33d4..6daa2ef 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -74,6 +74,14 @@ struct _stringatomobj; typedef struct _stringatomobj CpStringAtomObject; struct _constatomobj; typedef struct _constatomobj CpConstAtomObject; +struct _builtinatomobj; +typedef struct _builtinatomobj CpBuiltinAtomObject; +struct _repeatedatomobj; +typedef struct _repeatedatomobj CpRepeatedAtomObject; +struct _seqlayerobj; +typedef struct _seqlayerobj CpSeqLayerObject; +struct _objlayerobj; +typedef struct _objlayerobj CpObjLayerObject; #ifdef _CPMODULE @@ -110,6 +118,10 @@ extern PyTypeObject CpCharAtom_Type; extern PyTypeObject CpPaddingAtom_Type; extern PyTypeObject CpStringAtom_Type; extern PyTypeObject CpConstAtom_Type; +extern PyTypeObject CpBuiltinAtom_Type; +extern PyTypeObject CpRepeatedAtom_Type; +extern PyTypeObject CpSeqLayer_Type; +extern PyTypeObject CpObjLayer_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -129,7 +141,6 @@ int CpPack(PyObject* op, PyObject* atom, PyObject* io, PyObject* globals); int CpPack_Field(PyObject* op, CpFieldObject* field, CpLayerObject* layer); int CpPack_Common(PyObject* op, PyObject* atom, CpLayerObject* layer); int CpPack_Struct(PyObject* op, CpStructObject* struct_, CpLayerObject* layer); -int _Cp_Pack(PyObject* op, PyObject* atom, CpLayerObject* layer); int _CpPack_EvalLength(CpLayerObject* layer,PyObject* length,Py_ssize_t size,bool* greedy,Py_ssize_t* dstLength); PyObject* CpSizeOf(PyObject* op, PyObject* globals); PyObject* CpSizeOf_Field(CpFieldObject* field, CpLayerObject* layer); @@ -140,7 +151,6 @@ PyObject* CpUnpack(PyObject* atom, PyObject* io, PyObject* globals); PyObject* CpUnpack_Field(CpFieldObject* field, CpLayerObject* layer); PyObject* CpUnpack_Common(PyObject* op, CpLayerObject* layer); PyObject* CpUnpack_Struct(CpStructObject* struct_, CpLayerObject* layer); -PyObject* _Cp_Unpack(PyObject* atom, CpLayerObject* layer); int _CpUnpack_EvalLength(CpLayerObject* layer,PyObject* length,bool* seq_greedy,Py_ssize_t* seq_length); PyObject* CpUnpack_CAtom(CpCAtomObject* catom, CpLayerObject* layer); int CpPack_CAtom(PyObject* op, CpCAtomObject* catom, CpLayerObject* layer); @@ -155,7 +165,6 @@ PyObject* CpState_Write(CpStateObject* self, PyObject* value); int CpState_SetGlobals(CpStateObject* self, PyObject* globals); CpLayerObject* CpLayer_New(CpStateObject* state, CpLayerObject* parent); int CpLayer_Invalidate(CpLayerObject* self); -int CpLayer_SetSequence(CpLayerObject* self,PyObject* sequence,Py_ssize_t length,int8_t greedy); CpStructFieldInfoObject* CpStructFieldInfo_New(CpFieldObject* field); int CpStruct_AddFieldInfo(CpStructObject* o, CpStructFieldInfoObject* info); int CpStruct_AddField(CpStructObject* o, CpFieldObject* field, int exclude); @@ -165,6 +174,9 @@ int CpStruct_ReplaceType(CpStructObject* o, PyObject* name, PyObject* type); int CpStruct_HasOption(CpStructObject* o, PyObject* option); int CpStructModel_Check(PyObject* model, _modulestate* state); PyObject* CpStructModel_GetStruct(PyObject* model, _modulestate* state); +CpSeqLayerObject* CpSeqLayer_New(CpStateObject* state, CpLayerObject* parent); +int CpSeqLayer_SetSequence(CpSeqLayerObject* self,PyObject* sequence,Py_ssize_t length,int8_t greedy); +CpObjLayerObject* CpObjLayer_New(CpStateObject* state, CpLayerObject* parent); int CpIntAtom_Pack(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer); PyObject* CpIntAtom_Unpack(CpIntAtomObject* self, CpLayerObject* layer); int CpFloatAtom_Pack(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer); @@ -181,6 +193,9 @@ int CpStringAtom_Pack(CpStringAtomObject* self,PyObject* value,CpLayerObject* la PyObject* CpStringAtom_Unpack(CpStringAtomObject* self, CpLayerObject* layer); int CpConstAtom_Pack(CpConstAtomObject* self, PyObject* value, CpLayerObject* layer); PyObject* CpConstAtom_Unpack(CpConstAtomObject* self, CpLayerObject* layer); +int CpRepeatedAtom_Pack(CpRepeatedAtomObject* self,PyObject* op,CpLayerObject* layer); +PyObject* CpRepeatedAtom_Unpack(CpRepeatedAtomObject* self, CpLayerObject* layer); +PyObject* CpRepeatedAtom_GetLength(CpRepeatedAtomObject* self, PyObject* context); #else @@ -218,6 +233,10 @@ caterpillar_api.py #define CpPaddingAtom_Type (*(PyTypeObject *)Cp_API[25]) #define CpStringAtom_Type (*(PyTypeObject *)Cp_API[26]) #define CpConstAtom_Type (*(PyTypeObject *)Cp_API[27]) +#define CpBuiltinAtom_Type (*(PyTypeObject *)Cp_API[28]) +#define CpRepeatedAtom_Type (*(PyTypeObject *)Cp_API[29]) +#define CpSeqLayer_Type (*(PyTypeObject *)Cp_API[30]) +#define CpObjLayer_Type (*(PyTypeObject *)Cp_API[31]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -237,7 +256,6 @@ caterpillar_api.py #define CpPack_Field (*((int (*)(PyObject* op, CpFieldObject* field, CpLayerObject* layer)))Cp_API[68]) #define CpPack_Common (*((int (*)(PyObject* op, PyObject* atom, CpLayerObject* layer)))Cp_API[69]) #define CpPack_Struct (*((int (*)(PyObject* op, CpStructObject* struct_, CpLayerObject* layer)))Cp_API[70]) -#define _Cp_Pack (*((int (*)(PyObject* op, PyObject* atom, CpLayerObject* layer)))Cp_API[71]) #define _CpPack_EvalLength (*((int (*)(CpLayerObject* layer,PyObject* length,Py_ssize_t size,bool* greedy,Py_ssize_t* dstLength)))Cp_API[72]) #define CpSizeOf (*((PyObject* (*)(PyObject* op, PyObject* globals)))Cp_API[73]) #define CpSizeOf_Field (*((PyObject* (*)(CpFieldObject* field, CpLayerObject* layer)))Cp_API[74]) @@ -248,7 +266,6 @@ caterpillar_api.py #define CpUnpack_Field (*((PyObject* (*)(CpFieldObject* field, CpLayerObject* layer)))Cp_API[79]) #define CpUnpack_Common (*((PyObject* (*)(PyObject* op, CpLayerObject* layer)))Cp_API[80]) #define CpUnpack_Struct (*((PyObject* (*)(CpStructObject* struct_, CpLayerObject* layer)))Cp_API[81]) -#define _Cp_Unpack (*((PyObject* (*)(PyObject* atom, CpLayerObject* layer)))Cp_API[82]) #define _CpUnpack_EvalLength (*((int (*)(CpLayerObject* layer,PyObject* length,bool* seq_greedy,Py_ssize_t* seq_length)))Cp_API[83]) #define CpUnpack_CAtom (*((PyObject* (*)(CpCAtomObject* catom, CpLayerObject* layer)))Cp_API[84]) #define CpPack_CAtom (*((int (*)(PyObject* op, CpCAtomObject* catom, CpLayerObject* layer)))Cp_API[85]) @@ -263,7 +280,6 @@ caterpillar_api.py #define CpState_SetGlobals (*((int (*)(CpStateObject* self, PyObject* globals)))Cp_API[94]) #define CpLayer_New (*((CpLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[95]) #define CpLayer_Invalidate (*((int (*)(CpLayerObject* self)))Cp_API[96]) -#define CpLayer_SetSequence (*((int (*)(CpLayerObject* self,PyObject* sequence,Py_ssize_t length,int8_t greedy)))Cp_API[97]) #define CpStructFieldInfo_New (*((CpStructFieldInfoObject* (*)(CpFieldObject* field)))Cp_API[98]) #define CpStruct_AddFieldInfo (*((int (*)(CpStructObject* o, CpStructFieldInfoObject* info)))Cp_API[99]) #define CpStruct_AddField (*((int (*)(CpStructObject* o, CpFieldObject* field, int exclude)))Cp_API[100]) @@ -273,6 +289,9 @@ caterpillar_api.py #define CpStruct_HasOption (*((int (*)(CpStructObject* o, PyObject* option)))Cp_API[104]) #define CpStructModel_Check (*((int (*)(PyObject* model, _modulestate* state)))Cp_API[105]) #define CpStructModel_GetStruct (*((PyObject* (*)(PyObject* model, _modulestate* state)))Cp_API[106]) +#define CpSeqLayer_New (*((CpSeqLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[107]) +#define CpSeqLayer_SetSequence (*((int (*)(CpSeqLayerObject* self,PyObject* sequence,Py_ssize_t length,int8_t greedy)))Cp_API[108]) +#define CpObjLayer_New (*((CpObjLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[109]) #define CpIntAtom_Pack (*((int (*)(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer)))Cp_API[120]) #define CpIntAtom_Unpack (*((PyObject* (*)(CpIntAtomObject* self, CpLayerObject* layer)))Cp_API[121]) #define CpFloatAtom_Pack (*((int (*)(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[122]) @@ -289,6 +308,9 @@ caterpillar_api.py #define CpStringAtom_Unpack (*((PyObject* (*)(CpStringAtomObject* self, CpLayerObject* layer)))Cp_API[133]) #define CpConstAtom_Pack (*((int (*)(CpConstAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[134]) #define CpConstAtom_Unpack (*((PyObject* (*)(CpConstAtomObject* self, CpLayerObject* layer)))Cp_API[135]) +#define CpRepeatedAtom_Pack (*((int (*)(CpRepeatedAtomObject* self,PyObject* op,CpLayerObject* layer)))Cp_API[136]) +#define CpRepeatedAtom_Unpack (*((PyObject* (*)(CpRepeatedAtomObject* self, CpLayerObject* layer)))Cp_API[137]) +#define CpRepeatedAtom_GetLength (*((PyObject* (*)(CpRepeatedAtomObject* self, PyObject* context)))Cp_API[138]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillar/include/caterpillar/parsing.h b/src/caterpillar/include/caterpillar/parsing.h new file mode 100644 index 0000000..3a05b0c --- /dev/null +++ b/src/caterpillar/include/caterpillar/parsing.h @@ -0,0 +1,50 @@ +/** + * Copyright (C) MatrixEditor 2024 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef CP_PARSING_H +#define CP_PARSING_H + +#include "caterpillar/atomobj.h" +#include "caterpillar/module.h" +#include "caterpillar/state.h" + +static inline int +_Cp_Pack(PyObject* op, PyObject* atom, CpLayerObject* layer) +{ + if (CpCAtom_Check(atom)) { + return CpPack_CAtom(op, (CpCAtomObject*)atom, layer); + } + + _modulestate* mod = layer->m_state->mod; + PyObject* result = CpAtom_Pack(atom, mod->str___pack__, op, (PyObject*)layer); + if (result == NULL) { + return -1; + } + Py_DECREF(result); + return 0; +} + +static inline PyObject* +_Cp_Unpack(PyObject* op, CpLayerObject* layer) +{ + if (CpCAtom_Check(op)) { + return CpUnpack_CAtom((CpCAtomObject*)op, layer); + } + return PyObject_CallMethodOneArg( + op, layer->m_state->mod->str___unpack__, (PyObject*)layer); +} + +#endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/state.h b/src/caterpillar/include/caterpillar/state.h index 5133148..a1b40c2 100644 --- a/src/caterpillar/include/caterpillar/state.h +++ b/src/caterpillar/include/caterpillar/state.h @@ -74,37 +74,26 @@ struct _stateobj //----------------------------------------------------------------------------- // layer +enum { + CpLayerClass_Default = 0, + CpLayerClass_Sequence = 1, + CpLayerClass_Object = 2, +}; /** - * @brief TODO + * @brief Top level layer class with a reference to its parent and the global + * parsing state. */ struct _layerobj { PyObject_HEAD - /// The parent layer - struct _layerobj* m_parent; + /// The parent layer + struct _layerobj* m_parent; /// the global parsing state CpStateObject* m_state; - // --- Context sensitive variables --- - - /// In case a struct is linked to a field, the `Field` instance will always - /// set this layer to be accessible from within the underlying struct. - PyObject* m_field; - - /// When packing or unpacking objects, the current object attributes are - /// stored within an object context. This is a special context that allows - /// access to previously parsed fields or attributes of the input object. To - /// minimize the number of calls using this attribute, a shortcut named `this` - /// was defined, which automatically inserts a path to the object context. - PyObject* m_obj; - - /// In case a switch-case statement is activated in a field, the layer will - /// store the parsed value in this variable temporarily. - PyObject* m_value; - /// Although it is optional to provide the current parsing or building path, /// it is recommended. All nesting structures implement a behavior that /// automatically adds a sub-path while packing or unpacking. Special names @@ -112,24 +101,46 @@ struct _layerobj /// elements PyObject* m_path; - /// Same as `m_obj` but for the sequential elements. - PyObject* m_sequence; + // REVISIT: do we need them? + /// Lazily loaded context sensitive variables (this field remains NULL until + /// the first item gets added to it) + // PyObject *m_locals; - /// The length of the current collection. - Py_ssize_t m_length; + /// The type of the layer + int8_t m_class; - /// When packing or unpacking collections of elements, the current working - /// index is given under this layer variable. It is set only in this specific - /// situation. - Py_ssize_t m_index; + // --- Context sensitive variables --- - // --- Internal state variables --- - int8_t s_greedy; - int8_t s_sequential; -}; + // /// In case a struct is linked to a field, the `Field` instance will always + // /// set this layer to be accessible from within the underlying struct. + // PyObject* m_field; + + // /// When packing or unpacking objects, the current object attributes are + // /// stored within an object context. This is a special context that allows + // /// access to previously parsed fields or attributes of the input object. To + // /// minimize the number of calls using this attribute, a shortcut named `this` + // /// was defined, which automatically inserts a path to the object context. + // PyObject* m_obj; + + // /// In case a switch-case statement is activated in a field, the layer will + // /// store the parsed value in this variable temporarily. + // PyObject* m_value; + + // /// Same as `m_obj` but for the sequential elements. + // PyObject* m_sequence; -/// Layer type -// PyAPI_DATA(PyTypeObject) CpLayer_Type; + // /// The length of the current collection. + // Py_ssize_t m_length; + + // /// When packing or unpacking collections of elements, the current working + // /// index is given under this layer variable. It is set only in this specific + // /// situation. + // Py_ssize_t m_index; + + // // --- Internal state variables --- + // int8_t s_greedy; + // int8_t s_sequential; +}; /** * @brief Check whether the given object is a layer @@ -148,7 +159,7 @@ struct _layerobj * @return false if the object is not a layer */ #define CpLayer_Check(v) PyObject_TypeCheck((v), &CpLayer_Type) - +#define CpLayer_HEAD CpLayerObject ob_base; #define CpLayer_AppendPath(layer, newpath) \ Py_XSETREF( \ @@ -159,4 +170,60 @@ struct _layerobj : (layer)->m_path), \ _PyUnicode_AsString((newpath)))); +//----------------------------------------------------------------------------- +// seq layer + +struct _seqlayerobj +{ + CpLayer_HEAD + + /// Same as `m_obj` but for the sequential elements. + PyObject* m_sequence; + + /// The length of the current collection. + Py_ssize_t m_length; + + /// When packing or unpacking collections of elements, the current working + /// index is given under this layer variable. It is set only in this specific + /// situation. + Py_ssize_t m_index; + + int8_t s_greedy; +}; + +#define CpSeqLayer_NAME "seqlayer" +#define CpSeqLayer_CheckExact(v) Py_IS_TYPE((v), &CpSeqLayer_Type) +#define CpSeqLayer_Check(v) PyObject_TypeCheck((v), &CpSeqLayer_Type) + +//----------------------------------------------------------------------------- +// obj layer + +struct _objlayerobj +{ + CpLayer_HEAD + + /// When packing or unpacking objects, the current object attributes are + /// stored within an object context. This is a special context that allows + /// access to previously parsed fields or attributes of the input object. To + /// minimize the number of calls using this attribute, a shortcut named `this` + /// was defined, which automatically inserts a path to the object context. + PyObject* m_obj; +}; + +#define CpObjLayer_NAME "objlayer" +#define CpObjLayer_CheckExact(v) Py_IS_TYPE((v), &CpObjLayer_Type) +#define CpObjLayer_Check(v) PyObject_TypeCheck((v), &CpObjLayer_Type) + +//----------------------------------------------------------------------------- +// field layer + +struct _fieldlayerobj +{ + CpLayer_HEAD + + /// In case a struct is linked to a field, the `Field` instance will always + /// set this layer to be accessible from within the underlying struct. + PyObject* m_field; +}; + #endif \ No newline at end of file diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index 15bcbab..aecfc89 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -31,10 +31,10 @@ void *Cp_API[] = { (void *) &CpPaddingAtom_Type, (void *) &CpStringAtom_Type, (void *) &CpConstAtom_Type, - NULL, - NULL, - NULL, - NULL, + (void *) &CpBuiltinAtom_Type, + (void *) &CpRepeatedAtom_Type, + (void *) &CpSeqLayer_Type, + (void *) &CpObjLayer_Type, NULL, NULL, NULL, @@ -74,7 +74,7 @@ void *Cp_API[] = { (void *) &CpPack_Field, (void *) &CpPack_Common, (void *) &CpPack_Struct, - (void *) &_Cp_Pack, + NULL, (void *) &_CpPack_EvalLength, (void *) &CpSizeOf, (void *) &CpSizeOf_Field, @@ -85,7 +85,7 @@ void *Cp_API[] = { (void *) &CpUnpack_Field, (void *) &CpUnpack_Common, (void *) &CpUnpack_Struct, - (void *) &_Cp_Unpack, + NULL, (void *) &_CpUnpack_EvalLength, (void *) &CpUnpack_CAtom, (void *) &CpPack_CAtom, @@ -100,7 +100,7 @@ void *Cp_API[] = { (void *) &CpState_SetGlobals, (void *) &CpLayer_New, (void *) &CpLayer_Invalidate, - (void *) &CpLayer_SetSequence, + NULL, (void *) &CpStructFieldInfo_New, (void *) &CpStruct_AddFieldInfo, (void *) &CpStruct_AddField, @@ -110,9 +110,9 @@ void *Cp_API[] = { (void *) &CpStruct_HasOption, (void *) &CpStructModel_Check, (void *) &CpStructModel_GetStruct, - NULL, - NULL, - NULL, + (void *) &CpSeqLayer_New, + (void *) &CpSeqLayer_SetSequence, + (void *) &CpObjLayer_New, NULL, NULL, NULL, @@ -138,6 +138,9 @@ void *Cp_API[] = { (void *) &CpStringAtom_Pack, (void *) &CpStringAtom_Unpack, (void *) &CpConstAtom_Pack, - (void *) &CpConstAtom_Unpack + (void *) &CpConstAtom_Unpack, + (void *) &CpRepeatedAtom_Pack, + (void *) &CpRepeatedAtom_Unpack, + (void *) &CpRepeatedAtom_GetLength }; diff --git a/src/code_gen/caterpillar_api.py b/src/code_gen/caterpillar_api.py index db0f3b0..caa68bd 100644 --- a/src/code_gen/caterpillar_api.py +++ b/src/code_gen/caterpillar_api.py @@ -42,6 +42,10 @@ "_paddingatomobj": "CpPaddingAtomObject", "_stringatomobj": "CpStringAtomObject", "_constatomobj": "CpConstAtomObject", + "_builtinatomobj": "CpBuiltinAtomObject", + "_repeatedatomobj": "CpRepeatedAtomObject", + "_seqlayerobj": "CpSeqLayerObject", + "_objlayerobj": "CpObjLayerObject", } cp_type_api = { @@ -73,6 +77,10 @@ "CpPaddingAtom_Type": (25,), "CpStringAtom_Type": (26,), "CpConstAtom_Type": (27,), + "CpBuiltinAtom_Type": (28,), + "CpRepeatedAtom_Type": (29,), + "CpSeqLayer_Type": (30,), + "CpObjLayer_Type": (31,), } cp_func_api = { @@ -98,7 +106,6 @@ "CpPack_Field": 68, "CpPack_Common": 69, "CpPack_Struct": 70, - "_Cp_Pack": 71, "_CpPack_EvalLength": 72, "CpSizeOf": 73, "CpSizeOf_Field": 74, @@ -109,7 +116,6 @@ "CpUnpack_Field": 79, "CpUnpack_Common": 80, "CpUnpack_Struct": 81, - "_Cp_Unpack": 82, "_CpUnpack_EvalLength": 83, "CpUnpack_CAtom": 84, "CpPack_CAtom": 85, @@ -124,7 +130,6 @@ "CpState_SetGlobals": 94, "CpLayer_New": 95, "CpLayer_Invalidate": 96, - "CpLayer_SetSequence": 97, "CpStructFieldInfo_New" : 98, "CpStruct_AddFieldInfo": 99, "CpStruct_AddField": 100, @@ -134,6 +139,10 @@ "CpStruct_HasOption": 104, "CpStructModel_Check": 105, "CpStructModel_GetStruct": 106, + "CpSeqLayer_New": 107, + "CpSeqLayer_SetSequence": 108, + "CpObjLayer_New": 109, + # atom api "CpIntAtom_Pack": 120, @@ -152,6 +161,9 @@ "CpStringAtom_Unpack": 133, "CpConstAtom_Pack": 134, "CpConstAtom_Unpack": 135, + "CpRepeatedAtom_Pack": 136, + "CpRepeatedAtom_Unpack": 137, + "CpRepeatedAtom_GetLength": 138, } API_SRC = [ @@ -162,6 +174,7 @@ "option.c", "struct.c", "state.c", + "layer.c", "parsing_pack.c", "parsing_unpack.c", "parsing_typeof.c", @@ -173,6 +186,8 @@ "atomimpl/padatomobj.c", "atomimpl/stringatomobj.c", "atomimpl/constatomobj.c", + "atomimpl/builtins/builtinatomobj.c", + "atomimpl/builtins/repeatedatomobj.c", ] diff --git a/src/layer.c b/src/layer.c new file mode 100644 index 0000000..63b26b1 --- /dev/null +++ b/src/layer.c @@ -0,0 +1,380 @@ +/* layer implementation */ +#include "caterpillar/caterpillar.h" + +#include + +/* top level layer impl */ +static PyObject* +cp_layer_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + CpLayerObject* self; + self = (CpLayerObject*)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + + self->m_path = NULL; + self->m_parent = NULL; + self->m_state = NULL; + self->m_class = CpLayerClass_Default; + return (PyObject*)self; +} + +static void +cp_layer_dealloc(CpLayerObject* self) +{ + Py_XDECREF(self->m_path); + Py_XDECREF(self->m_state); + Py_XDECREF(self->m_parent); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_layer_init(CpLayerObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "state", "path", "parent", NULL }; + PyObject *state = NULL, *path = NULL, *parent = NULL; + if (!PyArg_ParseTupleAndKeywords( + args, kw, "O|OO", kwlist, &state, &path, &parent)) + return -1; + + if (!PyObject_IsInstance(state, (PyObject*)(&CpState_Type))) { + PyErr_SetString(PyExc_TypeError, "state must be an instance of CpState"); + return -1; + } + + Py_XSETREF(self->m_state, (CpStateObject*)Py_NewRef(state)); + _Cp_SetObj(self->m_path, path); + Py_XSETREF(self->m_parent, (CpLayerObject*)Py_XNewRef(parent)); + return 0; +} + +/*CpAPI*/ +CpLayerObject* +CpLayer_New(CpStateObject* state, CpLayerObject* parent) +{ + CpLayerObject* self = + (CpLayerObject*)CpObject_Create(&CpLayer_Type, "O", state); + if (!self) { + return NULL; + } + if (parent) { + Py_XSETREF(self->m_parent, (CpLayerObject*)Py_XNewRef(parent)); + Py_XSETREF(self->m_path, Py_XNewRef(parent->m_path)); + } + return self; +} + +/*CpAPI*/ +int +CpLayer_Invalidate(CpLayerObject* self) +{ + if (!self) { + PyErr_SetString(PyExc_ValueError, "layer must be non-null"); + return -1; + } + + Py_XSETREF(self->m_parent, NULL); + Py_XSETREF(self->m_state, NULL); + Py_XSETREF(self->m_path, NULL); + Py_CLEAR(self); + return 0; +} + +/* docs */ + +/* type */ +static PyMemberDef CpLayer_Members[] = { + { "state", T_OBJECT, offsetof(CpLayerObject, m_state), READONLY, "state" }, + { "path", T_OBJECT, offsetof(CpLayerObject, m_path), 0, "path" }, + { "parent", T_OBJECT, offsetof(CpLayerObject, m_parent), 0, "parent" }, + { NULL } /* Sentinel */ +}; + +PyTypeObject CpLayer_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(layer), + .tp_basicsize = sizeof(CpLayerObject), + .tp_dealloc = (destructor)cp_layer_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = NULL, + .tp_members = CpLayer_Members, + .tp_init = (initproc)cp_layer_init, + .tp_new = (newfunc)cp_layer_new, +}; + +//------------------------------------------------------------------------------- +// seq layer + +static PyObject* +cp_seqlayer_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + CpSeqLayerObject* self = (CpSeqLayerObject*)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + self->ob_base.m_path = NULL; + self->m_sequence = NULL; + self->m_length = -1; + self->m_index = -1; + self->s_greedy = false; + self->ob_base.m_parent = NULL; + self->ob_base.m_state = NULL; + self->ob_base.m_class = CpLayerClass_Sequence; + return (PyObject*)self; +} + +static void +cp_seqlayer_dealloc(CpSeqLayerObject* self) +{ + Py_XDECREF(self->ob_base.m_path); + Py_XDECREF(self->m_sequence); + Py_XDECREF(self->ob_base.m_parent); + Py_XDECREF(self->ob_base.m_state); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_seqlayer_init(CpSeqLayerObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "state", "path", "parent", NULL }; + PyObject *state = NULL, *sequence = NULL, *path = NULL, *parent = NULL; + if (!PyArg_ParseTupleAndKeywords( + args, kw, "O|OOO", kwlist, &state, &sequence, &path, &parent)) + return -1; + + if (!PyObject_IsInstance(state, (PyObject*)(&CpState_Type))) { + PyErr_SetString(PyExc_TypeError, "state must be an instance of CpState"); + return -1; + } + + Py_XSETREF(self->ob_base.m_state, (CpStateObject*)Py_NewRef(state)); + _Cp_SetObj(self->ob_base.m_path, path); + Py_XSETREF(self->ob_base.m_parent, (CpLayerObject*)Py_XNewRef(parent)); + return 0; +} + +/*CpAPI*/ +int +CpSeqLayer_SetSequence(CpSeqLayerObject* self, + PyObject* sequence, + Py_ssize_t length, + int8_t greedy) +{ + if (!sequence || !self) { + PyErr_SetString(PyExc_ValueError, "sequence and layer must be non-null"); + return -1; + } + + Py_XSETREF(self->m_sequence, Py_NewRef(sequence)); + self->m_length = length; + self->s_greedy = greedy; + self->m_index = 0; + return 0; +} + +/*CpAPI*/ +CpSeqLayerObject* +CpSeqLayer_New(CpStateObject* state, CpLayerObject* parent) +{ + CpSeqLayerObject* self = + (CpSeqLayerObject*)CpObject_Create(&CpSeqLayer_Type, "O", state); + if (!self) { + return NULL; + } + if (parent) { + Py_XSETREF(self->ob_base.m_parent, (CpLayerObject*)Py_XNewRef(parent)); + Py_XSETREF(self->ob_base.m_path, Py_XNewRef(parent->m_path)); + } + return self; +} + +/* docs */ + +/* type */ +static PyMemberDef CpSeqLayer_Members[] = { + { + "sequence", + T_OBJECT, + offsetof(CpSeqLayerObject, m_sequence), + READONLY, + }, + { + "length", + T_PYSSIZET, + offsetof(CpSeqLayerObject, m_length), + 0, + }, + { + "index", + T_PYSSIZET, + offsetof(CpSeqLayerObject, m_index), + 0, + }, + { NULL } /* Sentinel */ +}; + +PyTypeObject CpSeqLayer_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpSeqLayer_NAME), + .tp_basicsize = sizeof(CpSeqLayerObject), + .tp_dealloc = (destructor)cp_seqlayer_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = NULL, + .tp_members = CpSeqLayer_Members, + .tp_init = (initproc)cp_seqlayer_init, + .tp_new = (newfunc)cp_seqlayer_new, +}; + +//------------------------------------------------------------------------------- +// obj layer + +static PyObject* +cp_objlayer_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + CpObjLayerObject* self = (CpObjLayerObject*)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + self->ob_base.m_path = NULL; + self->m_obj = NULL; + self->ob_base.m_parent = NULL; + self->ob_base.m_state = NULL; + self->ob_base.m_class = CpLayerClass_Object; + return (PyObject*)self; +} + +static void +cp_objlayer_dealloc(CpObjLayerObject* self) +{ + Py_XDECREF(self->ob_base.m_path); + Py_XDECREF(self->m_obj); + Py_XDECREF(self->ob_base.m_parent); + Py_XDECREF(self->ob_base.m_state); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_objlayer_init(CpObjLayerObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "state", "path", "parent", NULL }; + PyObject *state = NULL, *path = NULL, *parent = NULL; + if (!PyArg_ParseTupleAndKeywords( + args, kw, "O|OO", kwlist, &state, &path, &parent)) + return -1; + + if (!PyObject_IsInstance(state, (PyObject*)(&CpState_Type))) { + PyErr_SetString(PyExc_TypeError, "state must be an instance of CpState"); + return -1; + } + + Py_XSETREF(self->ob_base.m_state, (CpStateObject*)Py_NewRef(state)); + Py_XSETREF(self->m_obj, (PyObject*)CpContext_New()); + Py_XSETREF(self->ob_base.m_path, Py_NewRef(path)); + Py_XSETREF(self->ob_base.m_parent, (CpLayerObject*)Py_NewRef(parent)); + return 0; +} + +static PyObject* +cp_objlayer_context_getattr(CpLayerObject* self, PyObject* args) +{ + _modulestate* state = get_global_module_state(); + PyObject *tmp = NULL, *lineValue = NULL, *lastKey = NULL, *key = NULL, + *obj = NULL; + if (!PyArg_ParseTuple(args, "O", &lineValue)) { + return NULL; + } + + PyObject* values = PyUnicode_Split(lineValue, state->str_path_delim, -1); + if (!values) { + return NULL; + } + + size_t length = PyList_Size(values); + if (length == 0) { + Py_XDECREF(values); + PyErr_SetString(PyExc_ValueError, "Empty path"); + return NULL; + } + + key = PyList_GetItem(values, 0); + if (!key) { + Py_XDECREF(values); + return NULL; + } + + tmp = PyObject_GenericGetAttr((PyObject*)self, key); + if (!tmp) { + PyErr_Clear(); + PyErr_Format(PyExc_ValueError, "CpObjLayer has no attribute '%s'", key); + Py_XDECREF(values); + return NULL; + } + + obj = tmp; + Py_XSETREF(lastKey, Py_XNewRef(key)); + for (size_t i = 1; i < length; i++) { + key = PyList_GetItem(values, i); + if (!key) { + Py_XDECREF(values); + return NULL; + } + + tmp = PyObject_GetAttr(obj, key); + if (!tmp) { + PyErr_Clear(); + PyErr_Format( + PyExc_ValueError, "'%s' has no attribute '%s'", lastKey, key); + Py_XDECREF(values); + return NULL; + } + + Py_XSETREF(obj, tmp); + Py_XSETREF(lastKey, Py_XNewRef(key)); + } + Py_XDECREF(values); + return obj; +} + +/*CpAPI*/ +CpObjLayerObject* +CpObjLayer_New(CpStateObject* state, CpLayerObject* parent) +{ + CpObjLayerObject* self = + (CpObjLayerObject*)CpObject_Create(&CpObjLayer_Type, "O", state); + if (!self) { + return NULL; + } + if (parent) { + Py_XSETREF(self->ob_base.m_parent, (CpLayerObject*)Py_XNewRef(parent)); + Py_XSETREF(self->ob_base.m_path, Py_XNewRef(parent->m_path)); + } + return self; +} + +/* docs */ + +/* type */ +static PyMemberDef CpObjLayer_Members[] = { + { + "obj", + T_OBJECT, + offsetof(CpObjLayerObject, m_obj), + 0, + }, + { NULL } /* Sentinel */ +}; + +static PyMethodDef CpObjLayer_Methods[] = { + { "__context_getattr__", + (PyCFunction)cp_objlayer_context_getattr, + METH_VARARGS }, + { NULL } /* Sentinel */ +}; + +PyTypeObject CpObjLayer_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpObjLayer_NAME), + .tp_basicsize = sizeof(CpLayerObject), + .tp_dealloc = (destructor)cp_objlayer_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = NULL, + .tp_methods = CpObjLayer_Methods, + .tp_members = CpObjLayer_Members, + .tp_init = (initproc)cp_objlayer_init, + .tp_new = (newfunc)cp_objlayer_new, +}; \ No newline at end of file diff --git a/src/module.c b/src/module.c index 9631c46..02fb1da 100644 --- a/src/module.c +++ b/src/module.c @@ -380,13 +380,22 @@ PyInit__C(void) CpModule_SetupType(&CpFieldAtom_Type); CpModule_SetupType(&CpFieldCAtom_Type); CpModule_SetupType(&CpLayer_Type); + CpModule_SetupType(&CpSeqLayer_Type); + CpModule_SetupType(&CpObjLayer_Type); CpModule_SetupType(&CpState_Type); CpModule_SetupType(&CpStructFieldInfo_Type); CpStruct_Type.tp_base = &CpFieldAtom_Type; CpModule_SetupType(&CpStruct_Type); - CpIntAtom_Type.tp_base = &CpFieldCAtom_Type; + // builtins setup + CpBuiltinAtom_Type.tp_base = &CpCAtom_Type; + CpModule_SetupType(&CpBuiltinAtom_Type); + + CpRepeatedAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpModule_SetupType(&CpRepeatedAtom_Type); + + CpIntAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpIntAtom_Type); CpFloatAtom_Type.tp_base = &CpFieldCAtom_Type; @@ -407,6 +416,8 @@ PyInit__C(void) CpConstAtom_Type.tp_base = &CpFieldCAtom_Type; CpModule_SetupType(&CpConstAtom_Type); + + // module setup m = PyModule_Create(&CpModule); if (!m) { @@ -430,10 +441,15 @@ PyInit__C(void) CpModule_AddObject("fieldatom", &CpFieldAtom_Type); CpModule_AddObject("fieldcatom", &CpFieldCAtom_Type); CpModule_AddObject("layer", &CpLayer_Type); + CpModule_AddObject(CpSeqLayer_NAME, &CpSeqLayer_Type); + CpModule_AddObject(CpObjLayer_NAME, &CpObjLayer_Type); CpModule_AddObject("State", &CpState_Type); CpModule_AddObject("fieldinfo", &CpStructFieldInfo_Type); CpModule_AddObject("Struct", &CpStruct_Type); + CpModule_AddObject(CpBuiltinAtom_NAME, &CpBuiltinAtom_Type); + CpModule_AddObject(CpRepeatedAtom_NAME, &CpRepeatedAtom_Type); + CpModule_AddObject(CpIntAtom_NAME, &CpIntAtom_Type); CpModule_AddObject(CpFloatAtom_NAME, &CpFloatAtom_Type); CpModule_AddObject(CpBoolAtom_NAME, &CpBoolAtom_Type); diff --git a/src/parsing_pack.c b/src/parsing_pack.c index 289ca9f..ea73f6e 100644 --- a/src/parsing_pack.c +++ b/src/parsing_pack.c @@ -82,10 +82,7 @@ _CpPack_EvalLength(CpLayerObject* layer, if (!sizeobj) { return -1; } - // TODO: explain why - layer->s_sequential = false; success = _Cp_Pack(sizeobj, start, layer); - layer->s_sequential = true; Py_DECREF(sizeobj); Py_DECREF(start); if (success < 0) { @@ -117,88 +114,90 @@ _CpPack_EvalLength(CpLayerObject* layer, int CpPack_Field(PyObject* op, CpFieldObject* field, CpLayerObject* layer) { - // we can assert that all provided objects are of the correct type - // REVISIT: really? - if (!op) { - PyErr_SetString(PyExc_ValueError, "object is NULL"); - return -1; - } - - CpLayer_AppendPath(layer, field->m_name); - if (!layer->m_path) { - return -1; - } - - int res = CpField_IsEnabled(field, (PyObject*)layer); - if (!res) { - // disabled fields are not packed - return 0; - } - if (res < 0) { - return -1; - } +#if 0 + // we can assert that all provided objects are of the correct type + // REVISIT: really? + if (!op) { + PyErr_SetString(PyExc_ValueError, "object is NULL"); + return -1; + } - PyObject *base_stream = NULL, *fallback = NULL; - CpStateObject* state = layer->m_state; + CpLayer_AppendPath(layer, field->m_name); + if (!layer->m_path) { + return -1; + } - Py_XSETREF(layer->m_field, Py_NewRef((PyObject*)field)); - layer->s_sequential = field->s_sequential; + int res = CpField_IsEnabled(field, (PyObject*)layer); + if (!res) { + // disabled fields are not packed + return 0; + } + if (res < 0) { + return -1; + } - Py_ssize_t offset = CpField_GetOffset(field, (PyObject*)layer); - if (offset < 0 && PyErr_Occurred()) { - return -1; - } + PyObject *base_stream = NULL, *fallback = NULL; + CpStateObject* state = layer->m_state; - if (offset == -1 || !field->s_keep_pos) { - if (!(fallback = CpState_Tell(layer->m_state))) { - return -1; - }; - } + Py_XSETREF(layer->m_field, Py_NewRef((PyObject*)field)); + layer->s_sequential = field->s_sequential; - if (offset >= 0) { - // We write the current field into a temporary memory buffer - // and add it after all processing has finished. - base_stream = Py_XNewRef(layer->m_state->m_io); - layer->m_state->m_io = PyObject_CallNoArgs(state->mod->BytesIO_Type); - if (!state->m_io) { + Py_ssize_t offset = CpField_GetOffset(field, (PyObject*)layer); + if (offset < 0 && PyErr_Occurred()) { return -1; } - } - if (!PyCallable_Check(field->m_atom)) { - return _Cp_Pack(op, field->m_atom, layer); - } else { - PyObject* res = PyObject_CallOneArg(field->m_atom, (PyObject*)layer); - if (!res) { - return -1; + if (offset == -1 || !field->s_keep_pos) { + if (!(fallback = CpState_Tell(layer->m_state))) { + return -1; + }; } - if (field->m_switch && field->m_switch != Py_None) { - PyObject* atom = CpField_EvalSwitch(field, res, (PyObject*)layer); - Py_DECREF(res); - if (!atom) { + if (offset >= 0) { + // We write the current field into a temporary memory buffer + // and add it after all processing has finished. + base_stream = Py_XNewRef(layer->m_state->m_io); + layer->m_state->m_io = PyObject_CallNoArgs(state->mod->BytesIO_Type); + if (!state->m_io) { return -1; } - if (_Cp_Pack(op, atom, layer) < 0) { + } + + if (!PyCallable_Check(field->m_atom)) { + return _Cp_Pack(op, field->m_atom, layer); + } else { + PyObject* res = PyObject_CallOneArg(field->m_atom, (PyObject*)layer); + if (!res) { return -1; } - } else - Py_DECREF(res); - } - if (offset == -1 || !field->s_keep_pos) { - if (!CpState_Seek(state, fallback, 0)) { - return -1; + if (field->m_switch && field->m_switch != Py_None) { + PyObject* atom = CpField_EvalSwitch(field, res, (PyObject*)layer); + Py_DECREF(res); + if (!atom) { + return -1; + } + if (_Cp_Pack(op, atom, layer) < 0) { + return -1; + } + } else + Py_DECREF(res); } - } - if (offset >= 0) { - if (PyObject_SetItem( - state->m_offset_table, PyLong_FromSsize_t(offset), state->m_io) < 0) - return -1; + if (offset == -1 || !field->s_keep_pos) { + if (!CpState_Seek(state, fallback, 0)) { + return -1; + } + } - Py_XSETREF(state->m_io, base_stream); - } + if (offset >= 0) { + if (PyObject_SetItem( + state->m_offset_table, PyLong_FromSsize_t(offset), state->m_io) < 0) + return -1; + + Py_XSETREF(state->m_io, base_stream); + } +#endif return 0; } @@ -206,99 +205,102 @@ CpPack_Field(PyObject* op, CpFieldObject* field, CpLayerObject* layer) int CpPack_Common(PyObject* op, PyObject* atom, CpLayerObject* layer) { - - int success; - if (!layer->s_sequential) { - return PyObject_CallMethod(atom, "__pack__", "OO", op, layer) ? 0 : -1; - } - - CpStateObject* state = layer->m_state; - if (PyObject_HasAttr(atom, state->mod->str___pack_many__)) { - // class explicitly defines __pack_many__ -> use it - PyObject* res = - CpAtom_Pack(atom, state->mod->str___pack_many__, op, (PyObject*)layer); - PyObject* exc = NULL; - if ((exc = PyErr_GetRaisedException(), - exc && PyErr_GivenExceptionMatches(exc, PyExc_NotImplementedError))) { - // Make sure this method continues to pack the given object - Py_XDECREF(exc); - PyErr_Clear(); - Py_XDECREF(res); - } else { - success = res ? 0 : -1; - Py_XDECREF(res); - Py_XDECREF(exc); - return success; +#if 0 + int success; + if (!layer->s_sequential) { + return PyObject_CallMethod(atom, "__pack__", "OO", op, layer) ? 0 : -1; } - } - if (!PySequence_Check(op)) { - PyErr_Format(PyExc_ValueError, "input object (%R) is not a sequence", op); - return -1; - } - - // TODO: explain why - CpLayerObject* seq_layer = CpLayer_New(state, layer); - if (!seq_layer) { - return -1; - } - seq_layer->s_sequential = false; - - Py_ssize_t size = PySequence_Size(op); - bool greedy = false; - Py_ssize_t layer_length = 0; - PyObject* length = - CpField_GetLength((CpFieldObject*)layer->m_field, (PyObject*)layer); - if (!length) { - return -1; - } - if (_CpPack_EvalLength(layer, length, size, &greedy, &layer_length) < 0) { - Py_XDECREF(length); - return -1; - } - Py_XDECREF(length); + CpStateObject* state = layer->m_state; + if (PyObject_HasAttr(atom, state->mod->str___pack_many__)) { + // class explicitly defines __pack_many__ -> use it + PyObject* res = + CpAtom_Pack(atom, state->mod->str___pack_many__, op, (PyObject*)layer); + PyObject* exc = NULL; + if ((exc = PyErr_GetRaisedException(), + exc && + PyErr_GivenExceptionMatches(exc, PyExc_NotImplementedError))) { + // Make sure this method continues to pack the given object + Py_XDECREF(exc); + PyErr_Clear(); + Py_XDECREF(res); + } else { + success = res ? 0 : -1; + Py_XDECREF(res); + Py_XDECREF(exc); + return success; + } + } - if (layer_length <= 0) { - // continue packing, here's nothing to store - Py_XDECREF(seq_layer); - return 0; - } - CpLayer_SetSequence(seq_layer, op, layer_length, greedy); + if (!PySequence_Check(op)) { + PyErr_Format(PyExc_ValueError, "input object (%R) is not a sequence", op); + return -1; + } - PyObject* obj = NULL; - for (seq_layer->m_index = 0; seq_layer->m_index < seq_layer->m_length; - seq_layer->m_index++) { - obj = PySequence_GetItem(op, seq_layer->m_index); - if (!obj) { + // TODO: explain why + CpLayerObject* seq_layer = CpLayer_New(state, layer); + if (!seq_layer) { return -1; } - seq_layer->m_path = PyUnicode_FromFormat( - "%s.%d", _PyUnicode_AsString(layer->m_path), seq_layer->m_index); - if (!seq_layer->m_path) { - Py_XDECREF(obj); + seq_layer->s_sequential = false; + + Py_ssize_t size = PySequence_Size(op); + bool greedy = false; + Py_ssize_t layer_length = 0; + PyObject* length = + CpField_GetLength((CpFieldObject*)layer->m_field, (PyObject*)layer); + if (!length) { return -1; } - success = _Cp_Pack(obj, atom, seq_layer); - Py_XSETREF(obj, NULL); - if (success < 0) { + if (_CpPack_EvalLength(layer, length, size, &greedy, &layer_length) < 0) { + Py_XDECREF(length); return -1; } - } - CpLayer_Invalidate(seq_layer); - return 0; + Py_XDECREF(length); -fail: - if (seq_layer) { + if (layer_length <= 0) { + // continue packing, here's nothing to store + Py_XDECREF(seq_layer); + return 0; + } + CpLayer_SetSequence(seq_layer, op, layer_length, greedy); + + PyObject* obj = NULL; + for (seq_layer->m_index = 0; seq_layer->m_index < seq_layer->m_length; + seq_layer->m_index++) { + obj = PySequence_GetItem(op, seq_layer->m_index); + if (!obj) { + return -1; + } + seq_layer->m_path = PyUnicode_FromFormat( + "%s.%d", _PyUnicode_AsString(layer->m_path), seq_layer->m_index); + if (!seq_layer->m_path) { + Py_XDECREF(obj); + return -1; + } + success = _Cp_Pack(obj, atom, seq_layer); + Py_XSETREF(obj, NULL); + if (success < 0) { + return -1; + } + } CpLayer_Invalidate(seq_layer); - } - return -1; + return 0; + + fail: + if (seq_layer) { + CpLayer_Invalidate(seq_layer); + } + return -1; +#endif + return 0; } /*CpAPI*/ int CpPack_Struct(PyObject* op, CpStructObject* struct_, CpLayerObject* layer) { - +#if 0 if (layer->s_sequential) { // TODO: explain why return CpPack_Common(op, (PyObject*)struct_, layer); @@ -394,14 +396,22 @@ CpPack_Struct(PyObject* op, CpStructObject* struct_, CpLayerObject* layer) if (obj_layer) { CpLayer_Invalidate(obj_layer); } - return res; +#endif + return 0; } /*CpAPI*/ int CpPack_CAtom(PyObject* op, CpCAtomObject* catom, CpLayerObject* layer) { - + if (catom->ob_pack == NULL) { + PyErr_Format(PyExc_NotImplementedError, + "The atom of type '%s' cannot be packed (missing __pack__)", + Py_TYPE(catom)->tp_name); + return -1; + } + return catom->ob_pack((PyObject*)catom, op, (PyObject*)layer); +#if 0 if (!layer->s_sequential) { if (catom->ob_pack == NULL) { PyErr_Format(PyExc_NotImplementedError, @@ -418,31 +428,10 @@ CpPack_CAtom(PyObject* op, CpCAtomObject* catom, CpLayerObject* layer) // "The atom of type '%s' cannot be packed (missing __pack_many__)", // Py_TYPE(catom)->tp_name); // return -1; - return CpPack_Common(op, (PyObject *)catom, layer); + return CpPack_Common(op, (PyObject*)catom, layer); } return catom->ob_pack_many((PyObject*)catom, op, (PyObject*)layer); -} - -/*CpAPI*/ -int -_Cp_Pack(PyObject* op, PyObject* atom, CpLayerObject* layer) -{ - // 1. the current context-sensitive variables must be stored - // elsewhere. - // - // 2. the current context-sensitive variables must be restored - // to their original values. - int success; - if (CpField_CheckExact(atom)) { - success = CpPack_Field(op, (CpFieldObject*)atom, layer); - } else if (CpStruct_CheckExact(atom)) { - success = CpPack_Struct(op, (CpStructObject*)atom, layer); - } else if (CpCAtom_Check(atom)) { - return CpPack_CAtom(op, (CpCAtomObject*)atom, layer); - } else { - success = CpPack_Common(op, atom, layer); - } - return success; +#endif } /*CpAPI*/ diff --git a/src/parsing_sizeof.c b/src/parsing_sizeof.c index 41ff7e3..5e77de7 100644 --- a/src/parsing_sizeof.c +++ b/src/parsing_sizeof.c @@ -60,7 +60,7 @@ CpSizeOf_Field(CpFieldObject* field, CpLayerObject* layer) } // prepare context - Py_XSETREF(layer->m_field, Py_NewRef(field)); + // Py_XSETREF(layer->m_field, Py_NewRef(field)); if (field->s_sequential) { count = CpField_GetLength(field, (PyObject*)layer); if (!count) { diff --git a/src/parsing_unpack.c b/src/parsing_unpack.c index 5287f88..9c16172 100644 --- a/src/parsing_unpack.c +++ b/src/parsing_unpack.c @@ -44,9 +44,7 @@ _CpUnpack_EvalLength(CpLayerObject* layer, PyErr_SetString(PyExc_ValueError, "start is None"); return -1; } - layer->s_sequential = false; Py_XSETREF(length, _Cp_Unpack(start, layer)); - layer->s_sequential = true; Py_DECREF(start); if (!length) { return -1; @@ -81,6 +79,7 @@ _CpUnpack_EvalLength(CpLayerObject* layer, PyObject* CpUnpack_Common(PyObject* op, CpLayerObject* layer) { +#if 0 if (!op || !layer) { PyErr_SetString(PyExc_ValueError, "input or layer object is NULL!"); return NULL; @@ -172,12 +171,16 @@ CpUnpack_Common(PyObject* op, CpLayerObject* layer) CpLayer_Invalidate(seq_layer); } return NULL; + +#endif + Py_RETURN_NONE; } /*CpAPI*/ PyObject* CpUnpack_Field(CpFieldObject* field, CpLayerObject* layer) { +#if 0 if (!field || !layer) { PyErr_SetString(PyExc_ValueError, "input or layer object is NULL!"); return NULL; @@ -261,12 +264,15 @@ CpUnpack_Field(CpFieldObject* field, CpLayerObject* layer) cleanup: Py_XDECREF(fallback); return obj; +#endif + Py_RETURN_NONE; } /*CpAPI*/ PyObject* CpUnpack_Struct(CpStructObject* struct_, CpLayerObject* layer) { +#if 0 if (layer->s_sequential) { // TODO: explain why return CpUnpack_Common((PyObject*)struct_, layer); @@ -350,27 +356,23 @@ CpUnpack_Struct(CpStructObject* struct_, CpLayerObject* layer) CpLayer_Invalidate(obj_layer); } return obj; -} - -/*CpAPI*/ -PyObject* -_Cp_Unpack(PyObject* atom, CpLayerObject* layer) -{ - if (CpField_CheckExact(atom)) { - return CpUnpack_Field((CpFieldObject*)atom, layer); - } else if (CpStruct_CheckExact(atom)) { - return CpUnpack_Struct((CpStructObject*)atom, layer); - } else if (CpCAtom_Check(atom)) { - return CpUnpack_CAtom((CpCAtomObject*)atom, layer); - } else { - return CpUnpack_Common(atom, layer); - } +#endif + Py_RETURN_NONE; } /*CpAPI*/ PyObject* CpUnpack_CAtom(CpCAtomObject* catom, CpLayerObject* layer) { + if (catom->ob_unpack == NULL) { + PyErr_Format( + PyExc_NotImplementedError, + "The atom of type '%s' cannot be unpacked (missing __unpack__)", + Py_TYPE(catom)->tp_name); + return NULL; + } + return catom->ob_unpack((PyObject*)catom, (PyObject*)layer); +#if 0 PyObject* result = NULL; if (!layer->s_sequential) { if (catom->ob_unpack == NULL) { @@ -394,6 +396,8 @@ CpUnpack_CAtom(CpCAtomObject* catom, CpLayerObject* layer) result = catom->ob_unpack_many((PyObject *)catom, (PyObject *)layer); } return result; +#endif + Py_RETURN_NONE; } /*CpAPI*/ diff --git a/src/state.c b/src/state.c index 2c5ce42..81e2fcf 100644 --- a/src/state.c +++ b/src/state.c @@ -3,248 +3,6 @@ #include "caterpillar/context.h" #include "structmember.h" -/* layer implementation */ -static PyObject* -cp_layer_new(PyTypeObject* type, PyObject* args, PyObject* kw) -{ - CpLayerObject* self; - self = (CpLayerObject*)type->tp_alloc(type, 0); - if (self == NULL) - return NULL; - - self->m_field = NULL; - self->m_obj = NULL; - self->m_value = NULL; - self->m_path = NULL; - self->m_sequence = NULL; - self->m_length = -1; - self->m_index = -1; - self->s_greedy = false; - self->s_sequential = false; - self->m_parent = NULL; - self->m_state = NULL; - return (PyObject*)self; -} - -static void -cp_layer_dealloc(CpLayerObject* self) -{ - Py_XDECREF(self->m_field); - Py_XDECREF(self->m_obj); - Py_XDECREF(self->m_value); - Py_XDECREF(self->m_path); - Py_XDECREF(self->m_sequence); - Py_XDECREF(self->m_state); - Py_XDECREF(self->m_parent); - Py_TYPE(self)->tp_free((PyObject*)self); -} - -static int -cp_layer_init(CpLayerObject* self, PyObject* args, PyObject* kw) -{ - static char* kwlist[] = { "state", "field", "obj", "value", - "path", "sequence", "parent", NULL }; - PyObject *state = NULL, *field = NULL, *obj = NULL, *value = NULL, - *path = NULL, *sequence = NULL, *parent = NULL, *next = NULL; - if (!PyArg_ParseTupleAndKeywords(args, - kw, - "O|OOOOOOO", - kwlist, - &state, - &field, - &obj, - &value, - &path, - &sequence, - &parent, - &next)) - return -1; - - if (!PyObject_IsInstance(state, (PyObject*)(&CpState_Type))) { - PyErr_SetString(PyExc_TypeError, "state must be an instance of CpState"); - return -1; - } - - Py_XSETREF(self->m_state, (CpStateObject*)Py_NewRef(state)); - _Cp_SetObj(self->m_field, field); - _Cp_SetObj(self->m_obj, obj); - _Cp_SetObj(self->m_value, value); - _Cp_SetObj(self->m_path, path); - _Cp_SetObj(self->m_sequence, sequence); - Py_XSETREF(self->m_parent, (CpLayerObject*)Py_XNewRef(parent)); - self->s_greedy = false; - self->m_index = -1; - self->m_length = -1; - self->s_sequential = false; - return 0; -} - -static PyObject* -cp_layer__context_getattr__(CpLayerObject* self, PyObject* args) -{ - _modulestate* state = get_global_module_state(); - PyObject *tmp = NULL, *lineValue = NULL, *lastKey = NULL, *key = NULL, - *obj = NULL; - if (!PyArg_ParseTuple(args, "O", &lineValue)) { - return NULL; - } - - PyObject* values = PyUnicode_Split(lineValue, state->str_path_delim, -1); - if (!values) { - return NULL; - } - - size_t length = PyList_Size(values); - if (length == 0) { - Py_XDECREF(values); - PyErr_SetString(PyExc_ValueError, "Empty path"); - return NULL; - } - - key = PyList_GetItem(values, 0); - if (!key) { - Py_XDECREF(values); - return NULL; - } - - tmp = PyObject_GenericGetAttr((PyObject*)self, key); - if (!tmp) { - PyErr_Clear(); - PyErr_Format(PyExc_ValueError, "CpLayer has no attribute '%s'", key); - Py_XDECREF(values); - return NULL; - } - - obj = tmp; - Py_XSETREF(lastKey, Py_XNewRef(key)); - for (size_t i = 1; i < length; i++) { - key = PyList_GetItem(values, i); - if (!key) { - Py_XDECREF(values); - return NULL; - } - - tmp = PyObject_GetAttr(obj, key); - if (!tmp) { - PyErr_Clear(); - PyErr_Format( - PyExc_ValueError, "'%s' has no attribute '%s'", lastKey, key); - Py_XDECREF(values); - return NULL; - } - - Py_XSETREF(obj, tmp); - Py_XSETREF(lastKey, Py_XNewRef(key)); - } - Py_XDECREF(values); - return obj; -} - -/* PUblic API */ - -/*CpAPI*/ -CpLayerObject* -CpLayer_New(CpStateObject* state, CpLayerObject* parent) -{ - CpLayerObject* self = - (CpLayerObject*)CpObject_Create(&CpLayer_Type, "O", state); - if (!self) { - return NULL; - } - if (parent) { - Py_XSETREF(self->m_parent, (CpLayerObject*)Py_XNewRef(parent)); - // automatically inherit field object - // TODO: document this - Py_XSETREF(self->m_field, Py_XNewRef(parent->m_field)); - } - return self; -} - -/*CpAPI*/ -int -CpLayer_SetSequence(CpLayerObject* self, - PyObject* sequence, - Py_ssize_t length, - int8_t greedy) -{ - if (!sequence || !self) { - PyErr_SetString(PyExc_ValueError, "sequence and layer must be non-null"); - return -1; - } - - Py_XSETREF(self->m_sequence, Py_NewRef(sequence)); - self->m_length = length; - self->s_greedy = greedy; - self->m_index = 0; - self->s_sequential = false; - return 0; -} - -/*CpAPI*/ -int -CpLayer_Invalidate(CpLayerObject* self) -{ - if (!self) { - PyErr_SetString(PyExc_ValueError, "layer must be non-null"); - return -1; - } - - Py_XSETREF(self->m_parent, NULL); - Py_XSETREF(self->m_field, NULL); - Py_XSETREF(self->m_value, NULL); - Py_XSETREF(self->m_sequence, NULL); - Py_XSETREF(self->m_obj, NULL); - Py_XSETREF(self->m_state, NULL); - Py_XSETREF(self->m_path, NULL); - Py_CLEAR(self); - return 0; -} - -/* docs */ - -/* type */ -static PyMemberDef CpLayer_Members[] = { - { "state", T_OBJECT, offsetof(CpLayerObject, m_state), READONLY, "state" }, - { "field", T_OBJECT, offsetof(CpLayerObject, m_field), 0, "field" }, - { "obj", T_OBJECT, offsetof(CpLayerObject, m_obj), 0, "obj" }, - { "value", T_OBJECT, offsetof(CpLayerObject, m_value), 0, "value" }, - { "path", T_OBJECT, offsetof(CpLayerObject, m_path), 0, "path" }, - { "sequence", T_OBJECT, offsetof(CpLayerObject, m_sequence), 0, "sequence" }, - { "length", - T_PYSSIZET, - offsetof(CpLayerObject, m_length), - READONLY, - "length" }, - { "index", T_PYSSIZET, offsetof(CpLayerObject, m_index), READONLY, "index" }, - { "greedy", T_BOOL, offsetof(CpLayerObject, s_greedy), READONLY, "greedy" }, - { "sequential", - T_INT, - offsetof(CpLayerObject, s_sequential), - READONLY, - "sequential" }, - { "parent", T_OBJECT, offsetof(CpLayerObject, m_parent), 0, "parent" }, - { NULL } /* Sentinel */ -}; - -static PyMethodDef CpLayer_Methods[] = { - { "__context_getattr__", - (PyCFunction)cp_layer__context_getattr__, - METH_VARARGS }, - { NULL } /* Sentinel */ -}; - -PyTypeObject CpLayer_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(layer), - .tp_basicsize = sizeof(CpLayerObject), - .tp_dealloc = (destructor)cp_layer_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = NULL, - .tp_methods = CpLayer_Methods, - .tp_members = CpLayer_Members, - .tp_init = (initproc)cp_layer_init, - .tp_new = (newfunc)cp_layer_new, -}; - //------------------------------------------------------------------------------- // state static PyObject* From 6e00b01290d8304910b32e08f63026dd00b91780 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 8 Sep 2024 09:06:23 +0200 Subject: [PATCH 05/29] CpConditionAtom implementation --- CMakeLists.txt | 1 + src/arch.c | 12 +- src/atomimpl/builtins/conditionatomobj.c | 190 ++++++++++++++++++ src/atomimpl/builtins/repeatedatomobj.c | 16 +- src/atomimpl/floatatomobj.c | 20 +- src/atomimpl/intatomobj.c | 21 +- src/caterpillar/include/caterpillar/arch.h | 31 +-- .../include/caterpillar/atoms/builtins.h | 24 +++ .../include/caterpillar/caterpillarapi.h | 10 + src/caterpillar/include/caterpillar/module.h | 1 + src/caterpillarapi.c | 7 +- src/code_gen/caterpillar_api.py | 6 + src/module.c | 13 +- 13 files changed, 305 insertions(+), 47 deletions(-) create mode 100644 src/atomimpl/builtins/conditionatomobj.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 157f2a2..3764f00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ python_add_library( src/atomimpl/builtins/builtinatomobj.c src/atomimpl/builtins/repeatedatomobj.c + src/atomimpl/builtins/conditionatomobj.c WITH_SOABI ) diff --git a/src/arch.c b/src/arch.c index f832e47..823058c 100644 --- a/src/arch.c +++ b/src/arch.c @@ -1,6 +1,6 @@ /* CpArch and CpEndian */ -#include "caterpillar/caterpillar.h" #include "caterpillar/arch.h" +#include "caterpillar/caterpillar.h" #include "caterpillar/field.h" #include "structmember.h" @@ -101,7 +101,7 @@ PyTypeObject CpArch_Type = { .tp_dealloc = (destructor)cp_arch_dealloc, .tp_repr = (reprfunc)cp_arch_repr, .tp_hash = (hashfunc)cp_arch_hash, - .tp_richcompare =(richcmpfunc)cp_arch_richcmp, + .tp_richcompare = (richcmpfunc)cp_arch_richcmp, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_doc = cp_arch_doc, .tp_members = CpArch_Members, @@ -197,12 +197,12 @@ cp_endian_hash(CpEndianObject* self) static PyObject* cp_endian_as_number_add(CpEndianObject* self, PyObject* atom) { - CpFieldObject* field = (CpFieldObject*)CpField_New(atom); - if (!field) { + if (!atom) { + PyErr_SetString(PyExc_ValueError, "atom must be non-null"); return NULL; } - _Cp_SetObj(field->m_endian, self); - return (PyObject*)field; + + return CpEndian_SetEndian(atom, self); } /* Doc strings */ diff --git a/src/atomimpl/builtins/conditionatomobj.c b/src/atomimpl/builtins/conditionatomobj.c new file mode 100644 index 0000000..8dd2a83 --- /dev/null +++ b/src/atomimpl/builtins/conditionatomobj.c @@ -0,0 +1,190 @@ +/* condition atom implementation */ + +#include "caterpillar/caterpillar.h" + +#include + +/* impl */ + +static PyObject* +cp_conditionatom_type(CpConditionAtomObject* self) +{ + PyObject* atom_type = CpTypeOf(self->m_atom); + if (!atom_type) + return NULL; + + _modulestate* state = get_global_module_state(); + Py_XSETREF(atom_type, PyObject_GetItem(state->Optional_Type, atom_type)); + return atom_type; +} + +static PyObject* +cp_conditionatom_size(CpConditionAtomObject* self, CpLayerObject* layer) +{ + return _Cp_SizeOf(self->m_atom, layer); +} + +static PyObject* +cp_conditionatom_repr(CpConditionAtomObject* self) +{ + return PyUnicode_FromFormat( + "", self->m_condition, self->m_atom); +} + +static PyObject* +cp_conditionatom_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + CpConditionAtomObject* self = (CpConditionAtomObject*)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpConditionAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpConditionAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_conditionatom_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_conditionatom_type; + return (PyObject*)self; +} + +static void +cp_conditionatom_dealloc(CpConditionAtomObject* self) +{ + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = NULL; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = NULL; + CpBuiltinAtom_CATOM(self).ob_type = NULL; + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_conditionatom_init(CpConditionAtomObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "condition", "atom", NULL }; + PyObject *condition = NULL, *atom = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kw, "OO", kwlist, &condition, &atom)) + return -1; + + _Cp_SetObj(self->m_atom, atom); + _Cp_SetObj(self->m_condition, condition); + return 0; +} + +static PyObject* +cp_conditionatom_set_byteorder(CpConditionAtomObject* self, + PyObject* args, + PyObject* kw) +{ + static char* kwlist[] = { "byteorder", NULL }; + PyObject* byteorder = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &byteorder)) { + return NULL; + } + if (!CpEndian_Check(byteorder)) { + PyErr_SetString(PyExc_TypeError, "byteorder must be an Endian object"); + return NULL; + } + + PyObject* new_atom = + CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder); + if (!new_atom) { + return NULL; + } + _Cp_SetObj(self->m_atom, new_atom); + return (PyObject*)self; +} + +static PyObject* +cp_conditionatom_is_enabled(CpConditionAtomObject* self, PyObject* args) +{ + PyObject* layer = NULL; + if (!PyArg_ParseTuple(args, "O", &layer)) { + return NULL; + } + + if (CpConditionAtom_IsEnabled(self, layer)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +/* Public API */ + +/*CpAPI*/ +int +CpConditionAtom_IsEnabled(CpConditionAtomObject* self, PyObject* context) +{ + if (self->m_condition == NULL) { + return true; + } + + if (PyCallable_Check(self->m_condition)) { + PyObject* result = PyObject_CallOneArg(self->m_condition, context); + if (result == NULL) { + return -1; + } + int truth = PyObject_IsTrue(result); + Py_DECREF(result); + return truth; + } + + return PyObject_IsTrue(self->m_condition); +} + +/*CpAPI*/ +int +CpConditionAtom_Pack(CpConditionAtomObject* self, PyObject* op, PyObject* layer) +{ + if (!CpConditionAtom_IsEnabled(self, layer)) { + return 0; + } + + return _Cp_Pack(op, self->m_atom, (CpLayerObject*)layer); +} + +/*CpAPI*/ +PyObject* +CpConditionAtom_Unpack(CpConditionAtomObject* self, CpLayerObject* layer) +{ + if (!CpConditionAtom_IsEnabled(self, (PyObject*)layer)) { + return NULL; + } + return _Cp_Unpack((PyObject*)self, layer); +} + +/* docs */ + +/* members */ +static PyMemberDef CpRepeatedAtom_Members[] = { + { "condition", + T_OBJECT, + offsetof(CpConditionAtomObject, m_condition), + READONLY }, + { "atom", T_OBJECT, offsetof(CpConditionAtomObject, m_atom), READONLY }, + { NULL } +}; + +/* methods */ +static PyMethodDef CpConditionAtom_Methods[] = { + _CpEndian_ImplSetByteorder_MethDef(conditionatom, NULL), + { "is_enabled", (PyCFunction)cp_conditionatom_is_enabled, METH_VARARGS }, + { NULL } +}; + +PyTypeObject CpConditionAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpConditionAtom_NAME), + .tp_basicsize = sizeof(CpConditionAtomObject), + .tp_dealloc = (destructor)cp_conditionatom_dealloc, + .tp_init = (initproc)cp_conditionatom_init, + .tp_members = CpRepeatedAtom_Members, + .tp_methods = CpConditionAtom_Methods, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = NULL, + .tp_new = (newfunc)cp_conditionatom_new, + .tp_repr = (reprfunc)cp_conditionatom_repr, +}; diff --git a/src/atomimpl/builtins/repeatedatomobj.c b/src/atomimpl/builtins/repeatedatomobj.c index abcb283..387f6c7 100644 --- a/src/atomimpl/builtins/repeatedatomobj.c +++ b/src/atomimpl/builtins/repeatedatomobj.c @@ -78,7 +78,7 @@ cp_repeatedatomobj_init(CpRepeatedAtomObject* self, } static PyObject* -cp_repeatedatomobj_set_byteorder(CpRepeatedAtomObject* self, +cp_repeatedatom_set_byteorder(CpRepeatedAtomObject* self, PyObject* args, PyObject* kw) { @@ -92,8 +92,13 @@ cp_repeatedatomobj_set_byteorder(CpRepeatedAtomObject* self, return NULL; } - return CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder) ? NULL - : Py_None; + PyObject* new_atom = + CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder); + if (!new_atom) { + return NULL; + } + _Cp_SetObj(self->m_atom, new_atom); + return (PyObject*)self; } /* Public API */ @@ -286,10 +291,7 @@ static PyMemberDef CpRepeatedAtom_Members[] = { }; static PyMethodDef CpRepeatedAtom_Methods[] = { - { "__set_byteorder__", - (PyCFunction)cp_repeatedatomobj_set_byteorder, - METH_VARARGS, - NULL }, + _CpEndian_ImplSetByteorder_MethDef(repeatedatom, NULL), { NULL, } diff --git a/src/atomimpl/floatatomobj.c b/src/atomimpl/floatatomobj.c index a858c55..d4b5038 100644 --- a/src/atomimpl/floatatomobj.c +++ b/src/atomimpl/floatatomobj.c @@ -69,7 +69,25 @@ cp_floatatom_init(CpFloatAtomObject* self, PyObject* args, PyObject* kwds) return 0; } -_CpEndian_ImplSetByteorder(floatatom, self->_m_little_endian); +static PyObject* +cp_floatatom_set_byteorder(CpFloatAtomObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "byteorder", NULL }; + PyObject* byteorder = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &byteorder)) { + return NULL; + } + if (!CpEndian_Check(byteorder)) { + PyErr_SetString(PyExc_TypeError, "byteorder must be an Endian object"); + return NULL; + } + + return CpObject_Create(&CpIntAtom_Type, + "Ii", + self->_m_bits, + CpEndian_IsLittleEndian((CpEndianObject*)byteorder, + get_global_module_state())); +} /* Public API */ /*CpAPI*/ diff --git a/src/atomimpl/intatomobj.c b/src/atomimpl/intatomobj.c index 4932a2b..9959a64 100644 --- a/src/atomimpl/intatomobj.c +++ b/src/atomimpl/intatomobj.c @@ -89,7 +89,26 @@ cp_intatom_repr(CpIntAtomObject* self) return PyUnicode_FromFormat("<%ce %cint%d>", endian, sign, self->_m_bits); } -_CpEndian_ImplSetByteorder(intatom, self->_m_little_endian); +static PyObject* +cp_intatom_set_byteorder(CpIntAtomObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "byteorder", NULL }; + PyObject* byteorder = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &byteorder)) { + return NULL; + } + if (!CpEndian_Check(byteorder)) { + PyErr_SetString(PyExc_TypeError, "byteorder must be an Endian object"); + return NULL; + } + + return CpObject_Create(&CpIntAtom_Type, + "Iii", + (unsigned int)self->_m_bits, + self->_m_signed, + CpEndian_IsLittleEndian((CpEndianObject*)byteorder, + get_global_module_state())); +} /* Public API */ /*CpAPI*/ diff --git a/src/caterpillar/include/caterpillar/arch.h b/src/caterpillar/include/caterpillar/arch.h index f420e39..6916978 100644 --- a/src/caterpillar/include/caterpillar/arch.h +++ b/src/caterpillar/include/caterpillar/arch.h @@ -95,44 +95,25 @@ struct _endianobj * @param endian the new byteorder * @return 0 on success, -1 on error */ -inline int +static inline PyObject* CpEndian_SetEndian(PyObject* op, CpEndianObject* endian) { PyObject* attr = PyObject_GetAttrString(op, "__set_byteorder__"); if (!attr) { - return -1; + return NULL; } PyObject* ret = PyObject_CallOneArg(attr, (PyObject*)endian); if (!ret) { - return -1; + return NULL; } - Py_DECREF(ret); Py_DECREF(attr); - return 0; + return ret; } -#define _CpEndian_ImplSetByteorder(name, field) \ - static PyObject* cp_##name##_set_byteorder( \ - CpFloatAtomObject* self, PyObject* args, PyObject* kw) \ - { \ - static char* kwlist[] = { "byteorder", NULL }; \ - PyObject* byteorder = NULL; \ - if (!PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &byteorder)) { \ - return NULL; \ - } \ - if (!CpEndian_Check(byteorder)) { \ - PyErr_SetString(PyExc_TypeError, "byteorder must be an Endian object"); \ - return NULL; \ - } \ - field = CpEndian_IsLittleEndian((CpEndianObject*)byteorder, \ - get_global_module_state()); \ - Py_RETURN_NONE; \ - } - #define _CpEndian_ImplSetByteorder_MethDef(name, docs) \ { \ - "__set_byteorder__", (PyCFunction)cp_##name##_set_byteorder, METH_VARARGS, \ - (docs) \ + "__set_byteorder__", (PyCFunction)cp_##name##_set_byteorder, \ + METH_VARARGS | METH_KEYWORDS, (docs) \ } #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/builtins.h b/src/caterpillar/include/caterpillar/atoms/builtins.h index 6136d70..ccc311b 100644 --- a/src/caterpillar/include/caterpillar/atoms/builtins.h +++ b/src/caterpillar/include/caterpillar/atoms/builtins.h @@ -59,4 +59,28 @@ CpRepeatedAtom_New(PyObject* atom, PyObject* length) return (CpRepeatedAtomObject*)CpObject_Create(&CpRepeatedAtom_Type, "OO", atom, length); } +//------------------------------------------------------------------------------ +// Conditional +struct _conditionatomobj +{ + CpBuiltinAtom_HEAD + + /// Stores a reference to the actual parsing struct that will be used + /// to parse or build our data. This attribute is never null. + PyObject *m_atom; + + /// A constant or dynamic value to represent the condition. + PyObject *m_condition; +}; + +#define CpConditionAtom_NAME "condition" +#define CpConditionAtom_CheckExact(op) Py_IS_TYPE((op), &CpConditionAtom_Type) +#define CpConditionAtom_Check(op) PyObject_IsType((op), &CpConditionAtom_Type) + +static inline CpConditionAtomObject * +CpConditionAtom_New(PyObject* atom, PyObject* condition) +{ + return (CpConditionAtomObject*)CpObject_Create(&CpConditionAtom_Type, "OO", atom, condition); +} + #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index 6daa2ef..4387c7e 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -82,6 +82,8 @@ struct _seqlayerobj; typedef struct _seqlayerobj CpSeqLayerObject; struct _objlayerobj; typedef struct _objlayerobj CpObjLayerObject; +struct _conditionatomobj; +typedef struct _conditionatomobj CpConditionAtomObject; #ifdef _CPMODULE @@ -122,6 +124,7 @@ extern PyTypeObject CpBuiltinAtom_Type; extern PyTypeObject CpRepeatedAtom_Type; extern PyTypeObject CpSeqLayer_Type; extern PyTypeObject CpObjLayer_Type; +extern PyTypeObject CpConditionAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -196,6 +199,9 @@ PyObject* CpConstAtom_Unpack(CpConstAtomObject* self, CpLayerObject* layer); int CpRepeatedAtom_Pack(CpRepeatedAtomObject* self,PyObject* op,CpLayerObject* layer); PyObject* CpRepeatedAtom_Unpack(CpRepeatedAtomObject* self, CpLayerObject* layer); PyObject* CpRepeatedAtom_GetLength(CpRepeatedAtomObject* self, PyObject* context); +int CpConditionAtom_Pack(CpConditionAtomObject* self, PyObject* op, PyObject* layer); +PyObject* CpConditionAtom_Unpack(CpConditionAtomObject* self, CpLayerObject* layer); +int CpConditionAtom_IsEnabled(CpConditionAtomObject* self, PyObject* context); #else @@ -237,6 +243,7 @@ caterpillar_api.py #define CpRepeatedAtom_Type (*(PyTypeObject *)Cp_API[29]) #define CpSeqLayer_Type (*(PyTypeObject *)Cp_API[30]) #define CpObjLayer_Type (*(PyTypeObject *)Cp_API[31]) +#define CpConditionAtom_Type (*(PyTypeObject *)Cp_API[32]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -311,6 +318,9 @@ caterpillar_api.py #define CpRepeatedAtom_Pack (*((int (*)(CpRepeatedAtomObject* self,PyObject* op,CpLayerObject* layer)))Cp_API[136]) #define CpRepeatedAtom_Unpack (*((PyObject* (*)(CpRepeatedAtomObject* self, CpLayerObject* layer)))Cp_API[137]) #define CpRepeatedAtom_GetLength (*((PyObject* (*)(CpRepeatedAtomObject* self, PyObject* context)))Cp_API[138]) +#define CpConditionAtom_Pack (*((int (*)(CpConditionAtomObject* self, PyObject* op, PyObject* layer)))Cp_API[139]) +#define CpConditionAtom_Unpack (*((PyObject* (*)(CpConditionAtomObject* self, CpLayerObject* layer)))Cp_API[140]) +#define CpConditionAtom_IsEnabled (*((int (*)(CpConditionAtomObject* self, PyObject* context)))Cp_API[141]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillar/include/caterpillar/module.h b/src/caterpillar/include/caterpillar/module.h index 2d83c00..66ee389 100644 --- a/src/caterpillar/include/caterpillar/module.h +++ b/src/caterpillar/include/caterpillar/module.h @@ -52,6 +52,7 @@ struct _modulestate // typing constants PyObject* Any_Type; PyObject* List_Type; + PyObject* Optional_Type; PyObject* Union_Type; PyObject *BytesIO_Type; diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index aecfc89..317edd9 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -35,7 +35,7 @@ void *Cp_API[] = { (void *) &CpRepeatedAtom_Type, (void *) &CpSeqLayer_Type, (void *) &CpObjLayer_Type, - NULL, + (void *) &CpConditionAtom_Type, NULL, NULL, NULL, @@ -141,6 +141,9 @@ void *Cp_API[] = { (void *) &CpConstAtom_Unpack, (void *) &CpRepeatedAtom_Pack, (void *) &CpRepeatedAtom_Unpack, - (void *) &CpRepeatedAtom_GetLength + (void *) &CpRepeatedAtom_GetLength, + (void *) &CpConditionAtom_Pack, + (void *) &CpConditionAtom_Unpack, + (void *) &CpConditionAtom_IsEnabled }; diff --git a/src/code_gen/caterpillar_api.py b/src/code_gen/caterpillar_api.py index caa68bd..4406a38 100644 --- a/src/code_gen/caterpillar_api.py +++ b/src/code_gen/caterpillar_api.py @@ -46,6 +46,7 @@ "_repeatedatomobj": "CpRepeatedAtomObject", "_seqlayerobj": "CpSeqLayerObject", "_objlayerobj": "CpObjLayerObject", + "_conditionatomobj": "CpConditionAtomObject", } cp_type_api = { @@ -81,6 +82,7 @@ "CpRepeatedAtom_Type": (29,), "CpSeqLayer_Type": (30,), "CpObjLayer_Type": (31,), + "CpConditionAtom_Type": (32,), } cp_func_api = { @@ -164,6 +166,9 @@ "CpRepeatedAtom_Pack": 136, "CpRepeatedAtom_Unpack": 137, "CpRepeatedAtom_GetLength": 138, + "CpConditionAtom_Pack": 139, + "CpConditionAtom_Unpack": 140, + "CpConditionAtom_IsEnabled": 141, } API_SRC = [ @@ -188,6 +193,7 @@ "atomimpl/constatomobj.c", "atomimpl/builtins/builtinatomobj.c", "atomimpl/builtins/repeatedatomobj.c", + "atomimpl/builtins/conditionatomobj.c", ] diff --git a/src/module.c b/src/module.c index 02fb1da..63f0028 100644 --- a/src/module.c +++ b/src/module.c @@ -274,6 +274,7 @@ cp_module_clear(PyObject* m) Py_CLEAR(state->Any_Type); Py_CLEAR(state->List_Type); Py_CLEAR(state->Union_Type); + Py_CLEAR(state->Optional_Type); // sttings Py_CLEAR(state->str_path_delim); @@ -388,12 +389,14 @@ PyInit__C(void) CpStruct_Type.tp_base = &CpFieldAtom_Type; CpModule_SetupType(&CpStruct_Type); - // builtins setup + // builtins setup CpBuiltinAtom_Type.tp_base = &CpCAtom_Type; - CpModule_SetupType(&CpBuiltinAtom_Type); - CpRepeatedAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpConditionAtom_Type.tp_base = &CpBuiltinAtom_Type; + + CpModule_SetupType(&CpBuiltinAtom_Type); CpModule_SetupType(&CpRepeatedAtom_Type); + CpModule_SetupType(&CpConditionAtom_Type); CpIntAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpIntAtom_Type); @@ -416,8 +419,6 @@ PyInit__C(void) CpConstAtom_Type.tp_base = &CpFieldCAtom_Type; CpModule_SetupType(&CpConstAtom_Type); - - // module setup m = PyModule_Create(&CpModule); if (!m) { @@ -449,6 +450,7 @@ PyInit__C(void) CpModule_AddObject(CpBuiltinAtom_NAME, &CpBuiltinAtom_Type); CpModule_AddObject(CpRepeatedAtom_NAME, &CpRepeatedAtom_Type); + CpModule_AddObject(CpConditionAtom_NAME, &CpConditionAtom_Type); CpModule_AddObject(CpIntAtom_NAME, &CpIntAtom_Type); CpModule_AddObject(CpFloatAtom_NAME, &CpFloatAtom_Type); @@ -611,6 +613,7 @@ PyInit__C(void) CpModuleState_Set(Any_Type, PyObject_GetAttrString(typing, "Any")); CpModuleState_Set(List_Type, PyObject_GetAttrString(typing, "List")); CpModuleState_Set(Union_Type, PyObject_GetAttrString(typing, "Union")); + CpModuleState_Set(Optional_Type, PyObject_GetAttrString(typing, "Optional")); Py_XDECREF(typing); // regex setup From bf727f2101be52e23a3115dba2cee9e0ca679063 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 8 Sep 2024 12:23:50 +0200 Subject: [PATCH 06/29] Changed CAPI definition file --- + Implementation for native switch and condition --- CMakeLists.txt | 1 + docs/sphinx/source/conf.py | 2 +- .../sphinx/source/extensions/c_annotations.py | 18 +- src/atomimpl/builtins/builtinatomobj.c | 35 ++- src/atomimpl/builtins/conditionatomobj.c | 21 +- src/atomimpl/builtins/repeatedatomobj.c | 11 +- src/atomimpl/builtins/switchatomobj.c | 199 +++++++++++++++ src/capi.dat | 175 +++++++++++++ src/caterpillar/include/caterpillar/arch.h | 11 + .../include/caterpillar/atoms/builtins.h | 29 ++- .../include/caterpillar/caterpillarapi.h | 50 ++-- src/caterpillarapi.c | 9 +- src/code_gen/caterpillar_api.py | 232 ++++-------------- src/module.c | 3 + 14 files changed, 548 insertions(+), 248 deletions(-) create mode 100644 src/atomimpl/builtins/switchatomobj.c create mode 100644 src/capi.dat diff --git a/CMakeLists.txt b/CMakeLists.txt index 3764f00..187242d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,7 @@ python_add_library( src/atomimpl/builtins/builtinatomobj.c src/atomimpl/builtins/repeatedatomobj.c src/atomimpl/builtins/conditionatomobj.c + src/atomimpl/builtins/switchatomobj.c WITH_SOABI ) diff --git a/docs/sphinx/source/conf.py b/docs/sphinx/source/conf.py index aa9a9ce..c6c7136 100644 --- a/docs/sphinx/source/conf.py +++ b/docs/sphinx/source/conf.py @@ -61,7 +61,7 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False -refcount_file = 'extensions/refcounts.dat' +refcount_file = '../../../src/capi.dat' # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/docs/sphinx/source/extensions/c_annotations.py b/docs/sphinx/source/extensions/c_annotations.py index 4d01d2c..419691c 100644 --- a/docs/sphinx/source/extensions/c_annotations.py +++ b/docs/sphinx/source/extensions/c_annotations.py @@ -27,20 +27,20 @@ def __init__(self, refcount_filename): if line[:1] in ("", "#"): # blank lines and comments continue - parts = line.split(":", 2) - if len(parts) != 3: - raise ValueError("Wrong field count in %r" % line) - function, type, refcount = parts + def_type, *parts = line.split(":") + if def_type != "func": + continue + + index, name, rtype, refcount = parts # Get the entry, creating it if needed: - try: - entry = self.refcount_data[function] - except KeyError: - entry = self.refcount_data[function] = RCEntry(function) + entry = self.refcount_data.get(name) + if not entry: + entry = self.refcount_data[name] = RCEntry(name) if not refcount or refcount == "null": refcount = None else: refcount = int(refcount) - entry.result_type = type + entry.result_type = rtype entry.result_refs = refcount def add_annotations(self, app, doctree): diff --git a/src/atomimpl/builtins/builtinatomobj.c b/src/atomimpl/builtins/builtinatomobj.c index f45cc2a..d3f3ed0 100644 --- a/src/atomimpl/builtins/builtinatomobj.c +++ b/src/atomimpl/builtins/builtinatomobj.c @@ -6,7 +6,7 @@ /* impl */ static PyObject* -cp_builtinatomobj_new(PyTypeObject* type, PyObject* args, PyObject* kw) +cp_builtinatom_new(PyTypeObject* type, PyObject* args, PyObject* kw) { CpBuiltinAtomObject* self; self = (CpBuiltinAtomObject*)type->tp_alloc(type, 0); @@ -23,7 +23,7 @@ cp_builtinatomobj_new(PyTypeObject* type, PyObject* args, PyObject* kw) } static void -cp_builtinatomobj_dealloc(CpBuiltinAtomObject* self) +cp_builtinatom_dealloc(CpBuiltinAtomObject* self) { self->ob_base.ob_bits = NULL; self->ob_base.ob_pack = NULL; @@ -36,28 +36,45 @@ cp_builtinatomobj_dealloc(CpBuiltinAtomObject* self) } static int -cp_builtinatomobj_init(CpBuiltinAtomObject* self, PyObject* args, PyObject* kw) +cp_builtinatom_init(CpBuiltinAtomObject* self, PyObject* args, PyObject* kw) { _Cp_InitNoArgs(CpBuiltinAtomObject, args, kw); } // TODO member methods static PyObject* -cp_builtinatomobj_as_mapping_getitem(CpBuiltinAtomObject* self, - PyObject* length) +cp_builtinatom_as_mapping_getitem(CpBuiltinAtomObject* self, PyObject* length) { return (PyObject*)CpRepeatedAtom_New((PyObject*)self, length); } +static PyObject* +cp_builtinatom_as_number_rshift(PyObject* self, PyObject* other) +{ + return (PyObject*)CpSwitchAtom_New(self, other); +} + +static PyObject* +cp_builtinatom_as_number_floordiv(PyObject* self, PyObject* other) +{ + return (PyObject*)CpConditionAtom_New(self, other); +} + static PyMappingMethods CpFieldAtom_MappingMethods = { - .mp_subscript = (binaryfunc)cp_builtinatomobj_as_mapping_getitem, + .mp_subscript = (binaryfunc)cp_builtinatom_as_mapping_getitem, +}; + +static PyNumberMethods CpField_NumberMethods = { + .nb_rshift = (binaryfunc)cp_builtinatom_as_number_rshift, + .nb_floor_divide = (binaryfunc)cp_builtinatom_as_number_floordiv, }; PyTypeObject CpBuiltinAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) "CpBuiltinAtom", // tp_name .tp_basicsize = sizeof(CpBuiltinAtomObject), - .tp_dealloc = (destructor)cp_builtinatomobj_dealloc, - .tp_init = (initproc)cp_builtinatomobj_init, - .tp_new = (newfunc)cp_builtinatomobj_new, + .tp_dealloc = (destructor)cp_builtinatom_dealloc, + .tp_init = (initproc)cp_builtinatom_init, + .tp_new = (newfunc)cp_builtinatom_new, .tp_as_mapping = &CpFieldAtom_MappingMethods, + .tp_as_number = &CpField_NumberMethods, }; diff --git a/src/atomimpl/builtins/conditionatomobj.c b/src/atomimpl/builtins/conditionatomobj.c index 8dd2a83..ef3da17 100644 --- a/src/atomimpl/builtins/conditionatomobj.c +++ b/src/atomimpl/builtins/conditionatomobj.c @@ -28,7 +28,7 @@ static PyObject* cp_conditionatom_repr(CpConditionAtomObject* self) { return PyUnicode_FromFormat( - "", self->m_condition, self->m_atom); + "] %R>", Py_TYPE(self->m_condition)->tp_name, self->m_atom); } static PyObject* @@ -58,6 +58,8 @@ cp_conditionatom_dealloc(CpConditionAtomObject* self) CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; CpBuiltinAtom_CATOM(self).ob_size = NULL; CpBuiltinAtom_CATOM(self).ob_type = NULL; + Py_CLEAR(self->m_atom); + Py_CLEAR(self->m_condition); Py_TYPE(self)->tp_free((PyObject*)self); } @@ -80,16 +82,7 @@ cp_conditionatom_set_byteorder(CpConditionAtomObject* self, PyObject* args, PyObject* kw) { - static char* kwlist[] = { "byteorder", NULL }; - PyObject* byteorder = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &byteorder)) { - return NULL; - } - if (!CpEndian_Check(byteorder)) { - PyErr_SetString(PyExc_TypeError, "byteorder must be an Endian object"); - return NULL; - } - + _CpEndian_KwArgsGetByteorder(NULL); PyObject* new_atom = CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder); if (!new_atom) { @@ -154,13 +147,13 @@ CpConditionAtom_Unpack(CpConditionAtomObject* self, CpLayerObject* layer) if (!CpConditionAtom_IsEnabled(self, (PyObject*)layer)) { return NULL; } - return _Cp_Unpack((PyObject*)self, layer); + return _Cp_Unpack(self->m_atom, layer); } /* docs */ /* members */ -static PyMemberDef CpRepeatedAtom_Members[] = { +static PyMemberDef CpConditionAtom_Members[] = { { "condition", T_OBJECT, offsetof(CpConditionAtomObject, m_condition), @@ -181,7 +174,7 @@ PyTypeObject CpConditionAtom_Type = { .tp_basicsize = sizeof(CpConditionAtomObject), .tp_dealloc = (destructor)cp_conditionatom_dealloc, .tp_init = (initproc)cp_conditionatom_init, - .tp_members = CpRepeatedAtom_Members, + .tp_members = CpConditionAtom_Members, .tp_methods = CpConditionAtom_Methods, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_doc = NULL, diff --git a/src/atomimpl/builtins/repeatedatomobj.c b/src/atomimpl/builtins/repeatedatomobj.c index 387f6c7..b268a0e 100644 --- a/src/atomimpl/builtins/repeatedatomobj.c +++ b/src/atomimpl/builtins/repeatedatomobj.c @@ -82,16 +82,7 @@ cp_repeatedatom_set_byteorder(CpRepeatedAtomObject* self, PyObject* args, PyObject* kw) { - static char* kwlist[] = { "byteorder", NULL }; - PyObject* byteorder = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &byteorder)) { - return NULL; - } - if (!CpEndian_Check(byteorder)) { - PyErr_SetString(PyExc_TypeError, "byteorder must be an Endian object"); - return NULL; - } - + _CpEndian_KwArgsGetByteorder(NULL); PyObject* new_atom = CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder); if (!new_atom) { diff --git a/src/atomimpl/builtins/switchatomobj.c b/src/atomimpl/builtins/switchatomobj.c new file mode 100644 index 0000000..50f34be --- /dev/null +++ b/src/atomimpl/builtins/switchatomobj.c @@ -0,0 +1,199 @@ +/* switch atom implementation */ + +#include "caterpillar/caterpillar.h" + +#include + +/* impl */ +static PyObject* +cp_switchatom_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + CpSwitchAtomObject* self = (CpSwitchAtomObject*)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpSwitchAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpSwitchAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = NULL; + CpBuiltinAtom_CATOM(self).ob_type = NULL; + self->s_callable = false; + return (PyObject*)self; +} + +static void +cp_switchatom_dealloc(CpSwitchAtomObject* self) +{ + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = NULL; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = NULL; + CpBuiltinAtom_CATOM(self).ob_type = NULL; + Py_CLEAR(self->m_cases); + Py_CLEAR(self->m_atom); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_switchatom_init(CpSwitchAtomObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "atom", "cases", NULL }; + PyObject* cases = NULL; + PyObject* atom = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "OO", kwlist, &atom, &cases)) { + return -1; + } + _Cp_SetObj(self->m_cases, cases); + _Cp_SetObj(self->m_atom, atom); + self->s_callable = PyCallable_Check(self->m_atom); + return 0; +} + +static PyObject* +cp_switchatom_set_byteorder(CpSwitchAtomObject* self, + PyObject* args, + PyObject* kw) +{ + _CpEndian_KwArgsGetByteorder(NULL); + PyObject* new_atom = + CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder); + if (!new_atom) { + return NULL; + } + _Cp_SetObj(self->m_atom, new_atom); + return (PyObject*)self; +} + +static PyObject* +cp_switchatom_get_next(CpSwitchAtomObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "atom", "context", NULL }; + PyObject* op = NULL; + PyObject* context = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "OO", kwlist, &op, &context)) { + return NULL; + } + + return CpSwitchAtom_GetNext(self, op, context); +} + +static PyObject* +cp_switchatom_repr(CpSwitchAtomObject* self) +{ + return PyUnicode_FromFormat("] %R>", Py_TYPE(self->m_cases)->tp_name, self->m_atom); +} + +/* Public API */ + +/*CpAPI*/ +PyObject* +CpSwitchAtom_GetNext(CpSwitchAtomObject* self, PyObject* op, PyObject* context) +{ + PyObject* result = NULL; + if (PyCallable_Check(self->m_cases)) { + result = PyObject_CallOneArg(self->m_cases, context); + if (!result) + return NULL; + } else { + result = PyObject_GetItem(self->m_cases, op); + if (!result) { + result = PyObject_GetItem(self->m_cases, CpInvalidDefault); + if (!result) + return NULL; + } + } + // TODO: check for nested struct + return result; +} + +/*CpAPI*/ +PyObject* +CpSwitchAtom_Unpack(CpSwitchAtomObject* self, CpLayerObject* layer) +{ + PyObject* value = _Cp_Unpack(self->m_atom, layer); + if (!value) { + return NULL; + } + + PyObject* next_atom = CpSwitchAtom_GetNext(self, value, (PyObject*)layer); + if (!next_atom) { + Py_DECREF(value); + return NULL; + } + + if (Py_IsNone(next_atom)) { + // TODO: document this behaviour + return value; + } + + PyObject* result = _Cp_Unpack(next_atom, layer); + Py_DECREF(value); + Py_DECREF(next_atom); + return result; +} + +/*CpAPI*/ +int +CpSwitchAtom_Pack(CpSwitchAtomObject* self, PyObject* obj, CpLayerObject* layer) +{ + if (!self->s_callable) { + PyErr_SetString( + PyExc_TypeError, + ("Switch atom currently supports only callable atoms when used as a " + "condition to select the final atom to be used to pack the given " + "value.")); + return -1; + } + + PyObject* value = PyObject_CallOneArg(self->m_atom, (PyObject*)layer); + if (!value) { + return -1; + } + + PyObject* next_atom = CpSwitchAtom_GetNext(self, value, (PyObject*)layer); + if (!next_atom) { + Py_DECREF(value); + return -1; + } + + int result = _Cp_Pack(obj, next_atom, layer); + Py_DECREF(next_atom); + Py_DECREF(value); + return result; +} + +/* docs */ + +/* members */ +static PyMemberDef CpSwitchAtom_Members[] = { + { "cases", T_OBJECT, offsetof(CpSwitchAtomObject, m_cases), READONLY }, + { "atom", T_OBJECT, offsetof(CpSwitchAtomObject, m_atom), READONLY }, + { NULL } +}; + +/* methods */ +static PyMethodDef CpSwitchAtom_Methods[] = { + _CpEndian_ImplSetByteorder_MethDef(switchatom, NULL), + { "get_next", + (PyCFunction)cp_switchatom_get_next, + METH_VARARGS | METH_KEYWORDS, + NULL }, + { NULL } +}; + +PyTypeObject CpSwitchAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpConditionAtom_NAME), + .tp_basicsize = sizeof(CpConditionAtomObject), + .tp_dealloc = (destructor)cp_switchatom_dealloc, + .tp_init = (initproc)cp_switchatom_init, + .tp_members = CpSwitchAtom_Members, + .tp_methods = CpSwitchAtom_Methods, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = NULL, + .tp_new = (newfunc)cp_switchatom_new, + .tp_repr = (reprfunc)cp_switchatom_repr, +}; diff --git a/src/capi.dat b/src/capi.dat new file mode 100644 index 0000000..00e8e18 --- /dev/null +++ b/src/capi.dat @@ -0,0 +1,175 @@ +# Caterpillar C API +# +# All types and API functions are described in this file. They are used to +# generate the Caterpillar C API headers and to build the documentation. +# +# modified format from sphinx documentation +# see https://github.com/python/cpython/blob/main/Doc/data/refcounts.dat +# +# There are currently four different types of statements: +# +# type:INDEX:STRUCT_NAME:TYPEDEF_NAME:CAPI_TYPE +# Defines a C API type for a C structure. The index is optional and +# the CAPI_TYPE will be inferred as PyTypeObject if none set +# +# obj:INDEX:NAME:TYPE +# Defines a C API object. +# +# func:INDEX:NAME:RETURN_TYPE:REFCOUNT +# Defines a C API function. The function must be present within the +# source set of this file. +# +# src:FILE +# Defines the source file (relative to this file) that contains the +# function definitions. + +src:field.c +src:context.c +src:arch.c +src:atomobj.c +src:option.c +src:struct.c +src:state.c +src:layer.c +src:parsing_pack.c +src:parsing_unpack.c +src:parsing_typeof.c +src:parsing_sizeof.c +src:atomimpl/boolatomobj.c +src:atomimpl/floatatomobj.c +src:atomimpl/charatomobj.c +src:atomimpl/intatomobj.c +src:atomimpl/padatomobj.c +src:atomimpl/stringatomobj.c +src:atomimpl/constatomobj.c +src:atomimpl/builtins/builtinatomobj.c +src:atomimpl/builtins/repeatedatomobj.c +src:atomimpl/builtins/conditionatomobj.c +src:atomimpl/builtins/switchatomobj.c + +# First index is reserved for the global module reference +obj:null:CpModule:PyModuleDef + +type:-:_modulestate:_modulestate:- + +type:1:_atomobj:CpAtomObject:- +type:2:_catomobj:CpCAtomObject:- +type:3:_archobj:CpArchObject:- +type:4:_endianobj:CpEndianObject:- +type:5:_contextobj:CpContextObject:- +type:6:_unaryexpr:CpUnaryExprObject:- +type:7:_binaryexpr:CpBinaryExprObject:- +type:8:_contextpath:CpContextPathObject:- +type:9:_fieldobj:CpFieldObject:- +type:10:_fieldatomobj:CpFieldAtomObject:- +type:11:_fieldcatomobj:CpFieldCAtomObject:- + +obj:12:CpInvalidDefault_Type:- +obj:13:CpDefaultOption_Type:- +obj:14:_CpInvalidDefault_Object:PyObject +obj:15:_CpDefaultOption_Object:PyObject + +type:16:_option:CpOptionObject:- +type:17:_stateobj:CpStateObject:- +type:18:_layerobj:CpLayerObject:- + +# REVISIT: maybe rename to _structfieldinfo +type:19:CpStructFieldInfo:CpStructFieldInfoObject:- +type:20:_structobj:CpStructObject:- +type:21:_floatatomobj:CpFloatAtomObject:- +type:22:_intatomobj:CpIntAtomObject:- +type:23:_boolatomobj:CpBoolAtomObject:- +type:24:_charatomobj:CpCharAtomObject:- +type:25:_paddingatomobj:CpPaddingAtomObject:- +type:26:_stringatomobj:CpStringAtomObject:- +type:27:_constatomobj:CpConstAtomObject:- +type:28:_builtinatomobj:CpBuiltinAtomObject:- +type:29:_repeatedatomobj:CpRepeatedAtomObject:- +type:30:_seqlayerobj:CpSeqLayerObject:- +type:31:_objlayerobj:CpObjLayerObject:- +type:32:_conditionatomobj:CpConditionAtomObject:- +type:33:_switchatomobj:CpSwitchAtomObject:- + +func:50:CpEndian_IsLittleEndian:int:null +func:53:CpContext_New:CpContextObject*:+1 +func:54:CpUnaryExpr_New:CpUnaryExprObject*:+1 +func:55:CpBinaryExpr_New:CpBinaryExprObject*:+1 +func:56:CpContextPath_New:CpContextPathObject*:+1 +func:57:CpContextPath_FromString:CpContextPathObject*:+1 +func:58:CpField_New:CpFieldObject*:null +func:59:CpField_HasCondition:int:null +func:60:CpField_IsEnabled:int:null +func:61:CpField_GetOffset:Py_ssize_t:null +func:62:CpField_EvalSwitch:PyObject*:+1 +func:63:CpField_GetLength:PyObject*:+1 +func:64:CpTypeOf:PyObject*:+1 +func:65:CpTypeOf_Field:PyObject*:+1 +func:66:CpTypeOf_Common:PyObject*:+1 +func:67:CpPack:int:null +func:68:CpPack_Field:int:null +func:69:CpPack_Common:int:null +func:70:CpPack_Struct:int:null +func:72:_CpPack_EvalLength:int:null +func:73:CpSizeOf:PyObject*:+1 +func:74:CpSizeOf_Field:PyObject*:+1 +func:75:CpSizeOf_Struct:PyObject*:+1 +func:76:CpSizeOf_Common:PyObject*:+1 +func:77:_Cp_SizeOf:PyObject*:+1 +func:78:CpUnpack:PyObject*:+1 +func:79:CpUnpack_Field:PyObject*:+1 +func:80:CpUnpack_Common:PyObject*:+1 +func:81:CpUnpack_Struct:PyObject*:+1 +func:83:_CpUnpack_EvalLength:int:null +func:84:CpUnpack_CAtom:PyObject*:+1 +func:85:CpPack_CAtom:int:null +func:86:CpSizeOf_CAtom:PyObject*:+1 +func:87:CpTypeOf_CAtom:PyObject*:+1 +func:88:CpState_New:CpStateObject*:null +func:89:CpState_Tell:PyObject*:+1 +func:90:CpState_Seek:PyObject*:+1 +func:91:CpState_Read:PyObject*:+1 +func:92:CpState_ReadFully:PyObject*:+1 +func:93:CpState_Write:PyObject*:+1 +func:94:CpState_SetGlobals:int:null +func:95:CpLayer_New:CpLayerObject*:+1 +func:96:CpLayer_Invalidate:int:null +func:98:CpStructFieldInfo_New:CpStructFieldInfoObject*:+1 +func:99:CpStruct_AddFieldInfo:int:null +func:100:CpStruct_AddField:int:null +func:101:CpStruct_New:CpStructObject*:+1 +func:102:CpStruct_GetAnnotations:PyObject*:+1 +func:103:CpStruct_ReplaceType:int:null +func:104:CpStruct_HasOption:int:null +func:105:CpStructModel_Check:int:null +func:106:CpStructModel_GetStruct:PyObject*:+1 +func:107:CpSeqLayer_New:CpSeqLayerObject*:+1 +func:108:CpSeqLayer_SetSequence:int:null +func:109:CpObjLayer_New:CpObjLayerObject*:+1 + +# atom api +func:120:CpIntAtom_Pack:int:null +func:121:CpIntAtom_Unpack:PyObject*:+1 +func:122:CpFloatAtom_Pack:int:null +func:123:CpFloatAtom_Unpack:PyObject*:+1 +func:124:CpBoolAtom_Pack:int:null +func:125:CpBoolAtom_Unpack:PyObject*:+1 +func:126:CpCharAtom_Pack:int:null +func:127:CpCharAtom_Unpack:PyObject*:+1 +func:128:CpPaddingAtom_Pack:int:null +func:129:CpPaddingAtom_PackMany:int:null +func:130:CpPaddingAtom_Unpack:PyObject*:+1 +func:131:CpPaddingAtom_UnpackMany:PyObject*:+1 +func:132:CpStringAtom_Pack:int:null +func:133:CpStringAtom_Unpack:PyObject*:+1 +func:134:CpConstAtom_Pack:int:null +func:135:CpConstAtom_Unpack:PyObject*:+1 +func:136:CpRepeatedAtom_Pack:int:null +func:137:CpRepeatedAtom_Unpack:PyObject*:+1 +func:138:CpRepeatedAtom_GetLength:PyObject*:+1 +func:139:CpConditionAtom_Pack:int:null +func:140:CpConditionAtom_Unpack:PyObject*:+1 +func:141:CpConditionAtom_IsEnabled:int:null +func:142:CpSwitchAtom_GetNext:PyObject*:+1 +func:143:CpSwitchAtom_Pack:int:null +func:144:CpSwitchAtom_Unpack:PyObject*:+1 + diff --git a/src/caterpillar/include/caterpillar/arch.h b/src/caterpillar/include/caterpillar/arch.h index 6916978..6d18399 100644 --- a/src/caterpillar/include/caterpillar/arch.h +++ b/src/caterpillar/include/caterpillar/arch.h @@ -116,4 +116,15 @@ CpEndian_SetEndian(PyObject* op, CpEndianObject* endian) METH_VARARGS | METH_KEYWORDS, (docs) \ } +#define _CpEndian_KwArgsGetByteorder(ret) \ + static char* kwlist[] = { "byteorder", NULL }; \ + PyObject* byteorder = NULL; \ + if (!PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &byteorder)) { \ + return ret; \ + } \ + if (!CpEndian_Check(byteorder)) { \ + PyErr_SetString(PyExc_TypeError, "byteorder must be an Endian object"); \ + return ret; \ + } + #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/builtins.h b/src/caterpillar/include/caterpillar/atoms/builtins.h index ccc311b..45bdaab 100644 --- a/src/caterpillar/include/caterpillar/atoms/builtins.h +++ b/src/caterpillar/include/caterpillar/atoms/builtins.h @@ -80,7 +80,34 @@ struct _conditionatomobj static inline CpConditionAtomObject * CpConditionAtom_New(PyObject* atom, PyObject* condition) { - return (CpConditionAtomObject*)CpObject_Create(&CpConditionAtom_Type, "OO", atom, condition); + return (CpConditionAtomObject*)CpObject_Create(&CpConditionAtom_Type, "OO", condition, atom); +} + +//------------------------------------------------------------------------------ +// Switch +struct _switchatomobj +{ + CpBuiltinAtom_HEAD + + /// Stores a reference to the actual parsing struct that will be used + /// to parse or build our data. This attribute is never null. + PyObject *m_atom; + + // A dictionary or dynamic value to represent the cases. + PyObject *m_cases; + + // -- internal --- + int s_callable; +}; + +#define CpSwitchAtom_NAME "switch" +#define CpSwitchAtom_CheckExact(op) Py_IS_TYPE((op), &CpSwitchAtom_Type) +#define CpSwitchAtom_Check(op) PyObject_IsType((op), &CpSwitchAtom_Type) + +static inline CpSwitchAtomObject * +CpSwitchAtom_New(PyObject* atom, PyObject* cases) +{ + return (CpSwitchAtomObject*)CpObject_Create(&CpSwitchAtom_Type, "OO", atom, cases); } #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index 4387c7e..abaee40 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -28,14 +28,14 @@ /* struct and typedefs */ struct _modulestate; typedef struct _modulestate _modulestate; -struct _endianobj; -typedef struct _endianobj CpEndianObject; -struct _archobj; -typedef struct _archobj CpArchObject; struct _atomobj; typedef struct _atomobj CpAtomObject; struct _catomobj; typedef struct _catomobj CpCAtomObject; +struct _archobj; +typedef struct _archobj CpArchObject; +struct _endianobj; +typedef struct _endianobj CpEndianObject; struct _contextobj; typedef struct _contextobj CpContextObject; struct _unaryexpr; @@ -84,6 +84,8 @@ struct _objlayerobj; typedef struct _objlayerobj CpObjLayerObject; struct _conditionatomobj; typedef struct _conditionatomobj CpConditionAtomObject; +struct _switchatomobj; +typedef struct _switchatomobj CpSwitchAtomObject; #ifdef _CPMODULE @@ -93,6 +95,7 @@ are then used and implemented in the internal API. */ extern PyModuleDef CpModule; +extern PyTypeObject CpAtom_Type; extern PyTypeObject CpCAtom_Type; extern PyTypeObject CpArch_Type; extern PyTypeObject CpEndian_Type; @@ -107,7 +110,6 @@ extern PyTypeObject CpInvalidDefault_Type; extern PyTypeObject CpDefaultOption_Type; extern PyObject _CpInvalidDefault_Object; extern PyObject _CpDefaultOption_Object; -extern PyTypeObject CpAtom_Type; extern PyTypeObject CpOption_Type; extern PyTypeObject CpState_Type; extern PyTypeObject CpLayer_Type; @@ -125,6 +127,7 @@ extern PyTypeObject CpRepeatedAtom_Type; extern PyTypeObject CpSeqLayer_Type; extern PyTypeObject CpObjLayer_Type; extern PyTypeObject CpConditionAtom_Type; +extern PyTypeObject CpSwitchAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -202,6 +205,9 @@ PyObject* CpRepeatedAtom_GetLength(CpRepeatedAtomObject* self, PyObject* context int CpConditionAtom_Pack(CpConditionAtomObject* self, PyObject* op, PyObject* layer); PyObject* CpConditionAtom_Unpack(CpConditionAtomObject* self, CpLayerObject* layer); int CpConditionAtom_IsEnabled(CpConditionAtomObject* self, PyObject* context); +PyObject* CpSwitchAtom_GetNext(CpSwitchAtomObject* self, PyObject* op, PyObject* context); +int CpSwitchAtom_Pack(CpSwitchAtomObject* self, PyObject* obj, CpLayerObject* layer); +PyObject* CpSwitchAtom_Unpack(CpSwitchAtomObject* self, CpLayerObject* layer); #else @@ -212,21 +218,21 @@ internal API functions and types. Their indices are static and defined in caterpillar_api.py */ #define CpModule (*(PyModuleDef *)Cp_API[0]) -#define CpCAtom_Type (*(PyTypeObject *)Cp_API[1]) -#define CpArch_Type (*(PyTypeObject *)Cp_API[2]) -#define CpEndian_Type (*(PyTypeObject *)Cp_API[3]) -#define CpContext_Type (*(PyTypeObject *)Cp_API[4]) -#define CpUnaryExpr_Type (*(PyTypeObject *)Cp_API[5]) -#define CpBinaryExpr_Type (*(PyTypeObject *)Cp_API[6]) -#define CpContextPath_Type (*(PyTypeObject *)Cp_API[7]) -#define CpField_Type (*(PyTypeObject *)Cp_API[8]) -#define CpFieldAtom_Type (*(PyTypeObject *)Cp_API[9]) -#define CpFieldCAtom_Type (*(PyTypeObject *)Cp_API[10]) -#define CpInvalidDefault_Type (*(PyTypeObject *)Cp_API[11]) -#define CpDefaultOption_Type (*(PyTypeObject *)Cp_API[12]) -#define _CpInvalidDefault_Object (*(PyObject *)Cp_API[13]) -#define _CpDefaultOption_Object (*(PyObject *)Cp_API[14]) -#define CpAtom_Type (*(PyTypeObject *)Cp_API[15]) +#define CpAtom_Type (*(PyTypeObject *)Cp_API[1]) +#define CpCAtom_Type (*(PyTypeObject *)Cp_API[2]) +#define CpArch_Type (*(PyTypeObject *)Cp_API[3]) +#define CpEndian_Type (*(PyTypeObject *)Cp_API[4]) +#define CpContext_Type (*(PyTypeObject *)Cp_API[5]) +#define CpUnaryExpr_Type (*(PyTypeObject *)Cp_API[6]) +#define CpBinaryExpr_Type (*(PyTypeObject *)Cp_API[7]) +#define CpContextPath_Type (*(PyTypeObject *)Cp_API[8]) +#define CpField_Type (*(PyTypeObject *)Cp_API[9]) +#define CpFieldAtom_Type (*(PyTypeObject *)Cp_API[10]) +#define CpFieldCAtom_Type (*(PyTypeObject *)Cp_API[11]) +#define CpInvalidDefault_Type (*(PyTypeObject *)Cp_API[12]) +#define CpDefaultOption_Type (*(PyTypeObject *)Cp_API[13]) +#define _CpInvalidDefault_Object (*(PyObject *)Cp_API[14]) +#define _CpDefaultOption_Object (*(PyObject *)Cp_API[15]) #define CpOption_Type (*(PyTypeObject *)Cp_API[16]) #define CpState_Type (*(PyTypeObject *)Cp_API[17]) #define CpLayer_Type (*(PyTypeObject *)Cp_API[18]) @@ -244,6 +250,7 @@ caterpillar_api.py #define CpSeqLayer_Type (*(PyTypeObject *)Cp_API[30]) #define CpObjLayer_Type (*(PyTypeObject *)Cp_API[31]) #define CpConditionAtom_Type (*(PyTypeObject *)Cp_API[32]) +#define CpSwitchAtom_Type (*(PyTypeObject *)Cp_API[33]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -321,6 +328,9 @@ caterpillar_api.py #define CpConditionAtom_Pack (*((int (*)(CpConditionAtomObject* self, PyObject* op, PyObject* layer)))Cp_API[139]) #define CpConditionAtom_Unpack (*((PyObject* (*)(CpConditionAtomObject* self, CpLayerObject* layer)))Cp_API[140]) #define CpConditionAtom_IsEnabled (*((int (*)(CpConditionAtomObject* self, PyObject* context)))Cp_API[141]) +#define CpSwitchAtom_GetNext (*((PyObject* (*)(CpSwitchAtomObject* self, PyObject* op, PyObject* context)))Cp_API[142]) +#define CpSwitchAtom_Pack (*((int (*)(CpSwitchAtomObject* self, PyObject* obj, CpLayerObject* layer)))Cp_API[143]) +#define CpSwitchAtom_Unpack (*((PyObject* (*)(CpSwitchAtomObject* self, CpLayerObject* layer)))Cp_API[144]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index 317edd9..30c4c62 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -4,6 +4,7 @@ void *Cp_API[] = { (void *) &CpModule, + (void *) &CpAtom_Type, (void *) &CpCAtom_Type, (void *) &CpArch_Type, (void *) &CpEndian_Type, @@ -18,7 +19,6 @@ void *Cp_API[] = { (void *) &CpDefaultOption_Type, (void *) &_CpInvalidDefault_Object, (void *) &_CpDefaultOption_Object, - (void *) &CpAtom_Type, (void *) &CpOption_Type, (void *) &CpState_Type, (void *) &CpLayer_Type, @@ -36,7 +36,7 @@ void *Cp_API[] = { (void *) &CpSeqLayer_Type, (void *) &CpObjLayer_Type, (void *) &CpConditionAtom_Type, - NULL, + (void *) &CpSwitchAtom_Type, NULL, NULL, NULL, @@ -144,6 +144,9 @@ void *Cp_API[] = { (void *) &CpRepeatedAtom_GetLength, (void *) &CpConditionAtom_Pack, (void *) &CpConditionAtom_Unpack, - (void *) &CpConditionAtom_IsEnabled + (void *) &CpConditionAtom_IsEnabled, + (void *) &CpSwitchAtom_GetNext, + (void *) &CpSwitchAtom_Pack, + (void *) &CpSwitchAtom_Unpack }; diff --git a/src/code_gen/caterpillar_api.py b/src/code_gen/caterpillar_api.py index 4406a38..e87bf06 100644 --- a/src/code_gen/caterpillar_api.py +++ b/src/code_gen/caterpillar_api.py @@ -11,190 +11,60 @@ # !! heavily inspired by numpy !! import os +import pathlib + +CAPI_PATH = pathlib.Path(__file__).parent.parent / "capi.dat" +if not CAPI_PATH.exists(): + raise FileNotFoundError(f"File not found: {CAPI_PATH}") # Reserved for future use __reserved__ = "__reserved__" -cp_types = { - # struct and typedefs - "_modulestate": "_modulestate", - "_endianobj": "CpEndianObject", - "_archobj": "CpArchObject", - "_atomobj": "CpAtomObject", - "_catomobj": "CpCAtomObject", - "_contextobj": "CpContextObject", - "_unaryexpr": "CpUnaryExprObject", - "_binaryexpr": "CpBinaryExprObject", - "_contextpath": "CpContextPathObject", - "_fieldobj": "CpFieldObject", - "_fieldatomobj": "CpFieldAtomObject", - "_fieldcatomobj": "CpFieldCAtomObject", - "_option": "CpOptionObject", - "_stateobj": "CpStateObject", - "_layerobj": "CpLayerObject", - # REVISIT: maybe rename to _structfieldinfo - "CpStructFieldInfo": "CpStructFieldInfoObject", - "_structobj": "CpStructObject", - "_floatatomobj": "CpFloatAtomObject", - "_intatomobj": "CpIntAtomObject", - "_boolatomobj": "CpBoolAtomObject", - "_charatomobj": "CpCharAtomObject", - "_paddingatomobj": "CpPaddingAtomObject", - "_stringatomobj": "CpStringAtomObject", - "_constatomobj": "CpConstAtomObject", - "_builtinatomobj": "CpBuiltinAtomObject", - "_repeatedatomobj": "CpRepeatedAtomObject", - "_seqlayerobj": "CpSeqLayerObject", - "_objlayerobj": "CpObjLayerObject", - "_conditionatomobj": "CpConditionAtomObject", -} - -cp_type_api = { - "CpModule": (0, "PyModuleDef"), - "CpCAtom_Type": (1,), - "CpArch_Type": (2,), - "CpEndian_Type": (3,), - "CpContext_Type": (4,), - "CpUnaryExpr_Type": (5,), - "CpBinaryExpr_Type": (6,), - "CpContextPath_Type": (7,), - "CpField_Type": (8,), - "CpFieldAtom_Type": (9,), - "CpFieldCAtom_Type": (10,), - "CpInvalidDefault_Type": (11,), - "CpDefaultOption_Type": (12,), - "_CpInvalidDefault_Object": (13, "PyObject"), - "_CpDefaultOption_Object": (14, "PyObject"), - "CpAtom_Type": (15,), - "CpOption_Type": (16,), - "CpState_Type": (17,), - "CpLayer_Type": (18,), - "CpStructFieldInfo_Type": (19,), - "CpStruct_Type": (20,), - "CpFloatAtom_Type": (21,), - "CpIntAtom_Type": (22,), - "CpBoolAtom_Type": (23,), - "CpCharAtom_Type": (24,), - "CpPaddingAtom_Type": (25,), - "CpStringAtom_Type": (26,), - "CpConstAtom_Type": (27,), - "CpBuiltinAtom_Type": (28,), - "CpRepeatedAtom_Type": (29,), - "CpSeqLayer_Type": (30,), - "CpObjLayer_Type": (31,), - "CpConditionAtom_Type": (32,), -} - -cp_func_api = { - # : <<-->> : (rtype, [args]) - "CpEndian_IsLittleEndian": 50, - # "CpContext_GetAttr": 51, - # "CpContext_GetAttrString": 52, - "CpContext_New": 53, - "CpUnaryExpr_New": 54, - "CpBinaryExpr_New": 55, - "CpContextPath_New": 56, - "CpContextPath_FromString": 57, - "CpField_New": 58, - "CpField_HasCondition": 59, - "CpField_IsEnabled": 60, - "CpField_GetOffset": 61, - "CpField_EvalSwitch": 62, - "CpField_GetLength": 63, - "CpTypeOf": 64, - "CpTypeOf_Field": 65, - "CpTypeOf_Common": 66, - "CpPack": 67, - "CpPack_Field": 68, - "CpPack_Common": 69, - "CpPack_Struct": 70, - "_CpPack_EvalLength": 72, - "CpSizeOf": 73, - "CpSizeOf_Field": 74, - "CpSizeOf_Struct": 75, - "CpSizeOf_Common": 76, - "_Cp_SizeOf": 77, - "CpUnpack": 78, - "CpUnpack_Field": 79, - "CpUnpack_Common": 80, - "CpUnpack_Struct": 81, - "_CpUnpack_EvalLength": 83, - "CpUnpack_CAtom": 84, - "CpPack_CAtom": 85, - "CpSizeOf_CAtom": 86, - "CpTypeOf_CAtom": 87, - "CpState_New": 88, - "CpState_Tell": 89, - "CpState_Seek": 90, - "CpState_Read": 91, - "CpState_ReadFully": 92, - "CpState_Write": 93, - "CpState_SetGlobals": 94, - "CpLayer_New": 95, - "CpLayer_Invalidate": 96, - "CpStructFieldInfo_New" : 98, - "CpStruct_AddFieldInfo": 99, - "CpStruct_AddField": 100, - "CpStruct_New": 101, - "CpStruct_GetAnnotations": 102, - "CpStruct_ReplaceType": 103, - "CpStruct_HasOption": 104, - "CpStructModel_Check": 105, - "CpStructModel_GetStruct": 106, - "CpSeqLayer_New": 107, - "CpSeqLayer_SetSequence": 108, - "CpObjLayer_New": 109, - - - # atom api - "CpIntAtom_Pack": 120, - "CpIntAtom_Unpack": 121, - "CpFloatAtom_Pack": 122, - "CpFloatAtom_Unpack": 123, - "CpBoolAtom_Pack": 124, - "CpBoolAtom_Unpack": 125, - "CpCharAtom_Pack": 126, - "CpCharAtom_Unpack": 127, - "CpPaddingAtom_Pack": 128, - "CpPaddingAtom_PackMany": 129, - "CpPaddingAtom_Unpack": 130, - "CpPaddingAtom_UnpackMany": 131, - "CpStringAtom_Pack": 132, - "CpStringAtom_Unpack": 133, - "CpConstAtom_Pack": 134, - "CpConstAtom_Unpack": 135, - "CpRepeatedAtom_Pack": 136, - "CpRepeatedAtom_Unpack": 137, - "CpRepeatedAtom_GetLength": 138, - "CpConditionAtom_Pack": 139, - "CpConditionAtom_Unpack": 140, - "CpConditionAtom_IsEnabled": 141, -} - -API_SRC = [ - "field.c", - "context.c", - "arch.c", - "atomobj.c", - "option.c", - "struct.c", - "state.c", - "layer.c", - "parsing_pack.c", - "parsing_unpack.c", - "parsing_typeof.c", - "parsing_sizeof.c", - "atomimpl/boolatomobj.c", - "atomimpl/floatatomobj.c", - "atomimpl/charatomobj.c", - "atomimpl/intatomobj.c", - "atomimpl/padatomobj.c", - "atomimpl/stringatomobj.c", - "atomimpl/constatomobj.c", - "atomimpl/builtins/builtinatomobj.c", - "atomimpl/builtins/repeatedatomobj.c", - "atomimpl/builtins/conditionatomobj.c", -] +cp_types = {} # struct and typedefs +cp_type_api = {} +cp_api_src = [] +cp_func_api = {} # : <<-->> : (rtype, [args]) + +for line in CAPI_PATH.read_text("utf-8").splitlines(): + if line.startswith("#"): + continue + + line = line.strip() + if not line: + continue + + def_type, *parts = line.split(":") + match def_type: + case "obj": + # obj:INDEX:NAME:TYPE + # Defines a C API object. + index, name, type_ = parts + cp_type_api[name] = (int(index), type_ if type_ != "-" else "PyTypeObject") + + case "type": + # type:INDEX:STRUCT_NAME:TYPEDEF_NAME:CAPI_TYPE + # Defines a C API type for a C structure. The index is optional and + # the CAPI_TYPE will be inferred as PyTypeObject if none set + index, struct_name, typedef_name, c_api_type = parts + cp_types[struct_name] = typedef_name + if index != '-': + if typedef_name.endswith("Object"): + typedef_name = typedef_name[:-6] + "_Type" + cp_type_api[typedef_name] = (int(index), c_api_type if c_api_type != "-" else "PyTypeObject") + + case "src": + # src:FILE + # Defines the source file (relative to this file) that contains the + # function definitions. + cp_api_src.append(parts[0]) + + case "func": + # func:INDEX:NAME:RETURN_TYPE:REFCOUNT + # Defines a C API function. The function must be present within the + # source set of this file. + index, name, *_ = parts + if index != '-': + cp_func_api[name] = int(index) def cp_api_functions() -> dict[str, tuple]: @@ -202,7 +72,7 @@ def cp_api_functions() -> dict[str, tuple]: func_api = {} # REVISIT: dirty parsing algorithm - for f in API_SRC: + for f in cp_api_src: with open(os.path.join(base_path, "..", f), "r", encoding="utf-8") as c_src: while True: line = c_src.readline() diff --git a/src/module.c b/src/module.c index 63f0028..06b4b85 100644 --- a/src/module.c +++ b/src/module.c @@ -393,10 +393,12 @@ PyInit__C(void) CpBuiltinAtom_Type.tp_base = &CpCAtom_Type; CpRepeatedAtom_Type.tp_base = &CpBuiltinAtom_Type; CpConditionAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpSwitchAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpBuiltinAtom_Type); CpModule_SetupType(&CpRepeatedAtom_Type); CpModule_SetupType(&CpConditionAtom_Type); + CpModule_SetupType(&CpSwitchAtom_Type); CpIntAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpIntAtom_Type); @@ -451,6 +453,7 @@ PyInit__C(void) CpModule_AddObject(CpBuiltinAtom_NAME, &CpBuiltinAtom_Type); CpModule_AddObject(CpRepeatedAtom_NAME, &CpRepeatedAtom_Type); CpModule_AddObject(CpConditionAtom_NAME, &CpConditionAtom_Type); + CpModule_AddObject(CpSwitchAtom_NAME, &CpSwitchAtom_Type); CpModule_AddObject(CpIntAtom_NAME, &CpIntAtom_Type); CpModule_AddObject(CpFloatAtom_NAME, &CpFloatAtom_Type); From 8983f2666932ba5c6942db372f478de8da6f8dda Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 8 Sep 2024 14:22:31 +0200 Subject: [PATCH 07/29] Added CpOffsetAtom --- + fixed issues with CpLayer* classes + changed struct to builtinatom --- CMakeLists.txt | 1 + docs/sphinx/source/development/roadmap.rst | 2 +- .../source/reference/capi/objects/state.rst | 2 +- src/atomimpl/builtins/builtinatomobj.c | 7 + src/atomimpl/builtins/offsetatomobj.c | 205 +++++++++++++++++ src/atomimpl/builtins/repeatedatomobj.c | 25 +- src/atomimpl/builtins/switchatomobj.c | 85 ++++++- src/capi.dat | 27 ++- .../include/caterpillar/atoms/builtins.h | 89 ++++++-- .../include/caterpillar/caterpillarapi.h | 36 ++- src/caterpillar/include/caterpillar/struct.h | 20 +- src/caterpillarapi.c | 11 +- src/layer.c | 12 +- src/module.c | 10 +- src/state.c | 6 +- src/struct.c | 213 ++++++++++++------ 16 files changed, 608 insertions(+), 143 deletions(-) create mode 100644 src/atomimpl/builtins/offsetatomobj.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 187242d..3a6ec7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ python_add_library( src/atomimpl/builtins/repeatedatomobj.c src/atomimpl/builtins/conditionatomobj.c src/atomimpl/builtins/switchatomobj.c + src/atomimpl/builtins/offsetatomobj.c WITH_SOABI ) diff --git a/docs/sphinx/source/development/roadmap.rst b/docs/sphinx/source/development/roadmap.rst index 465bc83..89bb664 100644 --- a/docs/sphinx/source/development/roadmap.rst +++ b/docs/sphinx/source/development/roadmap.rst @@ -25,7 +25,7 @@ C API ----- - |check_| Implementation of parsing process (unpack, pack) -- |check_| Struct class (:c:type:`CpStructObject`) +- |uncheck_| Struct class (:c:type:`CpStructObject`) - |uncheck_| Struct wrapper function - |uncheck_| Python docs diff --git a/docs/sphinx/source/reference/capi/objects/state.rst b/docs/sphinx/source/reference/capi/objects/state.rst index 755978d..e8220b0 100644 --- a/docs/sphinx/source/reference/capi/objects/state.rst +++ b/docs/sphinx/source/reference/capi/objects/state.rst @@ -41,7 +41,7 @@ State Objects Returns the current position of the state. Returns *NULL* if an error occurs. -.. c:function:: PyObject *CpState_Seek(CpStateObject *state, Py_ssize_t pos, int whence) +.. c:function:: PyObject *CpState_Seek(CpStateObject *state, Py_ssize_t pos, PyObject whence) Seeks to the given position. Returns :code:`0` if successful. Returns :code:`-1` if an error occurs. diff --git a/src/atomimpl/builtins/builtinatomobj.c b/src/atomimpl/builtins/builtinatomobj.c index d3f3ed0..76c8c3a 100644 --- a/src/atomimpl/builtins/builtinatomobj.c +++ b/src/atomimpl/builtins/builtinatomobj.c @@ -60,6 +60,12 @@ cp_builtinatom_as_number_floordiv(PyObject* self, PyObject* other) return (PyObject*)CpConditionAtom_New(self, other); } +static PyObject* +cp_builtinatom_as_number_matmul(PyObject* self, PyObject* other) +{ + return (PyObject*)CpOffsetAtom_New(self, other); +} + static PyMappingMethods CpFieldAtom_MappingMethods = { .mp_subscript = (binaryfunc)cp_builtinatom_as_mapping_getitem, }; @@ -67,6 +73,7 @@ static PyMappingMethods CpFieldAtom_MappingMethods = { static PyNumberMethods CpField_NumberMethods = { .nb_rshift = (binaryfunc)cp_builtinatom_as_number_rshift, .nb_floor_divide = (binaryfunc)cp_builtinatom_as_number_floordiv, + .nb_matrix_multiply = (binaryfunc)cp_builtinatom_as_number_matmul, }; PyTypeObject CpBuiltinAtom_Type = { diff --git a/src/atomimpl/builtins/offsetatomobj.c b/src/atomimpl/builtins/offsetatomobj.c new file mode 100644 index 0000000..2cb0ddb --- /dev/null +++ b/src/atomimpl/builtins/offsetatomobj.c @@ -0,0 +1,205 @@ +/* offset atom implementation */ + +#include "caterpillar/caterpillar.h" + +#include + +/* impl */ +static PyObject* +cp_offsetatom_type(CpOffsetAtomObject* self) +{ + return CpTypeOf(self->m_atom); +} + +static PyObject* +cp_offsetatom_size(CpOffsetAtomObject* self, CpLayerObject* layer) +{ + return _Cp_SizeOf(self->m_atom, layer); +} + +static PyObject* +cp_offsetatom_repr(CpOffsetAtomObject* self) +{ + return PyUnicode_FromFormat( + "] %R>", Py_TYPE(self->m_offset)->tp_name, self->m_atom); +} + +static PyObject* +cp_offsetatom_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + CpOffsetAtomObject* self = (CpOffsetAtomObject*)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpOffsetAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpOffsetAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_offsetatom_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_offsetatom_type; + self->m_offset = 0; + self->m_atom = NULL; + return (PyObject*)self; +} + +static void +cp_offsetatom_dealloc(CpOffsetAtomObject* self) +{ + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = NULL; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = NULL; + CpBuiltinAtom_CATOM(self).ob_type = NULL; + Py_CLEAR(self->m_atom); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_offsetatom_init(CpOffsetAtomObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "offset", "atom", "whence", NULL }; + PyObject *offset = NULL, *atom = NULL; + int whence = 0; + if (!PyArg_ParseTupleAndKeywords( + args, kw, "OO|i", kwlist, &atom, &offset, &whence)) + return -1; + + _Cp_SetObj(self->m_atom, atom); + _Cp_SetObj(self->m_offset, offset); + self->m_whence = PyLong_FromLong(whence); + if (!self->m_whence) { + return -1; + } + self->s_is_number = PyNumber_Check(self->m_offset); + return 0; +} + +static PyObject* +cp_offsetatom_set_byteorder(CpOffsetAtomObject* self, + PyObject* args, + PyObject* kw) +{ + _CpEndian_KwArgsGetByteorder(NULL); + PyObject* new_atom = + CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder); + if (!new_atom) { + return NULL; + } + _Cp_SetObj(self->m_atom, new_atom); + return (PyObject*)self; +} + +static PyObject* +cp_offsetatom_get_offset(CpOffsetAtomObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "layer", NULL }; + PyObject* layer = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &layer)) + return NULL; + return CpOffsetAtom_GetOffset(self, layer); +} + +/* Public API */ + +/*CpAPI*/ +PyObject* +CpOffsetAtom_GetOffset(CpOffsetAtomObject* self, PyObject* layer) +{ + if (self->s_is_number) { + return Py_NewRef(self->m_offset); + } + + // asset offset is callable + return PyObject_CallOneArg(self->m_offset, layer); +} + +/*CpAPI*/ +int +CpOffsetAtom_Pack(CpOffsetAtomObject* self, PyObject* obj, CpLayerObject* layer) +{ + // TODO: implement offset table + PyObject* fallback = CpState_Tell(layer->m_state); + if (!fallback) { + return -1; + } + + PyObject* res = CpState_Seek(layer->m_state, self->m_offset, self->m_whence); + if (!res) { + return -1; + } + Py_DECREF(res); + + if (_Cp_Pack(self->m_atom, obj, layer) < 0) { + return -1; + } + + res = CpState_Seek(layer->m_state, fallback, PyLong_FromLong(0)); + if (!res) { + return -1; + } + Py_DECREF(res); + return 0; +} + +/*CpAPI*/ +PyObject* +CpOffsetAtom_Unpack(CpOffsetAtomObject* self, CpLayerObject* layer) +{ + PyObject* fallback = CpState_Tell(layer->m_state); + if (!fallback) { + return NULL; + } + + PyObject* res = CpState_Seek(layer->m_state, self->m_offset, self->m_whence); + if (!res) { + return NULL; + } + Py_DECREF(res); + + PyObject* result = _Cp_Unpack(self->m_atom, layer); + if (!result) { + return NULL; + } + + res = CpState_Seek(layer->m_state, fallback, PyLong_FromLong(0)); + if (!res) { + return NULL; + } + Py_DECREF(res); + return result; +} + +/* docs */ + +/* members */ + +static PyMemberDef CpOffsetAtom_Members[] = { + { "offset", T_OBJECT_EX, offsetof(CpOffsetAtomObject, m_offset), READONLY }, + { "whence", T_OBJECT_EX, offsetof(CpOffsetAtomObject, m_whence), READONLY }, + { NULL } /* Sentinel */ +}; + +static PyMethodDef CpOffsetAtom_Methods[] = { + _CpEndian_ImplSetByteorder_MethDef(offsetatom, NULL), + { "get_offset", + (PyCFunction)cp_offsetatom_get_offset, + METH_VARARGS | METH_KEYWORDS, + NULL }, + { NULL } /* Sentinel */ +}; + +PyTypeObject CpOffsetAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpOffsetAtom_NAME), + .tp_basicsize = sizeof(CpOffsetAtomObject), + .tp_members = CpOffsetAtom_Members, + .tp_methods = CpOffsetAtom_Methods, + .tp_init = (initproc)cp_offsetatom_init, + .tp_dealloc = (destructor)cp_offsetatom_dealloc, + .tp_repr = (reprfunc)cp_offsetatom_repr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = NULL, + .tp_new = (newfunc)cp_offsetatom_new, +}; diff --git a/src/atomimpl/builtins/repeatedatomobj.c b/src/atomimpl/builtins/repeatedatomobj.c index b268a0e..26cb4e4 100644 --- a/src/atomimpl/builtins/repeatedatomobj.c +++ b/src/atomimpl/builtins/repeatedatomobj.c @@ -7,7 +7,7 @@ /* impl */ static PyObject* -cp_repeatedatomobj__type__(CpRepeatedAtomObject* self) +cp_repeatedatomobj_type(CpRepeatedAtomObject* self) { PyObject* atom_type = CpTypeOf(self->m_atom); if (!atom_type) @@ -19,9 +19,20 @@ cp_repeatedatomobj__type__(CpRepeatedAtomObject* self) } static PyObject* -cp_repeatedatomobj__size__(CpRepeatedAtomObject* self, CpLayerObject* layer) +cp_repeatedatomobj_size(CpRepeatedAtomObject* self, CpLayerObject* layer) { - return _Cp_SizeOf(self->m_atom, layer); + PyObject* length = CpRepeatedAtom_GetLength(self, (PyObject*)layer); + if (!length) { + return NULL; + } + PyObject* atom_size = _Cp_SizeOf(self->m_atom, layer); + if (!atom_size) { + return NULL; + } + + PyObject* result = PyNumber_Multiply(atom_size, length); + Py_DECREF(length); + return result; } static PyObject* @@ -43,8 +54,8 @@ cp_repeatedatomobj_new(PyTypeObject* type, PyObject* args, PyObject* kw) CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpRepeatedAtom_Unpack; CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; - CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_repeatedatomobj__size__; - CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_repeatedatomobj__type__; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_repeatedatomobj_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_repeatedatomobj_type; return (PyObject*)self; } @@ -79,8 +90,8 @@ cp_repeatedatomobj_init(CpRepeatedAtomObject* self, static PyObject* cp_repeatedatom_set_byteorder(CpRepeatedAtomObject* self, - PyObject* args, - PyObject* kw) + PyObject* args, + PyObject* kw) { _CpEndian_KwArgsGetByteorder(NULL); PyObject* new_atom = diff --git a/src/atomimpl/builtins/switchatomobj.c b/src/atomimpl/builtins/switchatomobj.c index 50f34be..88cc332 100644 --- a/src/atomimpl/builtins/switchatomobj.c +++ b/src/atomimpl/builtins/switchatomobj.c @@ -5,6 +5,84 @@ #include /* impl */ +static PyObject* +cp_switchatom_type(CpSwitchAtomObject* self) +{ + _modulestate* mod = get_global_module_state(); + if (!self->s_callable) { + return Py_NewRef(mod->Any_Type); + } + + PyObject* types = PyList_New(0); + if (!types) { + return NULL; + } + + PyObject* atomType = CpTypeOf(self->m_atom); + if (!atomType) { + Py_DECREF(types); + return NULL; + } + + if (PyList_Append(types, atomType) < 0) { + Py_DECREF(types); + return NULL; + } + + PyObject* values = PyDict_Values(self->m_cases); + if (!values) { + Py_XDECREF(types); + Py_XDECREF(atomType); + return NULL; + } + + Py_ssize_t length = PyList_GET_SIZE(values); + PyObject* switch_type = NULL; + for (Py_ssize_t i = 0; i < length; i++) { + PyObject* value = PyList_GetItem(values, i); + if (!value) { + Py_XDECREF(values); + Py_XDECREF(types); + Py_XDECREF(switch_type); + return NULL; + } + + switch_type = CpTypeOf(value); + if (!switch_type) { + Py_XDECREF(values); + Py_XDECREF(types); + return NULL; + } + + if (!PySequence_Contains(types, switch_type)) { + PyList_Append(types, switch_type); + } + Py_XSETREF(switch_type, NULL); + } + + Py_XDECREF(values); + PyObject* tuple = PyList_AsTuple(types); + Py_XDECREF(types); + if (!tuple) { + Py_XDECREF(atomType); + Py_XDECREF(switch_type); + return NULL; + } + + PyObject* type = PyObject_GetItem(mod->Union_Type, tuple); + Py_XDECREF(tuple); + Py_XDECREF(atomType); + Py_XDECREF(switch_type); + return type; +} + +static PyObject* +cp_switchatom_size(CpSwitchAtomObject* self, CpLayerObject* layer) +{ + PyErr_SetString(PyExc_TypeError, "Switch atoms does not have a static size!"); + return NULL; +} + static PyObject* cp_switchatom_new(PyTypeObject* type, PyObject* args, PyObject* kw) { @@ -17,8 +95,8 @@ cp_switchatom_new(PyTypeObject* type, PyObject* args, PyObject* kw) CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpSwitchAtom_Unpack; CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; - CpBuiltinAtom_CATOM(self).ob_size = NULL; - CpBuiltinAtom_CATOM(self).ob_type = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_switchatom_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_switchatom_type; self->s_callable = false; return (PyObject*)self; } @@ -84,7 +162,8 @@ cp_switchatom_get_next(CpSwitchAtomObject* self, PyObject* args, PyObject* kw) static PyObject* cp_switchatom_repr(CpSwitchAtomObject* self) { - return PyUnicode_FromFormat("] %R>", Py_TYPE(self->m_cases)->tp_name, self->m_atom); + return PyUnicode_FromFormat( + "] %R>", Py_TYPE(self->m_cases)->tp_name, self->m_atom); } /* Public API */ diff --git a/src/capi.dat b/src/capi.dat index 00e8e18..3d98fc7 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -46,9 +46,10 @@ src:atomimpl/builtins/builtinatomobj.c src:atomimpl/builtins/repeatedatomobj.c src:atomimpl/builtins/conditionatomobj.c src:atomimpl/builtins/switchatomobj.c +src:atomimpl/builtins/offsetatomobj.c # First index is reserved for the global module reference -obj:null:CpModule:PyModuleDef +obj:0:CpModule:PyModuleDef type:-:_modulestate:_modulestate:- @@ -74,7 +75,7 @@ type:17:_stateobj:CpStateObject:- type:18:_layerobj:CpLayerObject:- # REVISIT: maybe rename to _structfieldinfo -type:19:CpStructFieldInfo:CpStructFieldInfoObject:- +type:19:_fieldinfoobj:CpStructFieldInfoObject:- type:20:_structobj:CpStructObject:- type:21:_floatatomobj:CpFloatAtomObject:- type:22:_intatomobj:CpIntAtomObject:- @@ -89,6 +90,7 @@ type:30:_seqlayerobj:CpSeqLayerObject:- type:31:_objlayerobj:CpObjLayerObject:- type:32:_conditionatomobj:CpConditionAtomObject:- type:33:_switchatomobj:CpSwitchAtomObject:- +type:34:_offsetatomobj:CpOffsetAtomObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 @@ -140,11 +142,13 @@ func:101:CpStruct_New:CpStructObject*:+1 func:102:CpStruct_GetAnnotations:PyObject*:+1 func:103:CpStruct_ReplaceType:int:null func:104:CpStruct_HasOption:int:null -func:105:CpStructModel_Check:int:null -func:106:CpStructModel_GetStruct:PyObject*:+1 -func:107:CpSeqLayer_New:CpSeqLayerObject*:+1 -func:108:CpSeqLayer_SetSequence:int:null -func:109:CpObjLayer_New:CpObjLayerObject*:+1 +func:105:CpStruct_Pack:int:null +func:106:CpStruct_Unpack:PyObject*:+1 +func:107:CpStructModel_Check:int:null +func:108:CpStructModel_GetStruct:PyObject*:+1 +func:109:CpSeqLayer_New:CpSeqLayerObject*:+1 +func:110:CpSeqLayer_SetSequence:int:null +func:111:CpObjLayer_New:CpObjLayerObject*:+1 # atom api func:120:CpIntAtom_Pack:int:null @@ -163,13 +167,20 @@ func:132:CpStringAtom_Pack:int:null func:133:CpStringAtom_Unpack:PyObject*:+1 func:134:CpConstAtom_Pack:int:null func:135:CpConstAtom_Unpack:PyObject*:+1 +func:-:CpRepeatedAtom_New:CpRepeatedAtomObject*:+1 func:136:CpRepeatedAtom_Pack:int:null func:137:CpRepeatedAtom_Unpack:PyObject*:+1 func:138:CpRepeatedAtom_GetLength:PyObject*:+1 +func:-:CpConditionAtom_New:CpConditionAtomObject*:+1 func:139:CpConditionAtom_Pack:int:null func:140:CpConditionAtom_Unpack:PyObject*:+1 func:141:CpConditionAtom_IsEnabled:int:null +func:-:CpSwitchAtom_New:CpSwitchAtomObject*:+1 func:142:CpSwitchAtom_GetNext:PyObject*:+1 func:143:CpSwitchAtom_Pack:int:null func:144:CpSwitchAtom_Unpack:PyObject*:+1 - +func:-:CpOffsetAtom_FromSsize_t:CpOffsetAtomObject*:+1 +func:-:CpOffsetAtom_New:CpOffsetAtomObject*:+1 +func:145:CpOffsetAtom_Pack:int:null +func:146:CpOffsetAtom_Unpack:PyObject*:+1 +func:147:CpOffsetAtom_GetOffset:PyObject*:+1 \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/builtins.h b/src/caterpillar/include/caterpillar/atoms/builtins.h index 45bdaab..0eb0a8b 100644 --- a/src/caterpillar/include/caterpillar/atoms/builtins.h +++ b/src/caterpillar/include/caterpillar/atoms/builtins.h @@ -17,15 +17,15 @@ #ifndef BUILTINS_H #define BUILTINS_H -#include "caterpillar/caterpillarapi.h" #include "caterpillar/atomobj.h" +#include "caterpillar/caterpillarapi.h" // PROPOSAL: Builtin atoms are similar to the Python class FieldMixin // but implement all parts of a field separately. For instance, the // builtin atom for a sequence only implements the sequence part. struct _builtinatomobj { - CpCAtom_HEAD + CpCAtom_HEAD }; #define CpBuiltinAtom_NAME "builtinatom" @@ -38,76 +38,119 @@ struct _builtinatomobj // Repeated struct _repeatedatomobj { - CpBuiltinAtom_HEAD + CpBuiltinAtom_HEAD /// Stores a reference to the actual parsing struct that will be used /// to parse or build our data. This attribute is never null. - PyObject *m_atom; + PyObject* m_atom; - /// A constant or dynamic value to represent the amount of structs. Zero - /// indicates there are no sequence types associated with this field. - PyObject* m_length; + /// A constant or dynamic value to represent the amount of structs. Zero + /// indicates there are no sequence types associated with this field. + PyObject* m_length; }; #define CpRepeatedAtom_NAME "repeated" #define CpRepeatedAtom_CheckExact(op) Py_IS_TYPE((op), &CpRepeatedAtom_Type) #define CpRepeatedAtom_Check(op) PyObject_IsType((op), &CpRepeatedAtom_Type) -inline CpRepeatedAtomObject * +inline CpRepeatedAtomObject* CpRepeatedAtom_New(PyObject* atom, PyObject* length) { - return (CpRepeatedAtomObject*)CpObject_Create(&CpRepeatedAtom_Type, "OO", atom, length); + return (CpRepeatedAtomObject*)CpObject_Create( + &CpRepeatedAtom_Type, "OO", atom, length); } //------------------------------------------------------------------------------ // Conditional struct _conditionatomobj { - CpBuiltinAtom_HEAD + CpBuiltinAtom_HEAD /// Stores a reference to the actual parsing struct that will be used /// to parse or build our data. This attribute is never null. - PyObject *m_atom; + PyObject* m_atom; - /// A constant or dynamic value to represent the condition. - PyObject *m_condition; + /// A constant or dynamic value to represent the condition. + PyObject* m_condition; }; #define CpConditionAtom_NAME "condition" #define CpConditionAtom_CheckExact(op) Py_IS_TYPE((op), &CpConditionAtom_Type) #define CpConditionAtom_Check(op) PyObject_IsType((op), &CpConditionAtom_Type) -static inline CpConditionAtomObject * +static inline CpConditionAtomObject* CpConditionAtom_New(PyObject* atom, PyObject* condition) { - return (CpConditionAtomObject*)CpObject_Create(&CpConditionAtom_Type, "OO", condition, atom); + return (CpConditionAtomObject*)CpObject_Create( + &CpConditionAtom_Type, "OO", condition, atom); } //------------------------------------------------------------------------------ // Switch struct _switchatomobj { - CpBuiltinAtom_HEAD + CpBuiltinAtom_HEAD /// Stores a reference to the actual parsing struct that will be used /// to parse or build our data. This attribute is never null. - PyObject *m_atom; + PyObject* m_atom; - // A dictionary or dynamic value to represent the cases. - PyObject *m_cases; + // A dictionary or dynamic value to represent the cases. + PyObject* m_cases; - // -- internal --- - int s_callable; + // -- internal --- + int s_callable; }; #define CpSwitchAtom_NAME "switch" #define CpSwitchAtom_CheckExact(op) Py_IS_TYPE((op), &CpSwitchAtom_Type) #define CpSwitchAtom_Check(op) PyObject_IsType((op), &CpSwitchAtom_Type) -static inline CpSwitchAtomObject * +static inline CpSwitchAtomObject* CpSwitchAtom_New(PyObject* atom, PyObject* cases) { - return (CpSwitchAtomObject*)CpObject_Create(&CpSwitchAtom_Type, "OO", atom, cases); + return (CpSwitchAtomObject*)CpObject_Create( + &CpSwitchAtom_Type, "OO", atom, cases); +} + +//------------------------------------------------------------------------------ +// Offset +struct _offsetatomobj +{ + CpBuiltinAtom_HEAD + + /// Stores a reference to the actual parsing struct that will be used + /// to parse or build our data. This attribute is never null. + PyObject* m_atom; + + PyObject* m_offset; + PyObject* m_whence; + + // -- internal --- + int s_is_number; +}; + +#define CpOffsetAtom_NAME "atoffset" +#define CpOffsetAtom_CheckExact(op) Py_IS_TYPE((op), &CpOffsetAtom_Type) +#define CpOffsetAtom_Check(op) PyObject_IsType((op), &CpOffsetAtom_Type) + +static inline CpOffsetAtomObject* +CpOffsetAtom_FromSsize_t(PyObject* atom, Py_ssize_t offset) +{ + PyObject* o = PyLong_FromSsize_t(offset); + if (o == NULL) + return NULL; + CpOffsetAtomObject* a = + (CpOffsetAtomObject*)CpObject_Create(&CpOffsetAtom_Type, "OO", atom, o); + Py_DECREF(o); + return a; +} + +static inline CpOffsetAtomObject* +CpOffsetAtom_New(PyObject* atom, PyObject* offset) +{ + return (CpOffsetAtomObject*)CpObject_Create( + &CpOffsetAtom_Type, "OO", atom, offset); } #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index abaee40..03b0fae 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -56,8 +56,8 @@ struct _stateobj; typedef struct _stateobj CpStateObject; struct _layerobj; typedef struct _layerobj CpLayerObject; -struct CpStructFieldInfo; -typedef struct CpStructFieldInfo CpStructFieldInfoObject; +struct _fieldinfoobj; +typedef struct _fieldinfoobj CpStructFieldInfoObject; struct _structobj; typedef struct _structobj CpStructObject; struct _floatatomobj; @@ -86,6 +86,8 @@ struct _conditionatomobj; typedef struct _conditionatomobj CpConditionAtomObject; struct _switchatomobj; typedef struct _switchatomobj CpSwitchAtomObject; +struct _offsetatomobj; +typedef struct _offsetatomobj CpOffsetAtomObject; #ifdef _CPMODULE @@ -128,6 +130,7 @@ extern PyTypeObject CpSeqLayer_Type; extern PyTypeObject CpObjLayer_Type; extern PyTypeObject CpConditionAtom_Type; extern PyTypeObject CpSwitchAtom_Type; +extern PyTypeObject CpOffsetAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -164,20 +167,22 @@ PyObject* CpSizeOf_CAtom(CpCAtomObject* catom, CpLayerObject* layer); PyObject * CpTypeOf_CAtom(CpCAtomObject* op); CpStateObject* CpState_New(PyObject* io); PyObject* CpState_Tell(CpStateObject* self); -PyObject* CpState_Seek(CpStateObject* self, PyObject* offset, int whence); +PyObject* CpState_Seek(CpStateObject* self, PyObject* offset, PyObject* whence); PyObject* CpState_Read(CpStateObject* self, Py_ssize_t size); PyObject* CpState_ReadFully(CpStateObject* self); PyObject* CpState_Write(CpStateObject* self, PyObject* value); int CpState_SetGlobals(CpStateObject* self, PyObject* globals); CpLayerObject* CpLayer_New(CpStateObject* state, CpLayerObject* parent); int CpLayer_Invalidate(CpLayerObject* self); -CpStructFieldInfoObject* CpStructFieldInfo_New(CpFieldObject* field); +CpStructFieldInfoObject* CpStructFieldInfo_New(PyObject* name, PyObject* field); int CpStruct_AddFieldInfo(CpStructObject* o, CpStructFieldInfoObject* info); int CpStruct_AddField(CpStructObject* o, CpFieldObject* field, int exclude); CpStructObject* CpStruct_New(PyObject* model); PyObject* CpStruct_GetAnnotations(CpStructObject* o, int eval); int CpStruct_ReplaceType(CpStructObject* o, PyObject* name, PyObject* type); int CpStruct_HasOption(CpStructObject* o, PyObject* option); +int CpStruct_Pack(CpStructObject* self, PyObject* obj, CpLayerObject* layer); +PyObject* CpStruct_Unpack(CpStructObject* self, CpLayerObject* layer); int CpStructModel_Check(PyObject* model, _modulestate* state); PyObject* CpStructModel_GetStruct(PyObject* model, _modulestate* state); CpSeqLayerObject* CpSeqLayer_New(CpStateObject* state, CpLayerObject* parent); @@ -208,6 +213,9 @@ int CpConditionAtom_IsEnabled(CpConditionAtomObject* self, PyObject* context); PyObject* CpSwitchAtom_GetNext(CpSwitchAtomObject* self, PyObject* op, PyObject* context); int CpSwitchAtom_Pack(CpSwitchAtomObject* self, PyObject* obj, CpLayerObject* layer); PyObject* CpSwitchAtom_Unpack(CpSwitchAtomObject* self, CpLayerObject* layer); +int CpOffsetAtom_Pack(CpOffsetAtomObject* self, PyObject* obj, CpLayerObject* layer); +PyObject* CpOffsetAtom_Unpack(CpOffsetAtomObject* self, CpLayerObject* layer); +PyObject* CpOffsetAtom_GetOffset(CpOffsetAtomObject* self, PyObject* layer); #else @@ -251,6 +259,7 @@ caterpillar_api.py #define CpObjLayer_Type (*(PyTypeObject *)Cp_API[31]) #define CpConditionAtom_Type (*(PyTypeObject *)Cp_API[32]) #define CpSwitchAtom_Type (*(PyTypeObject *)Cp_API[33]) +#define CpOffsetAtom_Type (*(PyTypeObject *)Cp_API[34]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -287,25 +296,27 @@ caterpillar_api.py #define CpTypeOf_CAtom (*((PyObject * (*)(CpCAtomObject* op)))Cp_API[87]) #define CpState_New (*((CpStateObject* (*)(PyObject* io)))Cp_API[88]) #define CpState_Tell (*((PyObject* (*)(CpStateObject* self)))Cp_API[89]) -#define CpState_Seek (*((PyObject* (*)(CpStateObject* self, PyObject* offset, int whence)))Cp_API[90]) +#define CpState_Seek (*((PyObject* (*)(CpStateObject* self, PyObject* offset, PyObject* whence)))Cp_API[90]) #define CpState_Read (*((PyObject* (*)(CpStateObject* self, Py_ssize_t size)))Cp_API[91]) #define CpState_ReadFully (*((PyObject* (*)(CpStateObject* self)))Cp_API[92]) #define CpState_Write (*((PyObject* (*)(CpStateObject* self, PyObject* value)))Cp_API[93]) #define CpState_SetGlobals (*((int (*)(CpStateObject* self, PyObject* globals)))Cp_API[94]) #define CpLayer_New (*((CpLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[95]) #define CpLayer_Invalidate (*((int (*)(CpLayerObject* self)))Cp_API[96]) -#define CpStructFieldInfo_New (*((CpStructFieldInfoObject* (*)(CpFieldObject* field)))Cp_API[98]) +#define CpStructFieldInfo_New (*((CpStructFieldInfoObject* (*)(PyObject* name, PyObject* field)))Cp_API[98]) #define CpStruct_AddFieldInfo (*((int (*)(CpStructObject* o, CpStructFieldInfoObject* info)))Cp_API[99]) #define CpStruct_AddField (*((int (*)(CpStructObject* o, CpFieldObject* field, int exclude)))Cp_API[100]) #define CpStruct_New (*((CpStructObject* (*)(PyObject* model)))Cp_API[101]) #define CpStruct_GetAnnotations (*((PyObject* (*)(CpStructObject* o, int eval)))Cp_API[102]) #define CpStruct_ReplaceType (*((int (*)(CpStructObject* o, PyObject* name, PyObject* type)))Cp_API[103]) #define CpStruct_HasOption (*((int (*)(CpStructObject* o, PyObject* option)))Cp_API[104]) -#define CpStructModel_Check (*((int (*)(PyObject* model, _modulestate* state)))Cp_API[105]) -#define CpStructModel_GetStruct (*((PyObject* (*)(PyObject* model, _modulestate* state)))Cp_API[106]) -#define CpSeqLayer_New (*((CpSeqLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[107]) -#define CpSeqLayer_SetSequence (*((int (*)(CpSeqLayerObject* self,PyObject* sequence,Py_ssize_t length,int8_t greedy)))Cp_API[108]) -#define CpObjLayer_New (*((CpObjLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[109]) +#define CpStruct_Pack (*((int (*)(CpStructObject* self, PyObject* obj, CpLayerObject* layer)))Cp_API[105]) +#define CpStruct_Unpack (*((PyObject* (*)(CpStructObject* self, CpLayerObject* layer)))Cp_API[106]) +#define CpStructModel_Check (*((int (*)(PyObject* model, _modulestate* state)))Cp_API[107]) +#define CpStructModel_GetStruct (*((PyObject* (*)(PyObject* model, _modulestate* state)))Cp_API[108]) +#define CpSeqLayer_New (*((CpSeqLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[109]) +#define CpSeqLayer_SetSequence (*((int (*)(CpSeqLayerObject* self,PyObject* sequence,Py_ssize_t length,int8_t greedy)))Cp_API[110]) +#define CpObjLayer_New (*((CpObjLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[111]) #define CpIntAtom_Pack (*((int (*)(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer)))Cp_API[120]) #define CpIntAtom_Unpack (*((PyObject* (*)(CpIntAtomObject* self, CpLayerObject* layer)))Cp_API[121]) #define CpFloatAtom_Pack (*((int (*)(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[122]) @@ -331,6 +342,9 @@ caterpillar_api.py #define CpSwitchAtom_GetNext (*((PyObject* (*)(CpSwitchAtomObject* self, PyObject* op, PyObject* context)))Cp_API[142]) #define CpSwitchAtom_Pack (*((int (*)(CpSwitchAtomObject* self, PyObject* obj, CpLayerObject* layer)))Cp_API[143]) #define CpSwitchAtom_Unpack (*((PyObject* (*)(CpSwitchAtomObject* self, CpLayerObject* layer)))Cp_API[144]) +#define CpOffsetAtom_Pack (*((int (*)(CpOffsetAtomObject* self, PyObject* obj, CpLayerObject* layer)))Cp_API[145]) +#define CpOffsetAtom_Unpack (*((PyObject* (*)(CpOffsetAtomObject* self, CpLayerObject* layer)))Cp_API[146]) +#define CpOffsetAtom_GetOffset (*((PyObject* (*)(CpOffsetAtomObject* self, PyObject* layer)))Cp_API[147]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillar/include/caterpillar/struct.h b/src/caterpillar/include/caterpillar/struct.h index 4d3f0b5..3fe351b 100644 --- a/src/caterpillar/include/caterpillar/struct.h +++ b/src/caterpillar/include/caterpillar/struct.h @@ -18,19 +18,25 @@ #define CP_STRUCT_H #include "caterpillar/caterpillarapi.h" -#include "caterpillar/field.h" +#include "caterpillar/atoms/builtins.h" /* Caterpillar Struct C implementation */ /** * @brief Internal class used to store information about a field */ -struct CpStructFieldInfo +struct _fieldinfoobj { PyObject_HEAD - /// the referenced field object - CpFieldObject* m_field; + /// the referenced field object + PyObject *m_field; + + /// the name of the field + PyObject *m_name; + + /// The configured default value. + PyObject* m_default; // Excluded: True if the field is included in the struct int8_t s_excluded; @@ -70,7 +76,8 @@ struct CpStructFieldInfo */ struct _structobj { - CpFieldAtom_HEAD PyTypeObject* m_model; // underlying class + CpBuiltinAtom_HEAD + PyTypeObject* m_model; // underlying class PyObject* m_members; // Dict[str, FieldInfo] PyObject* m_options; // set[CpOption] @@ -89,9 +96,6 @@ struct _structobj _modulestate* s_mod; }; -/// Struct object type -// PyAPI_DATA(PyTypeObject) CpStruct_Type; - /** * @brief Checks if the given object is a struct object * diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index 30c4c62..f78f843 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -37,7 +37,7 @@ void *Cp_API[] = { (void *) &CpObjLayer_Type, (void *) &CpConditionAtom_Type, (void *) &CpSwitchAtom_Type, - NULL, + (void *) &CpOffsetAtom_Type, NULL, NULL, NULL, @@ -108,6 +108,8 @@ void *Cp_API[] = { (void *) &CpStruct_GetAnnotations, (void *) &CpStruct_ReplaceType, (void *) &CpStruct_HasOption, + (void *) &CpStruct_Pack, + (void *) &CpStruct_Unpack, (void *) &CpStructModel_Check, (void *) &CpStructModel_GetStruct, (void *) &CpSeqLayer_New, @@ -121,8 +123,6 @@ void *Cp_API[] = { NULL, NULL, NULL, - NULL, - NULL, (void *) &CpIntAtom_Pack, (void *) &CpIntAtom_Unpack, (void *) &CpFloatAtom_Pack, @@ -147,6 +147,9 @@ void *Cp_API[] = { (void *) &CpConditionAtom_IsEnabled, (void *) &CpSwitchAtom_GetNext, (void *) &CpSwitchAtom_Pack, - (void *) &CpSwitchAtom_Unpack + (void *) &CpSwitchAtom_Unpack, + (void *) &CpOffsetAtom_Pack, + (void *) &CpOffsetAtom_Unpack, + (void *) &CpOffsetAtom_GetOffset }; diff --git a/src/layer.c b/src/layer.c index 63b26b1..af390b4 100644 --- a/src/layer.c +++ b/src/layer.c @@ -265,8 +265,8 @@ cp_objlayer_init(CpObjLayerObject* self, PyObject* args, PyObject* kw) Py_XSETREF(self->ob_base.m_state, (CpStateObject*)Py_NewRef(state)); Py_XSETREF(self->m_obj, (PyObject*)CpContext_New()); - Py_XSETREF(self->ob_base.m_path, Py_NewRef(path)); - Py_XSETREF(self->ob_base.m_parent, (CpLayerObject*)Py_NewRef(parent)); + Py_XSETREF(self->ob_base.m_path, Py_XNewRef(path)); + Py_XSETREF(self->ob_base.m_parent, (CpLayerObject*)Py_XNewRef(parent)); return 0; } @@ -335,8 +335,8 @@ cp_objlayer_context_getattr(CpLayerObject* self, PyObject* args) CpObjLayerObject* CpObjLayer_New(CpStateObject* state, CpLayerObject* parent) { - CpObjLayerObject* self = - (CpObjLayerObject*)CpObject_Create(&CpObjLayer_Type, "O", state); + CpObjLayerObject* self = (CpObjLayerObject*)CpObject_CreateOneArg( + &CpObjLayer_Type, (PyObject*)state); if (!self) { return NULL; } @@ -363,13 +363,13 @@ static PyMemberDef CpObjLayer_Members[] = { static PyMethodDef CpObjLayer_Methods[] = { { "__context_getattr__", (PyCFunction)cp_objlayer_context_getattr, - METH_VARARGS }, + METH_VARARGS}, { NULL } /* Sentinel */ }; PyTypeObject CpObjLayer_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpObjLayer_NAME), - .tp_basicsize = sizeof(CpLayerObject), + .tp_basicsize = sizeof(CpObjLayerObject), .tp_dealloc = (destructor)cp_objlayer_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_doc = NULL, diff --git a/src/module.c b/src/module.c index 06b4b85..108d1c5 100644 --- a/src/module.c +++ b/src/module.c @@ -381,24 +381,29 @@ PyInit__C(void) CpModule_SetupType(&CpFieldAtom_Type); CpModule_SetupType(&CpFieldCAtom_Type); CpModule_SetupType(&CpLayer_Type); + + CpSeqLayer_Type.tp_base = &CpLayer_Type; + CpObjLayer_Type.tp_base = &CpLayer_Type; CpModule_SetupType(&CpSeqLayer_Type); CpModule_SetupType(&CpObjLayer_Type); CpModule_SetupType(&CpState_Type); CpModule_SetupType(&CpStructFieldInfo_Type); - CpStruct_Type.tp_base = &CpFieldAtom_Type; - CpModule_SetupType(&CpStruct_Type); // builtins setup CpBuiltinAtom_Type.tp_base = &CpCAtom_Type; CpRepeatedAtom_Type.tp_base = &CpBuiltinAtom_Type; CpConditionAtom_Type.tp_base = &CpBuiltinAtom_Type; CpSwitchAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpOffsetAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpStruct_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpBuiltinAtom_Type); CpModule_SetupType(&CpRepeatedAtom_Type); CpModule_SetupType(&CpConditionAtom_Type); CpModule_SetupType(&CpSwitchAtom_Type); + CpModule_SetupType(&CpOffsetAtom_Type); + CpModule_SetupType(&CpStruct_Type); CpIntAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpIntAtom_Type); @@ -454,6 +459,7 @@ PyInit__C(void) CpModule_AddObject(CpRepeatedAtom_NAME, &CpRepeatedAtom_Type); CpModule_AddObject(CpConditionAtom_NAME, &CpConditionAtom_Type); CpModule_AddObject(CpSwitchAtom_NAME, &CpSwitchAtom_Type); + CpModule_AddObject(CpOffsetAtom_NAME, &CpOffsetAtom_Type); CpModule_AddObject(CpIntAtom_NAME, &CpIntAtom_Type); CpModule_AddObject(CpFloatAtom_NAME, &CpFloatAtom_Type); diff --git a/src/state.c b/src/state.c index 81e2fcf..f22b1de 100644 --- a/src/state.c +++ b/src/state.c @@ -138,8 +138,8 @@ static PyObject* cp_state_seek(CpStateObject* self, PyObject* args) { PyObject* offset = NULL; - int whence = 0; - if (!PyArg_ParseTuple(args, "O|i", &offset, &whence)) { + PyObject* whence = NULL; + if (!PyArg_ParseTuple(args, "OO", &offset, &whence)) { return NULL; } return CpState_Seek(self, offset, whence); @@ -170,7 +170,7 @@ CpState_Tell(CpStateObject* self) /*CpAPI*/ PyObject* -CpState_Seek(CpStateObject* self, PyObject* offset, int whence) +CpState_Seek(CpStateObject* self, PyObject* offset, PyObject* whence) { return PyObject_CallMethodObjArgs( self->m_io, self->mod->str_seek, offset, whence); diff --git a/src/struct.c b/src/struct.c index 214c0af..f248c77 100644 --- a/src/struct.c +++ b/src/struct.c @@ -1,9 +1,8 @@ /* struct and struct-field-info implementation */ #include -#include "caterpillar/arch.h" -#include "caterpillar/module.h" -#include "caterpillar/struct.h" +#include "caterpillar/caterpillar.h" + #include /* struct-field-info implementation */ @@ -16,6 +15,7 @@ cp_struct_fieldinfo_new(PyTypeObject* type, PyObject* args, PyObject* kw) return NULL; } self->m_field = NULL; + self->m_name = NULL; self->s_excluded = false; return (PyObject*)self; } @@ -24,6 +24,7 @@ static void cp_struct_fieldinfo_dealloc(CpStructFieldInfoObject* self) { Py_XDECREF(self->m_field); + Py_XDECREF(self->m_name); Py_TYPE(self)->tp_free((PyObject*)self); } @@ -32,12 +33,12 @@ cp_struct_fieldinfo_init(CpStructFieldInfoObject* self, PyObject* args, PyObject* kw) { - static char* kwlist[] = { "field", "excluded", NULL }; - PyObject* field = NULL; + static char* kwlist[] = { "name", "field", "excluded", NULL }; + PyObject *field = NULL, *name = NULL; int excluded = false; if (!PyArg_ParseTupleAndKeywords( - args, kw, "O|p", kwlist, &field, &excluded)) { + args, kw, "OO|p", kwlist, &name, &field, &excluded)) { return -1; } @@ -46,12 +47,8 @@ cp_struct_fieldinfo_init(CpStructFieldInfoObject* self, return -1; } - if (!CpField_Check(field)) { - PyErr_SetString(PyExc_ValueError, "field is not a field!"); - return -1; - } - - Py_XSETREF(self->m_field, (CpFieldObject*)Py_XNewRef(field)); + Py_XSETREF(self->m_field, Py_XNewRef(field)); + Py_XSETREF(self->m_name, (PyObject*)Py_XNewRef(name)); self->s_excluded = excluded; return 0; } @@ -59,18 +56,17 @@ cp_struct_fieldinfo_init(CpStructFieldInfoObject* self, static PyObject* cp_struct_fieldinfo_repr(CpStructFieldInfoObject* self) { - return PyUnicode_FromFormat("", - self->m_field->m_name); + return PyUnicode_FromFormat("", self->m_name, self->m_field); } /* public API */ /*CpAPI*/ CpStructFieldInfoObject* -CpStructFieldInfo_New(CpFieldObject* field) +CpStructFieldInfo_New(PyObject* name, PyObject* field) { - return (CpStructFieldInfoObject*)CpObject_CreateOneArg( - &CpStructFieldInfo_Type, (PyObject*)field); + return (CpStructFieldInfoObject*)CpObject_Create( + &CpStructFieldInfo_Type, "OO", name, field); } /* docs */ @@ -82,6 +78,11 @@ static PyMemberDef CpStructFieldInfo_Members[] = { offsetof(CpStructFieldInfoObject, m_field), READONLY, NULL }, + { "name", + T_OBJECT, + offsetof(CpStructFieldInfoObject, m_name), + READONLY, + NULL }, { "excluded", T_BOOL, offsetof(CpStructFieldInfoObject, s_excluded), @@ -204,6 +205,13 @@ cp_struct_new(PyTypeObject* type, PyObject* args, PyObject* kw) if (!self->s_std_init_fields) { return NULL; } + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpStruct_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpStruct_Unpack; + CpBuiltinAtom_CATOM(self).ob_size = NULL; + CpBuiltinAtom_CATOM(self).ob_type = NULL; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; return (PyObject*)self; } @@ -571,7 +579,7 @@ cp_struct_process_annotation(CpStructObject* self, // It is worthwhile to note that the default value specified here will be // assigned to the created field instance. There is no destinction between // class-level defaults and field-level defaults. - CpFieldObject* field = NULL; + PyObject* field = NULL; _modulestate* state = self->s_mod; { // 1. Annotated field: @@ -579,7 +587,7 @@ cp_struct_process_annotation(CpStructObject* self, // it. Note thate we check against the *exact* type, which means that we // will not accept any subclass of CpField. if (CpField_CheckExact(annotation)) { - field = (CpFieldObject*)Py_NewRef(annotation); + field = Py_NewRef(annotation); } // 2. Atom object or protocol @@ -590,10 +598,7 @@ cp_struct_process_annotation(CpStructObject* self, // only against packing, unpacking and size calculation. The type function // is optional and won't be covered here. else if (CpAtom_Check(annotation)) { - field = (CpFieldObject*)CpField_New(annotation); - if (!field) { - return -1; - } + field = Py_NewRef(annotation); } // 3. Type @@ -605,11 +610,7 @@ cp_struct_process_annotation(CpStructObject* self, if (!struct_) { return -1; } - field = (CpFieldObject*)CpField_New(struct_); - Py_XDECREF(struct_); - if (!field) { - return -1; - } + field = Py_NewRef(struct_); } } @@ -617,43 +618,48 @@ cp_struct_process_annotation(CpStructObject* self, // The annotation is a callable object and conforms to the ContextLambda // protocol. Note that we assume the right function signature. else if (PyCallable_Check(annotation)) { - field = (CpFieldObject*)CpField_New(annotation); - if (!field) { - return -1; - } + field = Py_NewRef(annotation); + } + + else { + // REVISIT: Add support for other types here. + PyErr_Format(PyExc_ValueError, + ("Field %R could not be created, because the placed " + "annotation does not " + "conform to any of the supported types.\n" + "annotation: %R"), + name, + annotation); + return -1; } } - // Currently, there is no extension support for other types. That is - // a future feature. - if (!field) { - PyErr_Format( - PyExc_ValueError, - ("Field %R could not be created, because the placed annotation does not " - "conform to any of the supported types.\n" - "annotation: %R"), - name, - annotation); - return -1; + // TODO: arch + PyObject* next_field = + CpEndian_SetEndian(field, (CpEndianObject*)self->m_endian); + if (next_field) { + Py_SETREF(field, next_field); + } else { + PyErr_Clear(); } - _Cp_SetObj(field->m_arch, self->m_arch); - _Cp_SetObj(field->m_endian, self->m_endian); - _Cp_SetObj(field->m_default, default_value); - _Cp_SetObj(field->m_name, name); - if (_PySet_Update(field->m_options, self->m_field_options) < 0) { + CpStructFieldInfoObject* field_info = CpStructFieldInfo_New(name, field); + if (!field_info) { Py_XDECREF(field); return -1; } + field_info->s_excluded = exclude; + _Cp_SetObj(field_info->m_default, default_value); - int res = CpStruct_AddField(self, field, exclude); + int res = CpStruct_AddFieldInfo(self, field_info); if (res < 0) { + Py_XDECREF(field_info); Py_XDECREF(field); return -1; } if (!Cp_IsInvalidDefault(default_value)) { - if (CpStructModel_SetDefault(self, field->m_name, default_value) < 0) { + if (CpStructModel_SetDefault(self, name, default_value) < 0) { Py_XDECREF(field); return -1; } @@ -668,7 +674,7 @@ cp_struct_process_annotation(CpStructObject* self, } if (res) { - PyObject* type = CpTypeOf_Field(field); + PyObject* type = CpTypeOf(field); if (!type) { Py_XDECREF(field); return -1; @@ -709,15 +715,15 @@ _cp_struct_model__init__(PyObject* self, PyObject* args, PyObject* kwnames) if (i >= argc) { // If the current positional field is defined as a keyword argument, // we should retrieve it from the keyword arguments. - if (!kwnames || !PyDict_Contains(kwnames, info->m_field->m_name)) { + if (!kwnames || !PyDict_Contains(kwnames, info->m_name)) { Py_XDECREF(struct_); PyErr_Format(PyExc_ValueError, ("Missing argument for positional field %R"), - info->m_field->m_name); + info->m_name); return -1; } - value = PyDict_GetItem(kwnames, info->m_field->m_name); + value = PyDict_GetItem(kwnames, info->m_name); } else value = PyTuple_GetItem(args, i); @@ -728,7 +734,7 @@ _cp_struct_model__init__(PyObject* self, PyObject* args, PyObject* kwnames) // as we store a borrowed reference in 'value', we don't need // to decref it. - if (PyObject_SetAttr(self, info->m_field->m_name, value) < 0) { + if (PyObject_SetAttr(self, info->m_name, value) < 0) { Py_XDECREF(struct_); return -1; } @@ -747,10 +753,10 @@ _cp_struct_model__init__(PyObject* self, PyObject* args, PyObject* kwnames) // A custom value is only provided if it is in the given keyword // arguments. - if (!kwnames || !PyDict_Contains(kwnames, info->m_field->m_name)) { - value = info->m_field->m_default; + if (!kwnames || !PyDict_Contains(kwnames, info->m_name)) { + value = info->m_default; } else { - value = PyDict_GetItem(kwnames, info->m_field->m_name); + value = PyDict_GetItem(kwnames, info->m_name); } if (!value) { @@ -759,7 +765,7 @@ _cp_struct_model__init__(PyObject* self, PyObject* args, PyObject* kwnames) } // same as before, we don't need to decref the value - if (PyObject_SetAttr(self, info->m_field->m_name, value) < 0) { + if (PyObject_SetAttr(self, info->m_name, value) < 0) { Py_XDECREF(struct_); return -1; } @@ -980,7 +986,7 @@ cp_struct_add_slots(CpStructObject* self) static PyObject* cp_struct_repr(CpStructObject* self) { - return PyUnicode_FromFormat("", self->m_model->tp_name); + return PyUnicode_FromFormat(">", self->m_model->tp_name); } /* public API */ @@ -994,15 +1000,14 @@ CpStruct_AddFieldInfo(CpStructObject* o, CpStructFieldInfoObject* info) return -1; } - CpFieldObject* field = info->m_field; - if (PyMapping_HasKey(o->m_members, field->m_name)) { + if (PyMapping_HasKey(o->m_members, info->m_name)) { PyErr_Format( - PyExc_ValueError, "field with name %R already exists", field->m_name); + PyExc_ValueError, "field with name %R already exists", info->m_name); return -1; } if (!info->s_excluded) { - PyObject* list = Cp_IsInvalidDefault(field->m_default) + PyObject* list = Cp_IsInvalidDefault(info->m_default) ? o->s_std_init_fields : o->s_kwonly_init_fields; @@ -1010,7 +1015,7 @@ CpStruct_AddFieldInfo(CpStructObject* o, CpStructFieldInfoObject* info) return -1; } } - return PyObject_SetItem(o->m_members, field->m_name, (PyObject*)info); + return PyObject_SetItem(o->m_members, info->m_name, (PyObject*)info); } /*CpAPI*/ @@ -1022,7 +1027,8 @@ CpStruct_AddField(CpStructObject* o, CpFieldObject* field, int exclude) return -1; } - CpStructFieldInfoObject* info = CpStructFieldInfo_New(field); + CpStructFieldInfoObject* info = + CpStructFieldInfo_New(field->m_name, (PyObject*)field); if (!info) { return -1; } @@ -1129,6 +1135,81 @@ CpStruct_New(PyObject* model) return (CpStructObject*)CpObject_CreateOneArg(&CpStruct_Type, model); } +/*CpAPI*/ +int +CpStruct_Pack(CpStructObject* self, PyObject* obj, CpLayerObject* layer) +{ + CpObjLayerObject* obj_layer = CpObjLayer_New(layer->m_state, layer); + if (!obj_layer) { + return -1; + } + + _Cp_SetObj(obj_layer->m_obj, Py_NewRef(obj)); + CpStructFieldInfoObject* info = NULL; + // all borrowed references + PyObject *name = NULL, *value = NULL; + int res = 0; + + Py_ssize_t pos = 0; + while (PyDict_Next(self->m_members, &pos, &name, (PyObject**)&info)) { + value = PyObject_GetAttr(obj, name); + if (!value) { + goto failure; + } + + res = _Cp_Pack(value, (PyObject*)info->m_field, (CpLayerObject*)obj_layer); + Py_XSETREF(value, NULL); + if (res < 0) { + goto failure; + } + } + + CpLayer_Invalidate((CpLayerObject*)obj_layer); + return 0; + +failure: + CpLayer_Invalidate((CpLayerObject*)obj_layer); + Py_XDECREF(value); + return -1; +} + +/*CpAPI*/ +PyObject* +CpStruct_Unpack(CpStructObject* self, CpLayerObject* layer) +{ + printf("creating layer....\n"); + CpObjLayerObject* obj_layer = CpObjLayer_New(layer->m_state, layer); + if (!obj_layer) { + return NULL; + } + + CpStructFieldInfoObject* info = NULL; + Py_ssize_t pos = 0; + PyObject *name = NULL, *value = NULL; + while (PyDict_Next(self->m_members, &pos, &name, (PyObject**)&info)) { + if (!info->m_field) { + return NULL; + } + + printf("getting %s....\n", PyUnicode_AsUTF8(name)); + value = _Cp_Unpack((PyObject*)info->m_field, (CpLayerObject*)obj_layer); + CpLayer_AppendPath(&obj_layer->ob_base, name); + if (!value || PyDict_SetItem(obj_layer->m_obj, name, value) < 0) { + Py_XDECREF(value); + Py_XDECREF(obj_layer); + return NULL; + } + Py_XDECREF(value); + } + printf("finished parsing....\n"); + PyObject* args = PyTuple_New(0); + PyObject* obj = + PyObject_Call((PyObject*)self->m_model, args, obj_layer->m_obj); + Py_XDECREF(args); + Py_XDECREF(obj_layer); + return obj; +} + /* docs */ /* members */ From 039aea7eb1e850073812ac1f8e2158baa51f0f4e Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 8 Sep 2024 15:45:39 +0200 Subject: [PATCH 08/29] New primitive atom for python classes --- + CpPrimitiveAtom <-> patom + Updated _C.pyi + updated tests (padding_atom excluded) + Added CpStruct_Sizeof + Fixed CpState_Seek + Builtin-Atoms now use builtinatom as base class --- CMakeLists.txt | 1 + src/atomimpl/boolatomobj.c | 14 +-- src/atomimpl/builtins/builtinatomobj.c | 12 +- src/atomimpl/builtins/offsetatomobj.c | 2 +- src/atomimpl/builtins/primitiveatomobj.c | 83 +++++++++++++ src/atomimpl/charatomobj.c | 14 +-- src/atomimpl/constatomobj.c | 14 +-- src/atomimpl/floatatomobj.c | 14 +-- src/atomimpl/intatomobj.c | 14 +-- src/atomimpl/padatomobj.c | 14 +-- src/atomimpl/stringatomobj.c | 14 +-- src/capi.dat | 13 +- src/caterpillar/_C.pyi | 66 +++++++++- .../include/caterpillar/atoms/builtins.h | 10 +- .../include/caterpillar/atoms/const.h | 5 +- .../include/caterpillar/atoms/float.h | 5 +- .../include/caterpillar/atoms/primitive.h | 23 +++- .../include/caterpillar/atoms/string.h | 5 +- .../include/caterpillar/caterpillarapi.h | 16 ++- src/caterpillarapi.c | 4 +- src/layer.c | 10 +- src/module.c | 24 ++-- src/parsing_sizeof.c | 4 +- src/state.c | 2 +- src/struct.c | 115 +++++++++++------- test/_C/test_parsing.py | 15 ++- test/_C/test_state.py | 8 +- test/_C/test_struct.py | 2 +- 28 files changed, 354 insertions(+), 169 deletions(-) create mode 100644 src/atomimpl/builtins/primitiveatomobj.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a6ec7f..ad2be11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,7 @@ python_add_library( src/atomimpl/builtins/conditionatomobj.c src/atomimpl/builtins/switchatomobj.c src/atomimpl/builtins/offsetatomobj.c + src/atomimpl/builtins/primitiveatomobj.c WITH_SOABI ) diff --git a/src/atomimpl/boolatomobj.c b/src/atomimpl/boolatomobj.c index 5332a7a..d2214ed 100644 --- a/src/atomimpl/boolatomobj.c +++ b/src/atomimpl/boolatomobj.c @@ -20,13 +20,13 @@ cp_boolatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { CpBoolAtomObject* self = (CpBoolAtomObject*)type->tp_alloc(type, 0); if (self != NULL) { - CpFieldCAtom_CATOM(self).ob_pack = (packfunc)CpBoolAtom_Pack; - CpFieldCAtom_CATOM(self).ob_unpack = (unpackfunc)CpBoolAtom_Unpack; - CpFieldCAtom_CATOM(self).ob_pack_many = NULL; - CpFieldCAtom_CATOM(self).ob_unpack_many = NULL; - CpFieldCAtom_CATOM(self).ob_size = (sizefunc)cp_boolatom__size__; - CpFieldCAtom_CATOM(self).ob_type = (typefunc)cp_boolatom__type__; - CpFieldCAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpBoolAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpBoolAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_boolatom__size__; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_boolatom__type__; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; } return (PyObject*)self; } diff --git a/src/atomimpl/builtins/builtinatomobj.c b/src/atomimpl/builtins/builtinatomobj.c index 76c8c3a..3799b8c 100644 --- a/src/atomimpl/builtins/builtinatomobj.c +++ b/src/atomimpl/builtins/builtinatomobj.c @@ -66,22 +66,24 @@ cp_builtinatom_as_number_matmul(PyObject* self, PyObject* other) return (PyObject*)CpOffsetAtom_New(self, other); } -static PyMappingMethods CpFieldAtom_MappingMethods = { +static PyMappingMethods CpBuiltinAtom_MappingMethods = { .mp_subscript = (binaryfunc)cp_builtinatom_as_mapping_getitem, }; -static PyNumberMethods CpField_NumberMethods = { +static PyNumberMethods CpBuiltinAtom_NumberMethods = { .nb_rshift = (binaryfunc)cp_builtinatom_as_number_rshift, .nb_floor_divide = (binaryfunc)cp_builtinatom_as_number_floordiv, .nb_matrix_multiply = (binaryfunc)cp_builtinatom_as_number_matmul, }; PyTypeObject CpBuiltinAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "CpBuiltinAtom", // tp_name + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpBuiltinAtom_NAME), .tp_basicsize = sizeof(CpBuiltinAtomObject), .tp_dealloc = (destructor)cp_builtinatom_dealloc, .tp_init = (initproc)cp_builtinatom_init, .tp_new = (newfunc)cp_builtinatom_new, - .tp_as_mapping = &CpFieldAtom_MappingMethods, - .tp_as_number = &CpField_NumberMethods, + .tp_as_mapping = &CpBuiltinAtom_MappingMethods, + .tp_as_number = &CpBuiltinAtom_NumberMethods, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = NULL, }; diff --git a/src/atomimpl/builtins/offsetatomobj.c b/src/atomimpl/builtins/offsetatomobj.c index 2cb0ddb..6b9155b 100644 --- a/src/atomimpl/builtins/offsetatomobj.c +++ b/src/atomimpl/builtins/offsetatomobj.c @@ -132,7 +132,7 @@ CpOffsetAtom_Pack(CpOffsetAtomObject* self, PyObject* obj, CpLayerObject* layer) } Py_DECREF(res); - if (_Cp_Pack(self->m_atom, obj, layer) < 0) { + if (_Cp_Pack(obj, self->m_atom, layer) < 0) { return -1; } diff --git a/src/atomimpl/builtins/primitiveatomobj.c b/src/atomimpl/builtins/primitiveatomobj.c new file mode 100644 index 0000000..85a8da7 --- /dev/null +++ b/src/atomimpl/builtins/primitiveatomobj.c @@ -0,0 +1,83 @@ +/* primitive atom C implementation */ + +#include "caterpillar/caterpillar.h" + +#include + +/* impl */ +static PyObject* +cp_primitiveatom_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + CpPrimitiveAtomObject* self; + self = (CpPrimitiveAtomObject*)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + return (PyObject*)self; +} + +static void +cp_primitiveatom_dealloc(CpPrimitiveAtomObject* self) +{ + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_primitiveatom_init(CpPrimitiveAtomObject* self, PyObject* args, PyObject* kw) +{ + _Cp_InitNoArgs(CpPrimitiveAtomObject, args, kw); +} + + +static PyObject * +cp_primitiveatom_repr(CpPrimitiveAtomObject *self) +{ + return PyUnicode_FromString(""); +} + +// TODO member methods +static PyObject* +cp_primitiveatom_as_mapping_getitem(CpPrimitiveAtomObject* self, PyObject* length) +{ + return (PyObject*)CpRepeatedAtom_New((PyObject*)self, length); +} + +static PyObject* +cp_primitiveatom_as_number_rshift(PyObject* self, PyObject* other) +{ + return (PyObject*)CpSwitchAtom_New(self, other); +} + +static PyObject* +cp_primitiveatom_as_number_floordiv(PyObject* self, PyObject* other) +{ + return (PyObject*)CpConditionAtom_New(self, other); +} + +static PyObject* +cp_primitiveatom_as_number_matmul(PyObject* self, PyObject* other) +{ + return (PyObject*)CpOffsetAtom_New(self, other); +} + +static PyMappingMethods CpPrimitiveAtom_MappingMethods = { + .mp_subscript = (binaryfunc)cp_primitiveatom_as_mapping_getitem, +}; + +static PyNumberMethods CpPrimitiveAtom_NumberMethods = { + .nb_rshift = (binaryfunc)cp_primitiveatom_as_number_rshift, + .nb_floor_divide = (binaryfunc)cp_primitiveatom_as_number_floordiv, + .nb_matrix_multiply = (binaryfunc)cp_primitiveatom_as_number_matmul, +}; + +PyTypeObject CpPrimitiveAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpPrimitiveAtom_NAME), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_basicsize = sizeof(CpPrimitiveAtomObject), + .tp_dealloc = (destructor)cp_primitiveatom_dealloc, + .tp_init = (initproc)cp_primitiveatom_init, + .tp_repr = (reprfunc)cp_primitiveatom_repr, + .tp_new = (newfunc)cp_primitiveatom_new, + .tp_doc = NULL, + .tp_as_mapping = &CpPrimitiveAtom_MappingMethods, + .tp_as_number = &CpPrimitiveAtom_NumberMethods, +}; \ No newline at end of file diff --git a/src/atomimpl/charatomobj.c b/src/atomimpl/charatomobj.c index 3804f36..bc5d0ce 100644 --- a/src/atomimpl/charatomobj.c +++ b/src/atomimpl/charatomobj.c @@ -20,13 +20,13 @@ cp_charatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { CpCharAtomObject* self = (CpCharAtomObject*)type->tp_alloc(type, 0); if (self != NULL) { - CpFieldCAtom_CATOM(self).ob_pack = (packfunc)CpCharAtom_Pack; - CpFieldCAtom_CATOM(self).ob_unpack = (unpackfunc)CpCharAtom_Unpack; - CpFieldCAtom_CATOM(self).ob_pack_many = NULL; - CpFieldCAtom_CATOM(self).ob_unpack_many = NULL; - CpFieldCAtom_CATOM(self).ob_size = (sizefunc)cp_charatom__size__; - CpFieldCAtom_CATOM(self).ob_type = (typefunc)cp_charatom__type__; - CpFieldCAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpCharAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpCharAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_charatom__size__; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_charatom__type__; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; } return (PyObject*)self; } diff --git a/src/atomimpl/constatomobj.c b/src/atomimpl/constatomobj.c index 26c3a9c..8a4c648 100644 --- a/src/atomimpl/constatomobj.c +++ b/src/atomimpl/constatomobj.c @@ -20,13 +20,13 @@ cp_constatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { CpConstAtomObject* self = (CpConstAtomObject*)type->tp_alloc(type, 0); if (self != NULL) { - CpFieldCAtom_CATOM(self).ob_pack = (packfunc)CpConstAtom_Pack; - CpFieldCAtom_CATOM(self).ob_unpack = (unpackfunc)CpConstAtom_Unpack; - CpFieldCAtom_CATOM(self).ob_pack_many = NULL; - CpFieldCAtom_CATOM(self).ob_unpack_many = NULL; - CpFieldCAtom_CATOM(self).ob_size = (sizefunc)cp_constatom__size__; - CpFieldCAtom_CATOM(self).ob_type = (typefunc)cp_constatom__type__; - CpFieldCAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpConstAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpConstAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_constatom__size__; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_constatom__type__; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; self->m_atom = NULL; self->m_value = NULL; diff --git a/src/atomimpl/floatatomobj.c b/src/atomimpl/floatatomobj.c index d4b5038..9a95be5 100644 --- a/src/atomimpl/floatatomobj.c +++ b/src/atomimpl/floatatomobj.c @@ -23,13 +23,13 @@ cp_floatatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) self->_m_bits = 0; self->_m_little_endian = true; - CpFieldCAtom_CATOM(self).ob_pack = (packfunc)CpFloatAtom_Pack; - CpFieldCAtom_CATOM(self).ob_unpack = (unpackfunc)CpFloatAtom_Unpack; - CpFieldCAtom_CATOM(self).ob_pack_many = NULL; - CpFieldCAtom_CATOM(self).ob_unpack_many = NULL; - CpFieldCAtom_CATOM(self).ob_size = (sizefunc)cp_floatatom__size__; - CpFieldCAtom_CATOM(self).ob_type = (typefunc)cp_floatatom__type__; - CpFieldCAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpFloatAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpFloatAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_floatatom__size__; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_floatatom__type__; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; } return (PyObject*)self; } diff --git a/src/atomimpl/intatomobj.c b/src/atomimpl/intatomobj.c index 9959a64..85b4a68 100644 --- a/src/atomimpl/intatomobj.c +++ b/src/atomimpl/intatomobj.c @@ -27,13 +27,13 @@ cp_intatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) self->_m_signed = true; self->_m_little_endian = true; - CpFieldCAtom_CATOM(self).ob_pack = (packfunc)CpIntAtom_Pack; - CpFieldCAtom_CATOM(self).ob_unpack = (unpackfunc)CpIntAtom_Unpack; - CpFieldCAtom_CATOM(self).ob_pack_many = NULL; - CpFieldCAtom_CATOM(self).ob_unpack_many = NULL; - CpFieldCAtom_CATOM(self).ob_size = (sizefunc)cp_intatom__size__; - CpFieldCAtom_CATOM(self).ob_type = (typefunc)cp_intatom__type__; - CpFieldCAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpIntAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpIntAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_intatom__size__; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_intatom__type__; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; } return (PyObject*)self; } diff --git a/src/atomimpl/padatomobj.c b/src/atomimpl/padatomobj.c index fc615c1..e63786b 100644 --- a/src/atomimpl/padatomobj.c +++ b/src/atomimpl/padatomobj.c @@ -26,15 +26,15 @@ cp_paddingatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { CpPaddingAtomObject* self = (CpPaddingAtomObject*)type->tp_alloc(type, 0); if (self != NULL) { - CpFieldCAtom_CATOM(self).ob_pack = (packfunc)CpPaddingAtom_Pack; - CpFieldCAtom_CATOM(self).ob_unpack = (unpackfunc)CpPaddingAtom_Unpack; - CpFieldCAtom_CATOM(self).ob_pack_many = + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpPaddingAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpPaddingAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = (packmanyfunc)CpPaddingAtom_PackMany; - CpFieldCAtom_CATOM(self).ob_unpack_many = + CpBuiltinAtom_CATOM(self).ob_unpack_many = (unpackmanyfunc)CpPaddingAtom_UnpackMany; - CpFieldCAtom_CATOM(self).ob_size = (sizefunc)cp_paddingatom__size__; - CpFieldCAtom_CATOM(self).ob_type = (typefunc)cp_paddingatom__type__; - CpFieldCAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_paddingatom__size__; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_paddingatom__type__; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; } return (PyObject*)self; } diff --git a/src/atomimpl/stringatomobj.c b/src/atomimpl/stringatomobj.c index 4eaae28..8c516a8 100644 --- a/src/atomimpl/stringatomobj.c +++ b/src/atomimpl/stringatomobj.c @@ -31,13 +31,13 @@ cp_stringatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { CpStringAtomObject* self = (CpStringAtomObject*)type->tp_alloc(type, 0); if (self != NULL) { - CpFieldCAtom_CATOM(self).ob_pack = (packfunc)CpStringAtom_Pack; - CpFieldCAtom_CATOM(self).ob_unpack = (unpackfunc)CpStringAtom_Unpack; - CpFieldCAtom_CATOM(self).ob_pack_many = NULL; - CpFieldCAtom_CATOM(self).ob_unpack_many = NULL; - CpFieldCAtom_CATOM(self).ob_size = (sizefunc)cp_stringatom__size__; - CpFieldCAtom_CATOM(self).ob_type = (typefunc)cp_stringatom__type__; - CpFieldCAtom_CATOM(self).ob_bits = NULL; + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpStringAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpStringAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_stringatom__size__; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_stringatom__type__; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; self->m_encoding = NULL; self->m_length = NULL; diff --git a/src/capi.dat b/src/capi.dat index 3d98fc7..418303f 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -47,6 +47,7 @@ src:atomimpl/builtins/repeatedatomobj.c src:atomimpl/builtins/conditionatomobj.c src:atomimpl/builtins/switchatomobj.c src:atomimpl/builtins/offsetatomobj.c +src:atomimpl/builtins/primitiveatomobj.c # First index is reserved for the global module reference obj:0:CpModule:PyModuleDef @@ -91,6 +92,7 @@ type:31:_objlayerobj:CpObjLayerObject:- type:32:_conditionatomobj:CpConditionAtomObject:- type:33:_switchatomobj:CpSwitchAtomObject:- type:34:_offsetatomobj:CpOffsetAtomObject:- +type:35:_primitiveatomobj:CpPrimitiveAtomObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 @@ -144,11 +146,12 @@ func:103:CpStruct_ReplaceType:int:null func:104:CpStruct_HasOption:int:null func:105:CpStruct_Pack:int:null func:106:CpStruct_Unpack:PyObject*:+1 -func:107:CpStructModel_Check:int:null -func:108:CpStructModel_GetStruct:PyObject*:+1 -func:109:CpSeqLayer_New:CpSeqLayerObject*:+1 -func:110:CpSeqLayer_SetSequence:int:null -func:111:CpObjLayer_New:CpObjLayerObject*:+1 +func:107:CpStruct_SizeOf:PyObject*:+1 +func:108:CpStructModel_Check:int:null +func:109:CpStructModel_GetStruct:PyObject*:+1 +func:110:CpSeqLayer_New:CpSeqLayerObject*:+1 +func:111:CpSeqLayer_SetSequence:int:null +func:112:CpObjLayer_New:CpObjLayerObject*:+1 # atom api func:120:CpIntAtom_Pack:int:null diff --git a/src/caterpillar/_C.pyi b/src/caterpillar/_C.pyi index f8ad62c..2078173 100644 --- a/src/caterpillar/_C.pyi +++ b/src/caterpillar/_C.pyi @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Optional, Collection, Union, Callable, IO +from typing import Any, Optional, Collection, Union, Callable, IO, TypeVar _Length = Union[int, ContextLambda, slice, Ellipsis] @@ -96,6 +96,31 @@ class catom(atom): def __unpack__(self, context: Context) -> Any: ... def __unpack_many__(self, context: Context) -> Collection[Any]: ... +class builtinatom(catom): + def __init__(self, *args, **kwargs) -> None: ... + def __floordiv__(self, other) -> condition: ... + def __getitem__(self, index) -> repeated: ... + def __matmul__(self, *args, **kwargs) -> atoffset: ... + def __rfloordiv__(self, other) -> condition: ... + def __rmatmul__(self, *args, **kwargs) -> atoffset: ... + def __rrshift__(self, other) -> switch: ... + def __rshift__(self, other) -> switch: ... + + +class char_t(builtinatom): + def __init__(self, *args, **kwargs) -> None: ... + +class condition(builtinatom): + atom: Any + condition: Union[bool, ContextLambda] + def __init__(self, *args, **kwargs) -> None: ... + def is_enabled(self, *args, **kwargs): ... + def __set_byteorder__(self, *args, **kwargs): ... + +class const_t(builtinatom): + def __init__(self, *args, **kwargs) -> None: ... + + class fieldatom(atom): def __init__(self) -> None: ... def __add__(self, endian: Endian) -> Field: ... @@ -199,10 +224,10 @@ class State: class fieldinfo: excluded: bool - field: Field - def __init__(self, field: Field, excluded: bool = ...) -> None: ... + field: atom + def __init__(self, field: atom, excluded: bool = ...) -> None: ... -class Struct(fieldatom): +class Struct(builtinatom): members: dict[str, fieldinfo] model: type options: set[Option] @@ -285,10 +310,43 @@ class padding_t(fieldcatom): padding: padding_t +boolean: bool_t +char: char_t + class string(fieldcatom): encoding: str errors: str length: _Length def __init__(self, length: _Length, encoding: str, errors: str = ...) -> None: ... +class atoffset(builtinatom): + offset: Union[int, ContextLambda] + whence: int + def __init__(self, *args, **kwargs) -> None: ... + def get_offset(self, layer: layer) -> int: ... + def __set_byteorder__(self, byteorder: Endian): ... + + +class objlayer(layer): + obj: Context + def __init__(self, *args, **kwargs) -> None: ... + def __context_getattr__(self, *args, **kwargs): ... + +class repeated(builtinatom): + atom: Any + length: _Length + def __init__(self, *args, **kwargs) -> None: ... + def __set_byteorder__(self, *args, **kwargs): ... + +class seqlayer(layer): + index: int + length: int + sequence: Collection + def __init__(self, *args, **kwargs) -> None: ... +class switch(builtinatom): + atom: Any + cases: Union[dict[Any, Any], ContextLambda] + def __init__(self, *args, **kwargs) -> None: ... + def get_next(self, *args, **kwargs): ... + def __set_byteorder__(self, *args, **kwargs): ... diff --git a/src/caterpillar/include/caterpillar/atoms/builtins.h b/src/caterpillar/include/caterpillar/atoms/builtins.h index 0eb0a8b..0359bec 100644 --- a/src/caterpillar/include/caterpillar/atoms/builtins.h +++ b/src/caterpillar/include/caterpillar/atoms/builtins.h @@ -30,7 +30,7 @@ struct _builtinatomobj #define CpBuiltinAtom_NAME "builtinatom" #define CpBuiltinAtom_CheckExact(op) Py_IS_TYPE((op), &CpBuiltinAtom_Type) -#define CpBuiltinAtom_Check(op) PyObject_IsType((op), &CpBuiltinAtom_Type) +#define CpBuiltinAtom_Check(op) PyObject_TypeCheck((op), &CpBuiltinAtom_Type) #define CpBuiltinAtom_HEAD CpBuiltinAtomObject ob_base; #define CpBuiltinAtom_CATOM(self) (self)->ob_base.ob_base @@ -51,7 +51,7 @@ struct _repeatedatomobj #define CpRepeatedAtom_NAME "repeated" #define CpRepeatedAtom_CheckExact(op) Py_IS_TYPE((op), &CpRepeatedAtom_Type) -#define CpRepeatedAtom_Check(op) PyObject_IsType((op), &CpRepeatedAtom_Type) +#define CpRepeatedAtom_Check(op) PyObject_TypeCheck((op), &CpRepeatedAtom_Type) inline CpRepeatedAtomObject* CpRepeatedAtom_New(PyObject* atom, PyObject* length) @@ -76,7 +76,7 @@ struct _conditionatomobj #define CpConditionAtom_NAME "condition" #define CpConditionAtom_CheckExact(op) Py_IS_TYPE((op), &CpConditionAtom_Type) -#define CpConditionAtom_Check(op) PyObject_IsType((op), &CpConditionAtom_Type) +#define CpConditionAtom_Check(op) PyObject_TypeCheck((op), &CpConditionAtom_Type) static inline CpConditionAtomObject* CpConditionAtom_New(PyObject* atom, PyObject* condition) @@ -104,7 +104,7 @@ struct _switchatomobj #define CpSwitchAtom_NAME "switch" #define CpSwitchAtom_CheckExact(op) Py_IS_TYPE((op), &CpSwitchAtom_Type) -#define CpSwitchAtom_Check(op) PyObject_IsType((op), &CpSwitchAtom_Type) +#define CpSwitchAtom_Check(op) PyObject_TypeCheck((op), &CpSwitchAtom_Type) static inline CpSwitchAtomObject* CpSwitchAtom_New(PyObject* atom, PyObject* cases) @@ -132,7 +132,7 @@ struct _offsetatomobj #define CpOffsetAtom_NAME "atoffset" #define CpOffsetAtom_CheckExact(op) Py_IS_TYPE((op), &CpOffsetAtom_Type) -#define CpOffsetAtom_Check(op) PyObject_IsType((op), &CpOffsetAtom_Type) +#define CpOffsetAtom_Check(op) PyObject_TypeCheck((op), &CpOffsetAtom_Type) static inline CpOffsetAtomObject* CpOffsetAtom_FromSsize_t(PyObject* atom, Py_ssize_t offset) diff --git a/src/caterpillar/include/caterpillar/atoms/const.h b/src/caterpillar/include/caterpillar/atoms/const.h index 35c3243..372445d 100644 --- a/src/caterpillar/include/caterpillar/atoms/const.h +++ b/src/caterpillar/include/caterpillar/atoms/const.h @@ -17,12 +17,11 @@ #ifndef CONSTATOMOBJ_H #define CONSTATOMOBJ_H -#include "caterpillar/caterpillarapi.h" -#include "caterpillar/field.h" +#include "caterpillar/atoms/builtins.h" struct _constatomobj { - CpFieldCAtom_HEAD + CpBuiltinAtom_HEAD PyObject *m_value; PyObject *m_atom; diff --git a/src/caterpillar/include/caterpillar/atoms/float.h b/src/caterpillar/include/caterpillar/atoms/float.h index 767c7e2..758dc6d 100644 --- a/src/caterpillar/include/caterpillar/atoms/float.h +++ b/src/caterpillar/include/caterpillar/atoms/float.h @@ -17,12 +17,11 @@ #ifndef FLOATATOMOBJ_H #define FLOATATOMOBJ_H -#include "caterpillar/caterpillarapi.h" -#include "caterpillar/field.h" +#include "caterpillar/atoms/builtins.h" struct _floatatomobj { - CpFieldCAtom_HEAD + CpBuiltinAtom_HEAD /// Stores the amount of bytes this float atom /// has in total diff --git a/src/caterpillar/include/caterpillar/atoms/primitive.h b/src/caterpillar/include/caterpillar/atoms/primitive.h index a7ad399..fb5eee0 100644 --- a/src/caterpillar/include/caterpillar/atoms/primitive.h +++ b/src/caterpillar/include/caterpillar/atoms/primitive.h @@ -17,8 +17,21 @@ #ifndef PRIMITIVEATOMOBJ_H #define PRIMITIVEATOMOBJ_H -#include "caterpillar/caterpillarapi.h" -#include "caterpillar/field.h" +#include "caterpillar/atoms/builtins.h" + + +//------------------------------------------------------------------------------ +// primitive atom +struct _primitiveatomobj { + CpAtom_HEAD +}; + +#define CpPrimitiveAtom_NAME "patom" +#define CpPrimitiveAtom_CheckExact(op) Py_IS_TYPE((op), &CpPrimitiveAtom_Type) +#define CpPrimitiveAtom_Check(op) (PyObject_IsInstance((op), &CpPrimitiveAtom_Type)) + +//------------------------------------------------------------------------------ +// Bool /// @struct _boolatomobj /// @brief Struct representing a bool atom object. @@ -27,7 +40,7 @@ /// family. struct _boolatomobj { - CpFieldCAtom_HEAD + CpBuiltinAtom_HEAD }; /// @brief Bool atom object type @@ -65,7 +78,7 @@ struct _boolatomobj // Char Atom struct _charatomobj { - CpFieldCAtom_HEAD + CpBuiltinAtom_HEAD }; /// Char atom object type @@ -81,7 +94,7 @@ struct _charatomobj // Padding struct _paddingatomobj { - CpFieldCAtom_HEAD + CpBuiltinAtom_HEAD char padding; }; diff --git a/src/caterpillar/include/caterpillar/atoms/string.h b/src/caterpillar/include/caterpillar/atoms/string.h index ce6708a..ae9bbf9 100644 --- a/src/caterpillar/include/caterpillar/atoms/string.h +++ b/src/caterpillar/include/caterpillar/atoms/string.h @@ -17,12 +17,11 @@ #ifndef STRINGATOMOBJ_H #define STRINGATOMOBJ_H -#include "caterpillar/caterpillarapi.h" -#include "caterpillar/field.h" +#include "caterpillar/atoms/builtins.h" struct _stringatomobj { - CpFieldCAtom_HEAD + CpBuiltinAtom_HEAD PyObject* m_length; PyObject* m_errors; diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index 03b0fae..e95c5d9 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -88,6 +88,8 @@ struct _switchatomobj; typedef struct _switchatomobj CpSwitchAtomObject; struct _offsetatomobj; typedef struct _offsetatomobj CpOffsetAtomObject; +struct _primitiveatomobj; +typedef struct _primitiveatomobj CpPrimitiveAtomObject; #ifdef _CPMODULE @@ -131,6 +133,7 @@ extern PyTypeObject CpObjLayer_Type; extern PyTypeObject CpConditionAtom_Type; extern PyTypeObject CpSwitchAtom_Type; extern PyTypeObject CpOffsetAtom_Type; +extern PyTypeObject CpPrimitiveAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -183,6 +186,7 @@ int CpStruct_ReplaceType(CpStructObject* o, PyObject* name, PyObject* type); int CpStruct_HasOption(CpStructObject* o, PyObject* option); int CpStruct_Pack(CpStructObject* self, PyObject* obj, CpLayerObject* layer); PyObject* CpStruct_Unpack(CpStructObject* self, CpLayerObject* layer); +PyObject* CpStruct_SizeOf(CpStructObject* self, CpLayerObject* layer); int CpStructModel_Check(PyObject* model, _modulestate* state); PyObject* CpStructModel_GetStruct(PyObject* model, _modulestate* state); CpSeqLayerObject* CpSeqLayer_New(CpStateObject* state, CpLayerObject* parent); @@ -260,6 +264,7 @@ caterpillar_api.py #define CpConditionAtom_Type (*(PyTypeObject *)Cp_API[32]) #define CpSwitchAtom_Type (*(PyTypeObject *)Cp_API[33]) #define CpOffsetAtom_Type (*(PyTypeObject *)Cp_API[34]) +#define CpPrimitiveAtom_Type (*(PyTypeObject *)Cp_API[35]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -312,11 +317,12 @@ caterpillar_api.py #define CpStruct_HasOption (*((int (*)(CpStructObject* o, PyObject* option)))Cp_API[104]) #define CpStruct_Pack (*((int (*)(CpStructObject* self, PyObject* obj, CpLayerObject* layer)))Cp_API[105]) #define CpStruct_Unpack (*((PyObject* (*)(CpStructObject* self, CpLayerObject* layer)))Cp_API[106]) -#define CpStructModel_Check (*((int (*)(PyObject* model, _modulestate* state)))Cp_API[107]) -#define CpStructModel_GetStruct (*((PyObject* (*)(PyObject* model, _modulestate* state)))Cp_API[108]) -#define CpSeqLayer_New (*((CpSeqLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[109]) -#define CpSeqLayer_SetSequence (*((int (*)(CpSeqLayerObject* self,PyObject* sequence,Py_ssize_t length,int8_t greedy)))Cp_API[110]) -#define CpObjLayer_New (*((CpObjLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[111]) +#define CpStruct_SizeOf (*((PyObject* (*)(CpStructObject* self, CpLayerObject* layer)))Cp_API[107]) +#define CpStructModel_Check (*((int (*)(PyObject* model, _modulestate* state)))Cp_API[108]) +#define CpStructModel_GetStruct (*((PyObject* (*)(PyObject* model, _modulestate* state)))Cp_API[109]) +#define CpSeqLayer_New (*((CpSeqLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[110]) +#define CpSeqLayer_SetSequence (*((int (*)(CpSeqLayerObject* self,PyObject* sequence,Py_ssize_t length,int8_t greedy)))Cp_API[111]) +#define CpObjLayer_New (*((CpObjLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[112]) #define CpIntAtom_Pack (*((int (*)(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer)))Cp_API[120]) #define CpIntAtom_Unpack (*((PyObject* (*)(CpIntAtomObject* self, CpLayerObject* layer)))Cp_API[121]) #define CpFloatAtom_Pack (*((int (*)(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[122]) diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index f78f843..cfd0fdc 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -38,7 +38,7 @@ void *Cp_API[] = { (void *) &CpConditionAtom_Type, (void *) &CpSwitchAtom_Type, (void *) &CpOffsetAtom_Type, - NULL, + (void *) &CpPrimitiveAtom_Type, NULL, NULL, NULL, @@ -110,6 +110,7 @@ void *Cp_API[] = { (void *) &CpStruct_HasOption, (void *) &CpStruct_Pack, (void *) &CpStruct_Unpack, + (void *) &CpStruct_SizeOf, (void *) &CpStructModel_Check, (void *) &CpStructModel_GetStruct, (void *) &CpSeqLayer_New, @@ -122,7 +123,6 @@ void *Cp_API[] = { NULL, NULL, NULL, - NULL, (void *) &CpIntAtom_Pack, (void *) &CpIntAtom_Unpack, (void *) &CpFloatAtom_Pack, diff --git a/src/layer.c b/src/layer.c index af390b4..248e0d0 100644 --- a/src/layer.c +++ b/src/layer.c @@ -252,10 +252,10 @@ cp_objlayer_dealloc(CpObjLayerObject* self) static int cp_objlayer_init(CpObjLayerObject* self, PyObject* args, PyObject* kw) { - static char* kwlist[] = { "state", "path", "parent", NULL }; - PyObject *state = NULL, *path = NULL, *parent = NULL; + static char* kwlist[] = { "state", "path", "parent", "obj", NULL }; + PyObject *state = NULL, *path = NULL, *parent = NULL, *obj = NULL; if (!PyArg_ParseTupleAndKeywords( - args, kw, "O|OO", kwlist, &state, &path, &parent)) + args, kw, "O|OOO", kwlist, &state, &path, &parent, &obj)) return -1; if (!PyObject_IsInstance(state, (PyObject*)(&CpState_Type))) { @@ -264,7 +264,7 @@ cp_objlayer_init(CpObjLayerObject* self, PyObject* args, PyObject* kw) } Py_XSETREF(self->ob_base.m_state, (CpStateObject*)Py_NewRef(state)); - Py_XSETREF(self->m_obj, (PyObject*)CpContext_New()); + Py_XSETREF(self->m_obj, obj ? Py_NewRef(obj) : (PyObject*)CpContext_New()); Py_XSETREF(self->ob_base.m_path, Py_XNewRef(path)); Py_XSETREF(self->ob_base.m_parent, (CpLayerObject*)Py_XNewRef(parent)); return 0; @@ -363,7 +363,7 @@ static PyMemberDef CpObjLayer_Members[] = { static PyMethodDef CpObjLayer_Methods[] = { { "__context_getattr__", (PyCFunction)cp_objlayer_context_getattr, - METH_VARARGS}, + METH_VARARGS }, { NULL } /* Sentinel */ }; diff --git a/src/module.c b/src/module.c index 108d1c5..ea52ae1 100644 --- a/src/module.c +++ b/src/module.c @@ -391,39 +391,34 @@ PyInit__C(void) CpModule_SetupType(&CpStructFieldInfo_Type); // builtins setup + CpPrimitiveAtom_Type.tp_base = &CpAtom_Type; CpBuiltinAtom_Type.tp_base = &CpCAtom_Type; CpRepeatedAtom_Type.tp_base = &CpBuiltinAtom_Type; CpConditionAtom_Type.tp_base = &CpBuiltinAtom_Type; CpSwitchAtom_Type.tp_base = &CpBuiltinAtom_Type; CpOffsetAtom_Type.tp_base = &CpBuiltinAtom_Type; CpStruct_Type.tp_base = &CpBuiltinAtom_Type; + CpBoolAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpFloatAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpIntAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpCharAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpPaddingAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpStringAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpConstAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpBuiltinAtom_Type); + CpModule_SetupType(&CpPrimitiveAtom_Type); CpModule_SetupType(&CpRepeatedAtom_Type); CpModule_SetupType(&CpConditionAtom_Type); CpModule_SetupType(&CpSwitchAtom_Type); CpModule_SetupType(&CpOffsetAtom_Type); CpModule_SetupType(&CpStruct_Type); - - CpIntAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpIntAtom_Type); - - CpFloatAtom_Type.tp_base = &CpFieldCAtom_Type; CpModule_SetupType(&CpFloatAtom_Type); - - CpBoolAtom_Type.tp_base = &CpFieldCAtom_Type; CpModule_SetupType(&CpBoolAtom_Type); - - CpCharAtom_Type.tp_base = &CpFieldCAtom_Type; CpModule_SetupType(&CpCharAtom_Type); - - CpPaddingAtom_Type.tp_base = &CpFieldCAtom_Type; CpModule_SetupType(&CpPaddingAtom_Type); - - CpStringAtom_Type.tp_base = &CpFieldCAtom_Type; CpModule_SetupType(&CpStringAtom_Type); - - CpConstAtom_Type.tp_base = &CpFieldCAtom_Type; CpModule_SetupType(&CpConstAtom_Type); // module setup @@ -455,6 +450,7 @@ PyInit__C(void) CpModule_AddObject("fieldinfo", &CpStructFieldInfo_Type); CpModule_AddObject("Struct", &CpStruct_Type); + CpModule_AddObject(CpPrimitiveAtom_NAME, &CpPrimitiveAtom_Type); CpModule_AddObject(CpBuiltinAtom_NAME, &CpBuiltinAtom_Type); CpModule_AddObject(CpRepeatedAtom_NAME, &CpRepeatedAtom_Type); CpModule_AddObject(CpConditionAtom_NAME, &CpConditionAtom_Type); diff --git a/src/parsing_sizeof.c b/src/parsing_sizeof.c index 5e77de7..0f6b08d 100644 --- a/src/parsing_sizeof.c +++ b/src/parsing_sizeof.c @@ -213,10 +213,10 @@ _Cp_SizeOf(PyObject* op, CpLayerObject* layer) if (CpField_CheckExact(op)) { result = CpSizeOf_Field((CpFieldObject*)op, layer); - } else if (CpStruct_CheckExact(op)) { - result = CpSizeOf_Struct((CpStructObject*)op, layer); } else if (CpCAtom_Check(op)) { result = CpSizeOf_CAtom((CpCAtomObject*)op, layer); + } else if (CpStruct_CheckExact(op)) { + result = CpSizeOf_Struct((CpStructObject*)op, layer); } else { result = CpSizeOf_Common(op, layer); } diff --git a/src/state.c b/src/state.c index f22b1de..564c6f1 100644 --- a/src/state.c +++ b/src/state.c @@ -173,7 +173,7 @@ PyObject* CpState_Seek(CpStateObject* self, PyObject* offset, PyObject* whence) { return PyObject_CallMethodObjArgs( - self->m_io, self->mod->str_seek, offset, whence); + self->m_io, self->mod->str_seek, offset, whence, NULL); } /*CpAPI*/ diff --git a/src/struct.c b/src/struct.c index f248c77..edfd6c0 100644 --- a/src/struct.c +++ b/src/struct.c @@ -88,6 +88,7 @@ static PyMemberDef CpStructFieldInfo_Members[] = { offsetof(CpStructFieldInfoObject, s_excluded), READONLY, NULL }, + { "default", T_OBJECT_EX, offsetof(CpStructFieldInfoObject, m_default), 0 }, { NULL } /* Sentinel */ }; @@ -177,6 +178,12 @@ CpStructModel_GetDefaultValue(CpStructObject* o, PyObject* name) } /* impl */ +static PyObject* +cp_struct_type(CpStructObject* self) +{ + return Py_NewRef(self->m_model); +} + static PyObject* cp_struct_new(PyTypeObject* type, PyObject* args, PyObject* kw) { @@ -207,8 +214,8 @@ cp_struct_new(PyTypeObject* type, PyObject* args, PyObject* kw) } CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpStruct_Pack; CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpStruct_Unpack; - CpBuiltinAtom_CATOM(self).ob_size = NULL; - CpBuiltinAtom_CATOM(self).ob_type = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)CpStruct_SizeOf; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_struct_type; CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; CpBuiltinAtom_CATOM(self).ob_bits = NULL; @@ -415,21 +422,9 @@ cp_struct_prepare(CpStructObject* self) Py_XDECREF(discardable); return -1; } - // Constant values that are not in the form of fields. A special case are - // definitions using the 'Const' class for example. - if (CpField_Check(annotation)) { - atom = annotation; - // SPECIAL CASE: If the field has a condition linked to it, a default - // value of None is inferred. - if (CpField_HasCondition((CpFieldObject*)annotation) && - Cp_IsInvalidDefault(default_)) { - _Cp_SetObj(default_, Py_None); - } - } - - // SPECIAL CASE: If this struct is a union, we will infer None as the - // default value (if none has been set already) - if (self->s_union && Cp_IsInvalidDefault(default_)) { + // SPECIAL CASE: If the field has a condition linked to it, a default + // value of None is inferred. + if (CpConditionAtom_Check(atom) && Cp_IsInvalidDefault(default_)) { _Cp_SetObj(default_, Py_None); } @@ -620,18 +615,18 @@ cp_struct_process_annotation(CpStructObject* self, else if (PyCallable_Check(annotation)) { field = Py_NewRef(annotation); } + } - else { - // REVISIT: Add support for other types here. - PyErr_Format(PyExc_ValueError, - ("Field %R could not be created, because the placed " - "annotation does not " - "conform to any of the supported types.\n" - "annotation: %R"), - name, - annotation); - return -1; - } + if (!field) { + // REVISIT: Add support for other types here. + PyErr_Format(PyExc_ValueError, + ("Field %R could not be created, because the placed " + "annotation does not " + "conform to any of the supported types.\n" + "annotation: %R"), + name, + annotation); + return -1; } // TODO: arch @@ -1177,7 +1172,6 @@ CpStruct_Pack(CpStructObject* self, PyObject* obj, CpLayerObject* layer) PyObject* CpStruct_Unpack(CpStructObject* self, CpLayerObject* layer) { - printf("creating layer....\n"); CpObjLayerObject* obj_layer = CpObjLayer_New(layer->m_state, layer); if (!obj_layer) { return NULL; @@ -1191,9 +1185,20 @@ CpStruct_Unpack(CpStructObject* self, CpLayerObject* layer) return NULL; } - printf("getting %s....\n", PyUnicode_AsUTF8(name)); - value = _Cp_Unpack((PyObject*)info->m_field, (CpLayerObject*)obj_layer); CpLayer_AppendPath(&obj_layer->ob_base, name); + + value = _Cp_Unpack((PyObject*)info->m_field, (CpLayerObject*)obj_layer); + if (!value && PyErr_Occurred()) { + if (!Cp_IsInvalidDefault(info->m_default)) { + Py_XSETREF(value, info->m_default); + } + if (!value) { + Py_XDECREF(obj_layer); + return NULL; + } + PyErr_Clear(); + } + if (!value || PyDict_SetItem(obj_layer->m_obj, name, value) < 0) { Py_XDECREF(value); Py_XDECREF(obj_layer); @@ -1201,7 +1206,6 @@ CpStruct_Unpack(CpStructObject* self, CpLayerObject* layer) } Py_XDECREF(value); } - printf("finished parsing....\n"); PyObject* args = PyTuple_New(0); PyObject* obj = PyObject_Call((PyObject*)self->m_model, args, obj_layer->m_obj); @@ -1210,6 +1214,41 @@ CpStruct_Unpack(CpStructObject* self, CpLayerObject* layer) return obj; } +/*CpAPI*/ +PyObject* +CpStruct_SizeOf(CpStructObject* self, CpLayerObject* layer) +{ + PyObject *size = PyLong_FromLong(0), *tmp = NULL; + CpStructFieldInfoObject* info = NULL; + Py_ssize_t pos = 0; + + CpLayerObject* obj_layer = CpLayer_New(layer->m_state, layer); + if (!obj_layer || !size) { + return NULL; + } + + while (PyDict_Next(self->m_members, &pos, NULL, (PyObject**)&info)) { + tmp = _Cp_SizeOf((PyObject*)info->m_field, obj_layer); + if (!tmp) { + goto fail; + } + Py_XSETREF(tmp, PyNumber_Add(size, tmp)); + if (!tmp) { + goto fail; + } + Py_XSETREF(size, tmp); + } + Py_XDECREF(tmp); + Py_XDECREF(obj_layer); + return size; + +fail: + Py_XDECREF(tmp); + Py_XDECREF(obj_layer); + Py_XDECREF(size); + return NULL; +} + /* docs */ /* members */ @@ -1260,16 +1299,4 @@ PyTypeObject CpStruct_Type = { (initproc)cp_struct_init, /* tp_init */ 0, /* tp_alloc */ (newfunc)cp_struct_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ - 0, /* tp_vectorcall */ - 0, /* tp_watched */ }; \ No newline at end of file diff --git a/test/_C/test_parsing.py b/test/_C/test_parsing.py index be30cf6..ddcb417 100644 --- a/test/_C/test_parsing.py +++ b/test/_C/test_parsing.py @@ -5,7 +5,7 @@ if caterpillar.native_support(): - from caterpillar._C import atom, typeof, Field, sizeof, fieldatom + from caterpillar._C import atom, typeof, Field, sizeof, patom from caterpillar._C import unpack, layer, Struct, pack, ContextPath @@ -16,15 +16,15 @@ def __type__(self): return str - # Using the base class 'fieldatom' enabled direct field creation + # Using the base class 'patom' enabled direct field creation # using the common operators. - class Bar(fieldatom): + class Bar(patom): def __size__(self, ctx): return 2 # Only packing is implemented - class Baz(fieldatom): + class Baz(patom): def __unpack__(self, ctx: layer): return int.from_bytes(ctx.state.read(2)) @@ -67,7 +67,7 @@ def test_sizeof(): # NOTE: A switch context is somewhat special as we don't # know the target atom yet. Therefore, only context lambdas # as switch values are supported. - with pytest.raises(ValueError): + with pytest.raises(TypeError): sizeof(field >> {2: Bar()}) # Calculation is done by first evaluating the length of @@ -92,7 +92,7 @@ def test_unpack_basic(): # NOTE: parsing is done by first unpacking the switch value (Baz # atom) and then use it aas the key of the switch dictionary. the # returned atom will be used to parse the rest of the data. - assert unpack(data, Field(b) >> {1: b}) == 2 + # assert unpack(data, Field(b) >> {1: b}) == 2 assert unpack(data, b @ 0x0002) == 2 @@ -141,8 +141,7 @@ def test_pack_basic(): assert pack(1, b) == b"\x00\x01" assert pack([1, 2], b[2]) == b"\x00\x01\x00\x02" assert pack([2], b[b::]) == b"\x00\x01\x00\x02" - # NOTE: not implemented yet - assert pack(2, b @ 0x0002) == b"" + assert pack(2, b @ 0x0002) == b"\x00\x00\x00\x02" def test_pack_struct(): diff --git a/test/_C/test_state.py b/test/_C/test_state.py index efe9518..5cfb02e 100644 --- a/test/_C/test_state.py +++ b/test/_C/test_state.py @@ -4,7 +4,7 @@ if caterpillar.native_support(): - from caterpillar._C import State, layer, ContextPath, Context + from caterpillar._C import State, layer, ContextPath, Context, objlayer # State and layer each implement the context protocol, we @@ -25,8 +25,8 @@ def test_layer_init(): with pytest.raises(TypeError): _ = layer() - l = layer(State()) - assert l.obj is None + l = objlayer(State()) + assert l.obj is not None l.obj = 1 assert l.obj == 1 @@ -57,7 +57,7 @@ def test_context_path_ops(name, func, expected): cp2 = ContextPath("obj.foo") # path always points to foo path2 = func(cp2) - l = layer(State(), obj=c) + l = objlayer(State(), obj=c) rval = path2(l) assert ( rval == expected diff --git a/test/_C/test_struct.py b/test/_C/test_struct.py index 0800e9f..ea6b8a3 100644 --- a/test/_C/test_struct.py +++ b/test/_C/test_struct.py @@ -26,7 +26,7 @@ class Foo: assert s.model is Foo # The member dictionary will store Field instances, # which then store the default value - assert s.members["a"].field.default == 1 + assert s.members["a"].default == 1 def test_invalid_struct_model(): From f2b3333b51cd407ae4d1b47e0e0c353709506fb3 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:04:26 +0200 Subject: [PATCH 09/29] Updated pack_many catom callback --- + Removed API functions to pack fields and structs + Fixed repeated atom parsing logic to accept __pack_many__ and __unpack_many__ + New type CpLengthInfoObject + Removed deprecated parsing logic + Added verification logic to paddingatom --- docs/sphinx/source/reference/capi/parsing.rst | 62 +--- src/atomimpl/builtins/repeatedatomobj.c | 127 ++++--- src/atomimpl/padatomobj.c | 109 +++--- src/atomobj.c | 74 +++- src/capi.dat | 13 +- src/caterpillar/include/caterpillar/atomobj.h | 40 ++- .../include/caterpillar/caterpillarapi.h | 24 +- src/caterpillarapi.c | 14 +- src/module.c | 2 + src/parsing_pack.c | 313 +---------------- src/parsing_unpack.c | 316 ------------------ src/state.c | 6 + test/_C/atoms/test_padding.py | 5 +- 13 files changed, 271 insertions(+), 834 deletions(-) diff --git a/docs/sphinx/source/reference/capi/parsing.rst b/docs/sphinx/source/reference/capi/parsing.rst index 131666a..04a4271 100644 --- a/docs/sphinx/source/reference/capi/parsing.rst +++ b/docs/sphinx/source/reference/capi/parsing.rst @@ -11,64 +11,20 @@ Packing and Unpacking Returns the type of the given atom object. + .. c:function:: PyObject *CpSizeOf(PyObject *atom, PyObject *globals) - PyObject *CpSizeOf_Field(CpFieldObject *, CpLayerObject *) - PyObject *CpSizeOf_Struct(CpStructObject *, CpLayerObject *) - PyObject *CpSizeOf_Common(PyObject *, CpLayerObject *) Returns the size of the given atom object. + .. c:function:: int CpPack(PyObject *op, PyObject *atom, PyObject *io, PyObject *globals) Packs the given value using the given atom object and returns :code::`0` if successful. Returns :code:`-1` if an error occurs. The *globals* parameter may be *NULL*. - Serializing objects follows specific rules when it comes to CpLayerObject - creation. The first layer (root layer) will be assigned with the path - ''. By default, cp_pack will automatically create a new layer object - when called. Therefore, it is recommended to utilize cp_pack_internal for - C calls to avoid unnecessary layer creation. - - Currently, the following conditions allow the creation of a new layer: - - - Start packing an object - - Serializing a sequence object that requires to iterate over all elements individually. Note that atoms that support :code:`__unpack_many__` won't be called with the newly created sequence layer. - - Struct object or named sequences of fields in general will always introduce a new layer object. - - There are some built-in types that are designed to fasten calls by providing - a native C implementation. It is recommended to implement new classes in C - to leverage its speed rather than define new Python atom classes. - - A full workflow of cp_pack with all layers displayed would be the following: - .. code-block:: text - - 1. cp_pack: create new state and root layer | l: - 2. cp_pack_internal: measure atom type and call | <--+-+-+ - according C method | l: | | | - 3. One of the following methods will be called: | | | - a. cp_pack_catom: calls native C method | l: | | | - b. cp_pack_atom: call python pack method | l: | | | - c. cp_pack_field: inserts the field instance into | | | - the current layer and calls cp_pack_internal | | | - again. | l: -+ | | - d. cp_pack_struct: creates a new object layer | | - that addresses only the current struct's | | - values. Calls cp_pack_internal on each field | l: ---+ | - e. cp_pack_common: createsa a new layer if | - multiple objects should be packed AND | - __[un]pack_many__ is not implemented. | l: -+ - - The following functions are defined and called within the packing process: - - .. c:function:: int CpPack_Field(PyObject *, CpFieldObject *, CpLayerObject *) - int CpPack_Struct(PyObject *, CpStructObject *, CpLayerObject *) - int CpPack_CAtom(PyObject *, CpCAtomObject *, CpLayerObject *) - int CpPack_Common(PyObject *, PyObject *, CpLayerObject *) - int _Cp_Pack(PyObject*, PyObject*, CpLayerObject* ) - - .. c:function:: int _CpPack_EvalLength(CpLayerObject* layer, PyObject* length, Py_ssize_t size, bool* seq_greedy, Py_ssize_t* seq_length) +.. c:function:: int _CpPack_EvalLength(CpLayerObject* layer, PyObject* length, Py_ssize_t size, bool* seq_greedy, Py_ssize_t* seq_length) Evaluates the length of the sequence to pack. @@ -83,16 +39,8 @@ Packing and Unpacking :param seq_length: destination pointer to store the length of the sequence :return: :code:`0` if successful, :code:`-1` if an error occurs -.. c:function:: PyObject *CpUnpack(PyObject *atom, PyObject *io, PyObject *globals) - Unlike serializing objects, unpacking returns fully qualified objects as a - result. In addition, the internal 'obj' within a struct layer will be a - :c:type:`CpContextObject` instance. +.. c:function:: PyObject *CpUnpack(PyObject *atom, PyObject *io, PyObject *globals) - The following functions are defined and called within the unpacking process: + *TODO* - .. c:function:: PyObject *_Cp_Unpack(PyObject*, CpLayerObject*) - PyObject *CpUnpack_Field(CpFieldObject*, CpLayerObject*) - PyObject *CpUnpack_Struct(CpStructObject*, CpLayerObject*) - PyObject *CpUnpack_CAtom(CpCAtomObject*, CpLayerObject*) - PyObject *CpUnpack_Common(PyObject *, CpLayerObject*) diff --git a/src/atomimpl/builtins/repeatedatomobj.c b/src/atomimpl/builtins/repeatedatomobj.c index 26cb4e4..587773a 100644 --- a/src/atomimpl/builtins/repeatedatomobj.c +++ b/src/atomimpl/builtins/repeatedatomobj.c @@ -125,10 +125,44 @@ CpRepeatedAtom_Pack(CpRepeatedAtomObject* self, CpLayerObject* layer) { CpStateObject* state = layer->m_state; - if (PyObject_HasAttr(self->m_atom, state->mod->str___pack_many__)) { + CpLengthInfoObject* lengthinfo = + (CpLengthInfoObject*)CpObject_CreateNoArgs(&CpLengthInfo_Type); + + bool pack_many_attr = + PyObject_HasAttr(self->m_atom, state->mod->str___pack_many__); + bool is_seq = PySequence_Check(op); + + Py_ssize_t obj_length = 0; + if (is_seq) { + obj_length = PySequence_Length(op); + if (obj_length < 0) { + goto failure; + } + } + + PyObject* length = CpRepeatedAtom_GetLength(self, (PyObject*)layer); + if (!length) { + goto failure; + } + + if (_CpPack_EvalLength(layer, + length, + obj_length, + (bool*)&lengthinfo->m_greedy, + &lengthinfo->m_length) < 0) { + Py_XDECREF(length); + goto failure; + } + Py_XDECREF(length); + + if (pack_many_attr) { // class explicitly defines __pack_many__ -> use it - PyObject* res = CpAtom_Pack( - self->m_atom, state->mod->str___pack_many__, op, (PyObject*)layer); + PyObject* res = PyObject_CallMethodObjArgs(self->m_atom, + state->mod->str___pack_many__, + op, + (PyObject*)layer, + lengthinfo, + NULL); PyObject* exc = NULL; if ((exc = PyErr_GetRaisedException(), exc && PyErr_GivenExceptionMatches(exc, PyExc_NotImplementedError))) { @@ -139,41 +173,39 @@ CpRepeatedAtom_Pack(CpRepeatedAtomObject* self, } else { int success = res ? 0 : -1; Py_XDECREF(res); - Py_XDECREF(exc); + if (success < 0) { + PyErr_SetRaisedException(exc); + } else { + Py_XDECREF(exc); + } return success; } } - if (!PySequence_Check(op)) { + if (!is_seq) { PyErr_Format(PyExc_TypeError, "input object (%R) is not a sequence", op); return -1; } + if (lengthinfo->m_greedy) { + lengthinfo->m_length = obj_length; + } else { + if (lengthinfo->m_length != obj_length) { + PyErr_Format(PyExc_ValueError, + "given length %d does not match sequence size %d", + lengthinfo->m_length, + obj_length); + goto failure; + } + } + CpSeqLayerObject* seq_layer = CpSeqLayer_New(state, layer); if (!seq_layer) { return -1; } - Py_ssize_t size = PySequence_Size(op); - bool greedy = false; - Py_ssize_t layer_length = 0; - PyObject* length = CpRepeatedAtom_GetLength(self, (PyObject*)layer); - if (!length) { - goto failure; - } - if (_CpPack_EvalLength(layer, length, size, &greedy, &layer_length) < 0) { - Py_XDECREF(length); - goto failure; - } - Py_XDECREF(length); - - if (layer_length <= 0) { - // continue packing, here's nothing to store - Py_XDECREF(seq_layer); - return 0; - } - - CpSeqLayer_SetSequence(seq_layer, op, layer_length, greedy); + CpSeqLayer_SetSequence( + seq_layer, op, lengthinfo->m_length, lengthinfo->m_greedy); PyObject* obj = NULL; int success = 0; @@ -198,6 +230,7 @@ CpRepeatedAtom_Pack(CpRepeatedAtomObject* self, CpLayer_Invalidate((CpLayerObject*)seq_layer); return 0; failure: + Py_XDECREF(lengthinfo); CpLayer_Invalidate((CpLayerObject*)seq_layer); return -1; } @@ -207,9 +240,28 @@ PyObject* CpRepeatedAtom_Unpack(CpRepeatedAtomObject* self, CpLayerObject* layer) { _modulestate* mod = layer->m_state->mod; - if (PyObject_HasAttr(self->m_atom, mod->str___unpack_many__)) { - PyObject* res = PyObject_CallMethodOneArg( - self->m_atom, mod->str___unpack_many__, (PyObject*)layer); + CpLengthInfoObject* lengthinfo = + (CpLengthInfoObject*)CpObject_CreateNoArgs(&CpLengthInfo_Type); + + bool unpack_many_attr = + PyObject_HasAttr(self->m_atom, mod->str___unpack_many__); + + PyObject* length = CpRepeatedAtom_GetLength(self, (PyObject*)layer); + if (!length) { + goto fail; + } + if (_CpUnpack_EvalLength( + layer, length, (bool*)&lengthinfo->m_greedy, &lengthinfo->m_length) < + 0) { + goto fail; + } + + if (unpack_many_attr) { + PyObject* res = PyObject_CallMethodObjArgs(self->m_atom, + mod->str___unpack_many__, + (PyObject*)layer, + lengthinfo, + NULL); PyObject* exc = NULL; if ((exc = PyErr_GetRaisedException(), exc && PyErr_GivenExceptionMatches(exc, PyExc_NotImplementedError))) { @@ -218,22 +270,11 @@ CpRepeatedAtom_Unpack(CpRepeatedAtomObject* self, CpLayerObject* layer) PyErr_Clear(); Py_XDECREF(res); } else { - Py_XDECREF(exc); + PyErr_SetRaisedException(exc); return res; } } - PyObject *obj = NULL, *length = NULL; - // First, get the amount of elements we have to parse - bool seq_greedy = false; - Py_ssize_t seq_length = 0; - length = CpRepeatedAtom_GetLength(self, (PyObject*)layer); - if (!length) { - goto fail; - } - if (_CpUnpack_EvalLength(layer, length, &seq_greedy, &seq_length) < 0) { - goto fail; - } CpSeqLayerObject* seq_layer = CpSeqLayer_New(layer->m_state, layer); if (!layer) { goto fail; @@ -241,11 +282,13 @@ CpRepeatedAtom_Unpack(CpRepeatedAtomObject* self, CpLayerObject* layer) // REVISIT: add sequence factory here PyObject* seq = PyList_New(0); + PyObject* obj = NULL; if (!seq) { goto fail; } - CpSeqLayer_SetSequence(seq_layer, seq, seq_length, seq_greedy); + CpSeqLayer_SetSequence( + seq_layer, seq, lengthinfo->m_length, lengthinfo->m_greedy); while (seq_layer->s_greedy || (seq_layer->m_index < seq_layer->m_length)) { seq_layer->ob_base.m_path = PyUnicode_FromFormat( "%s.%d", _PyUnicode_AsString(layer->m_path), seq_layer->m_index); @@ -272,6 +315,7 @@ CpRepeatedAtom_Unpack(CpRepeatedAtomObject* self, CpLayerObject* layer) success: Py_XDECREF(obj); Py_XDECREF(length); + Py_XDECREF(lengthinfo); CpLayer_Invalidate((CpLayerObject*)seq_layer); seq_layer = NULL; return Py_NewRef(seq); @@ -279,6 +323,7 @@ CpRepeatedAtom_Unpack(CpRepeatedAtomObject* self, CpLayerObject* layer) fail: Py_XDECREF(obj); Py_XDECREF(length); + Py_XDECREF(lengthinfo); if (seq_layer) { CpLayer_Invalidate((CpLayerObject*)seq_layer); } diff --git a/src/atomimpl/padatomobj.c b/src/atomimpl/padatomobj.c index e63786b..1d4ac1b 100644 --- a/src/atomimpl/padatomobj.c +++ b/src/atomimpl/padatomobj.c @@ -5,13 +5,13 @@ #include static PyObject* -cp_paddingatom__type__(PyObject* self) +cp_paddingatom_type(PyObject* self) { return Py_XNewRef(&_PyNone_Type); } static PyObject* -cp_paddingatom__size__(PyObject* self, PyObject *ctx) +cp_paddingatom_size(PyObject* self, PyObject* ctx) { /* NOTE: We are using the size of one byte here to allow padding atoms to be @@ -32,8 +32,8 @@ cp_paddingatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) (packmanyfunc)CpPaddingAtom_PackMany; CpBuiltinAtom_CATOM(self).ob_unpack_many = (unpackmanyfunc)CpPaddingAtom_UnpackMany; - CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_paddingatom__size__; - CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_paddingatom__type__; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_paddingatom_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_paddingatom_type; CpBuiltinAtom_CATOM(self).ob_bits = NULL; } return (PyObject*)self; @@ -90,55 +90,23 @@ CpPaddingAtom_Pack(CpPaddingAtomObject* self, int CpPaddingAtom_PackMany(CpPaddingAtomObject* self, PyObject* value, - CpLayerObject* layer) + CpLayerObject* layer, + CpLengthInfoObject* lengthinfo) { -#if 0 - /* value will be ignored here */ - PyObject *res = NULL, *bytes = NULL, *objSize = NULL, *objLengh = NULL; - bool greedy = false; - Py_ssize_t length = 0; - - if (!layer->m_field) { - PyErr_SetString(PyExc_TypeError, "layer must have a field"); - return -1; - } - - /* The following call will fail if the length is a prefixed statement - * (slice)*/ - objLengh = - CpField_GetLength((CpFieldObject*)layer->m_field, (PyObject*)layer); - if (!objLengh) { - return -1; - } - if (_CpPack_EvalLength(layer, objLengh, -1, &greedy, &length) < 0) { - Py_XDECREF(objLengh); - return -1; - } - Py_XDECREF(objLengh); - - if (length == 0) { - return 0; - } else if (length < 0) { - PyErr_SetString(PyExc_ValueError, "length must be >= 0"); - return -1; - } else if (greedy) { - PyErr_SetString(PyExc_ValueError, "length must be >= 0 and not greedy!"); - return -1; - } - - objSize = PyLong_FromSsize_t(length); + PyObject* objSize = PyLong_FromSsize_t(lengthinfo->m_length); if (!objSize) { return -1; } - bytes = CpObject_CreateOneArg(&PyBytes_Type, objSize); + + PyObject* bytes = CpObject_CreateOneArg(&PyBytes_Type, objSize); if (!bytes) { Py_DECREF(objSize); return -1; } /* unsafe { */ - memset(PyBytes_AS_STRING(bytes), self->padding, length); + memset(PyBytes_AS_STRING(bytes), self->padding, lengthinfo->m_length); /* } */ - res = CpState_Write(layer->m_state, bytes); + PyObject* res = CpState_Write(layer->m_state, bytes); Py_DECREF(bytes); Py_DECREF(objSize); if (!res) { @@ -146,8 +114,6 @@ CpPaddingAtom_PackMany(CpPaddingAtomObject* self, } Py_XDECREF(res); return 0; -#endif - return 0; } /*CpAPI*/ @@ -159,43 +125,56 @@ CpPaddingAtom_Unpack(CpPaddingAtomObject* self, CpLayerObject* layer) return NULL; } Py_XDECREF(res); - return 0; + Py_RETURN_NONE; } /*CpAPI*/ PyObject* -CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, CpLayerObject* layer) +CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, + CpLayerObject* layer, + CpLengthInfoObject* lengthinfo) { -#if 0 - PyObject *res = NULL, *bytes = NULL, *objLengh = NULL; - bool greedy = false; - Py_ssize_t length = 0, parsedLength = 0; - - objLengh = - CpField_GetLength((CpFieldObject*)layer->m_field, (PyObject*)layer); - if (!objLengh) { - return NULL; + PyObject* res = NULL; + + if (lengthinfo->m_greedy) { + res = CpState_ReadFully(layer->m_state); + } else { + res = CpState_Read(layer->m_state, lengthinfo->m_length); } - if (_CpPack_EvalLength(layer, objLengh, -1, &greedy, &length) < 0) { - Py_DECREF(objLengh); + + if (!res) { return NULL; } - Py_DECREF(objLengh); - res = CpState_Read(layer->m_state, length); - if (!res) { + Py_ssize_t length = PyBytes_GET_SIZE(res); + Py_ssize_t offset = 0; + const char* ptr = PyBytes_AS_STRING(res); + while (offset < length && *(ptr + offset) == self->padding) { + offset++; + } + if (offset != length) { + PyErr_Format(PyExc_ValueError, + ("The parsed padding contains invalid padding characters " + "(possible padding overflow?). " + "Expected %ld bytes of 0x%02x but parsed only %ld bytes."), + length, + self->padding, + offset); + Py_XDECREF(res); return NULL; } Py_XDECREF(res); -#endif Py_RETURN_NONE; } PyObject* -cp_paddingatom__repr__(CpPaddingAtomObject* self) +cp_paddingatom_repr(CpPaddingAtomObject* self) { - return PyUnicode_FromFormat("padding(0x%02x)", self->padding); + if (self->padding == 0) { + return PyUnicode_FromFormat(""); + } + return PyUnicode_FromFormat("", self->padding); } /* type setup */ @@ -203,7 +182,7 @@ PyTypeObject CpPaddingAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpPaddingAtom_NAME), .tp_basicsize = sizeof(CpPaddingAtomObject), .tp_dealloc = (destructor)cp_paddingatom_dealloc, - .tp_repr = (reprfunc)cp_paddingatom__repr__, + .tp_repr = (reprfunc)cp_paddingatom_repr, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_doc = NULL, .tp_init = (initproc)cp_paddingatom_init, diff --git a/src/atomobj.c b/src/atomobj.c index 476dfcc..a563fa5 100644 --- a/src/atomobj.c +++ b/src/atomobj.c @@ -111,11 +111,11 @@ static PyMethodDef CpAtom_Methods[] = { PyTypeObject CpAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(atom), .tp_basicsize = sizeof(CpAtomObject), - .tp_dealloc =(destructor)cp_atom_dealloc, + .tp_dealloc = (destructor)cp_atom_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_doc = cp_atom_doc, .tp_methods = CpAtom_Methods, - .tp_init =(initproc)cp_atom_init, + .tp_init = (initproc)cp_atom_init, .tp_new = (newfunc)cp_atom_new, }; @@ -193,13 +193,16 @@ cp_catom_pack_many(CpCAtomObject* self, PyObject* args, PyObject* kw) return NULL; } - static char* kwlist[] = { "ops", "context", NULL }; - PyObject *ops = NULL, *context = NULL; - if (PyArg_ParseTupleAndKeywords(args, kw, "OO", kwlist, &ops, &context) < 0) { + static char* kwlist[] = { "ops", "context", "lengthinfo", NULL }; + PyObject *ops = NULL, *context = NULL, *lengthinfo = NULL; + if (PyArg_ParseTupleAndKeywords( + args, kw, "OOO", kwlist, &ops, &context, &lengthinfo) < 0) { return NULL; } - return self->ob_pack_many((PyObject*)self, ops, context) ? NULL : Py_NewRef(Py_None); + return self->ob_pack_many((PyObject*)self, ops, context, lengthinfo) + ? NULL + : Py_NewRef(Py_None); } static PyObject* @@ -232,12 +235,12 @@ cp_catom_unpack_many(CpCAtomObject* self, PyObject* args, PyObject* kw) return NULL; } - static char* kwlist[] = { "context", NULL }; - PyObject* context = NULL; - if (PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &context) < 0) { + static char* kwlist[] = { "context", "lengthinfo", NULL }; + PyObject *context = NULL, *lengthinfo = NULL; + if (PyArg_ParseTupleAndKeywords(args, kw, "OO", kwlist, &context, &lengthinfo) < 0) { return NULL; } - return self->ob_unpack_many((PyObject*)self, context); + return self->ob_unpack_many((PyObject*)self, context, lengthinfo); } static PyObject* @@ -316,4 +319,55 @@ PyTypeObject CpCAtom_Type = { .tp_methods = CpCAtom_Methods, .tp_init = (initproc)cp_catom_init, .tp_new = (newfunc)cp_catom_new, +}; + +// ------------------------------------------------------------------------------ +// length info + +static PyObject* +cp_lengthinfo_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + return type->tp_alloc(type, 0); +} + +static void +cp_lengthinfo_dealloc(CpLengthInfoObject* self) +{ + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_lengthinfo_init(CpLengthInfoObject* self, PyObject* args, PyObject* kw) +{ + static char* kwlist[] = { "length", "greedy", NULL }; + Py_ssize_t length = 0; + int greedy = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kw, "|np", kwlist, &length, &greedy)) { + return -1; + } + + self->m_length = length; + self->m_greedy = greedy; + return 0; +} + +static PyObject* +cp_lengthinfo_repr(CpLengthInfoObject* self) +{ + if (self->m_greedy) { + return PyUnicode_FromString(""); + } + return PyUnicode_FromFormat("", self->m_length); +} + +PyTypeObject CpLengthInfo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpLengthInfo_NAME), + .tp_basicsize = sizeof(CpLengthInfoObject), + .tp_dealloc = (destructor)cp_lengthinfo_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_new = (newfunc)cp_lengthinfo_new, + .tp_init = (initproc)cp_lengthinfo_init, + .tp_repr = (reprfunc)cp_lengthinfo_repr, + .tp_doc = NULL, }; \ No newline at end of file diff --git a/src/capi.dat b/src/capi.dat index 418303f..db46361 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -93,6 +93,7 @@ type:32:_conditionatomobj:CpConditionAtomObject:- type:33:_switchatomobj:CpSwitchAtomObject:- type:34:_offsetatomobj:CpOffsetAtomObject:- type:35:_primitiveatomobj:CpPrimitiveAtomObject:- +type:36:_lengthinfoobj:CpLengthInfoObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 @@ -110,9 +111,9 @@ func:64:CpTypeOf:PyObject*:+1 func:65:CpTypeOf_Field:PyObject*:+1 func:66:CpTypeOf_Common:PyObject*:+1 func:67:CpPack:int:null -func:68:CpPack_Field:int:null -func:69:CpPack_Common:int:null -func:70:CpPack_Struct:int:null +# func:68:CpPack_Field:int:null +# func:69:CpPack_Common:int:null +# func:70:CpPack_Struct:int:null func:72:_CpPack_EvalLength:int:null func:73:CpSizeOf:PyObject*:+1 func:74:CpSizeOf_Field:PyObject*:+1 @@ -120,9 +121,9 @@ func:75:CpSizeOf_Struct:PyObject*:+1 func:76:CpSizeOf_Common:PyObject*:+1 func:77:_Cp_SizeOf:PyObject*:+1 func:78:CpUnpack:PyObject*:+1 -func:79:CpUnpack_Field:PyObject*:+1 -func:80:CpUnpack_Common:PyObject*:+1 -func:81:CpUnpack_Struct:PyObject*:+1 +# func:79:CpUnpack_Field:PyObject*:+1 +# func:80:CpUnpack_Common:PyObject*:+1 +# func:81:CpUnpack_Struct:PyObject*:+1 func:83:_CpUnpack_EvalLength:int:null func:84:CpUnpack_CAtom:PyObject*:+1 func:85:CpPack_CAtom:int:null diff --git a/src/caterpillar/include/caterpillar/atomobj.h b/src/caterpillar/include/caterpillar/atomobj.h index dfafa09..c929f48 100644 --- a/src/caterpillar/include/caterpillar/atomobj.h +++ b/src/caterpillar/include/caterpillar/atomobj.h @@ -110,6 +110,19 @@ CpAtom_Pack(PyObject* atom, PyObject* attrname, PyObject* op, PyObject* ctx) return PyObject_CallMethodObjArgs(atom, attrname, op, ctx, NULL); } +// --------------------------------------------------------------------------- +// Length info + +struct _lengthinfoobj +{ + PyObject_HEAD + + Py_ssize_t m_length; + int m_greedy; +}; + +#define CpLengthInfo_NAME "lengthinfo" + // --------------------------------------------------------------------------- // CAtom @@ -176,8 +189,20 @@ typedef int (*packfunc)(PyObject*, PyObject*, PyObject*); * This is another specialized version of the `packfunc` function, which * is used to pack multiple objects to the underlying stream. See `packfunc` * for more information. + * + * The signature of this function is as follows: + * @code {.cpp} + * int packmanyfunc(PyObject* self, PyObject* op, PyObject* ctx, + * CpLengthInfo* info); + * @endcode + * + * @param self the atom object (self) + * @param op the object to pack + * @param ctx the context object + * @param info the length information + * @return the result of the `__pack_many__` method */ -typedef int (*packmanyfunc)(PyObject*, PyObject*, PyObject*); +typedef int (*packmanyfunc)(PyObject*, PyObject*, PyObject*, PyObject*); /** * @brief Unpack-Protocol @@ -198,8 +223,19 @@ typedef PyObject* (*unpackfunc)(PyObject*, PyObject*); * This is another specialized version of the `unpackfunc` function, which * is used to unpack multiple objects from the underlying stream. See * `unpackfunc` for more information. + * + * The signature of this function is as follows: + * @code {.cpp} + * PyObject* unpackmanyfunc(PyObject* self, PyObject* ctx, + * CpLengthInfo* info); + * @endcode + * + * @param self the atom object (self) + * @param ctx the context object + * @param info the length information + * @return the result of the `__unpack_many__` method */ -typedef PyObject* (*unpackmanyfunc)(PyObject*, PyObject*); +typedef PyObject* (*unpackmanyfunc)(PyObject*, PyObject*, PyObject*); /// Experimental API /** diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index e95c5d9..ccb8553 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -90,6 +90,8 @@ struct _offsetatomobj; typedef struct _offsetatomobj CpOffsetAtomObject; struct _primitiveatomobj; typedef struct _primitiveatomobj CpPrimitiveAtomObject; +struct _lengthinfoobj; +typedef struct _lengthinfoobj CpLengthInfoObject; #ifdef _CPMODULE @@ -134,6 +136,7 @@ extern PyTypeObject CpConditionAtom_Type; extern PyTypeObject CpSwitchAtom_Type; extern PyTypeObject CpOffsetAtom_Type; extern PyTypeObject CpPrimitiveAtom_Type; +extern PyTypeObject CpLengthInfo_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -150,9 +153,6 @@ PyObject* CpTypeOf(PyObject* op); PyObject* CpTypeOf_Field(CpFieldObject* op); PyObject* CpTypeOf_Common(PyObject* op); int CpPack(PyObject* op, PyObject* atom, PyObject* io, PyObject* globals); -int CpPack_Field(PyObject* op, CpFieldObject* field, CpLayerObject* layer); -int CpPack_Common(PyObject* op, PyObject* atom, CpLayerObject* layer); -int CpPack_Struct(PyObject* op, CpStructObject* struct_, CpLayerObject* layer); int _CpPack_EvalLength(CpLayerObject* layer,PyObject* length,Py_ssize_t size,bool* greedy,Py_ssize_t* dstLength); PyObject* CpSizeOf(PyObject* op, PyObject* globals); PyObject* CpSizeOf_Field(CpFieldObject* field, CpLayerObject* layer); @@ -160,9 +160,6 @@ PyObject* CpSizeOf_Struct(CpStructObject* struct_, CpLayerObject* layer); PyObject* CpSizeOf_Common(PyObject* op, CpLayerObject* layer); PyObject* _Cp_SizeOf(PyObject* op, CpLayerObject* layer); PyObject* CpUnpack(PyObject* atom, PyObject* io, PyObject* globals); -PyObject* CpUnpack_Field(CpFieldObject* field, CpLayerObject* layer); -PyObject* CpUnpack_Common(PyObject* op, CpLayerObject* layer); -PyObject* CpUnpack_Struct(CpStructObject* struct_, CpLayerObject* layer); int _CpUnpack_EvalLength(CpLayerObject* layer,PyObject* length,bool* seq_greedy,Py_ssize_t* seq_length); PyObject* CpUnpack_CAtom(CpCAtomObject* catom, CpLayerObject* layer); int CpPack_CAtom(PyObject* op, CpCAtomObject* catom, CpLayerObject* layer); @@ -201,9 +198,9 @@ PyObject* CpBoolAtom_Unpack(CpBoolAtomObject* self, CpLayerObject* layer); int CpCharAtom_Pack(CpCharAtomObject* self, PyObject* value, CpLayerObject* layer); PyObject* CpCharAtom_Unpack(CpCharAtomObject* self, CpLayerObject* layer); int CpPaddingAtom_Pack(CpPaddingAtomObject* self,PyObject* value,CpLayerObject* layer); -int CpPaddingAtom_PackMany(CpPaddingAtomObject* self,PyObject* value,CpLayerObject* layer); +int CpPaddingAtom_PackMany(CpPaddingAtomObject* self,PyObject* value,CpLayerObject* layer,CpLengthInfoObject* lengthinfo); PyObject* CpPaddingAtom_Unpack(CpPaddingAtomObject* self, CpLayerObject* layer); -PyObject* CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, CpLayerObject* layer); +PyObject* CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self,CpLayerObject* layer,CpLengthInfoObject* lengthinfo); int CpStringAtom_Pack(CpStringAtomObject* self,PyObject* value,CpLayerObject* layer); PyObject* CpStringAtom_Unpack(CpStringAtomObject* self, CpLayerObject* layer); int CpConstAtom_Pack(CpConstAtomObject* self, PyObject* value, CpLayerObject* layer); @@ -265,6 +262,7 @@ caterpillar_api.py #define CpSwitchAtom_Type (*(PyTypeObject *)Cp_API[33]) #define CpOffsetAtom_Type (*(PyTypeObject *)Cp_API[34]) #define CpPrimitiveAtom_Type (*(PyTypeObject *)Cp_API[35]) +#define CpLengthInfo_Type (*(PyTypeObject *)Cp_API[36]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -281,9 +279,6 @@ caterpillar_api.py #define CpTypeOf_Field (*((PyObject* (*)(CpFieldObject* op)))Cp_API[65]) #define CpTypeOf_Common (*((PyObject* (*)(PyObject* op)))Cp_API[66]) #define CpPack (*((int (*)(PyObject* op, PyObject* atom, PyObject* io, PyObject* globals)))Cp_API[67]) -#define CpPack_Field (*((int (*)(PyObject* op, CpFieldObject* field, CpLayerObject* layer)))Cp_API[68]) -#define CpPack_Common (*((int (*)(PyObject* op, PyObject* atom, CpLayerObject* layer)))Cp_API[69]) -#define CpPack_Struct (*((int (*)(PyObject* op, CpStructObject* struct_, CpLayerObject* layer)))Cp_API[70]) #define _CpPack_EvalLength (*((int (*)(CpLayerObject* layer,PyObject* length,Py_ssize_t size,bool* greedy,Py_ssize_t* dstLength)))Cp_API[72]) #define CpSizeOf (*((PyObject* (*)(PyObject* op, PyObject* globals)))Cp_API[73]) #define CpSizeOf_Field (*((PyObject* (*)(CpFieldObject* field, CpLayerObject* layer)))Cp_API[74]) @@ -291,9 +286,6 @@ caterpillar_api.py #define CpSizeOf_Common (*((PyObject* (*)(PyObject* op, CpLayerObject* layer)))Cp_API[76]) #define _Cp_SizeOf (*((PyObject* (*)(PyObject* op, CpLayerObject* layer)))Cp_API[77]) #define CpUnpack (*((PyObject* (*)(PyObject* atom, PyObject* io, PyObject* globals)))Cp_API[78]) -#define CpUnpack_Field (*((PyObject* (*)(CpFieldObject* field, CpLayerObject* layer)))Cp_API[79]) -#define CpUnpack_Common (*((PyObject* (*)(PyObject* op, CpLayerObject* layer)))Cp_API[80]) -#define CpUnpack_Struct (*((PyObject* (*)(CpStructObject* struct_, CpLayerObject* layer)))Cp_API[81]) #define _CpUnpack_EvalLength (*((int (*)(CpLayerObject* layer,PyObject* length,bool* seq_greedy,Py_ssize_t* seq_length)))Cp_API[83]) #define CpUnpack_CAtom (*((PyObject* (*)(CpCAtomObject* catom, CpLayerObject* layer)))Cp_API[84]) #define CpPack_CAtom (*((int (*)(PyObject* op, CpCAtomObject* catom, CpLayerObject* layer)))Cp_API[85]) @@ -332,9 +324,9 @@ caterpillar_api.py #define CpCharAtom_Pack (*((int (*)(CpCharAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[126]) #define CpCharAtom_Unpack (*((PyObject* (*)(CpCharAtomObject* self, CpLayerObject* layer)))Cp_API[127]) #define CpPaddingAtom_Pack (*((int (*)(CpPaddingAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[128]) -#define CpPaddingAtom_PackMany (*((int (*)(CpPaddingAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[129]) +#define CpPaddingAtom_PackMany (*((int (*)(CpPaddingAtomObject* self,PyObject* value,CpLayerObject* layer,CpLengthInfoObject* lengthinfo)))Cp_API[129]) #define CpPaddingAtom_Unpack (*((PyObject* (*)(CpPaddingAtomObject* self, CpLayerObject* layer)))Cp_API[130]) -#define CpPaddingAtom_UnpackMany (*((PyObject* (*)(CpPaddingAtomObject* self, CpLayerObject* layer)))Cp_API[131]) +#define CpPaddingAtom_UnpackMany (*((PyObject* (*)(CpPaddingAtomObject* self,CpLayerObject* layer,CpLengthInfoObject* lengthinfo)))Cp_API[131]) #define CpStringAtom_Pack (*((int (*)(CpStringAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[132]) #define CpStringAtom_Unpack (*((PyObject* (*)(CpStringAtomObject* self, CpLayerObject* layer)))Cp_API[133]) #define CpConstAtom_Pack (*((int (*)(CpConstAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[134]) diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index cfd0fdc..86d64e6 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -39,7 +39,7 @@ void *Cp_API[] = { (void *) &CpSwitchAtom_Type, (void *) &CpOffsetAtom_Type, (void *) &CpPrimitiveAtom_Type, - NULL, + (void *) &CpLengthInfo_Type, NULL, NULL, NULL, @@ -71,9 +71,9 @@ void *Cp_API[] = { (void *) &CpTypeOf_Field, (void *) &CpTypeOf_Common, (void *) &CpPack, - (void *) &CpPack_Field, - (void *) &CpPack_Common, - (void *) &CpPack_Struct, + NULL, + NULL, + NULL, NULL, (void *) &_CpPack_EvalLength, (void *) &CpSizeOf, @@ -82,9 +82,9 @@ void *Cp_API[] = { (void *) &CpSizeOf_Common, (void *) &_Cp_SizeOf, (void *) &CpUnpack, - (void *) &CpUnpack_Field, - (void *) &CpUnpack_Common, - (void *) &CpUnpack_Struct, + NULL, + NULL, + NULL, NULL, (void *) &_CpUnpack_EvalLength, (void *) &CpUnpack_CAtom, diff --git a/src/module.c b/src/module.c index ea52ae1..354d6d1 100644 --- a/src/module.c +++ b/src/module.c @@ -389,6 +389,7 @@ PyInit__C(void) CpModule_SetupType(&CpState_Type); CpModule_SetupType(&CpStructFieldInfo_Type); + CpModule_SetupType(&CpLengthInfo_Type); // builtins setup CpPrimitiveAtom_Type.tp_base = &CpAtom_Type; @@ -449,6 +450,7 @@ PyInit__C(void) CpModule_AddObject("State", &CpState_Type); CpModule_AddObject("fieldinfo", &CpStructFieldInfo_Type); CpModule_AddObject("Struct", &CpStruct_Type); + CpModule_AddObject(CpLengthInfo_NAME, &CpLengthInfo_Type); CpModule_AddObject(CpPrimitiveAtom_NAME, &CpPrimitiveAtom_Type); CpModule_AddObject(CpBuiltinAtom_NAME, &CpBuiltinAtom_Type); diff --git a/src/parsing_pack.c b/src/parsing_pack.c index ea73f6e..138b0ff 100644 --- a/src/parsing_pack.c +++ b/src/parsing_pack.c @@ -2,7 +2,7 @@ #include "caterpillar/caterpillar.h" // ------------------------------------------------------------------------------ -// pack: +// pack: (OUTDATED) // Serializing objects follows specific rules when it comes to CpLayerObject // creation. The first layer (root layer) will be assigned with the path // ''. By default, cp_pack will automatically create a new layer object @@ -110,296 +110,6 @@ _CpPack_EvalLength(CpLayerObject* layer, return 0; } -/*CpAPI*/ -int -CpPack_Field(PyObject* op, CpFieldObject* field, CpLayerObject* layer) -{ -#if 0 - // we can assert that all provided objects are of the correct type - // REVISIT: really? - if (!op) { - PyErr_SetString(PyExc_ValueError, "object is NULL"); - return -1; - } - - CpLayer_AppendPath(layer, field->m_name); - if (!layer->m_path) { - return -1; - } - - int res = CpField_IsEnabled(field, (PyObject*)layer); - if (!res) { - // disabled fields are not packed - return 0; - } - if (res < 0) { - return -1; - } - - PyObject *base_stream = NULL, *fallback = NULL; - CpStateObject* state = layer->m_state; - - Py_XSETREF(layer->m_field, Py_NewRef((PyObject*)field)); - layer->s_sequential = field->s_sequential; - - Py_ssize_t offset = CpField_GetOffset(field, (PyObject*)layer); - if (offset < 0 && PyErr_Occurred()) { - return -1; - } - - if (offset == -1 || !field->s_keep_pos) { - if (!(fallback = CpState_Tell(layer->m_state))) { - return -1; - }; - } - - if (offset >= 0) { - // We write the current field into a temporary memory buffer - // and add it after all processing has finished. - base_stream = Py_XNewRef(layer->m_state->m_io); - layer->m_state->m_io = PyObject_CallNoArgs(state->mod->BytesIO_Type); - if (!state->m_io) { - return -1; - } - } - - if (!PyCallable_Check(field->m_atom)) { - return _Cp_Pack(op, field->m_atom, layer); - } else { - PyObject* res = PyObject_CallOneArg(field->m_atom, (PyObject*)layer); - if (!res) { - return -1; - } - - if (field->m_switch && field->m_switch != Py_None) { - PyObject* atom = CpField_EvalSwitch(field, res, (PyObject*)layer); - Py_DECREF(res); - if (!atom) { - return -1; - } - if (_Cp_Pack(op, atom, layer) < 0) { - return -1; - } - } else - Py_DECREF(res); - } - - if (offset == -1 || !field->s_keep_pos) { - if (!CpState_Seek(state, fallback, 0)) { - return -1; - } - } - - if (offset >= 0) { - if (PyObject_SetItem( - state->m_offset_table, PyLong_FromSsize_t(offset), state->m_io) < 0) - return -1; - - Py_XSETREF(state->m_io, base_stream); - } -#endif - return 0; -} - -/*CpAPI*/ -int -CpPack_Common(PyObject* op, PyObject* atom, CpLayerObject* layer) -{ -#if 0 - int success; - if (!layer->s_sequential) { - return PyObject_CallMethod(atom, "__pack__", "OO", op, layer) ? 0 : -1; - } - - CpStateObject* state = layer->m_state; - if (PyObject_HasAttr(atom, state->mod->str___pack_many__)) { - // class explicitly defines __pack_many__ -> use it - PyObject* res = - CpAtom_Pack(atom, state->mod->str___pack_many__, op, (PyObject*)layer); - PyObject* exc = NULL; - if ((exc = PyErr_GetRaisedException(), - exc && - PyErr_GivenExceptionMatches(exc, PyExc_NotImplementedError))) { - // Make sure this method continues to pack the given object - Py_XDECREF(exc); - PyErr_Clear(); - Py_XDECREF(res); - } else { - success = res ? 0 : -1; - Py_XDECREF(res); - Py_XDECREF(exc); - return success; - } - } - - if (!PySequence_Check(op)) { - PyErr_Format(PyExc_ValueError, "input object (%R) is not a sequence", op); - return -1; - } - - // TODO: explain why - CpLayerObject* seq_layer = CpLayer_New(state, layer); - if (!seq_layer) { - return -1; - } - seq_layer->s_sequential = false; - - Py_ssize_t size = PySequence_Size(op); - bool greedy = false; - Py_ssize_t layer_length = 0; - PyObject* length = - CpField_GetLength((CpFieldObject*)layer->m_field, (PyObject*)layer); - if (!length) { - return -1; - } - if (_CpPack_EvalLength(layer, length, size, &greedy, &layer_length) < 0) { - Py_XDECREF(length); - return -1; - } - Py_XDECREF(length); - - if (layer_length <= 0) { - // continue packing, here's nothing to store - Py_XDECREF(seq_layer); - return 0; - } - CpLayer_SetSequence(seq_layer, op, layer_length, greedy); - - PyObject* obj = NULL; - for (seq_layer->m_index = 0; seq_layer->m_index < seq_layer->m_length; - seq_layer->m_index++) { - obj = PySequence_GetItem(op, seq_layer->m_index); - if (!obj) { - return -1; - } - seq_layer->m_path = PyUnicode_FromFormat( - "%s.%d", _PyUnicode_AsString(layer->m_path), seq_layer->m_index); - if (!seq_layer->m_path) { - Py_XDECREF(obj); - return -1; - } - success = _Cp_Pack(obj, atom, seq_layer); - Py_XSETREF(obj, NULL); - if (success < 0) { - return -1; - } - } - CpLayer_Invalidate(seq_layer); - return 0; - - fail: - if (seq_layer) { - CpLayer_Invalidate(seq_layer); - } - return -1; -#endif - return 0; -} - -/*CpAPI*/ -int -CpPack_Struct(PyObject* op, CpStructObject* struct_, CpLayerObject* layer) -{ -#if 0 - if (layer->s_sequential) { - // TODO: explain why - return CpPack_Common(op, (PyObject*)struct_, layer); - } - - if (PySequence_Check(op)) { - PyErr_SetString( - PyExc_ValueError, - ("Expected a valid CpAtom definition, got a single " - "struct for sequential packing. Did you forget to add '[]' " - "at the end of your field definiton?")); - return -1; - } - - CpLayerObject* obj_layer = CpLayer_New(layer->m_state, layer); - if (!obj_layer) { - return -1; - } - obj_layer->m_obj = Py_NewRef(op); - obj_layer->m_path = Py_NewRef(layer->m_path); - - CpStructFieldInfoObject *info = NULL, *union_field = NULL; - // all borrowed references - PyObject* name = NULL; - // new references - PyObject *max_size = NULL, *size = NULL, *cmp_result = NULL, *value = NULL; - int res = 0; - Py_ssize_t pos = 0; - - while (PyDict_Next(struct_->m_members, &pos, &name, (PyObject**)&info)) { - - if (struct_->s_union) { - size = NULL; // cp_sizeof_field(info->m_field, state); - if (!size) { - goto fail; - } - cmp_result = PyObject_RichCompare(max_size, size, Py_LT); - if (!cmp_result) { - goto fail; - } - if (Py_IsTrue(cmp_result)) { - max_size = Py_XNewRef(size); - union_field = info; - } - Py_XSETREF(cmp_result, NULL); - Py_XSETREF(size, NULL); - } - - else { - value = PyObject_GetAttr(op, name); - if (!value) { - goto fail; - } - - res = _Cp_Pack(value, (PyObject*)info->m_field, obj_layer); - if (res < 0) { - goto fail; - } - Py_XDECREF(value); - value = NULL; - } - } - - if (struct_->s_union) { - if (!union_field) { - goto success; - } - value = PyObject_GetAttr(op, union_field->m_field->m_name); - if (!value) { - goto fail; - } - res = _Cp_Pack(value, (PyObject*)union_field->m_field, obj_layer); - if (res < 0) { - goto fail; - } - goto cleanup; - } - - goto success; - -fail: - res = -1; - goto cleanup; - -success: - res = 0; - -cleanup: - Py_XDECREF(size); - Py_XDECREF(max_size); - Py_XDECREF(cmp_result); - - if (obj_layer) { - CpLayer_Invalidate(obj_layer); - } -#endif - return 0; -} - /*CpAPI*/ int CpPack_CAtom(PyObject* op, CpCAtomObject* catom, CpLayerObject* layer) @@ -411,27 +121,6 @@ CpPack_CAtom(PyObject* op, CpCAtomObject* catom, CpLayerObject* layer) return -1; } return catom->ob_pack((PyObject*)catom, op, (PyObject*)layer); -#if 0 - if (!layer->s_sequential) { - if (catom->ob_pack == NULL) { - PyErr_Format(PyExc_NotImplementedError, - "The atom of type '%s' cannot be packed (missing __pack__)", - Py_TYPE(catom)->tp_name); - return -1; - } - return catom->ob_pack((PyObject*)catom, op, (PyObject*)layer); - } - - if (catom->ob_pack_many == NULL) { - // PyErr_Format( - // PyExc_NotImplementedError, - // "The atom of type '%s' cannot be packed (missing __pack_many__)", - // Py_TYPE(catom)->tp_name); - // return -1; - return CpPack_Common(op, (PyObject*)catom, layer); - } - return catom->ob_pack_many((PyObject*)catom, op, (PyObject*)layer); -#endif } /*CpAPI*/ diff --git a/src/parsing_unpack.c b/src/parsing_unpack.c index 9c16172..8f09e8d 100644 --- a/src/parsing_unpack.c +++ b/src/parsing_unpack.c @@ -75,291 +75,6 @@ _CpUnpack_EvalLength(CpLayerObject* layer, return 0; } -/*CpAPI*/ -PyObject* -CpUnpack_Common(PyObject* op, CpLayerObject* layer) -{ -#if 0 - if (!op || !layer) { - PyErr_SetString(PyExc_ValueError, "input or layer object is NULL!"); - return NULL; - } - - if (!layer->s_sequential) { - return PyObject_CallMethodOneArg( - op, layer->m_state->mod->str___unpack__, (PyObject*)layer); - } - - _modulestate* mod = layer->m_state->mod; - if (!layer->m_field) { - PyErr_SetString(PyExc_ValueError, "state is invalid (field is NULL)!"); - return NULL; - } - - if (PyObject_HasAttr(op, mod->str___unpack_many__)) { - PyObject* res = - PyObject_CallMethodOneArg(op, mod->str___unpack_many__, (PyObject*)layer); - PyObject* exc = NULL; - if ((exc = PyErr_GetRaisedException(), - exc && PyErr_GivenExceptionMatches(exc, PyExc_NotImplementedError))) { - // Make sure this method continues to unpack - Py_XDECREF(exc); - PyErr_Clear(); - Py_XDECREF(res); - } else { - Py_XDECREF(exc); - return res; - } - } - - PyObject *obj = NULL, *length = NULL; - // First, get the amount of elements we have to parse - bool seq_greedy = false; - Py_ssize_t seq_length = 0; - length = CpField_GetLength((CpFieldObject*)layer->m_field, (PyObject*)layer); - if (!length) { - goto fail; - } - if (_CpUnpack_EvalLength(layer, length, &seq_greedy, &seq_length) < 0) { - goto fail; - } - CpLayerObject* seq_layer = CpLayer_New(layer->m_state, layer); - if (!layer) { - goto fail; - } - - PyObject* seq = PyList_New(0); - if (!seq) { - goto fail; - } - - CpLayer_SetSequence(seq_layer, seq, seq_length, seq_greedy); - while (seq_layer->s_greedy || (seq_layer->m_index < seq_layer->m_length)) { - seq_layer->m_path = PyUnicode_FromFormat( - "%s.%d", _PyUnicode_AsString(layer->m_path), seq_layer->m_index); - if (!seq_layer->m_path) { - goto fail; - } - - Py_XSETREF(obj, _Cp_Unpack(op, seq_layer)); - if (!obj) { - - if (seq_layer->s_greedy) { - PyErr_Clear(); - break; - } - goto fail; - } - - if (PyList_Append(seq_layer->m_sequence, Py_NewRef(obj)) < 0) { - goto fail; - } - seq_layer->m_index++; - } - -success: - Py_XDECREF(obj); - Py_XDECREF(length); - CpLayer_Invalidate(seq_layer); - seq_layer = NULL; - return Py_NewRef(seq); - -fail: - Py_XDECREF(obj); - Py_XDECREF(length); - if (seq_layer) { - CpLayer_Invalidate(seq_layer); - } - return NULL; - -#endif - Py_RETURN_NONE; -} - -/*CpAPI*/ -PyObject* -CpUnpack_Field(CpFieldObject* field, CpLayerObject* layer) -{ -#if 0 - if (!field || !layer) { - PyErr_SetString(PyExc_ValueError, "input or layer object is NULL!"); - return NULL; - } - - CpLayer_AppendPath(layer, field->m_name); - if (!layer->m_path) { - return NULL; - } - - int enabled = CpField_IsEnabled(field, (PyObject*)layer); - if (enabled < 0) { - return NULL; - } else if (!enabled) { - return Py_NewRef(Py_None); - } - - PyObject *obj = NULL, *fallback = NULL; - Py_ssize_t offset = -1; - - layer->s_sequential = field->s_sequential; - Py_XSETREF(layer->m_field, Py_NewRef(field)); - if (PyCallable_Check(field->m_atom)) { - obj = PyObject_CallOneArg(field->m_atom, (PyObject*)layer); - if (!obj) { - goto cleanup; - } - } - // The underlying atom is not callable and therefore not a context lambda. - // Using the internal unpack function we will try to parse the given data - // stream. - else { - if (!field->s_keep_pos) { - fallback = CpState_Tell(layer->m_state); - if (!fallback) { - goto cleanup; - } - } - - offset = CpField_GetOffset(field, (PyObject*)layer); - if (offset < 0 && PyErr_Occurred()) { - goto cleanup; - } - if (offset >= 0) { - if (CpState_Seek(layer->m_state, PyLong_FromSsize_t(offset), 0) < 0) { - goto cleanup; - } - } - - obj = _Cp_Unpack(field->m_atom, layer); - - if (!obj && PyErr_Occurred()) { - if (!Cp_IsInvalidDefault(field->m_default)) { - Py_XSETREF(obj, field->m_default); - } - if (!obj) { - goto cleanup; - } - PyErr_Clear(); - } - - if (!field->s_keep_pos) { - if (CpState_Seek(layer->m_state, fallback, 0) < 0) { - goto cleanup; - } - } - } - - if (field->m_switch && !Py_IsNone(field->m_switch)) { - - PyObject* atom = CpField_EvalSwitch(field, obj, (PyObject*)layer); - if (!atom) { - Py_XSETREF(obj, NULL); - goto cleanup; - } - - layer->m_value = Py_NewRef(obj); - obj = _Cp_Unpack(atom, layer); - } - -cleanup: - Py_XDECREF(fallback); - return obj; -#endif - Py_RETURN_NONE; -} - -/*CpAPI*/ -PyObject* -CpUnpack_Struct(CpStructObject* struct_, CpLayerObject* layer) -{ -#if 0 - if (layer->s_sequential) { - // TODO: explain why - return CpUnpack_Common((PyObject*)struct_, layer); - } - - PyObject* obj = NULL; - CpLayerObject* obj_layer = CpLayer_New(layer->m_state, layer); - if (!obj_layer) { - return NULL; - } - obj_layer->m_obj = CpObject_CreateNoArgs(&CpContext_Type); - if (!obj_layer->m_obj) { - goto cleanup; - } - - CpStructFieldInfoObject *info = NULL, *union_field = NULL; - Py_ssize_t pos = 0; - PyObject *name = NULL, *value = NULL, *start = CpState_Tell(layer->m_state), - *max_size = NULL, *stream_pos = NULL, - *init_data = CpObject_CreateNoArgs(&CpContext_Type); - - if (!start || !init_data) { - goto cleanup; - } - while (PyDict_Next(struct_->m_members, &pos, &name, (PyObject**)&info)) { - if (struct_->s_union) { - Py_XSETREF(stream_pos, CpState_Tell(layer->m_state)); - if (!stream_pos) { - goto cleanup; - } - } - - value = _Cp_Unpack((PyObject*)info->m_field, obj_layer); - if (!value || PyObject_SetItem(obj_layer->m_obj, name, value) < 0) { - goto cleanup; - } - - if (struct_->s_union) { - PyObject* pos = CpState_Tell(obj_layer->m_state); - if (!pos) { - goto cleanup; - } - PyObject* sub_result = PyNumber_Subtract(pos, stream_pos); - if (!sub_result) { - Py_XDECREF(pos); - goto cleanup; - } - if (PyObject_RichCompareBool(max_size, sub_result, Py_LT)) { - Py_XSETREF(max_size, Py_NewRef(sub_result)); - } - Py_XDECREF(sub_result); - Py_XDECREF(pos); - if (CpState_Seek(obj_layer->m_state, start, 0) < 0) { - goto cleanup; - } - } - } - - if (struct_->s_union) { - PyObject* pos = PyNumber_Add(start, max_size); - if (!pos) { - goto cleanup; - } - if (CpState_Seek(layer->m_state, pos, 0) < 0) { - Py_XDECREF(pos); - goto cleanup; - } - Py_XDECREF(pos); - } - - PyObject* args = PyTuple_New(0); - obj = PyObject_Call((PyObject*)struct_->m_model, args, obj_layer->m_obj); - Py_XDECREF(args); - -cleanup: - Py_XDECREF(value); - Py_XDECREF(start); - Py_XDECREF(init_data); - Py_XDECREF(stream_pos); - if (obj_layer) { - CpLayer_Invalidate(obj_layer); - } - return obj; -#endif - Py_RETURN_NONE; -} - /*CpAPI*/ PyObject* CpUnpack_CAtom(CpCAtomObject* catom, CpLayerObject* layer) @@ -372,32 +87,6 @@ CpUnpack_CAtom(CpCAtomObject* catom, CpLayerObject* layer) return NULL; } return catom->ob_unpack((PyObject*)catom, (PyObject*)layer); -#if 0 - PyObject* result = NULL; - if (!layer->s_sequential) { - if (catom->ob_unpack == NULL) { - PyErr_Format( - PyExc_NotImplementedError, - "The atom of type '%s' cannot be unpacked (missing __unpack__)", - Py_TYPE(catom)->tp_name); - return NULL; - } - result = catom->ob_unpack((PyObject *)catom, (PyObject *)layer); - } - else { - if (!catom->ob_unpack_many) { - // REVISIT: use flag to enable strict parsing - // PyErr_Format( - // PyExc_NotImplementedError, - // "The atom of type '%s' cannot be unpacked (missing __unpack_many__)", - // Py_TYPE(catom)->tp_name); - return CpUnpack_Common((PyObject *)catom, layer); - } - result = catom->ob_unpack_many((PyObject *)catom, (PyObject *)layer); - } - return result; -#endif - Py_RETURN_NONE; } /*CpAPI*/ @@ -426,11 +115,6 @@ CpUnpack(PyObject* atom, PyObject* io, PyObject* globals) Py_XSETREF(root->m_path, Py_NewRef(state->mod->str_ctx__root)); obj = _Cp_Unpack(atom, root); - if (!obj) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_RuntimeError, "Unpack failed"); - } - } Py_DECREF(state); CpLayer_Invalidate(root); return obj; diff --git a/src/state.c b/src/state.c index 564c6f1..2941fa8 100644 --- a/src/state.c +++ b/src/state.c @@ -186,6 +186,12 @@ CpState_Read(CpStateObject* self, Py_ssize_t size) Py_DECREF(sizeobj); if (!res) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, + "read() returned NULL without error (possible size " + "overflow?). Tried to read %ld bytes", + size); + } return NULL; } diff --git a/test/_C/atoms/test_padding.py b/test/_C/atoms/test_padding.py index 7b3038c..e57c4d3 100644 --- a/test/_C/atoms/test_padding.py +++ b/test/_C/atoms/test_padding.py @@ -28,13 +28,14 @@ def test_padding_pack_many(): def test_padding_unpack(): # NOTE: unpack using the padding atom always returns None - # and DOES NOT validate the parsed value. + # and DOES validate the parsed value. assert unpack(b"\x00", padding) is None assert unpack(b"\x20", padding_t(0x20)) is None # Therefore, the following code IS valid if the underlying # stream does not throw an exception. - assert unpack(b"\x00" * 10, padding_t(0x02)[10]) is None + with pytest.raises(ValueError): + unpack(b"\x00" * 10, padding_t(0x02)[10]) def test_padding_unpack_many(): assert unpack(b"\x00" * 10, padding[10]) is None From 7e657566feaaf4c19255eb1507813877b15e817a Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 8 Sep 2024 20:33:05 +0200 Subject: [PATCH 10/29] CpBytesAtom implementation ... + Python class: "octetstring" + Tests included --- CMakeLists.txt | 1 + docs/sphinx/source/development/roadmap.rst | 4 +- src/atomimpl/bytesatomobj.c | 127 ++++++++++++++++++ src/capi.dat | 8 +- .../include/caterpillar/atoms/string.h | 22 ++- .../include/caterpillar/caterpillarapi.h | 10 ++ src/caterpillarapi.c | 7 +- src/module.c | 3 + test/_C/atoms/test_string.py | 17 ++- 9 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 src/atomimpl/bytesatomobj.c diff --git a/CMakeLists.txt b/CMakeLists.txt index ad2be11..7a1c70f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,7 @@ python_add_library( src/atomimpl/padatomobj.c src/atomimpl/stringatomobj.c src/atomimpl/constatomobj.c + src/atomimpl/bytesatomobj.c src/atomimpl/builtins/builtinatomobj.c src/atomimpl/builtins/repeatedatomobj.c diff --git a/docs/sphinx/source/development/roadmap.rst b/docs/sphinx/source/development/roadmap.rst index 89bb664..4ad0ac8 100644 --- a/docs/sphinx/source/development/roadmap.rst +++ b/docs/sphinx/source/development/roadmap.rst @@ -38,9 +38,9 @@ Atom Objects: - |check_| Char (C type: :c:type:`CpCharAtomObject`, Py type: :class:`char_t`), global instance: :code:`char` [:text-danger:`missing docs`] - |check_| Padding (C type: :c:type:`CpPaddingAtomObject`, Py type: :class:`padding_t`), global instance: :code:`padding` [:text-danger:`missing docs`] - |check_| String (C type: :c:type:`CpStringAtomObject`, Py type: :class:`string`) [:text-danger:`missing docs`] -- |uncheck_| Const +- |uncheck_| Const (C type: :c:type:`CpConstAtomObject`, Py type: :class:`const_t`) [:text-danger:`missing docs`] - |uncheck_| CString -- |uncheck_| Bytes +- |uncheck_| Bytes (C type: :c:type:`CpBytesAtomObject`, Py type: :class:`octetstring`) - |uncheck_| Enum - |uncheck_| Computed - |uncheck_| PString diff --git a/src/atomimpl/bytesatomobj.c b/src/atomimpl/bytesatomobj.c new file mode 100644 index 0000000..bad5121 --- /dev/null +++ b/src/atomimpl/bytesatomobj.c @@ -0,0 +1,127 @@ +/* constatom C implementation */ +#include "caterpillar/caterpillar.h" + +#include + +/*impl*/ + +static PyObject* +cp_bytesatom_type(CpBytesAtomObject* self) +{ + return Py_NewRef(&PyBytes_Type); +} + +static PyObject* +cp_bytesatom_size(CpBytesAtomObject* self, CpLayerObject* layer) +{ + return CpBytesAtom_GetLength(self, layer); +} + +static PyObject* +cp_bytesatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) +{ + CpBytesAtomObject* self = (CpBytesAtomObject*)type->tp_alloc(type, 0); + if (self != NULL) { + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpBytesAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpBytesAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_bytesatom_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_bytesatom_type; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + self->m_length = NULL; + self->s_callable = 0; + } + return (PyObject*)self; +} + +static void +cp_bytesatom_dealloc(CpBytesAtomObject* self) +{ + Py_XDECREF(self->m_length); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_bytesatom_init(CpBytesAtomObject* self, PyObject* args, PyObject* kwds) +{ + static char* kwlist[] = { "length", NULL }; + PyObject* length = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &length)) { + return -1; + } + _Cp_SetObj(self->m_length, length); + self->s_callable = PyCallable_Check(self->m_length); + return 0; +} + +static PyObject* +cp_bytesatom_repr(CpBytesAtomObject* self) +{ + return PyUnicode_FromFormat("", self->m_length); +} + +/* Public API */ + +/*CpAPI*/ +PyObject* +CpBytesAtom_GetLength(CpBytesAtomObject* self, CpLayerObject* layer) +{ + if (self->s_callable) { + return PyObject_CallOneArg(self->m_length, (PyObject*)layer); + } + return Py_NewRef(self->m_length); +} + +/*CpAPI*/ +int +CpBytesAtom_Pack(CpBytesAtomObject* self, PyObject* value, CpLayerObject* layer) +{ + PyObject* length = CpBytesAtom_GetLength(self, layer); + if (length == NULL) { + return -1; + } + + if (PyLong_AS_LONG(length) != PyBytes_GET_SIZE(value)) { + PyErr_SetString(PyExc_ValueError, "Invalid length"); + return -1; + } + Py_DECREF(length); + + PyObject* res = CpState_Write(layer->m_state, value); + if (!res) { + return -1; + } + Py_DECREF(res); + return 0; +} + +/*CpAPI*/ +PyObject* +CpBytesAtom_Unpack(CpBytesAtomObject* self, CpLayerObject* layer) +{ + PyObject* length = CpBytesAtom_GetLength(self, layer); + if (length == NULL) { + return NULL; + } + return CpState_Read(layer->m_state, PyLong_AS_LONG(length)); +} + +/* docs */ + +/* type setup */ +static PyMemberDef CpBytesAtom_Members[] = { + { "length", T_OBJECT, offsetof(CpBytesAtomObject, m_length), READONLY }, + { NULL } +}; + +PyTypeObject CpBytesAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpBytesAtom_NAME), + .tp_basicsize = sizeof(CpBytesAtomObject), + .tp_members = CpBytesAtom_Members, + .tp_new = (newfunc)cp_bytesatom_new, + .tp_init = (initproc)cp_bytesatom_init, + .tp_dealloc = (destructor)cp_bytesatom_dealloc, + .tp_repr = (reprfunc)cp_bytesatom_repr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; \ No newline at end of file diff --git a/src/capi.dat b/src/capi.dat index db46361..9b61cdb 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -42,6 +42,7 @@ src:atomimpl/intatomobj.c src:atomimpl/padatomobj.c src:atomimpl/stringatomobj.c src:atomimpl/constatomobj.c +src:atomimpl/bytesatomobj.c src:atomimpl/builtins/builtinatomobj.c src:atomimpl/builtins/repeatedatomobj.c src:atomimpl/builtins/conditionatomobj.c @@ -94,6 +95,7 @@ type:33:_switchatomobj:CpSwitchAtomObject:- type:34:_offsetatomobj:CpOffsetAtomObject:- type:35:_primitiveatomobj:CpPrimitiveAtomObject:- type:36:_lengthinfoobj:CpLengthInfoObject:- +type:37:_bytesatomobj:CpBytesAtomObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 @@ -187,4 +189,8 @@ func:-:CpOffsetAtom_FromSsize_t:CpOffsetAtomObject*:+1 func:-:CpOffsetAtom_New:CpOffsetAtomObject*:+1 func:145:CpOffsetAtom_Pack:int:null func:146:CpOffsetAtom_Unpack:PyObject*:+1 -func:147:CpOffsetAtom_GetOffset:PyObject*:+1 \ No newline at end of file +func:147:CpOffsetAtom_GetOffset:PyObject*:+1 +func:-:CpBytesAtom_New:CpBytesAtomObject*:+1 +func:148:CpBytesAtom_GetLength:PyObject*:+1 +func:149:CpBytesAtom_Pack:int:null +func:150:CpBytesAtom_Unpack:PyObject*:+1 \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/string.h b/src/caterpillar/include/caterpillar/atoms/string.h index ae9bbf9..76f784d 100644 --- a/src/caterpillar/include/caterpillar/atoms/string.h +++ b/src/caterpillar/include/caterpillar/atoms/string.h @@ -28,10 +28,28 @@ struct _stringatomobj PyObject* m_encoding; }; -// PyAPI_DATA(PyTypeObject) CpStringAtom_Type; #define CpStringAtom_NAME "string" - #define CpStringAtom_CheckExact(op) Py_IS_TYPE((op), &CpStringAtom_Type) #define CpStringAtom_Check(op) PyObject_TypeCheck((op), &CpStringAtom_Type) +// TODO: CString, PString + +struct _bytesatomobj +{ + CpBuiltinAtom_HEAD + + PyObject* m_length; + int s_callable; +}; + +#define CpBytesAtom_NAME "octetstring" +#define CpBytesAtom_CheckExact(op) Py_IS_TYPE((op), &CpBytesAtom_Type) +#define CpBytesAtom_Check(op) PyObject_TypeCheck((op), &CpBytesAtom_Type) + +static inline CpBytesAtomObject* +CpBytesAtom_New(PyObject* length) +{ + return (CpBytesAtomObject*)CpObject_CreateOneArg(&CpBytesAtom_Type, length); +} + #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index ccb8553..c03a2b6 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -92,6 +92,8 @@ struct _primitiveatomobj; typedef struct _primitiveatomobj CpPrimitiveAtomObject; struct _lengthinfoobj; typedef struct _lengthinfoobj CpLengthInfoObject; +struct _bytesatomobj; +typedef struct _bytesatomobj CpBytesAtomObject; #ifdef _CPMODULE @@ -137,6 +139,7 @@ extern PyTypeObject CpSwitchAtom_Type; extern PyTypeObject CpOffsetAtom_Type; extern PyTypeObject CpPrimitiveAtom_Type; extern PyTypeObject CpLengthInfo_Type; +extern PyTypeObject CpBytesAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -217,6 +220,9 @@ PyObject* CpSwitchAtom_Unpack(CpSwitchAtomObject* self, CpLayerObject* layer); int CpOffsetAtom_Pack(CpOffsetAtomObject* self, PyObject* obj, CpLayerObject* layer); PyObject* CpOffsetAtom_Unpack(CpOffsetAtomObject* self, CpLayerObject* layer); PyObject* CpOffsetAtom_GetOffset(CpOffsetAtomObject* self, PyObject* layer); +PyObject* CpBytesAtom_GetLength(CpBytesAtomObject* self, CpLayerObject* layer); +int CpBytesAtom_Pack(CpBytesAtomObject* self, PyObject* value, CpLayerObject* layer); +PyObject* CpBytesAtom_Unpack(CpBytesAtomObject* self, CpLayerObject* layer); #else @@ -263,6 +269,7 @@ caterpillar_api.py #define CpOffsetAtom_Type (*(PyTypeObject *)Cp_API[34]) #define CpPrimitiveAtom_Type (*(PyTypeObject *)Cp_API[35]) #define CpLengthInfo_Type (*(PyTypeObject *)Cp_API[36]) +#define CpBytesAtom_Type (*(PyTypeObject *)Cp_API[37]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -343,6 +350,9 @@ caterpillar_api.py #define CpOffsetAtom_Pack (*((int (*)(CpOffsetAtomObject* self, PyObject* obj, CpLayerObject* layer)))Cp_API[145]) #define CpOffsetAtom_Unpack (*((PyObject* (*)(CpOffsetAtomObject* self, CpLayerObject* layer)))Cp_API[146]) #define CpOffsetAtom_GetOffset (*((PyObject* (*)(CpOffsetAtomObject* self, PyObject* layer)))Cp_API[147]) +#define CpBytesAtom_GetLength (*((PyObject* (*)(CpBytesAtomObject* self, CpLayerObject* layer)))Cp_API[148]) +#define CpBytesAtom_Pack (*((int (*)(CpBytesAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[149]) +#define CpBytesAtom_Unpack (*((PyObject* (*)(CpBytesAtomObject* self, CpLayerObject* layer)))Cp_API[150]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index 86d64e6..edf33a0 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -40,7 +40,7 @@ void *Cp_API[] = { (void *) &CpOffsetAtom_Type, (void *) &CpPrimitiveAtom_Type, (void *) &CpLengthInfo_Type, - NULL, + (void *) &CpBytesAtom_Type, NULL, NULL, NULL, @@ -150,6 +150,9 @@ void *Cp_API[] = { (void *) &CpSwitchAtom_Unpack, (void *) &CpOffsetAtom_Pack, (void *) &CpOffsetAtom_Unpack, - (void *) &CpOffsetAtom_GetOffset + (void *) &CpOffsetAtom_GetOffset, + (void *) &CpBytesAtom_GetLength, + (void *) &CpBytesAtom_Pack, + (void *) &CpBytesAtom_Unpack }; diff --git a/src/module.c b/src/module.c index 354d6d1..6ce708b 100644 --- a/src/module.c +++ b/src/module.c @@ -406,6 +406,7 @@ PyInit__C(void) CpPaddingAtom_Type.tp_base = &CpBuiltinAtom_Type; CpStringAtom_Type.tp_base = &CpBuiltinAtom_Type; CpConstAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpBytesAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpBuiltinAtom_Type); CpModule_SetupType(&CpPrimitiveAtom_Type); @@ -421,6 +422,7 @@ PyInit__C(void) CpModule_SetupType(&CpPaddingAtom_Type); CpModule_SetupType(&CpStringAtom_Type); CpModule_SetupType(&CpConstAtom_Type); + CpModule_SetupType(&CpBytesAtom_Type); // module setup m = PyModule_Create(&CpModule); @@ -466,6 +468,7 @@ PyInit__C(void) CpModule_AddObject(CpPaddingAtom_NAME, &CpPaddingAtom_Type); CpModule_AddObject(CpStringAtom_NAME, &CpStringAtom_Type); CpModule_AddObject(CpConstAtom_NAME, &CpConstAtom_Type); + CpModule_AddObject(CpBytesAtom_NAME, &CpBytesAtom_Type); /* setup custom intatoms */ #define CpModule_DefAtom(name, ...) \ diff --git a/test/_C/atoms/test_string.py b/test/_C/atoms/test_string.py index 90218e2..d80793f 100644 --- a/test/_C/atoms/test_string.py +++ b/test/_C/atoms/test_string.py @@ -3,7 +3,8 @@ import caterpillar if caterpillar.native_support(): - from caterpillar._C import unpack, pack, string + from caterpillar._C import unpack, pack, string, octetstring + from caterpillar._C import BIG_ENDIAN as be def test_stringatom_unpack(): @@ -11,4 +12,18 @@ def test_stringatom_unpack(): # bytes and unicode objects. assert unpack(b"foo", string(3, "utf-8")) == "foo" + def test_stringatom_pack(): + # The string atom works as a basic converter between + # bytes and unicode objects. + assert pack("foo", string(3, "utf-8")) == b"foo" + + # TODO: + # we can even apply the endian here. It will append + # "-be" or "-le" at the end of the codec string. + # assert unpack(b"\x00f\x00o\x00o", be + string(3, "utf-16")) == "foo" + def test_bytesatom_unpack(): + # To parse raw bytes you can use the octetstring, which + # is basically the C equivalent for caterpillar.py.Bytes + assert unpack(b"foo", octetstring(3)) == b"foo" + assert unpack(b"foobar", octetstring(3)[2]) == [b"foo", b"bar"] \ No newline at end of file From 214d108e25fdbf46f14c436f8aaf0eafa6cb4c44 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 8 Sep 2024 21:15:49 +0200 Subject: [PATCH 11/29] Pascal-String C implementation --- + Python type: `pstring` + New API functions: CpState_ReadSsize_t, CpPStringAtom_Pack and CpPStringAtom_Unpack + Tests included + Added macro to implement byteorder operator (_CpEndian_ImplSetByteorder) --- CMakeLists.txt | 1 + src/atomimpl/boolatomobj.c | 2 +- src/atomimpl/builtins/conditionatomobj.c | 15 +- src/atomimpl/builtins/offsetatomobj.c | 15 +- src/atomimpl/builtins/repeatedatomobj.c | 15 +- src/atomimpl/builtins/switchatomobj.c | 15 +- src/atomimpl/bytesatomobj.c | 2 +- src/atomimpl/charatomobj.c | 2 +- src/atomimpl/constatomobj.c | 22 +++ src/atomimpl/floatatomobj.c | 2 +- src/atomimpl/intatomobj.c | 2 +- src/atomimpl/padatomobj.c | 4 +- src/atomimpl/pstringatomobj.c | 165 ++++++++++++++++++ src/atomimpl/stringatomobj.c | 2 +- src/capi.dat | 48 ++--- src/caterpillar/include/caterpillar/arch.h | 13 ++ .../include/caterpillar/atoms/string.h | 45 ++++- .../include/caterpillar/caterpillarapi.h | 54 +++--- src/caterpillarapi.c | 11 +- src/module.c | 3 + src/state.c | 25 ++- test/_C/atoms/test_string.py | 21 ++- 22 files changed, 360 insertions(+), 124 deletions(-) create mode 100644 src/atomimpl/pstringatomobj.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a1c70f..d1c2e8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,7 @@ python_add_library( src/atomimpl/stringatomobj.c src/atomimpl/constatomobj.c src/atomimpl/bytesatomobj.c + src/atomimpl/pstringatomobj.c src/atomimpl/builtins/builtinatomobj.c src/atomimpl/builtins/repeatedatomobj.c diff --git a/src/atomimpl/boolatomobj.c b/src/atomimpl/boolatomobj.c index d2214ed..9428473 100644 --- a/src/atomimpl/boolatomobj.c +++ b/src/atomimpl/boolatomobj.c @@ -67,7 +67,7 @@ PyObject* CpBoolAtom_Unpack(CpBoolAtomObject* self, CpLayerObject* layer) { PyObject* res; - PyObject* value = CpState_Read(layer->m_state, 1); + PyObject* value = CpState_ReadSsize_t(layer->m_state, 1); if (!value) { return NULL; } diff --git a/src/atomimpl/builtins/conditionatomobj.c b/src/atomimpl/builtins/conditionatomobj.c index ef3da17..a50f533 100644 --- a/src/atomimpl/builtins/conditionatomobj.c +++ b/src/atomimpl/builtins/conditionatomobj.c @@ -77,20 +77,7 @@ cp_conditionatom_init(CpConditionAtomObject* self, PyObject* args, PyObject* kw) return 0; } -static PyObject* -cp_conditionatom_set_byteorder(CpConditionAtomObject* self, - PyObject* args, - PyObject* kw) -{ - _CpEndian_KwArgsGetByteorder(NULL); - PyObject* new_atom = - CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder); - if (!new_atom) { - return NULL; - } - _Cp_SetObj(self->m_atom, new_atom); - return (PyObject*)self; -} +_CpEndian_ImplSetByteorder(CpConditionAtomObject, conditionatom, self->m_atom); static PyObject* cp_conditionatom_is_enabled(CpConditionAtomObject* self, PyObject* args) diff --git a/src/atomimpl/builtins/offsetatomobj.c b/src/atomimpl/builtins/offsetatomobj.c index 6b9155b..2d1a897 100644 --- a/src/atomimpl/builtins/offsetatomobj.c +++ b/src/atomimpl/builtins/offsetatomobj.c @@ -77,20 +77,7 @@ cp_offsetatom_init(CpOffsetAtomObject* self, PyObject* args, PyObject* kw) return 0; } -static PyObject* -cp_offsetatom_set_byteorder(CpOffsetAtomObject* self, - PyObject* args, - PyObject* kw) -{ - _CpEndian_KwArgsGetByteorder(NULL); - PyObject* new_atom = - CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder); - if (!new_atom) { - return NULL; - } - _Cp_SetObj(self->m_atom, new_atom); - return (PyObject*)self; -} +_CpEndian_ImplSetByteorder(CpOffsetAtomObject, offsetatom, self->m_atom); static PyObject* cp_offsetatom_get_offset(CpOffsetAtomObject* self, PyObject* args, PyObject* kw) diff --git a/src/atomimpl/builtins/repeatedatomobj.c b/src/atomimpl/builtins/repeatedatomobj.c index 587773a..21c5ffd 100644 --- a/src/atomimpl/builtins/repeatedatomobj.c +++ b/src/atomimpl/builtins/repeatedatomobj.c @@ -88,20 +88,7 @@ cp_repeatedatomobj_init(CpRepeatedAtomObject* self, return 0; } -static PyObject* -cp_repeatedatom_set_byteorder(CpRepeatedAtomObject* self, - PyObject* args, - PyObject* kw) -{ - _CpEndian_KwArgsGetByteorder(NULL); - PyObject* new_atom = - CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder); - if (!new_atom) { - return NULL; - } - _Cp_SetObj(self->m_atom, new_atom); - return (PyObject*)self; -} +_CpEndian_ImplSetByteorder(CpRepeatedAtomObject, repeatedatom, self->m_atom); /* Public API */ diff --git a/src/atomimpl/builtins/switchatomobj.c b/src/atomimpl/builtins/switchatomobj.c index 88cc332..7548e56 100644 --- a/src/atomimpl/builtins/switchatomobj.c +++ b/src/atomimpl/builtins/switchatomobj.c @@ -131,20 +131,7 @@ cp_switchatom_init(CpSwitchAtomObject* self, PyObject* args, PyObject* kw) return 0; } -static PyObject* -cp_switchatom_set_byteorder(CpSwitchAtomObject* self, - PyObject* args, - PyObject* kw) -{ - _CpEndian_KwArgsGetByteorder(NULL); - PyObject* new_atom = - CpEndian_SetEndian(self->m_atom, (CpEndianObject*)byteorder); - if (!new_atom) { - return NULL; - } - _Cp_SetObj(self->m_atom, new_atom); - return (PyObject*)self; -} +_CpEndian_ImplSetByteorder(CpSwitchAtomObject, switchatom, self->m_atom); static PyObject* cp_switchatom_get_next(CpSwitchAtomObject* self, PyObject* args, PyObject* kw) diff --git a/src/atomimpl/bytesatomobj.c b/src/atomimpl/bytesatomobj.c index bad5121..6a42a38 100644 --- a/src/atomimpl/bytesatomobj.c +++ b/src/atomimpl/bytesatomobj.c @@ -104,7 +104,7 @@ CpBytesAtom_Unpack(CpBytesAtomObject* self, CpLayerObject* layer) if (length == NULL) { return NULL; } - return CpState_Read(layer->m_state, PyLong_AS_LONG(length)); + return CpState_Read(layer->m_state, length); } /* docs */ diff --git a/src/atomimpl/charatomobj.c b/src/atomimpl/charatomobj.c index bc5d0ce..8087d00 100644 --- a/src/atomimpl/charatomobj.c +++ b/src/atomimpl/charatomobj.c @@ -70,7 +70,7 @@ CpCharAtom_Pack(CpCharAtomObject* self, PyObject* value, CpLayerObject* layer) PyObject* CpCharAtom_Unpack(CpCharAtomObject* self, CpLayerObject* layer) { - PyObject* res = CpState_Read(layer->m_state, 1); + PyObject* res = CpState_ReadSsize_t(layer->m_state, 1); if (!res) { return NULL; } diff --git a/src/atomimpl/constatomobj.c b/src/atomimpl/constatomobj.c index 8a4c648..6bb2446 100644 --- a/src/atomimpl/constatomobj.c +++ b/src/atomimpl/constatomobj.c @@ -56,6 +56,14 @@ cp_constatom_init(CpConstAtomObject* self, PyObject* args, PyObject* kwds) return 0; } +static PyObject* +cp_constatom_repr(CpConstAtomObject* self) +{ + return PyUnicode_FromFormat("", self->m_value, self->m_atom); +} + +_CpEndian_ImplSetByteorder(CpConstAtomObject, constatom, self->m_atom); + /* Public API */ /*CpAPI*/ @@ -88,6 +96,17 @@ CpConstAtom_Unpack(CpConstAtomObject* self, CpLayerObject* layer) /* type setup */ +static PyMemberDef cp_constatom_members[] = { + { "atom", T_OBJECT_EX, offsetof(CpConstAtomObject, m_atom), READONLY }, + { "value", T_OBJECT_EX, offsetof(CpConstAtomObject, m_value), READONLY }, + { NULL }, +}; + +static PyMethodDef cp_constatom_methods[] = { + _CpEndian_ImplSetByteorder_MethDef(constatom, NULL), + { NULL } +}; + PyTypeObject CpConstAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpConstAtom_NAME), .tp_basicsize = sizeof(CpConstAtomObject), @@ -95,4 +114,7 @@ PyTypeObject CpConstAtom_Type = { .tp_init = (initproc)cp_constatom_init, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = (newfunc)cp_constatom_new, + .tp_repr = (reprfunc)cp_constatom_repr, + .tp_members = cp_constatom_members, + .tp_methods = cp_constatom_methods }; \ No newline at end of file diff --git a/src/atomimpl/floatatomobj.c b/src/atomimpl/floatatomobj.c index 9a95be5..7f8d8cd 100644 --- a/src/atomimpl/floatatomobj.c +++ b/src/atomimpl/floatatomobj.c @@ -146,7 +146,7 @@ CpFloatAtom_Pack(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer) PyObject* CpFloatAtom_Unpack(CpFloatAtomObject* self, CpLayerObject* layer) { - PyObject* bytes = CpState_Read(layer->m_state, self->_m_byte_count); + PyObject* bytes = CpState_ReadSsize_t(layer->m_state, self->_m_byte_count); if (!bytes) { return NULL; } diff --git a/src/atomimpl/intatomobj.c b/src/atomimpl/intatomobj.c index 85b4a68..3ec90f6 100644 --- a/src/atomimpl/intatomobj.c +++ b/src/atomimpl/intatomobj.c @@ -147,7 +147,7 @@ CpIntAtom_Pack(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer) PyObject* CpIntAtom_Unpack(CpIntAtomObject* self, CpLayerObject* layer) { - PyObject* bytes = CpState_Read(layer->m_state, self->_m_byte_count); + PyObject* bytes = CpState_ReadSsize_t(layer->m_state, self->_m_byte_count); if (!bytes) { return NULL; } diff --git a/src/atomimpl/padatomobj.c b/src/atomimpl/padatomobj.c index 1d4ac1b..5fcd083 100644 --- a/src/atomimpl/padatomobj.c +++ b/src/atomimpl/padatomobj.c @@ -120,7 +120,7 @@ CpPaddingAtom_PackMany(CpPaddingAtomObject* self, PyObject* CpPaddingAtom_Unpack(CpPaddingAtomObject* self, CpLayerObject* layer) { - PyObject* res = CpState_Read(layer->m_state, 1); + PyObject* res = CpState_ReadSsize_t(layer->m_state, 1); if (!res) { return NULL; } @@ -139,7 +139,7 @@ CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, if (lengthinfo->m_greedy) { res = CpState_ReadFully(layer->m_state); } else { - res = CpState_Read(layer->m_state, lengthinfo->m_length); + res = CpState_ReadSsize_t(layer->m_state, lengthinfo->m_length); } if (!res) { diff --git a/src/atomimpl/pstringatomobj.c b/src/atomimpl/pstringatomobj.c new file mode 100644 index 0000000..45bc2a1 --- /dev/null +++ b/src/atomimpl/pstringatomobj.c @@ -0,0 +1,165 @@ +/* pascal string implementation */ + +#include "caterpillar/caterpillar.h" + +#include + +/*impl*/ +static PyObject* +cp_pstringatom_type(PyObject* self) +{ + return Py_XNewRef(&PyUnicode_Type); +} + +static PyObject* +cp_pstringatom_size(CpPStringAtomObject* self, CpLayerObject* layer) +{ + PyErr_SetString(PyExc_TypeError, "Pascal strings do not have a static size!"); + return NULL; +} + +static PyObject* +cp_pstringatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) +{ + CpPStringAtomObject* self = (CpPStringAtomObject*)type->tp_alloc(type, 0); + if (self != NULL) { + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpPStringAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpPStringAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_pstringatom_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_pstringatom_type; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + + self->m_encoding = NULL; + self->m_errors = NULL; + self->m_atom = NULL; + } + return (PyObject*)self; +} + +static void +cp_pstringatom_dealloc(CpPStringAtomObject* self) +{ + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_pstringatom_init(CpPStringAtomObject* self, PyObject* args, PyObject* kwds) +{ + static char* kwlist[] = { "atom", "encoding", "errors", NULL }; + PyObject *encoding = NULL, *errors = NULL, *atom = NULL; + + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "OO|O", kwlist, &atom, &encoding, &errors)) { + return -1; + } + + _Cp_SetObj(self->m_atom, atom); + _Cp_SetObj(self->m_encoding, encoding); + _Cp_SetObj(self->m_errors, errors); + if (!self->m_errors) { + _modulestate* mod = get_global_module_state(); + _Cp_SetObj(self->m_errors, mod->str_strict); + } + return 0; +} + +static PyObject* +cp_pstringatom_repr(CpPStringAtomObject* self) +{ + return PyUnicode_FromFormat( + "", self->m_encoding, self->m_errors, self->m_atom); +} + +_CpEndian_ImplSetByteorder(CpPStringAtomObject, pstringatom, self->m_atom); + +/* Public API */ + +/*CpAPI*/ +int +CpPStringAtom_Pack(CpPStringAtomObject* self, + PyObject* value, + CpLayerObject* layer) +{ + if (!PyUnicode_Check(value)) { + PyErr_Format(PyExc_TypeError, + "Pascal strings must be packed from unicode - got %R", + value); + return -1; + } + + PyObject* length = PyLong_FromSsize_t(PyUnicode_GET_LENGTH(value)); + if (!length) { + return -1; + } + + if (_Cp_Pack(length, self->m_atom, layer)) { + Py_DECREF(length); + return -1; + } + Py_DECREF(length); + + PyObject* bytes = PyUnicode_AsEncodedString( + value, PyUnicode_DATA(self->m_encoding), PyUnicode_DATA(self->m_errors)); + if (!bytes) { + return -1; + } + PyObject* res = CpState_Write(layer->m_state, bytes); + Py_DECREF(bytes); + if (!res) { + return -1; + } + Py_XDECREF(res); + return 0; +} + +/*CpAPI*/ +PyObject* +CpPStringAtom_Unpack(CpPStringAtomObject* self, CpLayerObject* layer) +{ + PyObject* length = _Cp_Unpack(self->m_atom, layer); + if (length == NULL) { + return NULL; + } + + PyObject* bytes = CpState_Read(layer->m_state, length); + Py_DECREF(length); + if (!bytes) { + return NULL; + } + PyObject* string = PyUnicode_Decode(PyBytes_AS_STRING(bytes), + PyBytes_GET_SIZE(bytes), + PyUnicode_DATA(self->m_encoding), + PyUnicode_DATA(self->m_errors)); + Py_DECREF(bytes); + return string; +} + +/* docs */ + +/* type */ +static PyMemberDef CpPStringAtom_Members[] = { + { "atom", T_OBJECT, offsetof(CpPStringAtomObject, m_atom), 0, NULL }, + { "encoding", T_STRING, offsetof(CpPStringAtomObject, m_encoding), 0, NULL }, + { "errors", T_STRING, offsetof(CpPStringAtomObject, m_errors), 0, NULL }, + { NULL } /* Sentinel */ +}; + +static PyMethodDef CpPStringAtom_Methods[] = { + _CpEndian_ImplSetByteorder_MethDef(pstringatom, NULL), + { NULL } /* Sentinel */ +}; + +PyTypeObject CpPStringAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpPStringAtom_NAME), + .tp_basicsize = sizeof(CpPStringAtomObject), + .tp_dealloc = (destructor)cp_pstringatom_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = NULL, + .tp_members = CpPStringAtom_Members, + .tp_init = (initproc)cp_pstringatom_init, + .tp_new = (newfunc)cp_pstringatom_new, + .tp_repr = (reprfunc)cp_pstringatom_repr, + .tp_methods = CpPStringAtom_Methods, +}; \ No newline at end of file diff --git a/src/atomimpl/stringatomobj.c b/src/atomimpl/stringatomobj.c index 8c516a8..37730f1 100644 --- a/src/atomimpl/stringatomobj.c +++ b/src/atomimpl/stringatomobj.c @@ -110,7 +110,7 @@ CpStringAtom_Unpack(CpStringAtomObject* self, CpLayerObject* layer) if (greedy) { res = CpState_ReadFully(layer->m_state); } else { - res = CpState_Read(layer->m_state, length); + res = CpState_ReadSsize_t(layer->m_state, length); } if (!res) { return NULL; diff --git a/src/capi.dat b/src/capi.dat index 9b61cdb..9ea3c48 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -43,6 +43,7 @@ src:atomimpl/padatomobj.c src:atomimpl/stringatomobj.c src:atomimpl/constatomobj.c src:atomimpl/bytesatomobj.c +src:atomimpl/pstringatomobj.c src:atomimpl/builtins/builtinatomobj.c src:atomimpl/builtins/repeatedatomobj.c src:atomimpl/builtins/conditionatomobj.c @@ -96,6 +97,7 @@ type:34:_offsetatomobj:CpOffsetAtomObject:- type:35:_primitiveatomobj:CpPrimitiveAtomObject:- type:36:_lengthinfoobj:CpLengthInfoObject:- type:37:_bytesatomobj:CpBytesAtomObject:- +type:38:_pstringatomobj:CpPStringAtomObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 @@ -135,26 +137,27 @@ func:88:CpState_New:CpStateObject*:null func:89:CpState_Tell:PyObject*:+1 func:90:CpState_Seek:PyObject*:+1 func:91:CpState_Read:PyObject*:+1 -func:92:CpState_ReadFully:PyObject*:+1 -func:93:CpState_Write:PyObject*:+1 -func:94:CpState_SetGlobals:int:null -func:95:CpLayer_New:CpLayerObject*:+1 -func:96:CpLayer_Invalidate:int:null -func:98:CpStructFieldInfo_New:CpStructFieldInfoObject*:+1 -func:99:CpStruct_AddFieldInfo:int:null -func:100:CpStruct_AddField:int:null -func:101:CpStruct_New:CpStructObject*:+1 -func:102:CpStruct_GetAnnotations:PyObject*:+1 -func:103:CpStruct_ReplaceType:int:null -func:104:CpStruct_HasOption:int:null -func:105:CpStruct_Pack:int:null -func:106:CpStruct_Unpack:PyObject*:+1 -func:107:CpStruct_SizeOf:PyObject*:+1 -func:108:CpStructModel_Check:int:null -func:109:CpStructModel_GetStruct:PyObject*:+1 -func:110:CpSeqLayer_New:CpSeqLayerObject*:+1 -func:111:CpSeqLayer_SetSequence:int:null -func:112:CpObjLayer_New:CpObjLayerObject*:+1 +func:92:CpState_ReadSsize_t:PyObject*:+1 +func:93:CpState_ReadFully:PyObject*:+1 +func:94:CpState_Write:PyObject*:+1 +func:95:CpState_SetGlobals:int:null +func:96:CpLayer_New:CpLayerObject*:+1 +func:98:CpLayer_Invalidate:int:null +func:99:CpStructFieldInfo_New:CpStructFieldInfoObject*:+1 +func:100:CpStruct_AddFieldInfo:int:null +func:101:CpStruct_AddField:int:null +func:102:CpStruct_New:CpStructObject*:+1 +func:103:CpStruct_GetAnnotations:PyObject*:+1 +func:104:CpStruct_ReplaceType:int:null +func:105:CpStruct_HasOption:int:null +func:106:CpStruct_Pack:int:null +func:107:CpStruct_Unpack:PyObject*:+1 +func:108:CpStruct_SizeOf:PyObject*:+1 +func:109:CpStructModel_Check:int:null +func:110:CpStructModel_GetStruct:PyObject*:+1 +func:111:CpSeqLayer_New:CpSeqLayerObject*:+1 +func:112:CpSeqLayer_SetSequence:int:null +func:113:CpObjLayer_New:CpObjLayerObject*:+1 # atom api func:120:CpIntAtom_Pack:int:null @@ -193,4 +196,7 @@ func:147:CpOffsetAtom_GetOffset:PyObject*:+1 func:-:CpBytesAtom_New:CpBytesAtomObject*:+1 func:148:CpBytesAtom_GetLength:PyObject*:+1 func:149:CpBytesAtom_Pack:int:null -func:150:CpBytesAtom_Unpack:PyObject*:+1 \ No newline at end of file +func:150:CpBytesAtom_Unpack:PyObject*:+1 +func:-:CpPStringAtom_New:CpPStringAtomObject*:+1 +func:152:CpPStringAtom_Pack:int:null +func:153:CpPStringAtom_Unpack:PyObject*:+1 \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/arch.h b/src/caterpillar/include/caterpillar/arch.h index 6d18399..a79b6ca 100644 --- a/src/caterpillar/include/caterpillar/arch.h +++ b/src/caterpillar/include/caterpillar/arch.h @@ -127,4 +127,17 @@ CpEndian_SetEndian(PyObject* op, CpEndianObject* endian) return ret; \ } +#define _CpEndian_ImplSetByteorder(typename, name, field) \ + static PyObject* cp_##name##_set_byteorder( \ + typename* self, PyObject* args, PyObject* kw) \ + { \ + _CpEndian_KwArgsGetByteorder(NULL); \ + PyObject* ret = CpEndian_SetEndian(field, (CpEndianObject*)byteorder); \ + if (!ret) { \ + return NULL; \ + } \ + _Cp_SetObj(field, ret); \ + return Py_NewRef((PyObject*)self); \ + } + #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/string.h b/src/caterpillar/include/caterpillar/atoms/string.h index 76f784d..c2f3114 100644 --- a/src/caterpillar/include/caterpillar/atoms/string.h +++ b/src/caterpillar/include/caterpillar/atoms/string.h @@ -19,12 +19,20 @@ #include "caterpillar/atoms/builtins.h" +// --------------------------------------------------------------------------- +// Default String struct _stringatomobj { CpBuiltinAtom_HEAD + /// The length of this string. It can be a constant integer + /// object or a ContextLambda. PyObject* m_length; + + /// Whether this string should ignore errors when parsing PyObject* m_errors; + + /// The encoding of this string PyObject* m_encoding; }; @@ -34,14 +42,21 @@ struct _stringatomobj // TODO: CString, PString +// --------------------------------------------------------------------------- +// Bytes struct _bytesatomobj { CpBuiltinAtom_HEAD + /// The length of this string. It can be a constant integer + /// object or a ContextLambda. PyObject* m_length; - int s_callable; + + // -- internal --- + int s_callable; }; +// REVISIT: The name of this atom should be something similar to 'bytes' #define CpBytesAtom_NAME "octetstring" #define CpBytesAtom_CheckExact(op) Py_IS_TYPE((op), &CpBytesAtom_Type) #define CpBytesAtom_Check(op) PyObject_TypeCheck((op), &CpBytesAtom_Type) @@ -52,4 +67,32 @@ CpBytesAtom_New(PyObject* length) return (CpBytesAtomObject*)CpObject_CreateOneArg(&CpBytesAtom_Type, length); } +// --------------------------------------------------------------------------- +// PString + +struct _pstringatomobj +{ + CpBuiltinAtom_HEAD + + /// The atom that parses the length of this string + PyObject* m_atom; + + /// Whether this string should ignore errors when parsing + PyObject* m_errors; + + /// The encoding of this string + PyObject* m_encoding; +}; + +#define CpPStringAtom_NAME "pstring" +#define CpPStringAtom_CheckExact(op) Py_IS_TYPE((op), &CpPStringAtom_Type) +#define CpPStringAtom_Check(op) PyObject_TypeCheck((op), &CpPStringAtom_Type) + +static inline CpPStringAtomObject* +CpPStringAtom_New(PyObject* atom, PyObject* encoding) +{ + return (CpPStringAtomObject*)CpObject_Create( + &CpPStringAtom_Type, "OO", atom, encoding); +} + #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index c03a2b6..3f15b67 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -94,6 +94,8 @@ struct _lengthinfoobj; typedef struct _lengthinfoobj CpLengthInfoObject; struct _bytesatomobj; typedef struct _bytesatomobj CpBytesAtomObject; +struct _pstringatomobj; +typedef struct _pstringatomobj CpPStringAtomObject; #ifdef _CPMODULE @@ -140,6 +142,7 @@ extern PyTypeObject CpOffsetAtom_Type; extern PyTypeObject CpPrimitiveAtom_Type; extern PyTypeObject CpLengthInfo_Type; extern PyTypeObject CpBytesAtom_Type; +extern PyTypeObject CpPStringAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -171,7 +174,8 @@ PyObject * CpTypeOf_CAtom(CpCAtomObject* op); CpStateObject* CpState_New(PyObject* io); PyObject* CpState_Tell(CpStateObject* self); PyObject* CpState_Seek(CpStateObject* self, PyObject* offset, PyObject* whence); -PyObject* CpState_Read(CpStateObject* self, Py_ssize_t size); +PyObject* CpState_Read(CpStateObject* self, PyObject* sizeobj); +PyObject* CpState_ReadSsize_t(CpStateObject* self, Py_ssize_t size); PyObject* CpState_ReadFully(CpStateObject* self); PyObject* CpState_Write(CpStateObject* self, PyObject* value); int CpState_SetGlobals(CpStateObject* self, PyObject* globals); @@ -223,6 +227,8 @@ PyObject* CpOffsetAtom_GetOffset(CpOffsetAtomObject* self, PyObject* layer); PyObject* CpBytesAtom_GetLength(CpBytesAtomObject* self, CpLayerObject* layer); int CpBytesAtom_Pack(CpBytesAtomObject* self, PyObject* value, CpLayerObject* layer); PyObject* CpBytesAtom_Unpack(CpBytesAtomObject* self, CpLayerObject* layer); +int CpPStringAtom_Pack(CpPStringAtomObject* self,PyObject* value,CpLayerObject* layer); +PyObject* CpPStringAtom_Unpack(CpPStringAtomObject* self, CpLayerObject* layer); #else @@ -270,6 +276,7 @@ caterpillar_api.py #define CpPrimitiveAtom_Type (*(PyTypeObject *)Cp_API[35]) #define CpLengthInfo_Type (*(PyTypeObject *)Cp_API[36]) #define CpBytesAtom_Type (*(PyTypeObject *)Cp_API[37]) +#define CpPStringAtom_Type (*(PyTypeObject *)Cp_API[38]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -301,27 +308,28 @@ caterpillar_api.py #define CpState_New (*((CpStateObject* (*)(PyObject* io)))Cp_API[88]) #define CpState_Tell (*((PyObject* (*)(CpStateObject* self)))Cp_API[89]) #define CpState_Seek (*((PyObject* (*)(CpStateObject* self, PyObject* offset, PyObject* whence)))Cp_API[90]) -#define CpState_Read (*((PyObject* (*)(CpStateObject* self, Py_ssize_t size)))Cp_API[91]) -#define CpState_ReadFully (*((PyObject* (*)(CpStateObject* self)))Cp_API[92]) -#define CpState_Write (*((PyObject* (*)(CpStateObject* self, PyObject* value)))Cp_API[93]) -#define CpState_SetGlobals (*((int (*)(CpStateObject* self, PyObject* globals)))Cp_API[94]) -#define CpLayer_New (*((CpLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[95]) -#define CpLayer_Invalidate (*((int (*)(CpLayerObject* self)))Cp_API[96]) -#define CpStructFieldInfo_New (*((CpStructFieldInfoObject* (*)(PyObject* name, PyObject* field)))Cp_API[98]) -#define CpStruct_AddFieldInfo (*((int (*)(CpStructObject* o, CpStructFieldInfoObject* info)))Cp_API[99]) -#define CpStruct_AddField (*((int (*)(CpStructObject* o, CpFieldObject* field, int exclude)))Cp_API[100]) -#define CpStruct_New (*((CpStructObject* (*)(PyObject* model)))Cp_API[101]) -#define CpStruct_GetAnnotations (*((PyObject* (*)(CpStructObject* o, int eval)))Cp_API[102]) -#define CpStruct_ReplaceType (*((int (*)(CpStructObject* o, PyObject* name, PyObject* type)))Cp_API[103]) -#define CpStruct_HasOption (*((int (*)(CpStructObject* o, PyObject* option)))Cp_API[104]) -#define CpStruct_Pack (*((int (*)(CpStructObject* self, PyObject* obj, CpLayerObject* layer)))Cp_API[105]) -#define CpStruct_Unpack (*((PyObject* (*)(CpStructObject* self, CpLayerObject* layer)))Cp_API[106]) -#define CpStruct_SizeOf (*((PyObject* (*)(CpStructObject* self, CpLayerObject* layer)))Cp_API[107]) -#define CpStructModel_Check (*((int (*)(PyObject* model, _modulestate* state)))Cp_API[108]) -#define CpStructModel_GetStruct (*((PyObject* (*)(PyObject* model, _modulestate* state)))Cp_API[109]) -#define CpSeqLayer_New (*((CpSeqLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[110]) -#define CpSeqLayer_SetSequence (*((int (*)(CpSeqLayerObject* self,PyObject* sequence,Py_ssize_t length,int8_t greedy)))Cp_API[111]) -#define CpObjLayer_New (*((CpObjLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[112]) +#define CpState_Read (*((PyObject* (*)(CpStateObject* self, PyObject* sizeobj)))Cp_API[91]) +#define CpState_ReadSsize_t (*((PyObject* (*)(CpStateObject* self, Py_ssize_t size)))Cp_API[92]) +#define CpState_ReadFully (*((PyObject* (*)(CpStateObject* self)))Cp_API[93]) +#define CpState_Write (*((PyObject* (*)(CpStateObject* self, PyObject* value)))Cp_API[94]) +#define CpState_SetGlobals (*((int (*)(CpStateObject* self, PyObject* globals)))Cp_API[95]) +#define CpLayer_New (*((CpLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[96]) +#define CpLayer_Invalidate (*((int (*)(CpLayerObject* self)))Cp_API[98]) +#define CpStructFieldInfo_New (*((CpStructFieldInfoObject* (*)(PyObject* name, PyObject* field)))Cp_API[99]) +#define CpStruct_AddFieldInfo (*((int (*)(CpStructObject* o, CpStructFieldInfoObject* info)))Cp_API[100]) +#define CpStruct_AddField (*((int (*)(CpStructObject* o, CpFieldObject* field, int exclude)))Cp_API[101]) +#define CpStruct_New (*((CpStructObject* (*)(PyObject* model)))Cp_API[102]) +#define CpStruct_GetAnnotations (*((PyObject* (*)(CpStructObject* o, int eval)))Cp_API[103]) +#define CpStruct_ReplaceType (*((int (*)(CpStructObject* o, PyObject* name, PyObject* type)))Cp_API[104]) +#define CpStruct_HasOption (*((int (*)(CpStructObject* o, PyObject* option)))Cp_API[105]) +#define CpStruct_Pack (*((int (*)(CpStructObject* self, PyObject* obj, CpLayerObject* layer)))Cp_API[106]) +#define CpStruct_Unpack (*((PyObject* (*)(CpStructObject* self, CpLayerObject* layer)))Cp_API[107]) +#define CpStruct_SizeOf (*((PyObject* (*)(CpStructObject* self, CpLayerObject* layer)))Cp_API[108]) +#define CpStructModel_Check (*((int (*)(PyObject* model, _modulestate* state)))Cp_API[109]) +#define CpStructModel_GetStruct (*((PyObject* (*)(PyObject* model, _modulestate* state)))Cp_API[110]) +#define CpSeqLayer_New (*((CpSeqLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[111]) +#define CpSeqLayer_SetSequence (*((int (*)(CpSeqLayerObject* self,PyObject* sequence,Py_ssize_t length,int8_t greedy)))Cp_API[112]) +#define CpObjLayer_New (*((CpObjLayerObject* (*)(CpStateObject* state, CpLayerObject* parent)))Cp_API[113]) #define CpIntAtom_Pack (*((int (*)(CpIntAtomObject* self, PyObject* op, CpLayerObject* layer)))Cp_API[120]) #define CpIntAtom_Unpack (*((PyObject* (*)(CpIntAtomObject* self, CpLayerObject* layer)))Cp_API[121]) #define CpFloatAtom_Pack (*((int (*)(CpFloatAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[122]) @@ -353,6 +361,8 @@ caterpillar_api.py #define CpBytesAtom_GetLength (*((PyObject* (*)(CpBytesAtomObject* self, CpLayerObject* layer)))Cp_API[148]) #define CpBytesAtom_Pack (*((int (*)(CpBytesAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[149]) #define CpBytesAtom_Unpack (*((PyObject* (*)(CpBytesAtomObject* self, CpLayerObject* layer)))Cp_API[150]) +#define CpPStringAtom_Pack (*((int (*)(CpPStringAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[152]) +#define CpPStringAtom_Unpack (*((PyObject* (*)(CpPStringAtomObject* self, CpLayerObject* layer)))Cp_API[153]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index edf33a0..b7dbde7 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -41,7 +41,7 @@ void *Cp_API[] = { (void *) &CpPrimitiveAtom_Type, (void *) &CpLengthInfo_Type, (void *) &CpBytesAtom_Type, - NULL, + (void *) &CpPStringAtom_Type, NULL, NULL, NULL, @@ -95,12 +95,13 @@ void *Cp_API[] = { (void *) &CpState_Tell, (void *) &CpState_Seek, (void *) &CpState_Read, + (void *) &CpState_ReadSsize_t, (void *) &CpState_ReadFully, (void *) &CpState_Write, (void *) &CpState_SetGlobals, (void *) &CpLayer_New, - (void *) &CpLayer_Invalidate, NULL, + (void *) &CpLayer_Invalidate, (void *) &CpStructFieldInfo_New, (void *) &CpStruct_AddFieldInfo, (void *) &CpStruct_AddField, @@ -122,7 +123,6 @@ void *Cp_API[] = { NULL, NULL, NULL, - NULL, (void *) &CpIntAtom_Pack, (void *) &CpIntAtom_Unpack, (void *) &CpFloatAtom_Pack, @@ -153,6 +153,9 @@ void *Cp_API[] = { (void *) &CpOffsetAtom_GetOffset, (void *) &CpBytesAtom_GetLength, (void *) &CpBytesAtom_Pack, - (void *) &CpBytesAtom_Unpack + (void *) &CpBytesAtom_Unpack, + NULL, + (void *) &CpPStringAtom_Pack, + (void *) &CpPStringAtom_Unpack }; diff --git a/src/module.c b/src/module.c index 6ce708b..eafd6fa 100644 --- a/src/module.c +++ b/src/module.c @@ -407,6 +407,7 @@ PyInit__C(void) CpStringAtom_Type.tp_base = &CpBuiltinAtom_Type; CpConstAtom_Type.tp_base = &CpBuiltinAtom_Type; CpBytesAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpPStringAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpBuiltinAtom_Type); CpModule_SetupType(&CpPrimitiveAtom_Type); @@ -423,6 +424,7 @@ PyInit__C(void) CpModule_SetupType(&CpStringAtom_Type); CpModule_SetupType(&CpConstAtom_Type); CpModule_SetupType(&CpBytesAtom_Type); + CpModule_SetupType(&CpPStringAtom_Type); // module setup m = PyModule_Create(&CpModule); @@ -469,6 +471,7 @@ PyInit__C(void) CpModule_AddObject(CpStringAtom_NAME, &CpStringAtom_Type); CpModule_AddObject(CpConstAtom_NAME, &CpConstAtom_Type); CpModule_AddObject(CpBytesAtom_NAME, &CpBytesAtom_Type); + CpModule_AddObject(CpPStringAtom_NAME, &CpPStringAtom_Type); /* setup custom intatoms */ #define CpModule_DefAtom(name, ...) \ diff --git a/src/state.c b/src/state.c index 2941fa8..1807566 100644 --- a/src/state.c +++ b/src/state.c @@ -120,8 +120,8 @@ cp_state_write(CpStateObject* self, PyObject* args) static PyObject* cp_state_read(CpStateObject* self, PyObject* args) { - Py_ssize_t size = 0; - if (!PyArg_ParseTuple(args, "n", &size)) { + PyObject* size = 0; + if (!PyArg_ParseTuple(args, "O", &size)) { return NULL; } @@ -178,9 +178,18 @@ CpState_Seek(CpStateObject* self, PyObject* offset, PyObject* whence) /*CpAPI*/ PyObject* -CpState_Read(CpStateObject* self, Py_ssize_t size) +CpState_ReadSsize_t(CpStateObject* self, Py_ssize_t size) { PyObject* sizeobj = PyLong_FromSsize_t(size); + PyObject* res = CpState_Read(self, sizeobj); + Py_DECREF(sizeobj); + return res; +} + +/*CpAPI*/ +PyObject* +CpState_Read(CpStateObject* self, PyObject* sizeobj) +{ PyObject* res = PyObject_CallMethodOneArg(self->m_io, self->mod->str_read, sizeobj); Py_DECREF(sizeobj); @@ -189,18 +198,18 @@ CpState_Read(CpStateObject* self, Py_ssize_t size) if (!PyErr_Occurred()) { PyErr_Format(PyExc_ValueError, "read() returned NULL without error (possible size " - "overflow?). Tried to read %ld bytes", - size); + "overflow?). Tried to read %R bytes", + sizeobj); } return NULL; } Py_ssize_t length = 0; - if ((length = PyObject_Length(res)) != size) { + if ((length = PyObject_Length(res)) != PyLong_AsSsize_t(sizeobj)) { Py_DECREF(res); PyErr_Format(PyExc_ValueError, - "read() expected to return buffer with length %ld, got %ld", - size, + "read() expected to return buffer with length %R, got %ld", + sizeobj, length); return NULL; } diff --git a/test/_C/atoms/test_string.py b/test/_C/atoms/test_string.py index d80793f..5859af3 100644 --- a/test/_C/atoms/test_string.py +++ b/test/_C/atoms/test_string.py @@ -3,9 +3,8 @@ import caterpillar if caterpillar.native_support(): - from caterpillar._C import unpack, pack, string, octetstring - from caterpillar._C import BIG_ENDIAN as be - + from caterpillar._C import unpack, pack, string, octetstring, pstring + from caterpillar._C import BIG_ENDIAN as be, u16, u8 def test_stringatom_unpack(): # The string atom works as a basic converter between @@ -26,4 +25,18 @@ def test_bytesatom_unpack(): # To parse raw bytes you can use the octetstring, which # is basically the C equivalent for caterpillar.py.Bytes assert unpack(b"foo", octetstring(3)) == b"foo" - assert unpack(b"foobar", octetstring(3)[2]) == [b"foo", b"bar"] \ No newline at end of file + assert unpack(b"foobar", octetstring(3)[2]) == [b"foo", b"bar"] + + def test_pstringatom_pack(): + # P-Strings or (Pascal-String) are a special case of + # string atoms where the length of the string is stored + # in the first byte(s). + assert pack("foo", pstring(u8, "utf-8")) == b"\x03foo" + # NOTE that the byteorder operator will be applied to + # the underlying struct. + assert pack("bar", be + pstring(u16, "utf-8")) == b"\x00\x03bar" + + def test_pstringatom_unpack(): + # Same applies to parsing data + assert unpack(b"\x03foo", pstring(u8, "utf-8")) == "foo" + assert unpack(b"\x00\x03bar", be + pstring(u16, "utf-8")) == "bar" \ No newline at end of file From a9a2ddad8be381919555f6b291b0a212fe5bbaec Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 8 Sep 2024 22:08:55 +0200 Subject: [PATCH 12/29] Implementation for C enum atoms --- + Python class: `enumeration` + Tests included + Updated roadmap accordingly --- CMakeLists.txt | 3 + docs/sphinx/source/development/roadmap.rst | 53 +++-- src/atomimpl/constatomobj.c | 2 +- src/atomimpl/enumatomobj.c | 187 ++++++++++++++++++ src/capi.dat | 7 +- .../include/caterpillar/atoms/enum.h | 51 +++++ .../include/caterpillar/caterpillar.h | 1 + .../include/caterpillar/caterpillarapi.h | 8 + src/caterpillar/include/caterpillar/module.h | 9 +- src/caterpillarapi.c | 6 +- src/module.c | 9 +- test/_C/atoms/test_enum.py | 26 +++ 12 files changed, 342 insertions(+), 20 deletions(-) create mode 100644 src/atomimpl/enumatomobj.c create mode 100644 src/caterpillar/include/caterpillar/atoms/enum.h create mode 100644 test/_C/atoms/test_enum.py diff --git a/CMakeLists.txt b/CMakeLists.txt index d1c2e8c..aeb8a64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ python_add_library( src/atomimpl/constatomobj.c src/atomimpl/bytesatomobj.c src/atomimpl/pstringatomobj.c + src/atomimpl/enumatomobj.c src/atomimpl/builtins/builtinatomobj.c src/atomimpl/builtins/repeatedatomobj.c @@ -57,6 +58,8 @@ message(STATUS "Python ${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}") add_custom_target( genapi ALL + # execute python ./src/code_gen/genapi.py ./src/caterpillar/include/caterpillar/caterpillarapi.h.in ./src/caterpillarapi.c.in + # in the root directory COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/src/code_gen/genapi.py ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillarapi.c.in BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillarapi.c COMMENT "Generating Public Caterpillar API" diff --git a/docs/sphinx/source/development/roadmap.rst b/docs/sphinx/source/development/roadmap.rst index 4ad0ac8..cd536a7 100644 --- a/docs/sphinx/source/development/roadmap.rst +++ b/docs/sphinx/source/development/roadmap.rst @@ -14,12 +14,15 @@ Roadmap .. role:: text-danger +.. role:: text-warning + Python API ---------- - |check_| Implementation of parsing process (unpack, pack) - |check_| Struct class (:class:`Struct`) with wrapper function (:code:`@struct`) - +- |uncheck_| Python docs and examples +- |uncheck_| Python tests C API ----- @@ -32,18 +35,46 @@ C API Atom Objects: ^^^^^^^^^^^^^ -- |check_| Integer (uint8-128 and int8-128) (C type: :c:type:`CpIntAtomObject`, Py type: :class:`int_t`) [:text-danger:`missing docs`] -- |check_| Float, Double (C type: :c:type:`CpFloatAtomObject`, Py type: :class:`float_t`) [:text-danger:`missing docs`] -- |check_| Boolean (C type: :c:type:`CpBoolAtomObject`, Py type: :class:`bool_t`), global instance: :code:`boolean` [:text-danger:`missing docs`] -- |check_| Char (C type: :c:type:`CpCharAtomObject`, Py type: :class:`char_t`), global instance: :code:`char` [:text-danger:`missing docs`] -- |check_| Padding (C type: :c:type:`CpPaddingAtomObject`, Py type: :class:`padding_t`), global instance: :code:`padding` [:text-danger:`missing docs`] -- |check_| String (C type: :c:type:`CpStringAtomObject`, Py type: :class:`string`) [:text-danger:`missing docs`] -- |uncheck_| Const (C type: :c:type:`CpConstAtomObject`, Py type: :class:`const_t`) [:text-danger:`missing docs`] +- |check_| Integer (uint8-128 and int8-128) (C type: :c:type:`CpIntAtomObject`, Py type: :class:`int_t`) + [:text-danger:`missing docs`] + +- |check_| Float, Double (C type: :c:type:`CpFloatAtomObject`, Py type: :class:`float_t`) + [:text-danger:`missing docs`] + +- |check_| Boolean (C type: :c:type:`CpBoolAtomObject`, Py type: :class:`bool_t`) + Global instance: :code:`boolean`, + [:text-danger:`missing docs`] + +- |check_| Char (C type: :c:type:`CpCharAtomObject`, Py type: :class:`char_t`) + Global instance: :code:`char`, + [:text-danger:`missing docs`] + +- |check_| Padding (C type: :c:type:`CpPaddingAtomObject`, Py type: :class:`padding_t`) + Global instance: :code:`padding`, + [:text-danger:`missing docs`] + +- |check_| String (C type: :c:type:`CpStringAtomObject`, Py type: :class:`string`) + [:text-danger:`missing docs`], + [:text-warning:`missing perftest`] + +- |check_| Const (C type: :c:type:`CpConstAtomObject`, Py type: :class:`const_t`) + [:text-danger:`missing docs`], + [:text-warning:`missing perftest`] + - |uncheck_| CString -- |uncheck_| Bytes (C type: :c:type:`CpBytesAtomObject`, Py type: :class:`octetstring`) -- |uncheck_| Enum +- |check_| Bytes (C type: :c:type:`CpBytesAtomObject`, Py type: :class:`octetstring`) + [:text-danger:`missing docs`], + [:text-warning:`missing perftest`] + +- |check_| Enum (C type: :c:type:`CpEnumAtomObject`, Py type: :class:`enumeration`) + [:text-danger:`missing docs`], + [:text-warning:`missing perftest`] + - |uncheck_| Computed -- |uncheck_| PString +- |check_| PString (C type: :c:type:`CpPStringAtomObject`, Py type: :class:`pstring`) + [:text-danger:`missing docs`], + [:text-warning:`missing perftest`] + - |uncheck_| Prefixed - |uncheck_| Lazy - |uncheck_| uuid diff --git a/src/atomimpl/constatomobj.c b/src/atomimpl/constatomobj.c index 6bb2446..27c476b 100644 --- a/src/atomimpl/constatomobj.c +++ b/src/atomimpl/constatomobj.c @@ -112,7 +112,7 @@ PyTypeObject CpConstAtom_Type = { .tp_basicsize = sizeof(CpConstAtomObject), .tp_dealloc = (destructor)cp_constatom_dealloc, .tp_init = (initproc)cp_constatom_init, - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_new = (newfunc)cp_constatom_new, .tp_repr = (reprfunc)cp_constatom_repr, .tp_members = cp_constatom_members, diff --git a/src/atomimpl/enumatomobj.c b/src/atomimpl/enumatomobj.c new file mode 100644 index 0000000..266778e --- /dev/null +++ b/src/atomimpl/enumatomobj.c @@ -0,0 +1,187 @@ +/* enum atom C implementation */ + +#include "caterpillar/caterpillar.h" + +#include + +/* impl */ +static PyObject* +cp_enumatom_type(CpEnumAtomObject* self) +{ + if (self->m_enum_type) { + return Py_NewRef(self->m_enum_type); + } + return CpTypeOf(self->m_atom); +} + +static PyObject* +cp_enumatom_size(CpEnumAtomObject* self, CpLayerObject* layer) +{ + return _Cp_SizeOf(self->m_atom, layer); +} + +static PyObject* +cp_enumatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) +{ + CpEnumAtomObject* atom = (CpEnumAtomObject*)type->tp_alloc(type, 0); + if (atom != NULL) { + CpBuiltinAtom_CATOM(atom).ob_pack = (packfunc)CpEnumAtom_Pack; + CpBuiltinAtom_CATOM(atom).ob_unpack = (unpackfunc)CpEnumAtom_Unpack; + CpBuiltinAtom_CATOM(atom).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(atom).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(atom).ob_size = (sizefunc)cp_enumatom_size; + CpBuiltinAtom_CATOM(atom).ob_type = (typefunc)cp_enumatom_type; + CpBuiltinAtom_CATOM(atom).ob_bits = NULL; + + atom->m_atom = NULL; + atom->m_enum_type = NULL; + atom->m_members = NULL; + atom->m_value2member_map = NULL; + atom->m_default = Py_NewRef(CpInvalidDefault); + } + return (PyObject*)atom; +} + +static void +cp_enumatom_dealloc(CpEnumAtomObject* self) +{ + Py_XDECREF(self->m_atom); + Py_XDECREF(self->m_members); + Py_XDECREF(self->m_enum_type); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_enumatom_init(CpEnumAtomObject* self, PyObject* args, PyObject* kwds) +{ + static char* kwlist[] = { "atom", "enum_type", "default", NULL }; + PyObject *atom = NULL, *enum_type = NULL, *default_value = NULL; + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "OO|O", kwlist, &atom, &enum_type, &default_value)) { + return -1; + } + _Cp_SetObj(self->m_atom, atom); + _Cp_SetObj(self->m_enum_type, enum_type); + _Cp_SetObj(self->m_default, default_value); + + _modulestate* mod = get_global_module_state(); + PyObject* members = + PyObject_GetAttr(self->m_enum_type, mod->str__member_map_); + if (!members) { + return -1; + } + + _Cp_SetObj(self->m_members, members); + PyObject* value2member_map = + PyObject_GetAttr(self->m_enum_type, mod->str__value2member_map_); + if (!value2member_map) { + return -1; + } + _Cp_SetObj(self->m_value2member_map, value2member_map); + + return 0; +} + +_CpEndian_ImplSetByteorder(CpEnumAtomObject, enumatom, self->m_atom); + +static PyObject* +cp_enumatom_repr(CpEnumAtomObject* self) +{ + return PyUnicode_FromFormat( + "] %R>", ((PyTypeObject *)self->m_enum_type)->tp_name, self->m_atom); +} + +/* Public API */ + +/*CpAPI*/ +int +CpEnumAtom_Pack(CpEnumAtomObject* self, PyObject* value, CpLayerObject* layer) +{ + // TODO: check enum type - add strict flag + // if (!PyObject_TypeCheck(value, (PyTypeObject*)self->m_enum_type)) { + // PyErr_Format(PyExc_TypeError, + // "Expected a %s, got %s", + // Py_TYPE(self->m_enum_type)->tp_name, + // Py_TYPE(value)->tp_name); + // return -1; + // } + int res = 0; + if (PyObject_TypeCheck(value, (PyTypeObject*)self->m_enum_type)) { + PyObject* real_value = PyObject_GetAttrString(value, "value"); + if (!real_value) { + return -1; + } + res = _Cp_Pack(real_value, self->m_atom, layer); + Py_DECREF(real_value); + } else { + res = _Cp_Pack(value, self->m_atom, layer); + } + + return res; +} + +/*CpAPI*/ +PyObject* +CpEnumAtom_Unpack(CpEnumAtomObject* self, CpLayerObject* layer) +{ + PyObject* value = _Cp_Unpack(self->m_atom, layer); + if (!value) { + return NULL; + } + + PyObject* by_name = PyDict_GetItem(self->m_members, value); + if (by_name) { + Py_DECREF(value); + return Py_NewRef(by_name); + } + // not found + PyErr_Clear(); + + PyObject* by_value = PyDict_GetItem(self->m_value2member_map, value); + if (by_value) { + Py_DECREF(value); + return Py_NewRef(by_value); + } + + // not found, fallback to default value + PyErr_Clear(); + + if (Cp_IsInvalidDefault(self->m_default)) { + // TODO: add strict parsing flag here + return value; + } + Py_DECREF(value); + return Py_NewRef(self->m_default); +} + +/* docs */ + +/* type setup */ + +static PyMemberDef CpEnumAtom_Members[] = { + { "atom", T_OBJECT, offsetof(CpEnumAtomObject, m_atom), READONLY }, + { "enum_type", T_OBJECT, offsetof(CpEnumAtomObject, m_enum_type), READONLY }, + { "members", T_OBJECT, offsetof(CpEnumAtomObject, m_members), READONLY }, + { "default", T_OBJECT, offsetof(CpEnumAtomObject, m_default), 0 }, + { NULL } +}; + +static PyMethodDef CpEnumAtom_Methods[] = { + _CpEndian_ImplSetByteorder_MethDef(enumatom, NULL), + { + NULL, + } +}; + +PyTypeObject CpEnumAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpEnumAtom_NAME), + .tp_basicsize = sizeof(CpEnumAtomObject), + .tp_dealloc = (destructor)cp_enumatom_dealloc, + .tp_init = (initproc)cp_enumatom_init, + .tp_repr = (reprfunc)cp_enumatom_repr, + .tp_members = CpEnumAtom_Members, + .tp_methods = CpEnumAtom_Methods, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = NULL, + .tp_new = (newfunc)cp_enumatom_new, +}; diff --git a/src/capi.dat b/src/capi.dat index 9ea3c48..d703690 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -44,6 +44,7 @@ src:atomimpl/stringatomobj.c src:atomimpl/constatomobj.c src:atomimpl/bytesatomobj.c src:atomimpl/pstringatomobj.c +src:atomimpl/enumatomobj.c src:atomimpl/builtins/builtinatomobj.c src:atomimpl/builtins/repeatedatomobj.c src:atomimpl/builtins/conditionatomobj.c @@ -98,6 +99,7 @@ type:35:_primitiveatomobj:CpPrimitiveAtomObject:- type:36:_lengthinfoobj:CpLengthInfoObject:- type:37:_bytesatomobj:CpBytesAtomObject:- type:38:_pstringatomobj:CpPStringAtomObject:- +type:39:_enumatomobj:CpEnumAtomObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 @@ -199,4 +201,7 @@ func:149:CpBytesAtom_Pack:int:null func:150:CpBytesAtom_Unpack:PyObject*:+1 func:-:CpPStringAtom_New:CpPStringAtomObject*:+1 func:152:CpPStringAtom_Pack:int:null -func:153:CpPStringAtom_Unpack:PyObject*:+1 \ No newline at end of file +func:153:CpPStringAtom_Unpack:PyObject*:+1 +func:-:CpEnumAtom_New:CpEnumAtomObject*:+1 +func:154:CpEnumAtom_Pack:int:null +func:155:CpEnumAtom_Unpack:PyObject*:+1 \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/enum.h b/src/caterpillar/include/caterpillar/atoms/enum.h new file mode 100644 index 0000000..9458730 --- /dev/null +++ b/src/caterpillar/include/caterpillar/atoms/enum.h @@ -0,0 +1,51 @@ +/** + * Copyright (C) MatrixEditor 2024 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef ENUMATOMOBJ_H +#define ENUMATOMOBJ_H + +#include "caterpillar/atoms/builtins.h" + +struct _enumatomobj +{ + CpBuiltinAtom_HEAD + + /// Stores the mapping of the underlying enum type. + PyObject* m_members; + + PyObject* m_value2member_map; + PyObject* m_default; + + /// Reference to the atom that is responsible for + /// parsing the enum value + PyObject* m_atom; + + /// Reference to the enum type + PyObject* m_enum_type; +}; + +#define CpEnumAtom_NAME "enumeration" +#define CpEnumAtom_CheckExact(op) Py_IS_TYPE((op), &CpEnumAtom_Type) +#define CpEnumAtom_Check(op) (PyObject_TypeCheck((op), &CpEnumAtom_Type)) + +static inline CpEnumAtomObject* +CpEnumAtom_New(PyObject* members, PyObject* atom) +{ + return (CpEnumAtomObject*)CpObject_Create( + &CpEnumAtom_Type, "OO", members, atom); +} + +#endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillar.h b/src/caterpillar/include/caterpillar/caterpillar.h index b965070..a2a611f 100644 --- a/src/caterpillar/include/caterpillar/caterpillar.h +++ b/src/caterpillar/include/caterpillar/caterpillar.h @@ -22,5 +22,6 @@ #include "caterpillar/atoms/primitive.h" #include "caterpillar/atoms/string.h" #include "caterpillar/atoms/const.h" +#include "caterpillar/atoms/enum.h" #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index 3f15b67..a04dfe7 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -96,6 +96,8 @@ struct _bytesatomobj; typedef struct _bytesatomobj CpBytesAtomObject; struct _pstringatomobj; typedef struct _pstringatomobj CpPStringAtomObject; +struct _enumatomobj; +typedef struct _enumatomobj CpEnumAtomObject; #ifdef _CPMODULE @@ -143,6 +145,7 @@ extern PyTypeObject CpPrimitiveAtom_Type; extern PyTypeObject CpLengthInfo_Type; extern PyTypeObject CpBytesAtom_Type; extern PyTypeObject CpPStringAtom_Type; +extern PyTypeObject CpEnumAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -229,6 +232,8 @@ int CpBytesAtom_Pack(CpBytesAtomObject* self, PyObject* value, CpLayerObject* la PyObject* CpBytesAtom_Unpack(CpBytesAtomObject* self, CpLayerObject* layer); int CpPStringAtom_Pack(CpPStringAtomObject* self,PyObject* value,CpLayerObject* layer); PyObject* CpPStringAtom_Unpack(CpPStringAtomObject* self, CpLayerObject* layer); +int CpEnumAtom_Pack(CpEnumAtomObject* self, PyObject* value, CpLayerObject* layer); +PyObject* CpEnumAtom_Unpack(CpEnumAtomObject* self, CpLayerObject* layer); #else @@ -277,6 +282,7 @@ caterpillar_api.py #define CpLengthInfo_Type (*(PyTypeObject *)Cp_API[36]) #define CpBytesAtom_Type (*(PyTypeObject *)Cp_API[37]) #define CpPStringAtom_Type (*(PyTypeObject *)Cp_API[38]) +#define CpEnumAtom_Type (*(PyTypeObject *)Cp_API[39]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -363,6 +369,8 @@ caterpillar_api.py #define CpBytesAtom_Unpack (*((PyObject* (*)(CpBytesAtomObject* self, CpLayerObject* layer)))Cp_API[150]) #define CpPStringAtom_Pack (*((int (*)(CpPStringAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[152]) #define CpPStringAtom_Unpack (*((PyObject* (*)(CpPStringAtomObject* self, CpLayerObject* layer)))Cp_API[153]) +#define CpEnumAtom_Pack (*((int (*)(CpEnumAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[154]) +#define CpEnumAtom_Unpack (*((PyObject* (*)(CpEnumAtomObject* self, CpLayerObject* layer)))Cp_API[155]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillar/include/caterpillar/module.h b/src/caterpillar/include/caterpillar/module.h index 66ee389..0912d68 100644 --- a/src/caterpillar/include/caterpillar/module.h +++ b/src/caterpillar/include/caterpillar/module.h @@ -54,7 +54,7 @@ struct _modulestate PyObject* List_Type; PyObject* Optional_Type; PyObject* Union_Type; - PyObject *BytesIO_Type; + PyObject* BytesIO_Type; // string constants // strings @@ -79,6 +79,8 @@ struct _modulestate PyObject* str___weakref__; PyObject* str___dict__; PyObject* str___qualname__; + PyObject* str__member_map_; + PyObject* str__value2member_map_; PyObject* str_start; PyObject* str_ctx__root; @@ -92,11 +94,10 @@ struct _modulestate PyObject* inspect_getannotations; // cached objects - PyObject *cp_bytes__true; - PyObject *cp_bytes__false; + PyObject* cp_bytes__true; + PyObject* cp_bytes__false; }; - /** * @brief Get the module state object * diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index b7dbde7..cdf7294 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -42,7 +42,7 @@ void *Cp_API[] = { (void *) &CpLengthInfo_Type, (void *) &CpBytesAtom_Type, (void *) &CpPStringAtom_Type, - NULL, + (void *) &CpEnumAtom_Type, NULL, NULL, NULL, @@ -156,6 +156,8 @@ void *Cp_API[] = { (void *) &CpBytesAtom_Unpack, NULL, (void *) &CpPStringAtom_Pack, - (void *) &CpPStringAtom_Unpack + (void *) &CpPStringAtom_Unpack, + (void *) &CpEnumAtom_Pack, + (void *) &CpEnumAtom_Unpack }; diff --git a/src/module.c b/src/module.c index eafd6fa..356192e 100644 --- a/src/module.c +++ b/src/module.c @@ -276,7 +276,7 @@ cp_module_clear(PyObject* m) Py_CLEAR(state->Union_Type); Py_CLEAR(state->Optional_Type); - // sttings + // strings Py_CLEAR(state->str_path_delim); Py_CLEAR(state->str___pack__); Py_CLEAR(state->str___unpack__); @@ -304,6 +304,8 @@ cp_module_clear(PyObject* m) Py_CLEAR(state->str___weakref__); Py_CLEAR(state->str___qualname__); Py_CLEAR(state->str_strict); + Py_CLEAR(state->str__value2member_map_); + Py_CLEAR(state->str__member_map_); Py_CLEAR(state->cp_regex__unnamed); @@ -408,6 +410,7 @@ PyInit__C(void) CpConstAtom_Type.tp_base = &CpBuiltinAtom_Type; CpBytesAtom_Type.tp_base = &CpBuiltinAtom_Type; CpPStringAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpEnumAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpBuiltinAtom_Type); CpModule_SetupType(&CpPrimitiveAtom_Type); @@ -425,6 +428,7 @@ PyInit__C(void) CpModule_SetupType(&CpConstAtom_Type); CpModule_SetupType(&CpBytesAtom_Type); CpModule_SetupType(&CpPStringAtom_Type); + CpModule_SetupType(&CpEnumAtom_Type); // module setup m = PyModule_Create(&CpModule); @@ -472,6 +476,7 @@ PyInit__C(void) CpModule_AddObject(CpConstAtom_NAME, &CpConstAtom_Type); CpModule_AddObject(CpBytesAtom_NAME, &CpBytesAtom_Type); CpModule_AddObject(CpPStringAtom_NAME, &CpPStringAtom_Type); + CpModule_AddObject(CpEnumAtom_NAME, &CpEnumAtom_Type); /* setup custom intatoms */ #define CpModule_DefAtom(name, ...) \ @@ -603,6 +608,8 @@ PyInit__C(void) CACHED_STRING(str___qualname__, "__qualname__"); CACHED_STRING(str_path_delim, "."); CACHED_STRING(str_strict, "strict"); + CACHED_STRING(str__member_map_, "_member_map_"); + CACHED_STRING(str__value2member_map_, "_value2member_map_"); #undef CACHED_STRING diff --git a/test/_C/atoms/test_enum.py b/test/_C/atoms/test_enum.py new file mode 100644 index 0000000..5af8f98 --- /dev/null +++ b/test/_C/atoms/test_enum.py @@ -0,0 +1,26 @@ +# pylint: disable=unused-import,no-name-in-module,import-error +import pytest +import caterpillar +import enum + +if caterpillar.native_support(): + + from caterpillar._C import enumeration, pack, unpack, u8 + + def test_enum(): + # TODO: The Struct class should infer the enum + # type automatically. + class TestEnum(enum.Enum): + A = 1 + B = 2 + C = 3 + + # We can use an enumeration atom to parse data and translate + # it to an enum value. By default, invalid values won't get + # reported (TODO: strict parsing flag) + enum_atom = enumeration(u8, TestEnum) + assert unpack(b"\x01", enum_atom) == TestEnum.A + assert pack(TestEnum.B, enum_atom) == b"\x02" + + # NOTE: byteorder operations will be delegated to the underlying + # atom. \ No newline at end of file From d5312752308117a0894dec0058025621edafd9e5 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 8 Sep 2024 22:09:26 +0200 Subject: [PATCH 13/29] Change version to 2.2.0-rc --- pyproject.toml | 2 +- src/caterpillar/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 97cde92..20ee5a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ wheel.py-api = "cp312" [project] name = "caterpillar" -version = "2.1.5" +version = "2.2.0-rc" description="Library to pack and unpack structurized binary data." authors = [ diff --git a/src/caterpillar/__init__.py b/src/caterpillar/__init__.py index df1f0fe..f67352c 100644 --- a/src/caterpillar/__init__.py +++ b/src/caterpillar/__init__.py @@ -13,7 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -__version__ = "2.1.5" +__version__ = "2.2.0-rc" __release__ = None __author__ = "MatrixEditor" From 64802f3d36fd2c65a15f1adb90ca18707159150d Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Mon, 9 Sep 2024 22:47:04 +0200 Subject: [PATCH 14/29] C varint implementation --- + CpVarIntAtom for both little endian and bigendian + Added C installation candidate to subpackage + Tests included + New C API functions: CpVarIntAtom_New, CpVarIntAtom_Pack, CpVarIntAtom_Unpack, CpVarIntAtom_BSwap, CpVarIntAtom_BSwapUnsignedLongLong, CpVarIntAtom_BSwapLongLong, CpVarIntAtom_BSwapSsize_t + Changed CpEndian and CpArch representation --- CMakeLists.txt | 15 +- pyproject.toml | 1 + src/arch.c | 20 +- src/atomimpl/intatomobj.c | 2 +- src/atomimpl/varintatomobj.c | 343 ++++++++++++++++++ src/capi.dat | 11 +- .../include/caterpillar/atoms/float.h | 2 +- .../include/caterpillar/atoms/int.h | 30 +- .../include/caterpillar/atoms/primitive.h | 8 +- .../include/caterpillar/caterpillarapi.h | 16 + src/caterpillarapi.c | 10 +- src/ccaterpillar/pyproject.toml | 50 +++ src/module.c | 5 + test/_C/atoms/test_varint.py | 41 +++ 14 files changed, 534 insertions(+), 20 deletions(-) create mode 100644 src/atomimpl/varintatomobj.c create mode 100644 src/ccaterpillar/pyproject.toml create mode 100644 test/_C/atoms/test_varint.py diff --git a/CMakeLists.txt b/CMakeLists.txt index aeb8a64..40ebe0d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,8 +8,10 @@ find_package( add_compile_definitions(_CPMODULE) +# TODO: document this change +set(CP_C_CONFIGURED DEFINED ENV{CP_ENABLE_NATIVE} OR DEFINED CP_ENABLE_NATIVE) -if (DEFINED ENV{CP_ENABLE_NATIVE}) +if (CP_C_CONFIGURED) python_add_library( _C MODULE @@ -40,6 +42,7 @@ python_add_library( src/atomimpl/bytesatomobj.c src/atomimpl/pstringatomobj.c src/atomimpl/enumatomobj.c + src/atomimpl/varintatomobj.c src/atomimpl/builtins/builtinatomobj.c src/atomimpl/builtins/repeatedatomobj.c @@ -56,12 +59,18 @@ target_include_directories(_C PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpilla message(STATUS "CMake ${CMAKE_VERSION}") message(STATUS "Python ${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}") +set (CP_GENAPI_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/src/code_gen/genapi.py) +set (CP_CAPI_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h.in) +set (CP_CAPI_CSRC ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillarapi.c.in) +set (CP_CAPI_HEADER_OUT ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h) +set (CP_CAPI_CSRC_OUT ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillarapi.c) + add_custom_target( genapi ALL # execute python ./src/code_gen/genapi.py ./src/caterpillar/include/caterpillar/caterpillarapi.h.in ./src/caterpillarapi.c.in # in the root directory - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/src/code_gen/genapi.py ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillarapi.c.in - BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillarapi.c + COMMAND ${PYTHON_EXECUTABLE} ${CP_GENAPI_SCRIPT} ${CP_CAPI_HEADER} ${CP_CAPI_CSRC} + BYPRODUCTS ${CP_CAPI_HEADER_OUT} ${CP_CAPI_CSRC_OUT} COMMENT "Generating Public Caterpillar API" ) diff --git a/pyproject.toml b/pyproject.toml index 20ee5a6..2bd591b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ build-backend = "scikit_build_core.build" build-dir = "build/{wheel_tag}" sdist.include = ["*.pyi", "*.h"] wheel.py-api = "cp312" +cmake.source-dir = "." [project] name = "caterpillar" diff --git a/src/arch.c b/src/arch.c index 823058c..2ca84b8 100644 --- a/src/arch.c +++ b/src/arch.c @@ -1,8 +1,7 @@ /* CpArch and CpEndian */ -#include "caterpillar/arch.h" #include "caterpillar/caterpillar.h" -#include "caterpillar/field.h" -#include "structmember.h" + +#include /* CpArch */ static PyObject* @@ -54,8 +53,7 @@ cp_arch_init(CpArchObject* self, PyObject* args, PyObject* kw) static PyObject* cp_arch_repr(CpArchObject* self) { - return PyUnicode_FromFormat( - "CpArch(name=%R, ptr_size=%i)", self->name, self->pointer_size); + return PyUnicode_FromFormat("", self->pointer_size, self->name); } static PyObject* @@ -174,8 +172,16 @@ cp_endian_init(CpEndianObject* self, PyObject* args, PyObject* kw) static PyObject* cp_endian_repr(CpEndianObject* self) { - return PyUnicode_FromFormat( - "CpEndian(name=%R, ch='%c')", self->name, self->id); + switch (self->id) { + case '=': + return PyUnicode_FromFormat(""); + case '<': + return PyUnicode_FromFormat(""); + case '>': + return PyUnicode_FromFormat(""); + default: + return PyUnicode_FromFormat("", self->id, self->name); + } } static PyObject* diff --git a/src/atomimpl/intatomobj.c b/src/atomimpl/intatomobj.c index 3ec90f6..839310a 100644 --- a/src/atomimpl/intatomobj.c +++ b/src/atomimpl/intatomobj.c @@ -192,4 +192,4 @@ PyTypeObject CpIntAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_new = (newfunc)cp_intatom_new, .tp_init = (initproc)cp_intatom_init, .tp_repr = (reprfunc)cp_intatom_repr, - .tp_methods = CpIntAtom_Methods }; \ No newline at end of file + .tp_methods = CpIntAtom_Methods }; diff --git a/src/atomimpl/varintatomobj.c b/src/atomimpl/varintatomobj.c new file mode 100644 index 0000000..bdfaf0b --- /dev/null +++ b/src/atomimpl/varintatomobj.c @@ -0,0 +1,343 @@ +/* varint implementation for Little Endian and Big Endian */ + +#include "caterpillar/caterpillar.h" + +#include + +// Inspired by https://github.com/google/cityhash/blob/master/src/city.cc +#ifdef _MSC_VER +#include +#define CP_BSWAP_U32(x) _byteswap_ulong(x)(x) +#define CP_BSWAP_U64(x) _byteswap_uint64(x) + +#elif defined(__APPLE__) +#include +#define CP_BSWAP_U32(x) OSSwapInt32(x) +#define CP_BSWAP_U64(x) OSSwapInt64(x) + +#elif defined(sun) || defined(__sun) +#include +#define CP_BSWAP_U32(x) BSWAP_32(x) +#define CP_BSWAP_U64(x) BSWAP_64(x) + +#elif defined(__FreeBSD__) +#include +#define CP_BSWAP_U32(x) bswap32(x) +#define CP_BSWAP_U64(x) bswap64(x) + +#elif defined(__OpenBSD__) +#include +#define CP_BSWAP_U32(x) swap32(x) +#define CP_BSWAP_U64(x) swap64(x) + +#else +#include +#define CP_BSWAP_U32(x) bswap_32(x) +#define CP_BSWAP_U64(x) bswap_64(x) +#endif + +/* impl */ +static PyObject* +cp_varintatom_type(PyObject* self) +{ + return Py_XNewRef(&PyLong_Type); +} + +static PyObject* +cp_varintatom_size(CpVarIntAtomObject* self, PyObject* ctx) +{ + PyErr_SetString(PyExc_TypeError, "VarInts do not have a static size!"); + return NULL; +} + +static PyObject* +cp_varintatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) +{ + CpVarIntAtomObject* self = (CpVarIntAtomObject*)type->tp_alloc(type, 0); + if (self != NULL) { + self->_m_little_endian = true; + self->_m_lsb = false; + + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpVarIntAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpVarIntAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_varintatom_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_varintatom_type; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + } + return (PyObject*)self; +} + +static void +cp_varintatom_dealloc(CpVarIntAtomObject* self) +{ + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_varintatom_init(CpVarIntAtomObject* self, PyObject* args, PyObject* kwds) +{ + static char* kwlist[] = { "little_endian", "lsb", NULL }; + int little_endian = true; + int lsb = false; + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "|pp", kwlist, &little_endian, &lsb)) { + return -1; + } + self->_m_little_endian = little_endian; + self->_m_lsb = lsb; + return 0; +} + +static PyObject* +cp_varintatom_repr(CpVarIntAtomObject* self) +{ + char endian = 'l'; + if (!self->_m_little_endian) { + endian = 'b'; + } + if (self->_m_lsb) { + return PyUnicode_FromFormat("<%ce varint [lsb]>", endian); + } + return PyUnicode_FromFormat("<%ce varint>", endian); +} + +static PyObject* +cp_varintatom_set_byteorder(CpVarIntAtomObject* self, + PyObject* args, + PyObject* kw) +{ + _CpEndian_KwArgsGetByteorder(NULL); + + _modulestate* mod = get_global_module_state(); + int little_endian = CpEndian_IsLittleEndian((CpEndianObject*)byteorder, mod); + PyObject* new_varint = + (PyObject*)CpVarIntAtom_New(little_endian, self->_m_lsb); + + Py_DECREF(byteorder); + return new_varint; +} + +/* Public API */ + +#define _CP_VARINT_BSWAP_IMPL(name, type) \ + PyObject* result_int = PyLong_From##name(number); \ + if (!result_int) { \ + return 0; \ + } \ + PyObject* result = CpVarIntAtom_BSwap(result_int, little_endian); \ + Py_DECREF(result_int); \ + if (!result) { \ + return 0; \ + } \ + type res = PyLong_As##name(result); \ + Py_DECREF(result); \ + return res; + +/*CpAPI*/ +unsigned long long +CpVarIntAtom_BSwapUnsignedLongLong(unsigned long long number, + bool little_endian) +{ + _CP_VARINT_BSWAP_IMPL(UnsignedLongLong, unsigned long long); +} + +/*CpAPI*/ +long long +CpVarIntAtom_BSwapLongLong(long long number, bool little_endian) +{ + _CP_VARINT_BSWAP_IMPL(LongLong, long long); +} + +/*CpAPI*/ +Py_ssize_t +CpVarIntAtom_BSwapSsize_t(Py_ssize_t number, bool little_endian) +{ + _CP_VARINT_BSWAP_IMPL(Ssize_t, Py_ssize_t); +} + +#undef _CP_VARINT_BSWAP_IMPL + +/*CpAPI*/ +PyObject* +CpVarIntAtom_BSwap(PyObject* number, bool little_endian) +{ + size_t nbytes = _PyLong_NumBits(number) / 8; + PyObject* bytesSize = PyLong_FromSize_t(nbytes); + if (!bytesSize) { + return NULL; + } + + PyObject* bytes = CpObject_CreateOneArg(&PyBytes_Type, bytesSize); + if (!bytes) { + Py_DECREF(bytesSize); + return NULL; + } + + int res = _PyLong_AsByteArray((PyLongObject*)number, + (unsigned char*)PyBytes_AS_STRING(bytes), + nbytes, + little_endian, + false); + if (res == -1) { + Py_DECREF(bytesSize); + Py_DECREF(bytes); + return NULL; + } + + PyObject* result = _PyLong_FromByteArray( + (unsigned char*)PyBytes_AS_STRING(bytes), nbytes, !little_endian, false); + + Py_DECREF(bytesSize); + Py_DECREF(bytes); + return result; +} + +PyObject* +_CpVarIntAtom_Length(unsigned long long number) +{ + if (number == 0) { + return PyLong_FromLong(1); + } + int length = 1; + while ((number >> 7) != 0) { + number >>= 7; + length++; + } + return PyLong_FromLong(length); +} + +/*CpAPI*/ +int +CpVarIntAtom_Pack(CpVarIntAtomObject* self, + PyObject* value, + CpLayerObject* layer) +{ + unsigned long long raw_value = PyLong_AsUnsignedLongLongMask(value); + if (PyErr_Occurred()) { + return -1; + } + + int little_endian = self->_m_little_endian; + int lsb = self->_m_lsb; + + PyObject* length = _CpVarIntAtom_Length(raw_value); + if (!length) { + return -1; + } + + PyObject* bytes = CpObject_CreateOneArg(&PyBytes_Type, length); + Py_DECREF(length); + if (!bytes) { + return -1; + } + + int raw_length = PyBytes_GET_SIZE(bytes); + for (int i = 0; i < raw_length; i++) { + unsigned char byte = 0; + if (little_endian) { + byte = (raw_value >> (i * 7)) & 0x7f; + } else { + byte = (raw_value >> ((raw_length - 1 - i) * 7)) & 0x7f; + } + if (!lsb && i != raw_length - 1) { + byte |= 0x80; + } else if (lsb && (i == raw_length - 1)) { + byte |= 0x80; + } + + PyBytes_AS_STRING(bytes)[i] = byte; + } + + PyObject* result = CpState_Write(layer->m_state, bytes); + Py_DECREF(bytes); + if (!result) { + return -1; + } + Py_DECREF(result); + return 0; +} + +/*CpAPI*/ +PyObject* +CpVarIntAtom_Unpack(CpVarIntAtomObject* self, CpLayerObject* layer) +{ + PyObject* tmp_val = NULL; + unsigned long long result = 0, tmp = 0; + int shift = 0; + int little_endian = self->_m_little_endian; + int lsb = self->_m_lsb; + int nbytes = 0; + + while (true) { + tmp_val = CpState_ReadSsize_t(layer->m_state, 1); + if (!tmp_val) { + return NULL; + } + // only one byte + tmp = *(PyBytes_AS_STRING(tmp_val)); + Py_DECREF(tmp_val); + + result |= (tmp & 0x7f) << shift; + shift += 7; + nbytes++; + if ((lsb && (tmp & 0x80) == 1) || (!lsb && (tmp & 0x80) == 0)) { + break; + } + } + + if (!little_endian) { + if (nbytes == 1) { + // no swap needed + } else if (nbytes == 2) { + // manually swap + // TODO: explain why we use 7 bits here + result = ((result >> 7) & 0xff) | ((result << 7) & 0xff00); + } else if (nbytes <= 4) { + // swap using compiler directive + result = CP_BSWAP_U32(result); + } else if (nbytes <= 8) { + // compiler directive + result = CP_BSWAP_U64(result); + } else { + // use Python methods + result = CpVarIntAtom_BSwapUnsignedLongLong(result, true); + if (PyErr_Occurred()) { + return NULL; + } + } + } + + return PyLong_FromUnsignedLongLong(result); +} + +/* docs */ + +/* type setup */ +static PyMemberDef CpVarIntAtom_Members[] = { + { "little_endian", + T_BOOL, + offsetof(CpVarIntAtomObject, _m_little_endian), + READONLY }, + { "lsb", T_BOOL, offsetof(CpVarIntAtomObject, _m_lsb), READONLY }, + { NULL } +}; + +static PyMethodDef CpVarIntAtom_Methods[] = { + _CpEndian_ImplSetByteorder_MethDef(varintatom, NULL), + { NULL } /* Sentinel */ +}; + +PyTypeObject CpVarIntAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpVarIntAtom_NAME), + .tp_basicsize = sizeof(CpVarIntAtomObject), + .tp_dealloc = (destructor)cp_varintatom_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = NULL, + .tp_members = CpVarIntAtom_Members, + .tp_methods = CpVarIntAtom_Methods, + .tp_init = (initproc)cp_varintatom_init, + .tp_new = (newfunc)cp_varintatom_new, + .tp_repr = (reprfunc)cp_varintatom_repr, +}; diff --git a/src/capi.dat b/src/capi.dat index d703690..2b71f84 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -45,6 +45,7 @@ src:atomimpl/constatomobj.c src:atomimpl/bytesatomobj.c src:atomimpl/pstringatomobj.c src:atomimpl/enumatomobj.c +src:atomimpl/varintatomobj.c src:atomimpl/builtins/builtinatomobj.c src:atomimpl/builtins/repeatedatomobj.c src:atomimpl/builtins/conditionatomobj.c @@ -100,6 +101,7 @@ type:36:_lengthinfoobj:CpLengthInfoObject:- type:37:_bytesatomobj:CpBytesAtomObject:- type:38:_pstringatomobj:CpPStringAtomObject:- type:39:_enumatomobj:CpEnumAtomObject:- +type:40:_varintatomobj:CpVarIntAtomObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 @@ -204,4 +206,11 @@ func:152:CpPStringAtom_Pack:int:null func:153:CpPStringAtom_Unpack:PyObject*:+1 func:-:CpEnumAtom_New:CpEnumAtomObject*:+1 func:154:CpEnumAtom_Pack:int:null -func:155:CpEnumAtom_Unpack:PyObject*:+1 \ No newline at end of file +func:155:CpEnumAtom_Unpack:PyObject*:+1 +func:-:CpVarIntAtom_New:CpVarIntAtomObject*:+1 +func:156:CpVarIntAtom_Pack:int:null +func:157:CpVarIntAtom_Unpack:PyObject*:+1 +func:158:CpVarIntAtom_BSwap:PyObject*:+1 +func:159:CpVarIntAtom_BSwapUnsignedLongLong:ulonglong:null +func:160:CpVarIntAtom_BSwapLongLong:longlong:null +func:161:CpVarIntAtom_BSwapSsize_t:Py_ssize_t:null \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/float.h b/src/caterpillar/include/caterpillar/atoms/float.h index 758dc6d..752d253 100644 --- a/src/caterpillar/include/caterpillar/atoms/float.h +++ b/src/caterpillar/include/caterpillar/atoms/float.h @@ -39,6 +39,6 @@ struct _floatatomobj /** @brief Checks if the given object is an integer atom object */ #define CpFloatAtom_CheckExact(op) Py_IS_TYPE((op), &CpFloatAtom_Type) /** @brief Checks if the given object is an integer atom object */ -#define CpFloatAtom_Check(op) (PyObject_IsInstance((op), &CpFloatAtom_Type)) +#define CpFloatAtom_Check(op) (PyObject_TypeCheck((op), &CpFloatAtom_Type)) #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/int.h b/src/caterpillar/include/caterpillar/atoms/int.h index 02f741d..1c142b9 100644 --- a/src/caterpillar/include/caterpillar/atoms/int.h +++ b/src/caterpillar/include/caterpillar/atoms/int.h @@ -44,6 +44,34 @@ struct _intatomobj /** @brief Checks if the given object is an integer atom object */ #define CpIntAtom_CheckExact(op) Py_IS_TYPE((op), &CpIntAtom_Type) /** @brief Checks if the given object is an integer atom object */ -#define CpIntAtom_Check(op) (PyObject_IsInstance((op), &CpIntAtom_Type)) +#define CpIntAtom_Check(op) (PyObject_TypeCheck((op), &CpIntAtom_Type)) + +//------------------------------------------------------------------------------ +// varint atom + +struct _varintatomobj +{ + CpBuiltinAtom_HEAD + + /// Stores whether or not the integer is little endian + int _m_little_endian; + + /// Specifies that the last significant byte will use a ``1`` to identify + /// the end of the varint. Otherwise, zero will be used (which is the + /// default setting). + int _m_lsb; +}; + +#define CpVarIntAtom_NAME "varint_t" +#define CpVarIntAtom_CheckExact(op) Py_IS_TYPE((op), &CpVarIntAtom_Type) +#define CpVarIntAtom_Check(op) (PyObject_TypeCheck((op), &CpVarIntAtom_Type)) + +static inline CpVarIntAtomObject* +CpVarIntAtom_New(bool little_endian, bool lsb) +{ + return (CpVarIntAtomObject*)CpObject_Create( + &CpVarIntAtom_Type, "II", little_endian, lsb); +} + #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/primitive.h b/src/caterpillar/include/caterpillar/atoms/primitive.h index fb5eee0..1f75a05 100644 --- a/src/caterpillar/include/caterpillar/atoms/primitive.h +++ b/src/caterpillar/include/caterpillar/atoms/primitive.h @@ -28,7 +28,7 @@ struct _primitiveatomobj { #define CpPrimitiveAtom_NAME "patom" #define CpPrimitiveAtom_CheckExact(op) Py_IS_TYPE((op), &CpPrimitiveAtom_Type) -#define CpPrimitiveAtom_Check(op) (PyObject_IsInstance((op), &CpPrimitiveAtom_Type)) +#define CpPrimitiveAtom_Check(op) (PyObject_TypeCheck((op), &CpPrimitiveAtom_Type)) //------------------------------------------------------------------------------ // Bool @@ -72,7 +72,7 @@ struct _boolatomobj * @return True if the object is of type CpBoolAtom_Type or a subtype, * false otherwise. */ -#define CpBoolAtom_Check(op) (PyObject_IsInstance((op), &CpBoolAtom_Type)) +#define CpBoolAtom_Check(op) (PyObject_TypeCheck((op), &CpBoolAtom_Type)) //------------------------------------------------------------------------------ // Char Atom @@ -88,7 +88,7 @@ struct _charatomobj /** @brief Checks if the given object is a char atom object */ #define CpCharAtom_CheckExact(op) Py_IS_TYPE((op), &CpCharAtom_Type) /** @brief Checks if the given object is a char atom object */ -#define CpCharAtom_Check(op) (PyObject_IsInstance((op), &CpCharAtom_Type)) +#define CpCharAtom_Check(op) (PyObject_TypeCheck((op), &CpCharAtom_Type)) //------------------------------------------------------------------------------ // Padding @@ -106,6 +106,6 @@ struct _paddingatomobj /** @brief Checks if the given object is a padding atom object */ #define CpPaddingAtom_CheckExact(op) Py_IS_TYPE((op), &CpPaddingAtom_Type) /** @brief Checks if the given object is a padding atom object */ -#define CpPaddingAtom_Check(op) (PyObject_IsInstance((op), &CpPaddingAtom_Type)) +#define CpPaddingAtom_Check(op) (PyObject_TypeCheck((op), &CpPaddingAtom_Type)) #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index a04dfe7..b9f0879 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -98,6 +98,8 @@ struct _pstringatomobj; typedef struct _pstringatomobj CpPStringAtomObject; struct _enumatomobj; typedef struct _enumatomobj CpEnumAtomObject; +struct _varintatomobj; +typedef struct _varintatomobj CpVarIntAtomObject; #ifdef _CPMODULE @@ -146,6 +148,7 @@ extern PyTypeObject CpLengthInfo_Type; extern PyTypeObject CpBytesAtom_Type; extern PyTypeObject CpPStringAtom_Type; extern PyTypeObject CpEnumAtom_Type; +extern PyTypeObject CpVarIntAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -234,6 +237,12 @@ int CpPStringAtom_Pack(CpPStringAtomObject* self,PyObject* value,CpLayerObject* PyObject* CpPStringAtom_Unpack(CpPStringAtomObject* self, CpLayerObject* layer); int CpEnumAtom_Pack(CpEnumAtomObject* self, PyObject* value, CpLayerObject* layer); PyObject* CpEnumAtom_Unpack(CpEnumAtomObject* self, CpLayerObject* layer); +int CpVarIntAtom_Pack(CpVarIntAtomObject* self,PyObject* value,CpLayerObject* layer); +PyObject* CpVarIntAtom_Unpack(CpVarIntAtomObject* self, CpLayerObject* layer); +PyObject* CpVarIntAtom_BSwap(PyObject* number, bool little_endian); +unsigned long long CpVarIntAtom_BSwapUnsignedLongLong(unsigned long long number,bool little_endian); +long long CpVarIntAtom_BSwapLongLong(long long number, bool little_endian); +Py_ssize_t CpVarIntAtom_BSwapSsize_t(Py_ssize_t number, bool little_endian); #else @@ -283,6 +292,7 @@ caterpillar_api.py #define CpBytesAtom_Type (*(PyTypeObject *)Cp_API[37]) #define CpPStringAtom_Type (*(PyTypeObject *)Cp_API[38]) #define CpEnumAtom_Type (*(PyTypeObject *)Cp_API[39]) +#define CpVarIntAtom_Type (*(PyTypeObject *)Cp_API[40]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -371,6 +381,12 @@ caterpillar_api.py #define CpPStringAtom_Unpack (*((PyObject* (*)(CpPStringAtomObject* self, CpLayerObject* layer)))Cp_API[153]) #define CpEnumAtom_Pack (*((int (*)(CpEnumAtomObject* self, PyObject* value, CpLayerObject* layer)))Cp_API[154]) #define CpEnumAtom_Unpack (*((PyObject* (*)(CpEnumAtomObject* self, CpLayerObject* layer)))Cp_API[155]) +#define CpVarIntAtom_Pack (*((int (*)(CpVarIntAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[156]) +#define CpVarIntAtom_Unpack (*((PyObject* (*)(CpVarIntAtomObject* self, CpLayerObject* layer)))Cp_API[157]) +#define CpVarIntAtom_BSwap (*((PyObject* (*)(PyObject* number, bool little_endian)))Cp_API[158]) +#define CpVarIntAtom_BSwapUnsignedLongLong (*((unsigned long long (*)(unsigned long long number,bool little_endian)))Cp_API[159]) +#define CpVarIntAtom_BSwapLongLong (*((long long (*)(long long number, bool little_endian)))Cp_API[160]) +#define CpVarIntAtom_BSwapSsize_t (*((Py_ssize_t (*)(Py_ssize_t number, bool little_endian)))Cp_API[161]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index cdf7294..a9d27a0 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -43,7 +43,7 @@ void *Cp_API[] = { (void *) &CpBytesAtom_Type, (void *) &CpPStringAtom_Type, (void *) &CpEnumAtom_Type, - NULL, + (void *) &CpVarIntAtom_Type, NULL, NULL, NULL, @@ -158,6 +158,12 @@ void *Cp_API[] = { (void *) &CpPStringAtom_Pack, (void *) &CpPStringAtom_Unpack, (void *) &CpEnumAtom_Pack, - (void *) &CpEnumAtom_Unpack + (void *) &CpEnumAtom_Unpack, + (void *) &CpVarIntAtom_Pack, + (void *) &CpVarIntAtom_Unpack, + (void *) &CpVarIntAtom_BSwap, + (void *) &CpVarIntAtom_BSwapUnsignedLongLong, + (void *) &CpVarIntAtom_BSwapLongLong, + (void *) &CpVarIntAtom_BSwapSsize_t }; diff --git a/src/ccaterpillar/pyproject.toml b/src/ccaterpillar/pyproject.toml new file mode 100644 index 0000000..b0f1798 --- /dev/null +++ b/src/ccaterpillar/pyproject.toml @@ -0,0 +1,50 @@ +# Installation via pip: +# pip install "caterpillar[all]@git+https://github.com/MatrixEditor/caterpillar/#subdirectory=src/ccaterpillar" + +[build-system] +requires = ["scikit-build-core"] +build-backend = "scikit_build_core.build" + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +sdist.include = ["*.pyi", "*.h"] +wheel.py-api = "cp312" +cmake.source-dir = "../.." +wheel.packages = ["../../src/caterpillar"] + +[tool.scikit-build.cmake.define] +# We have to set this variable in order to enable the C API. +CP_ENABLE_NATIVE = "1" + +[project] +name = "caterpillar" +version = "2.2.0-rc" + +description="Library to pack and unpack structurized binary data." +authors = [ + { name="MatrixEditor", email="not@supported.com" }, +] +readme = "../../README.md" +classifiers = [ + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: MIT License', + + 'Programming Language :: Python :: 3.12', + ] + +[project.urls] +"Homepage" = "https://github.com/MatrixEditor/caterpillar" +"API-Docs" = "https://matrixeditor.github.io/caterpillar" + +[project.optional-dependencies] +# compression +lzo = [ + "lzallright" +] +crypt = [ + "cryptography" +] +all = [ + "lzallright", + "cryptography" +] \ No newline at end of file diff --git a/src/module.c b/src/module.c index 356192e..76b6b20 100644 --- a/src/module.c +++ b/src/module.c @@ -411,6 +411,7 @@ PyInit__C(void) CpBytesAtom_Type.tp_base = &CpBuiltinAtom_Type; CpPStringAtom_Type.tp_base = &CpBuiltinAtom_Type; CpEnumAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpVarIntAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpBuiltinAtom_Type); CpModule_SetupType(&CpPrimitiveAtom_Type); @@ -429,6 +430,7 @@ PyInit__C(void) CpModule_SetupType(&CpBytesAtom_Type); CpModule_SetupType(&CpPStringAtom_Type); CpModule_SetupType(&CpEnumAtom_Type); + CpModule_SetupType(&CpVarIntAtom_Type); // module setup m = PyModule_Create(&CpModule); @@ -477,6 +479,7 @@ PyInit__C(void) CpModule_AddObject(CpBytesAtom_NAME, &CpBytesAtom_Type); CpModule_AddObject(CpPStringAtom_NAME, &CpPStringAtom_Type); CpModule_AddObject(CpEnumAtom_NAME, &CpEnumAtom_Type); + CpModule_AddObject(CpVarIntAtom_NAME, &CpVarIntAtom_Type); /* setup custom intatoms */ #define CpModule_DefAtom(name, ...) \ @@ -518,6 +521,8 @@ PyInit__C(void) CpModule_AddObject("boolean", CpObject_CreateNoArgs(&CpBoolAtom_Type)); CpModule_AddObject("char", CpObject_CreateNoArgs(&CpCharAtom_Type)); CpModule_AddObject("padding", CpObject_CreateNoArgs(&CpPaddingAtom_Type)); + CpModule_AddObject("varint", CpVarIntAtom_New(true, false)); + CpModule_AddObject("lsbvarint", CpVarIntAtom_New(true, true)); /* setup state */ _modulestate* state = get_module_state(m); diff --git a/test/_C/atoms/test_varint.py b/test/_C/atoms/test_varint.py new file mode 100644 index 0000000..c23349e --- /dev/null +++ b/test/_C/atoms/test_varint.py @@ -0,0 +1,41 @@ +# pylint: disable=unused-import,no-name-in-module,import-error +import pytest +import caterpillar + + +if caterpillar.native_support(): + + from caterpillar._C import unpack, pack, varint, varint_t + from caterpillar._C import BIG_ENDIAN as be + + def test_varint_init(): + # The C varint atom implements a variable-length unsigned + # integer for big-endian AND little-endian encoding. The + # byteorder operator '+' can be applied as usual. + assert varint_t(True).little_endian is True + assert varint_t(False).little_endian is False + + # We can even control the least significant byte using + # the 'lsb' flag. + assert varint_t(lsb=True).lsb is True + assert varint_t(lsb=False).lsb is False + + # These parameters are virible through the representation: + # little-endian: + # big-endian: + # + # little-endian + lsb: + # big-endian + lsb: + + def test_varint_pack(): + value = 1024 + # be + varint should result in b'\x88\x00' + assert pack(value, be + varint) == b"\x88\x00" + # le + varint should result in b'\x80\x08' + assert pack(value, varint) == b'\x80\x08' + + def test_varint_unpack(): + value = 1024 + assert unpack(b"\x80\x08", varint) == value + assert unpack(b"\x88\x00", be + varint) == value + From 162b20841372fd4594aeb522f9e69817a7748442 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Mon, 9 Sep 2024 22:56:29 +0200 Subject: [PATCH 15/29] Removed CpContext_GetAttr* methods --- docs/sphinx/source/extensions/refcounts.dat | 3 --- .../source/reference/capi/objects/contextobj.rst | 11 ----------- src/caterpillar/include/caterpillar/context.h | 15 --------------- src/context.c | 9 --------- 4 files changed, 38 deletions(-) diff --git a/docs/sphinx/source/extensions/refcounts.dat b/docs/sphinx/source/extensions/refcounts.dat index d4bb573..ec563fe 100644 --- a/docs/sphinx/source/extensions/refcounts.dat +++ b/docs/sphinx/source/extensions/refcounts.dat @@ -10,9 +10,6 @@ CpStruct_GetAnnotations:PyObject*:+1 CpStructFieldInfo_New:CpStructFieldInfoObject*:+1 CpCharAtom_Unpack:PyObject*:+1 - -CpContext_GetAttr:PyObject*:+1 -CpContext_GetAttrString:PyObject*:+1 CpContext_New:CpContextObject*:+1 CpUnaryExpr_New:CpUnaryExprObject*:+1 diff --git a/docs/sphinx/source/reference/capi/objects/contextobj.rst b/docs/sphinx/source/reference/capi/objects/contextobj.rst index 20cc060..897b7d7 100644 --- a/docs/sphinx/source/reference/capi/objects/contextobj.rst +++ b/docs/sphinx/source/reference/capi/objects/contextobj.rst @@ -26,17 +26,6 @@ Context Objects Checks if the given object is instance of an :c:type:`CpContextObject`. -.. c:function:: PyObject* CpContext_GetAttr(PyObject* context, PyObject* key, _coremodulestate* state) - - Get an attribute from a context by using :code:`__context_getattr__`. Returns - *NULL* if the attribute is not found. - - -.. c:function:: PyObject *CpContext_GetAttrString(PyObject* context, const char* key) - - Get an attribute from a context by using :code:`__context_getattr__`. Returns - *NULL* if the attribute is not found. - .. c:function:: CpContextObject *CpContext_New(void) Creates a new context and returns it. Returns *NULL* if an error occurs. diff --git a/src/caterpillar/include/caterpillar/context.h b/src/caterpillar/include/caterpillar/context.h index 905ece0..e43019b 100644 --- a/src/caterpillar/include/caterpillar/context.h +++ b/src/caterpillar/include/caterpillar/context.h @@ -59,21 +59,6 @@ struct _contextobj */ #define CpContext_Check(op) PyObject_TypeCheck((op), &CpContext_Type) -/** - * @brief Get an attribute from a context by using __context_getattr__. - * - * @return the attribute, or NULL on failure - */ -PyAPI_FUNC(PyObject*) - CpContext_GetAttr(PyObject* ctx, PyObject* key, _modulestate* m); - -/** - * @brief Get an attribute from a context by using __context_getattr__. - * - * @return the attribute, or NULL on failure - */ -PyAPI_FUNC(PyObject*) CpContext_GetAttrString(PyObject* ctx, const char* key); - // ----------------------------------------------------------------------------- // unary expression enum diff --git a/src/context.c b/src/context.c index 680b350..0309d6e 100644 --- a/src/context.c +++ b/src/context.c @@ -99,15 +99,6 @@ cp_context__getattro__(CpContextObject* self, PyObject* name) } /* public API */ -/*CpAPI*/ -PyObject* -CpContext_GetAttr(PyObject* ctx, PyObject* key, _modulestate* m) -{ - // TODO - // return PyObject_CallMethodOneArg(ctx, m->str, PyObject *arg) - PyErr_SetString(PyExc_NotImplementedError, "GetAttr not implemented yet"); - return NULL; -} /*CpAPI*/ CpContextObject* From 2fa88177fe0f0493253747d153700f96727dac17 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Mon, 9 Sep 2024 23:00:38 +0200 Subject: [PATCH 16/29] Fix CpVarIntAtom on Windows --- + Enabled tests for C implementation --- .github/workflows/python-test.yml | 6 +++--- src/atomimpl/varintatomobj.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 61d57f2..0156440 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -13,17 +13,17 @@ jobs: steps: - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: x64 - name: Install run: | - pip install -e . + pip install -ve ./src/ccaterpillar pip install -r requirements.txt pip install -r test/requirements.txt diff --git a/src/atomimpl/varintatomobj.c b/src/atomimpl/varintatomobj.c index bdfaf0b..c8f05a2 100644 --- a/src/atomimpl/varintatomobj.c +++ b/src/atomimpl/varintatomobj.c @@ -7,7 +7,7 @@ // Inspired by https://github.com/google/cityhash/blob/master/src/city.cc #ifdef _MSC_VER #include -#define CP_BSWAP_U32(x) _byteswap_ulong(x)(x) +#define CP_BSWAP_U32(x) _byteswap_ulong(x) #define CP_BSWAP_U64(x) _byteswap_uint64(x) #elif defined(__APPLE__) From c85b45ef5131c2c7dc7c3bcefcc6b4c13a41db5a Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Wed, 11 Sep 2024 19:42:34 +0200 Subject: [PATCH 17/29] Renamed atom implementation files --- CMakeLists.txt | 36 +++++++++---------- src/atomimpl/{boolatomobj.c => bool.c} | 0 .../builtins/{offsetatomobj.c => atoffset.c} | 0 .../builtins/{builtinatomobj.c => builtin.c} | 0 .../{conditionatomobj.c => condition.c} | 0 .../{primitiveatomobj.c => primitive.c} | 0 .../{repeatedatomobj.c => repeated.c} | 0 .../builtins/{switchatomobj.c => switch.c} | 0 src/atomimpl/{bytesatomobj.c => bytes.c} | 0 src/atomimpl/{charatomobj.c => char.c} | 0 src/atomimpl/{constatomobj.c => const.c} | 0 src/atomimpl/{enumatomobj.c => enum.c} | 0 src/atomimpl/{floatatomobj.c => float.c} | 0 src/atomimpl/{intatomobj.c => int.c} | 0 src/atomimpl/{padatomobj.c => pad.c} | 18 +++++----- src/atomimpl/{pstringatomobj.c => pstring.c} | 0 src/atomimpl/{stringatomobj.c => string.c} | 0 src/atomimpl/{varintatomobj.c => varint.c} | 0 src/capi.dat | 35 +++++++++--------- .../include/caterpillar/atoms/primitive.h | 8 ++--- 20 files changed, 47 insertions(+), 50 deletions(-) rename src/atomimpl/{boolatomobj.c => bool.c} (100%) rename src/atomimpl/builtins/{offsetatomobj.c => atoffset.c} (100%) rename src/atomimpl/builtins/{builtinatomobj.c => builtin.c} (100%) rename src/atomimpl/builtins/{conditionatomobj.c => condition.c} (100%) rename src/atomimpl/builtins/{primitiveatomobj.c => primitive.c} (100%) rename src/atomimpl/builtins/{repeatedatomobj.c => repeated.c} (100%) rename src/atomimpl/builtins/{switchatomobj.c => switch.c} (100%) rename src/atomimpl/{bytesatomobj.c => bytes.c} (100%) rename src/atomimpl/{charatomobj.c => char.c} (100%) rename src/atomimpl/{constatomobj.c => const.c} (100%) rename src/atomimpl/{enumatomobj.c => enum.c} (100%) rename src/atomimpl/{floatatomobj.c => float.c} (100%) rename src/atomimpl/{intatomobj.c => int.c} (100%) rename src/atomimpl/{padatomobj.c => pad.c} (90%) rename src/atomimpl/{pstringatomobj.c => pstring.c} (100%) rename src/atomimpl/{stringatomobj.c => string.c} (100%) rename src/atomimpl/{varintatomobj.c => varint.c} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 40ebe0d..5446d6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,24 +32,24 @@ python_add_library( src/parsing_pack.c src/parsing_unpack.c - src/atomimpl/intatomobj.c - src/atomimpl/floatatomobj.c - src/atomimpl/boolatomobj.c - src/atomimpl/charatomobj.c - src/atomimpl/padatomobj.c - src/atomimpl/stringatomobj.c - src/atomimpl/constatomobj.c - src/atomimpl/bytesatomobj.c - src/atomimpl/pstringatomobj.c - src/atomimpl/enumatomobj.c - src/atomimpl/varintatomobj.c - - src/atomimpl/builtins/builtinatomobj.c - src/atomimpl/builtins/repeatedatomobj.c - src/atomimpl/builtins/conditionatomobj.c - src/atomimpl/builtins/switchatomobj.c - src/atomimpl/builtins/offsetatomobj.c - src/atomimpl/builtins/primitiveatomobj.c + src/atomimpl/int.c + src/atomimpl/float.c + "src/atomimpl/bool.c" + src/atomimpl/char.c + src/atomimpl/pad.c + "src/atomimpl/string.c" + src/atomimpl/const.c + src/atomimpl/bytes.c + src/atomimpl/pstring.c + src/atomimpl/enum.c + src/atomimpl/varint.c + + src/atomimpl/builtins/builtin.c + src/atomimpl/builtins/repeated.c + src/atomimpl/builtins/condition.c + src/atomimpl/builtins/switch.c + src/atomimpl/builtins/atoffset.c + src/atomimpl/builtins/primitive.c WITH_SOABI ) diff --git a/src/atomimpl/boolatomobj.c b/src/atomimpl/bool.c similarity index 100% rename from src/atomimpl/boolatomobj.c rename to src/atomimpl/bool.c diff --git a/src/atomimpl/builtins/offsetatomobj.c b/src/atomimpl/builtins/atoffset.c similarity index 100% rename from src/atomimpl/builtins/offsetatomobj.c rename to src/atomimpl/builtins/atoffset.c diff --git a/src/atomimpl/builtins/builtinatomobj.c b/src/atomimpl/builtins/builtin.c similarity index 100% rename from src/atomimpl/builtins/builtinatomobj.c rename to src/atomimpl/builtins/builtin.c diff --git a/src/atomimpl/builtins/conditionatomobj.c b/src/atomimpl/builtins/condition.c similarity index 100% rename from src/atomimpl/builtins/conditionatomobj.c rename to src/atomimpl/builtins/condition.c diff --git a/src/atomimpl/builtins/primitiveatomobj.c b/src/atomimpl/builtins/primitive.c similarity index 100% rename from src/atomimpl/builtins/primitiveatomobj.c rename to src/atomimpl/builtins/primitive.c diff --git a/src/atomimpl/builtins/repeatedatomobj.c b/src/atomimpl/builtins/repeated.c similarity index 100% rename from src/atomimpl/builtins/repeatedatomobj.c rename to src/atomimpl/builtins/repeated.c diff --git a/src/atomimpl/builtins/switchatomobj.c b/src/atomimpl/builtins/switch.c similarity index 100% rename from src/atomimpl/builtins/switchatomobj.c rename to src/atomimpl/builtins/switch.c diff --git a/src/atomimpl/bytesatomobj.c b/src/atomimpl/bytes.c similarity index 100% rename from src/atomimpl/bytesatomobj.c rename to src/atomimpl/bytes.c diff --git a/src/atomimpl/charatomobj.c b/src/atomimpl/char.c similarity index 100% rename from src/atomimpl/charatomobj.c rename to src/atomimpl/char.c diff --git a/src/atomimpl/constatomobj.c b/src/atomimpl/const.c similarity index 100% rename from src/atomimpl/constatomobj.c rename to src/atomimpl/const.c diff --git a/src/atomimpl/enumatomobj.c b/src/atomimpl/enum.c similarity index 100% rename from src/atomimpl/enumatomobj.c rename to src/atomimpl/enum.c diff --git a/src/atomimpl/floatatomobj.c b/src/atomimpl/float.c similarity index 100% rename from src/atomimpl/floatatomobj.c rename to src/atomimpl/float.c diff --git a/src/atomimpl/intatomobj.c b/src/atomimpl/int.c similarity index 100% rename from src/atomimpl/intatomobj.c rename to src/atomimpl/int.c diff --git a/src/atomimpl/padatomobj.c b/src/atomimpl/pad.c similarity index 90% rename from src/atomimpl/padatomobj.c rename to src/atomimpl/pad.c index 5fcd083..a9fa402 100644 --- a/src/atomimpl/padatomobj.c +++ b/src/atomimpl/pad.c @@ -42,7 +42,7 @@ cp_paddingatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) static void cp_paddingatom_dealloc(CpPaddingAtomObject* self) { - self->padding = 0; + self->_m_padding = 0; Py_TYPE(self)->tp_free((PyObject*)self); } @@ -55,7 +55,7 @@ cp_paddingatom_init(CpPaddingAtomObject* self, PyObject* args, PyObject* kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "|b", kwlist, &value)) return -1; - self->padding = value; + self->_m_padding = value; return 0; } @@ -68,10 +68,10 @@ CpPaddingAtom_Pack(CpPaddingAtomObject* self, { /* value will be ignored here */ PyObject* res; - if (!self->padding) { + if (!self->_m_padding) { res = CpState_Write(layer->m_state, layer->m_state->mod->cp_bytes__false); } else { - PyObject* bytes = PyBytes_FromStringAndSize((char*)&self->padding, 1); + PyObject* bytes = PyBytes_FromStringAndSize((char*)&self->_m_padding, 1); if (!bytes) { return -1; } @@ -104,7 +104,7 @@ CpPaddingAtom_PackMany(CpPaddingAtomObject* self, return -1; } /* unsafe { */ - memset(PyBytes_AS_STRING(bytes), self->padding, lengthinfo->m_length); + memset(PyBytes_AS_STRING(bytes), self->_m_padding, lengthinfo->m_length); /* } */ PyObject* res = CpState_Write(layer->m_state, bytes); Py_DECREF(bytes); @@ -149,7 +149,7 @@ CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, Py_ssize_t length = PyBytes_GET_SIZE(res); Py_ssize_t offset = 0; const char* ptr = PyBytes_AS_STRING(res); - while (offset < length && *(ptr + offset) == self->padding) { + while (offset < length && *(ptr + offset) == self->_m_padding) { offset++; } if (offset != length) { @@ -158,7 +158,7 @@ CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, "(possible padding overflow?). " "Expected %ld bytes of 0x%02x but parsed only %ld bytes."), length, - self->padding, + self->_m_padding, offset); Py_XDECREF(res); return NULL; @@ -171,10 +171,10 @@ CpPaddingAtom_UnpackMany(CpPaddingAtomObject* self, PyObject* cp_paddingatom_repr(CpPaddingAtomObject* self) { - if (self->padding == 0) { + if (self->_m_padding == 0) { return PyUnicode_FromFormat(""); } - return PyUnicode_FromFormat("", self->padding); + return PyUnicode_FromFormat("", self->_m_padding); } /* type setup */ diff --git a/src/atomimpl/pstringatomobj.c b/src/atomimpl/pstring.c similarity index 100% rename from src/atomimpl/pstringatomobj.c rename to src/atomimpl/pstring.c diff --git a/src/atomimpl/stringatomobj.c b/src/atomimpl/string.c similarity index 100% rename from src/atomimpl/stringatomobj.c rename to src/atomimpl/string.c diff --git a/src/atomimpl/varintatomobj.c b/src/atomimpl/varint.c similarity index 100% rename from src/atomimpl/varintatomobj.c rename to src/atomimpl/varint.c diff --git a/src/capi.dat b/src/capi.dat index 2b71f84..56063be 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -35,23 +35,23 @@ src:parsing_pack.c src:parsing_unpack.c src:parsing_typeof.c src:parsing_sizeof.c -src:atomimpl/boolatomobj.c -src:atomimpl/floatatomobj.c -src:atomimpl/charatomobj.c -src:atomimpl/intatomobj.c -src:atomimpl/padatomobj.c -src:atomimpl/stringatomobj.c -src:atomimpl/constatomobj.c -src:atomimpl/bytesatomobj.c -src:atomimpl/pstringatomobj.c -src:atomimpl/enumatomobj.c -src:atomimpl/varintatomobj.c -src:atomimpl/builtins/builtinatomobj.c -src:atomimpl/builtins/repeatedatomobj.c -src:atomimpl/builtins/conditionatomobj.c -src:atomimpl/builtins/switchatomobj.c -src:atomimpl/builtins/offsetatomobj.c -src:atomimpl/builtins/primitiveatomobj.c +src:atomimpl/bool.c +src:atomimpl/float.c +src:atomimpl/char.c +src:atomimpl/int.c +src:atomimpl/pad.c +src:atomimpl/string.c +src:atomimpl/const.c +src:atomimpl/bytes.c +src:atomimpl/pstring.c +src:atomimpl/enum.c +src:atomimpl/varint.c +src:atomimpl/builtins/builtin.c +src:atomimpl/builtins/repeated.c +src:atomimpl/builtins/condition.c +src:atomimpl/builtins/switch.c +src:atomimpl/builtins/atoffset.c +src:atomimpl/builtins/primitive.c # First index is reserved for the global module reference obj:0:CpModule:PyModuleDef @@ -102,6 +102,7 @@ type:37:_bytesatomobj:CpBytesAtomObject:- type:38:_pstringatomobj:CpPStringAtomObject:- type:39:_enumatomobj:CpEnumAtomObject:- type:40:_varintatomobj:CpVarIntAtomObject:- +type:41:_prefixedatomobj:CpPrefixedAtomObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 diff --git a/src/caterpillar/include/caterpillar/atoms/primitive.h b/src/caterpillar/include/caterpillar/atoms/primitive.h index 1f75a05..8620ad6 100644 --- a/src/caterpillar/include/caterpillar/atoms/primitive.h +++ b/src/caterpillar/include/caterpillar/atoms/primitive.h @@ -96,16 +96,12 @@ struct _paddingatomobj { CpBuiltinAtom_HEAD - char padding; + char _m_padding; }; -/// Padding atom object type -// PyAPI_DATA(PyTypeObject) CpPaddingAtom_Type; #define CpPaddingAtom_NAME "padding_t" - -/** @brief Checks if the given object is a padding atom object */ #define CpPaddingAtom_CheckExact(op) Py_IS_TYPE((op), &CpPaddingAtom_Type) -/** @brief Checks if the given object is a padding atom object */ #define CpPaddingAtom_Check(op) (PyObject_TypeCheck((op), &CpPaddingAtom_Type)) + #endif \ No newline at end of file From 06a278a9e22510eb106dae183b86ced2fed75652 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:17:39 +0200 Subject: [PATCH 18/29] CpComputedAtom implementation --- + Tests included + Python type: `computed` --- CMakeLists.txt | 1 + docs/sphinx/source/development/roadmap.rst | 10 +- src/atomimpl/computed.c | 109 ++++++++++++++++++ src/capi.dat | 9 +- .../include/caterpillar/atoms/primitive.h | 37 +++++- .../include/caterpillar/caterpillarapi.h | 8 ++ src/caterpillarapi.c | 6 +- src/module.c | 3 + test/_C/atoms/test_computed.py | 22 ++++ 9 files changed, 196 insertions(+), 9 deletions(-) create mode 100644 src/atomimpl/computed.c create mode 100644 test/_C/atoms/test_computed.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 5446d6a..4a8a6ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,7 @@ python_add_library( src/atomimpl/pstring.c src/atomimpl/enum.c src/atomimpl/varint.c + src/atomimpl/computed.c src/atomimpl/builtins/builtin.c src/atomimpl/builtins/repeated.c diff --git a/docs/sphinx/source/development/roadmap.rst b/docs/sphinx/source/development/roadmap.rst index cd536a7..8e96a0c 100644 --- a/docs/sphinx/source/development/roadmap.rst +++ b/docs/sphinx/source/development/roadmap.rst @@ -70,7 +70,10 @@ Atom Objects: [:text-danger:`missing docs`], [:text-warning:`missing perftest`] -- |uncheck_| Computed +- |check_| Computed (C type: :c:type:`CpComputedAtomObject`, Py type: :class:`computed`) + [:text-danger:`missing docs`], + [:text-warning:`missing perftest`] + - |check_| PString (C type: :c:type:`CpPStringAtomObject`, Py type: :class:`pstring`) [:text-danger:`missing docs`], [:text-warning:`missing perftest`] @@ -79,5 +82,8 @@ Atom Objects: - |uncheck_| Lazy - |uncheck_| uuid - |uncheck_| Conditional: If, Else, ElseIf -- |uncheck_| VarInt +- |check_| VarInt (C type: :c:type:`CpVarIntAtomObject`, Py type: :class:`varint_t`) + [:text-danger:`missing docs`], + [:text-warning:`missing perftest`] + - |uncheck_| While \ No newline at end of file diff --git a/src/atomimpl/computed.c b/src/atomimpl/computed.c new file mode 100644 index 0000000..ae141f7 --- /dev/null +++ b/src/atomimpl/computed.c @@ -0,0 +1,109 @@ +/* computed atom C implementation */ + +#include "caterpillar/caterpillar.h" + +#include + +/* impl */ +static PyObject* +cp_computedatom_type(CpComputedAtomObject* self) +{ + if (self->s_callable) { + return Py_XNewRef(get_global_module_state()->Any_Type); + } + return Py_NewRef(Py_TYPE(self->m_value)); +} + +static PyObject* +cp_computedatom_size(CpComputedAtomObject* self, PyObject* ctx) +{ + // TODO: explain why computed atom objects are of size 0 + return PyLong_FromSize_t(0); +} + +static PyObject* +cp_computedatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) +{ + CpComputedAtomObject* self = (CpComputedAtomObject*)type->tp_alloc(type, 0); + if (self != NULL) { + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpComputedAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpComputedAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_computedatom_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_computedatom_type; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + + self->m_value = NULL; + self->s_callable = false; + } + return (PyObject*)self; +} + +static void +cp_computedatom_dealloc(CpComputedAtomObject* self) +{ + Py_XDECREF(self->m_value); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_computedatom_init(CpComputedAtomObject* self, PyObject* args, PyObject* kwds) +{ + static char* kwlist[] = { "value", NULL }; + PyObject* value = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &value)) { + return -1; + } + + _Cp_SetObj(self->m_value, value); + self->s_callable = PyCallable_Check(self->m_value); + return 0; +} + +static PyObject* +cp_computedatom_repr(CpComputedAtomObject* self) +{ + return PyUnicode_FromFormat( + "", + self->s_callable ? "" : Py_TYPE(self->m_value)->tp_name); +} + +/* Public API */ + +/*CpAPI*/ +int +CpComputedAtom_Pack(CpComputedAtomObject* self, + PyObject* obj, + CpLayerObject* layer) +{ + return 0; +} + +/*CpAPI*/ +PyObject* +CpComputedAtom_Unpack(CpComputedAtomObject* self, CpLayerObject* layer) +{ + return CpComputedAtom_Value(self, layer); +} + +/* docs */ + +/* type */ + +static PyMemberDef cp_computedatom_members[] = { + { "value", T_OBJECT, offsetof(CpComputedAtomObject, m_value), READONLY, "value" }, + { NULL } +}; + +PyTypeObject CpComputedAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpComputedAtom_NAME), + .tp_basicsize = sizeof(CpComputedAtomObject), + .tp_dealloc = (destructor)cp_computedatom_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = NULL, + .tp_init = (initproc)cp_computedatom_init, + .tp_new = (newfunc)cp_computedatom_new, + .tp_members = cp_computedatom_members, + .tp_repr = (reprfunc)cp_computedatom_repr, +}; \ No newline at end of file diff --git a/src/capi.dat b/src/capi.dat index 56063be..8dccaee 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -46,6 +46,7 @@ src:atomimpl/bytes.c src:atomimpl/pstring.c src:atomimpl/enum.c src:atomimpl/varint.c +src:atomimpl/computed.c src:atomimpl/builtins/builtin.c src:atomimpl/builtins/repeated.c src:atomimpl/builtins/condition.c @@ -102,7 +103,7 @@ type:37:_bytesatomobj:CpBytesAtomObject:- type:38:_pstringatomobj:CpPStringAtomObject:- type:39:_enumatomobj:CpEnumAtomObject:- type:40:_varintatomobj:CpVarIntAtomObject:- -type:41:_prefixedatomobj:CpPrefixedAtomObject:- +type:41:_computedatomobj:CpComputedAtomObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 @@ -214,4 +215,8 @@ func:157:CpVarIntAtom_Unpack:PyObject*:+1 func:158:CpVarIntAtom_BSwap:PyObject*:+1 func:159:CpVarIntAtom_BSwapUnsignedLongLong:ulonglong:null func:160:CpVarIntAtom_BSwapLongLong:longlong:null -func:161:CpVarIntAtom_BSwapSsize_t:Py_ssize_t:null \ No newline at end of file +func:161:CpVarIntAtom_BSwapSsize_t:Py_ssize_t:null +func:-:CpComputedAtom_New:CpComputedAtomObject*:+1 +func:162:CpComputedAtom_Pack:int:null +func:163:CpComputedAtom_Unpack:PyObject*:+1 +func:-:CpComputedAtom_Value:PyObject*:+1 \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/primitive.h b/src/caterpillar/include/caterpillar/atoms/primitive.h index 8620ad6..ce1a704 100644 --- a/src/caterpillar/include/caterpillar/atoms/primitive.h +++ b/src/caterpillar/include/caterpillar/atoms/primitive.h @@ -19,16 +19,17 @@ #include "caterpillar/atoms/builtins.h" - //------------------------------------------------------------------------------ // primitive atom -struct _primitiveatomobj { +struct _primitiveatomobj +{ CpAtom_HEAD }; #define CpPrimitiveAtom_NAME "patom" #define CpPrimitiveAtom_CheckExact(op) Py_IS_TYPE((op), &CpPrimitiveAtom_Type) -#define CpPrimitiveAtom_Check(op) (PyObject_TypeCheck((op), &CpPrimitiveAtom_Type)) +#define CpPrimitiveAtom_Check(op) \ + (PyObject_TypeCheck((op), &CpPrimitiveAtom_Type)) //------------------------------------------------------------------------------ // Bool @@ -103,5 +104,35 @@ struct _paddingatomobj #define CpPaddingAtom_CheckExact(op) Py_IS_TYPE((op), &CpPaddingAtom_Type) #define CpPaddingAtom_Check(op) (PyObject_TypeCheck((op), &CpPaddingAtom_Type)) +//------------------------------------------------------------------------------ +// Computed +struct _computedatomobj +{ + CpBuiltinAtom_HEAD + + PyObject* m_value; + int s_callable; +}; + +#define CpComputedAtom_NAME "computed" +#define CpComputedAtom_CheckExact(op) Py_IS_TYPE((op), &CpComputedAtom_Type) +#define CpComputedAtom_Check(op) \ + (PyObject_TypeCheck((op), &CpComputedAtom_Type)) + +#define CpComputedAtom_VALUE(op) (((CpComputedAtomObject*)(op))->m_value) + +static inline PyObject* +CpComputedAtom_Value(CpComputedAtomObject* self, CpLayerObject* layer) +{ + return self->s_callable ? PyObject_CallOneArg(self->m_value, (PyObject*)layer) + : Py_NewRef(self->m_value); +} + +static inline CpComputedAtomObject* +CpComputedAtom_New(PyObject* value) +{ + return (CpComputedAtomObject*)CpObject_Create( + &CpComputedAtom_Type, "O", value); +} #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index b9f0879..f7f0ef2 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -100,6 +100,8 @@ struct _enumatomobj; typedef struct _enumatomobj CpEnumAtomObject; struct _varintatomobj; typedef struct _varintatomobj CpVarIntAtomObject; +struct _computedatomobj; +typedef struct _computedatomobj CpComputedAtomObject; #ifdef _CPMODULE @@ -149,6 +151,7 @@ extern PyTypeObject CpBytesAtom_Type; extern PyTypeObject CpPStringAtom_Type; extern PyTypeObject CpEnumAtom_Type; extern PyTypeObject CpVarIntAtom_Type; +extern PyTypeObject CpComputedAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -243,6 +246,8 @@ PyObject* CpVarIntAtom_BSwap(PyObject* number, bool little_endian); unsigned long long CpVarIntAtom_BSwapUnsignedLongLong(unsigned long long number,bool little_endian); long long CpVarIntAtom_BSwapLongLong(long long number, bool little_endian); Py_ssize_t CpVarIntAtom_BSwapSsize_t(Py_ssize_t number, bool little_endian); +int CpComputedAtom_Pack(CpComputedAtomObject* self,PyObject* obj,CpLayerObject* layer); +PyObject* CpComputedAtom_Unpack(CpComputedAtomObject* self, CpLayerObject* layer); #else @@ -293,6 +298,7 @@ caterpillar_api.py #define CpPStringAtom_Type (*(PyTypeObject *)Cp_API[38]) #define CpEnumAtom_Type (*(PyTypeObject *)Cp_API[39]) #define CpVarIntAtom_Type (*(PyTypeObject *)Cp_API[40]) +#define CpComputedAtom_Type (*(PyTypeObject *)Cp_API[41]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -387,6 +393,8 @@ caterpillar_api.py #define CpVarIntAtom_BSwapUnsignedLongLong (*((unsigned long long (*)(unsigned long long number,bool little_endian)))Cp_API[159]) #define CpVarIntAtom_BSwapLongLong (*((long long (*)(long long number, bool little_endian)))Cp_API[160]) #define CpVarIntAtom_BSwapSsize_t (*((Py_ssize_t (*)(Py_ssize_t number, bool little_endian)))Cp_API[161]) +#define CpComputedAtom_Pack (*((int (*)(CpComputedAtomObject* self,PyObject* obj,CpLayerObject* layer)))Cp_API[162]) +#define CpComputedAtom_Unpack (*((PyObject* (*)(CpComputedAtomObject* self, CpLayerObject* layer)))Cp_API[163]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index a9d27a0..aeec14f 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -44,7 +44,7 @@ void *Cp_API[] = { (void *) &CpPStringAtom_Type, (void *) &CpEnumAtom_Type, (void *) &CpVarIntAtom_Type, - NULL, + (void *) &CpComputedAtom_Type, NULL, NULL, NULL, @@ -164,6 +164,8 @@ void *Cp_API[] = { (void *) &CpVarIntAtom_BSwap, (void *) &CpVarIntAtom_BSwapUnsignedLongLong, (void *) &CpVarIntAtom_BSwapLongLong, - (void *) &CpVarIntAtom_BSwapSsize_t + (void *) &CpVarIntAtom_BSwapSsize_t, + (void *) &CpComputedAtom_Pack, + (void *) &CpComputedAtom_Unpack }; diff --git a/src/module.c b/src/module.c index 76b6b20..6d22b88 100644 --- a/src/module.c +++ b/src/module.c @@ -412,6 +412,7 @@ PyInit__C(void) CpPStringAtom_Type.tp_base = &CpBuiltinAtom_Type; CpEnumAtom_Type.tp_base = &CpBuiltinAtom_Type; CpVarIntAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpComputedAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpBuiltinAtom_Type); CpModule_SetupType(&CpPrimitiveAtom_Type); @@ -431,6 +432,7 @@ PyInit__C(void) CpModule_SetupType(&CpPStringAtom_Type); CpModule_SetupType(&CpEnumAtom_Type); CpModule_SetupType(&CpVarIntAtom_Type); + CpModule_SetupType(&CpComputedAtom_Type); // module setup m = PyModule_Create(&CpModule); @@ -480,6 +482,7 @@ PyInit__C(void) CpModule_AddObject(CpPStringAtom_NAME, &CpPStringAtom_Type); CpModule_AddObject(CpEnumAtom_NAME, &CpEnumAtom_Type); CpModule_AddObject(CpVarIntAtom_NAME, &CpVarIntAtom_Type); + CpModule_AddObject(CpComputedAtom_NAME, &CpComputedAtom_Type); /* setup custom intatoms */ #define CpModule_DefAtom(name, ...) \ diff --git a/test/_C/atoms/test_computed.py b/test/_C/atoms/test_computed.py new file mode 100644 index 0000000..0edc9fe --- /dev/null +++ b/test/_C/atoms/test_computed.py @@ -0,0 +1,22 @@ +# pylint: disable=unused-import,no-name-in-module,import-error +import pytest +import caterpillar +import enum +import typing + +if caterpillar.native_support(): + + from caterpillar._C import pack, unpack, computed, typeof + + def test_computed(): + value = 0xDEADBEEF + # computed atoms don't affect the underlying stream + assert unpack(b"", computed(value)) == 0xDEADBEEF + assert unpack(b"", computed(lambda ctx: value)) == 0xDEADBEEF + + def test_computed_type(): + value = 0xDEADBEEF + # Type will be inferred using the given value. A callable object + # will use the Any type. + assert typeof(computed(value)) == int + assert typeof(computed(lambda ctx: value)) == typing.Any From f5d49312279e212fdf3d0be319fb3c2ac38ce0ed Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:53:09 +0200 Subject: [PATCH 19/29] CpLazyAtom implementation --- + Python type: `lazy` + Tests included + README installation update --- CMakeLists.txt | 1 + README.md | 17 +- docs/sphinx/source/development/roadmap.rst | 9 +- src/atomimpl/lazy.c | 163 ++++++++++++++++++ src/capi.dat | 8 +- .../include/caterpillar/atoms/primitive.h | 44 +++++ .../include/caterpillar/caterpillarapi.h | 8 + src/caterpillarapi.c | 6 +- src/module.c | 3 + test/_C/atoms/test_lazy.py | 29 ++++ 10 files changed, 273 insertions(+), 15 deletions(-) create mode 100644 src/atomimpl/lazy.c create mode 100644 test/_C/atoms/test_lazy.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a8a6ca..3c71326 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,7 @@ python_add_library( src/atomimpl/enum.c src/atomimpl/varint.c src/atomimpl/computed.c + src/atomimpl/lazy.c src/atomimpl/builtins/builtin.c src/atomimpl/builtins/repeated.c diff --git a/README.md b/README.md index bf68ee5..8d45d45 100644 --- a/README.md +++ b/README.md @@ -61,24 +61,19 @@ on its powerful features, explore the official [documentation](https://matrixedi > As of Caterpillar v2.1.2 it is possible to install the library without the need of > compiling the C extension. -Simply use pip to install the package with all tools, all extra packages and the C extension: +### Python-only installation + ```bash -pip install "caterpillar[all]@git+https://github.com/MatrixEditor/caterpillar.git" +pip install "caterpillar[all]@git+https://github.com/MatrixEditor/caterpillar" ``` -If you want to use the native C extension you can specify the following environment variables: - -* `CP_ENABLE_NATIVE` - Enables installation of the native C extension -* `CP_ENABLE_TOOLS` - Enables installation of extra tools (proposed) +### C-extension installation -For instance, the following command will install the `caterpillar` package with -a native interface and with extra tools: ```bash -CP_ENABLE_NATIVE=1 \ -CP_ENABLE_TOOLS=1 \ -pip install git+https://github.com/MatrixEditor/caterpillar +pip install "caterpillar[all]@git+https://github.com/MatrixEditor/caterpillar/#subdirectory=src/ccaterpillar" ``` + ## Starting Point Please visit the [Documentation](https://matrixeditor.github.io/caterpillar/), it contains a complete tutorial on how to use this library. diff --git a/docs/sphinx/source/development/roadmap.rst b/docs/sphinx/source/development/roadmap.rst index 8e96a0c..6b44b25 100644 --- a/docs/sphinx/source/development/roadmap.rst +++ b/docs/sphinx/source/development/roadmap.rst @@ -53,6 +53,8 @@ Atom Objects: Global instance: :code:`padding`, [:text-danger:`missing docs`] +- |uncheck_| Bytes + - |check_| String (C type: :c:type:`CpStringAtomObject`, Py type: :class:`string`) [:text-danger:`missing docs`], [:text-warning:`missing perftest`] @@ -79,7 +81,12 @@ Atom Objects: [:text-warning:`missing perftest`] - |uncheck_| Prefixed -- |uncheck_| Lazy + [:text-danger:`create issue (discuss relevance)`] + +- |check_| Lazy + [:text-danger:`missing docs`], + [:text-warning:`missing perftest`] + - |uncheck_| uuid - |uncheck_| Conditional: If, Else, ElseIf - |check_| VarInt (C type: :c:type:`CpVarIntAtomObject`, Py type: :class:`varint_t`) diff --git a/src/atomimpl/lazy.c b/src/atomimpl/lazy.c new file mode 100644 index 0000000..1eb0dee --- /dev/null +++ b/src/atomimpl/lazy.c @@ -0,0 +1,163 @@ +/* lazy C atom implementation */ + +#include "caterpillar/caterpillar.h" + +#include + +/* impl */ +static PyObject* +cp_lazyatom_type(CpLazyAtomObject* self) +{ + PyObject* atom = CpLazyAtom_Atom(self); + if (!atom) + return NULL; + + PyObject* type = CpTypeOf(atom); + Py_DECREF(atom); + return type; +} + +static PyObject* +cp_lazyatom_size(CpLazyAtomObject* self, CpLayerObject* layer) +{ + PyObject* atom = CpLazyAtom_Atom(self); + if (!atom) + return NULL; + + PyObject* size = _Cp_SizeOf(atom, layer); + Py_DECREF(atom); + return size; +} + +static PyObject* +cp_lazyatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) +{ + CpLazyAtomObject* self = (CpLazyAtomObject*)type->tp_alloc(type, 0); + if (self != NULL) { + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpLazyAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpLazyAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_lazyatom_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_lazyatom_type; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + + self->m_atom = NULL; + self->m_fn = NULL; + self->s_always_lazy = false; + } + return (PyObject*)self; +} + +static void +cp_lazyatom_dealloc(CpLazyAtomObject* self) +{ + Py_XDECREF(self->m_atom); + Py_XDECREF(self->m_fn); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_lazyatom_init(CpLazyAtomObject* self, PyObject* args, PyObject* kwds) +{ + static char* kwlist[] = { "fn", "always_lazy", NULL }; + PyObject* fn = NULL; + int always_lazy = 0; + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "O|p", kwlist, &fn, &always_lazy)) { + return -1; + } + + _Cp_SetObj(self->m_fn, fn); + self->s_always_lazy = always_lazy; + self->m_atom = NULL; + return 0; +} + +static PyObject* +cp_lazyatom_repr(CpLazyAtomObject* self) +{ + return PyUnicode_FromFormat("<%slazy %s>", + self->s_always_lazy ? "always_" : "", + self->m_atom ? Py_TYPE(self->m_atom)->tp_name + : ""); +} + +static PyObject* +cp_lazyatom_set_byteorder(CpLazyAtomObject* self, PyObject* args, PyObject* kw) +{ + _CpEndian_KwArgsGetByteorder(NULL); + + PyObject* atom = CpLazyAtom_Atom(self); + if (!atom) { + return NULL; + } + PyObject* ret = CpEndian_SetEndian(atom, (CpEndianObject*)byteorder); + if (!ret) { + Py_DECREF(atom); + return NULL; + } + _Cp_SetObj(self->m_atom, ret); + Py_DECREF(atom); + Py_DECREF(ret); + return Py_NewRef((PyObject*)self); +} + +/* Public API */ + +/*CpAPI*/ +int +CpLazyAtom_Pack(CpLazyAtomObject* self, PyObject* obj, CpLayerObject* layer) +{ + PyObject* atom = CpLazyAtom_Atom(self); + if (!atom) { + return -1; + } + int res = _Cp_Pack(obj, atom, layer); + Py_DECREF(atom); + return res; +} + +/*CpAPI*/ +PyObject* +CpLazyAtom_Unpack(CpLazyAtomObject* self, CpLayerObject* layer) +{ + PyObject* atom = CpLazyAtom_Atom(self); + if (!atom) { + return NULL; + } + PyObject* res = _Cp_Unpack(atom, layer); + Py_DECREF(atom); + return res; +} + +/* docs */ + +/* type */ + +static PyMemberDef CpLazyAtom_Members[] = { + { "fn", T_OBJECT_EX, offsetof(CpLazyAtomObject, m_fn), READONLY }, + { "always_lazy", + T_BOOL, + offsetof(CpLazyAtomObject, s_always_lazy), + READONLY }, + { NULL } /* Sentinel */ +}; + +static PyMethodDef CpLazyAtom_Methods[] = { + _CpEndian_ImplSetByteorder_MethDef(lazyatom, NULL), + { NULL } /* Sentinel */ +}; + +PyTypeObject CpLazyAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpLazyAtom_NAME), + .tp_basicsize = sizeof(CpLazyAtomObject), + .tp_dealloc = (destructor)cp_lazyatom_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = NULL, + .tp_init = (initproc)cp_lazyatom_init, + .tp_new = (newfunc)cp_lazyatom_new, + .tp_repr = (reprfunc)cp_lazyatom_repr, + .tp_members = CpLazyAtom_Members, + .tp_methods = CpLazyAtom_Methods, +}; \ No newline at end of file diff --git a/src/capi.dat b/src/capi.dat index 8dccaee..471ed54 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -47,6 +47,7 @@ src:atomimpl/pstring.c src:atomimpl/enum.c src:atomimpl/varint.c src:atomimpl/computed.c +src:atomimpl/lazy.c src:atomimpl/builtins/builtin.c src:atomimpl/builtins/repeated.c src:atomimpl/builtins/condition.c @@ -104,6 +105,7 @@ type:38:_pstringatomobj:CpPStringAtomObject:- type:39:_enumatomobj:CpEnumAtomObject:- type:40:_varintatomobj:CpVarIntAtomObject:- type:41:_computedatomobj:CpComputedAtomObject:- +type:42:_lazyatomobj:CpLazyAtomObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 @@ -219,4 +221,8 @@ func:161:CpVarIntAtom_BSwapSsize_t:Py_ssize_t:null func:-:CpComputedAtom_New:CpComputedAtomObject*:+1 func:162:CpComputedAtom_Pack:int:null func:163:CpComputedAtom_Unpack:PyObject*:+1 -func:-:CpComputedAtom_Value:PyObject*:+1 \ No newline at end of file +func:-:CpComputedAtom_Value:PyObject*:+1 +func:-:CpLazyAtom_New:CpLazyAtomObject*:+1 +func:164:CpLazyAtom_Pack:int:null +func:165:CpLazyAtom_Unpack:PyObject*:+1 +func:-:CpLazyAtom_Atom:PyObject*:+1 \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/primitive.h b/src/caterpillar/include/caterpillar/atoms/primitive.h index ce1a704..19b3e8b 100644 --- a/src/caterpillar/include/caterpillar/atoms/primitive.h +++ b/src/caterpillar/include/caterpillar/atoms/primitive.h @@ -122,6 +122,7 @@ struct _computedatomobj #define CpComputedAtom_VALUE(op) (((CpComputedAtomObject*)(op))->m_value) +/*CpAPI*/ static inline PyObject* CpComputedAtom_Value(CpComputedAtomObject* self, CpLayerObject* layer) { @@ -129,10 +130,53 @@ CpComputedAtom_Value(CpComputedAtomObject* self, CpLayerObject* layer) : Py_NewRef(self->m_value); } +/*CpAPI*/ static inline CpComputedAtomObject* CpComputedAtom_New(PyObject* value) { return (CpComputedAtomObject*)CpObject_Create( &CpComputedAtom_Type, "O", value); } + +//------------------------------------------------------------------------------ +// Lazy + +struct _lazyatomobj +{ + CpBuiltinAtom_HEAD + + PyObject* m_fn; + PyObject* m_atom; + int s_always_lazy; +}; + +#define CpLazyAtom_NAME "lazy" +#define CpLazyAtom_CheckExact(op) Py_IS_TYPE((op), &CpLazyAtom_Type) +#define CpLazyAtom_Check(op) (PyObject_TypeCheck((op), &CpLazyAtom_Type)) +#define CpLazyAtom_ATOM(op) (((CpLazyAtomObject*)(op))->m_atom) + +/*CpAPI*/ +static inline PyObject* +CpLazyAtom_Atom(CpLazyAtomObject* self) +{ + if (self->s_always_lazy) + return PyObject_CallNoArgs(self->m_fn); + + if (!self->m_atom) { + Py_XSETREF(self->m_atom, PyObject_CallNoArgs(self->m_fn)); + if (!self->m_atom) + return NULL; + } + + return Py_NewRef(self->m_atom); +} + +/*CpAPI*/ +static inline CpLazyAtomObject* +CpLazyAtom_New(PyObject* fn, int always_lazy) +{ + return (CpLazyAtomObject*)CpObject_Create( + &CpLazyAtom_Type, "Oi", fn, always_lazy); +} + #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index f7f0ef2..e1a1dba 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -102,6 +102,8 @@ struct _varintatomobj; typedef struct _varintatomobj CpVarIntAtomObject; struct _computedatomobj; typedef struct _computedatomobj CpComputedAtomObject; +struct _lazyatomobj; +typedef struct _lazyatomobj CpLazyAtomObject; #ifdef _CPMODULE @@ -152,6 +154,7 @@ extern PyTypeObject CpPStringAtom_Type; extern PyTypeObject CpEnumAtom_Type; extern PyTypeObject CpVarIntAtom_Type; extern PyTypeObject CpComputedAtom_Type; +extern PyTypeObject CpLazyAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -248,6 +251,8 @@ long long CpVarIntAtom_BSwapLongLong(long long number, bool little_endian); Py_ssize_t CpVarIntAtom_BSwapSsize_t(Py_ssize_t number, bool little_endian); int CpComputedAtom_Pack(CpComputedAtomObject* self,PyObject* obj,CpLayerObject* layer); PyObject* CpComputedAtom_Unpack(CpComputedAtomObject* self, CpLayerObject* layer); +int CpLazyAtom_Pack(CpLazyAtomObject* self, PyObject* obj, CpLayerObject* layer); +PyObject* CpLazyAtom_Unpack(CpLazyAtomObject* self, CpLayerObject* layer); #else @@ -299,6 +304,7 @@ caterpillar_api.py #define CpEnumAtom_Type (*(PyTypeObject *)Cp_API[39]) #define CpVarIntAtom_Type (*(PyTypeObject *)Cp_API[40]) #define CpComputedAtom_Type (*(PyTypeObject *)Cp_API[41]) +#define CpLazyAtom_Type (*(PyTypeObject *)Cp_API[42]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -395,6 +401,8 @@ caterpillar_api.py #define CpVarIntAtom_BSwapSsize_t (*((Py_ssize_t (*)(Py_ssize_t number, bool little_endian)))Cp_API[161]) #define CpComputedAtom_Pack (*((int (*)(CpComputedAtomObject* self,PyObject* obj,CpLayerObject* layer)))Cp_API[162]) #define CpComputedAtom_Unpack (*((PyObject* (*)(CpComputedAtomObject* self, CpLayerObject* layer)))Cp_API[163]) +#define CpLazyAtom_Pack (*((int (*)(CpLazyAtomObject* self, PyObject* obj, CpLayerObject* layer)))Cp_API[164]) +#define CpLazyAtom_Unpack (*((PyObject* (*)(CpLazyAtomObject* self, CpLayerObject* layer)))Cp_API[165]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index aeec14f..6e3d652 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -45,7 +45,7 @@ void *Cp_API[] = { (void *) &CpEnumAtom_Type, (void *) &CpVarIntAtom_Type, (void *) &CpComputedAtom_Type, - NULL, + (void *) &CpLazyAtom_Type, NULL, NULL, NULL, @@ -166,6 +166,8 @@ void *Cp_API[] = { (void *) &CpVarIntAtom_BSwapLongLong, (void *) &CpVarIntAtom_BSwapSsize_t, (void *) &CpComputedAtom_Pack, - (void *) &CpComputedAtom_Unpack + (void *) &CpComputedAtom_Unpack, + (void *) &CpLazyAtom_Pack, + (void *) &CpLazyAtom_Unpack }; diff --git a/src/module.c b/src/module.c index 6d22b88..68d9ee7 100644 --- a/src/module.c +++ b/src/module.c @@ -413,6 +413,7 @@ PyInit__C(void) CpEnumAtom_Type.tp_base = &CpBuiltinAtom_Type; CpVarIntAtom_Type.tp_base = &CpBuiltinAtom_Type; CpComputedAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpLazyAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpBuiltinAtom_Type); CpModule_SetupType(&CpPrimitiveAtom_Type); @@ -433,6 +434,7 @@ PyInit__C(void) CpModule_SetupType(&CpEnumAtom_Type); CpModule_SetupType(&CpVarIntAtom_Type); CpModule_SetupType(&CpComputedAtom_Type); + CpModule_SetupType(&CpLazyAtom_Type); // module setup m = PyModule_Create(&CpModule); @@ -483,6 +485,7 @@ PyInit__C(void) CpModule_AddObject(CpEnumAtom_NAME, &CpEnumAtom_Type); CpModule_AddObject(CpVarIntAtom_NAME, &CpVarIntAtom_Type); CpModule_AddObject(CpComputedAtom_NAME, &CpComputedAtom_Type); + CpModule_AddObject(CpLazyAtom_NAME, &CpLazyAtom_Type); /* setup custom intatoms */ #define CpModule_DefAtom(name, ...) \ diff --git a/test/_C/atoms/test_lazy.py b/test/_C/atoms/test_lazy.py new file mode 100644 index 0000000..176c5be --- /dev/null +++ b/test/_C/atoms/test_lazy.py @@ -0,0 +1,29 @@ +# pylint: disable=unused-import,no-name-in-module,import-error +import pytest +import caterpillar +import enum +import typing + +if caterpillar.native_support(): + + from caterpillar._C import pack, unpack, lazy, typeof, u8 + + def test_lazy(): + # Here, we can add a forward reference to types that may + # appear after this line + atom = lazy(lambda: u8) + assert unpack(b"\x01", atom) == 1 + assert pack(1, atom) == b"\x01" + + # By default, the atom returned by the lambda funtion will be + # cached internally. to prevent that, use always_lazy: + atom = lazy(lambda: u8, always_lazy=True) + assert unpack(b"\x01", atom) == 1 + assert unpack(b"\x01\x01\x01", atom[3]) == [1, 1, 1] + + # TODO: support pack_many + # TODO: describe __repr__ + + def test_lazy_type(): + atom = lazy(lambda: u8) + assert typeof(atom) == int From 792f15c7b23e22cc4c54fef4f5f50761a3444f5c Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sat, 14 Sep 2024 13:37:49 +0200 Subject: [PATCH 20/29] CpCStringAtom implementation --- + Python Type: `cstring` + Tests included + Changed encoding parameter to be optional --- CMakeLists.txt | 1 + docs/sphinx/source/development/roadmap.rst | 16 +- src/atomimpl/cstring.c | 307 ++++++++++++++++++ src/atomimpl/pstring.c | 7 +- src/atomimpl/string.c | 8 +- src/capi.dat | 6 +- .../include/caterpillar/atoms/string.h | 47 +++ .../include/caterpillar/caterpillarapi.h | 8 + src/caterpillar/include/caterpillar/module.h | 2 + src/caterpillarapi.c | 6 +- src/module.c | 12 + test/_C/atoms/test_string.py | 23 +- 12 files changed, 428 insertions(+), 15 deletions(-) create mode 100644 src/atomimpl/cstring.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c71326..82bedd2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,7 @@ python_add_library( src/atomimpl/varint.c src/atomimpl/computed.c src/atomimpl/lazy.c + src/atomimpl/cstring.c src/atomimpl/builtins/builtin.c src/atomimpl/builtins/repeated.c diff --git a/docs/sphinx/source/development/roadmap.rst b/docs/sphinx/source/development/roadmap.rst index 6b44b25..fb32685 100644 --- a/docs/sphinx/source/development/roadmap.rst +++ b/docs/sphinx/source/development/roadmap.rst @@ -53,8 +53,6 @@ Atom Objects: Global instance: :code:`padding`, [:text-danger:`missing docs`] -- |uncheck_| Bytes - - |check_| String (C type: :c:type:`CpStringAtomObject`, Py type: :class:`string`) [:text-danger:`missing docs`], [:text-warning:`missing perftest`] @@ -63,7 +61,10 @@ Atom Objects: [:text-danger:`missing docs`], [:text-warning:`missing perftest`] -- |uncheck_| CString +- |uncheck_| CString (C type: :c:type:`CpCStringAtomObject`, Py type: :class:`cstring`) + [:text-danger:`missing docs`], + [:text-warning:`missing perftest`] + - |check_| Bytes (C type: :c:type:`CpBytesAtomObject`, Py type: :class:`octetstring`) [:text-danger:`missing docs`], [:text-warning:`missing perftest`] @@ -81,16 +82,21 @@ Atom Objects: [:text-warning:`missing perftest`] - |uncheck_| Prefixed - [:text-danger:`create issue (discuss relevance)`] + .. seealso:: *link issue here* - |check_| Lazy [:text-danger:`missing docs`], [:text-warning:`missing perftest`] - |uncheck_| uuid + .. seealso:: *link issue here* + - |uncheck_| Conditional: If, Else, ElseIf + .. seealso:: *link issue here* + - |check_| VarInt (C type: :c:type:`CpVarIntAtomObject`, Py type: :class:`varint_t`) [:text-danger:`missing docs`], [:text-warning:`missing perftest`] -- |uncheck_| While \ No newline at end of file +- |uncheck_| While + .. seealso:: *link issue here* \ No newline at end of file diff --git a/src/atomimpl/cstring.c b/src/atomimpl/cstring.c new file mode 100644 index 0000000..a6d90b7 --- /dev/null +++ b/src/atomimpl/cstring.c @@ -0,0 +1,307 @@ +/* c-string atom implementation */ + +#include "caterpillar/caterpillar.h" + +#include +#include + +/* impl */ +static PyObject* +cp_cstringatom_type(CpCStringAtomObject* self) +{ + return Py_NewRef(&PyUnicode_Type); +} + +static PyObject* +cp_cstringatom_size(CpCStringAtomObject* self, CpLayerObject* layer) +{ + if (self->s_number) { + return Py_NewRef(self->m_length); + } + PyErr_SetString(PyExc_ValueError, + "dynamic-sized c-string atoms do not have a static size"); + return NULL; +} + +static PyObject* +cp_cstringatom_new(PyTypeObject* type, PyObject* args, PyObject* kwds) +{ + CpCStringAtomObject* self = (CpCStringAtomObject*)type->tp_alloc(type, 0); + if (self != NULL) { + CpBuiltinAtom_CATOM(self).ob_pack = (packfunc)CpCStringAtom_Pack; + CpBuiltinAtom_CATOM(self).ob_unpack = (unpackfunc)CpCStringAtom_Unpack; + CpBuiltinAtom_CATOM(self).ob_pack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; + CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_cstringatom_size; + CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_cstringatom_type; + CpBuiltinAtom_CATOM(self).ob_bits = NULL; + + self->m_length = NULL; + self->m_encoding = NULL; + self->m_errors = NULL; + self->m_terminator = NULL; + self->_m_terminator_bytes = NULL; + self->s_keep_terminator = false; + self->s_number = false; + self->s_callable = false; + self->s_greedy = false; + } + return (PyObject*)self; +} + +static void +cp_cstringatom_dealloc(CpCStringAtomObject* self) +{ + Py_XDECREF(self->m_length); + Py_XDECREF(self->m_encoding); + Py_XDECREF(self->m_errors); + Py_XDECREF(self->m_terminator); + Py_XDECREF(self->_m_terminator_bytes); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static int +cp_cstringatom_init(CpCStringAtomObject* self, PyObject* args, PyObject* kwds) +{ + static char* kwlist[] = { "length", "encoding", "errors", + "terminator", "keep", NULL }; + PyObject* length = NULL; + PyObject* encoding = NULL; + PyObject* errors = NULL; + PyObject* terminator = NULL; + int keep_terminator = false; + if (!PyArg_ParseTupleAndKeywords(args, + kwds, + "O|OOOp", + kwlist, + &length, + &encoding, + &errors, + &terminator, + &keep_terminator)) { + return -1; + } + + _modulestate* mod = get_global_module_state(); + + _Cp_SetObj(self->m_length, length); + _Cp_SetObj(self->m_encoding, encoding); + if (!self->m_encoding) { + Py_XSETREF(self->m_encoding, mod->str_utf8); + } + + _Cp_SetObj(self->m_errors, errors); + if (!self->m_errors) { + Py_XSETREF(self->m_errors, mod->str_strict); + } + + _Cp_SetObj(self->m_terminator, terminator); + if (!self->m_terminator) { + _Cp_SetObj(self->_m_terminator_bytes, mod->cp_bytes__false); + _Cp_SetObj(self->m_terminator, mod->str_cstring_default_pad); + } else { + if (!PyUnicode_Check(self->m_terminator)) { + PyErr_SetString(PyExc_TypeError, "terminator must be a string"); + return -1; + } + self->_m_terminator_bytes = + PyUnicode_AsEncodedString(self->m_terminator, + PyUnicode_AsUTF8AndSize(self->m_encoding, NULL), + PyUnicode_AsUTF8AndSize(self->m_errors, NULL)); + if (!self->_m_terminator_bytes) { + return -1; + } + } + + self->s_keep_terminator = keep_terminator; + self->s_number = PyLong_Check(self->m_length); + self->s_callable = PyCallable_Check(self->m_length); + self->s_greedy = Py_IS_TYPE(self->m_length, &PyEllipsis_Type); + return 0; +} + +static PyObject* +cp_cstringatom_repr(CpCStringAtomObject* self) +{ + return PyUnicode_FromFormat( + "", self->m_encoding, self->m_errors); +} + +/* Public API */ + +/*CpAPI*/ +int +CpCStringAtom_Pack(CpCStringAtomObject* self, + PyObject* value, + CpLayerObject* layer) +{ + if (!PyUnicode_Check(value)) { + PyErr_Format( + PyExc_TypeError, "C strings must be packed from unicode - got %R", value); + return -1; + } + PyObject* bytes = PyUnicode_AsEncodedString( + value, PyUnicode_DATA(self->m_encoding), PyUnicode_DATA(self->m_errors)); + if (!bytes) { + return -1; + } + + if (!self->s_callable && !self->s_number && !self->s_greedy) { + // is atom, must pack length as prefix + PyObject* length = PyLong_FromSsize_t(PyUnicode_GET_LENGTH(value) + 1); + if (!length) { + return -1; + } + if (_Cp_Pack(length, self->m_length, layer)) { + Py_DECREF(length); + return -1; + } + Py_DECREF(length); + } + + PyObject* res = CpState_Write(layer->m_state, bytes); + Py_DECREF(bytes); + if (!res) { + return -1; + } + Py_XDECREF(res); + + if (self->s_greedy || (!self->s_callable && !self->s_number)) { + res = CpState_Write(layer->m_state, self->_m_terminator_bytes); + if (!res) { + return -1; + } + Py_XDECREF(res); + } else { + // NOTE: we know unpack won't be called + PyObject* length = CpCStringAtom_Length(self, layer); + if (length == NULL) { + return -1; + } + PyObject* objLength = PyLong_FromSsize_t(PyUnicode_GET_LENGTH(value)); + if (!objLength) { + Py_DECREF(length); + return -1; + } + + PyObject* realLength = PyNumber_Subtract(length, objLength); + if (!realLength) { + Py_DECREF(length); + return -1; + } + + PyObject* trailer = + PyNumber_Multiply(self->_m_terminator_bytes, realLength); + Py_DECREF(objLength); + Py_DECREF(realLength); + Py_DECREF(length); + if (!trailer) { + return -1; + } + + res = CpState_Write(layer->m_state, trailer); + Py_DECREF(trailer); + if (!res) { + return -1; + } + Py_XDECREF(res); + } + return 0; +} + +/*CpAPI*/ +PyObject* +CpCStringAtom_Unpack(CpCStringAtomObject* self, CpLayerObject* layer) +{ + PyObject* length = CpCStringAtom_Length(self, layer); + if (length == NULL) { + return NULL; + } + + PyObject* result = NULL; + if (Py_IsNone(length)) { + // greedy parsing + result = CpObject_CreateNoArgs(&PyBytes_Type); + if (!result) { + return NULL; + } + // REVISIT: is this efficient? + while (true) { + PyObject* bytes = CpState_ReadSsize_t(layer->m_state, 1); + if (!bytes) { + return NULL; + } + if (PyBytes_AS_STRING(bytes)[0] == + PyBytes_AS_STRING(self->_m_terminator_bytes)[0]) { + break; + } + PyBytes_Concat(&result, bytes); + Py_DECREF(bytes); + if (!result) { + return NULL; + } + } + } else { + result = CpState_Read(layer->m_state, length); + Py_DECREF(length); + if (!result) { + return NULL; + } + } + if (!result) { + return NULL; + } + PyObject* string = PyUnicode_Decode(PyBytes_AS_STRING(result), + PyBytes_GET_SIZE(result), + PyUnicode_DATA(self->m_encoding), + PyUnicode_DATA(self->m_errors)); + Py_XDECREF(result); + if (!string) { + return NULL; + } + + if (!self->s_keep_terminator) { + Py_XSETREF(string, _PyUnicode_XStrip(string, 1, self->m_terminator)); + } + return string; +} + +/* docs */ + +/* type */ +static PyMemberDef CpCStringAtom_Members[] = { + { "encoding", + T_OBJECT_EX, + offsetof(CpCStringAtomObject, m_encoding), + 0, + "the encoding of the string" }, + { "errors", + T_OBJECT_EX, + offsetof(CpCStringAtomObject, m_errors), + 0, + "the error handling for the string" }, + { "terminator", + T_OBJECT_EX, + offsetof(CpCStringAtomObject, m_terminator), + READONLY, + "the terminator for the string" }, + { "keep_terminator", + T_BOOL, + offsetof(CpCStringAtomObject, s_keep_terminator), + 0, + "whether to keep the terminator" }, + { NULL } +}; + +PyTypeObject CpCStringAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpStringAtom_NAME), + .tp_basicsize = sizeof(CpCStringAtomObject), + .tp_itemsize = 0, + .tp_dealloc = (destructor)cp_cstringatom_dealloc, + .tp_repr = (reprfunc)cp_cstringatom_repr, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = (newfunc)cp_cstringatom_new, + .tp_members = CpCStringAtom_Members, + .tp_init = (initproc)cp_cstringatom_init, + .tp_doc = NULL, +}; \ No newline at end of file diff --git a/src/atomimpl/pstring.c b/src/atomimpl/pstring.c index 45bc2a1..deea57d 100644 --- a/src/atomimpl/pstring.c +++ b/src/atomimpl/pstring.c @@ -51,15 +51,18 @@ cp_pstringatom_init(CpPStringAtomObject* self, PyObject* args, PyObject* kwds) PyObject *encoding = NULL, *errors = NULL, *atom = NULL; if (!PyArg_ParseTupleAndKeywords( - args, kwds, "OO|O", kwlist, &atom, &encoding, &errors)) { + args, kwds, "O|OO", kwlist, &atom, &encoding, &errors)) { return -1; } + _modulestate* mod = get_global_module_state(); _Cp_SetObj(self->m_atom, atom); _Cp_SetObj(self->m_encoding, encoding); + if (!self->m_encoding) { + _Cp_SetObj(self->m_encoding, mod->str_utf8); + } _Cp_SetObj(self->m_errors, errors); if (!self->m_errors) { - _modulestate* mod = get_global_module_state(); _Cp_SetObj(self->m_errors, mod->str_strict); } return 0; diff --git a/src/atomimpl/string.c b/src/atomimpl/string.c index 37730f1..7a98a94 100644 --- a/src/atomimpl/string.c +++ b/src/atomimpl/string.c @@ -61,15 +61,19 @@ cp_stringatom_init(CpStringAtomObject* self, PyObject* args, PyObject* kwds) static char* kwlist[] = { "length", "encoding", "errors", NULL }; PyObject *length = NULL, *encoding = NULL, *errors = NULL; if (!PyArg_ParseTupleAndKeywords( - args, kwds, "OU|U", kwlist, &length, &encoding, &errors)) { + args, kwds, "O|OO", kwlist, &length, &encoding, &errors)) { return -1; } + _modulestate* mod = get_global_module_state(); + _Cp_SetObj(self->m_length, length); _Cp_SetObj(self->m_encoding, encoding); + if (!self->m_encoding) { + _Cp_SetObj(self->m_encoding, mod->str_utf8); + } _Cp_SetObj(self->m_errors, errors); if (!self->m_errors) { - _modulestate* mod = get_global_module_state(); _Cp_SetObj(self->m_errors, mod->str_strict); } return 0; diff --git a/src/capi.dat b/src/capi.dat index 471ed54..65977b6 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -48,6 +48,7 @@ src:atomimpl/enum.c src:atomimpl/varint.c src:atomimpl/computed.c src:atomimpl/lazy.c +src:atomimpl/cstring.c src:atomimpl/builtins/builtin.c src:atomimpl/builtins/repeated.c src:atomimpl/builtins/condition.c @@ -106,6 +107,7 @@ type:39:_enumatomobj:CpEnumAtomObject:- type:40:_varintatomobj:CpVarIntAtomObject:- type:41:_computedatomobj:CpComputedAtomObject:- type:42:_lazyatomobj:CpLazyAtomObject:- +type:43:_cstringatomobj:CpCStringAtomObject:- func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 @@ -225,4 +227,6 @@ func:-:CpComputedAtom_Value:PyObject*:+1 func:-:CpLazyAtom_New:CpLazyAtomObject*:+1 func:164:CpLazyAtom_Pack:int:null func:165:CpLazyAtom_Unpack:PyObject*:+1 -func:-:CpLazyAtom_Atom:PyObject*:+1 \ No newline at end of file +func:-:CpLazyAtom_Atom:PyObject*:+1 +func:166:CpCStringAtom_Pack:int:null +func:167:CpCStringAtom_Unpack:PyObject*:+1 \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/string.h b/src/caterpillar/include/caterpillar/atoms/string.h index c2f3114..80a1052 100644 --- a/src/caterpillar/include/caterpillar/atoms/string.h +++ b/src/caterpillar/include/caterpillar/atoms/string.h @@ -18,6 +18,7 @@ #define STRINGATOMOBJ_H #include "caterpillar/atoms/builtins.h" +#include "caterpillar/parsing.h" // --------------------------------------------------------------------------- // Default String @@ -95,4 +96,50 @@ CpPStringAtom_New(PyObject* atom, PyObject* encoding) &CpPStringAtom_Type, "OO", atom, encoding); } +// --------------------------------------------------------------------------- +// CString + +struct _cstringatomobj +{ + CpBuiltinAtom_HEAD + + PyObject *m_length; + PyObject *m_encoding; + PyObject *m_errors; + PyObject *m_terminator; + PyObject *_m_terminator_bytes; + + int s_callable; + int s_number; + int s_keep_terminator; + int s_greedy; +}; + +#define CpCStringAtom_NAME "cstring" +#define CpCStringAtom_CheckExact(op) Py_IS_TYPE((op), &CpCStringAtom_Type) +#define CpCStringAtom_Check(op) PyObject_TypeCheck((op), &CpCStringAtom_Type) + +static inline CpCStringAtomObject* +CpCStringAtom_New(PyObject* length, PyObject* encoding, PyObject* errors, + PyObject* terminator) +{ + return (CpCStringAtomObject*)CpObject_Create( + &CpCStringAtom_Type, "OOOO", length, encoding, errors, terminator); +} + +static inline PyObject* +CpCStringAtom_Length(CpCStringAtomObject* self, CpLayerObject *layer) +{ + if (self->s_number) { + return Py_NewRef(self->m_length); + } + if (self->s_greedy) { + Py_RETURN_NONE; + } + if (self->s_callable) { + return PyObject_CallOneArg(self->m_length, (PyObject *)layer); + } + return _Cp_Unpack(self->m_length, layer); +} + #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index e1a1dba..e0e6112 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -104,6 +104,8 @@ struct _computedatomobj; typedef struct _computedatomobj CpComputedAtomObject; struct _lazyatomobj; typedef struct _lazyatomobj CpLazyAtomObject; +struct _cstringatomobj; +typedef struct _cstringatomobj CpCStringAtomObject; #ifdef _CPMODULE @@ -155,6 +157,7 @@ extern PyTypeObject CpEnumAtom_Type; extern PyTypeObject CpVarIntAtom_Type; extern PyTypeObject CpComputedAtom_Type; extern PyTypeObject CpLazyAtom_Type; +extern PyTypeObject CpCStringAtom_Type; int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -253,6 +256,8 @@ int CpComputedAtom_Pack(CpComputedAtomObject* self,PyObject* obj,CpLayerObject* PyObject* CpComputedAtom_Unpack(CpComputedAtomObject* self, CpLayerObject* layer); int CpLazyAtom_Pack(CpLazyAtomObject* self, PyObject* obj, CpLayerObject* layer); PyObject* CpLazyAtom_Unpack(CpLazyAtomObject* self, CpLayerObject* layer); +int CpCStringAtom_Pack(CpCStringAtomObject* self,PyObject* value,CpLayerObject* layer); +PyObject* CpCStringAtom_Unpack(CpCStringAtomObject* self, CpLayerObject* layer); #else @@ -305,6 +310,7 @@ caterpillar_api.py #define CpVarIntAtom_Type (*(PyTypeObject *)Cp_API[40]) #define CpComputedAtom_Type (*(PyTypeObject *)Cp_API[41]) #define CpLazyAtom_Type (*(PyTypeObject *)Cp_API[42]) +#define CpCStringAtom_Type (*(PyTypeObject *)Cp_API[43]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) @@ -403,6 +409,8 @@ caterpillar_api.py #define CpComputedAtom_Unpack (*((PyObject* (*)(CpComputedAtomObject* self, CpLayerObject* layer)))Cp_API[163]) #define CpLazyAtom_Pack (*((int (*)(CpLazyAtomObject* self, PyObject* obj, CpLayerObject* layer)))Cp_API[164]) #define CpLazyAtom_Unpack (*((PyObject* (*)(CpLazyAtomObject* self, CpLayerObject* layer)))Cp_API[165]) +#define CpCStringAtom_Pack (*((int (*)(CpCStringAtomObject* self,PyObject* value,CpLayerObject* layer)))Cp_API[166]) +#define CpCStringAtom_Unpack (*((PyObject* (*)(CpCStringAtomObject* self, CpLayerObject* layer)))Cp_API[167]) /** * @brief Public C API for extension modules as reference table diff --git a/src/caterpillar/include/caterpillar/module.h b/src/caterpillar/include/caterpillar/module.h index 0912d68..3bbd970 100644 --- a/src/caterpillar/include/caterpillar/module.h +++ b/src/caterpillar/include/caterpillar/module.h @@ -88,6 +88,8 @@ struct _modulestate PyObject* str_bytesio_getvalue; PyObject* str_builder_process; PyObject* str_pattern_match; + PyObject* str_cstring_default_pad; + PyObject* str_utf8; // compiled regex for unnamed fields PyObject* cp_regex__unnamed; diff --git a/src/caterpillarapi.c b/src/caterpillarapi.c index 6e3d652..1d3d448 100644 --- a/src/caterpillarapi.c +++ b/src/caterpillarapi.c @@ -46,7 +46,7 @@ void *Cp_API[] = { (void *) &CpVarIntAtom_Type, (void *) &CpComputedAtom_Type, (void *) &CpLazyAtom_Type, - NULL, + (void *) &CpCStringAtom_Type, NULL, NULL, NULL, @@ -168,6 +168,8 @@ void *Cp_API[] = { (void *) &CpComputedAtom_Pack, (void *) &CpComputedAtom_Unpack, (void *) &CpLazyAtom_Pack, - (void *) &CpLazyAtom_Unpack + (void *) &CpLazyAtom_Unpack, + (void *) &CpCStringAtom_Pack, + (void *) &CpCStringAtom_Unpack }; diff --git a/src/module.c b/src/module.c index 68d9ee7..1f0e339 100644 --- a/src/module.c +++ b/src/module.c @@ -306,6 +306,8 @@ cp_module_clear(PyObject* m) Py_CLEAR(state->str_strict); Py_CLEAR(state->str__value2member_map_); Py_CLEAR(state->str__member_map_); + Py_CLEAR(state->str_utf8); + Py_CLEAR(state->str_cstring_default_pad); Py_CLEAR(state->cp_regex__unnamed); @@ -414,6 +416,7 @@ PyInit__C(void) CpVarIntAtom_Type.tp_base = &CpBuiltinAtom_Type; CpComputedAtom_Type.tp_base = &CpBuiltinAtom_Type; CpLazyAtom_Type.tp_base = &CpBuiltinAtom_Type; + CpCStringAtom_Type.tp_base = &CpBuiltinAtom_Type; CpModule_SetupType(&CpBuiltinAtom_Type); CpModule_SetupType(&CpPrimitiveAtom_Type); @@ -435,6 +438,7 @@ PyInit__C(void) CpModule_SetupType(&CpVarIntAtom_Type); CpModule_SetupType(&CpComputedAtom_Type); CpModule_SetupType(&CpLazyAtom_Type); + CpModule_SetupType(&CpCStringAtom_Type); // module setup m = PyModule_Create(&CpModule); @@ -486,6 +490,7 @@ PyInit__C(void) CpModule_AddObject(CpVarIntAtom_NAME, &CpVarIntAtom_Type); CpModule_AddObject(CpComputedAtom_NAME, &CpComputedAtom_Type); CpModule_AddObject(CpLazyAtom_NAME, &CpLazyAtom_Type); + CpModule_AddObject(CpCStringAtom_NAME, &CpCStringAtom_Type); /* setup custom intatoms */ #define CpModule_DefAtom(name, ...) \ @@ -621,6 +626,7 @@ PyInit__C(void) CACHED_STRING(str_strict, "strict"); CACHED_STRING(str__member_map_, "_member_map_"); CACHED_STRING(str__value2member_map_, "_value2member_map_"); + CACHED_STRING(str_utf8, "utf-8"); #undef CACHED_STRING @@ -634,6 +640,12 @@ PyInit__C(void) #undef CACHED_BYTES + if ((state->str_cstring_default_pad = PyUnicode_Decode( + PyBytes_AS_STRING(state->cp_bytes__false), 1, "utf-8", "replace")) == + NULL) { + return NULL; + } + // setup typing constants PyObject* typing = PyImport_ImportModule("typing"); if (!typing) { diff --git a/test/_C/atoms/test_string.py b/test/_C/atoms/test_string.py index 5859af3..ab1eb64 100644 --- a/test/_C/atoms/test_string.py +++ b/test/_C/atoms/test_string.py @@ -4,12 +4,13 @@ if caterpillar.native_support(): from caterpillar._C import unpack, pack, string, octetstring, pstring - from caterpillar._C import BIG_ENDIAN as be, u16, u8 + from caterpillar._C import BIG_ENDIAN as be, u16, u8, cstring def test_stringatom_unpack(): # The string atom works as a basic converter between # bytes and unicode objects. assert unpack(b"foo", string(3, "utf-8")) == "foo" + assert unpack(b"foo", string(3)) == "foo" def test_stringatom_pack(): # The string atom works as a basic converter between @@ -34,9 +35,25 @@ def test_pstringatom_pack(): assert pack("foo", pstring(u8, "utf-8")) == b"\x03foo" # NOTE that the byteorder operator will be applied to # the underlying struct. - assert pack("bar", be + pstring(u16, "utf-8")) == b"\x00\x03bar" + assert pack("bar", be + pstring(u16)) == b"\x00\x03bar" def test_pstringatom_unpack(): # Same applies to parsing data assert unpack(b"\x03foo", pstring(u8, "utf-8")) == "foo" - assert unpack(b"\x00\x03bar", be + pstring(u16, "utf-8")) == "bar" \ No newline at end of file + assert unpack(b"\x00\x03bar", be + pstring(u16, "utf-8")) == "bar" + + def test_cstringatom_pack(): + # C-Strings represent a string with a padding + s = cstring(..., "utf-8") + assert pack("foo", s) == b"foo\x00" + assert pack("bar", cstring(3, "utf-8")) == b"bar" + assert pack("baz", cstring(u8, "utf-8")) == b"\x04baz\x00" + assert pack("abc", cstring(4, terminator="\x20")) == b"abc\x20" + + def test_cstringatom_unpack(): + assert unpack(b"foo\x00", cstring(...)) == "foo" + assert unpack(b"ba\x00r", cstring(...)) == "ba" + assert unpack(b"baz\x00", cstring(3)) == "baz" + assert unpack(b"baz\x00", cstring(4)) == "baz" + assert unpack(b"\x04baz\x00", cstring(u8)) == "baz" + assert unpack(b"abc\x20", cstring(4, terminator="\x20")) == "abc" From 9966a7ea44425f54a83b4a71dbee7bc133f2c520 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 15 Sep 2024 13:26:25 +0200 Subject: [PATCH 21/29] Renamed *_t python types --- + int_t -> Int + float_t -> Float + varint_t -> VarInt + const_t -> const + padding_t -> Padding + bool_t -> Bool + char_t -> Char + added missing __bits__ method string to internal module state --- docs/sphinx/source/development/roadmap.rst | 9 +++++++++ docs/sphinx/source/reference/index.rst | 5 +++-- src/caterpillar/include/caterpillar/atoms/const.h | 2 +- src/caterpillar/include/caterpillar/atoms/float.h | 2 +- src/caterpillar/include/caterpillar/atoms/int.h | 2 +- .../include/caterpillar/atoms/primitive.h | 6 +++--- src/caterpillar/include/caterpillar/module.h | 1 + src/module.c | 2 ++ test/_C/atoms/test_padding.py | 12 ++++++------ 9 files changed, 27 insertions(+), 14 deletions(-) diff --git a/docs/sphinx/source/development/roadmap.rst b/docs/sphinx/source/development/roadmap.rst index fb32685..a2ad41a 100644 --- a/docs/sphinx/source/development/roadmap.rst +++ b/docs/sphinx/source/development/roadmap.rst @@ -32,6 +32,15 @@ C API - |uncheck_| Struct wrapper function - |uncheck_| Python docs +Struct Objects: +^^^^^^^^^^^^^^^ + +- |check_| Struct (C type: :c:type:`CpStructObject`, Py type: :class:`Struct`) + [:text-danger:`missing docs`], + [:text-warning:`missing perftest`] + +- |uncheck_| Bitfield + Atom Objects: ^^^^^^^^^^^^^ diff --git a/docs/sphinx/source/reference/index.rst b/docs/sphinx/source/reference/index.rst index 0b9875a..5fdf64a 100644 --- a/docs/sphinx/source/reference/index.rst +++ b/docs/sphinx/source/reference/index.rst @@ -23,14 +23,15 @@ documentation. If you want to switch to the C API instead, please refer to the sections described in the CAPI reference. .. toctree:: - :maxdepth: 3 + :maxdepth: 2 :numbered: :caption: C API Reference + capi/extension capi/abstract capi/objects/objimpl capi/parsing capi/macros capi/module - capi/extension + diff --git a/src/caterpillar/include/caterpillar/atoms/const.h b/src/caterpillar/include/caterpillar/atoms/const.h index 372445d..3631595 100644 --- a/src/caterpillar/include/caterpillar/atoms/const.h +++ b/src/caterpillar/include/caterpillar/atoms/const.h @@ -27,7 +27,7 @@ struct _constatomobj PyObject *m_atom; }; -#define CpConstAtom_NAME "const_t" +#define CpConstAtom_NAME "const" #define CpConstAtom_CheckExact(op) Py_IS_TYPE((op), &CpConstAtom_Type) #define CpConstAtom_Check(op) PyObject_TypeCheck((op), &CpConstAtom_Type) diff --git a/src/caterpillar/include/caterpillar/atoms/float.h b/src/caterpillar/include/caterpillar/atoms/float.h index 752d253..6c2ad8b 100644 --- a/src/caterpillar/include/caterpillar/atoms/float.h +++ b/src/caterpillar/include/caterpillar/atoms/float.h @@ -34,7 +34,7 @@ struct _floatatomobj int _m_little_endian; }; -#define CpFloatAtom_NAME "float_t" +#define CpFloatAtom_NAME "Float" /** @brief Checks if the given object is an integer atom object */ #define CpFloatAtom_CheckExact(op) Py_IS_TYPE((op), &CpFloatAtom_Type) diff --git a/src/caterpillar/include/caterpillar/atoms/int.h b/src/caterpillar/include/caterpillar/atoms/int.h index 1c142b9..9e19050 100644 --- a/src/caterpillar/include/caterpillar/atoms/int.h +++ b/src/caterpillar/include/caterpillar/atoms/int.h @@ -39,7 +39,7 @@ struct _intatomobj int _m_little_endian; }; -#define CpIntAtom_NAME "int_t" +#define CpIntAtom_NAME "Int" /** @brief Checks if the given object is an integer atom object */ #define CpIntAtom_CheckExact(op) Py_IS_TYPE((op), &CpIntAtom_Type) diff --git a/src/caterpillar/include/caterpillar/atoms/primitive.h b/src/caterpillar/include/caterpillar/atoms/primitive.h index 19b3e8b..4a15b68 100644 --- a/src/caterpillar/include/caterpillar/atoms/primitive.h +++ b/src/caterpillar/include/caterpillar/atoms/primitive.h @@ -49,7 +49,7 @@ struct _boolatomobj /// This variable defines the type object for bool atom objects, allowing /// them to be used across the library. // PyAPI_DATA(PyTypeObject) CpBoolAtom_Type; -#define CpBoolAtom_NAME "bool_t" +#define CpBoolAtom_NAME "Bool" /** * @brief Checks if the given object is a bool atom object. @@ -84,7 +84,7 @@ struct _charatomobj /// Char atom object type // PyAPI_DATA(PyTypeObject) CpCharAtom_Type; -#define CpCharAtom_NAME "char_t" +#define CpCharAtom_NAME "Char" /** @brief Checks if the given object is a char atom object */ #define CpCharAtom_CheckExact(op) Py_IS_TYPE((op), &CpCharAtom_Type) @@ -100,7 +100,7 @@ struct _paddingatomobj char _m_padding; }; -#define CpPaddingAtom_NAME "padding_t" +#define CpPaddingAtom_NAME "Padding" #define CpPaddingAtom_CheckExact(op) Py_IS_TYPE((op), &CpPaddingAtom_Type) #define CpPaddingAtom_Check(op) (PyObject_TypeCheck((op), &CpPaddingAtom_Type)) diff --git a/src/caterpillar/include/caterpillar/module.h b/src/caterpillar/include/caterpillar/module.h index 3bbd970..63d3a7f 100644 --- a/src/caterpillar/include/caterpillar/module.h +++ b/src/caterpillar/include/caterpillar/module.h @@ -71,6 +71,7 @@ struct _modulestate PyObject* str___unpack_many__; PyObject* str___size__; PyObject* str___type__; + PyObject* str___bits__; PyObject* str___annotations__; PyObject* str___mro__; PyObject* str___struct__; diff --git a/src/module.c b/src/module.c index 1f0e339..602618f 100644 --- a/src/module.c +++ b/src/module.c @@ -308,6 +308,7 @@ cp_module_clear(PyObject* m) Py_CLEAR(state->str__member_map_); Py_CLEAR(state->str_utf8); Py_CLEAR(state->str_cstring_default_pad); + Py_CLEAR(state->str___bits__); Py_CLEAR(state->cp_regex__unnamed); @@ -627,6 +628,7 @@ PyInit__C(void) CACHED_STRING(str__member_map_, "_member_map_"); CACHED_STRING(str__value2member_map_, "_value2member_map_"); CACHED_STRING(str_utf8, "utf-8"); + CACHED_STRING(str___bits__, "__bits__"); #undef CACHED_STRING diff --git a/test/_C/atoms/test_padding.py b/test/_C/atoms/test_padding.py index e57c4d3..72f8c83 100644 --- a/test/_C/atoms/test_padding.py +++ b/test/_C/atoms/test_padding.py @@ -5,7 +5,7 @@ if caterpillar.native_support(): - from caterpillar._C import unpack, pack, padding, padding_t + from caterpillar._C import unpack, pack, padding, Padding def test_padding_pack(): # The padding atom is special, because it does not need @@ -14,13 +14,13 @@ def test_padding_pack(): # In addition, we can define custom padding values, # which will affect only packing data. - assert pack(None, padding_t(0x20)) == b"\x20" + assert pack(None, Padding(0x20)) == b"\x20" def test_padding_pack_many(): # The padding atom also implements __pack_many__ and # __unpack_many__. (Time measurements are work in progress). assert pack(None, padding[10]) == b"\x00" * 10 - assert pack(None, padding_t(0x20)[10]) == b"\x20" * 10 + assert pack(None, Padding(0x20)[10]) == b"\x20" * 10 # REVISIT: this should return an exception # with pytest.raises(ValueError): @@ -30,13 +30,13 @@ def test_padding_unpack(): # NOTE: unpack using the padding atom always returns None # and DOES validate the parsed value. assert unpack(b"\x00", padding) is None - assert unpack(b"\x20", padding_t(0x20)) is None + assert unpack(b"\x20", Padding(0x20)) is None # Therefore, the following code IS valid if the underlying # stream does not throw an exception. with pytest.raises(ValueError): - unpack(b"\x00" * 10, padding_t(0x02)[10]) + unpack(b"\x00" * 10, Padding(0x02)[10]) def test_padding_unpack_many(): assert unpack(b"\x00" * 10, padding[10]) is None - assert unpack(b"\x20" * 10, padding_t(0x20)[10]) is None + assert unpack(b"\x20" * 10, Padding(0x20)[10]) is None From 875f5205be72fd936a768a27ff9878ac7b9cc710 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 15 Sep 2024 13:36:23 +0200 Subject: [PATCH 22/29] Moved C sources into ccaterpillar directory --- + CpInvalidDefault and CpDefaultSwitchOption now in separate source file --- CMakeLists.txt | 79 ++++++++++--------- src/capi.dat | 64 +++++++-------- .../include/caterpillar/caterpillar.h | 1 + src/caterpillar/include/caterpillar/default.h | 30 +++++++ src/caterpillar/include/caterpillar/module.h | 12 --- src/{ => ccaterpillar}/arch.c | 0 src/{ => ccaterpillar}/atomimpl/bool.c | 0 .../atomimpl/builtins/atoffset.c | 0 .../atomimpl/builtins/builtin.c | 0 .../atomimpl/builtins/condition.c | 0 .../atomimpl/builtins/primitive.c | 0 .../atomimpl/builtins/repeated.c | 0 .../atomimpl/builtins/switch.c | 0 src/{ => ccaterpillar}/atomimpl/bytes.c | 0 src/{ => ccaterpillar}/atomimpl/char.c | 0 src/{ => ccaterpillar}/atomimpl/computed.c | 0 src/{ => ccaterpillar}/atomimpl/const.c | 0 src/{ => ccaterpillar}/atomimpl/cstring.c | 0 src/{ => ccaterpillar}/atomimpl/enum.c | 0 src/{ => ccaterpillar}/atomimpl/float.c | 0 src/{ => ccaterpillar}/atomimpl/int.c | 0 src/{ => ccaterpillar}/atomimpl/lazy.c | 0 src/{ => ccaterpillar}/atomimpl/pad.c | 0 src/{ => ccaterpillar}/atomimpl/pstring.c | 0 src/{ => ccaterpillar}/atomimpl/string.c | 0 src/{ => ccaterpillar}/atomimpl/varint.c | 0 src/{ => ccaterpillar}/atomobj.c | 0 src/{ => ccaterpillar}/caterpillarapi.c | 0 src/{ => ccaterpillar}/caterpillarapi.c.in | 0 src/{ => ccaterpillar}/context.c | 0 src/ccaterpillar/default.c | 63 +++++++++++++++ src/{ => ccaterpillar}/field.c | 4 +- src/{ => ccaterpillar}/layer.c | 0 src/{ => ccaterpillar}/module.c | 60 -------------- src/{ => ccaterpillar}/option.c | 0 src/{ => ccaterpillar}/parsing_pack.c | 0 src/{ => ccaterpillar}/parsing_sizeof.c | 0 src/{ => ccaterpillar}/parsing_typeof.c | 0 src/{ => ccaterpillar}/parsing_unpack.c | 0 src/{ => ccaterpillar}/state.c | 0 src/{ => ccaterpillar}/struct.c | 0 41 files changed, 168 insertions(+), 145 deletions(-) create mode 100644 src/caterpillar/include/caterpillar/default.h rename src/{ => ccaterpillar}/arch.c (100%) rename src/{ => ccaterpillar}/atomimpl/bool.c (100%) rename src/{ => ccaterpillar}/atomimpl/builtins/atoffset.c (100%) rename src/{ => ccaterpillar}/atomimpl/builtins/builtin.c (100%) rename src/{ => ccaterpillar}/atomimpl/builtins/condition.c (100%) rename src/{ => ccaterpillar}/atomimpl/builtins/primitive.c (100%) rename src/{ => ccaterpillar}/atomimpl/builtins/repeated.c (100%) rename src/{ => ccaterpillar}/atomimpl/builtins/switch.c (100%) rename src/{ => ccaterpillar}/atomimpl/bytes.c (100%) rename src/{ => ccaterpillar}/atomimpl/char.c (100%) rename src/{ => ccaterpillar}/atomimpl/computed.c (100%) rename src/{ => ccaterpillar}/atomimpl/const.c (100%) rename src/{ => ccaterpillar}/atomimpl/cstring.c (100%) rename src/{ => ccaterpillar}/atomimpl/enum.c (100%) rename src/{ => ccaterpillar}/atomimpl/float.c (100%) rename src/{ => ccaterpillar}/atomimpl/int.c (100%) rename src/{ => ccaterpillar}/atomimpl/lazy.c (100%) rename src/{ => ccaterpillar}/atomimpl/pad.c (100%) rename src/{ => ccaterpillar}/atomimpl/pstring.c (100%) rename src/{ => ccaterpillar}/atomimpl/string.c (100%) rename src/{ => ccaterpillar}/atomimpl/varint.c (100%) rename src/{ => ccaterpillar}/atomobj.c (100%) rename src/{ => ccaterpillar}/caterpillarapi.c (100%) rename src/{ => ccaterpillar}/caterpillarapi.c.in (100%) rename src/{ => ccaterpillar}/context.c (100%) create mode 100644 src/ccaterpillar/default.c rename src/{ => ccaterpillar}/field.c (99%) rename src/{ => ccaterpillar}/layer.c (100%) rename src/{ => ccaterpillar}/module.c (92%) rename src/{ => ccaterpillar}/option.c (100%) rename src/{ => ccaterpillar}/parsing_pack.c (100%) rename src/{ => ccaterpillar}/parsing_sizeof.c (100%) rename src/{ => ccaterpillar}/parsing_typeof.c (100%) rename src/{ => ccaterpillar}/parsing_unpack.c (100%) rename src/{ => ccaterpillar}/state.c (100%) rename src/{ => ccaterpillar}/struct.c (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 82bedd2..99d1c35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,43 +16,44 @@ python_add_library( _C MODULE - src/arch.c - src/atomobj.c - src/context.c - src/field.c - src/option.c - src/module.c - src/context.c - src/state.c - src/struct.c - src/layer.c - - src/parsing_typeof.c - src/parsing_sizeof.c - src/parsing_pack.c - src/parsing_unpack.c - - src/atomimpl/int.c - src/atomimpl/float.c - "src/atomimpl/bool.c" - src/atomimpl/char.c - src/atomimpl/pad.c - "src/atomimpl/string.c" - src/atomimpl/const.c - src/atomimpl/bytes.c - src/atomimpl/pstring.c - src/atomimpl/enum.c - src/atomimpl/varint.c - src/atomimpl/computed.c - src/atomimpl/lazy.c - src/atomimpl/cstring.c - - src/atomimpl/builtins/builtin.c - src/atomimpl/builtins/repeated.c - src/atomimpl/builtins/condition.c - src/atomimpl/builtins/switch.c - src/atomimpl/builtins/atoffset.c - src/atomimpl/builtins/primitive.c + src/ccaterpillar/arch.c + src/ccaterpillar/atomobj.c + src/ccaterpillar/context.c + src/ccaterpillar/field.c + src/ccaterpillar/option.c + src/ccaterpillar/module.c + src/ccaterpillar/context.c + src/ccaterpillar/state.c + src/ccaterpillar/struct.c + src/ccaterpillar/layer.c + src/ccaterpillar/default.c + + src/ccaterpillar/parsing_typeof.c + src/ccaterpillar/parsing_sizeof.c + src/ccaterpillar/parsing_pack.c + src/ccaterpillar/parsing_unpack.c + + src/ccaterpillar/atomimpl/int.c + src/ccaterpillar/atomimpl/float.c + src/ccaterpillar/atomimpl/bool.c + src/ccaterpillar/atomimpl/char.c + src/ccaterpillar/atomimpl/pad.c + src/ccaterpillar/atomimpl/string.c + src/ccaterpillar/atomimpl/const.c + src/ccaterpillar/atomimpl/bytes.c + src/ccaterpillar/atomimpl/pstring.c + src/ccaterpillar/atomimpl/enum.c + src/ccaterpillar/atomimpl/varint.c + src/ccaterpillar/atomimpl/computed.c + src/ccaterpillar/atomimpl/lazy.c + src/ccaterpillar/atomimpl/cstring.c + + src/ccaterpillar/atomimpl/builtins/builtin.c + src/ccaterpillar/atomimpl/builtins/repeated.c + src/ccaterpillar/atomimpl/builtins/condition.c + src/ccaterpillar/atomimpl/builtins/switch.c + src/ccaterpillar/atomimpl/builtins/atoffset.c + src/ccaterpillar/atomimpl/builtins/primitive.c WITH_SOABI ) @@ -64,9 +65,9 @@ message(STATUS "Python ${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}") set (CP_GENAPI_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/src/code_gen/genapi.py) set (CP_CAPI_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h.in) -set (CP_CAPI_CSRC ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillarapi.c.in) +set (CP_CAPI_CSRC ${CMAKE_CURRENT_SOURCE_DIR}/src/ccaterpillar/caterpillarapi.c.in) set (CP_CAPI_HEADER_OUT ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h) -set (CP_CAPI_CSRC_OUT ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillarapi.c) +set (CP_CAPI_CSRC_OUT ${CMAKE_CURRENT_SOURCE_DIR}/src/ccaterpillar/caterpillarapi.c) add_custom_target( genapi ALL diff --git a/src/capi.dat b/src/capi.dat index 65977b6..31c3207 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -23,38 +23,38 @@ # Defines the source file (relative to this file) that contains the # function definitions. -src:field.c -src:context.c -src:arch.c -src:atomobj.c -src:option.c -src:struct.c -src:state.c -src:layer.c -src:parsing_pack.c -src:parsing_unpack.c -src:parsing_typeof.c -src:parsing_sizeof.c -src:atomimpl/bool.c -src:atomimpl/float.c -src:atomimpl/char.c -src:atomimpl/int.c -src:atomimpl/pad.c -src:atomimpl/string.c -src:atomimpl/const.c -src:atomimpl/bytes.c -src:atomimpl/pstring.c -src:atomimpl/enum.c -src:atomimpl/varint.c -src:atomimpl/computed.c -src:atomimpl/lazy.c -src:atomimpl/cstring.c -src:atomimpl/builtins/builtin.c -src:atomimpl/builtins/repeated.c -src:atomimpl/builtins/condition.c -src:atomimpl/builtins/switch.c -src:atomimpl/builtins/atoffset.c -src:atomimpl/builtins/primitive.c +src:ccaterpillar/field.c +src:ccaterpillar/context.c +src:ccaterpillar/arch.c +src:ccaterpillar/atomobj.c +src:ccaterpillar/option.c +src:ccaterpillar/struct.c +src:ccaterpillar/state.c +src:ccaterpillar/layer.c +src:ccaterpillar/parsing_pack.c +src:ccaterpillar/parsing_unpack.c +src:ccaterpillar/parsing_typeof.c +src:ccaterpillar/parsing_sizeof.c +src:ccaterpillar/atomimpl/bool.c +src:ccaterpillar/atomimpl/float.c +src:ccaterpillar/atomimpl/char.c +src:ccaterpillar/atomimpl/int.c +src:ccaterpillar/atomimpl/pad.c +src:ccaterpillar/atomimpl/string.c +src:ccaterpillar/atomimpl/const.c +src:ccaterpillar/atomimpl/bytes.c +src:ccaterpillar/atomimpl/pstring.c +src:ccaterpillar/atomimpl/enum.c +src:ccaterpillar/atomimpl/varint.c +src:ccaterpillar/atomimpl/computed.c +src:ccaterpillar/atomimpl/lazy.c +src:ccaterpillar/atomimpl/cstring.c +src:ccaterpillar/atomimpl/builtins/builtin.c +src:ccaterpillar/atomimpl/builtins/repeated.c +src:ccaterpillar/atomimpl/builtins/condition.c +src:ccaterpillar/atomimpl/builtins/switch.c +src:ccaterpillar/atomimpl/builtins/atoffset.c +src:ccaterpillar/atomimpl/builtins/primitive.c # First index is reserved for the global module reference obj:0:CpModule:PyModuleDef diff --git a/src/caterpillar/include/caterpillar/caterpillar.h b/src/caterpillar/include/caterpillar/caterpillar.h index a2a611f..17aea62 100644 --- a/src/caterpillar/include/caterpillar/caterpillar.h +++ b/src/caterpillar/include/caterpillar/caterpillar.h @@ -14,6 +14,7 @@ #include "caterpillar/field.h" #include "caterpillar/state.h" #include "caterpillar/parsing.h" +#include "caterpillar/default.h" // Atom Objects #include "caterpillar/atoms/builtins.h" diff --git a/src/caterpillar/include/caterpillar/default.h b/src/caterpillar/include/caterpillar/default.h new file mode 100644 index 0000000..eb47cd0 --- /dev/null +++ b/src/caterpillar/include/caterpillar/default.h @@ -0,0 +1,30 @@ +/** + * Copyright (C) MatrixEditor 2024 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef CP_DEFAULT_H +#define CP_DEFAULT_H + +#include "caterpillarapi.h" + +/* immortal objects */ + +#define CpInvalidDefault (&_CpInvalidDefault_Object) +#define Cp_IsInvalidDefault(o) ((o) == CpInvalidDefault) + +#define CpDefaultOption (&_CpDefaultOption_Object) +#define Cp_IsDefaultOption(o) ((o) == CpDefaultOption) + +#endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/module.h b/src/caterpillar/include/caterpillar/module.h index 63d3a7f..bdb5b75 100644 --- a/src/caterpillar/include/caterpillar/module.h +++ b/src/caterpillar/include/caterpillar/module.h @@ -126,18 +126,6 @@ get_global_module_state(void) return get_module_state(PyState_FindModule(&CpModule)); } -/* immortal objects */ -// PyAPI_DATA(PyTypeObject) CpInvalidDefault_Type; -// PyAPI_DATA(PyTypeObject) CpDefaultOption_Type; - -// PyAPI_DATA(PyObject) _CpInvalidDefault_Object; -#define CpInvalidDefault (&_CpInvalidDefault_Object) -#define Cp_IsInvalidDefault(o) ((o) == CpInvalidDefault) - -// PyAPI_DATA(PyObject) _CpDefaultOption_Object; -#define CpDefaultOption (&_CpDefaultOption_Object) -#define Cp_IsDefaultOption(o) ((o) == CpDefaultOption) - /* utility macros */ #define CpModule_SetupType(op) \ if (PyType_Ready(op) < 0) \ diff --git a/src/arch.c b/src/ccaterpillar/arch.c similarity index 100% rename from src/arch.c rename to src/ccaterpillar/arch.c diff --git a/src/atomimpl/bool.c b/src/ccaterpillar/atomimpl/bool.c similarity index 100% rename from src/atomimpl/bool.c rename to src/ccaterpillar/atomimpl/bool.c diff --git a/src/atomimpl/builtins/atoffset.c b/src/ccaterpillar/atomimpl/builtins/atoffset.c similarity index 100% rename from src/atomimpl/builtins/atoffset.c rename to src/ccaterpillar/atomimpl/builtins/atoffset.c diff --git a/src/atomimpl/builtins/builtin.c b/src/ccaterpillar/atomimpl/builtins/builtin.c similarity index 100% rename from src/atomimpl/builtins/builtin.c rename to src/ccaterpillar/atomimpl/builtins/builtin.c diff --git a/src/atomimpl/builtins/condition.c b/src/ccaterpillar/atomimpl/builtins/condition.c similarity index 100% rename from src/atomimpl/builtins/condition.c rename to src/ccaterpillar/atomimpl/builtins/condition.c diff --git a/src/atomimpl/builtins/primitive.c b/src/ccaterpillar/atomimpl/builtins/primitive.c similarity index 100% rename from src/atomimpl/builtins/primitive.c rename to src/ccaterpillar/atomimpl/builtins/primitive.c diff --git a/src/atomimpl/builtins/repeated.c b/src/ccaterpillar/atomimpl/builtins/repeated.c similarity index 100% rename from src/atomimpl/builtins/repeated.c rename to src/ccaterpillar/atomimpl/builtins/repeated.c diff --git a/src/atomimpl/builtins/switch.c b/src/ccaterpillar/atomimpl/builtins/switch.c similarity index 100% rename from src/atomimpl/builtins/switch.c rename to src/ccaterpillar/atomimpl/builtins/switch.c diff --git a/src/atomimpl/bytes.c b/src/ccaterpillar/atomimpl/bytes.c similarity index 100% rename from src/atomimpl/bytes.c rename to src/ccaterpillar/atomimpl/bytes.c diff --git a/src/atomimpl/char.c b/src/ccaterpillar/atomimpl/char.c similarity index 100% rename from src/atomimpl/char.c rename to src/ccaterpillar/atomimpl/char.c diff --git a/src/atomimpl/computed.c b/src/ccaterpillar/atomimpl/computed.c similarity index 100% rename from src/atomimpl/computed.c rename to src/ccaterpillar/atomimpl/computed.c diff --git a/src/atomimpl/const.c b/src/ccaterpillar/atomimpl/const.c similarity index 100% rename from src/atomimpl/const.c rename to src/ccaterpillar/atomimpl/const.c diff --git a/src/atomimpl/cstring.c b/src/ccaterpillar/atomimpl/cstring.c similarity index 100% rename from src/atomimpl/cstring.c rename to src/ccaterpillar/atomimpl/cstring.c diff --git a/src/atomimpl/enum.c b/src/ccaterpillar/atomimpl/enum.c similarity index 100% rename from src/atomimpl/enum.c rename to src/ccaterpillar/atomimpl/enum.c diff --git a/src/atomimpl/float.c b/src/ccaterpillar/atomimpl/float.c similarity index 100% rename from src/atomimpl/float.c rename to src/ccaterpillar/atomimpl/float.c diff --git a/src/atomimpl/int.c b/src/ccaterpillar/atomimpl/int.c similarity index 100% rename from src/atomimpl/int.c rename to src/ccaterpillar/atomimpl/int.c diff --git a/src/atomimpl/lazy.c b/src/ccaterpillar/atomimpl/lazy.c similarity index 100% rename from src/atomimpl/lazy.c rename to src/ccaterpillar/atomimpl/lazy.c diff --git a/src/atomimpl/pad.c b/src/ccaterpillar/atomimpl/pad.c similarity index 100% rename from src/atomimpl/pad.c rename to src/ccaterpillar/atomimpl/pad.c diff --git a/src/atomimpl/pstring.c b/src/ccaterpillar/atomimpl/pstring.c similarity index 100% rename from src/atomimpl/pstring.c rename to src/ccaterpillar/atomimpl/pstring.c diff --git a/src/atomimpl/string.c b/src/ccaterpillar/atomimpl/string.c similarity index 100% rename from src/atomimpl/string.c rename to src/ccaterpillar/atomimpl/string.c diff --git a/src/atomimpl/varint.c b/src/ccaterpillar/atomimpl/varint.c similarity index 100% rename from src/atomimpl/varint.c rename to src/ccaterpillar/atomimpl/varint.c diff --git a/src/atomobj.c b/src/ccaterpillar/atomobj.c similarity index 100% rename from src/atomobj.c rename to src/ccaterpillar/atomobj.c diff --git a/src/caterpillarapi.c b/src/ccaterpillar/caterpillarapi.c similarity index 100% rename from src/caterpillarapi.c rename to src/ccaterpillar/caterpillarapi.c diff --git a/src/caterpillarapi.c.in b/src/ccaterpillar/caterpillarapi.c.in similarity index 100% rename from src/caterpillarapi.c.in rename to src/ccaterpillar/caterpillarapi.c.in diff --git a/src/context.c b/src/ccaterpillar/context.c similarity index 100% rename from src/context.c rename to src/ccaterpillar/context.c diff --git a/src/ccaterpillar/default.c b/src/ccaterpillar/default.c new file mode 100644 index 0000000..cc02764 --- /dev/null +++ b/src/ccaterpillar/default.c @@ -0,0 +1,63 @@ +/* immortal objects */ + +#include "caterpillar/caterpillar.h" + +static PyObject* +cp_invaliddefault_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + if (PyTuple_GET_SIZE(args) || PyDict_GET_SIZE(kw)) { + PyErr_SetString(PyExc_TypeError, + "InvalidDefaultType does not accept arguments"); + return NULL; + } + + Py_INCREF(CpInvalidDefault); + return CpInvalidDefault; +} + +static PyObject* +cp_invaliddefault_repr(PyObject* self) +{ + return PyUnicode_FromString(""); +} + +PyTypeObject CpInvalidDefault_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(InvalidDefaultType), + .tp_repr = (reprfunc)cp_invaliddefault_repr, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = (newfunc)cp_invaliddefault_new, +}; + +PyObject _CpInvalidDefault_Object = { _PyObject_EXTRA_INIT{ + _Py_IMMORTAL_REFCNT /* ob_refcnt */ }, + &CpInvalidDefault_Type /* ob_type */ }; + +static PyObject* +cp_defaultoption_new(PyTypeObject* type, PyObject* args, PyObject* kw) +{ + if (PyTuple_GET_SIZE(args) || PyDict_GET_SIZE(kw)) { + PyErr_SetString(PyExc_TypeError, + "InvalidDefaultType does not accept arguments"); + return NULL; + } + + Py_INCREF(CpDefaultOption); + return CpDefaultOption; +} + +static PyObject* +cp_defaultoption_repr(PyObject* self) +{ + return PyUnicode_FromString(""); +} + +PyTypeObject CpDefaultOption_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(DefaultOptionType), + .tp_repr = (reprfunc)cp_defaultoption_repr, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = (newfunc)cp_defaultoption_new, +}; + +PyObject _CpDefaultOption_Object = { _PyObject_EXTRA_INIT{ + _Py_IMMORTAL_REFCNT /* ob_refcnt */ }, + &CpDefaultOption_Type /* ob_type */ }; diff --git a/src/field.c b/src/ccaterpillar/field.c similarity index 99% rename from src/field.c rename to src/ccaterpillar/field.c index d8e0f0f..8a6346b 100644 --- a/src/field.c +++ b/src/ccaterpillar/field.c @@ -1,8 +1,8 @@ /* field implementation */ #include -#include "caterpillar/field.h" -#include "caterpillar/module.h" +#include "caterpillar//caterpillar.h" + #include "structmember.h" /* impl */ diff --git a/src/layer.c b/src/ccaterpillar/layer.c similarity index 100% rename from src/layer.c rename to src/ccaterpillar/layer.c diff --git a/src/module.c b/src/ccaterpillar/module.c similarity index 92% rename from src/module.c rename to src/ccaterpillar/module.c index 602618f..8fc384f 100644 --- a/src/module.c +++ b/src/ccaterpillar/module.c @@ -33,66 +33,6 @@ return NULL; \ } -/* immortal objects */ -static PyObject* -cp_invaliddefault_new(PyTypeObject* type, PyObject* args, PyObject* kw) -{ - if (PyTuple_GET_SIZE(args) || PyDict_GET_SIZE(kw)) { - PyErr_SetString(PyExc_TypeError, - "InvalidDefaultType does not accept arguments"); - return NULL; - } - - Py_INCREF(CpInvalidDefault); - return CpInvalidDefault; -} - -static PyObject* -cp_invaliddefault_repr(PyObject* self) -{ - return PyUnicode_FromString(""); -} - -PyTypeObject CpInvalidDefault_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(InvalidDefaultType), - .tp_repr = (reprfunc)cp_invaliddefault_repr, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = (newfunc)cp_invaliddefault_new, -}; - -PyObject _CpInvalidDefault_Object = { _PyObject_EXTRA_INIT{ - _Py_IMMORTAL_REFCNT /* ob_refcnt */ }, - &CpInvalidDefault_Type /* ob_type */ }; - -static PyObject* -cp_defaultoption_new(PyTypeObject* type, PyObject* args, PyObject* kw) -{ - if (PyTuple_GET_SIZE(args) || PyDict_GET_SIZE(kw)) { - PyErr_SetString(PyExc_TypeError, - "InvalidDefaultType does not accept arguments"); - return NULL; - } - - Py_INCREF(CpDefaultOption); - return CpDefaultOption; -} - -static PyObject* -cp_defaultoption_repr(PyObject* self) -{ - return PyUnicode_FromString(""); -} - -PyTypeObject CpDefaultOption_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(DefaultOptionType), - .tp_repr = (reprfunc)cp_defaultoption_repr, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = (newfunc)cp_defaultoption_new, -}; - -PyObject _CpDefaultOption_Object = { _PyObject_EXTRA_INIT{ - _Py_IMMORTAL_REFCNT /* ob_refcnt */ }, - &CpDefaultOption_Type /* ob_type */ }; // ------------------------------------------------------------------------------ // module diff --git a/src/option.c b/src/ccaterpillar/option.c similarity index 100% rename from src/option.c rename to src/ccaterpillar/option.c diff --git a/src/parsing_pack.c b/src/ccaterpillar/parsing_pack.c similarity index 100% rename from src/parsing_pack.c rename to src/ccaterpillar/parsing_pack.c diff --git a/src/parsing_sizeof.c b/src/ccaterpillar/parsing_sizeof.c similarity index 100% rename from src/parsing_sizeof.c rename to src/ccaterpillar/parsing_sizeof.c diff --git a/src/parsing_typeof.c b/src/ccaterpillar/parsing_typeof.c similarity index 100% rename from src/parsing_typeof.c rename to src/ccaterpillar/parsing_typeof.c diff --git a/src/parsing_unpack.c b/src/ccaterpillar/parsing_unpack.c similarity index 100% rename from src/parsing_unpack.c rename to src/ccaterpillar/parsing_unpack.c diff --git a/src/state.c b/src/ccaterpillar/state.c similarity index 100% rename from src/state.c rename to src/ccaterpillar/state.c diff --git a/src/struct.c b/src/ccaterpillar/struct.c similarity index 100% rename from src/struct.c rename to src/ccaterpillar/struct.c From 29d91e9ddebc2526d28e876b6967298a104b0618 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 15 Sep 2024 14:47:13 +0200 Subject: [PATCH 23/29] CpIntAtomObject Docs --- CMakeLists.txt | 2 +- docs/sphinx/source/library/ctypes/int.rst | 62 +++++++++++++++ docs/sphinx/source/library/index.rst | 16 ++-- .../reference/capi/objects/atoms/intatom.rst | 60 +++++++++++---- src/ccaterpillar/atomimpl/int.c | 75 +++++++++++++++---- 5 files changed, 173 insertions(+), 42 deletions(-) create mode 100644 docs/sphinx/source/library/ctypes/int.rst diff --git a/CMakeLists.txt b/CMakeLists.txt index 99d1c35..7e664f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,7 +71,7 @@ set (CP_CAPI_CSRC_OUT ${CMAKE_CURRENT_SOURCE_DIR}/src/ccaterpillar/caterpillarap add_custom_target( genapi ALL - # execute python ./src/code_gen/genapi.py ./src/caterpillar/include/caterpillar/caterpillarapi.h.in ./src/caterpillarapi.c.in + # execute python ./src/code_gen/genapi.py ./src/caterpillar/include/caterpillar/caterpillarapi.h.in ./src/ccaterpillar/caterpillarapi.c.in # in the root directory COMMAND ${PYTHON_EXECUTABLE} ${CP_GENAPI_SCRIPT} ${CP_CAPI_HEADER} ${CP_CAPI_CSRC} BYPRODUCTS ${CP_CAPI_HEADER_OUT} ${CP_CAPI_CSRC_OUT} diff --git a/docs/sphinx/source/library/ctypes/int.rst b/docs/sphinx/source/library/ctypes/int.rst new file mode 100644 index 0000000..55872f5 --- /dev/null +++ b/docs/sphinx/source/library/ctypes/int.rst @@ -0,0 +1,62 @@ +.. _api-ctypes_int: + +************* +Integer Types +************* + +.. autoclass:: caterpillar.c.Int + :members: + + +.. rubric:: Pre-Defined Integer Types (global) + +All defined integer types are set to use little-endian by default. Apply +the byteorder operation to change endianess. + +.. py:data:: caterpillar.c.u8 + + Unsigned 8-bit integer type. + +.. py:data:: caterpillar.c.u16 + + Unsigned 16-bit integer type. + +.. py:data:: caterpillar.c.u24 + + Unsigned 24-bit integer type. + +.. py:data:: caterpillar.c.u32 + + Unsigned 32-bit integer type. + +.. py:data:: caterpillar.c.u64 + + Unsigned 64-bit integer type. + +.. py:data:: caterpillar.c.u128 + + Unsigned 128-bit integer type. + +.. py:data:: caterpillar.c.i8 + + Signed 8-bit integer type. + +.. py:data:: caterpillar.c.i16 + + Signed 16-bit integer type. + +.. py:data:: caterpillar.c.i24 + + Signed 24-bit integer type. + +.. py:data:: caterpillar.c.i32 + + Signed 32-bit integer type. + +.. py:data:: caterpillar.c.i64 + + Signed 64-bit integer type. + +.. py:data:: caterpillar.c.i128 + + Signed 128-bit integer type. diff --git a/docs/sphinx/source/library/index.rst b/docs/sphinx/source/library/index.rst index 7e1c14c..cada89b 100644 --- a/docs/sphinx/source/library/index.rst +++ b/docs/sphinx/source/library/index.rst @@ -8,25 +8,19 @@ Library .. toctree:: :maxdepth: 2 - :caption: Basic Core classes + :caption: Python API byteorder.rst options.rst abc.rst - - -.. toctree:: - :maxdepth: 2 - :caption: Context and Exceptions - context.rst exceptions.rst + model.rst + fields/index.rst .. toctree:: :maxdepth: 2 - :caption: Struct model - - model.rst - fields/index.rst + :caption: C API Python Types + ctypes/int.rst diff --git a/docs/sphinx/source/reference/capi/objects/atoms/intatom.rst b/docs/sphinx/source/reference/capi/objects/atoms/intatom.rst index b1e0043..94423d0 100644 --- a/docs/sphinx/source/reference/capi/objects/atoms/intatom.rst +++ b/docs/sphinx/source/reference/capi/objects/atoms/intatom.rst @@ -11,7 +11,7 @@ counterpart (:class:`FormatField`), this C-based class focuses solely on integer .. c:var:: PyTypeObject CpIntAtom_Type - The type object for the :c:type:`int_t` class. + The type object for the :class:`~caterpillar.c.Int` class. This implementation utilizes :code:`int.from_bytes` and :code:`int.to_bytes`. Direct C calls are optimized, reducing runtime overhead compared to Python. @@ -53,26 +53,56 @@ calls are optimized, reducing runtime overhead compared to Python. Checks if the given object is an instance of an :c:type:`CpIntAtomObject` -Recommendations ---------------- -The following examples illustrate how to effectively utilize the :c:type:`int_t` class -and the associated methods: +.. rubric:: Runtime Performance + +Measurements represent the accumulated runtime of one million calls to +:code:`unpack` or :code:`pack` using the corresponding implementation +in seconds. + +.. list-table:: + :header-rows: 1 + :widths: 20 20 20 20 20 + + * - Function + - Caterpillar [0]_ + - Caterpillar G [1]_ + - Caterpillar C [2]_ + - Construct [3]_ + * - :code:`unpack` + - 4.002179 + - 2.782663 + - 0.815902 + - 1.581962 + * - :code:`pack` + - 3.866999 + - 2.707753 + - 0.926041 + - 1.587046 + + +.. [0] local field instance +.. [1] global field instance +.. [2] C implementation +.. [3] Construct :code:`Int32sn` + +The *benchmark* has been performed using the following code snippets: .. code-block:: python :linenos: - from caterpillar._C import LITTLE_ENDIAN as le - from caterpillar._C import int_t, unpack, i16 + from caterpillar import _C, model, fields + from construct import Int32sn - # Define a global 16-bit little-endian signed integer atom - I16_LE = le + i16 - _I16_LE = int_t(16, signed=True, little_endian=True) + # Caterpillar + model.unpack(fields.Field(fields.int32), b"\x00\xFF\x00\xFF") - unpack(b"\x01\x02", _I16_LE) - unpack(b"\x01\x02", int_t(16, signed=True, little_endian=True)) - unpack(b"\x01\x02", I16_LE) - unpack(b"\x01\x02", le + i16) + # Caterpillar (Global) + I32_G = fields.Field(fields.int32) + model.unpack(I32_G, b"\x00\xFF\x00\xFF") -*TODO: describe the impact on runtime overhead of these four methods* + # Caterpillar (C) + _C.unpack(b"\x00\xFF\x00\xFF", _C.i32) + # Construct + Int32sn.parse(b"\x00\xFF\x00\xFF") diff --git a/src/ccaterpillar/atomimpl/int.c b/src/ccaterpillar/atomimpl/int.c index 839310a..461fb4b 100644 --- a/src/ccaterpillar/atomimpl/int.c +++ b/src/ccaterpillar/atomimpl/int.c @@ -163,6 +163,38 @@ CpIntAtom_Unpack(CpIntAtomObject* self, CpLayerObject* layer) } /* docs */ +// clang-format off +PyDoc_STRVAR(cp_intatom_member__nbytes__doc, +"Stores the amount of bytes in the atom (not the number of bits)."); + +PyDoc_STRVAR(cp_intatom_member__nbits__doc, +"Stores the amount of bits in the atom."); + +PyDoc_STRVAR(cp_intatom_member__signed__doc, +"Stores whether the atom is signed or not."); + +PyDoc_STRVAR(cp_intatom_member__little_endian__doc, +"Stores whether the atom is little endian or not."); + +PyDoc_STRVAR(cp_intatom__doc, +"Int(bits, signed=True, little_endian=True)\n" +"--\n" +"\n" +"Implements an integer atom with a variable amount of bits.\n" +"\n" +"bits\n" +" The amount of bits in the atom.\n" +"signed\n" +" Whether the atom is signed or not.\n" +"little_endian\n" +" Whether the atom is little endian or not.\n" +"\n" +"Unlike its Python counterpart (caterpillar.py.FormatField), this C-based " +"class focuses solely on integer operations. It utilizes `int.from_bytes` and " +"`int.to_bytes` internally, optimizing runtime performance. However, it is " +"recommended to use the global instances of this class instead of creating " +"new objects directly."); +// clang-format on /* type */ static PyMemberDef CpIntAtom_Members[] = { @@ -170,10 +202,22 @@ static PyMemberDef CpIntAtom_Members[] = { T_PYSSIZET, offsetof(CpIntAtomObject, _m_byte_count), READONLY, - NULL }, - { "nbits", T_PYSSIZET, offsetof(CpIntAtomObject, _m_bits), READONLY, NULL }, - { "signed", T_BOOL, offsetof(CpIntAtomObject, _m_signed), READONLY, NULL }, - { "little_endian", T_BOOL, offsetof(CpIntAtomObject, _m_little_endian) }, + cp_intatom_member__nbytes__doc }, + { "nbits", + T_PYSSIZET, + offsetof(CpIntAtomObject, _m_bits), + READONLY, + cp_intatom_member__nbits__doc }, + { "signed", + T_BOOL, + offsetof(CpIntAtomObject, _m_signed), + READONLY, + cp_intatom_member__signed__doc }, + { "little_endian", + T_BOOL, + offsetof(CpIntAtomObject, _m_little_endian), + READONLY, + cp_intatom_member__little_endian__doc }, { NULL } /* Sentinel */ }; @@ -182,14 +226,15 @@ static PyMethodDef CpIntAtom_Methods[] = { { NULL } /* Sentinel */ }; -PyTypeObject CpIntAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) - _Cp_NameStr(CpIntAtom_NAME), - .tp_basicsize = sizeof(CpIntAtomObject), - .tp_dealloc = (destructor)cp_intatom_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = NULL, - .tp_members = CpIntAtom_Members, - .tp_new = (newfunc)cp_intatom_new, - .tp_init = (initproc)cp_intatom_init, - .tp_repr = (reprfunc)cp_intatom_repr, - .tp_methods = CpIntAtom_Methods }; +/* --- */ PyTypeObject CpIntAtom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpIntAtom_NAME), + .tp_basicsize = sizeof(CpIntAtomObject), + .tp_dealloc = (destructor)cp_intatom_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = cp_intatom__doc, + .tp_members = CpIntAtom_Members, + .tp_new = (newfunc)cp_intatom_new, + .tp_init = (initproc)cp_intatom_init, + .tp_repr = (reprfunc)cp_intatom_repr, + .tp_methods = CpIntAtom_Methods +}; From 4eac7b185b8a2117586ede541f12ebbe9add60c4 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 15 Sep 2024 21:24:06 +0200 Subject: [PATCH 24/29] Added Python class names of C types to capi.dat --- + c.Int Python type documentation + added copybutton to docs --- docs/requirements.txt | 3 +- docs/sphinx/source/conf.py | 3 +- .../sphinx/source/extensions/c_annotations.py | 113 +++++++++----- docs/sphinx/source/installing/index.rst | 10 +- docs/sphinx/source/library/ctypes/int.rst | 140 +++++++++++++++++- src/capi.dat | 82 +++++----- src/caterpillar/include/caterpillar/atomobj.h | 2 - .../include/caterpillar/atoms/builtins.h | 5 - .../include/caterpillar/atoms/const.h | 1 - .../include/caterpillar/atoms/enum.h | 1 - .../include/caterpillar/atoms/float.h | 4 - .../include/caterpillar/atoms/int.h | 3 - .../include/caterpillar/atoms/primitive.h | 8 - .../include/caterpillar/atoms/string.h | 4 - .../include/caterpillar/caterpillarapi.h | 117 ++++++++++----- src/caterpillar/include/caterpillar/state.h | 2 - src/ccaterpillar/module.c | 32 ++-- src/code_gen/genapi.py | 6 +- 18 files changed, 360 insertions(+), 176 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index c1500b1..ad16558 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,5 @@ sphinx pydata-sphinx-theme sphinx-design -breathe \ No newline at end of file +breathe +sphinx-copybutton \ No newline at end of file diff --git a/docs/sphinx/source/conf.py b/docs/sphinx/source/conf.py index c6c7136..efcc88a 100644 --- a/docs/sphinx/source/conf.py +++ b/docs/sphinx/source/conf.py @@ -27,7 +27,8 @@ "sphinx.ext.viewcode", "sphinx_design", "breathe", - "c_annotations" + "c_annotations", + "sphinx_copybutton" ] templates_path = ["_templates"] diff --git a/docs/sphinx/source/extensions/c_annotations.py b/docs/sphinx/source/extensions/c_annotations.py index 419691c..9ef9c98 100644 --- a/docs/sphinx/source/extensions/c_annotations.py +++ b/docs/sphinx/source/extensions/c_annotations.py @@ -6,9 +6,10 @@ from os import path import docutils from docutils import nodes +from sphinx.addnodes import pending_xref from sphinx.locale import _ as sphinx_gettext -from sphinx import addnodes +from sphinx import addnodes, application class RCEntry: @@ -21,6 +22,7 @@ def __init__(self, name): class Annotations: def __init__(self, refcount_filename): self.refcount_data = {} + self.typedef_data = {} with open(refcount_filename, "r", encoding="utf-8") as fp: for line in fp: line = line.strip() @@ -28,49 +30,82 @@ def __init__(self, refcount_filename): # blank lines and comments continue def_type, *parts = line.split(":") - if def_type != "func": - continue + match def_type: + case "func": + index, name, rtype, refcount = parts + # Get the entry, creating it if needed: + entry = self.refcount_data.get(name) + if not entry: + entry = self.refcount_data[name] = RCEntry(name) + if not refcount or refcount == "null": + refcount = None + else: + refcount = int(refcount) + entry.result_type = rtype + entry.result_refs = refcount + + case "type": + index, struct_name, typedef_name, py_type = parts + if py_type != "-": + self.typedef_data[py_type] = typedef_name + + def add_py_annotations( + self, app: application.Sphinx, node: docutils.nodes.Element, par + ): + + if not par[0].has_key("ids") or not par[0]["ids"]: + return - index, name, rtype, refcount = parts - # Get the entry, creating it if needed: - entry = self.refcount_data.get(name) - if not entry: - entry = self.refcount_data[name] = RCEntry(name) - if not refcount or refcount == "null": - refcount = None - else: - refcount = int(refcount) - entry.result_type = rtype - entry.result_refs = refcount + name = par[0]["ids"][0] + if not name.startswith("caterpillar.c."): + return + + if par["objtype"] != "class": + return + + name = name[14:] + if name in self.typedef_data: + typedef_name = self.typedef_data[name] + + rc = sphinx_gettext(f"C API Type: {typedef_name}") + node.insert(0, nodes.emphasis(rc, rc, classes=["refcount", "text-info"])) + node.insert(1, nodes.line()) + + def add_c_annotation(self, app, node, par): + if not par[0].has_key("ids") or not par[0]["ids"]: + return + + name = par[0]["ids"][0] + if name.startswith("c."): + name = name[2:] + + objtype = par["objtype"] + if objtype != "function": + return + entry = self.refcount_data.get(name) + if not entry: + return + + if not entry.result_type.endswith("Object*"): + return + + if entry.result_refs is None: + rc = sphinx_gettext("Return value: Always NULL.") + elif entry.result_refs: + rc = sphinx_gettext("Return value: New reference.") + else: + rc = sphinx_gettext("Return value: Borrowed reference.") + node.insert(0, nodes.emphasis(rc, rc, classes=["refcount", "text-info"])) def add_annotations(self, app, doctree): for node in doctree.findall(addnodes.desc_content): par = node.parent - if par["domain"] != "c": - continue - if not par[0].has_key("ids") or not par[0]["ids"]: - continue - name = par[0]["ids"][0] - if name.startswith("c."): - name = name[2:] - - objtype = par["objtype"] - if objtype != "function": - continue - entry = self.refcount_data.get(name) - if not entry: - continue - - if not entry.result_type.endswith("Object*"): - continue - - if entry.result_refs is None: - rc = sphinx_gettext("Return value: Always NULL.") - elif entry.result_refs: - rc = sphinx_gettext("Return value: New reference.") - else: - rc = sphinx_gettext("Return value: Borrowed reference.") - node.insert(0, nodes.emphasis(rc, rc, classes=["refcount", "text-info"])) + match par["domain"]: + case "c": + self.add_c_annotation(app, node, par) + + case "py": + self.add_py_annotations(app, node, par) def init_annotations(app): diff --git a/docs/sphinx/source/installing/index.rst b/docs/sphinx/source/installing/index.rst index b93a1a1..3930ee8 100644 --- a/docs/sphinx/source/installing/index.rst +++ b/docs/sphinx/source/installing/index.rst @@ -9,9 +9,8 @@ to install it by providing the Git link. This library has no fixed dependencies, out of the box. There are several installation options you can use to install a desired part of the library. -*caterpillar* ships with a native C extension and additional tools, that are using native -extensions too. As not everyone wants to use C extensions, it's default installation candidate -does not include the native extension. +*caterpillar* ships with a native C extension and additional tools. As not everyone wants to +use C extensions, it's default installation candidate does not include the native extension. .. code-block:: bash @@ -28,6 +27,11 @@ If you want to use the native C extension you can specify the following environm * `CP_ENABLE_NATIVE` - Enables installation of the native C extension * `CP_ENABLE_TOOLS` - Enables installation of extra tools (proposed) +**or** you can use the following command to install it directly: + +.. code-block:: bash + + pip install git+https://github.com/MatrixEditor/caterpillar/#subdirectory=src/ccaterpillar This project comes with packaging extras. Therefore, if you want to enable specific structs, you have to install the corresponding extra. It can be done using pip again: diff --git a/docs/sphinx/source/library/ctypes/int.rst b/docs/sphinx/source/library/ctypes/int.rst index 55872f5..ebbd881 100644 --- a/docs/sphinx/source/library/ctypes/int.rst +++ b/docs/sphinx/source/library/ctypes/int.rst @@ -4,14 +4,140 @@ Integer Types ************* -.. autoclass:: caterpillar.c.Int - :members: +This document provides detailed API documentation for integer types in the :code:`caterpillar.c` module. These +integer types are low-level representations of integers with customizable bit-length, signedness, and byte +order. They are particularly useful when interacting with binary data or when a fixed-width integer +representation is required. +The integer types offered here are optimized for performance, making them more efficient than their Python +equivalents (:class:`FormatField`) in almost every context. The global predefined integer types (e.g., +:data:`u8`, :data:`i32`) are convenient for most common cases, but you can also define custom integer types +with specific bit-widths, signedness, and endianness using the :class:`Int` class. -.. rubric:: Pre-Defined Integer Types (global) +.. py:class:: caterpillar.c.Int(bits, signed=True, little_endian=True) -All defined integer types are set to use little-endian by default. Apply -the byteorder operation to change endianess. + This class implements an integer type with a variable number of bits, + supporting signed and unsigned integers, and little-endian or big-endian + byte orders. + + Unlike its Python counterpart (:class:`FormatField`), this C-based class + focuses solely on integer operations, optimizing performance through direct + use of the `int.from_bytes` and `int.to_bytes` methods. Although you can + instantiate the class directly, it is recommended to use the global predefined + instances for common integer types (see below for more details). + + .. note:: + For most use cases, utilize the pre-defined global instances like :data:`u8`, + :data:`i16`, etc., for simplicity and efficiency. + + .. csv-table:: Supported Operations + :header: "Action", "Operator", "Result" + :widths: 15, 30, 30 + + "Byteorder", ":code:` + `", "new :class:`Int` object with corresponding byteorder" + "Sequence", ":code:`[]`", ":class:`repeated` object" + "Offset", ":code:` @ `", ":class:`atoffset` object" + "Condition", ":code:` // `", ":class:`condition` object" + "Switch", ":code:` >> `", ":class:`switch` object" + + .. versionadded:: 2.2.0 + + + .. py:method:: __init__(self, bits, signed=True, little_endian=True) + + Initializes the integer type with the given number of bits, sign specification, + and byte order. + + :param int bits: The number of bits for the integer. Must be a multiple of 8. + :param bool signed: Optional; whether the integer is signed. Defaults to True. + :param bool little_endian: Optional; whether the integer should be stored in little-endian byte order. Defaults to True. + + .. code-block:: python + :caption: Examples of using :class:`Int` + + from caterpillar.c import * + + # Unsigned 8-bit integer + assert unpack(b"\x01", u8) == 1 + + # Signed 8-bit integer + assert pack(1, i8) == b"\x01" + + # Unsigned 16-bit little-endian integer + assert unpack(b"\x01\x00", u16) == 1 + + # Signed 16-bit big-endian integer + assert pack(-1, Int(16, signed=True, little_endian=False)) == b"\xff\xff" + + .. py:method:: __repr__(self) -> str + + Returns a string representation of the integer type. The format of the returned string + is as follows: + + .. code-block:: + + := '<' ('le' | 'be') ['u'] 'int' '>' + + >>> assert repr(u32) == "" + + :return: A string representation of the integer type, indicating the byte order, sign, and number of bits. + + .. py:method:: __type__(self) -> type + + Returns the Python type for this class. This is typically :code:`int`. + + :return: The Python type for this class. + + .. py:method:: __size__(self, ctx) -> int + + Returns the size in bytes of the integer type. + + >>> assert sizeof(u64) == 8 + + :param ctx: The context object. (must not be null) + :return: The size in bytes of the integer type. + + .. py:attribute:: nbytes + :type: int + + The number of bytes in the integer, calculated from :attr:`nbits`. + + .. py:attribute:: nbits + :type: int + + The number of bits in the integer. + + >>> assert u32.nbits == 32 + + .. py:attribute:: signed + :type: bool + + Indicates whether the integer is signed (True) or unsigned (False). + + >>> assert i8.signed is True + + .. py:attribute:: little_endian + :type: bool + + Indicates whether the integer is stored in little-endian byte order (True) or big-endian byte order (False). + + >>> assert Int(32, little_endian=False).little_endian is False + >>> assert (BIG_ENDIAN + u32).little_endian is False + + +Pre-Defined Integer Types (global) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. raw:: html + +
+ +The following pre-defined integer types are globally available, optimized for +common use cases. By default, they use little-endian byte order. To switch to +big-endian, use the byteorder operation (e.g., :code:`BIG_ENDIAN + u16`). + +Unsigned Integer Types: +~~~~~~~~~~~~~~~~~~~~~~~ .. py:data:: caterpillar.c.u8 @@ -37,6 +163,10 @@ the byteorder operation to change endianess. Unsigned 128-bit integer type. + +Signed Integer Types: +~~~~~~~~~~~~~~~~~~~~~ + .. py:data:: caterpillar.c.i8 Signed 8-bit integer type. diff --git a/src/capi.dat b/src/capi.dat index 31c3207..5ffb4ad 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -8,9 +8,9 @@ # # There are currently four different types of statements: # -# type:INDEX:STRUCT_NAME:TYPEDEF_NAME:CAPI_TYPE +# type:INDEX:STRUCT_NAME:TYPEDEF_NAME:CAPI_TYPE_NAME # Defines a C API type for a C structure. The index is optional and -# the CAPI_TYPE will be inferred as PyTypeObject if none set +# the CAPI_TYPE_NAME will add an extra macro to the header file. # # obj:INDEX:NAME:TYPE # Defines a C API object. @@ -61,53 +61,53 @@ obj:0:CpModule:PyModuleDef type:-:_modulestate:_modulestate:- -type:1:_atomobj:CpAtomObject:- -type:2:_catomobj:CpCAtomObject:- -type:3:_archobj:CpArchObject:- -type:4:_endianobj:CpEndianObject:- -type:5:_contextobj:CpContextObject:- -type:6:_unaryexpr:CpUnaryExprObject:- -type:7:_binaryexpr:CpBinaryExprObject:- -type:8:_contextpath:CpContextPathObject:- -type:9:_fieldobj:CpFieldObject:- -type:10:_fieldatomobj:CpFieldAtomObject:- -type:11:_fieldcatomobj:CpFieldCAtomObject:- +type:1:_atomobj:CpAtomObject:atom +type:2:_catomobj:CpCAtomObject:catom +type:3:_archobj:CpArchObject:Arch +type:4:_endianobj:CpEndianObject:Endian +type:5:_contextobj:CpContextObject:Context +type:6:_unaryexpr:CpUnaryExprObject:unaryexpr +type:7:_binaryexpr:CpBinaryExprObject:binaryexpr +type:8:_contextpath:CpContextPathObject:ContextPath +type:9:_fieldobj:CpFieldObject:Field +type:10:_fieldatomobj:CpFieldAtomObject:fieldatom +type:11:_fieldcatomobj:CpFieldCAtomObject:fieldcatom obj:12:CpInvalidDefault_Type:- obj:13:CpDefaultOption_Type:- obj:14:_CpInvalidDefault_Object:PyObject obj:15:_CpDefaultOption_Object:PyObject -type:16:_option:CpOptionObject:- -type:17:_stateobj:CpStateObject:- -type:18:_layerobj:CpLayerObject:- +type:16:_option:CpOptionObject:Option +type:17:_stateobj:CpStateObject:State +type:18:_layerobj:CpLayerObject:layer # REVISIT: maybe rename to _structfieldinfo -type:19:_fieldinfoobj:CpStructFieldInfoObject:- -type:20:_structobj:CpStructObject:- -type:21:_floatatomobj:CpFloatAtomObject:- -type:22:_intatomobj:CpIntAtomObject:- -type:23:_boolatomobj:CpBoolAtomObject:- -type:24:_charatomobj:CpCharAtomObject:- -type:25:_paddingatomobj:CpPaddingAtomObject:- -type:26:_stringatomobj:CpStringAtomObject:- -type:27:_constatomobj:CpConstAtomObject:- -type:28:_builtinatomobj:CpBuiltinAtomObject:- -type:29:_repeatedatomobj:CpRepeatedAtomObject:- -type:30:_seqlayerobj:CpSeqLayerObject:- -type:31:_objlayerobj:CpObjLayerObject:- -type:32:_conditionatomobj:CpConditionAtomObject:- -type:33:_switchatomobj:CpSwitchAtomObject:- -type:34:_offsetatomobj:CpOffsetAtomObject:- -type:35:_primitiveatomobj:CpPrimitiveAtomObject:- -type:36:_lengthinfoobj:CpLengthInfoObject:- -type:37:_bytesatomobj:CpBytesAtomObject:- -type:38:_pstringatomobj:CpPStringAtomObject:- -type:39:_enumatomobj:CpEnumAtomObject:- -type:40:_varintatomobj:CpVarIntAtomObject:- -type:41:_computedatomobj:CpComputedAtomObject:- -type:42:_lazyatomobj:CpLazyAtomObject:- -type:43:_cstringatomobj:CpCStringAtomObject:- +type:19:_fieldinfoobj:CpStructFieldInfoObject:fieldinfo +type:20:_structobj:CpStructObject:Struct +type:21:_floatatomobj:CpFloatAtomObject:Float +type:22:_intatomobj:CpIntAtomObject:Int +type:23:_boolatomobj:CpBoolAtomObject:Bool +type:24:_charatomobj:CpCharAtomObject:Char +type:25:_paddingatomobj:CpPaddingAtomObject:Padding +type:26:_stringatomobj:CpStringAtomObject:string +type:27:_constatomobj:CpConstAtomObject:const +type:28:_builtinatomobj:CpBuiltinAtomObject:builtinatom +type:29:_repeatedatomobj:CpRepeatedAtomObject:repeated +type:30:_seqlayerobj:CpSeqLayerObject:seqlayer +type:31:_objlayerobj:CpObjLayerObject:objlayer +type:32:_conditionatomobj:CpConditionAtomObject:condition +type:33:_switchatomobj:CpSwitchAtomObject:switch +type:34:_offsetatomobj:CpOffsetAtomObject:atoffset +type:35:_primitiveatomobj:CpPrimitiveAtomObject:patom +type:36:_lengthinfoobj:CpLengthInfoObject:lengthinfo +type:37:_bytesatomobj:CpBytesAtomObject:octetstring +type:38:_pstringatomobj:CpPStringAtomObject:pstring +type:39:_enumatomobj:CpEnumAtomObject:enumeration +type:40:_varintatomobj:CpVarIntAtomObject:VarInt +type:41:_computedatomobj:CpComputedAtomObject:computed +type:42:_lazyatomobj:CpLazyAtomObject:lazy +type:43:_cstringatomobj:CpCStringAtomObject:cstring func:50:CpEndian_IsLittleEndian:int:null func:53:CpContext_New:CpContextObject*:+1 diff --git a/src/caterpillar/include/caterpillar/atomobj.h b/src/caterpillar/include/caterpillar/atomobj.h index c929f48..b002961 100644 --- a/src/caterpillar/include/caterpillar/atomobj.h +++ b/src/caterpillar/include/caterpillar/atomobj.h @@ -121,8 +121,6 @@ struct _lengthinfoobj int m_greedy; }; -#define CpLengthInfo_NAME "lengthinfo" - // --------------------------------------------------------------------------- // CAtom diff --git a/src/caterpillar/include/caterpillar/atoms/builtins.h b/src/caterpillar/include/caterpillar/atoms/builtins.h index 0359bec..60288ca 100644 --- a/src/caterpillar/include/caterpillar/atoms/builtins.h +++ b/src/caterpillar/include/caterpillar/atoms/builtins.h @@ -28,7 +28,6 @@ struct _builtinatomobj CpCAtom_HEAD }; -#define CpBuiltinAtom_NAME "builtinatom" #define CpBuiltinAtom_CheckExact(op) Py_IS_TYPE((op), &CpBuiltinAtom_Type) #define CpBuiltinAtom_Check(op) PyObject_TypeCheck((op), &CpBuiltinAtom_Type) #define CpBuiltinAtom_HEAD CpBuiltinAtomObject ob_base; @@ -49,7 +48,6 @@ struct _repeatedatomobj PyObject* m_length; }; -#define CpRepeatedAtom_NAME "repeated" #define CpRepeatedAtom_CheckExact(op) Py_IS_TYPE((op), &CpRepeatedAtom_Type) #define CpRepeatedAtom_Check(op) PyObject_TypeCheck((op), &CpRepeatedAtom_Type) @@ -74,7 +72,6 @@ struct _conditionatomobj PyObject* m_condition; }; -#define CpConditionAtom_NAME "condition" #define CpConditionAtom_CheckExact(op) Py_IS_TYPE((op), &CpConditionAtom_Type) #define CpConditionAtom_Check(op) PyObject_TypeCheck((op), &CpConditionAtom_Type) @@ -102,7 +99,6 @@ struct _switchatomobj int s_callable; }; -#define CpSwitchAtom_NAME "switch" #define CpSwitchAtom_CheckExact(op) Py_IS_TYPE((op), &CpSwitchAtom_Type) #define CpSwitchAtom_Check(op) PyObject_TypeCheck((op), &CpSwitchAtom_Type) @@ -130,7 +126,6 @@ struct _offsetatomobj int s_is_number; }; -#define CpOffsetAtom_NAME "atoffset" #define CpOffsetAtom_CheckExact(op) Py_IS_TYPE((op), &CpOffsetAtom_Type) #define CpOffsetAtom_Check(op) PyObject_TypeCheck((op), &CpOffsetAtom_Type) diff --git a/src/caterpillar/include/caterpillar/atoms/const.h b/src/caterpillar/include/caterpillar/atoms/const.h index 3631595..e311d88 100644 --- a/src/caterpillar/include/caterpillar/atoms/const.h +++ b/src/caterpillar/include/caterpillar/atoms/const.h @@ -27,7 +27,6 @@ struct _constatomobj PyObject *m_atom; }; -#define CpConstAtom_NAME "const" #define CpConstAtom_CheckExact(op) Py_IS_TYPE((op), &CpConstAtom_Type) #define CpConstAtom_Check(op) PyObject_TypeCheck((op), &CpConstAtom_Type) diff --git a/src/caterpillar/include/caterpillar/atoms/enum.h b/src/caterpillar/include/caterpillar/atoms/enum.h index 9458730..d1ea921 100644 --- a/src/caterpillar/include/caterpillar/atoms/enum.h +++ b/src/caterpillar/include/caterpillar/atoms/enum.h @@ -37,7 +37,6 @@ struct _enumatomobj PyObject* m_enum_type; }; -#define CpEnumAtom_NAME "enumeration" #define CpEnumAtom_CheckExact(op) Py_IS_TYPE((op), &CpEnumAtom_Type) #define CpEnumAtom_Check(op) (PyObject_TypeCheck((op), &CpEnumAtom_Type)) diff --git a/src/caterpillar/include/caterpillar/atoms/float.h b/src/caterpillar/include/caterpillar/atoms/float.h index 6c2ad8b..11bf6f7 100644 --- a/src/caterpillar/include/caterpillar/atoms/float.h +++ b/src/caterpillar/include/caterpillar/atoms/float.h @@ -34,11 +34,7 @@ struct _floatatomobj int _m_little_endian; }; -#define CpFloatAtom_NAME "Float" - -/** @brief Checks if the given object is an integer atom object */ #define CpFloatAtom_CheckExact(op) Py_IS_TYPE((op), &CpFloatAtom_Type) -/** @brief Checks if the given object is an integer atom object */ #define CpFloatAtom_Check(op) (PyObject_TypeCheck((op), &CpFloatAtom_Type)) #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/atoms/int.h b/src/caterpillar/include/caterpillar/atoms/int.h index 9e19050..5776fde 100644 --- a/src/caterpillar/include/caterpillar/atoms/int.h +++ b/src/caterpillar/include/caterpillar/atoms/int.h @@ -39,8 +39,6 @@ struct _intatomobj int _m_little_endian; }; -#define CpIntAtom_NAME "Int" - /** @brief Checks if the given object is an integer atom object */ #define CpIntAtom_CheckExact(op) Py_IS_TYPE((op), &CpIntAtom_Type) /** @brief Checks if the given object is an integer atom object */ @@ -62,7 +60,6 @@ struct _varintatomobj int _m_lsb; }; -#define CpVarIntAtom_NAME "varint_t" #define CpVarIntAtom_CheckExact(op) Py_IS_TYPE((op), &CpVarIntAtom_Type) #define CpVarIntAtom_Check(op) (PyObject_TypeCheck((op), &CpVarIntAtom_Type)) diff --git a/src/caterpillar/include/caterpillar/atoms/primitive.h b/src/caterpillar/include/caterpillar/atoms/primitive.h index 4a15b68..d030fc1 100644 --- a/src/caterpillar/include/caterpillar/atoms/primitive.h +++ b/src/caterpillar/include/caterpillar/atoms/primitive.h @@ -49,7 +49,6 @@ struct _boolatomobj /// This variable defines the type object for bool atom objects, allowing /// them to be used across the library. // PyAPI_DATA(PyTypeObject) CpBoolAtom_Type; -#define CpBoolAtom_NAME "Bool" /** * @brief Checks if the given object is a bool atom object. @@ -82,10 +81,6 @@ struct _charatomobj CpBuiltinAtom_HEAD }; -/// Char atom object type -// PyAPI_DATA(PyTypeObject) CpCharAtom_Type; -#define CpCharAtom_NAME "Char" - /** @brief Checks if the given object is a char atom object */ #define CpCharAtom_CheckExact(op) Py_IS_TYPE((op), &CpCharAtom_Type) /** @brief Checks if the given object is a char atom object */ @@ -100,7 +95,6 @@ struct _paddingatomobj char _m_padding; }; -#define CpPaddingAtom_NAME "Padding" #define CpPaddingAtom_CheckExact(op) Py_IS_TYPE((op), &CpPaddingAtom_Type) #define CpPaddingAtom_Check(op) (PyObject_TypeCheck((op), &CpPaddingAtom_Type)) @@ -115,7 +109,6 @@ struct _computedatomobj int s_callable; }; -#define CpComputedAtom_NAME "computed" #define CpComputedAtom_CheckExact(op) Py_IS_TYPE((op), &CpComputedAtom_Type) #define CpComputedAtom_Check(op) \ (PyObject_TypeCheck((op), &CpComputedAtom_Type)) @@ -150,7 +143,6 @@ struct _lazyatomobj int s_always_lazy; }; -#define CpLazyAtom_NAME "lazy" #define CpLazyAtom_CheckExact(op) Py_IS_TYPE((op), &CpLazyAtom_Type) #define CpLazyAtom_Check(op) (PyObject_TypeCheck((op), &CpLazyAtom_Type)) #define CpLazyAtom_ATOM(op) (((CpLazyAtomObject*)(op))->m_atom) diff --git a/src/caterpillar/include/caterpillar/atoms/string.h b/src/caterpillar/include/caterpillar/atoms/string.h index 80a1052..9fd5b30 100644 --- a/src/caterpillar/include/caterpillar/atoms/string.h +++ b/src/caterpillar/include/caterpillar/atoms/string.h @@ -37,7 +37,6 @@ struct _stringatomobj PyObject* m_encoding; }; -#define CpStringAtom_NAME "string" #define CpStringAtom_CheckExact(op) Py_IS_TYPE((op), &CpStringAtom_Type) #define CpStringAtom_Check(op) PyObject_TypeCheck((op), &CpStringAtom_Type) @@ -58,7 +57,6 @@ struct _bytesatomobj }; // REVISIT: The name of this atom should be something similar to 'bytes' -#define CpBytesAtom_NAME "octetstring" #define CpBytesAtom_CheckExact(op) Py_IS_TYPE((op), &CpBytesAtom_Type) #define CpBytesAtom_Check(op) PyObject_TypeCheck((op), &CpBytesAtom_Type) @@ -85,7 +83,6 @@ struct _pstringatomobj PyObject* m_encoding; }; -#define CpPStringAtom_NAME "pstring" #define CpPStringAtom_CheckExact(op) Py_IS_TYPE((op), &CpPStringAtom_Type) #define CpPStringAtom_Check(op) PyObject_TypeCheck((op), &CpPStringAtom_Type) @@ -115,7 +112,6 @@ struct _cstringatomobj int s_greedy; }; -#define CpCStringAtom_NAME "cstring" #define CpCStringAtom_CheckExact(op) Py_IS_TYPE((op), &CpCStringAtom_Type) #define CpCStringAtom_Check(op) PyObject_TypeCheck((op), &CpCStringAtom_Type) diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index e0e6112..3fb35b1 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -116,48 +116,87 @@ are then used and implemented in the internal API. extern PyModuleDef CpModule; extern PyTypeObject CpAtom_Type; +#define CpAtom_NAME "atom" extern PyTypeObject CpCAtom_Type; +#define CpCAtom_NAME "catom" extern PyTypeObject CpArch_Type; +#define CpArch_NAME "Arch" extern PyTypeObject CpEndian_Type; +#define CpEndian_NAME "Endian" extern PyTypeObject CpContext_Type; +#define CpContext_NAME "Context" extern PyTypeObject CpUnaryExpr_Type; +#define CpUnaryExpr_NAME "unaryexpr" extern PyTypeObject CpBinaryExpr_Type; +#define CpBinaryExpr_NAME "binaryexpr" extern PyTypeObject CpContextPath_Type; +#define CpContextPath_NAME "ContextPath" extern PyTypeObject CpField_Type; +#define CpField_NAME "Field" extern PyTypeObject CpFieldAtom_Type; +#define CpFieldAtom_NAME "fieldatom" extern PyTypeObject CpFieldCAtom_Type; +#define CpFieldCAtom_NAME "fieldcatom" extern PyTypeObject CpInvalidDefault_Type; extern PyTypeObject CpDefaultOption_Type; extern PyObject _CpInvalidDefault_Object; extern PyObject _CpDefaultOption_Object; extern PyTypeObject CpOption_Type; +#define CpOption_NAME "Option" extern PyTypeObject CpState_Type; +#define CpState_NAME "State" extern PyTypeObject CpLayer_Type; +#define CpLayer_NAME "layer" extern PyTypeObject CpStructFieldInfo_Type; +#define CpStructFieldInfo_NAME "fieldinfo" extern PyTypeObject CpStruct_Type; +#define CpStruct_NAME "Struct" extern PyTypeObject CpFloatAtom_Type; +#define CpFloatAtom_NAME "Float" extern PyTypeObject CpIntAtom_Type; +#define CpIntAtom_NAME "Int" extern PyTypeObject CpBoolAtom_Type; +#define CpBoolAtom_NAME "Bool" extern PyTypeObject CpCharAtom_Type; +#define CpCharAtom_NAME "Char" extern PyTypeObject CpPaddingAtom_Type; +#define CpPaddingAtom_NAME "Padding" extern PyTypeObject CpStringAtom_Type; +#define CpStringAtom_NAME "string" extern PyTypeObject CpConstAtom_Type; +#define CpConstAtom_NAME "const" extern PyTypeObject CpBuiltinAtom_Type; +#define CpBuiltinAtom_NAME "builtinatom" extern PyTypeObject CpRepeatedAtom_Type; +#define CpRepeatedAtom_NAME "repeated" extern PyTypeObject CpSeqLayer_Type; +#define CpSeqLayer_NAME "seqlayer" extern PyTypeObject CpObjLayer_Type; +#define CpObjLayer_NAME "objlayer" extern PyTypeObject CpConditionAtom_Type; +#define CpConditionAtom_NAME "condition" extern PyTypeObject CpSwitchAtom_Type; +#define CpSwitchAtom_NAME "switch" extern PyTypeObject CpOffsetAtom_Type; +#define CpOffsetAtom_NAME "atoffset" extern PyTypeObject CpPrimitiveAtom_Type; +#define CpPrimitiveAtom_NAME "patom" extern PyTypeObject CpLengthInfo_Type; +#define CpLengthInfo_NAME "lengthinfo" extern PyTypeObject CpBytesAtom_Type; +#define CpBytesAtom_NAME "octetstring" extern PyTypeObject CpPStringAtom_Type; +#define CpPStringAtom_NAME "pstring" extern PyTypeObject CpEnumAtom_Type; +#define CpEnumAtom_NAME "enumeration" extern PyTypeObject CpVarIntAtom_Type; +#define CpVarIntAtom_NAME "VarInt" extern PyTypeObject CpComputedAtom_Type; +#define CpComputedAtom_NAME "computed" extern PyTypeObject CpLazyAtom_Type; +#define CpLazyAtom_NAME "lazy" extern PyTypeObject CpCStringAtom_Type; +#define CpCStringAtom_NAME "cstring" int CpEndian_IsLittleEndian(CpEndianObject* endian, _modulestate* mod); CpContextObject* CpContext_New(void); CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); @@ -268,49 +307,49 @@ internal API functions and types. Their indices are static and defined in caterpillar_api.py */ #define CpModule (*(PyModuleDef *)Cp_API[0]) -#define CpAtom_Type (*(PyTypeObject *)Cp_API[1]) -#define CpCAtom_Type (*(PyTypeObject *)Cp_API[2]) -#define CpArch_Type (*(PyTypeObject *)Cp_API[3]) -#define CpEndian_Type (*(PyTypeObject *)Cp_API[4]) -#define CpContext_Type (*(PyTypeObject *)Cp_API[5]) -#define CpUnaryExpr_Type (*(PyTypeObject *)Cp_API[6]) -#define CpBinaryExpr_Type (*(PyTypeObject *)Cp_API[7]) -#define CpContextPath_Type (*(PyTypeObject *)Cp_API[8]) -#define CpField_Type (*(PyTypeObject *)Cp_API[9]) -#define CpFieldAtom_Type (*(PyTypeObject *)Cp_API[10]) -#define CpFieldCAtom_Type (*(PyTypeObject *)Cp_API[11]) +#define CpAtom_Type (*(atom *)Cp_API[1]) +#define CpCAtom_Type (*(catom *)Cp_API[2]) +#define CpArch_Type (*(Arch *)Cp_API[3]) +#define CpEndian_Type (*(Endian *)Cp_API[4]) +#define CpContext_Type (*(Context *)Cp_API[5]) +#define CpUnaryExpr_Type (*(unaryexpr *)Cp_API[6]) +#define CpBinaryExpr_Type (*(binaryexpr *)Cp_API[7]) +#define CpContextPath_Type (*(ContextPath *)Cp_API[8]) +#define CpField_Type (*(Field *)Cp_API[9]) +#define CpFieldAtom_Type (*(fieldatom *)Cp_API[10]) +#define CpFieldCAtom_Type (*(fieldcatom *)Cp_API[11]) #define CpInvalidDefault_Type (*(PyTypeObject *)Cp_API[12]) #define CpDefaultOption_Type (*(PyTypeObject *)Cp_API[13]) #define _CpInvalidDefault_Object (*(PyObject *)Cp_API[14]) #define _CpDefaultOption_Object (*(PyObject *)Cp_API[15]) -#define CpOption_Type (*(PyTypeObject *)Cp_API[16]) -#define CpState_Type (*(PyTypeObject *)Cp_API[17]) -#define CpLayer_Type (*(PyTypeObject *)Cp_API[18]) -#define CpStructFieldInfo_Type (*(PyTypeObject *)Cp_API[19]) -#define CpStruct_Type (*(PyTypeObject *)Cp_API[20]) -#define CpFloatAtom_Type (*(PyTypeObject *)Cp_API[21]) -#define CpIntAtom_Type (*(PyTypeObject *)Cp_API[22]) -#define CpBoolAtom_Type (*(PyTypeObject *)Cp_API[23]) -#define CpCharAtom_Type (*(PyTypeObject *)Cp_API[24]) -#define CpPaddingAtom_Type (*(PyTypeObject *)Cp_API[25]) -#define CpStringAtom_Type (*(PyTypeObject *)Cp_API[26]) -#define CpConstAtom_Type (*(PyTypeObject *)Cp_API[27]) -#define CpBuiltinAtom_Type (*(PyTypeObject *)Cp_API[28]) -#define CpRepeatedAtom_Type (*(PyTypeObject *)Cp_API[29]) -#define CpSeqLayer_Type (*(PyTypeObject *)Cp_API[30]) -#define CpObjLayer_Type (*(PyTypeObject *)Cp_API[31]) -#define CpConditionAtom_Type (*(PyTypeObject *)Cp_API[32]) -#define CpSwitchAtom_Type (*(PyTypeObject *)Cp_API[33]) -#define CpOffsetAtom_Type (*(PyTypeObject *)Cp_API[34]) -#define CpPrimitiveAtom_Type (*(PyTypeObject *)Cp_API[35]) -#define CpLengthInfo_Type (*(PyTypeObject *)Cp_API[36]) -#define CpBytesAtom_Type (*(PyTypeObject *)Cp_API[37]) -#define CpPStringAtom_Type (*(PyTypeObject *)Cp_API[38]) -#define CpEnumAtom_Type (*(PyTypeObject *)Cp_API[39]) -#define CpVarIntAtom_Type (*(PyTypeObject *)Cp_API[40]) -#define CpComputedAtom_Type (*(PyTypeObject *)Cp_API[41]) -#define CpLazyAtom_Type (*(PyTypeObject *)Cp_API[42]) -#define CpCStringAtom_Type (*(PyTypeObject *)Cp_API[43]) +#define CpOption_Type (*(Option *)Cp_API[16]) +#define CpState_Type (*(State *)Cp_API[17]) +#define CpLayer_Type (*(layer *)Cp_API[18]) +#define CpStructFieldInfo_Type (*(fieldinfo *)Cp_API[19]) +#define CpStruct_Type (*(Struct *)Cp_API[20]) +#define CpFloatAtom_Type (*(Float *)Cp_API[21]) +#define CpIntAtom_Type (*(Int *)Cp_API[22]) +#define CpBoolAtom_Type (*(Bool *)Cp_API[23]) +#define CpCharAtom_Type (*(Char *)Cp_API[24]) +#define CpPaddingAtom_Type (*(Padding *)Cp_API[25]) +#define CpStringAtom_Type (*(string *)Cp_API[26]) +#define CpConstAtom_Type (*(const *)Cp_API[27]) +#define CpBuiltinAtom_Type (*(builtinatom *)Cp_API[28]) +#define CpRepeatedAtom_Type (*(repeated *)Cp_API[29]) +#define CpSeqLayer_Type (*(seqlayer *)Cp_API[30]) +#define CpObjLayer_Type (*(objlayer *)Cp_API[31]) +#define CpConditionAtom_Type (*(condition *)Cp_API[32]) +#define CpSwitchAtom_Type (*(switch *)Cp_API[33]) +#define CpOffsetAtom_Type (*(atoffset *)Cp_API[34]) +#define CpPrimitiveAtom_Type (*(patom *)Cp_API[35]) +#define CpLengthInfo_Type (*(lengthinfo *)Cp_API[36]) +#define CpBytesAtom_Type (*(octetstring *)Cp_API[37]) +#define CpPStringAtom_Type (*(pstring *)Cp_API[38]) +#define CpEnumAtom_Type (*(enumeration *)Cp_API[39]) +#define CpVarIntAtom_Type (*(VarInt *)Cp_API[40]) +#define CpComputedAtom_Type (*(computed *)Cp_API[41]) +#define CpLazyAtom_Type (*(lazy *)Cp_API[42]) +#define CpCStringAtom_Type (*(cstring *)Cp_API[43]) #define CpEndian_IsLittleEndian (*((int (*)(CpEndianObject* endian, _modulestate* mod)))Cp_API[50]) #define CpContext_New (*((CpContextObject* (*)(void)))Cp_API[53]) #define CpUnaryExpr_New (*((CpUnaryExprObject* (*)(int op, PyObject* value)))Cp_API[54]) diff --git a/src/caterpillar/include/caterpillar/state.h b/src/caterpillar/include/caterpillar/state.h index a1b40c2..68411fe 100644 --- a/src/caterpillar/include/caterpillar/state.h +++ b/src/caterpillar/include/caterpillar/state.h @@ -191,7 +191,6 @@ struct _seqlayerobj int8_t s_greedy; }; -#define CpSeqLayer_NAME "seqlayer" #define CpSeqLayer_CheckExact(v) Py_IS_TYPE((v), &CpSeqLayer_Type) #define CpSeqLayer_Check(v) PyObject_TypeCheck((v), &CpSeqLayer_Type) @@ -210,7 +209,6 @@ struct _objlayerobj PyObject* m_obj; }; -#define CpObjLayer_NAME "objlayer" #define CpObjLayer_CheckExact(v) Py_IS_TYPE((v), &CpObjLayer_Type) #define CpObjLayer_Check(v) PyObject_TypeCheck((v), &CpObjLayer_Type) diff --git a/src/ccaterpillar/module.c b/src/ccaterpillar/module.c index 8fc384f..e067eb1 100644 --- a/src/ccaterpillar/module.c +++ b/src/ccaterpillar/module.c @@ -387,28 +387,28 @@ PyInit__C(void) return NULL; } - CpModule_AddObject("atom", &CpAtom_Type); - CpModule_AddObject("catom", &CpCAtom_Type); - CpModule_AddObject("Option", &CpOption_Type); - CpModule_AddObject("Arch", &CpArch_Type); - CpModule_AddObject("Endian", &CpEndian_Type); - CpModule_AddObject("Context", &CpContext_Type); - CpModule_AddObject("BinaryExpr", &CpBinaryExpr_Type); - CpModule_AddObject("UnaryExpr", &CpUnaryExpr_Type); - CpModule_AddObject("ContextPath", &CpContextPath_Type); + CpModule_AddObject(CpAtom_NAME, &CpAtom_Type); + CpModule_AddObject(CpCAtom_NAME, &CpCAtom_Type); + CpModule_AddObject(CpOption_NAME, &CpOption_Type); + CpModule_AddObject(CpArch_NAME, &CpArch_Type); + CpModule_AddObject(CpEndian_NAME, &CpEndian_Type); + CpModule_AddObject(CpContext_NAME, &CpContext_Type); + CpModule_AddObject(CpBinaryExpr_NAME, &CpBinaryExpr_Type); + CpModule_AddObject(CpUnaryExpr_NAME, &CpUnaryExpr_Type); + CpModule_AddObject(CpContextPath_NAME, &CpContextPath_Type); CpModule_AddObject("InvalidDefaultType", &CpInvalidDefault_Type); CpModule_AddObject("InvalidDefault", CpInvalidDefault); CpModule_AddObject("DefaultOptionType", &CpDefaultOption_Type); CpModule_AddObject("DefaultOption", CpDefaultOption); - CpModule_AddObject("Field", &CpField_Type); - CpModule_AddObject("fieldatom", &CpFieldAtom_Type); - CpModule_AddObject("fieldcatom", &CpFieldCAtom_Type); - CpModule_AddObject("layer", &CpLayer_Type); + CpModule_AddObject(CpField_NAME, &CpField_Type); + CpModule_AddObject(CpFieldAtom_NAME, &CpFieldAtom_Type); + CpModule_AddObject(CpFieldCAtom_NAME, &CpFieldCAtom_Type); + CpModule_AddObject(CpLayer_NAME, &CpLayer_Type); CpModule_AddObject(CpSeqLayer_NAME, &CpSeqLayer_Type); CpModule_AddObject(CpObjLayer_NAME, &CpObjLayer_Type); - CpModule_AddObject("State", &CpState_Type); - CpModule_AddObject("fieldinfo", &CpStructFieldInfo_Type); - CpModule_AddObject("Struct", &CpStruct_Type); + CpModule_AddObject(CpState_NAME, &CpState_Type); + CpModule_AddObject(CpStructFieldInfo_NAME, &CpStructFieldInfo_Type); + CpModule_AddObject(CpStruct_NAME, &CpStruct_Type); CpModule_AddObject(CpLengthInfo_NAME, &CpLengthInfo_Type); CpModule_AddObject(CpPrimitiveAtom_NAME, &CpPrimitiveAtom_Type); diff --git a/src/code_gen/genapi.py b/src/code_gen/genapi.py index ef94834..4eaac01 100644 --- a/src/code_gen/genapi.py +++ b/src/code_gen/genapi.py @@ -40,7 +40,11 @@ def __init__(self, name: str, index: int, api_name: str, type_: str = None) -> N def cp_internal_def(self) -> str: # REVISIT: what about struct definitions? - return f"extern {self.type} {self.name};" + if self.type.startswith("Py"): + return f"extern {self.type} {self.name};" + + extra_def = f"#define {self.name.replace('_Type', '')}_NAME \"{self.type}\"" + return f"extern PyTypeObject {self.name};\n{extra_def}" def cp_external_def(self): """Return the external definition for this API object From 3aa6bae2f0b32857e69594aace55b7df20979ec8 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 15 Sep 2024 22:56:36 +0200 Subject: [PATCH 25/29] Updated Tutorial documentation --- + Fixed issue with STRUCT_OPTIONS referenced two times + cstring now parses with greedy option by default + Documentation updates + modified tests adapt to fixed bugs and naming changes --- docs/sphinx/source/index.rst | 36 +- docs/sphinx/source/library/fields/common.rst | 68 ++-- .../source/library/fields/compression.rst | 10 +- .../source/library/fields/field_model.rst | 12 +- docs/sphinx/source/tutorial/basics.rst | 362 ++++++++++++++---- docs/sphinx/source/tutorial/first_steps.rst | 165 ++++++-- docs/sphinx/source/tutorial/index.rst | 3 +- src/ccaterpillar/atomimpl/cstring.c | 11 +- src/ccaterpillar/module.c | 3 +- test/_C/atoms/test_string.py | 8 +- test/_C/atoms/test_varint.py | 10 +- test/_C/test_option.py | 1 + 12 files changed, 508 insertions(+), 181 deletions(-) diff --git a/docs/sphinx/source/index.rst b/docs/sphinx/source/index.rst index f9a0d10..fb8595d 100644 --- a/docs/sphinx/source/index.rst +++ b/docs/sphinx/source/index.rst @@ -16,15 +16,33 @@ efficient. structs that adjust their size based on the current context. This framework enables you to write complex structures in a compact and readable manner. -.. code-block:: - :caption: Simple example of a custom struct - - @struct - class Format: - magic: b"Foo" # constant values - name: CString(...) # C-String without a fixed length - value: le + uint16 # little endian encoding - entries: be + CString[uint32::] # arrays with big-endian prefixed length +.. tab-set:: + + .. tab-item:: Python + + .. code-block:: + :caption: Simple example of a custom struct + + @struct + class Format: + magic: b"Foo" # constant values + name: CString(...) # C-String without a fixed length + value: le + uint16 # little endian encoding + entries: be + CString[uint32::] # arrays with big-endian prefixed length + + .. tab-item:: Caterpillar C + + .. code-block:: python + :caption: Simple example of a custom struct + + from caterpillar.c import BIG_ENDIAN as be + + @struct + class Format: + magic: const(b"Foo", octetstring(3)) # constant values + name: cstring(...) # C-String without a fixed length + value: u16 # little endian encoding + entries: cstring(...)[be + u32::] # arrays with big-endian prefixed length .. admonition:: Hold up, wait a minute! diff --git a/docs/sphinx/source/library/fields/common.rst b/docs/sphinx/source/library/fields/common.rst index 6a4c45b..7642d40 100644 --- a/docs/sphinx/source/library/fields/common.rst +++ b/docs/sphinx/source/library/fields/common.rst @@ -7,93 +7,93 @@ Common Structs Numeric Structs --------------- -.. autoclass:: caterpillar.fields.FormatField +.. autoclass:: caterpillar.py.FormatField :members: -.. autoattribute:: caterpillar.fields.uint8 +.. autoattribute:: caterpillar.py.uint8 -.. autoattribute:: caterpillar.fields.int8 +.. autoattribute:: caterpillar.py.int8 -.. autoattribute:: caterpillar.fields.uint16 +.. autoattribute:: caterpillar.py.uint16 -.. autoattribute:: caterpillar.fields.int16 +.. autoattribute:: caterpillar.py.int16 -.. autoattribute:: caterpillar.fields.uint32 +.. autoattribute:: caterpillar.py.uint32 -.. autoattribute:: caterpillar.fields.int32 +.. autoattribute:: caterpillar.py.int32 -.. autoattribute:: caterpillar.fields.uint64 +.. autoattribute:: caterpillar.py.uint64 -.. autoattribute:: caterpillar.fields.int64 +.. autoattribute:: caterpillar.py.int64 -.. autoattribute:: caterpillar.fields.size_t +.. autoattribute:: caterpillar.py.size_t -.. autoattribute:: caterpillar.fields.ssize_t +.. autoattribute:: caterpillar.py.ssize_t -.. autoattribute:: caterpillar.fields.float16 +.. autoattribute:: caterpillar.py.float16 -.. autoattribute:: caterpillar.fields.float32 +.. autoattribute:: caterpillar.py.float32 -.. autoattribute:: caterpillar.fields.float64 +.. autoattribute:: caterpillar.py.float64 -.. autoattribute:: caterpillar.fields.void_ptr +.. autoattribute:: caterpillar.py.void_ptr -.. autoattribute:: caterpillar.fields.char +.. autoattribute:: caterpillar.py.char -.. autoattribute:: caterpillar.fields.boolean +.. autoattribute:: caterpillar.py.boolean -.. autoattribute:: caterpillar.fields.padding +.. autoattribute:: caterpillar.py.padding -.. autoclass:: caterpillar.fields.Int +.. autoclass:: caterpillar.py.Int :members: -.. autoclass:: caterpillar.fields.UInt +.. autoclass:: caterpillar.py.UInt :members: -.. autoattribute:: caterpillar.fields.vint +.. autoattribute:: caterpillar.py.vint -.. autoclass:: caterpillar.fields.VarInt +.. autoclass:: caterpillar.py.VarInt :members: Bytes, Strings -------------- -.. autoclass:: caterpillar.fields.Memory +.. autoclass:: caterpillar.py.Memory :members: -.. autoclass:: caterpillar.fields.Bytes +.. autoclass:: caterpillar.py.Bytes :members: -.. autoclass:: caterpillar.fields.String +.. autoclass:: caterpillar.py.String :members: -.. autoclass:: caterpillar.fields.Prefixed +.. autoclass:: caterpillar.py.Prefixed :members: -.. autoclass:: caterpillar.fields.CString +.. autoclass:: caterpillar.py.CString :members: -.. autoclass:: caterpillar.fields.ConstString +.. autoclass:: caterpillar.py.ConstString :members: -.. autoclass:: caterpillar.fields.ConstBytes +.. autoclass:: caterpillar.py.ConstBytes :members: Special Structs --------------- -.. autoattribute:: caterpillar.fields.Pass +.. autoattribute:: caterpillar.py.Pass -.. autoclass:: caterpillar.fields.Computed +.. autoclass:: caterpillar.py.Computed :members: -.. autoclass:: caterpillar.fields.Transformer +.. autoclass:: caterpillar.py.Transformer :members: -.. autoclass:: caterpillar.fields.Enum +.. autoclass:: caterpillar.py.Enum :members: -.. autoclass:: caterpillar.fields.Const +.. autoclass:: caterpillar.py.Const :members: \ No newline at end of file diff --git a/docs/sphinx/source/library/fields/compression.rst b/docs/sphinx/source/library/fields/compression.rst index 6fbbaf4..cf7296a 100644 --- a/docs/sphinx/source/library/fields/compression.rst +++ b/docs/sphinx/source/library/fields/compression.rst @@ -7,16 +7,16 @@ Compression Structs Basic structs ------------- -.. autoclass:: caterpillar.fields.Compressed +.. autoclass:: caterpillar.py.Compressed :members: Supported compression types --------------------------- -.. autofunction:: caterpillar.fields.ZLibCompressed +.. autofunction:: caterpillar.py.ZLibCompressed -.. autofunction:: caterpillar.fields.Bz2Compressed +.. autofunction:: caterpillar.py.Bz2Compressed -.. autofunction:: caterpillar.fields.LZMACompressed +.. autofunction:: caterpillar.py.LZMACompressed -.. autofunction:: caterpillar.fields.LZOCompressed +.. autofunction:: caterpillar.py.LZOCompressed diff --git a/docs/sphinx/source/library/fields/field_model.rst b/docs/sphinx/source/library/fields/field_model.rst index f0b3c68..e737cc0 100644 --- a/docs/sphinx/source/library/fields/field_model.rst +++ b/docs/sphinx/source/library/fields/field_model.rst @@ -4,24 +4,24 @@ Field Model *********** -.. autoclass:: caterpillar.fields.Field() +.. autoclass:: caterpillar.py.Field() :members: :private-members: :no-undoc-members: -.. autoclass:: caterpillar.fields.FieldMixin +.. autoclass:: caterpillar.py.FieldMixin :special-members: -.. autoclass:: caterpillar.fields.FieldStruct +.. autoclass:: caterpillar.py.FieldStruct :members: :special-members: -.. autoclass:: caterpillar.fields.Chain +.. autoclass:: caterpillar.py.Chain :members: :special-members: -.. autoclass:: caterpillar.fields.If +.. autoclass:: caterpillar.py.If :members: -.. autoclass:: caterpillar.fields.ElseIf +.. autoclass:: caterpillar.py.ElseIf :members: \ No newline at end of file diff --git a/docs/sphinx/source/tutorial/basics.rst b/docs/sphinx/source/tutorial/basics.rst index 82b1511..99c706e 100644 --- a/docs/sphinx/source/tutorial/basics.rst +++ b/docs/sphinx/source/tutorial/basics.rst @@ -27,24 +27,52 @@ When dealing with binary data, numbers play a crucial role. Besides the default (e.g., uint8, uint16, etc.), *caterpillar* introduces some special integer formats. The default types include: -* Unsigned (:code:`u...`) and signed: - :attr:`~caterpillar.fields.int8`, :attr:`~caterpillar.fields.uint8`, - :attr:`~caterpillar.fields.int16`, :attr:`~caterpillar.fields.uint16`, - :attr:`~caterpillar.fields.int24`, :attr:`~caterpillar.fields.uint24`, - :attr:`~caterpillar.fields.int32`, :attr:`~caterpillar.fields.uint32`, - :attr:`~caterpillar.fields.int64`, :attr:`~caterpillar.fields.uint64`, - :attr:`~caterpillar.fields.ssize_t`, :attr:`~caterpillar.fields.size_t`, +.. tab-set:: -* Floating point: + .. tab-item:: Python - :attr:`~caterpillar.fields.float16`, :attr:`~caterpillar.fields.float32`, - :attr:`~caterpillar.fields.float64` + * Unsigned (:code:`u...`) and signed: + + :attr:`~caterpillar.fields.int8`, :attr:`~caterpillar.fields.uint8`, + :attr:`~caterpillar.fields.int16`, :attr:`~caterpillar.fields.uint16`, + :attr:`~caterpillar.fields.int24`, :attr:`~caterpillar.fields.uint24`, + :attr:`~caterpillar.fields.int32`, :attr:`~caterpillar.fields.uint32`, + :attr:`~caterpillar.fields.int64`, :attr:`~caterpillar.fields.uint64`, + :attr:`~caterpillar.fields.ssize_t`, :attr:`~caterpillar.fields.size_t`, + + * Floating point: + + :attr:`~caterpillar.fields.float16`, :attr:`~caterpillar.fields.float32`, + :attr:`~caterpillar.fields.float64` + + * Special primitives: + + :attr:`~caterpillar.fields.boolean`, :attr:`~caterpillar.fields.char`, + :attr:`~caterpillar.fields.void_ptr` + + + .. tab-item:: Caterpillar C + + * Unsigned (:code:`u...`) and signed: + + :attr:`~caterpillar.c.i8`, :attr:`~caterpillar.c.u8`, + :attr:`~caterpillar.c.i16`, :attr:`~caterpillar.c.u16`, + :attr:`~caterpillar.c.i24`, :attr:`~caterpillar.c.u24`, + :attr:`~caterpillar.c.i32`, :attr:`~caterpillar.c.u32`, + :attr:`~caterpillar.c.i64`, :attr:`~caterpillar.c.u64`, + :attr:`~caterpillar.c.i128`, :attr:`~caterpillar.c.u128`, + + * Floating point: + + :attr:`~caterpillar.c.f16`, :attr:`~caterpillar.c.f32`, + :attr:`~caterpillar.c.f64` + + * Special primitives: + + :attr:`~caterpillar.c.boolean`, :attr:`~caterpillar.c.char` -* Special primitives: - :attr:`~caterpillar.fields.boolean`, :attr:`~caterpillar.fields.char`, - :attr:`~caterpillar.fields.void_ptr` Custom-sized integer ~~~~~~~~~~~~~~~~~~~~ @@ -53,18 +81,38 @@ It's also possible to use integers with a custom size (in bits). However, it's i that you have to define the struct with the bit count, and internally, only the occupied bytes will be used. For example: ->>> field = F(Int(24)) # three-byte signed integer ->>> field = F(UInt(40)) # five-byte unsigned integer +.. tab-set:: + + .. tab-item:: Python + + >>> field = F(Int(24)) # three-byte signed integer + >>> field = F(UInt(40)) # five-byte unsigned integer + + + .. tab-item:: Caterpillar C + + >>> i48 = Int(48) # six-byte signed integer + >>> u40 = Int(40, signed=False) # five-byte unsigned integer Variable-sized integer ~~~~~~~~~~~~~~~~~~~~~~ -The built-in struct :class:`~caterpillar.fields.VarInt` supports parsing and building integers with variable length. Its +The built-in struct :class:`.py.VarInt`/:class:`.c.VarInt` supports parsing and building integers with variable length. Its documentation provides a detailed explanation of all different configurations. ->>> field = F(vint) # or F(VarInt()) +.. tab-set:: + + .. tab-item:: Python + + >>> field = F(vint) # or F(VarInt()) + + .. tab-item:: Caterpillar C + + >>> # use 'varint' directly or use VarInt() + >>> be_varint = BIG_ENDIAN + varint + >>> le_varint = VarInt(little_endian=True) Enumerations @@ -76,21 +124,43 @@ standard Python enumerations - classes extending :code:`enum.Enum` - with ease. Let's revisit `pHYS `_ chunk to add an enum to the last field. -.. code-block:: python - :caption: Simple enumeration in a struct definition +.. tab-set:: - import enum + .. tab-item:: Python - class PHYSUnit(enum.IntEnum): # <-- the enum value doesn't have to be int - __struct__ = uint8 # <-- to make the code even more compact, use this - UNKNOWN = 0 - METRE = 1 + .. code-block:: python + :caption: Simple enumeration in a struct definition + + import enum + + class PHYSUnit(enum.IntEnum): # <-- the enum value doesn't have to be int + __struct__ = uint8 # <-- to make the code even more compact, use this + UNKNOWN = 0 + METRE = 1 + + @struct(order=BigEndian) # <-- same as before + class PHYSChunk: + pixels_per_unit_x: uint32 + pixels_per_unit_y: uint32 + unit: PHYSUnit # <-- now we have an auto-enumeration + + + .. tab-item:: Caterpillar C - @struct(order=BigEndian) # <-- same as before - class PHYSChunk: - pixels_per_unit_x: uint32 - pixels_per_unit_y: uint32 - unit: PHYSUnit # <-- now we have an auto-enumeration + .. code-block:: python + :caption: Simple enumeration in a struct definition + + import enum + + class PHYSUnit(enum.IntEnum): # <-- the enum value doesn't have to be int + UNKNOWN = 0 + METRE = 1 + + @struct(endian=BIG_ENDIAN) # <-- same as before + class PHYSChunk: + pixels_per_unit_x: u32 + pixels_per_unit_y: u32 + unit: enumeration(u8, PHYSUnit) # <-- atom is required here .. important:: It's worth noting that a default value can be specified for the field as a fallback. If @@ -107,7 +177,19 @@ simplifies this with item access for defining arrays of static or dynamic size. We started with the `PLTE `_ chunk, which stores three-byte sequences. We can define an array of RGB objects as follows: ->>> PLTEChunk = RGB[this.length / 3] +.. tab-set:: + + .. tab-item:: Python + + >>> PLTEChunk = RGB[this.length / 3] + + .. tab-item:: Caterpillar C + + >>> PLTEChunk = RGB.__struct__[ContextPath("obj.length") / 3] + + .. versionadded:: 2.2.0 + The syntax will be changed once `__class_getitem__` is implemented by + any :class:`.c.Struct` instance. Since this chunk has only one field, the array specifier is used to make it a list type. The length is calculated based on the chunk's length field divided by three because the RGB class @@ -123,18 +205,50 @@ CString The CString in this library extends beyond a mere reference to C strings. It provides additional functionality, as demonstrated in the structure of the next chunk. -.. code-block:: python - :caption: The `tEXt `_ chunk structure +.. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + :caption: The `tEXt `_ chunk structure + + from caterpillar.py import * + from caterpillar.shortcuts import lenof + + @struct + class TEXTChunk: + # dynamic sized string that ends with a null-byte + keyword: CString(encoding="ISO-8859-1") + # static sized string based on the current context. some notes: + # - parent.length is the current chunkt's length + # - lenof(...) is the runtime length of the context variable + # - 1 because of the extra null-byte that is stripped from keyword + text: CString(encoding="ISO-8859-1", length=parent.length - lenof(this.keword) - 1) + + .. tab-item:: Caterpillar C + + .. code-block:: python + :caption: The `tEXt` chunk structure + + from caterpillar.c import * # <-- main difference + from caterpillar.shortcuts import lenof + # NOTE: lenof works here, because Caterpillar C's Context implements + # the 'Context Protocol'. + + parent = ContextPath("parent.obj") + this = ContextPath("obj") + + @struct + class TEXTChunk: + # dynamic sized string that ends with a null-byte + keyword: cstring(encoding="ISO-8859-1") + # static sized string based on the current context. some notes: + # - parent.length is the current chunkt's length + # - lenof(...) is the runtime length of the context variable + # - 1 because of the extra null-byte that is stripped from keyword + text: cstring(encoding="ISO-8859-1", length=parent.length - lenof(this.keword) - 1) + - @struct - class TEXTChunk: - # dynamic sized string that ends with a null-byte - keyword: CString(encoding="ISO-8859-1") - # static sized string based on the current context. some notes: - # - parent.length is the current chunkt's length - # - lenof(...) is the runtime length of the context variable - # - 1 because of the extra null-byte that is stripped from keyword - text: CString(encoding="ISO-8859-1", length=parent.length - lenof(this.keword) - 1) .. admonition:: Challenge @@ -145,25 +259,62 @@ additional functionality, as demonstrated in the structure of the next chunk. This solution serves as an example and isn't the only way to approach it! - .. code-block:: python - - @struct - class ITXTChunk: - keyword: CString(encoding="utf-8") - compression_flag: uint8 - # we actually don't need an Enum here - compression_method: uint8 - language_tag: CString(encoding="ASCII") - translated_keyword: CString(encoding="utf-8") - # length is calculated with parent.length - len(keyword)+len(b"\x00") - ... - text: CString( - encoding="utf-8", - length=parent.length - lenof(this.translated_keyword) - lenof(this.keyword) - 5, - ) + .. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + :linenos: + + @struct + class ITXTChunk: + keyword: CString(encoding="utf-8") + compression_flag: uint8 + # we actually don't need an Enum here + compression_method: uint8 + language_tag: CString(encoding="ASCII") + translated_keyword: CString(encoding="utf-8") + # length is calculated with parent.length - len(keyword)+len(b"\x00") - ... + text: CString( + encoding="utf-8", + length=parent.length - lenof(this.translated_keyword) - lenof(this.keyword) - 5, + ) + + .. tab-item:: Caterpillar C + + .. code-block:: python + :linenos: + + from caterpillar.c import * # <-- main difference + from caterpillar.shortcuts import lenof + + parent = ContextPath("parent.obj") + this = ContextPath("obj") + + @struct + class ITXTChunk: + keyword: cstring() # default encoding is "utf-8" + compression_flag: u8 + # we actually don't need an Enum here + compression_method: u8 + language_tag: cstring(encoding="ASCII") + translated_keyword: cstring(...) # explicit greedy parsing + # length is calculated with parent.length - len(keyword)+len(b"\x00") - ... + text: cstring( + parent.length - lenof(this.translated_keyword) - lenof(this.keyword) - 5, + ) You can also apply your own termination character, for example: ->>> struct = CString(pad="\x0A") +.. tab-set:: + + .. tab-item:: Python + + >>> struct = CString(pad="\x0A") + + .. tab-item:: Caterpillar C + + >>> s = cstring(sep="\x0A") This struct will use a space as the termination character and strip all trailing padding bytes. @@ -174,7 +325,16 @@ String Besides special the special *c strings* there's a default :class:`~caterpillar.fields.String` class that implements the basic behaviour of a string. It's crucial to specify the length for this struct. ->>> struct = String(100 or this.length) # static integer or context lambda +.. tab-set:: + + .. tab-item:: Python + + >>> struct = String(100 or this.length) # static integer or context lambda + + .. tab-item:: Caterpillar C + + >>> # takes static length, context lambda, another atom or ... for greedy parsing + >>> s = cstring(100) Prefixed @@ -185,11 +345,23 @@ encoding is specified, the returned value will be of type :code:`bytes`. This cl using the given struct and then retrieves the corresponding number of bytes from the stream returned by that struct. ->>> field = F(Prefixed(uint8, encoding="utf-8")) ->>> pack("Hello, World!", field) -b'\rHello, World!' ->>> unpack(field, _) -'Hello, World!' +.. tab-set:: + + .. tab-item:: Python + + >>> field = F(Prefixed(uint8, encoding="utf-8")) + >>> pack("Hello, World!", field) + b'\rHello, World!' + >>> unpack(field, _) + 'Hello, World!' + + .. tab-item:: Caterpillar C + + >>> s = pstring(u8) + >>> pack("Hello, World!", s) + b'\rHello, World!' + >>> unpack(_, s) + 'Hello, World!' Byte Sequences @@ -201,11 +373,19 @@ Memory When dealing with data that can be stored in memory and you intend to print out your unpacked object, the :class:`~caterpillar.fields.Memory` struct is recommended. ->>> m = F(Memory(5)) # static size; dynamic size is allowed too ->>> pack(bytes([i for i in range(5)], m)) -b'\x00\x01\x02\x03\x04' ->>> unpack(m, _) - +.. tab-set:: + + .. tab-item:: Python + + >>> m = F(Memory(5)) # static size; dynamic size is allowed too + >>> pack(bytes([i for i in range(5)], m)) + b'\x00\x01\x02\x03\x04' + >>> unpack(m, _) + + + .. tab-item:: Caterpillar C + + *Not supported yet.* Bytes ~~~~~ @@ -214,20 +394,48 @@ If direct access to the bytes is what you need, the :class:`~caterpillar.fields. converts the :code:`memoryview` to :code:`bytes`. Additionally, as mentioned earlier, you can use the :class:`~caterpillar.fields.Prefixed` class to unpack bytes of a prefixed size. ->>> field = F(Bytes(5)) # static, dynamic and greedy size allowed + +.. tab-set:: + + .. tab-item:: Python + + >>> field = F(Bytes(5)) # static, dynamic and greedy size allowed + + + .. tab-item:: Caterpillar C + + >>> b = octetstring(5) # static, dynamic size allowed With the gained knowledge, let's implement the struct for the `fDAT `_ chunk of our PNG format. It should look like this: -.. code-block:: python - :caption: Implementation for the frame data chunk - @struct(order=BigEndian) # <-- endianess as usual - class FDATChunk: - sequence_number: uint32 - # We rather use a memory instance here instead of Bytes() - frame_data: Memory(parent.length - 4) +.. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + :caption: Implementation for the frame data chunk + + @struct(order=BigEndian) # <-- endianess as usual + class FDATChunk: + sequence_number: uint32 + # We rather use a memory instance here instead of Bytes() + frame_data: Memory(parent.length - 4) + + .. tab-item:: Caterpillar C + + .. code-block:: python + :caption: Implementation for the frame data chunk + + parent = ContextPath("parent.obj") + + @struct(endian=BIG_ENDIAN) + class FDATChunk: + sequence_number: u32 + frame_data: octetstring(parent.length - 4) + .. admonition:: Challenge @@ -236,6 +444,8 @@ chunk of our PNG format. It should look like this: .. dropdown:: Solution + Python API only: + .. code-block:: python :caption: Sample implementation of the *zTXt* chunk diff --git a/docs/sphinx/source/tutorial/first_steps.rst b/docs/sphinx/source/tutorial/first_steps.rst index dc071ca..4231436 100644 --- a/docs/sphinx/source/tutorial/first_steps.rst +++ b/docs/sphinx/source/tutorial/first_steps.rst @@ -31,19 +31,35 @@ Given the important role of structs in this library, let's start by understandin starting point is the `PLTE `_ chunk, which uses three-byte entries for its data. -.. code-block:: python - :caption: RGB struct for the PLTE chunk +.. tab-set:: - from caterpillar.model import struct # <-- these imports are important - from caterpillar.fields import * + .. tab-item:: Python - @struct # <-- just decorate the class with the struct() function - class RGB: - r: uint8 # <-- a field can be defined just like this - g: uint8 - b: uint8 + .. code-block:: python + :caption: RGB struct for the PLTE chunk + + from caterpillar.py import * # <-- just import everything + + @struct # <-- just decorate the class with the struct() function + class RGB: + r: uint8 # <-- a field can be defined just like this + g: uint8 + b: uint8 + .. tab-item:: Caterpillar C + + .. code-block:: python + :caption: RGB struct for the PLTE chunk (using Caterpillar C) + + from caterpillar.c import * # <-- just import everything + + @struct # <-- just decorate the class with the struct() function + class RGB: + r: u8 # <-- a field can be defined just like this + g: u8 + b: u8 + With this simple annotation, the struct class becomes universally applicable. You can integrate it into other struct definitions or instantiate objects of the class. @@ -51,7 +67,8 @@ integrate it into other struct definitions or instantiate objects of the class. .. note:: To optimize memory space and get faster attribute access times, you have to explicitly - enable the :attr:`~caterpillar.options.S_SLOTS` option. More information can be taken from :ref:`options`. + enable the :attr:`~caterpillar.options.S_SLOTS` option. More information can be taken from + :ref:`options`. Wow, thats it? That was less than expected? Let's move directly to working with the defined class. @@ -71,9 +88,20 @@ module. When packing data, a struct and an input object are needed. Thanks to the RGB class encapsulating its struct instance, explicitly stating the struct to use becomes unnecessary. ->>> obj = RGB(r=1, g=2, b=3) ->>> pack(obj) # equivalent to pack(obj, RGB) -b'\x01\x02\x03' +.. tab-set:: + + .. tab-item:: Python + + >>> obj = RGB(r=1, g=2, b=3) + >>> pack(obj) # equivalent to pack(obj, RGB) + b'\x01\x02\x03' + + .. tab-item:: Caterpillar C + + >>> obj = RGB(r=1, g=2, b=3) + >>> pack(obj, RGB.__struct__) # required as of version 2.2.0 + b'\x01\x02\x03' + Unpacking data ^^^^^^^^^^^^^^ @@ -81,8 +109,17 @@ Unpacking data Recreating data from binary streams is as easy as serializing objects. Here, providing the struct directly or our struct class is necessary. ->>> unpack(RGB, b"\x01\x02\x03") -RGB(r=1, g=2, b=3) +.. tab-set:: + + .. tab-item:: Python + + >>> unpack(RGB, b"\x01\x02\x03") + RGB(r=1, g=2, b=3) + + .. tab-item:: Caterpillar C + + >>> unpack(b"\x01\x02\x03", RGB.__struct__) + RGB(r=1, g=2, b=3) And no, we're not done yet - we've just wrapped up the warm-up! @@ -100,14 +137,36 @@ configure the struct to correctly decode these integer fields. `StackOverflow `_ or `Wikipedia `_ -.. code-block:: python - :caption: Configuring a struct-wide endianess - @struct(order=BigEndian) # <-- extra argument to apply the order to all fields. - class PHYSChunk: - pixels_per_unit_x: uint32 # <-- same definition as above - pixels_per_unit_y: uint32 - unit: uint8 # <-- endianess meaningless, only one byte +.. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + :caption: Configuring a struct-wide endianess + + @struct(order=BigEndian) # <-- extra argument to apply the order to all fields. + class PHYSChunk: + pixels_per_unit_x: uint32 # <-- same definition as above + pixels_per_unit_y: uint32 + unit: uint8 # <-- endianess meaningless, only one byte + + .. tab-item:: Caterpillar C + + .. code-block:: python + :caption: Configuring a struct-wide endianess + + @struct(endian=BIG_ENDIAN) # <-- extra argument to apply the order to all fields. + class PHYSChunk: + pixels_per_unit_x: u32 # <-- same definition as above + pixels_per_unit_y: u32 + unit: u8 # <-- endianess meaningless, only one byte + + .. note:: + Even though, there will be :code:`` visible in the annotations + of the class, the created struct stores the modified big endian integer + atom. + If your structs depend on the architecture associated with the binary, you can also specify a struct-wide :class:`~caterpillar.byteorder.Arch`. @@ -120,17 +179,37 @@ struct-wide :class:`~caterpillar.byteorder.Arch`. .. dropdown:: Solution :icon: check - .. code-block:: python - :caption: Example implementation + Example implementation + + .. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + :linenos: + + @struct(order=BigEndian) + class TIMEChunk: + year: uint16 # <-- we could also use: BigEndian + uint16 + month: uint8 + day: uint8 + hour: uint8 + minute: uint8 + second: uint8 + + .. tab-item:: Caterpillar C - @struct(order=BigEndian) - class TIMEChunk: - year: uint16 # <-- we could also use: BigEndian + uint16 - month: uint8 - day: uint8 - hour: uint8 - minute: uint8 - second: uint8 + .. code-block:: python + :linenos: + + @struct(endian=BIG_ENDIAN) + class TIMEChunk: + year: u16 # <-- we could also use: BIG_ENDIAN + u16 + month: u8 + day: u8 + hour: u8 + minute: u8 + second: u8 Note that we can integrate this struct later on. @@ -142,12 +221,26 @@ To minimize changes to your codebase or require as little adaptation as possible this library, there's a documentation feature. By utilizing the ability to globally apply options, you just need the following code: -.. code-block:: python - :caption: Enable documentation feature +.. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + :caption: Enable documentation feature + + from caterpillar.shortcuts import opt + + opt.set_struct_flags(opt.S_REPLACE_TYPES) + + + .. tab-item:: Caterpillar C + + .. code-block:: python + :caption: Enable documentation feature - from caterpillar.shortcuts import opt + from caterpillar.c import * - opt.set_struct_flags(opt.S_REPLACE_TYPES) + STRUCT_OPTIONS.add(S_REPLACE_TYPES) .. tip:: If you are working with `Sphinx `_, you might need diff --git a/docs/sphinx/source/tutorial/index.rst b/docs/sphinx/source/tutorial/index.rst index 430552b..8d2a40d 100644 --- a/docs/sphinx/source/tutorial/index.rst +++ b/docs/sphinx/source/tutorial/index.rst @@ -22,10 +22,11 @@ For those ready to dig deeper or contribute to the framework, the :ref:`library- is an important resource to start from. .. toctree:: - :caption: Roadmap + :caption: Python :numbered: :maxdepth: 4 first_steps.rst basics.rst advanced.rst + diff --git a/src/ccaterpillar/atomimpl/cstring.c b/src/ccaterpillar/atomimpl/cstring.c index a6d90b7..3804a7b 100644 --- a/src/ccaterpillar/atomimpl/cstring.c +++ b/src/ccaterpillar/atomimpl/cstring.c @@ -64,7 +64,7 @@ static int cp_cstringatom_init(CpCStringAtomObject* self, PyObject* args, PyObject* kwds) { static char* kwlist[] = { "length", "encoding", "errors", - "terminator", "keep", NULL }; + "sep", "keep", NULL }; PyObject* length = NULL; PyObject* encoding = NULL; PyObject* errors = NULL; @@ -72,7 +72,7 @@ cp_cstringatom_init(CpCStringAtomObject* self, PyObject* args, PyObject* kwds) int keep_terminator = false; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|OOOp", + "|OOOOp", kwlist, &length, &encoding, @@ -85,6 +85,11 @@ cp_cstringatom_init(CpCStringAtomObject* self, PyObject* args, PyObject* kwds) _modulestate* mod = get_global_module_state(); _Cp_SetObj(self->m_length, length); + if (!self->m_length) { + // TODO: document this + _Cp_SetObj(self->m_length, Py_Ellipsis); + } + _Cp_SetObj(self->m_encoding, encoding); if (!self->m_encoding) { Py_XSETREF(self->m_encoding, mod->str_utf8); @@ -294,7 +299,7 @@ static PyMemberDef CpCStringAtom_Members[] = { }; PyTypeObject CpCStringAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpStringAtom_NAME), + PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpCStringAtom_NAME), .tp_basicsize = sizeof(CpCStringAtomObject), .tp_itemsize = 0, .tp_dealloc = (destructor)cp_cstringatom_dealloc, diff --git a/src/ccaterpillar/module.c b/src/ccaterpillar/module.c index e067eb1..47325fe 100644 --- a/src/ccaterpillar/module.c +++ b/src/ccaterpillar/module.c @@ -33,7 +33,6 @@ return NULL; \ } - // ------------------------------------------------------------------------------ // module static PyObject* @@ -516,7 +515,7 @@ PyInit__C(void) CpModuleState_AddObject( cp_option__global_struct_options, "STRUCT_OPTIONS", PySet_New(NULL)); CpModuleState_AddObject( - cp_option__global_struct_options, "FIELD_OPTIONS", PySet_New(NULL)); + cp_option__global_field_options, "FIELD_OPTIONS", PySet_New(NULL)); /* setup arch and endian */ CpModuleState_AddObject(cp_endian__native, diff --git a/test/_C/atoms/test_string.py b/test/_C/atoms/test_string.py index ab1eb64..68bde61 100644 --- a/test/_C/atoms/test_string.py +++ b/test/_C/atoms/test_string.py @@ -48,12 +48,12 @@ def test_cstringatom_pack(): assert pack("foo", s) == b"foo\x00" assert pack("bar", cstring(3, "utf-8")) == b"bar" assert pack("baz", cstring(u8, "utf-8")) == b"\x04baz\x00" - assert pack("abc", cstring(4, terminator="\x20")) == b"abc\x20" + assert pack("abc", cstring(4, sep="\x20")) == b"abc\x20" def test_cstringatom_unpack(): - assert unpack(b"foo\x00", cstring(...)) == "foo" - assert unpack(b"ba\x00r", cstring(...)) == "ba" + assert unpack(b"foo\x00", cstring()) == "foo" + assert unpack(b"ba\x00r", cstring()) == "ba" assert unpack(b"baz\x00", cstring(3)) == "baz" assert unpack(b"baz\x00", cstring(4)) == "baz" assert unpack(b"\x04baz\x00", cstring(u8)) == "baz" - assert unpack(b"abc\x20", cstring(4, terminator="\x20")) == "abc" + assert unpack(b"abc\x20", cstring(4, sep="\x20")) == "abc" diff --git a/test/_C/atoms/test_varint.py b/test/_C/atoms/test_varint.py index c23349e..700c2ac 100644 --- a/test/_C/atoms/test_varint.py +++ b/test/_C/atoms/test_varint.py @@ -5,20 +5,20 @@ if caterpillar.native_support(): - from caterpillar._C import unpack, pack, varint, varint_t + from caterpillar._C import unpack, pack, varint, VarInt from caterpillar._C import BIG_ENDIAN as be def test_varint_init(): # The C varint atom implements a variable-length unsigned # integer for big-endian AND little-endian encoding. The # byteorder operator '+' can be applied as usual. - assert varint_t(True).little_endian is True - assert varint_t(False).little_endian is False + assert VarInt(True).little_endian is True + assert VarInt(False).little_endian is False # We can even control the least significant byte using # the 'lsb' flag. - assert varint_t(lsb=True).lsb is True - assert varint_t(lsb=False).lsb is False + assert VarInt(lsb=True).lsb is True + assert VarInt(lsb=False).lsb is False # These parameters are virible through the representation: # little-endian: diff --git a/test/_C/test_option.py b/test/_C/test_option.py index 126932f..5c08040 100644 --- a/test/_C/test_option.py +++ b/test/_C/test_option.py @@ -33,6 +33,7 @@ def test_option(option: Option): STRUCT_OPTIONS.add(option) assert option in STRUCT_OPTIONS + STRUCT_OPTIONS.remove(option) # necessary for other tests @pytest.mark.parametrize( From 2ec7343fa31a090621d1c49228de29fe6c87de43 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sat, 28 Sep 2024 07:37:08 +0200 Subject: [PATCH 26/29] Removed all field-* related types --- + Renamed atom documentation files and removed the "atom" suffix --- CMakeLists.txt | 1 - .../objects/atoms/{boolatom.rst => bool.rst} | 0 .../objects/atoms/{charatom.rst => char.rst} | 0 .../atoms/{floatatom.rst => float.rst} | 0 .../objects/atoms/{intatom.rst => int.rst} | 0 .../atoms/{paddingatom.rst => padding.rst} | 0 docs/sphinx/source/tutorial/basics.rst | 27 +- src/capi.dat | 25 +- .../include/caterpillar/caterpillar.h | 1 - .../include/caterpillar/caterpillarapi.h | 33 - src/caterpillar/include/caterpillar/field.h | 180 ----- src/ccaterpillar/caterpillarapi.c | 24 +- src/ccaterpillar/context.c | 25 + src/ccaterpillar/field.c | 748 ------------------ src/ccaterpillar/module.c | 9 - src/ccaterpillar/parsing_sizeof.c | 116 +-- src/ccaterpillar/parsing_typeof.c | 100 +-- src/ccaterpillar/struct.c | 27 +- 18 files changed, 76 insertions(+), 1240 deletions(-) rename docs/sphinx/source/reference/capi/objects/atoms/{boolatom.rst => bool.rst} (100%) rename docs/sphinx/source/reference/capi/objects/atoms/{charatom.rst => char.rst} (100%) rename docs/sphinx/source/reference/capi/objects/atoms/{floatatom.rst => float.rst} (100%) rename docs/sphinx/source/reference/capi/objects/atoms/{intatom.rst => int.rst} (100%) rename docs/sphinx/source/reference/capi/objects/atoms/{paddingatom.rst => padding.rst} (100%) delete mode 100644 src/caterpillar/include/caterpillar/field.h delete mode 100644 src/ccaterpillar/field.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e664f5..f179498 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,6 @@ python_add_library( src/ccaterpillar/arch.c src/ccaterpillar/atomobj.c src/ccaterpillar/context.c - src/ccaterpillar/field.c src/ccaterpillar/option.c src/ccaterpillar/module.c src/ccaterpillar/context.c diff --git a/docs/sphinx/source/reference/capi/objects/atoms/boolatom.rst b/docs/sphinx/source/reference/capi/objects/atoms/bool.rst similarity index 100% rename from docs/sphinx/source/reference/capi/objects/atoms/boolatom.rst rename to docs/sphinx/source/reference/capi/objects/atoms/bool.rst diff --git a/docs/sphinx/source/reference/capi/objects/atoms/charatom.rst b/docs/sphinx/source/reference/capi/objects/atoms/char.rst similarity index 100% rename from docs/sphinx/source/reference/capi/objects/atoms/charatom.rst rename to docs/sphinx/source/reference/capi/objects/atoms/char.rst diff --git a/docs/sphinx/source/reference/capi/objects/atoms/floatatom.rst b/docs/sphinx/source/reference/capi/objects/atoms/float.rst similarity index 100% rename from docs/sphinx/source/reference/capi/objects/atoms/floatatom.rst rename to docs/sphinx/source/reference/capi/objects/atoms/float.rst diff --git a/docs/sphinx/source/reference/capi/objects/atoms/intatom.rst b/docs/sphinx/source/reference/capi/objects/atoms/int.rst similarity index 100% rename from docs/sphinx/source/reference/capi/objects/atoms/intatom.rst rename to docs/sphinx/source/reference/capi/objects/atoms/int.rst diff --git a/docs/sphinx/source/reference/capi/objects/atoms/paddingatom.rst b/docs/sphinx/source/reference/capi/objects/atoms/padding.rst similarity index 100% rename from docs/sphinx/source/reference/capi/objects/atoms/paddingatom.rst rename to docs/sphinx/source/reference/capi/objects/atoms/padding.rst diff --git a/docs/sphinx/source/tutorial/basics.rst b/docs/sphinx/source/tutorial/basics.rst index 99c706e..d7b5cd6 100644 --- a/docs/sphinx/source/tutorial/basics.rst +++ b/docs/sphinx/source/tutorial/basics.rst @@ -482,14 +482,29 @@ reference in detail. The current object that is being packed or parsed can be referenced with a shortcut :attr:`~caterpillar.context.this`. Additionally, the parent object (if any) can be referenced by using :attr:`~caterpillar.context.parent`. +.. tab-set:: -.. code-block:: python - :caption: Understanding the *context* + .. tab-item:: Python + + .. code-block:: python + :caption: Understanding the *context* + + @struct + class Format: + length: uint8 + foo: CString(this.length) # <-- just reference the length field - @struct - class Format: - length: uint8 - foo: CString(this.length) # <-- just reference the length field + .. tab-item:: Caterpillar C + + .. code-block:: python + :caption: Understanding the *context* + + this = ContextPath("obj") + + @struct + class Format: + length: u8 + foo: cstring(this.length) .. note:: You can apply any operation on context paths. However, be aware that conditional branches must diff --git a/src/capi.dat b/src/capi.dat index 5ffb4ad..fc5c63e 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -23,7 +23,6 @@ # Defines the source file (relative to this file) that contains the # function definitions. -src:ccaterpillar/field.c src:ccaterpillar/context.c src:ccaterpillar/arch.c src:ccaterpillar/atomobj.c @@ -69,9 +68,9 @@ type:5:_contextobj:CpContextObject:Context type:6:_unaryexpr:CpUnaryExprObject:unaryexpr type:7:_binaryexpr:CpBinaryExprObject:binaryexpr type:8:_contextpath:CpContextPathObject:ContextPath -type:9:_fieldobj:CpFieldObject:Field -type:10:_fieldatomobj:CpFieldAtomObject:fieldatom -type:11:_fieldcatomobj:CpFieldCAtomObject:fieldcatom +# type:9:_fieldobj:CpFieldObject:Field +# type:10:_fieldatomobj:CpFieldAtomObject:fieldatom +# type:11:_fieldcatomobj:CpFieldCAtomObject:fieldcatom obj:12:CpInvalidDefault_Type:- obj:13:CpDefaultOption_Type:- @@ -115,14 +114,14 @@ func:54:CpUnaryExpr_New:CpUnaryExprObject*:+1 func:55:CpBinaryExpr_New:CpBinaryExprObject*:+1 func:56:CpContextPath_New:CpContextPathObject*:+1 func:57:CpContextPath_FromString:CpContextPathObject*:+1 -func:58:CpField_New:CpFieldObject*:null -func:59:CpField_HasCondition:int:null -func:60:CpField_IsEnabled:int:null -func:61:CpField_GetOffset:Py_ssize_t:null -func:62:CpField_EvalSwitch:PyObject*:+1 -func:63:CpField_GetLength:PyObject*:+1 +# func:58:CpField_New:CpFieldObject*:null +# func:59:CpField_HasCondition:int:null +# func:60:CpField_IsEnabled:int:null +# func:61:CpField_GetOffset:Py_ssize_t:null +# func:62:CpField_EvalSwitch:PyObject*:+1 +# func:63:CpField_GetLength:PyObject*:+1 func:64:CpTypeOf:PyObject*:+1 -func:65:CpTypeOf_Field:PyObject*:+1 +# func:65:CpTypeOf_Field:PyObject*:+1 func:66:CpTypeOf_Common:PyObject*:+1 func:67:CpPack:int:null # func:68:CpPack_Field:int:null @@ -130,7 +129,7 @@ func:67:CpPack:int:null # func:70:CpPack_Struct:int:null func:72:_CpPack_EvalLength:int:null func:73:CpSizeOf:PyObject*:+1 -func:74:CpSizeOf_Field:PyObject*:+1 +# func:74:CpSizeOf_Field:PyObject*:+1 func:75:CpSizeOf_Struct:PyObject*:+1 func:76:CpSizeOf_Common:PyObject*:+1 func:77:_Cp_SizeOf:PyObject*:+1 @@ -155,7 +154,7 @@ func:96:CpLayer_New:CpLayerObject*:+1 func:98:CpLayer_Invalidate:int:null func:99:CpStructFieldInfo_New:CpStructFieldInfoObject*:+1 func:100:CpStruct_AddFieldInfo:int:null -func:101:CpStruct_AddField:int:null +# func:101:CpStruct_AddField:int:null func:102:CpStruct_New:CpStructObject*:+1 func:103:CpStruct_GetAnnotations:PyObject*:+1 func:104:CpStruct_ReplaceType:int:null diff --git a/src/caterpillar/include/caterpillar/caterpillar.h b/src/caterpillar/include/caterpillar/caterpillar.h index 17aea62..1124a9b 100644 --- a/src/caterpillar/include/caterpillar/caterpillar.h +++ b/src/caterpillar/include/caterpillar/caterpillar.h @@ -11,7 +11,6 @@ #include "caterpillar/struct.h" #include "caterpillar/context.h" #include "caterpillar/atomobj.h" -#include "caterpillar/field.h" #include "caterpillar/state.h" #include "caterpillar/parsing.h" #include "caterpillar/default.h" diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index 3fb35b1..38ef90c 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -44,12 +44,6 @@ struct _binaryexpr; typedef struct _binaryexpr CpBinaryExprObject; struct _contextpath; typedef struct _contextpath CpContextPathObject; -struct _fieldobj; -typedef struct _fieldobj CpFieldObject; -struct _fieldatomobj; -typedef struct _fieldatomobj CpFieldAtomObject; -struct _fieldcatomobj; -typedef struct _fieldcatomobj CpFieldCAtomObject; struct _option; typedef struct _option CpOptionObject; struct _stateobj; @@ -131,12 +125,6 @@ extern PyTypeObject CpBinaryExpr_Type; #define CpBinaryExpr_NAME "binaryexpr" extern PyTypeObject CpContextPath_Type; #define CpContextPath_NAME "ContextPath" -extern PyTypeObject CpField_Type; -#define CpField_NAME "Field" -extern PyTypeObject CpFieldAtom_Type; -#define CpFieldAtom_NAME "fieldatom" -extern PyTypeObject CpFieldCAtom_Type; -#define CpFieldCAtom_NAME "fieldcatom" extern PyTypeObject CpInvalidDefault_Type; extern PyTypeObject CpDefaultOption_Type; extern PyObject _CpInvalidDefault_Object; @@ -203,19 +191,11 @@ CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); CpBinaryExprObject* CpBinaryExpr_New(int op, PyObject* left, PyObject* right); CpContextPathObject* CpContextPath_New(PyObject* path); CpContextPathObject* CpContextPath_FromString(const char* path); -CpFieldObject* CpField_New(PyObject* atom); -int CpField_HasCondition(CpFieldObject* self); -int CpField_IsEnabled(CpFieldObject* self, PyObject* context); -Py_ssize_t CpField_GetOffset(CpFieldObject* self, PyObject* context); -PyObject* CpField_EvalSwitch(CpFieldObject* self, PyObject* op, PyObject* context); -PyObject* CpField_GetLength(CpFieldObject* self, PyObject* context); PyObject* CpTypeOf(PyObject* op); -PyObject* CpTypeOf_Field(CpFieldObject* op); PyObject* CpTypeOf_Common(PyObject* op); int CpPack(PyObject* op, PyObject* atom, PyObject* io, PyObject* globals); int _CpPack_EvalLength(CpLayerObject* layer,PyObject* length,Py_ssize_t size,bool* greedy,Py_ssize_t* dstLength); PyObject* CpSizeOf(PyObject* op, PyObject* globals); -PyObject* CpSizeOf_Field(CpFieldObject* field, CpLayerObject* layer); PyObject* CpSizeOf_Struct(CpStructObject* struct_, CpLayerObject* layer); PyObject* CpSizeOf_Common(PyObject* op, CpLayerObject* layer); PyObject* _Cp_SizeOf(PyObject* op, CpLayerObject* layer); @@ -237,7 +217,6 @@ CpLayerObject* CpLayer_New(CpStateObject* state, CpLayerObject* parent); int CpLayer_Invalidate(CpLayerObject* self); CpStructFieldInfoObject* CpStructFieldInfo_New(PyObject* name, PyObject* field); int CpStruct_AddFieldInfo(CpStructObject* o, CpStructFieldInfoObject* info); -int CpStruct_AddField(CpStructObject* o, CpFieldObject* field, int exclude); CpStructObject* CpStruct_New(PyObject* model); PyObject* CpStruct_GetAnnotations(CpStructObject* o, int eval); int CpStruct_ReplaceType(CpStructObject* o, PyObject* name, PyObject* type); @@ -315,9 +294,6 @@ caterpillar_api.py #define CpUnaryExpr_Type (*(unaryexpr *)Cp_API[6]) #define CpBinaryExpr_Type (*(binaryexpr *)Cp_API[7]) #define CpContextPath_Type (*(ContextPath *)Cp_API[8]) -#define CpField_Type (*(Field *)Cp_API[9]) -#define CpFieldAtom_Type (*(fieldatom *)Cp_API[10]) -#define CpFieldCAtom_Type (*(fieldcatom *)Cp_API[11]) #define CpInvalidDefault_Type (*(PyTypeObject *)Cp_API[12]) #define CpDefaultOption_Type (*(PyTypeObject *)Cp_API[13]) #define _CpInvalidDefault_Object (*(PyObject *)Cp_API[14]) @@ -356,19 +332,11 @@ caterpillar_api.py #define CpBinaryExpr_New (*((CpBinaryExprObject* (*)(int op, PyObject* left, PyObject* right)))Cp_API[55]) #define CpContextPath_New (*((CpContextPathObject* (*)(PyObject* path)))Cp_API[56]) #define CpContextPath_FromString (*((CpContextPathObject* (*)(const char* path)))Cp_API[57]) -#define CpField_New (*((CpFieldObject* (*)(PyObject* atom)))Cp_API[58]) -#define CpField_HasCondition (*((int (*)(CpFieldObject* self)))Cp_API[59]) -#define CpField_IsEnabled (*((int (*)(CpFieldObject* self, PyObject* context)))Cp_API[60]) -#define CpField_GetOffset (*((Py_ssize_t (*)(CpFieldObject* self, PyObject* context)))Cp_API[61]) -#define CpField_EvalSwitch (*((PyObject* (*)(CpFieldObject* self, PyObject* op, PyObject* context)))Cp_API[62]) -#define CpField_GetLength (*((PyObject* (*)(CpFieldObject* self, PyObject* context)))Cp_API[63]) #define CpTypeOf (*((PyObject* (*)(PyObject* op)))Cp_API[64]) -#define CpTypeOf_Field (*((PyObject* (*)(CpFieldObject* op)))Cp_API[65]) #define CpTypeOf_Common (*((PyObject* (*)(PyObject* op)))Cp_API[66]) #define CpPack (*((int (*)(PyObject* op, PyObject* atom, PyObject* io, PyObject* globals)))Cp_API[67]) #define _CpPack_EvalLength (*((int (*)(CpLayerObject* layer,PyObject* length,Py_ssize_t size,bool* greedy,Py_ssize_t* dstLength)))Cp_API[72]) #define CpSizeOf (*((PyObject* (*)(PyObject* op, PyObject* globals)))Cp_API[73]) -#define CpSizeOf_Field (*((PyObject* (*)(CpFieldObject* field, CpLayerObject* layer)))Cp_API[74]) #define CpSizeOf_Struct (*((PyObject* (*)(CpStructObject* struct_, CpLayerObject* layer)))Cp_API[75]) #define CpSizeOf_Common (*((PyObject* (*)(PyObject* op, CpLayerObject* layer)))Cp_API[76]) #define _Cp_SizeOf (*((PyObject* (*)(PyObject* op, CpLayerObject* layer)))Cp_API[77]) @@ -390,7 +358,6 @@ caterpillar_api.py #define CpLayer_Invalidate (*((int (*)(CpLayerObject* self)))Cp_API[98]) #define CpStructFieldInfo_New (*((CpStructFieldInfoObject* (*)(PyObject* name, PyObject* field)))Cp_API[99]) #define CpStruct_AddFieldInfo (*((int (*)(CpStructObject* o, CpStructFieldInfoObject* info)))Cp_API[100]) -#define CpStruct_AddField (*((int (*)(CpStructObject* o, CpFieldObject* field, int exclude)))Cp_API[101]) #define CpStruct_New (*((CpStructObject* (*)(PyObject* model)))Cp_API[102]) #define CpStruct_GetAnnotations (*((PyObject* (*)(CpStructObject* o, int eval)))Cp_API[103]) #define CpStruct_ReplaceType (*((int (*)(CpStructObject* o, PyObject* name, PyObject* type)))Cp_API[104]) diff --git a/src/caterpillar/include/caterpillar/field.h b/src/caterpillar/include/caterpillar/field.h deleted file mode 100644 index 3ac4fd3..0000000 --- a/src/caterpillar/include/caterpillar/field.h +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Copyright (C) MatrixEditor 2024 - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#ifndef CP_FIELD_H -#define CP_FIELD_H - -#include "atomobj.h" - -/* Caterpillar Field definition */ - -/** - * @brief C implementation of the Python equivalent. - * - * Represents a field in a data structure. - */ -struct _fieldobj -{ - PyObject_HEAD - - /// The name of this field - PyObject* m_name; - - /// Stores a reference to the actual parsing struct that will be used - /// to parse or build our data. This attribute is never null. - PyObject* m_atom; - - /// An automatically inferred or explicitly specified byte order. Note that - /// this attribute may have no impact on the underlying struct. The default - /// byte order is `NATIVE_ENDIAN`. - PyObject* m_endian; - - /// Using the `@` operator an offset can be assigned to a field. If set, the - /// stream will be reset and set to the original position. - /// - /// The minus one indicates that no offset has been associated with this - /// field. - /// - /// @see CpField_GetOffset - PyObject* m_offset; - - /// Additional options that can be enabled using the logical OR operator - /// `|`. - /// - /// Note that there are default options that will be set automatically: - /// * `field:keep_position`: - /// Persists the streams position after parsing data using the - /// underlying struct. In relation to `offset`, this option will reset - /// the stream to its original position if deactivated. - /// * `field:dynamic`: - /// Specifies that this field does not store a constant size. - /// * `field:sequential`: - /// An automatic flag that indicates this field stores a sequential - /// struct. - PyObject* m_options; - - /// A constant or dynamic value to represent the amount of structs. Zero - /// indicates there are no sequence types associated with this field. - PyObject* m_length; - - /// An extra attribute that stores additional options that can be translates - /// as a switch statement. - PyObject* m_switch; - - /// Given optional execution this attribute should be used to return a boolean - /// value that decides whether the value of this field should be set. Using - /// `//` the condition can be set during class declaration. - PyObject* m_condition; - - /// The field's architecture (inferred or explicitly specified). - PyObject* m_arch; - - /// The configured default value. - PyObject* m_default; - - // internal state variables - int8_t s_size; - int8_t s_type; - int8_t s_sequential; - int8_t s_keep_pos; -}; - -/// Field type -// PyAPI_DATA(PyTypeObject) CpField_Type; - -/** - * @brief Check whether the given object is afield - * - * @param v the object to check - * @return true if the object is a field - * @return false if the object is not a field - */ -#define CpField_CheckExact(v) Py_IS_TYPE((v), &CpField_Type) - -/** - * @brief Check whether the given object is a field - * - * @param v the object to check - * @return true if the object is a field - * @return false if the object is not a field - */ -#define CpField_Check(v) PyObject_TypeCheck((v), &CpField_Type) - -// ----------------------------------------------------------------------------- -// field atom - -/** - * @brief C implementation of the Python equivalent (FieldMixin). - * - * A simple mixin to support operators used to create `Field` instances. - */ -struct _fieldatomobj -{ - CpAtom_HEAD -}; - -/// Field atom type -// PyAPI_DATA(PyTypeObject) CpFieldAtom_Type; - -/** - * @brief Check whether the given object is a field atom - * - * @param v the object to check - * @return true if the object is a field atom - * @return false if the object is not a field atom - */ -#define CpFieldAtom_CheckExact(v) Py_IS_TYPE((v), &CpFieldAtom_Type) - -/** - * @brief Check whether the given object is a field atom - * - * @param v the object to check - * @return true if the object is a field atom - * @return false if the object is not a field atom - */ -#define CpFieldAtom_Check(v) PyObject_TypeCheck((v), &CpFieldAtom_Type) - -#define CpFieldAtom_HEAD CpFieldAtomObject ob_base; - -// ----------------------------------------------------------------------------- -// field C atom -struct _fieldcatomobj -{ - CpCAtom_HEAD -}; - -/** - * @brief Check whether the given object is a field C atom - * - * @param v the object to check - * @return true if the object is a field C atom - * @return false if the object is not a field C atom - */ -#define CpFieldCAtom_CheckExact(v) Py_IS_TYPE((v), &CpFieldCAtom_Type) - -/** - * @brief Check whether the given object is a field C atom - * - * @param v the object to check - * @return true if the object is a field C atom - * @return false if the object is not a field C atom - */ -#define CpFieldCAtom_Check(v) PyObject_TypeCheck((v), &CpFieldCAtom_Type) - -#define CpFieldCAtom_HEAD CpFieldCAtomObject ob_base; -#define CpFieldCAtom_CATOM(x) (x)->ob_base.ob_base - -#endif \ No newline at end of file diff --git a/src/ccaterpillar/caterpillarapi.c b/src/ccaterpillar/caterpillarapi.c index 1d3d448..b3cae8a 100644 --- a/src/ccaterpillar/caterpillarapi.c +++ b/src/ccaterpillar/caterpillarapi.c @@ -12,9 +12,9 @@ void *Cp_API[] = { (void *) &CpUnaryExpr_Type, (void *) &CpBinaryExpr_Type, (void *) &CpContextPath_Type, - (void *) &CpField_Type, - (void *) &CpFieldAtom_Type, - (void *) &CpFieldCAtom_Type, + NULL, + NULL, + NULL, (void *) &CpInvalidDefault_Type, (void *) &CpDefaultOption_Type, (void *) &_CpInvalidDefault_Object, @@ -61,14 +61,14 @@ void *Cp_API[] = { (void *) &CpBinaryExpr_New, (void *) &CpContextPath_New, (void *) &CpContextPath_FromString, - (void *) &CpField_New, - (void *) &CpField_HasCondition, - (void *) &CpField_IsEnabled, - (void *) &CpField_GetOffset, - (void *) &CpField_EvalSwitch, - (void *) &CpField_GetLength, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, (void *) &CpTypeOf, - (void *) &CpTypeOf_Field, + NULL, (void *) &CpTypeOf_Common, (void *) &CpPack, NULL, @@ -77,7 +77,7 @@ void *Cp_API[] = { NULL, (void *) &_CpPack_EvalLength, (void *) &CpSizeOf, - (void *) &CpSizeOf_Field, + NULL, (void *) &CpSizeOf_Struct, (void *) &CpSizeOf_Common, (void *) &_Cp_SizeOf, @@ -104,7 +104,7 @@ void *Cp_API[] = { (void *) &CpLayer_Invalidate, (void *) &CpStructFieldInfo_New, (void *) &CpStruct_AddFieldInfo, - (void *) &CpStruct_AddField, + NULL, (void *) &CpStruct_New, (void *) &CpStruct_GetAnnotations, (void *) &CpStruct_ReplaceType, diff --git a/src/ccaterpillar/context.c b/src/ccaterpillar/context.c index 0309d6e..214f38b 100644 --- a/src/ccaterpillar/context.c +++ b/src/ccaterpillar/context.c @@ -542,6 +542,31 @@ cp_binaryexpr__call__(CpBinaryExprObject* self, PyObject* args, PyObject* kw) return result; } +/* operations */ +#define _CpBinaryExpr_BinaryNumberMethod(name, op) \ + static PyObject* cp_binaryexpr_as_number_##name(PyObject* self, \ + PyObject* other) \ + { \ + return (PyObject*)CpBinaryExpr_New(op, self, other); \ + } + +_CpBinaryExpr_BinaryNumberMethod(add, CpBinaryExpr_OpAdd); +_CpBinaryExpr_BinaryNumberMethod(sub, CpBinaryExpr_OpSub); +_CpBinaryExpr_BinaryNumberMethod(mul, CpBinaryExpr_OpMul); +_CpBinaryExpr_BinaryNumberMethod(div, CpBinaryExpr_OpTrueDiv); +_CpBinaryExpr_BinaryNumberMethod(truediv, CpBinaryExpr_OpTrueDiv); +_CpBinaryExpr_BinaryNumberMethod(floordiv, CpBinaryExpr_OpFloorDiv); +_CpBinaryExpr_BinaryNumberMethod(mod, CpBinaryExpr_OpMod); +_CpBinaryExpr_BinaryNumberMethod(lshift, CpBinaryExpr_OpLShift); +_CpBinaryExpr_BinaryNumberMethod(rshift, CpBinaryExpr_OpRShift); +_CpBinaryExpr_BinaryNumberMethod(and, CpBinaryExpr_OpBitAnd); +_CpBinaryExpr_BinaryNumberMethod(xor, CpBinaryExpr_OpBitXor); +_CpBinaryExpr_BinaryNumberMethod(or, CpBinaryExpr_OpBitOr); +_CpBinaryExpr_BinaryNumberMethod(pow, CpBinaryExpr_OpPow); +_CpBinaryExpr_BinaryNumberMethod(matmul, CpBinaryExpr_OpMatMul); + +#undef _CpContextPath_BinaryNumberMethod + /*CpAPI*/ CpBinaryExprObject* CpBinaryExpr_New(int op, PyObject* left, PyObject* right) diff --git a/src/ccaterpillar/field.c b/src/ccaterpillar/field.c deleted file mode 100644 index 8a6346b..0000000 --- a/src/ccaterpillar/field.c +++ /dev/null @@ -1,748 +0,0 @@ -/* field implementation */ -#include - -#include "caterpillar//caterpillar.h" - -#include "structmember.h" - -/* impl */ -static PyObject* -cp_field_new(PyTypeObject* type, PyObject* args, PyObject* kw) -{ - CpFieldObject* self; - self = (CpFieldObject*)type->tp_alloc(type, 0); - if (self == NULL) - return NULL; - - self->m_name = PyUnicode_FromString("_"); - self->m_atom = NULL; - self->m_endian = NULL; - self->m_offset = NULL; - self->m_arch = NULL; - self->m_length = NULL; - - Py_XINCREF(CpInvalidDefault); - self->m_default = CpInvalidDefault; - self->m_switch = NULL; - self->m_options = NULL; - self->m_condition = Py_NewRef(Py_True); - - // internal state - self->s_size = false; - self->s_type = false; - self->s_sequential = false; - self->s_keep_pos = true; - return (PyObject*)self; -} - -static void -cp_field_dealloc(CpFieldObject* self) -{ - - Py_XDECREF(self->m_name); - Py_XDECREF(self->m_endian); - Py_XDECREF(self->m_offset); - Py_XDECREF(self->m_arch); - Py_XDECREF(self->m_length); - Py_XDECREF(self->m_default); - Py_XDECREF(self->m_switch); - Py_XDECREF(self->m_options); - Py_XDECREF(self->m_condition); - Py_XDECREF(self->m_atom); - Py_TYPE(self)->tp_free((PyObject*)self); -} - -static PyObject* -cp_field_repr(CpFieldObject* self) -{ - return PyUnicode_FromFormat("", self->m_name); -} - -static int -cp_field_set_length(CpFieldObject* self, PyObject* value, void* closure); - -static int -cp_field_set_offset(CpFieldObject* self, PyObject* value, void* closure); - -static int -cp_field_init(CpFieldObject* self, PyObject* args, PyObject* kw) -{ - static char* kwlist[] = { "atom", "name", "endian", "offset", - "arch", "length", "default", "switch", - "options", "condition", NULL }; - PyObject *name = NULL, *endian = NULL, *offset = NULL, *arch = NULL, - *length = NULL, *default_ = NULL, *switch_ = NULL, *options = NULL, - *condition = NULL, *atom = NULL; - if (!PyArg_ParseTupleAndKeywords(args, - kw, - "O|UOOOOOOOO", - kwlist, - &atom, - &name, - &endian, - &offset, - &arch, - &length, - &default_, - &switch_, - &options, - &condition)) - return -1; - - _modulestate* state = get_global_module_state(); - - _Cp_SetObj(self->m_name, name); - _Cp_SetObj(self->m_endian, endian); - _Cp_SetObj(self->m_arch, arch); - - _Cp_SetObj(self->m_default, default_); - _Cp_SetObj(self->m_switch, switch_); - _Cp_SetObj(self->m_options, options); - _Cp_SetObj(self->m_condition, condition); - _Cp_SetObj(self->m_atom, atom); - - if (!self->m_options) { - self->m_options = PySet_New(NULL); - if (!self->m_options) - return -1; - - if (PySet_Add(self->m_options, state->cp_option__keep_position) < 0) { - return -1; - }; - } - - // tyoe checks - if (!self->m_atom || self->m_atom == Py_None) { - PyErr_SetString(PyExc_ValueError, "atom is required"); - return -1; - } - - if (!self->m_options || !PySet_Check(self->m_options)) { - PyErr_SetString(PyExc_ValueError, "options must be a set"); - return -1; - } - - if (!PyObject_IsInstance(self->m_atom, (PyObject*)&CpAtom_Type)) { - if (!PyCallable_Check(self->m_atom)) { - PyErr_SetString(PyExc_ValueError, - "atom must be an instace of atom or a callable function"); - return -1; - } - } - - if (!self->m_arch) { - _Cp_SetObj(self->m_arch, state->cp_arch__host); - } - if (!self->m_endian) { - _Cp_SetObj(self->m_endian, state->cp_endian__native); - } - - if (length) - if (cp_field_set_length(self, length, NULL) < 0) - return -1; - if (offset) - if (cp_field_set_offset(self, offset, NULL) < 0) - return -1; - - self->s_size = CpAtom_HasSize(self->m_atom); - self->s_type = CpAtom_HasType(self->m_atom); - return 0; -} - -static PyObject* -cp_field_get_length(CpFieldObject* self) -{ - return Py_NewRef(self->m_length ? self->m_length : Py_None); -} - -static int -cp_field_set_length(CpFieldObject* self, PyObject* value, void* closure) -{ - if (!value || value == Py_None || - (!PyLong_Check(value) && !PyCallable_Check(value) && - !PyObject_IsInstance(value, (PyObject*)&PyEllipsis_Type) && - !PyObject_IsInstance(value, (PyObject*)&PySlice_Type))) { - PyErr_SetString(PyExc_TypeError, - "length must be an integer, context lambda or a special " - "type"); - return -1; - } - - _Cp_SetObj(self->m_length, value); - _modulestate* state = get_global_module_state(); - - int8_t is_number = PyLong_Check(value); - - if (is_number && PyLong_AsSize_t(self->m_length) <= 1) { - // remove sequential option automatically - PySet_Discard(self->m_options, state->cp_option__sequential); - self->s_sequential = false; - } else { - PySet_Add(self->m_options, state->cp_option__sequential); - self->s_sequential = true; - } - - if (!is_number) { - PySet_Add(self->m_options, state->cp_option__dynamic); - } else { - PySet_Discard(self->m_options, state->cp_option__dynamic); - } - return 0; -} - -static PyObject* -cp_field_get_offset(CpFieldObject* self) -{ - return Py_NewRef(self->m_offset ? self->m_offset : Py_None); -} - -static int -cp_field_set_offset(CpFieldObject* self, PyObject* value, void* closure) -{ - if (!value || value == Py_None || - (!PyLong_Check(value) && !PyCallable_Check(value))) { - PyErr_SetString(PyExc_TypeError, - "offset must be an integer or a context lambda"); - return -1; - } - - _Cp_SetObj(self->m_offset, value); - _modulestate* state = get_global_module_state(); - if (PyLong_Check(self->m_offset) && PyLong_AsSize_t(self->m_offset) != -1) { - PySet_Discard(self->m_options, state->cp_option__keep_position); - self->s_keep_pos = false; - } else { - PySet_Add(self->m_options, state->cp_option__keep_position); - self->s_keep_pos = true; - } - return 0; -} - -static PyObject* -cp_field_get_switch(CpFieldObject* self) -{ - return Py_NewRef(self->m_switch ? self->m_switch : Py_None); -} - -static int -cp_field_set_switch(CpFieldObject* self, PyObject* value, void* closure) -{ - if (!value || value == Py_None || - (!PyCallable_Check(value) && !PyDict_Check(value))) { - PyErr_SetString(PyExc_TypeError, - "switch must be a callable or a dictionary"); - return -1; - } - - _Cp_SetObj(self->m_switch, value); - return 0; -} - -static PyObject* -cp_field_get_condition(CpFieldObject* self) -{ - return Py_NewRef(self->m_condition ? self->m_condition : Py_None); -} - -static int -cp_field_set_condition(CpFieldObject* self, PyObject* value, void* closure) -{ - if (!value || value == Py_None || - (!PyCallable_Check(value) && !PyBool_Check(value))) { - PyErr_SetString(PyExc_TypeError, - "condition must be a callable or a boolean"); - return -1; - } - - _Cp_SetObj(self->m_condition, value); - return 0; -} - -static PyObject* -cp_field_as_number_matmul(CpFieldObject* self, PyObject* other) -{ - if (cp_field_set_offset(self, other, NULL) < 0) { - return NULL; - } - return Py_NewRef((PyObject*)self); -} - -static PyObject* -cp_field_as_number_or(CpFieldObject* self, PyObject* other) -{ - if (!other || other->ob_type != &CpOption_Type) { - Py_RETURN_NOTIMPLEMENTED; - } - - PySet_Add(self->m_options, other); - return Py_NewRef((PyObject*)self); -} - -static PyObject* -cp_field_as_number_xor(CpFieldObject* self, PyObject* other) -{ - if (!other || other->ob_type != &CpOption_Type) { - Py_RETURN_NOTIMPLEMENTED; - } - - PySet_Discard(self->m_options, other); - return Py_NewRef((PyObject*)self); -} - -static PyObject* -cp_field_as_number_rshift(CpFieldObject* self, PyObject* other) -{ - if (cp_field_set_switch(self, other, NULL) < 0) { - return NULL; - } - return Py_NewRef((PyObject*)self); -} - -static PyObject* -cp_field_as_number_floordiv(CpFieldObject* self, PyObject* other) -{ - if (cp_field_set_condition(self, other, NULL) < 0) { - return NULL; - } - return Py_NewRef((PyObject*)self); -} - -static PyObject* -cp_field_as_number_add(CpFieldObject* self, PyObject* other) -{ - if (!other || other->ob_type != &CpEndian_Type) { - Py_RETURN_NOTIMPLEMENTED; - } - _Cp_SetObj(self->m_endian, other); - return Py_NewRef((PyObject*)self); -} - -static PyObject* -cp_field_as_mapping_getitem(CpFieldObject* self, PyObject* key) -{ - // allow multiple dimensions - if (cp_field_set_length(self, key, NULL) < 0) { - return NULL; - } - return Py_NewRef((PyObject*)self); -} - -/*CpAPI*/ -CpFieldObject* -CpField_New(PyObject* atom) -{ - return (CpFieldObject*)CpObject_CreateOneArg(&CpField_Type, atom); -} - -/*CpAPI*/ -int -CpField_HasCondition(CpFieldObject* self) -{ - return !Py_IsTrue(self->m_condition); -} - -/*CpAPI*/ -int -CpField_IsEnabled(CpFieldObject* self, PyObject* context) -{ - if (self->m_condition == NULL) { - return true; - } - - if (PyCallable_Check(self->m_condition)) { - PyObject* result = PyObject_CallOneArg(self->m_condition, context); - if (result == NULL) { - return -1; - } - int truth = PyObject_IsTrue(result); - Py_DECREF(result); - return truth; - } - - return PyObject_IsTrue(self->m_condition); -} - -/*CpAPI*/ -Py_ssize_t -CpField_GetOffset(CpFieldObject* self, PyObject* context) -{ - Py_ssize_t offset = -1; - if (self->m_offset != NULL) { - - if (PyCallable_Check(self->m_offset)) { - PyObject* result = PyObject_CallOneArg(self->m_offset, context); - if (result == NULL) { - return -1; - } - offset = PyLong_AsSsize_t(result); - Py_DECREF(result); - } else { - offset = PyLong_AsSsize_t(self->m_offset); - } - } - return offset; -} - -/*CpAPI*/ -PyObject* -CpField_EvalSwitch(CpFieldObject* self, PyObject* op, PyObject* context) -{ - if (self->m_switch == NULL) { - PyErr_SetString(PyExc_TypeError, "field does not have a switch"); - return NULL; - } - - PyObject* result = NULL; - if (PyCallable_Check(self->m_switch)) { - result = PyObject_CallOneArg(self->m_switch, context); - if (!result) - return NULL; - } else { - result = PyObject_GetItem(self->m_switch, op); - if (!result) { - result = PyObject_GetItem(self->m_switch, CpInvalidDefault); - if (!result) - return NULL; - } - } - - // TODO: check for nested struct - return result; -} - -/*CpAPI*/ -PyObject* -CpField_GetLength(CpFieldObject* self, PyObject* context) -{ - if (self->m_length == NULL) { - PyErr_SetString(PyExc_TypeError, "field does not have a length"); - return NULL; - } - - if (PyCallable_Check(self->m_length)) { - PyObject* result = PyObject_CallOneArg(self->m_length, context); - if (!result) - return NULL; - return result; - } - return Py_NewRef(self->m_length); -} - -/* docs */ - -/* members */ -static PyMemberDef CpField_Members[] = { - { "name", T_OBJECT, offsetof(CpFieldObject, m_name), 0 }, - { "default", T_OBJECT, offsetof(CpFieldObject, m_default), 0 }, - { "options", T_OBJECT, offsetof(CpFieldObject, m_options), 0 }, - { "atom", T_OBJECT, offsetof(CpFieldObject, m_atom), READONLY }, - { "endian", T_OBJECT, offsetof(CpFieldObject, m_endian), 0 }, - { "arch", T_OBJECT, offsetof(CpFieldObject, m_arch), 0 }, - { NULL } /* Sentinel */ -}; - -static PyGetSetDef CpField_GetSetters[] = { - { "length", - (getter)cp_field_get_length, - (setter)cp_field_set_length, - NULL, - NULL }, - { "offset", - (getter)cp_field_get_offset, - (setter)cp_field_set_offset, - NULL, - NULL }, - { "switch", - (getter)cp_field_get_switch, - (setter)cp_field_set_switch, - NULL, - NULL }, - { "condition", - (getter)cp_field_get_condition, - (setter)cp_field_set_condition, - NULL, - NULL }, - { NULL } /* Sentinel */ -}; - -static PyNumberMethods CpField_NumberMethods = { - .nb_matrix_multiply = (binaryfunc)cp_field_as_number_matmul, - .nb_or = (binaryfunc)cp_field_as_number_or, - .nb_xor = (binaryfunc)cp_field_as_number_xor, - .nb_rshift = (binaryfunc)cp_field_as_number_rshift, - .nb_floor_divide = (binaryfunc)cp_field_as_number_floordiv, - .nb_add = (binaryfunc)cp_field_as_number_add, -}; - -static PyMappingMethods CpField_MappingMethods = { - .mp_subscript = (binaryfunc)cp_field_as_mapping_getitem, -}; - -static PyMethodDef CpField_Methods[] = { - // { "__type__", (PyCFunction)cp_typeof_field, METH_NOARGS }, - { NULL } /* Sentinel */ -}; - -/* type */ -PyTypeObject CpField_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(Field), - .tp_basicsize = sizeof(CpFieldObject), - .tp_dealloc = (destructor)cp_field_dealloc, - .tp_repr = (reprfunc)cp_field_repr, - .tp_as_number = &CpField_NumberMethods, - .tp_as_mapping = &CpField_MappingMethods, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = NULL, - .tp_methods = CpField_Methods, - .tp_members = CpField_Members, - .tp_getset = CpField_GetSetters, - .tp_init = (initproc)cp_field_init, - .tp_new = (newfunc)cp_field_new, -}; - -// ----------------------------------------------------------------------------- -// field atom -static PyObject* -cp_fieldatom_new(PyTypeObject* type, PyObject* args, PyObject* kw) -{ - CpFieldAtomObject* self; - self = (CpFieldAtomObject*)type->tp_alloc(type, 0); - if (self == NULL) - return NULL; - return (PyObject*)self; -} - -static void -cp_fieldatom_dealloc(CpFieldAtomObject* self) -{ - Py_TYPE(self)->tp_free((PyObject*)self); -} - -static int -cp_fieldatom_init(CpFieldAtomObject* self, PyObject* args, PyObject* kw) -{ - if ((args && PyTuple_Size(args)) || (kw && PyDict_Size(kw))) { - PyErr_SetString(PyExc_TypeError, - "fieldatoms cannot be initialized with arguments"); - return -1; - } - return 0; -} - -#define _CpFieldAtom_DefMethod(name, func) \ - static PyObject* cp_fieldatom_##name(CpFieldAtomObject* self, \ - PyObject* other) \ - { \ - CpFieldObject* field = CpField_New((PyObject*)self); \ - if (!field) { \ - return NULL; \ - } \ - if (func(field, other, NULL) < 0) { \ - Py_XDECREF(field); \ - return NULL; \ - } \ - return (PyObject*)field; \ - } - -_CpFieldAtom_DefMethod(as_number_matmul, cp_field_set_offset); -_CpFieldAtom_DefMethod(as_number_floordiv, cp_field_set_condition); -_CpFieldAtom_DefMethod(as_number_rshift, cp_field_set_switch); -_CpFieldAtom_DefMethod(as_mapping_getitem, cp_field_set_length); - -#undef _CpFieldAtom_DefMethod - -static PyObject* -cp_fieldatom_as_number_add(CpFieldAtomObject* self, PyObject* other) -{ - if (other->ob_type != &CpEndian_Type) { - Py_RETURN_NOTIMPLEMENTED; - } - - CpFieldObject* field = (CpFieldObject*)CpField_New((PyObject*)self); - if (!field) { - return NULL; - } - _Cp_SetObj(field->m_endian, other); - return (PyObject*)field; -} - -static PyObject* -cp_fieldatom_as_number_or(CpFieldAtomObject* self, PyObject* other) -{ - if (other->ob_type != &CpOption_Type) { - Py_RETURN_NOTIMPLEMENTED; - } - - CpFieldObject* field = (CpFieldObject*)CpField_New((PyObject*)self); - if (!field) { - return NULL; - } - PySet_Add(field->m_options, other); - return (PyObject*)field; -} - -static PyObject* -cp_fieldatom_as_number_xor(CpFieldAtomObject* self, PyObject* other) -{ - if (other->ob_type != &CpOption_Type) { - Py_RETURN_NOTIMPLEMENTED; - } - - CpFieldObject* field = (CpFieldObject*)CpField_New((PyObject*)self); - if (!field) { - return NULL; - } - PySet_Discard(field->m_options, other); - return (PyObject*)field; -} - -/* docs */ - -/* members */ - -static PyNumberMethods CpFieldAtom_NumberMethods = { - .nb_matrix_multiply = (binaryfunc)cp_fieldatom_as_number_matmul, - .nb_floor_divide = (binaryfunc)cp_fieldatom_as_number_floordiv, - .nb_add = (binaryfunc)cp_fieldatom_as_number_add, - .nb_inplace_add = (binaryfunc)cp_fieldatom_as_number_add, - .nb_or = (binaryfunc)cp_fieldatom_as_number_or, - .nb_xor = (binaryfunc)cp_fieldatom_as_number_xor, -}; - -static PyMappingMethods CpFieldAtom_MappingMethods = { - .mp_subscript = (binaryfunc)cp_fieldatom_as_mapping_getitem, -}; - -/* type */ -PyTypeObject CpFieldAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(fieldatom), - .tp_basicsize = sizeof(CpFieldAtomObject), - .tp_dealloc = (destructor)cp_fieldatom_dealloc, - .tp_as_number = &CpFieldAtom_NumberMethods, - .tp_as_mapping = &CpFieldAtom_MappingMethods, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_doc = NULL, - .tp_init = (initproc)cp_fieldatom_init, - .tp_new = (newfunc)cp_fieldatom_new, -}; - -// ----------------------------------------------------------------------------- -// field C atom -static PyObject* -cp_fieldcatom_new(PyTypeObject* type, PyObject* args, PyObject* kw) -{ - CpFieldAtomObject* self; - self = (CpFieldAtomObject*)type->tp_alloc(type, 0); - if (self == NULL) - return NULL; - return (PyObject*)self; -} - -static void -cp_fieldcatom_dealloc(CpFieldAtomObject* self) -{ - Py_TYPE(self)->tp_free((PyObject*)self); -} - -static int -cp_fieldcatom_init(CpFieldAtomObject* self, PyObject* args, PyObject* kw) -{ - return 0; -} - -#define _CpFieldCAtom_DefMethod(name, func) \ - static PyObject* cp_fieldcatom_##name(CpFieldAtomObject* self, \ - PyObject* other) \ - { \ - CpFieldObject* field = CpField_New((PyObject*)self); \ - if (!field) { \ - return NULL; \ - } \ - if (func(field, other, NULL) < 0) { \ - Py_XDECREF(field); \ - return NULL; \ - } \ - return (PyObject*)field; \ - } - -_CpFieldCAtom_DefMethod(as_number_matmul, cp_field_set_offset); -_CpFieldCAtom_DefMethod(as_number_floordiv, cp_field_set_condition); -_CpFieldCAtom_DefMethod(as_number_rshift, cp_field_set_switch); -_CpFieldCAtom_DefMethod(as_mapping_getitem, cp_field_set_length); - -#undef _CpFieldCAtom_DefMethod - -static PyObject* -cp_fieldcatom_as_number_add(CpFieldAtomObject* self, PyObject* other) -{ - if (other->ob_type != &CpEndian_Type) { - Py_RETURN_NOTIMPLEMENTED; - } - - CpFieldObject* field = (CpFieldObject*)CpField_New((PyObject*)self); - if (!field) { - return NULL; - } - _Cp_SetObj(field->m_endian, other); - return (PyObject*)field; -} - -static PyObject* -cp_fieldcatom_as_number_or(CpFieldAtomObject* self, PyObject* other) -{ - if (other->ob_type != &CpOption_Type) { - Py_RETURN_NOTIMPLEMENTED; - } - - CpFieldObject* field = (CpFieldObject*)CpField_New((PyObject*)self); - if (!field) { - return NULL; - } - PySet_Add(field->m_options, other); - return (PyObject*)field; -} - -static PyObject* -cp_fieldcatom_as_number_xor(CpFieldAtomObject* self, PyObject* other) -{ - if (other->ob_type != &CpOption_Type) { - Py_RETURN_NOTIMPLEMENTED; - } - - CpFieldObject* field = (CpFieldObject*)CpField_New((PyObject*)self); - if (!field) { - return NULL; - } - PySet_Discard(field->m_options, other); - return (PyObject*)field; -} - -/* docs */ - -/* members */ - -static PyNumberMethods CpFieldCAtom_NumberMethods = { - .nb_matrix_multiply = (binaryfunc)cp_fieldcatom_as_number_matmul, - .nb_floor_divide = (binaryfunc)cp_fieldcatom_as_number_floordiv, - .nb_add = (binaryfunc)cp_fieldcatom_as_number_add, - .nb_rshift = (binaryfunc)cp_fieldcatom_as_number_rshift, - .nb_or = (binaryfunc)cp_fieldcatom_as_number_or, - .nb_xor = (binaryfunc)cp_fieldcatom_as_number_xor, -}; - -static PyMappingMethods CpFieldCAtom_MappingMethods = { - .mp_subscript = (binaryfunc)cp_fieldcatom_as_mapping_getitem, -}; - -/* type */ -PyTypeObject CpFieldCAtom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(fieldcatom), - .tp_basicsize = sizeof(CpFieldCAtomObject), - .tp_dealloc = (destructor)cp_fieldcatom_dealloc, - .tp_as_number = &CpFieldCAtom_NumberMethods, - .tp_as_mapping = &CpFieldCAtom_MappingMethods, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_doc = NULL, - .tp_init = (initproc)cp_fieldcatom_init, - .tp_new = (newfunc)cp_fieldcatom_new, -}; diff --git a/src/ccaterpillar/module.c b/src/ccaterpillar/module.c index 47325fe..1ce6c26 100644 --- a/src/ccaterpillar/module.c +++ b/src/ccaterpillar/module.c @@ -318,12 +318,6 @@ PyInit__C(void) CpModule_SetupType(&CpBinaryExpr_Type); CpModule_SetupType(&CpUnaryExpr_Type); CpModule_SetupType(&CpContextPath_Type); - CpModule_SetupType(&CpField_Type); - - CpFieldAtom_Type.tp_base = &CpAtom_Type; - CpFieldCAtom_Type.tp_base = &CpCAtom_Type; - CpModule_SetupType(&CpFieldAtom_Type); - CpModule_SetupType(&CpFieldCAtom_Type); CpModule_SetupType(&CpLayer_Type); CpSeqLayer_Type.tp_base = &CpLayer_Type; @@ -399,9 +393,6 @@ PyInit__C(void) CpModule_AddObject("InvalidDefault", CpInvalidDefault); CpModule_AddObject("DefaultOptionType", &CpDefaultOption_Type); CpModule_AddObject("DefaultOption", CpDefaultOption); - CpModule_AddObject(CpField_NAME, &CpField_Type); - CpModule_AddObject(CpFieldAtom_NAME, &CpFieldAtom_Type); - CpModule_AddObject(CpFieldCAtom_NAME, &CpFieldCAtom_Type); CpModule_AddObject(CpLayer_NAME, &CpLayer_Type); CpModule_AddObject(CpSeqLayer_NAME, &CpSeqLayer_Type); CpModule_AddObject(CpObjLayer_NAME, &CpObjLayer_Type); diff --git a/src/ccaterpillar/parsing_sizeof.c b/src/ccaterpillar/parsing_sizeof.c index 0f6b08d..031278e 100644 --- a/src/ccaterpillar/parsing_sizeof.c +++ b/src/ccaterpillar/parsing_sizeof.c @@ -15,118 +15,6 @@ CpSizeOf_Common(PyObject* op, CpLayerObject* layer) return PyObject_CallMethodOneArg(op, state->str___size__, (PyObject*)layer); } -/*CpAPI*/ -PyObject* -CpSizeOf_Field(CpFieldObject* field, CpLayerObject* layer) -{ - // Size calculation is done in a special way. The following situations have - // to be considered: - // - // 1. Disabled field: return 0 - // 2. Dynamic field: raise an error - // 3. Sequential field: multiply the size of the underlying atom by the - // number of elements - // 4. Constant field: just return the size of the atom - if (!field) { - PyErr_SetString(PyExc_ValueError, "field is NULL!"); - return NULL; - } - if (!layer) { - PyErr_SetString(PyExc_ValueError, "state is NULL!"); - return NULL; - } - - // set path - CpLayer_AppendPath(layer, field->m_name); - if (!layer->m_path) { - return NULL; - } - - if (!CpField_IsEnabled(field, (PyObject*)layer)) { - // see case 1. - return PyLong_FromLong(0); - } - - _modulestate* mod = layer->m_state->mod; - PyObject *count = PyLong_FromLong(1), *size = NULL, *extraSize = NULL; - if (!count) { - return NULL; - } - - if (PySet_Contains(field->m_options, mod->cp_option__dynamic)) { - // see case 2. - PyErr_SetString(PyExc_ValueError, "dynamic fields are not supported!"); - return NULL; - } - - // prepare context - // Py_XSETREF(layer->m_field, Py_NewRef(field)); - if (field->s_sequential) { - count = CpField_GetLength(field, (PyObject*)layer); - if (!count) { - return NULL; - } - - if (!PyLong_Check(count)) { - Py_XDECREF(count); - PyErr_SetString(PyExc_ValueError, "length is not an integer!"); - return NULL; - } - } - - PyObject* atom = Py_NewRef(field->m_atom); - size = _Cp_SizeOf(atom, layer); - if (!size) { - goto fail; - } - - if (field->m_switch) { - // Static switch structures are supported only if the predecessor - // is a context lambda. - if (!PyCallable_Check(field->m_switch)) { - PyErr_SetString( - PyExc_ValueError, - "Switch statement without ContextLambda is danymic sized!"); - goto fail; - } - - PyObject* value = PyObject_CallOneArg(field->m_switch, (PyObject*)layer); - if (!value) { - goto fail; - } - Py_XSETREF(extraSize, size); - size = _Cp_SizeOf(value, layer); - if (!size) { - goto fail; - } - Py_XDECREF(value); - } - - PyObject* result = PyNumber_Multiply(size, count); - if (!result) { - goto fail; - } - if (extraSize) { - Py_XSETREF(result, PyNumber_Add(result, extraSize)); - if (!result) { - goto fail; - } - } - - Py_XDECREF(size); - Py_XDECREF(count); - Py_XDECREF(atom); - Py_XDECREF(extraSize); - return result; - -fail: - Py_XDECREF(atom); - Py_XDECREF(count); - Py_XDECREF(size); - Py_XDECREF(extraSize); - return NULL; -} - /*CpAPI*/ PyObject* CpSizeOf_Struct(CpStructObject* struct_, CpLayerObject* layer) @@ -211,9 +99,7 @@ _Cp_SizeOf(PyObject* op, CpLayerObject* layer) return NULL; } - if (CpField_CheckExact(op)) { - result = CpSizeOf_Field((CpFieldObject*)op, layer); - } else if (CpCAtom_Check(op)) { + if (CpCAtom_Check(op)) { result = CpSizeOf_CAtom((CpCAtomObject*)op, layer); } else if (CpStruct_CheckExact(op)) { result = CpSizeOf_Struct((CpStructObject*)op, layer); diff --git a/src/ccaterpillar/parsing_typeof.c b/src/ccaterpillar/parsing_typeof.c index cc1ed8e..b924325 100644 --- a/src/ccaterpillar/parsing_typeof.c +++ b/src/ccaterpillar/parsing_typeof.c @@ -30,102 +30,6 @@ CpTypeOf_Common(PyObject* op) return type; } -/*CpAPI*/ -PyObject* -CpTypeOf_Field(CpFieldObject* op) -{ - PyObject *type = NULL, *switch_types = NULL; - if (!op) { - PyErr_SetString(PyExc_ValueError, "input object is NULL!"); - return NULL; - } - - _modulestate* state = get_global_module_state(); - if (!op->s_type) { - return Py_NewRef(state->Any_Type); - } - - if (!op->m_switch || Py_IsNone(op->m_switch)) { - // new ref - type = CpTypeOf(op->m_atom); - if (!type) { - goto err; - } - } else if (PyCallable_Check(op->m_switch)) { - Py_XSETREF(type, Py_NewRef(state->Any_Type)); - } else { - PyObject* types = PyList_New(0); - if (!types) { - goto err; - } - PyObject *atomType = CpTypeOf(op->m_atom); - if (!atomType) { - Py_XDECREF(types); - goto err; - } - if (PyList_Append(types, atomType) < 0) { - Py_XDECREF(types); - goto err; - } - - PyObject* values = PyDict_Values(op->m_switch); - if (!values) { - Py_XDECREF(types); - goto err; - } - - Py_ssize_t length = PyList_GET_SIZE(values); - PyObject* switch_type = NULL; - for (Py_ssize_t i = 0; i < length; i++) { - PyObject* value = PyList_GetItem(values, i); - if (!value) { - Py_XDECREF(values); - Py_XDECREF(types); - goto err; - } - - switch_type = CpTypeOf(value); - if (!switch_type) { - Py_XDECREF(values); - Py_XDECREF(types); - goto err; - } - - if (!PySequence_Contains(types, switch_type)) { - PyList_Append(types, switch_type); - } - Py_XDECREF(switch_type); - switch_type = NULL; - } - - Py_XDECREF(values); - PyObject* tuple = PyList_AsTuple(types); - Py_XDECREF(types); - if (!tuple) { - goto err; - } - - type = PyObject_GetItem(state->Union_Type, tuple); - Py_XDECREF(tuple); - } - - if (!type) { - goto err; - } - - if (op->s_sequential) { - Py_XSETREF(type, PyObject_GetItem(state->List_Type, type)); - if (!type) { - goto err; - } - } - - return type; -err: - Py_XDECREF(type); - return NULL; -} - /*CpAPI*/ PyObject * CpTypeOf_CAtom(CpCAtomObject* op) @@ -145,9 +49,7 @@ CpTypeOf(PyObject* op) return NULL; } - if (CpField_CheckExact(op)) { - return CpTypeOf_Field((CpFieldObject*)op); - } else if (CpStruct_CheckExact(op)) { + if (CpStruct_CheckExact(op)) { return Py_NewRef(((CpStructObject*)op)->m_model); } else if (CpCAtom_CheckExact(op)) { return CpTypeOf_CAtom((CpCAtomObject*)op); diff --git a/src/ccaterpillar/struct.c b/src/ccaterpillar/struct.c index edfd6c0..55e16e0 100644 --- a/src/ccaterpillar/struct.c +++ b/src/ccaterpillar/struct.c @@ -581,9 +581,9 @@ cp_struct_process_annotation(CpStructObject* self, // The annotation is already a CpField object, so we don't need to convert // it. Note thate we check against the *exact* type, which means that we // will not accept any subclass of CpField. - if (CpField_CheckExact(annotation)) { - field = Py_NewRef(annotation); - } + // if (CpField_CheckExact(annotation)) { + // field = Py_NewRef(annotation); + // } // 2. Atom object or protocol // The annotation is an instance of a subclass of the CpAtom class OR it @@ -592,7 +592,7 @@ cp_struct_process_annotation(CpStructObject* self, // The annotation conforms to the Atom protocol. Note that we check here // only against packing, unpacking and size calculation. The type function // is optional and won't be covered here. - else if (CpAtom_Check(annotation)) { + if (CpAtom_Check(annotation)) { field = Py_NewRef(annotation); } @@ -1013,25 +1013,6 @@ CpStruct_AddFieldInfo(CpStructObject* o, CpStructFieldInfoObject* info) return PyObject_SetItem(o->m_members, info->m_name, (PyObject*)info); } -/*CpAPI*/ -int -CpStruct_AddField(CpStructObject* o, CpFieldObject* field, int exclude) -{ - if (!field) { - PyErr_SetString(PyExc_TypeError, "field must be a Field"); - return -1; - } - - CpStructFieldInfoObject* info = - CpStructFieldInfo_New(field->m_name, (PyObject*)field); - if (!info) { - return -1; - } - info->s_excluded = exclude; - int res = CpStruct_AddFieldInfo(o, info); - return res; -} - /*CpAPI*/ PyObject* CpStruct_GetAnnotations(CpStructObject* o, int eval) From a1cde82bc23a8b2404480396413359c5c1280493 Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sat, 28 Sep 2024 08:31:08 +0200 Subject: [PATCH 27/29] Updated stub file --- + Removed field related tests --- src/caterpillar/_C.pyi | 333 ++++++++++-------- src/ccaterpillar/atomimpl/builtins/atoffset.c | 1 + src/ccaterpillar/atomimpl/builtins/switch.c | 2 +- src/ccaterpillar/atomimpl/cstring.c | 7 +- src/ccaterpillar/atomimpl/pstring.c | 2 +- test/_C/test_atom.py | 4 +- test/_C/test_field.py | 100 ------ test/_C/test_parsing.py | 19 +- 8 files changed, 201 insertions(+), 267 deletions(-) delete mode 100644 test/_C/test_field.py diff --git a/src/caterpillar/_C.pyi b/src/caterpillar/_C.pyi index 2078173..9e6703a 100644 --- a/src/caterpillar/_C.pyi +++ b/src/caterpillar/_C.pyi @@ -1,69 +1,95 @@ from __future__ import annotations from typing import Any, Optional, Collection, Union, Callable, IO, TypeVar +from enum import EnumType _Length = Union[int, ContextLambda, slice, Ellipsis] - -DefaultOption: DefaultOptionType = ... -FIELD_OPTIONS: set = ... -F_DYNAMIC: Option = ... -F_SEQUENTIAL: Option = ... -HOST_ARCH: Arch = ... -InvalidDefault: InvalidDefaultType = ... -NATIVE_ENDIAN: Endian = ... -LITTLE_ENDIAN: Endian = ... -BIG_ENDIAN: Endian = ... -STRUCT_OPTIONS: set = ... -S_DISCARD_CONST: Option = ... -S_DISCARD_UNNAMED: Option = ... -S_EVAL_ANNOTATIONS: Option = ... -S_REPLACE_TYPES: Option = ... -S_SLOTS: Option = ... -S_UNION: Option = ... +ContextLambda = Callable[[Context], Any] +_ConstType = Union[Any, ContextLambda] + +BIG_ENDIAN: Endian +DefaultOption: DefaultOptionType +FIELD_OPTIONS: set +F_DYNAMIC: Option +F_SEQUENTIAL: Option +HOST_ARCH: Arch +InvalidDefault: InvalidDefaultType +LITTLE_ENDIAN: Endian +NATIVE_ENDIAN: Endian +STRUCT_OPTIONS: set +S_DISCARD_CONST: Option +S_DISCARD_UNNAMED: Option +S_EVAL_ANNOTATIONS: Option +S_REPLACE_TYPES: Option +S_SLOTS: Option +S_UNION: Option +boolean: Bool +char: Char +f16: Float +f32: Float +f64: Float +i128: Int +i16: Int +i24: Int +i32: Int +i64: Int +i8: Int +lsbvarint: VarInt +padding: Padding +u128: Int +u16: Int +u24: Int +u32: Int +u64: Int +u8: Int +varint: VarInt class Context(dict): - def __init__(self, *args, **kwargs) -> None: ... + def __init__(self, /, **kwargs) -> None: ... def __context_getattr__(self, *args, **kwargs): ... -ContextLambda = Callable[[Context], Any] - class Arch: name: str ptr_size: int def __init__(self, name: str, ptr_size: int) -> None: ... def __hash__(self) -> int: ... -class BinaryExpr: +class binaryexpr: expr: int lhs: Union[ContextLambda, Any] rhs: Union[ContextLambda, Any] - def __init__(self, *args, **kwargs) -> None: ... - def __call__(self, *args, **kwargs): ... + def __init__( + self, + expr: int, + left: Union[ContextLambda, Any], + right: Union[ContextLambda, Any], + ) -> None: ... + def __call__(self, *args, **kwargs) -> Any: ... class ContextPath: path: str def __init__(self, path: str) -> None: ... - def __add__(self, other) -> BinaryExpr: ... - def __and__(self, other) -> BinaryExpr: ... + def __add__(self, other) -> binaryexpr: ... + def __and__(self, other) -> binaryexpr: ... def __call__(self, context: Context) -> Any: ... - def __eq__(self, other: object) -> BinaryExpr: ... - def __floordiv__(self, other) -> BinaryExpr: ... - def __ge__(self, other: object) -> BinaryExpr: ... - def __gt__(self, other: object) -> BinaryExpr: ... + def __eq__(self, other: object) -> binaryexpr: ... + def __floordiv__(self, other) -> binaryexpr: ... + def __ge__(self, other: object) -> binaryexpr: ... + def __gt__(self, other: object) -> binaryexpr: ... def __hash__(self) -> int: ... - def __invert__(self) -> UnaryExpr: ... - def __le__(self, other: object) -> BinaryExpr: ... - def __lshift__(self, other) -> BinaryExpr: ... - def __lt__(self, other: object) -> BinaryExpr: ... - def __mod__(self, other) -> BinaryExpr: ... - def __mul__(self, other) -> BinaryExpr: ... - def __ne__(self, other: object) -> BinaryExpr: ... - def __neg__(self) -> UnaryExpr: ... - def __or__(self, other) -> BinaryExpr: ... - def __sub__(self, other) -> UnaryExpr: ... - def __truediv__(self, other) -> BinaryExpr: ... + def __invert__(self) -> unaryexpr: ... + def __le__(self, other: object) -> binaryexpr: ... + def __lshift__(self, other) -> binaryexpr: ... + def __lt__(self, other: object) -> binaryexpr: ... + def __mod__(self, other) -> binaryexpr: ... + def __mul__(self, other) -> binaryexpr: ... + def __ne__(self, other: object) -> binaryexpr: ... + def __neg__(self) -> unaryexpr: ... + def __or__(self, other) -> binaryexpr: ... + def __sub__(self, other) -> unaryexpr: ... + def __truediv__(self, other) -> binaryexpr: ... def __type__(self) -> type: ... - def __xor__(self, other) -> BinaryExpr: ... + def __xor__(self, other) -> binaryexpr: ... class DefaultOptionType: def __init__(self) -> None: ... @@ -82,19 +108,19 @@ class Endian: class atom: def __init__(self) -> None: ... - def __pack__(self, obj: Any, context: Context) -> None: ... - def __size__(self, context: Context) -> Any: ... + def __pack__(self, obj: Any, context: layer) -> None: ... + def __size__(self, context: layer) -> Any: ... def __type__(self) -> Any: ... - def __unpack__(self, context: Context) -> Any: ... + def __unpack__(self, context: layer) -> Any: ... class catom(atom): def __init__(self) -> None: ... - def __pack__(self, obj: Any, context: Context) -> None: ... - def __pack_many__(self, obj: Collection[Any], context: Context) -> None: ... - def __size__(self, context: Context) -> Any: ... + def __pack__(self, obj: Any, context: layer) -> None: ... + def __pack_many__(self, obj: Collection[Any], layer: layer) -> None: ... + def __size__(self, context: layer) -> Any: ... def __type__(self) -> Any: ... - def __unpack__(self, context: Context) -> Any: ... - def __unpack_many__(self, context: Context) -> Collection[Any]: ... + def __unpack__(self, context: layer) -> Any: ... + def __unpack_many__(self, context: layer, lengthinfo) -> Collection[Any]: ... class builtinatom(catom): def __init__(self, *args, **kwargs) -> None: ... @@ -106,86 +132,24 @@ class builtinatom(catom): def __rrshift__(self, other) -> switch: ... def __rshift__(self, other) -> switch: ... - -class char_t(builtinatom): +class Char(builtinatom): def __init__(self, *args, **kwargs) -> None: ... class condition(builtinatom): atom: Any condition: Union[bool, ContextLambda] - def __init__(self, *args, **kwargs) -> None: ... - def is_enabled(self, *args, **kwargs): ... - def __set_byteorder__(self, *args, **kwargs): ... + def __init__(self, condition: Union[bool, ContextLambda], atom: Any) -> None: ... + def is_enabled(self, context: layer) -> bool: ... + def __set_byteorder__(self, byteorder: Endian) -> condition: ... -class const_t(builtinatom): +class const(builtinatom): def __init__(self, *args, **kwargs) -> None: ... - -class fieldatom(atom): - def __init__(self) -> None: ... - def __add__(self, endian: Endian) -> Field: ... - def __floordiv__(self, condition: Union[bool, ContextLambda]) -> Field: ... - def __getitem__( - self, length: _Length - ) -> Field: ... - def __matmul__(self, offset: Union[ContextLambda, int]) -> Field: ... - def __or__(self, option: Option) -> Field: ... - def __radd__(self, endian: Endian) -> Field: ... - def __rfloordiv__(self, condition: Union[bool, ContextLambda]) -> Field: ... - def __rmatmul__(self, offset: Union[ContextLambda, int]) -> Field: ... - def __ror__(self, option: Option) -> Field: ... - def __rrshift__(self, switch: Union[dict, ContextLambda]) -> Field: ... - def __rshift__(self, switch: Union[dict, ContextLambda]) -> Field: ... - def __rxor__(self, option: Option) -> Field: ... - def __xor__(self, option: Option) -> Field: ... - -class fieldcatom(catom): +class builtinatom(catom): def __init__(self) -> None: ... def __add__(self, endian: Endian) -> Field: ... def __floordiv__(self, condition: Union[bool, ContextLambda]) -> Field: ... - def __getitem__( - self, length: _Length - ) -> Field: ... - def __matmul__(self, offset: Union[ContextLambda, int]) -> Field: ... - def __or__(self, option: Option) -> Field: ... - def __radd__(self, endian: Endian) -> Field: ... - def __rfloordiv__(self, condition: Union[bool, ContextLambda]) -> Field: ... - def __rmatmul__(self, offset: Union[ContextLambda, int]) -> Field: ... - def __ror__(self, option: Option) -> Field: ... - def __rrshift__(self, switch: Union[dict, ContextLambda]) -> Field: ... - def __rshift__(self, switch: Union[dict, ContextLambda]) -> Field: ... - def __rxor__(self, option: Option) -> Field: ... - def __xor__(self, option: Option) -> Field: ... - -class Field: - arch: Arch - atom: atom - condition: Union[bool, ContextLambda] - default: Union[Any, InvalidDefaultType] - endian: Endian - length: _Length - name: str - offset: Union[ContextLambda, int] - options: set[Option] - switch: Union[dict, ContextLambda] - def __init__( - self, - atom: atom, - name: str = ..., - endian: Endian = ..., - offset: Union[ContextLambda, int] = ..., - arch: Arch = ..., - length: _Length = ..., - default: Union[Any, InvalidDefaultType] = ..., - switch: Union[dict, ContextLambda] = ..., - options: set[Option] = ..., - condition: Union[bool, ContextLambda] = ..., - ) -> None: ... - def __add__(self, endian: Endian) -> Field: ... - def __floordiv__(self, condition: Union[bool, ContextLambda]) -> Field: ... - def __getitem__( - self, length: _Length - ) -> Field: ... + def __getitem__(self, length: _Length) -> Field: ... def __matmul__(self, offset: Union[ContextLambda, int]) -> Field: ... def __or__(self, option: Option) -> Field: ... def __radd__(self, endian: Endian) -> Field: ... @@ -224,9 +188,14 @@ class State: class fieldinfo: excluded: bool + default: Any field: atom + name: str def __init__(self, field: atom, excluded: bool = ...) -> None: ... +class lengthinfo: + def __init__(self, *args, **kwargs) -> None: ... + class Struct(builtinatom): members: dict[str, fieldinfo] model: type @@ -240,7 +209,7 @@ class Struct(builtinatom): alter_model: bool = ..., ) -> None: ... -class UnaryExpr: +class unaryexpr: expr: int value: Union[ContextLambda, Any] def __init__(self, expr: int, value: Union[ContextLambda, Any]) -> Any: ... @@ -277,43 +246,25 @@ def pack(__obj: Any, __struct: atom, **globals) -> bytes: ... def sizeof(obj: atom, globals: Optional[dict | Context] = ...): ... def unpack(__io: Any, __struct: atom, **globals) -> Any: ... -class int_t(fieldcatom): +class Int(builtinatom): little_endian: bool nbits: int nbytes: int signed: bool - def __init__(self, nbits: int, signed: bool = ..., little_endian: bool = ...) -> None: ... - -i16: int_t -i24: int_t -i32: int_t -i64: int_t -i8: int_t -u16: int_t -u24: int_t -u32: int_t -u64: int_t -u8: int_t - -class float_t(fieldcatom): + def __init__( + self, nbits: int, signed: bool = ..., little_endian: bool = ... + ) -> None: ... + +class Float(builtinatom): little_endian: bool nbits: int nbytes: int def __init__(self, nbits: int, little_endian: bool = ...) -> None: ... -f16: float_t -f32: float_t -f64: float_t - -class padding_t(fieldcatom): +class Padding(builtinatom): def __init__(self, pad: int) -> None: ... -padding: padding_t - -boolean: bool_t -char: char_t - -class string(fieldcatom): +class string(builtinatom): encoding: str errors: str length: _Length @@ -322,10 +273,11 @@ class string(fieldcatom): class atoffset(builtinatom): offset: Union[int, ContextLambda] whence: int - def __init__(self, *args, **kwargs) -> None: ... + def __init__( + self, offset: Union[int, ContextLambda], atom: atom, whence: int = ... + ) -> None: ... def get_offset(self, layer: layer) -> int: ... - def __set_byteorder__(self, byteorder: Endian): ... - + def __set_byteorder__(self, byteorder: Endian) -> atoffset: ... class objlayer(layer): obj: Context @@ -335,8 +287,8 @@ class objlayer(layer): class repeated(builtinatom): atom: Any length: _Length - def __init__(self, *args, **kwargs) -> None: ... - def __set_byteorder__(self, *args, **kwargs): ... + def __init__(self, atom: atom, length: _Length) -> None: ... + def __set_byteorder__(self, byteorder: Endian) -> repeated: ... class seqlayer(layer): index: int @@ -347,6 +299,81 @@ class seqlayer(layer): class switch(builtinatom): atom: Any cases: Union[dict[Any, Any], ContextLambda] - def __init__(self, *args, **kwargs) -> None: ... - def get_next(self, *args, **kwargs): ... - def __set_byteorder__(self, *args, **kwargs): ... + def __init__( + self, atom: atom, cases: Union[dict[Any, Any], ContextLambda] + ) -> None: ... + def get_next(self, obj: Any, context: layer) -> Any: ... + def __set_byteorder__(self, byteorder: Endian) -> switch: ... + +class cstring(builtinatom): + length: Union[_Length, atom] + encoding: str + errors: str + terminator: str + keep_terminator: bool + + def __init__( + self, + length: Union[_Length, atom], + encoding: str = ..., + errors: str = ..., + sep: str = ..., + keep_terminator: bool = ..., + ) -> None: ... + +class octetstring(builtinatom): + length: _Length + def __init__(self, length: _Length) -> None: ... + +class enumeration(builtinatom): + atom: Any + enum_type: EnumType + members: dict[Any, Any] + default: Any + def __init__( + self, + atom: atom, + enum_type: EnumType, + default: Any = ..., + ) -> None: ... + def __set_byteorder__(self, byteorder: Endian) -> enumeration: ... + +class pstring(builtinatom): + atom: Any + encoding: str + errors: str + def __init__(self, atom: atom, encoding: str = ..., errors: str = ...) -> None: ... + def __set_byteorder__(self, byteorder: Endian) -> pstring: ... + +class VarInt(builtinatom): + little_endian: bool + lsb: bool + def __init__(self, little_endian: bool = ..., lsb: bool = ...) -> None: ... + def __set_byteorder__(self, byteorder: Endian) -> VarInt: ... + +class computed(builtinatom): + value: _ConstType + def __init__(self, value: _ConstType) -> None: ... + +class objlayer(layer): + obj: Context + def __init__( + self, state: State, path: str = ..., parent: layer = ..., obj: Context = ... + ) -> None: ... + def __context_getattr__(self, path: str) -> Any: ... + +class lazy(builtinatom): + always_lazy: bool + fn: Callable[[], atom] + def __init__(self, fn: Callable[[], atom], always_lazy: bool = ...) -> None: ... + def __set_byteorder__(self, byteorder: Endian) -> lazy: ... + +class patom(atom): + def __init__(self) -> None: ... + def __floordiv__(self, other): ... + def __getitem__(self, index): ... + def __matmul__(self, *args, **kwargs): ... + def __rfloordiv__(self, other): ... + def __rmatmul__(self, *args, **kwargs): ... + def __rrshift__(self, other): ... + def __rshift__(self, other): ... diff --git a/src/ccaterpillar/atomimpl/builtins/atoffset.c b/src/ccaterpillar/atomimpl/builtins/atoffset.c index 2d1a897..7b11102 100644 --- a/src/ccaterpillar/atomimpl/builtins/atoffset.c +++ b/src/ccaterpillar/atomimpl/builtins/atoffset.c @@ -166,6 +166,7 @@ CpOffsetAtom_Unpack(CpOffsetAtomObject* self, CpLayerObject* layer) static PyMemberDef CpOffsetAtom_Members[] = { { "offset", T_OBJECT_EX, offsetof(CpOffsetAtomObject, m_offset), READONLY }, { "whence", T_OBJECT_EX, offsetof(CpOffsetAtomObject, m_whence), READONLY }, + { "atom", T_OBJECT_EX, offsetof(CpOffsetAtomObject, m_atom), READONLY }, { NULL } /* Sentinel */ }; diff --git a/src/ccaterpillar/atomimpl/builtins/switch.c b/src/ccaterpillar/atomimpl/builtins/switch.c index 7548e56..5f341a5 100644 --- a/src/ccaterpillar/atomimpl/builtins/switch.c +++ b/src/ccaterpillar/atomimpl/builtins/switch.c @@ -136,7 +136,7 @@ _CpEndian_ImplSetByteorder(CpSwitchAtomObject, switchatom, self->m_atom); static PyObject* cp_switchatom_get_next(CpSwitchAtomObject* self, PyObject* args, PyObject* kw) { - static char* kwlist[] = { "atom", "context", NULL }; + static char* kwlist[] = { "obj", "context", NULL }; PyObject* op = NULL; PyObject* context = NULL; if (!PyArg_ParseTupleAndKeywords(args, kw, "OO", kwlist, &op, &context)) { diff --git a/src/ccaterpillar/atomimpl/cstring.c b/src/ccaterpillar/atomimpl/cstring.c index 3804a7b..25e3795 100644 --- a/src/ccaterpillar/atomimpl/cstring.c +++ b/src/ccaterpillar/atomimpl/cstring.c @@ -295,6 +295,11 @@ static PyMemberDef CpCStringAtom_Members[] = { offsetof(CpCStringAtomObject, s_keep_terminator), 0, "whether to keep the terminator" }, + { "length", + T_OBJECT_EX, + offsetof(CpCStringAtomObject, m_length), + 0, + "the length of the string" }, { NULL } }; @@ -304,7 +309,7 @@ PyTypeObject CpCStringAtom_Type = { .tp_itemsize = 0, .tp_dealloc = (destructor)cp_cstringatom_dealloc, .tp_repr = (reprfunc)cp_cstringatom_repr, - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_new = (newfunc)cp_cstringatom_new, .tp_members = CpCStringAtom_Members, .tp_init = (initproc)cp_cstringatom_init, diff --git a/src/ccaterpillar/atomimpl/pstring.c b/src/ccaterpillar/atomimpl/pstring.c index deea57d..7fe1cd1 100644 --- a/src/ccaterpillar/atomimpl/pstring.c +++ b/src/ccaterpillar/atomimpl/pstring.c @@ -158,7 +158,7 @@ PyTypeObject CpPStringAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpPStringAtom_NAME), .tp_basicsize = sizeof(CpPStringAtomObject), .tp_dealloc = (destructor)cp_pstringatom_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_doc = NULL, .tp_members = CpPStringAtom_Members, .tp_init = (initproc)cp_pstringatom_init, diff --git a/test/_C/test_atom.py b/test/_C/test_atom.py index cda36d1..c58eaf0 100644 --- a/test/_C/test_atom.py +++ b/test/_C/test_atom.py @@ -4,7 +4,7 @@ if caterpillar.native_support(): - from caterpillar._C import atom, catom, fieldatom, fieldcatom + from caterpillar._C import atom, catom def test_atom_init(): @@ -26,8 +26,6 @@ class Foo(atom): assert issubclass(Foo, atom) assert issubclass(catom, atom) - assert issubclass(fieldatom, atom) - assert issubclass(fieldcatom, catom) def test_atom_methods(): diff --git a/test/_C/test_field.py b/test/_C/test_field.py deleted file mode 100644 index 449e2ed..0000000 --- a/test/_C/test_field.py +++ /dev/null @@ -1,100 +0,0 @@ -# pylint: disable=unused-import,no-name-in-module,import-error -import pytest - -import caterpillar - -if caterpillar.native_support(): - - from caterpillar._C import Field, fieldatom, atom - from caterpillar._C import NATIVE_ENDIAN, HOST_ARCH, F_DYNAMIC - - - def test_field_init(): - # Just verify that we can instantiate a field - with pytest.raises(TypeError): - Field() - - Field(atom()) - - # use all parameters - Field( - atom(), - name="foo", - offset=1, - length=2, - endian=NATIVE_ENDIAN, - default=3, - switch={1: 2}, - condition=False, - options=set(), - arch=HOST_ARCH, - ) - - - def test_field_offset(): - # The offset may be set using the constructor or by applying - # the special '@' operator. We can apply a constant value or - # a context lambda (callable function) here. - assert Field(atom()).offset is None - assert Field(atom(), offset=1).offset == 1 - assert (Field(atom()) @ 1).offset == 1 - assert (Field(atom()) @ (lambda ctx: 1)).offset(None) == 1 - - - def test_field_length(): - # The length may be set using the constructor or by applying - # the special '[]' operation. Be aware, thatz multiple dimensions - # are not supported yet. - # The length can take the following types: int, ellipsis, slice - # or a context lambda (callable function). - field = Field(atom()) - assert field.length is None - assert Field(atom(), length=1).length == 1 - assert field[1].length == 1 - assert field[lambda ctx: 1].length(None) == 1 - assert field[lambda x: 2 : :].length.start(None) == 2 - assert field[...].length is Ellipsis - - with pytest.raises(TypeError): - # type not supported - _ = field["foo"] - - - def test_field_condition(): - # The condition is somewhat special as it will be deprecated in the - # future. It may be a contant boolean value or a context lambda (callable - # function). Use the '//' operatorto apply a new condition. - field = Field(atom()) - assert field.condition is True # the default value is True - assert Field(atom(), condition=False).condition is False - assert (field // (lambda ctx: False)).condition(None) is False - assert (field // (lambda ctx: True)).condition(None) is True - - with pytest.raises(TypeError): - # type not supported - _ = field // "foo" - - - def test_field_switch(): - # A switch can be defined using the '>>' operation and the - # required value must be a dictionary or a context lambda. - field = Field(atom()) - assert field.switch is None - assert Field(atom(), switch={1: 2}).switch == {1: 2} - - with pytest.raises(TypeError): - # type not supported - _ = field >> "foo" - - # the stored value must be an atom instance, that can be - # called directly. - assert (field >> {1: atom()}).switch[1].__class__ == atom - assert (field >> (lambda x: atom())).switch(None).__class__ == atom - - - def test_field_options(): - # Options can be added with `|` and removed with `^`. - field = Field(atom()) - assert len(field.options) == 1 # keep_pos is inferred - assert F_DYNAMIC in (field | F_DYNAMIC).options - assert F_DYNAMIC not in (field ^ F_DYNAMIC).options diff --git a/test/_C/test_parsing.py b/test/_C/test_parsing.py index ddcb417..1dd7802 100644 --- a/test/_C/test_parsing.py +++ b/test/_C/test_parsing.py @@ -5,7 +5,8 @@ if caterpillar.native_support(): - from caterpillar._C import atom, typeof, Field, sizeof, patom + from caterpillar._C import atom, typeof, sizeof, patom, repeated + from caterpillar._C import switch from caterpillar._C import unpack, layer, Struct, pack, ContextPath @@ -42,16 +43,16 @@ def test_typeof(): # That will change if we have a field with a length or a # switch statement. - field = Field(f, length=2) + field = repeated(f, 2) assert typeof(field) == typing.List[str] - field2 = Field(f, switch={1: atom()}) + field2 = switch(f, {1: atom()})[2] # We can't know the type of a switch atom which does # not implement the __type__ method - assert typeof(Field(atom())) == typing.Any - assert typeof(field2) == typing.Union[typing.Any, str] + assert typeof(atom()) == typing.Any + assert typeof(field2) == typing.List[typing.Any] # switch and length can be combined as well - assert typeof(field2[2]) == typing.List[typing.Union[typing.Any, str]] + assert typeof(field2[2]) == typing.List[typing.List[typing.Any]] def test_sizeof(): @@ -73,8 +74,10 @@ def test_sizeof(): # Calculation is done by first evaluating the length of # the field's atom and then multiply the length of the evalutated # switch atom by the field's length. - field = Field(b, length=2, switch=(lambda ctx: Bar())) - assert sizeof(field) == 2 + (2 * 2) + # REVISIT: switch atoms does not have a static size + with pytest.raises(TypeError): + field = switch(b, (lambda ctx: Bar()))[2] + assert sizeof(field) == 2 + (2 * 2) def test_unpack_basic(): From d21f29eb1cc08ffc54af0edf5298a8019fb9566d Mon Sep 17 00:00:00 2001 From: MatrixEditor <58256046+MatrixEditor@users.noreply.github.com> Date: Sun, 29 Sep 2024 08:44:46 +0200 Subject: [PATCH 28/29] New feature: global type handlers for capi structs --- + Fixed refcount issue in _CpPack_EvalLength and _CpUnpack_EvalLength + Added Py_TPFLAGS_BASETYPE to all C types + Struct model classes can now be used in pack() and unpack() calls + Added missing operations to CpBinaryExpr and CpUnaryExpr --- docs/sphinx/source/index.rst | 35 ++-- .../capi/objects/atoms/fieldatom.rst | 108 ----------- .../reference/capi/objects/atoms/index.rst | 11 +- .../reference/capi/objects/fieldobj.rst | 59 ------ .../source/reference/capi/objects/objimpl.rst | 1 - src/capi.dat | 2 + src/caterpillar/_C.pyi | 8 +- .../include/caterpillar/atoms/const.h | 11 +- .../include/caterpillar/caterpillarapi.h | 4 + src/caterpillar/include/caterpillar/module.h | 3 + src/ccaterpillar/atomimpl/builtins/repeated.c | 5 + src/ccaterpillar/atomimpl/float.c | 2 +- src/ccaterpillar/atomimpl/int.c | 2 +- src/ccaterpillar/atomimpl/lazy.c | 2 +- src/ccaterpillar/atomimpl/pad.c | 2 +- src/ccaterpillar/atomimpl/string.c | 2 +- src/ccaterpillar/atomimpl/varint.c | 1 - src/ccaterpillar/atomobj.c | 7 + src/ccaterpillar/caterpillarapi.c | 4 +- src/ccaterpillar/context.c | 181 ++++++++++++++---- src/ccaterpillar/module.c | 52 ++++- src/ccaterpillar/option.c | 2 +- src/ccaterpillar/parsing_pack.c | 3 - src/ccaterpillar/parsing_unpack.c | 2 +- src/ccaterpillar/struct.c | 101 +++++++++- test/_C/test_parsing.py | 2 +- test/_C/test_struct.py | 13 +- 27 files changed, 371 insertions(+), 254 deletions(-) delete mode 100644 docs/sphinx/source/reference/capi/objects/atoms/fieldatom.rst delete mode 100644 docs/sphinx/source/reference/capi/objects/fieldobj.rst diff --git a/docs/sphinx/source/index.rst b/docs/sphinx/source/index.rst index fb8595d..97b0893 100644 --- a/docs/sphinx/source/index.rst +++ b/docs/sphinx/source/index.rst @@ -39,10 +39,11 @@ to write complex structures in a compact and readable manner. @struct class Format: - magic: const(b"Foo", octetstring(3)) # constant values + magic: b"Foo" # constant values name: cstring(...) # C-String without a fixed length value: u16 # little endian encoding entries: cstring(...)[be + u32::] # arrays with big-endian prefixed length + value2: bool # python type mapping configurable .. admonition:: Hold up, wait a minute! @@ -52,27 +53,39 @@ to write complex structures in a compact and readable manner. Working with defined classes is as straightforward as working with normal classes. *All constant values are created automatically!* + >>> obj = Format(name="Hello, World!", value=10, entries=["Bar", "Baz"]) >>> print(obj) Format(magic=b'Foo', name='Hello, World!', value=10, entries=['Bar', 'Baz']) Packing and unpacking have never been easier: ->>> pack(obj) -b'FooHello, World!\x00\n\x00\x00\x00\x00\x02Bar\x00Baz\x00' ->>> unpack(Format, _) -Format(magic=b'Foo', name='Hello, World!', value=10, entries=['Bar', 'Baz']) -- What about documentation? +.. tab-set:: + + .. tab-item:: Python - There are specialized options created only for documentation purposes, so you don't - have to worry about documenting fields. Just apply the documentation as usual. + >>> pack(obj) + b'FooHello, World!\x00\n\x00\x00\x00\x00\x02Bar\x00Baz\x00' + >>> unpack(Format, _) + Format(magic=b'Foo', name='Hello, World!', value=10, entries=['Bar', 'Baz']) + + .. tab-item:: Caterpillar C + + >>> pack(obj, Format) + b'FooHello, World!\x00\n\x00\x00\x00\x00\x02Bar\x00Baz\x00\x01' + >>> unpack(_, Format) + + +- What about documentation? + There are specialized options created only for documentation purposes, so you don't + have to worry about documenting fields. Just apply the documentation as usual. - You want to optimize memory space? + No problem! It is possible to shrink the memory space occupied by unpacked objects up + to 4 times. More information are provided when discussing available configuration + :ref:`options`. - No problem! It is possible to shrink the memory space occupied by unpacked objects up - to 4 times. More information are provided when discussing available configuration - :ref:`options`. Where to start? --------------- diff --git a/docs/sphinx/source/reference/capi/objects/atoms/fieldatom.rst b/docs/sphinx/source/reference/capi/objects/atoms/fieldatom.rst deleted file mode 100644 index 1f1fdfe..0000000 --- a/docs/sphinx/source/reference/capi/objects/atoms/fieldatom.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. _reference-capi_fieldatom: - -****************** -Field Atom Objects -****************** - -Python Field Atoms ------------------- - -.. c:var:: PyTypeObject CpFieldAtom_Type - - The type object for the :c:type:`CpFieldAtomObject` class. - - -.. c:type:: CpFieldAtomObject - - C implementation of the Python equivalent (:class:`~caterpillar.fields.FieldMixin`). - - A simple mixin to support operators used to create `Field` instances. - - -.. c:function:: int CpFieldAtom_CheckExact(PyObject *op) - - Checks if the given object is an :c:type:`CpFieldAtomObject`. - - -.. c:function:: int CpFieldAtom_Check(PyObject *op) - - Checks if the given object is instance of an :c:type:`CpFieldAtomObject` - - -.. c:macro:: CpFieldAtom_HEAD - - Inspired by :code:`PyObject_HEAD`, places the :c:type:`CpFieldAtomObject` at the - beginning of the object's memory block. For instance the following code: - - .. code-block:: c - - struct FooAtom { - CpFieldAtom_HEAD - - // other fields - }; - - extern PyTypeObject FooAtom_Type; - - The defined class is now a subclass of :c:type:`CpFieldAtomObject` and inherits - all methods. Make sure you don't forget to initialize the type correctly in your - module code: - - .. code-block:: c - - // ... - FooAtom_Type.tp_base = &CpFieldAtom_Type; - CpModule_SetupType(&FooAtom_Type); - // ... - - -C Field Atoms -------------- - -.. c:var:: PyTypeObject CpFieldCAtom_Type - - The type object for the :c:type:`CpFieldCAtomObject` class. - - -.. c:type:: CpFieldCAtomObject - - Field atom for C fields. Inherits from :c:type:`CpCAtomObject`. - - -.. c:function:: int CpFieldCAtom_CheckExact(PyObject *op) - - Checks if the given object is an :c:type:`CpFieldCAtomObject`. - - -.. c:function:: int CpFieldCAtom_Check(PyObject *op) - - Checks if the given object is instance of an :c:type:`CpFieldCAtomObject` - - -.. c:macro:: CpFieldCAtom_HEAD - - Inspired by :code:`PyObject_HEAD`, places the :c:type:`CpFieldCAtomObject` at the - beginning of the object's memory block. - - -.. c:macro:: CpFieldCAtom_CATOM(self) - - Returns the :c:type:`CpCAtomObject` from the given object which must be a - :c:type:`CpFieldCAtomObject`. This macro can be used to set the packing and - unpacking implementations: - - .. code-block:: c - - static PyObject* - fooatom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) - { - FooAtom *self = (FooAtom *)type->tp_alloc(type, 0); - if (self != NULL) { - // initialize the fields - CpFieldCAtom_CATOM(self).ob_pack = (packfunc)fooatom_pack; - CpFieldCAtom_CATOM(self).ob_unpack = (unpackfunc)fooatom_unpack; - // ... - } - return (PyObject*)self; - } - diff --git a/docs/sphinx/source/reference/capi/objects/atoms/index.rst b/docs/sphinx/source/reference/capi/objects/atoms/index.rst index d2d1d36..38fe60b 100644 --- a/docs/sphinx/source/reference/capi/objects/atoms/index.rst +++ b/docs/sphinx/source/reference/capi/objects/atoms/index.rst @@ -7,9 +7,8 @@ Atom Objects .. toctree:: :maxdepth: 1 - fieldatom.rst - intatom.rst - charatom.rst - floatatom.rst - boolatom.rst - paddingatom.rst + int + char + float + bool + padding diff --git a/docs/sphinx/source/reference/capi/objects/fieldobj.rst b/docs/sphinx/source/reference/capi/objects/fieldobj.rst deleted file mode 100644 index a4b8df9..0000000 --- a/docs/sphinx/source/reference/capi/objects/fieldobj.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. _reference-capi_field: - -************* -Field Objects -************* - -.. c:var:: PyTypeObject CpField_Type - - The type object for the :c:type:`CpFieldObject` class. - - -.. c:type:: CpFieldObject - - C implementation of the Python equivalent. Represents a field in a data structure. - - -.. c:function:: int CpField_CheckExact(PyObject *op) - - Checks if the given object is an :c:type:`CpFieldObject`. - - -.. c:function:: int CpField_Check(PyObject *op) - - Checks if the given object is instance of an :c:type:`CpFieldObject` - - -.. c:function:: CpFieldObject *CpField_New(PyObject *atom) - - Creates a new field and returns it. Returns *NULL* if an error occurs. - - -.. c:function:: int CpField_HasCondition(CpFieldObject *field) - - Returns whether the field has a condition. This function always succeeds. - - -.. c:function:: int CpField_IsEnabled(CpFieldObject *field, PyObject *context) - - Returns whether the field is enabled. The context object must conform to - the context protocol.Returns :code:`1` if the field is enabled, :code:`0` - otherwise and returns :code:`-1` if an error occurs. - - -.. c:function:: Py_ssize_t CpField_GetOffset(CpFieldObject *field, PyObject *context) - - Returns the offset of the field. The context object must conform to the - context protocol. Returns :code:`-1` if an error occurs. - - -.. c:function:: PyObject *CpField_EvalSwitch(CpFieldObject *field, PyObject *value, PyObject *context) - - Evaluates a switch field (if configured to do so). The context object must conform to the - context protocol. Returns *NULL* if an error occurs. - - -.. c:function:: PyObject *CpField_GetLength(CpFieldObject *field, PyObject *context) - - Returns the length of the field. The context object must conform to the - context protocol. Returns *NULL* if an error occurs. \ No newline at end of file diff --git a/docs/sphinx/source/reference/capi/objects/objimpl.rst b/docs/sphinx/source/reference/capi/objects/objimpl.rst index f9edabf..45d1de0 100644 --- a/docs/sphinx/source/reference/capi/objects/objimpl.rst +++ b/docs/sphinx/source/reference/capi/objects/objimpl.rst @@ -14,6 +14,5 @@ Concrete Objects Layer contextpath state layer - fieldobj struct atoms/index.rst \ No newline at end of file diff --git a/src/capi.dat b/src/capi.dat index fc5c63e..6e213c9 100644 --- a/src/capi.dat +++ b/src/capi.dat @@ -114,6 +114,8 @@ func:54:CpUnaryExpr_New:CpUnaryExprObject*:+1 func:55:CpBinaryExpr_New:CpBinaryExprObject*:+1 func:56:CpContextPath_New:CpContextPathObject*:+1 func:57:CpContextPath_FromString:CpContextPathObject*:+1 +func:58:CpTypeMap_Lookup:PyObject*:+1 +func:59:CpTypeMap_Register:int:null # func:58:CpField_New:CpFieldObject*:null # func:59:CpField_HasCondition:int:null # func:60:CpField_IsEnabled:int:null diff --git a/src/caterpillar/_C.pyi b/src/caterpillar/_C.pyi index 9e6703a..813ee8c 100644 --- a/src/caterpillar/_C.pyi +++ b/src/caterpillar/_C.pyi @@ -194,7 +194,9 @@ class fieldinfo: def __init__(self, field: atom, excluded: bool = ...) -> None: ... class lengthinfo: - def __init__(self, *args, **kwargs) -> None: ... + length: int + greedy: bool + def __init__(self, length: int = ..., greedy: bool = ...) -> None: ... class Struct(builtinatom): members: dict[str, fieldinfo] @@ -279,10 +281,6 @@ class atoffset(builtinatom): def get_offset(self, layer: layer) -> int: ... def __set_byteorder__(self, byteorder: Endian) -> atoffset: ... -class objlayer(layer): - obj: Context - def __init__(self, *args, **kwargs) -> None: ... - def __context_getattr__(self, *args, **kwargs): ... class repeated(builtinatom): atom: Any diff --git a/src/caterpillar/include/caterpillar/atoms/const.h b/src/caterpillar/include/caterpillar/atoms/const.h index e311d88..f7afc70 100644 --- a/src/caterpillar/include/caterpillar/atoms/const.h +++ b/src/caterpillar/include/caterpillar/atoms/const.h @@ -23,11 +23,18 @@ struct _constatomobj { CpBuiltinAtom_HEAD - PyObject *m_value; - PyObject *m_atom; + PyObject* m_value; + PyObject* m_atom; }; #define CpConstAtom_CheckExact(op) Py_IS_TYPE((op), &CpConstAtom_Type) #define CpConstAtom_Check(op) PyObject_TypeCheck((op), &CpConstAtom_Type) +static inline CpConstAtomObject* +CpConstAtom_New(PyObject* value, PyObject* atom) +{ + return (CpConstAtomObject*)CpObject_Create( + &CpConstAtom_Type, "OO", atom, value); +} + #endif \ No newline at end of file diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h b/src/caterpillar/include/caterpillar/caterpillarapi.h index 38ef90c..916e0e0 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h @@ -191,6 +191,8 @@ CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value); CpBinaryExprObject* CpBinaryExpr_New(int op, PyObject* left, PyObject* right); CpContextPathObject* CpContextPath_New(PyObject* path); CpContextPathObject* CpContextPath_FromString(const char* path); +PyObject* CpTypeMap_Lookup(PyObject* annotation, _modulestate* mod); +int CpTypeMap_Register(PyObject* annotation, PyObject* handler, _modulestate* mod); PyObject* CpTypeOf(PyObject* op); PyObject* CpTypeOf_Common(PyObject* op); int CpPack(PyObject* op, PyObject* atom, PyObject* io, PyObject* globals); @@ -332,6 +334,8 @@ caterpillar_api.py #define CpBinaryExpr_New (*((CpBinaryExprObject* (*)(int op, PyObject* left, PyObject* right)))Cp_API[55]) #define CpContextPath_New (*((CpContextPathObject* (*)(PyObject* path)))Cp_API[56]) #define CpContextPath_FromString (*((CpContextPathObject* (*)(const char* path)))Cp_API[57]) +#define CpTypeMap_Lookup (*((PyObject* (*)(PyObject* annotation, _modulestate* mod)))Cp_API[58]) +#define CpTypeMap_Register (*((int (*)(PyObject* annotation, PyObject* handler, _modulestate* mod)))Cp_API[59]) #define CpTypeOf (*((PyObject* (*)(PyObject* op)))Cp_API[64]) #define CpTypeOf_Common (*((PyObject* (*)(PyObject* op)))Cp_API[66]) #define CpPack (*((int (*)(PyObject* op, PyObject* atom, PyObject* io, PyObject* globals)))Cp_API[67]) diff --git a/src/caterpillar/include/caterpillar/module.h b/src/caterpillar/include/caterpillar/module.h index bdb5b75..70bb7b9 100644 --- a/src/caterpillar/include/caterpillar/module.h +++ b/src/caterpillar/include/caterpillar/module.h @@ -99,6 +99,9 @@ struct _modulestate // cached objects PyObject* cp_bytes__true; PyObject* cp_bytes__false; + + // type handler map + PyObject* cp_typehandler_map; }; /** diff --git a/src/ccaterpillar/atomimpl/builtins/repeated.c b/src/ccaterpillar/atomimpl/builtins/repeated.c index 21c5ffd..bf09cdc 100644 --- a/src/ccaterpillar/atomimpl/builtins/repeated.c +++ b/src/ccaterpillar/atomimpl/builtins/repeated.c @@ -56,6 +56,8 @@ cp_repeatedatomobj_new(PyTypeObject* type, PyObject* args, PyObject* kw) CpBuiltinAtom_CATOM(self).ob_unpack_many = NULL; CpBuiltinAtom_CATOM(self).ob_size = (sizefunc)cp_repeatedatomobj_size; CpBuiltinAtom_CATOM(self).ob_type = (typefunc)cp_repeatedatomobj_type; + self->m_length = NULL; + self->m_atom = NULL; return (PyObject*)self; } @@ -229,6 +231,9 @@ CpRepeatedAtom_Unpack(CpRepeatedAtomObject* self, CpLayerObject* layer) _modulestate* mod = layer->m_state->mod; CpLengthInfoObject* lengthinfo = (CpLengthInfoObject*)CpObject_CreateNoArgs(&CpLengthInfo_Type); + if (!lengthinfo) { + return NULL; + } bool unpack_many_attr = PyObject_HasAttr(self->m_atom, mod->str___unpack_many__); diff --git a/src/ccaterpillar/atomimpl/float.c b/src/ccaterpillar/atomimpl/float.c index 7f8d8cd..575052c 100644 --- a/src/ccaterpillar/atomimpl/float.c +++ b/src/ccaterpillar/atomimpl/float.c @@ -199,7 +199,7 @@ PyTypeObject CpFloatAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpFloatAtom_NAME), .tp_basicsize = sizeof(CpFloatAtomObject), .tp_dealloc = (destructor)cp_floatatom_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_doc = NULL, .tp_members = CpFloatAtom_Members, .tp_methods = CpFloatAtom_Methods, diff --git a/src/ccaterpillar/atomimpl/int.c b/src/ccaterpillar/atomimpl/int.c index 461fb4b..7714cd2 100644 --- a/src/ccaterpillar/atomimpl/int.c +++ b/src/ccaterpillar/atomimpl/int.c @@ -230,7 +230,7 @@ static PyMethodDef CpIntAtom_Methods[] = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpIntAtom_NAME), .tp_basicsize = sizeof(CpIntAtomObject), .tp_dealloc = (destructor)cp_intatom_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_doc = cp_intatom__doc, .tp_members = CpIntAtom_Members, .tp_new = (newfunc)cp_intatom_new, diff --git a/src/ccaterpillar/atomimpl/lazy.c b/src/ccaterpillar/atomimpl/lazy.c index 1eb0dee..564f27f 100644 --- a/src/ccaterpillar/atomimpl/lazy.c +++ b/src/ccaterpillar/atomimpl/lazy.c @@ -153,7 +153,7 @@ PyTypeObject CpLazyAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpLazyAtom_NAME), .tp_basicsize = sizeof(CpLazyAtomObject), .tp_dealloc = (destructor)cp_lazyatom_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_doc = NULL, .tp_init = (initproc)cp_lazyatom_init, .tp_new = (newfunc)cp_lazyatom_new, diff --git a/src/ccaterpillar/atomimpl/pad.c b/src/ccaterpillar/atomimpl/pad.c index a9fa402..6793354 100644 --- a/src/ccaterpillar/atomimpl/pad.c +++ b/src/ccaterpillar/atomimpl/pad.c @@ -183,7 +183,7 @@ PyTypeObject CpPaddingAtom_Type = { .tp_basicsize = sizeof(CpPaddingAtomObject), .tp_dealloc = (destructor)cp_paddingatom_dealloc, .tp_repr = (reprfunc)cp_paddingatom_repr, - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_doc = NULL, .tp_init = (initproc)cp_paddingatom_init, .tp_new = (newfunc)cp_paddingatom_new, diff --git a/src/ccaterpillar/atomimpl/string.c b/src/ccaterpillar/atomimpl/string.c index 7a98a94..56589c5 100644 --- a/src/ccaterpillar/atomimpl/string.c +++ b/src/ccaterpillar/atomimpl/string.c @@ -140,7 +140,7 @@ PyTypeObject CpStringAtom_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpStringAtom_NAME), .tp_basicsize = sizeof(CpStringAtomObject), .tp_dealloc = (destructor)cp_stringatom_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_doc = NULL, .tp_members = CpStringAtom_Members, .tp_init = (initproc)cp_stringatom_init, diff --git a/src/ccaterpillar/atomimpl/varint.c b/src/ccaterpillar/atomimpl/varint.c index c8f05a2..6f47fd1 100644 --- a/src/ccaterpillar/atomimpl/varint.c +++ b/src/ccaterpillar/atomimpl/varint.c @@ -115,7 +115,6 @@ cp_varintatom_set_byteorder(CpVarIntAtomObject* self, PyObject* new_varint = (PyObject*)CpVarIntAtom_New(little_endian, self->_m_lsb); - Py_DECREF(byteorder); return new_varint; } diff --git a/src/ccaterpillar/atomobj.c b/src/ccaterpillar/atomobj.c index a563fa5..3948553 100644 --- a/src/ccaterpillar/atomobj.c +++ b/src/ccaterpillar/atomobj.c @@ -3,6 +3,7 @@ #include "caterpillar/atomobj.h" #include "caterpillar/macros.h" +#include #include static PyObject* @@ -361,6 +362,12 @@ cp_lengthinfo_repr(CpLengthInfoObject* self) return PyUnicode_FromFormat("", self->m_length); } +static PyMemberDef CpLengthInfo_Members[] = { + { "length", T_PYSSIZET, offsetof(CpLengthInfoObject, m_length), 0, NULL }, + { "greedy", T_BOOL, offsetof(CpLengthInfoObject, m_greedy), 0, NULL }, + { NULL } /* Sentinel */ +}; + PyTypeObject CpLengthInfo_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_NameStr(CpLengthInfo_NAME), .tp_basicsize = sizeof(CpLengthInfoObject), diff --git a/src/ccaterpillar/caterpillarapi.c b/src/ccaterpillar/caterpillarapi.c index b3cae8a..e7b5ef5 100644 --- a/src/ccaterpillar/caterpillarapi.c +++ b/src/ccaterpillar/caterpillarapi.c @@ -61,8 +61,8 @@ void *Cp_API[] = { (void *) &CpBinaryExpr_New, (void *) &CpContextPath_New, (void *) &CpContextPath_FromString, - NULL, - NULL, + (void *) &CpTypeMap_Lookup, + (void *) &CpTypeMap_Register, NULL, NULL, NULL, diff --git a/src/ccaterpillar/context.c b/src/ccaterpillar/context.c index 214f38b..f23dd8d 100644 --- a/src/ccaterpillar/context.c +++ b/src/ccaterpillar/context.c @@ -262,6 +262,49 @@ cp_unaryexpr__call__(CpUnaryExprObject* self, PyObject* args, PyObject* kw) return result; } +static PyObject* +cp_unaryexpr_as_number_neg(PyObject* self) +{ + return (PyObject*)CpUnaryExpr_New(CpUnaryExpr_OpNeg, self); +} + +static PyObject* +cp_unaryexpr_as_number_pos(PyObject* self) +{ + return (PyObject*)CpUnaryExpr_New(CpUnaryExpr_OpPos, self); +} + +static PyObject* +cp_unaryexpr_as_number_not(PyObject* self) +{ + return (PyObject*)CpUnaryExpr_New(CpUnaryExpr_OpNot, self); +} + +/* operations */ +#define _CpUnaryExpr_BinaryNumberMethod(name, op) \ + static PyObject* cp_unaryexpr_as_number_##name(PyObject* self, \ + PyObject* other) \ + { \ + return (PyObject*)CpBinaryExpr_New(op, self, other); \ + } + +_CpUnaryExpr_BinaryNumberMethod(add, CpBinaryExpr_OpAdd); +_CpUnaryExpr_BinaryNumberMethod(sub, CpBinaryExpr_OpSub); +_CpUnaryExpr_BinaryNumberMethod(mul, CpBinaryExpr_OpMul); +_CpUnaryExpr_BinaryNumberMethod(div, CpBinaryExpr_OpTrueDiv); +_CpUnaryExpr_BinaryNumberMethod(truediv, CpBinaryExpr_OpTrueDiv); +_CpUnaryExpr_BinaryNumberMethod(floordiv, CpBinaryExpr_OpFloorDiv); +_CpUnaryExpr_BinaryNumberMethod(mod, CpBinaryExpr_OpMod); +_CpUnaryExpr_BinaryNumberMethod(lshift, CpBinaryExpr_OpLShift); +_CpUnaryExpr_BinaryNumberMethod(rshift, CpBinaryExpr_OpRShift); +_CpUnaryExpr_BinaryNumberMethod(and, CpBinaryExpr_OpBitAnd); +_CpUnaryExpr_BinaryNumberMethod(xor, CpBinaryExpr_OpBitXor); +_CpUnaryExpr_BinaryNumberMethod(or, CpBinaryExpr_OpBitOr); +_CpUnaryExpr_BinaryNumberMethod(pow, CpBinaryExpr_OpPow); +_CpUnaryExpr_BinaryNumberMethod(matmul, CpBinaryExpr_OpMatMul); + +#undef _CpUnaryExpr_BinaryNumberMethod + /*CpAPI*/ CpUnaryExprObject* CpUnaryExpr_New(int op, PyObject* value) @@ -285,6 +328,27 @@ static PyMemberDef CpUnaryExpr_Members[] = { { NULL } /* Sentinel */ }; +static PyNumberMethods CpBinaryExpr_NumberMethods = { + // unary + .nb_negative = (unaryfunc)cp_unaryexpr_as_number_neg, + .nb_positive = (unaryfunc)cp_unaryexpr_as_number_pos, + .nb_invert = (unaryfunc)cp_unaryexpr_as_number_not, + // binary + .nb_add = (binaryfunc)cp_unaryexpr_as_number_add, + .nb_subtract = (binaryfunc)cp_unaryexpr_as_number_sub, + .nb_multiply = (binaryfunc)cp_unaryexpr_as_number_mul, + .nb_true_divide = (binaryfunc)cp_unaryexpr_as_number_truediv, + .nb_floor_divide = (binaryfunc)cp_unaryexpr_as_number_floordiv, + .nb_remainder = (binaryfunc)cp_unaryexpr_as_number_mod, + .nb_power = (ternaryfunc)cp_unaryexpr_as_number_pow, + .nb_lshift = (binaryfunc)cp_unaryexpr_as_number_lshift, + .nb_rshift = (binaryfunc)cp_unaryexpr_as_number_rshift, + .nb_and = (binaryfunc)cp_unaryexpr_as_number_and, + .nb_xor = (binaryfunc)cp_unaryexpr_as_number_xor, + .nb_or = (binaryfunc)cp_unaryexpr_as_number_or, + .nb_matrix_multiply = (binaryfunc)cp_unaryexpr_as_number_matmul, +}; + PyTypeObject CpUnaryExpr_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(UnaryExpr), .tp_basicsize = sizeof(CpUnaryExprObject), @@ -297,6 +361,7 @@ PyTypeObject CpUnaryExpr_Type = { .tp_members = CpUnaryExpr_Members, .tp_init = (initproc)cp_unaryexpr_init, .tp_new = (newfunc)cp_unaryexpr_new, + .tp_as_number = &CpBinaryExpr_NumberMethods, }; //------------------------------------------------------------------------------ @@ -542,30 +607,48 @@ cp_binaryexpr__call__(CpBinaryExprObject* self, PyObject* args, PyObject* kw) return result; } +static PyObject* +cp_binaryexpr_as_number_neg(PyObject* self) +{ + return (PyObject*)CpUnaryExpr_New(CpUnaryExpr_OpNeg, self); +} + +static PyObject* +cp_binaryexpr_as_number_pos(PyObject* self) +{ + return (PyObject*)CpUnaryExpr_New(CpUnaryExpr_OpPos, self); +} + +static PyObject* +cp_binaryexpr_as_number_not(PyObject* self) +{ + return (PyObject*)CpUnaryExpr_New(CpUnaryExpr_OpNot, self); +} + /* operations */ -#define _CpBinaryExpr_BinaryNumberMethod(name, op) \ - static PyObject* cp_binaryexpr_as_number_##name(PyObject* self, \ - PyObject* other) \ +#define _CpUnaryExpr_BinaryNumberMethod(name, op) \ + static PyObject* cp_binaryexpr_as_number_##name(PyObject* self, \ + PyObject* other) \ { \ return (PyObject*)CpBinaryExpr_New(op, self, other); \ } -_CpBinaryExpr_BinaryNumberMethod(add, CpBinaryExpr_OpAdd); -_CpBinaryExpr_BinaryNumberMethod(sub, CpBinaryExpr_OpSub); -_CpBinaryExpr_BinaryNumberMethod(mul, CpBinaryExpr_OpMul); -_CpBinaryExpr_BinaryNumberMethod(div, CpBinaryExpr_OpTrueDiv); -_CpBinaryExpr_BinaryNumberMethod(truediv, CpBinaryExpr_OpTrueDiv); -_CpBinaryExpr_BinaryNumberMethod(floordiv, CpBinaryExpr_OpFloorDiv); -_CpBinaryExpr_BinaryNumberMethod(mod, CpBinaryExpr_OpMod); -_CpBinaryExpr_BinaryNumberMethod(lshift, CpBinaryExpr_OpLShift); -_CpBinaryExpr_BinaryNumberMethod(rshift, CpBinaryExpr_OpRShift); -_CpBinaryExpr_BinaryNumberMethod(and, CpBinaryExpr_OpBitAnd); -_CpBinaryExpr_BinaryNumberMethod(xor, CpBinaryExpr_OpBitXor); -_CpBinaryExpr_BinaryNumberMethod(or, CpBinaryExpr_OpBitOr); -_CpBinaryExpr_BinaryNumberMethod(pow, CpBinaryExpr_OpPow); -_CpBinaryExpr_BinaryNumberMethod(matmul, CpBinaryExpr_OpMatMul); - -#undef _CpContextPath_BinaryNumberMethod +_CpUnaryExpr_BinaryNumberMethod(add, CpBinaryExpr_OpAdd); +_CpUnaryExpr_BinaryNumberMethod(sub, CpBinaryExpr_OpSub); +_CpUnaryExpr_BinaryNumberMethod(mul, CpBinaryExpr_OpMul); +_CpUnaryExpr_BinaryNumberMethod(div, CpBinaryExpr_OpTrueDiv); +_CpUnaryExpr_BinaryNumberMethod(truediv, CpBinaryExpr_OpTrueDiv); +_CpUnaryExpr_BinaryNumberMethod(floordiv, CpBinaryExpr_OpFloorDiv); +_CpUnaryExpr_BinaryNumberMethod(mod, CpBinaryExpr_OpMod); +_CpUnaryExpr_BinaryNumberMethod(lshift, CpBinaryExpr_OpLShift); +_CpUnaryExpr_BinaryNumberMethod(rshift, CpBinaryExpr_OpRShift); +_CpUnaryExpr_BinaryNumberMethod(and, CpBinaryExpr_OpBitAnd); +_CpUnaryExpr_BinaryNumberMethod(xor, CpBinaryExpr_OpBitXor); +_CpUnaryExpr_BinaryNumberMethod(or, CpBinaryExpr_OpBitOr); +_CpUnaryExpr_BinaryNumberMethod(pow, CpBinaryExpr_OpPow); +_CpUnaryExpr_BinaryNumberMethod(matmul, CpBinaryExpr_OpMatMul); + +#undef _CpUnaryExpr_BinaryNumberMethod /*CpAPI*/ CpBinaryExprObject* @@ -598,6 +681,27 @@ static PyMemberDef CpBinaryExpr_Members[] = { { NULL } /* Sentinel */ }; +static PyNumberMethods CpUnaryExpr_NumberMethods = { + // unary + .nb_negative = (unaryfunc)cp_binaryexpr_as_number_neg, + .nb_positive = (unaryfunc)cp_binaryexpr_as_number_pos, + .nb_invert = (unaryfunc)cp_binaryexpr_as_number_not, + // binary + .nb_add = (binaryfunc)cp_binaryexpr_as_number_add, + .nb_subtract = (binaryfunc)cp_binaryexpr_as_number_sub, + .nb_multiply = (binaryfunc)cp_binaryexpr_as_number_mul, + .nb_true_divide = (binaryfunc)cp_binaryexpr_as_number_truediv, + .nb_floor_divide = (binaryfunc)cp_binaryexpr_as_number_floordiv, + .nb_remainder = (binaryfunc)cp_binaryexpr_as_number_mod, + .nb_power = (ternaryfunc)cp_binaryexpr_as_number_pow, + .nb_lshift = (binaryfunc)cp_binaryexpr_as_number_lshift, + .nb_rshift = (binaryfunc)cp_binaryexpr_as_number_rshift, + .nb_and = (binaryfunc)cp_binaryexpr_as_number_and, + .nb_xor = (binaryfunc)cp_binaryexpr_as_number_xor, + .nb_or = (binaryfunc)cp_binaryexpr_as_number_or, + .nb_matrix_multiply = (binaryfunc)cp_binaryexpr_as_number_matmul, +}; + PyTypeObject CpBinaryExpr_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(BinaryExpr), .tp_basicsize = sizeof(CpBinaryExprObject), @@ -609,6 +713,7 @@ PyTypeObject CpBinaryExpr_Type = { .tp_members = CpBinaryExpr_Members, .tp_init = (initproc)cp_binaryexpr_init, .tp_new = (newfunc)cp_binaryexpr_new, + .tp_as_number = &CpUnaryExpr_NumberMethods, }; //------------------------------------------------------------------------------ @@ -655,7 +760,13 @@ cp_contextpath_init(CpContextPathObject* self, PyObject* args, PyObject* kw) static PyObject* cp_contextpath_repr(CpContextPathObject* self) { - return PyUnicode_FromFormat("CpPath(%R)", self->m_path); + return PyUnicode_FromFormat("", self->m_path); +} + +static PyObject* +cp_contextpath_str(CpContextPathObject* self) +{ + return Py_XNewRef(self->m_path); } static Py_hash_t @@ -784,7 +895,6 @@ CpContextPath_FromString(const char* path) return (CpContextPathObject*)CpObject_Create(&CpContextPath_Type, "s", path); } - /* docs */ PyDoc_STRVAR(cp_contextpath__doc__, "\ ContextPath(path)\n\ @@ -832,18 +942,19 @@ static PyNumberMethods CpContextPath_NumberMethods = { /* type */ PyTypeObject CpContextPath_Type = { PyVarObject_HEAD_INIT(NULL, 0) _Cp_Name(ContextPath), - .tp_basicsize =sizeof(CpContextPathObject), - .tp_dealloc =(destructor)cp_contextpath_dealloc, - .tp_getattr =(getattrfunc)cp_contextpath__getattr__, - .tp_repr =(reprfunc)cp_contextpath_repr, - .tp_as_number =&CpContextPath_NumberMethods, - .tp_hash =(hashfunc)cp_contextpath_hash, - .tp_call =(ternaryfunc)cp_contextpath__call__, - .tp_flags =Py_TPFLAGS_DEFAULT, - .tp_doc =cp_contextpath__doc__, - .tp_richcompare =(richcmpfunc)cp_contextpath_richcmp, - .tp_methods =CpContextPath_Methods, - .tp_members =CpContextPath_Members, - .tp_init =(initproc)cp_contextpath_init, - .tp_new =(newfunc)cp_contextpath_new, + .tp_basicsize = sizeof(CpContextPathObject), + .tp_dealloc = (destructor)cp_contextpath_dealloc, + .tp_getattr = (getattrfunc)cp_contextpath__getattr__, + .tp_repr = (reprfunc)cp_contextpath_repr, + .tp_as_number = &CpContextPath_NumberMethods, + .tp_hash = (hashfunc)cp_contextpath_hash, + .tp_call = (ternaryfunc)cp_contextpath__call__, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = cp_contextpath__doc__, + .tp_richcompare = (richcmpfunc)cp_contextpath_richcmp, + .tp_methods = CpContextPath_Methods, + .tp_members = CpContextPath_Members, + .tp_init = (initproc)cp_contextpath_init, + .tp_new = (newfunc)cp_contextpath_new, + .tp_str = (reprfunc)cp_contextpath_str, }; \ No newline at end of file diff --git a/src/ccaterpillar/module.c b/src/ccaterpillar/module.c index 1ce6c26..69847d8 100644 --- a/src/ccaterpillar/module.c +++ b/src/ccaterpillar/module.c @@ -75,7 +75,19 @@ cp_core_pack_into(PyObject* m, PyObject* args, PyObject* kw) PyErr_SetString(PyExc_ValueError, "output stream not set!"); goto finish; } - res = CpPack(op, atom, io, globals); + + _modulestate* state = get_module_state(m); + if (CpStructModel_Check(op, state)) { + atom = CpStructModel_GetStruct(op, state); + if (!atom) { + goto finish; + } + res = CpPack(op, atom, io, globals); + Py_DECREF(atom); + } else { + res = CpPack(op, atom, io, globals); + } + finish: Py_XDECREF(globals); return res < 0 ? NULL : Py_None; @@ -112,7 +124,17 @@ cp_core_pack(PyObject* m, PyObject* args, PyObject* kw) goto finish; } - res = CpPack(op, atom, io, globals); + if (CpStructModel_Check(op, state)) { + atom = CpStructModel_GetStruct(op, state); + if (!atom) { + goto finish; + } + res = CpPack(op, atom, io, globals); + Py_DECREF(atom); + } else { + res = CpPack(op, atom, io, globals); + } + finish: Py_XDECREF(globals); if (res < 0) { @@ -132,6 +154,16 @@ cp_core_sizeof(PyObject* m, PyObject* args, PyObject* kw) if (!PyArg_ParseTupleAndKeywords(args, kw, "O|O", kwlist, &op, &globals)) { return NULL; } + _modulestate* state = get_module_state(m); + if (CpStructModel_Check(op, state)) { + PyObject* atom = CpStructModel_GetStruct(op, state); + if (!atom) { + return NULL; + } + PyObject* res = CpSizeOf(atom, globals); + Py_DECREF(atom); + return res; + } return CpSizeOf(op, globals); } @@ -174,7 +206,17 @@ cp_core_unpack(PyObject* m, PyObject* args, PyObject* kw) goto finish; } - res = CpUnpack(atom, io, globals); + if (CpStructModel_Check(atom, state)) { + atom = CpStructModel_GetStruct(atom, state); + if (!atom) { + goto finish; + } + res = CpUnpack(atom, io, globals); + Py_DECREF(atom); + } else { + res = CpUnpack(atom, io, globals); + } + finish: Py_XDECREF(globals); if (wrapped_io) { @@ -253,6 +295,7 @@ cp_module_clear(PyObject* m) Py_CLEAR(state->cp_bytes__false); Py_CLEAR(state->cp_bytes__true); + Py_CLEAR(state->cp_typehandler_map); } return 0; } @@ -508,6 +551,9 @@ PyInit__C(void) CpModuleState_AddObject( cp_option__global_field_options, "FIELD_OPTIONS", PySet_New(NULL)); + /* typehandler map */ + CpModuleState_AddObject(cp_typehandler_map, "TYPE_MAP", PyDict_New()); + /* setup arch and endian */ CpModuleState_AddObject(cp_endian__native, "NATIVE_ENDIAN", diff --git a/src/ccaterpillar/option.c b/src/ccaterpillar/option.c index f39665c..1c8e1c6 100644 --- a/src/ccaterpillar/option.c +++ b/src/ccaterpillar/option.c @@ -37,7 +37,7 @@ static PyObject* cp_option_repr(CpOptionObject* self) { return PyUnicode_FromFormat( - "CpOption(name=%R value=%R)", self->name, self->value); + "