Merge pull request #2 from ameharoo/dev

Python backend + change type in examples/cpp

README.md

@@ -2,7 +2,7 @@
 are extensible and language-neutral Mess(-ages Embedded Style Serialization).  
 Supported backends:
 - C++
-- Python (in progress)
+- Python
 - C# (in progress)
 - PHP (in progress)
 - ImHex Pattern Language

backend.py

@@ -116,3 +116,10 @@ class ImHexBackend(Backend):
 
     def render_message(self, message:  messages_base.Message):
         return super().render_message(message)
+
+class PythonBackend(Backend):
+    name = "python"
+    path_to_templates = "python/"
+
+    def render_message(self, message:  messages_base.Message):
+        return super().render_message(message)

examples/cpp/test_message.ini

@@ -16,6 +16,6 @@ string = VarArray<Int8>
 colors = VarArray<Color>
 
 [Color]
-r = Int8
-g = Int8
-b = Int8
+r = Uint8
+g = Uint8
+b = Uint8

messages/python.py

@@ -0,0 +1,155 @@
+from messages import base
+
+
+class Message(base.Message):
+    def __init__(self, name=None, fields=None, generic_args=None, docs=None):
+        super().__init__(name, fields, generic_args, docs)
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "Type.j2"
+
+
+class Int8(base.Int8):
+    name = "Int8"
+
+    def __init__(self):
+        super().__init__(self.name)
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "UniformInt.j2"
+
+
+class Int16(base.Int16):
+    name = "Int16"
+
+    def __init__(self):
+        super().__init__(self.name)
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "UniformInt.j2"
+
+
+class Int32(base.Int32):
+    name = "Int32"
+
+    def __init__(self):
+        super().__init__(self.name)
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "UniformInt.j2"
+
+
+class Uint8(base.Uint8):
+    name = "Uint8"
+
+    def __init__(self):
+        super().__init__(self.name)
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "UniformInt.j2"
+
+
+class Uint16(base.Uint16):
+    name = "Uint16"
+
+    def __init__(self):
+        super().__init__(self.name)
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "UniformInt.j2"
+
+
+class Uint32(base.Uint32):
+    name = "Uint32"
+
+    def __init__(self):
+        super().__init__(self.name)
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "UniformInt.j2"
+
+
+class Float(base.Float):
+    name = "Float"
+
+    def __init__(self):
+        super().__init__(self.name)
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "Float.j2"
+
+
+class Fixed16(base.Fixed16):
+    name = "Fixed16"
+
+    def __init__(self):
+        super().__init__()
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "Fixed.j2"
+
+
+class Fixed32(base.Fixed32):
+    name = "Fixed32"
+
+    def __init__(self):
+        super().__init__()
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "Fixed.j2"
+
+
+class VarArray(base.VarArray):
+    name = "VarArray"
+    is_variative = True
+
+    def __init__(self):
+        super().__init__(self.name)
+
+    def render(self, backend: 'Backend'):
+        return backend.get_template(self.get_render_template()) \
+                      .render(data={'message': self, 'backend': backend})
+
+    def get_render_template(self) -> str:
+        return "VarArray.j2"
+

templates/python/Fixed.j2

@@ -0,0 +1,25 @@
+{%- set msg = message %}
+{% set msg_type = msg | message_pure_type %}
+{% set msg_mangled_type = msg | message_type %}
+
+{% set size_in_bytes = 0 %}
+
+{% if msg_type == 'Fixed16' %}
+    {% set size_in_bytes = 2 %}
+{% elif msg_type == 'Fixed32' %}
+    {% set size_in_bytes = 4 %}
+{% endif %}
+
+@dataclasses.dataclass
+class {{ msg_mangled_type }}(BaseMessage):
+    value: float
+
+    def to_bytes(self) -> bytes:
+        return int(self.value * (1 << {{ msg.bitness // 2 }})).to_bytes({{ size_in_bytes }}, "little")
+
+    @classmethod
+    def from_bytes(cls, buffer: bytes) -> tuple[Self, int]:
+        raw = int.from_bytes(buffer[:{{ size_in_bytes }}], "little")
+        return cls(float(raw) / (1 << {{ msg.bitness // 2 }})),{{ size_in_bytes }}
+
+

templates/python/Float.j2

@@ -0,0 +1,15 @@
+{% set msg = message %}
+{% set msg_type = msg | message_pure_type -%}
+{% set msg_mangled_type = msg | message_type -%}
+
+
+@dataclasses.dataclass
+class {{ msg_mangled_type }}(BaseMessage):
+    value: float
+
+    def to_bytes(self) -> bytes:
+        return struct.pack("<f", self.value)
+    
+    @classmethod
+    def from_bytes(cls, buffer: bytes) -> tuple[Self, int]:
+        return {{ msg_mangled_type }}(struct.unpack("<f", buffer[:4])[0]),4

templates/python/Message.j2

@@ -0,0 +1,49 @@
+@dataclasses.dataclass(kw_only=True)
+class BaseMessage:
+    {% block hash %}
+    protocol_hash: int = {{ data.protocol_hash }}
+    {%- endblock %} 
+
+    def __init__(self):
+        pass
+
+    # Validation and implicit cast if possible
+    def __setattr__(self, name, value):
+        if name != 'protocol_hash' and type(value) != self.__annotations__[name]:
+            value = self.__annotations__[name](value)
+
+        self.__dict__[name] = value
+
+    def to_bytes(self) -> bytes:
+        buffer = bytearray()
+        buffer += int(self.protocol_hash).to_bytes({{ data.protocol_hash_bytes_count }}, "little")
+
+        field_types = {field.name: field.type for field in dataclasses.fields(type(self))}
+        for field_name, field_type in field_types.items():
+            if field_name == "protocol_hash":
+                continue
+
+            field_message: Self = getattr(self, field_name)
+            assert issubclass(type(field_message), BaseMessage), f"Wrong type {field_message} must be {field_type} ihnerited from BaseMessage"
+            buffer += field_message.to_bytes()
+
+        return bytes(buffer)
+
+    @classmethod
+    def from_bytes(cls, buffer: bytes) -> tuple[Self, int]:
+        _protocol_hash = int.from_bytes(buffer[:{{ data.protocol_hash_bytes_count }}], "little")
+
+        offset = {{ data.protocol_hash_bytes_count }}
+
+        field_values = []
+        field_types: dict[str, Self] = {field.name: field.type for field in dataclasses.fields(cls)}
+        for field_name, field_type in field_types.items():
+            if field_name == "protocol_hash":
+                continue
+
+            assert issubclass(field_type, BaseMessage), f"Wrong type {field_type} must be ihnerited from BaseMessage"
+            field_message, _offset = field_type.from_bytes(buffer[offset:])
+            field_values.append(field_message)
+            offset += _offset
+
+        return cls(*field_values, protocol_hash=_protocol_hash), offset

templates/python/Output.j2

@@ -0,0 +1,36 @@
+{% block includes -%}
+import struct
+import dataclasses
+from typing import Self
+{% endblock %}
+
+{% include 'python/Watermark.j2' %}
+
+{% block messages_block %}
+# todo namespaces in python backend
+# Namespace: {{data.protocol_name or 'Default'}}
+
+# Hash of protocol
+{%+ block hash -%}
+    protocol_hash: int = {{ data.protocol_hash }}
+{% endblock %}
+
+{% include 'python/Message.j2' +%}
+
+{% block messages_definitions %}
+{% for template_name, message in data.messages %}
+    {%- do data.hook_on_message_render(message) -%}
+    {% filter trim() %}
+    {% include 'python/' + template_name %}
+    {% endfilter +%}
+
+{% endfor %}
+{% endblock%}
+
+{% for template_name, message in data.messages -%}
+    {%- if not message.is_user_defined %}{% continue %}{% endif -%}
+    {%- do data.hook_on_message_usings(message) -%}
+    {{ message | message_pure_type }} = {{ message | message_type }}
+{% endfor %}
+{%endblock%}
+

templates/python/Type.j2

@@ -0,0 +1,42 @@
+{% macro doc_comment(msg) -%}
+{% set doc = msg|message_docs %}
+{% set doc_lines = doc.split('\n') %}
+{% if doc_lines | length > 1%}
+{{ "# " + doc_lines[:-1] | map('trim') | join('\n# ') }}
+{% endif %}
+{% endmacro -%}
+
+
+{% set msg = message %}
+{# First simple fields (non variative), then complex, variative #}
+{% set sorted_fields = msg.fields | sort(attribute="message.is_variative") %}
+
+
+{% block main %}
+
+{% block before_definition %}
+{% endblock %}
+
+{%+ block definition_doc -%}
+    {% if msg | message_docs | length > 0 %}
+    {{ doc_comment(msg) }}
+    {% endif %}
+{% endblock %}
+
+{%- block definition -%}
+@dataclasses.dataclass
+class {{ msg | message_type }}(BaseMessage):
+    {% block fields_definition %}
+    {% for field in sorted_fields %}
+    {% if field | message_docs | length > 0 %}
+    {{ doc_comment(field) | indent()}}
+    {%- endif %}
+    {{ field | field_name }}: {{ field | field_type }}
+    {% endfor %}
+    {% endblock %} 
+{%+ endblock %}
+
+{% endblock %}
+
+{% block after_definition %}
+{% endblock %}