POC pydantic resource

src/inmanta/resources.py

@@ -36,11 +36,14 @@
     cast,
 )
 
+from pydantic import BaseModel, Extra, PrivateAttr, constr, validator
+
 import inmanta.util
 from inmanta import plugins
 from inmanta.ast import CompilerException, ExplicitPluginException, ExternalException
 from inmanta.data.model import ResourceIdStr, ResourceVersionIdStr
 from inmanta.execute import proxy, util
+from inmanta.execute.proxy import SequenceProxy
 from inmanta.stable_api import stable_api
 from inmanta.types import JsonType
 
@@ -96,7 +99,8 @@ def __call__(self, cls: Type[T]) -> Type[T]:
     @classmethod
     def validate(cls) -> None:
         for resource, _ in cls._resources.values():
-            resource.validate()
+            if not issubclass(resource, PydanticResource):
+                resource.validate()
 
     @classmethod
     def get_entity_resources(cls) -> Iterable[str]:
@@ -191,8 +195,12 @@ def __new__(cls, class_name, bases, dct):
 RESERVED_FOR_RESOURCE = {"id", "version", "model", "requires", "unknowns", "set_version", "clone", "is_type", "serialize"}
 
 
+class BaseResource:
+    pass
+
+
 @stable_api
-class Resource(metaclass=ResourceMeta):
+class Resource(BaseResource, metaclass=ResourceMeta):
     """
     Plugins should inherit resource from this class so a resource from a model can be serialized and deserialized.
 
@@ -329,20 +337,21 @@ def create_from_model(cls, exporter: "export.Exporter", entity_name: str, model_
         Build a resource from a given configuration model entity
         """
         resource_cls, options = resource.get_class(entity_name)
-
         if resource_cls is None or options is None:
             raise TypeError("No resource class registered for entity %s" % entity_name)
+        obj = resource_cls.construct_from_model(entity_name, exporter, model_object, options)
 
-        # build the id of the object
-        obj_id = resource_cls.object_to_id(model_object, entity_name, options["name"], options["agent"])
+        return obj
 
+    @classmethod
+    def construct_from_model(cls, entity_name, exporter, model_object, options):
+        # build the id of the object
+        obj_id = cls.object_to_id(model_object, entity_name, options["name"], options["agent"])
         # map all fields
-        fields = {field: resource_cls.map_field(exporter, entity_name, field, model_object) for field in resource_cls.fields}
-
-        obj = resource_cls(obj_id)
+        fields = {field: cls.map_field(exporter, entity_name, field, model_object) for field in cls.fields}
+        obj = cls(obj_id)
         obj.populate(fields)
         obj.model = model_object
-
         return obj
 
     @classmethod
@@ -361,9 +370,14 @@ def deserialize(cls, obj_map: JsonType, use_generic: bool = False) -> "Resource"
                 cls_resource = cls
                 force_fields = True
 
-        obj = cls_resource(obj_id)
-        obj.populate(obj_map, force_fields)
+        obj = cls_resource.do_deserialize(force_fields, obj_id, obj_map)
+
+        return obj
 
+    @classmethod
+    def do_deserialize(cls, force_fields: bool, obj_id: "Id", obj_map: JsonType):
+        obj = cls(obj_id)
+        obj.populate(obj_map, force_fields)
         return obj
 
     @classmethod
@@ -478,10 +492,14 @@ def get_managed(exporter: "export.Exporter", obj: "ManagedResource") -> bool:
         return obj.managed
 
 
-PARSE_ID_REGEX = re.compile(
+PARSE_ID_REGEX_RAW = (
     r"^(?P<id>(?P<type>(?P<ns>[\w-]+(::[\w-]+)*)::(?P<class>[\w-]+))\[(?P<hostname>[^,]+),"
     r"(?P<attr>[^=]+)=(?P<value>[^\]]+)\])(,v=(?P<version>[0-9]+))?$"
 )
+PARSE_ID_REGEX = re.compile(PARSE_ID_REGEX_RAW)
+
+ResourceVersionId_pd = constr(regex=PARSE_ID_REGEX_RAW)
+
 
 PARSE_RVID_REGEX = re.compile(
     r"^(?P<id>(?P<type>(?P<ns>[\w-]+(::[\w-]+)*)::(?P<class>[\w-]+))\[(?P<hostname>[^,]+),"
@@ -666,3 +684,79 @@ def to_action(self) -> "ResourceAction":
         ra.data = {"host": self.hostname, "user": self.user, "error": self.error}
 
         return ra
+
+
+# TODO reserved keywords
+# todo parsed id
+class PydanticResource(BaseModel):
+    class Config:
+        orm_mode = True
+        underscore_attrs_are_private = True
+        extra = Extra.allow  # todo: resource_requires
+
+    # ID forming data
+    _entity_name: str
+    _attribute_name: str
+    _agent_attribute: str
+    _model: object
+    _id: Optional[Id] = None
+
+    requires: List[ResourceVersionId_pd] = []
+    version: int = 0
+    unknowns: List[str] = []  # ???
+
+    @classmethod
+    def construct_from_model(cls, entity_name, exporter, model_object, options):
+        out = cls.from_orm(model_object)
+        out._entity_name = entity_name
+        out._attribute_name = options["name"]
+        out._agent_attribute = options["agent"]
+        out.id
+        out._model = model_object
+        return out
+
+    @classmethod
+    def do_deserialize(cls, force_fields: bool, obj_id: "Id", obj_map: JsonType):
+        out = cls(**obj_map)
+        out._id = obj_id
+        return out
+
+    @validator("requires", pre=True)
+    def validate_requires(cls, value) -> list[ResourceVersionId_pd]:
+        if isinstance(value, SequenceProxy):
+            # From orm, let it go
+            return []
+        return value
+
+    def set_version(self, version: int) -> None:
+        self.version = version
+
+    @property
+    def id(self):
+        if self._id is None:
+            self._id = Id(
+                self._entity_name,
+                getattr(self, self._agent_attribute),
+                self._attribute_name,
+                getattr(self, self._attribute_name),
+            )
+        return self._id
+
+    @property
+    def model(self):
+        return self._model
+
+    def __hash__(self):
+        return hash(self.id)
+
+    def serialize(self) -> JsonType:
+        """
+        Serialize this resource to its dictionary representation
+        """
+        dictionary = self.dict()
+
+        dictionary["requires"] = [str(x) for x in self.requires]  # do we need this?
+        dictionary["id"] = self.id.resource_version_str()
+        del dictionary["resource_requires"]
+
+        return dictionary

tests/agent_server/conftest.py

@@ -29,7 +29,7 @@
 from inmanta.agent.agent import Agent
 from inmanta.agent.handler import CRUDHandler, HandlerContext, ResourceHandler, ResourcePurged, SkipResource, provider
 from inmanta.data.model import ResourceIdStr
-from inmanta.resources import IgnoreResourceException, PurgeableResource, Resource, resource
+from inmanta.resources import IgnoreResourceException, PurgeableResource, PydanticResource, Resource, resource
 from inmanta.server import SLICE_AGENT_MANAGER
 from inmanta.util import get_compiler_version
 from utils import retry_limited
@@ -110,12 +110,14 @@ async def in_progress():
 def resource_container():
     @resource("test::Resource", agent="agent", id_attribute="key")
     class MyResource(Resource):
-        """
-        A file on a filesystem
-        """
-
         fields = ("key", "value", "purged")
 
+    @resource("test::ResourceP", agent="agent", id_attribute="key")
+    class MyResourceP(PydanticResource):
+        key: str
+        value: str
+        purged: bool
+
     @resource("test::Fact", agent="agent", id_attribute="key")
     class FactResource(Resource):
         """
@@ -205,6 +207,7 @@ class DeployR(Resource):
         fields = ("key", "value", "set_state_to_deployed", "purged")
 
     @provider("test::Resource", name="test_resource")
+    @provider("test::ResourceP", name="test_resource")
     class Provider(ResourceHandler):
         def check_resource(self, ctx, resource):
             self.read(resource.id.get_agent_name(), resource.key)

tests/agent_server/test_server_agent.py

@@ -3443,3 +3443,76 @@ async def deploy_resource(set_state_to_deployed_in_handler: bool = False) -> con
     # Exception is raised by handler
     resource_container.Provider.set_fail("agent1", "key1", 1)
     assert const.ResourceState.failed == await deploy_resource()
+
+
+async def test_deploy_pydantic_resource(server, client, resource_container, async_finalizer, no_agent_backoff):
+    """
+    Test deploy of resource with undefined
+    """
+    agentmanager = server.get_slice(SLICE_AGENT_MANAGER)
+
+    Config.set("config", "agent-deploy-interval", "100")
+
+    resource_container.Provider.reset()
+    result = await client.create_project("env-test")
+    project_id = result.result["project"]["id"]
+
+    result = await client.create_environment(project_id=project_id, name="dev")
+    env_id = result.result["environment"]["id"]
+
+    await client.set_setting(env_id, data.AUTO_DEPLOY, False)
+    await client.set_setting(env_id, data.PUSH_ON_AUTO_DEPLOY, False)
+    await client.set_setting(env_id, data.AGENT_TRIGGER_METHOD_ON_AUTO_DEPLOY, const.AgentTriggerMethod.push_full_deploy)
+
+    agent = Agent(
+        hostname="node1", environment=env_id, agent_map={"agent1": "localhost", "agent2": "localhost"}, code_loader=False
+    )
+    async_finalizer.add(agent.stop)
+    await agent.add_end_point_name("agent2")
+    await agent.start()
+
+    await retry_limited(lambda: len(agentmanager.sessions) == 1, 10)
+
+    clienthelper = ClientHelper(client, env_id)
+
+    version = await clienthelper.get_version()
+
+    resources = [
+        {
+            "key": "key1",
+            "value": "value1",
+            "id": "test::ResourceP[agent2,key=key1],v=%d" % version,
+            "send_event": False,
+            "purged": False,
+            "requires": [],
+        },
+    ]
+
+    result = await client.put_version(
+        tid=env_id,
+        version=version,
+        resources=resources,
+        resource_state={},
+        unknowns=[],
+        version_info={},
+        compiler_version=get_compiler_version(),
+    )
+    assert result.code == 200
+
+    # do a deploy
+    result = await client.release_version(env_id, version, True, const.AgentTriggerMethod.push_full_deploy)
+    assert result.code == 200
+    assert not result.result["model"]["deployed"]
+    assert result.result["model"]["released"]
+    assert result.result["model"]["total"] == len(resources)
+    assert result.result["model"]["result"] == "deploying"
+
+    # The server will mark the full version as deployed even though the agent has not done anything yet.
+    result = await client.get_version(env_id, version)
+    assert result.code == 200
+
+    await _wait_until_deployment_finishes(client, env_id, version)
+
+    result = await client.get_version(env_id, version)
+    assert result.result["model"]["done"] == len(resources)
+    assert result.code == 200

tests/test_export.py

@@ -718,3 +718,43 @@ class Res(Resource):
             "the_resource_z": None,
         },
     )
+
+
+def test_resource_pydantic(snippetcompiler, modules_dir: str, tmpdir, environment) -> None:
+    init_cf = """
+entity Res extends std::Resource:
+    string name
+end
+
+implement Res using std::none
+
+Res(name="the_resource_a")
+"""
+    init_py = """
+from inmanta.resources import (
+    PydanticResource,
+    resource,
+)
+@resource("modulev1::Res", agent="name", id_attribute="name")
+class Res(PydanticResource):
+    name: str
+"""
+    module_name: str = "minimalv1module"
+    module_path: str = str(tmpdir.join("modulev1"))
+    v1_module_from_template(
+        os.path.join(modules_dir, module_name),
+        module_path,
+        new_content_init_cf=init_cf,
+        new_content_init_py=init_py,
+        new_name="modulev1",
+    )
+    snippetcompiler.setup_for_snippet(
+        """
+import modulev1
+        """,
+        add_to_module_path=[str(tmpdir)],
+    )
+
+    _version, json_value = snippetcompiler.do_export()
+    string_keyed = {str(k): v for k, v in json_value.items()}
+    string_keyed["modulev1::Res[the_resource_a,name=the_resource_a]"].name == "the_resource_a"