diff --git a/src/inmanta/resources.py b/src/inmanta/resources.py index b0c796a487..bc3ee451bd 100644 --- a/src/inmanta/resources.py +++ b/src/inmanta/resources.py @@ -36,11 +36,14 @@ cast, ) +from pydantic import BaseModel, Extra, PrivateAttr, constr, validator + import inmanta.util from inmanta import plugins from inmanta.ast import CompilerException, ExplicitPluginException, ExternalException from inmanta.data.model import ResourceIdStr, ResourceVersionIdStr from inmanta.execute import proxy, util +from inmanta.execute.proxy import SequenceProxy from inmanta.stable_api import stable_api from inmanta.types import JsonType @@ -96,7 +99,8 @@ def __call__(self, cls: Type[T]) -> Type[T]: @classmethod def validate(cls) -> None: for resource, _ in cls._resources.values(): - resource.validate() + if not issubclass(resource, PydanticResource): + resource.validate() @classmethod def get_entity_resources(cls) -> Iterable[str]: @@ -191,8 +195,12 @@ def __new__(cls, class_name, bases, dct): RESERVED_FOR_RESOURCE = {"id", "version", "model", "requires", "unknowns", "set_version", "clone", "is_type", "serialize"} +class BaseResource: + pass + + @stable_api -class Resource(metaclass=ResourceMeta): +class Resource(BaseResource, metaclass=ResourceMeta): """ Plugins should inherit resource from this class so a resource from a model can be serialized and deserialized. @@ -329,20 +337,21 @@ def create_from_model(cls, exporter: "export.Exporter", entity_name: str, model_ Build a resource from a given configuration model entity """ resource_cls, options = resource.get_class(entity_name) - if resource_cls is None or options is None: raise TypeError("No resource class registered for entity %s" % entity_name) + obj = resource_cls.construct_from_model(entity_name, exporter, model_object, options) - # build the id of the object - obj_id = resource_cls.object_to_id(model_object, entity_name, options["name"], options["agent"]) + return obj + @classmethod + def construct_from_model(cls, entity_name, exporter, model_object, options): + # build the id of the object + obj_id = cls.object_to_id(model_object, entity_name, options["name"], options["agent"]) # map all fields - fields = {field: resource_cls.map_field(exporter, entity_name, field, model_object) for field in resource_cls.fields} - - obj = resource_cls(obj_id) + fields = {field: cls.map_field(exporter, entity_name, field, model_object) for field in cls.fields} + obj = cls(obj_id) obj.populate(fields) obj.model = model_object - return obj @classmethod @@ -361,9 +370,14 @@ def deserialize(cls, obj_map: JsonType, use_generic: bool = False) -> "Resource" cls_resource = cls force_fields = True - obj = cls_resource(obj_id) - obj.populate(obj_map, force_fields) + obj = cls_resource.do_deserialize(force_fields, obj_id, obj_map) + + return obj + @classmethod + def do_deserialize(cls, force_fields: bool, obj_id: "Id", obj_map: JsonType): + obj = cls(obj_id) + obj.populate(obj_map, force_fields) return obj @classmethod @@ -478,10 +492,14 @@ def get_managed(exporter: "export.Exporter", obj: "ManagedResource") -> bool: return obj.managed -PARSE_ID_REGEX = re.compile( +PARSE_ID_REGEX_RAW = ( r"^(?P(?P(?P[\w-]+(::[\w-]+)*)::(?P[\w-]+))\[(?P[^,]+)," r"(?P[^=]+)=(?P[^\]]+)\])(,v=(?P[0-9]+))?$" ) +PARSE_ID_REGEX = re.compile(PARSE_ID_REGEX_RAW) + +ResourceVersionId_pd = constr(regex=PARSE_ID_REGEX_RAW) + PARSE_RVID_REGEX = re.compile( r"^(?P(?P(?P[\w-]+(::[\w-]+)*)::(?P[\w-]+))\[(?P[^,]+)," @@ -666,3 +684,79 @@ def to_action(self) -> "ResourceAction": ra.data = {"host": self.hostname, "user": self.user, "error": self.error} return ra + + +# TODO reserved keywords +# todo parsed id +class PydanticResource(BaseModel): + class Config: + orm_mode = True + underscore_attrs_are_private = True + extra = Extra.allow # todo: resource_requires + + # ID forming data + _entity_name: str + _attribute_name: str + _agent_attribute: str + _model: object + _id: Optional[Id] = None + + requires: List[ResourceVersionId_pd] = [] + version: int = 0 + unknowns: List[str] = [] # ??? + + @classmethod + def construct_from_model(cls, entity_name, exporter, model_object, options): + out = cls.from_orm(model_object) + out._entity_name = entity_name + out._attribute_name = options["name"] + out._agent_attribute = options["agent"] + out.id + out._model = model_object + return out + + @classmethod + def do_deserialize(cls, force_fields: bool, obj_id: "Id", obj_map: JsonType): + out = cls(**obj_map) + out._id = obj_id + return out + + @validator("requires", pre=True) + def validate_requires(cls, value) -> list[ResourceVersionId_pd]: + if isinstance(value, SequenceProxy): + # From orm, let it go + return [] + return value + + def set_version(self, version: int) -> None: + self.version = version + + @property + def id(self): + if self._id is None: + self._id = Id( + self._entity_name, + getattr(self, self._agent_attribute), + self._attribute_name, + getattr(self, self._attribute_name), + ) + return self._id + + @property + def model(self): + return self._model + + def __hash__(self): + return hash(self.id) + + def serialize(self) -> JsonType: + """ + Serialize this resource to its dictionary representation + """ + dictionary = self.dict() + + dictionary["requires"] = [str(x) for x in self.requires] # do we need this? + dictionary["id"] = self.id.resource_version_str() + del dictionary["resource_requires"] + + return dictionary diff --git a/tests/agent_server/conftest.py b/tests/agent_server/conftest.py index 49787de2fd..5c075740a8 100644 --- a/tests/agent_server/conftest.py +++ b/tests/agent_server/conftest.py @@ -29,7 +29,7 @@ from inmanta.agent.agent import Agent from inmanta.agent.handler import CRUDHandler, HandlerContext, ResourceHandler, ResourcePurged, SkipResource, provider from inmanta.data.model import ResourceIdStr -from inmanta.resources import IgnoreResourceException, PurgeableResource, Resource, resource +from inmanta.resources import IgnoreResourceException, PurgeableResource, PydanticResource, Resource, resource from inmanta.server import SLICE_AGENT_MANAGER from inmanta.util import get_compiler_version from utils import retry_limited @@ -110,12 +110,14 @@ async def in_progress(): def resource_container(): @resource("test::Resource", agent="agent", id_attribute="key") class MyResource(Resource): - """ - A file on a filesystem - """ - fields = ("key", "value", "purged") + @resource("test::ResourceP", agent="agent", id_attribute="key") + class MyResourceP(PydanticResource): + key: str + value: str + purged: bool + @resource("test::Fact", agent="agent", id_attribute="key") class FactResource(Resource): """ @@ -205,6 +207,7 @@ class DeployR(Resource): fields = ("key", "value", "set_state_to_deployed", "purged") @provider("test::Resource", name="test_resource") + @provider("test::ResourceP", name="test_resource") class Provider(ResourceHandler): def check_resource(self, ctx, resource): self.read(resource.id.get_agent_name(), resource.key) diff --git a/tests/agent_server/test_server_agent.py b/tests/agent_server/test_server_agent.py index 492ae56e30..e2eca5ffd9 100644 --- a/tests/agent_server/test_server_agent.py +++ b/tests/agent_server/test_server_agent.py @@ -3443,3 +3443,76 @@ async def deploy_resource(set_state_to_deployed_in_handler: bool = False) -> con # Exception is raised by handler resource_container.Provider.set_fail("agent1", "key1", 1) assert const.ResourceState.failed == await deploy_resource() + + +async def test_deploy_pydantic_resource(server, client, resource_container, async_finalizer, no_agent_backoff): + """ + Test deploy of resource with undefined + """ + agentmanager = server.get_slice(SLICE_AGENT_MANAGER) + + Config.set("config", "agent-deploy-interval", "100") + + resource_container.Provider.reset() + result = await client.create_project("env-test") + project_id = result.result["project"]["id"] + + result = await client.create_environment(project_id=project_id, name="dev") + env_id = result.result["environment"]["id"] + + await client.set_setting(env_id, data.AUTO_DEPLOY, False) + await client.set_setting(env_id, data.PUSH_ON_AUTO_DEPLOY, False) + await client.set_setting(env_id, data.AGENT_TRIGGER_METHOD_ON_AUTO_DEPLOY, const.AgentTriggerMethod.push_full_deploy) + + agent = Agent( + hostname="node1", environment=env_id, agent_map={"agent1": "localhost", "agent2": "localhost"}, code_loader=False + ) + async_finalizer.add(agent.stop) + await agent.add_end_point_name("agent2") + await agent.start() + + await retry_limited(lambda: len(agentmanager.sessions) == 1, 10) + + clienthelper = ClientHelper(client, env_id) + + version = await clienthelper.get_version() + + resources = [ + { + "key": "key1", + "value": "value1", + "id": "test::ResourceP[agent2,key=key1],v=%d" % version, + "send_event": False, + "purged": False, + "requires": [], + }, + ] + + result = await client.put_version( + tid=env_id, + version=version, + resources=resources, + resource_state={}, + unknowns=[], + version_info={}, + compiler_version=get_compiler_version(), + ) + assert result.code == 200 + + # do a deploy + result = await client.release_version(env_id, version, True, const.AgentTriggerMethod.push_full_deploy) + assert result.code == 200 + assert not result.result["model"]["deployed"] + assert result.result["model"]["released"] + assert result.result["model"]["total"] == len(resources) + assert result.result["model"]["result"] == "deploying" + + # The server will mark the full version as deployed even though the agent has not done anything yet. + result = await client.get_version(env_id, version) + assert result.code == 200 + + await _wait_until_deployment_finishes(client, env_id, version) + + result = await client.get_version(env_id, version) + assert result.result["model"]["done"] == len(resources) + assert result.code == 200 diff --git a/tests/test_export.py b/tests/test_export.py index 4df81a92ca..a703289d95 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -718,3 +718,43 @@ class Res(Resource): "the_resource_z": None, }, ) + + +def test_resource_pydantic(snippetcompiler, modules_dir: str, tmpdir, environment) -> None: + init_cf = """ +entity Res extends std::Resource: + string name +end + +implement Res using std::none + +Res(name="the_resource_a") +""" + init_py = """ +from inmanta.resources import ( + PydanticResource, + resource, +) +@resource("modulev1::Res", agent="name", id_attribute="name") +class Res(PydanticResource): + name: str +""" + module_name: str = "minimalv1module" + module_path: str = str(tmpdir.join("modulev1")) + v1_module_from_template( + os.path.join(modules_dir, module_name), + module_path, + new_content_init_cf=init_cf, + new_content_init_py=init_py, + new_name="modulev1", + ) + snippetcompiler.setup_for_snippet( + """ +import modulev1 + """, + add_to_module_path=[str(tmpdir)], + ) + + _version, json_value = snippetcompiler.do_export() + string_keyed = {str(k): v for k, v in json_value.items()} + string_keyed["modulev1::Res[the_resource_a,name=the_resource_a]"].name == "the_resource_a"