From 2abd93c0074c7ab49473f4940c74e6de325d22ca Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Fri, 4 Nov 2022 12:43:18 +0100 Subject: [PATCH 01/20] [CNA] assume role test resource and role ARN retrieval from AWS account (#2927) this implements the example-aws-assumerole CNA type to test assume role functionality. the role ARNs are stored in the aws account Signed-off-by: Gerd Oberlechner --- reconcile/cna/assets/asset.py | 2 + reconcile/cna/assets/asset_factory.py | 19 +- reconcile/cna/assets/aws_assume_role.py | 65 +++++ reconcile/cna/state.py | 5 +- .../cna/queries/aws_account_fragment.gql | 12 + .../cna/queries/aws_account_fragment.py | 46 ++++ .../cna/queries/cna_resources.gql | 9 + .../cna/queries/cna_resources.py | 46 +++- reconcile/gql_definitions/introspection.json | 227 ++++++++++++++++++ 9 files changed, 425 insertions(+), 6 deletions(-) create mode 100644 reconcile/cna/assets/aws_assume_role.py create mode 100644 reconcile/gql_definitions/cna/queries/aws_account_fragment.gql create mode 100644 reconcile/gql_definitions/cna/queries/aws_account_fragment.py diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index 347e892ce8..146294c5b6 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -21,9 +21,11 @@ class AssetError(Exception): class AssetType(Enum): NULL = "null" + EXAMPLE_AWS_ASSUMEROLE = "example-aws-assumerole" class AssetStatus(Enum): + READY = "Ready" TERMINATED = "Terminated" PENDING = "Pending" RUNNING = "Running" diff --git a/reconcile/cna/assets/asset_factory.py b/reconcile/cna/assets/asset_factory.py index 7dede2b3d4..a6f238bd44 100644 --- a/reconcile/cna/assets/asset_factory.py +++ b/reconcile/cna/assets/asset_factory.py @@ -1,3 +1,4 @@ +from typing import Any, Optional from collections.abc import Mapping from typing import Any @@ -8,20 +9,32 @@ from reconcile.cna.assets.null import NullAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, + CNAAssumeRoleAssetV1, CNAssetV1, ) +from reconcile.cna.assets.null import NullAsset +from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset +from reconcile.cna.assets.asset import Asset, AssetError, AssetType def asset_factory_from_schema(schema_asset: CNAssetV1) -> Asset: if isinstance(schema_asset, CNANullAssetV1): return NullAsset.from_query_class(schema_asset) + elif isinstance(schema_asset, CNAAssumeRoleAssetV1): + return AWSAssumeRoleAsset.from_query_class(schema_asset) else: raise AssetError(f"Unknown schema asset type {schema_asset}") -def asset_factory_from_raw_data(data_asset: Mapping[str, Any]) -> Asset: +def asset_factory_from_raw_data(data_asset: Mapping[str, Any]) -> Optional[Asset]: asset_type = data_asset.get("asset_type") - if asset_type == "null": + if asset_type == AssetType.NULL.value: return NullAsset.from_api_mapping(data_asset) + elif asset_type == AssetType.EXAMPLE_AWS_ASSUMEROLE.value: + return AWSAssumeRoleAsset.from_api_mapping(data_asset) else: - raise AssetError(f"Unknown data asset type {data_asset}") + href = data_asset.get("href") + logging.warning( + f"Ignoring unknown data asset type '{asset_type}' - href: {href}" + ) + return None diff --git a/reconcile/cna/assets/aws_assume_role.py b/reconcile/cna/assets/aws_assume_role.py new file mode 100644 index 0000000000..6ef0f7e139 --- /dev/null +++ b/reconcile/cna/assets/aws_assume_role.py @@ -0,0 +1,65 @@ +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Mapping + +from reconcile.cna.assets.asset import Asset, AssetError, AssetStatus, AssetType +from reconcile.gql_definitions.cna.queries.cna_resources import CNAAssumeRoleAssetV1 + + +@dataclass(frozen=True) +class AWSAssumeRoleAsset(Asset): + slug: str + role_arn: str + + def api_payload(self) -> dict[str, Any]: + return { + "asset_type": AssetType.EXAMPLE_AWS_ASSUMEROLE, + "name": self.name, + "parameters": { + "slug": self.slug, + "role_arn": self.role_arn, + }, + } + + def update_from(self, asset: Asset) -> Asset: + if not isinstance(asset, AWSAssumeRoleAsset): + raise AssetError(f"Cannot create AWSAssumeRoleAsset from {asset}") + return AWSAssumeRoleAsset( + uuid=self.uuid, + href=self.href, + status=self.status, + name=self.name, + kind=self.kind, + slug=asset.slug, + role_arn=asset.role_arn, + ) + + @staticmethod + def from_query_class(asset: CNAAssumeRoleAssetV1) -> AWSAssumeRoleAsset: + role_arn = asset.aws_assume_role.account.cna.default_role_arn + for module_config in asset.aws_assume_role.account.cna.module_role_arns: + if module_config.module == AssetType.EXAMPLE_AWS_ASSUMEROLE: + role_arn = module_config.role_arn + break + + return AWSAssumeRoleAsset( + uuid=None, + href=None, + status=None, + kind=AssetType.EXAMPLE_AWS_ASSUMEROLE, + name=asset.name, + slug=asset.aws_assume_role.slug, + role_arn=role_arn, + ) + + @staticmethod + def from_api_mapping(asset: Mapping[str, Any]) -> AWSAssumeRoleAsset: + return AWSAssumeRoleAsset( + uuid=asset.get("id"), + href=asset.get("href"), + status=AssetStatus(asset.get("status")), + kind=AssetType.NULL, + name=asset.get("name", ""), + slug=asset.get("slug"), + role_arn=asset.get("role_arn"), + ) diff --git a/reconcile/cna/state.py b/reconcile/cna/state.py index 266779eeb4..5b414b4ea0 100644 --- a/reconcile/cna/state.py +++ b/reconcile/cna/state.py @@ -68,8 +68,9 @@ def add_asset(self, asset: Asset): def add_raw_data(self, data: Iterable[Mapping[str, Any]]): for cna in data: asset = asset_factory_from_raw_data(cna) - self._validate_addition(asset=asset) - self._assets[asset.kind][asset.name] = asset + if asset: + self._validate_addition(asset=asset) + self._assets[asset.kind][asset.name] = asset def required_updates_to_reach(self, other: State) -> State: """ diff --git a/reconcile/gql_definitions/cna/queries/aws_account_fragment.gql b/reconcile/gql_definitions/cna/queries/aws_account_fragment.gql new file mode 100644 index 0000000000..67f2026d6b --- /dev/null +++ b/reconcile/gql_definitions/cna/queries/aws_account_fragment.gql @@ -0,0 +1,12 @@ +# qenerate: plugin=pydantic_v1 + +fragment CNAAWSAccountRoleARNs on AWSAccount_v1 { + name + cna { + defaultRoleARN + moduleRoleARNS { + module + arn + } + } +} diff --git a/reconcile/gql_definitions/cna/queries/aws_account_fragment.py b/reconcile/gql_definitions/cna/queries/aws_account_fragment.py new file mode 100644 index 0000000000..f654cc5192 --- /dev/null +++ b/reconcile/gql_definitions/cna/queries/aws_account_fragment.py @@ -0,0 +1,46 @@ +""" +Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY! +""" +from enum import Enum # noqa: F401 # pylint: disable=W0611 +from typing import ( # noqa: F401 # pylint: disable=W0611 + Any, + Callable, + Optional, + Union, +) + +from pydantic import ( # noqa: F401 # pylint: disable=W0611 + BaseModel, + Extra, + Field, + Json, +) + + +class CNAModuleAWSARNV1(BaseModel): + module: str = Field(..., alias="module") + arn: str = Field(..., alias="arn") + + class Config: + smart_union = True + extra = Extra.forbid + + +class CNAAWSSpecV1(BaseModel): + default_role_arn: Optional[str] = Field(..., alias="defaultRoleARN") + module_role_arns: Optional[list[CNAModuleAWSARNV1]] = Field( + ..., alias="moduleRoleARNS" + ) + + class Config: + smart_union = True + extra = Extra.forbid + + +class CNAAWSAccountRoleARNs(BaseModel): + name: str = Field(..., alias="name") + cna: Optional[CNAAWSSpecV1] = Field(..., alias="cna") + + class Config: + smart_union = True + extra = Extra.forbid diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.gql b/reconcile/gql_definitions/cna/queries/cna_resources.gql index fff8ff7f15..b56d31a890 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.gql +++ b/reconcile/gql_definitions/cna/queries/cna_resources.gql @@ -15,6 +15,15 @@ query CNAssets { name: identifier addr_block } + ... on CNAAssumeRoleAsset_v1{ + name: identifier + aws_assume_role { + slug + account { + ... CNAAWSAccountRoleARNs + } + } + } } } } diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.py b/reconcile/gql_definitions/cna/queries/cna_resources.py index 52218751ff..5dee1ef8cd 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.py +++ b/reconcile/gql_definitions/cna/queries/cna_resources.py @@ -16,8 +16,23 @@ Json, ) +from reconcile.gql_definitions.cna.queries.aws_account_fragment import ( + CNAAWSAccountRoleARNs, +) + DEFINITION = """ +fragment CNAAWSAccountRoleARNs on AWSAccount_v1 { + name + cna { + defaultRoleARN + moduleRoleARNS { + module + arn + } + } +} + query CNAssets { namespaces: namespaces_v1 { name @@ -33,6 +48,15 @@ name: identifier addr_block } + ... on CNAAssumeRoleAsset_v1{ + name: identifier + aws_assume_role { + slug + account { + ... CNAAWSAccountRoleARNs + } + } + } } } } @@ -75,8 +99,28 @@ class Config: extra = Extra.forbid +class CNAAssumeRoleAssetConfigV1(BaseModel): + slug: str = Field(..., alias="slug") + account: CNAAWSAccountRoleARNs = Field(..., alias="account") + + class Config: + smart_union = True + extra = Extra.forbid + + +class CNAAssumeRoleAssetV1(CNAssetV1): + name: str = Field(..., alias="name") + aws_assume_role: CNAAssumeRoleAssetConfigV1 = Field(..., alias="aws_assume_role") + + class Config: + smart_union = True + extra = Extra.forbid + + class NamespaceCNAssetV1(NamespaceExternalResourceV1): - resources: list[Union[CNANullAssetV1, CNAssetV1]] = Field(..., alias="resources") + resources: list[Union[CNANullAssetV1, CNAAssumeRoleAssetV1, CNAssetV1]] = Field( + ..., alias="resources" + ) class Config: smart_union = True diff --git a/reconcile/gql_definitions/introspection.json b/reconcile/gql_definitions/introspection.json index e0c7c4e873..0ee7c874b9 100644 --- a/reconcile/gql_definitions/introspection.json +++ b/reconcile/gql_definitions/introspection.json @@ -7438,6 +7438,18 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "cna", + "description": null, + "args": [], + "type": { + "kind": "OBJECT", + "name": "CNAAWSSpec_v1", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "ecrs", "description": null, @@ -9908,6 +9920,92 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "CNAAWSSpec_v1", + "description": null, + "fields": [ + { + "name": "defaultRoleARN", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "moduleRoleARNS", + "description": null, + "args": [], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "CNAModuleAWSARN_v1", + "ofType": null + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CNAModuleAWSARN_v1", + "description": null, + "fields": [ + { + "name": "module", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "arn", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "AWSECR_v1", @@ -28168,12 +28266,33 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "identifier", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, "interfaces": null, "enumValues": null, "possibleTypes": [ + { + "kind": "OBJECT", + "name": "CNAAssumeRoleAsset_v1", + "ofType": null + }, { "kind": "OBJECT", "name": "CNANullAsset_v1", @@ -28181,6 +28300,114 @@ } ] }, + { + "kind": "OBJECT", + "name": "CNAAssumeRoleAsset_v1", + "description": null, + "fields": [ + { + "name": "provider", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "identifier", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "aws_assume_role", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "CNAAssumeRoleAssetConfig_v1", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + { + "kind": "INTERFACE", + "name": "CNAsset_v1", + "ofType": null + } + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CNAAssumeRoleAssetConfig_v1", + "description": null, + "fields": [ + { + "name": "slug", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "account", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "AWSAccount_v1", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "CNANullAsset_v1", From 1bb7a65ce0543b713abff05a6522c839ba607d72 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Mon, 7 Nov 2022 14:59:25 +0100 Subject: [PATCH 02/20] rely on asset field metadata for CNA API and asset class conversion (#2932) * rely on asset field metadata for CNA API and asset class conversion * use registration process to make asset classes, their provider and CNA kind known to the integration * use pydantic `alias` fields to map the difference from CNA api and python dataclasses * drive dataclass<->API conversion through this metadata * load CNA metadata into the client so we can test for dataclass<->API compatibility * renamed kind to asset_type (kind means something different in the CNA API) * creator filtering --- reconcile/cna/assets/__init__.py | 7 + reconcile/cna/assets/asset.py | 218 ++++++++++++++++++--- reconcile/cna/assets/asset_factory.py | 66 ++++--- reconcile/cna/assets/aws_assume_role.py | 93 ++++----- reconcile/cna/assets/aws_utils.py | 15 ++ reconcile/cna/assets/null.py | 67 +++---- reconcile/cna/client.py | 57 +++++- reconcile/cna/integration.py | 27 ++- reconcile/cna/state.py | 75 +++---- reconcile/test/cna/test_asset.py | 214 ++++++++++++++++++++ reconcile/test/cna/test_aws_assume_role.py | 31 +++ reconcile/test/cna/test_client.py | 45 +++++ reconcile/test/cna/test_integration.py | 40 ++-- reconcile/test/cna/test_null.py | 16 ++ reconcile/test/cna/test_state_assembly.py | 19 +- reconcile/test/cna/test_state_diff.py | 52 ++--- reconcile/test/cna/test_state_overrides.py | 13 +- 17 files changed, 799 insertions(+), 256 deletions(-) create mode 100644 reconcile/cna/assets/aws_utils.py create mode 100644 reconcile/test/cna/test_asset.py create mode 100644 reconcile/test/cna/test_aws_assume_role.py create mode 100644 reconcile/test/cna/test_client.py create mode 100644 reconcile/test/cna/test_null.py diff --git a/reconcile/cna/assets/__init__.py b/reconcile/cna/assets/__init__.py index e69de29bb2..b5e8a75044 100644 --- a/reconcile/cna/assets/__init__.py +++ b/reconcile/cna/assets/__init__.py @@ -0,0 +1,7 @@ +from reconcile.cna.assets.asset_factory import register_asset_dataclass +from reconcile.cna.assets.null import NullAsset +from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset + + +register_asset_dataclass(NullAsset) +register_asset_dataclass(AWSAssumeRoleAsset) diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index 146294c5b6..7630e6a2b0 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -1,48 +1,222 @@ from __future__ import annotations +from abc import ABC, abstractmethod -from abc import ( - ABC, - abstractmethod, -) -from dataclasses import ( - dataclass, - field, -) +from pydantic.dataclasses import dataclass +from pydantic.fields import FieldInfo from enum import Enum -from typing import ( - Any, - Optional, -) +from typing import Any, Mapping, Optional, Type + +from reconcile.gql_definitions.cna.queries.cna_resources import CNAssetV1 + + +ASSET_TYPE_FIELD = "asset_type" +ASSET_PARAMETERS_FIELD = "parameters" class AssetError(Exception): pass -class AssetType(Enum): +class UnknownAssetTypeError(Exception): + pass + + +class AssetType(str, Enum): NULL = "null" EXAMPLE_AWS_ASSUMEROLE = "example-aws-assumerole" +def asset_type_by_id(asset_type_id: str) -> Optional[AssetType]: + try: + return AssetType(asset_type_id) + except ValueError: + return None + + +def asset_type_from_raw_asset(raw_assset: Mapping[str, Any]) -> Optional[AssetType]: + return asset_type_by_id(raw_assset.get(ASSET_TYPE_FIELD, "")) + + +class AssetTypeVariableType(Enum): + STRING = "${string}" + NUMBER = "${number}" + LIST_STRING = "${list(string)}" + LIST_NUMBER = "${list(number)}" + + +@dataclass(frozen=True) +class AssetTypeVariable: + name: str + type: AssetTypeVariableType + optional: bool = False + default: Optional[str] = None + + +@dataclass +class AssetTypeMetadata: + id: AssetType + bindable: bool + variables: set[AssetTypeVariable] + + class AssetStatus(Enum): + UNKNOWN = None READY = "Ready" TERMINATED = "Terminated" PENDING = "Pending" RUNNING = "Running" -@dataclass(frozen=True) +class AssetModelConfig: + allow_population_by_field_name = True + extra = "forbid" + + +@dataclass(frozen=True, config=AssetModelConfig) class Asset(ABC): - uuid: Optional[str] = field(compare=False, hash=True) - href: Optional[str] = field(compare=False, hash=True) - status: Optional[AssetStatus] = field(compare=False, hash=True) name: str - kind: AssetType + id: Optional[str] + href: Optional[str] + status: Optional[AssetStatus] + @staticmethod + def bindable() -> bool: + return True + + @classmethod + def type_metadata(cls) -> AssetTypeMetadata: + return asset_type_metadata_from_asset_dataclass(cls) + + @staticmethod @abstractmethod - def api_payload(self) -> dict[str, Any]: - raise NotImplementedError() + def asset_type() -> AssetType: + ... + + @staticmethod + @abstractmethod + def provider() -> str: + ... + @staticmethod @abstractmethod - def update_from(self, asset: Asset) -> Asset: - raise NotImplementedError() + def from_query_class(asset: CNAssetV1) -> Asset: + ... + + @staticmethod + def asset_type_from_raw_asset(raw_asset: Mapping[str, Any]) -> Optional[AssetType]: + asset_type_value = raw_asset[ASSET_TYPE_FIELD] + return asset_type_by_id(asset_type_value) + + def asset_metadata(self) -> dict[str, Any]: + return { + "id": self.id, + "href": self.href, + "status": self.status.value if self.status else None, + "name": self.name, + ASSET_TYPE_FIELD: self.asset_type().value, + } + + def api_payload(self) -> dict[str, Any]: + return { + ASSET_TYPE_FIELD: self.asset_type().value, + "name": self.name, + ASSET_PARAMETERS_FIELD: self.raw_asset_parameters(omit_empty=False), + } + + def raw_asset_parameters(self, omit_empty: bool) -> dict[str, Any]: + raw_asset_params = {} + for var in self.type_metadata().variables: + python_property_name = _property_for_asset_parameter_alias( + type(self), var.name + ) + var_value = getattr(self, python_property_name) + if not var.optional and var_value is None: + raise AssetError( + f"Required variable {var.name} not set for asset {self.name}" + ) + if var_value is not None or not omit_empty: + raw_asset_params[var.name] = var_value + return raw_asset_params + + def update_from( + self, + asset: Asset, + ) -> Asset: + assert isinstance(asset, type(self)) + return type(asset)( + id=self.id, + href=self.href, + status=self.status, + name=self.name, + **asset.asset_properties(), + ) + + def asset_properties(self) -> dict[str, Any]: + return {p: getattr(self, p) for p in self.__annotations__.keys()} + + @staticmethod + def from_api_mapping( + raw_asset: Mapping[str, Any], + cna_dataclass: Type[Asset], + ) -> Asset: + params = {} + raw_asset_params = raw_asset.get(ASSET_PARAMETERS_FIELD) or {} + for var in cna_dataclass.type_metadata().variables: + var_value = raw_asset_params.get(var.name) + if not var.optional and not var_value: + raise AssetError( + f"Inconsistent asset from CNA API {raw_asset}: required parameter {var.name} is missing in CNA" + ) + property_name = _property_for_asset_parameter_alias(cna_dataclass, var.name) + params[property_name] = var_value + + return cna_dataclass( + id=raw_asset.get("id"), + href=raw_asset.get("href"), + status=AssetStatus(raw_asset.get("status")), + name=raw_asset.get("name", ""), + **params, + ) + + +def asset_type_metadata_from_asset_dataclass( + asset_dataclass: Type[Asset], +) -> AssetTypeMetadata: + variables = { + _asset_type_metadata_variable_from_type_annotation( + property_name, type_hint, getattr(asset_dataclass, property_name) + ) + for property_name, type_hint in asset_dataclass.__annotations__.items() + } + return AssetTypeMetadata( + id=asset_dataclass.asset_type(), + bindable=asset_dataclass.bindable(), + variables=variables, + ) + + +def _asset_type_metadata_variable_from_type_annotation( + property_name: str, + type_hint: str, + field_info: FieldInfo, +) -> AssetTypeVariable: + optional = type_hint.startswith("Optional[") + if type_hint == "str" or type_hint.endswith("[str]"): + asset_type = AssetTypeVariableType.STRING + elif type_hint == "int" or type_hint.endswith("[int]"): + asset_type = AssetTypeVariableType.NUMBER + else: + raise AssetError(f"Unsupported type hint {type_hint} for {property_name}") + # TODO handle list types + return AssetTypeVariable( + name=field_info.alias or property_name, + optional=optional, + type=asset_type, + ) + + +def _property_for_asset_parameter_alias(cna_dataclass: Type[Asset], alias: str) -> str: + for property_name in cna_dataclass.__annotations__.keys(): + if alias in (getattr(cna_dataclass, property_name).alias, property_name): + return property_name + raise AssetError(f"Cannot find property for alias {alias} in {cna_dataclass}") diff --git a/reconcile/cna/assets/asset_factory.py b/reconcile/cna/assets/asset_factory.py index a6f238bd44..ec4451dfcc 100644 --- a/reconcile/cna/assets/asset_factory.py +++ b/reconcile/cna/assets/asset_factory.py @@ -1,4 +1,4 @@ -from typing import Any, Optional +from typing import Any, Optional, Type from collections.abc import Mapping from typing import Any @@ -8,33 +8,47 @@ ) from reconcile.cna.assets.null import NullAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( - CNANullAssetV1, - CNAAssumeRoleAssetV1, CNAssetV1, ) -from reconcile.cna.assets.null import NullAsset -from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset -from reconcile.cna.assets.asset import Asset, AssetError, AssetType +from reconcile.cna.assets.asset import ( + Asset, + AssetType, + UnknownAssetTypeError, + asset_type_from_raw_asset, +) + + +_ASSET_TYPE_SCHEME: dict[AssetType, Type[Asset]] = {} +_PROVIDER_SCHEME: dict[str, Type[Asset]] = {} + + +def register_asset_dataclass(asset_dataclass: Type[Asset]) -> None: + _ASSET_TYPE_SCHEME[asset_dataclass.asset_type()] = asset_dataclass + _PROVIDER_SCHEME[asset_dataclass.provider()] = asset_dataclass + + +def _dataclass_for_asset_type(asset_type: AssetType) -> Type[Asset]: + if asset_type in _ASSET_TYPE_SCHEME: + return _ASSET_TYPE_SCHEME[asset_type] + raise UnknownAssetTypeError(f"Unknown asset type {asset_type}") + + +def _dataclass_for_provider(provider: str) -> Type[Asset]: + return _PROVIDER_SCHEME[provider] + + +def asset_type_for_provider(provider: str) -> AssetType: + return _dataclass_for_provider(provider).asset_type() def asset_factory_from_schema(schema_asset: CNAssetV1) -> Asset: - if isinstance(schema_asset, CNANullAssetV1): - return NullAsset.from_query_class(schema_asset) - elif isinstance(schema_asset, CNAAssumeRoleAssetV1): - return AWSAssumeRoleAsset.from_query_class(schema_asset) - else: - raise AssetError(f"Unknown schema asset type {schema_asset}") - - -def asset_factory_from_raw_data(data_asset: Mapping[str, Any]) -> Optional[Asset]: - asset_type = data_asset.get("asset_type") - if asset_type == AssetType.NULL.value: - return NullAsset.from_api_mapping(data_asset) - elif asset_type == AssetType.EXAMPLE_AWS_ASSUMEROLE.value: - return AWSAssumeRoleAsset.from_api_mapping(data_asset) - else: - href = data_asset.get("href") - logging.warning( - f"Ignoring unknown data asset type '{asset_type}' - href: {href}" - ) - return None + cna_dataclass = _dataclass_for_provider(schema_asset.provider) + return cna_dataclass.from_query_class(schema_asset) + + +def asset_factory_from_raw_data(raw_asset: Mapping[str, Any]) -> Asset: + asset_type = asset_type_from_raw_asset(raw_asset) + if asset_type: + cna_dataclass = _dataclass_for_asset_type(asset_type) + return Asset.from_api_mapping(raw_asset, cna_dataclass) + raise UnknownAssetTypeError(f"Unknown asset type found in {raw_asset}") diff --git a/reconcile/cna/assets/aws_assume_role.py b/reconcile/cna/assets/aws_assume_role.py index 6ef0f7e139..c22181a3ff 100644 --- a/reconcile/cna/assets/aws_assume_role.py +++ b/reconcile/cna/assets/aws_assume_role.py @@ -1,65 +1,52 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Mapping - -from reconcile.cna.assets.asset import Asset, AssetError, AssetStatus, AssetType -from reconcile.gql_definitions.cna.queries.cna_resources import CNAAssumeRoleAssetV1 - - -@dataclass(frozen=True) +from pydantic.dataclasses import dataclass +from pydantic import Field +from typing import Optional + +from reconcile.cna.assets.asset import ( + Asset, + AssetError, + AssetType, + AssetStatus, + AssetModelConfig, +) +from reconcile.cna.assets.aws_utils import aws_role_arn_for_module +from reconcile.gql_definitions.cna.queries.cna_resources import ( + CNAAssumeRoleAssetV1, + CNAssetV1, +) + + +@dataclass(frozen=True, config=AssetModelConfig) class AWSAssumeRoleAsset(Asset): - slug: str - role_arn: str + verify_slug: Optional[str] = Field(None, alias="verify-slug") + role_arn: str = Field(alias="role_arn") - def api_payload(self) -> dict[str, Any]: - return { - "asset_type": AssetType.EXAMPLE_AWS_ASSUMEROLE, - "name": self.name, - "parameters": { - "slug": self.slug, - "role_arn": self.role_arn, - }, - } + @staticmethod + def provider() -> str: + return "aws-assume-role" - def update_from(self, asset: Asset) -> Asset: - if not isinstance(asset, AWSAssumeRoleAsset): - raise AssetError(f"Cannot create AWSAssumeRoleAsset from {asset}") - return AWSAssumeRoleAsset( - uuid=self.uuid, - href=self.href, - status=self.status, - name=self.name, - kind=self.kind, - slug=asset.slug, - role_arn=asset.role_arn, - ) + @staticmethod + def asset_type() -> AssetType: + return AssetType.EXAMPLE_AWS_ASSUMEROLE @staticmethod - def from_query_class(asset: CNAAssumeRoleAssetV1) -> AWSAssumeRoleAsset: - role_arn = asset.aws_assume_role.account.cna.default_role_arn - for module_config in asset.aws_assume_role.account.cna.module_role_arns: - if module_config.module == AssetType.EXAMPLE_AWS_ASSUMEROLE: - role_arn = module_config.role_arn - break + def from_query_class(asset: CNAssetV1) -> Asset: + assert isinstance(asset, CNAAssumeRoleAssetV1) + aws_cna_cfg = asset.aws_assume_role.account.cna + role_arn = aws_role_arn_for_module( + aws_cna_cfg, AssetType.EXAMPLE_AWS_ASSUMEROLE.value + ) + if role_arn is None: + raise AssetError( + f"No CNA roles configured for AWS account {asset.aws_assume_role.account.name}" + ) return AWSAssumeRoleAsset( - uuid=None, + id=None, href=None, - status=None, - kind=AssetType.EXAMPLE_AWS_ASSUMEROLE, + status=AssetStatus.UNKNOWN, name=asset.name, - slug=asset.aws_assume_role.slug, + verify_slug=asset.aws_assume_role.slug, role_arn=role_arn, ) - - @staticmethod - def from_api_mapping(asset: Mapping[str, Any]) -> AWSAssumeRoleAsset: - return AWSAssumeRoleAsset( - uuid=asset.get("id"), - href=asset.get("href"), - status=AssetStatus(asset.get("status")), - kind=AssetType.NULL, - name=asset.get("name", ""), - slug=asset.get("slug"), - role_arn=asset.get("role_arn"), - ) diff --git a/reconcile/cna/assets/aws_utils.py b/reconcile/cna/assets/aws_utils.py new file mode 100644 index 0000000000..f9a1cb3fb4 --- /dev/null +++ b/reconcile/cna/assets/aws_utils.py @@ -0,0 +1,15 @@ +from typing import Optional +from reconcile.gql_definitions.cna.queries.aws_account_fragment import CNAAWSSpecV1 + + +def aws_role_arn_for_module( + aws_cna_cfg: Optional[CNAAWSSpecV1], module: str +) -> Optional[str]: + if aws_cna_cfg is None: + return None + role_arn = aws_cna_cfg.default_role_arn + for module_config in aws_cna_cfg.module_role_arns or []: + if module_config.module == module: + role_arn = module_config.arn + break + return role_arn diff --git a/reconcile/cna/assets/null.py b/reconcile/cna/assets/null.py index 7bba6e352e..c6aefd1735 100644 --- a/reconcile/cna/assets/null.py +++ b/reconcile/cna/assets/null.py @@ -1,64 +1,41 @@ from __future__ import annotations - -from collections.abc import Mapping from dataclasses import dataclass -from typing import ( - Any, - Optional, -) +from typing import Any, Optional +from collections.abc import Mapping +from pydantic.dataclasses import dataclass +from pydantic import Field from reconcile.cna.assets.asset import ( Asset, - AssetError, - AssetStatus, AssetType, + AssetStatus, + AssetModelConfig, +) +from reconcile.gql_definitions.cna.queries.cna_resources import ( + CNANullAssetV1, + CNAssetV1, ) -from reconcile.gql_definitions.cna.queries.cna_resources import CNANullAssetV1 -@dataclass(frozen=True) +@dataclass(frozen=True, config=AssetModelConfig) class NullAsset(Asset): - addr_block: Optional[str] + addr_block: Optional[str] = Field(None, alias="AddrBlock") - def api_payload(self) -> dict[str, Any]: - return { - "asset_type": "null", - "name": self.name, - "parameters": { - "addr_block": self.addr_block, - }, - } + @staticmethod + def provider() -> str: + return "null-asset" - def update_from(self, asset: Asset) -> Asset: - if not isinstance(asset, NullAsset): - raise AssetError(f"Cannot create NullAsset from {asset}") - return NullAsset( - uuid=self.uuid, - href=self.href, - status=self.status, - name=self.name, - kind=self.kind, - addr_block=asset.addr_block, - ) + @staticmethod + def asset_type() -> AssetType: + return AssetType.NULL @staticmethod - def from_query_class(asset: CNANullAssetV1) -> NullAsset: + def from_query_class(asset: CNAssetV1) -> Asset: + assert isinstance(asset, CNANullAssetV1) return NullAsset( - uuid=None, + id=None, href=None, - status=None, - kind=AssetType.NULL, + status=AssetStatus.UNKNOWN, name=asset.name, addr_block=asset.addr_block, ) - - @staticmethod - def from_api_mapping(asset: Mapping[str, Any]) -> NullAsset: - return NullAsset( - uuid=asset.get("id"), - href=asset.get("href"), - status=AssetStatus(asset.get("status")), - kind=AssetType.NULL, - name=asset.get("name", ""), - addr_block=asset.get("addr_block"), - ) diff --git a/reconcile/cna/client.py b/reconcile/cna/client.py index 6913eff1d5..e5bc25fbb4 100644 --- a/reconcile/cna/client.py +++ b/reconcile/cna/client.py @@ -1,7 +1,13 @@ import logging from typing import Any - -from reconcile.cna.assets.asset import Asset +from reconcile.cna.assets.asset import ( + Asset, + AssetType, + AssetTypeMetadata, + AssetTypeVariable, + AssetTypeVariableType, + asset_type_by_id, +) from reconcile.utils.ocm_base_client import OCMBaseClient @@ -11,8 +17,46 @@ class CNAClient: https://gitlab.cee.redhat.com/service/cna-management/-/blob/main/openapi/openapi.yaml#/ """ - def __init__(self, ocm_client: OCMBaseClient): + def __init__(self, ocm_client: OCMBaseClient, init_metadata: bool = False): self._ocm_client = ocm_client + self._metadata = self._init_metadata() if init_metadata else None + + def _init_metadata(self) -> dict[AssetType, AssetTypeMetadata]: + asset_types_metadata: dict[AssetType, AssetTypeMetadata] = {} + for asset_type_ref in self._ocm_client.get( + api_path="/api/cna-management/v1/asset_types" + )["items"]: + raw_asset_type_metadata = self._ocm_client.get( + api_path=asset_type_ref["href"] + ) + asset_type = asset_type_by_id(raw_asset_type_metadata["id"]) + if asset_type: + asset_types_metadata[asset_type] = AssetTypeMetadata( + id=asset_type, + bindable=raw_asset_type_metadata.get("bindable", False), + variables={ + AssetTypeVariable( + name=var["name"], + optional=var.get("default") is not None, + type=AssetTypeVariableType(var["type"]), + default=var.get("default"), + ) + for var in raw_asset_type_metadata.get("variables", []) + }, + ) + + return asset_types_metadata + + def service_account_name(self) -> str: + account = self._ocm_client.get(api_path="/api/accounts_mgmt/v1/current_account") + return account["username"] + + def list_assets_for_creator(self, creator_username: str) -> list[dict[str, Any]]: + return [ + c + for c in self.list_assets() + if c.get("creator", {}).get("username") == creator_username + ] def list_assets(self) -> list[dict[str, Any]]: """ @@ -25,7 +69,12 @@ def list_assets(self) -> list[dict[str, Any]]: def create(self, asset: Asset, dry_run: bool = False): if dry_run: - logging.info("CREATE %s", asset) + logging.info( + "CREATE %s %s %s", + asset.asset_type().value, + asset.name, + asset.raw_asset_parameters(True), + ) return self._ocm_client.post( api_path="/api/cna-management/v1/cnas", diff --git a/reconcile/cna/integration.py b/reconcile/cna/integration.py index ddd5c4a459..1a6f3ad0b0 100644 --- a/reconcile/cna/integration.py +++ b/reconcile/cna/integration.py @@ -1,11 +1,6 @@ from collections import defaultdict -from collections.abc import ( - Iterable, - Mapping, -) -from typing import Optional +from collections.abc import Iterable, Mapping -from reconcile.cna.assets.asset_factory import asset_factory_from_schema from reconcile.cna.client import CNAClient from reconcile.cna.state import State from reconcile.gql_definitions.cna.queries.cna_provisioners import ( @@ -31,6 +26,12 @@ create_secret_reader, ) from reconcile.utils.semver_helper import make_semver +from reconcile.cna.assets.asset import UnknownAssetTypeError +from reconcile.cna.assets.asset_factory import ( + asset_factory_from_schema, + asset_factory_from_raw_data, +) + QONTRACT_INTEGRATION = "cna_resources" QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0) @@ -68,9 +69,18 @@ def assemble_desired_states(self): def assemble_current_states(self): self._current_states = defaultdict(State) for name, client in self._cna_clients.items(): - cnas = client.list_assets() state = State() - state.add_raw_data(cnas) + for raw_asset in client.list_assets_for_creator( + client.service_account_name() + ): + try: + state.add_asset( + asset_factory_from_raw_data( + raw_asset, + ) + ) + except UnknownAssetTypeError as e: + logging.warning(e) self._current_states[name] = state def provision(self, dry_run: bool = False): @@ -104,6 +114,7 @@ def build_cna_clients( secret_data = secret_reader.read_all_secret( provisioner.ocm.access_token_client_secret ) + # todo verify schema compatibility ocm_client = OCMBaseClient( url=provisioner.ocm.url, access_token_client_secret=secret_data["client_secret"], diff --git a/reconcile/cna/state.py b/reconcile/cna/state.py index 5b414b4ea0..58fed59118 100644 --- a/reconcile/cna/state.py +++ b/reconcile/cna/state.py @@ -1,20 +1,7 @@ from __future__ import annotations - -from collections.abc import ( - Iterable, - Mapping, -) -from typing import ( - Any, - Optional, -) - -from reconcile.cna.assets.asset import ( - Asset, - AssetStatus, - AssetType, -) -from reconcile.cna.assets.asset_factory import asset_factory_from_raw_data +from typing import Optional +from collections.abc import Iterable, Mapping +from reconcile.cna.assets.asset import Asset, AssetStatus, AssetType class CNAStateError(Exception): @@ -31,21 +18,27 @@ class State: def __init__(self, assets: Optional[dict[AssetType, dict[str, Asset]]] = None): self._assets: dict[AssetType, dict[str, Asset]] = {} - for kind in AssetType: - self._assets[kind] = {} if assets: self._assets = assets + for asset_type in AssetType: + if asset_type not in self._assets: + self._assets[asset_type] = {} def __eq__(self, other: object) -> bool: if not isinstance(other, State): return False if not set(list(self._assets.keys())) == set(list(other._assets.keys())): return False - for kind in list(self._assets.keys()): - if not set(list(self._assets[kind])) == set(list(other._assets[kind])): + for asset_type in list(self._assets.keys()): + if not set(list(self._assets[asset_type])) == set( + list(other._assets[asset_type]) + ): return False - for name, asset in self._assets[kind].items(): - if asset != other._assets[kind][name]: + for name, asset in self._assets[asset_type].items(): + if ( + asset.asset_properties() + != other._assets[asset_type][name].asset_properties() + ): return False return True @@ -54,23 +47,17 @@ def __repr__(self) -> str: return str(self._assets) def _validate_addition(self, asset: Asset): - if asset.kind not in self._assets: - raise CNAStateError(f"State doesn't know asset_kind {asset.kind}") - if asset.name in self._assets[asset.kind]: + asset_type = asset.asset_type() + if asset_type not in self._assets: + raise CNAStateError(f"State doesn't know asset_type {asset_type}") + if asset.name in self._assets[asset_type]: raise CNAStateError( - f"Duplicate asset name found in state: kind={asset.kind}, name={asset.name}" + f"Duplicate asset name found in state: asset_type={asset_type}, name={asset.name}" ) def add_asset(self, asset: Asset): self._validate_addition(asset=asset) - self._assets[asset.kind][asset.name] = asset - - def add_raw_data(self, data: Iterable[Mapping[str, Any]]): - for cna in data: - asset = asset_factory_from_raw_data(cna) - if asset: - self._validate_addition(asset=asset) - self._assets[asset.kind][asset.name] = asset + self._assets[asset.asset_type()][asset.name] = asset def required_updates_to_reach(self, other: State) -> State: """ @@ -81,14 +68,14 @@ def required_updates_to_reach(self, other: State) -> State: I.e., actual.required_updates_to_reach(desired) """ ans = State() - for kind in AssetType: - for asset_name, other_asset in other._assets[kind].items(): - if asset_name not in self._assets[kind]: + for asset_type in AssetType: + for asset_name, other_asset in other._assets[asset_type].items(): + if asset_name not in self._assets[asset_type]: continue - asset = self._assets[kind][asset_name] + asset = self._assets[asset_type][asset_name] if asset.status in (AssetStatus.TERMINATED, AssetStatus.PENDING): continue - if asset == other_asset: + if asset.asset_properties() == other_asset.asset_properties(): # There is no diff - no need to update continue ans.add_asset(asset=asset.update_from(other_asset)) @@ -105,11 +92,11 @@ def __sub__(self, other: State) -> State: deletions = other - self """ ans = State() - for kind in AssetType: - for asset_name, asset in self._assets[kind].items(): + for asset_type in AssetType: + for asset_name, asset in self._assets[asset_type].items(): if asset.status in (AssetStatus.TERMINATED, AssetStatus.PENDING): continue - if other_asset := other._assets[kind].get(asset_name): + if other_asset := other._assets[asset_type].get(asset_name): if other_asset.status == AssetStatus.TERMINATED: raise CNAStateError( f"Trying to create/update terminated asset {asset}. Currently not possible." @@ -121,8 +108,8 @@ def __sub__(self, other: State) -> State: def __iter__(self) -> State: self._i = 0 self._assets_list: list[Asset] = [] - for kind in AssetType: - self._assets_list += list(self._assets[kind].values()) + for asset_type in AssetType: + self._assets_list += list(self._assets[asset_type].values()) return self def __next__(self) -> Asset: diff --git a/reconcile/test/cna/test_asset.py b/reconcile/test/cna/test_asset.py new file mode 100644 index 0000000000..522419e7b9 --- /dev/null +++ b/reconcile/test/cna/test_asset.py @@ -0,0 +1,214 @@ +from typing import Any, Mapping +from reconcile.cna.assets.asset import ( + Asset, + AssetStatus, + AssetType, + AssetTypeMetadata, + AssetTypeVariable, + AssetTypeVariableType, + AssetError, + asset_type_metadata_from_asset_dataclass, +) +from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset + +import pytest + + +@pytest.fixture +def aws_assumerole_asset_type_metadata() -> AssetTypeMetadata: + return AssetTypeMetadata( + id=AssetType.EXAMPLE_AWS_ASSUMEROLE, + bindable=True, + variables={ + AssetTypeVariable( + name="role_arn", + type=AssetTypeVariableType.STRING, + ), + AssetTypeVariable( + name="verify-slug", + type=AssetTypeVariableType.STRING, + default="verify-slug", + ), + }, + ) + + +def raw_asset( + id: str, + name: str, + asset_type: AssetType, + status: AssetStatus, + parameters: dict[str, str], +) -> dict[str, Any]: + return { + "id": id, + "kind": "CNA", + "href": f"/api/cna-management/v1/cnas/{id}", + "asset_type": asset_type.value, + "name": name, + "status": status.value, + "parameters": parameters, + "creator": { + "name": "App SRE OCM bot", + "email": "sd-app-sre+ocm@redhat.com", + "username": "sd-app-sre-ocm-bot", + }, + "created_at": "2022-10-27T12:08:27.98559Z", + "updated_at": "2022-10-27T12:08:27.98559Z", + } + + +@pytest.fixture +def raw_aws_assumerole_asset() -> dict[str, Any]: + return raw_asset( + "123", + "test", + AssetType.EXAMPLE_AWS_ASSUMEROLE, + AssetStatus.READY, + {"role_arn": "1234", "verify-slug": "verify-slug"}, + ) + + +@pytest.fixture +def aws_assumerole_asset( + raw_aws_assumerole_asset: Mapping[str, Any], +) -> AWSAssumeRoleAsset: + asset = Asset.from_api_mapping( + raw_aws_assumerole_asset, + AWSAssumeRoleAsset, + ) + assert isinstance(asset, AWSAssumeRoleAsset) + return asset + + +def test_asset_type_extraction_from_raw(raw_aws_assumerole_asset: Mapping[str, Any]): + assert AssetType.EXAMPLE_AWS_ASSUMEROLE == Asset.asset_type_from_raw_asset( + raw_aws_assumerole_asset + ) + + +def test_from_api_mapping( + raw_aws_assumerole_asset: Mapping[str, Any], +): + asset = Asset.from_api_mapping(raw_aws_assumerole_asset, AWSAssumeRoleAsset) + assert isinstance(asset, AWSAssumeRoleAsset) + assert asset.id == raw_aws_assumerole_asset["id"] + assert asset.href == raw_aws_assumerole_asset["href"] + assert asset.name == raw_aws_assumerole_asset["name"] + assert asset.status == AssetStatus(raw_aws_assumerole_asset["status"]) + assert asset.role_arn == raw_aws_assumerole_asset["parameters"]["role_arn"] + assert asset.verify_slug == raw_aws_assumerole_asset["parameters"]["verify-slug"] + + +def test_from_api_mapping_required_parameter_missing( + raw_aws_assumerole_asset: Mapping[str, Any], +): + raw_aws_assumerole_asset["parameters"].pop("role_arn") + with pytest.raises(AssetError) as e: + Asset.from_api_mapping( + raw_aws_assumerole_asset, + AWSAssumeRoleAsset, + ) + assert str(e.value).startswith("Inconsistent asset from CNA API") + + +def test_api_payload(aws_assumerole_asset: AWSAssumeRoleAsset): + assert aws_assumerole_asset.api_payload() == { + "asset_type": aws_assumerole_asset.asset_type().value, + "name": aws_assumerole_asset.name, + "parameters": { + "role_arn": aws_assumerole_asset.role_arn, + "verify-slug": aws_assumerole_asset.verify_slug, + }, + } + + +def test_asset_type_metadata_from_asset_dataclass(): + expected = AssetTypeMetadata( + id=AssetType.EXAMPLE_AWS_ASSUMEROLE, + bindable=True, + variables={ + AssetTypeVariable( + name="role_arn", + type=AssetTypeVariableType.STRING, + ), + AssetTypeVariable( + name="verify-slug", type=AssetTypeVariableType.STRING, optional=True + ), + }, + ) + actual = asset_type_metadata_from_asset_dataclass(AWSAssumeRoleAsset) + assert expected == actual + + +def test_update_from(): + id = "1234" + name = "name" + href = "href" + status = AssetStatus.READY + + desired_asset = AWSAssumeRoleAsset( + id=None, + href=None, + status=None, + name=name, + role_arn="new_arn", + verify_slug="new_verify_slug", + ) + + current_asset = AWSAssumeRoleAsset( + id=id, + href=href, + name=name, + status=status, + role_arn="old_arn", + verify_slug="old_verify_slug", + ) + + update_asset = current_asset.update_from(desired_asset) + assert isinstance(update_asset, AWSAssumeRoleAsset) + assert update_asset.id == id + assert update_asset.href == href + assert update_asset.name == name + assert update_asset.status == status + assert update_asset.role_arn == "new_arn" + assert update_asset.verify_slug == "new_verify_slug" + + +def test_asset_comparion_ignorable_fields(): + name = "name" + arn = "arn" + verify_slug = "verify-slug" + asset_1 = AWSAssumeRoleAsset( + id="id", + href="href", + status=AssetStatus.READY, + name=name, + role_arn=arn, + verify_slug=verify_slug, + ) + + asset_2 = AWSAssumeRoleAsset( + id=None, + href=None, + status=AssetStatus.TERMINATED, + name=name, + role_arn=arn, + verify_slug=verify_slug, + ) + + assert asset_1.asset_properties() == asset_2.asset_properties() + + +def test_asset_properties_extration(): + AWSAssumeRoleAsset( + id=None, + href=None, + status=AssetStatus.TERMINATED, + name="name", + role_arn="arn", + verify_slug="slug", + ).asset_properties() == { + "role_arn": "arn", + "verify_slug": "slug", + } diff --git a/reconcile/test/cna/test_aws_assume_role.py b/reconcile/test/cna/test_aws_assume_role.py new file mode 100644 index 0000000000..d74a235f83 --- /dev/null +++ b/reconcile/test/cna/test_aws_assume_role.py @@ -0,0 +1,31 @@ +from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset +from reconcile.gql_definitions.cna.queries.cna_resources import ( + CNAAssumeRoleAssetV1, + CNAAssumeRoleAssetConfigV1, +) +from reconcile.gql_definitions.cna.queries.aws_account_fragment import ( + CNAAWSAccountRoleARNs, + CNAAWSSpecV1, +) + + +def test_from_query_class(): + name = "name" + slug = "slug" + arn = "arn" + query_asset = CNAAssumeRoleAssetV1( + provider=AWSAssumeRoleAsset.provider(), + name=name, + aws_assume_role=CNAAssumeRoleAssetConfigV1( + slug=slug, + account=CNAAWSAccountRoleARNs( + name="acc", + cna=CNAAWSSpecV1(defaultRoleARN=arn, moduleRoleARNS=None), + ), + ), + ) + asset = AWSAssumeRoleAsset.from_query_class(query_asset) + assert isinstance(asset, AWSAssumeRoleAsset) + assert asset.name == name + assert asset.verify_slug == slug + assert asset.role_arn == arn diff --git a/reconcile/test/cna/test_client.py b/reconcile/test/cna/test_client.py new file mode 100644 index 0000000000..0d1b1df123 --- /dev/null +++ b/reconcile/test/cna/test_client.py @@ -0,0 +1,45 @@ +from reconcile.cna.client import CNAClient + + +def test_client_asset_type_metadata_init(): + pass + + +def test_client_list_assets_for_creator(mocker): + creator = "creator" + listed_assets = [ + { + "asset_type": "null", + "id": "123", + "href": "url/123", + "status": "Running", + "creator": {"username": creator}, + }, + { + "asset_type": "null", + "id": "456", + "href": "url/456", + "status": "Running", + "creator": {}, + }, + { + "asset_type": "null", + "id": "789", + "href": "url/789", + "status": "Running", + }, + { + "asset_type": "null", + "id": "000", + "href": "url/000", + "status": "Running", + "creator": {"username": "another_user"}, + }, + ] + + mocker.patch.object(CNAClient, "list_assets", return_value=listed_assets) + cna_client = CNAClient(None) # type: ignore + + creator_assets = cna_client.list_assets_for_creator(creator) + for asset in creator_assets: + assert asset["creator"]["username"] == creator diff --git a/reconcile/test/cna/test_integration.py b/reconcile/test/cna/test_integration.py index 2b643ea00f..56d389b825 100644 --- a/reconcile/test/cna/test_integration.py +++ b/reconcile/test/cna/test_integration.py @@ -18,6 +18,11 @@ from reconcile.cna.assets.null import NullAsset from reconcile.cna.client import CNAClient from reconcile.cna.integration import CNAIntegration +from reconcile.cna.assets.asset import ( + AssetStatus, + AssetType, +) +from reconcile.cna.assets.null import NullAsset from reconcile.cna.state import State from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, @@ -52,11 +57,10 @@ def namespace(assets: list[CNANullAssetV1]) -> NamespaceV1: def null_asset(name: str, addr_block: Optional[str]) -> NullAsset: return NullAsset( - uuid=None, + id=None, href=None, status=None, name=name, - kind=AssetType.NULL, addr_block=addr_block, ) @@ -67,7 +71,7 @@ def null_asset(name: str, addr_block: Optional[str]) -> NullAsset: ( # Empty state [], - State(assets={AssetType.NULL: {}}), + State(), ), ( # Single asset @@ -78,16 +82,17 @@ def null_asset(name: str, addr_block: Optional[str]) -> NullAsset: "href": "url/123", "status": "Running", "name": "null-test", + "parameters": {}, + "creator": {"username": "creator"}, } ], State( assets={ AssetType.NULL: { "null-test": NullAsset( - uuid="123", + id="123", status=AssetStatus.RUNNING, name="null-test", - kind=AssetType.NULL, href="url/123", addr_block=None, ) @@ -104,6 +109,7 @@ def null_asset(name: str, addr_block: Optional[str]) -> NullAsset: "href": "url/123", "status": "Running", "name": "null-test", + "creator": {"username": "creator"}, }, { "asset_type": "null", @@ -111,24 +117,23 @@ def null_asset(name: str, addr_block: Optional[str]) -> NullAsset: "href": "url/456", "status": "Running", "name": "null-test2", + "creator": {"username": "creator"}, }, ], State( assets={ AssetType.NULL: { "null-test": NullAsset( - uuid="123", + id="123", status=AssetStatus.RUNNING, name="null-test", - kind=AssetType.NULL, href="url/123", addr_block=None, ), "null-test2": NullAsset( - uuid="456", + id="456", status=AssetStatus.RUNNING, name="null-test2", - kind=AssetType.NULL, href="url/456", addr_block=None, ), @@ -144,12 +149,17 @@ def null_asset(name: str, addr_block: Optional[str]) -> NullAsset: ], ) def test_integration_assemble_current_states( - cna_clients: Mapping[str, CNAClient], + mocker, listed_assets: Iterable[Mapping[str, Any]], expected_state: State, ): - cna_clients["test"].list_assets.side_effect = [listed_assets] # type: ignore - integration = CNAIntegration(cna_clients=cna_clients, namespaces=[]) + mocker.patch.object( + CNAClient, "list_assets", create_autospec=True, return_value=listed_assets + ) + mocker.patch.object( + CNAClient, "service_account_name", create_autospec=True, return_value="creator" + ) + integration = CNAIntegration(cna_clients={"test": CNAClient(None)}, namespaces=[]) integration.assemble_current_states() assert integration._current_states == {"test": expected_state} @@ -210,8 +220,10 @@ def test_integration_assemble_current_states( ], ) def test_integration_assemble_desired_states( - namespaces: list[NamespaceV1], expected_state: State + cna_clients: Mapping[str, CNAClient], + namespaces: list[NamespaceV1], + expected_state: State, ): - integration = CNAIntegration(cna_clients={}, namespaces=namespaces) + integration = CNAIntegration(cna_clients=cna_clients, namespaces=namespaces) integration.assemble_desired_states() assert integration._desired_states == {"test": expected_state} diff --git a/reconcile/test/cna/test_null.py b/reconcile/test/cna/test_null.py new file mode 100644 index 0000000000..f83975a839 --- /dev/null +++ b/reconcile/test/cna/test_null.py @@ -0,0 +1,16 @@ +from reconcile.cna.assets.null import NullAsset +from reconcile.gql_definitions.cna.queries.cna_resources import ( + CNANullAssetV1, +) + + +def test_from_query_class(): + name = "name" + addr_block = "addr_block" + query_asset = CNANullAssetV1( + provider=NullAsset.provider(), name=name, addr_block=addr_block + ) + asset = NullAsset.from_query_class(query_asset) + assert isinstance(asset, NullAsset) + assert asset.name == name + assert asset.addr_block == addr_block diff --git a/reconcile/test/cna/test_state_assembly.py b/reconcile/test/cna/test_state_assembly.py index 6f78c0e383..0daf169317 100644 --- a/reconcile/test/cna/test_state_assembly.py +++ b/reconcile/test/cna/test_state_assembly.py @@ -10,6 +10,7 @@ AssetStatus, AssetType, ) +from reconcile.cna.assets.asset_factory import asset_factory_from_raw_data from reconcile.cna.assets.null import NullAsset from reconcile.cna.state import ( CNAStateError, @@ -19,9 +20,8 @@ def null_asset(name: str, addr_block: Optional[str] = None) -> NullAsset: return NullAsset( - uuid=None, + id=None, href=None, - kind=AssetType.NULL, status=AssetStatus.RUNNING, name=name, addr_block=addr_block, @@ -40,7 +40,6 @@ def test_assemble_state_with_assets(): } for asset in list(assets.values()): state.add_asset(asset) - state.add_raw_data([]) assert state == State( assets=cast(dict[AssetType, dict[str, Asset]], {AssetType.NULL: assets}) @@ -66,13 +65,19 @@ def test_assemble_state_raw_data(): "addr_block": "1234", }, ] - assets = { + assets: dict[AssetType, dict[str, Asset]] = { AssetType.NULL: { - asset.get("name", ""): NullAsset.from_api_mapping(asset) for asset in data + raw_asset.get("name", ""): Asset.from_api_mapping( + raw_asset, + NullAsset, + ) + for raw_asset in data } } - state.add_raw_data(data) + for raw_asset in data: + state.add_asset(asset_factory_from_raw_data(raw_asset)) + # todo check if cast is needed assert state == State(assets=cast(dict[AssetType, dict[str, Asset]], assets)) @@ -88,5 +93,5 @@ def test_assemble_raises_duplicate_error(): assert ( str(err.value) - == "Duplicate asset name found in state: kind=AssetType.NULL, name=test" + == "Duplicate asset name found in state: asset_type=null, name=test" ) diff --git a/reconcile/test/cna/test_state_diff.py b/reconcile/test/cna/test_state_diff.py index 7657e6d262..dba096ea67 100644 --- a/reconcile/test/cna/test_state_diff.py +++ b/reconcile/test/cna/test_state_diff.py @@ -1,4 +1,10 @@ from typing import Optional +import pytest +from reconcile.cna.assets.asset import ( + AssetStatus, + AssetType, +) +from reconcile.cna.assets.null import NullAsset import pytest @@ -17,20 +23,19 @@ def null_asset( name: str, status: Optional[AssetStatus] = None, addr_block: Optional[str] = None, - uuid: Optional[str] = None, + id: Optional[str] = None, ) -> NullAsset: return NullAsset( - uuid=uuid, + id=id, href=None, status=status, - kind=AssetType.NULL, addr_block=addr_block, name=name, ) @pytest.mark.parametrize( - "desired, actual, expected_additions, expected_deletions, expected_updates", + "desired, current, expected_additions, expected_deletions, expected_updates", [ ( # Empty states @@ -68,20 +73,12 @@ def null_asset( # Do not add/update already existing resource State( assets={ - AssetType.NULL: { - "test": null_asset( - name="test", - ) - } + AssetType.NULL: {"test": null_asset(name="test", addr_block="addr")} } ), State( assets={ - AssetType.NULL: { - "test": null_asset( - name="test", - ) - } + AssetType.NULL: {"test": null_asset(name="test", addr_block="addr")} } ), State(assets={}), @@ -154,7 +151,7 @@ def null_asset( ), ( # Delete, create and update resources - State( + State( # desired assets={ AssetType.NULL: { "test1": null_asset( @@ -167,20 +164,21 @@ def null_asset( } } ), - State( + State( # current assets={ AssetType.NULL: { "test": null_asset( name="test", + id="test", ), "test1": null_asset( name="test1", - uuid="123", + id="test1", ), } } ), - State( + State( # expected additions assets={ AssetType.NULL: { "test2": null_asset( @@ -189,22 +187,23 @@ def null_asset( } } ), - State( + State( # expected deletions assets={ AssetType.NULL: { "test": null_asset( name="test", + id="test", ) } } ), - State( + State( # expected updates assets={ AssetType.NULL: { "test1": null_asset( name="test1", addr_block="123", - uuid="123", + id="test1", ) } } @@ -253,14 +252,14 @@ def null_asset( ) def test_state_create_delete_update( desired: State, - actual: State, + current: State, expected_additions: State, expected_deletions: State, expected_updates: State, ): - additions = desired - actual - deletions = actual - desired - updates = actual.required_updates_to_reach(desired) + additions = desired - current + deletions = current - desired + updates = current.required_updates_to_reach(desired) assert additions == expected_additions assert deletions == expected_deletions assert updates == expected_updates @@ -271,7 +270,7 @@ def test_state_create_delete_update( Currently CNA does not support addressing w/o use of internal uuid. """ - assert update.uuid == expected_update.uuid + assert update.id == expected_update.id def test_state_create_update_terminated(): @@ -293,6 +292,7 @@ def test_state_create_update_terminated(): AssetType.NULL: { "test": null_asset( name="test", + id="test", status=AssetStatus.TERMINATED, ) } diff --git a/reconcile/test/cna/test_state_overrides.py b/reconcile/test/cna/test_state_overrides.py index bf155c2678..9f3d716f25 100644 --- a/reconcile/test/cna/test_state_overrides.py +++ b/reconcile/test/cna/test_state_overrides.py @@ -13,14 +13,13 @@ def null_asset( name: str, href: Optional[str] = None, - uuid: Optional[str] = None, + id: Optional[str] = None, addr_block: Optional[str] = None, status: Optional[AssetStatus] = None, ) -> NullAsset: return NullAsset( - uuid=uuid, + id=id, href=href, - kind=AssetType.NULL, status=status, name=name, addr_block=addr_block, @@ -65,7 +64,7 @@ def null_asset( ), ), ( - # uuid and href do not count towards equality + # id and href do not count towards equality State( assets={ AssetType.NULL: { @@ -87,7 +86,7 @@ def null_asset( ), "test2": null_asset( name="test2", - uuid="123", + id="123", href="/123", ), } @@ -98,7 +97,7 @@ def null_asset( ids=[ "Empty states are equal", "Status does not count towards equality", - "uuid and href do not count towards equality", + "id and href do not count towards equality", ], ) def test_state_eq(a: State, b: State): @@ -153,7 +152,7 @@ def test_state_eq(a: State, b: State): AssetType.NULL: { "test2": null_asset( name="test2", - uuid="123", + id="123", href="/123", ), } From ed55a3661460781d0561feecaa74ef690693b954 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Mon, 14 Nov 2022 13:01:02 +0100 Subject: [PATCH 03/20] [CNA] add support for aws-rds CNA module (#2936) [CNA] add support for aws-rds CNA module --- reconcile/cna/assets/__init__.py | 2 + reconcile/cna/assets/asset.py | 88 ++++++-- reconcile/cna/assets/asset_factory.py | 28 ++- reconcile/cna/assets/aws_assume_role.py | 14 +- reconcile/cna/assets/aws_rds.py | 90 ++++++++ reconcile/cna/assets/null.py | 10 +- reconcile/cna/client.py | 3 +- reconcile/cna/integration.py | 27 ++- .../cna/queries/cna_resources.gql | 40 +++- .../cna/queries/cna_resources.py | 111 ++++++++-- .../fragments/resource_file.gql | 6 + .../fragments/resource_file.py | 26 +++ reconcile/test/cna/test_asset.py | 114 +++++++++- reconcile/test/cna/test_aws_assume_role.py | 15 +- reconcile/test/cna/test_aws_rds.py | 94 ++++++++ reconcile/test/cna/test_integration.py | 23 +- reconcile/test/cna/test_null.py | 20 +- .../test/test_utils_external_resources.py | 165 +++++++++++++- reconcile/utils/external_resource_spec.py | 202 +++++++++++++++++- reconcile/utils/external_resources.py | 52 ++++- 20 files changed, 1010 insertions(+), 120 deletions(-) create mode 100644 reconcile/cna/assets/aws_rds.py create mode 100644 reconcile/gql_definitions/fragments/resource_file.gql create mode 100644 reconcile/gql_definitions/fragments/resource_file.py create mode 100644 reconcile/test/cna/test_aws_rds.py diff --git a/reconcile/cna/assets/__init__.py b/reconcile/cna/assets/__init__.py index b5e8a75044..b4c5fbce57 100644 --- a/reconcile/cna/assets/__init__.py +++ b/reconcile/cna/assets/__init__.py @@ -1,7 +1,9 @@ from reconcile.cna.assets.asset_factory import register_asset_dataclass from reconcile.cna.assets.null import NullAsset from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset +from reconcile.cna.assets.aws_rds import AWSRDSAsset register_asset_dataclass(NullAsset) register_asset_dataclass(AWSAssumeRoleAsset) +register_asset_dataclass(AWSRDSAsset) diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index 7630e6a2b0..958e1d8b97 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -4,13 +4,21 @@ from pydantic.dataclasses import dataclass from pydantic.fields import FieldInfo from enum import Enum -from typing import Any, Mapping, Optional, Type +from typing import Any, Generic, Mapping, Optional, Type, TypeVar, get_args +import copy from reconcile.gql_definitions.cna.queries.cna_resources import CNAssetV1 +from reconcile.utils.external_resource_spec import TypedExternalResourceSpec +ASSET_ID_FIELD = "id" ASSET_TYPE_FIELD = "asset_type" +ASSET_NAME_FIELD = "name" +ASSET_HREF_FIELD = "href" +ASSET_STATUS_FIELD = "status" ASSET_PARAMETERS_FIELD = "parameters" +ASSET_OUTPUTS_FIELD = "outputs" +ASSET_CREATOR_FIELD = "creator" class AssetError(Exception): @@ -24,6 +32,7 @@ class UnknownAssetTypeError(Exception): class AssetType(str, Enum): NULL = "null" EXAMPLE_AWS_ASSUMEROLE = "example-aws-assumerole" + AWS_RDS = "aws-rds" def asset_type_by_id(asset_type_id: str) -> Optional[AssetType]: @@ -33,8 +42,16 @@ def asset_type_by_id(asset_type_id: str) -> Optional[AssetType]: return None -def asset_type_from_raw_asset(raw_assset: Mapping[str, Any]) -> Optional[AssetType]: - return asset_type_by_id(raw_assset.get(ASSET_TYPE_FIELD, "")) +def asset_type_id_from_raw_asset(raw_asset: Mapping[str, Any]) -> Optional[str]: + return raw_asset.get(ASSET_TYPE_FIELD) + + +def asset_type_from_raw_asset(raw_asset: Mapping[str, Any]) -> Optional[AssetType]: + asset_type_id = asset_type_id_from_raw_asset(raw_asset) + if asset_type_id: + return asset_type_by_id(asset_type_id) + else: + return None class AssetTypeVariableType(Enum): @@ -69,11 +86,13 @@ class AssetStatus(Enum): class AssetModelConfig: allow_population_by_field_name = True - extra = "forbid" + + +AssetQueryClass = TypeVar("AssetQueryClass", bound=CNAssetV1) @dataclass(frozen=True, config=AssetModelConfig) -class Asset(ABC): +class Asset(ABC, Generic[AssetQueryClass]): name: str id: Optional[str] href: Optional[str] @@ -99,27 +118,37 @@ def provider() -> str: @staticmethod @abstractmethod - def from_query_class(asset: CNAssetV1) -> Asset: + def from_query_class(asset: AssetQueryClass) -> Asset: ... - @staticmethod - def asset_type_from_raw_asset(raw_asset: Mapping[str, Any]) -> Optional[AssetType]: - asset_type_value = raw_asset[ASSET_TYPE_FIELD] - return asset_type_by_id(asset_type_value) + @classmethod + def from_external_resources( + cls, + external_resource: TypedExternalResourceSpec[CNAssetV1], + ) -> Asset: + cls_arg = get_args(cls.__orig_bases__[0])[0] # type: ignore[attr-defined] + resolved = external_resource.resolve() + if isinstance(resolved.spec, cls_arg): + return cls.from_query_class(resolved.spec) + else: + raise AssetError( + f"CNA type {cls_arg} does not match " + f"external resource type {type(external_resource)}" + ) def asset_metadata(self) -> dict[str, Any]: return { - "id": self.id, - "href": self.href, - "status": self.status.value if self.status else None, - "name": self.name, + ASSET_ID_FIELD: self.id, + ASSET_HREF_FIELD: self.href, + ASSET_STATUS_FIELD: self.status.value if self.status else None, + ASSET_NAME_FIELD: self.name, ASSET_TYPE_FIELD: self.asset_type().value, } def api_payload(self) -> dict[str, Any]: return { ASSET_TYPE_FIELD: self.asset_type().value, - "name": self.name, + ASSET_NAME_FIELD: self.name, ASSET_PARAMETERS_FIELD: self.raw_asset_parameters(omit_empty=False), } @@ -160,21 +189,34 @@ def from_api_mapping( cna_dataclass: Type[Asset], ) -> Asset: params = {} + inconsistency_errors = [] raw_asset_params = raw_asset.get(ASSET_PARAMETERS_FIELD) or {} for var in cna_dataclass.type_metadata().variables: var_value = raw_asset_params.get(var.name) if not var.optional and not var_value: - raise AssetError( - f"Inconsistent asset from CNA API {raw_asset}: required parameter {var.name} is missing in CNA" + inconsistency_errors.append( + f" - required parameter {var.name} is missing" + ) + else: + property_name = _property_for_asset_parameter_alias( + cna_dataclass, var.name ) - property_name = _property_for_asset_parameter_alias(cna_dataclass, var.name) - params[property_name] = var_value + params[property_name] = var_value + + if inconsistency_errors: + errors = "\n".join(inconsistency_errors) + redacted_raw_asset = dict(copy.deepcopy(raw_asset)) + redacted_raw_asset.pop(ASSET_OUTPUTS_FIELD, None) + redacted_raw_asset.pop(ASSET_CREATOR_FIELD, None) + raise AssetError( + f"Inconsistent asset {redacted_raw_asset} found on CNA:\n{errors}" + ) return cna_dataclass( - id=raw_asset.get("id"), - href=raw_asset.get("href"), - status=AssetStatus(raw_asset.get("status")), - name=raw_asset.get("name", ""), + id=raw_asset.get(ASSET_ID_FIELD), + href=raw_asset.get(ASSET_HREF_FIELD), + status=AssetStatus(raw_asset.get(ASSET_STATUS_FIELD)), + name=raw_asset.get(ASSET_NAME_FIELD, ""), **params, ) diff --git a/reconcile/cna/assets/asset_factory.py b/reconcile/cna/assets/asset_factory.py index ec4451dfcc..90301dd4be 100644 --- a/reconcile/cna/assets/asset_factory.py +++ b/reconcile/cna/assets/asset_factory.py @@ -14,8 +14,12 @@ Asset, AssetType, UnknownAssetTypeError, - asset_type_from_raw_asset, + asset_type_id_from_raw_asset, + asset_type_by_id, + ASSET_HREF_FIELD, + ASSET_NAME_FIELD, ) +from reconcile.utils.external_resource_spec import TypedExternalResourceSpec _ASSET_TYPE_SCHEME: dict[AssetType, Type[Asset]] = {} @@ -41,14 +45,20 @@ def asset_type_for_provider(provider: str) -> AssetType: return _dataclass_for_provider(provider).asset_type() -def asset_factory_from_schema(schema_asset: CNAssetV1) -> Asset: - cna_dataclass = _dataclass_for_provider(schema_asset.provider) - return cna_dataclass.from_query_class(schema_asset) +def asset_factory_from_schema( + external_resource_spec: TypedExternalResourceSpec[CNAssetV1], +) -> Asset: + cna_dataclass = _dataclass_for_provider(external_resource_spec.provider) + return cna_dataclass.from_external_resources(external_resource_spec) def asset_factory_from_raw_data(raw_asset: Mapping[str, Any]) -> Asset: - asset_type = asset_type_from_raw_asset(raw_asset) - if asset_type: - cna_dataclass = _dataclass_for_asset_type(asset_type) - return Asset.from_api_mapping(raw_asset, cna_dataclass) - raise UnknownAssetTypeError(f"Unknown asset type found in {raw_asset}") + asset_type_id = asset_type_id_from_raw_asset(raw_asset) + if asset_type_id: + asset_type = asset_type_by_id(asset_type_id) + if asset_type: + cna_dataclass = _dataclass_for_asset_type(asset_type) + return Asset.from_api_mapping(raw_asset, cna_dataclass) + raise UnknownAssetTypeError( + f"Unknown asset type {asset_type_id} found in {raw_asset.get(ASSET_NAME_FIELD, '')} - {raw_asset.get(ASSET_HREF_FIELD, '')}" + ) diff --git a/reconcile/cna/assets/aws_assume_role.py b/reconcile/cna/assets/aws_assume_role.py index c22181a3ff..46bfca9355 100644 --- a/reconcile/cna/assets/aws_assume_role.py +++ b/reconcile/cna/assets/aws_assume_role.py @@ -13,12 +13,11 @@ from reconcile.cna.assets.aws_utils import aws_role_arn_for_module from reconcile.gql_definitions.cna.queries.cna_resources import ( CNAAssumeRoleAssetV1, - CNAssetV1, ) @dataclass(frozen=True, config=AssetModelConfig) -class AWSAssumeRoleAsset(Asset): +class AWSAssumeRoleAsset(Asset[CNAAssumeRoleAssetV1]): verify_slug: Optional[str] = Field(None, alias="verify-slug") role_arn: str = Field(alias="role_arn") @@ -31,22 +30,21 @@ def asset_type() -> AssetType: return AssetType.EXAMPLE_AWS_ASSUMEROLE @staticmethod - def from_query_class(asset: CNAssetV1) -> Asset: - assert isinstance(asset, CNAAssumeRoleAssetV1) - aws_cna_cfg = asset.aws_assume_role.account.cna + def from_query_class(asset: CNAAssumeRoleAssetV1) -> Asset: + aws_cna_cfg = asset.account.cna role_arn = aws_role_arn_for_module( aws_cna_cfg, AssetType.EXAMPLE_AWS_ASSUMEROLE.value ) if role_arn is None: raise AssetError( - f"No CNA roles configured for AWS account {asset.aws_assume_role.account.name}" + f"No CNA roles configured for AWS account {asset.account.name}" ) return AWSAssumeRoleAsset( id=None, href=None, status=AssetStatus.UNKNOWN, - name=asset.name, - verify_slug=asset.aws_assume_role.slug, + name=asset.identifier, + verify_slug=asset.overrides.slug if asset.overrides else None, role_arn=role_arn, ) diff --git a/reconcile/cna/assets/aws_rds.py b/reconcile/cna/assets/aws_rds.py new file mode 100644 index 0000000000..63f23ed0ac --- /dev/null +++ b/reconcile/cna/assets/aws_rds.py @@ -0,0 +1,90 @@ +from __future__ import annotations +from typing import Optional +from pydantic.dataclasses import dataclass +from pydantic import Field + +from reconcile.cna.assets.asset import ( + Asset, + AssetError, + AssetType, + AssetModelConfig, + AssetStatus, +) +from reconcile.cna.assets.aws_utils import aws_role_arn_for_module +from reconcile.gql_definitions.cna.queries.cna_resources import ( + CNARDSInstanceV1, +) + + +@dataclass(frozen=True, config=AssetModelConfig) +class AWSRDSAsset(Asset[CNARDSInstanceV1]): + identifier: str = Field(alias="identifier") + vpc_id: str = Field(alias="vpc_id") + role_arn: str = Field(alias="role_arn") + db_subnet_group_name: str = Field(alias="db_subnet_group_name") + instance_class: str = Field(alias="instance_class") + allocated_storage: str = Field(alias="allocated_storage") + max_allocated_storage: str = Field(alias="max_allocated_storage") + engine: Optional[str] = Field(None, alias="engine") + engine_version: Optional[str] = Field(None, alias="engine_version") + region: Optional[str] = Field(None, alias="region") + backup_retention_period: Optional[int] = Field( + None, alias="backup_retention_period" + ) + backup_window: Optional[str] = Field(None, alias="backup_window") + maintenance_window: Optional[str] = Field(None, alias="maintenance_window") + + @staticmethod + def provider() -> str: + return "aws-rds" + + @staticmethod + def asset_type() -> AssetType: + return AssetType.AWS_RDS + + @staticmethod + def from_query_class(asset: CNARDSInstanceV1) -> Asset: + aws_cna_cfg = asset.vpc.account.cna + role_arn = aws_role_arn_for_module(aws_cna_cfg, AssetType.AWS_RDS.value) + if role_arn is None: + raise AssetError( + f"No CNA roles configured for AWS account {asset.vpc.account.name}" + ) + + if not asset.overrides: + raise AssetError("No overrides provided for RDS instance") + + if not (db_subnet_group_name := asset.overrides.db_subnet_group_name): + raise AssetError( + f"No db_subnet_group_name provided for RDS instance {asset.identifier}" + ) + if not (instance_class := asset.overrides.instance_class): + raise AssetError( + f"No instance_class provided for RDS instance {asset.identifier}" + ) + if not (engine := asset.overrides.engine): + raise AssetError(f"No engine provided for RDS instance {asset.identifier}") + if not (max_allocated_storage := asset.overrides.max_allocated_storage): + raise AssetError( + f"No max_allocated_storage provided for RDS instance {asset.identifier}" + ) + + return AWSRDSAsset( + id=None, + href=None, + status=AssetStatus.UNKNOWN, + name=asset.identifier, + identifier=asset.overrides.name or asset.identifier, + vpc_id=asset.vpc.vpc_id, + role_arn=role_arn, + db_subnet_group_name=db_subnet_group_name, + engine=engine, + engine_version=asset.overrides.engine_version, + instance_class=instance_class, + allocated_storage=str(asset.overrides.allocated_storage), + max_allocated_storage=str(max_allocated_storage), + region=asset.vpc.region, + backup_retention_period=asset.overrides.backup_retention_period, + backup_window=None, + maintenance_window=None, + ) diff --git a/reconcile/cna/assets/null.py b/reconcile/cna/assets/null.py index c6aefd1735..d476d6380b 100644 --- a/reconcile/cna/assets/null.py +++ b/reconcile/cna/assets/null.py @@ -13,12 +13,11 @@ ) from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, - CNAssetV1, ) @dataclass(frozen=True, config=AssetModelConfig) -class NullAsset(Asset): +class NullAsset(Asset[CNANullAssetV1]): addr_block: Optional[str] = Field(None, alias="AddrBlock") @staticmethod @@ -30,12 +29,11 @@ def asset_type() -> AssetType: return AssetType.NULL @staticmethod - def from_query_class(asset: CNAssetV1) -> Asset: - assert isinstance(asset, CNANullAssetV1) + def from_query_class(asset: CNANullAssetV1) -> Asset: return NullAsset( id=None, href=None, status=AssetStatus.UNKNOWN, - name=asset.name, - addr_block=asset.addr_block, + name=asset.identifier, + addr_block=asset.overrides.addr_block if asset.overrides else None, ) diff --git a/reconcile/cna/client.py b/reconcile/cna/client.py index e5bc25fbb4..0e3d74812a 100644 --- a/reconcile/cna/client.py +++ b/reconcile/cna/client.py @@ -7,6 +7,7 @@ AssetTypeVariable, AssetTypeVariableType, asset_type_by_id, + ASSET_CREATOR_FIELD, ) from reconcile.utils.ocm_base_client import OCMBaseClient @@ -55,7 +56,7 @@ def list_assets_for_creator(self, creator_username: str) -> list[dict[str, Any]] return [ c for c in self.list_assets() - if c.get("creator", {}).get("username") == creator_username + if c.get(ASSET_CREATOR_FIELD, {}).get("username") == creator_username ] def list_assets(self) -> list[dict[str, Any]]: diff --git a/reconcile/cna/integration.py b/reconcile/cna/integration.py index 1a6f3ad0b0..b0d9f60a67 100644 --- a/reconcile/cna/integration.py +++ b/reconcile/cna/integration.py @@ -3,6 +3,13 @@ from reconcile.cna.client import CNAClient from reconcile.cna.state import State + +from reconcile.utils import gql +from reconcile.utils.external_resources import ( + get_external_resource_specs_for_namespace, + PROVIDER_CNA_EXPERIMENTAL, +) +from reconcile.utils.ocm_base_client import OCMBaseClient from reconcile.gql_definitions.cna.queries.cna_provisioners import ( CNAExperimentalProvisionerV1, ) @@ -10,7 +17,7 @@ query as cna_provisioners_query, ) from reconcile.gql_definitions.cna.queries.cna_resources import ( - NamespaceCNAssetV1, + CNAssetV1, NamespaceV1, ) from reconcile.gql_definitions.cna.queries.cna_resources import ( @@ -26,7 +33,7 @@ create_secret_reader, ) from reconcile.utils.semver_helper import make_semver -from reconcile.cna.assets.asset import UnknownAssetTypeError +from reconcile.cna.assets.asset import UnknownAssetTypeError, AssetError from reconcile.cna.assets.asset_factory import ( asset_factory_from_schema, asset_factory_from_raw_data, @@ -57,14 +64,11 @@ def __init__( def assemble_desired_states(self): self._desired_states = defaultdict(State) for namespace in self._namespaces: - for provider in namespace.external_resources or []: - # TODO: this should probably be filtered within the query already - if not isinstance(provider, NamespaceCNAssetV1): - continue - for resource in provider.resources or []: - self._desired_states[provider.provisioner.name].add_asset( - asset_factory_from_schema(resource) - ) + for spec in get_external_resource_specs_for_namespace( + namespace, CNAssetV1, PROVIDER_CNA_EXPERIMENTAL + ): + asset = asset_factory_from_schema(spec) + self._desired_states[spec.provisioner_name].add_asset(asset) def assemble_current_states(self): self._current_states = defaultdict(State) @@ -81,6 +85,9 @@ def assemble_current_states(self): ) except UnknownAssetTypeError as e: logging.warning(e) + except AssetError as e: + # TODO: remember this somehow in the state so we don't try to update/create this asset but skip it instead + logging.error(e) self._current_states[name] = state def provision(self, dry_run: bool = False): diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.gql b/reconcile/gql_definitions/cna/queries/cna_resources.gql index b56d31a890..934c79780d 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.gql +++ b/reconcile/gql_definitions/cna/queries/cna_resources.gql @@ -3,6 +3,7 @@ query CNAssets { namespaces: namespaces_v1 { name + managedExternalResources externalResources { provider provisioner { @@ -11,18 +12,45 @@ query CNAssets { ... on NamespaceCNAsset_v1 { resources { provider + identifier ... on CNANullAsset_v1 { - name: identifier - addr_block + overrides { + addr_block + } } - ... on CNAAssumeRoleAsset_v1{ - name: identifier - aws_assume_role { - slug + ... on CNARDSInstance_v1 { + vpc { + vpc_id + region account { ... CNAAWSAccountRoleARNs } } + defaults { + ... ResourceFile + } + overrides { + name + engine + engine_version + username + instance_class + allocated_storage + max_allocated_storage + backup_retention_period + db_subnet_group_name + } + } + ... on CNAAssumeRoleAsset_v1{ + account { + ... CNAAWSAccountRoleARNs + } + overrides { + slug + } + defaults { + ... ResourceFile + } } } } diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.py b/reconcile/gql_definitions/cna/queries/cna_resources.py index 5dee1ef8cd..a93fbde04f 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.py +++ b/reconcile/gql_definitions/cna/queries/cna_resources.py @@ -19,6 +19,7 @@ from reconcile.gql_definitions.cna.queries.aws_account_fragment import ( CNAAWSAccountRoleARNs, ) +from reconcile.gql_definitions.fragments.resource_file import ResourceFile DEFINITION = """ @@ -33,9 +34,15 @@ } } +fragment ResourceFile on Resource_v1 { + resourceFileSchema: schema + content +} + query CNAssets { namespaces: namespaces_v1 { name + managedExternalResources externalResources { provider provisioner { @@ -44,18 +51,45 @@ ... on NamespaceCNAsset_v1 { resources { provider + identifier ... on CNANullAsset_v1 { - name: identifier - addr_block + overrides { + addr_block + } } - ... on CNAAssumeRoleAsset_v1{ - name: identifier - aws_assume_role { - slug + ... on CNARDSInstance_v1 { + vpc { + vpc_id + region account { ... CNAAWSAccountRoleARNs } } + defaults { + ... ResourceFile + } + overrides { + name + engine + engine_version + username + instance_class + allocated_storage + max_allocated_storage + backup_retention_period + db_subnet_group_name + } + } + ... on CNAAssumeRoleAsset_v1{ + account { + ... CNAAWSAccountRoleARNs + } + overrides { + slug + } + defaults { + ... ResourceFile + } } } } @@ -84,14 +118,14 @@ class Config: class CNAssetV1(BaseModel): provider: str = Field(..., alias="provider") + identifier: str = Field(..., alias="identifier") class Config: smart_union = True extra = Extra.forbid -class CNANullAssetV1(CNAssetV1): - name: str = Field(..., alias="name") +class CNANullAssetOverridesV1(BaseModel): addr_block: Optional[str] = Field(..., alias="addr_block") class Config: @@ -99,8 +133,17 @@ class Config: extra = Extra.forbid -class CNAAssumeRoleAssetConfigV1(BaseModel): - slug: str = Field(..., alias="slug") +class CNANullAssetV1(CNAssetV1): + overrides: Optional[CNANullAssetOverridesV1] = Field(..., alias="overrides") + + class Config: + smart_union = True + extra = Extra.forbid + + +class AWSVPCV1(BaseModel): + vpc_id: str = Field(..., alias="vpc_id") + region: str = Field(..., alias="region") account: CNAAWSAccountRoleARNs = Field(..., alias="account") class Config: @@ -108,9 +151,44 @@ class Config: extra = Extra.forbid +class CNARDSInstanceOverridesV1(BaseModel): + name: Optional[str] = Field(..., alias="name") + engine: Optional[str] = Field(..., alias="engine") + engine_version: Optional[str] = Field(..., alias="engine_version") + username: Optional[str] = Field(..., alias="username") + instance_class: Optional[str] = Field(..., alias="instance_class") + allocated_storage: Optional[int] = Field(..., alias="allocated_storage") + max_allocated_storage: Optional[int] = Field(..., alias="max_allocated_storage") + backup_retention_period: Optional[int] = Field(..., alias="backup_retention_period") + db_subnet_group_name: Optional[str] = Field(..., alias="db_subnet_group_name") + + class Config: + smart_union = True + extra = Extra.forbid + + +class CNARDSInstanceV1(CNAssetV1): + vpc: AWSVPCV1 = Field(..., alias="vpc") + defaults: Optional[ResourceFile] = Field(..., alias="defaults") + overrides: Optional[CNARDSInstanceOverridesV1] = Field(..., alias="overrides") + + class Config: + smart_union = True + extra = Extra.forbid + + +class CNAAssumeRoleAssetOverridesV1(BaseModel): + slug: Optional[str] = Field(..., alias="slug") + + class Config: + smart_union = True + extra = Extra.forbid + + class CNAAssumeRoleAssetV1(CNAssetV1): - name: str = Field(..., alias="name") - aws_assume_role: CNAAssumeRoleAssetConfigV1 = Field(..., alias="aws_assume_role") + account: CNAAWSAccountRoleARNs = Field(..., alias="account") + overrides: Optional[CNAAssumeRoleAssetOverridesV1] = Field(..., alias="overrides") + defaults: Optional[ResourceFile] = Field(..., alias="defaults") class Config: smart_union = True @@ -118,9 +196,9 @@ class Config: class NamespaceCNAssetV1(NamespaceExternalResourceV1): - resources: list[Union[CNANullAssetV1, CNAAssumeRoleAssetV1, CNAssetV1]] = Field( - ..., alias="resources" - ) + resources: list[ + Union[CNARDSInstanceV1, CNAAssumeRoleAssetV1, CNANullAssetV1, CNAssetV1] + ] = Field(..., alias="resources") class Config: smart_union = True @@ -129,6 +207,9 @@ class Config: class NamespaceV1(BaseModel): name: str = Field(..., alias="name") + managed_external_resources: Optional[bool] = Field( + ..., alias="managedExternalResources" + ) external_resources: Optional[ list[Union[NamespaceCNAssetV1, NamespaceExternalResourceV1]] ] = Field(..., alias="externalResources") diff --git a/reconcile/gql_definitions/fragments/resource_file.gql b/reconcile/gql_definitions/fragments/resource_file.gql new file mode 100644 index 0000000000..771d8d5fb9 --- /dev/null +++ b/reconcile/gql_definitions/fragments/resource_file.gql @@ -0,0 +1,6 @@ +# qenerate: plugin=pydantic_v1 + +fragment ResourceFile on Resource_v1 { + resourceFileSchema: schema + content +} diff --git a/reconcile/gql_definitions/fragments/resource_file.py b/reconcile/gql_definitions/fragments/resource_file.py new file mode 100644 index 0000000000..3ed2ef61fd --- /dev/null +++ b/reconcile/gql_definitions/fragments/resource_file.py @@ -0,0 +1,26 @@ +""" +Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY! +""" +from enum import Enum # noqa: F401 # pylint: disable=W0611 +from typing import ( # noqa: F401 # pylint: disable=W0611 + Any, + Callable, + Optional, + Union, +) + +from pydantic import ( # noqa: F401 # pylint: disable=W0611 + BaseModel, + Extra, + Field, + Json, +) + + +class ResourceFile(BaseModel): + resource_file_schema: Optional[str] = Field(..., alias="resourceFileSchema") + content: str = Field(..., alias="content") + + class Config: + smart_union = True + extra = Extra.forbid diff --git a/reconcile/test/cna/test_asset.py b/reconcile/test/cna/test_asset.py index 522419e7b9..362495bde8 100644 --- a/reconcile/test/cna/test_asset.py +++ b/reconcile/test/cna/test_asset.py @@ -1,4 +1,4 @@ -from typing import Any, Mapping +from typing import Any, Mapping, MutableMapping, Optional from reconcile.cna.assets.asset import ( Asset, AssetStatus, @@ -8,8 +8,21 @@ AssetTypeVariableType, AssetError, asset_type_metadata_from_asset_dataclass, + asset_type_from_raw_asset, ) from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset +from reconcile.cna.assets.null import NullAsset + +from reconcile.gql_definitions.cna.queries.cna_resources import ( + CNAAssumeRoleAssetV1, + CNAssetV1, + NamespaceV1, + NamespaceCNAssetV1, +) +from reconcile.utils.external_resource_spec import ( + TypedExternalResourceSpec, +) +from reconcile.utils.external_resources import PROVIDER_CNA_EXPERIMENTAL import pytest @@ -71,7 +84,7 @@ def raw_aws_assumerole_asset() -> dict[str, Any]: @pytest.fixture def aws_assumerole_asset( - raw_aws_assumerole_asset: Mapping[str, Any], + raw_aws_assumerole_asset: MutableMapping[str, Any], ) -> AWSAssumeRoleAsset: asset = Asset.from_api_mapping( raw_aws_assumerole_asset, @@ -82,13 +95,13 @@ def aws_assumerole_asset( def test_asset_type_extraction_from_raw(raw_aws_assumerole_asset: Mapping[str, Any]): - assert AssetType.EXAMPLE_AWS_ASSUMEROLE == Asset.asset_type_from_raw_asset( + assert AssetType.EXAMPLE_AWS_ASSUMEROLE == asset_type_from_raw_asset( raw_aws_assumerole_asset ) def test_from_api_mapping( - raw_aws_assumerole_asset: Mapping[str, Any], + raw_aws_assumerole_asset: MutableMapping[str, Any], ): asset = Asset.from_api_mapping(raw_aws_assumerole_asset, AWSAssumeRoleAsset) assert isinstance(asset, AWSAssumeRoleAsset) @@ -101,7 +114,7 @@ def test_from_api_mapping( def test_from_api_mapping_required_parameter_missing( - raw_aws_assumerole_asset: Mapping[str, Any], + raw_aws_assumerole_asset: MutableMapping[str, Any], ): raw_aws_assumerole_asset["parameters"].pop("role_arn") with pytest.raises(AssetError) as e: @@ -109,7 +122,7 @@ def test_from_api_mapping_required_parameter_missing( raw_aws_assumerole_asset, AWSAssumeRoleAsset, ) - assert str(e.value).startswith("Inconsistent asset from CNA API") + assert str(e.value).startswith("Inconsistent asset") def test_api_payload(aws_assumerole_asset: AWSAssumeRoleAsset): @@ -200,7 +213,7 @@ def test_asset_comparion_ignorable_fields(): assert asset_1.asset_properties() == asset_2.asset_properties() -def test_asset_properties_extration(): +def test_asset_properties_extraction(): AWSAssumeRoleAsset( id=None, href=None, @@ -212,3 +225,90 @@ def test_asset_properties_extration(): "role_arn": "arn", "verify_slug": "slug", } + + +def build_assume_role_typed_external_resource( + identifier: str, + role_arn: str, + verify_slug_override: Optional[str], + verify_slug_default: Optional[str], +) -> TypedExternalResourceSpec[CNAssetV1]: + resource = { + "provider": AWSAssumeRoleAsset.provider(), + "identifier": identifier, + "account": { + "name": "acc", + "cna": {"defaultRoleARN": role_arn, "moduleRoleARNS": None}, + }, + "overrides": { + "slug": verify_slug_override, + }, + "defaults": { + "resourceFileSchema": "schema", + "content": f"slug: {verify_slug_default}" if verify_slug_default else "", + }, + } + namespace_resource = { + "provider": PROVIDER_CNA_EXPERIMENTAL, + "provisioner": {"name": "some-ocm-org"}, + "resources": [resource], + } + namespace = { + "name": "ns-name", + "managedExternalResources": True, + "externalResources": [namespace_resource], + } + return TypedExternalResourceSpec[CNAssetV1]( + namespace_spec=NamespaceV1(**namespace), + namespace_external_resource=NamespaceCNAssetV1(**namespace_resource), + spec=CNAAssumeRoleAssetV1(**resource), + ) + + +def test_from_external_resources_with_default(): + identifier = "my_id" + role_arn = "arn" + verify_slug_default = "slug-default" + spec = build_assume_role_typed_external_resource( + identifier, role_arn, None, verify_slug_default + ) + asset = AWSAssumeRoleAsset.from_external_resources(spec) + + assert isinstance(asset, AWSAssumeRoleAsset) + assert asset.name == identifier + assert asset.role_arn == role_arn + assert asset.verify_slug == verify_slug_default + + +def test_from_external_resources_with_no_override_and_no_default(): + identifier = "my_id" + role_arn = "arn" + spec = build_assume_role_typed_external_resource(identifier, role_arn, None, None) + asset = AWSAssumeRoleAsset.from_external_resources(spec) + + assert isinstance(asset, AWSAssumeRoleAsset) + assert asset.name == identifier + assert asset.role_arn == role_arn + assert asset.verify_slug is None + + +def test_from_external_resources_with_override(): + identifier = "my_id" + role_arn = "arn" + verify_slug_override = "slug-override" + verify_slug_default = "slug-default" + spec = build_assume_role_typed_external_resource( + identifier, role_arn, verify_slug_override, verify_slug_default + ) + asset = AWSAssumeRoleAsset.from_external_resources(spec) + + assert isinstance(asset, AWSAssumeRoleAsset) + assert asset.name == identifier + assert asset.role_arn == role_arn + assert asset.verify_slug == verify_slug_override + + +def test_from_external_resource_wrong_class(): + spec = build_assume_role_typed_external_resource("id", "arn", "slug", "def") + with pytest.raises(AssetError): + NullAsset.from_external_resources(spec) diff --git a/reconcile/test/cna/test_aws_assume_role.py b/reconcile/test/cna/test_aws_assume_role.py index d74a235f83..1140b42ac6 100644 --- a/reconcile/test/cna/test_aws_assume_role.py +++ b/reconcile/test/cna/test_aws_assume_role.py @@ -1,7 +1,7 @@ from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( CNAAssumeRoleAssetV1, - CNAAssumeRoleAssetConfigV1, + CNAAssumeRoleAssetOverridesV1, ) from reconcile.gql_definitions.cna.queries.aws_account_fragment import ( CNAAWSAccountRoleARNs, @@ -15,13 +15,14 @@ def test_from_query_class(): arn = "arn" query_asset = CNAAssumeRoleAssetV1( provider=AWSAssumeRoleAsset.provider(), - name=name, - aws_assume_role=CNAAssumeRoleAssetConfigV1( + identifier=name, + overrides=CNAAssumeRoleAssetOverridesV1( slug=slug, - account=CNAAWSAccountRoleARNs( - name="acc", - cna=CNAAWSSpecV1(defaultRoleARN=arn, moduleRoleARNS=None), - ), + ), + defaults=None, + account=CNAAWSAccountRoleARNs( + name="acc", + cna=CNAAWSSpecV1(defaultRoleARN=arn, moduleRoleARNS=None), ), ) asset = AWSAssumeRoleAsset.from_query_class(query_asset) diff --git a/reconcile/test/cna/test_aws_rds.py b/reconcile/test/cna/test_aws_rds.py new file mode 100644 index 0000000000..11cf12a149 --- /dev/null +++ b/reconcile/test/cna/test_aws_rds.py @@ -0,0 +1,94 @@ +from reconcile.cna.assets.aws_rds import AWSRDSAsset +from reconcile.gql_definitions.cna.queries.cna_resources import ( + CNARDSInstanceV1, + CNARDSInstanceOverridesV1, + AWSVPCV1, +) +from reconcile.gql_definitions.cna.queries.aws_account_fragment import ( + CNAAWSAccountRoleARNs, + CNAAWSSpecV1, +) + + +def test_from_query_class(): + asset_identifier = "identifier" + db_name = "instance-name" + vpc_id = "vpc-id" + region = "region" + arn = "arn" + engine = "engine" + engine_version = "14.2" + allocated_storage = 10 + max_allocated_storage = 20 + instance_class = "instance-class" + db_subnet_group_name = "db-subnet-group-name" + backup_retention_period = 7 + query_asset = CNARDSInstanceV1( + provider=AWSRDSAsset.provider(), + identifier=asset_identifier, + vpc=AWSVPCV1( + vpc_id=vpc_id, + region=region, + account=CNAAWSAccountRoleARNs( + name="acc", + cna=CNAAWSSpecV1(defaultRoleARN=arn, moduleRoleARNS=None), + ), + ), + defaults=None, + overrides=CNARDSInstanceOverridesV1( + name=db_name, + engine=engine, + engine_version=engine_version, + allocated_storage=allocated_storage, + max_allocated_storage=max_allocated_storage, + instance_class=instance_class, + db_subnet_group_name=db_subnet_group_name, + username=None, + backup_retention_period=backup_retention_period, + ), + ) + asset = AWSRDSAsset.from_query_class(query_asset) + assert isinstance(asset, AWSRDSAsset) + assert asset.name == asset_identifier + assert asset.region == region + assert asset.identifier == db_name + assert asset.vpc_id == vpc_id + assert asset.db_subnet_group_name == db_subnet_group_name + assert asset.instance_class == instance_class + assert asset.engine == engine + assert asset.engine_version == engine_version + assert asset.allocated_storage == str(allocated_storage) + assert asset.max_allocated_storage == str(max_allocated_storage) + assert asset.role_arn == arn + assert asset.backup_retention_period == backup_retention_period + + +def test_from_query_class_db_name_default(): + asset_identifier = "identifier" + query_asset = CNARDSInstanceV1( + provider=AWSRDSAsset.provider(), + identifier="identifier", + vpc=AWSVPCV1( + vpc_id="vpc-id", + region="region", + account=CNAAWSAccountRoleARNs( + name="acc", + cna=CNAAWSSpecV1(defaultRoleARN="arn", moduleRoleARNS=None), + ), + ), + defaults=None, + overrides=CNARDSInstanceOverridesV1( + name=None, + engine="postgres", + engine_version="14.2", + allocated_storage=10, + max_allocated_storage=20, + instance_class="instance-class", + db_subnet_group_name="db-subnet-group-name", + username=None, + backup_retention_period=7, + ), + ) + asset = AWSRDSAsset.from_query_class(query_asset) + assert isinstance(asset, AWSRDSAsset) + assert asset.identifier == asset_identifier diff --git a/reconcile/test/cna/test_integration.py b/reconcile/test/cna/test_integration.py index 56d389b825..310a036929 100644 --- a/reconcile/test/cna/test_integration.py +++ b/reconcile/test/cna/test_integration.py @@ -26,10 +26,12 @@ from reconcile.cna.state import State from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, + CNANullAssetOverridesV1, ExternalResourcesProvisionerV1, NamespaceCNAssetV1, NamespaceV1, ) +from reconcile.utils.external_resources import PROVIDER_CNA_EXPERIMENTAL @fixture @@ -43,9 +45,10 @@ def cna_clients() -> dict[str, CNAClient]: def namespace(assets: list[CNANullAssetV1]) -> NamespaceV1: return NamespaceV1( name="test", + managedExternalResources=True, externalResources=[ NamespaceCNAssetV1( - provider="null-asset", + provider=PROVIDER_CNA_EXPERIMENTAL, provisioner=ExternalResourcesProvisionerV1( name="test", ), @@ -159,7 +162,7 @@ def test_integration_assemble_current_states( mocker.patch.object( CNAClient, "service_account_name", create_autospec=True, return_value="creator" ) - integration = CNAIntegration(cna_clients={"test": CNAClient(None)}, namespaces=[]) + integration = CNAIntegration(cna_clients={"test": CNAClient(None)}, namespaces=[]) # type: ignore integration.assemble_current_states() assert integration._current_states == {"test": expected_state} @@ -174,8 +177,10 @@ def test_integration_assemble_current_states( assets=[ CNANullAssetV1( provider="null-asset", - name="test", - addr_block="123", + identifier="test", + overrides=CNANullAssetOverridesV1( + addr_block="123", + ), ) ] ) @@ -193,13 +198,13 @@ def test_integration_assemble_current_states( assets=[ CNANullAssetV1( provider="null-asset", - name="test", - addr_block="123", + identifier="test", + overrides=CNANullAssetOverridesV1( + addr_block="123", + ), ), CNANullAssetV1( - provider="null-asset", - name="test2", - addr_block=None, + provider="null-asset", identifier="test2", overrides=None ), ] ) diff --git a/reconcile/test/cna/test_null.py b/reconcile/test/cna/test_null.py index f83975a839..2bc7fa2338 100644 --- a/reconcile/test/cna/test_null.py +++ b/reconcile/test/cna/test_null.py @@ -1,16 +1,30 @@ from reconcile.cna.assets.null import NullAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, + CNANullAssetOverridesV1, ) def test_from_query_class(): - name = "name" + identifier = "name" addr_block = "addr_block" query_asset = CNANullAssetV1( - provider=NullAsset.provider(), name=name, addr_block=addr_block + provider=NullAsset.provider(), + identifier=identifier, + overrides=CNANullAssetOverridesV1(addr_block=addr_block), ) asset = NullAsset.from_query_class(query_asset) assert isinstance(asset, NullAsset) - assert asset.name == name + assert asset.name == identifier assert asset.addr_block == addr_block + + +def test_from_query_class_no_overrides(): + identifier = "name" + query_asset = CNANullAssetV1( + provider=NullAsset.provider(), identifier=identifier, overrides=None + ) + asset = NullAsset.from_query_class(query_asset) + assert isinstance(asset, NullAsset) + assert asset.name == identifier + assert asset.addr_block is None diff --git a/reconcile/test/test_utils_external_resources.py b/reconcile/test/test_utils_external_resources.py index ca08318335..bf555f1de7 100644 --- a/reconcile/test/test_utils_external_resources.py +++ b/reconcile/test/test_utils_external_resources.py @@ -1,9 +1,15 @@ import json +from typing import Optional, Union import pytest +from pydantic import BaseModel +from reconcile.utils.external_resource_spec import ( + ExternalResourceSpec, + TypedExternalResourceSpec, +) import reconcile.utils.external_resources as uer -from reconcile.utils.external_resource_spec import ExternalResourceSpec +from reconcile.gql_definitions.fragments.resource_file import ResourceFile @pytest.fixture @@ -245,3 +251,160 @@ def test_resource_value_resolver_overrides_and_defaults(mocker): "default_2": "override_data2", "default_3": "default_data3", } + + +class TestProvisionier(BaseModel): + name: str + + +class MyResource(BaseModel): + provider: str + identifier: str + + +class ResourceOverrides(BaseModel): + field_1: Optional[str] + field_2: Optional[str] + + +class OverrideableResource(BaseModel): + provider: str + identifier: str + overrides: Optional[ResourceOverrides] + defaults: Optional[ResourceFile] + + +class TestNamespaceExternalResource(BaseModel): + provider: str + provisioner: TestProvisionier + resources: list[Union[MyResource, OverrideableResource]] + + +class TestNamespace(BaseModel): + name: str + managed_external_resources: bool + external_resources: Optional[list[TestNamespaceExternalResource]] + + +@pytest.fixture +def namespace() -> TestNamespace: + return TestNamespace( + name="ns", + managed_external_resources=True, + external_resources=[ + TestNamespaceExternalResource( + provider="pp", + provisioner=TestProvisionier(name="pn"), + resources=[ + MyResource(provider="rp", identifier="ri"), + ], + ) + ], + ) + + +def test_get_external_resource_specs_for_namespace( + namespace: TestNamespace, +): + external_resources = uer.get_external_resource_specs_for_namespace( + namespace, MyResource, None + ) + assert len(external_resources) == 1 + + assert external_resources[0].provision_provider == "pp" + assert external_resources[0].provisioner_name == "pn" + assert external_resources[0].namespace_name == "ns" + assert external_resources[0].provider == "rp" + assert external_resources[0].identifier == "ri" + + +def test_get_external_resource_specs_for_namespace_provisioning_provider_filter( + namespace: TestNamespace, +): + external_resources = uer.get_external_resource_specs_for_namespace( + namespace, MyResource, "another-provisioning-provider" + ) + assert len(external_resources) == 0 + + +def test_get_external_resource_specs_for_namespace_wrong_type(namespace: TestNamespace): + with pytest.raises(ValueError): + uer.get_external_resource_specs_for_namespace( + namespace, OverrideableResource, None + ) + + +def test_typed_external_resource_resolve_no_defaults(namespace: TestNamespace): + """ + In this scenario, the resource has no defaults, so overrides remain untouched. + """ + assert namespace.external_resources is not None + spec = TypedExternalResourceSpec[OverrideableResource]( + namespace_spec=namespace, + namespace_external_resource=namespace.external_resources[0], + spec=OverrideableResource( + provider="p", + identifier="i", + overrides=ResourceOverrides(field_1="f1", field_2="f2"), + defaults=None, + ), + ) + resolved_spec = spec.resolve() + assert resolved_spec.spec.overrides == spec.spec.overrides + + +def test_typed_external_resource_resolve_defaults_overrides( + namespace: TestNamespace, +): + """ + This scenario tests defaults overwriting undefined overrides. + """ + overwrite_f2 = "f2_override" + default_f1 = "f1_default" + default_f2 = "f2_default" + assert namespace.external_resources is not None + spec = TypedExternalResourceSpec[OverrideableResource]( + namespace_spec=namespace, + namespace_external_resource=namespace.external_resources[0], + spec=OverrideableResource( + provider="p", + identifier="i", + overrides=ResourceOverrides(field_1=None, field_2=overwrite_f2), + defaults=ResourceFile( + resourceFileSchema=None, + content=f"field_1: {default_f1}\nfield_2: {default_f2}", + ), + ), + ) + resolved_spec = spec.resolve() + assert resolved_spec.spec.overrides is not None + assert resolved_spec.spec.overrides.field_1 == default_f1 + assert resolved_spec.spec.overrides.field_2 == overwrite_f2 + + +def test_typed_external_resource_resolve_override_none( + namespace: TestNamespace, +): + """ + This scenario tests that a missing override is created from defaults. + """ + default_f1 = "f1_default" + default_f2 = "f2_default" + assert namespace.external_resources is not None + spec = TypedExternalResourceSpec[OverrideableResource]( + namespace_spec=namespace, + namespace_external_resource=namespace.external_resources[0], + spec=OverrideableResource( + provider="p", + identifier="i", + overrides=None, + defaults=ResourceFile( + resourceFileSchema=None, + content=f"field_1: {default_f1}\nfield_2: {default_f2}", + ), + ), + ) + resolved_spec = spec.resolve() + assert resolved_spec.spec.overrides is not None + assert resolved_spec.spec.overrides.field_1 == default_f1 + assert resolved_spec.spec.overrides.field_2 == default_f2 diff --git a/reconcile/utils/external_resource_spec.py b/reconcile/utils/external_resource_spec.py index 039a4b8ab8..b0d239aa9a 100644 --- a/reconcile/utils/external_resource_spec.py +++ b/reconcile/utils/external_resource_spec.py @@ -1,26 +1,32 @@ import json -from abc import abstractmethod -from collections.abc import ( - Mapping, - MutableMapping, -) -from dataclasses import field from typing import ( Any, + Generic, Optional, + Protocol, + TypeVar, + runtime_checkable, + Union, cast, + get_args, + get_origin, ) +from collections.abc import Mapping, MutableMapping, Sequence import yaml -from pydantic.dataclasses import dataclass - -from reconcile import openshift_resources_base from reconcile.utils.openshift_resource import ( - SECRET_MAX_KEY_LENGTH, OpenshiftResource, build_secret, + SECRET_MAX_KEY_LENGTH, ) +import yaml +from pydantic.dataclasses import dataclass + +from reconcile import openshift_resources_base +from reconcile.gql_definitions.fragments.resource_file import ResourceFile +import anymarkup + class OutputFormatProcessor: @abstractmethod @@ -84,6 +90,88 @@ def render(self, vars: Mapping[str, str]) -> dict[str, str]: return self._formatter.render(vars) +class ExternalResourceProvisioner(Protocol): + @property + def name(self) -> str: + ... + + @abstractmethod + def dict(self, *args, **kwargs) -> dict[str, Any]: + ... + + +@runtime_checkable +class ExternalResource(Protocol): + @property + def provider(self) -> str: + ... + + @property + def identifier(self) -> str: + ... + + @abstractmethod + def dict(self, *args, **kwargs) -> dict[str, Any]: + ... + + +@runtime_checkable +class OverridableExternalResource(ExternalResource, Protocol): + @property + def overrides(self) -> Optional[Any]: + ... + + @abstractmethod + def dict(self, *args, **kwargs) -> dict[str, Any]: + ... + + +@runtime_checkable +class DefaultableExternalResource(ExternalResource, Protocol): + @property + def defaults(self) -> Optional[ResourceFile]: + ... + + @abstractmethod + def dict(self, *args, **kwargs) -> dict[str, Any]: + ... + + +@runtime_checkable +class NamespaceExternalResource(Protocol): + @property + def provider(self) -> str: + ... + + @property + def provisioner(self) -> ExternalResourceProvisioner: + ... + + @property + def resources(self) -> Sequence[ExternalResource]: + ... + + +class Namespace(Protocol): + @property + def name(self) -> str: + ... + + @property + def managed_external_resources(self) -> Optional[bool]: + ... + + @property + def external_resources( + self, + ) -> Optional[Sequence[Union[NamespaceExternalResource, Any]]]: + ... + + @abstractmethod + def dict(self, *args, **kwargs) -> dict[str, Any]: + ... + + @dataclass class ExternalResourceSpec: @@ -191,3 +279,97 @@ def from_spec(spec: ExternalResourceSpec) -> "ExternalResourceUniqueKey": ExternalResourceSpecInventory = MutableMapping[ ExternalResourceUniqueKey, ExternalResourceSpec ] + + +T = TypeVar("T", bound=ExternalResource) + + +class MyConfig: + arbitrary_types_allowed = True + + +EXTERNAL_RESOURCE_SPEC_DEFAULTS_PROPERTY = "defaults" +EXTERNAL_RESOURCE_SPEC_OVERRIDES_PROPERTY = "overrides" + + +@dataclass(config=MyConfig) +class TypedExternalResourceSpec(ExternalResourceSpec, Generic[T]): + + namespace_spec: Namespace + namespace_external_resource: NamespaceExternalResource + spec: T + + def __init__( + self, + namespace_spec: Namespace, + namespace_external_resource: NamespaceExternalResource, + spec: T, + ): + self.namespace_spec = namespace_spec + self.namespace_external_resource = namespace_external_resource + self.spec = spec + super().__init__( + provision_provider=self.namespace_external_resource.provider, + provisioner=self.namespace_external_resource.provisioner.dict( + by_alias=True + ), + resource=self.spec.dict(by_alias=True), + namespace=self.namespace_spec.dict(by_alias=True), + ) + + def get_defaults_data(self) -> dict[str, Any]: + if isinstance(self.spec, DefaultableExternalResource) and self.spec.defaults: + try: + defaults_values = anymarkup.parse( + self.spec.defaults.content, force_types=None + ) + defaults_values.pop("$schema", None) + return defaults_values + except anymarkup.AnyMarkupError: + # todo error handling + raise Exception("Could not parse data. Skipping resource") + return {} + + def get_overrides_data(self) -> dict[str, Any]: + if not isinstance(self.spec, OverridableExternalResource): + return {} + if self.spec.overrides is None: + return {} + return self.spec.overrides.dict(by_alias=True) + + def is_overridable(self) -> bool: + return isinstance(self.spec, OverridableExternalResource) + + def get_overridable_fields(self) -> Sequence[str]: + if isinstance(self.spec, OverridableExternalResource): + overrides_class = self.spec.__annotations__[ + EXTERNAL_RESOURCE_SPEC_OVERRIDES_PROPERTY + ] + is_optional = get_origin(overrides_class) is Union and type( + None + ) in get_args(overrides_class) + if is_optional: + overrides_class = get_args(overrides_class)[0] + return overrides_class.__annotations__.keys() + else: + raise ValueError("resource is not overridable") + + def resolve(self) -> "TypedExternalResourceSpec[T]": + if self.is_overridable(): + overrides_data = self.get_overrides_data() + defaults_data = self.get_defaults_data() + + for field_name in self.get_overridable_fields(): + if overrides_data.get(field_name) is None: + overrides_data[field_name] = defaults_data.get(field_name) + else: + overrides_data = {} + + new_spec_attr = self.spec.dict(by_alias=True) + new_spec_attr[EXTERNAL_RESOURCE_SPEC_OVERRIDES_PROPERTY] = overrides_data + new_spec = type(self.spec)(**new_spec_attr) + return TypedExternalResourceSpec( + namespace_spec=self.namespace_spec, + namespace_external_resource=self.namespace_external_resource, + spec=new_spec, + ) diff --git a/reconcile/utils/external_resources.py b/reconcile/utils/external_resources.py index 784c8f5b99..0808366724 100644 --- a/reconcile/utils/external_resources.py +++ b/reconcile/utils/external_resources.py @@ -1,21 +1,63 @@ import json -from collections.abc import ( - Mapping, - MutableMapping, -) from typing import ( Any, Optional, + Type, + TypeVar, ) +from collections.abc import Mapping, MutableMapping, Set, List import anymarkup from reconcile.utils import gql from reconcile.utils.exceptions import FetchResourceError -from reconcile.utils.external_resource_spec import ExternalResourceSpec +from reconcile.utils.external_resource_spec import ( + ExternalResourceSpec, + TypedExternalResourceSpec, + ExternalResource, + Namespace, + NamespaceExternalResource, +) PROVIDER_AWS = "aws" PROVIDER_CLOUDFLARE = "cloudflare" +PROVIDER_CNA_EXPERIMENTAL = "cna-experimental" + +T = TypeVar("T", bound=ExternalResource) + + +def get_external_resource_specs_for_namespace( + namespace: Namespace, + resource_type: Type[T], + provision_provider: Optional[str] = None, +) -> list[TypedExternalResourceSpec[T]]: + if not namespace.managed_external_resources: + return [] + specs: List[TypedExternalResourceSpec[T]] = [] + for e in namespace.external_resources or []: + if isinstance(e, NamespaceExternalResource): + for r in e.resources: + if isinstance(r, resource_type): + specs.append( + TypedExternalResourceSpec[T]( + namespace_spec=namespace, + namespace_external_resource=e, + spec=r, + ) + ) + else: + raise ValueError( + f"expected resource of type {resource_type}, got {type(r)}" + ) + + if provision_provider: + specs = [ + s + for s in specs + if s.namespace_external_resource.provider == provision_provider + ] + + return specs def get_external_resource_specs( From 6dc3188786fb881e881d00f565ee4d9ec23edad4 Mon Sep 17 00:00:00 2001 From: Karl Fischer Date: Wed, 16 Nov 2022 09:54:15 +0100 Subject: [PATCH 04/20] runtime_checkable for Namespace (#2960) --- reconcile/utils/external_resource_spec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reconcile/utils/external_resource_spec.py b/reconcile/utils/external_resource_spec.py index b0d239aa9a..1572d50df7 100644 --- a/reconcile/utils/external_resource_spec.py +++ b/reconcile/utils/external_resource_spec.py @@ -152,6 +152,7 @@ def resources(self) -> Sequence[ExternalResource]: ... +@runtime_checkable class Namespace(Protocol): @property def name(self) -> str: From 956b1c1c3a5b1b6297ebacd7fa58fd126fd214b7 Mon Sep 17 00:00:00 2001 From: Karl Fischer Date: Wed, 16 Nov 2022 12:37:29 +0100 Subject: [PATCH 05/20] CNA bindings (#2962) --- reconcile/cna/assets/asset.py | 22 +++++--- reconcile/cna/assets/aws_assume_role.py | 5 +- reconcile/cna/assets/aws_rds.py | 1 + reconcile/cna/assets/null.py | 1 + reconcile/cna/client.py | 26 +++++++++ reconcile/cna/integration.py | 56 ++++++++++++++++++- reconcile/cna/state.py | 51 +++++++++++++---- .../cna/queries/cna_resources.gql | 7 ++- .../cna/queries/cna_resources.py | 26 ++++++++- reconcile/test/cna/test_asset.py | 10 +++- reconcile/test/cna/test_aws_assume_role.py | 2 +- reconcile/test/cna/test_integration.py | 12 ++++ reconcile/test/cna/test_state_assembly.py | 1 + reconcile/test/cna/test_state_diff.py | 1 + reconcile/test/cna/test_state_overrides.py | 6 +- 15 files changed, 200 insertions(+), 27 deletions(-) diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index 958e1d8b97..e146ff0be2 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -1,4 +1,3 @@ -from __future__ import annotations from abc import ABC, abstractmethod from pydantic.dataclasses import dataclass @@ -91,12 +90,19 @@ class AssetModelConfig: AssetQueryClass = TypeVar("AssetQueryClass", bound=CNAssetV1) +@dataclass(frozen=True) +class Binding: + cluster_id: str + namespace: str + + @dataclass(frozen=True, config=AssetModelConfig) class Asset(ABC, Generic[AssetQueryClass]): name: str id: Optional[str] href: Optional[str] status: Optional[AssetStatus] + bindings: set[Binding] @staticmethod def bindable() -> bool: @@ -118,14 +124,14 @@ def provider() -> str: @staticmethod @abstractmethod - def from_query_class(asset: AssetQueryClass) -> Asset: + def from_query_class(asset: AssetQueryClass) -> "Asset": ... @classmethod def from_external_resources( cls, external_resource: TypedExternalResourceSpec[CNAssetV1], - ) -> Asset: + ) -> "Asset": cls_arg = get_args(cls.__orig_bases__[0])[0] # type: ignore[attr-defined] resolved = external_resource.resolve() if isinstance(resolved.spec, cls_arg): @@ -169,14 +175,15 @@ def raw_asset_parameters(self, omit_empty: bool) -> dict[str, Any]: def update_from( self, - asset: Asset, - ) -> Asset: + asset: "Asset", + ) -> "Asset": assert isinstance(asset, type(self)) return type(asset)( id=self.id, href=self.href, status=self.status, name=self.name, + bindings=asset.bindings, **asset.asset_properties(), ) @@ -186,8 +193,8 @@ def asset_properties(self) -> dict[str, Any]: @staticmethod def from_api_mapping( raw_asset: Mapping[str, Any], - cna_dataclass: Type[Asset], - ) -> Asset: + cna_dataclass: "Type[Asset]", + ) -> "Asset": params = {} inconsistency_errors = [] raw_asset_params = raw_asset.get(ASSET_PARAMETERS_FIELD) or {} @@ -217,6 +224,7 @@ def from_api_mapping( href=raw_asset.get(ASSET_HREF_FIELD), status=AssetStatus(raw_asset.get(ASSET_STATUS_FIELD)), name=raw_asset.get(ASSET_NAME_FIELD, ""), + bindings=set(), **params, ) diff --git a/reconcile/cna/assets/aws_assume_role.py b/reconcile/cna/assets/aws_assume_role.py index 46bfca9355..42b4fb017e 100644 --- a/reconcile/cna/assets/aws_assume_role.py +++ b/reconcile/cna/assets/aws_assume_role.py @@ -31,13 +31,13 @@ def asset_type() -> AssetType: @staticmethod def from_query_class(asset: CNAAssumeRoleAssetV1) -> Asset: - aws_cna_cfg = asset.account.cna + aws_cna_cfg = asset.aws_account.cna role_arn = aws_role_arn_for_module( aws_cna_cfg, AssetType.EXAMPLE_AWS_ASSUMEROLE.value ) if role_arn is None: raise AssetError( - f"No CNA roles configured for AWS account {asset.account.name}" + f"No CNA roles configured for AWS account {asset.aws_account.name}" ) return AWSAssumeRoleAsset( @@ -45,6 +45,7 @@ def from_query_class(asset: CNAAssumeRoleAssetV1) -> Asset: href=None, status=AssetStatus.UNKNOWN, name=asset.identifier, + bindings=set(), verify_slug=asset.overrides.slug if asset.overrides else None, role_arn=role_arn, ) diff --git a/reconcile/cna/assets/aws_rds.py b/reconcile/cna/assets/aws_rds.py index 63f23ed0ac..550deb02fd 100644 --- a/reconcile/cna/assets/aws_rds.py +++ b/reconcile/cna/assets/aws_rds.py @@ -73,6 +73,7 @@ def from_query_class(asset: CNARDSInstanceV1) -> Asset: id=None, href=None, status=AssetStatus.UNKNOWN, + bindings=set(), name=asset.identifier, identifier=asset.overrides.name or asset.identifier, vpc_id=asset.vpc.vpc_id, diff --git a/reconcile/cna/assets/null.py b/reconcile/cna/assets/null.py index d476d6380b..916ea8dac4 100644 --- a/reconcile/cna/assets/null.py +++ b/reconcile/cna/assets/null.py @@ -34,6 +34,7 @@ def from_query_class(asset: CNANullAssetV1) -> Asset: id=None, href=None, status=AssetStatus.UNKNOWN, + bindings=set(), name=asset.identifier, addr_block=asset.overrides.addr_block if asset.overrides else None, ) diff --git a/reconcile/cna/client.py b/reconcile/cna/client.py index 0e3d74812a..085cee9fcc 100644 --- a/reconcile/cna/client.py +++ b/reconcile/cna/client.py @@ -1,4 +1,5 @@ import logging +from dataclasses import asdict from typing import Any from reconcile.cna.assets.asset import ( Asset, @@ -8,6 +9,7 @@ AssetTypeVariableType, asset_type_by_id, ASSET_CREATOR_FIELD, + Binding, ) from reconcile.utils.ocm_base_client import OCMBaseClient @@ -68,6 +70,15 @@ def list_assets(self) -> list[dict[str, Any]]: cnas = self._ocm_client.get(api_path="/api/cna-management/v1/cnas") return cnas.get("items", []) + def fetch_bindings_for_asset(self, asset: Asset) -> list[dict[str, str]]: + """ + Currently bindings can only be retrieved per asset. + I.e., we will need one GET call per asset to aquire + all bindings. + """ + bindings = self._ocm_client.get(api_path=f"{asset.href}/bind") + return bindings.get("items", []) + def create(self, asset: Asset, dry_run: bool = False): if dry_run: logging.info( @@ -82,6 +93,21 @@ def create(self, asset: Asset, dry_run: bool = False): data=asset.api_payload(), ) + def bind(self, asset: Asset, dry_run: bool = False): + for binding in asset.bindings: + if dry_run: + logging.info( + "BIND %s %s %s", + asset.asset_type().value, + asset.name, + binding, + ) + continue + self._ocm_client.post( + api_path=f"{asset.href}/bind", + data=asdict(binding), + ) + def delete(self, asset: Asset, dry_run: bool = False): if dry_run: logging.info("DELETE %s", asset) diff --git a/reconcile/cna/integration.py b/reconcile/cna/integration.py index b0d9f60a67..35420ba50b 100644 --- a/reconcile/cna/integration.py +++ b/reconcile/cna/integration.py @@ -33,7 +33,7 @@ create_secret_reader, ) from reconcile.utils.semver_helper import make_semver -from reconcile.cna.assets.asset import UnknownAssetTypeError, AssetError +from reconcile.cna.assets.asset import UnknownAssetTypeError, AssetError, Binding from reconcile.cna.assets.asset_factory import ( asset_factory_from_schema, asset_factory_from_raw_data, @@ -70,8 +70,30 @@ def assemble_desired_states(self): asset = asset_factory_from_schema(spec) self._desired_states[spec.provisioner_name].add_asset(asset) + # For now we assume that if an asset is bindable, then it + # always binds to its defining namespace + # TODO: probably this should also be done by passing the required namespace vars + # to the factory method. + if not asset.bindable(): + continue + if not (namespace.cluster.spec and namespace.cluster.spec.q_id): + logging.warning( + "cannot bind asset %s because namespace %s does not have a cluster spec with a cluster id.", + asset, + namespace.name, + ) + continue + asset.bindings.add( + Binding( + cluster_id=namespace.cluster.spec.q_id, + namespace=namespace.name, + ) + ) + def assemble_current_states(self): self._current_states = defaultdict(State) + + # We fetch all assets from API for name, client in self._cna_clients.items(): state = State() for raw_asset in client.list_assets_for_creator( @@ -90,11 +112,29 @@ def assemble_current_states(self): logging.error(e) self._current_states[name] = state + # For each asset, we fetch its bindings from API + for _, state in self._current_states.items(): + for asset in state: + for binding in client.fetch_bindings_for_asset(asset): + asset.bindings.add( + Binding( + cluster_id=binding.get("cluster_id", ""), + namespace=binding.get("namespace", ""), + ) + ) + def provision(self, dry_run: bool = False): for provisioner_name, cna_client in self._cna_clients.items(): desired_state = self._desired_states[provisioner_name] current_state = self._current_states[provisioner_name] + terminated_assets = current_state.get_terminated_assets() + for asset in terminated_assets: + # We want to purge all terminated assets + # A DELETE call to a terminated asset will + # purge it from CNA database + cna_client.delete(asset=asset) + additions = desired_state - current_state for asset in additions: cna_client.create(asset=asset, dry_run=dry_run) @@ -107,6 +147,20 @@ def provision(self, dry_run: bool = False): for assets in updates: cna_client.update(asset=assets, dry_run=dry_run) + bindings = current_state.required_bindings_to_reach(desired_state) + for asset in bindings: + # TODO: a MR check will not show any bindings in the diff, + # because resources are either first created or updated before + # any binding happens on a subsequent reconcile iteration. + # We keep this for now, as the API might change in how bindings + # are created. E.g., an asset creation call might also take + # bindings as parameters. + if updates.contains(asset=asset): + # We dont want to bind if there is currently + # an update in progress that changes the asset state + continue + cna_client.bind(asset=asset, dry_run=dry_run) + def build_cna_clients( secret_reader: SecretReaderBase, diff --git a/reconcile/cna/state.py b/reconcile/cna/state.py index 58fed59118..600384e6a1 100644 --- a/reconcile/cna/state.py +++ b/reconcile/cna/state.py @@ -1,7 +1,6 @@ from __future__ import annotations from typing import Optional -from collections.abc import Iterable, Mapping -from reconcile.cna.assets.asset import Asset, AssetStatus, AssetType +from reconcile.cna.assets.asset import Asset, AssetStatus, AssetType, Binding class CNAStateError(Exception): @@ -55,17 +54,21 @@ def _validate_addition(self, asset: Asset): f"Duplicate asset name found in state: asset_type={asset_type}, name={asset.name}" ) + def contains(self, asset: Asset) -> bool: + return asset.name in self._assets[asset.asset_type()] + def add_asset(self, asset: Asset): self._validate_addition(asset=asset) self._assets[asset.asset_type()][asset.name] = asset - def required_updates_to_reach(self, other: State) -> State: + def _diff(self, other: State, compare_bindings: bool) -> State: """ This operation is NOT commutative, i.e.,: - a.required_updates_to_reach(b) != b.required_updates_to_reach(a) + a._diff(b) != b._diff(a) - This is supposed to be called on actual state (self). - I.e., actual.required_updates_to_reach(desired) + This is supposed to be used on actual state (self). + I.e., actual._diff(desired) is supposed to show diff + from actual to reach desired. """ ans = State() for asset_type in AssetType: @@ -75,12 +78,40 @@ def required_updates_to_reach(self, other: State) -> State: asset = self._assets[asset_type][asset_name] if asset.status in (AssetStatus.TERMINATED, AssetStatus.PENDING): continue - if asset.asset_properties() == other_asset.asset_properties(): - # There is no diff - no need to update - continue - ans.add_asset(asset=asset.update_from(other_asset)) + required_bindings: set[Binding] = set() + if compare_bindings: + for binding in other_asset.bindings: + if binding not in asset.bindings: + required_bindings.add(binding) + if not required_bindings: + # Bindings are the same - no need to bind + continue + else: + if asset.asset_properties() == other_asset.asset_properties(): + # There is no diff - no need to update + continue + updated_asset = asset.update_from(other_asset) + if compare_bindings: + # We want to make sure the missing bindings are created + updated_asset.bindings.clear() + for binding in required_bindings: + updated_asset.bindings.add(binding) + ans.add_asset(asset=updated_asset) return ans + def required_updates_to_reach(self, other: State) -> State: + return self._diff(other=other, compare_bindings=False) + + def required_bindings_to_reach(self, other: State) -> State: + return self._diff(other=other, compare_bindings=True) + + def get_terminated_assets(self) -> list[Asset]: + """ + Return a list of assets in terminated state. + Those should be deleted again to be purged. + """ + return [asset for asset in self if asset.status == AssetStatus.TERMINATED] + def __sub__(self, other: State) -> State: """ This is used to determine creations and deletions diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.gql b/reconcile/gql_definitions/cna/queries/cna_resources.gql index 934c79780d..2d234780cf 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.gql +++ b/reconcile/gql_definitions/cna/queries/cna_resources.gql @@ -3,6 +3,11 @@ query CNAssets { namespaces: namespaces_v1 { name + cluster { + spec { + id + } + } managedExternalResources externalResources { provider @@ -42,7 +47,7 @@ query CNAssets { } } ... on CNAAssumeRoleAsset_v1{ - account { + aws_account { ... CNAAWSAccountRoleARNs } overrides { diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.py b/reconcile/gql_definitions/cna/queries/cna_resources.py index a93fbde04f..fa20ef2095 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.py +++ b/reconcile/gql_definitions/cna/queries/cna_resources.py @@ -42,6 +42,11 @@ query CNAssets { namespaces: namespaces_v1 { name + cluster { + spec { + id + } + } managedExternalResources externalResources { provider @@ -81,7 +86,7 @@ } } ... on CNAAssumeRoleAsset_v1{ - account { + aws_account { ... CNAAWSAccountRoleARNs } overrides { @@ -99,6 +104,22 @@ """ +class ClusterSpecV1(BaseModel): + q_id: Optional[str] = Field(..., alias="id") + + class Config: + smart_union = True + extra = Extra.forbid + + +class ClusterV1(BaseModel): + spec: Optional[ClusterSpecV1] = Field(..., alias="spec") + + class Config: + smart_union = True + extra = Extra.forbid + + class ExternalResourcesProvisionerV1(BaseModel): name: str = Field(..., alias="name") @@ -186,7 +207,7 @@ class Config: class CNAAssumeRoleAssetV1(CNAssetV1): - account: CNAAWSAccountRoleARNs = Field(..., alias="account") + aws_account: CNAAWSAccountRoleARNs = Field(..., alias="aws_account") overrides: Optional[CNAAssumeRoleAssetOverridesV1] = Field(..., alias="overrides") defaults: Optional[ResourceFile] = Field(..., alias="defaults") @@ -207,6 +228,7 @@ class Config: class NamespaceV1(BaseModel): name: str = Field(..., alias="name") + cluster: ClusterV1 = Field(..., alias="cluster") managed_external_resources: Optional[bool] = Field( ..., alias="managedExternalResources" ) diff --git a/reconcile/test/cna/test_asset.py b/reconcile/test/cna/test_asset.py index 362495bde8..4894892503 100644 --- a/reconcile/test/cna/test_asset.py +++ b/reconcile/test/cna/test_asset.py @@ -165,6 +165,7 @@ def test_update_from(): href=None, status=None, name=name, + bindings=set(), role_arn="new_arn", verify_slug="new_verify_slug", ) @@ -174,6 +175,7 @@ def test_update_from(): href=href, name=name, status=status, + bindings=set(), role_arn="old_arn", verify_slug="old_verify_slug", ) @@ -196,6 +198,7 @@ def test_asset_comparion_ignorable_fields(): id="id", href="href", status=AssetStatus.READY, + bindings=set(), name=name, role_arn=arn, verify_slug=verify_slug, @@ -205,6 +208,7 @@ def test_asset_comparion_ignorable_fields(): id=None, href=None, status=AssetStatus.TERMINATED, + bindings=set(), name=name, role_arn=arn, verify_slug=verify_slug, @@ -218,6 +222,7 @@ def test_asset_properties_extraction(): id=None, href=None, status=AssetStatus.TERMINATED, + bindings=set(), name="name", role_arn="arn", verify_slug="slug", @@ -236,7 +241,7 @@ def build_assume_role_typed_external_resource( resource = { "provider": AWSAssumeRoleAsset.provider(), "identifier": identifier, - "account": { + "aws_account": { "name": "acc", "cna": {"defaultRoleARN": role_arn, "moduleRoleARNS": None}, }, @@ -256,6 +261,9 @@ def build_assume_role_typed_external_resource( namespace = { "name": "ns-name", "managedExternalResources": True, + "cluster": { + "spec": None, + }, "externalResources": [namespace_resource], } return TypedExternalResourceSpec[CNAssetV1]( diff --git a/reconcile/test/cna/test_aws_assume_role.py b/reconcile/test/cna/test_aws_assume_role.py index 1140b42ac6..12af252b01 100644 --- a/reconcile/test/cna/test_aws_assume_role.py +++ b/reconcile/test/cna/test_aws_assume_role.py @@ -20,7 +20,7 @@ def test_from_query_class(): slug=slug, ), defaults=None, - account=CNAAWSAccountRoleARNs( + aws_account=CNAAWSAccountRoleARNs( name="acc", cna=CNAAWSSpecV1(defaultRoleARN=arn, moduleRoleARNS=None), ), diff --git a/reconcile/test/cna/test_integration.py b/reconcile/test/cna/test_integration.py index 310a036929..ecfbe757a0 100644 --- a/reconcile/test/cna/test_integration.py +++ b/reconcile/test/cna/test_integration.py @@ -29,6 +29,7 @@ CNANullAssetOverridesV1, ExternalResourcesProvisionerV1, NamespaceCNAssetV1, + ClusterV1, NamespaceV1, ) from reconcile.utils.external_resources import PROVIDER_CNA_EXPERIMENTAL @@ -45,6 +46,7 @@ def cna_clients() -> dict[str, CNAClient]: def namespace(assets: list[CNANullAssetV1]) -> NamespaceV1: return NamespaceV1( name="test", + cluster=ClusterV1(spec=None), managedExternalResources=True, externalResources=[ NamespaceCNAssetV1( @@ -63,6 +65,7 @@ def null_asset(name: str, addr_block: Optional[str]) -> NullAsset: id=None, href=None, status=None, + bindings=set(), name=name, addr_block=addr_block, ) @@ -97,6 +100,7 @@ def null_asset(name: str, addr_block: Optional[str]) -> NullAsset: status=AssetStatus.RUNNING, name="null-test", href="url/123", + bindings=set(), addr_block=None, ) } @@ -131,6 +135,7 @@ def null_asset(name: str, addr_block: Optional[str]) -> NullAsset: status=AssetStatus.RUNNING, name="null-test", href="url/123", + bindings=set(), addr_block=None, ), "null-test2": NullAsset( @@ -138,6 +143,7 @@ def null_asset(name: str, addr_block: Optional[str]) -> NullAsset: status=AssetStatus.RUNNING, name="null-test2", href="url/456", + bindings=set(), addr_block=None, ), } @@ -159,6 +165,12 @@ def test_integration_assemble_current_states( mocker.patch.object( CNAClient, "list_assets", create_autospec=True, return_value=listed_assets ) + mocker.patch.object( + CNAClient, + "fetch_bindings_for_asset", + create_autospec=True, + return_value=[], + ) mocker.patch.object( CNAClient, "service_account_name", create_autospec=True, return_value="creator" ) diff --git a/reconcile/test/cna/test_state_assembly.py b/reconcile/test/cna/test_state_assembly.py index 0daf169317..d57008ff3e 100644 --- a/reconcile/test/cna/test_state_assembly.py +++ b/reconcile/test/cna/test_state_assembly.py @@ -25,6 +25,7 @@ def null_asset(name: str, addr_block: Optional[str] = None) -> NullAsset: status=AssetStatus.RUNNING, name=name, addr_block=addr_block, + bindings=set(), ) diff --git a/reconcile/test/cna/test_state_diff.py b/reconcile/test/cna/test_state_diff.py index dba096ea67..c1fc6b99c3 100644 --- a/reconcile/test/cna/test_state_diff.py +++ b/reconcile/test/cna/test_state_diff.py @@ -30,6 +30,7 @@ def null_asset( href=None, status=status, addr_block=addr_block, + bindings=set(), name=name, ) diff --git a/reconcile/test/cna/test_state_overrides.py b/reconcile/test/cna/test_state_overrides.py index 9f3d716f25..fd38a5215b 100644 --- a/reconcile/test/cna/test_state_overrides.py +++ b/reconcile/test/cna/test_state_overrides.py @@ -23,6 +23,7 @@ def null_asset( status=status, name=name, addr_block=addr_block, + bindings=set(), ) @@ -181,5 +182,6 @@ def test_state_iter(): state = State(assets={AssetType.NULL: {asset.name: asset for asset in assets}}) iterated_assets = [asset for asset in state] - assert len(assets) == len(iterated_assets) - assert set(assets) == set(iterated_assets) + assert sorted(assets, key=lambda x: x.name) == sorted( + iterated_assets, key=lambda x: x.name + ) From 726694e6020af58fec14bbfd22965e17ec4696f3 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Thu, 17 Nov 2022 09:36:08 +0100 Subject: [PATCH 06/20] move CNA binding fetching into the asset creation loop (#2969) a minor change: this way the we can use the same `client` to fetch the `bindings` we used to fetch the actual assets. Signed-off-by: Gerd Oberlechner --- reconcile/cna/integration.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/reconcile/cna/integration.py b/reconcile/cna/integration.py index 35420ba50b..f149405895 100644 --- a/reconcile/cna/integration.py +++ b/reconcile/cna/integration.py @@ -100,11 +100,15 @@ def assemble_current_states(self): client.service_account_name() ): try: - state.add_asset( - asset_factory_from_raw_data( - raw_asset, + asset = asset_factory_from_raw_data(raw_asset) + for binding in client.fetch_bindings_for_asset(asset): + asset.bindings.add( + Binding( + cluster_id=binding.get("cluster_id", ""), + namespace=binding.get("namespace", ""), + ) ) - ) + state.add_asset(asset) except UnknownAssetTypeError as e: logging.warning(e) except AssetError as e: @@ -112,17 +116,6 @@ def assemble_current_states(self): logging.error(e) self._current_states[name] = state - # For each asset, we fetch its bindings from API - for _, state in self._current_states.items(): - for asset in state: - for binding in client.fetch_bindings_for_asset(asset): - asset.bindings.add( - Binding( - cluster_id=binding.get("cluster_id", ""), - namespace=binding.get("namespace", ""), - ) - ) - def provision(self, dry_run: bool = False): for provisioner_name, cna_client in self._cna_clients.items(): desired_state = self._desired_states[provisioner_name] From fac431a2a1939eb4a0a4bd8d2e26ca810e349c8f Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Thu, 17 Nov 2022 10:42:05 +0100 Subject: [PATCH 07/20] [CNA] override and default same schema objects (#2968) all CNA assets must have a `defaults` and an `overrides` section, as a lot of the terraform-resources do. the difference is, that each CNA type declares a special `XXXConfig_v1` type that is used for both fields. ```yaml name: CNARDSInstance_v1 interface: CNAsset_v1 fields: - { name: provider, type: string, isRequired: true } - { name: identifier, type: string, isRequired: true, isUnique: true } - { name: name, type: string } - { name: defaults, type: CNARDSInstanceConfig_v1 } <-- - { name: overrides, type: CNARDSInstanceConfig_v1 } <-- ``` having defaults following a strict schema and overrides being able to override all of the defaults, makes writing testable and verifiable code a lot easier. --- reconcile/cna/assets/asset.py | 50 +++++-- reconcile/cna/assets/aws_assume_role.py | 10 +- reconcile/cna/assets/aws_rds.py | 46 +++--- reconcile/cna/assets/aws_utils.py | 2 +- reconcile/cna/assets/null.py | 10 +- reconcile/cna/client.py | 1 - .../{aws_account_fragment.gql => aws_arn.gql} | 0 .../{aws_account_fragment.py => aws_arn.py} | 0 .../cna/queries/aws_assume_role_config.gql | 5 + .../cna/queries/aws_assume_role_config.py | 25 ++++ .../cna/queries/aws_rds_instance_config.gql | 25 ++++ .../cna/queries/aws_rds_instance_config.py | 51 +++++++ .../cna/queries/cna_resources.gql | 33 ++--- .../cna/queries/cna_resources.py | 131 ++++++++---------- .../cna/queries/null_asset_config.gql | 5 + .../cna/queries/null_asset_config.py | 25 ++++ reconcile/test/cna/test_asset.py | 3 +- reconcile/test/cna/test_aws_assume_role.py | 6 +- reconcile/test/cna/test_aws_rds.py | 119 ++++++++-------- reconcile/test/cna/test_integration.py | 13 +- reconcile/test/cna/test_null.py | 10 +- .../test/test_utils_external_resources.py | 84 +---------- reconcile/utils/external_resource_spec.py | 40 +----- 23 files changed, 377 insertions(+), 317 deletions(-) rename reconcile/gql_definitions/cna/queries/{aws_account_fragment.gql => aws_arn.gql} (100%) rename reconcile/gql_definitions/cna/queries/{aws_account_fragment.py => aws_arn.py} (100%) create mode 100644 reconcile/gql_definitions/cna/queries/aws_assume_role_config.gql create mode 100644 reconcile/gql_definitions/cna/queries/aws_assume_role_config.py create mode 100644 reconcile/gql_definitions/cna/queries/aws_rds_instance_config.gql create mode 100644 reconcile/gql_definitions/cna/queries/aws_rds_instance_config.py create mode 100644 reconcile/gql_definitions/cna/queries/null_asset_config.gql create mode 100644 reconcile/gql_definitions/cna/queries/null_asset_config.py diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index e146ff0be2..aa88098c4e 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -88,6 +88,7 @@ class AssetModelConfig: AssetQueryClass = TypeVar("AssetQueryClass", bound=CNAssetV1) +ConfigClass = TypeVar("ConfigClass") @dataclass(frozen=True) @@ -97,7 +98,7 @@ class Binding: @dataclass(frozen=True, config=AssetModelConfig) -class Asset(ABC, Generic[AssetQueryClass]): +class Asset(ABC, Generic[AssetQueryClass, ConfigClass]): name: str id: Optional[str] href: Optional[str] @@ -122,9 +123,9 @@ def asset_type() -> AssetType: def provider() -> str: ... - @staticmethod + @classmethod @abstractmethod - def from_query_class(asset: AssetQueryClass) -> "Asset": + def from_query_class(cls, asset: AssetQueryClass) -> "Asset": ... @classmethod @@ -132,13 +133,12 @@ def from_external_resources( cls, external_resource: TypedExternalResourceSpec[CNAssetV1], ) -> "Asset": - cls_arg = get_args(cls.__orig_bases__[0])[0] # type: ignore[attr-defined] - resolved = external_resource.resolve() - if isinstance(resolved.spec, cls_arg): - return cls.from_query_class(resolved.spec) + query_class = cls._get_query_class_type() + if isinstance(external_resource.spec, query_class): + return cls.from_query_class(external_resource.spec) else: raise AssetError( - f"CNA type {cls_arg} does not match " + f"CNA type {query_class} does not match " f"external resource type {type(external_resource)}" ) @@ -228,6 +228,40 @@ def from_api_mapping( **params, ) + @classmethod + def _get_query_class_type(cls) -> Type[AssetQueryClass]: + return get_args(cls.__orig_bases__[0])[0] # type: ignore[attr-defined] + + @classmethod + def _get_config_class_type(cls) -> Type[ConfigClass]: + return get_args(cls.__orig_bases__[0])[1] # type: ignore[attr-defined] + + @classmethod + def _get_config( + cls, attribute: str, spec: AssetQueryClass + ) -> Optional[ConfigClass]: + if not (configs := getattr(spec, attribute, None)): + return None + config_class = cls._get_config_class_type() + if isinstance(configs, config_class): + return configs + else: + raise AssetError( + f"{attribute} for asset {spec.provider}:{spec.identifier} are not of expected type {config_class}" + ) + + @classmethod + def aggregate_config(cls, spec: AssetQueryClass) -> ConfigClass: + defaults = cls._get_config("defaults", spec) + overrides = cls._get_config("overrides", spec) + config_class = cls._get_config_class_type() + data = { + property: getattr(overrides, property, None) + or getattr(defaults, property, None) + for property in config_class.__annotations__.keys() + } + return config_class(**data) + def asset_type_metadata_from_asset_dataclass( asset_dataclass: Type[Asset], diff --git a/reconcile/cna/assets/aws_assume_role.py b/reconcile/cna/assets/aws_assume_role.py index 42b4fb017e..63626fbda4 100644 --- a/reconcile/cna/assets/aws_assume_role.py +++ b/reconcile/cna/assets/aws_assume_role.py @@ -13,11 +13,12 @@ from reconcile.cna.assets.aws_utils import aws_role_arn_for_module from reconcile.gql_definitions.cna.queries.cna_resources import ( CNAAssumeRoleAssetV1, + CNAAssumeRoleAssetConfig, ) @dataclass(frozen=True, config=AssetModelConfig) -class AWSAssumeRoleAsset(Asset[CNAAssumeRoleAssetV1]): +class AWSAssumeRoleAsset(Asset[CNAAssumeRoleAssetV1, CNAAssumeRoleAssetConfig]): verify_slug: Optional[str] = Field(None, alias="verify-slug") role_arn: str = Field(alias="role_arn") @@ -29,8 +30,9 @@ def provider() -> str: def asset_type() -> AssetType: return AssetType.EXAMPLE_AWS_ASSUMEROLE - @staticmethod - def from_query_class(asset: CNAAssumeRoleAssetV1) -> Asset: + @classmethod + def from_query_class(cls, asset: CNAAssumeRoleAssetV1) -> Asset: + config = cls.aggregate_config(asset) aws_cna_cfg = asset.aws_account.cna role_arn = aws_role_arn_for_module( aws_cna_cfg, AssetType.EXAMPLE_AWS_ASSUMEROLE.value @@ -46,6 +48,6 @@ def from_query_class(asset: CNAAssumeRoleAssetV1) -> Asset: status=AssetStatus.UNKNOWN, name=asset.identifier, bindings=set(), - verify_slug=asset.overrides.slug if asset.overrides else None, + verify_slug=config.slug, role_arn=role_arn, ) diff --git a/reconcile/cna/assets/aws_rds.py b/reconcile/cna/assets/aws_rds.py index 550deb02fd..d0a088928c 100644 --- a/reconcile/cna/assets/aws_rds.py +++ b/reconcile/cna/assets/aws_rds.py @@ -13,11 +13,12 @@ from reconcile.cna.assets.aws_utils import aws_role_arn_for_module from reconcile.gql_definitions.cna.queries.cna_resources import ( CNARDSInstanceV1, + CNARDSInstanceConfig, ) @dataclass(frozen=True, config=AssetModelConfig) -class AWSRDSAsset(Asset[CNARDSInstanceV1]): +class AWSRDSAsset(Asset[CNARDSInstanceV1, CNARDSInstanceConfig]): identifier: str = Field(alias="identifier") vpc_id: str = Field(alias="vpc_id") role_arn: str = Field(alias="role_arn") @@ -27,12 +28,17 @@ class AWSRDSAsset(Asset[CNARDSInstanceV1]): max_allocated_storage: str = Field(alias="max_allocated_storage") engine: Optional[str] = Field(None, alias="engine") engine_version: Optional[str] = Field(None, alias="engine_version") + major_engine_version: Optional[str] = Field(None, alias="major_engine_version") + username: Optional[str] = Field(None, alias="username") region: Optional[str] = Field(None, alias="region") backup_retention_period: Optional[int] = Field( None, alias="backup_retention_period" ) backup_window: Optional[str] = Field(None, alias="backup_window") maintenance_window: Optional[str] = Field(None, alias="maintenance_window") + multi_az: Optional[bool] = Field(None, alias="multi_az") + deletion_protection: Optional[bool] = Field(None, alias="deletion_protection") + apply_immediately: Optional[bool] = Field(None, alias="apply_immediately") @staticmethod def provider() -> str: @@ -42,29 +48,30 @@ def provider() -> str: def asset_type() -> AssetType: return AssetType.AWS_RDS - @staticmethod - def from_query_class(asset: CNARDSInstanceV1) -> Asset: - aws_cna_cfg = asset.vpc.account.cna + @classmethod + def from_query_class(cls, asset: CNARDSInstanceV1) -> Asset: + config = cls.aggregate_config(asset) + if config.vpc is None: + raise AssetError("Missing VPC configuration for asset") + + aws_cna_cfg = config.vpc.account.cna role_arn = aws_role_arn_for_module(aws_cna_cfg, AssetType.AWS_RDS.value) if role_arn is None: raise AssetError( - f"No CNA roles configured for AWS account {asset.vpc.account.name}" + f"No CNA roles configured for AWS account {config.vpc.account.name}" ) - if not asset.overrides: - raise AssetError("No overrides provided for RDS instance") - - if not (db_subnet_group_name := asset.overrides.db_subnet_group_name): + if not (db_subnet_group_name := config.db_subnet_group_name): raise AssetError( f"No db_subnet_group_name provided for RDS instance {asset.identifier}" ) - if not (instance_class := asset.overrides.instance_class): + if not (instance_class := config.instance_class): raise AssetError( f"No instance_class provided for RDS instance {asset.identifier}" ) - if not (engine := asset.overrides.engine): + if not (engine := config.engine): raise AssetError(f"No engine provided for RDS instance {asset.identifier}") - if not (max_allocated_storage := asset.overrides.max_allocated_storage): + if not (max_allocated_storage := config.max_allocated_storage): raise AssetError( f"No max_allocated_storage provided for RDS instance {asset.identifier}" ) @@ -75,17 +82,20 @@ def from_query_class(asset: CNARDSInstanceV1) -> Asset: status=AssetStatus.UNKNOWN, bindings=set(), name=asset.identifier, - identifier=asset.overrides.name or asset.identifier, - vpc_id=asset.vpc.vpc_id, + identifier=asset.name or asset.identifier, + vpc_id=config.vpc.vpc_id, role_arn=role_arn, db_subnet_group_name=db_subnet_group_name, engine=engine, - engine_version=asset.overrides.engine_version, + engine_version=config.engine_version, instance_class=instance_class, - allocated_storage=str(asset.overrides.allocated_storage), + allocated_storage=str(config.allocated_storage), max_allocated_storage=str(max_allocated_storage), - region=asset.vpc.region, - backup_retention_period=asset.overrides.backup_retention_period, + region=config.vpc.region, + backup_retention_period=config.backup_retention_period, backup_window=None, maintenance_window=None, + apply_immediately=config.apply_immediately, + deletion_protection=config.deletion_protection, + multi_az=config.multi_az, ) diff --git a/reconcile/cna/assets/aws_utils.py b/reconcile/cna/assets/aws_utils.py index f9a1cb3fb4..af09f1ab34 100644 --- a/reconcile/cna/assets/aws_utils.py +++ b/reconcile/cna/assets/aws_utils.py @@ -1,5 +1,5 @@ from typing import Optional -from reconcile.gql_definitions.cna.queries.aws_account_fragment import CNAAWSSpecV1 +from reconcile.gql_definitions.cna.queries.aws_arn import CNAAWSSpecV1 def aws_role_arn_for_module( diff --git a/reconcile/cna/assets/null.py b/reconcile/cna/assets/null.py index 916ea8dac4..b3bd7095bf 100644 --- a/reconcile/cna/assets/null.py +++ b/reconcile/cna/assets/null.py @@ -13,11 +13,12 @@ ) from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, + CNANullAssetConfig, ) @dataclass(frozen=True, config=AssetModelConfig) -class NullAsset(Asset[CNANullAssetV1]): +class NullAsset(Asset[CNANullAssetV1, CNANullAssetConfig]): addr_block: Optional[str] = Field(None, alias="AddrBlock") @staticmethod @@ -28,13 +29,14 @@ def provider() -> str: def asset_type() -> AssetType: return AssetType.NULL - @staticmethod - def from_query_class(asset: CNANullAssetV1) -> Asset: + @classmethod + def from_query_class(cls, asset: CNANullAssetV1) -> Asset: + config = cls.aggregate_config(asset) return NullAsset( id=None, href=None, status=AssetStatus.UNKNOWN, bindings=set(), name=asset.identifier, - addr_block=asset.overrides.addr_block if asset.overrides else None, + addr_block=config.addr_block, ) diff --git a/reconcile/cna/client.py b/reconcile/cna/client.py index 085cee9fcc..0b07dfa1f3 100644 --- a/reconcile/cna/client.py +++ b/reconcile/cna/client.py @@ -9,7 +9,6 @@ AssetTypeVariableType, asset_type_by_id, ASSET_CREATOR_FIELD, - Binding, ) from reconcile.utils.ocm_base_client import OCMBaseClient diff --git a/reconcile/gql_definitions/cna/queries/aws_account_fragment.gql b/reconcile/gql_definitions/cna/queries/aws_arn.gql similarity index 100% rename from reconcile/gql_definitions/cna/queries/aws_account_fragment.gql rename to reconcile/gql_definitions/cna/queries/aws_arn.gql diff --git a/reconcile/gql_definitions/cna/queries/aws_account_fragment.py b/reconcile/gql_definitions/cna/queries/aws_arn.py similarity index 100% rename from reconcile/gql_definitions/cna/queries/aws_account_fragment.py rename to reconcile/gql_definitions/cna/queries/aws_arn.py diff --git a/reconcile/gql_definitions/cna/queries/aws_assume_role_config.gql b/reconcile/gql_definitions/cna/queries/aws_assume_role_config.gql new file mode 100644 index 0000000000..182f3be0e6 --- /dev/null +++ b/reconcile/gql_definitions/cna/queries/aws_assume_role_config.gql @@ -0,0 +1,5 @@ +# qenerate: plugin=pydantic_v1 + +fragment CNAAssumeRoleAssetConfig on CNAAssumeRoleAssetConfig_v1 { + slug +} diff --git a/reconcile/gql_definitions/cna/queries/aws_assume_role_config.py b/reconcile/gql_definitions/cna/queries/aws_assume_role_config.py new file mode 100644 index 0000000000..d6eb4ed004 --- /dev/null +++ b/reconcile/gql_definitions/cna/queries/aws_assume_role_config.py @@ -0,0 +1,25 @@ +""" +Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY! +""" +from enum import Enum # noqa: F401 # pylint: disable=W0611 +from typing import ( # noqa: F401 # pylint: disable=W0611 + Any, + Callable, + Optional, + Union, +) + +from pydantic import ( # noqa: F401 # pylint: disable=W0611 + BaseModel, + Extra, + Field, + Json, +) + + +class CNAAssumeRoleAssetConfig(BaseModel): + slug: Optional[str] = Field(..., alias="slug") + + class Config: + smart_union = True + extra = Extra.forbid diff --git a/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.gql b/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.gql new file mode 100644 index 0000000000..905c771a89 --- /dev/null +++ b/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.gql @@ -0,0 +1,25 @@ +# qenerate: plugin=pydantic_v1 + +fragment CNARDSInstanceConfig on CNARDSInstanceConfig_v1 { + vpc { + vpc_id + region + account { + ... CNAAWSAccountRoleARNs + } + } + db_subnet_group_name + instance_class + allocated_storage + max_allocated_storage + engine + engine_version + major_engine_version + username + maintenance_window + backup_retention_period + backup_window + multi_az + deletion_protection + apply_immediately +} diff --git a/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.py b/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.py new file mode 100644 index 0000000000..d09245d088 --- /dev/null +++ b/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.py @@ -0,0 +1,51 @@ +""" +Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY! +""" +from enum import Enum # noqa: F401 # pylint: disable=W0611 +from typing import ( # noqa: F401 # pylint: disable=W0611 + Any, + Callable, + Optional, + Union, +) + +from pydantic import ( # noqa: F401 # pylint: disable=W0611 + BaseModel, + Extra, + Field, + Json, +) + +from reconcile.gql_definitions.cna.queries.aws_arn import CNAAWSAccountRoleARNs + + +class AWSVPCV1(BaseModel): + vpc_id: str = Field(..., alias="vpc_id") + region: str = Field(..., alias="region") + account: CNAAWSAccountRoleARNs = Field(..., alias="account") + + class Config: + smart_union = True + extra = Extra.forbid + + +class CNARDSInstanceConfig(BaseModel): + vpc: Optional[AWSVPCV1] = Field(..., alias="vpc") + db_subnet_group_name: Optional[str] = Field(..., alias="db_subnet_group_name") + instance_class: Optional[str] = Field(..., alias="instance_class") + allocated_storage: Optional[int] = Field(..., alias="allocated_storage") + max_allocated_storage: Optional[int] = Field(..., alias="max_allocated_storage") + engine: Optional[str] = Field(..., alias="engine") + engine_version: Optional[str] = Field(..., alias="engine_version") + major_engine_version: Optional[str] = Field(..., alias="major_engine_version") + username: Optional[str] = Field(..., alias="username") + maintenance_window: Optional[str] = Field(..., alias="maintenance_window") + backup_retention_period: Optional[int] = Field(..., alias="backup_retention_period") + backup_window: Optional[str] = Field(..., alias="backup_window") + multi_az: Optional[bool] = Field(..., alias="multi_az") + deletion_protection: Optional[bool] = Field(..., alias="deletion_protection") + apply_immediately: Optional[bool] = Field(..., alias="apply_immediately") + + class Config: + smart_union = True + extra = Extra.forbid diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.gql b/reconcile/gql_definitions/cna/queries/cna_resources.gql index 2d234780cf..5e8f3e1822 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.gql +++ b/reconcile/gql_definitions/cna/queries/cna_resources.gql @@ -19,42 +19,31 @@ query CNAssets { provider identifier ... on CNANullAsset_v1 { + defaults { + ... CNANullAssetConfig + } overrides { - addr_block + ... CNANullAssetConfig } } ... on CNARDSInstance_v1 { - vpc { - vpc_id - region - account { - ... CNAAWSAccountRoleARNs - } - } + name defaults { - ... ResourceFile + ... CNARDSInstanceConfig } overrides { - name - engine - engine_version - username - instance_class - allocated_storage - max_allocated_storage - backup_retention_period - db_subnet_group_name + ... CNARDSInstanceConfig } } ... on CNAAssumeRoleAsset_v1{ aws_account { ... CNAAWSAccountRoleARNs } - overrides { - slug - } defaults { - ... ResourceFile + ... CNAAssumeRoleAssetConfig + } + overrides { + ... CNAAssumeRoleAssetConfig } } } diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.py b/reconcile/gql_definitions/cna/queries/cna_resources.py index fa20ef2095..d77bc9a017 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.py +++ b/reconcile/gql_definitions/cna/queries/cna_resources.py @@ -16,10 +16,14 @@ Json, ) -from reconcile.gql_definitions.cna.queries.aws_account_fragment import ( - CNAAWSAccountRoleARNs, +from reconcile.gql_definitions.cna.queries.aws_arn import CNAAWSAccountRoleARNs +from reconcile.gql_definitions.cna.queries.aws_assume_role_config import ( + CNAAssumeRoleAssetConfig, +) +from reconcile.gql_definitions.cna.queries.null_asset_config import CNANullAssetConfig +from reconcile.gql_definitions.cna.queries.aws_rds_instance_config import ( + CNARDSInstanceConfig, ) -from reconcile.gql_definitions.fragments.resource_file import ResourceFile DEFINITION = """ @@ -34,9 +38,36 @@ } } -fragment ResourceFile on Resource_v1 { - resourceFileSchema: schema - content +fragment CNAAssumeRoleAssetConfig on CNAAssumeRoleAssetConfig_v1 { + slug +} + +fragment CNANullAssetConfig on CNANullAssetConfig_v1 { + addr_block +} + +fragment CNARDSInstanceConfig on CNARDSInstanceConfig_v1 { + vpc { + vpc_id + region + account { + ... CNAAWSAccountRoleARNs + } + } + db_subnet_group_name + instance_class + allocated_storage + max_allocated_storage + engine + engine_version + major_engine_version + username + maintenance_window + backup_retention_period + backup_window + multi_az + deletion_protection + apply_immediately } query CNAssets { @@ -58,42 +89,31 @@ provider identifier ... on CNANullAsset_v1 { + defaults { + ... CNANullAssetConfig + } overrides { - addr_block + ... CNANullAssetConfig } } ... on CNARDSInstance_v1 { - vpc { - vpc_id - region - account { - ... CNAAWSAccountRoleARNs - } - } + name defaults { - ... ResourceFile + ... CNARDSInstanceConfig } overrides { - name - engine - engine_version - username - instance_class - allocated_storage - max_allocated_storage - backup_retention_period - db_subnet_group_name + ... CNARDSInstanceConfig } } ... on CNAAssumeRoleAsset_v1{ aws_account { ... CNAAWSAccountRoleARNs } - overrides { - slug - } defaults { - ... ResourceFile + ... CNAAssumeRoleAssetConfig + } + overrides { + ... CNAAssumeRoleAssetConfig } } } @@ -146,42 +166,9 @@ class Config: extra = Extra.forbid -class CNANullAssetOverridesV1(BaseModel): - addr_block: Optional[str] = Field(..., alias="addr_block") - - class Config: - smart_union = True - extra = Extra.forbid - - class CNANullAssetV1(CNAssetV1): - overrides: Optional[CNANullAssetOverridesV1] = Field(..., alias="overrides") - - class Config: - smart_union = True - extra = Extra.forbid - - -class AWSVPCV1(BaseModel): - vpc_id: str = Field(..., alias="vpc_id") - region: str = Field(..., alias="region") - account: CNAAWSAccountRoleARNs = Field(..., alias="account") - - class Config: - smart_union = True - extra = Extra.forbid - - -class CNARDSInstanceOverridesV1(BaseModel): - name: Optional[str] = Field(..., alias="name") - engine: Optional[str] = Field(..., alias="engine") - engine_version: Optional[str] = Field(..., alias="engine_version") - username: Optional[str] = Field(..., alias="username") - instance_class: Optional[str] = Field(..., alias="instance_class") - allocated_storage: Optional[int] = Field(..., alias="allocated_storage") - max_allocated_storage: Optional[int] = Field(..., alias="max_allocated_storage") - backup_retention_period: Optional[int] = Field(..., alias="backup_retention_period") - db_subnet_group_name: Optional[str] = Field(..., alias="db_subnet_group_name") + defaults: Optional[CNANullAssetConfig] = Field(..., alias="defaults") + overrides: Optional[CNANullAssetConfig] = Field(..., alias="overrides") class Config: smart_union = True @@ -189,17 +176,9 @@ class Config: class CNARDSInstanceV1(CNAssetV1): - vpc: AWSVPCV1 = Field(..., alias="vpc") - defaults: Optional[ResourceFile] = Field(..., alias="defaults") - overrides: Optional[CNARDSInstanceOverridesV1] = Field(..., alias="overrides") - - class Config: - smart_union = True - extra = Extra.forbid - - -class CNAAssumeRoleAssetOverridesV1(BaseModel): - slug: Optional[str] = Field(..., alias="slug") + name: Optional[str] = Field(..., alias="name") + defaults: Optional[CNARDSInstanceConfig] = Field(..., alias="defaults") + overrides: Optional[CNARDSInstanceConfig] = Field(..., alias="overrides") class Config: smart_union = True @@ -208,8 +187,8 @@ class Config: class CNAAssumeRoleAssetV1(CNAssetV1): aws_account: CNAAWSAccountRoleARNs = Field(..., alias="aws_account") - overrides: Optional[CNAAssumeRoleAssetOverridesV1] = Field(..., alias="overrides") - defaults: Optional[ResourceFile] = Field(..., alias="defaults") + defaults: Optional[CNAAssumeRoleAssetConfig] = Field(..., alias="defaults") + overrides: Optional[CNAAssumeRoleAssetConfig] = Field(..., alias="overrides") class Config: smart_union = True diff --git a/reconcile/gql_definitions/cna/queries/null_asset_config.gql b/reconcile/gql_definitions/cna/queries/null_asset_config.gql new file mode 100644 index 0000000000..48e008560d --- /dev/null +++ b/reconcile/gql_definitions/cna/queries/null_asset_config.gql @@ -0,0 +1,5 @@ +# qenerate: plugin=pydantic_v1 + +fragment CNANullAssetConfig on CNANullAssetConfig_v1 { + addr_block +} diff --git a/reconcile/gql_definitions/cna/queries/null_asset_config.py b/reconcile/gql_definitions/cna/queries/null_asset_config.py new file mode 100644 index 0000000000..cadb55ab6a --- /dev/null +++ b/reconcile/gql_definitions/cna/queries/null_asset_config.py @@ -0,0 +1,25 @@ +""" +Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY! +""" +from enum import Enum # noqa: F401 # pylint: disable=W0611 +from typing import ( # noqa: F401 # pylint: disable=W0611 + Any, + Callable, + Optional, + Union, +) + +from pydantic import ( # noqa: F401 # pylint: disable=W0611 + BaseModel, + Extra, + Field, + Json, +) + + +class CNANullAssetConfig(BaseModel): + addr_block: Optional[str] = Field(..., alias="addr_block") + + class Config: + smart_union = True + extra = Extra.forbid diff --git a/reconcile/test/cna/test_asset.py b/reconcile/test/cna/test_asset.py index 4894892503..03905113b9 100644 --- a/reconcile/test/cna/test_asset.py +++ b/reconcile/test/cna/test_asset.py @@ -249,8 +249,7 @@ def build_assume_role_typed_external_resource( "slug": verify_slug_override, }, "defaults": { - "resourceFileSchema": "schema", - "content": f"slug: {verify_slug_default}" if verify_slug_default else "", + "slug": verify_slug_default, }, } namespace_resource = { diff --git a/reconcile/test/cna/test_aws_assume_role.py b/reconcile/test/cna/test_aws_assume_role.py index 12af252b01..0ac4d55b94 100644 --- a/reconcile/test/cna/test_aws_assume_role.py +++ b/reconcile/test/cna/test_aws_assume_role.py @@ -1,9 +1,9 @@ from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( CNAAssumeRoleAssetV1, - CNAAssumeRoleAssetOverridesV1, + CNAAssumeRoleAssetConfig, ) -from reconcile.gql_definitions.cna.queries.aws_account_fragment import ( +from reconcile.gql_definitions.cna.queries.aws_arn import ( CNAAWSAccountRoleARNs, CNAAWSSpecV1, ) @@ -16,7 +16,7 @@ def test_from_query_class(): query_asset = CNAAssumeRoleAssetV1( provider=AWSAssumeRoleAsset.provider(), identifier=name, - overrides=CNAAssumeRoleAssetOverridesV1( + overrides=CNAAssumeRoleAssetConfig( slug=slug, ), defaults=None, diff --git a/reconcile/test/cna/test_aws_rds.py b/reconcile/test/cna/test_aws_rds.py index 11cf12a149..81bf8ec280 100644 --- a/reconcile/test/cna/test_aws_rds.py +++ b/reconcile/test/cna/test_aws_rds.py @@ -1,53 +1,74 @@ +from reconcile.cna.assets.asset import AssetError from reconcile.cna.assets.aws_rds import AWSRDSAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( CNARDSInstanceV1, - CNARDSInstanceOverridesV1, - AWSVPCV1, + CNARDSInstanceConfig, ) -from reconcile.gql_definitions.cna.queries.aws_account_fragment import ( +from reconcile.gql_definitions.cna.queries.aws_arn import ( CNAAWSAccountRoleARNs, CNAAWSSpecV1, ) +from reconcile.gql_definitions.cna.queries.aws_rds_instance_config import AWSVPCV1 +import pytest -def test_from_query_class(): - asset_identifier = "identifier" - db_name = "instance-name" - vpc_id = "vpc-id" - region = "region" - arn = "arn" - engine = "engine" - engine_version = "14.2" - allocated_storage = 10 - max_allocated_storage = 20 - instance_class = "instance-class" - db_subnet_group_name = "db-subnet-group-name" - backup_retention_period = 7 - query_asset = CNARDSInstanceV1( +asset_identifier = "identifier" +db_name = "instance-name" +vpc_id = "vpc-id" +region = "region" +arn = "arn" +engine = "engine" +engine_version = "14.2" +major_engine_version = "14" +allocated_storage = 10 +max_allocated_storage = 20 +instance_class = "instance-class" +db_subnet_group_name = "db-subnet-group-name" +backup_retention_period = 7 +backup_window = "backup-window" +maintenance_window = "maintenance_window" +username = "username" +apply_immediately = True +multi_az = True +deletion_protection = True + + +@pytest.fixture +def rds_query_asset() -> CNARDSInstanceV1: + return CNARDSInstanceV1( provider=AWSRDSAsset.provider(), identifier=asset_identifier, - vpc=AWSVPCV1( - vpc_id=vpc_id, - region=region, - account=CNAAWSAccountRoleARNs( - name="acc", - cna=CNAAWSSpecV1(defaultRoleARN=arn, moduleRoleARNS=None), - ), - ), + name=db_name, defaults=None, - overrides=CNARDSInstanceOverridesV1( - name=db_name, + overrides=CNARDSInstanceConfig( + vpc=AWSVPCV1( + vpc_id=vpc_id, + region=region, + account=CNAAWSAccountRoleARNs( + name="acc", + cna=CNAAWSSpecV1(defaultRoleARN=arn, moduleRoleARNS=None), + ), + ), engine=engine, engine_version=engine_version, + major_engine_version=major_engine_version, allocated_storage=allocated_storage, max_allocated_storage=max_allocated_storage, instance_class=instance_class, db_subnet_group_name=db_subnet_group_name, - username=None, + username=username, + maintenance_window=maintenance_window, backup_retention_period=backup_retention_period, + backup_window=backup_window, + multi_az=multi_az, + deletion_protection=deletion_protection, + apply_immediately=apply_immediately, ), ) - asset = AWSRDSAsset.from_query_class(query_asset) + + +def test_from_query_class(rds_query_asset: CNARDSInstanceV1): + asset = AWSRDSAsset.from_query_class(rds_query_asset) assert isinstance(asset, AWSRDSAsset) assert asset.name == asset_identifier assert asset.region == region @@ -61,34 +82,20 @@ def test_from_query_class(): assert asset.max_allocated_storage == str(max_allocated_storage) assert asset.role_arn == arn assert asset.backup_retention_period == backup_retention_period + assert asset.apply_immediately == apply_immediately + assert asset.multi_az == multi_az + assert asset.deletion_protection == deletion_protection -def test_from_query_class_db_name_default(): - asset_identifier = "identifier" - query_asset = CNARDSInstanceV1( - provider=AWSRDSAsset.provider(), - identifier="identifier", - vpc=AWSVPCV1( - vpc_id="vpc-id", - region="region", - account=CNAAWSAccountRoleARNs( - name="acc", - cna=CNAAWSSpecV1(defaultRoleARN="arn", moduleRoleARNS=None), - ), - ), - defaults=None, - overrides=CNARDSInstanceOverridesV1( - name=None, - engine="postgres", - engine_version="14.2", - allocated_storage=10, - max_allocated_storage=20, - instance_class="instance-class", - db_subnet_group_name="db-subnet-group-name", - username=None, - backup_retention_period=7, - ), - ) - asset = AWSRDSAsset.from_query_class(query_asset) +def test_from_query_class_db_name_default(rds_query_asset: CNARDSInstanceV1): + rds_query_asset.name = None + asset = AWSRDSAsset.from_query_class(rds_query_asset) assert isinstance(asset, AWSRDSAsset) assert asset.identifier == asset_identifier + + +def test_from_query_class_no_overrides_no_defaults(rds_query_asset: CNARDSInstanceV1): + rds_query_asset.overrides = None + rds_query_asset.defaults = None + with pytest.raises(AssetError): + AWSRDSAsset.from_query_class(rds_query_asset) diff --git a/reconcile/test/cna/test_integration.py b/reconcile/test/cna/test_integration.py index ecfbe757a0..2fc3bc4f29 100644 --- a/reconcile/test/cna/test_integration.py +++ b/reconcile/test/cna/test_integration.py @@ -26,7 +26,7 @@ from reconcile.cna.state import State from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, - CNANullAssetOverridesV1, + CNANullAssetConfig, ExternalResourcesProvisionerV1, NamespaceCNAssetV1, ClusterV1, @@ -190,9 +190,10 @@ def test_integration_assemble_current_states( CNANullAssetV1( provider="null-asset", identifier="test", - overrides=CNANullAssetOverridesV1( + overrides=CNANullAssetConfig( addr_block="123", ), + defaults=None, ) ] ) @@ -211,12 +212,16 @@ def test_integration_assemble_current_states( CNANullAssetV1( provider="null-asset", identifier="test", - overrides=CNANullAssetOverridesV1( + overrides=CNANullAssetConfig( addr_block="123", ), + defaults=None, ), CNANullAssetV1( - provider="null-asset", identifier="test2", overrides=None + provider="null-asset", + identifier="test2", + overrides=None, + defaults=None, ), ] ) diff --git a/reconcile/test/cna/test_null.py b/reconcile/test/cna/test_null.py index 2bc7fa2338..a727a465e2 100644 --- a/reconcile/test/cna/test_null.py +++ b/reconcile/test/cna/test_null.py @@ -1,7 +1,7 @@ from reconcile.cna.assets.null import NullAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, - CNANullAssetOverridesV1, + CNANullAssetConfig, ) @@ -11,7 +11,8 @@ def test_from_query_class(): query_asset = CNANullAssetV1( provider=NullAsset.provider(), identifier=identifier, - overrides=CNANullAssetOverridesV1(addr_block=addr_block), + overrides=CNANullAssetConfig(addr_block=addr_block), + defaults=None, ) asset = NullAsset.from_query_class(query_asset) assert isinstance(asset, NullAsset) @@ -22,7 +23,10 @@ def test_from_query_class(): def test_from_query_class_no_overrides(): identifier = "name" query_asset = CNANullAssetV1( - provider=NullAsset.provider(), identifier=identifier, overrides=None + provider=NullAsset.provider(), + identifier=identifier, + overrides=None, + defaults=None, ) asset = NullAsset.from_query_class(query_asset) assert isinstance(asset, NullAsset) diff --git a/reconcile/test/test_utils_external_resources.py b/reconcile/test/test_utils_external_resources.py index bf555f1de7..273a4060eb 100644 --- a/reconcile/test/test_utils_external_resources.py +++ b/reconcile/test/test_utils_external_resources.py @@ -6,10 +6,8 @@ from reconcile.utils.external_resource_spec import ( ExternalResourceSpec, - TypedExternalResourceSpec, ) import reconcile.utils.external_resources as uer -from reconcile.gql_definitions.fragments.resource_file import ResourceFile @pytest.fixture @@ -262,7 +260,7 @@ class MyResource(BaseModel): identifier: str -class ResourceOverrides(BaseModel): +class ResourceConfig(BaseModel): field_1: Optional[str] field_2: Optional[str] @@ -270,8 +268,8 @@ class ResourceOverrides(BaseModel): class OverrideableResource(BaseModel): provider: str identifier: str - overrides: Optional[ResourceOverrides] - defaults: Optional[ResourceFile] + overrides: Optional[ResourceConfig] + defaults: Optional[ResourceConfig] class TestNamespaceExternalResource(BaseModel): @@ -332,79 +330,3 @@ def test_get_external_resource_specs_for_namespace_wrong_type(namespace: TestNam uer.get_external_resource_specs_for_namespace( namespace, OverrideableResource, None ) - - -def test_typed_external_resource_resolve_no_defaults(namespace: TestNamespace): - """ - In this scenario, the resource has no defaults, so overrides remain untouched. - """ - assert namespace.external_resources is not None - spec = TypedExternalResourceSpec[OverrideableResource]( - namespace_spec=namespace, - namespace_external_resource=namespace.external_resources[0], - spec=OverrideableResource( - provider="p", - identifier="i", - overrides=ResourceOverrides(field_1="f1", field_2="f2"), - defaults=None, - ), - ) - resolved_spec = spec.resolve() - assert resolved_spec.spec.overrides == spec.spec.overrides - - -def test_typed_external_resource_resolve_defaults_overrides( - namespace: TestNamespace, -): - """ - This scenario tests defaults overwriting undefined overrides. - """ - overwrite_f2 = "f2_override" - default_f1 = "f1_default" - default_f2 = "f2_default" - assert namespace.external_resources is not None - spec = TypedExternalResourceSpec[OverrideableResource]( - namespace_spec=namespace, - namespace_external_resource=namespace.external_resources[0], - spec=OverrideableResource( - provider="p", - identifier="i", - overrides=ResourceOverrides(field_1=None, field_2=overwrite_f2), - defaults=ResourceFile( - resourceFileSchema=None, - content=f"field_1: {default_f1}\nfield_2: {default_f2}", - ), - ), - ) - resolved_spec = spec.resolve() - assert resolved_spec.spec.overrides is not None - assert resolved_spec.spec.overrides.field_1 == default_f1 - assert resolved_spec.spec.overrides.field_2 == overwrite_f2 - - -def test_typed_external_resource_resolve_override_none( - namespace: TestNamespace, -): - """ - This scenario tests that a missing override is created from defaults. - """ - default_f1 = "f1_default" - default_f2 = "f2_default" - assert namespace.external_resources is not None - spec = TypedExternalResourceSpec[OverrideableResource]( - namespace_spec=namespace, - namespace_external_resource=namespace.external_resources[0], - spec=OverrideableResource( - provider="p", - identifier="i", - overrides=None, - defaults=ResourceFile( - resourceFileSchema=None, - content=f"field_1: {default_f1}\nfield_2: {default_f2}", - ), - ), - ) - resolved_spec = spec.resolve() - assert resolved_spec.spec.overrides is not None - assert resolved_spec.spec.overrides.field_1 == default_f1 - assert resolved_spec.spec.overrides.field_2 == default_f2 diff --git a/reconcile/utils/external_resource_spec.py b/reconcile/utils/external_resource_spec.py index 1572d50df7..aaf8aae76a 100644 --- a/reconcile/utils/external_resource_spec.py +++ b/reconcile/utils/external_resource_spec.py @@ -24,8 +24,6 @@ from pydantic.dataclasses import dataclass from reconcile import openshift_resources_base -from reconcile.gql_definitions.fragments.resource_file import ResourceFile -import anymarkup class OutputFormatProcessor: @@ -129,7 +127,7 @@ def dict(self, *args, **kwargs) -> dict[str, Any]: @runtime_checkable class DefaultableExternalResource(ExternalResource, Protocol): @property - def defaults(self) -> Optional[ResourceFile]: + def defaults(self) -> Optional[Any]: ... @abstractmethod @@ -319,17 +317,11 @@ def __init__( ) def get_defaults_data(self) -> dict[str, Any]: - if isinstance(self.spec, DefaultableExternalResource) and self.spec.defaults: - try: - defaults_values = anymarkup.parse( - self.spec.defaults.content, force_types=None - ) - defaults_values.pop("$schema", None) - return defaults_values - except anymarkup.AnyMarkupError: - # todo error handling - raise Exception("Could not parse data. Skipping resource") - return {} + if not isinstance(self.spec, DefaultableExternalResource): + return {} + if self.spec.defaults is None: + return {} + return self.spec.defaults.dict(by_alias=True) def get_overrides_data(self) -> dict[str, Any]: if not isinstance(self.spec, OverridableExternalResource): @@ -354,23 +346,3 @@ def get_overridable_fields(self) -> Sequence[str]: return overrides_class.__annotations__.keys() else: raise ValueError("resource is not overridable") - - def resolve(self) -> "TypedExternalResourceSpec[T]": - if self.is_overridable(): - overrides_data = self.get_overrides_data() - defaults_data = self.get_defaults_data() - - for field_name in self.get_overridable_fields(): - if overrides_data.get(field_name) is None: - overrides_data[field_name] = defaults_data.get(field_name) - else: - overrides_data = {} - - new_spec_attr = self.spec.dict(by_alias=True) - new_spec_attr[EXTERNAL_RESOURCE_SPEC_OVERRIDES_PROPERTY] = overrides_data - new_spec = type(self.spec)(**new_spec_attr) - return TypedExternalResourceSpec( - namespace_spec=self.namespace_spec, - namespace_external_resource=self.namespace_external_resource, - spec=new_spec, - ) From 8b72f232df6aadce77339ba12b21d85ef6037ffc Mon Sep 17 00:00:00 2001 From: Karl Fischer Date: Fri, 18 Nov 2022 16:02:51 +0100 Subject: [PATCH 08/20] set secret_name (#2976) --- reconcile/cna/assets/asset.py | 1 + reconcile/cna/integration.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index aa88098c4e..1899cf619b 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -95,6 +95,7 @@ class AssetModelConfig: class Binding: cluster_id: str namespace: str + secret_name: str @dataclass(frozen=True, config=AssetModelConfig) diff --git a/reconcile/cna/integration.py b/reconcile/cna/integration.py index f149405895..18d32aeb1d 100644 --- a/reconcile/cna/integration.py +++ b/reconcile/cna/integration.py @@ -33,7 +33,7 @@ create_secret_reader, ) from reconcile.utils.semver_helper import make_semver -from reconcile.cna.assets.asset import UnknownAssetTypeError, AssetError, Binding +from reconcile.cna.assets.asset import UnknownAssetTypeError, AssetError, Binding, Asset from reconcile.cna.assets.asset_factory import ( asset_factory_from_schema, asset_factory_from_raw_data, @@ -87,6 +87,8 @@ def assemble_desired_states(self): Binding( cluster_id=namespace.cluster.spec.q_id, namespace=namespace.name, + # For now secret_name is implicit. + secret_name=f"{asset.asset_type()}-{asset.name}", ) ) @@ -106,6 +108,7 @@ def assemble_current_states(self): Binding( cluster_id=binding.get("cluster_id", ""), namespace=binding.get("namespace", ""), + secret_name=binding.get("secret_name", ""), ) ) state.add_asset(asset) From 4f602f6b1b1055a927a39f3160d2ca5d5b2ffce3 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Thu, 24 Nov 2022 08:22:25 +0100 Subject: [PATCH 09/20] [cna] RDS overrides as json with all optional (#2982) revert experiment where overrides and defaults are exactly the same schema. the result was that all fields in overrides and defaults needed to be optional so they can be used in both places without making overrides mandatory. in this PR, overrides are now all optional, follow a schema in jsonschema but none in GQL where they are just JSON --- reconcile/cna/assets/asset.py | 23 +-- reconcile/cna/assets/aws_assume_role.py | 4 +- reconcile/cna/assets/aws_rds.py | 30 +--- reconcile/cna/assets/null.py | 4 +- reconcile/cna/integration.py | 2 +- .../cna/queries/aws_assume_role_config.gql | 5 - .../cna/queries/aws_assume_role_config.py | 25 ---- .../cna/queries/aws_rds_instance_config.gql | 25 ---- .../cna/queries/aws_rds_instance_config.py | 51 ------- .../cna/queries/cna_resources.gql | 37 +++-- .../cna/queries/cna_resources.py | 135 ++++++++++-------- .../cna/queries/null_asset_config.gql | 5 - .../cna/queries/null_asset_config.py | 25 ---- reconcile/test/cna/test_asset.py | 9 +- reconcile/test/cna/test_aws_assume_role.py | 6 +- reconcile/test/cna/test_aws_rds.py | 27 ++-- reconcile/test/cna/test_integration.py | 16 ++- reconcile/test/cna/test_null.py | 8 +- reconcile/utils/external_resource_spec.py | 10 +- 19 files changed, 173 insertions(+), 274 deletions(-) delete mode 100644 reconcile/gql_definitions/cna/queries/aws_assume_role_config.gql delete mode 100644 reconcile/gql_definitions/cna/queries/aws_assume_role_config.py delete mode 100644 reconcile/gql_definitions/cna/queries/aws_rds_instance_config.gql delete mode 100644 reconcile/gql_definitions/cna/queries/aws_rds_instance_config.py delete mode 100644 reconcile/gql_definitions/cna/queries/null_asset_config.gql delete mode 100644 reconcile/gql_definitions/cna/queries/null_asset_config.py diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index 1899cf619b..6c469f5367 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -238,27 +238,32 @@ def _get_config_class_type(cls) -> Type[ConfigClass]: return get_args(cls.__orig_bases__[0])[1] # type: ignore[attr-defined] @classmethod - def _get_config( - cls, attribute: str, spec: AssetQueryClass - ) -> Optional[ConfigClass]: - if not (configs := getattr(spec, attribute, None)): + def _get_defaults(cls, spec: AssetQueryClass) -> Optional[ConfigClass]: + if not (configs := getattr(spec, "defaults", None)): return None config_class = cls._get_config_class_type() if isinstance(configs, config_class): return configs else: raise AssetError( - f"{attribute} for asset {spec.provider}:{spec.identifier} are not of expected type {config_class}" + f"defaults for asset {spec.provider}:{spec.identifier} are not of expected type {config_class}" ) + @classmethod + def _get_overrides(cls, spec: AssetQueryClass) -> dict[str, Any]: + overrides = getattr(spec, "overrides", None) + if overrides and isinstance(overrides, dict): + return overrides + else: + return {} + @classmethod def aggregate_config(cls, spec: AssetQueryClass) -> ConfigClass: - defaults = cls._get_config("defaults", spec) - overrides = cls._get_config("overrides", spec) + defaults = cls._get_defaults(spec) + overrides = cls._get_overrides(spec) config_class = cls._get_config_class_type() data = { - property: getattr(overrides, property, None) - or getattr(defaults, property, None) + property: overrides.get(property) or getattr(defaults, property, None) for property in config_class.__annotations__.keys() } return config_class(**data) diff --git a/reconcile/cna/assets/aws_assume_role.py b/reconcile/cna/assets/aws_assume_role.py index 63626fbda4..7ef476cef1 100644 --- a/reconcile/cna/assets/aws_assume_role.py +++ b/reconcile/cna/assets/aws_assume_role.py @@ -13,12 +13,12 @@ from reconcile.cna.assets.aws_utils import aws_role_arn_for_module from reconcile.gql_definitions.cna.queries.cna_resources import ( CNAAssumeRoleAssetV1, - CNAAssumeRoleAssetConfig, + CNAAssumeRoleAssetConfigV1, ) @dataclass(frozen=True, config=AssetModelConfig) -class AWSAssumeRoleAsset(Asset[CNAAssumeRoleAssetV1, CNAAssumeRoleAssetConfig]): +class AWSAssumeRoleAsset(Asset[CNAAssumeRoleAssetV1, CNAAssumeRoleAssetConfigV1]): verify_slug: Optional[str] = Field(None, alias="verify-slug") role_arn: str = Field(alias="role_arn") diff --git a/reconcile/cna/assets/aws_rds.py b/reconcile/cna/assets/aws_rds.py index d0a088928c..f697ba2af8 100644 --- a/reconcile/cna/assets/aws_rds.py +++ b/reconcile/cna/assets/aws_rds.py @@ -13,12 +13,12 @@ from reconcile.cna.assets.aws_utils import aws_role_arn_for_module from reconcile.gql_definitions.cna.queries.cna_resources import ( CNARDSInstanceV1, - CNARDSInstanceConfig, + CNARDSInstanceDefaultsV1, ) @dataclass(frozen=True, config=AssetModelConfig) -class AWSRDSAsset(Asset[CNARDSInstanceV1, CNARDSInstanceConfig]): +class AWSRDSAsset(Asset[CNARDSInstanceV1, CNARDSInstanceDefaultsV1]): identifier: str = Field(alias="identifier") vpc_id: str = Field(alias="vpc_id") role_arn: str = Field(alias="role_arn") @@ -51,8 +51,6 @@ def asset_type() -> AssetType: @classmethod def from_query_class(cls, asset: CNARDSInstanceV1) -> Asset: config = cls.aggregate_config(asset) - if config.vpc is None: - raise AssetError("Missing VPC configuration for asset") aws_cna_cfg = config.vpc.account.cna role_arn = aws_role_arn_for_module(aws_cna_cfg, AssetType.AWS_RDS.value) @@ -61,21 +59,6 @@ def from_query_class(cls, asset: CNARDSInstanceV1) -> Asset: f"No CNA roles configured for AWS account {config.vpc.account.name}" ) - if not (db_subnet_group_name := config.db_subnet_group_name): - raise AssetError( - f"No db_subnet_group_name provided for RDS instance {asset.identifier}" - ) - if not (instance_class := config.instance_class): - raise AssetError( - f"No instance_class provided for RDS instance {asset.identifier}" - ) - if not (engine := config.engine): - raise AssetError(f"No engine provided for RDS instance {asset.identifier}") - if not (max_allocated_storage := config.max_allocated_storage): - raise AssetError( - f"No max_allocated_storage provided for RDS instance {asset.identifier}" - ) - return AWSRDSAsset( id=None, href=None, @@ -85,12 +68,13 @@ def from_query_class(cls, asset: CNARDSInstanceV1) -> Asset: identifier=asset.name or asset.identifier, vpc_id=config.vpc.vpc_id, role_arn=role_arn, - db_subnet_group_name=db_subnet_group_name, - engine=engine, + db_subnet_group_name=config.db_subnet_group_name, + engine=config.engine, engine_version=config.engine_version, - instance_class=instance_class, + major_engine_version=config.engine_version.split(".")[0], + instance_class=config.instance_class, allocated_storage=str(config.allocated_storage), - max_allocated_storage=str(max_allocated_storage), + max_allocated_storage=str(config.max_allocated_storage), region=config.vpc.region, backup_retention_period=config.backup_retention_period, backup_window=None, diff --git a/reconcile/cna/assets/null.py b/reconcile/cna/assets/null.py index b3bd7095bf..0b33bc4f46 100644 --- a/reconcile/cna/assets/null.py +++ b/reconcile/cna/assets/null.py @@ -13,12 +13,12 @@ ) from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, - CNANullAssetConfig, + CNANullAssetConfigV1, ) @dataclass(frozen=True, config=AssetModelConfig) -class NullAsset(Asset[CNANullAssetV1, CNANullAssetConfig]): +class NullAsset(Asset[CNANullAssetV1, CNANullAssetConfigV1]): addr_block: Optional[str] = Field(None, alias="AddrBlock") @staticmethod diff --git a/reconcile/cna/integration.py b/reconcile/cna/integration.py index 18d32aeb1d..6d3a5e7364 100644 --- a/reconcile/cna/integration.py +++ b/reconcile/cna/integration.py @@ -33,7 +33,7 @@ create_secret_reader, ) from reconcile.utils.semver_helper import make_semver -from reconcile.cna.assets.asset import UnknownAssetTypeError, AssetError, Binding, Asset +from reconcile.cna.assets.asset import UnknownAssetTypeError, AssetError, Binding from reconcile.cna.assets.asset_factory import ( asset_factory_from_schema, asset_factory_from_raw_data, diff --git a/reconcile/gql_definitions/cna/queries/aws_assume_role_config.gql b/reconcile/gql_definitions/cna/queries/aws_assume_role_config.gql deleted file mode 100644 index 182f3be0e6..0000000000 --- a/reconcile/gql_definitions/cna/queries/aws_assume_role_config.gql +++ /dev/null @@ -1,5 +0,0 @@ -# qenerate: plugin=pydantic_v1 - -fragment CNAAssumeRoleAssetConfig on CNAAssumeRoleAssetConfig_v1 { - slug -} diff --git a/reconcile/gql_definitions/cna/queries/aws_assume_role_config.py b/reconcile/gql_definitions/cna/queries/aws_assume_role_config.py deleted file mode 100644 index d6eb4ed004..0000000000 --- a/reconcile/gql_definitions/cna/queries/aws_assume_role_config.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY! -""" -from enum import Enum # noqa: F401 # pylint: disable=W0611 -from typing import ( # noqa: F401 # pylint: disable=W0611 - Any, - Callable, - Optional, - Union, -) - -from pydantic import ( # noqa: F401 # pylint: disable=W0611 - BaseModel, - Extra, - Field, - Json, -) - - -class CNAAssumeRoleAssetConfig(BaseModel): - slug: Optional[str] = Field(..., alias="slug") - - class Config: - smart_union = True - extra = Extra.forbid diff --git a/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.gql b/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.gql deleted file mode 100644 index 905c771a89..0000000000 --- a/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.gql +++ /dev/null @@ -1,25 +0,0 @@ -# qenerate: plugin=pydantic_v1 - -fragment CNARDSInstanceConfig on CNARDSInstanceConfig_v1 { - vpc { - vpc_id - region - account { - ... CNAAWSAccountRoleARNs - } - } - db_subnet_group_name - instance_class - allocated_storage - max_allocated_storage - engine - engine_version - major_engine_version - username - maintenance_window - backup_retention_period - backup_window - multi_az - deletion_protection - apply_immediately -} diff --git a/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.py b/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.py deleted file mode 100644 index d09245d088..0000000000 --- a/reconcile/gql_definitions/cna/queries/aws_rds_instance_config.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY! -""" -from enum import Enum # noqa: F401 # pylint: disable=W0611 -from typing import ( # noqa: F401 # pylint: disable=W0611 - Any, - Callable, - Optional, - Union, -) - -from pydantic import ( # noqa: F401 # pylint: disable=W0611 - BaseModel, - Extra, - Field, - Json, -) - -from reconcile.gql_definitions.cna.queries.aws_arn import CNAAWSAccountRoleARNs - - -class AWSVPCV1(BaseModel): - vpc_id: str = Field(..., alias="vpc_id") - region: str = Field(..., alias="region") - account: CNAAWSAccountRoleARNs = Field(..., alias="account") - - class Config: - smart_union = True - extra = Extra.forbid - - -class CNARDSInstanceConfig(BaseModel): - vpc: Optional[AWSVPCV1] = Field(..., alias="vpc") - db_subnet_group_name: Optional[str] = Field(..., alias="db_subnet_group_name") - instance_class: Optional[str] = Field(..., alias="instance_class") - allocated_storage: Optional[int] = Field(..., alias="allocated_storage") - max_allocated_storage: Optional[int] = Field(..., alias="max_allocated_storage") - engine: Optional[str] = Field(..., alias="engine") - engine_version: Optional[str] = Field(..., alias="engine_version") - major_engine_version: Optional[str] = Field(..., alias="major_engine_version") - username: Optional[str] = Field(..., alias="username") - maintenance_window: Optional[str] = Field(..., alias="maintenance_window") - backup_retention_period: Optional[int] = Field(..., alias="backup_retention_period") - backup_window: Optional[str] = Field(..., alias="backup_window") - multi_az: Optional[bool] = Field(..., alias="multi_az") - deletion_protection: Optional[bool] = Field(..., alias="deletion_protection") - apply_immediately: Optional[bool] = Field(..., alias="apply_immediately") - - class Config: - smart_union = True - extra = Extra.forbid diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.gql b/reconcile/gql_definitions/cna/queries/cna_resources.gql index 5e8f3e1822..5e59b7c414 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.gql +++ b/reconcile/gql_definitions/cna/queries/cna_resources.gql @@ -20,31 +20,44 @@ query CNAssets { identifier ... on CNANullAsset_v1 { defaults { - ... CNANullAssetConfig - } - overrides { - ... CNANullAssetConfig + addr_block } + overrides } ... on CNARDSInstance_v1 { name defaults { - ... CNARDSInstanceConfig - } - overrides { - ... CNARDSInstanceConfig + vpc { + vpc_id + region + account { + ... CNAAWSAccountRoleARNs + } + } + db_subnet_group_name + instance_class + allocated_storage + max_allocated_storage + engine + engine_version + username + maintenance_window + backup_retention_period + backup_window + multi_az + deletion_protection + apply_immediately } + overrides } ... on CNAAssumeRoleAsset_v1{ aws_account { ... CNAAWSAccountRoleARNs } defaults { - ... CNAAssumeRoleAssetConfig - } - overrides { - ... CNAAssumeRoleAssetConfig + slug } + overrides } } } diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.py b/reconcile/gql_definitions/cna/queries/cna_resources.py index d77bc9a017..054d68ef17 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.py +++ b/reconcile/gql_definitions/cna/queries/cna_resources.py @@ -17,13 +17,6 @@ ) from reconcile.gql_definitions.cna.queries.aws_arn import CNAAWSAccountRoleARNs -from reconcile.gql_definitions.cna.queries.aws_assume_role_config import ( - CNAAssumeRoleAssetConfig, -) -from reconcile.gql_definitions.cna.queries.null_asset_config import CNANullAssetConfig -from reconcile.gql_definitions.cna.queries.aws_rds_instance_config import ( - CNARDSInstanceConfig, -) DEFINITION = """ @@ -38,38 +31,6 @@ } } -fragment CNAAssumeRoleAssetConfig on CNAAssumeRoleAssetConfig_v1 { - slug -} - -fragment CNANullAssetConfig on CNANullAssetConfig_v1 { - addr_block -} - -fragment CNARDSInstanceConfig on CNARDSInstanceConfig_v1 { - vpc { - vpc_id - region - account { - ... CNAAWSAccountRoleARNs - } - } - db_subnet_group_name - instance_class - allocated_storage - max_allocated_storage - engine - engine_version - major_engine_version - username - maintenance_window - backup_retention_period - backup_window - multi_az - deletion_protection - apply_immediately -} - query CNAssets { namespaces: namespaces_v1 { name @@ -90,31 +51,44 @@ identifier ... on CNANullAsset_v1 { defaults { - ... CNANullAssetConfig - } - overrides { - ... CNANullAssetConfig + addr_block } + overrides } ... on CNARDSInstance_v1 { name defaults { - ... CNARDSInstanceConfig - } - overrides { - ... CNARDSInstanceConfig + vpc { + vpc_id + region + account { + ... CNAAWSAccountRoleARNs + } + } + db_subnet_group_name + instance_class + allocated_storage + max_allocated_storage + engine + engine_version + username + maintenance_window + backup_retention_period + backup_window + multi_az + deletion_protection + apply_immediately } + overrides } ... on CNAAssumeRoleAsset_v1{ aws_account { ... CNAAWSAccountRoleARNs } defaults { - ... CNAAssumeRoleAssetConfig - } - overrides { - ... CNAAssumeRoleAssetConfig + slug } + overrides } } } @@ -166,9 +140,48 @@ class Config: extra = Extra.forbid +class CNANullAssetConfigV1(BaseModel): + addr_block: Optional[str] = Field(..., alias="addr_block") + + class Config: + smart_union = True + extra = Extra.forbid + + class CNANullAssetV1(CNAssetV1): - defaults: Optional[CNANullAssetConfig] = Field(..., alias="defaults") - overrides: Optional[CNANullAssetConfig] = Field(..., alias="overrides") + defaults: Optional[CNANullAssetConfigV1] = Field(..., alias="defaults") + overrides: Optional[Json] = Field(..., alias="overrides") + + class Config: + smart_union = True + extra = Extra.forbid + + +class AWSVPCV1(BaseModel): + vpc_id: str = Field(..., alias="vpc_id") + region: str = Field(..., alias="region") + account: CNAAWSAccountRoleARNs = Field(..., alias="account") + + class Config: + smart_union = True + extra = Extra.forbid + + +class CNARDSInstanceDefaultsV1(BaseModel): + vpc: AWSVPCV1 = Field(..., alias="vpc") + db_subnet_group_name: str = Field(..., alias="db_subnet_group_name") + instance_class: str = Field(..., alias="instance_class") + allocated_storage: int = Field(..., alias="allocated_storage") + max_allocated_storage: int = Field(..., alias="max_allocated_storage") + engine: str = Field(..., alias="engine") + engine_version: str = Field(..., alias="engine_version") + username: Optional[str] = Field(..., alias="username") + maintenance_window: Optional[str] = Field(..., alias="maintenance_window") + backup_retention_period: Optional[int] = Field(..., alias="backup_retention_period") + backup_window: Optional[str] = Field(..., alias="backup_window") + multi_az: Optional[bool] = Field(..., alias="multi_az") + deletion_protection: Optional[bool] = Field(..., alias="deletion_protection") + apply_immediately: Optional[bool] = Field(..., alias="apply_immediately") class Config: smart_union = True @@ -177,8 +190,16 @@ class Config: class CNARDSInstanceV1(CNAssetV1): name: Optional[str] = Field(..., alias="name") - defaults: Optional[CNARDSInstanceConfig] = Field(..., alias="defaults") - overrides: Optional[CNARDSInstanceConfig] = Field(..., alias="overrides") + defaults: Optional[CNARDSInstanceDefaultsV1] = Field(..., alias="defaults") + overrides: Optional[Json] = Field(..., alias="overrides") + + class Config: + smart_union = True + extra = Extra.forbid + + +class CNAAssumeRoleAssetConfigV1(BaseModel): + slug: Optional[str] = Field(..., alias="slug") class Config: smart_union = True @@ -187,8 +208,8 @@ class Config: class CNAAssumeRoleAssetV1(CNAssetV1): aws_account: CNAAWSAccountRoleARNs = Field(..., alias="aws_account") - defaults: Optional[CNAAssumeRoleAssetConfig] = Field(..., alias="defaults") - overrides: Optional[CNAAssumeRoleAssetConfig] = Field(..., alias="overrides") + defaults: Optional[CNAAssumeRoleAssetConfigV1] = Field(..., alias="defaults") + overrides: Optional[Json] = Field(..., alias="overrides") class Config: smart_union = True diff --git a/reconcile/gql_definitions/cna/queries/null_asset_config.gql b/reconcile/gql_definitions/cna/queries/null_asset_config.gql deleted file mode 100644 index 48e008560d..0000000000 --- a/reconcile/gql_definitions/cna/queries/null_asset_config.gql +++ /dev/null @@ -1,5 +0,0 @@ -# qenerate: plugin=pydantic_v1 - -fragment CNANullAssetConfig on CNANullAssetConfig_v1 { - addr_block -} diff --git a/reconcile/gql_definitions/cna/queries/null_asset_config.py b/reconcile/gql_definitions/cna/queries/null_asset_config.py deleted file mode 100644 index cadb55ab6a..0000000000 --- a/reconcile/gql_definitions/cna/queries/null_asset_config.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY! -""" -from enum import Enum # noqa: F401 # pylint: disable=W0611 -from typing import ( # noqa: F401 # pylint: disable=W0611 - Any, - Callable, - Optional, - Union, -) - -from pydantic import ( # noqa: F401 # pylint: disable=W0611 - BaseModel, - Extra, - Field, - Json, -) - - -class CNANullAssetConfig(BaseModel): - addr_block: Optional[str] = Field(..., alias="addr_block") - - class Config: - smart_union = True - extra = Extra.forbid diff --git a/reconcile/test/cna/test_asset.py b/reconcile/test/cna/test_asset.py index 03905113b9..b2f4d416a6 100644 --- a/reconcile/test/cna/test_asset.py +++ b/reconcile/test/cna/test_asset.py @@ -1,4 +1,5 @@ from typing import Any, Mapping, MutableMapping, Optional +import json from reconcile.cna.assets.asset import ( Asset, AssetStatus, @@ -245,9 +246,11 @@ def build_assume_role_typed_external_resource( "name": "acc", "cna": {"defaultRoleARN": role_arn, "moduleRoleARNS": None}, }, - "overrides": { - "slug": verify_slug_override, - }, + "overrides": json.dumps( + { + "slug": verify_slug_override, + } + ), "defaults": { "slug": verify_slug_default, }, diff --git a/reconcile/test/cna/test_aws_assume_role.py b/reconcile/test/cna/test_aws_assume_role.py index 0ac4d55b94..963c75942b 100644 --- a/reconcile/test/cna/test_aws_assume_role.py +++ b/reconcile/test/cna/test_aws_assume_role.py @@ -1,7 +1,7 @@ +import json from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( CNAAssumeRoleAssetV1, - CNAAssumeRoleAssetConfig, ) from reconcile.gql_definitions.cna.queries.aws_arn import ( CNAAWSAccountRoleARNs, @@ -16,9 +16,7 @@ def test_from_query_class(): query_asset = CNAAssumeRoleAssetV1( provider=AWSAssumeRoleAsset.provider(), identifier=name, - overrides=CNAAssumeRoleAssetConfig( - slug=slug, - ), + overrides=json.dumps({"slug": slug}), defaults=None, aws_account=CNAAWSAccountRoleARNs( name="acc", diff --git a/reconcile/test/cna/test_aws_rds.py b/reconcile/test/cna/test_aws_rds.py index 81bf8ec280..a776d67209 100644 --- a/reconcile/test/cna/test_aws_rds.py +++ b/reconcile/test/cna/test_aws_rds.py @@ -1,14 +1,15 @@ +import json from reconcile.cna.assets.asset import AssetError from reconcile.cna.assets.aws_rds import AWSRDSAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( CNARDSInstanceV1, - CNARDSInstanceConfig, + CNARDSInstanceDefaultsV1, + AWSVPCV1, ) from reconcile.gql_definitions.cna.queries.aws_arn import ( CNAAWSAccountRoleARNs, CNAAWSSpecV1, ) -from reconcile.gql_definitions.cna.queries.aws_rds_instance_config import AWSVPCV1 import pytest @@ -19,7 +20,6 @@ arn = "arn" engine = "engine" engine_version = "14.2" -major_engine_version = "14" allocated_storage = 10 max_allocated_storage = 20 instance_class = "instance-class" @@ -39,8 +39,8 @@ def rds_query_asset() -> CNARDSInstanceV1: provider=AWSRDSAsset.provider(), identifier=asset_identifier, name=db_name, - defaults=None, - overrides=CNARDSInstanceConfig( + overrides=None, + defaults=CNARDSInstanceDefaultsV1( vpc=AWSVPCV1( vpc_id=vpc_id, region=region, @@ -51,7 +51,6 @@ def rds_query_asset() -> CNARDSInstanceV1: ), engine=engine, engine_version=engine_version, - major_engine_version=major_engine_version, allocated_storage=allocated_storage, max_allocated_storage=max_allocated_storage, instance_class=instance_class, @@ -73,11 +72,11 @@ def test_from_query_class(rds_query_asset: CNARDSInstanceV1): assert asset.name == asset_identifier assert asset.region == region assert asset.identifier == db_name - assert asset.vpc_id == vpc_id assert asset.db_subnet_group_name == db_subnet_group_name assert asset.instance_class == instance_class assert asset.engine == engine assert asset.engine_version == engine_version + assert asset.major_engine_version == "14" assert asset.allocated_storage == str(allocated_storage) assert asset.max_allocated_storage == str(max_allocated_storage) assert asset.role_arn == arn @@ -94,8 +93,12 @@ def test_from_query_class_db_name_default(rds_query_asset: CNARDSInstanceV1): assert asset.identifier == asset_identifier -def test_from_query_class_no_overrides_no_defaults(rds_query_asset: CNARDSInstanceV1): - rds_query_asset.overrides = None - rds_query_asset.defaults = None - with pytest.raises(AssetError): - AWSRDSAsset.from_query_class(rds_query_asset) +def test_from_query_class_engine_version_override(rds_query_asset: CNARDSInstanceV1): + engine_version_override = "15.2" + rds_query_asset.name = None + rds_query_asset.overrides = { + "engine_version": engine_version_override + } # type: ignore + asset = AWSRDSAsset.from_query_class(rds_query_asset) + assert isinstance(asset, AWSRDSAsset) + assert asset.engine_version == engine_version_override diff --git a/reconcile/test/cna/test_integration.py b/reconcile/test/cna/test_integration.py index 2fc3bc4f29..47f0fcb378 100644 --- a/reconcile/test/cna/test_integration.py +++ b/reconcile/test/cna/test_integration.py @@ -7,7 +7,8 @@ Optional, ) from unittest.mock import create_autospec - +from pytest import fixture +import json import pytest from pytest import fixture @@ -26,7 +27,6 @@ from reconcile.cna.state import State from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, - CNANullAssetConfig, ExternalResourcesProvisionerV1, NamespaceCNAssetV1, ClusterV1, @@ -190,8 +190,10 @@ def test_integration_assemble_current_states( CNANullAssetV1( provider="null-asset", identifier="test", - overrides=CNANullAssetConfig( - addr_block="123", + overrides=json.dumps( + { + "addr_block": "123", + } ), defaults=None, ) @@ -212,8 +214,10 @@ def test_integration_assemble_current_states( CNANullAssetV1( provider="null-asset", identifier="test", - overrides=CNANullAssetConfig( - addr_block="123", + overrides=json.dumps( + { + "addr_block": "123", + } ), defaults=None, ), diff --git a/reconcile/test/cna/test_null.py b/reconcile/test/cna/test_null.py index a727a465e2..ffcd49707f 100644 --- a/reconcile/test/cna/test_null.py +++ b/reconcile/test/cna/test_null.py @@ -1,8 +1,8 @@ from reconcile.cna.assets.null import NullAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( CNANullAssetV1, - CNANullAssetConfig, ) +import json def test_from_query_class(): @@ -11,7 +11,11 @@ def test_from_query_class(): query_asset = CNANullAssetV1( provider=NullAsset.provider(), identifier=identifier, - overrides=CNANullAssetConfig(addr_block=addr_block), + overrides=json.dumps( + { + "addr_block": addr_block, + } + ), defaults=None, ) asset = NullAsset.from_query_class(query_asset) diff --git a/reconcile/utils/external_resource_spec.py b/reconcile/utils/external_resource_spec.py index aaf8aae76a..6f4e44db42 100644 --- a/reconcile/utils/external_resource_spec.py +++ b/reconcile/utils/external_resource_spec.py @@ -94,7 +94,7 @@ def name(self) -> str: ... @abstractmethod - def dict(self, *args, **kwargs) -> dict[str, Any]: + def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: ... @@ -109,7 +109,7 @@ def identifier(self) -> str: ... @abstractmethod - def dict(self, *args, **kwargs) -> dict[str, Any]: + def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: ... @@ -120,7 +120,7 @@ def overrides(self) -> Optional[Any]: ... @abstractmethod - def dict(self, *args, **kwargs) -> dict[str, Any]: + def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: ... @@ -131,7 +131,7 @@ def defaults(self) -> Optional[Any]: ... @abstractmethod - def dict(self, *args, **kwargs) -> dict[str, Any]: + def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: ... @@ -167,7 +167,7 @@ def external_resources( ... @abstractmethod - def dict(self, *args, **kwargs) -> dict[str, Any]: + def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: ... From fabf67e6c67601ca73b327ada6bbc2deec2ac64f Mon Sep 17 00:00:00 2001 From: Karl Fischer Date: Fri, 25 Nov 2022 13:51:33 +0100 Subject: [PATCH 10/20] allow bool as asset type variable (#3005) --- reconcile/cna/assets/asset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index 6c469f5367..b064a60970 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -56,8 +56,10 @@ def asset_type_from_raw_asset(raw_asset: Mapping[str, Any]) -> Optional[AssetTyp class AssetTypeVariableType(Enum): STRING = "${string}" NUMBER = "${number}" + BOOL = "${bool}" LIST_STRING = "${list(string)}" LIST_NUMBER = "${list(number)}" + LIST_BOOL = "${list(bool)}" @dataclass(frozen=True) @@ -295,6 +297,8 @@ def _asset_type_metadata_variable_from_type_annotation( asset_type = AssetTypeVariableType.STRING elif type_hint == "int" or type_hint.endswith("[int]"): asset_type = AssetTypeVariableType.NUMBER + elif type_hint == "bool" or type_hint.endswith("[bool]"): + asset_type = AssetTypeVariableType.BOOL else: raise AssetError(f"Unsupported type hint {type_hint} for {property_name}") # TODO handle list types From 62a16479cc95154ee5d555b79dba1da14386e228 Mon Sep 17 00:00:00 2001 From: Karl Fischer Date: Mon, 28 Nov 2022 11:08:58 +0100 Subject: [PATCH 11/20] allow Error state (#3006) --- reconcile/cna/assets/asset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index b064a60970..5ecce8581c 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -83,6 +83,7 @@ class AssetStatus(Enum): TERMINATED = "Terminated" PENDING = "Pending" RUNNING = "Running" + ERROR = "Error" class AssetModelConfig: From 4633d937bfe41b512b15e04ed0183b6f933e6365 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Mon, 28 Nov 2022 12:41:31 +0100 Subject: [PATCH 12/20] fix import issues due to rebase Signed-off-by: Gerd Oberlechner --- reconcile/cna/assets/asset_factory.py | 2 +- reconcile/cna/assets/null.py | 4 +--- reconcile/cna/integration.py | 2 ++ reconcile/test/cna/test_aws_rds.py | 2 -- reconcile/utils/external_resources.py | 4 ++-- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/reconcile/cna/assets/asset_factory.py b/reconcile/cna/assets/asset_factory.py index 90301dd4be..a89d0dd680 100644 --- a/reconcile/cna/assets/asset_factory.py +++ b/reconcile/cna/assets/asset_factory.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Type +from typing import Any, Type from collections.abc import Mapping from typing import Any diff --git a/reconcile/cna/assets/null.py b/reconcile/cna/assets/null.py index 0b33bc4f46..5a637a34be 100644 --- a/reconcile/cna/assets/null.py +++ b/reconcile/cna/assets/null.py @@ -1,7 +1,5 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Optional -from collections.abc import Mapping +from typing import Optional from pydantic.dataclasses import dataclass from pydantic import Field diff --git a/reconcile/cna/integration.py b/reconcile/cna/integration.py index 6d3a5e7364..c90251b1cf 100644 --- a/reconcile/cna/integration.py +++ b/reconcile/cna/integration.py @@ -1,6 +1,8 @@ from collections import defaultdict from collections.abc import Iterable, Mapping +import logging + from reconcile.cna.client import CNAClient from reconcile.cna.state import State diff --git a/reconcile/test/cna/test_aws_rds.py b/reconcile/test/cna/test_aws_rds.py index a776d67209..8e66cfd163 100644 --- a/reconcile/test/cna/test_aws_rds.py +++ b/reconcile/test/cna/test_aws_rds.py @@ -1,5 +1,3 @@ -import json -from reconcile.cna.assets.asset import AssetError from reconcile.cna.assets.aws_rds import AWSRDSAsset from reconcile.gql_definitions.cna.queries.cna_resources import ( CNARDSInstanceV1, diff --git a/reconcile/utils/external_resources.py b/reconcile/utils/external_resources.py index 0808366724..35c0f9b8c2 100644 --- a/reconcile/utils/external_resources.py +++ b/reconcile/utils/external_resources.py @@ -5,7 +5,7 @@ Type, TypeVar, ) -from collections.abc import Mapping, MutableMapping, Set, List +from collections.abc import Mapping, MutableMapping import anymarkup @@ -33,7 +33,7 @@ def get_external_resource_specs_for_namespace( ) -> list[TypedExternalResourceSpec[T]]: if not namespace.managed_external_resources: return [] - specs: List[TypedExternalResourceSpec[T]] = [] + specs: list[TypedExternalResourceSpec[T]] = [] for e in namespace.external_resources or []: if isinstance(e, NamespaceExternalResource): for r in e.resources: From b863cedbc7cc35006d781a919b2888e6b8ff211c Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Tue, 29 Nov 2022 14:38:03 +0100 Subject: [PATCH 13/20] [cna] revert externalresourcespec (#3018) the changes have been extracted from the cna-integration branch into this PR https://github.com/app-sre/qontract-reconcile/pull/2990 --- reconcile/cna/assets/asset.py | 7 +- reconcile/cna/assets/asset_factory.py | 3 +- reconcile/cna/integration.py | 56 +++--- reconcile/test/cna/test_asset.py | 27 +-- .../test/test_utils_external_resources.py | 87 +-------- reconcile/utils/external_resource_spec.py | 166 +----------------- reconcile/utils/external_resources.py | 51 +----- 7 files changed, 39 insertions(+), 358 deletions(-) diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index 5ecce8581c..1585b3cb51 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -7,7 +7,6 @@ import copy from reconcile.gql_definitions.cna.queries.cna_resources import CNAssetV1 -from reconcile.utils.external_resource_spec import TypedExternalResourceSpec ASSET_ID_FIELD = "id" @@ -135,11 +134,11 @@ def from_query_class(cls, asset: AssetQueryClass) -> "Asset": @classmethod def from_external_resources( cls, - external_resource: TypedExternalResourceSpec[CNAssetV1], + external_resource: CNAssetV1, ) -> "Asset": query_class = cls._get_query_class_type() - if isinstance(external_resource.spec, query_class): - return cls.from_query_class(external_resource.spec) + if isinstance(external_resource, query_class): + return cls.from_query_class(external_resource) else: raise AssetError( f"CNA type {query_class} does not match " diff --git a/reconcile/cna/assets/asset_factory.py b/reconcile/cna/assets/asset_factory.py index a89d0dd680..609d2e0180 100644 --- a/reconcile/cna/assets/asset_factory.py +++ b/reconcile/cna/assets/asset_factory.py @@ -19,7 +19,6 @@ ASSET_HREF_FIELD, ASSET_NAME_FIELD, ) -from reconcile.utils.external_resource_spec import TypedExternalResourceSpec _ASSET_TYPE_SCHEME: dict[AssetType, Type[Asset]] = {} @@ -46,7 +45,7 @@ def asset_type_for_provider(provider: str) -> AssetType: def asset_factory_from_schema( - external_resource_spec: TypedExternalResourceSpec[CNAssetV1], + external_resource_spec: CNAssetV1, ) -> Asset: cna_dataclass = _dataclass_for_provider(external_resource_spec.provider) return cna_dataclass.from_external_resources(external_resource_spec) diff --git a/reconcile/cna/integration.py b/reconcile/cna/integration.py index c90251b1cf..49fcde454c 100644 --- a/reconcile/cna/integration.py +++ b/reconcile/cna/integration.py @@ -8,7 +8,6 @@ from reconcile.utils import gql from reconcile.utils.external_resources import ( - get_external_resource_specs_for_namespace, PROVIDER_CNA_EXPERIMENTAL, ) from reconcile.utils.ocm_base_client import OCMBaseClient @@ -19,10 +18,8 @@ query as cna_provisioners_query, ) from reconcile.gql_definitions.cna.queries.cna_resources import ( - CNAssetV1, NamespaceV1, -) -from reconcile.gql_definitions.cna.queries.cna_resources import ( + NamespaceCNAssetV1, query as namespaces_query, ) from reconcile.typed_queries.app_interface_vault_settings import ( @@ -66,33 +63,36 @@ def __init__( def assemble_desired_states(self): self._desired_states = defaultdict(State) for namespace in self._namespaces: - for spec in get_external_resource_specs_for_namespace( - namespace, CNAssetV1, PROVIDER_CNA_EXPERIMENTAL - ): - asset = asset_factory_from_schema(spec) - self._desired_states[spec.provisioner_name].add_asset(asset) - - # For now we assume that if an asset is bindable, then it - # always binds to its defining namespace - # TODO: probably this should also be done by passing the required namespace vars - # to the factory method. - if not asset.bindable(): + for provider in namespace.external_resources or []: + if provider.provider != PROVIDER_CNA_EXPERIMENTAL: continue - if not (namespace.cluster.spec and namespace.cluster.spec.q_id): - logging.warning( - "cannot bind asset %s because namespace %s does not have a cluster spec with a cluster id.", - asset, - namespace.name, - ) + if not isinstance(provider, NamespaceCNAssetV1): continue - asset.bindings.add( - Binding( - cluster_id=namespace.cluster.spec.q_id, - namespace=namespace.name, - # For now secret_name is implicit. - secret_name=f"{asset.asset_type()}-{asset.name}", + for resource in provider.resources or []: + asset = asset_factory_from_schema(resource) + self._desired_states[provider.provisioner.name].add_asset(asset) + + # For now we assume that if an asset is bindable, then it + # always binds to its defining namespace + # TODO: probably this should also be done by passing the required namespace vars + # to the factory method. + if not asset.bindable(): + continue + if not (namespace.cluster.spec and namespace.cluster.spec.q_id): + logging.warning( + "cannot bind asset %s because namespace %s does not have a cluster spec with a cluster id.", + asset, + namespace.name, + ) + continue + asset.bindings.add( + Binding( + cluster_id=namespace.cluster.spec.q_id, + namespace=namespace.name, + # For now secret_name is implicit. + secret_name=f"{asset.asset_type()}-{asset.name}", + ) ) - ) def assemble_current_states(self): self._current_states = defaultdict(State) diff --git a/reconcile/test/cna/test_asset.py b/reconcile/test/cna/test_asset.py index b2f4d416a6..462120f5f8 100644 --- a/reconcile/test/cna/test_asset.py +++ b/reconcile/test/cna/test_asset.py @@ -17,13 +17,7 @@ from reconcile.gql_definitions.cna.queries.cna_resources import ( CNAAssumeRoleAssetV1, CNAssetV1, - NamespaceV1, - NamespaceCNAssetV1, ) -from reconcile.utils.external_resource_spec import ( - TypedExternalResourceSpec, -) -from reconcile.utils.external_resources import PROVIDER_CNA_EXPERIMENTAL import pytest @@ -238,7 +232,7 @@ def build_assume_role_typed_external_resource( role_arn: str, verify_slug_override: Optional[str], verify_slug_default: Optional[str], -) -> TypedExternalResourceSpec[CNAssetV1]: +) -> CNAssetV1: resource = { "provider": AWSAssumeRoleAsset.provider(), "identifier": identifier, @@ -255,24 +249,7 @@ def build_assume_role_typed_external_resource( "slug": verify_slug_default, }, } - namespace_resource = { - "provider": PROVIDER_CNA_EXPERIMENTAL, - "provisioner": {"name": "some-ocm-org"}, - "resources": [resource], - } - namespace = { - "name": "ns-name", - "managedExternalResources": True, - "cluster": { - "spec": None, - }, - "externalResources": [namespace_resource], - } - return TypedExternalResourceSpec[CNAssetV1]( - namespace_spec=NamespaceV1(**namespace), - namespace_external_resource=NamespaceCNAssetV1(**namespace_resource), - spec=CNAAssumeRoleAssetV1(**resource), - ) + return CNAAssumeRoleAssetV1(**resource) def test_from_external_resources_with_default(): diff --git a/reconcile/test/test_utils_external_resources.py b/reconcile/test/test_utils_external_resources.py index 273a4060eb..e60b3c4d60 100644 --- a/reconcile/test/test_utils_external_resources.py +++ b/reconcile/test/test_utils_external_resources.py @@ -1,12 +1,8 @@ import json -from typing import Optional, Union import pytest -from pydantic import BaseModel +from reconcile.utils.external_resource_spec import ExternalResourceSpec -from reconcile.utils.external_resource_spec import ( - ExternalResourceSpec, -) import reconcile.utils.external_resources as uer @@ -249,84 +245,3 @@ def test_resource_value_resolver_overrides_and_defaults(mocker): "default_2": "override_data2", "default_3": "default_data3", } - - -class TestProvisionier(BaseModel): - name: str - - -class MyResource(BaseModel): - provider: str - identifier: str - - -class ResourceConfig(BaseModel): - field_1: Optional[str] - field_2: Optional[str] - - -class OverrideableResource(BaseModel): - provider: str - identifier: str - overrides: Optional[ResourceConfig] - defaults: Optional[ResourceConfig] - - -class TestNamespaceExternalResource(BaseModel): - provider: str - provisioner: TestProvisionier - resources: list[Union[MyResource, OverrideableResource]] - - -class TestNamespace(BaseModel): - name: str - managed_external_resources: bool - external_resources: Optional[list[TestNamespaceExternalResource]] - - -@pytest.fixture -def namespace() -> TestNamespace: - return TestNamespace( - name="ns", - managed_external_resources=True, - external_resources=[ - TestNamespaceExternalResource( - provider="pp", - provisioner=TestProvisionier(name="pn"), - resources=[ - MyResource(provider="rp", identifier="ri"), - ], - ) - ], - ) - - -def test_get_external_resource_specs_for_namespace( - namespace: TestNamespace, -): - external_resources = uer.get_external_resource_specs_for_namespace( - namespace, MyResource, None - ) - assert len(external_resources) == 1 - - assert external_resources[0].provision_provider == "pp" - assert external_resources[0].provisioner_name == "pn" - assert external_resources[0].namespace_name == "ns" - assert external_resources[0].provider == "rp" - assert external_resources[0].identifier == "ri" - - -def test_get_external_resource_specs_for_namespace_provisioning_provider_filter( - namespace: TestNamespace, -): - external_resources = uer.get_external_resource_specs_for_namespace( - namespace, MyResource, "another-provisioning-provider" - ) - assert len(external_resources) == 0 - - -def test_get_external_resource_specs_for_namespace_wrong_type(namespace: TestNamespace): - with pytest.raises(ValueError): - uer.get_external_resource_specs_for_namespace( - namespace, OverrideableResource, None - ) diff --git a/reconcile/utils/external_resource_spec.py b/reconcile/utils/external_resource_spec.py index 6f4e44db42..11af0220ce 100644 --- a/reconcile/utils/external_resource_spec.py +++ b/reconcile/utils/external_resource_spec.py @@ -1,17 +1,6 @@ import json -from typing import ( - Any, - Generic, - Optional, - Protocol, - TypeVar, - runtime_checkable, - Union, - cast, - get_args, - get_origin, -) -from collections.abc import Mapping, MutableMapping, Sequence +from typing import Any, Optional, cast +from collections.abc import Mapping, MutableMapping import yaml from reconcile.utils.openshift_resource import ( @@ -88,89 +77,6 @@ def render(self, vars: Mapping[str, str]) -> dict[str, str]: return self._formatter.render(vars) -class ExternalResourceProvisioner(Protocol): - @property - def name(self) -> str: - ... - - @abstractmethod - def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - ... - - -@runtime_checkable -class ExternalResource(Protocol): - @property - def provider(self) -> str: - ... - - @property - def identifier(self) -> str: - ... - - @abstractmethod - def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - ... - - -@runtime_checkable -class OverridableExternalResource(ExternalResource, Protocol): - @property - def overrides(self) -> Optional[Any]: - ... - - @abstractmethod - def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - ... - - -@runtime_checkable -class DefaultableExternalResource(ExternalResource, Protocol): - @property - def defaults(self) -> Optional[Any]: - ... - - @abstractmethod - def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - ... - - -@runtime_checkable -class NamespaceExternalResource(Protocol): - @property - def provider(self) -> str: - ... - - @property - def provisioner(self) -> ExternalResourceProvisioner: - ... - - @property - def resources(self) -> Sequence[ExternalResource]: - ... - - -@runtime_checkable -class Namespace(Protocol): - @property - def name(self) -> str: - ... - - @property - def managed_external_resources(self) -> Optional[bool]: - ... - - @property - def external_resources( - self, - ) -> Optional[Sequence[Union[NamespaceExternalResource, Any]]]: - ... - - @abstractmethod - def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - ... - - @dataclass class ExternalResourceSpec: @@ -278,71 +184,3 @@ def from_spec(spec: ExternalResourceSpec) -> "ExternalResourceUniqueKey": ExternalResourceSpecInventory = MutableMapping[ ExternalResourceUniqueKey, ExternalResourceSpec ] - - -T = TypeVar("T", bound=ExternalResource) - - -class MyConfig: - arbitrary_types_allowed = True - - -EXTERNAL_RESOURCE_SPEC_DEFAULTS_PROPERTY = "defaults" -EXTERNAL_RESOURCE_SPEC_OVERRIDES_PROPERTY = "overrides" - - -@dataclass(config=MyConfig) -class TypedExternalResourceSpec(ExternalResourceSpec, Generic[T]): - - namespace_spec: Namespace - namespace_external_resource: NamespaceExternalResource - spec: T - - def __init__( - self, - namespace_spec: Namespace, - namespace_external_resource: NamespaceExternalResource, - spec: T, - ): - self.namespace_spec = namespace_spec - self.namespace_external_resource = namespace_external_resource - self.spec = spec - super().__init__( - provision_provider=self.namespace_external_resource.provider, - provisioner=self.namespace_external_resource.provisioner.dict( - by_alias=True - ), - resource=self.spec.dict(by_alias=True), - namespace=self.namespace_spec.dict(by_alias=True), - ) - - def get_defaults_data(self) -> dict[str, Any]: - if not isinstance(self.spec, DefaultableExternalResource): - return {} - if self.spec.defaults is None: - return {} - return self.spec.defaults.dict(by_alias=True) - - def get_overrides_data(self) -> dict[str, Any]: - if not isinstance(self.spec, OverridableExternalResource): - return {} - if self.spec.overrides is None: - return {} - return self.spec.overrides.dict(by_alias=True) - - def is_overridable(self) -> bool: - return isinstance(self.spec, OverridableExternalResource) - - def get_overridable_fields(self) -> Sequence[str]: - if isinstance(self.spec, OverridableExternalResource): - overrides_class = self.spec.__annotations__[ - EXTERNAL_RESOURCE_SPEC_OVERRIDES_PROPERTY - ] - is_optional = get_origin(overrides_class) is Union and type( - None - ) in get_args(overrides_class) - if is_optional: - overrides_class = get_args(overrides_class)[0] - return overrides_class.__annotations__.keys() - else: - raise ValueError("resource is not overridable") diff --git a/reconcile/utils/external_resources.py b/reconcile/utils/external_resources.py index 35c0f9b8c2..cc06253a95 100644 --- a/reconcile/utils/external_resources.py +++ b/reconcile/utils/external_resources.py @@ -1,64 +1,17 @@ import json -from typing import ( - Any, - Optional, - Type, - TypeVar, -) +from typing import Any, Optional from collections.abc import Mapping, MutableMapping import anymarkup from reconcile.utils import gql from reconcile.utils.exceptions import FetchResourceError -from reconcile.utils.external_resource_spec import ( - ExternalResourceSpec, - TypedExternalResourceSpec, - ExternalResource, - Namespace, - NamespaceExternalResource, -) +from reconcile.utils.external_resource_spec import ExternalResourceSpec PROVIDER_AWS = "aws" PROVIDER_CLOUDFLARE = "cloudflare" PROVIDER_CNA_EXPERIMENTAL = "cna-experimental" -T = TypeVar("T", bound=ExternalResource) - - -def get_external_resource_specs_for_namespace( - namespace: Namespace, - resource_type: Type[T], - provision_provider: Optional[str] = None, -) -> list[TypedExternalResourceSpec[T]]: - if not namespace.managed_external_resources: - return [] - specs: list[TypedExternalResourceSpec[T]] = [] - for e in namespace.external_resources or []: - if isinstance(e, NamespaceExternalResource): - for r in e.resources: - if isinstance(r, resource_type): - specs.append( - TypedExternalResourceSpec[T]( - namespace_spec=namespace, - namespace_external_resource=e, - spec=r, - ) - ) - else: - raise ValueError( - f"expected resource of type {resource_type}, got {type(r)}" - ) - - if provision_provider: - specs = [ - s - for s in specs - if s.namespace_external_resource.provider == provision_provider - ] - - return specs - def get_external_resource_specs( namespace_info: Mapping[str, Any], provision_provider: Optional[str] = None From 4876411506889e63f3eb5c2db5f8edf270b3b50e Mon Sep 17 00:00:00 2001 From: Karl Fischer Date: Tue, 29 Nov 2022 14:45:26 +0100 Subject: [PATCH 14/20] make RDS work (#3022) --- reconcile/cna/assets/aws_rds.py | 17 ++++++++++++++++- reconcile/cna/state.py | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/reconcile/cna/assets/aws_rds.py b/reconcile/cna/assets/aws_rds.py index f697ba2af8..3fc174f5af 100644 --- a/reconcile/cna/assets/aws_rds.py +++ b/reconcile/cna/assets/aws_rds.py @@ -29,7 +29,7 @@ class AWSRDSAsset(Asset[CNARDSInstanceV1, CNARDSInstanceDefaultsV1]): engine: Optional[str] = Field(None, alias="engine") engine_version: Optional[str] = Field(None, alias="engine_version") major_engine_version: Optional[str] = Field(None, alias="major_engine_version") - username: Optional[str] = Field(None, alias="username") + username: str = Field(None, alias="username") region: Optional[str] = Field(None, alias="region") backup_retention_period: Optional[int] = Field( None, alias="backup_retention_period" @@ -39,6 +39,10 @@ class AWSRDSAsset(Asset[CNARDSInstanceV1, CNARDSInstanceDefaultsV1]): multi_az: Optional[bool] = Field(None, alias="multi_az") deletion_protection: Optional[bool] = Field(None, alias="deletion_protection") apply_immediately: Optional[bool] = Field(None, alias="apply_immediately") + + # Those values are implicit and not set in app-interface + is_production: bool = Field(None, alias="is_production") + family: Optional[str] = Field(None, alias="family") @staticmethod def provider() -> str: @@ -48,6 +52,14 @@ def provider() -> str: def asset_type() -> AssetType: return AssetType.AWS_RDS + @staticmethod + def determine_family(config: CNARDSInstanceDefaultsV1) -> str: + """ + The engine family for the parameter group is implicitly + determined based on the engine_version + """ + return f"postgres{config.engine_version.split('.')[0]}" + @classmethod def from_query_class(cls, asset: CNARDSInstanceV1) -> Asset: config = cls.aggregate_config(asset) @@ -82,4 +94,7 @@ def from_query_class(cls, asset: CNARDSInstanceV1) -> Asset: apply_immediately=config.apply_immediately, deletion_protection=config.deletion_protection, multi_az=config.multi_az, + username=config.username, + is_production=True, + family=AWSRDSAsset.determine_family(config=config), ) diff --git a/reconcile/cna/state.py b/reconcile/cna/state.py index 600384e6a1..4133acc2e9 100644 --- a/reconcile/cna/state.py +++ b/reconcile/cna/state.py @@ -76,7 +76,7 @@ def _diff(self, other: State, compare_bindings: bool) -> State: if asset_name not in self._assets[asset_type]: continue asset = self._assets[asset_type][asset_name] - if asset.status in (AssetStatus.TERMINATED, AssetStatus.PENDING): + if asset.status in (AssetStatus.TERMINATED, AssetStatus.PENDING, AssetStatus.ERROR): continue required_bindings: set[Binding] = set() if compare_bindings: From 935f0ea8c40af5501a4d6cd1667c19afc22488ba Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Fri, 9 Dec 2022 11:38:11 +0100 Subject: [PATCH 15/20] reformat and lint after rebase Signed-off-by: Gerd Oberlechner --- reconcile/cna/assets/__init__.py | 3 +- reconcile/cna/assets/asset.py | 20 ++++++++--- reconcile/cna/assets/asset_factory.py | 22 +++++------- reconcile/cna/assets/aws_assume_role.py | 12 ++++--- reconcile/cna/assets/aws_rds.py | 10 +++--- reconcile/cna/assets/aws_utils.py | 1 + reconcile/cna/assets/null.py | 10 +++--- reconcile/cna/client.py | 3 +- reconcile/cna/integration.py | 35 ++++++++++--------- reconcile/cna/state.py | 15 ++++++-- .../cna/queries/cna_resources.py | 2 +- reconcile/test/cna/test_asset.py | 17 +++++---- reconcile/test/cna/test_aws_assume_role.py | 5 ++- reconcile/test/cna/test_aws_rds.py | 14 ++++---- reconcile/test/cna/test_integration.py | 11 ++---- reconcile/test/cna/test_null.py | 7 ++-- reconcile/test/cna/test_state_diff.py | 6 ---- .../test/test_utils_external_resources.py | 2 +- reconcile/utils/external_resource_spec.py | 23 +++++++----- reconcile/utils/external_resources.py | 10 ++++-- 20 files changed, 129 insertions(+), 99 deletions(-) diff --git a/reconcile/cna/assets/__init__.py b/reconcile/cna/assets/__init__.py index b4c5fbce57..d1a2466ddf 100644 --- a/reconcile/cna/assets/__init__.py +++ b/reconcile/cna/assets/__init__.py @@ -1,8 +1,7 @@ from reconcile.cna.assets.asset_factory import register_asset_dataclass -from reconcile.cna.assets.null import NullAsset from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset from reconcile.cna.assets.aws_rds import AWSRDSAsset - +from reconcile.cna.assets.null import NullAsset register_asset_dataclass(NullAsset) register_asset_dataclass(AWSAssumeRoleAsset) diff --git a/reconcile/cna/assets/asset.py b/reconcile/cna/assets/asset.py index 1585b3cb51..3c4da916b9 100644 --- a/reconcile/cna/assets/asset.py +++ b/reconcile/cna/assets/asset.py @@ -1,14 +1,24 @@ -from abc import ABC, abstractmethod +import copy +from abc import ( + ABC, + abstractmethod, +) +from enum import Enum +from typing import ( + Any, + Generic, + Mapping, + Optional, + Type, + TypeVar, + get_args, +) from pydantic.dataclasses import dataclass from pydantic.fields import FieldInfo -from enum import Enum -from typing import Any, Generic, Mapping, Optional, Type, TypeVar, get_args -import copy from reconcile.gql_definitions.cna.queries.cna_resources import CNAssetV1 - ASSET_ID_FIELD = "id" ASSET_TYPE_FIELD = "asset_type" ASSET_NAME_FIELD = "name" diff --git a/reconcile/cna/assets/asset_factory.py b/reconcile/cna/assets/asset_factory.py index 609d2e0180..07505b2082 100644 --- a/reconcile/cna/assets/asset_factory.py +++ b/reconcile/cna/assets/asset_factory.py @@ -1,25 +1,19 @@ -from typing import Any, Type from collections.abc import Mapping -from typing import Any - -from reconcile.cna.assets.asset import ( - Asset, - AssetError, -) -from reconcile.cna.assets.null import NullAsset -from reconcile.gql_definitions.cna.queries.cna_resources import ( - CNAssetV1, +from typing import ( + Any, + Type, ) + from reconcile.cna.assets.asset import ( + ASSET_HREF_FIELD, + ASSET_NAME_FIELD, Asset, AssetType, UnknownAssetTypeError, - asset_type_id_from_raw_asset, asset_type_by_id, - ASSET_HREF_FIELD, - ASSET_NAME_FIELD, + asset_type_id_from_raw_asset, ) - +from reconcile.gql_definitions.cna.queries.cna_resources import CNAssetV1 _ASSET_TYPE_SCHEME: dict[AssetType, Type[Asset]] = {} _PROVIDER_SCHEME: dict[str, Type[Asset]] = {} diff --git a/reconcile/cna/assets/aws_assume_role.py b/reconcile/cna/assets/aws_assume_role.py index 7ef476cef1..6c957e8972 100644 --- a/reconcile/cna/assets/aws_assume_role.py +++ b/reconcile/cna/assets/aws_assume_role.py @@ -1,19 +1,21 @@ from __future__ import annotations -from pydantic.dataclasses import dataclass -from pydantic import Field + from typing import Optional +from pydantic import Field +from pydantic.dataclasses import dataclass + from reconcile.cna.assets.asset import ( Asset, AssetError, - AssetType, - AssetStatus, AssetModelConfig, + AssetStatus, + AssetType, ) from reconcile.cna.assets.aws_utils import aws_role_arn_for_module from reconcile.gql_definitions.cna.queries.cna_resources import ( - CNAAssumeRoleAssetV1, CNAAssumeRoleAssetConfigV1, + CNAAssumeRoleAssetV1, ) diff --git a/reconcile/cna/assets/aws_rds.py b/reconcile/cna/assets/aws_rds.py index 3fc174f5af..a1eeaf91ca 100644 --- a/reconcile/cna/assets/aws_rds.py +++ b/reconcile/cna/assets/aws_rds.py @@ -1,19 +1,21 @@ from __future__ import annotations + from typing import Optional -from pydantic.dataclasses import dataclass + from pydantic import Field +from pydantic.dataclasses import dataclass from reconcile.cna.assets.asset import ( Asset, AssetError, - AssetType, AssetModelConfig, AssetStatus, + AssetType, ) from reconcile.cna.assets.aws_utils import aws_role_arn_for_module from reconcile.gql_definitions.cna.queries.cna_resources import ( - CNARDSInstanceV1, CNARDSInstanceDefaultsV1, + CNARDSInstanceV1, ) @@ -39,7 +41,7 @@ class AWSRDSAsset(Asset[CNARDSInstanceV1, CNARDSInstanceDefaultsV1]): multi_az: Optional[bool] = Field(None, alias="multi_az") deletion_protection: Optional[bool] = Field(None, alias="deletion_protection") apply_immediately: Optional[bool] = Field(None, alias="apply_immediately") - + # Those values are implicit and not set in app-interface is_production: bool = Field(None, alias="is_production") family: Optional[str] = Field(None, alias="family") diff --git a/reconcile/cna/assets/aws_utils.py b/reconcile/cna/assets/aws_utils.py index af09f1ab34..90f1581acb 100644 --- a/reconcile/cna/assets/aws_utils.py +++ b/reconcile/cna/assets/aws_utils.py @@ -1,4 +1,5 @@ from typing import Optional + from reconcile.gql_definitions.cna.queries.aws_arn import CNAAWSSpecV1 diff --git a/reconcile/cna/assets/null.py b/reconcile/cna/assets/null.py index 5a637a34be..811ec9a3a6 100644 --- a/reconcile/cna/assets/null.py +++ b/reconcile/cna/assets/null.py @@ -1,17 +1,19 @@ from __future__ import annotations + from typing import Optional -from pydantic.dataclasses import dataclass + from pydantic import Field +from pydantic.dataclasses import dataclass from reconcile.cna.assets.asset import ( Asset, - AssetType, - AssetStatus, AssetModelConfig, + AssetStatus, + AssetType, ) from reconcile.gql_definitions.cna.queries.cna_resources import ( - CNANullAssetV1, CNANullAssetConfigV1, + CNANullAssetV1, ) diff --git a/reconcile/cna/client.py b/reconcile/cna/client.py index 0b07dfa1f3..a5437d0aa4 100644 --- a/reconcile/cna/client.py +++ b/reconcile/cna/client.py @@ -1,14 +1,15 @@ import logging from dataclasses import asdict from typing import Any + from reconcile.cna.assets.asset import ( + ASSET_CREATOR_FIELD, Asset, AssetType, AssetTypeMetadata, AssetTypeVariable, AssetTypeVariableType, asset_type_by_id, - ASSET_CREATOR_FIELD, ) from reconcile.utils.ocm_base_client import OCMBaseClient diff --git a/reconcile/cna/integration.py b/reconcile/cna/integration.py index 49fcde454c..8c18289864 100644 --- a/reconcile/cna/integration.py +++ b/reconcile/cna/integration.py @@ -1,16 +1,22 @@ -from collections import defaultdict -from collections.abc import Iterable, Mapping - import logging +from collections import defaultdict +from collections.abc import ( + Iterable, + Mapping, +) +from typing import Optional +from reconcile.cna.assets.asset import ( + AssetError, + Binding, + UnknownAssetTypeError, +) +from reconcile.cna.assets.asset_factory import ( + asset_factory_from_raw_data, + asset_factory_from_schema, +) from reconcile.cna.client import CNAClient from reconcile.cna.state import State - -from reconcile.utils import gql -from reconcile.utils.external_resources import ( - PROVIDER_CNA_EXPERIMENTAL, -) -from reconcile.utils.ocm_base_client import OCMBaseClient from reconcile.gql_definitions.cna.queries.cna_provisioners import ( CNAExperimentalProvisionerV1, ) @@ -18,26 +24,23 @@ query as cna_provisioners_query, ) from reconcile.gql_definitions.cna.queries.cna_resources import ( - NamespaceV1, NamespaceCNAssetV1, + NamespaceV1, +) +from reconcile.gql_definitions.cna.queries.cna_resources import ( query as namespaces_query, ) from reconcile.typed_queries.app_interface_vault_settings import ( get_app_interface_vault_settings, ) from reconcile.utils import gql +from reconcile.utils.external_resources import PROVIDER_CNA_EXPERIMENTAL from reconcile.utils.ocm_base_client import OCMBaseClient from reconcile.utils.secret_reader import ( SecretReaderBase, create_secret_reader, ) from reconcile.utils.semver_helper import make_semver -from reconcile.cna.assets.asset import UnknownAssetTypeError, AssetError, Binding -from reconcile.cna.assets.asset_factory import ( - asset_factory_from_schema, - asset_factory_from_raw_data, -) - QONTRACT_INTEGRATION = "cna_resources" QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0) diff --git a/reconcile/cna/state.py b/reconcile/cna/state.py index 4133acc2e9..583a21cd59 100644 --- a/reconcile/cna/state.py +++ b/reconcile/cna/state.py @@ -1,6 +1,13 @@ from __future__ import annotations + from typing import Optional -from reconcile.cna.assets.asset import Asset, AssetStatus, AssetType, Binding + +from reconcile.cna.assets.asset import ( + Asset, + AssetStatus, + AssetType, + Binding, +) class CNAStateError(Exception): @@ -76,7 +83,11 @@ def _diff(self, other: State, compare_bindings: bool) -> State: if asset_name not in self._assets[asset_type]: continue asset = self._assets[asset_type][asset_name] - if asset.status in (AssetStatus.TERMINATED, AssetStatus.PENDING, AssetStatus.ERROR): + if asset.status in ( + AssetStatus.TERMINATED, + AssetStatus.PENDING, + AssetStatus.ERROR, + ): continue required_bindings: set[Binding] = set() if compare_bindings: diff --git a/reconcile/gql_definitions/cna/queries/cna_resources.py b/reconcile/gql_definitions/cna/queries/cna_resources.py index 054d68ef17..a44b9a9379 100644 --- a/reconcile/gql_definitions/cna/queries/cna_resources.py +++ b/reconcile/gql_definitions/cna/queries/cna_resources.py @@ -175,7 +175,7 @@ class CNARDSInstanceDefaultsV1(BaseModel): max_allocated_storage: int = Field(..., alias="max_allocated_storage") engine: str = Field(..., alias="engine") engine_version: str = Field(..., alias="engine_version") - username: Optional[str] = Field(..., alias="username") + username: str = Field(..., alias="username") maintenance_window: Optional[str] = Field(..., alias="maintenance_window") backup_retention_period: Optional[int] = Field(..., alias="backup_retention_period") backup_window: Optional[str] = Field(..., alias="backup_window") diff --git a/reconcile/test/cna/test_asset.py b/reconcile/test/cna/test_asset.py index 462120f5f8..a656dba393 100644 --- a/reconcile/test/cna/test_asset.py +++ b/reconcile/test/cna/test_asset.py @@ -1,26 +1,31 @@ -from typing import Any, Mapping, MutableMapping, Optional import json +from typing import ( + Any, + Mapping, + MutableMapping, + Optional, +) + +import pytest + from reconcile.cna.assets.asset import ( Asset, + AssetError, AssetStatus, AssetType, AssetTypeMetadata, AssetTypeVariable, AssetTypeVariableType, - AssetError, - asset_type_metadata_from_asset_dataclass, asset_type_from_raw_asset, + asset_type_metadata_from_asset_dataclass, ) from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset from reconcile.cna.assets.null import NullAsset - from reconcile.gql_definitions.cna.queries.cna_resources import ( CNAAssumeRoleAssetV1, CNAssetV1, ) -import pytest - @pytest.fixture def aws_assumerole_asset_type_metadata() -> AssetTypeMetadata: diff --git a/reconcile/test/cna/test_aws_assume_role.py b/reconcile/test/cna/test_aws_assume_role.py index 963c75942b..ad7cdafd00 100644 --- a/reconcile/test/cna/test_aws_assume_role.py +++ b/reconcile/test/cna/test_aws_assume_role.py @@ -1,12 +1,11 @@ import json + from reconcile.cna.assets.aws_assume_role import AWSAssumeRoleAsset -from reconcile.gql_definitions.cna.queries.cna_resources import ( - CNAAssumeRoleAssetV1, -) from reconcile.gql_definitions.cna.queries.aws_arn import ( CNAAWSAccountRoleARNs, CNAAWSSpecV1, ) +from reconcile.gql_definitions.cna.queries.cna_resources import CNAAssumeRoleAssetV1 def test_from_query_class(): diff --git a/reconcile/test/cna/test_aws_rds.py b/reconcile/test/cna/test_aws_rds.py index 8e66cfd163..38da1c66fe 100644 --- a/reconcile/test/cna/test_aws_rds.py +++ b/reconcile/test/cna/test_aws_rds.py @@ -1,15 +1,15 @@ +import pytest + from reconcile.cna.assets.aws_rds import AWSRDSAsset -from reconcile.gql_definitions.cna.queries.cna_resources import ( - CNARDSInstanceV1, - CNARDSInstanceDefaultsV1, - AWSVPCV1, -) from reconcile.gql_definitions.cna.queries.aws_arn import ( CNAAWSAccountRoleARNs, CNAAWSSpecV1, ) - -import pytest +from reconcile.gql_definitions.cna.queries.cna_resources import ( + AWSVPCV1, + CNARDSInstanceDefaultsV1, + CNARDSInstanceV1, +) asset_identifier = "identifier" db_name = "instance-name" diff --git a/reconcile/test/cna/test_integration.py b/reconcile/test/cna/test_integration.py index 47f0fcb378..e2174fb9ba 100644 --- a/reconcile/test/cna/test_integration.py +++ b/reconcile/test/cna/test_integration.py @@ -1,3 +1,4 @@ +import json from collections.abc import ( Iterable, Mapping, @@ -7,8 +8,7 @@ Optional, ) from unittest.mock import create_autospec -from pytest import fixture -import json + import pytest from pytest import fixture @@ -19,17 +19,12 @@ from reconcile.cna.assets.null import NullAsset from reconcile.cna.client import CNAClient from reconcile.cna.integration import CNAIntegration -from reconcile.cna.assets.asset import ( - AssetStatus, - AssetType, -) -from reconcile.cna.assets.null import NullAsset from reconcile.cna.state import State from reconcile.gql_definitions.cna.queries.cna_resources import ( + ClusterV1, CNANullAssetV1, ExternalResourcesProvisionerV1, NamespaceCNAssetV1, - ClusterV1, NamespaceV1, ) from reconcile.utils.external_resources import PROVIDER_CNA_EXPERIMENTAL diff --git a/reconcile/test/cna/test_null.py b/reconcile/test/cna/test_null.py index ffcd49707f..5177076172 100644 --- a/reconcile/test/cna/test_null.py +++ b/reconcile/test/cna/test_null.py @@ -1,9 +1,8 @@ -from reconcile.cna.assets.null import NullAsset -from reconcile.gql_definitions.cna.queries.cna_resources import ( - CNANullAssetV1, -) import json +from reconcile.cna.assets.null import NullAsset +from reconcile.gql_definitions.cna.queries.cna_resources import CNANullAssetV1 + def test_from_query_class(): identifier = "name" diff --git a/reconcile/test/cna/test_state_diff.py b/reconcile/test/cna/test_state_diff.py index c1fc6b99c3..2b9c7bce88 100644 --- a/reconcile/test/cna/test_state_diff.py +++ b/reconcile/test/cna/test_state_diff.py @@ -1,10 +1,4 @@ from typing import Optional -import pytest -from reconcile.cna.assets.asset import ( - AssetStatus, - AssetType, -) -from reconcile.cna.assets.null import NullAsset import pytest diff --git a/reconcile/test/test_utils_external_resources.py b/reconcile/test/test_utils_external_resources.py index e60b3c4d60..ca08318335 100644 --- a/reconcile/test/test_utils_external_resources.py +++ b/reconcile/test/test_utils_external_resources.py @@ -1,9 +1,9 @@ import json import pytest -from reconcile.utils.external_resource_spec import ExternalResourceSpec import reconcile.utils.external_resources as uer +from reconcile.utils.external_resource_spec import ExternalResourceSpec @pytest.fixture diff --git a/reconcile/utils/external_resource_spec.py b/reconcile/utils/external_resource_spec.py index 11af0220ce..039a4b8ab8 100644 --- a/reconcile/utils/external_resource_spec.py +++ b/reconcile/utils/external_resource_spec.py @@ -1,18 +1,25 @@ import json -from typing import Any, Optional, cast -from collections.abc import Mapping, MutableMapping - -import yaml -from reconcile.utils.openshift_resource import ( - OpenshiftResource, - build_secret, - SECRET_MAX_KEY_LENGTH, +from abc import abstractmethod +from collections.abc import ( + Mapping, + MutableMapping, +) +from dataclasses import field +from typing import ( + Any, + Optional, + cast, ) import yaml from pydantic.dataclasses import dataclass from reconcile import openshift_resources_base +from reconcile.utils.openshift_resource import ( + SECRET_MAX_KEY_LENGTH, + OpenshiftResource, + build_secret, +) class OutputFormatProcessor: diff --git a/reconcile/utils/external_resources.py b/reconcile/utils/external_resources.py index cc06253a95..996ba176bd 100644 --- a/reconcile/utils/external_resources.py +++ b/reconcile/utils/external_resources.py @@ -1,6 +1,12 @@ import json -from typing import Any, Optional -from collections.abc import Mapping, MutableMapping +from collections.abc import ( + Mapping, + MutableMapping, +) +from typing import ( + Any, + Optional, +) import anymarkup From e70a1dd6e0a3f5663d1af8e8b4ea50b2c1bf59c3 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Tue, 13 Dec 2022 15:19:06 +0100 Subject: [PATCH 16/20] running qenerate one more time Signed-off-by: Gerd Oberlechner --- .../gql_definitions/cna/queries/aws_arn.py | 2 +- .../fragments/resource_file.py | 2 +- reconcile/gql_definitions/introspection.json | 358 +++++++++++++++++- 3 files changed, 351 insertions(+), 11 deletions(-) diff --git a/reconcile/gql_definitions/cna/queries/aws_arn.py b/reconcile/gql_definitions/cna/queries/aws_arn.py index f654cc5192..94083b603b 100644 --- a/reconcile/gql_definitions/cna/queries/aws_arn.py +++ b/reconcile/gql_definitions/cna/queries/aws_arn.py @@ -1,10 +1,10 @@ """ Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY! """ +from collections.abc import Callable # noqa: F401 # pylint: disable=W0611 from enum import Enum # noqa: F401 # pylint: disable=W0611 from typing import ( # noqa: F401 # pylint: disable=W0611 Any, - Callable, Optional, Union, ) diff --git a/reconcile/gql_definitions/fragments/resource_file.py b/reconcile/gql_definitions/fragments/resource_file.py index 3ed2ef61fd..e7c1c4407b 100644 --- a/reconcile/gql_definitions/fragments/resource_file.py +++ b/reconcile/gql_definitions/fragments/resource_file.py @@ -1,10 +1,10 @@ """ Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY! """ +from collections.abc import Callable # noqa: F401 # pylint: disable=W0611 from enum import Enum # noqa: F401 # pylint: disable=W0611 from typing import ( # noqa: F401 # pylint: disable=W0611 Any, - Callable, Optional, Union, ) diff --git a/reconcile/gql_definitions/introspection.json b/reconcile/gql_definitions/introspection.json index 0ee7c874b9..dbc070a41d 100644 --- a/reconcile/gql_definitions/introspection.json +++ b/reconcile/gql_definitions/introspection.json @@ -28297,6 +28297,11 @@ "kind": "OBJECT", "name": "CNANullAsset_v1", "ofType": null + }, + { + "kind": "OBJECT", + "name": "CNARDSInstance_v1", + "ofType": null } ] }, @@ -28338,7 +28343,7 @@ "deprecationReason": null }, { - "name": "aws_assume_role", + "name": "aws_account", "description": null, "args": [], "type": { @@ -28346,12 +28351,36 @@ "name": null, "ofType": { "kind": "OBJECT", - "name": "CNAAssumeRoleAssetConfig_v1", + "name": "AWSAccount_v1", "ofType": null } }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "defaults", + "description": null, + "args": [], + "type": { + "kind": "OBJECT", + "name": "CNAAssumeRoleAssetConfig_v1", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "overrides", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "JSON", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -28374,6 +28403,29 @@ "name": "slug", "description": null, "args": [], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CNANullAsset_v1", + "description": null, + "fields": [ + { + "name": "provider", + "description": null, + "args": [], "type": { "kind": "NON_NULL", "name": null, @@ -28387,20 +28439,85 @@ "deprecationReason": null }, { - "name": "account", + "name": "identifier", "description": null, "args": [], "type": { "kind": "NON_NULL", "name": null, "ofType": { - "kind": "OBJECT", - "name": "AWSAccount_v1", + "kind": "SCALAR", + "name": "String", "ofType": null } }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "description", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "defaults", + "description": null, + "args": [], + "type": { + "kind": "OBJECT", + "name": "CNANullAssetConfig_v1", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "overrides", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "JSON", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + { + "kind": "INTERFACE", + "name": "CNAsset_v1", + "ofType": null + } + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CNANullAssetConfig_v1", + "description": null, + "fields": [ + { + "name": "addr_block", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -28410,7 +28527,7 @@ }, { "kind": "OBJECT", - "name": "CNANullAsset_v1", + "name": "CNARDSInstance_v1", "description": null, "fields": [ { @@ -28446,7 +28563,7 @@ "deprecationReason": null }, { - "name": "description", + "name": "name", "description": null, "args": [], "type": { @@ -28458,12 +28575,24 @@ "deprecationReason": null }, { - "name": "addr_block", + "name": "defaults", + "description": null, + "args": [], + "type": { + "kind": "OBJECT", + "name": "CNARDSInstanceDefaults_v1", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "overrides", "description": null, "args": [], "type": { "kind": "SCALAR", - "name": "String", + "name": "JSON", "ofType": null }, "isDeprecated": false, @@ -28481,6 +28610,217 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "CNARDSInstanceDefaults_v1", + "description": null, + "fields": [ + { + "name": "vpc", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "AWSVPC_v1", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "db_subnet_group_name", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "instance_class", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "allocated_storage", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "max_allocated_storage", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "engine", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "engine_version", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "username", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "maintenance_window", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "backup_retention_period", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "backup_window", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "multi_az", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "deletion_protection", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "apply_immediately", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "AWSAccountSharingOptionAMI_v1", From a577a1cf233e97dc47ea6fc2e2913092dcce737f Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Tue, 20 Dec 2022 08:51:22 +0100 Subject: [PATCH 17/20] dedup CNA API URL prefix Signed-off-by: Gerd Oberlechner --- reconcile/cna/client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/reconcile/cna/client.py b/reconcile/cna/client.py index a5437d0aa4..7e8bc03df9 100644 --- a/reconcile/cna/client.py +++ b/reconcile/cna/client.py @@ -27,7 +27,7 @@ def __init__(self, ocm_client: OCMBaseClient, init_metadata: bool = False): def _init_metadata(self) -> dict[AssetType, AssetTypeMetadata]: asset_types_metadata: dict[AssetType, AssetTypeMetadata] = {} for asset_type_ref in self._ocm_client.get( - api_path="/api/cna-management/v1/asset_types" + api_path=self._cna_api_v1_endpoint("/asset_types") )["items"]: raw_asset_type_metadata = self._ocm_client.get( api_path=asset_type_ref["href"] @@ -67,7 +67,7 @@ def list_assets(self) -> list[dict[str, Any]]: of our assets """ # TODO: properly handle paging - cnas = self._ocm_client.get(api_path="/api/cna-management/v1/cnas") + cnas = self._ocm_client.get(api_path=self._cna_api_v1_endpoint("/cnas")) return cnas.get("items", []) def fetch_bindings_for_asset(self, asset: Asset) -> list[dict[str, str]]: @@ -89,7 +89,7 @@ def create(self, asset: Asset, dry_run: bool = False): ) return self._ocm_client.post( - api_path="/api/cna-management/v1/cnas", + api_path=self._cna_api_v1_endpoint("/cnas"), data=asset.api_payload(), ) @@ -126,3 +126,6 @@ def update(self, asset: Asset, dry_run: bool = False): api_path=asset.href, data=asset.api_payload(), ) + + def _cna_api_v1_endpoint(self, path: str) -> str: + return f"/api/cna-management/v1{path}" From 443690b2af95f5b2b9ee7a07e6c68596aa7cb62d Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Tue, 20 Dec 2022 08:54:56 +0100 Subject: [PATCH 18/20] clarify failure behavior on CNA API errors Signed-off-by: Gerd Oberlechner --- reconcile/cna/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/reconcile/cna/client.py b/reconcile/cna/client.py index 7e8bc03df9..d05bc4e3c7 100644 --- a/reconcile/cna/client.py +++ b/reconcile/cna/client.py @@ -18,6 +18,9 @@ class CNAClient: """ Client used to interact with CNA. CNA API doc can be found here: https://gitlab.cee.redhat.com/service/cna-management/-/blob/main/openapi/openapi.yaml#/ + + All HTTP errors while communicating with the CNA API are raised + as a `requests.exceptions.HTTPError`. """ def __init__(self, ocm_client: OCMBaseClient, init_metadata: bool = False): From 408a3555bf69229f584cd73911f86bd5ea345ffc Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Tue, 20 Dec 2022 09:08:38 +0100 Subject: [PATCH 19/20] removed log statement von bind dry-run Signed-off-by: Gerd Oberlechner --- reconcile/cna/client.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/reconcile/cna/client.py b/reconcile/cna/client.py index d05bc4e3c7..11afd62438 100644 --- a/reconcile/cna/client.py +++ b/reconcile/cna/client.py @@ -99,12 +99,6 @@ def create(self, asset: Asset, dry_run: bool = False): def bind(self, asset: Asset, dry_run: bool = False): for binding in asset.bindings: if dry_run: - logging.info( - "BIND %s %s %s", - asset.asset_type().value, - asset.name, - binding, - ) continue self._ocm_client.post( api_path=f"{asset.href}/bind", From 727282a5a7de10add2458ce1ae105c08596b49c7 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Tue, 20 Dec 2022 12:18:38 +0100 Subject: [PATCH 20/20] autospec the patched list_asset function Signed-off-by: Gerd Oberlechner --- reconcile/test/cna/test_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reconcile/test/cna/test_client.py b/reconcile/test/cna/test_client.py index 0d1b1df123..e7a95af3ac 100644 --- a/reconcile/test/cna/test_client.py +++ b/reconcile/test/cna/test_client.py @@ -37,7 +37,9 @@ def test_client_list_assets_for_creator(mocker): }, ] - mocker.patch.object(CNAClient, "list_assets", return_value=listed_assets) + mocker.patch.object( + CNAClient, "list_assets", return_value=listed_assets, autospec=True + ) cna_client = CNAClient(None) # type: ignore creator_assets = cna_client.list_assets_for_creator(creator)