diff --git a/tdp/cli/commands/default_diff.py b/tdp/cli/commands/default_diff.py index 5401bd36..80a1a4b1 100644 --- a/tdp/cli/commands/default_diff.py +++ b/tdp/cli/commands/default_diff.py @@ -35,7 +35,7 @@ def default_diff(collections: Collections, vars: Path, service: Optional[str] = service_diff(collections, service_variables) -def service_diff(collections, service): +def service_diff(collections: Collections, service): """Computes the difference between the default variables from a service, and the variables from your service variables inside your tdp_vars. Args: @@ -44,12 +44,14 @@ def service_diff(collections, service): """ # key: filename with extension, value: PosixPath(filepath) default_service_vars_paths = OrderedDict() - for collection in collections.values(): - default_vars = collection.get_service_default_vars(service.name) - if not default_vars: + for collection_default_vars_dir in collections.default_vars_dirs.values(): + service_default_vars_dir = collection_default_vars_dir / service.name + if not service_default_vars_dir.exists(): continue - for name, path in default_vars: - default_service_vars_paths.setdefault(name, []).append(path) + for default_vars_path in service_default_vars_dir.iterdir(): + default_service_vars_paths.setdefault(default_vars_path.name, []).append( + default_vars_path + ) for ( default_service_vars_filename, diff --git a/tdp/cli/commands/vars/edit.py b/tdp/cli/commands/vars/edit.py index 331a8f7e..2be7bde1 100644 --- a/tdp/cli/commands/vars/edit.py +++ b/tdp/cli/commands/vars/edit.py @@ -96,7 +96,7 @@ def edit( # Check if component exists entity_name = parse_hostable_entity_name(variables_file.stem) if isinstance(entity_name, ServiceComponentName): - if entity_name not in collections.get_components_from_service(service_name): + if entity_name not in collections.hostable_entities[service_name]: raise click.ClickException( f"Error unknown component '{entity_name.component}' for service '{entity_name.service}'" ) diff --git a/tdp/cli/params/collections_option.py b/tdp/cli/params/collections_option.py index 6aa82132..6b29c3cb 100644 --- a/tdp/cli/params/collections_option.py +++ b/tdp/cli/params/collections_option.py @@ -6,7 +6,6 @@ import click from click.decorators import FC -from tdp.core.collection import Collection from tdp.core.collections import Collections @@ -29,10 +28,7 @@ def _collections_from_paths( if not value: raise click.BadParameter("cannot be empty", ctx=ctx, param=param) - collections_list = [Collection.from_path(path) for path in value] - collections = Collections.from_collection_list(collections_list) - - return collections + return Collections.from_collection_paths(value) def collections_option(func: FC) -> FC: diff --git a/tdp/cli/params/status/component_argument.py b/tdp/cli/params/status/component_argument.py index 04f1a1fc..42dcf150 100644 --- a/tdp/cli/params/status/component_argument.py +++ b/tdp/cli/params/status/component_argument.py @@ -19,8 +19,7 @@ def _check_component( collections: Collections = ctx.params["collections"] service: str = ctx.params["service"] if value and value not in [ - sc_name.component - for sc_name in collections.get_components_from_service(service) + sc_name.component for sc_name in collections.hostable_entities[service] ]: raise click.UsageError( f"Component '{value}' does not exists in service '{service}'." diff --git a/tdp/core/collection.py b/tdp/core/collection.py deleted file mode 100644 index 0f54f0b4..00000000 --- a/tdp/core/collection.py +++ /dev/null @@ -1,304 +0,0 @@ -# Copyright 2022 TOSIT.IO -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import logging -from collections.abc import Generator -from pathlib import Path -from typing import Optional - -import yaml -from pydantic import BaseModel, ConfigDict, ValidationError - -from tdp.core.constants import ( - DAG_DIRECTORY_NAME, - DEFAULT_VARS_DIRECTORY_NAME, - JSON_EXTENSION, - PLAYBOOKS_DIRECTORY_NAME, - SCHEMA_VARS_DIRECTORY_NAME, - YML_EXTENSION, -) -from tdp.core.entities.operation import Playbook -from tdp.core.inventory_reader import InventoryReader -from tdp.core.types import PathLike -from tdp.core.variables.schema import ServiceCollectionSchema -from tdp.core.variables.schema.exceptions import InvalidSchemaError - -try: - from yaml import CLoader as Loader -except ImportError: - from yaml import Loader - -MANDATORY_DIRECTORIES = [ - DAG_DIRECTORY_NAME, - DEFAULT_VARS_DIRECTORY_NAME, - PLAYBOOKS_DIRECTORY_NAME, -] - -logger = logging.getLogger(__name__) - - -class PathDoesNotExistsError(Exception): - pass - - -class PathIsNotADirectoryError(Exception): - pass - - -class MissingMandatoryDirectoryError(Exception): - pass - - -class Collection: - """An enriched version of an Ansible collection. - - A TDP Collection is a directory containing playbooks, DAGs, default variables and variables schemas. - """ - - def __init__( - self, - path: PathLike, - inventory_reader: Optional[InventoryReader] = None, - ): - """Initialize a collection. - - Args: - path: The path to the collection. - - Raises: - PathDoesNotExistsError: If the path does not exists. - PathIsNotADirectoryError: If the path is not a directory. - MissingMandatoryDirectoryError: If the collection does not contain a mandatory directory. - """ - self._path = Path(path) - check_collection_structure(self._path) - - self._inventory_reader = inventory_reader or InventoryReader() - self._dag_nodes = list(get_collection_dag_nodes(self._path)) - self._playbooks = get_collection_playbooks( - self._path, - inventory_reader=self._inventory_reader, - ) - self._schemas = get_collection_schemas(self._path) - - @staticmethod - def from_path(path: PathLike) -> Collection: - """Factory method to create a collection from a path. - - Args: - path: The path to the collection. - - Returns: A collection. - """ - return Collection(path=Path(path).expanduser().resolve()) - - @property - def name(self) -> str: - """Collection name.""" - return self._path.name - - @property - def path(self) -> Path: - """Path to the collection.""" - return self._path - - @property - def dag_directory(self) -> Path: - """Path to the DAG directory.""" - return self._path / DAG_DIRECTORY_NAME - - @property - def default_vars_directory(self) -> Path: - """Path to the default variables directory.""" - return self._path / DEFAULT_VARS_DIRECTORY_NAME - - @property - def playbooks_directory(self) -> Path: - """Path to the playbook directory.""" - return self._path / PLAYBOOKS_DIRECTORY_NAME - - @property - def schema_directory(self) -> Path: - """Path to the variables schema directory.""" - return self._path / SCHEMA_VARS_DIRECTORY_NAME - - @property - def dag_nodes(self) -> list[TDPLibDagNodeModel]: - """List of DAG files in the YAML format.""" - return self._dag_nodes # TODO: should return a generator - - @property - def playbooks(self) -> dict[str, Playbook]: - """Dictionary of playbooks.""" - return self._playbooks - - @property - def schemas(self) -> list[ServiceCollectionSchema]: - """List of schemas.""" - return self._schemas - - def get_service_default_vars(self, service_name: str) -> list[tuple[str, Path]]: - """Get the default variables for a service. - - Args: - service_name: The name of the service. - - Returns: - A list of tuples (name, path) of the default variables. - """ - service_path = self.default_vars_directory / service_name - if not service_path.exists(): - return [] - return [(path.name, path) for path in service_path.glob("*" + YML_EXTENSION)] - - -def check_collection_structure(path: Path) -> None: - """Check the structure of a collection. - - Args: - path: Path to the collection. - - Raises: - PathDoesNotExistsError: If the path does not exists. - PathIsNotADirectoryError: If the path is not a directory. - MissingMandatoryDirectoryError: If the collection does not contain a mandatory directory. - """ - if not path.exists(): - raise PathDoesNotExistsError(f"{path} does not exists.") - if not path.is_dir(): - raise PathIsNotADirectoryError(f"{path} is not a directory.") - for mandatory_directory in MANDATORY_DIRECTORIES: - mandatory_path = path / mandatory_directory - if not mandatory_path.exists() or not mandatory_path.is_dir(): - raise MissingMandatoryDirectoryError( - f"{path} does not contain the mandatory directory {mandatory_directory}.", - ) - - -def get_collection_schemas( - collection_path: Path, schemas_directory_name=SCHEMA_VARS_DIRECTORY_NAME -) -> list[ServiceCollectionSchema]: - """Get the schemas of a collection. - - This function is meant to be used only once during the initialization of a - collection object. - - Invalid schemas are ignored. - - Args: - collection_path: Path to the collection. - - Returns: - Dictionary of schemas. - """ - schemas: list[ServiceCollectionSchema] = [] - for schema_path in (collection_path / schemas_directory_name).glob( - "*" + JSON_EXTENSION - ): - try: - schemas.append(ServiceCollectionSchema.from_path(schema_path)) - except InvalidSchemaError as e: - logger.warning(f"{e}. Ignoring schema.") - return schemas - - -def get_collection_playbooks( - collection_path: Path, - playbooks_directory_name=PLAYBOOKS_DIRECTORY_NAME, - inventory_reader: Optional[InventoryReader] = None, -) -> dict[str, Playbook]: - """Get the playbooks of a collection. - - This function is meant to be used only once during the initialization of a - collection object. - - Args: - collection_path: Path to the collection. - playbook_directory: Name of the playbook directory. - inventory_reader: Inventory reader. - - Returns: - Dictionary of playbooks. - """ - return { - playbook_path.stem: Playbook( - playbook_path, - collection_path.name, - read_hosts_from_playbook(playbook_path, inventory_reader), - ) - for playbook_path in (collection_path / playbooks_directory_name).glob( - "*" + YML_EXTENSION - ) - } - - -def read_hosts_from_playbook( - playbook_path: Path, inventory_reader: Optional[InventoryReader] -) -> set[str]: - """Read the hosts from a playbook. - - Args: - playbook_path: Path to the playbook. - inventory_reader: Inventory reader. - - Returns: - Set of hosts. - """ - if not inventory_reader: - inventory_reader = InventoryReader() - try: - with playbook_path.open() as fd: - return inventory_reader.get_hosts_from_playbook(fd) - except Exception as e: - raise ValueError(f"Can't parse playbook {playbook_path}.") from e - - -def get_collection_dag_nodes( - collection_path: Path, dag_directory_name=DAG_DIRECTORY_NAME -) -> Generator[TDPLibDagNodeModel, None, None]: - """Get the DAG nodes of a collection. - - Args: - collection_path: Path to the collection. - dag_directory_name: Name of the DAG directory. - - Returns: - List of DAG nodes. - """ - for dag_file in (collection_path / dag_directory_name).glob("*" + YML_EXTENSION): - yield from read_dag_file(dag_file) - - -class TDPLibDagNodeModel(BaseModel): - """Model for a TDP operation defined in a tdp_lib_dag file.""" - - model_config = ConfigDict(extra="ignore") - - name: str - depends_on: list[str] = [] - - -class TDPLibDagModel(BaseModel): - """Model for a TDP DAG defined in a tdp_lib_dag file.""" - - model_config = ConfigDict(extra="ignore") - - operations: list[TDPLibDagNodeModel] - - -def read_dag_file( - dag_file_path: Path, -) -> Generator[TDPLibDagNodeModel, None, None]: - """Read a tdp_lib_dag file and return a list of DAG operations.""" - with dag_file_path.open("r") as operations_file: - file_content = yaml.load(operations_file, Loader=Loader) - - try: - tdp_lib_dag = TDPLibDagModel(operations=file_content) - for operation in tdp_lib_dag.operations: - yield operation - except ValidationError as e: - logger.error(f"Error while parsing tdp_lib_dag file {dag_file_path}: {e}") - raise diff --git a/tdp/core/collections/__init__.py b/tdp/core/collections/__init__.py new file mode 100644 index 00000000..953f287c --- /dev/null +++ b/tdp/core/collections/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +from .collections import Collections diff --git a/tdp/core/collections/collection_reader.py b/tdp/core/collections/collection_reader.py new file mode 100644 index 00000000..b98d9aac --- /dev/null +++ b/tdp/core/collections/collection_reader.py @@ -0,0 +1,221 @@ +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import logging +from collections.abc import Generator +from pathlib import Path +from typing import Optional + +import yaml +from pydantic import BaseModel, ConfigDict, ValidationError + +from tdp.core.constants import ( + DAG_DIRECTORY_NAME, + DEFAULT_VARS_DIRECTORY_NAME, + JSON_EXTENSION, + PLAYBOOKS_DIRECTORY_NAME, + SCHEMA_VARS_DIRECTORY_NAME, + YML_EXTENSION, +) +from tdp.core.entities.operation import Playbook +from tdp.core.inventory_reader import InventoryReader +from tdp.core.types import PathLike +from tdp.core.variables.schema import ServiceCollectionSchema +from tdp.core.variables.schema.exceptions import InvalidSchemaError + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + +MANDATORY_DIRECTORIES = [ + DAG_DIRECTORY_NAME, + DEFAULT_VARS_DIRECTORY_NAME, + PLAYBOOKS_DIRECTORY_NAME, +] + +logger = logging.getLogger(__name__) + + +class PathDoesNotExistsError(Exception): + pass + + +class PathIsNotADirectoryError(Exception): + pass + + +class MissingMandatoryDirectoryError(Exception): + pass + + +class TDPLibDagNodeModel(BaseModel): + """Model for a TDP operation defined in a tdp_lib_dag file.""" + + model_config = ConfigDict(extra="ignore") + + name: str + depends_on: list[str] = [] + + +class TDPLibDagModel(BaseModel): + """Model for a TDP DAG defined in a tdp_lib_dag file.""" + + model_config = ConfigDict(extra="ignore") + + operations: list[TDPLibDagNodeModel] + + +class CollectionReader: + """An enriched version of an Ansible collection. + + A TDP Collection is a directory containing playbooks, DAGs, default variables and variables schemas. + """ + + def __init__( + self, + path: PathLike, + inventory_reader: Optional[InventoryReader] = None, + ): + """Initialize a collection. + + Args: + path: The path to the collection. + + Raises: + PathDoesNotExistsError: If the path does not exists. + PathIsNotADirectoryError: If the path is not a directory. + MissingMandatoryDirectoryError: If the collection does not contain a mandatory directory. + """ + self._path = Path(path) + self._check_collection_structure(self._path) + self._inventory_reader = inventory_reader or InventoryReader() + + # ? Is this method really useful? + @staticmethod + def from_path( + path: PathLike, + inventory_reader: Optional[InventoryReader] = None, + ) -> CollectionReader: + """Factory method to create a collection from a path. + + Args: + path: The path to the collection. + + Returns: A collection. + """ + inventory_reader = inventory_reader or InventoryReader() + return CollectionReader(Path(path).expanduser().resolve(), inventory_reader) + + @property + def name(self) -> str: + """Collection name.""" + return self._path.name + + @property + def path(self) -> Path: + """Path to the collection.""" + return self._path + + @property + def dag_directory(self) -> Path: + """Path to the DAG directory.""" + return self._path / DAG_DIRECTORY_NAME + + @property + def default_vars_directory(self) -> Path: + """Path to the default variables directory.""" + return self._path / DEFAULT_VARS_DIRECTORY_NAME + + @property + def playbooks_directory(self) -> Path: + """Path to the playbook directory.""" + return self._path / PLAYBOOKS_DIRECTORY_NAME + + @property + def schema_directory(self) -> Path: + """Path to the variables schema directory.""" + return self._path / SCHEMA_VARS_DIRECTORY_NAME + + def read_dag_nodes(self) -> Generator[TDPLibDagNodeModel, None, None]: + """Read the DAG nodes stored in the dag_directory.""" + for dag_file in (self.dag_directory).glob("*" + YML_EXTENSION): + with dag_file.open("r") as operations_file: + file_content = yaml.load(operations_file, Loader=Loader) + + try: + tdp_lib_dag = TDPLibDagModel(operations=file_content) + for operation in tdp_lib_dag.operations: + yield operation + except ValidationError as e: + logger.error(f"Error while parsing tdp_lib_dag file {dag_file}: {e}") + raise + + def read_playbooks(self) -> dict[str, Playbook]: + """Read the playbooks stored in the playbooks_directory.""" + return { + playbook_path.stem: Playbook( + path=playbook_path, + collection_name=self.name, + hosts=read_hosts_from_playbook(playbook_path, self._inventory_reader), + ) + for playbook_path in (self.playbooks_directory).glob("*" + YML_EXTENSION) + } + + def read_schemas(self) -> list[ServiceCollectionSchema]: + """Read the schemas stored in the schema_directory. + + Invalid schemas are ignored. + """ + schemas: list[ServiceCollectionSchema] = [] + for schema_path in (self.schema_directory).glob("*" + JSON_EXTENSION): + try: + schemas.append(ServiceCollectionSchema.from_path(schema_path)) + except InvalidSchemaError as e: + logger.warning(f"{e}. Ignoring schema.") + return schemas + + def _check_collection_structure(self, path: Path) -> None: + """Check the structure of a collection. + + Args: + path: Path to the collection. + + Raises: + PathDoesNotExistsError: If the path does not exists. + PathIsNotADirectoryError: If the path is not a directory. + MissingMandatoryDirectoryError: If the collection does not contain a mandatory directory. + """ + if not path.exists(): + raise PathDoesNotExistsError(f"{path} does not exists.") + if not path.is_dir(): + raise PathIsNotADirectoryError(f"{path} is not a directory.") + for mandatory_directory in MANDATORY_DIRECTORIES: + mandatory_path = path / mandatory_directory + if not mandatory_path.exists() or not mandatory_path.is_dir(): + raise MissingMandatoryDirectoryError( + f"{path} does not contain the mandatory directory {mandatory_directory}.", + ) + + +def read_hosts_from_playbook( + playbook_path: Path, inventory_reader: Optional[InventoryReader] +) -> set[str]: + """Read the hosts from a playbook. + + Args: + playbook_path: Path to the playbook. + inventory_reader: Inventory reader. + + Returns: + Set of hosts. + """ + if not inventory_reader: + inventory_reader = InventoryReader() + try: + with playbook_path.open() as fd: + return inventory_reader.get_hosts_from_playbook(fd) + except Exception as e: + raise ValueError(f"Can't parse playbook {playbook_path}.") from e diff --git a/tdp/core/collections.py b/tdp/core/collections/collections.py similarity index 63% rename from tdp/core/collections.py rename to tdp/core/collections/collections.py index cde442d9..05024b34 100644 --- a/tdp/core/collections.py +++ b/tdp/core/collections/collections.py @@ -13,44 +13,30 @@ from __future__ import annotations import logging -from collections import OrderedDict -from collections.abc import Mapping, Sequence +from collections.abc import Iterable +from pathlib import Path +from typing import TYPE_CHECKING, Optional -from tdp.core.collection import Collection from tdp.core.entities.hostable_entity_name import ServiceComponentName -from tdp.core.entities.operation import Operations +from tdp.core.entities.operation import Operations, Playbook +from tdp.core.inventory_reader import InventoryReader from tdp.core.operation import Operation from tdp.core.variables.schema.service_schema import ServiceSchema -logger = logging.getLogger(__name__) - - -class Collections(Mapping[str, Collection]): - """A mapping of collection name to Collection instance. +from .collection_reader import CollectionReader - This class also gather operations from all collections and filter them by their - presence or not in the DAG. - """ +if TYPE_CHECKING: + from tdp.core.types import PathLike - def __init__(self, collections: Mapping[str, Collection]): - self._collections = collections - self._dag_operations, self._other_operations = self._init_operations( - self._collections - ) - self._schemas = self._init_schemas(self._collections) - def __getitem__(self, key): - return self._collections.__getitem__(key) +logger = logging.getLogger(__name__) - def __iter__(self): - return self._collections.__iter__() - def __len__(self): - return self._collections.__len__() +class Collections: + """Concatenation of in use collections.""" - @staticmethod - def from_collection_list(collections: Sequence[Collection]) -> Collections: - """Factory method to build Collections from a sequence of Collection. + def __init__(self, collections: Iterable[CollectionReader]): + """Build Collections from a sequence of Collection. Ordering of the sequence is what will determine the loading order of the operations. An operation can override a previous operation. @@ -59,14 +45,24 @@ def from_collection_list(collections: Sequence[Collection]) -> Collections: collections: Ordered Sequence of Collection object. Returns: - A Collections object. + A Collections object.""" + self._collection_readers = list(collections) - Raises: - ValueError: If a collection name is duplicated. - """ - return Collections( - OrderedDict((collection.name, collection) for collection in collections) - ) + self._playbooks = self._init_playbooks() + self._dag_operations, self._other_operations = self._init_operations() + self._default_var_directories = self._init_default_vars_dirs() + self._schemas = self._init_schemas() + self._services_components = self._init_hostable_entities() + + @staticmethod + def from_collection_paths( + paths: Iterable[PathLike], inventory_reader: Optional[InventoryReader] = None + ): + inventory_reader = inventory_reader or InventoryReader() + collection_readers = [ + CollectionReader.from_path(path, inventory_reader) for path in paths + ] + return Collections(collection_readers) @property def dag_operations(self) -> Operations: @@ -88,25 +84,61 @@ def operations(self) -> Operations: operations.update(self._other_operations) return operations + @property + def playbooks(self) -> dict[str, Playbook]: + """Mapping of playbook name to Playbook instance.""" + return self._playbooks + + @property + def default_vars_dirs(self) -> dict[str, Path]: + """Mapping of collection name to their default vars directory.""" + return self._default_var_directories + @property def schemas(self) -> dict[str, ServiceSchema]: """Mapping of service with their variable schemas.""" return self._schemas - def _init_operations( - self, collections: Mapping[str, Collection] - ) -> tuple[Operations, Operations]: + # ? The mapping is using service name as a string for convenience. Should we keep + # ? this or change it to ServiceName? + @property + def hostable_entities(self) -> dict[str, set[ServiceComponentName]]: + """Mapping of services to their set of components.""" + return self._services_components + + def _init_default_vars_dirs(self) -> dict[str, Path]: + """Mapping of collection name to their default vars directory.""" + default_var_directories = {} + for collection in self._collection_readers: + default_var_directories[collection.name] = collection.default_vars_directory + return default_var_directories + + def _init_playbooks(self) -> dict[str, Playbook]: + playbooks = {} + for collection in self._collection_readers: + playbooks.update(collection.read_playbooks()) + for [playbook_name, playbook] in collection.read_playbooks().items(): + if playbook_name in playbooks: + logger.debug( + f"'{playbook_name}' defined in " + f"'{playbooks[playbook_name].collection_name}' " + f"is overridden by '{collection.name}'" + ) + playbooks[playbook_name] = playbook + return playbooks + + def _init_operations(self) -> tuple[Operations, Operations]: dag_operations = Operations() other_operations = Operations() - for collection in collections.values(): + for collection in self._collection_readers: # Load DAG operations from the dag files - for dag_node in collection.dag_nodes: + for dag_node in collection.read_dag_nodes(): existing_operation = dag_operations.get(dag_node.name) # The read_operation is associated with a playbook defined in the # current collection - if playbook := collection.playbooks.get(dag_node.name): + if playbook := self.playbooks.get(dag_node.name): # TODO: would be nice to dissociate the Operation class from the playbook and store the playbook in the Operation dag_operation_to_register = Operation( name=dag_node.name, @@ -119,13 +151,6 @@ def _init_operations( dag_operation_to_register.depends_on.extend( dag_operations[dag_node.name].depends_on ) - # Print a warning if we override a playbook operation - if not existing_operation.noop: - logger.debug( - f"'{dag_node.name}' defined in " - f"'{existing_operation.collection_name}' " - f"is overridden by '{collection.name}'" - ) # Register the operation dag_operations[dag_node.name] = dag_operation_to_register continue @@ -180,10 +205,10 @@ def _init_operations( # We can't merge the two for loops to handle the case where a playbook operation # is defined in a first collection but not used in the DAG and then used in # the DAG in a second collection. - for collection in collections.values(): + for collection in self._collection_readers: # Load playbook operations to complete the operations list with the # operations that are not defined in the DAG files - for operation_name, playbook in collection.playbooks.items(): + for operation_name, playbook in self.playbooks.items(): if operation_name in dag_operations: continue if operation_name in other_operations: @@ -200,34 +225,20 @@ def _init_operations( return dag_operations, other_operations - def _init_schemas( - self, collections: Mapping[str, Collection] - ) -> dict[str, ServiceSchema]: + def _init_schemas(self) -> dict[str, ServiceSchema]: schemas: dict[str, ServiceSchema] = {} - for collection in collections.values(): - for schema in collection.schemas: + for collection in self._collection_readers: + for schema in collection.read_schemas(): schemas.setdefault(schema.service, ServiceSchema()).add_schema(schema) return schemas - def get_components_from_service( - self, service_name: str - ) -> set[ServiceComponentName]: - """Retrieve the distinct components associated with a specific service. - - This method fetches and returns the unique component names tied to a given - service. The input service is not returned. - - Args: - service_name: The name of the service for which to retrieve associated - components. - - Returns: - A set containing the unique names of components related to the provided - service. - """ - return { - ServiceComponentName(service_name, operation.component_name) - for operation in self.operations.values() - if operation.service_name == service_name - and not operation.is_service_operation() - } + def _init_hostable_entities(self) -> dict[str, set[ServiceComponentName]]: + services_components: dict[str, set[ServiceComponentName]] = {} + for operation in self.operations.values(): + service = services_components.setdefault(operation.service_name, set()) + if not operation.component_name: + continue + service.add( + ServiceComponentName(operation.service_name, operation.component_name) + ) + return services_components diff --git a/tdp/core/dag.py b/tdp/core/dag.py index a0e020dc..84584a00 100644 --- a/tdp/core/dag.py +++ b/tdp/core/dag.py @@ -330,7 +330,7 @@ def warning(collection_name: str, message: str) -> None: # Operations tagged with the noop flag should not have a playbook defined in the collection - if operation_name in collections[operation.collection_name].playbooks: + if operation_name in collections.playbooks: if operation.noop: c_warning( f"Operation '{operation_name}' is noop and the playbook should not exist" diff --git a/tdp/core/deployment/deployment_runner.py b/tdp/core/deployment/deployment_runner.py index f5e778bf..4ff52950 100644 --- a/tdp/core/deployment/deployment_runner.py +++ b/tdp/core/deployment/deployment_runner.py @@ -66,9 +66,7 @@ def _run_operation(self, operation_rec: OperationModel) -> None: return # Execute the operation - playbook_file = ( - self._collections[operation.collection_name].playbooks[operation.name].path - ) + playbook_file = self._collections.playbooks[operation.name].path state, logs = self._executor.execute( playbook=playbook_file, host=operation_rec.host, diff --git a/tdp/core/variables/cluster_variables.py b/tdp/core/variables/cluster_variables.py index 36cc2f02..59782b11 100644 --- a/tdp/core/variables/cluster_variables.py +++ b/tdp/core/variables/cluster_variables.py @@ -74,8 +74,11 @@ def initialize_cluster_variables( cluster_variables = {} collections_and_overrides = [ - (collection_name, collection.default_vars_directory.iterdir()) - for collection_name, collection in collections.items() + (collection_name, default_var_dir.iterdir()) + for [ + collection_name, + default_var_dir, + ] in collections.default_vars_dirs.items() ] for i, override_folder in enumerate(override_folders): diff --git a/tdp/core/variables/service_variables.py b/tdp/core/variables/service_variables.py index 395d522b..bd6cb286 100644 --- a/tdp/core/variables/service_variables.py +++ b/tdp/core/variables/service_variables.py @@ -10,8 +10,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Optional -from tdp.core.collection import YML_EXTENSION -from tdp.core.constants import SERVICE_NAME_MAX_LENGTH +from tdp.core.constants import SERVICE_NAME_MAX_LENGTH, YML_EXTENSION from tdp.core.types import PathLike from tdp.core.variables.schema.exceptions import SchemaValidationError from tdp.core.variables.variables import ( diff --git a/test_dag_order/conftest.py b/test_dag_order/conftest.py index 9da93cd0..5bcb9313 100644 --- a/test_dag_order/conftest.py +++ b/test_dag_order/conftest.py @@ -111,7 +111,7 @@ def collections(request: pytest.FixtureRequest) -> Collections: collections = cast( list["CollectionToTest"], request.config.getoption("collection_paths") ) - return Collections.from_collection_list(collections) + return Collections(collections) @pytest.fixture(scope="session") diff --git a/test_dag_order/helpers.py b/test_dag_order/helpers.py index 0f2d00ee..f33de363 100644 --- a/test_dag_order/helpers.py +++ b/test_dag_order/helpers.py @@ -10,13 +10,13 @@ import yaml -from tdp.core.collection import ( - Collection, +from tdp.core.collections import Collections +from tdp.core.collections.collection_reader import ( + CollectionReader, MissingMandatoryDirectoryError, PathDoesNotExistsError, PathIsNotADirectoryError, ) -from tdp.core.collections import Collections from tdp.core.constants import YML_EXTENSION from tdp.core.entities.hostable_entity_name import ( ServiceName, @@ -113,7 +113,7 @@ def __len__(self): return len(self.fixtures) -class CollectionToTest(Collection): +class CollectionToTest(CollectionReader): """A Collection containing a rules directory""" def __init__(self, path: Union[str, pathlib.Path]): @@ -187,7 +187,7 @@ def resolve_components( service_component_map: dict[str, str] = {} for service_component in service_components: if isinstance(parse_hostable_entity_name(service_component), ServiceName): - for component in collections.get_components_from_service(service_component): + for component in collections.hostable_entities[service_component]: resolved_components.add(component.name) service_component_map[component.name] = service_component else: diff --git a/tests/unit/core/conftest.py b/tests/unit/core/conftest.py index efa9ae84..51893936 100644 --- a/tests/unit/core/conftest.py +++ b/tests/unit/core/conftest.py @@ -4,7 +4,6 @@ import pytest from tdp.core.cluster_status import ClusterStatus -from tdp.core.collection import Collection from tdp.core.collections import Collections from tdp.core.dag import Dag from tdp.core.variables import ClusterVariables @@ -59,9 +58,7 @@ def mock_collections(tmp_path_factory: pytest.TempPathFactory) -> Collections: }, } generate_collection_at_path(path=temp_collection_path, dag=mock_dag, vars=mock_vars) - return Collections.from_collection_list( - [Collection.from_path(temp_collection_path)] - ) + return Collections.from_collection_paths([temp_collection_path]) @pytest.fixture(scope="session") diff --git a/tests/unit/core/models/test_deployment_log.py b/tests/unit/core/models/test_deployment_log.py index e81d3e4b..79a1ba81 100644 --- a/tests/unit/core/models/test_deployment_log.py +++ b/tests/unit/core/models/test_deployment_log.py @@ -9,7 +9,6 @@ import pytest from sqlalchemy import Engine -from tdp.core.collection import Collection from tdp.core.collections import ( Collections, ) @@ -163,8 +162,9 @@ def test_multiple_host(self, tmp_path_factory: pytest.TempPathFactory): ] } generate_collection_at_path(collection_path, dag_service_operations, {}) - collection = Collection(collection_path, MockInventoryReader(hosts)) - collections = Collections.from_collection_list([collection]) + collections = Collections.from_collection_paths( + [collection_path], MockInventoryReader(hosts) + ) deployment = DeploymentModel.from_operations( collections, operations_names, hosts diff --git a/tests/unit/core/test_collection.py b/tests/unit/core/test_collection.py deleted file mode 100644 index 1c4a8269..00000000 --- a/tests/unit/core/test_collection.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2022 TOSIT.IO -# SPDX-License-Identifier: Apache-2.0 - -from pathlib import Path - -import pytest -from pydantic import ValidationError - -from tdp.core.collection import ( - Collection, - MissingMandatoryDirectoryError, - PathDoesNotExistsError, - PathIsNotADirectoryError, - check_collection_structure, - get_collection_dag_nodes, - get_collection_playbooks, - read_dag_file, - read_hosts_from_playbook, -) -from tdp.core.constants import ( - DAG_DIRECTORY_NAME, - DEFAULT_VARS_DIRECTORY_NAME, - PLAYBOOKS_DIRECTORY_NAME, -) -from tests.conftest import generate_collection_at_path -from tests.unit.core.models.test_deployment_log import ( - MockInventoryReader, -) - - -def test_collection_from_path_does_not_exist(): - with pytest.raises(PathDoesNotExistsError): - Collection.from_path("foo") - - -def test_collection_from_path_is_not_a_directory(tmp_path: Path): - empty_file = tmp_path / "foo" - empty_file.touch() - with pytest.raises(PathIsNotADirectoryError): - Collection.from_path(empty_file) - - -def test_collection_from_path_missing_mandatory_directory(tmp_path: Path): - with pytest.raises(MissingMandatoryDirectoryError): - Collection.from_path(tmp_path) - - -def test_collection_from_path(tmp_path_factory: pytest.TempPathFactory): - collection_path = tmp_path_factory.mktemp("collection") - dag_service_operations = { - "service": [ - {"name": "service_install"}, - {"name": "service_config"}, - ], - } - service_vars = { - "service": { - "service": {}, - }, - } - generate_collection_at_path(collection_path, dag_service_operations, service_vars) - collection = Collection.from_path(collection_path) - assert "service_install" in collection.playbooks - assert "service_config" in collection.playbooks - - -def test_read_hosts_from_playbook(tmp_path: Path): - playbook_path = tmp_path / "playbook.yml" - playbook_path.write_text( - """--- -- name: Play 1 - hosts: host1, host2 - tasks: - - name: Task 1 - command: echo "Hello, World!" - -""" - ) - hosts = read_hosts_from_playbook( - playbook_path, MockInventoryReader(["host1", "host2"]) - ) - assert hosts == {"host1", "host2"} - - -def test_init_collection_playbooks(tmp_path: Path): - collection_path = tmp_path / "collection" - playbook_directory = "playbooks" - (playbook_directory_path := collection_path / playbook_directory).mkdir( - parents=True, exist_ok=True - ) - playbook_path_1 = playbook_directory_path / "playbook1.yml" - playbook_path_2 = playbook_directory_path / "playbook2.yml" - playbook_path_1.write_text( - """--- -- name: Play 1 - hosts: host1, host2 - tasks: - - name: Task 1 - command: echo "Hello, World!" -""" - ) - playbook_path_2.write_text( - """--- -- name: Play 2 - hosts: host3, host4 - tasks: - - name: Task 2 - command: echo "Hello, GitHub Copilot!" -""" - ) - playbooks = get_collection_playbooks(collection_path, playbook_directory) - assert len(playbooks) == 2 - assert "playbook1" in playbooks - assert "playbook2" in playbooks - assert playbooks["playbook1"].path == playbook_path_1 - assert playbooks["playbook1"].collection_name == collection_path.name - assert playbooks["playbook2"].path == playbook_path_2 - assert playbooks["playbook2"].collection_name == collection_path.name - - -def test_check_collection_structure_path_does_not_exist(tmp_path: Path): - with pytest.raises(PathDoesNotExistsError): - check_collection_structure(tmp_path / "nonexistent_directory") - - -def test_check_collection_structure_path_is_not_a_directory(tmp_path: Path): - empty_file = tmp_path / "foo" - empty_file.touch() - with pytest.raises(PathIsNotADirectoryError): - check_collection_structure(empty_file) - - -def test_check_collection_structure_missing_mandatory_directory(tmp_path: Path): - with pytest.raises(MissingMandatoryDirectoryError): - check_collection_structure(tmp_path) - - -def test_check_collection_structure_valid_collection(tmp_path: Path): - collection_path = tmp_path / "collection" - for mandatory_directory in ( - DAG_DIRECTORY_NAME, - DEFAULT_VARS_DIRECTORY_NAME, - PLAYBOOKS_DIRECTORY_NAME, - ): - (collection_path / mandatory_directory).mkdir(parents=True, exist_ok=True) - assert check_collection_structure(collection_path) is None - - -def test_read_dag_file(tmp_path: Path): - dag_file_path = tmp_path / "dag_file.yml" - dag_file_path.write_text( - """--- -- name: s1_c1_a - depends_on: - - sx_cx_a -- name: s2_c2_a - depends_on: - - s1_c1_a -- name: s3_c3_a - depends_on: - - sx_cx_a - - sy_cy_a -""" - ) - operations = list(read_dag_file(dag_file_path)) - assert len(operations) == 3 - assert operations[0].name == "s1_c1_a" - assert operations[0].depends_on == ["sx_cx_a"] - assert operations[1].name == "s2_c2_a" - assert operations[1].depends_on == ["s1_c1_a"] - assert operations[2].name == "s3_c3_a" - assert operations[2].depends_on == ["sx_cx_a", "sy_cy_a"] - - -def test_read_dag_file_empty(tmp_path: Path): - dag_file_path = tmp_path / "dag_file.yml" - dag_file_path.write_text("") - with pytest.raises(ValidationError): - list(read_dag_file(dag_file_path)) - - -def test_read_dag_file_with_additional_props(tmp_path: Path): - dag_file_path = tmp_path / "dag_file.yml" - dag_file_path.write_text( - """--- -- name: s1_c1_a - depends_on: - - sx_cx_a - foo: bar -""" - ) - operations = list(read_dag_file(dag_file_path)) - assert len(operations) == 1 - assert operations[0].name == "s1_c1_a" - assert operations[0].depends_on == ["sx_cx_a"] - - -def test_get_collection_dag_nodes(tmp_path: Path): - collection_path = tmp_path / "collection" - dag_directory = "dag" - (dag_directory_path := collection_path / dag_directory).mkdir( - parents=True, exist_ok=True - ) - dag_file_1 = dag_directory_path / "dag1.yml" - dag_file_2 = dag_directory_path / "dag2.yml" - dag_file_1.write_text( - """--- -- name: s1_c1_a - depends_on: - - sx_cx_a -""" - ) - dag_file_2.write_text( - """--- -- name: s2_c2_a - depends_on: - - s1_c1_a -""" - ) - dag_nodes = list(get_collection_dag_nodes(collection_path, dag_directory)) - assert len(dag_nodes) == 2 - assert any( - node.name == "s1_c1_a" and node.depends_on == ["sx_cx_a"] for node in dag_nodes - ) - assert any( - node.name == "s2_c2_a" and node.depends_on == ["s1_c1_a"] for node in dag_nodes - ) diff --git a/tests/unit/core/test_collection_reader.py b/tests/unit/core/test_collection_reader.py new file mode 100644 index 00000000..e5bb63bb --- /dev/null +++ b/tests/unit/core/test_collection_reader.py @@ -0,0 +1,160 @@ +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from tdp.core.collections.collection_reader import ( + CollectionReader, + MissingMandatoryDirectoryError, + PathDoesNotExistsError, + PathIsNotADirectoryError, + read_hosts_from_playbook, +) +from tdp.core.constants import ( + DAG_DIRECTORY_NAME, + DEFAULT_VARS_DIRECTORY_NAME, + PLAYBOOKS_DIRECTORY_NAME, +) +from tests.conftest import generate_collection_at_path +from tests.unit.core.models.test_deployment_log import ( + MockInventoryReader, +) + + +@pytest.fixture(scope="session") +def mock_empty_collection_reader(tmp_path_factory: pytest.TempPathFactory) -> Path: + temp_collection_path = tmp_path_factory.mktemp("mock_collection") + for directory in [ + DAG_DIRECTORY_NAME, + DEFAULT_VARS_DIRECTORY_NAME, + PLAYBOOKS_DIRECTORY_NAME, + ]: + (temp_collection_path / directory).mkdir(parents=True, exist_ok=True) + return temp_collection_path + + +def test_collection_from_path_does_not_exist(): + with pytest.raises(PathDoesNotExistsError): + CollectionReader.from_path("foo") + + +def test_collection_from_path_is_not_a_directory(tmp_path: Path): + empty_file = tmp_path / "foo" + empty_file.touch() + with pytest.raises(PathIsNotADirectoryError): + CollectionReader.from_path(empty_file) + + +def test_collection_from_path_missing_mandatory_directory(tmp_path: Path): + with pytest.raises(MissingMandatoryDirectoryError): + CollectionReader.from_path(tmp_path) + + +def test_collection_from_path(tmp_path_factory: pytest.TempPathFactory): + collection_path = tmp_path_factory.mktemp("collection") + dag_service_operations = { + "service": [ + {"name": "service_install"}, + {"name": "service_config"}, + ], + } + service_vars = { + "service": { + "service": {}, + }, + } + generate_collection_at_path(collection_path, dag_service_operations, service_vars) + playbooks = CollectionReader.from_path(collection_path).read_playbooks() + assert "service_install" in playbooks + assert "service_config" in playbooks + + +def test_read_hosts_from_playbook(tmp_path: Path): + playbook_path = tmp_path / "playbook.yml" + playbook_path.write_text( + """--- +- name: Play 1 + hosts: host1, host2 + tasks: + - name: Task 1 + command: echo "Hello, World!" + +""" + ) + hosts = read_hosts_from_playbook( + playbook_path, MockInventoryReader(["host1", "host2"]) + ) + assert hosts == {"host1", "host2"} + + +def test_collection_reader_read_playbooks(mock_empty_collection_reader: Path): + collection_reader = CollectionReader(mock_empty_collection_reader) + playbook_path_1 = collection_reader.playbooks_directory / "playbook1.yml" + playbook_path_2 = collection_reader.playbooks_directory / "playbook2.yml" + playbook_path_1.write_text( + """--- +- name: Play 1 + hosts: host1, host2 + tasks: + - name: Task 1 + command: echo "Hello, World!" +""" + ) + playbook_path_2.write_text( + """--- +- name: Play 2 + hosts: host3, host4 + tasks: + - name: Task 2 + command: echo "Hello, GitHub Copilot!" +""" + ) + playbooks = collection_reader.read_playbooks() + assert len(playbooks) == 2 + assert "playbook1" in playbooks + assert "playbook2" in playbooks + assert playbooks["playbook1"].path == playbook_path_1 + assert playbooks["playbook1"].collection_name == collection_reader.name + assert playbooks["playbook2"].path == playbook_path_2 + assert playbooks["playbook2"].collection_name == collection_reader.name + + +def test_collection_reader_read_dag_nodes(mock_empty_collection_reader: Path): + collection_reader = CollectionReader(mock_empty_collection_reader) + dag_file_1 = collection_reader.dag_directory / "dag1.yml" + dag_file_2 = collection_reader.dag_directory / "dag2.yml" + dag_file_1.write_text( + """--- +- name: s1_c1_a + depends_on: + - sx_cx_a +""" + ) + dag_file_2.write_text( + """--- +- name: s2_c2_a + depends_on: + - s1_c1_a +""" + ) + dag_nodes = list(collection_reader.read_dag_nodes()) + assert len(dag_nodes) == 2 + assert any( + node.name == "s1_c1_a" and node.depends_on == ["sx_cx_a"] for node in dag_nodes + ) + assert any( + node.name == "s2_c2_a" and node.depends_on == ["s1_c1_a"] for node in dag_nodes + ) + + +def test_collection_reader_read_dag_nodes_empty_file( + mock_empty_collection_reader: Path, +): + collection_reader = CollectionReader(mock_empty_collection_reader) + dag_file = collection_reader.dag_directory / "dag.yml" + dag_file.write_text("") + with pytest.raises(ValidationError): + list(collection_reader.read_dag_nodes()) diff --git a/tests/unit/core/test_collections.py b/tests/unit/core/test_collections.py index 43f11020..e3234375 100644 --- a/tests/unit/core/test_collections.py +++ b/tests/unit/core/test_collections.py @@ -3,7 +3,6 @@ import pytest -from tdp.core.collection import Collection from tdp.core.collections import Collections from tests.conftest import generate_collection_at_path @@ -45,9 +44,9 @@ def test_collections_from_collection_list(tmp_path_factory: pytest.TempPathFacto collection_path_2, dag_service_operations_2, service_vars_2 ) - collection1 = Collection.from_path(collection_path_1) - collection2 = Collection.from_path(collection_path_2) - collections = Collections.from_collection_list([collection1, collection2]) + collections = Collections.from_collection_paths( + [collection_path_1, collection_path_2] + ) assert collections.dag_operations is not None assert "service1_install" in collections.dag_operations