From fd771a224aa5d4608f7d58f77195eca76c0369de Mon Sep 17 00:00:00 2001
From: Paul Farault <diode-farault.consultant@dgfip.finances.gouv.fr>
Date: Mon, 15 Jul 2024 21:20:02 +0200
Subject: [PATCH 1/3] feat: handle run directory for the whole cli
---
tdp/cli/__main__.py | 10 ++++++++++
tdp/cli/commands/deploy.py | 9 ---------
tdp/core/deployment/executor.py | 2 --
tests/e2e/test_tdp_deploy.py | 2 --
4 files changed, 10 insertions(+), 13 deletions(-)
diff --git a/tdp/cli/__main__.py b/tdp/cli/__main__.py
index 4ed44822..d50acb97 100644
--- a/tdp/cli/__main__.py
+++ b/tdp/cli/__main__.py
@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
import logging
+from os import chdir
from pathlib import Path
from typing import Optional
@@ -50,6 +51,15 @@ def load_env(ctx: click.Context, param: click.Parameter, value: Path) -> Optiona
type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
help="Set the level of log output.",
)
+@click.option(
+ "--run-directory",
+ envvar="TDP_RUN_DIRECTORY",
+ type=click.Path(resolve_path=True, exists=True),
+ help="Working directory where the executor is launched (`ansible-playbook` for Ansible).",
+ required=True,
+ callback=lambda ctx, param, value: chdir(value),
+ expose_value=False,
+)
def cli(log_level: str):
setup_logging(log_level)
logging.info("Logging is configured.")
diff --git a/tdp/cli/commands/deploy.py b/tdp/cli/commands/deploy.py
index 2d582023..b61b277f 100644
--- a/tdp/cli/commands/deploy.py
+++ b/tdp/cli/commands/deploy.py
@@ -42,13 +42,6 @@
is_flag=True,
help="Mock the deploy, do not actually run the ansible playbook.",
)
-@click.option(
- "--run-directory",
- envvar="TDP_RUN_DIRECTORY",
- type=click.Path(resolve_path=True, path_type=Path, exists=True),
- help="Working directory where the executor is launched (`ansible-playbook` for Ansible).",
- required=True,
-)
@validate_option
@vars_option
def deploy(
@@ -57,7 +50,6 @@ def deploy(
db_engine: Engine,
force_stale_update: bool,
mock_deploy: bool,
- run_directory: Path,
validate: bool,
vars: Path,
):
@@ -77,7 +69,6 @@ def deploy(
deployment_iterator = DeploymentRunner(
collections=collections,
executor=Executor(
- run_directory=run_directory.absolute() if run_directory else None,
dry=dry or mock_deploy,
),
cluster_variables=cluster_variables,
diff --git a/tdp/core/deployment/executor.py b/tdp/core/deployment/executor.py
index ae6a54db..3138fa4a 100644
--- a/tdp/core/deployment/executor.py
+++ b/tdp/core/deployment/executor.py
@@ -28,7 +28,6 @@ def __init__(self, run_directory=None, dry: bool = False):
ExecutableNotFoundError: If the ansible-playbook command is not found in PATH.
"""
# TODO configurable via config file
- self._rundir = run_directory
self._dry = dry
# Resolve ansible-playbook command
@@ -57,7 +56,6 @@ def _execute_ansible_command(
command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
- cwd=self._rundir,
universal_newlines=True,
)
if res.stdout is None:
diff --git a/tests/e2e/test_tdp_deploy.py b/tests/e2e/test_tdp_deploy.py
index c79fd119..bdd3f234 100644
--- a/tests/e2e/test_tdp_deploy.py
+++ b/tests/e2e/test_tdp_deploy.py
@@ -34,8 +34,6 @@ def test_tdp_deploy_mock(
tdp_init.db_dsn,
"--vars",
str(tdp_init.vars),
- "--run-directory",
- str(tmp_path),
"--mock-deploy",
],
)
From 84fd5d3fc8add9e954dac525335c642764a472b9 Mon Sep 17 00:00:00 2001
From: Paul Farault <diode-farault.consultant@dgfip.finances.gouv.fr>
Date: Thu, 18 Jul 2024 12:14:00 +0200
Subject: [PATCH 2/3] feat: lazy load ansible imports
---
tdp/cli/commands/default_diff.py | 4 +-
tdp/core/ansible_loader.py | 102 +++++++++++++++++++++++++++++++
tdp/core/inventory_reader.py | 32 +++-------
tdp/core/variables/variables.py | 18 ++++--
4 files changed, 123 insertions(+), 33 deletions(-)
create mode 100644 tdp/core/ansible_loader.py
diff --git a/tdp/cli/commands/default_diff.py b/tdp/cli/commands/default_diff.py
index 5401bd36..c8d677e1 100644
--- a/tdp/cli/commands/default_diff.py
+++ b/tdp/cli/commands/default_diff.py
@@ -10,9 +10,9 @@
from typing import TYPE_CHECKING, Optional
import click
-from ansible.utils.vars import merge_hash
from tdp.cli.params import collections_option, vars_option
+from tdp.core.ansible_loader import AnsibleLoader
from tdp.core.constants import DEFAULT_VARS_DIRECTORY_NAME
from tdp.core.variables import ClusterVariables, Variables
@@ -73,7 +73,7 @@ def service_diff(collections, service):
with Variables(default_service_vars_filepath).open(
"r"
) as default_variables:
- default_service_varfile = merge_hash(
+ default_service_varfile = AnsibleLoader.load_merge_hash()(
default_service_varfile, default_variables
)
diff --git a/tdp/core/ansible_loader.py b/tdp/core/ansible_loader.py
new file mode 100644
index 00000000..cf130f32
--- /dev/null
+++ b/tdp/core/ansible_loader.py
@@ -0,0 +1,102 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+
+class AnsibleLoader:
+ """Lazy loader for Ansible classes and functions.
+
+ This class is required as ansible automatically generate a config when imported.
+ """
+
+ _merge_hash = None
+ _from_yaml = None
+ _AnsibleDumper = None
+ _InventoryCLI = None
+ _InventoryReader = None
+ _InventoryManager = None
+ _CustomInventoryCLI = None
+
+ @classmethod
+ def load_merge_hash(cls):
+ """Load the merge_hash function from ansible."""
+ if cls._merge_hash is None:
+ from ansible.utils.vars import merge_hash
+
+ cls._merge_hash = merge_hash
+
+ return cls._merge_hash
+
+ @classmethod
+ def load_from_yaml(cls):
+ """Load the from_yaml function from ansible."""
+ if cls._from_yaml is None:
+ from ansible.parsing.utils.yaml import from_yaml
+
+ cls._from_yaml = from_yaml
+
+ return cls._from_yaml
+
+ @classmethod
+ def load_AnsibleDumper(cls):
+ """Load the AnsibleDumper class from ansible."""
+ if cls._AnsibleDumper is None:
+ from ansible.parsing.yaml.dumper import AnsibleDumper
+
+ cls._AnsibleDumper = AnsibleDumper
+
+ return cls._AnsibleDumper
+
+ @classmethod
+ def load_InventoryCLI(cls):
+ """Load the InventoryCLI class from ansible."""
+ if cls._InventoryCLI is None:
+ from ansible.cli.inventory import InventoryCLI
+
+ cls._InventoryCLI = InventoryCLI
+
+ return cls._InventoryCLI
+
+ @classmethod
+ def load_InventoryReader(cls):
+ """Load the InventoryReader class from ansible."""
+ if cls._InventoryReader is None:
+ from tdp.core.inventory_reader import InventoryReader
+
+ cls._InventoryReader = InventoryReader
+
+ return cls._InventoryReader
+
+ @classmethod
+ def load_InventoryManager(cls):
+ """Load the InventoryManager class from ansible."""
+ if cls._InventoryManager is None:
+ from ansible.inventory.manager import InventoryManager
+
+ cls._InventoryManager = InventoryManager
+
+ return cls._InventoryManager
+
+ @classmethod
+ def get_CustomInventoryCLI(cls):
+ if cls._CustomInventoryCLI is None:
+
+ class CustomInventoryCLI(cls.load_InventoryCLI()):
+ """Represent a custom Ansible inventory CLI which does nothing.
+ This is used to load inventory files with Ansible code.
+ """
+
+ def __init__(self):
+ super().__init__(["program", "--list"])
+ # "run" must be called from CLI (the parent of InventoryCLI), to
+ # initialize context (reading ansible.cfg for example).
+ super(cls.load_InventoryCLI(), self).run()
+ # Get InventoryManager instance
+ _, self.inventory, _ = self._play_prereqs()
+
+ # Avoid call InventoryCLI "run", we do not need to run InventoryCLI
+ def run(self):
+ pass
+
+ cls._CustomInventoryCLI = CustomInventoryCLI()
+
+ return cls._CustomInventoryCLI
diff --git a/tdp/core/inventory_reader.py b/tdp/core/inventory_reader.py
index 7e58041c..eb5b975c 100644
--- a/tdp/core/inventory_reader.py
+++ b/tdp/core/inventory_reader.py
@@ -1,11 +1,11 @@
# Copyright 2022 TOSIT.IO
# SPDX-License-Identifier: Apache-2.0
-from typing import Optional, TextIO
+from typing import TYPE_CHECKING, Optional, TextIO
import yaml
-from ansible.cli.inventory import InventoryCLI
-from ansible.inventory.manager import InventoryManager
+
+from tdp.core.ansible_loader import AnsibleLoader
try:
from yaml import CLoader as Loader
@@ -13,33 +13,15 @@
from yaml import Loader
-# From ansible/cli/inventory.py
-class _CustomInventoryCLI(InventoryCLI):
- """Represent a custom Ansible inventory CLI which does nothing.
- This is used to load inventory files with Ansible code.
- """
-
- def __init__(self):
- super().__init__(["program", "--list"])
- # "run" must be called from CLI (the parent of InventoryCLI), to
- # initialize context (reading ansible.cfg for example).
- super(InventoryCLI, self).run()
- # Get InventoryManager instance
- _, self.inventory, _ = self._play_prereqs()
-
- # Avoid call InventoryCLI "run", we do not need to run InventoryCLI
- def run(self):
- pass
-
-
-custom_inventory_cli_instance = _CustomInventoryCLI()
+if TYPE_CHECKING:
+ from ansible.inventory.manager import InventoryManager
class InventoryReader:
"""Represent an Ansible inventory reader."""
- def __init__(self, inventory: Optional[InventoryManager] = None):
- self.inventory = inventory or custom_inventory_cli_instance.inventory
+ def __init__(self, inventory: Optional["InventoryManager"] = None):
+ self.inventory = inventory or AnsibleLoader.get_CustomInventoryCLI().inventory
def get_hosts(self, *args, **kwargs) -> list[str]:
"""Takes a pattern or list of patterns and returns a list of matching
diff --git a/tdp/core/variables/variables.py b/tdp/core/variables/variables.py
index fa3ba6bb..7a6afbce 100644
--- a/tdp/core/variables/variables.py
+++ b/tdp/core/variables/variables.py
@@ -10,10 +10,8 @@
from weakref import proxy
import yaml
-from ansible.parsing.utils.yaml import from_yaml
-from ansible.parsing.yaml.dumper import AnsibleDumper
-from ansible.utils.vars import merge_hash
+from tdp.core.ansible_loader import AnsibleLoader
from tdp.core.types import PathLike
@@ -90,7 +88,7 @@ def merge(self, mapping: MutableMapping) -> None:
Args:
mapping: Mapping to merge.
"""
- self._content = merge_hash(self._content, mapping)
+ self._content = AnsibleLoader.load_merge_hash()(self._content, mapping)
def __getitem__(self, key):
return self._content.__getitem__(key)
@@ -131,7 +129,10 @@ def __init__(self, path: Path, mode: Optional[str] = None):
self._file_path = path
self._file_descriptor = open(self._file_path, mode or "r+")
# Initialize the content of the variables file
- super().__init__(content=from_yaml(self._file_descriptor) or {}, name=path.name)
+ super().__init__(
+ content=AnsibleLoader.load_from_yaml()(self._file_descriptor) or {},
+ name=path.name,
+ )
def __enter__(self) -> _VariablesIOWrapper:
return proxy(self)
@@ -152,7 +153,12 @@ def _flush_on_disk(self) -> None:
# Write the content of the variables file on disk
self._file_descriptor.seek(0)
self._file_descriptor.write(
- yaml.dump(self._content, Dumper=AnsibleDumper, sort_keys=False, width=1000)
+ yaml.dump(
+ self._content,
+ Dumper=AnsibleLoader.load_AnsibleDumper(),
+ sort_keys=False,
+ width=1000,
+ )
)
self._file_descriptor.truncate()
self._file_descriptor.flush()
From 95f681656c881b4ebf3c9bb2512a5ca8ea8a0b80 Mon Sep 17 00:00:00 2001
From: Paul Farault <diode-farault.consultant@dgfip.finances.gouv.fr>
Date: Thu, 18 Jul 2024 15:01:40 +0200
Subject: [PATCH 3/3] feat: rename --run-directory to --cwd
---
tdp/cli/__main__.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tdp/cli/__main__.py b/tdp/cli/__main__.py
index d50acb97..597f6457 100644
--- a/tdp/cli/__main__.py
+++ b/tdp/cli/__main__.py
@@ -52,10 +52,10 @@ def load_env(ctx: click.Context, param: click.Parameter, value: Path) -> Optiona
help="Set the level of log output.",
)
@click.option(
- "--run-directory",
- envvar="TDP_RUN_DIRECTORY",
+ "--cwd",
+ envvar="TDP_CWD",
type=click.Path(resolve_path=True, exists=True),
- help="Working directory where the executor is launched (`ansible-playbook` for Ansible).",
+ help="Current working directory, where the command will be executed.",
required=True,
callback=lambda ctx, param, value: chdir(value),
expose_value=False,