Continue work on testing tool

pyproject.toml

@@ -16,6 +16,7 @@ build-backend = "poetry.core.masonry.api"
 python = "^3.10"
 pyyaml = "^6.0.1"
 pydantic = "^2.6.1"
+types-pyyaml = "^6.0.12.12"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.7.0"
@@ -97,6 +98,7 @@ src = ["src/", "tests/"]
 ignore_names = ["test_*", "fixture_*", "pytest_*"]
 paths = ["src/", "tests/"]
 exclude = [".venv/"]
+min_confidence = 90
 
 
 [tool.pytest.ini_options]

src/hpc_multibench/__main__.py

@@ -2,6 +2,6 @@
 # -*- coding: utf-8 -*-
 """A python template project."""
 
-from main import main
+from hpc_multibench.main import main
 
 main()

src/hpc_multibench/configuration.py

@@ -4,10 +4,9 @@
 
 from pathlib import Path
 from re import search as re_search
-from subprocess import PIPE
-from subprocess import run as subprocess_run
+from subprocess import PIPE  # nosec
+from subprocess import run as subprocess_run  # nosec
 from tempfile import NamedTemporaryFile
-from typing import Optional
 
 BASH_SHEBANG = "#!/bin/sh\n"
 JOB_ID_REGEX = r"Submitted batch job (\d+)"
@@ -22,10 +21,10 @@ def __init__(self, run_command: str):
         self.sbatch_config: dict[str, str] = {}
         self.module_loads: list[str] = []
         self.environment_variables: dict[str, str] = {}
-        self.directory: Optional[Path] = None
+        self.directory: Path | None = None
         self.build_commands: list[str] = []
         self.run_command: str = run_command
-        self.args: Optional[str] = None
+        self.args: str | None = None
 
     @property
     def sbatch_contents(self) -> str:
@@ -53,7 +52,7 @@ def sbatch_contents(self) -> str:
 
         sbatch_file += "\necho '===== RUN"
         if self.name != "":
-            sbatch_file += f"{self.name} "
+            sbatch_file += f" '{self.name}' "
         sbatch_file += " ====='\n"
         sbatch_file += f"time srun {self.run_command} {self.args}\n"
 
@@ -63,15 +62,17 @@ def __repr__(self) -> str:
         """Get the sbatch configuration file defining the run."""
         return self.sbatch_contents
 
-    def run(self) -> Optional[int]:
+    def run(self) -> int | None:
         """Run the specified run configuration."""
         with NamedTemporaryFile(
             suffix=".sbatch", dir=Path("./"), mode="w+"
         ) as sbatch_tmp:
             sbatch_tmp.write(self.sbatch_contents)
             sbatch_tmp.flush()
-            result = subprocess_run(
-                ["sbatch", Path(sbatch_tmp.name)], check=True, stdout=PIPE
+            result = subprocess_run(  # nosec
+                ["sbatch", Path(sbatch_tmp.name)],  # noqa: S603, S607
+                check=True,
+                stdout=PIPE,
             )
             job_id_search = re_search(JOB_ID_REGEX, result.stdout.decode("utf-8"))
             if job_id_search is None:

src/hpc_multibench/dsl_example.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""Manually construct run configurations via the Python DSL interface."""
+
+#
+# def get_cpp_reference_impl() -> RunConfiguration:
+#     """Build a run configuration for the reference implementation."""
+#     run.sbatch_config = {
+#         "ntasks-per-node": "1",
+#         "cpus-per-task": "1",
+#         "mem-per-cpu": "3700",  # max on avon?

src/hpc_multibench/main.py

@@ -4,31 +4,11 @@
 
 from pathlib import Path
 
-from yaml_ingest import absurd
-from configuration import RunConfiguration
-
-def get_greeting() -> str:
-    """Get a string greeting."""
-    return "Hello world!"
-
-
-def get_cpp_reference_impl() -> RunConfiguration:
-    """Build a run configuration for the reference implementation."""
-    run = RunConfiguration("./test_HPCCG")
-    run.build_commands = ["make -j 8"]
-    run.sbatch_config = {
-        "nodes": "1",
-        "ntasks-per-node": "1",
-        "cpus-per-task": "1",
-        "mem-per-cpu": "3700",  # max on avon?
-    }
-    run.module_loads = ["GCC/11.3.0"]
-    run.directory = Path("../0_cpp_versions/0_ref")
-    return run
+from hpc_multibench.yaml_ingest import get_benches
 
 
 def main() -> None:  # pragma: no cover
-    """Say hello."""
-    print(get_greeting())
-    print(absurd())
-    print(get_cpp_reference_impl())
+    """Run the tool."""
+    yaml_path = Path("./testing_plan.yaml")
+    benches = get_benches(yaml_path)
+    print(benches)

src/hpc_multibench/yaml_ingest.py

@@ -1,8 +1,105 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-"""Ingest YAML data."""
+"""
+Ingest YAML data.
 
+Sample code for adding defaults:
+```python
+class Defaults(BaseModel):
+    sbatch_config: Optional[list[str]] = None
+    module_loads: Optional[list[str]] = None
+    environment_variables: Optional[list[str]] = None
+    directory: Optional[Path] = None
+    build_commands: Optional[list[str]] = None
+    run_commands: Optional[list[str]] = None
+    args: Optional[str] = None
+```
+"""
 
-def absurd() -> int:
-    """Be strange."""
-    return 0
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel
+
+from hpc_multibench.configuration import RunConfiguration
+
+
+class Executable(BaseModel):
+    """."""
+
+    sbatch_config: dict[str, Any]
+    module_loads: list[str]
+    environment_variables: dict[str, Any]
+    directory: Path
+    build_commands: list[str]
+    run_command: str
+    args: str | None = None
+
+
+class Bench(BaseModel):
+    """."""
+
+    executables: list[str]
+    matrix: list[dict[str, list[Any]]]
+
+
+class TestPlan(BaseModel):
+    """."""
+
+    executables: dict[str, Executable]
+    benches: dict[str, Bench]
+
+
+def get_test_plan(config_file: Path) -> TestPlan:
+    """."""
+    with config_file.open(encoding="utf-8") as config_handle:
+        config_data = yaml.safe_load(config_handle)
+    return TestPlan(**config_data)
+
+
+def get_run_configuration(name: str, executable: Executable) -> RunConfiguration:
+    """."""
+    run = RunConfiguration(executable.run_command)
+    run.name = name
+    run.sbatch_config = executable.sbatch_config
+    run.module_loads = executable.module_loads
+    run.environment_variables = executable.environment_variables
+    run.directory = Path(executable.directory)
+    run.build_commands = executable.build_commands
+    run.args = executable.args
+    return run
+
+
+def get_bench(
+    bench_name: str, bench: Bench, executables: dict[str, Executable]
+) -> list[RunConfiguration]:
+    """."""
+    bench_run_configurations: list[RunConfiguration] = []
+    # Iterate through matrix
+    for executable_name in bench.executables:
+        if executable_name not in executables:
+            raise RuntimeError(
+                f"Executable '{executable_name}' not in list of defined executables!"
+            )
+        executable = executables[executable_name]
+
+        # Update executable based on matrix
+        bench_run_configurations.append(
+            get_run_configuration(f"{bench_name}/{executable_name}", executable)
+        )
+    return bench_run_configurations
+
+
+def get_benches(config_file: Path) -> dict[str, list[RunConfiguration]]:
+    """."""
+    test_plan = get_test_plan(config_file)
+
+    return {
+        bench_name: get_bench(bench_name, bench, test_plan.executables)
+        for (bench_name, bench) in test_plan.benches.items()
+    }
+
+    # for (bench_name, bench) in test_plan.benches.items():
+    #     for instance in variable:
+    #         for executable in test_plan.executables:

testing_plan.yaml

@@ -0,0 +1,25 @@
+#- defaults:
+#  - "default name":
+executables:
+  "executable name":
+    sbatch_config:
+      "nodes": 1
+      "ntasks-per-node": 1
+      "cpus-per-task": "1"
+      "mem-per-cpu": "3700"
+    module_loads:
+      - "GCC/11.3.0"
+    environment_variables: {}
+    directory: "../0_cpp_versions/0_ref"
+    build_commands:
+      - "make -j 8"
+    run_command: "./test_HPCCG"
+benches:
+ "bench name":
+   executables: ["executable name"]
+   matrix:
+     - args:
+       - "50 50 50"
+       - "100 100 100"
+     - environment_variables:
+       - OMP_NUM_THREADS: 8

tests/test_main.py

@@ -4,12 +4,7 @@
 
 import pytest
 
-from hpc_multibench.main import get_greeting, main
-
-
-def test_get_greeting() -> None:
-    """Test getting the greeting string."""
-    assert get_greeting().startswith("Hello")
+from hpc_multibench.main import main
 
 
 @pytest.mark.integration()