From 24ab6200d2e529c1ac843f45496611423484c0c6 Mon Sep 17 00:00:00 2001 From: Travis Byrne <42480803+Byrnetp@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:03:51 -0600 Subject: [PATCH 01/24] UW-627: Adding a test Jupyter notebook (#508) We are adding an example Jupyter notebook to test using a pull request. --- .gitignore | 1 + notebooks/example.ipynb | 83 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 notebooks/example.ipynb diff --git a/.gitignore b/.gitignore index fa050f56e..76191013a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ *.swp .coverage __pycache__ +.ipynb_checkpoints \ No newline at end of file diff --git a/notebooks/example.ipynb b/notebooks/example.ipynb new file mode 100644 index 000000000..53c48f765 --- /dev/null +++ b/notebooks/example.ipynb @@ -0,0 +1,83 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "afc23938-c4e4-4240-8fdc-66f0fff9ec06", + "metadata": {}, + "source": [ + "# Example Notebook\n", + "\n", + "## Example 1: Building YAML config from a Python dictionary\n", + "First, we need to import `uwtools.api.config` from the uwtools python package." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "a4a3fb02-06e0-4bf8-a78a-bfdb4fc474b3", + "metadata": {}, + "outputs": [], + "source": [ + "import uwtools.api.config as config" + ] + }, + { + "cell_type": "markdown", + "id": "ec5c7a56-d605-4a58-bd48-4eaba9cd25bc", + "metadata": {}, + "source": [ + "The `config.get_yaml_config` method can create a `YAMLconfig` object when given a Python dictionary." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "71957053-4264-42d5-a4e7-43b50a0ed4e4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "values:\n", + " date: 20240105\n", + " greeting: Good Night\n", + " recipient: Moon\n", + " repeat: 2" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# An example Python dictionary\n", + "dictionary = {\"values\":{\"date\":20240105, \"greeting\":\"Good Night\", \"recipient\":\"Moon\", \"repeat\":2}}\n", + "\n", + "# Build a YAMLconfig object from the dictionary\n", + "config.get_yaml_config(dictionary)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 429d5c12c48a9f199037d914f8c47bb07308496b Mon Sep 17 00:00:00 2001 From: Travis Byrne <42480803+Byrnetp@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:02:11 -0600 Subject: [PATCH 02/24] UW-644 Use testbook to test the example Jupyter notebook (#528) Adds a unit testing file to test the example Jupyter notebook using testbook Creates a make target for testing notebooks Adds cells to the notebook that demonstrate writing a file to disk by rendering a template and yaml files to facilitate this Adds cells to the notebook that include cell magics --- Makefile | 5 +- notebooks/Makefile | 9 ++ notebooks/example.ipynb | 165 ++++++++++++++++++++++++++-- notebooks/fixtures/user_config.yaml | 3 + notebooks/fixtures/user_values.yaml | 3 + notebooks/install-deps | 1 + notebooks/pyproject.toml | 2 + notebooks/tests/test_example.py | 52 +++++++++ 8 files changed, 228 insertions(+), 12 deletions(-) create mode 100644 notebooks/Makefile create mode 100644 notebooks/fixtures/user_config.yaml create mode 100644 notebooks/fixtures/user_values.yaml create mode 100644 notebooks/install-deps create mode 100644 notebooks/pyproject.toml create mode 100644 notebooks/tests/test_example.py diff --git a/Makefile b/Makefile index f42433637..d0899a757 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ CHANNELS = $(addprefix -c ,$(shell tr '\n' ' ' <$(RECIPE_DIR)/channels)) -c local METADEPS = $(addprefix $(RECIPE_DIR)/,meta.yaml) src/uwtools/resources/info.json METAJSON = $(RECIPE_DIR)/meta.json -TARGETS = clean-devenv devshell docs env format lint meta package test typecheck unittest +TARGETS = clean-devenv devshell docs env format lint meta package test test-nb typecheck unittest export RECIPE_DIR := $(shell cd ./recipe && pwd) @@ -41,6 +41,9 @@ package: meta test: recipe/run_test.sh +test-nb: + $(MAKE) -C notebooks test-nb + typecheck: recipe/run_test.sh typecheck diff --git a/notebooks/Makefile b/notebooks/Makefile new file mode 100644 index 000000000..f6c162470 --- /dev/null +++ b/notebooks/Makefile @@ -0,0 +1,9 @@ +TARGETS = test-nb + +.PHONY: $(TARGETS) + +all: + $(error Valid targets are: $(TARGETS)) + +test-nb: + pytest tests diff --git a/notebooks/example.ipynb b/notebooks/example.ipynb index 53c48f765..62c811302 100644 --- a/notebooks/example.ipynb +++ b/notebooks/example.ipynb @@ -18,7 +18,7 @@ "metadata": {}, "outputs": [], "source": [ - "import uwtools.api.config as config" + "from uwtools.api import config" ] }, { @@ -34,28 +34,171 @@ "execution_count": 2, "id": "71957053-4264-42d5-a4e7-43b50a0ed4e4", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "values:\n", + " date: 20240105\n", + " greeting: Good Night\n", + " recipient: Moon\n", + " repeat: 2\n" + ] + } + ], + "source": [ + "# An example Python dictionary\n", + "dictionary = {\"values\":{\"date\":20240105, \"greeting\":\"Good Night\", \"recipient\":\"Moon\", \"repeat\":2}}\n", + "\n", + "# Build a YAMLconfig object from the dictionary\n", + "config_yaml = config.get_yaml_config(dictionary)\n", + "print(config_yaml)" + ] + }, + { + "cell_type": "markdown", + "id": "dd7bcc1a-6402-42f8-884b-b22ebe1f04f5", + "metadata": {}, + "source": [ + "## Example 2: Rendering a template with uwtools\n", + "Next, let's look at using the `template` tool to render a Jinja2 template." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "bb672e22-d206-4d65-9dfd-b4b480d0937b", + "metadata": {}, + "outputs": [], + "source": [ + "from uwtools.api import template" + ] + }, + { + "cell_type": "markdown", + "id": "9328416e-b1b8-4062-9c56-5b37440cbc4a", + "metadata": {}, + "source": [ + "We have a Jinja2 template file in `fixtures/user_config.yaml` that looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "67ebded6-474e-465d-81f2-ca8f76b89f54", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "user:\n", + " name: {{ first }} {{ last }}\n", + " favorite_food: {{ food }}\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat ./fixtures/user_config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "8f29f1dc-a201-4356-a729-7b40565c8aa4", + "metadata": {}, + "source": [ + "We can use another yaml file that contains the values we want to add to the template to complete it:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "5df2f19a-6cb2-4c67-8fdf-2a5aa563b4e5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "first: John\n", + "last: Doe\n", + "food: burritos" + ] + } + ], + "source": [ + "%%bash\n", + "cat ./fixtures/user_values.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "cd2999db-c1e7-4ae3-9169-b807b63712e7", + "metadata": {}, + "source": [ + "Using `template.render` we can render the `user_config.yaml` file using the values supplied by the `user_values.yaml` to create a complete and ready to use config file." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ce7f9265-4536-4163-9e2f-53aaa6ff0c63", + "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "values:\n", - " date: 20240105\n", - " greeting: Good Night\n", - " recipient: Moon\n", - " repeat: 2" + "'user:\\n name: John Doe\\n favorite_food: burritos'" ] }, - "execution_count": 2, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# An example Python dictionary\n", - "dictionary = {\"values\":{\"date\":20240105, \"greeting\":\"Good Night\", \"recipient\":\"Moon\", \"repeat\":2}}\n", + "# path of Jinja2 template to update\n", + "source = \"./fixtures/user_config.yaml\"\n", "\n", - "# Build a YAMLconfig object from the dictionary\n", - "config.get_yaml_config(dictionary)" + "# values to add\n", + "vals = \"./fixtures/user_values.yaml\"\n", + "\n", + "# destination of the rendered file\n", + "target = \"./fixtures/rendered_config.yaml\"\n", + "\n", + "# render the template \n", + "print(template.render(values_src=vals, values_format=\"yaml\", input_file=source, output_file=target))" + ] + }, + { + "cell_type": "markdown", + "id": "f5d0b35f-d894-4758-a462-81ffa88760c0", + "metadata": {}, + "source": [ + "Let's take a look at the rendered file:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9fe7687f-5c6c-44fc-aaf7-92711ce97457", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "user:\n", + " name: John Doe\n", + " favorite_food: burritos\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat ./fixtures/rendered_config.yaml" ] } ], diff --git a/notebooks/fixtures/user_config.yaml b/notebooks/fixtures/user_config.yaml new file mode 100644 index 000000000..78ddc7977 --- /dev/null +++ b/notebooks/fixtures/user_config.yaml @@ -0,0 +1,3 @@ +user: + name: {{ first }} {{ last }} + favorite_food: {{ food }} diff --git a/notebooks/fixtures/user_values.yaml b/notebooks/fixtures/user_values.yaml new file mode 100644 index 000000000..5ae0cc34e --- /dev/null +++ b/notebooks/fixtures/user_values.yaml @@ -0,0 +1,3 @@ +first: John +last: Doe +food: burritos diff --git a/notebooks/install-deps b/notebooks/install-deps new file mode 100644 index 000000000..84dca4d86 --- /dev/null +++ b/notebooks/install-deps @@ -0,0 +1 @@ +conda install -q -y nb_conda_kernels testbook diff --git a/notebooks/pyproject.toml b/notebooks/pyproject.toml new file mode 100644 index 000000000..0ce02b608 --- /dev/null +++ b/notebooks/pyproject.toml @@ -0,0 +1,2 @@ +[tool.pytest.ini_options] +filterwarnings = ["ignore::DeprecationWarning"] diff --git a/notebooks/tests/test_example.py b/notebooks/tests/test_example.py new file mode 100644 index 000000000..13e9ff708 --- /dev/null +++ b/notebooks/tests/test_example.py @@ -0,0 +1,52 @@ +import os + +import yaml +from testbook import testbook + +# Run all cells of the example notebook. +@testbook("./example.ipynb", execute=True) +def test_get_yaml_config(tb): + + # Check output text of the cell that prints the YAMLconfig object. + assert ( + tb.cell_output_text(3) + == "values:\n date: 20240105\n greeting: Good Night\n recipient: Moon\n repeat: 2" + ) + + # Extract the config_yaml variable from the notebook and test its values. + nb_yaml = tb.ref("config_yaml") + assert nb_yaml["values"] == { + "date": 20240105, + "greeting": "Good Night", + "recipient": "Moon", + "repeat": 2, + } + + +def test_template_render(): + # Remove the rendered file if it exists. + rendered_path = "./fixtures/rendered_config.yaml" + if os.path.exists(rendered_path): + os.remove(rendered_path) + + # Run all cells of the example notebook. + with testbook("./example.ipynb", execute=True) as tb: + + # Check output text of cells with %%bash cell magics. + assert ( + tb.cell_output_text(7) + == "user:\n name: {{ first }} {{ last }}\n favorite_food: {{ food }}" + ) + assert tb.cell_output_text(9) == "first: John\nlast: Doe\nfood: burritos" + assert tb.cell_output_text(13) == "user:\n name: John Doe\n favorite_food: burritos" + + # Check that the rendered file was created in the correct directory. + assert os.path.exists(rendered_path) + + # Check the contents of the rendered file. + with open(rendered_path, "r") as f: + user_config = yaml.safe_load(f) + assert user_config["user"] == {"name": "John Doe", "favorite_food": "burritos"} + + # Clean up the temporary file after the test is run. + os.remove(rendered_path) From 0fdd366a318449075b67f65ccb71009780bb074b Mon Sep 17 00:00:00 2001 From: Travis Byrne <42480803+Byrnetp@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:12:15 -0600 Subject: [PATCH 03/24] UW-645 Automate the tests for Jupyter notebooks using GitHub Actions (#538) --- .github/scripts/test-nb.sh | 7 +++++++ .github/workflows/test.yaml | 2 ++ notebooks/install-deps | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/test-nb.sh diff --git a/.github/scripts/test-nb.sh b/.github/scripts/test-nb.sh new file mode 100644 index 000000000..e44216090 --- /dev/null +++ b/.github/scripts/test-nb.sh @@ -0,0 +1,7 @@ +set -ae +source $(dirname ${BASH_SOURCE[0]})/common.sh +ci_conda_activate +conda install -c ufs-community -c conda-forge --override-channels --repodata-fn repodata.json uwtools +cd notebooks +source install-deps +make test-nb diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cd94bd0a9..97eeff098 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,3 +23,5 @@ jobs: run: .github/scripts/format-check.sh - name: Test run: .github/scripts/test.sh + - name: Test Notebooks + run: .github/scripts/test-nb.sh diff --git a/notebooks/install-deps b/notebooks/install-deps index 84dca4d86..4f1226968 100644 --- a/notebooks/install-deps +++ b/notebooks/install-deps @@ -1 +1 @@ -conda install -q -y nb_conda_kernels testbook +conda install -q -y jupyterlab pytest testbook From 2733a1062b8f37e82aca493784ef99816c3b72eb Mon Sep 17 00:00:00 2001 From: lexbuko22 <119462215+lexbuko22@users.noreply.github.com> Date: Fri, 26 Jul 2024 08:58:47 -0600 Subject: [PATCH 04/24] Updated readme with binder links (#535) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d7a3b5c10..8458a2cdd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![pkgpage](https://anaconda.org/ufs-community/uwtools/badges/version.svg)](https://anaconda.org/ufs-community/uwtools) [![pkgfiles](https://anaconda.org/ufs-community/uwtools/badges/latest_release_date.svg)](https://anaconda.org/ufs-community/uwtools/files) [![docs](https://readthedocs.org/projects/uwtools/badge/?version=main)](https://uwtools.readthedocs.io/en/main/?badge=main) +[![pkgpage](https://anaconda.org/ufs-community/uwtools/badges/version.svg)](https://anaconda.org/ufs-community/uwtools) [![pkgfiles](https://anaconda.org/ufs-community/uwtools/badges/latest_release_date.svg)](https://anaconda.org/ufs-community/uwtools/files) [![docs](https://readthedocs.org/projects/uwtools/badge/?version=main)](https://uwtools.readthedocs.io/en/main/?badge=main) [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/ufs-community/uwtools/notebooks?labpath=notebooks%2Fexample.ipynb) # uwtools From f5977c9804e375afb01b5b03d5154da0d3eb5ef9 Mon Sep 17 00:00:00 2001 From: lexbuko22 <119462215+lexbuko22@users.noreply.github.com> Date: Fri, 26 Jul 2024 09:10:45 -0600 Subject: [PATCH 05/24] Added binder link to user guide (#544) --- docs/sections/user_guide/index.rst | 10 ++++++++++ notebooks/environment.yaml | 1 + 2 files changed, 11 insertions(+) create mode 100644 notebooks/environment.yaml diff --git a/docs/sections/user_guide/index.rst b/docs/sections/user_guide/index.rst index a385a1801..eff5d3b17 100644 --- a/docs/sections/user_guide/index.rst +++ b/docs/sections/user_guide/index.rst @@ -5,6 +5,16 @@ User Guide :maxdepth: 2 installation + +.. raw:: html + + + +.. toctree:: + :maxdepth: 2 + cli/index api/index yaml/index diff --git a/notebooks/environment.yaml b/notebooks/environment.yaml new file mode 100644 index 000000000..ea9d01798 --- /dev/null +++ b/notebooks/environment.yaml @@ -0,0 +1 @@ +uwtools From 765fe830f96ca082e6c034ec0e0a8e278adc679d Mon Sep 17 00:00:00 2001 From: lexbuko22 <119462215+lexbuko22@users.noreply.github.com> Date: Fri, 26 Jul 2024 10:16:07 -0600 Subject: [PATCH 06/24] Adding compare() example (#547) --- notebooks/binder/environment.yml | 6 + notebooks/example.ipynb | 235 +++++++++++++++++++++- notebooks/fixtures/config_test_file_a.nml | 1 + notebooks/fixtures/config_test_file_b.nml | 4 + notebooks/fixtures/config_test_file_c.nml | 4 + notebooks/fixtures/rendered_config.yaml | 3 + notebooks/tests/test_example.py | 9 + 7 files changed, 252 insertions(+), 10 deletions(-) create mode 100644 notebooks/binder/environment.yml create mode 100644 notebooks/fixtures/config_test_file_a.nml create mode 100644 notebooks/fixtures/config_test_file_b.nml create mode 100644 notebooks/fixtures/config_test_file_c.nml create mode 100644 notebooks/fixtures/rendered_config.yaml diff --git a/notebooks/binder/environment.yml b/notebooks/binder/environment.yml new file mode 100644 index 000000000..0ccdf58bd --- /dev/null +++ b/notebooks/binder/environment.yml @@ -0,0 +1,6 @@ +name: default +channels: + - conda-forge + - ufs-community +dependencies: + - uwtools diff --git a/notebooks/example.ipynb b/notebooks/example.ipynb index 62c811302..0f9b821bb 100644 --- a/notebooks/example.ipynb +++ b/notebooks/example.ipynb @@ -124,7 +124,7 @@ "text": [ "first: John\n", "last: Doe\n", - "food: burritos" + "food: burritos\n" ] } ], @@ -148,14 +148,13 @@ "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "'user:\\n name: John Doe\\n favorite_food: burritos'" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "user:\n", + " name: John Doe\n", + " favorite_food: burritos\n" + ] } ], "source": [ @@ -200,6 +199,222 @@ "%%bash\n", "cat ./fixtures/rendered_config.yaml" ] + }, + { + "cell_type": "markdown", + "id": "ea5eff92", + "metadata": {}, + "source": [ + "## Example 3: Comparing two config files\n", + "Let's explore using the `config.compare()` method to compare two config files." + ] + }, + { + "cell_type": "markdown", + "id": "71d27692", + "metadata": {}, + "source": [ + "We again need to start by importing the `uwtools.api.config` from the `uwtools` python package." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "7fb40514", + "metadata": {}, + "outputs": [], + "source": [ + "from uwtools.api import config\n", + "from uwtools.api.logging import use_uwtools_logger\n", + "use_uwtools_logger()" + ] + }, + { + "cell_type": "markdown", + "id": "b0103f44", + "metadata": {}, + "source": [ + "Please review the [config.compare() documentation](https://uwtools.readthedocs.io/en/main/sections/user_guide/api/config.html#uwtools.api.config.compare) for full information on this function's arguments." + ] + }, + { + "cell_type": "markdown", + "id": "b121dc29", + "metadata": {}, + "source": [ + "For example, let's compare two Fortran namelist files with differences:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8279b5c6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "&foo n=88, s=\"string\" /" + ] + } + ], + "source": [ + "%%bash\n", + "cat ./fixtures/config_test_file_a.nml" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e3232247", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "&FOO\n", + " S = \"string\"\n", + " N = 99\n", + "/" + ] + } + ], + "source": [ + "%%bash\n", + "cat ./fixtures/config_test_file_b.nml" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "5dcff125", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-07-18T19:27:40] INFO - fixtures/config_test_file_a.nml\n", + "[2024-07-18T19:27:40] INFO + fixtures/config_test_file_b.nml\n", + "[2024-07-18T19:27:40] INFO ---------------------------------------------------------------------\n", + "[2024-07-18T19:27:40] INFO foo: n: - 88 + 99\n" + ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "file_a = \"./fixtures/config_test_file_a.nml\"\n", + "different_file_b = \"./fixtures/config_test_file_b.nml\"\n", + "config.compare(file_a, different_file_b)" + ] + }, + { + "cell_type": "markdown", + "id": "c16c38d0", + "metadata": {}, + "source": [ + "The `config()` method returns `False` to denote the files are different. The UW logger shows the difference, of one file containing `n = 88`, and one file containing `n = 99`.\n", + "\n", + "Now to compare two semantically equivalent files:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "370149a3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "&foo n=88, s=\"string\" /" + ] + } + ], + "source": [ + "%%bash\n", + "cat ./fixtures/config_test_file_a.nml" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "bd444fba", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "&FOO\n", + " S = \"string\"\n", + " N = 88\n", + "/" + ] + } + ], + "source": [ + "%%bash\n", + "cat ./fixtures/config_test_file_c.nml" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "9d658b72", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-07-18T19:27:40] INFO - fixtures/config_test_file_a.nml\n", + "[2024-07-18T19:27:40] INFO + fixtures/config_test_file_c.nml\n", + "[2024-07-18T19:27:40] INFO ---------------------------------------------------------------------\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "file_a = \"./fixtures/config_test_file_a.nml\"\n", + "identical_file_c = \"./fixtures/config_test_file_c.nml\"\n", + "config.compare(file_a, identical_file_c)" + ] + }, + { + "cell_type": "markdown", + "id": "b84db243", + "metadata": {}, + "source": [ + "The `config()` method returns `True` to denote the files are semantically equivalent." + ] + }, + { + "cell_type": "markdown", + "id": "5b39d871", + "metadata": {}, + "source": [] } ], "metadata": { @@ -218,7 +433,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/fixtures/config_test_file_a.nml b/notebooks/fixtures/config_test_file_a.nml new file mode 100644 index 000000000..42e15827f --- /dev/null +++ b/notebooks/fixtures/config_test_file_a.nml @@ -0,0 +1 @@ +&foo n=88, s="string" / diff --git a/notebooks/fixtures/config_test_file_b.nml b/notebooks/fixtures/config_test_file_b.nml new file mode 100644 index 000000000..ecb87d942 --- /dev/null +++ b/notebooks/fixtures/config_test_file_b.nml @@ -0,0 +1,4 @@ +&FOO + S = "string" + N = 99 +/ diff --git a/notebooks/fixtures/config_test_file_c.nml b/notebooks/fixtures/config_test_file_c.nml new file mode 100644 index 000000000..e6603e87d --- /dev/null +++ b/notebooks/fixtures/config_test_file_c.nml @@ -0,0 +1,4 @@ +&FOO + S = "string" + N = 88 +/ diff --git a/notebooks/fixtures/rendered_config.yaml b/notebooks/fixtures/rendered_config.yaml new file mode 100644 index 000000000..2840700bf --- /dev/null +++ b/notebooks/fixtures/rendered_config.yaml @@ -0,0 +1,3 @@ +user: + name: John Doe + favorite_food: burritos diff --git a/notebooks/tests/test_example.py b/notebooks/tests/test_example.py index 13e9ff708..ea3113257 100644 --- a/notebooks/tests/test_example.py +++ b/notebooks/tests/test_example.py @@ -50,3 +50,12 @@ def test_template_render(): # Clean up the temporary file after the test is run. os.remove(rendered_path) + + +# Run all cells of the example notebook. +@testbook("./example.ipynb", execute=True) +def test_compare(tb): + + # Check output text of the cell prints the correct result + assert 'False' in tb.cell_output_text(21) + assert 'True' in tb.cell_output_text(25) From e1b37fafa5c9ce5a8c8b803506be5c3e54722457 Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:12:29 -0600 Subject: [PATCH 07/24] Move binder/environment.yml (#549) --- {notebooks/binder => binder}/environment.yml | 0 notebooks/environment.yaml | 1 - 2 files changed, 1 deletion(-) rename {notebooks/binder => binder}/environment.yml (100%) delete mode 100644 notebooks/environment.yaml diff --git a/notebooks/binder/environment.yml b/binder/environment.yml similarity index 100% rename from notebooks/binder/environment.yml rename to binder/environment.yml diff --git a/notebooks/environment.yaml b/notebooks/environment.yaml deleted file mode 100644 index ea9d01798..000000000 --- a/notebooks/environment.yaml +++ /dev/null @@ -1 +0,0 @@ -uwtools From 452139a9fc61f78ad1b8933b82f6b3ea92a08fe7 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 17 Aug 2024 17:55:22 +0000 Subject: [PATCH 08/24] Add formatting and linting --- notebooks/Makefile | 13 +++++++++++-- notebooks/pyproject.toml | 19 +++++++++++++++++++ notebooks/tests/test_example.py | 7 ++++--- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/notebooks/Makefile b/notebooks/Makefile index f6c162470..3a0635579 100644 --- a/notebooks/Makefile +++ b/notebooks/Makefile @@ -1,9 +1,18 @@ -TARGETS = test-nb +TARGETS = format lint test-nb unittest .PHONY: $(TARGETS) all: $(error Valid targets are: $(TARGETS)) -test-nb: +format: + black tests + isort tests + +lint: + pylint tests + +test-nb: lint unittest + +unittest: pytest tests diff --git a/notebooks/pyproject.toml b/notebooks/pyproject.toml index 0ce02b608..c3096a235 100644 --- a/notebooks/pyproject.toml +++ b/notebooks/pyproject.toml @@ -1,2 +1,21 @@ +[tool.black] +line-length = 100 + +[tool.isort] +line_length = 100 +profile = "black" + +[tool.pylint.main] +recursive = true + +[tool.pylint."messages control"] +disable = [ + "missing-function-docstring", + "missing-module-docstring", +] +enable = [ + "useless-suppression", +] + [tool.pytest.ini_options] filterwarnings = ["ignore::DeprecationWarning"] diff --git a/notebooks/tests/test_example.py b/notebooks/tests/test_example.py index ea3113257..d70fe05a0 100644 --- a/notebooks/tests/test_example.py +++ b/notebooks/tests/test_example.py @@ -3,6 +3,7 @@ import yaml from testbook import testbook + # Run all cells of the example notebook. @testbook("./example.ipynb", execute=True) def test_get_yaml_config(tb): @@ -44,7 +45,7 @@ def test_template_render(): assert os.path.exists(rendered_path) # Check the contents of the rendered file. - with open(rendered_path, "r") as f: + with open(rendered_path, "r", encoding="utf-8") as f: user_config = yaml.safe_load(f) assert user_config["user"] == {"name": "John Doe", "favorite_food": "burritos"} @@ -57,5 +58,5 @@ def test_template_render(): def test_compare(tb): # Check output text of the cell prints the correct result - assert 'False' in tb.cell_output_text(21) - assert 'True' in tb.cell_output_text(25) + assert "False" in tb.cell_output_text(21) + assert "True" in tb.cell_output_text(25) From 2aadaa3db10dab5ac17cf72079db4e5d88c56cf4 Mon Sep 17 00:00:00 2001 From: Travis Byrne <42480803+Byrnetp@users.noreply.github.com> Date: Tue, 20 Aug 2024 08:51:05 -0600 Subject: [PATCH 09/24] UW-651 Jupyter Notebook: Template Mode (#581) --- docs/sections/user_guide/api/template.rst | 3 + docs/sections/user_guide/index.rst | 13 + notebooks/example.ipynb | 72 +-- .../ex2-config.yaml} | 0 .../ex2-rendered-config.yaml} | 0 .../ex2-values.yaml} | 0 .../ex3-config-test-file-a.nml} | 0 .../ex3-config-test-file-b.nml} | 0 .../ex3-config-test-file-c.nml} | 0 .../fixtures/template/render-complete-1.yaml | 3 + .../fixtures/template/render-complete-2.yaml | 3 + .../fixtures/template/render-template.yaml | 3 + .../fixtures/template/render-values.yaml | 3 + .../fixtures/template/translate-complete.yaml | 3 + .../fixtures/template/translate-template.yaml | 3 + notebooks/template.ipynb | 511 ++++++++++++++++++ notebooks/tests/test_example.py | 8 +- notebooks/tests/test_template.py | 75 +++ 18 files changed, 654 insertions(+), 46 deletions(-) rename notebooks/fixtures/{user_config.yaml => example/ex2-config.yaml} (100%) rename notebooks/fixtures/{rendered_config.yaml => example/ex2-rendered-config.yaml} (100%) rename notebooks/fixtures/{user_values.yaml => example/ex2-values.yaml} (100%) rename notebooks/fixtures/{config_test_file_a.nml => example/ex3-config-test-file-a.nml} (100%) rename notebooks/fixtures/{config_test_file_b.nml => example/ex3-config-test-file-b.nml} (100%) rename notebooks/fixtures/{config_test_file_c.nml => example/ex3-config-test-file-c.nml} (100%) create mode 100644 notebooks/fixtures/template/render-complete-1.yaml create mode 100644 notebooks/fixtures/template/render-complete-2.yaml create mode 100644 notebooks/fixtures/template/render-template.yaml create mode 100644 notebooks/fixtures/template/render-values.yaml create mode 100644 notebooks/fixtures/template/translate-complete.yaml create mode 100644 notebooks/fixtures/template/translate-template.yaml create mode 100644 notebooks/template.ipynb create mode 100644 notebooks/tests/test_template.py diff --git a/docs/sections/user_guide/api/template.rst b/docs/sections/user_guide/api/template.rst index ac89e7002..b75de604f 100644 --- a/docs/sections/user_guide/api/template.rst +++ b/docs/sections/user_guide/api/template.rst @@ -1,5 +1,8 @@ ``uwtools.api.template`` ======================== +.. image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/ufs-community/uwtools/notebooks?labpath=notebooks%2Ftemplate.ipynb + .. automodule:: uwtools.api.template :members: diff --git a/docs/sections/user_guide/index.rst b/docs/sections/user_guide/index.rst index eff5d3b17..612a08f4d 100644 --- a/docs/sections/user_guide/index.rst +++ b/docs/sections/user_guide/index.rst @@ -17,4 +17,17 @@ User Guide cli/index api/index + +.. raw:: html + + + +.. toctree:: + :maxdepth: 2 + yaml/index diff --git a/notebooks/example.ipynb b/notebooks/example.ipynb index 0f9b821bb..167a62998 100644 --- a/notebooks/example.ipynb +++ b/notebooks/example.ipynb @@ -48,10 +48,8 @@ } ], "source": [ - "# An example Python dictionary\n", "dictionary = {\"values\":{\"date\":20240105, \"greeting\":\"Good Night\", \"recipient\":\"Moon\", \"repeat\":2}}\n", "\n", - "# Build a YAMLconfig object from the dictionary\n", "config_yaml = config.get_yaml_config(dictionary)\n", "print(config_yaml)" ] @@ -80,7 +78,7 @@ "id": "9328416e-b1b8-4062-9c56-5b37440cbc4a", "metadata": {}, "source": [ - "We have a Jinja2 template file in `fixtures/user_config.yaml` that looks like this:" + "We have a Jinja2 template file in `fixtures/example/ex2-config.yaml` that looks like this:" ] }, { @@ -101,7 +99,7 @@ ], "source": [ "%%bash\n", - "cat ./fixtures/user_config.yaml" + "cat fixtures/example/ex2-config.yaml" ] }, { @@ -130,7 +128,7 @@ ], "source": [ "%%bash\n", - "cat ./fixtures/user_values.yaml" + "cat fixtures/example/ex2-values.yaml" ] }, { @@ -138,7 +136,7 @@ "id": "cd2999db-c1e7-4ae3-9169-b807b63712e7", "metadata": {}, "source": [ - "Using `template.render` we can render the `user_config.yaml` file using the values supplied by the `user_values.yaml` to create a complete and ready to use config file." + "Using `template.render` we can render the `ex2-config.yaml` file using the values supplied by the `ex2-values.yaml` to create a complete and ready to use config file." ] }, { @@ -158,16 +156,12 @@ } ], "source": [ - "# path of Jinja2 template to update\n", - "source = \"./fixtures/user_config.yaml\"\n", + "source = \"fixtures/example/ex2-config.yaml\"\n", "\n", - "# values to add\n", - "vals = \"./fixtures/user_values.yaml\"\n", + "vals = \"fixtures/example/ex2-values.yaml\"\n", "\n", - "# destination of the rendered file\n", - "target = \"./fixtures/rendered_config.yaml\"\n", + "target = \"fixtures/example/ex2-rendered-config.yaml\"\n", "\n", - "# render the template \n", "print(template.render(values_src=vals, values_format=\"yaml\", input_file=source, output_file=target))" ] }, @@ -197,7 +191,7 @@ ], "source": [ "%%bash\n", - "cat ./fixtures/rendered_config.yaml" + "cat fixtures/example/ex2-rendered-config.yaml" ] }, { @@ -255,13 +249,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "&foo n=88, s=\"string\" /" + "&foo n=88, s=\"string\" /\n" ] } ], "source": [ "%%bash\n", - "cat ./fixtures/config_test_file_a.nml" + "cat fixtures/example/ex3-config-test-file-a.nml" ] }, { @@ -277,13 +271,13 @@ "&FOO\n", " S = \"string\"\n", " N = 99\n", - "/" + "/\n" ] } ], "source": [ "%%bash\n", - "cat ./fixtures/config_test_file_b.nml" + "cat fixtures/example/ex3-config-test-file-b.nml" ] }, { @@ -296,10 +290,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-07-18T19:27:40] INFO - fixtures/config_test_file_a.nml\n", - "[2024-07-18T19:27:40] INFO + fixtures/config_test_file_b.nml\n", - "[2024-07-18T19:27:40] INFO ---------------------------------------------------------------------\n", - "[2024-07-18T19:27:40] INFO foo: n: - 88 + 99\n" + "[2024-08-18T10:24:32] INFO - fixtures/example/ex3-config-test-file-a.nml\n", + "[2024-08-18T10:24:32] INFO + fixtures/example/ex3-config-test-file-b.nml\n", + "[2024-08-18T10:24:32] INFO ---------------------------------------------------------------------\n", + "[2024-08-18T10:24:32] INFO foo: n: - 88 + 99\n" ] }, { @@ -314,8 +308,8 @@ } ], "source": [ - "file_a = \"./fixtures/config_test_file_a.nml\"\n", - "different_file_b = \"./fixtures/config_test_file_b.nml\"\n", + "file_a = \"fixtures/example/ex3-config-test-file-a.nml\"\n", + "different_file_b = \"fixtures/example/ex3-config-test-file-b.nml\"\n", "config.compare(file_a, different_file_b)" ] }, @@ -339,13 +333,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "&foo n=88, s=\"string\" /" + "&foo n=88, s=\"string\" /\n" ] } ], "source": [ "%%bash\n", - "cat ./fixtures/config_test_file_a.nml" + "cat fixtures/example/ex3-config-test-file-a.nml" ] }, { @@ -361,13 +355,13 @@ "&FOO\n", " S = \"string\"\n", " N = 88\n", - "/" + "/\n" ] } ], "source": [ "%%bash\n", - "cat ./fixtures/config_test_file_c.nml" + "cat fixtures/example/ex3-config-test-file-c.nml" ] }, { @@ -380,9 +374,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-07-18T19:27:40] INFO - fixtures/config_test_file_a.nml\n", - "[2024-07-18T19:27:40] INFO + fixtures/config_test_file_c.nml\n", - "[2024-07-18T19:27:40] INFO ---------------------------------------------------------------------\n" + "[2024-08-18T10:24:35] INFO - fixtures/example/ex3-config-test-file-a.nml\n", + "[2024-08-18T10:24:35] INFO + fixtures/example/ex3-config-test-file-c.nml\n", + "[2024-08-18T10:24:35] INFO ---------------------------------------------------------------------\n" ] }, { @@ -397,8 +391,8 @@ } ], "source": [ - "file_a = \"./fixtures/config_test_file_a.nml\"\n", - "identical_file_c = \"./fixtures/config_test_file_c.nml\"\n", + "file_a = \"fixtures/example/ex3-config-test-file-a.nml\"\n", + "identical_file_c = \"fixtures/example/ex3-config-test-file-c.nml\"\n", "config.compare(file_a, identical_file_c)" ] }, @@ -409,19 +403,13 @@ "source": [ "The `config()` method returns `True` to denote the files are semantically equivalent." ] - }, - { - "cell_type": "markdown", - "id": "5b39d871", - "metadata": {}, - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python [conda env:DEV-uwtools] *", "language": "python", - "name": "python3" + "name": "conda-env-DEV-uwtools-py" }, "language_info": { "codemirror_mode": { @@ -433,7 +421,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/fixtures/user_config.yaml b/notebooks/fixtures/example/ex2-config.yaml similarity index 100% rename from notebooks/fixtures/user_config.yaml rename to notebooks/fixtures/example/ex2-config.yaml diff --git a/notebooks/fixtures/rendered_config.yaml b/notebooks/fixtures/example/ex2-rendered-config.yaml similarity index 100% rename from notebooks/fixtures/rendered_config.yaml rename to notebooks/fixtures/example/ex2-rendered-config.yaml diff --git a/notebooks/fixtures/user_values.yaml b/notebooks/fixtures/example/ex2-values.yaml similarity index 100% rename from notebooks/fixtures/user_values.yaml rename to notebooks/fixtures/example/ex2-values.yaml diff --git a/notebooks/fixtures/config_test_file_a.nml b/notebooks/fixtures/example/ex3-config-test-file-a.nml similarity index 100% rename from notebooks/fixtures/config_test_file_a.nml rename to notebooks/fixtures/example/ex3-config-test-file-a.nml diff --git a/notebooks/fixtures/config_test_file_b.nml b/notebooks/fixtures/example/ex3-config-test-file-b.nml similarity index 100% rename from notebooks/fixtures/config_test_file_b.nml rename to notebooks/fixtures/example/ex3-config-test-file-b.nml diff --git a/notebooks/fixtures/config_test_file_c.nml b/notebooks/fixtures/example/ex3-config-test-file-c.nml similarity index 100% rename from notebooks/fixtures/config_test_file_c.nml rename to notebooks/fixtures/example/ex3-config-test-file-c.nml diff --git a/notebooks/fixtures/template/render-complete-1.yaml b/notebooks/fixtures/template/render-complete-1.yaml new file mode 100644 index 000000000..2840700bf --- /dev/null +++ b/notebooks/fixtures/template/render-complete-1.yaml @@ -0,0 +1,3 @@ +user: + name: John Doe + favorite_food: burritos diff --git a/notebooks/fixtures/template/render-complete-2.yaml b/notebooks/fixtures/template/render-complete-2.yaml new file mode 100644 index 000000000..df6d5f0bd --- /dev/null +++ b/notebooks/fixtures/template/render-complete-2.yaml @@ -0,0 +1,3 @@ +user: + name: Jane Doe + favorite_food: tamales diff --git a/notebooks/fixtures/template/render-template.yaml b/notebooks/fixtures/template/render-template.yaml new file mode 100644 index 000000000..78ddc7977 --- /dev/null +++ b/notebooks/fixtures/template/render-template.yaml @@ -0,0 +1,3 @@ +user: + name: {{ first }} {{ last }} + favorite_food: {{ food }} diff --git a/notebooks/fixtures/template/render-values.yaml b/notebooks/fixtures/template/render-values.yaml new file mode 100644 index 000000000..5ae0cc34e --- /dev/null +++ b/notebooks/fixtures/template/render-values.yaml @@ -0,0 +1,3 @@ +first: John +last: Doe +food: burritos diff --git a/notebooks/fixtures/template/translate-complete.yaml b/notebooks/fixtures/template/translate-complete.yaml new file mode 100644 index 000000000..7841b3c8c --- /dev/null +++ b/notebooks/fixtures/template/translate-complete.yaml @@ -0,0 +1,3 @@ +flowers: + roses: {{ color1 }} + violets: {{ color2 }} diff --git a/notebooks/fixtures/template/translate-template.yaml b/notebooks/fixtures/template/translate-template.yaml new file mode 100644 index 000000000..0eab2bc00 --- /dev/null +++ b/notebooks/fixtures/template/translate-template.yaml @@ -0,0 +1,3 @@ +flowers: + roses: @[color1] + violets: @[color2] diff --git a/notebooks/template.ipynb b/notebooks/template.ipynb new file mode 100644 index 000000000..e0e9a2e9e --- /dev/null +++ b/notebooks/template.ipynb @@ -0,0 +1,511 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "50994576-4783-4e8c-a6b0-21b1f258685d", + "metadata": {}, + "source": [ + "# Template Tool\n", + "\n", + "The `uwtools` API's `template` module provides functions to render Jinja2 templates and to translate atparse templates to Jinja2.\n", + "\n", + "For more information, please see the uwtools.api.template Read the Docs page." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "cbec4cc0-369e-41ff-a8a6-8a2699cb5759", + "metadata": {}, + "outputs": [], + "source": [ + "from uwtools.api import template\n", + "from pathlib import Path" + ] + }, + { + "cell_type": "markdown", + "id": "017c777a-1ca0-4fef-873f-89924e209da8", + "metadata": {}, + "source": [ + "## render\n", + "\n", + "`template.render()` renders a Jinja2 template using values provided by the specified values source." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "54e88f1b-0b9f-4011-b070-df107f928cf9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function render in module uwtools.api.template:\n", + "\n", + "render(values_src: Union[dict, pathlib.Path, str, NoneType] = None, values_format: Optional[str] = None, input_file: Union[str, pathlib.Path, NoneType] = None, output_file: Union[str, pathlib.Path, NoneType] = None, overrides: Optional[dict[str, str]] = None, env: bool = False, searchpath: Optional[list[str]] = None, values_needed: bool = False, dry_run: bool = False, stdin_ok: bool = False) -> str\n", + " Render a Jinja2 template to a file, based on specified values.\n", + "\n", + " Primary values used to render the template are taken from the specified file. The format of the\n", + " values source will be deduced from the filename extension, if possible. This can be overridden\n", + " via the ``values_format`` argument. A ``dict`` object may alternatively be provided as the\n", + " primary values source. If no input file is specified, ``stdin`` is read. If no output file is\n", + " specified, ``stdout`` is written to.\n", + "\n", + " :param values_src: Source of values to render the template.\n", + " :param values_format: Format of values when sourced from file.\n", + " :param input_file: Raw input template file (``None`` => read ``stdin``).\n", + " :param output_file: Rendered template output file (``None`` => write to ``stdout``).\n", + " :param overrides: Supplemental override values.\n", + " :param env: Supplement values with environment variables?\n", + " :param searchpath: Paths to search for extra templates.\n", + " :param values_needed: Just report variables needed to render the template?\n", + " :param dry_run: Run in dry-run mode?\n", + " :param stdin_ok: OK to read from ``stdin``?\n", + " :return: The rendered template string.\n", + " :raises: UWTemplateRenderError if template could not be rendered.\n", + "\n" + ] + } + ], + "source": [ + "help(template.render)" + ] + }, + { + "cell_type": "markdown", + "id": "28a3415f-3f0e-42c7-8be2-bc94057e8510", + "metadata": {}, + "source": [ + "Consider the following template, to be rendered as YAML data:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "91bd29fd-77ba-4ea2-946f-cd7a2d9301f1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "user:\n", + " name: {{ first }} {{ last }}\n", + " favorite_food: {{ food }}\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/template/render-template.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "d335aec6-5fcd-4df2-ae2e-8afa1a510683", + "metadata": {}, + "source": [ + "The `values_needed` parameter can be used to display which values are needed to complete the template. A logger needs to be initialized for the log of the missing values to be displayed." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "040eceb1-0821-4e82-825a-5be18f06397d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-20T08:10:58] INFO Value(s) needed to render this template are:\n", + "[2024-08-20T08:10:58] INFO first\n", + "[2024-08-20T08:10:58] INFO food\n", + "[2024-08-20T08:10:58] INFO last\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "user:\n", + " name: {{ first }} {{ last }}\n", + " favorite_food: {{ food }}\n", + "\n" + ] + } + ], + "source": [ + "import uwtools.logging\n", + "uwtools.logging.setup_logging(verbose=False)\n", + "\n", + "print(\n", + " template.render(\n", + " input_file='fixtures/template/render-template.yaml',\n", + " values_needed=True\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "6d6913b9-a375-447a-b729-566afe84f694", + "metadata": {}, + "source": [ + "The log messages indicate that values are needed for keys `first`, `food`, and `last`. These values can be sourced from a Python dictionary or from a file. The following file provides the needed values:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7f794c66-8840-419a-adf5-20efddb85708", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "first: John\n", + "last: Doe\n", + "food: burritos\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/template/render-values.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "9c809a20-b09d-438a-a1da-3cb9986ce9fa", + "metadata": {}, + "source": [ + "With these values, we can render the template to a file. When the source of values is a file, its path can be given either as a string or a Path object. If it has an unrecognized (or no) extension, its format can be specified with `values_format`. The rendered template can be written to a file specified with `output_file`; otherwise, it will be written to `stdout`. " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "834b7a40-293e-4d35-81e8-121eed4cf8f2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "user:\n", + " name: John Doe\n", + " favorite_food: burritos\n" + ] + } + ], + "source": [ + "print(\n", + " template.render(\n", + " values_src=Path('fixtures/template/render-values.yaml'),\n", + " values_format='yaml',\n", + " input_file='fixtures/template/render-template.yaml',\n", + " output_file='fixtures/template/render-complete-1.yaml'\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c45b0ac1-23d6-4d25-a691-7bc4f482429e", + "metadata": {}, + "source": [ + "Values can be selectively overridden with a dictionary passed via the optional `overrides` argument." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "55eec4f4-4f91-4618-8382-78061907bd2a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "user:\n", + " name: Jane Doe\n", + " favorite_food: tamales\n" + ] + } + ], + "source": [ + "print(\n", + " template.render(\n", + " values_src=Path('fixtures/template/render-values.yaml'),\n", + " values_format='yaml',\n", + " input_file='fixtures/template/render-template.yaml',\n", + " output_file='fixtures/template/render-complete-2.yaml',\n", + " overrides={'first':'Jane', 'food':'tamales'}\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "861e1e96-58b6-4537-bc7d-7986b450e774", + "metadata": {}, + "source": [ + "Let's take a look at the two newly rendered files." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "8601d4d9-5e53-44b7-880c-666ab810d8b8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "user:\n", + " name: John Doe\n", + " favorite_food: burritos\n", + "---------------------------------------\n", + "user:\n", + " name: Jane Doe\n", + " favorite_food: tamales\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/template/render-complete-1.yaml\n", + "echo ---------------------------------------\n", + "cat fixtures/template/render-complete-2.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "58cbbfbf-b47d-485d-9ef3-80b067316d11", + "metadata": {}, + "source": [ + "## render_to_str\n", + "\n", + "`template.render_to_str()` is identical to `template.render()` except that it does not accept an `output_file` parameter: It returns the rendered template as a string and does not write to a file or to `stdout`." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "ff8b80b2-590c-476f-94f7-37c4f34932f7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function render_to_str in module uwtools.api.template:\n", + "\n", + "render_to_str(values_src: Union[dict, pathlib.Path, str, NoneType] = None, values_format: Optional[str] = None, input_file: Union[str, pathlib.Path, NoneType] = None, overrides: Optional[dict[str, str]] = None, env: bool = False, searchpath: Optional[list[str]] = None, values_needed: bool = False, dry_run: bool = False) -> str\n", + " Render a Jinja2 template to a string, based on specified values.\n", + "\n", + " See ``render()`` for details on arguments, etc.\n", + "\n" + ] + } + ], + "source": [ + "help(template.render_to_str)" + ] + }, + { + "cell_type": "markdown", + "id": "65905cf1-679d-46ef-96e6-23e0c952947c", + "metadata": {}, + "source": [ + "We can see the resulting string using the same template and values from the first `template.render()` example." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "1f2cec30-0761-42f4-85fc-05593e215b23", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "user:\n", + " name: John Doe\n", + " favorite_food: burritos\n" + ] + } + ], + "source": [ + "result = template.render_to_str(\n", + " values_src=Path('fixtures/template/render-values.yaml'),\n", + " values_format='yaml',\n", + " input_file='fixtures/template/render-template.yaml'\n", + ")\n", + "print(result)" + ] + }, + { + "cell_type": "markdown", + "id": "fe45be03-c1aa-4c50-a21b-3f35180569b4", + "metadata": {}, + "source": [ + "For more examples, please refer to the render section above.\n", + "\n", + "## translate\n", + "\n", + "This function can be used to translate atparse templates into Jinja2 templates by replacing `@[]` tokens with their corresponding `{{}}` Jinja2 equivalents. " + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "2ddcefac-030d-415c-a97f-eab9e176e811", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function translate in module uwtools.api.template:\n", + "\n", + "translate(input_file: Union[str, pathlib.Path, NoneType] = None, output_file: Union[str, pathlib.Path, NoneType] = None, dry_run: bool = False, stdin_ok: bool = False) -> bool\n", + " Translate an atparse template to a Jinja2 template.\n", + "\n", + " ``@[]`` tokens are replaced with Jinja2 ``{{}}`` equivalents. If no input file is specified,\n", + " ``stdin`` is read. If no output file is specified, ``stdout`` is written to. In ``dry_run``\n", + " mode, output is written to ``stderr``.\n", + "\n", + " :param input_file: Path to atparse file (``None`` => read ``stdin``).\n", + " :param output_file: Path to the file to write the converted template to.\n", + " :param dry_run: Run in dry-run mode?\n", + " :param stdin_ok: OK to read from ``stdin``?\n", + " :return: ``True``.\n", + "\n" + ] + } + ], + "source": [ + "help(template.translate)" + ] + }, + { + "cell_type": "markdown", + "id": "1340097f-5ace-482d-bd13-01b426e768a1", + "metadata": {}, + "source": [ + "The template tool works with atparse templates like the one shown below." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "66fbde65-2c4e-48fa-bc49-c4faec78f944", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "flowers:\n", + " roses: @[color1]\n", + " violets: @[color2]\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat 'fixtures/template/translate-template.yaml'" + ] + }, + { + "cell_type": "markdown", + "id": "62d87063-2cd0-40de-bf02-dee0ace11d5a", + "metadata": {}, + "source": [ + "We can translate this file to a Jinja2 template by passing appropriate `input_file` and `output_file` (either `str` or Path) values to `template.render()`." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "bab9026c-9f5a-435d-b8a2-71fa2a325109", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "template.translate(\n", + " input_file=Path('fixtures/template/translate-template.yaml'),\n", + " output_file='fixtures/template/translate-complete.yaml'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e8712324-39a1-49d1-b2c6-dce2907b149e", + "metadata": {}, + "source": [ + "Now we have created a Jinja2 template that can be rendered using `template.render()` or `template.render_to_str()`." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "5f30435c-e253-4f8a-a8e7-6bdbd8be92c9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "flowers:\n", + " roses: {{ color1 }}\n", + " violets: {{ color2 }}\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat 'fixtures/template/translate-complete.yaml'" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:DEV-uwtools] *", + "language": "python", + "name": "conda-env-DEV-uwtools-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/tests/test_example.py b/notebooks/tests/test_example.py index d70fe05a0..c573bab8e 100644 --- a/notebooks/tests/test_example.py +++ b/notebooks/tests/test_example.py @@ -5,7 +5,7 @@ # Run all cells of the example notebook. -@testbook("./example.ipynb", execute=True) +@testbook("example.ipynb", execute=True) def test_get_yaml_config(tb): # Check output text of the cell that prints the YAMLconfig object. @@ -26,12 +26,12 @@ def test_get_yaml_config(tb): def test_template_render(): # Remove the rendered file if it exists. - rendered_path = "./fixtures/rendered_config.yaml" + rendered_path = "fixtures/example/ex2-rendered-config.yaml" if os.path.exists(rendered_path): os.remove(rendered_path) # Run all cells of the example notebook. - with testbook("./example.ipynb", execute=True) as tb: + with testbook("example.ipynb", execute=True) as tb: # Check output text of cells with %%bash cell magics. assert ( @@ -54,7 +54,7 @@ def test_template_render(): # Run all cells of the example notebook. -@testbook("./example.ipynb", execute=True) +@testbook("example.ipynb", execute=True) def test_compare(tb): # Check output text of the cell prints the correct result diff --git a/notebooks/tests/test_template.py b/notebooks/tests/test_template.py new file mode 100644 index 000000000..d0395573f --- /dev/null +++ b/notebooks/tests/test_template.py @@ -0,0 +1,75 @@ +import os + +import yaml +from testbook import testbook + + +def test_render(): + + template = "fixtures/template/render-template.yaml" + values = "fixtures/template/render-values.yaml" + rendered_template1 = "fixtures/template/render-complete-1.yaml" + rendered_template2 = "fixtures/template/render-complete-2.yaml" + + for file in rendered_template1, rendered_template2: + if os.path.exists(file): + os.remove(file) + + with open(template, "r", encoding="utf-8") as f: + template_str = f.read().rstrip() + with open(values, "r", encoding="utf-8") as f: + values_str = f.read().rstrip() + + with testbook("template.ipynb", execute=True) as tb: + + with open(rendered_template1, "r", encoding="utf-8") as f: + rend_temp_str1 = f.read().rstrip() + temp_yaml1 = yaml.safe_load(rend_temp_str1) + assert temp_yaml1["user"] == {"name": "John Doe", "favorite_food": "burritos"} + + with open(rendered_template2, "r", encoding="utf-8") as f: + rend_temp_str2 = f.read().rstrip() + temp_yaml2 = yaml.safe_load(rend_temp_str2) + assert temp_yaml2["user"] == {"name": "Jane Doe", "favorite_food": "tamales"} + + assert tb.cell_output_text(5) == template_str + assert ( + "INFO first" in tb.cell_output_text(7) + and "INFO food" in tb.cell_output_text(7) + and "INFO last" in tb.cell_output_text(7) + ) + assert tb.cell_output_text(9) == values_str + assert tb.cell_output_text(11) == rend_temp_str1 + assert tb.cell_output_text(13) == rend_temp_str2 + assert rend_temp_str1 in tb.cell_output_text(15) and rend_temp_str2 in tb.cell_output_text( + 15 + ) + + +@testbook("template.ipynb", execute=True) +def test_render_to_str(tb): + + rend_temp_str = "user:\n name: John Doe\n favorite_food: burritos" + assert tb.ref("result") == rend_temp_str + assert tb.cell_output_text(19) == rend_temp_str + + +def test_translate(): + + atparse_template = "fixtures/template/translate-template.yaml" + translated_template = "fixtures/template/translate-complete.yaml" + + if os.path.exists(translated_template): + os.remove(translated_template) + + with open(atparse_template, "r", encoding="utf-8") as f: + atparse_str = f.read().rstrip() + + with testbook("template.ipynb", execute=True) as tb: + + with open(translated_template, "r", encoding="utf-8") as f: + translated_str = f.read().rstrip() + + assert tb.cell_output_text(23) == atparse_str + assert tb.cell_output_text(25) == "True" + assert tb.cell_output_text(27) == translated_str From a842fa310e79f0289f00c28dfbcf0f16af0fe430 Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Mon, 26 Aug 2024 10:57:31 -0600 Subject: [PATCH 10/24] notebooks update (#593) --- docs/Makefile | 2 +- docs/conf.py | 2 +- docs/index.rst | 15 +- docs/sections/user_guide/api/config.rst | 1 + docs/sections/user_guide/api/execute.rst | 6 + docs/sections/user_guide/api/file.rst | 5 - docs/sections/user_guide/api/fs.rst | 5 + docs/sections/user_guide/api/index.rst | 3 +- .../user_guide/cli/drivers/.gitignore | 1 + .../sections/user_guide/cli/drivers/cdeps.rst | 9 +- .../user_guide/cli/drivers/cdeps/help.out | 16 +- .../user_guide/cli/drivers/cdeps/run-help.out | 2 +- .../cli/drivers/cdeps/show-schema.cmd | 2 + .../cli/drivers/cdeps/show-schema.out | 21 ++ .../user_guide/cli/drivers/chgres_cube.rst | 8 + .../cli/drivers/chgres_cube/help.out | 4 +- .../cli/drivers/chgres_cube/show-schema.cmd | 2 + .../cli/drivers/chgres_cube/show-schema.out | 21 ++ .../user_guide/cli/drivers/esg_grid.rst | 8 + .../user_guide/cli/drivers/esg_grid/help.out | 4 +- .../cli/drivers/esg_grid/show-schema.cmd | 2 + .../cli/drivers/esg_grid/show-schema.out | 21 ++ .../user_guide/cli/drivers/filter_topo.rst | 8 + .../cli/drivers/filter_topo/help.out | 4 +- .../cli/drivers/filter_topo/show-schema.cmd | 2 + .../cli/drivers/filter_topo/show-schema.out | 21 ++ docs/sections/user_guide/cli/drivers/fv3.rst | 8 + .../user_guide/cli/drivers/fv3/help.out | 4 +- .../cli/drivers/fv3/show-schema.cmd | 2 + .../cli/drivers/fv3/show-schema.out | 21 ++ .../cli/drivers/global_equiv_resol.rst | 8 + .../cli/drivers/global_equiv_resol/help.out | 4 +- .../global_equiv_resol/show-schema.cmd | 2 + .../global_equiv_resol/show-schema.out | 21 ++ docs/sections/user_guide/cli/drivers/ioda.rst | 8 + .../user_guide/cli/drivers/ioda/help.out | 4 +- .../cli/drivers/ioda/show-schema.cmd | 2 + .../cli/drivers/ioda/show-schema.out | 21 ++ docs/sections/user_guide/cli/drivers/jedi.rst | 8 + .../user_guide/cli/drivers/jedi/help.out | 4 +- .../cli/drivers/jedi/show-schema.cmd | 2 + .../cli/drivers/jedi/show-schema.out | 21 ++ .../user_guide/cli/drivers/make_hgrid.rst | 8 + .../cli/drivers/make_hgrid/help.out | 4 +- .../cli/drivers/make_hgrid/show-schema.cmd | 2 + .../cli/drivers/make_hgrid/show-schema.out | 21 ++ .../cli/drivers/make_solo_mosaic.rst | 8 + .../cli/drivers/make_solo_mosaic/help.out | 4 +- .../drivers/make_solo_mosaic/show-schema.cmd | 2 + .../drivers/make_solo_mosaic/show-schema.out | 21 ++ docs/sections/user_guide/cli/drivers/mpas.rst | 8 + .../user_guide/cli/drivers/mpas/help.out | 4 +- .../cli/drivers/mpas/show-schema.cmd | 2 + .../cli/drivers/mpas/show-schema.out | 21 ++ .../user_guide/cli/drivers/mpas_init.rst | 8 + .../user_guide/cli/drivers/mpas_init/help.out | 4 +- .../cli/drivers/mpas_init/show-schema.cmd | 2 + .../cli/drivers/mpas_init/show-schema.out | 21 ++ .../user_guide/cli/drivers/orog_gsl.rst | 8 + .../user_guide/cli/drivers/orog_gsl/help.out | 4 +- .../cli/drivers/orog_gsl/show-schema.cmd | 2 + .../cli/drivers/orog_gsl/show-schema.out | 21 ++ .../user_guide/cli/drivers/sfc_climo_gen.rst | 8 + .../cli/drivers/sfc_climo_gen/help.out | 4 +- .../cli/drivers/sfc_climo_gen/show-schema.cmd | 2 + .../cli/drivers/sfc_climo_gen/show-schema.out | 21 ++ .../sections/user_guide/cli/drivers/shave.rst | 8 + .../user_guide/cli/drivers/shave/help.out | 4 +- .../cli/drivers/shave/show-schema.cmd | 2 + .../cli/drivers/shave/show-schema.out | 21 ++ .../user_guide/cli/drivers/ungrib.rst | 8 + .../user_guide/cli/drivers/ungrib/help.out | 4 +- .../cli/drivers/ungrib/show-schema.cmd | 2 + .../cli/drivers/ungrib/show-schema.out | 21 ++ docs/sections/user_guide/cli/drivers/upp.rst | 8 + .../user_guide/cli/drivers/upp/help.out | 4 +- .../cli/drivers/upp/show-schema.cmd | 2 + .../cli/drivers/upp/show-schema.out | 21 ++ .../user_guide/cli/tools/execute/rand.py | 4 +- docs/sections/user_guide/cli/tools/file.rst | 97 --------- .../file/copy-exec-no-target-dir-err.cmd | 1 - .../file/copy-exec-no-target-dir-err.out | 1 - .../cli/tools/file/copy-exec-timedep.cmd | 4 - .../user_guide/cli/tools/file/copy-exec.cmd | 4 - .../user_guide/cli/tools/file/copy-help.cmd | 1 - .../user_guide/cli/tools/file/help.cmd | 1 - .../file/link-exec-no-target-dir-err.cmd | 1 - .../file/link-exec-no-target-dir-err.out | 1 - .../cli/tools/file/link-exec-timedep.cmd | 4 - .../user_guide/cli/tools/file/link-exec.cmd | 4 - .../user_guide/cli/tools/file/link-help.cmd | 1 - docs/sections/user_guide/cli/tools/fs.rst | 135 ++++++++++++ .../cli/tools/{file => fs}/.gitignore | 0 .../cli/tools/{file => fs}/Makefile | 0 .../{file => fs}/copy-config-timedep.yaml | 0 .../cli/tools/{file => fs}/copy-config.yaml | 0 .../tools/fs/copy-exec-no-target-dir-err.cmd | 1 + .../tools/fs/copy-exec-no-target-dir-err.out | 3 + .../cli/tools/fs/copy-exec-timedep.cmd | 4 + .../tools/{file => fs}/copy-exec-timedep.out | 0 .../user_guide/cli/tools/fs/copy-exec.cmd | 4 + .../cli/tools/{file => fs}/copy-exec.out | 0 .../user_guide/cli/tools/fs/copy-help.cmd | 1 + .../cli/tools/{file => fs}/copy-help.out | 8 +- .../sections/user_guide/cli/tools/fs/help.cmd | 1 + .../cli/tools/{file => fs}/help.out | 6 +- .../{file => fs}/link-config-timedep.yaml | 0 .../cli/tools/{file => fs}/link-config.yaml | 0 .../tools/fs/link-exec-no-target-dir-err.cmd | 1 + .../tools/fs/link-exec-no-target-dir-err.out | 3 + .../cli/tools/fs/link-exec-timedep.cmd | 4 + .../tools/{file => fs}/link-exec-timedep.out | 0 .../user_guide/cli/tools/fs/link-exec.cmd | 4 + .../cli/tools/{file => fs}/link-exec.out | 0 .../user_guide/cli/tools/fs/link-help.cmd | 1 + .../cli/tools/{file => fs}/link-help.out | 8 +- .../cli/tools/fs/makedirs-config-timedep.yaml | 8 + .../cli/tools/fs/makedirs-config.yaml | 4 + .../fs/makedirs-exec-no-target-dir-err.cmd | 1 + .../fs/makedirs-exec-no-target-dir-err.out | 3 + .../cli/tools/fs/makedirs-exec-timedep.cmd | 4 + .../cli/tools/fs/makedirs-exec-timedep.out | 29 +++ .../user_guide/cli/tools/fs/makedirs-exec.cmd | 4 + .../user_guide/cli/tools/fs/makedirs-exec.out | 21 ++ .../user_guide/cli/tools/fs/makedirs-help.cmd | 1 + .../user_guide/cli/tools/fs/makedirs-help.out | 28 +++ .../{file => fs}/src/20240529/12/006/baz | 0 .../user_guide/cli/tools/{file => fs}/src/bar | 0 .../user_guide/cli/tools/{file => fs}/src/foo | 0 docs/sections/user_guide/cli/tools/index.rst | 2 +- docs/sections/user_guide/yaml/files.rst | 2 +- docs/sections/user_guide/yaml/index.rst | 5 +- docs/sections/user_guide/yaml/makedirs.rst | 22 ++ recipe/meta.json | 2 +- src/uwtools/api/cdeps.py | 24 ++- src/uwtools/api/chgres_cube.py | 24 ++- src/uwtools/api/config.py | 55 +++-- src/uwtools/api/driver.py | 153 -------------- src/uwtools/api/esg_grid.py | 24 ++- src/uwtools/api/execute.py | 157 ++++++++++++++ src/uwtools/api/filter_topo.py | 24 ++- src/uwtools/api/{file.py => fs.py} | 47 ++++- src/uwtools/api/fv3.py | 24 ++- src/uwtools/api/global_equiv_resol.py | 24 ++- src/uwtools/api/ioda.py | 24 ++- src/uwtools/api/jedi.py | 24 ++- src/uwtools/api/make_hgrid.py | 24 ++- src/uwtools/api/make_solo_mosaic.py | 24 ++- src/uwtools/api/mpas.py | 24 ++- src/uwtools/api/mpas_init.py | 24 ++- src/uwtools/api/orog_gsl.py | 24 ++- src/uwtools/api/schism.py | 24 ++- src/uwtools/api/sfc_climo_gen.py | 24 ++- src/uwtools/api/shave.py | 24 ++- src/uwtools/api/ungrib.py | 24 ++- src/uwtools/api/upp.py | 24 ++- src/uwtools/api/ww3.py | 24 ++- src/uwtools/cli.py | 167 +++++++++------ src/uwtools/config/atparse_to_jinja2.py | 2 +- src/uwtools/config/formats/base.py | 172 ++++++++-------- src/uwtools/config/formats/fieldtable.py | 57 +++--- src/uwtools/config/formats/ini.py | 46 ++--- src/uwtools/config/formats/nml.py | 45 ++-- src/uwtools/config/formats/sh.py | 46 ++--- src/uwtools/config/formats/yaml.py | 125 ++++++------ src/uwtools/config/jinja2.py | 6 +- src/uwtools/config/support.py | 4 +- src/uwtools/config/tools.py | 28 ++- src/uwtools/config/validator.py | 47 ++++- src/uwtools/drivers/cdeps.py | 18 +- src/uwtools/drivers/chgres_cube.py | 6 +- src/uwtools/drivers/driver.py | 74 ++++--- src/uwtools/drivers/esg_grid.py | 6 +- src/uwtools/drivers/filter_topo.py | 6 +- src/uwtools/drivers/fv3.py | 6 +- src/uwtools/drivers/global_equiv_resol.py | 8 +- src/uwtools/drivers/ioda.py | 10 +- src/uwtools/drivers/jedi.py | 10 +- src/uwtools/drivers/jedi_base.py | 2 +- src/uwtools/drivers/make_hgrid.py | 8 +- src/uwtools/drivers/make_solo_mosaic.py | 12 +- src/uwtools/drivers/mpas.py | 6 +- src/uwtools/drivers/mpas_init.py | 6 +- src/uwtools/drivers/orog_gsl.py | 8 +- src/uwtools/drivers/schism.py | 6 +- src/uwtools/drivers/sfc_climo_gen.py | 6 +- src/uwtools/drivers/shave.py | 8 +- src/uwtools/drivers/support.py | 6 +- src/uwtools/drivers/ungrib.py | 10 +- src/uwtools/drivers/upp.py | 10 +- src/uwtools/drivers/ww3.py | 6 +- src/uwtools/{file.py => fs.py} | 127 +++++++++--- src/uwtools/resources/info.json | 2 +- .../resources/jsonschema/makedirs.jsonschema | 16 ++ src/uwtools/rocoto.py | 14 +- src/uwtools/scheduler.py | 62 +++--- src/uwtools/strings.py | 7 +- src/uwtools/tests/api/test_config.py | 39 +++- src/uwtools/tests/api/test_driver.py | 193 +----------------- src/uwtools/tests/api/test_drivers.py | 11 +- src/uwtools/tests/api/test_execute.py | 190 +++++++++++++++++ .../tests/api/{test_file.py => test_fs.py} | 17 +- src/uwtools/tests/config/formats/test_base.py | 82 ++++---- .../tests/config/formats/test_fieldtable.py | 15 +- src/uwtools/tests/config/formats/test_ini.py | 24 +-- src/uwtools/tests/config/formats/test_nml.py | 68 +++--- src/uwtools/tests/config/formats/test_sh.py | 24 +-- src/uwtools/tests/config/formats/test_yaml.py | 56 ++--- src/uwtools/tests/config/test_tools.py | 2 +- src/uwtools/tests/config/test_validator.py | 29 ++- src/uwtools/tests/drivers/test_cdeps.py | 2 +- src/uwtools/tests/drivers/test_chgres_cube.py | 2 +- src/uwtools/tests/drivers/test_driver.py | 15 +- src/uwtools/tests/drivers/test_esg_grid.py | 2 +- src/uwtools/tests/drivers/test_filter_topo.py | 2 +- src/uwtools/tests/drivers/test_fv3.py | 2 +- .../tests/drivers/test_global_equiv_resol.py | 2 +- src/uwtools/tests/drivers/test_ioda.py | 2 +- src/uwtools/tests/drivers/test_jedi.py | 2 +- src/uwtools/tests/drivers/test_make_hgrid.py | 2 +- .../tests/drivers/test_make_solo_mosaic.py | 2 +- src/uwtools/tests/drivers/test_mpas.py | 2 +- src/uwtools/tests/drivers/test_mpas_init.py | 2 +- src/uwtools/tests/drivers/test_orog_gsl.py | 2 +- src/uwtools/tests/drivers/test_schism.py | 2 +- .../tests/drivers/test_sfc_climo_gen.py | 2 +- src/uwtools/tests/drivers/test_shave.py | 2 +- src/uwtools/tests/drivers/test_support.py | 4 +- src/uwtools/tests/drivers/test_ungrib.py | 2 +- src/uwtools/tests/drivers/test_upp.py | 2 +- src/uwtools/tests/drivers/test_ww3.py | 2 +- src/uwtools/tests/fixtures/testdriver.py | 4 +- src/uwtools/tests/support.py | 8 +- src/uwtools/tests/test_cli.py | 78 ++++--- .../tests/{test_file.py => test_fs.py} | 63 ++++-- src/uwtools/tests/test_rocoto.py | 9 + src/uwtools/tests/test_scheduler.py | 6 +- src/uwtools/tests/test_schemas.py | 17 +- src/uwtools/tests/utils/test_processing.py | 10 +- src/uwtools/tests/utils/test_tasks.py | 7 + src/uwtools/utils/api.py | 2 +- src/uwtools/utils/file.py | 14 +- src/uwtools/utils/memory.py | 6 +- src/uwtools/utils/processing.py | 12 +- src/uwtools/utils/tasks.py | 13 ++ 245 files changed, 2820 insertions(+), 1409 deletions(-) create mode 100644 docs/sections/user_guide/api/execute.rst delete mode 100644 docs/sections/user_guide/api/file.rst create mode 100644 docs/sections/user_guide/api/fs.rst create mode 100644 docs/sections/user_guide/cli/drivers/.gitignore create mode 100644 docs/sections/user_guide/cli/drivers/cdeps/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/cdeps/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/esg_grid/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/esg_grid/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/filter_topo/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/filter_topo/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/fv3/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/fv3/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/ioda/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/ioda/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/jedi/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/jedi/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/mpas/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/mpas/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/mpas_init/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/mpas_init/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/shave/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/shave/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/ungrib/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/ungrib/show-schema.out create mode 100644 docs/sections/user_guide/cli/drivers/upp/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/upp/show-schema.out delete mode 100644 docs/sections/user_guide/cli/tools/file.rst delete mode 100644 docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.out delete mode 100644 docs/sections/user_guide/cli/tools/file/copy-exec-timedep.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/copy-exec.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/copy-help.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/help.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.out delete mode 100644 docs/sections/user_guide/cli/tools/file/link-exec-timedep.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/link-exec.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/link-help.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs.rst rename docs/sections/user_guide/cli/tools/{file => fs}/.gitignore (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/Makefile (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/copy-config-timedep.yaml (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/copy-config.yaml (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/copy-exec-timedep.out (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-exec.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/copy-exec.out (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-help.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/copy-help.out (73%) create mode 100644 docs/sections/user_guide/cli/tools/fs/help.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/help.out (60%) rename docs/sections/user_guide/cli/tools/{file => fs}/link-config-timedep.yaml (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/link-config.yaml (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out create mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/link-exec-timedep.out (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/link-exec.out (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/link-help.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/link-help.out (73%) create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec.out create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-help.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-help.out rename docs/sections/user_guide/cli/tools/{file => fs}/src/20240529/12/006/baz (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/src/bar (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/src/foo (100%) create mode 100644 docs/sections/user_guide/yaml/makedirs.rst create mode 100644 src/uwtools/api/execute.py rename src/uwtools/api/{file.py => fs.py} (61%) rename src/uwtools/{file.py => fs.py} (52%) create mode 100644 src/uwtools/resources/jsonschema/makedirs.jsonschema create mode 100644 src/uwtools/tests/api/test_execute.py rename src/uwtools/tests/api/{test_file.py => test_fs.py} (77%) rename src/uwtools/tests/{test_file.py => test_fs.py} (69%) diff --git a/docs/Makefile b/docs/Makefile index 5f68e1b41..72693a881 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -2,7 +2,7 @@ BUILDDIR = build LINKCHECKDIR = $(BUILDDIR)/linkcheck SOURCEDIR = . SPHINXBUILD = sphinx-build -SPHINXOPTS = -a -n -W +SPHINXOPTS = -a -n -W --keep-going .PHONY: help clean docs examples linkcheck diff --git a/docs/conf.py b/docs/conf.py index b19d1b6f5..0087720b5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,7 +21,7 @@ html_theme = "sphinx_rtd_theme" intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} linkcheck_ignore = [r"https://github.com/.*#.*"] -nitpick_ignore_regex = [("py:class", r"^uwtools\..*")] +nitpick_ignore_regex = [("py:class", r"^uwtools\..*"), ("py:class", "f90nml.Namelist")] numfig = True numfig_format = {"figure": "Figure %s"} project = "Unified Workflow Tools" diff --git a/docs/index.rst b/docs/index.rst index b89e1cb05..c99bac998 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -71,22 +71,23 @@ This tool helps transform legacy configuration files templated with the atparse | :any:`CLI documentation with examples` -File Provisioning -^^^^^^^^^^^^^^^^^ +File/Directory Provisioning +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -This tool helps users define the source and destination of files to be copied or linked, in the same UW YAML language used by UW drivers. +| **CLI**: ``uw fs -h`` +| **API**: ``import uwtools.api.fs`` -| :any:`CLI documentation with examples` +This tool helps users define the source and destination of files to be copied or linked, or directories to be created, in the same UW YAML language used by UW drivers. -There is a video demonstration of the use of the ``uw file`` tool available via YouTube. +| :any:`CLI documentation with examples` + +There is a video demonstration of the use of the ``uw fs`` tool (formerly ``uw file``) available via YouTube. .. raw:: html -| - Rocoto Configurability ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/sections/user_guide/api/config.rst b/docs/sections/user_guide/api/config.rst index ddef1bb57..5f689e1e7 100644 --- a/docs/sections/user_guide/api/config.rst +++ b/docs/sections/user_guide/api/config.rst @@ -2,4 +2,5 @@ ====================== .. automodule:: uwtools.api.config + :inherited-members: UserDict :members: diff --git a/docs/sections/user_guide/api/execute.rst b/docs/sections/user_guide/api/execute.rst new file mode 100644 index 000000000..ddff82164 --- /dev/null +++ b/docs/sections/user_guide/api/execute.rst @@ -0,0 +1,6 @@ +``uwtools.api.execute`` +======================= + +.. automodule:: uwtools.api.execute + :inherited-members: + :members: diff --git a/docs/sections/user_guide/api/file.rst b/docs/sections/user_guide/api/file.rst deleted file mode 100644 index d705c81bc..000000000 --- a/docs/sections/user_guide/api/file.rst +++ /dev/null @@ -1,5 +0,0 @@ -``uwtools.api.file`` -==================== - -.. automodule:: uwtools.api.file - :members: diff --git a/docs/sections/user_guide/api/fs.rst b/docs/sections/user_guide/api/fs.rst new file mode 100644 index 000000000..0ac50fc87 --- /dev/null +++ b/docs/sections/user_guide/api/fs.rst @@ -0,0 +1,5 @@ +``uwtools.api.fs`` +================== + +.. automodule:: uwtools.api.fs + :members: diff --git a/docs/sections/user_guide/api/index.rst b/docs/sections/user_guide/api/index.rst index 8e4336103..cd29d7614 100644 --- a/docs/sections/user_guide/api/index.rst +++ b/docs/sections/user_guide/api/index.rst @@ -7,8 +7,9 @@ API config driver esg_grid - file + execute filter_topo + fs fv3 global_equiv_resol ioda diff --git a/docs/sections/user_guide/cli/drivers/.gitignore b/docs/sections/user_guide/cli/drivers/.gitignore new file mode 100644 index 000000000..1c12cf9bd --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/.gitignore @@ -0,0 +1 @@ +schema diff --git a/docs/sections/user_guide/cli/drivers/cdeps.rst b/docs/sections/user_guide/cli/drivers/cdeps.rst index c2e98694d..1044ad6c6 100644 --- a/docs/sections/user_guide/cli/drivers/cdeps.rst +++ b/docs/sections/user_guide/cli/drivers/cdeps.rst @@ -37,7 +37,6 @@ Its contents are described in depth in section :ref:`cdeps_yaml`. Each of the va The driver creates a ``datm_in`` Fortran namelist file and a ``datm.streams`` stream-configuration file in the directory specified by ``rundir:`` in the config. - * Specifying the ``--dry-run`` flag results in the driver logging messages about actions it would have taken, without actually taking any. .. code-block:: text @@ -45,3 +44,11 @@ Its contents are described in depth in section :ref:`cdeps_yaml`. Each of the va $ uw cdeps run --config-file config.yaml --cycle 2023-12-15T18 --batch --dry-run .. include:: /shared/key_path.rst + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: cdeps/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: cdeps/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/cdeps/help.out b/docs/sections/user_guide/cli/drivers/cdeps/help.out index 9bc97fa19..8fdcbbf8e 100644 --- a/docs/sections/user_guide/cli/drivers/cdeps/help.out +++ b/docs/sections/user_guide/cli/drivers/cdeps/help.out @@ -1,4 +1,4 @@ -usage: uw cdeps [-h] [--version] TASK ... +usage: uw cdeps [-h] [--version] [--show-schema] TASK ... Execute cdeps tasks @@ -7,20 +7,22 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK atm - Create data atmosphere configuration with all required content + The data atmosphere configuration with all required content atm_nml - Create data atmosphere Fortran namelist file (datm_in) + The data atmosphere Fortran namelist file (datm_in) atm_stream - Create data atmosphere stream config file (datm.streams) + The data atmosphere stream config file (datm.streams) ocn - Create data ocean configuration with all required content + The data ocean configuration with all required content ocn_nml - Create data ocean Fortran namelist file (docn_in) + The data ocean Fortran namelist file (docn_in) ocn_stream - Create data ocean stream config file (docn.streams) + The data ocean stream config file (docn.streams) validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/cdeps/run-help.out b/docs/sections/user_guide/cli/drivers/cdeps/run-help.out index 9eb45e799..7ddba5ecc 100644 --- a/docs/sections/user_guide/cli/drivers/cdeps/run-help.out +++ b/docs/sections/user_guide/cli/drivers/cdeps/run-help.out @@ -2,7 +2,7 @@ usage: uw cdeps atm --cycle CYCLE [-h] [--version] [--config-file PATH] [--dry-run] [--graph-file PATH] [--key-path KEY[.KEY...]] [--quiet] [--verbose] -Create data atmosphere configuration with all required content +The data atmosphere configuration with all required content Required arguments: --cycle CYCLE diff --git a/docs/sections/user_guide/cli/drivers/cdeps/show-schema.cmd b/docs/sections/user_guide/cli/drivers/cdeps/show-schema.cmd new file mode 100644 index 000000000..5be54d6cd --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/cdeps/show-schema.cmd @@ -0,0 +1,2 @@ +uw cdeps --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/cdeps/show-schema.out b/docs/sections/user_guide/cli/drivers/cdeps/show-schema.out new file mode 100644 index 000000000..b5e4a7c03 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/cdeps/show-schema.out @@ -0,0 +1,21 @@ +{ + "$defs": { + "streams": { + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + "^stream0[1-9]$": { + "additionalProperties": false, + "properties": { + "dtlimit": { +... + "required": [ + "rundir" + ] + } + }, + "required": [ + "cdeps" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube.rst b/docs/sections/user_guide/cli/drivers/chgres_cube.rst index 61f55661f..6d954a1c6 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube.rst +++ b/docs/sections/user_guide/cli/drivers/chgres_cube.rst @@ -58,3 +58,11 @@ Its contents are described in depth in section :ref:`chgres_cube_yaml`. Each of .. code-block:: text $ uw chgres_cube provisioned_rundir --config-file config.yaml --cycle 2023-12-15T18 --batch + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: chgres_cube/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: chgres_cube/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/help.out b/docs/sections/user_guide/cli/drivers/chgres_cube/help.out index 11879ab3c..7d632bb8b 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube/help.out +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/help.out @@ -1,4 +1,4 @@ -usage: uw chgres_cube [-h] [--version] TASK ... +usage: uw chgres_cube [-h] [--version] [--show-schema] TASK ... Execute chgres_cube tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.cmd b/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.cmd new file mode 100644 index 000000000..f539df7cc --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.cmd @@ -0,0 +1,2 @@ +uw chgres_cube --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.out b/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.out new file mode 100644 index 000000000..97d8138dd --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "chgres_cube": { + "additionalProperties": false, + "properties": { + "execution": { + "additionalProperties": false, + "properties": { + "batchargs": { + "additionalProperties": true, +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "chgres_cube" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/esg_grid.rst b/docs/sections/user_guide/cli/drivers/esg_grid.rst index 7bacfb75a..e04e8af80 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid.rst +++ b/docs/sections/user_guide/cli/drivers/esg_grid.rst @@ -58,3 +58,11 @@ The driver creates a ``runscript.esg_grid`` file in the directory specified by ` .. code-block:: text $ uw esg_grid provisioned_rundir --config-file config.yaml --batch + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: esg_grid/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: esg_grid/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/help.out b/docs/sections/user_guide/cli/drivers/esg_grid/help.out index 01a1825fe..0069553e6 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid/help.out +++ b/docs/sections/user_guide/cli/drivers/esg_grid/help.out @@ -1,4 +1,4 @@ -usage: uw esg_grid [-h] [--version] TASK ... +usage: uw esg_grid [-h] [--version] [--show-schema] TASK ... Execute esg_grid tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.cmd b/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.cmd new file mode 100644 index 000000000..a98ee17dc --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.cmd @@ -0,0 +1,2 @@ +uw esg_grid --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.out b/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.out new file mode 100644 index 000000000..8ac2cdd16 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.out @@ -0,0 +1,21 @@ +{ + "$defs": { + "base_file": { + "type": "string" + }, + "namelist_content": { + "additionalproperties": false, + "properties": { + "regional_grid_nml": { + "additionalProperties": false, +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "esg_grid" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/filter_topo.rst b/docs/sections/user_guide/cli/drivers/filter_topo.rst index 1787c9125..a7a77ff5d 100644 --- a/docs/sections/user_guide/cli/drivers/filter_topo.rst +++ b/docs/sections/user_guide/cli/drivers/filter_topo.rst @@ -52,3 +52,11 @@ Its contents are described in section :ref:`filter_topo_yaml`. $ uw filter_topo run --config-file config.yaml --batch --dry-run .. include:: /shared/key_path.rst + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: filter_topo/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: filter_topo/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/help.out b/docs/sections/user_guide/cli/drivers/filter_topo/help.out index a1fe57f9f..1c426c0f4 100644 --- a/docs/sections/user_guide/cli/drivers/filter_topo/help.out +++ b/docs/sections/user_guide/cli/drivers/filter_topo/help.out @@ -1,4 +1,4 @@ -usage: uw filter_topo [-h] [--version] TASK ... +usage: uw filter_topo [-h] [--version] [--show-schema] TASK ... Execute filter_topo tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.cmd b/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.cmd new file mode 100644 index 000000000..43883c9fa --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.cmd @@ -0,0 +1,2 @@ +uw filter_topo --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.out b/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.out new file mode 100644 index 000000000..d1ff44ffc --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "filter_topo": { + "additionalProperties": false, + "properties": { + "config": { + "additionalProperties": false, + "properties": { + "input_grid_file": { + "type": "string" +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "filter_topo" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/fv3.rst b/docs/sections/user_guide/cli/drivers/fv3.rst index 2e1e2d858..c601d7464 100644 --- a/docs/sections/user_guide/cli/drivers/fv3.rst +++ b/docs/sections/user_guide/cli/drivers/fv3.rst @@ -53,3 +53,11 @@ The examples use a configuration file named ``config.yaml``. Its contents are de .. code-block:: text $ uw fv3 provisioned_rundir --config-file config.yaml --cycle 2024-02-11T12 --batch + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: fv3/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: fv3/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/fv3/help.out b/docs/sections/user_guide/cli/drivers/fv3/help.out index e2fd2adf2..09420ef44 100644 --- a/docs/sections/user_guide/cli/drivers/fv3/help.out +++ b/docs/sections/user_guide/cli/drivers/fv3/help.out @@ -1,4 +1,4 @@ -usage: uw fv3 [-h] [--version] TASK ... +usage: uw fv3 [-h] [--version] [--show-schema] TASK ... Execute fv3 tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/fv3/show-schema.cmd b/docs/sections/user_guide/cli/drivers/fv3/show-schema.cmd new file mode 100644 index 000000000..25d915cea --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/fv3/show-schema.cmd @@ -0,0 +1,2 @@ +uw fv3 --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/fv3/show-schema.out b/docs/sections/user_guide/cli/drivers/fv3/show-schema.out new file mode 100644 index 000000000..6957302c1 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/fv3/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "fv3": { + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "domain": { + "const": "regional" +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "fv3" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst b/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst index f6a189727..f02e99760 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst @@ -52,3 +52,11 @@ Its contents are described in section :ref:`global_equiv_resol_yaml`. $ uw global_equiv_resol run --config-file config.yaml --batch --dry-run .. include:: /shared/key_path.rst + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: global_equiv_resol/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: global_equiv_resol/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.out b/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.out index 227ad0499..eb3d7b505 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.out +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.out @@ -1,4 +1,4 @@ -usage: uw global_equiv_resol [-h] [--version] TASK ... +usage: uw global_equiv_resol [-h] [--version] [--show-schema] TASK ... Execute global_equiv_resol tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.cmd b/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.cmd new file mode 100644 index 000000000..6d43fbfb2 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.cmd @@ -0,0 +1,2 @@ +uw global_equiv_resol --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.out b/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.out new file mode 100644 index 000000000..bfdd2e0e0 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "global_equiv_resol": { + "additionalProperties": false, + "properties": { + "execution": { + "additionalProperties": false, + "properties": { + "batchargs": { + "additionalProperties": true, +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "global_equiv_resol" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/ioda.rst b/docs/sections/user_guide/cli/drivers/ioda.rst index 132337a44..9ea6e745c 100644 --- a/docs/sections/user_guide/cli/drivers/ioda.rst +++ b/docs/sections/user_guide/cli/drivers/ioda.rst @@ -52,3 +52,11 @@ The driver creates a ``runscript.ioda`` file in the directory specified by ``run $ uw ioda run --config-file config.yaml --cycle 2024-05-22T12 --batch --dry-run .. include:: /shared/key_path.rst + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: ioda/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: ioda/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/ioda/help.out b/docs/sections/user_guide/cli/drivers/ioda/help.out index f7486be93..692138971 100644 --- a/docs/sections/user_guide/cli/drivers/ioda/help.out +++ b/docs/sections/user_guide/cli/drivers/ioda/help.out @@ -1,4 +1,4 @@ -usage: uw ioda [-h] [--version] TASK ... +usage: uw ioda [-h] [--version] [--show-schema] TASK ... Execute ioda tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/ioda/show-schema.cmd b/docs/sections/user_guide/cli/drivers/ioda/show-schema.cmd new file mode 100644 index 000000000..1b16cbe71 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/show-schema.cmd @@ -0,0 +1,2 @@ +uw ioda --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/ioda/show-schema.out b/docs/sections/user_guide/cli/drivers/ioda/show-schema.out new file mode 100644 index 000000000..f90bcf6a7 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "ioda": { + "additionalProperties": false, + "properties": { + "configuration_file": { + "additionalProperties": false, + "anyOf": [ + { + "required": [ +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "ioda" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/jedi.rst b/docs/sections/user_guide/cli/drivers/jedi.rst index 1aa90992f..464300b94 100644 --- a/docs/sections/user_guide/cli/drivers/jedi.rst +++ b/docs/sections/user_guide/cli/drivers/jedi.rst @@ -52,3 +52,11 @@ The driver creates a ``runscript.jedi`` file in the directory specified by ``run $ uw jedi run --config-file config.yaml --cycle 2024-05-22T12 --batch --dry-run .. include:: /shared/key_path.rst + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: jedi/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: jedi/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/jedi/help.out b/docs/sections/user_guide/cli/drivers/jedi/help.out index f35898de0..fe3d01ae0 100644 --- a/docs/sections/user_guide/cli/drivers/jedi/help.out +++ b/docs/sections/user_guide/cli/drivers/jedi/help.out @@ -1,4 +1,4 @@ -usage: uw jedi [-h] [--version] TASK ... +usage: uw jedi [-h] [--version] [--show-schema] TASK ... Execute jedi tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/jedi/show-schema.cmd b/docs/sections/user_guide/cli/drivers/jedi/show-schema.cmd new file mode 100644 index 000000000..0567b841e --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/jedi/show-schema.cmd @@ -0,0 +1,2 @@ +uw jedi --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/jedi/show-schema.out b/docs/sections/user_guide/cli/drivers/jedi/show-schema.out new file mode 100644 index 000000000..2dba61813 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/jedi/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "jedi": { + "additionalProperties": false, + "properties": { + "configuration_file": { + "additionalProperties": false, + "anyOf": [ + { + "required": [ +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "jedi" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid.rst b/docs/sections/user_guide/cli/drivers/make_hgrid.rst index 8d38fd152..dc6d79dd3 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid.rst +++ b/docs/sections/user_guide/cli/drivers/make_hgrid.rst @@ -60,3 +60,11 @@ Its contents are described in section :ref:`make_hgrid_yaml`. $ uw make_hgrid run --config-file config.yaml --batch --dry-run .. include:: /shared/key_path.rst + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: make_hgrid/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: make_hgrid/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/help.out b/docs/sections/user_guide/cli/drivers/make_hgrid/help.out index c61016726..7acf294ca 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid/help.out +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/help.out @@ -1,4 +1,4 @@ -usage: uw make_hgrid [-h] [--version] TASK ... +usage: uw make_hgrid [-h] [--version] [--show-schema] TASK ... Execute make_hgrid tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.cmd b/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.cmd new file mode 100644 index 000000000..e41d2138c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.cmd @@ -0,0 +1,2 @@ +uw make_hgrid --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.out b/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.out new file mode 100644 index 000000000..181977dd5 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "make_hgrid": { + "additionalProperties": false, + "properties": { + "config": { + "additionalProperties": false, + "allOf": [ + { + "if": { +... + "execution", + "rundir" + ] + } + }, + "required": [ + "make_hgrid" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic.rst b/docs/sections/user_guide/cli/drivers/make_solo_mosaic.rst index d3c95e77b..7d110acad 100644 --- a/docs/sections/user_guide/cli/drivers/make_solo_mosaic.rst +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic.rst @@ -61,3 +61,11 @@ Its contents are described in section :ref:`make_solo_mosaic_yaml`. $ uw make_solo_mosaic run --config-file config.yaml --batch --dry-run .. include:: /shared/key_path.rst + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: make_solo_mosaic/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: make_solo_mosaic/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.out b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.out index 1af7ff17e..0b899c013 100644 --- a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.out +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.out @@ -1,4 +1,4 @@ -usage: uw make_solo_mosaic [-h] [--version] TASK ... +usage: uw make_solo_mosaic [-h] [--version] [--show-schema] TASK ... Execute make_solo_mosaic tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.cmd b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.cmd new file mode 100644 index 000000000..8100c4a37 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.cmd @@ -0,0 +1,2 @@ +uw make_solo_mosaic --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.out b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.out new file mode 100644 index 000000000..61d30fd75 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "make_solo_mosaic": { + "additionalProperties": false, + "properties": { + "config": { + "additionalProperties": false, + "properties": { + "dir": { + "type": "string" +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "make_solo_mosaic" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/mpas.rst b/docs/sections/user_guide/cli/drivers/mpas.rst index 671d019a0..090d50a92 100644 --- a/docs/sections/user_guide/cli/drivers/mpas.rst +++ b/docs/sections/user_guide/cli/drivers/mpas.rst @@ -58,3 +58,11 @@ Its contents are described in depth in section :ref:`mpas_yaml`. .. code-block:: text $ uw mpas provisioned_rundir --config-file config.yaml --cycle 2025-02-12T12 --batch + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: mpas/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: mpas/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/mpas/help.out b/docs/sections/user_guide/cli/drivers/mpas/help.out index 000018cdf..cb7106e5f 100644 --- a/docs/sections/user_guide/cli/drivers/mpas/help.out +++ b/docs/sections/user_guide/cli/drivers/mpas/help.out @@ -1,4 +1,4 @@ -usage: uw mpas [-h] [--version] TASK ... +usage: uw mpas [-h] [--version] [--show-schema] TASK ... Execute mpas tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/mpas/show-schema.cmd b/docs/sections/user_guide/cli/drivers/mpas/show-schema.cmd new file mode 100644 index 000000000..d1293fe18 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/mpas/show-schema.cmd @@ -0,0 +1,2 @@ +uw mpas --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/mpas/show-schema.out b/docs/sections/user_guide/cli/drivers/mpas/show-schema.out new file mode 100644 index 000000000..561d89def --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/mpas/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "mpas": { + "additionalProperties": false, + "properties": { + "execution": { + "additionalProperties": false, + "properties": { + "batchargs": { + "additionalProperties": true, +... + "streams" + ], + "type": "object" + } + }, + "required": [ + "mpas" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/mpas_init.rst b/docs/sections/user_guide/cli/drivers/mpas_init.rst index 30be881e4..b8286a929 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init.rst +++ b/docs/sections/user_guide/cli/drivers/mpas_init.rst @@ -58,3 +58,11 @@ Its contents are described in depth in section :ref:`mpas_init_yaml`. .. code-block:: text $ uw mpas_init provisioned_rundir --config-file config.yaml --cycle 2023-12-18T00 --batch + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: mpas_init/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: mpas_init/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/help.out b/docs/sections/user_guide/cli/drivers/mpas_init/help.out index 7e51e71d4..c2f743422 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init/help.out +++ b/docs/sections/user_guide/cli/drivers/mpas_init/help.out @@ -1,4 +1,4 @@ -usage: uw mpas_init [-h] [--version] TASK ... +usage: uw mpas_init [-h] [--version] [--show-schema] TASK ... Execute mpas_init tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.cmd b/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.cmd new file mode 100644 index 000000000..32253d17d --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.cmd @@ -0,0 +1,2 @@ +uw mpas_init --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.out b/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.out new file mode 100644 index 000000000..998687f90 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "mpas_init": { + "additionalProperties": false, + "properties": { + "boundary_conditions": { + "additionalProperties": false, + "properties": { + "interval_hours": { + "minimum": 1, +... + "streams" + ], + "type": "object" + } + }, + "required": [ + "mpas_init" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl.rst b/docs/sections/user_guide/cli/drivers/orog_gsl.rst index 7f2490b2a..d589651b6 100644 --- a/docs/sections/user_guide/cli/drivers/orog_gsl.rst +++ b/docs/sections/user_guide/cli/drivers/orog_gsl.rst @@ -52,3 +52,11 @@ Its contents are described in section :ref:`orog_gsl_yaml`. $ uw orog_gsl run --config-file config.yaml --batch --dry-run .. include:: /shared/key_path.rst + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: orog_gsl/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: orog_gsl/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/help.out b/docs/sections/user_guide/cli/drivers/orog_gsl/help.out index 744e588ff..b49285d3d 100644 --- a/docs/sections/user_guide/cli/drivers/orog_gsl/help.out +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/help.out @@ -1,4 +1,4 @@ -usage: uw orog_gsl [-h] [--version] TASK ... +usage: uw orog_gsl [-h] [--version] [--show-schema] TASK ... Execute orog_gsl tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.cmd b/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.cmd new file mode 100644 index 000000000..3bdf61aac --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.cmd @@ -0,0 +1,2 @@ +uw orog_gsl --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.out b/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.out new file mode 100644 index 000000000..36a92dfb9 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "orog_gsl": { + "additionalProperties": false, + "properties": { + "config": { + "additionalProperties": false, + "properties": { + "halo": { + "type": "integer" +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "orog_gsl" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst b/docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst index 559316077..fe8939667 100644 --- a/docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst @@ -58,3 +58,11 @@ Its contents are described in depth in section :ref:`sfc_climo_gen_yaml`. .. code-block:: text $ uw sfc_climo_gen provisioned_rundir --config-file config.yaml --batch + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: sfc_climo_gen/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: sfc_climo_gen/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.out b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.out index 28b619718..f6170ae46 100644 --- a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.out +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.out @@ -1,4 +1,4 @@ -usage: uw sfc_climo_gen [-h] [--version] TASK ... +usage: uw sfc_climo_gen [-h] [--version] [--show-schema] TASK ... Execute sfc_climo_gen tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.cmd b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.cmd new file mode 100644 index 000000000..918a4c80b --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.cmd @@ -0,0 +1,2 @@ +uw sfc_climo_gen --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.out b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.out new file mode 100644 index 000000000..9cafdced3 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "sfc_climo_gen": { + "additionalProperties": false, + "properties": { + "execution": { + "additionalProperties": false, + "properties": { + "batchargs": { + "additionalProperties": true, +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "sfc_climo_gen" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/shave.rst b/docs/sections/user_guide/cli/drivers/shave.rst index d3bef10f2..ff528c5ab 100644 --- a/docs/sections/user_guide/cli/drivers/shave.rst +++ b/docs/sections/user_guide/cli/drivers/shave.rst @@ -52,3 +52,11 @@ Its contents are described in section :ref:`shave_yaml`. $ uw shave run --config-file config.yaml --batch --dry-run .. include:: /shared/key_path.rst + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: shave/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: shave/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/shave/help.out b/docs/sections/user_guide/cli/drivers/shave/help.out index 74056e0aa..7cd91374e 100644 --- a/docs/sections/user_guide/cli/drivers/shave/help.out +++ b/docs/sections/user_guide/cli/drivers/shave/help.out @@ -1,4 +1,4 @@ -usage: uw shave [-h] [--version] TASK ... +usage: uw shave [-h] [--version] [--show-schema] TASK ... Execute shave tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/shave/show-schema.cmd b/docs/sections/user_guide/cli/drivers/shave/show-schema.cmd new file mode 100644 index 000000000..783cabcfe --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/shave/show-schema.cmd @@ -0,0 +1,2 @@ +uw shave --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/shave/show-schema.out b/docs/sections/user_guide/cli/drivers/shave/show-schema.out new file mode 100644 index 000000000..b18e04f5f --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/shave/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "shave": { + "additionalProperties": false, + "properties": { + "config": { + "additionalProperties": false, + "properties": { + "input_grid_file": { + "type": "string" +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "shave" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/ungrib.rst b/docs/sections/user_guide/cli/drivers/ungrib.rst index a53a07a63..df25b4e1f 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib.rst +++ b/docs/sections/user_guide/cli/drivers/ungrib.rst @@ -58,3 +58,11 @@ Its contents are described in depth in section :ref:`ungrib_yaml`. .. code-block:: text $ uw ungrib provisioned_rundir --config-file config.yaml --cycle 2021-04-01T12 --batch + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: ungrib/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: ungrib/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/ungrib/help.out b/docs/sections/user_guide/cli/drivers/ungrib/help.out index 8d4fe4816..252272e05 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib/help.out +++ b/docs/sections/user_guide/cli/drivers/ungrib/help.out @@ -1,4 +1,4 @@ -usage: uw ungrib [-h] [--version] TASK ... +usage: uw ungrib [-h] [--version] [--show-schema] TASK ... Execute ungrib tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/ungrib/show-schema.cmd b/docs/sections/user_guide/cli/drivers/ungrib/show-schema.cmd new file mode 100644 index 000000000..04ae66e66 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ungrib/show-schema.cmd @@ -0,0 +1,2 @@ +uw ungrib --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/ungrib/show-schema.out b/docs/sections/user_guide/cli/drivers/ungrib/show-schema.out new file mode 100644 index 000000000..01684d40b --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ungrib/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "ungrib": { + "additionalProperties": false, + "properties": { + "execution": { + "additionalProperties": false, + "properties": { + "batchargs": { + "additionalProperties": true, +... + "vtable" + ], + "type": "object" + } + }, + "required": [ + "ungrib" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/drivers/upp.rst b/docs/sections/user_guide/cli/drivers/upp.rst index 22b06ecb4..ae8d24eb4 100644 --- a/docs/sections/user_guide/cli/drivers/upp.rst +++ b/docs/sections/user_guide/cli/drivers/upp.rst @@ -58,3 +58,11 @@ Its contents are described in depth in section :ref:`upp_yaml`. .. code-block:: text $ uw upp provisioned_rundir --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 --batch + +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + +.. literalinclude:: upp/show-schema.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: upp/show-schema.out + :language: text diff --git a/docs/sections/user_guide/cli/drivers/upp/help.out b/docs/sections/user_guide/cli/drivers/upp/help.out index 68d5748b7..7163d3f71 100644 --- a/docs/sections/user_guide/cli/drivers/upp/help.out +++ b/docs/sections/user_guide/cli/drivers/upp/help.out @@ -1,4 +1,4 @@ -usage: uw upp [-h] [--version] TASK ... +usage: uw upp [-h] [--version] [--show-schema] TASK ... Execute upp tasks @@ -7,6 +7,8 @@ Optional arguments: Show help and exit --version Show version info and exit + --show-schema + Show driver schema and exit Positional arguments: TASK diff --git a/docs/sections/user_guide/cli/drivers/upp/show-schema.cmd b/docs/sections/user_guide/cli/drivers/upp/show-schema.cmd new file mode 100644 index 000000000..c0a1b290d --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/upp/show-schema.cmd @@ -0,0 +1,2 @@ +uw upp --show-schema >schema +head schema && echo ... && tail schema diff --git a/docs/sections/user_guide/cli/drivers/upp/show-schema.out b/docs/sections/user_guide/cli/drivers/upp/show-schema.out new file mode 100644 index 000000000..11be8c436 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/upp/show-schema.out @@ -0,0 +1,21 @@ +{ + "properties": { + "upp": { + "additionalProperties": false, + "properties": { + "execution": { + "additionalProperties": false, + "properties": { + "batchargs": { + "additionalProperties": true, +... + "rundir" + ], + "type": "object" + } + }, + "required": [ + "upp" + ], + "type": "object" +} diff --git a/docs/sections/user_guide/cli/tools/execute/rand.py b/docs/sections/user_guide/cli/tools/execute/rand.py index 2a8e10b57..99e5853c9 100644 --- a/docs/sections/user_guide/cli/tools/execute/rand.py +++ b/docs/sections/user_guide/cli/tools/execute/rand.py @@ -23,6 +23,6 @@ def randfile(self): with open(path, "w", encoding="utf-8") as f: print(randint(self.config["lo"], self.config["hi"]), file=f) - @property - def driver_name(self): + @classmethod + def driver_name(cls): return "rand" diff --git a/docs/sections/user_guide/cli/tools/file.rst b/docs/sections/user_guide/cli/tools/file.rst deleted file mode 100644 index b7d34cbe3..000000000 --- a/docs/sections/user_guide/cli/tools/file.rst +++ /dev/null @@ -1,97 +0,0 @@ -``file`` -======== - -The ``uw`` mode for handling filesystem files. - -.. literalinclude:: file/help.cmd - :emphasize-lines: 1 -.. literalinclude:: file/help.out - :language: text - -.. _cli_file_copy_examples: - -``copy`` --------- - -The ``copy`` action stages files in a target directory by copying files. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. - -.. literalinclude:: file/copy-help.cmd - :emphasize-lines: 1 -.. literalinclude:: file/copy-help.out - :language: text - -Examples -^^^^^^^^ - -Given ``copy-config.yaml`` containing - -.. literalinclude:: file/copy-config.yaml - :language: yaml -.. literalinclude:: file/copy-exec.cmd - :emphasize-lines: 2 -.. literalinclude:: file/copy-exec.out - :language: text - -Here, ``foo`` and ``bar`` are copies of their respective source files. - -The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: - -.. literalinclude:: file/copy-config-timedep.yaml - :language: yaml -.. literalinclude:: file/copy-exec-timedep.cmd - :emphasize-lines: 2 -.. literalinclude:: file/copy-exec-timedep.out - :language: text - -The ``--target-dir`` option is optional when all destination paths are absolute, and will never be applied to absolute destination paths. If any destination paths are relative, however, it is an error not to provide a target directory: - -.. literalinclude:: file/copy-config.yaml - :language: yaml -.. literalinclude:: file/copy-exec-no-target-dir-err.cmd - :emphasize-lines: 1 -.. literalinclude:: file/copy-exec-no-target-dir-err.out - :language: text - -.. _cli_file_link_examples: - -``link`` --------- - -The ``link`` action stages files in a target directory by linking files, directories, or other symbolic links. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. - -.. literalinclude:: file/link-help.cmd - :emphasize-lines: 1 -.. literalinclude:: file/link-help.out - :language: text - -Examples -^^^^^^^^ - -Given ``link-config.yaml`` containing - -.. literalinclude:: file/link-config.yaml - :language: yaml -.. literalinclude:: file/link-exec.cmd - :emphasize-lines: 2 -.. literalinclude:: file/link-exec.out - :language: text - -Here, ``foo`` and ``bar`` are symbolic links. - -The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: - -.. literalinclude:: file/link-config-timedep.yaml - :language: yaml -.. literalinclude:: file/link-exec-timedep.cmd - :emphasize-lines: 2 -.. literalinclude:: file/link-exec-timedep.out - :language: text - -The ``--target-dir`` option is optional when all linkname paths are absolute, and will never be applied to absolute linkname paths. If any linkname paths are relative, however, it is an error not to provide a target directory: - -.. literalinclude:: file/link-config.yaml - :language: yaml -.. literalinclude:: file/link-exec-no-target-dir-err.cmd - :emphasize-lines: 1 -.. literalinclude:: file/link-exec-no-target-dir-err.out - :language: text diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.cmd deleted file mode 100644 index ed4fd1f90..000000000 --- a/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.cmd +++ /dev/null @@ -1 +0,0 @@ -uw file copy --config-file copy-config.yaml config files diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.out deleted file mode 100644 index 388dc0318..000000000 --- a/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.out +++ /dev/null @@ -1 +0,0 @@ -[2024-07-26T21:51:23] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.cmd deleted file mode 100644 index cec34d0a7..000000000 --- a/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf copy-dst-timedep -uw file copy --target-dir copy-dst-timedep --config-file copy-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files -echo -tree copy-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec.cmd b/docs/sections/user_guide/cli/tools/file/copy-exec.cmd deleted file mode 100644 index f74e38e33..000000000 --- a/docs/sections/user_guide/cli/tools/file/copy-exec.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf copy-dst -uw file copy --target-dir copy-dst --config-file copy-config.yaml config files -echo -tree copy-dst diff --git a/docs/sections/user_guide/cli/tools/file/copy-help.cmd b/docs/sections/user_guide/cli/tools/file/copy-help.cmd deleted file mode 100644 index 5a5188caa..000000000 --- a/docs/sections/user_guide/cli/tools/file/copy-help.cmd +++ /dev/null @@ -1 +0,0 @@ -uw file copy --help diff --git a/docs/sections/user_guide/cli/tools/file/help.cmd b/docs/sections/user_guide/cli/tools/file/help.cmd deleted file mode 100644 index e89f12e45..000000000 --- a/docs/sections/user_guide/cli/tools/file/help.cmd +++ /dev/null @@ -1 +0,0 @@ -uw file --help diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.cmd deleted file mode 100644 index 83b52f5e5..000000000 --- a/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.cmd +++ /dev/null @@ -1 +0,0 @@ -uw file link --config-file link-config.yaml config files diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.out deleted file mode 100644 index 388dc0318..000000000 --- a/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.out +++ /dev/null @@ -1 +0,0 @@ -[2024-07-26T21:51:23] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/file/link-exec-timedep.cmd deleted file mode 100644 index 915899c83..000000000 --- a/docs/sections/user_guide/cli/tools/file/link-exec-timedep.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf link-dst-timedep -uw file link --target-dir link-dst-timedep --config-file link-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files -echo -tree link-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/file/link-exec.cmd b/docs/sections/user_guide/cli/tools/file/link-exec.cmd deleted file mode 100644 index 81da785b4..000000000 --- a/docs/sections/user_guide/cli/tools/file/link-exec.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf link-dst -uw file link --target-dir link-dst --config-file link-config.yaml config files -echo -tree link-dst diff --git a/docs/sections/user_guide/cli/tools/file/link-help.cmd b/docs/sections/user_guide/cli/tools/file/link-help.cmd deleted file mode 100644 index a3ce9a824..000000000 --- a/docs/sections/user_guide/cli/tools/file/link-help.cmd +++ /dev/null @@ -1 +0,0 @@ -uw file link --help diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst new file mode 100644 index 000000000..cb61dd7bf --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -0,0 +1,135 @@ +``fs`` +====== + +.. _cli_fs_mode: + +The ``uw`` mode for handling filesystem items (files and directories). + +.. literalinclude:: fs/help.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/help.out + :language: text + +``copy`` +-------- + +The ``copy`` action stages files in a target directory by copying files. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. + +.. literalinclude:: fs/copy-help.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/copy-help.out + :language: text + +Examples +^^^^^^^^ + +Given ``copy-config.yaml`` containing + +.. literalinclude:: fs/copy-config.yaml + :language: yaml +.. literalinclude:: fs/copy-exec.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/copy-exec.out + :language: text + +Here, ``foo`` and ``bar`` are copies of their respective source files. + +The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: + +.. literalinclude:: fs/copy-config-timedep.yaml + :language: yaml +.. literalinclude:: fs/copy-exec-timedep.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/copy-exec-timedep.out + :language: text + +The ``--target-dir`` option need not be specified when all destination paths are absolute, and will never be applied to absolute destination paths. If any destination paths are relative, however, it is an error not to provide a target directory: + +.. literalinclude:: fs/copy-config.yaml + :language: yaml +.. literalinclude:: fs/copy-exec-no-target-dir-err.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/copy-exec-no-target-dir-err.out + :language: text + +``link`` +-------- + +The ``link`` action stages files in a target directory by linking files, directories, or other symbolic links. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. + +.. literalinclude:: fs/link-help.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/link-help.out + :language: text + +Examples +^^^^^^^^ + +Given ``link-config.yaml`` containing + +.. literalinclude:: fs/link-config.yaml + :language: yaml +.. literalinclude:: fs/link-exec.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/link-exec.out + :language: text + +Here, ``foo`` and ``bar`` are symbolic links. + +The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: + +.. literalinclude:: fs/link-config-timedep.yaml + :language: yaml +.. literalinclude:: fs/link-exec-timedep.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/link-exec-timedep.out + :language: text + +The ``--target-dir`` option need not be specified when all linkname paths are absolute, and will never be applied to absolute linkname paths. If any linkname paths are relative, however, it is an error not to provide a target directory: + +.. literalinclude:: fs/link-config.yaml + :language: yaml +.. literalinclude:: fs/link-exec-no-target-dir-err.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/link-exec-no-target-dir-err.out + :language: text + +``makedirs`` +------------ + +The ``makedirs`` action creates directories. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`makedirs block `. + +.. literalinclude:: fs/makedirs-help.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/makedirs-help.out + :language: text + +Examples +^^^^^^^^ + +Given ``makedirs-config.yaml`` containing + +.. literalinclude:: fs/makedirs-config.yaml + :language: yaml +.. literalinclude:: fs/makedirs-exec.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/makedirs-exec.out + :language: text + +The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: + +.. literalinclude:: fs/makedirs-config-timedep.yaml + :language: yaml +.. literalinclude:: fs/makedirs-exec-timedep.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/makedirs-exec-timedep.out + :language: text + +The ``--target-dir`` option need not be specified when all directory paths are absolute, and will never be applied to absolute paths. If any paths are relative, however, it is an error not to provide a target directory: + +.. literalinclude:: fs/makedirs-config.yaml + :language: yaml +.. literalinclude:: fs/makedirs-exec-no-target-dir-err.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/makedirs-exec-no-target-dir-err.out + :language: text diff --git a/docs/sections/user_guide/cli/tools/file/.gitignore b/docs/sections/user_guide/cli/tools/fs/.gitignore similarity index 100% rename from docs/sections/user_guide/cli/tools/file/.gitignore rename to docs/sections/user_guide/cli/tools/fs/.gitignore diff --git a/docs/sections/user_guide/cli/tools/file/Makefile b/docs/sections/user_guide/cli/tools/fs/Makefile similarity index 100% rename from docs/sections/user_guide/cli/tools/file/Makefile rename to docs/sections/user_guide/cli/tools/fs/Makefile diff --git a/docs/sections/user_guide/cli/tools/file/copy-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/copy-config-timedep.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/file/copy-config-timedep.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-config-timedep.yaml diff --git a/docs/sections/user_guide/cli/tools/file/copy-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy-config.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/file/copy-config.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-config.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd new file mode 100644 index 000000000..fb3d8b83f --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd @@ -0,0 +1 @@ +uw fs copy --config-file copy-config.yaml config files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out new file mode 100644 index 000000000..02c57878d --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out @@ -0,0 +1,3 @@ +[2024-08-14T15:19:59] INFO Validating config against internal schema: files-to-stage +[2024-08-14T15:19:59] INFO 0 UW schema-validation errors found +[2024-08-14T15:19:59] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd new file mode 100644 index 000000000..0f2a1911b --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd @@ -0,0 +1,4 @@ +rm -rf copy-dst-timedep +uw fs copy --target-dir copy-dst-timedep --config-file copy-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files +echo +tree copy-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out similarity index 100% rename from docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out rename to docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd new file mode 100644 index 000000000..175451030 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd @@ -0,0 +1,4 @@ +rm -rf copy-dst +uw fs copy --target-dir copy-dst --config-file copy-config.yaml config files +echo +tree copy-dst diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/file/copy-exec.out rename to docs/sections/user_guide/cli/tools/fs/copy-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-help.cmd b/docs/sections/user_guide/cli/tools/fs/copy-help.cmd new file mode 100644 index 000000000..4806f822a --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-help.cmd @@ -0,0 +1 @@ +uw fs copy --help diff --git a/docs/sections/user_guide/cli/tools/file/copy-help.out b/docs/sections/user_guide/cli/tools/fs/copy-help.out similarity index 73% rename from docs/sections/user_guide/cli/tools/file/copy-help.out rename to docs/sections/user_guide/cli/tools/fs/copy-help.out index bbc885512..ce73007a6 100644 --- a/docs/sections/user_guide/cli/tools/file/copy-help.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-help.out @@ -1,7 +1,7 @@ -usage: uw file copy [-h] [--version] [--config-file PATH] [--target-dir PATH] - [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] - [--quiet] [--verbose] - [KEY ...] +usage: uw fs copy [-h] [--version] [--config-file PATH] [--target-dir PATH] + [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] [--quiet] + [--verbose] + [KEY ...] Copy files diff --git a/docs/sections/user_guide/cli/tools/fs/help.cmd b/docs/sections/user_guide/cli/tools/fs/help.cmd new file mode 100644 index 000000000..7e46d94b9 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/help.cmd @@ -0,0 +1 @@ +uw fs --help diff --git a/docs/sections/user_guide/cli/tools/file/help.out b/docs/sections/user_guide/cli/tools/fs/help.out similarity index 60% rename from docs/sections/user_guide/cli/tools/file/help.out rename to docs/sections/user_guide/cli/tools/fs/help.out index 70252750d..6d573c404 100644 --- a/docs/sections/user_guide/cli/tools/file/help.out +++ b/docs/sections/user_guide/cli/tools/fs/help.out @@ -1,6 +1,6 @@ -usage: uw file [-h] [--version] ACTION ... +usage: uw fs [-h] [--version] ACTION ... -Handle files +Handle filesystem items (files and directories) Optional arguments: -h, --help @@ -14,3 +14,5 @@ Positional arguments: Copy files link Link files + makedirs + Make directories diff --git a/docs/sections/user_guide/cli/tools/file/link-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/file/link-config-timedep.yaml rename to docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml diff --git a/docs/sections/user_guide/cli/tools/file/link-config.yaml b/docs/sections/user_guide/cli/tools/fs/link-config.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/file/link-config.yaml rename to docs/sections/user_guide/cli/tools/fs/link-config.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd new file mode 100644 index 000000000..8446a4e8b --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd @@ -0,0 +1 @@ +uw fs link --config-file link-config.yaml config files diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out new file mode 100644 index 000000000..03c1247f5 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out @@ -0,0 +1,3 @@ +[2024-08-14T15:19:57] INFO Validating config against internal schema: files-to-stage +[2024-08-14T15:19:57] INFO 0 UW schema-validation errors found +[2024-08-14T15:19:57] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd new file mode 100644 index 000000000..f3e240397 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd @@ -0,0 +1,4 @@ +rm -rf link-dst-timedep +uw fs link --target-dir link-dst-timedep --config-file link-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files +echo +tree link-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out similarity index 100% rename from docs/sections/user_guide/cli/tools/file/link-exec-timedep.out rename to docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec.cmd new file mode 100644 index 000000000..f4f14059b --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-exec.cmd @@ -0,0 +1,4 @@ +rm -rf link-dst +uw fs link --target-dir link-dst --config-file link-config.yaml config files +echo +tree link-dst diff --git a/docs/sections/user_guide/cli/tools/file/link-exec.out b/docs/sections/user_guide/cli/tools/fs/link-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/file/link-exec.out rename to docs/sections/user_guide/cli/tools/fs/link-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/link-help.cmd b/docs/sections/user_guide/cli/tools/fs/link-help.cmd new file mode 100644 index 000000000..b36d8d6d2 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-help.cmd @@ -0,0 +1 @@ +uw fs link --help diff --git a/docs/sections/user_guide/cli/tools/file/link-help.out b/docs/sections/user_guide/cli/tools/fs/link-help.out similarity index 73% rename from docs/sections/user_guide/cli/tools/file/link-help.out rename to docs/sections/user_guide/cli/tools/fs/link-help.out index 8734b5ad4..be07a9eec 100644 --- a/docs/sections/user_guide/cli/tools/file/link-help.out +++ b/docs/sections/user_guide/cli/tools/fs/link-help.out @@ -1,7 +1,7 @@ -usage: uw file link [-h] [--version] [--config-file PATH] [--target-dir PATH] - [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] - [--quiet] [--verbose] - [KEY ...] +usage: uw fs link [-h] [--version] [--config-file PATH] [--target-dir PATH] + [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] [--quiet] + [--verbose] + [KEY ...] Link files diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml new file mode 100644 index 000000000..abf1001b8 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml @@ -0,0 +1,8 @@ +config: + makedirs: + - foo/{{ yyyymmdd }}/{{ hh }}/{{ nnn }}/bar + - baz/{{ yyyymmdd }}/{{ hh }}/{{ nnn }}/qux +yyyymmdd: "{{ cycle.strftime('%Y%m%d') }}" +hh: "{{ cycle.strftime('%H') }}" +nnn: "{{ '%03d' % (leadtime.total_seconds() // 3600) }}" +validtime: "{{ (cycle + leadtime).strftime('%Y-%m-%dT%H') }}" diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml new file mode 100644 index 000000000..6bf12f29a --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml @@ -0,0 +1,4 @@ +config: + makedirs: + - foo + - bar diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd new file mode 100644 index 000000000..80ff4150e --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd @@ -0,0 +1 @@ +uw fs makedirs --config-file makedirs-config.yaml config diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out new file mode 100644 index 000000000..ceefc693e --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out @@ -0,0 +1,3 @@ +[2024-08-14T15:19:58] INFO Validating config against internal schema: makedirs +[2024-08-14T15:19:58] INFO 0 UW schema-validation errors found +[2024-08-14T15:19:58] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd new file mode 100644 index 000000000..fa30614ec --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd @@ -0,0 +1,4 @@ +rm -rf makedirs-parent-timedep +uw fs makedirs --target-dir makedirs-parent-timedep --config-file makedirs-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config +echo +tree -F makedirs-parent-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out new file mode 100644 index 000000000..a89e33a29 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out @@ -0,0 +1,29 @@ +[2024-08-12T04:35:49] INFO Validating config against internal schema: makedirs +[2024-08-12T04:35:49] INFO 0 UW schema-validation errors found +[2024-08-12T04:35:49] INFO Directories: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directories: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Requirement(s) ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Executing +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Final state: Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Requirement(s) ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Executing +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Final state: Ready +[2024-08-12T04:35:49] INFO Directories: Final state: Ready + +makedirs-parent-timedep/ +├── baz/ +│   └── 20240529/ +│   └── 12/ +│   └── 006/ +│   └── qux/ +└── foo/ + └── 20240529/ + └── 12/ + └── 006/ + └── bar/ + +11 directories, 0 files diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd new file mode 100644 index 000000000..b99d8c3f4 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd @@ -0,0 +1,4 @@ +rm -rf makedirs-parent +uw fs makedirs --target-dir makedirs-parent --config-file makedirs-config.yaml config +echo +tree -F makedirs-parent diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out new file mode 100644 index 000000000..376518b59 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out @@ -0,0 +1,21 @@ +[2024-08-12T04:35:49] INFO Validating config against internal schema: makedirs +[2024-08-12T04:35:49] INFO 0 UW schema-validation errors found +[2024-08-12T04:35:49] INFO Directories: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directories: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Requirement(s) ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Executing +[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Final state: Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Requirement(s) ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Executing +[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Final state: Ready +[2024-08-12T04:35:49] INFO Directories: Final state: Ready + +makedirs-parent/ +├── bar/ +└── foo/ + +3 directories, 0 files diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-help.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-help.cmd new file mode 100644 index 000000000..f9cacc5e3 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-help.cmd @@ -0,0 +1 @@ +uw fs makedirs --help diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-help.out b/docs/sections/user_guide/cli/tools/fs/makedirs-help.out new file mode 100644 index 000000000..0e80d2f16 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-help.out @@ -0,0 +1,28 @@ +usage: uw fs makedirs [-h] [--version] [--config-file PATH] + [--target-dir PATH] [--cycle CYCLE] + [--leadtime LEADTIME] [--dry-run] [--quiet] [--verbose] + [KEY ...] + +Make directories + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) + --target-dir PATH + Root directory for relative destination paths + --cycle CYCLE + The cycle in ISO8601 format (e.g. 2024-08-12T00) + --leadtime LEADTIME + The leadtime as hours[:minutes[:seconds]] + --dry-run + Only log info, making no changes + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages + KEY + YAML key leading to file dst/src block diff --git a/docs/sections/user_guide/cli/tools/file/src/20240529/12/006/baz b/docs/sections/user_guide/cli/tools/fs/src/20240529/12/006/baz similarity index 100% rename from docs/sections/user_guide/cli/tools/file/src/20240529/12/006/baz rename to docs/sections/user_guide/cli/tools/fs/src/20240529/12/006/baz diff --git a/docs/sections/user_guide/cli/tools/file/src/bar b/docs/sections/user_guide/cli/tools/fs/src/bar similarity index 100% rename from docs/sections/user_guide/cli/tools/file/src/bar rename to docs/sections/user_guide/cli/tools/fs/src/bar diff --git a/docs/sections/user_guide/cli/tools/file/src/foo b/docs/sections/user_guide/cli/tools/fs/src/foo similarity index 100% rename from docs/sections/user_guide/cli/tools/file/src/foo rename to docs/sections/user_guide/cli/tools/fs/src/foo diff --git a/docs/sections/user_guide/cli/tools/index.rst b/docs/sections/user_guide/cli/tools/index.rst index 27d52c00f..238759a8b 100644 --- a/docs/sections/user_guide/cli/tools/index.rst +++ b/docs/sections/user_guide/cli/tools/index.rst @@ -6,6 +6,6 @@ Tools config execute - file + fs rocoto template diff --git a/docs/sections/user_guide/yaml/files.rst b/docs/sections/user_guide/yaml/files.rst index 878ffd382..0d34ba38d 100644 --- a/docs/sections/user_guide/yaml/files.rst +++ b/docs/sections/user_guide/yaml/files.rst @@ -3,7 +3,7 @@ File Blocks =========== -File blocks define files to be staged in a target directory as copies or symbolic links. Keys in such blocks specify destination paths relative to the target directory, and values specify source paths. +File blocks define files to be staged in a target directory as copies or symbolic links. Keys in such blocks specify either absolute destination paths, or destination paths relative to the target directory. Values specify source paths. Example block: diff --git a/docs/sections/user_guide/yaml/index.rst b/docs/sections/user_guide/yaml/index.rst index 8da086673..e8e1be934 100644 --- a/docs/sections/user_guide/yaml/index.rst +++ b/docs/sections/user_guide/yaml/index.rst @@ -5,9 +5,10 @@ UW YAML :maxdepth: 1 components/index - platform execution files - updating_values + makedirs + platform rocoto tags + updating_values diff --git a/docs/sections/user_guide/yaml/makedirs.rst b/docs/sections/user_guide/yaml/makedirs.rst new file mode 100644 index 000000000..e1744666a --- /dev/null +++ b/docs/sections/user_guide/yaml/makedirs.rst @@ -0,0 +1,22 @@ +.. _makedirs_yaml: + +Directory Blocks +================ + +Directory blocks define a sequence of one or more directories to be created, nested under a ``makedirs:`` key. Each value is either an absolute path, or a path relative to the target directory, specified either via the CLI or an API call. + +Example block with absolute paths: + +.. code-block:: yaml + + makedirs: + - /path/to/dir1 + - /path/to/dir2 + +Example block with relative paths: + +.. code-block:: yaml + + makedirs: + - /subdir/dir1 + - ../../dir2 diff --git a/recipe/meta.json b/recipe/meta.json index c37b5486b..faa658d13 100644 --- a/recipe/meta.json +++ b/recipe/meta.json @@ -33,5 +33,5 @@ "pyyaml =6.0.*" ] }, - "version": "2.4.1" + "version": "2.4.2" } diff --git a/src/uwtools/api/cdeps.py b/src/uwtools/api/cdeps.py index 0c91a6b06..f6e112309 100644 --- a/src/uwtools/api/cdeps.py +++ b/src/uwtools/api/cdeps.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``cdeps`` driver. """ -from functools import partial - from uwtools.drivers.cdeps import CDEPS -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = CDEPS execute = _make_execute(_driver, with_cycle=True) -tasks = partial(tasks, _driver) -__all__ = ["CDEPS", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["CDEPS", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/chgres_cube.py b/src/uwtools/api/chgres_cube.py index 51fd53f90..477f892e7 100644 --- a/src/uwtools/api/chgres_cube.py +++ b/src/uwtools/api/chgres_cube.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``chgres_cube`` driver. """ -from functools import partial - from uwtools.drivers.chgres_cube import ChgresCube -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = ChgresCube execute = _make_execute(_driver, with_cycle=True) -tasks = partial(tasks, _driver) -__all__ = ["ChgresCube", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["ChgresCube", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index 2d402b1be..ec05a0fa5 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -6,12 +6,12 @@ from pathlib import Path from typing import Optional, Union -from uwtools.config.formats.fieldtable import FieldTableConfig as _FieldTableConfig -from uwtools.config.formats.ini import INIConfig as _INIConfig -from uwtools.config.formats.nml import NMLConfig as _NMLConfig -from uwtools.config.formats.sh import SHConfig as _SHConfig -from uwtools.config.formats.yaml import Config as _Config -from uwtools.config.formats.yaml import YAMLConfig as _YAMLConfig +from uwtools.config.formats.base import Config as _Config +from uwtools.config.formats.fieldtable import FieldTableConfig +from uwtools.config.formats.ini import INIConfig +from uwtools.config.formats.nml import NMLConfig +from uwtools.config.formats.sh import SHConfig +from uwtools.config.formats.yaml import YAMLConfig from uwtools.config.tools import compare_configs as _compare from uwtools.config.tools import realize_config as _realize from uwtools.config.validator import validate_external as _validate_external @@ -42,7 +42,7 @@ def compare( def get_fieldtable_config( config: Union[dict, Optional[Union[Path, str]]] = None, stdin_ok=False -) -> _FieldTableConfig: +) -> FieldTableConfig: """ Get a ``FieldTableConfig`` object. @@ -50,13 +50,13 @@ def get_fieldtable_config( :param stdin_ok: OK to read from ``stdin``? :return: An initialized ``FieldTableConfig`` object. """ - return _FieldTableConfig(config=_ensure_data_source(_str2path(config), stdin_ok)) + return FieldTableConfig(config=_ensure_data_source(_str2path(config), stdin_ok)) def get_ini_config( config: Union[dict, Optional[Union[Path, str]]] = None, stdin_ok: bool = False, -) -> _INIConfig: +) -> INIConfig: """ Get an ``INIConfig`` object. @@ -64,13 +64,13 @@ def get_ini_config( :param stdin_ok: OK to read from ``stdin``? :return: An initialized ``INIConfig`` object. """ - return _INIConfig(config=_ensure_data_source(_str2path(config), stdin_ok)) + return INIConfig(config=_ensure_data_source(_str2path(config), stdin_ok)) def get_nml_config( config: Union[dict, Optional[Union[Path, str]]] = None, stdin_ok: bool = False, -) -> _NMLConfig: +) -> NMLConfig: """ Get an ``NMLConfig`` object. @@ -78,13 +78,13 @@ def get_nml_config( :param stdin_ok: OK to read from ``stdin``? :return: An initialized ``NMLConfig`` object. """ - return _NMLConfig(config=_ensure_data_source(_str2path(config), stdin_ok)) + return NMLConfig(config=_ensure_data_source(_str2path(config), stdin_ok)) def get_sh_config( config: Union[dict, Optional[Union[Path, str]]] = None, stdin_ok: bool = False, -) -> _SHConfig: +) -> SHConfig: """ Get an ``SHConfig`` object. @@ -92,13 +92,13 @@ def get_sh_config( :param stdin_ok: OK to read from ``stdin``? :return: An initialized ``SHConfig`` object. """ - return _SHConfig(config=_ensure_data_source(_str2path(config), stdin_ok)) + return SHConfig(config=_ensure_data_source(_str2path(config), stdin_ok)) def get_yaml_config( config: Union[dict, Optional[Union[Path, str]]] = None, stdin_ok: bool = False, -) -> _YAMLConfig: +) -> YAMLConfig: """ Get a ``YAMLConfig`` object. @@ -106,7 +106,7 @@ def get_yaml_config( :param stdin_ok: OK to read from ``stdin``? :return: An initialized ``YAMLConfig`` object. """ - return _YAMLConfig(config=_ensure_data_source(_str2path(config), stdin_ok)) + return YAMLConfig(config=_ensure_data_source(_str2path(config), stdin_ok)) def realize( @@ -125,10 +125,12 @@ def realize( """ NB: This docstring is dynamically replaced: See realize.__doc__ definition below. """ + if update_config is None and update_format is not None: # i.e. updates will be read from stdin + update_config = _ensure_data_source(update_config, stdin_ok) return _realize( input_config=_ensure_data_source(_str2path(input_config), stdin_ok), input_format=input_format, - update_config=_ensure_data_source(_str2path(update_config), stdin_ok), + update_config=_str2path(update_config), update_format=update_format, output_file=_str2path(output_file), output_format=output_format, @@ -160,7 +162,7 @@ def realize_to_dict( # pylint: disable=unused-argument def validate( schema_file: Union[Path, str], - config: Optional[Union[dict, _YAMLConfig, Path, str]] = None, + config: Optional[Union[dict, YAMLConfig, Path, str]] = None, stdin_ok: bool = False, ) -> bool: """ @@ -246,3 +248,20 @@ def validate( """.format( extensions=", ".join(_FORMAT.extensions()) ).strip() + +__all__ = [ + "FieldTableConfig", + "INIConfig", + "NMLConfig", + "SHConfig", + "YAMLConfig", + "compare", + "get_fieldtable_config", + "get_ini_config", + "get_nml_config", + "get_sh_config", + "get_yaml_config", + "realize", + "realize_to_dict", + "validate", +] diff --git a/src/uwtools/api/driver.py b/src/uwtools/api/driver.py index 71f9e946f..4b0138489 100644 --- a/src/uwtools/api/driver.py +++ b/src/uwtools/api/driver.py @@ -2,14 +2,6 @@ API access to the ``uwtools`` driver base classes. """ -from datetime import datetime, timedelta -from importlib import import_module -from importlib.util import module_from_spec, spec_from_file_location -from inspect import getfullargspec -from pathlib import Path -from types import ModuleType -from typing import Optional, Type, Union - from uwtools.drivers.driver import ( Assets, AssetsCycleBased, @@ -20,148 +12,6 @@ DriverCycleLeadtimeBased, DriverTimeInvariant, ) -from uwtools.drivers.support import graph -from uwtools.drivers.support import tasks as _tasks -from uwtools.logging import log -from uwtools.strings import STR -from uwtools.utils.api import ensure_data_source - - -def execute( - module: Union[Path, str], - classname: str, - task: str, - schema_file: Optional[str] = None, - config: Optional[Union[Path, str]] = None, - cycle: Optional[datetime] = None, # pylint: disable=unused-argument - leadtime: Optional[timedelta] = None, # pylint: disable=unused-argument - batch: Optional[bool] = False, # pylint: disable=unused-argument - dry_run: Optional[bool] = False, - graph_file: Optional[Union[Path, str]] = None, - key_path: Optional[list[str]] = None, - stdin_ok: Optional[bool] = False, -) -> bool: - """ - Execute a task. - - If ``batch`` is specified, a runscript will be written and submitted to the batch system. - Otherwise, the executable will be run directly on the current system. - - :param module: Path to driver module or name of module on sys.path. - :param classname: Name of driver class to instantiate. - :param task: Name of driver task to execute. - :param schema_file: Path to schema file. - :param config: Path to config file (read stdin if missing or None). - :param cycle: The cycle. - :param leadtime: The leadtime. - :param batch: Submit run to the batch system? - :param dry_run: Do not run the executable, just report what would have been done. - :param graph_file: Write Graphviz DOT output here. - :param key_path: Path of keys to subsection of config file. - :param stdin_ok: OK to read from stdin? - :return: ``True`` if task completes without raising an exception. - """ - class_, module_path = _get_driver_class(module, classname) - if not class_: - return False - assert module_path is not None - args = dict(locals()) - accepted = set(getfullargspec(class_).args) - non_optional = {STR.cycle, STR.leadtime} - for arg in sorted([STR.batch, *non_optional]): - if args.get(arg) and arg not in accepted: - log.error("%s does not accept argument '%s'", classname, arg) - return False - for arg in sorted(non_optional): - if arg in accepted and args[arg] is None: - log.error("%s requires argument '%s'", classname, arg) - return False - kwargs = dict( - config=ensure_data_source(config, bool(stdin_ok)), - dry_run=dry_run, - key_path=key_path, - schema_file=schema_file or module_path.with_suffix(".jsonschema"), - ) - required = non_optional & accepted - for arg in sorted([STR.batch, *required]): - if arg in accepted: - kwargs[arg] = args[arg] - driverobj = class_(**kwargs) - log.debug("Instantiated %s with: %s", classname, kwargs) - getattr(driverobj, task)() - if graph_file: - with open(graph_file, "w", encoding="utf-8") as f: - print(graph(), file=f) - return True - - -def tasks(module: Union[Path, str], classname: str) -> dict[str, str]: - """ - Returns a mapping from task names to their one-line descriptions. - - :param module: Name of driver module. - :param classname: Name of driver class to instantiate. - """ - class_, _ = _get_driver_class(module, classname) - if not class_: - log.error("Could not get tasks from class %s in module %s", classname, module) - return {} - return _tasks(class_) - - -def _get_driver_class( - module: Union[Path, str], classname: str -) -> tuple[Optional[Type], Optional[Path]]: - """ - Returns the driver class. - - :param module: Name of driver module to load. - :param classname: Name of driver class to instantiate. - """ - if not (m := _get_driver_module_explicit(Path(module))): - if not (m := _get_driver_module_implicit(str(module))): - log.error("Could not load module %s", module) - return None, None - assert m.__file__ is not None - module_path = Path(m.__file__) - if hasattr(m, classname): - c: Type = getattr(m, classname) - return c, module_path - log.error("Module %s has no class %s", module, classname) - return None, module_path - - -def _get_driver_module_explicit(module: Path) -> Optional[ModuleType]: - """ - Returns the named module found via explicit lookup of given path. - - :param module: Name of driver module to load. - """ - log.debug("Loading module %s", module) - if spec := spec_from_file_location(module.name, module): - m = module_from_spec(spec) - if loader := spec.loader: - try: - loader.exec_module(m) - log.debug("Loaded module %s", module) - return m - except Exception: # pylint: disable=broad-exception-caught - pass - return None - - -def _get_driver_module_implicit(module: str) -> Optional[ModuleType]: - """ - Returns the named module found via implicit (sys.path-based) lookup. - - :param module: Name of driver module to load. - """ - log.debug("Loading module %s from sys.path", module) - try: - return import_module(module) - except Exception: # pylint: disable=broad-exception-caught - return None - __all__ = [ "Assets", @@ -172,7 +22,4 @@ def _get_driver_module_implicit(module: str) -> Optional[ModuleType]: "DriverCycleBased", "DriverCycleLeadtimeBased", "DriverTimeInvariant", - "execute", - "graph", - "tasks", ] diff --git a/src/uwtools/api/esg_grid.py b/src/uwtools/api/esg_grid.py index 06e3bac41..9bbd7e6f5 100644 --- a/src/uwtools/api/esg_grid.py +++ b/src/uwtools/api/esg_grid.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``esg_grid`` driver. """ -from functools import partial - from uwtools.drivers.esg_grid import ESGGrid -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = ESGGrid execute = _make_execute(_driver) -tasks = partial(tasks, _driver) -__all__ = ["ESGGrid", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["ESGGrid", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/execute.py b/src/uwtools/api/execute.py new file mode 100644 index 000000000..24c16cee6 --- /dev/null +++ b/src/uwtools/api/execute.py @@ -0,0 +1,157 @@ +""" +API support for interacting with external drivers. +""" + +from datetime import datetime, timedelta +from importlib import import_module +from importlib.util import module_from_spec, spec_from_file_location +from inspect import getfullargspec +from pathlib import Path +from types import ModuleType +from typing import Optional, Type, Union + +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks +from uwtools.logging import log +from uwtools.strings import STR +from uwtools.utils.api import ensure_data_source + + +def execute( + module: Union[Path, str], + classname: str, + task: str, + schema_file: Optional[str] = None, + config: Optional[Union[Path, str]] = None, + cycle: Optional[datetime] = None, # pylint: disable=unused-argument + leadtime: Optional[timedelta] = None, # pylint: disable=unused-argument + batch: Optional[bool] = False, # pylint: disable=unused-argument + dry_run: Optional[bool] = False, + graph_file: Optional[Union[Path, str]] = None, + key_path: Optional[list[str]] = None, + stdin_ok: Optional[bool] = False, +) -> bool: + """ + Execute a driver task. + + If ``batch`` is specified and the driver is instructed to run, its runscript will be configured + for and submitted to the batch system. Otherwise, the executable will be run directly on the + current system. + + :param module: Path to driver module or name of module on sys.path. + :param classname: Name of driver class to instantiate. + :param task: Name of driver task to execute. + :param schema_file: Path to schema file. + :param config: Path to config file (read stdin if missing or None). + :param cycle: The cycle. + :param leadtime: The leadtime. + :param batch: Submit run to the batch system? + :param dry_run: Do not run the executable, just report what would have been done. + :param graph_file: Write Graphviz DOT output here. + :param key_path: Path of keys to subsection of config file. + :param stdin_ok: OK to read from stdin? + :return: ``True`` if task completes without raising an exception. + """ + class_, module_path = _get_driver_class(module, classname) + if not class_: + return False + assert module_path is not None + args = dict(locals()) + accepted = set(getfullargspec(class_).args) + non_optional = {STR.cycle, STR.leadtime} + for arg in sorted([STR.batch, *non_optional]): + if args.get(arg) and arg not in accepted: + log.error("%s does not accept argument '%s'", classname, arg) + return False + for arg in sorted(non_optional): + if arg in accepted and args[arg] is None: + log.error("%s requires argument '%s'", classname, arg) + return False + kwargs = dict( + config=ensure_data_source(config, bool(stdin_ok)), + dry_run=dry_run, + key_path=key_path, + schema_file=schema_file or module_path.with_suffix(".jsonschema"), + ) + required = non_optional & accepted + for arg in sorted([STR.batch, *required]): + if arg in accepted: + kwargs[arg] = args[arg] + driverobj = class_(**kwargs) + log.debug("Instantiated %s with: %s", classname, kwargs) + getattr(driverobj, task)() + if graph_file: + with open(graph_file, "w", encoding="utf-8") as f: + print(graph(), file=f) + return True + + +def tasks(module: Union[Path, str], classname: str) -> dict[str, str]: + """ + Return a mapping from driver task names to their one-line descriptions. + + :param module: Name of driver module. + :param classname: Name of driver class to instantiate. + """ + class_, _ = _get_driver_class(module, classname) + if not class_: + log.error("Could not get tasks from class %s in module %s", classname, module) + return {} + return _tasks(class_) + + +def _get_driver_class( + module: Union[Path, str], classname: str +) -> tuple[Optional[Type], Optional[Path]]: + """ + Return the driver class. + + :param module: Name of driver module to load. + :param classname: Name of driver class to instantiate. + """ + if not (m := _get_driver_module_explicit(Path(module))): + if not (m := _get_driver_module_implicit(str(module))): + log.error("Could not load module %s", module) + return None, None + assert m.__file__ is not None + module_path = Path(m.__file__) + if hasattr(m, classname): + c: Type = getattr(m, classname) + return c, module_path + log.error("Module %s has no class %s", module, classname) + return None, module_path + + +def _get_driver_module_explicit(module: Path) -> Optional[ModuleType]: + """ + Return the named module found via explicit lookup of given path. + + :param module: Name of driver module to load. + """ + log.debug("Loading module %s", module) + if spec := spec_from_file_location(module.name, module): + m = module_from_spec(spec) + if loader := spec.loader: + try: + loader.exec_module(m) + log.debug("Loaded module %s", module) + return m + except Exception: # pylint: disable=broad-exception-caught + pass + return None + + +def _get_driver_module_implicit(module: str) -> Optional[ModuleType]: + """ + Return the named module found via implicit (sys.path-based) lookup. + + :param module: Name of driver module to load. + """ + log.debug("Loading module %s from sys.path", module) + try: + return import_module(module) + except Exception: # pylint: disable=broad-exception-caught + return None + + +__all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/api/filter_topo.py b/src/uwtools/api/filter_topo.py index fac93ecf6..ded7d6cd6 100644 --- a/src/uwtools/api/filter_topo.py +++ b/src/uwtools/api/filter_topo.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``filter_topo`` driver. """ -from functools import partial - from uwtools.drivers.filter_topo import FilterTopo -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = FilterTopo execute = _make_execute(_driver) -tasks = partial(tasks, _driver) -__all__ = ["FilterTopo", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["FilterTopo", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/file.py b/src/uwtools/api/fs.py similarity index 61% rename from src/uwtools/api/file.py rename to src/uwtools/api/fs.py index 7bac2ddef..5b7d6102f 100644 --- a/src/uwtools/api/file.py +++ b/src/uwtools/api/fs.py @@ -1,5 +1,5 @@ """ -API access to ``uwtools`` file-management tools. +API access to ``uwtools`` file and directory management tools. """ import datetime as dt @@ -8,7 +8,7 @@ from iotaa import Asset -from uwtools.file import Copier, Linker +from uwtools.fs import Copier, Linker, MakeDirs from uwtools.utils.api import ensure_data_source as _ensure_data_source @@ -33,7 +33,7 @@ def copy( :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if all copies were created. """ - copier = Copier( + stager = Copier( target_dir=Path(target_dir) if target_dir else None, config=_ensure_data_source(config, stdin_ok), cycle=cycle, @@ -41,7 +41,7 @@ def copy( keys=keys, dry_run=dry_run, ) - assets: list[Asset] = copier.go() # type: ignore + assets: list[Asset] = stager.go() # type: ignore return all(asset.ready() for asset in assets) @@ -66,7 +66,7 @@ def link( :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if all links were created. """ - linker = Linker( + stager = Linker( target_dir=Path(target_dir) if target_dir else None, config=_ensure_data_source(config, stdin_ok), cycle=cycle, @@ -74,8 +74,41 @@ def link( keys=keys, dry_run=dry_run, ) - assets: list[Asset] = linker.go() # type: ignore + assets: list[Asset] = stager.go() # type: ignore return all(asset.ready() for asset in assets) -__all__ = ["Copier", "Linker", "copy", "link"] +def makedirs( + config: Optional[Union[Path, dict, str]] = None, + target_dir: Optional[Union[Path, str]] = None, + cycle: Optional[dt.datetime] = None, + leadtime: Optional[dt.timedelta] = None, + keys: Optional[list[str]] = None, + dry_run: bool = False, + stdin_ok: bool = False, +) -> bool: + """ + Make directories. + + :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``). + :param target_dir: Path to target directory. + :param cycle: A datetime object to make available for use in the config. + :param leadtime: A timedelta object to make available for use in the config. + :param keys: YAML keys leading to file dst/src block. + :param dry_run: Do not link files. + :param stdin_ok: OK to read from ``stdin``? + :return: ``True`` if all directories were made. + """ + stager = MakeDirs( + target_dir=Path(target_dir) if target_dir else None, + config=_ensure_data_source(config, stdin_ok), + cycle=cycle, + leadtime=leadtime, + keys=keys, + dry_run=dry_run, + ) + assets: list[Asset] = stager.go() # type: ignore + return all(asset.ready() for asset in assets) + + +__all__ = ["Copier", "Linker", "MakeDirs", "copy", "link", "makedirs"] diff --git a/src/uwtools/api/fv3.py b/src/uwtools/api/fv3.py index 4c613b2dd..b5a9401b0 100644 --- a/src/uwtools/api/fv3.py +++ b/src/uwtools/api/fv3.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``fv3`` driver. """ -from functools import partial - from uwtools.drivers.fv3 import FV3 -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = FV3 execute = _make_execute(_driver, with_cycle=True) -tasks = partial(tasks, _driver) -__all__ = ["FV3", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["FV3", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/global_equiv_resol.py b/src/uwtools/api/global_equiv_resol.py index 6bd3940b5..6a6484354 100644 --- a/src/uwtools/api/global_equiv_resol.py +++ b/src/uwtools/api/global_equiv_resol.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``global_equiv_resol`` driver. """ -from functools import partial - from uwtools.drivers.global_equiv_resol import GlobalEquivResol -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = GlobalEquivResol execute = _make_execute(_driver) -tasks = partial(tasks, _driver) -__all__ = ["GlobalEquivResol", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["GlobalEquivResol", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/ioda.py b/src/uwtools/api/ioda.py index 5abe332ca..cea6e5fa1 100644 --- a/src/uwtools/api/ioda.py +++ b/src/uwtools/api/ioda.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``ioda`` driver. """ -from functools import partial - from uwtools.drivers.ioda import IODA -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = IODA execute = _make_execute(_driver, with_cycle=True) -tasks = partial(tasks, _driver) -__all__ = ["IODA", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["IODA", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/jedi.py b/src/uwtools/api/jedi.py index c0986c277..1cb0f1ff1 100644 --- a/src/uwtools/api/jedi.py +++ b/src/uwtools/api/jedi.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``jedi`` driver. """ -from functools import partial - from uwtools.drivers.jedi import JEDI -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = JEDI execute = _make_execute(_driver, with_cycle=True) -tasks = partial(tasks, _driver) -__all__ = ["JEDI", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["JEDI", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/make_hgrid.py b/src/uwtools/api/make_hgrid.py index 63886ce03..68de11ff4 100644 --- a/src/uwtools/api/make_hgrid.py +++ b/src/uwtools/api/make_hgrid.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``make_hgrid`` driver. """ -from functools import partial - from uwtools.drivers.make_hgrid import MakeHgrid -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = MakeHgrid execute = _make_execute(_driver) -tasks = partial(tasks, _driver) -__all__ = ["MakeHgrid", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["MakeHgrid", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/make_solo_mosaic.py b/src/uwtools/api/make_solo_mosaic.py index d9dc4ebe7..c6ddb6bc9 100644 --- a/src/uwtools/api/make_solo_mosaic.py +++ b/src/uwtools/api/make_solo_mosaic.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``make_solo_mosaic`` driver. """ -from functools import partial - from uwtools.drivers.make_solo_mosaic import MakeSoloMosaic -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = MakeSoloMosaic execute = _make_execute(_driver) -tasks = partial(tasks, _driver) -__all__ = ["MakeSoloMosaic", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["MakeSoloMosaic", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/mpas.py b/src/uwtools/api/mpas.py index 21ccb9cdb..3ce1245ad 100644 --- a/src/uwtools/api/mpas.py +++ b/src/uwtools/api/mpas.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``mpas`` driver. """ -from functools import partial - from uwtools.drivers.mpas import MPAS -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = MPAS execute = _make_execute(_driver, with_cycle=True) -tasks = partial(tasks, _driver) -__all__ = ["MPAS", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["MPAS", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/mpas_init.py b/src/uwtools/api/mpas_init.py index 2654af782..2308a578f 100644 --- a/src/uwtools/api/mpas_init.py +++ b/src/uwtools/api/mpas_init.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``mpas_init`` driver. """ -from functools import partial - from uwtools.drivers.mpas_init import MPASInit -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = MPASInit execute = _make_execute(_driver, with_cycle=True) -tasks = partial(tasks, _driver) -__all__ = ["MPASInit", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["MPASInit", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/orog_gsl.py b/src/uwtools/api/orog_gsl.py index ff6973866..f56c7f67e 100644 --- a/src/uwtools/api/orog_gsl.py +++ b/src/uwtools/api/orog_gsl.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``orog_gsl`` driver. """ -from functools import partial - from uwtools.drivers.orog_gsl import OrogGSL -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = OrogGSL execute = _make_execute(_driver) -tasks = partial(tasks, _driver) -__all__ = ["OrogGSL", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["OrogGSL", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/schism.py b/src/uwtools/api/schism.py index 50cfe73f5..50e4e3ffb 100644 --- a/src/uwtools/api/schism.py +++ b/src/uwtools/api/schism.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``schism`` driver. """ -from functools import partial - from uwtools.drivers.schism import SCHISM -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = SCHISM execute = _make_execute(_driver, with_cycle=True) -tasks = partial(tasks, _driver) -__all__ = ["SCHISM", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["SCHISM", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/sfc_climo_gen.py b/src/uwtools/api/sfc_climo_gen.py index 7e881def3..d882d6357 100644 --- a/src/uwtools/api/sfc_climo_gen.py +++ b/src/uwtools/api/sfc_climo_gen.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``sfc_climo_gen`` driver. """ -from functools import partial - from uwtools.drivers.sfc_climo_gen import SfcClimoGen -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = SfcClimoGen execute = _make_execute(_driver) -tasks = partial(tasks, _driver) -__all__ = ["SfcClimoGen", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["SfcClimoGen", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/shave.py b/src/uwtools/api/shave.py index ff3b9911f..0dd01ffd8 100644 --- a/src/uwtools/api/shave.py +++ b/src/uwtools/api/shave.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``shave`` driver. """ -from functools import partial - from uwtools.drivers.shave import Shave -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.utils.api import make_execute as _make_execute _driver = Shave execute = _make_execute(_driver) -tasks = partial(tasks, _driver) -__all__ = ["Shave", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["Shave", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/ungrib.py b/src/uwtools/api/ungrib.py index 1bfa0c7fa..34a353dbd 100644 --- a/src/uwtools/api/ungrib.py +++ b/src/uwtools/api/ungrib.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``ungrib`` driver. """ -from functools import partial - -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.drivers.ungrib import Ungrib from uwtools.utils.api import make_execute as _make_execute _driver = Ungrib execute = _make_execute(_driver, with_cycle=True) -tasks = partial(tasks, _driver) -__all__ = ["Ungrib", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["Ungrib", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/upp.py b/src/uwtools/api/upp.py index 9a098df42..0def005f9 100644 --- a/src/uwtools/api/upp.py +++ b/src/uwtools/api/upp.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``upp`` driver. """ -from functools import partial - -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.drivers.upp import UPP from uwtools.utils.api import make_execute as _make_execute _driver = UPP execute = _make_execute(_driver, with_cycle=True, with_leadtime=True) -tasks = partial(tasks, _driver) -__all__ = ["UPP", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["UPP", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/ww3.py b/src/uwtools/api/ww3.py index dd1a4759f..25fe70e32 100644 --- a/src/uwtools/api/ww3.py +++ b/src/uwtools/api/ww3.py @@ -2,13 +2,27 @@ API access to the ``uwtools`` ``ww3`` driver. """ -from functools import partial - -from uwtools.drivers.support import graph, tasks +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks from uwtools.drivers.ww3 import WaveWatchIII from uwtools.utils.api import make_execute as _make_execute _driver = WaveWatchIII execute = _make_execute(_driver, with_cycle=True) -tasks = partial(tasks, _driver) -__all__ = ["WaveWatchIII", "execute", "graph", "tasks"] + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["WaveWatchIII", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index ce1941e8d..15f3dda36 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -18,7 +18,8 @@ import uwtools.api import uwtools.api.config import uwtools.api.driver -import uwtools.api.file +import uwtools.api.execute +import uwtools.api.fs import uwtools.api.rocoto import uwtools.api.template import uwtools.config.jinja2 @@ -51,9 +52,9 @@ def main() -> None: setup_logging(quiet=True) args, checks = _parse_args(sys.argv[1:]) args[STR.action] = args.get(STR.action, args[STR.mode]) - for check in checks[args[STR.mode]][args[STR.action]]: + for check in checks[args[STR.mode]].get(args[STR.action], []): check(args) - setup_logging(quiet=args[STR.quiet], verbose=args[STR.verbose]) + setup_logging(quiet=args.get(STR.quiet, False), verbose=args.get(STR.verbose, False)) except UWError as e: _abort(str(e)) try: @@ -61,7 +62,7 @@ def main() -> None: tools: dict[str, Callable[..., bool]] = { STR.config: _dispatch_config, STR.execute: _dispatch_execute, - STR.file: _dispatch_file, + STR.fs: _dispatch_fs, STR.rocoto: _dispatch_rocoto, STR.template: _dispatch_template, } @@ -99,7 +100,7 @@ def main() -> None: def _add_subparser_config(subparsers: Subparsers) -> ModeChecks: """ - Subparser for mode: config + Add subparser for mode: config :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -115,7 +116,7 @@ def _add_subparser_config(subparsers: Subparsers) -> ModeChecks: def _add_subparser_config_compare(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: config compare + Add subparser for mode: config compare :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -145,7 +146,7 @@ def _add_subparser_config_compare(subparsers: Subparsers) -> ActionChecks: def _add_subparser_config_realize(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: config realize + Add subparser for mode: config realize :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -171,7 +172,7 @@ def _add_subparser_config_realize(subparsers: Subparsers) -> ActionChecks: def _add_subparser_config_validate(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: config validate + Add subparser for mode: config validate :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -185,7 +186,7 @@ def _add_subparser_config_validate(subparsers: Subparsers) -> ActionChecks: def _dispatch_config(args: Args) -> bool: """ - Dispatch logic for config mode. + Define dispatch logic for config mode. :param args: Parsed command-line args. """ @@ -199,7 +200,7 @@ def _dispatch_config(args: Args) -> bool: def _dispatch_config_compare(args: Args) -> bool: """ - Dispatch logic for config compare action. + Define dispatch logic for config compare action. :param args: Parsed command-line args. """ @@ -213,7 +214,7 @@ def _dispatch_config_compare(args: Args) -> bool: def _dispatch_config_realize(args: Args) -> bool: """ - Dispatch logic for config realize action. + Define dispatch logic for config realize action. :param args: Parsed command-line args. """ @@ -241,7 +242,7 @@ def _dispatch_config_realize(args: Args) -> bool: def _dispatch_config_validate(args: Args) -> bool: """ - Dispatch logic for config validate action. + Define dispatch logic for config validate action. :param args: Parsed command-line args. """ @@ -257,7 +258,7 @@ def _dispatch_config_validate(args: Args) -> bool: def _add_subparser_execute(subparsers: Subparsers) -> ModeChecks: """ - Subparser for mode: execute + Add subparser for mode: execute :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -284,11 +285,11 @@ def _add_subparser_execute(subparsers: Subparsers) -> ModeChecks: def _dispatch_execute(args: Args) -> bool: """ - Dispatch logic for execute mode. + Define dispatch logic for execute mode. :param args: Parsed command-line args. """ - return uwtools.api.driver.execute( + return uwtools.api.execute.execute( classname=args[STR.classname], module=args[STR.module], task=args[STR.task], @@ -304,27 +305,28 @@ def _dispatch_execute(args: Args) -> bool: ) -# Mode file +# Mode fs -def _add_subparser_file(subparsers: Subparsers) -> ModeChecks: +def _add_subparser_fs(subparsers: Subparsers) -> ModeChecks: """ - Subparser for mode: file + Add subparser for mode: fs :param subparsers: Parent parser's subparsers, to add this subparser to. """ - parser = _add_subparser(subparsers, STR.file, "Handle files") + parser = _add_subparser(subparsers, STR.fs, "Handle filesystem items (files and directories)") _basic_setup(parser) subparsers = _add_subparsers(parser, STR.action, STR.action.upper()) return { - STR.copy: _add_subparser_file_copy(subparsers), - STR.link: _add_subparser_file_link(subparsers), + STR.copy: _add_subparser_fs_copy(subparsers), + STR.link: _add_subparser_fs_link(subparsers), + STR.makedirs: _add_subparser_fs_makedirs(subparsers), } -def _add_subparser_file_common(parser: Parser) -> ActionChecks: +def _add_subparser_fs_common(parser: Parser) -> ActionChecks: """ - Common subparser code for mode: file {copy link} + Perform common subparser setup for mode: fs {copy link makedirs} :param parser: The parser to configure. """ @@ -339,46 +341,57 @@ def _add_subparser_file_common(parser: Parser) -> ActionChecks: return checks -def _add_subparser_file_copy(subparsers: Subparsers) -> ActionChecks: +def _add_subparser_fs_copy(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: file copy + Add subparser for mode: fs copy :param subparsers: Parent parser's subparsers, to add this subparser to. """ parser = _add_subparser(subparsers, STR.copy, "Copy files") - return _add_subparser_file_common(parser) + return _add_subparser_fs_common(parser) -def _add_subparser_file_link(subparsers: Subparsers) -> ActionChecks: +def _add_subparser_fs_link(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: file link + Add subparser for mode: fs link :param subparsers: Parent parser's subparsers, to add this subparser to. """ parser = _add_subparser(subparsers, STR.link, "Link files") - return _add_subparser_file_common(parser) + return _add_subparser_fs_common(parser) -def _dispatch_file(args: Args) -> bool: +def _add_subparser_fs_makedirs(subparsers: Subparsers) -> ActionChecks: """ - Dispatch logic for file mode. + Add subparser for mode: fs makedirs + + :param subparsers: Parent parser's subparsers, to add this subparser to. + """ + parser = _add_subparser(subparsers, STR.makedirs, "Make directories") + return _add_subparser_fs_common(parser) + + +def _dispatch_fs(args: Args) -> bool: + """ + Define dispatch logic for fs mode. :param args: Parsed command-line args. """ actions = { - STR.copy: _dispatch_file_copy, - STR.link: _dispatch_file_link, + STR.copy: _dispatch_fs_copy, + STR.link: _dispatch_fs_link, + STR.makedirs: _dispatch_fs_makedirs, } return actions[args[STR.action]](args) -def _dispatch_file_copy(args: Args) -> bool: +def _dispatch_fs_copy(args: Args) -> bool: """ - Dispatch logic for file copy action. + Define dispatch logic for fs copy action. :param args: Parsed command-line args. """ - return uwtools.api.file.copy( + return uwtools.api.fs.copy( target_dir=args[STR.targetdir], config=args[STR.cfgfile], cycle=args[STR.cycle], @@ -389,13 +402,30 @@ def _dispatch_file_copy(args: Args) -> bool: ) -def _dispatch_file_link(args: Args) -> bool: +def _dispatch_fs_link(args: Args) -> bool: """ - Dispatch logic for file link action. + Define dispatch logic for fs link action. :param args: Parsed command-line args. """ - return uwtools.api.file.link( + return uwtools.api.fs.link( + target_dir=args[STR.targetdir], + config=args[STR.cfgfile], + cycle=args[STR.cycle], + leadtime=args[STR.leadtime], + keys=args[STR.keys], + dry_run=args[STR.dryrun], + stdin_ok=True, + ) + + +def _dispatch_fs_makedirs(args: Args) -> bool: + """ + Define dispatch logic for fs makedirs action. + + :param args: Parsed command-line args. + """ + return uwtools.api.fs.makedirs( target_dir=args[STR.targetdir], config=args[STR.cfgfile], cycle=args[STR.cycle], @@ -411,7 +441,7 @@ def _dispatch_file_link(args: Args) -> bool: def _add_subparser_rocoto(subparsers: Subparsers) -> ModeChecks: """ - Subparser for mode: rocoto + Add subparser for mode: rocoto :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -426,7 +456,7 @@ def _add_subparser_rocoto(subparsers: Subparsers) -> ModeChecks: def _add_subparser_rocoto_realize(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: rocoto realize + Add subparser for mode: rocoto realize :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -440,7 +470,7 @@ def _add_subparser_rocoto_realize(subparsers: Subparsers) -> ActionChecks: def _add_subparser_rocoto_validate(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: rocoto validate + Add subparser for mode: rocoto validate :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -453,7 +483,7 @@ def _add_subparser_rocoto_validate(subparsers: Subparsers) -> ActionChecks: def _dispatch_rocoto(args: Args) -> bool: """ - Dispatch logic for rocoto mode. + Define dispatch logic for rocoto mode. :param args: Parsed command-line args. """ @@ -466,7 +496,7 @@ def _dispatch_rocoto(args: Args) -> bool: def _dispatch_rocoto_realize(args: Args) -> bool: """ - Dispatch logic for rocoto realize action. Validate input and output. + Define dispatch logic for rocoto realize action. Validate input and output. :param args: Parsed command-line args. """ @@ -479,7 +509,7 @@ def _dispatch_rocoto_realize(args: Args) -> bool: def _dispatch_rocoto_validate(args: Args) -> bool: """ - Dispatch logic for rocoto validate action. + Define dispatch logic for rocoto validate action. :param args: Parsed command-line args. """ @@ -491,7 +521,7 @@ def _dispatch_rocoto_validate(args: Args) -> bool: def _add_subparser_template(subparsers: Subparsers) -> ModeChecks: """ - Subparser for mode: template + Add subparser for mode: template :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -506,7 +536,7 @@ def _add_subparser_template(subparsers: Subparsers) -> ModeChecks: def _add_subparser_template_translate(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: template translate + Add subparser for mode: template translate :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -520,7 +550,7 @@ def _add_subparser_template_translate(subparsers: Subparsers) -> ActionChecks: def _add_subparser_template_render(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: template render + Add subparser for mode: template render :param subparsers: Parent parser's subparsers, to add this subparser to. """ @@ -541,7 +571,7 @@ def _add_subparser_template_render(subparsers: Subparsers) -> ActionChecks: def _dispatch_template(args: Args) -> bool: """ - Dispatch logic for template mode. + Define dispatch logic for template mode. :param args: Parsed command-line args. """ @@ -554,7 +584,7 @@ def _dispatch_template(args: Args) -> bool: def _dispatch_template_render(args: Args) -> bool: """ - Dispatch logic for template render action. + Define dispatch logic for template render action. :param args: Parsed command-line args. """ @@ -581,7 +611,7 @@ def _dispatch_template_render(args: Args) -> bool: def _dispatch_template_translate(args: Args) -> bool: """ - Dispatch logic for template translate action. + Define dispatch logic for template translate action. :param args: Parsed command-line args. """ @@ -800,6 +830,14 @@ def _add_arg_search_path(group: Group) -> None: ) +def _add_arg_show_schema(group: Group) -> None: + group.add_argument( + _switch(STR.showschema), + action="store_true", + help="Show driver schema and exit", + ) + + def _add_arg_target_dir( group: Group, required: bool = False, helpmsg: Optional[str] = None ) -> None: @@ -938,7 +976,7 @@ def _add_subparser_for_driver( with_leadtime: Optional[bool] = False, ) -> ModeChecks: """ - Subparser for a standalone-driver mode. + Add subparser for a standalone-driver mode. :param name: Name of the driver whose subparser to configure. :param subparsers: Parent parser's subparsers, to add this subparser to. @@ -947,8 +985,9 @@ def _add_subparser_for_driver( :param with_leadtime: Does this driver require a leadtime? """ parser = _add_subparser(subparsers, name, "Execute %s tasks" % name) - _basic_setup(parser) - subparsers = _add_subparsers(parser, STR.action, STR.task.upper()) + optional = _basic_setup(parser) + _add_arg_show_schema(optional) + subparsers = _add_subparsers(parser, STR.action, STR.task.upper(), required=False) return { task: _add_subparser_for_driver_task( subparsers, task, helpmsg, with_batch, with_cycle, with_leadtime @@ -966,7 +1005,7 @@ def _add_subparser_for_driver_task( with_leadtime: Optional[bool] = False, ) -> ActionChecks: """ - Subparser for a driver action. + Add subparser for a driver action. :param subparsers: Parent parser's subparsers, to add this subparser to. :param task: The task to add a subparser for. @@ -996,7 +1035,7 @@ def _add_subparser_for_driver_task( return checks -def _add_subparsers(parser: Parser, dest: str, metavar: str) -> Subparsers: +def _add_subparsers(parser: Parser, dest: str, metavar: str, required: bool = True) -> Subparsers: """ Add subparsers to a parser. @@ -1006,7 +1045,7 @@ def _add_subparsers(parser: Parser, dest: str, metavar: str) -> Subparsers: :return: The new subparsers object. """ return parser.add_subparsers( - dest=dest, metavar=metavar, required=True, title="Positional arguments" + dest=dest, metavar=metavar, required=required, title="Positional arguments" ) @@ -1071,12 +1110,18 @@ def _dict_from_key_eq_val_strings(config_items: list[str]) -> dict[str, str]: def _dispatch_to_driver(name: str, args: Args) -> bool: """ - Dispatch logic for a driver mode. + Define dispatch logic for a driver mode. :param name: Name of the driver to dispatch to. :param args: Parsed command-line args. """ - execute: Callable[..., bool] = import_module("uwtools.api.%s" % name).execute + module = import_module("uwtools.api.%s" % name) + if args.get(STR.showschema): + print(json.dumps(module.schema(), sort_keys=True, indent=2)) + return True + if not args.get(STR.action): + _abort("No %s specified" % STR.task.upper()) + execute: Callable[..., bool] = module.execute kwargs = { "task": args[STR.action], "config": args[STR.cfgfile], @@ -1093,7 +1138,7 @@ def _dispatch_to_driver(name: str, args: Args) -> bool: def _formatter(prog: str) -> HelpFormatter: """ - A standard formatter for help messages. + Return a standard formatter for help messages. """ # max_help_positions sets the maximum starting column for option help text. return HelpFormatter(prog, max_help_position=6) @@ -1114,7 +1159,7 @@ def _parse_args(raw_args: list[str]) -> tuple[Args, Checks]: tools = { STR.config: partial(_add_subparser_config, subparsers), STR.execute: partial(_add_subparser_execute, subparsers), - STR.file: partial(_add_subparser_file, subparsers), + STR.fs: partial(_add_subparser_fs, subparsers), STR.rocoto: partial(_add_subparser_rocoto, subparsers), STR.template: partial(_add_subparser_template, subparsers), } diff --git a/src/uwtools/config/atparse_to_jinja2.py b/src/uwtools/config/atparse_to_jinja2.py index 049ee987d..22a387913 100644 --- a/src/uwtools/config/atparse_to_jinja2.py +++ b/src/uwtools/config/atparse_to_jinja2.py @@ -14,7 +14,7 @@ def convert( input_file: Optional[Path] = None, output_file: Optional[Path] = None, dry_run: bool = False ) -> None: """ - Replaces atparse @[] tokens with Jinja2 {{}} equivalents. + Replace atparse @[] tokens with Jinja2 {{}} equivalents. If no input file is given, stdin is used. If no output file is given, stdout is used. In dry-run mode, output is written to stderr. diff --git a/src/uwtools/config/formats/base.py b/src/uwtools/config/formats/base.py index 854449e11..c93209f07 100644 --- a/src/uwtools/config/formats/base.py +++ b/src/uwtools/config/formats/base.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import os import re from abc import ABC, abstractmethod @@ -25,8 +23,6 @@ class Config(ABC, UserDict): def __init__(self, config: Optional[Union[dict, str, Path]] = None) -> None: """ - Construct a Config object. - :param config: Config file to load (None => read from stdin), or initial dict. """ super().__init__() @@ -36,35 +32,86 @@ def __init__(self, config: Optional[Union[dict, str, Path]] = None) -> None: else: self._config_file = str2path(config) if config else None self.data = self._load(self._config_file) - if self.get_depth_threshold() and self.depth != self.get_depth_threshold(): + if self._get_depth_threshold() and self._depth != self._get_depth_threshold(): raise UWConfigError( "Cannot instantiate depth-%s %s with depth-%s config" - % (self.get_depth_threshold(), type(self).__name__, self.depth) + % (self._get_depth_threshold(), type(self).__name__, self._depth) ) def __repr__(self) -> str: """ - Returns the string representation of a Config object. + Return the string representation of a Config object. """ return self._dict_to_str(self.data) # Private methods + def _characterize_values(self, values: dict, parent: str) -> tuple[list, list]: + """ + Characterize values as complete or as template placeholders. + + :param values: The dictionary to examine. + :param parent: Parent key. + :return: Lists of of complete and template-placeholder values. + """ + complete: list[str] = [] + template: list[str] = [] + for key, val in values.items(): + if isinstance(val, dict): + complete.append(f"{INDENT}{parent}{key}") + c, t = self._characterize_values(val, f"{parent}{key}.") + complete, template = complete + c, template + t + elif isinstance(val, list): + for item in val: + if isinstance(item, dict): + c, t = self._characterize_values(item, parent) + complete, template = complete + c, template + t + complete.append(f"{INDENT}{parent}{key}") + elif "{{" in str(val) or "{%" in str(val): + template.append(f"{INDENT}{parent}{key}: {val}") + break + elif "{{" in str(val) or "{%" in str(val): + template.append(f"{INDENT}{parent}{key}: {val}") + else: + complete.append(f"{INDENT}{parent}{key}") + return complete, template + + @property + def _depth(self) -> int: + """ + The depth of this config's hierarchy. + """ + return depth(self.data) + @classmethod @abstractmethod def _dict_to_str(cls, cfg: dict) -> str: """ - Returns the string representation of the given dict. + Return the string representation of the given dict. :param cfg: A dict object. """ + @staticmethod + @abstractmethod + def _get_depth_threshold() -> Optional[int]: + """ + Return the config's depth threshold. + """ + + @staticmethod + @abstractmethod + def _get_format() -> str: + """ + Return the config's format name. + """ + @abstractmethod def _load(self, config_file: Optional[Path]) -> dict: """ - Reads and parses a config file. + Read and parse a config file. - Returns the result of loading and parsing the specified config file, or stdin if no file is + Return the result of loading and parsing the specified config file, or stdin if no file is given. :param config_file: Path to config file to load. @@ -88,37 +135,28 @@ def _load_paths(self, config_files: list[Path]) -> dict: cfg.update(self._load(config_file=config_file)) return cfg - # Public methods - - def characterize_values(self, values: dict, parent: str) -> tuple[list, list]: + def _parse_include(self, ref_dict: Optional[dict] = None) -> None: """ - Characterize values as complete or as template placeholders. + Recursively process include directives in a config object. - :param values: The dictionary to examine. - :param parent: Parent key. - :return: Lists of of complete and template-placeholder values. + Recursively traverse the dictionary, replacing include tags with the contents of the files + they specify. Assumes a section/key/value structure. YAML provides this functionality in its + own loader. + + :param ref_dict: A config object to process instead of the object's own data. """ - complete: list[str] = [] - template: list[str] = [] - for key, val in values.items(): - if isinstance(val, dict): - complete.append(f"{INDENT}{parent}{key}") - c, t = self.characterize_values(val, f"{parent}{key}.") - complete, template = complete + c, template + t - elif isinstance(val, list): - for item in val: - if isinstance(item, dict): - c, t = self.characterize_values(item, parent) - complete, template = complete + c, template + t - complete.append(f"{INDENT}{parent}{key}") - elif "{{" in str(val) or "{%" in str(val): - template.append(f"{INDENT}{parent}{key}: {val}") - break - elif "{{" in str(val) or "{%" in str(val): - template.append(f"{INDENT}{parent}{key}: {val}") - else: - complete.append(f"{INDENT}{parent}{key}") - return complete, template + if ref_dict is None: + ref_dict = self.data + for key, value in deepcopy(ref_dict).items(): + if isinstance(value, dict): + self._parse_include(ref_dict[key]) + elif isinstance(value, str): + if m := re.match(r"^\s*%s\s+(.*)" % INCLUDE_TAG, value): + filepaths = yaml.safe_load(m[1]) + self.update_from(self._load_paths(filepaths)) + del ref_dict[key] + + # Public methods def compare_config(self, dict1: dict, dict2: Optional[dict] = None) -> bool: """ @@ -127,7 +165,7 @@ def compare_config(self, dict1: dict, dict2: Optional[dict] = None) -> bool: Assumes a section/key/value structure. :param dict1: The first dictionary. - :param dict2: The second dictionary. + :param dict2: The second dictionary (default: this config). :return: True if the configs are identical, False otherwise. """ dict2 = self.data if dict2 is None else dict2 @@ -158,13 +196,6 @@ def compare_config(self, dict1: dict, dict2: Optional[dict] = None) -> bool: return not diffs - @property - def depth(self) -> int: - """ - Returns the depth of this config's hierarchy. - """ - return depth(self.data) - def dereference(self, context: Optional[dict] = None) -> None: """ Render as much Jinja2 syntax as possible. @@ -187,59 +218,24 @@ def logstate(state: str) -> None: @abstractmethod def dump(self, path: Optional[Path]) -> None: """ - Dumps the config to stdout or a file. - - :param path: Path to dump config to. - """ + Dump the config to stdout or a file. - @staticmethod - @abstractmethod - def get_depth_threshold() -> Optional[int]: - """ - Returns the config's depth threshold. + :param path: Path to dump config to (default: stdout). """ @staticmethod @abstractmethod def dump_dict(cfg: dict, path: Optional[Path] = None) -> None: """ - Dumps a provided config dictionary to stdout or a file. + Dump a provided config dictionary to stdout or a file. :param cfg: The in-memory config object to dump. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ - @staticmethod - @abstractmethod - def get_format() -> str: - """ - Returns the config's format name. - """ - - def parse_include(self, ref_dict: Optional[dict] = None) -> None: - """ - Recursively process include directives in a config object. - - Recursively traverse the dictionary, replacing include tags with the contents of the files - they specify. Assumes a section/key/value structure. YAML provides this functionality in its - own loader. - - :param ref_dict: A config object to process instead of the object's own data. - """ - if ref_dict is None: - ref_dict = self.data - for key, value in deepcopy(ref_dict).items(): - if isinstance(value, dict): - self.parse_include(ref_dict[key]) - elif isinstance(value, str): - if m := re.match(r"^\s*%s\s+(.*)" % INCLUDE_TAG, value): - filepaths = yaml.safe_load(m[1]) - self.update_values(self._load_paths(filepaths)) - del ref_dict[key] - - def update_values(self, src: Union[dict, Config]) -> None: + def update_from(self, src: Union[dict, UserDict]) -> None: """ - Updates a config. + Update a config. :param src: The dictionary with new data to use. """ @@ -254,4 +250,4 @@ def update(src: dict, dst: dict) -> None: else: dst[key] = val - update(src.data if isinstance(src, Config) else src, self.data) + update(src.data if isinstance(src, UserDict) else src, self.data) diff --git a/src/uwtools/config/formats/fieldtable.py b/src/uwtools/config/formats/fieldtable.py index 45705e77f..3e1ffd184 100644 --- a/src/uwtools/config/formats/fieldtable.py +++ b/src/uwtools/config/formats/fieldtable.py @@ -8,8 +8,7 @@ class FieldTableConfig(YAMLConfig): """ - This class exists to write out a field_table format given that its configuration has been set by - an input YAML file. + Work with configs in field_table format. """ # Private methods @@ -17,7 +16,7 @@ class FieldTableConfig(YAMLConfig): @classmethod def _dict_to_str(cls, cfg: dict) -> str: """ - Returns the field-table representation of the given dict. + Return the field-table representation of the given dict. :param cfg: A dict object. """ @@ -37,22 +36,36 @@ def _dict_to_str(cls, cfg: dict) -> str: lines[-1] += " /" return "\n".join(lines) + @staticmethod + def _get_depth_threshold() -> Optional[int]: + """ + Return the config's depth threshold. + """ + return None + + @staticmethod + def _get_format() -> str: + """ + Return the config's format name. + """ + return FORMAT.fieldtable + # Public methods def dump(self, path: Optional[Path] = None) -> None: """ - Dumps the config in Field Table format. + Dump the config in Field Table format. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ self.dump_dict(self.data, path) @classmethod def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None: """ - Dumps a provided config dictionary in Field Table format. + Dump a provided config dictionary in Field Table format. - FMS field and tracer managers must be registered in an ASCII table called 'field_table'. + FMS field and tracer managers must be registered in an ASCII table called ``field_table``. This table lists field type, target model and methods the querying model will ask for. See UFS documentation for more information: @@ -60,29 +73,17 @@ def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None: The example format for generating a field file is: - sphum: - longname: specific humidity - units: kg/kg - profile_type: - name: fixed - surface_value: 1.e30 + .. code-block:: + + sphum: + longname: specific humidity + units: kg/kg + profile_type: + name: fixed + surface_value: 1.e30 :param cfg: The in-memory config object to dump. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ with writable(path) as f: print(cls._dict_to_str(cfg), file=f) - - @staticmethod - def get_depth_threshold() -> Optional[int]: - """ - Returns the config's depth threshold. - """ - return None - - @staticmethod - def get_format() -> str: - """ - Returns the config's format name. - """ - return FORMAT.fieldtable diff --git a/src/uwtools/config/formats/ini.py b/src/uwtools/config/formats/ini.py index 87ed85a9e..01ac4ef54 100644 --- a/src/uwtools/config/formats/ini.py +++ b/src/uwtools/config/formats/ini.py @@ -11,24 +11,22 @@ class INIConfig(Config): """ - Concrete class to handle INI config files. + Work with INI configs. """ def __init__(self, config: Union[dict, Optional[Path]] = None): """ - Construct an INIConfig object. - :param config: Config file to load (None => read from stdin), or initial dict. """ super().__init__(config) - self.parse_include() + self._parse_include() # Private methods @classmethod def _dict_to_str(cls, cfg: dict) -> str: """ - Returns the INI representation of the given dict. + Return the INI representation of the given dict. :param cfg: A dict object. """ @@ -45,9 +43,23 @@ def _dict_to_str(cls, cfg: dict) -> str: parser.write(sio) return sio.getvalue().strip() + @staticmethod + def _get_depth_threshold() -> Optional[int]: + """ + Return the config's depth threshold. + """ + return 2 + + @staticmethod + def _get_format() -> str: + """ + Return the config's format name. + """ + return FORMAT.ini + def _load(self, config_file: Optional[Path]) -> dict: """ - Reads and parses an INI file. + Read and parse an INI file. See docs for Config._load(). @@ -62,33 +74,19 @@ def _load(self, config_file: Optional[Path]) -> dict: def dump(self, path: Optional[Path] = None) -> None: """ - Dumps the config in INI format. + Dump the config in INI format. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ self.dump_dict(self.data, path) @classmethod def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None: """ - Dumps a provided config dictionary in INI format. + Dump a provided config dictionary in INI format. :param cfg: The in-memory config object to dump. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ with writable(path) as f: print(cls._dict_to_str(cfg), file=f) - - @staticmethod - def get_depth_threshold() -> Optional[int]: - """ - Returns the config's depth threshold. - """ - return 2 - - @staticmethod - def get_format() -> str: - """ - Returns the config's format name. - """ - return FORMAT.ini diff --git a/src/uwtools/config/formats/nml.py b/src/uwtools/config/formats/nml.py index 34fbd24a4..93995d136 100644 --- a/src/uwtools/config/formats/nml.py +++ b/src/uwtools/config/formats/nml.py @@ -14,7 +14,7 @@ class NMLConfig(Config): """ - Concrete class to handle Fortran namelist files. + Work with Fortran namelist configs. """ def __init__(self, config: Union[dict, Optional[Path]] = None) -> None: @@ -24,14 +24,14 @@ def __init__(self, config: Union[dict, Optional[Path]] = None) -> None: :param config: Config file to load (None => read from stdin), or initial dict. """ super().__init__(config) - self.parse_include() + self._parse_include() # Private methods @classmethod def _dict_to_str(cls, cfg: dict) -> str: """ - Returns the field-table representation of the given dict. + Return the field-table representation of the given dict. :param cfg: A dict object. """ @@ -47,13 +47,28 @@ def to_od(d): nml.write(sio, sort=False) return sio.getvalue().strip() + @staticmethod + def _get_depth_threshold() -> Optional[int]: + """ + Return the config's depth threshold. + """ + return None + + @staticmethod + def _get_format() -> str: + """ + Return the config's format name. + """ + return FORMAT.nml + def _load(self, config_file: Optional[Path]) -> dict: """ - Reads and parses a Fortran namelist file. + Read and parse a Fortran namelist file. See docs for Config._load(). :param config_file: Path to config file to load. + :return: The parsed namelist data. """ with readable(config_file) as f: config: dict = f90nml.read(f) @@ -63,33 +78,19 @@ def _load(self, config_file: Optional[Path]) -> dict: def dump(self, path: Optional[Path]) -> None: """ - Dumps the config in Fortran namelist format. + Dump the config in Fortran namelist format. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ self.dump_dict(cfg=self.data, path=path) @classmethod def dump_dict(cls, cfg: Union[dict, Namelist], path: Optional[Path] = None) -> None: """ - Dumps a provided config dictionary in Fortran namelist format. + Dump a provided config dictionary in Fortran namelist format. :param cfg: The in-memory config object to dump. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ with writable(path) as f: print(cls._dict_to_str(cfg), file=f) - - @staticmethod - def get_depth_threshold() -> Optional[int]: - """ - Returns the config's depth threshold. - """ - return None - - @staticmethod - def get_format() -> str: - """ - Returns the config's format name. - """ - return FORMAT.nml diff --git a/src/uwtools/config/formats/sh.py b/src/uwtools/config/formats/sh.py index 0107c6d4d..290b8f4e2 100644 --- a/src/uwtools/config/formats/sh.py +++ b/src/uwtools/config/formats/sh.py @@ -12,24 +12,22 @@ class SHConfig(Config): """ - Concrete class to handle bash config files. + Work with key=value shell configs. """ def __init__(self, config: Union[dict, Optional[Path]] = None): """ - Construct a SHConfig object. - :param config: Config file to load (None => read from stdin), or initial dict. """ super().__init__(config) - self.parse_include() + self._parse_include() # Private methods @classmethod def _dict_to_str(cls, cfg: dict) -> str: """ - Returns the field-table representation of the given dict. + Return the field-table representation of the given dict. :param cfg: A dict object. """ @@ -39,9 +37,23 @@ def _dict_to_str(cls, cfg: dict) -> str: lines.append("%s=%s" % (key, shlex.quote(str(value)))) return "\n".join(lines) + @staticmethod + def _get_depth_threshold() -> Optional[int]: + """ + Return the config's depth threshold. + """ + return 1 + + @staticmethod + def _get_format() -> str: + """ + Return the config's format name. + """ + return FORMAT.sh + def _load(self, config_file: Optional[Path]) -> dict: """ - Reads and parses key=value lines from shell code. + Read and parse key=value lines from shell code. See docs for Config._load(). @@ -63,9 +75,9 @@ def _load(self, config_file: Optional[Path]) -> dict: def dump(self, path: Optional[Path]) -> None: """ - Dumps the config as key=value lines. + Dump the config as key=value lines. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ config_check_depths_dump(config_obj=self, target_format=FORMAT.sh) self.dump_dict(self.data, path) @@ -73,24 +85,10 @@ def dump(self, path: Optional[Path]) -> None: @classmethod def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None: """ - Dumps a provided config dictionary in bash format. + Dump a provided config dictionary in bash format. :param cfg: The in-memory config object to dump. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ with writable(path) as f: print(cls._dict_to_str(cfg), file=f) - - @staticmethod - def get_depth_threshold() -> Optional[int]: - """ - Returns the config's depth threshold. - """ - return 1 - - @staticmethod - def get_format() -> str: - """ - Returns the config's format name. - """ - return FORMAT.sh diff --git a/src/uwtools/config/formats/yaml.py b/src/uwtools/config/formats/yaml.py index faa0a67b9..c23ef6200 100644 --- a/src/uwtools/config/formats/yaml.py +++ b/src/uwtools/config/formats/yaml.py @@ -48,24 +48,47 @@ class YAMLConfig(Config): """ - Concrete class to handle YAML config files. + Work with YAML configs. """ # Private methods + @classmethod + def _add_yaml_representers(cls) -> None: + """ + Add representers to the YAML dumper for custom types. + """ + yaml.add_representer(UWYAMLConvert, UWYAMLConvert.represent) + yaml.add_representer(Namelist, cls._represent_namelist) + yaml.add_representer(OrderedDict, cls._represent_ordereddict) + @classmethod def _dict_to_str(cls, cfg: dict) -> str: """ - Returns the YAML representation of the given dict. + Return the YAML representation of the given dict. :param cfg: The in-memory config object. """ cls._add_yaml_representers() return yaml_to_str(cfg) + @staticmethod + def _get_depth_threshold() -> Optional[int]: + """ + Return the config's depth threshold. + """ + return None + + @staticmethod + def _get_format() -> str: + """ + Return the config's format name. + """ + return FORMAT.yaml + def _load(self, config_file: Optional[Path]) -> dict: """ - Reads and parses a YAML file. + Read and parse a YAML file. See docs for Config._load(). @@ -77,9 +100,10 @@ def _load(self, config_file: Optional[Path]) -> dict: config = yaml.load(f.read(), Loader=loader) if isinstance(config, dict): return config + t = type(config).__name__ raise UWConfigError( - "Parsed a %s value from %s, expected a dict" - % (type(config).__name__, config_file or "stdin") + "Parsed a%s %s value from %s, expected a dict" + % ("n" if t[0] in "aeiou" else "", t, config_file or "stdin") ) except yaml.constructor.ConstructorError as e: if e.problem: @@ -94,9 +118,35 @@ def _load(self, config_file: Optional[Path]) -> dict: msg = str(e) raise log_and_error(msg) from e + @classmethod + def _represent_namelist(cls, dumper: yaml.Dumper, data: Namelist) -> yaml.nodes.MappingNode: + """ + Convert an f90nml Namelist to an OrderedDict, then represent as a YAML mapping. + + :param dumper: The YAML dumper. + :param data: The f90nml Namelist to serialize. + :return: A YAML mapping. + """ + namelist_dict = data.todict() + return dumper.represent_mapping("tag:yaml.org,2002:map", namelist_dict) + + @classmethod + def _represent_ordereddict( + cls, dumper: yaml.Dumper, data: OrderedDict + ) -> yaml.nodes.MappingNode: + """ + Recursrively convert an OrderedDict to a dict, then represent as a YAML mapping. + + :param dumper: The YAML dumper. + :param data: The OrderedDict to serialize. + :return: A YAML mapping. + """ + + return dumper.represent_mapping("tag:yaml.org,2002:map", from_od(data)) + def _yaml_include(self, loader: yaml.Loader, node: yaml.SequenceNode) -> dict: """ - Returns a dictionary with include tags processed. + Return a dictionary with include tags processed. :param loader: The YAML loader. :param node: A YAML node. @@ -107,7 +157,7 @@ def _yaml_include(self, loader: yaml.Loader, node: yaml.SequenceNode) -> dict: @property def _yaml_loader(self) -> type[yaml.SafeLoader]: """ - Set up the loader with the appropriate constructors. + The loader, with appropriate constructors added. """ loader = yaml.SafeLoader loader.add_constructor(INCLUDE_TAG, self._yaml_include) @@ -120,78 +170,27 @@ def _yaml_loader(self) -> type[yaml.SafeLoader]: def dump(self, path: Optional[Path] = None) -> None: """ - Dumps the config in YAML format. + Dump the config in YAML format. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ self.dump_dict(self.data, path) @classmethod def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None: """ - Dumps a provided config dictionary in YAML format. + Dump a provided config dictionary in YAML format. :param cfg: The in-memory config object to dump. - :param path: Path to dump config to. + :param path: Path to dump config to (default: stdout). """ with writable(path) as f: print(cls._dict_to_str(cfg), file=f) - @staticmethod - def get_depth_threshold() -> Optional[int]: - """ - Returns the config's depth threshold. - """ - return None - - @staticmethod - def get_format() -> str: - """ - Returns the config's format name. - """ - return FORMAT.yaml - - # Private methods - - @classmethod - def _add_yaml_representers(cls) -> None: - """ - Add representers to the YAML dumper for custom types. - """ - yaml.add_representer(UWYAMLConvert, UWYAMLConvert.represent) - yaml.add_representer(Namelist, cls._represent_namelist) - yaml.add_representer(OrderedDict, cls._represent_ordereddict) - - @classmethod - def _represent_namelist(cls, dumper: yaml.Dumper, data: Namelist) -> yaml.nodes.MappingNode: - """ - Convert an f90nml Namelist to an OrderedDict, then represent as a YAML mapping. - - :param dumper: The YAML dumper. - :param data: The f90nml Namelist to serialize. - """ - namelist_dict = data.todict() - return dumper.represent_mapping("tag:yaml.org,2002:map", namelist_dict) - - @classmethod - def _represent_ordereddict( - cls, dumper: yaml.Dumper, data: OrderedDict - ) -> yaml.nodes.MappingNode: - """ - Recursrively convert an OrderedDict to a dict, then represent as a YAML mapping. - - :param dumper: The YAML dumper. - :param data: The OrderedDict to serialize. - """ - - return dumper.represent_mapping("tag:yaml.org,2002:map", from_od(data)) - def _write_plain_open_ended(self, *args, **kwargs) -> None: """ - Write YAML without ... - - end-of-stream marker. + Write YAML without the "..." end-of-stream marker. """ self.write_plain_base(*args, **kwargs) self.open_ended = False diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 2c766754f..58683a68a 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -19,7 +19,7 @@ class J2Template: """ - Reads Jinja2 templates from files or strings, and renders them using the user-provided values. + Read Jinja2 templates from files or strings, and render them using the user-provided values. """ def __init__( @@ -90,9 +90,7 @@ def render(self) -> str: @property def undeclared_variables(self) -> set[str]: """ - Returns the names of variables needed to render the template. - - :return: Names of variables needed to render the template. + The names of variables needed to render the template. """ j2_parsed = self._j2env.parse(self._template_str) return meta.find_undeclared_variables(j2_parsed) diff --git a/src/uwtools/config/support.py b/src/uwtools/config/support.py index 632405d89..96b433862 100644 --- a/src/uwtools/config/support.py +++ b/src/uwtools/config/support.py @@ -49,7 +49,7 @@ def format_to_config(fmt: str) -> Type: def from_od(d: Union[OrderedDict, dict]) -> dict: """ - Returns a (nested) dict with content equivalent to the given (nested) OrderedDict. + Return a (nested) dict with content equivalent to the given (nested) OrderedDict. :param d: A (possibly nested) OrderedDict. """ @@ -69,7 +69,7 @@ def log_and_error(msg: str) -> Exception: def yaml_to_str(cfg: dict) -> str: """ - Returns a uwtools-conventional YAML representation of the given dict. + Return a uwtools-conventional YAML representation of the given dict. :param cfg: A dict object. """ diff --git a/src/uwtools/config/tools.py b/src/uwtools/config/tools.py index dfda5b7cb..2d99e651d 100644 --- a/src/uwtools/config/tools.py +++ b/src/uwtools/config/tools.py @@ -149,7 +149,7 @@ def _ensure_format( :raises: UWError if the format cannot be determined. """ if isinstance(config, Config): - return config.get_format() + return config._get_format() # pylint: disable=protected-access if isinstance(config, Path): return fmt or get_file_format(config) if isinstance(config, dict): @@ -161,7 +161,7 @@ def _ensure_format( def _print_config_section(config: dict, key_path: list[str]) -> None: """ - Prints the contents of the located subtree as key=value pairs, one per line. + Print the contents of the located subtree as key=value pairs, one per line. :param config: A config. :param key_path: Path of keys to subsection of config file. @@ -211,7 +211,8 @@ def _realize_config_output_setup( """ output_format = _ensure_format("output", output_format, output_file) log.debug("Writing output to %s" % (output_file or "stdout")) - _validate_format("output", output_format, input_obj.get_format()) + fmt = input_obj._get_format() # pylint: disable=protected-access + _validate_format("output", output_format, fmt) output_data = input_obj.data if key_path is not None: for key in key_path: @@ -235,19 +236,21 @@ def _realize_config_update( """ if update_config or update_format: update_format = _ensure_format("update", update_format, update_config) + fmt = lambda x: x._get_format() # pylint: disable=protected-access + depth_ = lambda x: x._depth # pylint: disable=protected-access if not update_config: log.debug("Reading update from stdin") - _validate_format("update", update_format, input_obj.get_format()) + _validate_format("update", update_format, fmt(input_obj)) update_obj: Config = ( update_config if isinstance(update_config, Config) else format_to_config(update_format)(config=update_config) ) - log.debug("Initial input config depth: %s", input_obj.depth) - log.debug("Update config depth: %s", update_obj.depth) - config_check_depths_update(update_obj, input_obj.get_format()) - input_obj.update_values(update_obj) - log.debug("Final input config depth: %s", input_obj.depth) + log.debug("Initial input config depth: %s", depth_(input_obj)) + log.debug("Update config depth: %s", depth_(update_obj)) + config_check_depths_update(update_obj, fmt(input_obj)) + input_obj.update_from(update_obj) + log.debug("Final input config depth: %s", depth_(input_obj)) return input_obj @@ -257,7 +260,9 @@ def _realize_config_values_needed(input_obj: Config) -> None: :param input_obj: The config to update. """ - complete, template = input_obj.characterize_values(input_obj.data, parent="") + complete, template = input_obj._characterize_values( # pylint: disable=protected-access + input_obj.data, parent="" + ) if complete: log.info("Keys that are complete:") for var in complete: @@ -285,7 +290,8 @@ def _validate_depth( """ target_class = format_to_config(target_format) config = config_obj.data if isinstance(config_obj, Config) else config_obj - if bad_depth(target_class.get_depth_threshold(), depth(config)): + depth_threshold = target_class._get_depth_threshold() # pylint: disable=protected-access + if bad_depth(depth_threshold, depth(config)): raise UWConfigError( "Cannot %s depth-%s config to type-'%s' config" % (action, depth(config), target_format) ) diff --git a/src/uwtools/config/validator.py b/src/uwtools/config/validator.py index 0803598dd..42b723ed6 100644 --- a/src/uwtools/config/validator.py +++ b/src/uwtools/config/validator.py @@ -3,6 +3,7 @@ """ import json +from functools import cache from pathlib import Path from typing import Optional, Union @@ -19,9 +20,30 @@ # Public functions +def bundle(schema: dict) -> dict: + """ + Bundle a schema by dereferencing links to other schemas. + + :param schema: A JSON Schema. + :returns: The bundled schema. + """ + key = "$ref" + bundled = {} + for k, v in schema.items(): + if isinstance(v, dict): + if list(v.keys()) == [key] and v[key].startswith("urn:uwtools:"): + # i.e. the current key's value is of the form: {"$ref": "urn:uwtools:.*"} + bundled[k] = bundle(_registry().get_or_retrieve(v[key]).value.contents) + else: + bundled[k] = bundle(v) + else: + bundled[k] = v + return bundled + + def get_schema_file(schema_name: str) -> Path: """ - Returns the path to the JSON Schema file for a given name. + Return the path to the JSON Schema file for a given name. :param schema_name: Name of uwtools schema to validate the config against. """ @@ -95,13 +117,10 @@ def _prep_config(config: Union[dict, YAMLConfig, Optional[Path]]) -> YAMLConfig: return cfgobj -def _validation_errors(config: Union[dict, list], schema: dict) -> list[ValidationError]: +@cache +def _registry() -> Registry: """ - Identify schema-validation errors. - - :param config: A config to validate. - :param schema: JSON Schema to validate the config against. - :return: Any validation errors. + Return a JSON Schema registry resolving urn:uwtools:* references. """ # See https://github.com/python-jsonschema/referencing/issues/61 about typing issues. @@ -111,6 +130,16 @@ def retrieve(uri: str) -> Resource: with open(resource_path(f"jsonschema/{name}.jsonschema"), "r", encoding="utf-8") as f: return Resource(contents=json.load(f), specification=DRAFT202012) # type: ignore - registry = Registry(retrieve=retrieve) # type: ignore - validator = Draft202012Validator(schema, registry=registry) + return Registry(retrieve=retrieve) # type: ignore + + +def _validation_errors(config: Union[dict, list], schema: dict) -> list[ValidationError]: + """ + Identify schema-validation errors. + + :param config: A config to validate. + :param schema: JSON Schema to validate the config against. + :return: Any validation errors. + """ + validator = Draft202012Validator(schema, registry=_registry()) return list(validator.iter_errors(config)) diff --git a/src/uwtools/drivers/cdeps.py b/src/uwtools/drivers/cdeps.py index 6841c675a..a3a481797 100644 --- a/src/uwtools/drivers/cdeps.py +++ b/src/uwtools/drivers/cdeps.py @@ -24,7 +24,7 @@ class CDEPS(AssetsCycleBased): @tasks def atm(self): """ - Create data atmosphere configuration with all required content. + The data atmosphere configuration with all required content. """ yield self.taskname("data atmosphere configuration") yield [ @@ -35,7 +35,7 @@ def atm(self): @task def atm_nml(self): """ - Create data atmosphere Fortran namelist file (datm_in). + The data atmosphere Fortran namelist file (datm_in). """ fn = "datm_in" yield self.taskname(f"namelist file {fn}") @@ -47,7 +47,7 @@ def atm_nml(self): @task def atm_stream(self): """ - Create data atmosphere stream config file (datm.streams). + The data atmosphere stream config file (datm.streams). """ fn = "datm.streams" yield self.taskname(f"stream file {fn}") @@ -60,7 +60,7 @@ def atm_stream(self): @tasks def ocn(self): """ - Create data ocean configuration with all required content. + The data ocean configuration with all required content. """ yield self.taskname("data atmosphere configuration") yield [ @@ -71,7 +71,7 @@ def ocn(self): @task def ocn_nml(self): """ - Create data ocean Fortran namelist file (docn_in). + The data ocean Fortran namelist file (docn_in). """ fn = "docn_in" yield self.taskname(f"namelist file {fn}") @@ -83,7 +83,7 @@ def ocn_nml(self): @task def ocn_stream(self): """ - Create data ocean stream config file (docn.streams). + The data ocean stream config file (docn.streams). """ fn = "docn.streams" yield self.taskname(f"stream file {fn}") @@ -95,10 +95,10 @@ def ocn_stream(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.cdeps diff --git a/src/uwtools/drivers/chgres_cube.py b/src/uwtools/drivers/chgres_cube.py index cb110bb8e..669860b07 100644 --- a/src/uwtools/drivers/chgres_cube.py +++ b/src/uwtools/drivers/chgres_cube.py @@ -91,10 +91,10 @@ def runscript(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.chgrescube diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index c84abb088..1a60a6d14 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -19,13 +19,19 @@ from uwtools.config.formats.base import Config from uwtools.config.formats.yaml import YAMLConfig from uwtools.config.tools import walk_key_path -from uwtools.config.validator import get_schema_file, validate, validate_external, validate_internal +from uwtools.config.validator import ( + bundle, + get_schema_file, + validate, + validate_external, + validate_internal, +) from uwtools.exceptions import UWConfigError from uwtools.logging import log from uwtools.scheduler import JobScheduler from uwtools.strings import STR from uwtools.utils.file import writable -from uwtools.utils.processing import execute +from uwtools.utils.processing import run_shell_cmd # NB: Class docstrings are programmatically defined. @@ -56,9 +62,9 @@ def __init__( self._config_full: dict = config_input.data self._config_intermediate, _ = walk_key_path(self._config_full, key_path or []) try: - self._config: dict = self._config_intermediate[self.driver_name] + self._config: dict = self._config_intermediate[self.driver_name()] except KeyError as e: - raise UWConfigError("Required '%s' block missing in config" % self.driver_name) from e + raise UWConfigError("Required '%s' block missing in config" % self.driver_name()) from e if controller: self._config[STR.rundir] = self._config_intermediate[controller][STR.rundir] self._validate(schema_file) @@ -74,7 +80,7 @@ def __repr__(self) -> str: return " ".join(filter(None, [str(self), cycle, leadtime, "in", self.config[STR.rundir]])) def __str__(self) -> str: - return self.driver_name + return self.driver_name() @property def config(self) -> dict: @@ -97,9 +103,17 @@ def rundir(self) -> Path: """ return Path(self.config[STR.rundir]).absolute() + @classmethod + def schema(cls) -> dict: + """ + Return the driver's schema. + """ + with open(get_schema_file(schema_name=cls._schema_name()), "r", encoding="utf-8") as f: + return bundle(json.load(f)) + def taskname(self, suffix: str) -> str: """ - Returns a common tag for graph-task log messages. + Return a common tag for graph-task log messages. :param suffix: Log-string suffix. """ @@ -110,7 +124,7 @@ def taskname(self, suffix: str) -> str: if cycle and leadtime is not None else cycle.strftime("%Y%m%d %HZ") if cycle else None ) - return " ".join(filter(None, [timestr, self.driver_name, suffix])) + return " ".join(filter(None, [timestr, self.driver_name(), suffix])) # Workflow tasks @@ -139,7 +153,7 @@ def _create_user_updated_config( user_values = config_values.get(STR.updatevalues, {}) if base_file := config_values.get(STR.basefile): cfgobj = config_class(base_file) - cfgobj.update_values(user_values) + cfgobj.update_from(user_values) cfgobj.dereference() config = cfgobj.data dump = partial(cfgobj.dump, path) @@ -154,11 +168,11 @@ def _create_user_updated_config( # Public helper methods - @property + @classmethod @abstractmethod - def driver_name(self) -> str: + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ # Private helper methods @@ -167,7 +181,7 @@ def _namelist_schema( self, config_keys: Optional[list[str]] = None, schema_keys: Optional[list[str]] = None ) -> dict: """ - Returns the (sub)schema for validating the driver's namelist content. + Return the (sub)schema for validating the driver's namelist content. :param config_keys: Keys leading to the namelist block in the driver config. :param schema_keys: Keys leading to the namelist-validating (sub)schema. @@ -177,12 +191,12 @@ def _namelist_schema( for config_key in config_keys or [STR.namelist]: nmlcfg = nmlcfg[config_key] if nmlcfg.get(STR.validate, True): - schema_file = get_schema_file(schema_name=self.driver_name.replace("_", "-")) + schema_file = get_schema_file(schema_name=self._schema_name()) with open(schema_file, "r", encoding="utf-8") as f: schema = json.load(f) for schema_key in schema_keys or [ STR.properties, - self.driver_name, + self.driver_name(), STR.properties, STR.namelist, STR.properties, @@ -191,6 +205,13 @@ def _namelist_schema( schema = schema[schema_key] return schema + @classmethod + def _schema_name(cls) -> str: + """ + Return the filename stem for this driver's schema file. + """ + return cls.driver_name().replace("_", "-") + def _validate(self, schema_file: Optional[Path] = None) -> None: """ Perform all necessary schema validation. @@ -201,9 +222,7 @@ def _validate(self, schema_file: Optional[Path] = None) -> None: if schema_file: validate_external(schema_file=schema_file, config=self._config_intermediate) else: - validate_internal( - schema_name=self.driver_name.replace("_", "-"), config=self._config_intermediate - ) + validate_internal(schema_name=self._schema_name(), config=self._config_intermediate) class AssetsCycleBased(Assets): @@ -380,14 +399,14 @@ def _run_via_local_execution(self): yield asset(path, path.is_file) yield self.provisioned_rundir() cmd = "{x} >{x}.out 2>&1".format(x=self._runscript_path) - execute(cmd=cmd, cwd=self.rundir, log_output=True) + run_shell_cmd(cmd=cmd, cwd=self.rundir, log_output=True) # Private helper methods @property def _run_resources(self) -> dict[str, Any]: """ - Returns platform configuration data. + The platform configuration data. """ if not (platform := self._config_intermediate.get("platform")): raise UWConfigError("Required 'platform' block missing in config") @@ -404,7 +423,7 @@ def _run_resources(self) -> dict[str, Any]: @property def _runcmd(self) -> str: """ - Returns the full command-line component invocation. + The full command-line component invocation. """ execution = self.config.get(STR.execution, {}) mpiargs = execution.get(STR.mpiargs, []) @@ -423,7 +442,7 @@ def _runscript( scheduler: Optional[JobScheduler] = None, ) -> str: """ - Returns a driver runscript. + Return a driver runscript. :param execution: Statements to execute. :param envcmds: Shell commands to set up runtime environment. @@ -461,14 +480,14 @@ def _runscript_done_file(self): @property def _runscript_path(self) -> Path: """ - Returns the path to the runscript. + The path to the runscript. """ - return self.rundir / f"runscript.{self.driver_name}" + return self.rundir / f"runscript.{self.driver_name()}" @property def _scheduler(self) -> JobScheduler: """ - Returns the job scheduler specified by the platform information. + The job scheduler specified by the platform information. """ return JobScheduler.get_scheduler(self._run_resources) @@ -479,12 +498,7 @@ def _validate(self, schema_file: Optional[Path] = None) -> None: :param schema_file: The JSON Schema file to use for validation. :raises: UWConfigError if config fails validation. """ - if schema_file: - validate_external(schema_file=schema_file, config=self._config_intermediate) - else: - validate_internal( - schema_name=self.driver_name.replace("_", "-"), config=self._config_intermediate - ) + Assets._validate(self, schema_file) validate_internal(schema_name=STR.platform, config=self._config_intermediate) def _write_runscript(self, path: Path, envvars: Optional[dict[str, str]] = None) -> None: diff --git a/src/uwtools/drivers/esg_grid.py b/src/uwtools/drivers/esg_grid.py index 9b1082056..0d77a7da8 100644 --- a/src/uwtools/drivers/esg_grid.py +++ b/src/uwtools/drivers/esg_grid.py @@ -51,10 +51,10 @@ def provisioned_rundir(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.esggrid diff --git a/src/uwtools/drivers/filter_topo.py b/src/uwtools/drivers/filter_topo.py index 3b852a35d..3cb396a5d 100644 --- a/src/uwtools/drivers/filter_topo.py +++ b/src/uwtools/drivers/filter_topo.py @@ -62,10 +62,10 @@ def provisioned_rundir(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.filtertopo diff --git a/src/uwtools/drivers/fv3.py b/src/uwtools/drivers/fv3.py index 9d06e4de5..e417f58d1 100644 --- a/src/uwtools/drivers/fv3.py +++ b/src/uwtools/drivers/fv3.py @@ -178,10 +178,10 @@ def runscript(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.fv3 diff --git a/src/uwtools/drivers/global_equiv_resol.py b/src/uwtools/drivers/global_equiv_resol.py index 58264fdb5..25b3ef8a6 100644 --- a/src/uwtools/drivers/global_equiv_resol.py +++ b/src/uwtools/drivers/global_equiv_resol.py @@ -40,10 +40,10 @@ def provisioned_rundir(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.globalequivresol @@ -52,7 +52,7 @@ def driver_name(self) -> str: @property def _runcmd(self): """ - Returns the full command-line component invocation. + The full command-line component invocation. """ executable = self.config[STR.execution][STR.executable] input_file_path = self.config["input_grid_file"] diff --git a/src/uwtools/drivers/ioda.py b/src/uwtools/drivers/ioda.py index 1a740506b..533b7faa6 100644 --- a/src/uwtools/drivers/ioda.py +++ b/src/uwtools/drivers/ioda.py @@ -31,10 +31,10 @@ def provisioned_rundir(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.ioda @@ -43,14 +43,14 @@ def driver_name(self) -> str: @property def _config_fn(self) -> str: """ - Returns the name of the config file used in execution. + The name of the config file used in execution. """ return "ioda.yaml" @property def _runcmd(self) -> str: """ - Returns the full command-line component invocation. + The full command-line component invocation. """ executable = self.config[STR.execution][STR.executable] jedi_config = str(self.rundir / self._config_fn) diff --git a/src/uwtools/drivers/jedi.py b/src/uwtools/drivers/jedi.py index 12acbcf0e..a2467cdef 100644 --- a/src/uwtools/drivers/jedi.py +++ b/src/uwtools/drivers/jedi.py @@ -56,10 +56,10 @@ def validate_only(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.jedi @@ -68,14 +68,14 @@ def driver_name(self) -> str: @property def _config_fn(self) -> str: """ - Returns the name of the config file used in execution. + The name of the config file used in execution. """ return "jedi.yaml" @property def _runcmd(self) -> str: """ - Returns the full command-line component invocation. + The full command-line component invocation. """ execution = self.config[STR.execution] jedi_config = self.rundir / self._config_fn diff --git a/src/uwtools/drivers/jedi_base.py b/src/uwtools/drivers/jedi_base.py index f517a4ce5..1d4e1309a 100644 --- a/src/uwtools/drivers/jedi_base.py +++ b/src/uwtools/drivers/jedi_base.py @@ -72,5 +72,5 @@ def provisioned_rundir(self): @abstractmethod def _config_fn(self) -> str: """ - Returns the name of the config file used in execution. + The name of the config file used in execution. """ diff --git a/src/uwtools/drivers/make_hgrid.py b/src/uwtools/drivers/make_hgrid.py index 78f0f6baa..28f275953 100644 --- a/src/uwtools/drivers/make_hgrid.py +++ b/src/uwtools/drivers/make_hgrid.py @@ -26,10 +26,10 @@ def provisioned_rundir(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.makehgrid @@ -38,7 +38,7 @@ def driver_name(self) -> str: @property def _runcmd(self): """ - Returns the full command-line component invocation. + The full command-line component invocation. """ executable = self.config[STR.execution][STR.executable] config = self.config["config"] diff --git a/src/uwtools/drivers/make_solo_mosaic.py b/src/uwtools/drivers/make_solo_mosaic.py index 5b954333b..d9598edac 100644 --- a/src/uwtools/drivers/make_solo_mosaic.py +++ b/src/uwtools/drivers/make_solo_mosaic.py @@ -28,18 +28,18 @@ def provisioned_rundir(self): def taskname(self, suffix: str) -> str: """ - Returns a common tag for graph-task log messages. + Return a common tag for graph-task log messages. :param suffix: Log-string suffix. """ - return "%s %s" % (self.driver_name, suffix) + return "%s %s" % (self.driver_name(), suffix) # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.makesolomosaic @@ -48,7 +48,7 @@ def driver_name(self) -> str: @property def _runcmd(self): """ - Returns the full command-line component invocation. + The full command-line component invocation. """ executable = self.config[STR.execution][STR.executable] flags = " ".join(f"--{k} {v}" for k, v in self.config["config"].items()) diff --git a/src/uwtools/drivers/mpas.py b/src/uwtools/drivers/mpas.py index 2b250541a..c1953f822 100644 --- a/src/uwtools/drivers/mpas.py +++ b/src/uwtools/drivers/mpas.py @@ -68,10 +68,10 @@ def namelist_file(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.mpas diff --git a/src/uwtools/drivers/mpas_init.py b/src/uwtools/drivers/mpas_init.py index 547e88de3..47d954fbd 100644 --- a/src/uwtools/drivers/mpas_init.py +++ b/src/uwtools/drivers/mpas_init.py @@ -70,10 +70,10 @@ def namelist_file(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.mpasinit diff --git a/src/uwtools/drivers/orog_gsl.py b/src/uwtools/drivers/orog_gsl.py index 46d53beac..bfde683e7 100644 --- a/src/uwtools/drivers/orog_gsl.py +++ b/src/uwtools/drivers/orog_gsl.py @@ -72,10 +72,10 @@ def topo_data_30s(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.oroggsl @@ -84,7 +84,7 @@ def driver_name(self) -> str: @property def _runcmd(self): """ - Returns the full command-line component invocation. + The full command-line component invocation. """ inputs = [str(self.config["config"][k]) for k in ("tile", "resolution", "halo")] executable = self.config[STR.execution][STR.executable] diff --git a/src/uwtools/drivers/schism.py b/src/uwtools/drivers/schism.py index 76e71525e..1f6fa30ef 100644 --- a/src/uwtools/drivers/schism.py +++ b/src/uwtools/drivers/schism.py @@ -47,10 +47,10 @@ def provisioned_rundir(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.schism diff --git a/src/uwtools/drivers/sfc_climo_gen.py b/src/uwtools/drivers/sfc_climo_gen.py index 3d1ce53d4..035ccc07f 100644 --- a/src/uwtools/drivers/sfc_climo_gen.py +++ b/src/uwtools/drivers/sfc_climo_gen.py @@ -54,10 +54,10 @@ def provisioned_rundir(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.sfcclimogen diff --git a/src/uwtools/drivers/shave.py b/src/uwtools/drivers/shave.py index 018d39813..868d38982 100644 --- a/src/uwtools/drivers/shave.py +++ b/src/uwtools/drivers/shave.py @@ -26,10 +26,10 @@ def provisioned_rundir(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.shave @@ -38,7 +38,7 @@ def driver_name(self) -> str: @property def _runcmd(self): """ - Returns the full command-line component invocation. + The full command-line component invocation. """ executable = self.config[STR.execution][STR.executable] config = self.config["config"] diff --git a/src/uwtools/drivers/support.py b/src/uwtools/drivers/support.py index e01707835..012a6887b 100644 --- a/src/uwtools/drivers/support.py +++ b/src/uwtools/drivers/support.py @@ -12,14 +12,14 @@ def graph() -> str: """ - Returns Graphviz DOT code for the most recently executed task. + Return Graphviz DOT code for the most recently executed task. """ return _iotaa.graph() def set_driver_docstring(driver_class: Type) -> None: """ - Appends inherited parameter descriptions to the driver's own docstring. + Append inherited parameter descriptions to the driver's own docstring. :param driver_class: The class whose docstring to update. """ @@ -36,7 +36,7 @@ def set_driver_docstring(driver_class: Type) -> None: def tasks(driver_class: DriverT) -> dict[str, str]: """ - Returns a mapping from task names to their one-line descriptions. + Return a mapping from task names to their one-line descriptions. :param driver_class: Class of driver object to instantiate. """ diff --git a/src/uwtools/drivers/ungrib.py b/src/uwtools/drivers/ungrib.py index db3b7ab4c..22715fd22 100644 --- a/src/uwtools/drivers/ungrib.py +++ b/src/uwtools/drivers/ungrib.py @@ -104,10 +104,10 @@ def vtable(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.ungrib @@ -130,7 +130,9 @@ def _gribfile(self, infile: Path, link: Path): def _ext(n): """ - Maps integers to 3-letter string. + Return a 3-letter representation of the given integer. + + :param n: The integer to convert to a string representation. """ b = 26 return "{:A>3}".format(("" if n < b else _ext(n // b)) + chr(65 + n % b))[-3:] diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index d6fa799b4..eed851355 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -74,10 +74,10 @@ def provisioned_rundir(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.upp @@ -86,14 +86,14 @@ def driver_name(self) -> str: @property def _namelist_path(self) -> Path: """ - Path to the namelist file. + The path to the namelist file. """ return self.rundir / "itag" @property def _runcmd(self) -> str: """ - Returns the full command-line component invocation. + The full command-line component invocation. """ execution = self.config.get(STR.execution, {}) mpiargs = execution.get(STR.mpiargs, []) diff --git a/src/uwtools/drivers/ww3.py b/src/uwtools/drivers/ww3.py index 33b9af79c..a177eb438 100644 --- a/src/uwtools/drivers/ww3.py +++ b/src/uwtools/drivers/ww3.py @@ -61,10 +61,10 @@ def restart_directory(self): # Public helper methods - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: """ - Returns the name of this driver. + The name of this driver. """ return STR.ww3 diff --git a/src/uwtools/file.py b/src/uwtools/fs.py similarity index 52% rename from src/uwtools/file.py rename to src/uwtools/fs.py index 5b8162d8b..c5bd29b68 100644 --- a/src/uwtools/file.py +++ b/src/uwtools/fs.py @@ -1,9 +1,9 @@ """ -File handling. +File and directory staging. """ import datetime as dt -from functools import cached_property +from abc import ABC, abstractmethod from pathlib import Path from typing import Optional, Union @@ -13,13 +13,14 @@ from uwtools.config.validator import validate_internal from uwtools.exceptions import UWConfigError from uwtools.logging import log +from uwtools.strings import STR from uwtools.utils.api import str2path -from uwtools.utils.tasks import filecopy, symlink +from uwtools.utils.tasks import directory, filecopy, symlink -class Stager: +class Stager(ABC): """ - The base class for staging files. + The base class for staging files and directories. """ def __init__( @@ -32,7 +33,7 @@ def __init__( dry_run: bool = False, ) -> None: """ - Handle files. + Stage files and directories. :param config: YAML-file path, or dict (read stdin if missing or None). :param target_dir: Path to target directory. @@ -43,39 +44,41 @@ def __init__( :raises: UWConfigError if config fails validation. """ dryrun(enable=dry_run) - self._target_dir = str2path(target_dir) - self._config = YAMLConfig(config=str2path(config)) self._keys = keys or [] - self._config.dereference( + self._target_dir = str2path(target_dir) + yaml_config = YAMLConfig(config=str2path(config)) + yaml_config.dereference( context={ **({"cycle": cycle} if cycle else {}), **({"leadtime": leadtime} if leadtime else {}), - **self._config.data, + **yaml_config.data, } ) + self._config = yaml_config.data + self._set_config_block() self._validate() + self._check_paths() - def _check_dst_paths(self, cfg: dict[str, str]) -> None: + def _check_paths(self) -> None: """ - Check that all destination paths are absolute if no target directory is specified. + Check that all paths are absolute if no target directory is specified. - :parm cfg: The dst/linkname -> src/target map. + :parm paths: The paths to check. :raises: UWConfigError if no target directory is specified and a relative path is. """ if not self._target_dir: errmsg = "Relative path '%s' requires the target directory to be specified" - for dst in cfg.keys(): + for dst in self._dst_paths: if not Path(dst).is_absolute(): raise UWConfigError(errmsg % dst) - @cached_property - def _file_map(self) -> dict: + def _set_config_block(self) -> None: """ - Navigate keys to file dst/src config block. + Navigate keys to a config block. - :return: The dst/src file block from a potentially larger config. + :raises: UWConfigError if no target directory is specified and a relative path is. """ - cfg = self._config.data + cfg = self._config nav = [] for key in self._keys: nav.append(key) @@ -84,22 +87,54 @@ def _file_map(self) -> dict: log.debug("Following config key '%s'", key) cfg = cfg[key] if not isinstance(cfg, dict): - raise UWConfigError("No file map found at key path: %s" % " -> ".join(self._keys)) - self._check_dst_paths(cfg) - return cfg + msg = "Expected block not found at key path: %s" % " -> ".join(self._keys) + raise UWConfigError(msg) + self._config = cfg + + @property + @abstractmethod + def _dst_paths(self) -> list[str]: + """ + The paths to files or directories to create. + """ + + @property + @abstractmethod + def _schema(self) -> str: + """ + The name of the schema to use for config validation. + """ - def _validate(self) -> bool: + def _validate(self) -> None: """ Validate config against its schema. - :return: True if config passes validation. :raises: UWConfigError if config fails validation. """ - validate_internal(schema_name="files-to-stage", config=self._file_map) - return True + validate_internal(schema_name=self._schema, config=self._config) -class Copier(Stager): +class FileStager(Stager): + """ + Stage files. + """ + + @property + def _dst_paths(self) -> list[str]: + """ + The paths to files to create. + """ + return list(self._config.keys()) + + @property + def _schema(self) -> str: + """ + The name of the schema to use for config validation. + """ + return "files-to-stage" + + +class Copier(FileStager): """ Stage files by copying. """ @@ -111,10 +146,10 @@ def go(self): """ dst = lambda k: Path(self._target_dir / k if self._target_dir else k) yield "File copies" - yield [filecopy(src=Path(v), dst=dst(k)) for k, v in self._file_map.items()] + yield [filecopy(src=Path(v), dst=dst(k)) for k, v in self._config.items()] -class Linker(Stager): +class Linker(FileStager): """ Stage files by linking. """ @@ -126,4 +161,36 @@ def go(self): """ linkname = lambda k: Path(self._target_dir / k if self._target_dir else k) yield "File links" - yield [symlink(target=Path(v), linkname=linkname(k)) for k, v in self._file_map.items()] + yield [symlink(target=Path(v), linkname=linkname(k)) for k, v in self._config.items()] + + +class MakeDirs(Stager): + """ + Make directories. + """ + + @tasks + def go(self): + """ + Make directories. + """ + yield "Directories" + yield [ + directory(path=Path(self._target_dir / p if self._target_dir else p)) + for p in self._config[STR.makedirs] + ] + + @property + def _dst_paths(self) -> list[str]: + """ + The paths to directories to create. + """ + paths: list[str] = self._config[STR.makedirs] + return paths + + @property + def _schema(self) -> str: + """ + The name of the schema to use for config validation. + """ + return "makedirs" diff --git a/src/uwtools/resources/info.json b/src/uwtools/resources/info.json index 3fc5690e5..ecb08defa 100644 --- a/src/uwtools/resources/info.json +++ b/src/uwtools/resources/info.json @@ -1,4 +1,4 @@ { - "version": "2.4.1", + "version": "2.4.2", "buildnum": "0" } diff --git a/src/uwtools/resources/jsonschema/makedirs.jsonschema b/src/uwtools/resources/jsonschema/makedirs.jsonschema new file mode 100644 index 000000000..0fee1c309 --- /dev/null +++ b/src/uwtools/resources/jsonschema/makedirs.jsonschema @@ -0,0 +1,16 @@ +{ + "additionalProperties": false, + "properties": { + "makedirs": { + "items": { + "type": "string" + }, + "minItems": 1, + "type": "array" + } + }, + "required": [ + "makedirs" + ], + "type": "object" +} diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index a04cda373..4bfb6fb3b 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -218,6 +218,8 @@ def _add_task_dependency_child(self, e: _Element, config: dict, tag: str) -> Non self._add_task_dependency_datadep(e, config) elif tag == STR.taskdep: self._add_task_dependency_taskdep(e, config) + elif tag == STR.metataskdep: + self._add_task_dependency_metataskdep(e, config) elif tag == STR.taskvalid: self._add_task_dependency_taskvalid(e, config) elif tag == STR.timedep: @@ -235,6 +237,15 @@ def _add_task_dependency_datadep(self, e: _Element, config: dict) -> None: e = self._add_compound_time_string(e, config[STR.value], STR.datadep) self._set_attrs(e, config) + def _add_task_dependency_metataskdep(self, e: _Element, config: dict) -> None: + """ + Add a element to the . + + :param e: The parent element to add the new element to. + :param config: Configuration data for this element. + """ + self._set_attrs(SubElement(e, STR.metataskdep), config) + def _add_task_dependency_sh( self, e: _Element, config: dict, name_attr: Optional[str] = None ) -> None: @@ -353,7 +364,7 @@ def _config_validate(self, config: Union[dict, YAMLConfig, Optional[Path]]) -> N @property def _doctype(self) -> Optional[str]: """ - Generate the block with definitions. + The block with definitions. :return: The block if entities are defined, otherwise None. """ @@ -437,6 +448,7 @@ class STR: log: str = "log" memory: str = "memory" metatask: str = "metatask" + metataskdep: str = "metataskdep" name: str = "name" nand: str = "nand" native: str = "native" diff --git a/src/uwtools/scheduler.py b/src/uwtools/scheduler.py index ce4cc0755..7861387fd 100644 --- a/src/uwtools/scheduler.py +++ b/src/uwtools/scheduler.py @@ -14,7 +14,7 @@ from uwtools.exceptions import UWConfigError from uwtools.logging import log from uwtools.strings import STR -from uwtools.utils.processing import execute +from uwtools.utils.processing import run_shell_cmd class JobScheduler(ABC): @@ -32,7 +32,7 @@ def __init__(self, props: dict[str, Any]): @property def directives(self) -> list[str]: """ - Returns resource-request scheduler directives. + The resource-request scheduler directives. """ pre, sep = self._prefix, self._directive_separator ds = [] @@ -53,7 +53,7 @@ def directives(self) -> list[str]: @staticmethod def get_scheduler(props: Mapping) -> JobScheduler: """ - Returns a configured job scheduler. + Return a configured job scheduler. :param props: Configuration settings for job scheduler. :return: A configured job scheduler. @@ -71,7 +71,7 @@ def get_scheduler(props: Mapping) -> JobScheduler: def submit_job(self, runscript: Path, submit_file: Optional[Path] = None) -> bool: """ - Submits a job to the scheduler. + Submit a job to the scheduler. :param runscript: Path to the runscript. :param submit_file: Path to file to write output of submit command to. @@ -80,7 +80,7 @@ def submit_job(self, runscript: Path, submit_file: Optional[Path] = None) -> boo cmd = f"{self._submit_cmd} {runscript}" if submit_file: cmd += " 2>&1 | tee %s" % submit_file - success, _ = execute(cmd=cmd, cwd=f"{runscript.parent}") + success, _ = run_shell_cmd(cmd=cmd, cwd=f"{runscript.parent}") return success # Private methods @@ -89,34 +89,34 @@ def submit_job(self, runscript: Path, submit_file: Optional[Path] = None) -> boo @abstractmethod def _directive_separator(self) -> str: """ - Returns the character used to separate directive keys and values. + The character used to separate directive keys and values. """ @property @abstractmethod def _forbidden_directives(self) -> list[str]: """ - Returns directives that this scheduler does not support. + The directives that this scheduler does not support. """ @property @abstractmethod def _managed_directives(self) -> dict[str, Any]: """ - Returns a mapping from canonical names to scheduler-specific CLI switches. + A mapping from canonical names to scheduler-specific CLI switches. """ @property @abstractmethod def _prefix(self) -> str: """ - Returns the scheduler's resource-request prefix. + The scheduler's resource-request prefix. """ @property def _processed_props(self) -> dict[str, Any]: """ - Pre-process directives before converting to runscript. + Pre-processed runscript directives. """ return self._props @@ -124,7 +124,7 @@ def _processed_props(self) -> dict[str, Any]: @abstractmethod def _submit_cmd(self) -> str: """ - Returns the scheduler's job-submit executable name. + The scheduler's job-submit executable name. """ def _validate_props(self) -> None: @@ -149,21 +149,21 @@ class LSF(JobScheduler): @property def _directive_separator(self) -> str: """ - Returns the character used to separate directive keys and values. + The character used to separate directive keys and values. """ return " " @property def _forbidden_directives(self) -> list[str]: """ - Returns directives that this scheduler does not support. + Directives that this scheduler does not support. """ return [] @property def _managed_directives(self) -> dict[str, Any]: """ - Returns a mapping from canonical names to scheduler-specific CLI switches. + A mapping from canonical names to scheduler-specific CLI switches. """ return { _DirectivesOptional.JOB_NAME: "-J", @@ -181,12 +181,15 @@ def _managed_directives(self) -> dict[str, Any]: @property def _prefix(self) -> str: """ - Returns the scheduler's resource-request prefix. + The scheduler's resource-request prefix. """ return "#BSUB" @property def _processed_props(self) -> dict[str, Any]: + """ + Pre-processed runscript directives. + """ props = deepcopy(self._props) props[_DirectivesOptional.THREADS] = props.get(_DirectivesOptional.THREADS, 1) return props @@ -194,7 +197,7 @@ def _processed_props(self) -> dict[str, Any]: @property def _submit_cmd(self) -> str: """ - Returns the scheduler's job-submit executable name. + The scheduler's job-submit executable name. """ return "bsub" @@ -207,21 +210,21 @@ class PBS(JobScheduler): @property def _directive_separator(self) -> str: """ - Returns the character used to separate directive keys and values. + The character used to separate directive keys and values. """ return " " @property def _forbidden_directives(self) -> list[str]: """ - Returns directives that this scheduler does not support. + Directives that this scheduler does not support. """ return [] @property def _managed_directives(self) -> dict[str, Any]: """ - Returns a mapping from canonical names to scheduler-specific CLI switches. + A mapping from canonical names to scheduler-specific CLI switches. """ return { _DirectivesOptional.DEBUG: lambda x: f"-l debug={str(x).lower()}", @@ -240,7 +243,7 @@ def _managed_directives(self) -> dict[str, Any]: @staticmethod def _placement(items: dict[str, Any]) -> dict[str, Any]: """ - Placement logic. + Return provided items with scheduler-specific replacements. """ exclusive = items.get(_DirectivesOptional.EXCLUSIVE) placement = items.get(_DirectivesOptional.PLACEMENT) @@ -258,12 +261,15 @@ def _placement(items: dict[str, Any]) -> dict[str, Any]: @property def _prefix(self) -> str: """ - Returns the scheduler's resource-request prefix. + The scheduler's resource-request prefix. """ return "#PBS" @property def _processed_props(self) -> dict[str, Any]: + """ + Pre-processed runscript directives. + """ props = self._props props.update(self._select(props)) props.update(self._placement(props)) @@ -278,7 +284,7 @@ def _processed_props(self) -> dict[str, Any]: def _select(self, items: dict[str, Any]) -> dict[str, Any]: """ - Select logic. + Return provided items with scheduler-specific selections. """ select = [] if nodes := items.get(_DirectivesOptional.NODES): @@ -299,7 +305,7 @@ def _select(self, items: dict[str, Any]) -> dict[str, Any]: @property def _submit_cmd(self) -> str: """ - Returns the scheduler's job-submit executable name. + The scheduler's job-submit executable name. """ return "qsub" @@ -312,14 +318,14 @@ class Slurm(JobScheduler): @property def _forbidden_directives(self) -> list[str]: """ - Returns directives that this scheduler does not support. + Directives that this scheduler does not support. """ return [_DirectivesOptional.SHELL] @property def _managed_directives(self) -> dict[str, Any]: """ - Returns a mapping from canonical names to scheduler-specific CLI switches. + A mapping from canonical names to scheduler-specific CLI switches. """ return { _DirectivesOptional.CORES: "--ntasks", @@ -343,21 +349,21 @@ def _managed_directives(self) -> dict[str, Any]: @property def _directive_separator(self) -> str: """ - Returns the character used to separate directive keys and values. + The character used to separate directive keys and values. """ return "=" @property def _prefix(self) -> str: """ - Returns the scheduler's resource-request prefix. + The scheduler's resource-request prefix. """ return "#SBATCH" @property def _submit_cmd(self) -> str: """ - Returns the scheduler's job-submit executable name. + The scheduler's job-submit executable name. """ return "sbatch" diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 3319eb592..098724d62 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -38,14 +38,14 @@ class FORMAT: @staticmethod def extensions() -> list[str]: """ - Returns recognized filename extensions. + Return recognized filename extensions. """ return [FORMAT.ini, FORMAT.nml, FORMAT.sh, FORMAT.yaml] @staticmethod def formats() -> dict[str, str]: """ - Returns the recognized format names. + Return the recognized format names. """ return { field.name: str(getattr(FORMAT, field.name)) @@ -86,6 +86,7 @@ class STR: file2path: str = "file_2_path" file: str = "file" filtertopo: str = "filter_topo" + fs: str = "fs" fv3: str = "fv3" globalequivresol: str = "global_equiv_resol" graphfile: str = "graph_file" @@ -99,6 +100,7 @@ class STR: keyvalpairs: str = "key_eq_val_pairs" leadtime: str = "leadtime" link: str = "link" + makedirs: str = "makedirs" makehgrid: str = "make_hgrid" makesolomosaic: str = "make_solo_mosaic" mode: str = "mode" @@ -126,6 +128,7 @@ class STR: searchpath: str = "search_path" sfcclimogen: str = "sfc_climo_gen" shave: str = "shave" + showschema: str = "show_schema" stdout: str = "stdout" targetdir: str = "target_dir" task: str = "task" diff --git a/src/uwtools/tests/api/test_config.py b/src/uwtools/tests/api/test_config.py index c2cd84e64..803f35747 100644 --- a/src/uwtools/tests/api/test_config.py +++ b/src/uwtools/tests/api/test_config.py @@ -5,11 +5,11 @@ from unittest.mock import patch import yaml -from pytest import mark +from pytest import mark, raises from uwtools.api import config from uwtools.config.formats.yaml import YAMLConfig -from uwtools.exceptions import UWConfigError +from uwtools.exceptions import UWConfigError, UWError from uwtools.utils.file import FORMAT @@ -34,11 +34,11 @@ def test_compare(): @mark.parametrize( "classname,f", [ - ("_FieldTableConfig", config.get_fieldtable_config), - ("_INIConfig", config.get_ini_config), - ("_NMLConfig", config.get_nml_config), - ("_SHConfig", config.get_sh_config), - ("_YAMLConfig", config.get_yaml_config), + ("FieldTableConfig", config.get_fieldtable_config), + ("INIConfig", config.get_ini_config), + ("NMLConfig", config.get_nml_config), + ("SHConfig", config.get_sh_config), + ("YAMLConfig", config.get_yaml_config), ], ) def test_get_config(classname, f): @@ -92,6 +92,31 @@ def test_realize_to_dict(): ) +def test_realize_update_config_from_stdin(): + with raises(UWError) as e: + config.realize(input_config={}, output_file="output.yaml", update_format="yaml") + assert str(e.value) == "Set stdin_ok=True to permit read from stdin" + + +def test_realize_update_config_none(): + input_config = {"n": 88} + output_file = Path("output.yaml") + with patch.object(config, "_realize") as _realize: + config.realize(input_config=input_config, output_file=output_file) + _realize.assert_called_once_with( + input_config=input_config, + input_format=None, + update_config=None, + update_format=None, + output_file=output_file, + output_format=None, + key_path=None, + values_needed=False, + total=False, + dry_run=False, + ) + + @mark.parametrize("cfg", [{"foo": "bar"}, YAMLConfig(config={})]) def test_validate(cfg): kwargs: dict = {"schema_file": "schema-file", "config": cfg} diff --git a/src/uwtools/tests/api/test_driver.py b/src/uwtools/tests/api/test_driver.py index 1b49a9737..f1dc5a7c8 100644 --- a/src/uwtools/tests/api/test_driver.py +++ b/src/uwtools/tests/api/test_driver.py @@ -1,47 +1,9 @@ -# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +# pylint: disable=missing-function-docstring -import datetime as dt -import logging -import os -import sys -from pathlib import Path -from types import SimpleNamespace as ns -from unittest.mock import patch - -from pytest import fixture, mark, raises +from pytest import mark from uwtools.api import driver as driver_api from uwtools.drivers import driver as driver_lib -from uwtools.exceptions import UWError -from uwtools.logging import log -from uwtools.tests.support import fixture_path, logged, regex_logged - -# Fixtures - - -@fixture -def args(): - return ns( - classname="TestDriver", - config=fixture_path("testdriver.yaml"), - module=fixture_path("testdriver.py"), - schema_file=fixture_path("testdriver.jsonschema"), - task="eighty_eight", - ) - - -@fixture -def kwargs(args): - return dict( - classname=args.classname, - config=args.config, - module=args.module, - schema_file=args.schema_file, - task=args.task, - ) - - -# Tests @mark.parametrize( @@ -59,154 +21,3 @@ def kwargs(args): ) def test_driver(classname): assert getattr(driver_api, classname) is getattr(driver_lib, classname) - - -def test__get_driver_module_implicit(): - pass - - -@mark.parametrize("key,val", [("batch", True), ("leadtime", 6)]) -def test_execute_fail_bad_args(caplog, key, kwargs, val): - kwargs.update({"cycle": dt.datetime.now(), key: val}) - assert driver_api.execute(**kwargs) is False - assert logged(caplog, f"TestDriver does not accept argument '{key}'") - - -def test_execute_fail_stdin_not_ok(kwargs): - kwargs["config"] = None - kwargs["cycle"] = dt.datetime.now() - kwargs["stdin_ok"] = False - with raises(UWError) as e: - driver_api.execute(**kwargs) - assert str(e.value) == "Set stdin_ok=True to permit read from stdin" - - -@mark.parametrize("remove", ([], ["schema_file"])) -def test_execute_pass(caplog, kwargs, remove, tmp_path): - for kwarg in remove: - del kwargs[kwarg] - kwargs["cycle"] = dt.datetime.now() - log.setLevel(logging.DEBUG) - graph_file = tmp_path / "g.dot" - graph_code = "DOT code" - kwargs["graph_file"] = graph_file - with patch.object(driver_api, "graph", return_value=graph_code): - assert driver_api.execute(**kwargs) is True - assert regex_logged(caplog, "Instantiated %s with" % kwargs["classname"]) - with open(graph_file, "r", encoding="utf-8") as f: - assert f.read().strip() == graph_code - - -def test_execute_fail_cannot_load_driver_class(kwargs): - kwargs["module"] = "bad_module_name" - assert driver_api.execute(**kwargs) is False - - -def test_tasks_fail(args, caplog, tmp_path): - module = tmp_path / "not.py" - tasks = driver_api.tasks(classname=args.classname, module=module) - assert tasks == {} - assert logged( - caplog, "Could not get tasks from class %s in module %s" % (args.classname, module) - ) - - -def test_tasks_fail_no_cycle(args, caplog, kwargs): - log.setLevel(logging.DEBUG) - assert driver_api.execute(**kwargs) is False - assert logged(caplog, "%s requires argument '%s'" % (args.classname, "cycle")) - - -@mark.parametrize("f", [Path, str]) -def test_tasks_pass(args, f): - tasks = driver_api.tasks(classname=args.classname, module=f(args.module)) - assert tasks["eighty_eight"] == "88" - - -def test__get_driver_class_explicit_fail_bad_class(caplog, args): - log.setLevel(logging.DEBUG) - bad_class = "BadClass" - c, module_path = driver_api._get_driver_class(classname=bad_class, module=args.module) - assert c is None - assert module_path == args.module - assert logged(caplog, "Module %s has no class %s" % (args.module, bad_class)) - - -def test__get_driver_class_explicit_fail_bad_name(caplog, args): - log.setLevel(logging.DEBUG) - bad_name = Path("bad_name") - c, module_path = driver_api._get_driver_class(classname=args.classname, module=bad_name) - assert c is None - assert module_path is None - assert logged(caplog, "Could not load module %s" % bad_name) - - -def test__get_driver_class_explicit_fail_bad_path(caplog, args, tmp_path): - log.setLevel(logging.DEBUG) - module = tmp_path / "not.py" - c, module_path = driver_api._get_driver_class(classname=args.classname, module=module) - assert c is None - assert module_path is None - assert logged(caplog, "Could not load module %s" % module) - - -def test__get_driver_class_explicit_fail_bad_spec(caplog, args): - log.setLevel(logging.DEBUG) - with patch.object(driver_api, "spec_from_file_location", return_value=None): - c, module_path = driver_api._get_driver_class(classname=args.classname, module=args.module) - assert c is None - assert module_path is None - assert logged(caplog, "Could not load module %s" % args.module) - - -def test__get_driver_class_explicit_pass(args): - log.setLevel(logging.DEBUG) - c, module_path = driver_api._get_driver_class(classname=args.classname, module=args.module) - assert c - assert c.__name__ == "TestDriver" - assert module_path == args.module - - -def test__get_driver_class_implicit_pass(args): - log.setLevel(logging.DEBUG) - with patch.object(Path, "cwd", return_value=fixture_path()): - c, module_path = driver_api._get_driver_class(classname=args.classname, module=args.module) - assert c - assert c.__name__ == "TestDriver" - assert module_path == args.module - - -def test__get_driver_module_explicit_absolute_fail(args): - assert args.module.is_absolute() - module = args.module.with_suffix(".bad") - assert not driver_api._get_driver_module_explicit(module=module) - - -def test__get_driver_module_explicit_absolute_pass(args): - assert args.module.is_absolute() - assert driver_api._get_driver_module_explicit(module=args.module) - - -def test__get_driver_module_explicit_relative_fail(args): - args.module = Path(os.path.relpath(args.module)).with_suffix(".bad") - assert not args.module.is_absolute() - assert not driver_api._get_driver_module_explicit(module=args.module) - - -def test__get_driver_module_explicit_relative_pass(args): - args.module = Path(os.path.relpath(args.module)) - assert not args.module.is_absolute() - assert driver_api._get_driver_module_explicit(module=args.module) - - -def test__get_driver_module_implicit_pass_full_package(): - assert driver_api._get_driver_module_implicit("uwtools.tests.fixtures.testdriver") - - -def test__get_driver_module_implicit_pass(): - with patch.object(sys, "path", [str(fixture_path()), *sys.path]): - assert driver_api._get_driver_module_implicit("testdriver") - - -def test__get_driver_module_implicit_fail(): - assert not driver_api._get_driver_module_implicit("no.such.module") diff --git a/src/uwtools/tests/api/test_drivers.py b/src/uwtools/tests/api/test_drivers.py index 3b957f7ad..70bf280b3 100644 --- a/src/uwtools/tests/api/test_drivers.py +++ b/src/uwtools/tests/api/test_drivers.py @@ -10,8 +10,10 @@ cdeps, chgres_cube, esg_grid, + filter_topo, fv3, global_equiv_resol, + ioda, jedi, make_hgrid, make_solo_mosaic, @@ -32,8 +34,10 @@ cdeps, chgres_cube, esg_grid, + filter_topo, fv3, global_equiv_resol, + ioda, jedi, make_hgrid, make_solo_mosaic, @@ -47,7 +51,7 @@ upp, ww3, ] -with_cycle = [cdeps, chgres_cube, fv3, jedi, mpas, mpas_init, schism, ungrib, upp, ww3] +with_cycle = [cdeps, chgres_cube, fv3, ioda, jedi, mpas, mpas_init, schism, ungrib, upp, ww3] with_leadtime = [upp] @@ -82,6 +86,11 @@ def test_api_graph(module): assert module.graph is support.graph +@mark.parametrize("module", modules) +def test_api_schema(module): + assert module.schema() + + @mark.parametrize("module", modules) def test_api_tasks(module): with patch.object(iotaa, "tasknames") as tasknames: diff --git a/src/uwtools/tests/api/test_execute.py b/src/uwtools/tests/api/test_execute.py new file mode 100644 index 000000000..b81b39c76 --- /dev/null +++ b/src/uwtools/tests/api/test_execute.py @@ -0,0 +1,190 @@ +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name + +import datetime as dt +import logging +import os +import sys +from pathlib import Path +from types import SimpleNamespace as ns +from unittest.mock import patch + +from pytest import fixture, mark, raises + +from uwtools.api import execute +from uwtools.exceptions import UWError +from uwtools.logging import log +from uwtools.tests.support import fixture_path, logged, regex_logged + +# Fixtures + + +@fixture +def args(): + return ns( + classname="TestDriver", + config=fixture_path("testdriver.yaml"), + module=fixture_path("testdriver.py"), + schema_file=fixture_path("testdriver.jsonschema"), + task="eighty_eight", + ) + + +@fixture +def kwargs(args): + return dict( + classname=args.classname, + config=args.config, + module=args.module, + schema_file=args.schema_file, + task=args.task, + ) + + +# Tests + + +@mark.parametrize("key,val", [("batch", True), ("leadtime", 6)]) +def test_execute_fail_bad_args(caplog, key, kwargs, val): + kwargs.update({"cycle": dt.datetime.now(), key: val}) + assert execute.execute(**kwargs) is False + assert logged(caplog, f"TestDriver does not accept argument '{key}'") + + +def test_execute_fail_stdin_not_ok(kwargs): + kwargs["config"] = None + kwargs["cycle"] = dt.datetime.now() + kwargs["stdin_ok"] = False + with raises(UWError) as e: + execute.execute(**kwargs) + assert str(e.value) == "Set stdin_ok=True to permit read from stdin" + + +@mark.parametrize("remove", ([], ["schema_file"])) +def test_execute_pass(caplog, kwargs, remove, tmp_path): + for kwarg in remove: + del kwargs[kwarg] + kwargs["cycle"] = dt.datetime.now() + log.setLevel(logging.DEBUG) + graph_file = tmp_path / "g.dot" + graph_code = "DOT code" + kwargs["graph_file"] = graph_file + with patch.object(execute, "graph", return_value=graph_code): + assert execute.execute(**kwargs) is True + assert regex_logged(caplog, "Instantiated %s with" % kwargs["classname"]) + with open(graph_file, "r", encoding="utf-8") as f: + assert f.read().strip() == graph_code + + +def test_execute_fail_cannot_load_driver_class(kwargs): + kwargs["module"] = "bad_module_name" + assert execute.execute(**kwargs) is False + + +def test_tasks_fail(args, caplog, tmp_path): + module = tmp_path / "not.py" + tasks = execute.tasks(classname=args.classname, module=module) + assert tasks == {} + assert logged( + caplog, "Could not get tasks from class %s in module %s" % (args.classname, module) + ) + + +def test_tasks_fail_no_cycle(args, caplog, kwargs): + log.setLevel(logging.DEBUG) + assert execute.execute(**kwargs) is False + assert logged(caplog, "%s requires argument '%s'" % (args.classname, "cycle")) + + +@mark.parametrize("f", [Path, str]) +def test_tasks_pass(args, f): + tasks = execute.tasks(classname=args.classname, module=f(args.module)) + assert tasks["eighty_eight"] == "88" + + +def test__get_driver_class_explicit_fail_bad_class(caplog, args): + log.setLevel(logging.DEBUG) + bad_class = "BadClass" + c, module_path = execute._get_driver_class(classname=bad_class, module=args.module) + assert c is None + assert module_path == args.module + assert logged(caplog, "Module %s has no class %s" % (args.module, bad_class)) + + +def test__get_driver_class_explicit_fail_bad_name(caplog, args): + log.setLevel(logging.DEBUG) + bad_name = Path("bad_name") + c, module_path = execute._get_driver_class(classname=args.classname, module=bad_name) + assert c is None + assert module_path is None + assert logged(caplog, "Could not load module %s" % bad_name) + + +def test__get_driver_class_explicit_fail_bad_path(caplog, args, tmp_path): + log.setLevel(logging.DEBUG) + module = tmp_path / "not.py" + c, module_path = execute._get_driver_class(classname=args.classname, module=module) + assert c is None + assert module_path is None + assert logged(caplog, "Could not load module %s" % module) + + +def test__get_driver_class_explicit_fail_bad_spec(caplog, args): + log.setLevel(logging.DEBUG) + with patch.object(execute, "spec_from_file_location", return_value=None): + c, module_path = execute._get_driver_class(classname=args.classname, module=args.module) + assert c is None + assert module_path is None + assert logged(caplog, "Could not load module %s" % args.module) + + +def test__get_driver_class_explicit_pass(args): + log.setLevel(logging.DEBUG) + c, module_path = execute._get_driver_class(classname=args.classname, module=args.module) + assert c + assert c.__name__ == "TestDriver" + assert module_path == args.module + + +def test__get_driver_class_implicit_pass(args): + log.setLevel(logging.DEBUG) + with patch.object(Path, "cwd", return_value=fixture_path()): + c, module_path = execute._get_driver_class(classname=args.classname, module=args.module) + assert c + assert c.__name__ == "TestDriver" + assert module_path == args.module + + +def test__get_driver_module_explicit_absolute_fail(args): + assert args.module.is_absolute() + module = args.module.with_suffix(".bad") + assert not execute._get_driver_module_explicit(module=module) + + +def test__get_driver_module_explicit_absolute_pass(args): + assert args.module.is_absolute() + assert execute._get_driver_module_explicit(module=args.module) + + +def test__get_driver_module_explicit_relative_fail(args): + args.module = Path(os.path.relpath(args.module)).with_suffix(".bad") + assert not args.module.is_absolute() + assert not execute._get_driver_module_explicit(module=args.module) + + +def test__get_driver_module_explicit_relative_pass(args): + args.module = Path(os.path.relpath(args.module)) + assert not args.module.is_absolute() + assert execute._get_driver_module_explicit(module=args.module) + + +def test__get_driver_module_implicit_pass_full_package(): + assert execute._get_driver_module_implicit("uwtools.tests.fixtures.testdriver") + + +def test__get_driver_module_implicit_pass(): + with patch.object(sys, "path", [str(fixture_path()), *sys.path]): + assert execute._get_driver_module_implicit("testdriver") + + +def test__get_driver_module_implicit_fail(): + assert not execute._get_driver_module_implicit("no.such.module") diff --git a/src/uwtools/tests/api/test_file.py b/src/uwtools/tests/api/test_fs.py similarity index 77% rename from src/uwtools/tests/api/test_file.py rename to src/uwtools/tests/api/test_fs.py index a403bb99b..71d48b5cb 100644 --- a/src/uwtools/tests/api/test_file.py +++ b/src/uwtools/tests/api/test_fs.py @@ -5,7 +5,7 @@ from pytest import fixture -from uwtools.api import file +from uwtools.api import fs @fixture @@ -31,7 +31,7 @@ def test_copy_fail(kwargs): for p in paths: assert not Path(p).exists() Path(list(paths.values())[0]).unlink() - assert file.copy(**kwargs) is False + assert fs.copy(**kwargs) is False assert not Path(list(paths.keys())[0]).exists() assert Path(list(paths.keys())[1]).is_file() @@ -40,7 +40,7 @@ def test_copy_pass(kwargs): paths = kwargs["config"]["a"]["b"] for p in paths: assert not Path(p).exists() - assert file.copy(**kwargs) is True + assert fs.copy(**kwargs) is True for p in paths: assert Path(p).is_file() @@ -50,7 +50,7 @@ def test_link_fail(kwargs): for p in paths: assert not Path(p).exists() Path(list(paths.values())[0]).unlink() - assert file.link(**kwargs) is False + assert fs.link(**kwargs) is False assert not Path(list(paths.keys())[0]).exists() assert Path(list(paths.keys())[1]).is_symlink() @@ -59,6 +59,13 @@ def test_link_pass(kwargs): paths = kwargs["config"]["a"]["b"] for p in paths: assert not Path(p).exists() - assert file.link(**kwargs) is True + assert fs.link(**kwargs) is True for p in paths: assert Path(p).is_symlink() + + +def test_makedirs(tmp_path): + paths = [tmp_path / "foo" / x for x in ("bar", "baz")] + assert not any(path.is_dir() for path in paths) + assert fs.makedirs(config={"makedirs": [str(path) for path in paths]}) is True + assert all(path.is_dir() for path in paths) diff --git a/src/uwtools/tests/config/formats/test_base.py b/src/uwtools/tests/config/formats/test_base.py index fa2039b09..a4ba27c87 100644 --- a/src/uwtools/tests/config/formats/test_base.py +++ b/src/uwtools/tests/config/formats/test_base.py @@ -43,6 +43,14 @@ class ConcreteConfig(Config): def _dict_to_str(cls, cfg): pass + @staticmethod + def _get_depth_threshold(): + pass + + @staticmethod + def _get_format(): + pass + def _load(self, config_file): with readable(config_file) as f: return yaml.safe_load(f.read()) @@ -54,16 +62,19 @@ def dump(self, path=None): def dump_dict(cfg, path=None): pass - @staticmethod - def get_depth_threshold(): - pass - @staticmethod - def get_format(): - pass +# Tests -# Tests +def test__characterize_values(config): + values = {1: "", 2: None, 3: "{{ n }}", 4: {"a": 88}, 5: [{"b": 99}], 6: "string"} + complete, template = config._characterize_values(values=values, parent="p") + assert complete == [" p1", " p2", " p4", " p4.a", " pb", " p5", " p6"] + assert template == [" p3: {{ n }}"] + + +def test__depth(config): + assert config._depth == 1 def test__load_paths(config, tmp_path): @@ -76,11 +87,28 @@ def test__load_paths(config, tmp_path): assert cfg[path.name] == "present" -def test_characterize_values(config): - values = {1: "", 2: None, 3: "{{ n }}", 4: {"a": 88}, 5: [{"b": 99}], 6: "string"} - complete, template = config.characterize_values(values=values, parent="p") - assert complete == [" p1", " p2", " p4", " p4.a", " pb", " p5", " p6"] - assert template == [" p3: {{ n }}"] +def test__parse_include(config): + """ + Test that non-YAML handles include tags properly. + """ + del config["foo"] + # Create a symlink for the include file: + include_path = fixture_path("fruit_config.yaml") + config.data.update( + { + "config": { + "salad_include": f"!INCLUDE [{include_path}]", + "meat": "beef", + "dressing": "poppyseed", + } + } + ) + config._parse_include() + + assert config["fruit"] == "papaya" + assert config["how_many"] == 17 + assert config["config"]["meat"] == "beef" + assert len(config["config"]) == 2 @mark.parametrize("fmt", [FORMAT.ini, FORMAT.nml, FORMAT.yaml]) @@ -111,10 +139,6 @@ def test_compare_config(caplog, fmt, salad_base): assert logged(caplog, msg) -def test_depth(config): - assert config.depth == 1 - - def test_dereference(tmp_path): # Test demonstrates that: # - Config dereferencing ignores environment variables. @@ -177,31 +201,7 @@ def test_invalid_config(fmt2, tmp_path): assert f"Cannot dump depth-{depthin} config to type-'{fmt2}' config" in str(e.value) -def test_parse_include(config): - """ - Test that non-YAML handles include tags properly. - """ - del config["foo"] - # Create a symlink for the include file: - include_path = fixture_path("fruit_config.yaml") - config.data.update( - { - "config": { - "salad_include": f"!INCLUDE [{include_path}]", - "meat": "beef", - "dressing": "poppyseed", - } - } - ) - config.parse_include() - - assert config["fruit"] == "papaya" - assert config["how_many"] == 17 - assert config["config"]["meat"] == "beef" - assert len(config["config"]) == 2 - - -def test_update_values(config): +def test_update_from(config): """ Test that a config object can be updated. """ diff --git a/src/uwtools/tests/config/formats/test_fieldtable.py b/src/uwtools/tests/config/formats/test_fieldtable.py index edcfe9f5f..8776f84b5 100644 --- a/src/uwtools/tests/config/formats/test_fieldtable.py +++ b/src/uwtools/tests/config/formats/test_fieldtable.py @@ -1,4 +1,4 @@ -# pylint: disable=missing-function-docstring,redefined-outer-name +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name """ Tests for uwtools.config.formats.fieldtable module. """ @@ -9,7 +9,7 @@ from uwtools.tests.support import fixture_path from uwtools.utils.file import FORMAT -# Tests +# Fixtures @fixture(scope="module") @@ -23,12 +23,15 @@ def ref(): return f.read().strip() -def test_fieldtable_get_format(): - assert FieldTableConfig.get_format() == FORMAT.fieldtable +# Tests + + +def test_fieldtable__get_depth_threshold(): + assert FieldTableConfig._get_depth_threshold() is None -def test_fieldtable_get_depth_threshold(): - assert FieldTableConfig.get_depth_threshold() is None +def test_fieldtable__get_format(): + assert FieldTableConfig._get_format() == FORMAT.fieldtable def test_fieldtable_instantiation_depth(): diff --git a/src/uwtools/tests/config/formats/test_ini.py b/src/uwtools/tests/config/formats/test_ini.py index d12bdbd11..c6a9f9f7f 100644 --- a/src/uwtools/tests/config/formats/test_ini.py +++ b/src/uwtools/tests/config/formats/test_ini.py @@ -1,4 +1,4 @@ -# pylint: disable=missing-function-docstring +# pylint: disable=missing-function-docstring,protected-access """ Tests for uwtools.config.formats.ini module. """ @@ -15,21 +15,15 @@ # Tests -def test_ini_get_format(): - assert INIConfig.get_format() == FORMAT.ini +def test_ini__get_depth_threshold(): + assert INIConfig._get_depth_threshold() == 2 -def test_ini_get_depth_threshold(): - assert INIConfig.get_depth_threshold() == 2 +def test_ini__get_format(): + assert INIConfig._get_format() == FORMAT.ini -def test_ini_instantiation_depth(): - with raises(UWConfigError) as e: - INIConfig(config={1: {2: {3: 4}}}) - assert str(e.value) == "Cannot instantiate depth-2 INIConfig with depth-3 config" - - -def test_ini_parse_include(): +def test_ini__parse_include(): """ Test that an INI file handles include tags properly. """ @@ -40,6 +34,12 @@ def test_ini_parse_include(): assert len(cfgobj["config"]) == 5 +def test_ini_instantiation_depth(): + with raises(UWConfigError) as e: + INIConfig(config={1: {2: {3: 4}}}) + assert str(e.value) == "Cannot instantiate depth-2 INIConfig with depth-3 config" + + @mark.parametrize("func", [repr, str]) def test_ini_repr_str(func): config = fixture_path("simple.ini") diff --git a/src/uwtools/tests/config/formats/test_nml.py b/src/uwtools/tests/config/formats/test_nml.py index 59c817954..c91c16dde 100644 --- a/src/uwtools/tests/config/formats/test_nml.py +++ b/src/uwtools/tests/config/formats/test_nml.py @@ -1,4 +1,4 @@ -# pylint: disable=missing-function-docstring,redefined-outer-name +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name """ Tests for uwtools.config.formats.nml module. """ @@ -24,6 +24,39 @@ def data(): # Tests +def test_nml__get_depth_threshold(): + assert NMLConfig._get_depth_threshold() is None + + +def test_nml__get_format(): + assert NMLConfig._get_format() == FORMAT.nml + + +def test_nml__parse_include(): + """ + Test that non-YAML handles include tags properly. + """ + cfgobj = NMLConfig(fixture_path("include_files.nml")) + assert cfgobj["config"]["fruit"] == "papaya" + assert cfgobj["config"]["how_many"] == 17 + assert cfgobj["config"]["meat"] == "beef" + assert len(cfgobj["config"]) == 5 + + +def test_nml__parse_include_mult_sect(): + """ + Test that non-YAML handles include tags with files that have multiple sections in separate file. + """ + cfgobj = NMLConfig(fixture_path("include_files_with_sect.nml")) + assert cfgobj["config"]["fruit"] == "papaya" + assert cfgobj["config"]["how_many"] == 17 + assert cfgobj["config"]["meat"] == "beef" + assert cfgobj["config"]["dressing"] == "ranch" + assert cfgobj["setting"]["size"] == "large" + assert len(cfgobj["config"]) == 5 + assert len(cfgobj["setting"]) == 3 + + def test_nml_derived_type_dict(): nml = NMLConfig(config={"nl": {"o": {"i": 77, "j": 88}}}) assert nml["nl"]["o"] == {"i": 77, "j": 88} @@ -57,39 +90,6 @@ def test_nml_dump_dict_Namelist(data, tmp_path): assert nml == data -def test_nml_get_format(): - assert NMLConfig.get_format() == FORMAT.nml - - -def test_nml_get_depth_threshold(): - assert NMLConfig.get_depth_threshold() is None - - -def test_nml_parse_include(): - """ - Test that non-YAML handles include tags properly. - """ - cfgobj = NMLConfig(fixture_path("include_files.nml")) - assert cfgobj["config"]["fruit"] == "papaya" - assert cfgobj["config"]["how_many"] == 17 - assert cfgobj["config"]["meat"] == "beef" - assert len(cfgobj["config"]) == 5 - - -def test_nml_parse_include_mult_sect(): - """ - Test that non-YAML handles include tags with files that have multiple sections in separate file. - """ - cfgobj = NMLConfig(fixture_path("include_files_with_sect.nml")) - assert cfgobj["config"]["fruit"] == "papaya" - assert cfgobj["config"]["how_many"] == 17 - assert cfgobj["config"]["meat"] == "beef" - assert cfgobj["config"]["dressing"] == "ranch" - assert cfgobj["setting"]["size"] == "large" - assert len(cfgobj["config"]) == 5 - assert len(cfgobj["setting"]) == 3 - - @mark.parametrize("func", [repr, str]) def test_ini_repr_str(func): config = fixture_path("simple.nml") diff --git a/src/uwtools/tests/config/formats/test_sh.py b/src/uwtools/tests/config/formats/test_sh.py index 5d51a043e..ce6a0119e 100644 --- a/src/uwtools/tests/config/formats/test_sh.py +++ b/src/uwtools/tests/config/formats/test_sh.py @@ -1,4 +1,4 @@ -# pylint: disable=missing-function-docstring +# pylint: disable=missing-function-docstring,protected-access """ Tests for uwtools.config.formats.sh module. """ @@ -16,21 +16,15 @@ # Tests -def test_sh_get_format(): - assert SHConfig.get_format() == FORMAT.sh +def test_sh__get_depth_threshold(): + assert SHConfig._get_depth_threshold() == 1 -def test_sh_get_depth_threshold(): - assert SHConfig.get_depth_threshold() == 1 +def test_sh__get_format(): + assert SHConfig._get_format() == FORMAT.sh -def test_sh_instantiation_depth(): - with raises(UWConfigError) as e: - SHConfig(config={1: {2: {3: 4}}}) - assert str(e.value) == "Cannot instantiate depth-1 SHConfig with depth-3 config" - - -def test_sh_parse_include(): +def test_sh__parse_include(): """ Test that an sh file with no sections handles include tags properly. """ @@ -41,6 +35,12 @@ def test_sh_parse_include(): assert len(cfgobj) == 5 +def test_sh_instantiation_depth(): + with raises(UWConfigError) as e: + SHConfig(config={1: {2: {3: 4}}}) + assert str(e.value) == "Cannot instantiate depth-1 SHConfig with depth-3 config" + + @mark.parametrize("func", [repr, str]) def test_sh_repr_str(func): config = fixture_path("simple.sh") diff --git a/src/uwtools/tests/config/formats/test_yaml.py b/src/uwtools/tests/config/formats/test_yaml.py index 415785c2f..98a374d59 100644 --- a/src/uwtools/tests/config/formats/test_yaml.py +++ b/src/uwtools/tests/config/formats/test_yaml.py @@ -27,12 +27,34 @@ # Tests -def test_yaml_get_format(): - assert YAMLConfig.get_format() == FORMAT.yaml +def test_yaml__add_yaml_representers(): + YAMLConfig._add_yaml_representers() + representers = yaml.Dumper.yaml_representers + assert support.UWYAMLConvert in representers + assert OrderedDict in representers + assert f90nml.Namelist in representers + + +def test_yaml__get_depth_threshold(): + assert YAMLConfig._get_depth_threshold() is None + + +def test_yaml__get_format(): + assert YAMLConfig._get_format() == FORMAT.yaml + + +def test_yaml__represent_namelist(): + YAMLConfig._add_yaml_representers() + namelist = f90nml.reads("&namelist\n key = value\n/\n") + expected = "{namelist: {key: value}}" + assert yaml.dump(namelist, default_flow_style=True).strip() == expected -def test_yaml_get_depth_threshold(): - assert YAMLConfig.get_depth_threshold() is None +def test_yaml__represent_ordereddict(): + YAMLConfig._add_yaml_representers() + ordereddict_values = OrderedDict([("example", OrderedDict([("key", "value")]))]) + expected = "{example: {key: value}}" + assert yaml.dump(ordereddict_values, default_flow_style=True).strip() == expected def test_yaml_instantiation_depth(): @@ -133,10 +155,10 @@ def test_yaml_constructor_error_not_dict_from_file(tmp_path): def test_yaml_constructor_error_not_dict_from_stdin(): # Test that a useful exception is raised if the YAML stdin input is a non-dict value. - with StringIO("a string") as sio, patch.object(sys, "stdin", new=sio): + with StringIO("88") as sio, patch.object(sys, "stdin", new=sio): with raises(exceptions.UWConfigError) as e: YAMLConfig() - assert "Parsed a str value from stdin, expected a dict" in str(e.value) + assert "Parsed an int value from stdin, expected a dict" in str(e.value) def test_yaml_constructor_error_unregistered_constructor(tmp_path): @@ -186,25 +208,3 @@ def test_yaml_unexpected_error(tmp_path): with raises(UWConfigError) as e: YAMLConfig(config=cfgfile) assert msg in str(e.value) - - -def test_yaml__add_yaml_representers(): - YAMLConfig._add_yaml_representers() - representers = yaml.Dumper.yaml_representers - assert support.UWYAMLConvert in representers - assert OrderedDict in representers - assert f90nml.Namelist in representers - - -def test_yaml__represent_namelist(): - YAMLConfig._add_yaml_representers() - namelist = f90nml.reads("&namelist\n key = value\n/\n") - expected = "{namelist: {key: value}}" - assert yaml.dump(namelist, default_flow_style=True).strip() == expected - - -def test_yaml__represent_ordereddict(): - YAMLConfig._add_yaml_representers() - ordereddict_values = OrderedDict([("example", OrderedDict([("key", "value")]))]) - expected = "{example: {key: value}}" - assert yaml.dump(ordereddict_values, default_flow_style=True).strip() == expected diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 2964dbe30..c80be3d6d 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -74,7 +74,7 @@ def help_realize_config_fmt2fmt(input_file, input_format, update_file, update_fo ) cfgclass = tools.format_to_config(input_format) cfgobj = cfgclass(input_file) - cfgobj.update_values(cfgclass(update_file)) + cfgobj.update_from(cfgclass(update_file)) reference = tmpdir / f"expected{ext}" cfgobj.dump(reference) assert compare_files(reference, output_file) diff --git a/src/uwtools/tests/config/test_validator.py b/src/uwtools/tests/config/test_validator.py index 098cfe284..983c7abf0 100644 --- a/src/uwtools/tests/config/test_validator.py +++ b/src/uwtools/tests/config/test_validator.py @@ -6,7 +6,7 @@ import logging from pathlib import Path from typing import Any -from unittest.mock import patch +from unittest.mock import Mock, patch import yaml from pytest import fixture, raises @@ -124,6 +124,21 @@ def write_as_json(data: dict[str, Any], path: Path) -> Path: # Test functions +def test_bundle(): + schema = {"fruit": {"$ref": "urn:uwtools:a"}, "flowers": None} + with patch.object(validator, "_registry") as _registry: + outer, inner = Mock(), Mock() + outer.value.contents = {"a": {"$ref": "urn:uwtools:attrs"}, "b": {"name": "banana"}} + inner.value.contents = {"name": "apple"} + _registry().get_or_retrieve.side_effect = [outer, inner] + bundled = validator.bundle(schema) + assert bundled == {"fruit": {"a": {"name": "apple"}, "b": {"name": "banana"}}, "flowers": None} + assert [_registry().get_or_retrieve.mock_calls[i].args[0] for i in (0, 1)] == [ + "urn:uwtools:a", + "urn:uwtools:attrs", + ] + + def test_get_schema_file(): with patch.object(validator, "resource_path", return_value=Path("/foo/bar")): assert validator.get_schema_file("baz") == Path("/foo/bar/baz.jsonschema") @@ -191,6 +206,18 @@ def test__prep_config_file(prep_config_dict, tmp_path): assert cfgobj == {"roses": "red", "color": "red"} +def test__registry(tmp_path): + validator._registry.cache_clear() + d = {"foo": "bar"} + path = tmp_path / "foo-bar.jsonschema" + with open(path, "w", encoding="utf-8") as f: + json.dump(d, f) + with patch.object(validator, "resource_path", return_value=path) as resource_path: + r = validator._registry() + assert r.get_or_retrieve("urn:uwtools:foo-bar").value.contents == d + resource_path.assert_called_once_with("jsonschema/foo-bar.jsonschema") + + def test__validation_errors_bad_enum_value(config, schema): config["color"] = "yellow" assert len(validator._validation_errors(config, schema)) == 1 diff --git a/src/uwtools/tests/drivers/test_cdeps.py b/src/uwtools/tests/drivers/test_cdeps.py index e4ecfd5a6..66ad90867 100644 --- a/src/uwtools/tests/drivers/test_cdeps.py +++ b/src/uwtools/tests/drivers/test_cdeps.py @@ -112,7 +112,7 @@ def test_CDEPS_streams(driverobj, group): def test_CDEPS_driver_name(driverobj): - assert driverobj.driver_name == "cdeps" + assert driverobj.driver_name() == CDEPS.driver_name() == "cdeps" def test_CDEPS__model_namelist_file(driverobj): diff --git a/src/uwtools/tests/drivers/test_chgres_cube.py b/src/uwtools/tests/drivers/test_chgres_cube.py index 66ae79d4f..de399e7fe 100644 --- a/src/uwtools/tests/drivers/test_chgres_cube.py +++ b/src/uwtools/tests/drivers/test_chgres_cube.py @@ -148,7 +148,7 @@ def test_ChgresCube_runscript(driverobj): def test_ChgresCube_driver_name(driverobj): - assert driverobj.driver_name == "chgres_cube" + assert driverobj.driver_name() == ChgresCube.driver_name() == "chgres_cube" def test_ChgresCube_taskname(driverobj): diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index 4b1a74f91..9c80c7769 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -39,8 +39,8 @@ def atask(self): def provisioned_rundir(self): pass - @property - def driver_name(self) -> str: + @classmethod + def driver_name(cls) -> str: return "concrete" def _validate(self, schema_file: Optional[Path] = None) -> None: @@ -218,7 +218,7 @@ def test_Assets_key_path(config, tmp_path): assetsobj = ConcreteAssetsTimeInvariant( config=config_file, dry_run=False, key_path=["foo", "bar"] ) - assert assetsobj.config == config[assetsobj.driver_name] + assert assetsobj.config == config[assetsobj.driver_name()] def test_Assets_leadtime(config): @@ -273,6 +273,11 @@ def test_Assets__rundir(assetsobj): assert assetsobj.rundir == Path(assetsobj.config["rundir"]) +def test_Assets__schema_name(): + with patch.object(driver.Assets, "driver_name", return_value="a_driver"): + assert driver.Assets._schema_name() == "a-driver" + + def test_Assets__validate_internal(assetsobj): with patch.object(assetsobj, "_validate", driver.Assets._validate): with patch.object(driver, "validate_internal") as validate_internal: @@ -376,9 +381,9 @@ def test_Driver__run_via_local_execution(driverobj): executable = Path(driverobj.config["execution"]["executable"]) executable.touch() with patch.object(driverobj, "provisioned_rundir") as prd: - with patch.object(driver, "execute") as execute: + with patch.object(driver, "run_shell_cmd") as run_shell_cmd: driverobj._run_via_local_execution() - execute.assert_called_once_with( + run_shell_cmd.assert_called_once_with( cmd="{x} >{x}.out 2>&1".format(x=driverobj._runscript_path), cwd=driverobj.rundir, log_output=True, diff --git a/src/uwtools/tests/drivers/test_esg_grid.py b/src/uwtools/tests/drivers/test_esg_grid.py index 838c480a5..950ce2d6d 100644 --- a/src/uwtools/tests/drivers/test_esg_grid.py +++ b/src/uwtools/tests/drivers/test_esg_grid.py @@ -124,4 +124,4 @@ def test_ESGGrid_provisioned_rundir(driverobj): def test_FilterTopo_driver_name(driverobj): - assert driverobj.driver_name == "esg_grid" + assert driverobj.driver_name() == ESGGrid.driver_name() == "esg_grid" diff --git a/src/uwtools/tests/drivers/test_filter_topo.py b/src/uwtools/tests/drivers/test_filter_topo.py index e02bb24c3..54b243fdf 100644 --- a/src/uwtools/tests/drivers/test_filter_topo.py +++ b/src/uwtools/tests/drivers/test_filter_topo.py @@ -101,4 +101,4 @@ def test_FilterTopo_provisioned_rundir(driverobj): def test_FilterTopo_driver_name(driverobj): - assert driverobj.driver_name == "filter_topo" + assert driverobj.driver_name() == FilterTopo.driver_name() == "filter_topo" diff --git a/src/uwtools/tests/drivers/test_fv3.py b/src/uwtools/tests/drivers/test_fv3.py index c5997c768..ebc2a08bf 100644 --- a/src/uwtools/tests/drivers/test_fv3.py +++ b/src/uwtools/tests/drivers/test_fv3.py @@ -248,7 +248,7 @@ def test_FV3_runscript(driverobj): def test_FV3_driver_name(driverobj): - assert driverobj.driver_name == "fv3" + assert driverobj.driver_name() == FV3.driver_name() == "fv3" def test_FV3_taskname(driverobj): diff --git a/src/uwtools/tests/drivers/test_global_equiv_resol.py b/src/uwtools/tests/drivers/test_global_equiv_resol.py index 678b84bf7..f95201940 100644 --- a/src/uwtools/tests/drivers/test_global_equiv_resol.py +++ b/src/uwtools/tests/drivers/test_global_equiv_resol.py @@ -84,7 +84,7 @@ def test_GlobalEquivResol_provisioned_rundir(driverobj): def test_FilterTopo_driver_name(driverobj): - assert driverobj.driver_name == "global_equiv_resol" + assert driverobj.driver_name() == GlobalEquivResol.driver_name() == "global_equiv_resol" def test_GlobalEquivResol__runcmd(driverobj): diff --git a/src/uwtools/tests/drivers/test_ioda.py b/src/uwtools/tests/drivers/test_ioda.py index 2c0442bba..65527f86c 100644 --- a/src/uwtools/tests/drivers/test_ioda.py +++ b/src/uwtools/tests/drivers/test_ioda.py @@ -101,7 +101,7 @@ def test_IODA_provisioned_rundir(driverobj): def test_IODA_driver_name(driverobj): - assert driverobj.driver_name == "ioda" + assert driverobj.driver_name() == IODA.driver_name() == "ioda" def test_IODA__config_fn(driverobj): diff --git a/src/uwtools/tests/drivers/test_jedi.py b/src/uwtools/tests/drivers/test_jedi.py index 5e3ea8e6e..ebd7ecfcc 100644 --- a/src/uwtools/tests/drivers/test_jedi.py +++ b/src/uwtools/tests/drivers/test_jedi.py @@ -189,7 +189,7 @@ def file(path: Path): def test_JEDI_driver_name(driverobj): - assert driverobj.driver_name == "jedi" + assert driverobj.driver_name() == JEDI.driver_name() == "jedi" def test_JEDI__config_fn(driverobj): diff --git a/src/uwtools/tests/drivers/test_make_hgrid.py b/src/uwtools/tests/drivers/test_make_hgrid.py index 46a24d02e..74874e40d 100644 --- a/src/uwtools/tests/drivers/test_make_hgrid.py +++ b/src/uwtools/tests/drivers/test_make_hgrid.py @@ -77,7 +77,7 @@ def test_MakeHgrid_provisioned_rundir(driverobj): def test_MakeHgrid_driver_name(driverobj): - assert driverobj.driver_name == "make_hgrid" + assert driverobj.driver_name() == MakeHgrid.driver_name() == "make_hgrid" def test_MakeHgrid__runcmd(driverobj): diff --git a/src/uwtools/tests/drivers/test_make_solo_mosaic.py b/src/uwtools/tests/drivers/test_make_solo_mosaic.py index f4fd49339..bdd0f4573 100644 --- a/src/uwtools/tests/drivers/test_make_solo_mosaic.py +++ b/src/uwtools/tests/drivers/test_make_solo_mosaic.py @@ -71,7 +71,7 @@ def test_MakeSoloMosaic_provisioned_rundir(driverobj): def test_MakeSoloMosaic_driver_name(driverobj): - assert driverobj.driver_name == "make_solo_mosaic" + assert driverobj.driver_name() == MakeSoloMosaic.driver_name() == "make_solo_mosaic" def test_MakeSoloMosaic__runcmd(driverobj): diff --git a/src/uwtools/tests/drivers/test_mpas.py b/src/uwtools/tests/drivers/test_mpas.py index e0b252c49..b85a55588 100644 --- a/src/uwtools/tests/drivers/test_mpas.py +++ b/src/uwtools/tests/drivers/test_mpas.py @@ -237,7 +237,7 @@ def test_MPAS_provisioned_rundir(driverobj): def test_MPAS_driver_name(driverobj): - assert driverobj.driver_name == "mpas" + assert driverobj.driver_name() == MPAS.driver_name() == "mpas" def test_MPAS_streams_file(config, driverobj): diff --git a/src/uwtools/tests/drivers/test_mpas_init.py b/src/uwtools/tests/drivers/test_mpas_init.py index 679267d38..f9784755c 100644 --- a/src/uwtools/tests/drivers/test_mpas_init.py +++ b/src/uwtools/tests/drivers/test_mpas_init.py @@ -212,7 +212,7 @@ def test_MPASInit_provisioned_rundir(driverobj): def test_MPASInit_driver_name(driverobj): - assert driverobj.driver_name == "mpas_init" + assert driverobj.driver_name() == MPASInit.driver_name() == "mpas_init" def test_MPASInit_streams_file(config, driverobj): diff --git a/src/uwtools/tests/drivers/test_orog_gsl.py b/src/uwtools/tests/drivers/test_orog_gsl.py index 875f52fe2..a942aa687 100644 --- a/src/uwtools/tests/drivers/test_orog_gsl.py +++ b/src/uwtools/tests/drivers/test_orog_gsl.py @@ -103,7 +103,7 @@ def test_OrogGSL_topo_data_3os(driverobj): def test_OrogGSL_driver_name(driverobj): - assert driverobj.driver_name == "orog_gsl" + assert driverobj.driver_name() == OrogGSL.driver_name() == "orog_gsl" def test_OrogGSL__runcmd(driverobj): diff --git a/src/uwtools/tests/drivers/test_schism.py b/src/uwtools/tests/drivers/test_schism.py index d58fd0876..0c5fea845 100644 --- a/src/uwtools/tests/drivers/test_schism.py +++ b/src/uwtools/tests/drivers/test_schism.py @@ -72,4 +72,4 @@ def test_SCHISM_provisioned_rundir(driverobj): def test_SCHISM_driver_name(driverobj): - assert driverobj.driver_name == "schism" + assert driverobj.driver_name() == SCHISM.driver_name() == "schism" diff --git a/src/uwtools/tests/drivers/test_sfc_climo_gen.py b/src/uwtools/tests/drivers/test_sfc_climo_gen.py index f7e81c088..a9355f42d 100644 --- a/src/uwtools/tests/drivers/test_sfc_climo_gen.py +++ b/src/uwtools/tests/drivers/test_sfc_climo_gen.py @@ -139,4 +139,4 @@ def test_SfcClimoGen_provisioned_rundir(driverobj): def test_SfcClimoGen_driver_name(driverobj): - assert driverobj.driver_name == "sfc_climo_gen" + assert driverobj.driver_name() == SfcClimoGen.driver_name() == "sfc_climo_gen" diff --git a/src/uwtools/tests/drivers/test_shave.py b/src/uwtools/tests/drivers/test_shave.py index ec1ff4cc8..b6ac9c2a2 100644 --- a/src/uwtools/tests/drivers/test_shave.py +++ b/src/uwtools/tests/drivers/test_shave.py @@ -81,7 +81,7 @@ def test_Shave_provisioned_rundir(driverobj): def test_Shave_driver_name(driverobj): - assert driverobj.driver_name == "shave" + assert driverobj.driver_name() == Shave.driver_name() == "shave" def test_Shave__runcmd(driverobj): diff --git a/src/uwtools/tests/drivers/test_support.py b/src/uwtools/tests/drivers/test_support.py index aa643e728..d9cf9545b 100644 --- a/src/uwtools/tests/drivers/test_support.py +++ b/src/uwtools/tests/drivers/test_support.py @@ -59,8 +59,8 @@ def t2(self): def t3(self): "@tasks t3" - @property - def driver_name(self): + @classmethod + def driver_name(cls): pass @property diff --git a/src/uwtools/tests/drivers/test_ungrib.py b/src/uwtools/tests/drivers/test_ungrib.py index 52952fd3b..be2a15388 100644 --- a/src/uwtools/tests/drivers/test_ungrib.py +++ b/src/uwtools/tests/drivers/test_ungrib.py @@ -126,7 +126,7 @@ def test_Ungrib_vtable(driverobj): def test_Ungrib_driver_name(driverobj): - assert driverobj.driver_name == "ungrib" + assert driverobj.driver_name() == Ungrib.driver_name() == "ungrib" def test_Ungrib__gribfile(driverobj): diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py index 0747504da..55b34cb52 100644 --- a/src/uwtools/tests/drivers/test_upp.py +++ b/src/uwtools/tests/drivers/test_upp.py @@ -168,7 +168,7 @@ def test_UPP_provisioned_rundir(driverobj): def test_UPP_driver_name(driverobj): - assert driverobj.driver_name == "upp" + assert driverobj.driver_name() == UPP.driver_name() == "upp" def test_UPP__namelist_path(driverobj): diff --git a/src/uwtools/tests/drivers/test_ww3.py b/src/uwtools/tests/drivers/test_ww3.py index bf97854f9..253080387 100644 --- a/src/uwtools/tests/drivers/test_ww3.py +++ b/src/uwtools/tests/drivers/test_ww3.py @@ -80,4 +80,4 @@ def test_WaveWatchIII_restart_directory(driverobj): def test_WaveWatchIII_driver_name(driverobj): - assert driverobj.driver_name == "ww3" + assert driverobj.driver_name() == WaveWatchIII.driver_name() == "ww3" diff --git a/src/uwtools/tests/fixtures/testdriver.py b/src/uwtools/tests/fixtures/testdriver.py index e0320644a..025a7df55 100644 --- a/src/uwtools/tests/fixtures/testdriver.py +++ b/src/uwtools/tests/fixtures/testdriver.py @@ -17,6 +17,6 @@ def eighty_eight(self): yield asset(88, lambda: True) yield None - @property - def driver_name(self): + @classmethod + def driver_name(cls): return "testdriver" diff --git a/src/uwtools/tests/support.py b/src/uwtools/tests/support.py index 6759d266d..41364ed67 100644 --- a/src/uwtools/tests/support.py +++ b/src/uwtools/tests/support.py @@ -13,8 +13,8 @@ def compare_files(path1: Union[Path, str], path2: Union[Path, str]) -> bool: """ - Determines whether the two given files are identical up to any number of trailing newlines, - which are ignored. Print the contents of both files when they do not match. + Determine whether the two given files are identical up to any number of trailing newlines, which + are ignored. Print the contents of both files when they do not match. :param path1: Path to first file. :param path2: Path to second file. @@ -36,7 +36,7 @@ def compare_files(path1: Union[Path, str], path2: Union[Path, str]) -> bool: def fixture_pathobj(suffix: str = "") -> Path: """ - Returns a pathlib Path object to a test-fixture resource file. + Return a pathlib Path object to a test-fixture resource file. :param suffix: A subpath relative to the location of the unit-test fixture resource files. The prefix path to the resources files is known to Python and varies based on installation @@ -49,7 +49,7 @@ def fixture_pathobj(suffix: str = "") -> Path: def fixture_path(suffix: str = "") -> Path: """ - Returns a POSIX path to a test-fixture resource file. + Return a POSIX path to a test-fixture resource file. :param suffix: A subpath relative to the location of the unit-test fixture resource files. The prefix path to the resources files is known to Python and varies based on installation diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index 7d36e4349..09bab82e0 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -7,6 +7,7 @@ from argparse import ArgumentParser as Parser from argparse import _SubParsersAction from pathlib import Path +from textwrap import dedent from unittest.mock import Mock, patch from pytest import fixture, mark, raises @@ -52,7 +53,7 @@ def args_config_realize(): @fixture -def args_dispatch_file(): +def args_dispatch_fs(): return { "target_dir": "/target/dir", "config_file": "/config/file", @@ -102,17 +103,17 @@ def test__add_subparser_config_validate(subparsers): def test__add_subparser_file(subparsers): - cli._add_subparser_file(subparsers) - assert actions(subparsers.choices[STR.file]) == [STR.copy, STR.link] + cli._add_subparser_fs(subparsers) + assert actions(subparsers.choices[STR.fs]) == [STR.copy, STR.link, STR.makedirs] def test__add_subparser_file_copy(subparsers): - cli._add_subparser_file_copy(subparsers) + cli._add_subparser_fs_copy(subparsers) assert subparsers.choices[STR.copy] def test__add_subparser_file_link(subparsers): - cli._add_subparser_file_link(subparsers) + cli._add_subparser_fs_link(subparsers) assert subparsers.choices[STR.link] @@ -177,7 +178,7 @@ def test__dispatch_execute(): "task": "eighty_eight", "stdin_ok": True, } - with patch.object(cli.uwtools.api.driver, "execute") as execute: + with patch.object(cli.uwtools.api.execute, "execute") as execute: cli._dispatch_execute(args=args) execute.assert_called_once_with( classname="TestDriver", @@ -376,35 +377,26 @@ def test__dispatch_config_validate_config_obj(): @mark.parametrize( - "action, funcname", [(STR.copy, "_dispatch_file_copy"), (STR.link, "_dispatch_file_link")] + "action, funcname", + [ + (STR.copy, "_dispatch_fs_copy"), + (STR.link, "_dispatch_fs_link"), + (STR.makedirs, "_dispatch_fs_makedirs"), + ], ) -def test__dispatch_file(action, funcname): +def test__dispatch_fs(action, funcname): args = {STR.action: action} with patch.object(cli, funcname) as func: - cli._dispatch_file(args) + cli._dispatch_fs(args) func.assert_called_once_with(args) -def test__dispatch_file_copy(args_dispatch_file): - args = args_dispatch_file - with patch.object(cli.uwtools.api.file, "copy") as copy: - cli._dispatch_file_copy(args) - copy.assert_called_once_with( - target_dir=args["target_dir"], - config=args["config_file"], - cycle=args["cycle"], - leadtime=args["leadtime"], - keys=args["keys"], - dry_run=args["dry_run"], - stdin_ok=args["stdin_ok"], - ) - - -def test__dispatch_file_link(args_dispatch_file): - args = args_dispatch_file - with patch.object(cli.uwtools.api.file, "link") as link: - cli._dispatch_file_link(args) - link.assert_called_once_with( +@mark.parametrize("action", ["copy", "link", "makedirs"]) +def test__dispatch_fs_action(action, args_dispatch_fs): + args = args_dispatch_fs + with patch.object(cli.uwtools.api.fs, action) as a: + getattr(cli, f"_dispatch_fs_{action}")(args) + a.assert_called_once_with( target_dir=args["target_dir"], config=args["config_file"], cycle=args["cycle"], @@ -579,7 +571,6 @@ def test__dispatch_template_translate_no_optional(): @mark.parametrize("hours", [0, 24, 168]) def test__dispatch_to_driver(hours): - name = "adriver" cycle = dt.datetime.now() leadtime = dt.timedelta(hours=hours) args: dict = { @@ -591,11 +582,12 @@ def test__dispatch_to_driver(hours): "dry_run": False, "graph_file": None, "key_path": ["foo", "bar"], + "show_schema": False, "stdin_ok": True, } adriver = Mock() with patch.object(cli, "import_module", return_value=adriver): - cli._dispatch_to_driver(name=name, args=args) + cli._dispatch_to_driver(name="adriver", args=args) adriver.execute.assert_called_once_with( batch=True, config="/path/to/config", @@ -609,6 +601,30 @@ def test__dispatch_to_driver(hours): ) +def test__dispatch_to_driver_no_schema(capsys): + adriver = Mock() + with patch.object(cli, "import_module", return_value=adriver): + with raises(SystemExit): + cli._dispatch_to_driver(name="adriver", args={}) + assert "No TASK specified" in capsys.readouterr().err + + +def test__dispatch_to_driver_show_schema(capsys): + adriver = Mock() + adriver.schema.return_value = {"fruit": {"b": "banana", "a": "apple"}} + with patch.object(cli, "import_module", return_value=adriver): + assert cli._dispatch_to_driver(name="adriver", args={"show_schema": True}) is True + expected = """ + { + "fruit": { + "a": "apple", + "b": "banana" + } + } + """ + assert capsys.readouterr().out == dedent(expected).lstrip() + + @mark.parametrize("quiet", [False, True]) @mark.parametrize("verbose", [False, True]) def test_main_fail_checks(capsys, quiet, verbose): diff --git a/src/uwtools/tests/test_file.py b/src/uwtools/tests/test_fs.py similarity index 69% rename from src/uwtools/tests/test_file.py rename to src/uwtools/tests/test_fs.py index 9e3a94b3a..b83a4fbdf 100644 --- a/src/uwtools/tests/test_file.py +++ b/src/uwtools/tests/test_fs.py @@ -1,12 +1,16 @@ -# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +# pylint: disable=redefined-outer-name import iotaa import yaml from pytest import fixture, mark, raises -from uwtools import file +from uwtools import fs from uwtools.exceptions import UWConfigError +# Fixtures + @fixture def assets(tmp_path): @@ -25,13 +29,32 @@ def assets(tmp_path): return dstdir, cfgdict, cfgfile +# Helpers + + +class ConcreteStager(fs.Stager): + def _validate(self): + pass + + @property + def _dst_paths(self): + return list(self._config.keys()) + + @property + def _schema(self): + return "some-schema" + + +# Tests + + @mark.parametrize("source", ("dict", "file")) def test_Copier(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() - file.Copier(target_dir=dstdir, config=config, keys=["a", "b"]).go() + fs.Copier(target_dir=dstdir, config=config, keys=["a", "b"]).go() assert (dstdir / "foo").is_file() assert (dstdir / "subdir" / "bar").is_file() @@ -40,7 +63,7 @@ def test_Copier_config_file_dry_run(assets): dstdir, cfgdict, _ = assets assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() - file.Copier(target_dir=dstdir, config=cfgdict, keys=["a", "b"], dry_run=True).go() + fs.Copier(target_dir=dstdir, config=cfgdict, keys=["a", "b"], dry_run=True).go() assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() iotaa.dryrun(False) @@ -50,51 +73,49 @@ def test_Copier_no_targetdir_abspath_pass(assets): dstdir, cfgdict, _ = assets old = cfgdict["a"]["b"] cfgdict = {str(dstdir / "foo"): old["foo"], str(dstdir / "bar"): old["subdir/bar"]} - assets = file.Copier(config=cfgdict).go() + assets = fs.Copier(config=cfgdict).go() assert all(asset.ready() for asset in assets) # type: ignore def test_Copier_no_targetdir_relpath_fail(assets): _, cfgdict, _ = assets with raises(UWConfigError) as e: - file.Copier(config=cfgdict, keys=["a", "b"]).go() + fs.Copier(config=cfgdict, keys=["a", "b"]).go() errmsg = "Relative path '%s' requires the target directory to be specified" assert errmsg % "foo" in str(e.value) @mark.parametrize("source", ("dict", "file")) -def test_Linker(assets, source): +def test_FilerStager(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile - assert not (dstdir / "foo").exists() - assert not (dstdir / "subdir" / "bar").exists() - file.Linker(target_dir=dstdir, config=config, keys=["a", "b"]).go() - assert (dstdir / "foo").is_symlink() - assert (dstdir / "subdir" / "bar").is_symlink() + assert fs.FileStager(target_dir=dstdir, config=config, keys=["a", "b"]) @mark.parametrize("source", ("dict", "file")) -def test_Stager(assets, source): +def test_Linker(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile - stager = file.Stager(target_dir=dstdir, config=config, keys=["a", "b"]) - assert set(stager._file_map.keys()) == {"foo", "subdir/bar"} - assert stager._validate() is True + assert not (dstdir / "foo").exists() + assert not (dstdir / "subdir" / "bar").exists() + fs.Linker(target_dir=dstdir, config=config, keys=["a", "b"]).go() + assert (dstdir / "foo").is_symlink() + assert (dstdir / "subdir" / "bar").is_symlink() @mark.parametrize("source", ("dict", "file")) -def test_Stager_bad_key(assets, source): +def test_Stager__config_block_fail_bad_keypath(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile with raises(UWConfigError) as e: - file.Stager(target_dir=dstdir, config=config, keys=["a", "x"]) + ConcreteStager(target_dir=dstdir, config=config, keys=["a", "x"]) assert str(e.value) == "Failed following YAML key(s): a -> x" @mark.parametrize("val", [None, True, False, "str", 88, 3.14, [], tuple()]) -def test_Stager_empty_val(assets, val): +def test_Stager__config_block_fails_bad_type(assets, val): dstdir, cfgdict, _ = assets cfgdict["a"]["b"] = val with raises(UWConfigError) as e: - file.Stager(target_dir=dstdir, config=cfgdict, keys=["a", "b"]) - assert str(e.value) == "No file map found at key path: a -> b" + ConcreteStager(target_dir=dstdir, config=cfgdict, keys=["a", "b"]) + assert str(e.value) == "Expected block not found at key path: a -> b" diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 66032dc64..2c8b3fb67 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -206,6 +206,15 @@ def test__add_task_dependency_fail_bad_operand(self, instance, root): with raises(UWConfigError): instance._add_task_dependency(e=root, config=config) + def test__add_task_dependency_metataskdep(self, instance, root): + config = {"metataskdep": {"attrs": {"metatask": "foo"}}} + instance._add_task_dependency(e=root, config=config) + dependency = root[0] + assert dependency.tag == "dependency" + child = dependency[0] + assert child.tag == "metataskdep" + assert child.get("metatask") == "foo" + @mark.parametrize( "tag_config", [("and", {"strneq": {"attrs": {"left": "&RUN_GSI;", "right": "YES"}}})], diff --git a/src/uwtools/tests/test_scheduler.py b/src/uwtools/tests/test_scheduler.py index de41507e5..8a7413af4 100644 --- a/src/uwtools/tests/test_scheduler.py +++ b/src/uwtools/tests/test_scheduler.py @@ -104,10 +104,10 @@ def test_JobScheduler_get_scheduler_pass(props): def test_JobScheduler_submit_job(schedulerobj, tmp_path): runscript = tmp_path / "runscript" submit_file = tmp_path / "runscript.submit" - with patch.object(scheduler, "execute") as execute: - execute.return_value = (True, None) + with patch.object(scheduler, "run_shell_cmd") as run_shell_cmd: + run_shell_cmd.return_value = (True, None) assert schedulerobj.submit_job(runscript=runscript, submit_file=submit_file) is True - execute.assert_called_once_with( + run_shell_cmd.assert_called_once_with( cmd=f"sub {runscript} 2>&1 | tee {submit_file}", cwd=str(tmp_path) ) diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index d270d5cc2..ee0dd3043 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -739,7 +739,7 @@ def test_schema_execution_serial(): # files-to-stage -def test_schema_files_to_stage(): +def test_schema_stage_files(): errors = schema_validator("files-to-stage") # The input must be an dict: assert "is not of type 'object'\n" in errors([]) @@ -1212,6 +1212,21 @@ def test_schema_make_solo_mosaic_rundir(make_solo_mosaic_prop): assert "88 is not of type 'string'\n" in errors(88) +# makedirs + + +def test_schema_makedirs(): + errors = schema_validator("makedirs") + # The input must be an dict: + assert "is not of type 'object'\n" in errors([]) + # Basic correctness: + assert not errors({"makedirs": ["/path/to/dir1", "/path/to/dir2"]}) + # An empty array is not allowed: + assert "[] should be non-empty" in errors({"makedirs": []}) + # Non-string values are not allowed: + assert "True is not of type 'string'\n" in errors({"makedirs": [True]}) + + # mpas diff --git a/src/uwtools/tests/utils/test_processing.py b/src/uwtools/tests/utils/test_processing.py index cbabefdda..055891789 100644 --- a/src/uwtools/tests/utils/test_processing.py +++ b/src/uwtools/tests/utils/test_processing.py @@ -12,10 +12,10 @@ def test_run_failure(caplog): processing.log.setLevel(logging.INFO) cmd = "expr 1 / 0" - success, output = processing.execute(cmd=cmd) + success, output = processing.run_shell_cmd(cmd=cmd) assert "division by zero" in output assert success is False - assert logged(caplog, "Executing: %s" % cmd) + assert logged(caplog, "Running: %s" % cmd) assert logged(caplog, " Failed with status: 2") assert logged(caplog, " Output:") assert logged(caplog, " expr: division by zero") @@ -24,9 +24,11 @@ def test_run_failure(caplog): def test_run_success(caplog, tmp_path): processing.log.setLevel(logging.INFO) cmd = "echo hello $FOO" - success, _ = processing.execute(cmd=cmd, cwd=tmp_path, env={"FOO": "bar"}, log_output=True) + success, _ = processing.run_shell_cmd( + cmd=cmd, cwd=tmp_path, env={"FOO": "bar"}, log_output=True + ) assert success - assert logged(caplog, "Executing: %s" % cmd) + assert logged(caplog, "Running: %s" % cmd) assert logged(caplog, " in %s" % tmp_path) assert logged(caplog, " with environment variables:") assert logged(caplog, " FOO=bar") diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index f663957aa..aa606ed09 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -16,6 +16,13 @@ def ready(taskval): # Tests +def test_tasks_directory(tmp_path): + p = tmp_path / "foo" / "bar" + assert not p.is_dir() + assert ready(tasks.directory(path=p)) + assert p.is_dir() + + def test_tasks_executable(tmp_path): p = tmp_path / "program" # Ensure that only our temp directory is on the path: diff --git a/src/uwtools/utils/api.py b/src/uwtools/utils/api.py index 4df3d2ad7..548b3306b 100644 --- a/src/uwtools/utils/api.py +++ b/src/uwtools/utils/api.py @@ -38,7 +38,7 @@ def make_execute( with_leadtime: Optional[bool] = False, ) -> Callable[..., bool]: """ - Returns a function that executes tasks for the given driver. + Return a function that executes tasks for the given driver. :param driver_class: The driver class whose tasks to execute. :param with_cycle: Does the driver's constructor take a 'cycle' parameter? diff --git a/src/uwtools/utils/file.py b/src/uwtools/utils/file.py index 5c102827a..9efd2c324 100644 --- a/src/uwtools/utils/file.py +++ b/src/uwtools/utils/file.py @@ -11,13 +11,12 @@ from typing import IO, Any, Generator, Optional, Union from uwtools.exceptions import UWError -from uwtools.logging import log from uwtools.strings import FORMAT class StdinProxy: """ - Reads stdin once but permits multiple reads of its data. + Read stdin once and return its cached data. """ def __init__(self) -> None: @@ -43,7 +42,7 @@ def _stdinproxy(): def get_file_format(path: Path) -> str: """ - Returns a standardized file format name given a path/filename. + Return a standardized file format name given a path/filename. :param path: A path or filename. :return: One of a set of supported file-format names. @@ -52,15 +51,14 @@ def get_file_format(path: Path) -> str: suffix = Path(path).suffix.replace(".", "") try: return FORMAT.formats()[suffix] - except KeyError as e: + except KeyError: msg = f"Cannot deduce format of '{path}' from unknown extension '{suffix}'" - log.critical(msg) - raise UWError(msg) from e + raise UWError(msg) from None def path_if_it_exists(path: str) -> str: """ - Returns the given path as an absolute path if it exists, and raises an exception otherwise. + Return the given path as an absolute path if it exists, and raises an exception otherwise. :param path: The filesystem path to test. :return: The same filesystem path as an absolute path. @@ -93,7 +91,7 @@ def readable( def resource_path(suffix: str = "") -> Path: """ - Returns a pathlib Path object to a uwtools resource file. + Return a pathlib Path object to a uwtools resource file. :param suffix: A subpath relative to the location of the uwtools resource files. The prefix path to the resources files is known to Python and varies based on installation location. diff --git a/src/uwtools/utils/memory.py b/src/uwtools/utils/memory.py index 445f58a35..d2baca4dd 100644 --- a/src/uwtools/utils/memory.py +++ b/src/uwtools/utils/memory.py @@ -25,7 +25,7 @@ def __str__(self): @property def measurement(self): """ - Returns the measurement (MB, KB, etc.) + The measurement (MB, KB, etc.) """ if self._measurement is None: self._measurement = self._value[-2:] @@ -34,7 +34,7 @@ def measurement(self): @property def quantity(self): """ - Returns the quantity. + The quantity. """ if self._quantity is None: self._quantity = float(self._value[0:-2]) @@ -42,7 +42,7 @@ def quantity(self): def convert(self, measurement: str): """ - Converts the current representation to another measurement. + Convert the current representation to another measurement. """ quantity = (MAP[self.measurement] / MAP[measurement.upper()]) * self.quantity diff --git a/src/uwtools/utils/processing.py b/src/uwtools/utils/processing.py index 1408fe087..05c16efdf 100644 --- a/src/uwtools/utils/processing.py +++ b/src/uwtools/utils/processing.py @@ -9,23 +9,23 @@ from uwtools.logging import INDENT, log -def execute( +def run_shell_cmd( cmd: str, cwd: Optional[Union[Path, str]] = None, env: Optional[dict[str, str]] = None, log_output: Optional[bool] = False, ) -> tuple[bool, str]: """ - Execute a command in a subshell. + Run a command in a shell. - :param cmd: The command to execute. - :param cwd: Change to this directory before executing cmd. - :param env: Environment variables to set before executing cmd. + :param cmd: The command to run. + :param cwd: Change to this directory before running cmd. + :param env: Environment variables to set before running cmd. :param log_output: Log output from successful cmd? (Error output is always logged.) :return: A result object providing combined stder/stdout output and success values. """ - log.info("Executing: %s", cmd) + log.info("Running: %s", cmd) if cwd: log.info("%sin %s", INDENT, cwd) if env: diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 8a8555204..eef1c9deb 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -10,6 +10,19 @@ from iotaa import asset, external, task +@task +def directory(path: Path): + """ + A filesystem directory. + + :param path: Path to the directory. + """ + yield "Directory %s" % path + yield asset(path, path.is_dir) + yield None + path.mkdir(parents=True, exist_ok=True) + + @external def executable(program: Union[Path, str]): """ From c6d6040331d665cffab11bfc29bf9985fc085b15 Mon Sep 17 00:00:00 2001 From: Travis Byrne <42480803+Byrnetp@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:31:28 -0600 Subject: [PATCH 11/24] UW-666 Jupyter Notebook: File System Tool (#598) * update_values() -> update_from() (#578) * DRY out Driver._validate() (#579) * Move execute() to its own API namespace, disambiguate (#580) * Config classes to API (#582) * Simplify unknown-extension error message (#584) * UW-657 fs makedirs (#572) * Docstring cleanup (#585) * GH 586 bad stdin_ok prompt (#587) * Fix issue with creating a metatask dep. (#589) The rocoto schema is set up to allow metataskdep entries in the YAML, but the logic was not included in the tool to handle them. This addition fixes that. * Add --show-schema support to drivers (#588) * add file system notebook, unit tests, binder links * Add missing newlines * Add a table of contents * Makefile and .gitignore changes from review * Apply suggestions from code review Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> * Apply review suggestions to all sections of the notebook * Specific rmtree() targeting and output cell test update * Apply suggestions from code review Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> --------- Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Co-authored-by: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> --- .gitignore | 3 +- docs/sections/user_guide/api/fs.rst | 3 + docs/sections/user_guide/index.rst | 1 + notebooks/.gitignore | 2 + notebooks/fixtures/fs/copy-config.yaml | 3 + notebooks/fixtures/fs/copy-keys-config.yaml | 6 + notebooks/fixtures/fs/dir-config.yaml | 3 + notebooks/fixtures/fs/dir-keys-config.yaml | 6 + notebooks/fixtures/fs/file1.nml | 6 + notebooks/fixtures/fs/file2.txt | 1 + notebooks/fixtures/fs/file3.csv | 4 + notebooks/fixtures/fs/link-config.yaml | 3 + notebooks/fixtures/fs/link-keys-config.yaml | 6 + notebooks/fs.ipynb | 1586 +++++++++++++++++++ notebooks/tests/test_fs.py | 100 ++ 15 files changed, 1731 insertions(+), 2 deletions(-) create mode 100644 notebooks/.gitignore create mode 100644 notebooks/fixtures/fs/copy-config.yaml create mode 100644 notebooks/fixtures/fs/copy-keys-config.yaml create mode 100644 notebooks/fixtures/fs/dir-config.yaml create mode 100644 notebooks/fixtures/fs/dir-keys-config.yaml create mode 100644 notebooks/fixtures/fs/file1.nml create mode 100644 notebooks/fixtures/fs/file2.txt create mode 100644 notebooks/fixtures/fs/file3.csv create mode 100644 notebooks/fixtures/fs/link-config.yaml create mode 100644 notebooks/fixtures/fs/link-keys-config.yaml create mode 100644 notebooks/fs.ipynb create mode 100644 notebooks/tests/test_fs.py diff --git a/.gitignore b/.gitignore index 76191013a..001bce703 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,4 @@ *.egg-info *.swp .coverage -__pycache__ -.ipynb_checkpoints \ No newline at end of file +__pycache__ \ No newline at end of file diff --git a/docs/sections/user_guide/api/fs.rst b/docs/sections/user_guide/api/fs.rst index 0ac50fc87..63fe32be9 100644 --- a/docs/sections/user_guide/api/fs.rst +++ b/docs/sections/user_guide/api/fs.rst @@ -1,5 +1,8 @@ ``uwtools.api.fs`` ================== +.. image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/ufs-community/uwtools/notebooks?labpath=notebooks%2Ffs.ipynb + .. automodule:: uwtools.api.fs :members: diff --git a/docs/sections/user_guide/index.rst b/docs/sections/user_guide/index.rst index 612a08f4d..e0ecea2f3 100644 --- a/docs/sections/user_guide/index.rst +++ b/docs/sections/user_guide/index.rst @@ -24,6 +24,7 @@ User Guide
  • API Jupyter Notebooks diff --git a/notebooks/.gitignore b/notebooks/.gitignore new file mode 100644 index 000000000..8867348fd --- /dev/null +++ b/notebooks/.gitignore @@ -0,0 +1,2 @@ +.ipynb_checkpoints +tmp \ No newline at end of file diff --git a/notebooks/fixtures/fs/copy-config.yaml b/notebooks/fixtures/fs/copy-config.yaml new file mode 100644 index 000000000..ed3879d58 --- /dev/null +++ b/notebooks/fixtures/fs/copy-config.yaml @@ -0,0 +1,3 @@ +file1-copy.nml: fixtures/fs/file1.nml +data/file2-copy.txt: fixtures/fs/file2.txt +data/file3-copy.csv: fixtures/fs/file3.csv diff --git a/notebooks/fixtures/fs/copy-keys-config.yaml b/notebooks/fixtures/fs/copy-keys-config.yaml new file mode 100644 index 000000000..9c4cdc673 --- /dev/null +++ b/notebooks/fixtures/fs/copy-keys-config.yaml @@ -0,0 +1,6 @@ +files: + to: + copy: + file1-copy.nml: fixtures/fs/file1.nml + data/file2-copy.txt: fixtures/fs/file2.txt + data/file3-copy.csv: fixtures/fs/file3.csv diff --git a/notebooks/fixtures/fs/dir-config.yaml b/notebooks/fixtures/fs/dir-config.yaml new file mode 100644 index 000000000..1faec9f52 --- /dev/null +++ b/notebooks/fixtures/fs/dir-config.yaml @@ -0,0 +1,3 @@ +makedirs: + - foo + - bar/baz diff --git a/notebooks/fixtures/fs/dir-keys-config.yaml b/notebooks/fixtures/fs/dir-keys-config.yaml new file mode 100644 index 000000000..c18ae1478 --- /dev/null +++ b/notebooks/fixtures/fs/dir-keys-config.yaml @@ -0,0 +1,6 @@ +path: + to: + dirs: + makedirs: + - foo/bar + - baz diff --git a/notebooks/fixtures/fs/file1.nml b/notebooks/fixtures/fs/file1.nml new file mode 100644 index 000000000..f85ada67c --- /dev/null +++ b/notebooks/fixtures/fs/file1.nml @@ -0,0 +1,6 @@ +&animal + name = 'zebra' + num_legs = 4 + diet_type = 'herbivore' + location = 'Africa' +/ diff --git a/notebooks/fixtures/fs/file2.txt b/notebooks/fixtures/fs/file2.txt new file mode 100644 index 000000000..36fd2ad7e --- /dev/null +++ b/notebooks/fixtures/fs/file2.txt @@ -0,0 +1 @@ +Fun Fact: A group of zebras is called a "zeal" or a "dazzle". diff --git a/notebooks/fixtures/fs/file3.csv b/notebooks/fixtures/fs/file3.csv new file mode 100644 index 000000000..6556f0598 --- /dev/null +++ b/notebooks/fixtures/fs/file3.csv @@ -0,0 +1,4 @@ +id,location,age +B524,Botswana,12 +N290,Namibia,4 +K296,Kenya,23 diff --git a/notebooks/fixtures/fs/link-config.yaml b/notebooks/fixtures/fs/link-config.yaml new file mode 100644 index 000000000..401e77fda --- /dev/null +++ b/notebooks/fixtures/fs/link-config.yaml @@ -0,0 +1,3 @@ +file1-link.nml: fixtures/fs/file1.nml +file2-link.txt: fixtures/fs/file2.txt +data/file3-link.csv: fixtures/fs/file3.csv diff --git a/notebooks/fixtures/fs/link-keys-config.yaml b/notebooks/fixtures/fs/link-keys-config.yaml new file mode 100644 index 000000000..6abffc60c --- /dev/null +++ b/notebooks/fixtures/fs/link-keys-config.yaml @@ -0,0 +1,6 @@ +files: + to: + link: + file1-link.nml: fixtures/fs/file1.nml + file2-link.txt: fixtures/fs/file2.txt + data/file3-link.csv: fixtures/fs/file3.csv diff --git a/notebooks/fs.ipynb b/notebooks/fs.ipynb new file mode 100644 index 000000000..6f42297a0 --- /dev/null +++ b/notebooks/fs.ipynb @@ -0,0 +1,1586 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2c013e12-a6c6-4786-aa50-900f7da77e6b", + "metadata": {}, + "source": [ + "# File System Tool\n", + "\n", + "The `uwtools` API's `fs` module provides functions to copy and link files as well as create directories. \n", + "\n", + "For more information, please see the uwtools.api.fs Read the Docs page.\n", + "\n", + "## Table of Contents\n", + "\n", + "* [Copying Files](#Copying-Files)\n", + " * [Failing to copy](#Failing-to-copy)\n", + " * [Using the `keys` parameter](#Using-the-keys-parameter)\n", + " * [Using the `Copier` class](#Using-the-Copier-class)\n", + "* [Linking Files](#Linking-files)\n", + " * [Failing to link](#Failing-to-link)\n", + " * [Using the `keys` parameter](#Using-the-keys-parameter-)\n", + " * [Using the `Linker` class](#Using-the-Linker-class)\n", + "* [Creating directories](#Creating-directories)\n", + " * [Using the `keys` parameter](#Using-the-keys-parameter--)\n", + " * [Using the `MakeDirs` class](#Using-the-MakeDirs-class)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "77864d80-e6f4-48c2-a5d5-88fc512106a9", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from shutil import rmtree\n", + "from uwtools.api import fs\n", + "from uwtools.api.logging import use_uwtools_logger\n", + "\n", + "use_uwtools_logger()" + ] + }, + { + "cell_type": "markdown", + "id": "354cf476-720e-4352-8954-0752fd05250f", + "metadata": {}, + "source": [ + "## Copying Files\n", + "\n", + "The `copy()` function copies files, automatically creating parent directories as needed." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "834654da-dfa9-4997-bcc5-846420381b18", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function copy in module uwtools.api.fs:\n", + "\n", + "copy(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False, stdin_ok: bool = False) -> bool\n", + " Copy files.\n", + "\n", + " :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", + " :param target_dir: Path to target directory.\n", + " :param cycle: A datetime object to make available for use in the config.\n", + " :param leadtime: A timedelta object to make available for use in the config.\n", + " :param keys: YAML keys leading to file dst/src block.\n", + " :param dry_run: Do not copy files.\n", + " :param stdin_ok: OK to read from ``stdin``?\n", + " :return: ``True`` if all copies were created.\n", + "\n" + ] + } + ], + "source": [ + "help(fs.copy)" + ] + }, + { + "cell_type": "markdown", + "id": "0585971b-47c6-48aa-9f1f-d5890cbb2061", + "metadata": {}, + "source": [ + "Files to be copied are specified by a mapping from keys destination-pathname keys to source-pathname values, either in a YAML file or a a Python ``dict``." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a959522f-d769-48c6-918d-d42776b3600a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "file1-copy.nml: fixtures/fs/file1.nml\n", + "data/file2-copy.txt: fixtures/fs/file2.txt\n", + "data/file3-copy.csv: fixtures/fs/file3.csv\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/fs/copy-config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "5885fd76-78de-4814-ad7b-dfd6df18a07d", + "metadata": {}, + "source": [ + "With these instructions, `copy()` creates a copy of each given file with the given name and in the given subdirectory. Copies are created in the directory indicated by `target_dir`. Paths can be provided either as a string or Path object. Any directories in the targeted paths for copying will be created if they don't already exist. `True` is returned upon a successful copy." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a6aff6e3-815c-496e-81d7-d8756be9c232", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:21:27] INFO Validating config against internal schema: files-to-stage\n", + "[2024-08-30T15:21:27] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:21:27] INFO File copies: Initial state: Pending\n", + "[2024-08-30T15:21:27] INFO File copies: Checking requirements\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Initial state: Pending\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Checking requirements\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Requirement(s) ready\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Executing\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Final state: Ready\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Initial state: Pending\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Checking requirements\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Requirement(s) ready\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Executing\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Final state: Ready\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Initial state: Pending\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Checking requirements\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Requirement(s) ready\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Executing\n", + "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Final state: Ready\n", + "[2024-08-30T15:21:27] INFO File copies: Final state: Ready\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rmtree(\"tmp/copy-target\", ignore_errors=True)\n", + "fs.copy(\n", + " config=\"fixtures/fs/copy-config.yaml\",\n", + " target_dir=Path(\"tmp/copy-target\")\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "5c05369f-3cf6-4576-8689-df98ed9d151d", + "metadata": {}, + "source": [ + "Examining the target directory, we can see that the copies of the files have been made with their specified names and in their specified directories." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e67fdb49-beef-4006-9e36-1a22829f21fc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/copy-target\u001b[0m\n", + "├── \u001b[01;34mdata\u001b[0m\n", + "│   ├── file2-copy.txt\n", + "│   └── file3-copy.csv\n", + "└── file1-copy.nml\n", + "\n", + "2 directories, 3 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/copy-target" + ] + }, + { + "cell_type": "markdown", + "id": "90e17445-3d87-4894-8211-8c737f7579d6", + "metadata": {}, + "source": [ + "### Failing to copy\n", + "\n", + "A configuration can be provided as a dictionary instead as this example demonstrates. However, `missing-file.nml` does not exist. The function provides a warning and returns `False`." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b1fa6662-c4f3-4f7a-9b5d-8ee258cd6e0e", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:21:35] INFO Validating config against internal schema: files-to-stage\n", + "[2024-08-30T15:21:35] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:21:35] INFO File copies: Initial state: Pending\n", + "[2024-08-30T15:21:35] INFO File copies: Checking requirements\n", + "[2024-08-30T15:21:35] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Initial state: Pending\n", + "[2024-08-30T15:21:35] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Checking requirements\n", + "[2024-08-30T15:21:35] WARNING File fixtures/fs/missing-file.nml: State: Pending (EXTERNAL)\n", + "[2024-08-30T15:21:35] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Requirement(s) pending\n", + "[2024-08-30T15:21:35] WARNING Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Final state: Pending\n", + "[2024-08-30T15:21:35] WARNING File copies: Final state: Pending\n" + ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fs.copy(\n", + " config={\"missing-copy.nml\":\"fixtures/fs/missing-file.nml\"},\n", + " target_dir=\"tmp/copy-target\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "6f93ced5-0953-462b-9739-e74333c94e64", + "metadata": {}, + "source": [ + "The missing copy does not appear in the target directory." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ab122bde-f483-4981-8308-fc6d4a90e50d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/copy-target\u001b[0m\n", + "├── \u001b[01;34mdata\u001b[0m\n", + "│   ├── file2-copy.txt\n", + "│   └── file3-copy.csv\n", + "└── file1-copy.nml\n", + "\n", + "2 directories, 3 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/copy-target" + ] + }, + { + "cell_type": "markdown", + "id": "b2527839-c217-428d-a686-c684a682c0e8", + "metadata": {}, + "source": [ + "### Using the `keys` parameter\n", + "\n", + "Consider the following configuration, in which the destination/source mapping is not located at the top level of the configuration:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "1f567844-ff8d-4e7f-87be-dffae9e15643", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "files:\n", + " to:\n", + " copy:\n", + " file1-copy.nml: fixtures/fs/file1.nml\n", + " data/file2-copy.txt: fixtures/fs/file2.txt\n", + " data/file3-copy.csv: fixtures/fs/file3.csv\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/fs/copy-keys-config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "5311866f-a1f5-4243-81a8-2c52172e091a", + "metadata": {}, + "source": [ + "Without additional information, `copy()` would raise a `UWConfigError` given this configuration. However, the list of keys leading to the destination/source mapping can be provided with the `keys` parameter:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "dda3e407-a1a2-4b11-823a-3b6fdc39f67a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:21:47] INFO Validating config against internal schema: files-to-stage\n", + "[2024-08-30T15:21:47] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:21:47] INFO File copies: Initial state: Pending\n", + "[2024-08-30T15:21:47] INFO File copies: Checking requirements\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Initial state: Pending\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Checking requirements\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Requirement(s) ready\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Executing\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Final state: Ready\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Initial state: Pending\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Checking requirements\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Requirement(s) ready\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Executing\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Final state: Ready\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Initial state: Pending\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Checking requirements\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Requirement(s) ready\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Executing\n", + "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Final state: Ready\n", + "[2024-08-30T15:21:47] INFO File copies: Final state: Ready\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rmtree(\"tmp/copy-keys-target\", ignore_errors=True)\n", + "fs.copy(\n", + " config=\"fixtures/fs/copy-keys-config.yaml\",\n", + " target_dir=\"tmp/copy-keys-target\",\n", + " keys=[\"files\",\"to\",\"copy\"]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "d62ae369-69f9-4003-b5d3-d7b5908f23af", + "metadata": {}, + "source": [ + "With this information provided, the copy is successful." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "59c67e22-fe98-4e74-8b0b-b40e24a804e8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/copy-keys-target\u001b[0m\n", + "├── \u001b[01;34mdata\u001b[0m\n", + "│   ├── file2-copy.txt\n", + "│   └── file3-copy.csv\n", + "└── file1-copy.nml\n", + "\n", + "2 directories, 3 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/copy-keys-target" + ] + }, + { + "cell_type": "markdown", + "id": "1a1adba8-2daf-4fb6-b224-980b134f011c", + "metadata": {}, + "source": [ + "### Using the `Copier` class\n", + "\n", + "An alternative to using `copy()` is to instantiate a `Copier` object , then call its `go()` method." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7434dee7-fb52-4d9b-b2a1-d414165f3186", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on class Copier in module uwtools.fs:\n", + "\n", + "class Copier(FileStager)\n", + " | Copier(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " |\n", + " | Stage files by copying.\n", + " |\n", + " | Method resolution order:\n", + " | Copier\n", + " | FileStager\n", + " | Stager\n", + " | abc.ABC\n", + " | builtins.object\n", + " |\n", + " | Methods defined here:\n", + " |\n", + " | go = __iotaa_tasks__(*args, **kwargs) -> '_AssetT'\n", + " | Copy files.\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Data and other attributes defined here:\n", + " |\n", + " | __abstractmethods__ = frozenset()\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Methods inherited from Stager:\n", + " |\n", + " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " | Stage files and directories.\n", + " |\n", + " | :param config: YAML-file path, or dict (read stdin if missing or None).\n", + " | :param target_dir: Path to target directory.\n", + " | :param cycle: A datetime object to make available for use in the config.\n", + " | :param leadtime: A timedelta object to make available for use in the config.\n", + " | :param keys: YAML keys leading to file dst/src block.\n", + " | :param dry_run: Do not copy files.\n", + " | :raises: UWConfigError if config fails validation.\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Data descriptors inherited from Stager:\n", + " |\n", + " | __dict__\n", + " | dictionary for instance variables\n", + " |\n", + " | __weakref__\n", + " | list of weak references to the object\n", + "\n" + ] + } + ], + "source": [ + "help(fs.Copier)" + ] + }, + { + "cell_type": "markdown", + "id": "061ac341-96cb-4af6-94ce-4f1e4d342b63", + "metadata": {}, + "source": [ + "A `Copier` object is instantiated using the same parameters as `copy()`, but copying is not performed until `Copier.go()` is called." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "578cc091-c0eb-4293-8dbd-ee74a69a0940", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:21:56] INFO Validating config against internal schema: files-to-stage\n", + "[2024-08-30T15:21:56] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:21:56] INFO File copies: Initial state: Pending\n", + "[2024-08-30T15:21:56] INFO File copies: Checking requirements\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Initial state: Pending\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Checking requirements\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Requirement(s) ready\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Executing\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Final state: Ready\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Initial state: Pending\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Checking requirements\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Requirement(s) ready\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Executing\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Final state: Ready\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Initial state: Pending\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Checking requirements\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Requirement(s) ready\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Executing\n", + "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Final state: Ready\n", + "[2024-08-30T15:21:56] INFO File copies: Final state: Ready\n" + ] + }, + { + "data": { + "text/plain": [ + "[Asset(ref=PosixPath('tmp/copier-target/file1-copy.nml'), ready=),\n", + " Asset(ref=PosixPath('tmp/copier-target/data/file2-copy.txt'), ready=),\n", + " Asset(ref=PosixPath('tmp/copier-target/data/file3-copy.csv'), ready=)]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rmtree(\"tmp/copier-target\", ignore_errors=True)\n", + "copier = fs.Copier(\n", + " config=\"fixtures/fs/copy-config.yaml\",\n", + " target_dir=\"tmp/copier-target\"\n", + ")\n", + "copier.go()" + ] + }, + { + "cell_type": "markdown", + "id": "842f638a-bf97-4d40-bb40-0f37cc03ad9b", + "metadata": {}, + "source": [ + "Once `Copier.go()` is called, copies are created in the same way as they would have with `copy()`." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "c6aaac2b-bb72-433d-8ad4-349a1056cfa3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/copier-target\u001b[0m\n", + "├── \u001b[01;34mdata\u001b[0m\n", + "│   ├── file2-copy.txt\n", + "│   └── file3-copy.csv\n", + "└── file1-copy.nml\n", + "\n", + "2 directories, 3 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/copier-target" + ] + }, + { + "cell_type": "markdown", + "id": "76f144f9-0a2f-48ad-ae83-14bd7a97353e", + "metadata": {}, + "source": [ + "## Linking files\n", + "\n", + "The `link()` function creates symbolic links to files, automatically creating parent directories as needed." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "404d051e-18e1-4927-a24f-cbe98ab01ce9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function link in module uwtools.api.fs:\n", + "\n", + "link(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False, stdin_ok: bool = False) -> bool\n", + " Link files.\n", + "\n", + " :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", + " :param target_dir: Path to target directory.\n", + " :param cycle: A datetime object to make available for use in the config.\n", + " :param leadtime: A timedelta object to make available for use in the config.\n", + " :param keys: YAML keys leading to file dst/src block.\n", + " :param dry_run: Do not link files.\n", + " :param stdin_ok: OK to read from ``stdin``?\n", + " :return: ``True`` if all links were created.\n", + "\n" + ] + } + ], + "source": [ + "help(fs.link)" + ] + }, + { + "cell_type": "markdown", + "id": "710edac4-ba97-4599-a0f3-bc75ba2210e2", + "metadata": {}, + "source": [ + "Links to be created are specified by a mapping from keys destination-pathname keys to source-pathname values, either in a YAML file or a Python ``dict``." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "097b896c-aef4-48ac-aea5-eb2d463d172b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "file1-link.nml: fixtures/fs/file1.nml\n", + "file2-link.txt: fixtures/fs/file2.txt\n", + "data/file3-link.csv: fixtures/fs/file3.csv\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/fs/link-config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "1acd36eb-a5e7-4451-9d22-bfe8798cb4b0", + "metadata": {}, + "source": [ + "With these instructions, `link()` creates a symbolic link of each given file with the given name and in the given subdirectory. Links are created in the directory indicated by `target_dir`. Paths can be provided either as a string or Path object. Any directories in the targeted paths will be created if they don't already exist. `True` is returned upon a successful run." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "b6317f8a-c5fb-4114-93fa-236df3fd8805", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:22:08] INFO Validating config against internal schema: files-to-stage\n", + "[2024-08-30T15:22:08] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:22:08] INFO File links: Initial state: Pending\n", + "[2024-08-30T15:22:08] INFO File links: Checking requirements\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Pending\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Pending\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Pending\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", + "[2024-08-30T15:22:08] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", + "[2024-08-30T15:22:08] INFO File links: Final state: Ready\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rmtree(\"tmp/link-target\", ignore_errors=True)\n", + "fs.link(\n", + " config=Path(\"fixtures/fs/link-config.yaml\"),\n", + " target_dir=\"tmp/link-target\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "28fe0009-11cb-4ec4-b203-221e2a59cedb", + "metadata": {}, + "source": [ + "Examining the target directory, we can see that the links have been created with their specified names and in their specified directories." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "b31ca50e-01c4-4665-81e0-de70a75ceb2a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/link-target\u001b[0m\n", + "├── \u001b[01;34mdata\u001b[0m\n", + "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> ../../../fixtures/fs/file3.csv\n", + "├── \u001b[01;36mfile1-link.nml\u001b[0m -> ../../fixtures/fs/file1.nml\n", + "└── \u001b[01;36mfile2-link.txt\u001b[0m -> ../../fixtures/fs/file2.txt\n", + "\n", + "2 directories, 3 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/link-target" + ] + }, + { + "cell_type": "markdown", + "id": "e0661083-6e83-490f-82c5-19098b0f1b3c", + "metadata": {}, + "source": [ + "### Failing to link\n", + "\n", + "A configuration can be provided as a dictionary instead as this example demonstrates. However, `missing-file.nml` does not exist. The function provides a warning and returns `False`." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "59d93133-891d-4903-a965-23607cc72474", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:22:11] INFO Validating config against internal schema: files-to-stage\n", + "[2024-08-30T15:22:11] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:22:11] INFO File links: Initial state: Pending\n", + "[2024-08-30T15:22:11] INFO File links: Checking requirements\n", + "[2024-08-30T15:22:11] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Initial state: Pending\n", + "[2024-08-30T15:22:11] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Checking requirements\n", + "[2024-08-30T15:22:11] WARNING Filesystem item fixtures/fs/missing-file.nml: State: Pending (EXTERNAL)\n", + "[2024-08-30T15:22:11] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Requirement(s) pending\n", + "[2024-08-30T15:22:11] WARNING Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Final state: Pending\n", + "[2024-08-30T15:22:11] WARNING File links: Final state: Pending\n" + ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fs.link(\n", + " config={\"missing-link.nml\":\"fixtures/fs/missing-file.nml\"},\n", + " target_dir=\"tmp/link-target\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "7a3ca573-c160-4afa-8a96-4165b01eecfe", + "metadata": {}, + "source": [ + "The missing link does not appear in the target directory." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "7a6e94b9-1161-4f41-9333-55736aec07b3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/link-target\u001b[0m\n", + "├── \u001b[01;34mdata\u001b[0m\n", + "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> ../../../fixtures/fs/file3.csv\n", + "├── \u001b[01;36mfile1-link.nml\u001b[0m -> ../../fixtures/fs/file1.nml\n", + "└── \u001b[01;36mfile2-link.txt\u001b[0m -> ../../fixtures/fs/file2.txt\n", + "\n", + "2 directories, 3 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/link-target" + ] + }, + { + "cell_type": "markdown", + "id": "b887c95e-f71f-4a26-b709-d410a3c30c2e", + "metadata": {}, + "source": [ + "### Using the `keys` parameter \n", + "\n", + "Consider the following configuration, in which the destination/source mapping is not located at the top level of the configuration:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "1de6cbd4-3b10-4b18-a8a5-c0cd21064bd3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "files:\n", + " to:\n", + " link:\n", + " file1-link.nml: fixtures/fs/file1.nml\n", + " file2-link.txt: fixtures/fs/file2.txt\n", + " data/file3-link.csv: fixtures/fs/file3.csv\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/fs/link-keys-config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "9977ee46-17da-419e-821b-a32fac5139f8", + "metadata": {}, + "source": [ + "Without additional information, `link()` would raise a `UWConfigError` given this configuration. However, the list of keys leading to the destination/source mapping can be provided with the `keys` parameter:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "ee4bf2a3-4101-4d95-afd5-120e95e64550", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:22:22] INFO Validating config against internal schema: files-to-stage\n", + "[2024-08-30T15:22:22] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:22:22] INFO File links: Initial state: Pending\n", + "[2024-08-30T15:22:22] INFO File links: Checking requirements\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Pending\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Pending\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Pending\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", + "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", + "[2024-08-30T15:22:22] INFO File links: Final state: Ready\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rmtree(\"tmp/link-keys-target\", ignore_errors=True)\n", + "fs.link(\n", + " config=\"fixtures/fs/link-keys-config.yaml\",\n", + " target_dir=\"tmp/link-keys-target\",\n", + " keys=[\"files\",\"to\",\"link\"]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "d331d715-b6f1-4a9a-a207-2fb296aec4af", + "metadata": {}, + "source": [ + "With this information provided, the links are successfully created." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "1393ae73-798b-49c0-9b68-e8ed28ad1df0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/link-keys-target\u001b[0m\n", + "├── \u001b[01;34mdata\u001b[0m\n", + "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> ../../../fixtures/fs/file3.csv\n", + "├── \u001b[01;36mfile1-link.nml\u001b[0m -> ../../fixtures/fs/file1.nml\n", + "└── \u001b[01;36mfile2-link.txt\u001b[0m -> ../../fixtures/fs/file2.txt\n", + "\n", + "2 directories, 3 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/link-keys-target" + ] + }, + { + "cell_type": "markdown", + "id": "29a9457a-e4f3-460a-b873-cf1bf236c9de", + "metadata": {}, + "source": [ + "### Using the `Linker` class\n", + "\n", + "An alternative to using `link()` is to instantiate a `Linker` object , then call its `go()` method." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "b1e5d3a2-7003-4449-9483-440236f66df7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on class Linker in module uwtools.fs:\n", + "\n", + "class Linker(FileStager)\n", + " | Linker(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " |\n", + " | Stage files by linking.\n", + " |\n", + " | Method resolution order:\n", + " | Linker\n", + " | FileStager\n", + " | Stager\n", + " | abc.ABC\n", + " | builtins.object\n", + " |\n", + " | Methods defined here:\n", + " |\n", + " | go = __iotaa_tasks__(*args, **kwargs) -> '_AssetT'\n", + " | Link files.\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Data and other attributes defined here:\n", + " |\n", + " | __abstractmethods__ = frozenset()\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Methods inherited from Stager:\n", + " |\n", + " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " | Stage files and directories.\n", + " |\n", + " | :param config: YAML-file path, or dict (read stdin if missing or None).\n", + " | :param target_dir: Path to target directory.\n", + " | :param cycle: A datetime object to make available for use in the config.\n", + " | :param leadtime: A timedelta object to make available for use in the config.\n", + " | :param keys: YAML keys leading to file dst/src block.\n", + " | :param dry_run: Do not copy files.\n", + " | :raises: UWConfigError if config fails validation.\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Data descriptors inherited from Stager:\n", + " |\n", + " | __dict__\n", + " | dictionary for instance variables\n", + " |\n", + " | __weakref__\n", + " | list of weak references to the object\n", + "\n" + ] + } + ], + "source": [ + "help(fs.Linker)" + ] + }, + { + "cell_type": "markdown", + "id": "3312a98b-9f5d-41bd-ad02-f69d291cc947", + "metadata": {}, + "source": [ + "A `Linker` object is instantiated using the same parameters as `link()`, but links are not created until `Linker.go()` is called." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "ecfa9e89-9fbd-4352-babc-dfa5b91afe6a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:22:28] INFO Validating config against internal schema: files-to-stage\n", + "[2024-08-30T15:22:28] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:22:28] INFO File links: Initial state: Pending\n", + "[2024-08-30T15:22:28] INFO File links: Checking requirements\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Pending\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Pending\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Pending\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", + "[2024-08-30T15:22:28] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", + "[2024-08-30T15:22:28] INFO File links: Final state: Ready\n" + ] + }, + { + "data": { + "text/plain": [ + "[Asset(ref=PosixPath('tmp/linker-target/file1-link.nml'), ready=),\n", + " Asset(ref=PosixPath('tmp/linker-target/file2-link.txt'), ready=),\n", + " Asset(ref=PosixPath('tmp/linker-target/data/file3-link.csv'), ready=)]" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rmtree(\"tmp/linker-target\", ignore_errors=True)\n", + "linker = fs.Linker(\n", + " config=\"fixtures/fs/link-config.yaml\",\n", + " target_dir=\"tmp/linker-target\"\n", + ")\n", + "linker.go()" + ] + }, + { + "cell_type": "markdown", + "id": "8d2cbb32-cabb-498e-b4db-414e3ac2cf1d", + "metadata": {}, + "source": [ + "Once `Linker.go()` is called, links are created in the same way as they would have with `link()`." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "6695f7bb-7ab7-42d1-9d2c-0bef7341147d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/linker-target\u001b[0m\n", + "├── \u001b[01;34mdata\u001b[0m\n", + "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> ../../../fixtures/fs/file3.csv\n", + "├── \u001b[01;36mfile1-link.nml\u001b[0m -> ../../fixtures/fs/file1.nml\n", + "└── \u001b[01;36mfile2-link.txt\u001b[0m -> ../../fixtures/fs/file2.txt\n", + "\n", + "2 directories, 3 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/linker-target" + ] + }, + { + "cell_type": "markdown", + "id": "bd367e2a-e44c-4a5c-9600-4d86719f7d36", + "metadata": {}, + "source": [ + "## Creating directories\n", + "\n", + "The `makedirs()` function creates directories." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "43b381d1-8dc2-4ea6-924c-e21149f05e7f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function makedirs in module uwtools.api.fs:\n", + "\n", + "makedirs(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False, stdin_ok: bool = False) -> bool\n", + " Make directories.\n", + "\n", + " :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", + " :param target_dir: Path to target directory.\n", + " :param cycle: A datetime object to make available for use in the config.\n", + " :param leadtime: A timedelta object to make available for use in the config.\n", + " :param keys: YAML keys leading to file dst/src block.\n", + " :param dry_run: Do not link files.\n", + " :param stdin_ok: OK to read from ``stdin``?\n", + " :return: ``True`` if all directories were made.\n", + "\n" + ] + } + ], + "source": [ + "help(fs.makedirs)" + ] + }, + { + "cell_type": "markdown", + "id": "83b88d7e-f4cf-4358-98f9-106b47bd5d9f", + "metadata": {}, + "source": [ + "Directories to be created are specified by either a configuration YAML file or a Python ``dict``. A `makedirs` key must be included with a list of directories to create as its value." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "2f946927-509f-4cd6-a7ec-2d36f4d17318", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "makedirs:\n", + " - foo\n", + " - bar/baz\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/fs/dir-config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "eec8938c-1e2a-482b-aa0e-9e6e89dcf200", + "metadata": {}, + "source": [ + "With these instructions, `makedirs()` creates each directory in the list within the directory indicated by `target_dir`. Paths can be provided either as a string or Path object. `True` is returned upon a successful run." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "fdd4e832-3bc5-4c7a-9b31-e387a4e7d48b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:22:34] INFO Validating config against internal schema: makedirs\n", + "[2024-08-30T15:22:34] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:22:34] INFO Directories: Initial state: Pending\n", + "[2024-08-30T15:22:34] INFO Directories: Checking requirements\n", + "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/foo: Initial state: Pending\n", + "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/foo: Checking requirements\n", + "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/foo: Requirement(s) ready\n", + "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/foo: Executing\n", + "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/foo: Final state: Ready\n", + "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/bar/baz: Initial state: Pending\n", + "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/bar/baz: Checking requirements\n", + "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/bar/baz: Requirement(s) ready\n", + "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/bar/baz: Executing\n", + "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/bar/baz: Final state: Ready\n", + "[2024-08-30T15:22:34] INFO Directories: Final state: Ready\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rmtree(\"tmp/dir-target\", ignore_errors=True)\n", + "fs.makedirs(\n", + " config=\"fixtures/fs/dir-config.yaml\",\n", + " target_dir=Path(\"tmp/dir-target\")\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "df4245f6-7083-45ee-b56e-9697a50db5da", + "metadata": {}, + "source": [ + "Examining the target directory, we can see that the directories have been created with their specified names." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "c00ec8cc-964a-498e-bd8f-a3686a468dc3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/dir-target\u001b[0m\n", + "├── \u001b[01;34mbar\u001b[0m\n", + "│   └── \u001b[01;34mbaz\u001b[0m\n", + "└── \u001b[01;34mfoo\u001b[0m\n", + "\n", + "4 directories, 0 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/dir-target" + ] + }, + { + "cell_type": "markdown", + "id": "329e939d-0f6d-412c-a36a-4682fe99609a", + "metadata": {}, + "source": [ + "### Using the `keys` parameter \n", + "\n", + "Consider the following configuration, in which the destination/source mapping is not located at the top level of the configuration:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "3a93956d-0acf-4c37-87bf-83c0d5287644", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "path:\n", + " to:\n", + " dirs:\n", + " makedirs:\n", + " - foo/bar\n", + " - baz\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/fs/dir-keys-config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "909ff3ea-8577-4b91-94fc-6ce6effe4bec", + "metadata": {}, + "source": [ + "Without additional information, `makedirs()` would raise a `UWConfigError` given this configuration. However, the list of keys leading to the destination/source mapping can be provided with the `keys` parameter:" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "91549822-e85e-4d41-8860-1da05d713f75", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:22:42] INFO Validating config against internal schema: makedirs\n", + "[2024-08-30T15:22:42] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:22:42] INFO Directories: Initial state: Pending\n", + "[2024-08-30T15:22:42] INFO Directories: Checking requirements\n", + "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/foo/bar: Initial state: Pending\n", + "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/foo/bar: Checking requirements\n", + "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/foo/bar: Requirement(s) ready\n", + "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/foo/bar: Executing\n", + "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/foo/bar: Final state: Ready\n", + "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/baz: Initial state: Pending\n", + "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/baz: Checking requirements\n", + "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/baz: Requirement(s) ready\n", + "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/baz: Executing\n", + "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/baz: Final state: Ready\n", + "[2024-08-30T15:22:42] INFO Directories: Final state: Ready\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rmtree(\"tmp/dir-keys-target\", ignore_errors=True)\n", + "fs.makedirs(\n", + " config=\"fixtures/fs/dir-keys-config.yaml\",\n", + " target_dir=\"tmp/dir-keys-target\",\n", + " keys=[\"path\",\"to\",\"dirs\"]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "5b7ed362-2edb-438a-bb9f-e2dd7d505379", + "metadata": {}, + "source": [ + "With this information provided, the directories are successfully created." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "cb4ded9c-0de1-4010-af75-fbb7becd3fbc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/dir-keys-target\u001b[0m\n", + "├── \u001b[01;34mbaz\u001b[0m\n", + "└── \u001b[01;34mfoo\u001b[0m\n", + " └── \u001b[01;34mbar\u001b[0m\n", + "\n", + "4 directories, 0 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/dir-keys-target" + ] + }, + { + "cell_type": "markdown", + "id": "742ce55e-fded-4961-931d-49bd75c09901", + "metadata": {}, + "source": [ + "### Using the `MakeDirs` class\n", + "\n", + "An alternative to using `makedirs()` is to instantiate a `MakeDirs` object , then call its `go()` method." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "7fe53736-d8e8-4ca9-ab2b-87729934fc19", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on class MakeDirs in module uwtools.fs:\n", + "\n", + "class MakeDirs(Stager)\n", + " | MakeDirs(config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " |\n", + " | Make directories.\n", + " |\n", + " | Method resolution order:\n", + " | MakeDirs\n", + " | Stager\n", + " | abc.ABC\n", + " | builtins.object\n", + " |\n", + " | Methods defined here:\n", + " |\n", + " | go = __iotaa_tasks__(*args, **kwargs) -> '_AssetT'\n", + " | Make directories.\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Data and other attributes defined here:\n", + " |\n", + " | __abstractmethods__ = frozenset()\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Methods inherited from Stager:\n", + " |\n", + " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", + " | Stage files and directories.\n", + " |\n", + " | :param config: YAML-file path, or dict (read stdin if missing or None).\n", + " | :param target_dir: Path to target directory.\n", + " | :param cycle: A datetime object to make available for use in the config.\n", + " | :param leadtime: A timedelta object to make available for use in the config.\n", + " | :param keys: YAML keys leading to file dst/src block.\n", + " | :param dry_run: Do not copy files.\n", + " | :raises: UWConfigError if config fails validation.\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Data descriptors inherited from Stager:\n", + " |\n", + " | __dict__\n", + " | dictionary for instance variables\n", + " |\n", + " | __weakref__\n", + " | list of weak references to the object\n", + "\n" + ] + } + ], + "source": [ + "help(fs.MakeDirs)" + ] + }, + { + "cell_type": "markdown", + "id": "cf3b50de-4a8b-4f51-96bb-a477b2c53430", + "metadata": {}, + "source": [ + "A `MakeDirs` object is instantiated using the same parameters as `makedirs()`, but directories are not created until `MakeDirs.go()` is called." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "950d6b43-6db7-40df-b645-beaa1369cfa4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-08-30T15:22:48] INFO Validating config against internal schema: makedirs\n", + "[2024-08-30T15:22:48] INFO 0 UW schema-validation errors found\n", + "[2024-08-30T15:22:48] INFO Directories: Initial state: Pending\n", + "[2024-08-30T15:22:48] INFO Directories: Checking requirements\n", + "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/foo: Initial state: Pending\n", + "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/foo: Checking requirements\n", + "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/foo: Requirement(s) ready\n", + "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/foo: Executing\n", + "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/foo: Final state: Ready\n", + "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/bar/baz: Initial state: Pending\n", + "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/bar/baz: Checking requirements\n", + "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/bar/baz: Requirement(s) ready\n", + "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/bar/baz: Executing\n", + "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/bar/baz: Final state: Ready\n", + "[2024-08-30T15:22:48] INFO Directories: Final state: Ready\n" + ] + }, + { + "data": { + "text/plain": [ + "[Asset(ref=PosixPath('tmp/makedirs-target/foo'), ready=),\n", + " Asset(ref=PosixPath('tmp/makedirs-target/bar/baz'), ready=)]" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rmtree(\"tmp/makedirs-target\", ignore_errors=True)\n", + "dirs_stager = fs.MakeDirs(\n", + " config=\"fixtures/fs/dir-config.yaml\",\n", + " target_dir=\"tmp/makedirs-target\"\n", + ")\n", + "dirs_stager.go()" + ] + }, + { + "cell_type": "markdown", + "id": "9ae5f357-1d56-4670-a8d7-8546e73c4efa", + "metadata": {}, + "source": [ + "Once `MakeDirs.go()` is called, directories are created in the same way as they would have with `makedirs()`." + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "a61fb9ac-df2f-4e39-9f66-bfb789c39117", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mtmp/makedirs-target\u001b[0m\n", + "├── \u001b[01;34mbar\u001b[0m\n", + "│   └── \u001b[01;34mbaz\u001b[0m\n", + "└── \u001b[01;34mfoo\u001b[0m\n", + "\n", + "4 directories, 0 files\n" + ] + } + ], + "source": [ + "%%bash\n", + "tree tmp/makedirs-target" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:DEV-uwtools] *", + "language": "python", + "name": "conda-env-DEV-uwtools-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/tests/test_fs.py b/notebooks/tests/test_fs.py new file mode 100644 index 000000000..b5b0b2607 --- /dev/null +++ b/notebooks/tests/test_fs.py @@ -0,0 +1,100 @@ +import os + +import yaml +from testbook import testbook + + +def test_copy(): + # Get the config files as text and dictionaries. + with open("fixtures/fs/copy-config.yaml", "r", encoding="utf-8") as f: + config_str = f.read().rstrip() + config_dict = yaml.safe_load(config_str) + with open("fixtures/fs/copy-keys-config.yaml", "r", encoding="utf-8") as f: + config_keys_str = f.read().rstrip() + config_keys_dict = yaml.safe_load(config_keys_str) + with testbook("fs.ipynb", execute=True) as tb: + # Each key in each config should have created a copy of the file given by each value. + for item in config_dict.items(): + with open("tmp/copy-target/" + item[0], "r", encoding="utf-8") as f: + copy_dst_txt = f.read().rstrip() + with open("tmp/copier-target/" + item[0], "r", encoding="utf-8") as f: + copier_dst_txt = f.read().rstrip() + with open(item[1], "r", encoding="utf-8") as f: + src_txt = f.read().rstrip() + assert copy_dst_txt == src_txt + assert copier_dst_txt == src_txt + for item in config_keys_dict["files"]["to"]["copy"].items(): + with open("tmp/copy-keys-target/" + item[0], "r", encoding="utf-8") as f: + copy_keys_dst_txt = f.read().rstrip() + with open(item[1], "r", encoding="utf-8") as f: + src_txt = f.read().rstrip() + assert copy_keys_dst_txt == src_txt + # Ensure that cell output text matches expectations. + assert tb.cell_output_text(5) == config_str + assert "True" in tb.cell_output_text(7) + assert "False" in tb.cell_output_text(11) + assert tb.cell_output_text(13) == tb.cell_output_text(9) + assert tb.cell_output_text(15) == config_keys_str + assert "True" in tb.cell_output_text(17) + + +def test_link(): + # Get the config files as text and dictionaries. + with open("fixtures/fs/link-config.yaml", "r", encoding="utf-8") as f: + config_str = f.read().rstrip() + config_dict = yaml.safe_load(config_str) + with open("fixtures/fs/link-keys-config.yaml", "r", encoding="utf-8") as f: + config_keys_str = f.read().rstrip() + config_keys_dict = yaml.safe_load(config_keys_str) + with testbook("fs.ipynb", execute=True) as tb: + # Each key in each config should have created a symlink of the file given by each value. + for item in config_dict.items(): + link_path = "tmp/link-target/" + item[0] + linker_path = "tmp/linker-target/" + item[0] + with open(link_path, "r", encoding="utf-8") as f: + link_dst_txt = f.read().rstrip() + with open(linker_path, "r", encoding="utf-8") as f: + linker_dst_txt = f.read().rstrip() + with open(item[1], "r", encoding="utf-8") as f: + src_txt = f.read().rstrip() + assert os.path.islink(link_path) + assert link_dst_txt == src_txt + assert os.path.islink(linker_path) + assert linker_dst_txt == src_txt + for item in config_keys_dict["files"]["to"]["link"].items(): + link_keys_path = "tmp/link-keys-target/" + item[0] + with open(link_keys_path, "r", encoding="utf-8") as f: + link_keys_dst_txt = f.read().rstrip() + with open(item[1], "r", encoding="utf-8") as f: + src_txt = f.read().rstrip() + assert os.path.islink(link_keys_path) + assert link_keys_dst_txt == src_txt + # Ensure that cell output text matches expectations. + assert tb.cell_output_text(29) == config_str + assert "True" in tb.cell_output_text(31) + assert "False" in tb.cell_output_text(35) + assert tb.cell_output_text(37) == tb.cell_output_text(33) + assert tb.cell_output_text(39) == config_keys_str + assert "True" in tb.cell_output_text(41) + + +def test_makedirs(): + # Get the config files as text and dictionaries. + with open("fixtures/fs/dir-config.yaml", "r", encoding="utf-8") as f: + config_str = f.read().rstrip() + config_dict = yaml.safe_load(config_str) + with open("fixtures/fs/dir-keys-config.yaml", "r", encoding="utf-8") as f: + config_keys_str = f.read().rstrip() + config_keys_dict = yaml.safe_load(config_keys_str) + with testbook("fs.ipynb", execute=True) as tb: + # Each value in each config should have been created as one or more subdirectories. + for subdir in config_dict["makedirs"]: + assert os.path.exists("tmp/dir-target/" + subdir) + assert os.path.exists("tmp/makedirs-target/" + subdir) + for subdir in config_keys_dict["path"]["to"]["dirs"]["makedirs"]: + assert os.path.exists("tmp/dir-keys-target/" + subdir) + # Ensure that cell output text matches expectations. + assert tb.cell_output_text(53) == config_str + assert "True" in tb.cell_output_text(55) + assert tb.cell_output_text(59) == config_keys_str + assert "True" in tb.cell_output_text(61) From 3561964a823aba0597b9f6b92d85122b03731604 Mon Sep 17 00:00:00 2001 From: Travis Byrne <42480803+Byrnetp@users.noreply.github.com> Date: Wed, 25 Sep 2024 11:04:24 -0600 Subject: [PATCH 12/24] UW-667 Jupyter notebook: Config Tool Part I (#607) * WIP * WIP * WIP * Add config notebook, unit tests, Binder link & button * add newline * add markdown comment * add suggestions from review * apply suggestion from review * updating markdown comments and RtD links * Apply suggestions from code review Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> * change error demo style and update section links * Apply suggestions from review * update testing * update RtD links and add uwtools version --------- Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> --- docs/sections/user_guide/api/config.rst | 3 + docs/sections/user_guide/index.rst | 3 +- notebooks/config.ipynb | 996 +++++++++++++++++++++ notebooks/fixtures/config/base-config.nml | 5 + notebooks/fixtures/config/get-config.yaml | 2 + notebooks/fixtures/config/keys-config.yaml | 4 + notebooks/template.ipynb | 2 +- notebooks/tests/test_config.py | 73 ++ 8 files changed, 1086 insertions(+), 2 deletions(-) create mode 100644 notebooks/config.ipynb create mode 100644 notebooks/fixtures/config/base-config.nml create mode 100644 notebooks/fixtures/config/get-config.yaml create mode 100644 notebooks/fixtures/config/keys-config.yaml create mode 100644 notebooks/tests/test_config.py diff --git a/docs/sections/user_guide/api/config.rst b/docs/sections/user_guide/api/config.rst index 5f689e1e7..0de57f368 100644 --- a/docs/sections/user_guide/api/config.rst +++ b/docs/sections/user_guide/api/config.rst @@ -1,6 +1,9 @@ ``uwtools.api.config`` ====================== +.. image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/ufs-community/uwtools/notebooks?labpath=notebooks%2Fconfig.ipynb + .. automodule:: uwtools.api.config :inherited-members: UserDict :members: diff --git a/docs/sections/user_guide/index.rst b/docs/sections/user_guide/index.rst index e0ecea2f3..31b4eed42 100644 --- a/docs/sections/user_guide/index.rst +++ b/docs/sections/user_guide/index.rst @@ -23,8 +23,9 @@ User Guide diff --git a/notebooks/config.ipynb b/notebooks/config.ipynb new file mode 100644 index 000000000..bba503764 --- /dev/null +++ b/notebooks/config.ipynb @@ -0,0 +1,996 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d023d283-7e1d-4e75-95b8-5033bea42a59", + "metadata": {}, + "source": [ + "# Config Tool\n", + "\n", + "The `uwtools` API's `config` module provides functions to create and manipulate configuration files, objects, and dictionaries.\n", + "\n", + "Tested on `uwtools` version 2.4.2. For more information, please see the uwtools.api.config Read the Docs page.\n", + "\n", + "## Table of Contents\n", + "\n", + "* [Getting Config Objects](#Getting-Config-Objects)\n", + "* [Config Depth Limitations](#Config-Depth-Limitations)\n", + "* [Realizing Configs](#Realizing-Configs)\n", + " * [Updating Configs](#Updating-Configs)\n", + " * [Using the `key_path` Parameter](#Using-the-key_path-Parameter)\n", + " * [Using the `values_needed` Parameter](#Using-the-values_needed-Parameter)\n", + " * [Using the `total` Parameter](#Using-the-total-Parameter)\n", + "* [Realizing Configs to a Dictionary](#Realizing-Configs-to-a-Dictionary)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "6d02d033-0992-4990-861d-3f80d09d7083", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from uwtools.api import config\n", + "from uwtools.api.logging import use_uwtools_logger\n", + "\n", + "use_uwtools_logger()" + ] + }, + { + "cell_type": "markdown", + "id": "212594bb-379f-4441-805e-af0dbabe1815", + "metadata": {}, + "source": [ + "## Getting Config Objects\n", + "\n", + "The `config` tool can create configuration objects given a Python ``dict`` or a file in one of five different formats: FieldTable, INI, Fortran namelist, Shell, or YAML. `config.get_yaml_config` is demonstrated here, but the config module also has similar functions for each of the other supported formats: `get_fieldtable_config()`, `get_ini_config()`, `get_nml_config()`, and `get_sh_config()`.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ab0e21c3-a4b6-404c-bffd-e0d393d9b0a2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function get_yaml_config in module uwtools.api.config:\n", + "\n", + "get_yaml_config(config: Union[dict, str, pathlib.Path, NoneType] = None, stdin_ok: bool = False) -> uwtools.config.formats.yaml.YAMLConfig\n", + " Get a ``YAMLConfig`` object.\n", + "\n", + " :param config: YAML file or ``dict`` (``None`` => read ``stdin``).\n", + " :param stdin_ok: OK to read from ``stdin``?\n", + " :return: An initialized ``YAMLConfig`` object.\n", + "\n" + ] + } + ], + "source": [ + "help(config.get_yaml_config)" + ] + }, + { + "cell_type": "markdown", + "id": "606da5b3-4bff-4148-a9a5-908aa7dd5e8c", + "metadata": {}, + "source": [ + "The `stdin_ok` argument can be used to permit reads from `stdin`, but this is a rare use case beyond the scope of this notebook that will not be discussed here.\n", + "\n", + "`get_yaml_config()` can take input from a Python `dict` or a YAML file like the one below.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c6e049df-38f6-4879-8e0d-68356226d94b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "greeting: Hello\n", + "recipient: World\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/config/get-config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "c2f72448-d35e-4a0b-a371-cb47c7b3338b", + "metadata": {}, + "source": [ + "Paths to config files can be provided either as a string or Path object. Since `get_yaml_config()` is used here, a `YAMLConfig` object is returned.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cc3020a6-4eb4-4830-9263-a9fc8fac7450", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "greeting: Hello\n", + "recipient: World\n" + ] + } + ], + "source": [ + "config1 = config.get_yaml_config(\n", + " config=Path(\"fixtures/config/get-config.yaml\")\n", + ")\n", + "print(type(config1))\n", + "print(config1)" + ] + }, + { + "cell_type": "markdown", + "id": "b7bcd736-ff78-4e8b-957f-b348b812c5f6", + "metadata": {}, + "source": [ + "Providing a Python `dict` will create a UW `Config` object with format matching the function used.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f01ac223-4a02-40ba-822f-8e66ad39f313", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "message:\n", + " greeting: Hi\n", + " recipient: Earth\n" + ] + } + ], + "source": [ + "input_config = {\"message\": {\"greeting\":\"Hi\", \"recipient\":\"Earth\"}}\n", + "config2 = config.get_yaml_config(\n", + " config=input_config\n", + ")\n", + "print(config2)" + ] + }, + { + "cell_type": "markdown", + "id": "dc745e95-d1ce-435c-a488-13b761979e36", + "metadata": {}, + "source": [ + "## Config Depth Limitations\n", + "\n", + "Some config formats have limitations on the depth of their nested configs. Shell configs, for example, may only contain top-level, bash-syntax `key=value` pairs.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "28d23ac5-52a0-45bc-bfee-98d9ea518ca2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "greeting=Salutations\n", + "recipient=Mars" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.get_sh_config(\n", + " config={\"greeting\":\"Salutations\", \"recipient\":\"Mars\"}\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "3dd7292f-cbd1-4e45-b641-1e213a4ead07", + "metadata": {}, + "source": [ + "Shell configs cannot be nested, and any attempt to do so will raise a `UWConfigError`.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b3a0a5bc-9d1b-4d48-a05f-be6f94fb6e1d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cannot instantiate depth-1 SHConfig with depth-2 config\n" + ] + } + ], + "source": [ + "try: \n", + " config.get_sh_config(\n", + " config={\"message\": {\"greeting\":\"Salutations\", \"recipient\":\"Mars\"}}\n", + " )\n", + "except Exception as e:\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "id": "7677bdaa-8707-4dff-b812-91b4521f4820", + "metadata": {}, + "source": [ + "When creating INI configs, exactly one level of nesting is required so that each key-value pair is contained within a section. The top level keys become sections, which are contained within square brackets `[]`. Read more about INI configuration files here.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "6837e75b-bd20-4c3b-bd33-650e4b4f9f23", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[message]\n", + "greeting = Salutations\n", + "recipient = Mars" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.get_ini_config(\n", + " config={\"message\": {\"greeting\":\"Salutations\", \"recipient\":\"Mars\"}}\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "04ab0d52-75cc-4227-8058-a9a1faba7b54", + "metadata": {}, + "source": [ + "Either more or fewer levels of nesting will raise a `UWConfigError`.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "9655b36b-2d39-4fc1-b3b8-9cb3443cf4b8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cannot instantiate depth-2 INIConfig with depth-1 config\n" + ] + } + ], + "source": [ + "try:\n", + " config.get_ini_config(\n", + " config={\"greeting\":\"Salutations\", \"recipient\":\"Mars\"}\n", + " )\n", + "except Exception as e:\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "id": "e4403333-c247-465f-9a75-f96d1be914f6", + "metadata": {}, + "source": [ + "## Realizing Configs\n", + "\n", + "The `config.realize()` function writes config files to disk or `stdout` with the ability to render Jinja2 expressions and add/update values.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "2c3f1b75-b26f-4893-beb7-37a58c09f511", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function realize in module uwtools.api.config:\n", + "\n", + "realize(input_config: Union[uwtools.config.formats.base.Config, pathlib.Path, dict, str, NoneType] = None, input_format: Optional[str] = None, update_config: Union[uwtools.config.formats.base.Config, pathlib.Path, dict, str, NoneType] = None, update_format: Optional[str] = None, output_file: Union[str, pathlib.Path, NoneType] = None, output_format: Optional[str] = None, key_path: Optional[list[Union[str, int]]] = None, values_needed: bool = False, total: bool = False, dry_run: bool = False, stdin_ok: bool = False) -> dict\n", + " Realize a config based on a base input config and an optional update config.\n", + "\n", + " The input config may be specified as a filesystem path, a ``dict``, or a ``Config`` object. When it\n", + " is not, it will be read from ``stdin``.\n", + "\n", + " If an update config is specified, it is merged onto the input config, augmenting or overriding base\n", + " values. It may be specified as a filesystem path, a ``dict``, or a ``Config`` object. When it is\n", + " not, it will be read from ``stdin``.\n", + "\n", + " At most one of the input config or the update config may be left unspecified, in which case the\n", + " other will be read from ``stdin``. If neither filename or format is specified for the update config, no\n", + " update will be performed.\n", + "\n", + " The output destination may be specified as a filesystem path. When it is not, it will be written to\n", + " ``stdout``.\n", + "\n", + " If ``values_needed`` is ``True``, a report of values needed to realize the config is logged. In\n", + " ``dry_run`` mode, output is written to ``stderr``.\n", + "\n", + " If ``total`` is ``True``, an exception will be raised if any Jinja2 variables/expressions cannot be\n", + " rendered. Otherwise, such variables/expressions will be passed through unchanged in the output.\n", + "\n", + " Recognized file extensions are: ini, nml, sh, yaml\n", + "\n", + " :param input_config: Input config file (``None`` => read ``stdin``).\n", + " :param input_format: Format of the input config (optional if file's extension is recognized).\n", + " :param update_config: Update config file (``None`` => read ``stdin``).\n", + " :param update_format: Format of the update config (optional if file's extension is recognized).\n", + " :param output_file: Output config file (``None`` => write to ``stdout``).\n", + " :param output_format: Format of the output config (optional if file's extension is recognized).\n", + " :param key_path: Path through keys to the desired output block.\n", + " :param values_needed: Report complete, missing, and template values.\n", + " :param total: Require rendering of all Jinja2 variables/expressions.\n", + " :param dry_run: Log output instead of writing to output.\n", + " :param stdin_ok: OK to read from ``stdin``?\n", + " :return: The ``dict`` representation of the realized config.\n", + " :raises: UWConfigRealizeError if ``total`` is ``True`` and any Jinja2 variable/expression was not rendered.\n", + "\n" + ] + } + ], + "source": [ + "help(config.realize)" + ] + }, + { + "cell_type": "markdown", + "id": "501da514-a654-4511-928f-b2ad7db102b2", + "metadata": {}, + "source": [ + "The `input_config` parameter takes a config from a string path, Path object, Python `dict`, or UW `Config` object like the `YAMLConfig` object from the Getting Config Objects section. The `input_format` argument must be provided for `dict` inputs or for files without recognized extensions. Configs are written to `stdout` if `output_file` is unspecified or explicitly set to `None`, or to the file specified by `output_file`. The `output_format` argument must be provided when writing to `stdout` or to a file without a recognized extension. Recognized extensions are: `.ini`, `.nml`, `.sh`, and `.yaml`.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "77873e14-db3c-417d-be7a-2ba12c9a38f6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'greeting': 'Hello', 'recipient': 'World'}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.realize(\n", + " input_config=config1,\n", + " output_file=Path('tmp/config1.yaml')\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "fd1823c4-6c3d-4d4a-a614-4d1238588bdd", + "metadata": {}, + "source": [ + "The `realize()` method returns a dict version of the config regardless of input type, and the file is written in the YAML format as indicated by the file extension.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "4f237a73-da83-4632-990f-644632b15cd9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "greeting: Hello\n", + "recipient: World\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat tmp/config1.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "902c4863-e772-49c6-88f2-31e1ab76b418", + "metadata": {}, + "source": [ + "Input and output formats are not required to match. This can be used to convert some configs from one format to another. YAML configs can be converted to configs of other recognized formats so long as the depth restrictions of the output format are met. All configs of recognized formats can be converted into YAML configs. Keep in mind that some formats are unable to express some types (for example, Shell configs can't express a value as an `int` while a Fortran namelist can) so type information may be lost when converting between formats.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "b8854dc6-9dd2-4843-99e4-278b116b9767", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'greeting': 'Hello', 'recipient': 'World'}" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.realize(\n", + " input_config='fixtures/config/get-config.yaml',\n", + " input_format='yaml',\n", + " output_file='tmp/realize-config.sh',\n", + " output_format='sh'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "4032ab38-f6c7-45d1-bff7-f70d23832f26", + "metadata": {}, + "source": [ + "Here a Shell config is created from a YAML config.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0a59a3e8-27b5-4daa-a924-941aceaad157", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "greeting=Hello\n", + "recipient=World\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat tmp/realize-config.sh" + ] + }, + { + "cell_type": "markdown", + "id": "ef0136e4-f549-467f-9341-f77f84738bb0", + "metadata": {}, + "source": [ + "### Updating Configs\n", + "\n", + "Configs can be updated by providing a second config with the `update_config` parameter. If the update config contains keys that match the base config, the base config values for those keys will be overwritten. Once updated, if the config contains Jinja2 expressions, like the one below, they will be rendered in the config wherever possible.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "2d72bbc1-e438-48b2-8bd7-554b598c6f24", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "&memo\n", + " sender_id = \"{{ id }}\"\n", + " message = \"{{ greeting }}, {{ recipient }}!\"\n", + " sent = .FALSE.\n", + "/\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/config/base-config.nml" + ] + }, + { + "cell_type": "markdown", + "id": "67b07a34-1bea-402f-9cec-f33a2e519d27", + "metadata": {}, + "source": [ + "Here, the update config provides values that will update two of the Jinja2 expressions and override one key with a new value.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "7648fdd5-5752-4bf3-b366-db8da1eac601", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'memo': {'sender_id': '{{ id }}',\n", + " 'message': 'Salutations, Mars!',\n", + " 'sent': True,\n", + " 'greeting': 'Salutations',\n", + " 'recipient': 'Mars'}}" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.realize(\n", + " input_config='fixtures/config/base-config.nml',\n", + " update_config={\"memo\": {\"greeting\":\"Salutations\", \"recipient\":\"Mars\", \"sent\": True}},\n", + " output_file='tmp/updated-config.nml'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "bdac7f6c-4452-4137-9104-784449584100", + "metadata": {}, + "source": [ + "All of the key-value pairs were added to the updated config, and the base config was rendered where the appropriate values were provided. However, not all Jinja2 expressions are required to be rendered: An `id` key was not provided in the update config, so the expression referencing it was not rendered.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "f4538965-d3b0-4c0c-a878-b6852f8d8ab0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "&memo\n", + " sender_id = '{{ id }}'\n", + " message = 'Salutations, Mars!'\n", + " sent = .true.\n", + " greeting = 'Salutations'\n", + " recipient = 'Mars'\n", + "/\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat tmp/updated-config.nml" + ] + }, + { + "cell_type": "markdown", + "id": "a20024ac-0f33-4000-942d-29c99dc0502e", + "metadata": {}, + "source": [ + "### Using the `key_path` Parameter\n", + "\n", + "Consider the following config file, where the desired keys and values are not at the top level.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "7ce0e917-f0d0-4302-9c8c-b136ffc5410a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "keys:\n", + " to:\n", + " config:\n", + " message: \"{{ greeting }}, {{ recipient }}!\"\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/config/keys-config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "633778ca-391c-4ba4-b3b4-b3f599af5d41", + "metadata": {}, + "source": [ + "The `key_path` parameter allows only a portion of the config, identified by following a given list of keys, to be written to a file or, in this case, to `stdout`. Note that the key-value pairs from the update config are used to render values, but don't appear in the config written to `stdout`.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "00150efa-848c-44eb-ac0c-dab3845546b8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "message: Good morning, Venus!\n" + ] + } + ], + "source": [ + "_ = config.realize(\n", + " input_config=\"fixtures/config/keys-config.yaml\",\n", + " update_config={\"greeting\": \"Good morning\", \"recipient\": \"Venus\"},\n", + " output_file=None,\n", + " output_format='yaml',\n", + " key_path=['keys', 'to', 'config']\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "903ea23f-4625-48f0-9efc-1c0a106d5bf6", + "metadata": {}, + "source": [ + "### Using the `values_needed` Parameter\n", + "\n", + "Consider the config file below, which contains unrendered Jinja2 expressions.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "928a23d7-8ba9-4217-935d-01563bb36cb6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "&memo\n", + " sender_id = \"{{ id }}\"\n", + " message = \"{{ greeting }}, {{ recipient }}!\"\n", + " sent = .FALSE.\n", + "/\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/config/base-config.nml" + ] + }, + { + "cell_type": "markdown", + "id": "7d878080-55ae-4233-bb65-37cfa5ef7cff", + "metadata": {}, + "source": [ + "Setting `values_needed` to `True` will allow logging of keys that contain unrendered Jinja2 expressions and their values. A logger needs to be initialized for this information to be displayed. The config is not written and the returned `dict` is empty.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "d22c692d-e98e-4f88-bdac-a369f0a1962f", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-09-23T16:13:23] INFO Keys that are complete:\n", + "[2024-09-23T16:13:23] INFO memo\n", + "[2024-09-23T16:13:23] INFO memo.sent\n", + "[2024-09-23T16:13:23] INFO \n", + "[2024-09-23T16:13:23] INFO Keys with unrendered Jinja2 variables/expressions:\n", + "[2024-09-23T16:13:23] INFO memo.sender_id: {{ id }}\n", + "[2024-09-23T16:13:23] INFO memo.message: {{ greeting }}, {{ recipient }}!\n" + ] + }, + { + "data": { + "text/plain": [ + "{}" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.realize(\n", + " input_config='fixtures/config/base-config.nml',\n", + " output_file=None,\n", + " output_format='nml',\n", + " values_needed=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "ddbc989a-a9ed-4649-8e87-f74d1ff43b89", + "metadata": {}, + "source": [ + "### Using the `total` Parameter\n", + "\n", + "The `total` parameter is used to specify that all Jinja2 expressions must be rendered before the final config is written. Consider the config below which contains multiple expressions.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "647538d9-cd22-4f94-b15b-c34d68a324da", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "&memo\n", + " sender_id = \"{{ id }}\"\n", + " message = \"{{ greeting }}, {{ recipient }}!\"\n", + " sent = .FALSE.\n", + "/\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/config/base-config.nml" + ] + }, + { + "cell_type": "markdown", + "id": "7c0d64be-c97c-4d07-b636-54b1c8bb5d0c", + "metadata": {}, + "source": [ + "As was shown in the Updating Configs section, by default not all Jinja2 expressions are required to be rendered. However, when `total` is set to `True` and not enough values are provided to fully realize the config, a `UWConfigRealizeError` is raised. Notice that values are provided for `greeting` and `recipient`, but not for `id`.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "ba360102-f558-4dba-b0b6-c6f550c7d40f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Config could not be totally realized\n" + ] + } + ], + "source": [ + "try:\n", + " config.realize(\n", + " input_config='fixtures/config/base-config.nml',\n", + " update_config={\"memo\": {\"greeting\":\"Salutations\", \"recipient\":\"Mars\", \"sent\":True}},\n", + " output_file='tmp/config-total.nml',\n", + " total=True\n", + " )\n", + "except Exception as e:\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "id": "4048efdd-42f4-4400-bdba-1c8e0001d8f6", + "metadata": {}, + "source": [ + "With all values provided to fully render the config, `realize()` writes the complete config without error.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "e891a446-f699-460d-b4a8-568d9d4cf631", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'memo': {'sender_id': '321',\n", + " 'message': 'Salutations, Mars!',\n", + " 'sent': True,\n", + " 'greeting': 'Salutations',\n", + " 'recipient': 'Mars',\n", + " 'id': 321}}" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.realize(\n", + " input_config='fixtures/config/base-config.nml',\n", + " update_config={\"memo\": {\"greeting\":\"Salutations\", \"recipient\":\"Mars\", \"sent\":True, \"id\":321}},\n", + " output_file='tmp/config-total.nml',\n", + " total=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "07876d41-dad2-4c30-adba-f635050708ed", + "metadata": {}, + "source": [ + "The newly created config file is free from any unrendered Jinja2 expressions.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "aeab0ec5-7e6e-4309-b484-4de5dd9324b5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "&memo\n", + " sender_id = '321'\n", + " message = 'Salutations, Mars!'\n", + " sent = .true.\n", + " greeting = 'Salutations'\n", + " recipient = 'Mars'\n", + " id = 321\n", + "/\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat tmp/config-total.nml" + ] + }, + { + "cell_type": "markdown", + "id": "e121dab6-5d16-40b0-aa16-161c329a8e9a", + "metadata": {}, + "source": [ + "## Realizing Configs to a Dictionary\n", + "\n", + "The `config.realize_to_dict()` function has the ability to manipulate config values, and returns the config as a Python `dict` just as `realize()` does. However, a config won't be written to a file or to `stdout`. Like `realize()`, input or update configs can be Python dictionaries, UW `Config` objects, or files like the one below.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "bdc98ce1-e213-41ac-b1f2-03bd52238e30", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "greeting: Hello\n", + "recipient: World\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/config/get-config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "eeacc37f-b808-4d0e-9e29-ff8aa1fc2ff2", + "metadata": {}, + "source": [ + "`realize_to_dict()` has the same parameters as `realize()`, with the exception of `output_file` and `output_format`. Instead, configs can be manipulated or converted to a `dict` without the need to specify an output file or format.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "ae7f648e-6586-4700-87e7-492ca3a02a06", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': '456', 'greeting': 'Hello', 'recipient': 'World'}" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.realize_to_dict(\n", + " input_config={\"id\": \"456\"},\n", + " update_config=\"fixtures/config/get-config.yaml\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "adf2f518-f92c-4b67-a865-c01ce0b10a2e", + "metadata": {}, + "source": [ + "For more details on usage and parameters, see the Realizing Configs section above.\n", + "" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:DEV-uwtools] *", + "language": "python", + "name": "conda-env-DEV-uwtools-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/fixtures/config/base-config.nml b/notebooks/fixtures/config/base-config.nml new file mode 100644 index 000000000..9f30c6505 --- /dev/null +++ b/notebooks/fixtures/config/base-config.nml @@ -0,0 +1,5 @@ +&memo + sender_id = "{{ id }}" + message = "{{ greeting }}, {{ recipient }}!" + sent = .FALSE. +/ diff --git a/notebooks/fixtures/config/get-config.yaml b/notebooks/fixtures/config/get-config.yaml new file mode 100644 index 000000000..be310733b --- /dev/null +++ b/notebooks/fixtures/config/get-config.yaml @@ -0,0 +1,2 @@ +greeting: Hello +recipient: World diff --git a/notebooks/fixtures/config/keys-config.yaml b/notebooks/fixtures/config/keys-config.yaml new file mode 100644 index 000000000..a8f524b40 --- /dev/null +++ b/notebooks/fixtures/config/keys-config.yaml @@ -0,0 +1,4 @@ +keys: + to: + config: + message: "{{ greeting }}, {{ recipient }}!" diff --git a/notebooks/template.ipynb b/notebooks/template.ipynb index e0e9a2e9e..39a447f70 100644 --- a/notebooks/template.ipynb +++ b/notebooks/template.ipynb @@ -9,7 +9,7 @@ "\n", "The `uwtools` API's `template` module provides functions to render Jinja2 templates and to translate atparse templates to Jinja2.\n", "\n", - "For more information, please see the uwtools.api.template Read the Docs page." + "Tested on `uwtools` version 2.4.2. For more information, please see the uwtools.api.template Read the Docs page." ] }, { diff --git a/notebooks/tests/test_config.py b/notebooks/tests/test_config.py new file mode 100644 index 000000000..04260206a --- /dev/null +++ b/notebooks/tests/test_config.py @@ -0,0 +1,73 @@ +import yaml +from testbook import testbook + + +def test_get_config(): + with open("fixtures/config/get-config.yaml", "r", encoding="utf-8") as f: + config1_str = f.read().rstrip() + config1_dict = yaml.safe_load(config1_str) + with testbook("config.ipynb", execute=True) as tb: + assert tb.ref("config1") == config1_dict + assert tb.cell_output_text(5) == config1_str + assert config1_str in tb.cell_output_text(7) + assert tb.cell_output_text(9) == "message:\n greeting: Hi\n recipient: Earth" + + +def test_depth(): + with testbook("config.ipynb", execute=True) as tb: + assert tb.cell_output_text(11) == "greeting=Salutations\nrecipient=Mars" + assert tb.cell_output_text(13) == "Cannot instantiate depth-1 SHConfig with depth-2 config" + assert tb.cell_output_text(15) == "[message]\ngreeting = Salutations\nrecipient = Mars" + assert tb.cell_output_text(17) == "Cannot instantiate depth-2 INIConfig with depth-1 config" + + +def test_realize(): + # Get config file data to compare to cell output. + with open("fixtures/config/get-config.yaml", "r", encoding="utf-8") as f: + config_str = f.read().rstrip() + config_dict = yaml.safe_load(config_str) + with open("fixtures/config/base-config.nml", "r", encoding="utf-8") as f: + update_config_str = f.read().rstrip() + with open("fixtures/config/keys-config.yaml", "r", encoding="utf-8") as f: + keys_config_str = f.read().rstrip() + with testbook("config.ipynb", execute=True) as tb: + with open("tmp/updated-config.nml", "r", encoding="utf-8") as f: + updated_config = f.read().rstrip() + with open("tmp/config-total.nml", "r", encoding="utf-8") as f: + total_config = f.read().rstrip() + # Ensure that cell output text matches expectations. + assert tb.cell_output_text(21) == str(config_dict) + assert tb.cell_output_text(23) == config_str + assert tb.cell_output_text(25) == str(config_dict) + for item in config_dict.items(): + assert item[0] + "=" + item[1] in tb.cell_output_text(27) + assert tb.cell_output_text(29) == update_config_str + updated_dict = ( + "'sender_id': '{{ id }}'", + "'message': 'Salutations, Mars!'", + "'sent': True", + ) + assert all(x in tb.cell_output_text(31) for x in updated_dict) + assert tb.cell_output_text(33) == updated_config + assert tb.cell_output_text(35) == keys_config_str + assert tb.cell_output_text(37) == "message: Good morning, Venus!" + assert tb.cell_output_text(39) == update_config_str + expected_log = ( + "memo.sender_id: {{ id }}", + "memo.message: {{ greeting }}, {{ recipient }}!", + ) + assert all(x in tb.cell_output_text(41) for x in expected_log) + assert tb.cell_output_text(43) == update_config_str + assert tb.cell_output_text(45) == "Config could not be totally realized" + total_dict = ("'sender_id': '321'", "'message': 'Salutations, Mars!'", "'sent': True") + assert all(x in tb.cell_output_text(47) for x in total_dict) + assert tb.cell_output_text(49) == total_config + + +def test_realize_to_dict(): + with open("fixtures/config/get-config.yaml", "r", encoding="utf-8") as f: + config_str = f.read().rstrip() + with testbook("config.ipynb", execute=True) as tb: + assert tb.cell_output_text(51) == config_str + config_out = ("'id': '456'", "'greeting': 'Hello'", "'recipient': 'World'") + assert all(x in tb.cell_output_text(53) for x in config_out) From 2c75b30671854b703869da4125ae3b56f1e02bf4 Mon Sep 17 00:00:00 2001 From: Travis Byrne <42480803+Byrnetp@users.noreply.github.com> Date: Thu, 17 Oct 2024 08:37:43 -0600 Subject: [PATCH 13/24] Jupyter notebook: Config Tool Part II (#611) * update_values() -> update_from() (#578) * DRY out Driver._validate() (#579) * Move execute() to its own API namespace, disambiguate (#580) * Config classes to API (#582) * Simplify unknown-extension error message (#584) * UW-657 fs makedirs (#572) * Docstring cleanup (#585) * GH 586 bad stdin_ok prompt (#587) * Fix issue with creating a metatask dep. (#589) The rocoto schema is set up to allow metataskdep entries in the YAML, but the logic was not included in the tool to handle them. This addition fixes that. * Add --show-schema support to drivers (#588) * Config notebook p2 and example notebook removal * removing merge diff text * pin python version for RtD * improvements from code review * add config.compare() format mismatch example * Remove typo Co-authored-by: Emily Carpenter <137525341+elcarpenterNOAA@users.noreply.github.com> --------- Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Co-authored-by: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Co-authored-by: Emily Carpenter <137525341+elcarpenterNOAA@users.noreply.github.com> --- docs/environment.yml | 1 + notebooks/config.ipynb | 911 +++++++++++++++++- notebooks/example.ipynb | 429 --------- notebooks/fixtures/config/alt-config.nml | 5 + notebooks/fixtures/config/fruit-config.ini | 4 + notebooks/fixtures/config/validate.jsonschema | 15 + notebooks/fixtures/example/ex2-config.yaml | 3 - .../fixtures/example/ex2-rendered-config.yaml | 3 - notebooks/fixtures/example/ex2-values.yaml | 3 - .../example/ex3-config-test-file-a.nml | 1 - .../example/ex3-config-test-file-b.nml | 4 - .../example/ex3-config-test-file-c.nml | 4 - notebooks/tests/test_config.py | 63 ++ notebooks/tests/test_example.py | 62 -- 14 files changed, 991 insertions(+), 517 deletions(-) delete mode 100644 notebooks/example.ipynb create mode 100644 notebooks/fixtures/config/alt-config.nml create mode 100644 notebooks/fixtures/config/fruit-config.ini create mode 100644 notebooks/fixtures/config/validate.jsonschema delete mode 100644 notebooks/fixtures/example/ex2-config.yaml delete mode 100644 notebooks/fixtures/example/ex2-rendered-config.yaml delete mode 100644 notebooks/fixtures/example/ex2-values.yaml delete mode 100644 notebooks/fixtures/example/ex3-config-test-file-a.nml delete mode 100644 notebooks/fixtures/example/ex3-config-test-file-b.nml delete mode 100644 notebooks/fixtures/example/ex3-config-test-file-c.nml delete mode 100644 notebooks/tests/test_example.py diff --git a/docs/environment.yml b/docs/environment.yml index ea7494fd7..e71700ed5 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -2,6 +2,7 @@ name: readthedocs channels: - conda-forge dependencies: + - python=3.12 - sphinx_rtd_theme=2.0.* - sphinxcontrib-bibtex=2.6.* - tree diff --git a/notebooks/config.ipynb b/notebooks/config.ipynb index bba503764..c3aca9e55 100644 --- a/notebooks/config.ipynb +++ b/notebooks/config.ipynb @@ -21,6 +21,13 @@ " * [Using the `values_needed` Parameter](#Using-the-values_needed-Parameter)\n", " * [Using the `total` Parameter](#Using-the-total-Parameter)\n", "* [Realizing Configs to a Dictionary](#Realizing-Configs-to-a-Dictionary)\n", + "* [Comparing Configs](#Comparing-Configs)\n", + "* [Validating Configs](#Validating-Configs)\n", + "* [Working with Config Classes](#Working-with-Config-Classes)\n", + " * [Comparing Config Objects](#Comparing-Config-Objects)\n", + " * [Rendering Values](#Rendering-Values)\n", + " * [Writing Configs in a Specified Format](#Writing-Configs-in-a-Specified-Format)\n", + " * [Updating Values](#Updating-Values) \n", "" ] }, @@ -725,13 +732,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-09-23T16:13:23] INFO Keys that are complete:\n", - "[2024-09-23T16:13:23] INFO memo\n", - "[2024-09-23T16:13:23] INFO memo.sent\n", - "[2024-09-23T16:13:23] INFO \n", - "[2024-09-23T16:13:23] INFO Keys with unrendered Jinja2 variables/expressions:\n", - "[2024-09-23T16:13:23] INFO memo.sender_id: {{ id }}\n", - "[2024-09-23T16:13:23] INFO memo.message: {{ greeting }}, {{ recipient }}!\n" + "[2024-10-16T15:39:13] INFO Keys that are complete:\n", + "[2024-10-16T15:39:13] INFO memo\n", + "[2024-10-16T15:39:13] INFO memo.sent\n", + "[2024-10-16T15:39:13] INFO \n", + "[2024-10-16T15:39:13] INFO Keys with unrendered Jinja2 variables/expressions:\n", + "[2024-10-16T15:39:13] INFO memo.sender_id: {{ id }}\n", + "[2024-10-16T15:39:13] INFO memo.message: {{ greeting }}, {{ recipient }}!\n" ] }, { @@ -964,12 +971,900 @@ }, { "cell_type": "markdown", - "id": "adf2f518-f92c-4b67-a865-c01ce0b10a2e", + "id": "be66b870-a42c-46f2-a045-b769354403de", "metadata": {}, "source": [ "For more details on usage and parameters, see the Realizing Configs section above.\n", + "\n", + "## Comparing Configs\n", + "\n", + "The `config` tool can be used to compare two configuration files using `config.compare()`.\n", "" ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "249ae7a9-1ad3-4401-98b0-bc3c433f22f4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function compare in module uwtools.api.config:\n", + "\n", + "compare(config_1_path: Union[pathlib.Path, str], config_2_path: Union[pathlib.Path, str], config_1_format: Optional[str] = None, config_2_format: Optional[str] = None) -> bool\n", + " Compare two config files.\n", + "\n", + " Recognized file extensions are: ini, nml, sh, yaml\n", + "\n", + " :param config_1_path: Path to 1st config file.\n", + " :param config_2_path: Path to 2nd config file.\n", + " :param config_1_format: Format of 1st config file (optional if file's extension is recognized).\n", + " :param config_2_format: Format of 2nd config file (optional if file's extension is recognized).\n", + " :return: ``False`` if config files had differences, otherwise ``True``.\n", + "\n" + ] + } + ], + "source": [ + "help(config.compare)" + ] + }, + { + "cell_type": "markdown", + "id": "908b17fe-ae2d-4964-a659-0481d063c037", + "metadata": {}, + "source": [ + "Consider the following config files, which have similar values, with the exception of `sent`'s value.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "0218e5de-2c25-4d7a-a7b6-0f05ad81afb2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "&memo\n", + " sender_id = \"{{ id }}\"\n", + " message = \"{{ greeting }}, {{ recipient }}!\"\n", + " sent = .FALSE.\n", + "/\n", + "----------------------------------------------\n", + "&memo\n", + " sender_id = \"{{ id }}\"\n", + " message = \"{{ greeting }}, {{ recipient }}!\"\n", + " sent = .TRUE.\n", + "/\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/config/base-config.nml\n", + "echo ----------------------------------------------\n", + "cat fixtures/config/alt-config.nml " + ] + }, + { + "cell_type": "markdown", + "id": "c4361a7c-9c16-4bc0-b3e6-985d85f1450c", + "metadata": {}, + "source": [ + "`compare()` returns `True` if the configs contain identical key-value pairs, and `False` otherwise. If a logger has been initialized, information is logged on which files are being compared and the values that differ, if any. Files are passed to `config_1_path` and `config_2_path` as a string filename or Path object. Corresponding optional formats may be passed using `config_1_format` and `config_2_format` and are only needed if the format suffix is not recognized.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "988b72bc-7983-4d24-9d37-c9ba32a31ac4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-16T15:39:13] INFO - fixtures/config/base-config.nml\n", + "[2024-10-16T15:39:13] INFO + fixtures/config/alt-config.nml\n", + "[2024-10-16T15:39:13] INFO ---------------------------------------------------------------------\n", + "[2024-10-16T15:39:13] INFO memo: sent: - False + True\n" + ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.compare(\n", + " config_1_path=Path('fixtures/config/base-config.nml'),\n", + " config_2_path='fixtures/config/alt-config.nml',\n", + " config_1_format='nml',\n", + " config_2_format='nml'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "655717b7-b7db-4711-9dc4-c00557e6c2df", + "metadata": {}, + "source": [ + "To see the behavior of `compare()` when key-value pairs are identical, one of the configs from above is copied in the cell below.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "ceb7f893-398b-41e4-818a-49cd036c2bfe", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "cp fixtures/config/base-config.nml tmp/config-copy.nml" + ] + }, + { + "cell_type": "markdown", + "id": "02c75cf7-9f24-44a0-97c6-abd7b44fb60f", + "metadata": {}, + "source": [ + "When these two files are compared, `True` is returned and the log reports no differences.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "8b495735-2396-436e-87db-d886ac6769fa", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-16T15:39:13] INFO - fixtures/config/base-config.nml\n", + "[2024-10-16T15:39:13] INFO + tmp/config-copy.nml\n", + "[2024-10-16T15:39:13] INFO ---------------------------------------------------------------------\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.compare(\n", + " config_1_path='fixtures/config/base-config.nml',\n", + " config_2_path='tmp/config-copy.nml',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "eb4536f8-d198-46f8-ac74-ee9d2010704e", + "metadata": {}, + "source": [ + "If a comparison is attempted between two files whose formats that don't match, `compare()` returns `False` and the mismatch is reported.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "438ccd5b-30d5-49b4-ba12-6e6e89f46d28", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-16T15:39:13] ERROR Formats do not match: yaml vs nml\n" + ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.compare(\n", + " config_1_path=Path('fixtures/config/get-config.yaml'),\n", + " config_2_path=Path('fixtures/config/base-config.nml')\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e14c0158-baa4-45d0-a86c-d9a245021229", + "metadata": {}, + "source": [ + "## Validating Configs\n", + "\n", + "The `config.validate()` function checks if a given config conforms to a specified JSON schema.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "2eb5b2ce-bebd-449d-90a5-764484aa03aa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function validate in module uwtools.api.config:\n", + "\n", + "validate(schema_file: Union[pathlib.Path, str], config: Union[dict, str, uwtools.config.formats.yaml.YAMLConfig, pathlib.Path, NoneType] = None, stdin_ok: bool = False) -> bool\n", + " Check whether the specified config conforms to the specified JSON Schema spec.\n", + "\n", + " If no config is specified, ``stdin`` is read and will be parsed as YAML and then validated. A\n", + " ``dict`` or a YAMLConfig instance may also be provided for validation.\n", + "\n", + " :param schema_file: The JSON Schema file to use for validation.\n", + " :param config: The config to validate.\n", + " :param stdin_ok: OK to read from ``stdin``?\n", + " :return: ``True`` if the YAML file conforms to the schema, ``False`` otherwise.\n", + "\n" + ] + } + ], + "source": [ + "help(config.validate)" + ] + }, + { + "cell_type": "markdown", + "id": "57cdde4c-b3ab-40b1-8e52-1f99ea6d9696", + "metadata": {}, + "source": [ + "Consider the simple YAML config below. `validate()` used together with an appropriate JSON schema ensures that the config meets expectations before it's used elsewhere.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "dc7d2b80-ea92-4301-ae85-4edbf61bf510", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "greeting: Hello\n", + "recipient: World\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/config/get-config.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "1eeca5ad-e969-4670-bddd-8e20b38f3d6c", + "metadata": {}, + "source": [ + "Below is an example of a schema used to validate a config. It ensures that the required keys are present and the value types match expectations. For information on the keys used here and more, please refer to JSON Schema documentation.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "f9c66a24-2a7e-43be-839f-3ad5b136b646", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"additionalProperties\": false,\n", + " \"properties\": {\n", + " \"greeting\": {\n", + " \"type\": \"string\"\n", + " },\n", + " \"recipient\": {\n", + " \"type\": \"string\"\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"greeting\", \"recipient\"\n", + " ],\n", + " \"type\": \"object\"\n", + "}\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/config/validate.jsonschema" + ] + }, + { + "cell_type": "markdown", + "id": "8c61a2d2-473c-45c6-9c6c-6c07fc5bf940", + "metadata": {}, + "source": [ + "The schema file and config from above are passed to the respective `schema_file` and `config` parameters. Config file paths should be passed as a string or Path object. Files should be of YAML format, or parseable as YAML. Alternatively, a `YAMLConfig` object or a Python `dict` can be provided. `validate()` returns `True` if the config conforms to the JSON schema, and `False` otherwise. With a logger initialized, details about any validation errors are reported.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "f3873050-d857-490d-aeeb-f9217a7f808c", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-16T15:39:13] INFO 0 UW schema-validation errors found\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.validate(\n", + " schema_file='fixtures/config/validate.jsonschema',\n", + " config='fixtures/config/get-config.yaml'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8d151205-4a95-4b46-aa1d-30ec30d96e88", + "metadata": {}, + "source": [ + "The `config` argument also accepts a dictionary. In the next example, validation errors exist, and the logger reports the number of errors found along with their locations and details.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "b5664e58-8ddc-438c-8180-1e2911838744", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-16T15:39:13] ERROR 1 UW schema-validation error found\n", + "[2024-10-16T15:39:13] ERROR Error at recipient:\n", + "[2024-10-16T15:39:13] ERROR 47 is not of type 'string'\n" + ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.validate(\n", + " schema_file='fixtures/config/validate.jsonschema',\n", + " config={'greeting':'Hello', 'recipient':47}\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "fffeb768-7109-472f-916b-1407520b0f5e", + "metadata": {}, + "source": [ + "## Working with Config Classes\n", + "\n", + "The `config` tool provides five classes that can be used to work with configs in an object-oriented way. The five different classes each work with a single format: `config.FieldTableConfig`, `config.INIConfig`, `config.NMLConfig`, `config.SHConfig`, and `config.YAMLConfig`. `config.INIConfig` is demonstrated here, but the other classes all use methods of the same names for working with each respective format.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "6f4df804-cda2-48c4-bf42-a5b57de5e066", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on class INIConfig in module uwtools.config.formats.ini:\n", + "\n", + "class INIConfig(uwtools.config.formats.base.Config)\n", + " | INIConfig(config: Union[dict, pathlib.Path, NoneType] = None)\n", + " |\n", + " | Work with INI configs.\n", + " |\n", + " | Method resolution order:\n", + " | INIConfig\n", + " | uwtools.config.formats.base.Config\n", + " | abc.ABC\n", + " | collections.UserDict\n", + " | collections.abc.MutableMapping\n", + " | collections.abc.Mapping\n", + " | collections.abc.Collection\n", + " | collections.abc.Sized\n", + " | collections.abc.Iterable\n", + " | collections.abc.Container\n", + " | builtins.object\n", + " |\n", + " | Methods defined here:\n", + " |\n", + " | __init__(self, config: Union[dict, pathlib.Path, NoneType] = None)\n", + " | :param config: Config file to load (None => read from stdin), or initial dict.\n", + " |\n", + " | dump(self, path: Optional[pathlib.Path] = None) -> None\n", + " | Dump the config in INI format.\n", + " |\n", + " | :param path: Path to dump config to (default: stdout).\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Class methods defined here:\n", + " |\n", + " | dump_dict(cfg: dict, path: Optional[pathlib.Path] = None) -> None\n", + " | Dump a provided config dictionary in INI format.\n", + " |\n", + " | :param cfg: The in-memory config object to dump.\n", + " | :param path: Path to dump config to (default: stdout).\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Data and other attributes defined here:\n", + " |\n", + " | __abstractmethods__ = frozenset()\n", + " |\n", + " | __annotations__ = {}\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Methods inherited from uwtools.config.formats.base.Config:\n", + " |\n", + " | __repr__(self) -> str\n", + " | Return the string representation of a Config object.\n", + " |\n", + " | compare_config(self, dict1: dict, dict2: Optional[dict] = None) -> bool\n", + " | Compare two config dictionaries.\n", + " |\n", + " | Assumes a section/key/value structure.\n", + " |\n", + " | :param dict1: The first dictionary.\n", + " | :param dict2: The second dictionary (default: this config).\n", + " | :return: True if the configs are identical, False otherwise.\n", + " |\n", + " | dereference(self, context: Optional[dict] = None) -> None\n", + " | Render as much Jinja2 syntax as possible.\n", + " |\n", + " | update_from(self, src: Union[dict, collections.UserDict]) -> None\n", + " | Update a config.\n", + " |\n", + " | :param src: The dictionary with new data to use.\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Data descriptors inherited from uwtools.config.formats.base.Config:\n", + " |\n", + " | __dict__\n", + " | dictionary for instance variables\n", + " |\n", + " | __weakref__\n", + " | list of weak references to the object\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Methods inherited from collections.UserDict:\n", + " |\n", + " | __contains__(self, key)\n", + " | # Modify __contains__ and get() to work like dict\n", + " | # does when __missing__ is present.\n", + " |\n", + " | __copy__(self)\n", + " |\n", + " | __delitem__(self, key)\n", + " |\n", + " | __getitem__(self, key)\n", + " |\n", + " | __ior__(self, other)\n", + " |\n", + " | __iter__(self)\n", + " |\n", + " | __len__(self)\n", + " |\n", + " | __or__(self, other)\n", + " | Return self|value.\n", + " |\n", + " | __ror__(self, other)\n", + " | Return value|self.\n", + " |\n", + " | __setitem__(self, key, item)\n", + " |\n", + " | copy(self)\n", + " |\n", + " | get(self, key, default=None)\n", + " | D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Class methods inherited from collections.UserDict:\n", + " |\n", + " | fromkeys(iterable, value=None)\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Methods inherited from collections.abc.MutableMapping:\n", + " |\n", + " | clear(self)\n", + " | D.clear() -> None. Remove all items from D.\n", + " |\n", + " | pop(self, key, default=)\n", + " | D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n", + " | If key is not found, d is returned if given, otherwise KeyError is raised.\n", + " |\n", + " | popitem(self)\n", + " | D.popitem() -> (k, v), remove and return some (key, value) pair\n", + " | as a 2-tuple; but raise KeyError if D is empty.\n", + " |\n", + " | setdefault(self, key, default=None)\n", + " | D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D\n", + " |\n", + " | update(self, other=(), /, **kwds)\n", + " | D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.\n", + " | If E present and has a .keys() method, does: for k in E: D[k] = E[k]\n", + " | If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v\n", + " | In either case, this is followed by: for k, v in F.items(): D[k] = v\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Methods inherited from collections.abc.Mapping:\n", + " |\n", + " | __eq__(self, other)\n", + " | Return self==value.\n", + " |\n", + " | items(self)\n", + " | D.items() -> a set-like object providing a view on D's items\n", + " |\n", + " | keys(self)\n", + " | D.keys() -> a set-like object providing a view on D's keys\n", + " |\n", + " | values(self)\n", + " | D.values() -> an object providing a view on D's values\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Data and other attributes inherited from collections.abc.Mapping:\n", + " |\n", + " | __hash__ = None\n", + " |\n", + " | __reversed__ = None\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Class methods inherited from collections.abc.Collection:\n", + " |\n", + " | __subclasshook__(C)\n", + " | Abstract classes can override this to customize issubclass().\n", + " |\n", + " | This is invoked early on by abc.ABCMeta.__subclasscheck__().\n", + " | It should return True, False or NotImplemented. If it returns\n", + " | NotImplemented, the normal algorithm is used. Otherwise, it\n", + " | overrides the normal algorithm (and the outcome is cached).\n", + " |\n", + " | ----------------------------------------------------------------------\n", + " | Class methods inherited from collections.abc.Iterable:\n", + " |\n", + " | __class_getitem__ = GenericAlias(...)\n", + " | Represent a PEP 585 generic type\n", + " |\n", + " | E.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,).\n", + "\n" + ] + } + ], + "source": [ + "help(config.INIConfig)" + ] + }, + { + "cell_type": "markdown", + "id": "91a793e9-d1d1-41bb-84fd-c0c279606473", + "metadata": {}, + "source": [ + "An object can be initialized by providing a config either as a Python `dict` or a Path to the file.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "0e8d6c8a-cc3c-49d7-9b97-32187cd5f754", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[fruit count]\n", + "apples = 3\n", + "grapes = {{ grape_count }}\n", + "kiwis = 2\n" + ] + } + ], + "source": [ + "fruits = config.INIConfig(\n", + " config=Path('fixtures/config/fruit-config.ini')\n", + ")\n", + "print(fruits)" + ] + }, + { + "cell_type": "markdown", + "id": "1a50a362-d063-4ae1-9b6a-d98d0bf84c4f", + "metadata": {}, + "source": [ + "### Comparing Config Objects\n", + "\n", + "The `compare_config()` method compares two config `dict`s and returns `True` when they match and `False` otherwise. Two config `dict`s can be passed to the `dict1` and `dict2` parameters. Config objects of every format use the same method demonstrated here, and it stands as an alternative to `config.compare()`, which compares files rather than dictionaries. See the [Comparing Configs](#Comparing-Configs) section above for more details on `config.compare()`. The configs compared using `compare_config()` can be compared without regard for their intended format, since they are compared as dictionaries, but they must have a section/key/value structure.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "73925576-6b07-4af6-a9b0-7bd6f4235987", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fruits.compare_config(\n", + " dict1={'fruit count':{'apples':'3', 'grapes':'8', 'kiwis':'1'}},\n", + " dict2={'fruit count':{'apples':'3', 'grapes':'8', 'kiwis':'1'}}\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "3caa77b7-2eca-498b-841f-b37c91068974", + "metadata": {}, + "source": [ + "If `dict2` is left unspecified or set to `None`, the `dict1` config is compared to the config stored in the object itself. When there are differences between the two configs, as is the case here, `False` is returned. When a logger is initialized, the values that differ are displayed.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "9d94a325-93f6-43b4-bc56-0ce5a1830647", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-16T15:39:13] INFO fruit count: grapes: - {{ grape_count }} + 8\n", + "[2024-10-16T15:39:13] INFO fruit count: kiwis: - 2 + 1\n" + ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fruits.compare_config(\n", + " dict1={'fruit count':{'apples':'3', 'grapes':'8', 'kiwis':'1'}}\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a9c0020f-db59-4af3-a099-ebcdfbb017c8", + "metadata": {}, + "source": [ + "### Rendering Values\n", + "\n", + "If the object's config contains unrendered Jinja2 expressions, the `dereference()` method will render as many as possible. The optional `context` parameter can be used to provide additional values with a Python `dict`.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "65315885-28f7-4e34-b5e9-f07b51b85e42", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[fruit count]\n", + "apples = 3\n", + "grapes = 15\n", + "kiwis = 2\n" + ] + } + ], + "source": [ + "fruits.dereference(\n", + " context={'grape_count':'15'}\n", + ")\n", + "print(fruits)" + ] + }, + { + "cell_type": "markdown", + "id": "d9a330d1-35ff-4843-90fa-dd19e6eb98e9", + "metadata": {}, + "source": [ + "### Writing Configs in a Specified Format\n", + "\n", + "Each of the `config` tool's classes provide methods that write configs of their format. With the `fruits` object, which is an instance of `INIConfig`, INI configs are written. `dump()` is one of these methods, which writes the config stored in the object to a file specified by providing the `path` parameter with a Path object. If `path` is `None`, the config is written to `stdout`.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "7cd35285-9eb9-4796-b1d9-cf84160c0c0e", + "metadata": {}, + "outputs": [], + "source": [ + "fruits.dump(\n", + " path=Path('tmp/fruits.ini')\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "0c0170d2-16ab-4b61-a3e5-2840b4f59271", + "metadata": {}, + "source": [ + "Below we can see that the config was written in the INI format at the specified path.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "f2ce9e59-f03c-4203-b929-87b4c0eae9de", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[fruit count]\n", + "apples = 3\n", + "grapes = 15\n", + "kiwis = 2\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat tmp/fruits.ini" + ] + }, + { + "cell_type": "markdown", + "id": "83a9d6b2-161f-4bc0-891c-477bd5c7a013", + "metadata": {}, + "source": [ + "To write a config that is not stored in the object, the `dump_dict()` method is used. This method takes a config in the form of a `dict` and, like `dump()`, writes the config in the INI format to `stdout` if `path` is `None` or to the path that a Path object indicates.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "b896632e-6a5a-4756-a32b-0630bc02d504", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[fruit count]\n", + "oranges = 4\n", + "blueberries = 9\n" + ] + } + ], + "source": [ + "other_fruits = {'fruit count':{'oranges':4, 'blueberries':9}}\n", + "fruits.dump_dict(\n", + " cfg=other_fruits,\n", + " path=None\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "945d1e71-2704-49de-9a7c-dc8bf58c12fe", + "metadata": {}, + "source": [ + "### Updating Values\n", + "\n", + "The `update_from()` method adds new or updated key-value pairs to the stored config, and these are provided as a dictionary via the `src` parameter. \n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "7a3f8247-75f6-4cc6-9a9d-7e69896120e7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[fruit count]\n", + "apples = 3\n", + "grapes = 15\n", + "kiwis = 4\n", + "raspberries = 12\n" + ] + } + ], + "source": [ + "fruits.update_from(\n", + " src={'fruit count':{'kiwis': '4', 'raspberries': '12'}}\n", + ")\n", + "print(fruits)" + ] } ], "metadata": { diff --git a/notebooks/example.ipynb b/notebooks/example.ipynb deleted file mode 100644 index 167a62998..000000000 --- a/notebooks/example.ipynb +++ /dev/null @@ -1,429 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "afc23938-c4e4-4240-8fdc-66f0fff9ec06", - "metadata": {}, - "source": [ - "# Example Notebook\n", - "\n", - "## Example 1: Building YAML config from a Python dictionary\n", - "First, we need to import `uwtools.api.config` from the uwtools python package." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "a4a3fb02-06e0-4bf8-a78a-bfdb4fc474b3", - "metadata": {}, - "outputs": [], - "source": [ - "from uwtools.api import config" - ] - }, - { - "cell_type": "markdown", - "id": "ec5c7a56-d605-4a58-bd48-4eaba9cd25bc", - "metadata": {}, - "source": [ - "The `config.get_yaml_config` method can create a `YAMLconfig` object when given a Python dictionary." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "71957053-4264-42d5-a4e7-43b50a0ed4e4", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "values:\n", - " date: 20240105\n", - " greeting: Good Night\n", - " recipient: Moon\n", - " repeat: 2\n" - ] - } - ], - "source": [ - "dictionary = {\"values\":{\"date\":20240105, \"greeting\":\"Good Night\", \"recipient\":\"Moon\", \"repeat\":2}}\n", - "\n", - "config_yaml = config.get_yaml_config(dictionary)\n", - "print(config_yaml)" - ] - }, - { - "cell_type": "markdown", - "id": "dd7bcc1a-6402-42f8-884b-b22ebe1f04f5", - "metadata": {}, - "source": [ - "## Example 2: Rendering a template with uwtools\n", - "Next, let's look at using the `template` tool to render a Jinja2 template." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "bb672e22-d206-4d65-9dfd-b4b480d0937b", - "metadata": {}, - "outputs": [], - "source": [ - "from uwtools.api import template" - ] - }, - { - "cell_type": "markdown", - "id": "9328416e-b1b8-4062-9c56-5b37440cbc4a", - "metadata": {}, - "source": [ - "We have a Jinja2 template file in `fixtures/example/ex2-config.yaml` that looks like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "67ebded6-474e-465d-81f2-ca8f76b89f54", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "user:\n", - " name: {{ first }} {{ last }}\n", - " favorite_food: {{ food }}\n" - ] - } - ], - "source": [ - "%%bash\n", - "cat fixtures/example/ex2-config.yaml" - ] - }, - { - "cell_type": "markdown", - "id": "8f29f1dc-a201-4356-a729-7b40565c8aa4", - "metadata": {}, - "source": [ - "We can use another yaml file that contains the values we want to add to the template to complete it:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "5df2f19a-6cb2-4c67-8fdf-2a5aa563b4e5", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "first: John\n", - "last: Doe\n", - "food: burritos\n" - ] - } - ], - "source": [ - "%%bash\n", - "cat fixtures/example/ex2-values.yaml" - ] - }, - { - "cell_type": "markdown", - "id": "cd2999db-c1e7-4ae3-9169-b807b63712e7", - "metadata": {}, - "source": [ - "Using `template.render` we can render the `ex2-config.yaml` file using the values supplied by the `ex2-values.yaml` to create a complete and ready to use config file." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "ce7f9265-4536-4163-9e2f-53aaa6ff0c63", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "user:\n", - " name: John Doe\n", - " favorite_food: burritos\n" - ] - } - ], - "source": [ - "source = \"fixtures/example/ex2-config.yaml\"\n", - "\n", - "vals = \"fixtures/example/ex2-values.yaml\"\n", - "\n", - "target = \"fixtures/example/ex2-rendered-config.yaml\"\n", - "\n", - "print(template.render(values_src=vals, values_format=\"yaml\", input_file=source, output_file=target))" - ] - }, - { - "cell_type": "markdown", - "id": "f5d0b35f-d894-4758-a462-81ffa88760c0", - "metadata": {}, - "source": [ - "Let's take a look at the rendered file:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "9fe7687f-5c6c-44fc-aaf7-92711ce97457", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "user:\n", - " name: John Doe\n", - " favorite_food: burritos\n" - ] - } - ], - "source": [ - "%%bash\n", - "cat fixtures/example/ex2-rendered-config.yaml" - ] - }, - { - "cell_type": "markdown", - "id": "ea5eff92", - "metadata": {}, - "source": [ - "## Example 3: Comparing two config files\n", - "Let's explore using the `config.compare()` method to compare two config files." - ] - }, - { - "cell_type": "markdown", - "id": "71d27692", - "metadata": {}, - "source": [ - "We again need to start by importing the `uwtools.api.config` from the `uwtools` python package." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "7fb40514", - "metadata": {}, - "outputs": [], - "source": [ - "from uwtools.api import config\n", - "from uwtools.api.logging import use_uwtools_logger\n", - "use_uwtools_logger()" - ] - }, - { - "cell_type": "markdown", - "id": "b0103f44", - "metadata": {}, - "source": [ - "Please review the [config.compare() documentation](https://uwtools.readthedocs.io/en/main/sections/user_guide/api/config.html#uwtools.api.config.compare) for full information on this function's arguments." - ] - }, - { - "cell_type": "markdown", - "id": "b121dc29", - "metadata": {}, - "source": [ - "For example, let's compare two Fortran namelist files with differences:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "8279b5c6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "&foo n=88, s=\"string\" /\n" - ] - } - ], - "source": [ - "%%bash\n", - "cat fixtures/example/ex3-config-test-file-a.nml" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "e3232247", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "&FOO\n", - " S = \"string\"\n", - " N = 99\n", - "/\n" - ] - } - ], - "source": [ - "%%bash\n", - "cat fixtures/example/ex3-config-test-file-b.nml" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "5dcff125", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2024-08-18T10:24:32] INFO - fixtures/example/ex3-config-test-file-a.nml\n", - "[2024-08-18T10:24:32] INFO + fixtures/example/ex3-config-test-file-b.nml\n", - "[2024-08-18T10:24:32] INFO ---------------------------------------------------------------------\n", - "[2024-08-18T10:24:32] INFO foo: n: - 88 + 99\n" - ] - }, - { - "data": { - "text/plain": [ - "False" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "file_a = \"fixtures/example/ex3-config-test-file-a.nml\"\n", - "different_file_b = \"fixtures/example/ex3-config-test-file-b.nml\"\n", - "config.compare(file_a, different_file_b)" - ] - }, - { - "cell_type": "markdown", - "id": "c16c38d0", - "metadata": {}, - "source": [ - "The `config()` method returns `False` to denote the files are different. The UW logger shows the difference, of one file containing `n = 88`, and one file containing `n = 99`.\n", - "\n", - "Now to compare two semantically equivalent files:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "370149a3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "&foo n=88, s=\"string\" /\n" - ] - } - ], - "source": [ - "%%bash\n", - "cat fixtures/example/ex3-config-test-file-a.nml" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "bd444fba", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "&FOO\n", - " S = \"string\"\n", - " N = 88\n", - "/\n" - ] - } - ], - "source": [ - "%%bash\n", - "cat fixtures/example/ex3-config-test-file-c.nml" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "9d658b72", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2024-08-18T10:24:35] INFO - fixtures/example/ex3-config-test-file-a.nml\n", - "[2024-08-18T10:24:35] INFO + fixtures/example/ex3-config-test-file-c.nml\n", - "[2024-08-18T10:24:35] INFO ---------------------------------------------------------------------\n" - ] - }, - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "file_a = \"fixtures/example/ex3-config-test-file-a.nml\"\n", - "identical_file_c = \"fixtures/example/ex3-config-test-file-c.nml\"\n", - "config.compare(file_a, identical_file_c)" - ] - }, - { - "cell_type": "markdown", - "id": "b84db243", - "metadata": {}, - "source": [ - "The `config()` method returns `True` to denote the files are semantically equivalent." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python [conda env:DEV-uwtools] *", - "language": "python", - "name": "conda-env-DEV-uwtools-py" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/fixtures/config/alt-config.nml b/notebooks/fixtures/config/alt-config.nml new file mode 100644 index 000000000..487670727 --- /dev/null +++ b/notebooks/fixtures/config/alt-config.nml @@ -0,0 +1,5 @@ +&memo + sender_id = "{{ id }}" + message = "{{ greeting }}, {{ recipient }}!" + sent = .TRUE. +/ diff --git a/notebooks/fixtures/config/fruit-config.ini b/notebooks/fixtures/config/fruit-config.ini new file mode 100644 index 000000000..08973afb0 --- /dev/null +++ b/notebooks/fixtures/config/fruit-config.ini @@ -0,0 +1,4 @@ +[fruit count] +apples = 3 +grapes = {{ grape_count }} +kiwis = 2 diff --git a/notebooks/fixtures/config/validate.jsonschema b/notebooks/fixtures/config/validate.jsonschema new file mode 100644 index 000000000..8593430f8 --- /dev/null +++ b/notebooks/fixtures/config/validate.jsonschema @@ -0,0 +1,15 @@ +{ + "additionalProperties": false, + "properties": { + "greeting": { + "type": "string" + }, + "recipient": { + "type": "string" + } + }, + "required": [ + "greeting", "recipient" + ], + "type": "object" +} diff --git a/notebooks/fixtures/example/ex2-config.yaml b/notebooks/fixtures/example/ex2-config.yaml deleted file mode 100644 index 78ddc7977..000000000 --- a/notebooks/fixtures/example/ex2-config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -user: - name: {{ first }} {{ last }} - favorite_food: {{ food }} diff --git a/notebooks/fixtures/example/ex2-rendered-config.yaml b/notebooks/fixtures/example/ex2-rendered-config.yaml deleted file mode 100644 index 2840700bf..000000000 --- a/notebooks/fixtures/example/ex2-rendered-config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -user: - name: John Doe - favorite_food: burritos diff --git a/notebooks/fixtures/example/ex2-values.yaml b/notebooks/fixtures/example/ex2-values.yaml deleted file mode 100644 index 5ae0cc34e..000000000 --- a/notebooks/fixtures/example/ex2-values.yaml +++ /dev/null @@ -1,3 +0,0 @@ -first: John -last: Doe -food: burritos diff --git a/notebooks/fixtures/example/ex3-config-test-file-a.nml b/notebooks/fixtures/example/ex3-config-test-file-a.nml deleted file mode 100644 index 42e15827f..000000000 --- a/notebooks/fixtures/example/ex3-config-test-file-a.nml +++ /dev/null @@ -1 +0,0 @@ -&foo n=88, s="string" / diff --git a/notebooks/fixtures/example/ex3-config-test-file-b.nml b/notebooks/fixtures/example/ex3-config-test-file-b.nml deleted file mode 100644 index ecb87d942..000000000 --- a/notebooks/fixtures/example/ex3-config-test-file-b.nml +++ /dev/null @@ -1,4 +0,0 @@ -&FOO - S = "string" - N = 99 -/ diff --git a/notebooks/fixtures/example/ex3-config-test-file-c.nml b/notebooks/fixtures/example/ex3-config-test-file-c.nml deleted file mode 100644 index e6603e87d..000000000 --- a/notebooks/fixtures/example/ex3-config-test-file-c.nml +++ /dev/null @@ -1,4 +0,0 @@ -&FOO - S = "string" - N = 88 -/ diff --git a/notebooks/tests/test_config.py b/notebooks/tests/test_config.py index 04260206a..5e220288e 100644 --- a/notebooks/tests/test_config.py +++ b/notebooks/tests/test_config.py @@ -71,3 +71,66 @@ def test_realize_to_dict(): assert tb.cell_output_text(51) == config_str config_out = ("'id': '456'", "'greeting': 'Hello'", "'recipient': 'World'") assert all(x in tb.cell_output_text(53) for x in config_out) + + +def test_compare(): + with open("fixtures/config/base-config.nml", "r", encoding="utf-8") as f: + base_cfg = f.read().rstrip() + with open("fixtures/config/alt-config.nml", "r", encoding="utf-8") as f: + alt_cfg = f.read().rstrip() + with open("tmp/config-copy.nml", "r", encoding="utf-8") as f: + cp_cfg = f.read().rstrip() + with testbook("config.ipynb", execute=True) as tb: + assert base_cfg in tb.cell_output_text(57) + assert alt_cfg in tb.cell_output_text(57) + diff_cmp = ( + "INFO - fixtures/config/base-config.nml", + "INFO + fixtures/config/alt-config.nml", + "INFO memo: sent: - False + True", + "False", + ) + assert all(x in tb.cell_output_text(59) for x in diff_cmp) + assert base_cfg == cp_cfg # cell 61 creates this copy + same_cmp = ("INFO - fixtures/config/base-config.nml", "INFO + tmp/config-copy.nml", "True") + assert all(x in tb.cell_output_text(63) for x in same_cmp) + assert "ERROR Formats do not match: yaml vs nml" in tb.cell_output_text(65) + + +def test_validate(): + with open("fixtures/config/get-config.yaml", "r", encoding="utf-8") as f: + cfg = f.read().rstrip() + with open("fixtures/config/validate.jsonschema", "r", encoding="utf-8") as f: + schema = f.read().rstrip() + with testbook("config.ipynb", execute=True) as tb: + assert tb.cell_output_text(69) == cfg + assert tb.cell_output_text(71) == schema + valid_out = ("INFO 0 UW schema-validation errors found", "True") + assert all(x in tb.cell_output_text(73) for x in valid_out) + invalid_out = ( + "ERROR 1 UW schema-validation error found", + "ERROR 47 is not of type 'string'", + "False", + ) + assert all(x in tb.cell_output_text(75) for x in invalid_out) + + +def test_cfg_classes(): + with open("fixtures/config/fruit-config.ini", "r", encoding="utf-8") as f: + cfg = f.read().rstrip() + with testbook("config.ipynb", execute=True) as tb: + with open("tmp/fruits.ini", "r", encoding="utf-8") as f: + dump = f.read().rstrip() + assert tb.cell_output_text(79) == cfg + assert tb.cell_output_text(81) == "True" + diff_cmp = ( + "INFO fruit count: grapes: - {{ grape_count }} + 8", + "INFO fruit count: kiwis: - 2 + 1", + "False", + ) + assert all(x in tb.cell_output_text(83) for x in diff_cmp) + assert "grapes = 15" in tb.cell_output_text(85) + assert tb.cell_output_text(89) == dump + dump_dict = ("[fruit count]", "oranges = 4", "blueberries = 9") + assert all(x in tb.cell_output_text(91) for x in dump_dict) + updated_vals = ("kiwis = 4", "raspberries = 12") + assert all(x in tb.cell_output_text(93) for x in updated_vals) diff --git a/notebooks/tests/test_example.py b/notebooks/tests/test_example.py deleted file mode 100644 index c573bab8e..000000000 --- a/notebooks/tests/test_example.py +++ /dev/null @@ -1,62 +0,0 @@ -import os - -import yaml -from testbook import testbook - - -# Run all cells of the example notebook. -@testbook("example.ipynb", execute=True) -def test_get_yaml_config(tb): - - # Check output text of the cell that prints the YAMLconfig object. - assert ( - tb.cell_output_text(3) - == "values:\n date: 20240105\n greeting: Good Night\n recipient: Moon\n repeat: 2" - ) - - # Extract the config_yaml variable from the notebook and test its values. - nb_yaml = tb.ref("config_yaml") - assert nb_yaml["values"] == { - "date": 20240105, - "greeting": "Good Night", - "recipient": "Moon", - "repeat": 2, - } - - -def test_template_render(): - # Remove the rendered file if it exists. - rendered_path = "fixtures/example/ex2-rendered-config.yaml" - if os.path.exists(rendered_path): - os.remove(rendered_path) - - # Run all cells of the example notebook. - with testbook("example.ipynb", execute=True) as tb: - - # Check output text of cells with %%bash cell magics. - assert ( - tb.cell_output_text(7) - == "user:\n name: {{ first }} {{ last }}\n favorite_food: {{ food }}" - ) - assert tb.cell_output_text(9) == "first: John\nlast: Doe\nfood: burritos" - assert tb.cell_output_text(13) == "user:\n name: John Doe\n favorite_food: burritos" - - # Check that the rendered file was created in the correct directory. - assert os.path.exists(rendered_path) - - # Check the contents of the rendered file. - with open(rendered_path, "r", encoding="utf-8") as f: - user_config = yaml.safe_load(f) - assert user_config["user"] == {"name": "John Doe", "favorite_food": "burritos"} - - # Clean up the temporary file after the test is run. - os.remove(rendered_path) - - -# Run all cells of the example notebook. -@testbook("example.ipynb", execute=True) -def test_compare(tb): - - # Check output text of the cell prints the correct result - assert "False" in tb.cell_output_text(21) - assert "True" in tb.cell_output_text(25) From 04b802bdb5094f810dfb929f5d18b383115ff597 Mon Sep 17 00:00:00 2001 From: Travis Byrne <42480803+Byrnetp@users.noreply.github.com> Date: Thu, 31 Oct 2024 09:43:31 -0600 Subject: [PATCH 14/24] Jupyter Notebook: Rocoto Tool (#637) * Rocoto notebook, tests, and links * Apply suggestions from code review Co-authored-by: NaureenBharwaniNOAA <136371446+NaureenBharwaniNOAA@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Co-authored-by: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> * Changes from review * Update notebooks/rocoto.ipynb schemas typo fix Co-authored-by: Emily Carpenter <137525341+elcarpenterNOAA@users.noreply.github.com> * cyclestring markdown change * Remove unnecessary logger mentions * Add nested metatask example * Apply suggestions from code review Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> * Move nesting example to the end of metatask section --------- Co-authored-by: NaureenBharwaniNOAA <136371446+NaureenBharwaniNOAA@users.noreply.github.com> Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Co-authored-by: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Co-authored-by: Emily Carpenter <137525341+elcarpenterNOAA@users.noreply.github.com> --- README.md | 2 +- docs/sections/user_guide/api/rocoto.rst | 3 + docs/sections/user_guide/index.rst | 11 +- .../fixtures/rocoto/ent-cs-workflow.yaml | 16 + notebooks/fixtures/rocoto/ent-workflow.yaml | 14 + notebooks/fixtures/rocoto/err-workflow.xml | 10 + notebooks/fixtures/rocoto/err-workflow.yaml | 9 + .../fixtures/rocoto/meta-nested-workflow.yaml | 18 + notebooks/fixtures/rocoto/meta-workflow.yaml | 16 + notebooks/fixtures/rocoto/simple-workflow.xml | 11 + .../fixtures/rocoto/simple-workflow.yaml | 12 + .../fixtures/rocoto/tasks-deps-workflow.yaml | 31 + notebooks/fixtures/rocoto/tasks-workflow.yaml | 16 + notebooks/rocoto.ipynb | 1060 +++++++++++++++++ notebooks/tests/test_rocoto.py | 90 ++ 15 files changed, 1308 insertions(+), 11 deletions(-) create mode 100644 notebooks/fixtures/rocoto/ent-cs-workflow.yaml create mode 100644 notebooks/fixtures/rocoto/ent-workflow.yaml create mode 100644 notebooks/fixtures/rocoto/err-workflow.xml create mode 100644 notebooks/fixtures/rocoto/err-workflow.yaml create mode 100644 notebooks/fixtures/rocoto/meta-nested-workflow.yaml create mode 100644 notebooks/fixtures/rocoto/meta-workflow.yaml create mode 100644 notebooks/fixtures/rocoto/simple-workflow.xml create mode 100644 notebooks/fixtures/rocoto/simple-workflow.yaml create mode 100644 notebooks/fixtures/rocoto/tasks-deps-workflow.yaml create mode 100644 notebooks/fixtures/rocoto/tasks-workflow.yaml create mode 100644 notebooks/rocoto.ipynb create mode 100644 notebooks/tests/test_rocoto.py diff --git a/README.md b/README.md index ab9f574bf..c46c2aad5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Tests](https://github.com/ufs-community/uwtools/actions/workflows/test.yaml/badge.svg)](https://github.com/ufs-community/uwtools/actions) [![Release](https://github.com/ufs-community/uwtools/actions/workflows/release.yaml/badge.svg)](https://github.com/ufs-community/uwtools/releases) [![Docs](https://readthedocs.org/projects/uwtools/badge/?version=main)](https://uwtools.readthedocs.io/en/main/?badge=main) [![pkgpage](https://anaconda.org/ufs-community/uwtools/badges/version.svg)](https://anaconda.org/ufs-community/uwtools) [![pkgfiles](https://anaconda.org/ufs-community/uwtools/badges/latest_release_date.svg)](https://anaconda.org/ufs-community/uwtools/files) [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/ufs-community/uwtools/notebooks?labpath=notebooks%2Fexample.ipynb) +[![Tests](https://github.com/ufs-community/uwtools/actions/workflows/test.yaml/badge.svg)](https://github.com/ufs-community/uwtools/actions) [![Release](https://github.com/ufs-community/uwtools/actions/workflows/release.yaml/badge.svg)](https://github.com/ufs-community/uwtools/releases) [![Docs](https://readthedocs.org/projects/uwtools/badge/?version=main)](https://uwtools.readthedocs.io/en/main/?badge=main) [![pkgpage](https://anaconda.org/ufs-community/uwtools/badges/version.svg)](https://anaconda.org/ufs-community/uwtools) [![pkgfiles](https://anaconda.org/ufs-community/uwtools/badges/latest_release_date.svg)](https://anaconda.org/ufs-community/uwtools/files) # uwtools diff --git a/docs/sections/user_guide/api/rocoto.rst b/docs/sections/user_guide/api/rocoto.rst index dcacb9c0e..45dbfa06b 100644 --- a/docs/sections/user_guide/api/rocoto.rst +++ b/docs/sections/user_guide/api/rocoto.rst @@ -1,5 +1,8 @@ ``uwtools.api.rocoto`` ====================== +.. image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/ufs-community/uwtools/notebooks?labpath=notebooks%2Frocoto.ipynb + .. automodule:: uwtools.api.rocoto :members: diff --git a/docs/sections/user_guide/index.rst b/docs/sections/user_guide/index.rst index 31b4eed42..88d4c7d0e 100644 --- a/docs/sections/user_guide/index.rst +++ b/docs/sections/user_guide/index.rst @@ -5,16 +5,6 @@ User Guide :maxdepth: 2 installation - -.. raw:: html - - - -.. toctree:: - :maxdepth: 2 - cli/index api/index @@ -25,6 +15,7 @@ User Guide diff --git a/notebooks/fixtures/rocoto/ent-cs-workflow.yaml b/notebooks/fixtures/rocoto/ent-cs-workflow.yaml new file mode 100644 index 000000000..4aad5d5b8 --- /dev/null +++ b/notebooks/fixtures/rocoto/ent-cs-workflow.yaml @@ -0,0 +1,16 @@ +workflow: + attrs: + realtime: false + scheduler: slurm + cycledef: + - spec: 202410290000 202410300000 06:00:00 + entities: + LOG: "@Y-@m-@d/test@X.log" + log: + cyclestr: + value: logs/&LOG; + tasks: + task_greet: + command: echo Hello, World! + cores: 1 + walltime: 00:00:10 diff --git a/notebooks/fixtures/rocoto/ent-workflow.yaml b/notebooks/fixtures/rocoto/ent-workflow.yaml new file mode 100644 index 000000000..e6020aabb --- /dev/null +++ b/notebooks/fixtures/rocoto/ent-workflow.yaml @@ -0,0 +1,14 @@ +workflow: + attrs: + realtime: false + scheduler: slurm + cycledef: + - spec: 202410290000 202410300000 06:00:00 + entities: + LOG: "2024-10-29/test06:00:00.log" + log: logs/&LOG; + tasks: + task_greet: + command: echo Hello, World! + cores: 1 + walltime: 00:00:10 diff --git a/notebooks/fixtures/rocoto/err-workflow.xml b/notebooks/fixtures/rocoto/err-workflow.xml new file mode 100644 index 000000000..5f8f61473 --- /dev/null +++ b/notebooks/fixtures/rocoto/err-workflow.xml @@ -0,0 +1,10 @@ + + + logs/test.log + + 1 + 00:00:10 + echo Hello, World! + greet + + diff --git a/notebooks/fixtures/rocoto/err-workflow.yaml b/notebooks/fixtures/rocoto/err-workflow.yaml new file mode 100644 index 000000000..a6e37c180 --- /dev/null +++ b/notebooks/fixtures/rocoto/err-workflow.yaml @@ -0,0 +1,9 @@ +workflow: + attrs: + scheduler: slurm + cycledef: + - spec: 202410290000 202410300000 06:00:00 + tasks: + task_greet: + cores: 1 + walltime: 00:00:10 diff --git a/notebooks/fixtures/rocoto/meta-nested-workflow.yaml b/notebooks/fixtures/rocoto/meta-nested-workflow.yaml new file mode 100644 index 000000000..55a716d5f --- /dev/null +++ b/notebooks/fixtures/rocoto/meta-nested-workflow.yaml @@ -0,0 +1,18 @@ +workflow: + attrs: + realtime: false + scheduler: slurm + cycledef: + - spec: 202410290000 202410300000 06:00:00 + log: logs/test.log + tasks: + metatask_process: + var: + process: bake cool store + metatask_process_food: + var: + food: cookies cakes + task_#process#_#food#: + command: "echo It's time to #process# the #food#." + nodes: 1:ppn=4 + walltime: 00:00:30 diff --git a/notebooks/fixtures/rocoto/meta-workflow.yaml b/notebooks/fixtures/rocoto/meta-workflow.yaml new file mode 100644 index 000000000..1071b2653 --- /dev/null +++ b/notebooks/fixtures/rocoto/meta-workflow.yaml @@ -0,0 +1,16 @@ +workflow: + attrs: + realtime: false + scheduler: slurm + cycledef: + - spec: 202410290000 202410300000 06:00:00 + log: logs/test.log + tasks: + metatask_breakfast: + var: + food: biscuits OJ hashbrowns + prepare: bake pour fry + task_#food#: + command: "echo It's time for breakfast, #prepare# the #food#!" + cores: 1 + walltime: 00:00:03 diff --git a/notebooks/fixtures/rocoto/simple-workflow.xml b/notebooks/fixtures/rocoto/simple-workflow.xml new file mode 100644 index 000000000..62326060b --- /dev/null +++ b/notebooks/fixtures/rocoto/simple-workflow.xml @@ -0,0 +1,11 @@ + + + 202410290000 202410300000 06:00:00 + logs/test.log + + 1 + 00:00:10 + echo Hello, World! + greet + + diff --git a/notebooks/fixtures/rocoto/simple-workflow.yaml b/notebooks/fixtures/rocoto/simple-workflow.yaml new file mode 100644 index 000000000..47e7c55b6 --- /dev/null +++ b/notebooks/fixtures/rocoto/simple-workflow.yaml @@ -0,0 +1,12 @@ +workflow: + attrs: + realtime: false + scheduler: slurm + cycledef: + - spec: 202410290000 202410300000 06:00:00 + log: logs/test.log + tasks: + task_greet: + command: echo Hello, World! + cores: 1 + walltime: 00:00:10 diff --git a/notebooks/fixtures/rocoto/tasks-deps-workflow.yaml b/notebooks/fixtures/rocoto/tasks-deps-workflow.yaml new file mode 100644 index 000000000..3161e86c5 --- /dev/null +++ b/notebooks/fixtures/rocoto/tasks-deps-workflow.yaml @@ -0,0 +1,31 @@ +workflow: + attrs: + realtime: false + scheduler: slurm + cycledef: + - spec: 202410290000 202410300000 06:00:00 + log: logs/test.log + tasks: + task_bacon: + command: "echo Cooking bacon..." + cores: 1 + walltime: 00:00:10 + task_eggs: + command: "echo Cooking eggs..." + nodes: 1:ppn=4 + walltime: 00:00:10 + dependency: + datadep: + value: eggs_recipe.txt + task_serve: + command: "echo Serving breakfast..." + cores: 2 + walltime: 00:00:01 + dependency: + and: + taskdep_eggs: + attrs: + task: bacon + taskdep_bacon: + attrs: + task: eggs diff --git a/notebooks/fixtures/rocoto/tasks-workflow.yaml b/notebooks/fixtures/rocoto/tasks-workflow.yaml new file mode 100644 index 000000000..f415d700d --- /dev/null +++ b/notebooks/fixtures/rocoto/tasks-workflow.yaml @@ -0,0 +1,16 @@ +workflow: + attrs: + realtime: false + scheduler: slurm + cycledef: + - spec: 202410290000 202410300000 06:00:00 + log: logs/test.log + tasks: + task_bacon: + command: "echo Cooking bacon..." + cores: 1 + walltime: 00:00:10 + task_eggs: + command: "echo Cooking eggs..." + nodes: 1:ppn=4 + walltime: 00:00:10 diff --git a/notebooks/rocoto.ipynb b/notebooks/rocoto.ipynb new file mode 100644 index 000000000..984ff40d4 --- /dev/null +++ b/notebooks/rocoto.ipynb @@ -0,0 +1,1060 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c787c8df-e973-44d6-aa64-1f09c7a9a9d4", + "metadata": {}, + "source": [ + "# Rocoto Tool\n", + "\n", + "The `uwtools` API's `rocoto` module provides functions to build and validate Rocoto workflows. For more information on the UW YAML language than what is discussed here, see the Defining a Rocoto Workflow page. For more on Rocoto XML documents, see the Rocoto Documentation.\n", + "\n", + "Tested on `uwtools` version 2.4.2. For more information, please see the uwtools.api.rocoto Read the Docs page.\n", + "\n", + "## Table of Contents\n", + "* [Building Rocoto Workflows with UW YAML](#Building-Rocoto-Workflows-with-UW-YAML)\n", + " * [Entities and Cyclestrings](#Entities-and-Cyclestrings)\n", + " * [Tasks and Dependencies](#Tasks-and-Dependencies)\n", + " * [Metatasks](#Metatasks)\n", + "* [Validating Workflows](#Validating-Workflows)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "7dd67340-6553-40e9-be68-d79c1979280c", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from uwtools.api import rocoto\n", + "from uwtools.api.logging import use_uwtools_logger\n", + "\n", + "use_uwtools_logger()" + ] + }, + { + "cell_type": "markdown", + "id": "14ac9321-c59e-4149-bd08-7f2bcef1113e", + "metadata": {}, + "source": [ + "## Building Rocoto Workflows with UW YAML\n", + "\n", + "The `rocoto.realize()` function uses a UW YAML language to create Rocoto workflows in XML format.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "1d10d514-d918-4cbd-aa61-c2be8ee9e298", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function realize in module uwtools.api.rocoto:\n", + "\n", + "realize(config: Union[uwtools.config.formats.yaml.YAMLConfig, pathlib.Path, str, NoneType], output_file: Union[str, pathlib.Path, NoneType] = None, stdin_ok: bool = False) -> bool\n", + " Realize the Rocoto workflow defined in the given YAML as XML.\n", + "\n", + " If no input file is specified, ``stdin`` is read. A ``YAMLConfig`` object may also be provided\n", + " as input. If no output file is specified, ``stdout`` is written to. Both the input config and\n", + " output Rocoto XML will be validated against appropriate schcemas.\n", + "\n", + " :param config: YAML input file or ``YAMLConfig`` object (``None`` => read ``stdin``).\n", + " :param output_file: XML output file path (``None`` => write to ``stdout``).\n", + " :param stdin_ok: OK to read from ``stdin``?\n", + " :return: ``True``.\n", + "\n" + ] + } + ], + "source": [ + "help(rocoto.realize)" + ] + }, + { + "cell_type": "markdown", + "id": "65694bbf-cce1-4979-b872-17d4aac8ae84", + "metadata": {}, + "source": [ + "The following is an example of a simple workflow written in the UW YAML language. It uses a top-level `workflow:` block that contains all other blocks in the workflow. The workflow's global attributes are set within an `attrs:` block, and each workflow has two required attributes: `realtime` and `scheduler`. The `realtime` key indicates whether the workflow will be run in realtime or in retrospective mode, where a value of `true` means that the workflow will be run in realtime mode. The `scheduler` key tells Rocoto which batch system to use when submitting and monitoring jobs. Each workflow must contain a `cycledef:` block that defines one or more sets of cycles the workflow will iterate over. A set of cycles must be given using the `spec` key. This key may define a set of cycles using either the \"start stop step\" method or the \"crontab-like\" method. The \"start stop step\" method is used below. A `log:` block is required to define the path where Rocoto logs are written. At least one task must be defined in the `tasks:` block, which is discussed in the [Tasks and Dependencies](#Tasks-and-Dependencies) section of this notebook.\n", + "\n", + "The simple workflow below contains a minimal set of keys. For more on the UW YAML language, see the Defining a Rocoto Workflow page.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "9281ae4f-4d78-4401-bf6f-87d4b873e846", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workflow:\n", + " attrs:\n", + " realtime: false\n", + " scheduler: slurm\n", + " cycledef:\n", + " - spec: 202410290000 202410300000 06:00:00\n", + " log: logs/test.log\n", + " tasks:\n", + " task_greet:\n", + " command: echo Hello, World!\n", + " cores: 1\n", + " walltime: 00:00:10\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/rocoto/simple-workflow.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "da06b1f6-7699-4e89-bbd0-de6a5e9ea0b7", + "metadata": {}, + "source": [ + "Using `rocoto.realize()`, the UW YAML from above is translated to Rocoto XML. A `config` may be given as a string path, Path object, or `YAMLConfig` object. Likewise, the path to the XML output file may be defined by providing `output_file` with a string path or Path object. If `output_file` is omitted or set to `None`, the XML will be written to `stdout`. Both the input config and the output Rocoto XML are validated against appropriate schemas. The number of schema-validation errors, as well as details on the errors (if any), are reported.\n", + "\n", + "The `stdin_ok` argument can be used to permit configs to be read from `stdin` when `config` is omitted or set to `None`, but this is a rare use case beyond the scope of this notebook that will not be discussed here.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "983636f9-7c39-4f0e-a76e-e35129d2b9fe", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-31T09:34:05] INFO 0 UW schema-validation errors found\n", + "[2024-10-31T09:34:05] INFO 0 Rocoto validation errors found\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rocoto.realize(\n", + " config=Path('fixtures/rocoto/simple-workflow.yaml'),\n", + " output_file='tmp/simple-workflow.xml'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "12b07a2c-7f9c-4480-b165-9e3b5a0ced6d", + "metadata": {}, + "source": [ + "The resulting Rocoto XML file is shown below. An XML header is automatically added without the need to explicitly define it in the UW YAML. Note how blocks from UW YAML language have been transformed into XML tags along with their attributes and values. For example, attributes defined by the `attrs:` block in the UW YAML have become attributes of the `` tag in the XML.\n", + "\n", + "For more information on Rocoto workflows, including tags like the ones shown here and thier attributes, see the Rocoto Documentation.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d895e1cf-e8af-437b-9a38-2b03ec34f527", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + " 202410290000 202410300000 06:00:00\n", + " logs/test.log\n", + " \n", + " 1\n", + " 00:00:10\n", + " echo Hello, World!\n", + " greet\n", + " \n", + "\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat tmp/simple-workflow.xml" + ] + }, + { + "cell_type": "markdown", + "id": "39f50dbd-e3f3-4e1e-a10b-f622f635b16b", + "metadata": {}, + "source": [ + "The following workflow is missing required components: `workflow` doesn't contain a `realtime` attribute, a `log:` block isn't included, and `task_greet` doesn't include a `command`.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "56d04e6b-5fa2-4c93-ac2f-c95fa23c888e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workflow:\n", + " attrs:\n", + " scheduler: slurm\n", + " cycledef:\n", + " - spec: 202410290000 202410300000 06:00:00\n", + " tasks:\n", + " task_greet:\n", + " cores: 1\n", + " walltime: 00:00:10\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/rocoto/err-workflow.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "f102c6ca-c8a5-4f9c-8e40-b5fa040bbeab", + "metadata": {}, + "source": [ + "When validation errors occur, `realize()` raises an exception indicating what type of error occurred. Here, the YAML validation errors cause a `UWConfigError` to be raised. The number of validation errors present and their locations within the workflow structure are also shown.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "16704f19-cbca-4765-8c72-16512fc96e9b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-31T09:34:05] ERROR 3 UW schema-validation errors found\n", + "[2024-10-31T09:34:05] ERROR Error at workflow -> attrs:\n", + "[2024-10-31T09:34:05] ERROR 'realtime' is a required property\n", + "[2024-10-31T09:34:05] ERROR Error at workflow -> tasks -> task_greet:\n", + "[2024-10-31T09:34:05] ERROR 'command' is a required property\n", + "[2024-10-31T09:34:05] ERROR Error at workflow:\n", + "[2024-10-31T09:34:05] ERROR 'log' is a required property\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "YAML validation errors \n" + ] + } + ], + "source": [ + "try:\n", + " rocoto.realize(\n", + " config=Path('fixtures/rocoto/err-workflow.yaml'),\n", + " output_file='tmp/err-workflow.xml'\n", + " )\n", + "except Exception as e:\n", + " print(e, type(e))" + ] + }, + { + "cell_type": "markdown", + "id": "cbad7aef-02b6-43a3-9241-349affa2f71c", + "metadata": {}, + "source": [ + "### Entities and Cyclestrings\n", + "\n", + "Constants called entities may be defined so that their values can be referenced throughout the rest of the Rocoto XML. These are defined in an `entities:` block, with their names and values given as keys and values in the YAML. Below, an entity named `LOG` is defined with a string value. This value is referred elsewhere in the Rocoto XML with the syntax `&ENTITY_NAME;`. In this case, note the `&LOG;` entity within the `log:` block.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "074a6e7e-7c05-4037-b954-06eba8ae2241", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workflow:\n", + " attrs:\n", + " realtime: false\n", + " scheduler: slurm\n", + " cycledef:\n", + " - spec: 202410290000 202410300000 06:00:00\n", + " entities:\n", + " LOG: \"2024-10-29/test06:00:00.log\"\n", + " log: logs/&LOG;\n", + " tasks:\n", + " task_greet:\n", + " command: echo Hello, World!\n", + " cores: 1\n", + " walltime: 00:00:10\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/rocoto/ent-workflow.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "34cbb0e1-ac5f-4764-afbb-9574966542c6", + "metadata": {}, + "source": [ + "Cycle strings represent dynamic cycle time components that are represented by specific flags and are rendered when Rocoto runs the XML. Here, the `LOG` entity contains `@Y`, `@m`, `@d` and `@X` flags that represent the year, month, day, and time relative to a cycle defined by the `cycledefs:` entry. For more information on these flags, see the Rocoto Documentation. A `cyclestr:` block is used to mark a string containing cycle string flags for rendering when Rocoto runs. Here, since the `LOG` entity contains these flags, a `cyclestr:` block within the `log:` block indicates that the flags should be rendered when Rocoto runs. This string itself is contained in a `value` key.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "e4df500f-6b11-4c0e-ac44-3a5443d0ee02", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workflow:\n", + " attrs:\n", + " realtime: false\n", + " scheduler: slurm\n", + " cycledef:\n", + " - spec: 202410290000 202410300000 06:00:00\n", + " entities:\n", + " LOG: \"@Y-@m-@d/test@X.log\"\n", + " log: \n", + " cyclestr:\n", + " value: logs/&LOG;\n", + " tasks:\n", + " task_greet:\n", + " command: echo Hello, World!\n", + " cores: 1\n", + " walltime: 00:00:10\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/rocoto/ent-cs-workflow.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "305485ae-1676-473c-ad4e-5521145b5663", + "metadata": {}, + "source": [ + "As before, the `realize()` function transforms the UW YAML into Rocoto XML.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "5f1b9dcd-87cf-41c3-ab3c-e581e2967214", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-31T09:34:05] INFO 0 UW schema-validation errors found\n", + "[2024-10-31T09:34:05] INFO 0 Rocoto validation errors found\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rocoto.realize(\n", + " config='fixtures/rocoto/ent-cs-workflow.yaml',\n", + " output_file='tmp/ent-cs-workflow.xml'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "3f28d943-5e39-412b-8708-865a63030e29", + "metadata": {}, + "source": [ + "Here we see the Rocoto XML with the addition of an entity and a `` tag. The entity is defined in the header of the XML document, and the `` tag is added within the `` tag.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "1a1bea14-dbae-4e7d-96f7-0ec552b0e25a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "]>\n", + "\n", + " 202410290000 202410300000 06:00:00\n", + " \n", + " logs/&LOG;\n", + " \n", + " \n", + " 1\n", + " 00:00:10\n", + " echo Hello, World!\n", + " greet\n", + " \n", + "\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat tmp/ent-cs-workflow.xml" + ] + }, + { + "cell_type": "markdown", + "id": "6b339d55-8506-443e-9b7b-0d943aaf4641", + "metadata": {}, + "source": [ + "### Tasks and Dependencies\n", + "\n", + "A `tasks:` block defines all tasks in a Rocoto workflow. Each task is contained within its own block, where the key is `task_` followed by the name of the task. There are two tasks in the example below, `task_bacon` and `task_eggs`. In the Rocoto XML, two separate `` tags will be created with their `name` attributes set to \"bacon\" and \"eggs\" respectively. Each task must contain a command to execute indicated by the `command` key and an amount of time to request when submitting the task for execution indicated by the `walltime` key. Each task must also contain either a `cores`, `nodes`, or `native` key to request a given number of nodes/cores used to execute the task. The `task_bacon:` block below requests 1 core, while the `task_eggs:` block requests 4 cores on 1 node.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "33c67471-84a1-4b0f-b0bd-40f805e6615f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workflow:\n", + " attrs:\n", + " realtime: false\n", + " scheduler: slurm\n", + " cycledef:\n", + " - spec: 202410290000 202410300000 06:00:00\n", + " log: logs/test.log\n", + " tasks:\n", + " task_bacon:\n", + " command: \"echo Cooking bacon...\"\n", + " cores: 1\n", + " walltime: 00:00:10\n", + " task_eggs:\n", + " command: \"echo Cooking eggs...\"\n", + " nodes: 1:ppn=4\n", + " walltime: 00:00:10\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/rocoto/tasks-workflow.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "08e9c593-7ba1-45da-8f1a-99389c9dfdf5", + "metadata": {}, + "source": [ + "Each task may optionally have one or more dependencies that must be accounted for before a task runs. These are specified using a `dependency:` block within the `task_*` block that the dependencies apply to. Dependencies are structured as boolean expressions using a variety of keys that may define specific types of dependencies like task or data dependencies. They may also group dependencies together using boolean operators keys like `and` or `or`. For a full list of possible tags, see the Rocoto Documentation. \n", + "\n", + "Below, the `task_eggs:` block includes one data dependency indicated by the `datadep` key, plus a `value` key that identifies the required data. The `task_serve:` block includes two task dependencies for the bacon and eggs tasks. Since there are multiple dependencies here, they need to be contained within a boolean operator block that describes how to deal with the group of dependencies which may not all have the same level of completion. Here the `and:` block indicates that all of the individual tasks (i.e. `task_eggs`) within need to be completed. The two task dependencies must have unique names since they exist at the same level, and they are differentiated here using the `_name` suffix. To prevent circular dependencies, task dependencies must have a `task` attribute that indicates the name of a task that is already defined above it. Similar to the `workflow:` block, an `attrs:` block is used here to add attributes to `taskdep`, and the `task` key specifies the value of the task attribute.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "be4d4488-9ff6-4a6d-ace0-464e21f31116", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workflow:\n", + " attrs:\n", + " realtime: false\n", + " scheduler: slurm\n", + " cycledef:\n", + " - spec: 202410290000 202410300000 06:00:00\n", + " log: logs/test.log\n", + " tasks:\n", + " task_bacon:\n", + " command: \"echo Cooking bacon...\"\n", + " cores: 1\n", + " walltime: 00:00:10\n", + " task_eggs:\n", + " command: \"echo Cooking eggs...\"\n", + " nodes: 1:ppn=4\n", + " walltime: 00:00:10\n", + " dependency:\n", + " datadep:\n", + " value: eggs_recipe.txt\n", + " task_serve:\n", + " command: \"echo Serving breakfast...\"\n", + " cores: 2\n", + " walltime: 00:00:01\n", + " dependency:\n", + " and:\n", + " taskdep_eggs:\n", + " attrs:\n", + " task: bacon\n", + " taskdep_bacon:\n", + " attrs:\n", + " task: eggs\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/rocoto/tasks-deps-workflow.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "c6015712-f144-4640-9be8-1cd651df74b9", + "metadata": {}, + "source": [ + "Here, the `realize()` function transforms this UW YAML into Rocoto XML.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "001ac8ed-0f31-4012-bc63-55d63848e1d4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-31T09:34:05] INFO 0 UW schema-validation errors found\n", + "[2024-10-31T09:34:05] INFO 0 Rocoto validation errors found\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rocoto.realize(\n", + " config='fixtures/rocoto/tasks-deps-workflow.yaml',\n", + " output_file='tmp/tasks-deps-workflow.xml'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "dc87e5aa-aa21-4bbf-8204-389c355a1bbd", + "metadata": {}, + "source": [ + "Note how each task has its own tag in the Rocoto XML document, with name attributes that came from the unique suffixes of the `task_` keys. While the bacon task contains no `` tag, the eggs and serve tasks do. Within the serve task's dependencies, the `` tag describes the need for both of the two task dependencies to be fulfilled. Each `` task dependency uses the `task` attribute to point to a previously named task. \n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "7b9ecc04-9851-4e34-985f-d908285dc8e2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + " 202410290000 202410300000 06:00:00\n", + " logs/test.log\n", + " \n", + " 1\n", + " 00:00:10\n", + " echo Cooking bacon...\n", + " bacon\n", + " \n", + " \n", + " 1:ppn=4\n", + " 00:00:10\n", + " echo Cooking eggs...\n", + " eggs\n", + " \n", + " eggs_recipe.txt\n", + " \n", + " \n", + " \n", + " 2\n", + " 00:00:01\n", + " echo Serving breakfast...\n", + " serve\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat tmp/tasks-deps-workflow.xml" + ] + }, + { + "cell_type": "markdown", + "id": "f88d22e0-3433-482d-aeb3-44bbb8cf840f", + "metadata": {}, + "source": [ + "### Metatasks\n", + "\n", + "Metatasks define one or more tasks that are similar to one another using a substitution of values. Like tasks, metatask block keys use a suffix after an underscore to name a particular metatask. The metatask in the example below will have a `name=breakfast` attribute in its `` tag in the XML document. The values to substitute are defined in a `var:` block, and this block contains one or more keys representing the name of a list of values. The values in the list are separated by spaces. The number of tasks defined by a metatask is equal to the number of values in any list in the `var:` block. In the example below, two lists named `food` and `prepare` contain three values each, so three tasks are defined by this metatask. It is necessary that each list defined in a metatask has the same number of values. The values are referenced using the name of the list that contains the values bracketed by pound signs, as seen in the `task_#food#` key and in the following `command` string. \n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "6c8a8b8e-62b0-47f1-b5e3-763aef2e71ea", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workflow:\n", + " attrs:\n", + " realtime: false\n", + " scheduler: slurm\n", + " cycledef:\n", + " - spec: 202410290000 202410300000 06:00:00\n", + " log: logs/test.log\n", + " tasks:\n", + " metatask_breakfast:\n", + " var:\n", + " food: biscuits OJ hashbrowns\n", + " prepare: bake pour fry\n", + " task_#food#:\n", + " command: \"echo It's time for breakfast, #prepare# the #food#!\"\n", + " cores: 1\n", + " walltime: 00:00:03\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/rocoto/meta-workflow.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "8c2457c5-e305-4c4c-9020-3eb5553d2ba6", + "metadata": {}, + "source": [ + "Similar to previous examples, `realize()` transforms the metatask workflow to Rocoto XML.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "112a7586-5fef-46a0-83a5-f7016257fe9b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-31T09:34:05] INFO 0 UW schema-validation errors found\n", + "[2024-10-31T09:34:05] INFO 0 Rocoto validation errors found\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rocoto.realize(\n", + " config='fixtures/rocoto/meta-workflow.yaml',\n", + " output_file='tmp/meta-workflow.xml'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "0b078044-50fb-4158-b68e-aa0001621f7b", + "metadata": {}, + "source": [ + "The XML document below shows how the `` tag and each of its child tags efficiently define multiple similar tasks. Like previous examples, name attributes for task-related tags are created here from the suffixes of their keys in the UW YAML. Note that `` names were derived from full key names in the `var:` block. The ``, ``, and `` tags each contain strings that will receive substitute values wherever the placeholders `#food#` or `#prepare#` appear.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "3ac7f1af-44cf-440a-8b4f-6f63e4e98fee", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + " 202410290000 202410300000 06:00:00\n", + " logs/test.log\n", + " \n", + " biscuits OJ hashbrowns\n", + " bake pour fry\n", + " \n", + " 1\n", + " 00:00:03\n", + " echo It's time for breakfast, #prepare# the #food#!\n", + " #food#\n", + " \n", + " \n", + "\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat tmp/meta-workflow.xml" + ] + }, + { + "cell_type": "markdown", + "id": "d9c5ccad-07c4-4aad-aadd-e0343949f9fa", + "metadata": {}, + "source": [ + "Metatasks may be nested to create tasks using combinatorial lists of variables. This will create sets of tasks where each `var` value in a parent metatask applies to every child metatask. In the example below, a parent metatask contains a `var` named `process` with values `bake`, `cool`, and `store`. Its child metatask contains a `var` named `food` with values `cookies` and `cakes`. Tasks will be created to bake, cool, and store both cookies and cakes. Note that `var:` blocks at different levels do not necessarily contain the same number of values. \n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "69797f73-2475-449f-b036-2529f4379440", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workflow:\n", + " attrs:\n", + " realtime: false\n", + " scheduler: slurm\n", + " cycledef:\n", + " - spec: 202410290000 202410300000 06:00:00\n", + " log: logs/test.log\n", + " tasks:\n", + " metatask_process:\n", + " var:\n", + " process: bake cool store\n", + " metatask_process_food:\n", + " var:\n", + " food: cookies cakes\n", + " task_#process#_#food#:\n", + " command: \"echo It's time to #process# the #food#.\"\n", + " nodes: 1:ppn=4\n", + " walltime: 00:00:30\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/rocoto/meta-nested-workflow.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "40e85303-be55-4ebc-8f78-d9a0373cbf09", + "metadata": {}, + "source": [ + "## Validating Workflows\n", + "\n", + "The `rocoto.validate()` function checks the content of a Rocoto XML file against its schema, detecting and reporting any errors.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "6a681e6d-800c-4d9f-87a0-270e72dcb7be", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function validate in module uwtools.api.rocoto:\n", + "\n", + "validate(xml_file: Union[str, pathlib.Path, NoneType] = None, stdin_ok: bool = False) -> bool\n", + " Validate purported Rocoto XML file against its schema.\n", + "\n", + " :param xml_file: Path to XML file (``None`` or unspecified => read ``stdin``).\n", + " :param stdin_ok: OK to read from ``stdin``?\n", + " :return: ``True`` if the XML conforms to the schema, ``False`` otherwise.\n", + "\n" + ] + } + ], + "source": [ + "help(rocoto.validate)" + ] + }, + { + "cell_type": "markdown", + "id": "91f72e92-083c-4c36-90d8-1860133fe33b", + "metadata": {}, + "source": [ + "The following Rocoto XML is identical that generated in the [Building Rocoto Workflows with UW YAML](#Building-Rocoto-Workflows-with-UW-YAML) section above.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "22a8fe77-2094-4139-9ff2-91dc897c3af3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + " 202410290000 202410300000 06:00:00\n", + " logs/test.log\n", + " \n", + " 1\n", + " 00:00:10\n", + " echo Hello, World!\n", + " greet\n", + " \n", + "\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/rocoto/simple-workflow.xml" + ] + }, + { + "cell_type": "markdown", + "id": "bb04cf78-1afd-48eb-8673-803fbabac836", + "metadata": {}, + "source": [ + "`validate()` accepts Path objects or string paths passed via the `xml_file` parameter. (If `xml_file` is omitted or `None`, and `stdin_ok` is `True`, XML will be read from `stdin`, but this is a rare use case that won't be covered here.) The function returns `True` if the XML is validated without any errors, and `False` otherwise. The number of schema-validation errors, as well as details on the errors (if any), are reported.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "fcbf5ffd-7722-4801-b6f7-5867248d471d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-31T09:34:05] INFO 0 Rocoto validation errors found\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rocoto.validate(\n", + " xml_file=\"fixtures/rocoto/simple-workflow.xml\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "57e975e2-c46e-421b-8b5a-45c4f62afdd0", + "metadata": {}, + "source": [ + "The following Rocoto XML is missing two required components: ``'s `scheduler` attribute and a `` tag.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "9788207b-3c1f-4b60-bd4d-9c8a75666b24", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + " logs/test.log\n", + " \n", + " 1\n", + " 00:00:10\n", + " echo Hello, World!\n", + " greet\n", + " \n", + "\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/rocoto/err-workflow.xml" + ] + }, + { + "cell_type": "markdown", + "id": "f407c4a5-a5a6-4546-b23b-390c9bb52f9b", + "metadata": {}, + "source": [ + "When Rocoto validation errors are found, `validate()` returns `False`. Details are reported regarding the types of errors and number of errors found. For more information on required Rocoto XML components, see the Rocoto Documentation.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "54fcfb54-361d-47ef-9379-4b235fa54316", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-10-31T09:34:05] ERROR 4 Rocoto validation errors found\n", + "[2024-10-31T09:34:05] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_ATTRVALID: Element workflow failed to validate attributes\n", + "[2024-10-31T09:34:05] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_NOELEM: Expecting an element cycledef, got nothing\n", + "[2024-10-31T09:34:05] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_INTERSEQ: Invalid sequence in interleave\n", + "[2024-10-31T09:34:05] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_CONTENTVALID: Element workflow failed to validate content\n", + "[2024-10-31T09:34:05] ERROR Invalid Rocoto XML:\n", + "[2024-10-31T09:34:05] ERROR 1 \n", + "[2024-10-31T09:34:05] ERROR 2 \n", + "[2024-10-31T09:34:05] ERROR 3 logs/test.log\n", + "[2024-10-31T09:34:05] ERROR 4 \n", + "[2024-10-31T09:34:05] ERROR 5 1\n", + "[2024-10-31T09:34:05] ERROR 6 00:00:10\n", + "[2024-10-31T09:34:05] ERROR 7 echo Hello, World!\n", + "[2024-10-31T09:34:05] ERROR 8 greet\n", + "[2024-10-31T09:34:05] ERROR 9 \n", + "[2024-10-31T09:34:05] ERROR 10 \n" + ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rocoto.validate(\n", + " xml_file=Path(\"fixtures/rocoto/err-workflow.xml\")\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:DEV-uwtools] *", + "language": "python", + "name": "conda-env-DEV-uwtools-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/tests/test_rocoto.py b/notebooks/tests/test_rocoto.py new file mode 100644 index 000000000..2fe0e9adc --- /dev/null +++ b/notebooks/tests/test_rocoto.py @@ -0,0 +1,90 @@ +from testbook import testbook + + +def test_building_simple_workflow(): + with open("fixtures/rocoto/simple-workflow.yaml", "r", encoding="utf-8") as f: + simple_yaml = f.read().rstrip() + with open("fixtures/rocoto/err-workflow.yaml", "r", encoding="utf-8") as f: + err_yaml = f.read().rstrip() + with testbook("rocoto.ipynb", execute=True) as tb: + with open("tmp/simple-workflow.xml", "r", encoding="utf-8") as f: + simple_xml = f.read().rstrip() + assert tb.cell_output_text(5) == simple_yaml + valid_out = ( + "INFO 0 UW schema-validation errors found", + "INFO 0 Rocoto validation errors found", + "True", + ) + assert all(x in tb.cell_output_text(7) for x in valid_out) + assert tb.cell_output_text(9) == simple_xml + assert tb.cell_output_text(11) == err_yaml + err_out = ( + "ERROR 3 UW schema-validation errors found", + "ERROR Error at workflow -> attrs:", + "ERROR 'realtime' is a required property", + "ERROR Error at workflow -> tasks -> task_greet:", + "ERROR 'command' is a required property", + "ERROR Error at workflow:", + "ERROR 'log' is a required property", + "YAML validation errors", + ) + assert all(x in tb.cell_output_text(13) for x in err_out) + + +def test_building_workflows(): + with open("fixtures/rocoto/ent-workflow.yaml", "r", encoding="utf-8") as f: + ent_yaml = f.read().rstrip() + with open("fixtures/rocoto/ent-cs-workflow.yaml", "r", encoding="utf-8") as f: + ent_cs_yaml = f.read().rstrip() + with open("fixtures/rocoto/tasks-workflow.yaml", "r", encoding="utf-8") as f: + tasks_yaml = f.read().rstrip() + with open("fixtures/rocoto/tasks-deps-workflow.yaml", "r", encoding="utf-8") as f: + tasks_deps_yaml = f.read().rstrip() + with open("fixtures/rocoto/meta-workflow.yaml", "r", encoding="utf-8") as f: + meta_yaml = f.read().rstrip() + with open("fixtures/rocoto/meta-nested-workflow.yaml", "r", encoding="utf-8") as f: + meta_nested_yaml = f.read().rstrip() + with testbook("rocoto.ipynb", execute=True) as tb: + with open("tmp/ent-cs-workflow.xml", "r", encoding="utf-8") as f: + ent_cs_xml = f.read().rstrip() + with open("tmp/tasks-deps-workflow.xml", "r", encoding="utf-8") as f: + tasks_deps_xml = f.read().rstrip() + with open("tmp/meta-workflow.xml", "r", encoding="utf-8") as f: + meta_xml = f.read().rstrip() + assert tb.cell_output_text(15) == ent_yaml + assert tb.cell_output_text(17) == ent_cs_yaml + valid_out = ( + "INFO 0 UW schema-validation errors found", + "INFO 0 Rocoto validation errors found", + "True", + ) + assert all(x in tb.cell_output_text(19) for x in valid_out) + assert tb.cell_output_text(21) == ent_cs_xml + assert tb.cell_output_text(23) == tasks_yaml + assert tb.cell_output_text(25) == tasks_deps_yaml + assert all(x in tb.cell_output_text(27) for x in valid_out) + assert tb.cell_output_text(29) == tasks_deps_xml + assert tb.cell_output_text(31) == meta_yaml + assert all(x in tb.cell_output_text(33) for x in valid_out) + assert tb.cell_output_text(35) == meta_xml + assert tb.cell_output_text(37) == meta_nested_yaml + + +def test_validate(): + with open("fixtures/rocoto/simple-workflow.xml", "r", encoding="utf-8") as f: + simple_xml = f.read().rstrip() + with open("fixtures/rocoto/err-workflow.xml", "r", encoding="utf-8") as f: + err_xml = f.read().rstrip() + with testbook("rocoto.ipynb", execute=True) as tb: + assert tb.cell_output_text(41) == simple_xml + valid_out = ("INFO 0 Rocoto validation errors found", "True") + assert all(x in tb.cell_output_text(43) for x in valid_out) + assert tb.cell_output_text(45) == err_xml + err_out = ( + "ERROR 4 Rocoto validation errors found", + "Element workflow failed to validate attributes", + "Expecting an element cycledef, got nothing", + "Invalid sequence in interleave", + "Element workflow failed to validate content", + ) + assert all(x in tb.cell_output_text(47) for x in err_out) From e69e3dc46a7707413088fe6de724a03798359ee3 Mon Sep 17 00:00:00 2001 From: Travis Byrne <42480803+Byrnetp@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:40:45 -0700 Subject: [PATCH 15/24] Jupyter Notebook: Configure Experiment Cookbook (#653) * Add notebook, unit tests, Binder link * Apply suggestions from code review Co-authored-by: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> * Add user.PARMdir * Apply suggestions from code review Co-authored-by: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> * Update Binder links * Add validation example using chgres_cube driver * Move header to binder_links.rst * Apply suggestions from review * Apply suggestions from code review Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> * notebook name change and driver validation update --------- Co-authored-by: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> --- docs/index.rst | 2 + docs/sections/user_guide/index.rst | 18 +- docs/shared/binder_links.rst | 13 + notebooks/exp-config-cb.ipynb | 401 ++++++++++++++++++ notebooks/fixtures/exp-config/base-file.yaml | 27 ++ .../fixtures/exp-config/fv3-rap-physics.yaml | 6 + notebooks/fixtures/exp-config/user.yaml | 6 + notebooks/tests/test_exp_config_cb.py | 35 ++ 8 files changed, 492 insertions(+), 16 deletions(-) create mode 100644 docs/shared/binder_links.rst create mode 100644 notebooks/exp-config-cb.ipynb create mode 100644 notebooks/fixtures/exp-config/base-file.yaml create mode 100644 notebooks/fixtures/exp-config/fv3-rap-physics.yaml create mode 100644 notebooks/fixtures/exp-config/user.yaml create mode 100644 notebooks/tests/test_exp_config_cb.py diff --git a/docs/index.rst b/docs/index.rst index c99bac998..5f83e1004 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -236,6 +236,8 @@ mpas | **CLI**: ``uw mpas -h`` | **API**: ``import uwtools.api.mpas`` +.. include:: /shared/binder_links.rst + ------------------ **Disclaimer** diff --git a/docs/sections/user_guide/index.rst b/docs/sections/user_guide/index.rst index 88d4c7d0e..54ae9cd6d 100644 --- a/docs/sections/user_guide/index.rst +++ b/docs/sections/user_guide/index.rst @@ -7,20 +7,6 @@ User Guide installation cli/index api/index - -.. raw:: html - - - -.. toctree:: - :maxdepth: 2 - yaml/index + +.. include:: /shared/binder_links.rst diff --git a/docs/shared/binder_links.rst b/docs/shared/binder_links.rst new file mode 100644 index 000000000..7d0aba236 --- /dev/null +++ b/docs/shared/binder_links.rst @@ -0,0 +1,13 @@ +Jupyter Notebooks +----------------- + +* Jupyter Notebook Tutorials (API) + + * `Config Tool `_ + * `File System Tool `_ + * `Rocoto Tool `_ + * `Template Tool `_ + +* Cookbooks + + * `Configuring an Experiment with UW Tools `_ diff --git a/notebooks/exp-config-cb.ipynb b/notebooks/exp-config-cb.ipynb new file mode 100644 index 000000000..0b8ab9087 --- /dev/null +++ b/notebooks/exp-config-cb.ipynb @@ -0,0 +1,401 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "46a75a26-d2b7-4e44-8c35-01851975dd69", + "metadata": {}, + "source": [ + "# Building and Validating an Experiment Configuration\n", + "\n", + "
    Note: This notebook was tested using uwtools version 2.4.2.
    \n", + "\n", + "This notebook demonstrates how to build up a configuration file for generating FV3 initial conditions (ICs) from a hierarchy of smaller, purpose-specific files; dereferencing Jinja2 expressions in the configuration; and validating the final configuration to check for potential errors. A larger, more complex experimental setup could be built up by applying similar techniques.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "3a872d35-99f1-434c-927e-5c8fee3f0f2d", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from uwtools.api.config import get_yaml_config\n", + "from uwtools.api import chgres_cube\n", + "from uwtools.api.logging import use_uwtools_logger\n", + "\n", + "use_uwtools_logger()" + ] + }, + { + "cell_type": "markdown", + "id": "8834e2e9-8c1e-4791-b110-2f1916f7289e", + "metadata": {}, + "source": [ + "We start with a base file that configures the `chgres_cube` component to generate FV3 ICs for use with the default physics suite, controlled by the `varmap_file` key:\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "8b42bfca-18ae-48b1-a7ad-a1b76b9e24a8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_make_ics:\n", + " chgres_cube:\n", + " execution:\n", + " executable: \"execdir/chgres_cube\"\n", + " namelist:\n", + " update_values:\n", + " config:\n", + " cycle_day: !int \"{{ cycle.strftime('%d') }}\"\n", + " cycle_hour: !int \"{{ cycle.strftime('%H') }}\"\n", + " cycle_mon: !int \"{{ cycle.strftime('%m') }}\"\n", + " convert_atm: true\n", + " convert_nst: true\n", + " convert_sfc: true\n", + " data_dir_input_grid: \"{{ task_make_ics.chgres_cube.rundir }}\"\n", + " external_model: \"GFS\"\n", + " input_type: \"gaussian_nemsio\"\n", + " mosaic_file_target_grid: \"path/to/example_mosaic.halo.nc\"\n", + " tg3_from_soil: false\n", + " tracers:\n", + " - sphum\n", + " - liq_wat\n", + " tracers_input:\n", + " - spfh\n", + " - clwmr\n", + " varmap_file: \"{{ user.PARMdir }}/ufs_utils/varmap_tables/GFSphys_var_map.txt\"\n", + " vcoord_file_target_grid: \"path/to/global_hyblev.165.txt\"\n", + " rundir: '{{ workflow.EXPTDIR }}/make_ics'\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/exp-config/base-file.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "dece324c-ee0e-4c0b-9913-79d63c36ec4e", + "metadata": {}, + "source": [ + "To produce ICs compatible with the FV3_RAP physics suite instead, this partial configuration can be used to update the base:\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e7fbe2c2-90af-446c-b398-621d91c763c9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_make_ics:\n", + " chgres_cube:\n", + " namelist:\n", + " update_values:\n", + " config:\n", + " varmap_file: \"{{ user.PARMdir }}/ufs_utils/varmap_tables/GSDphys_var_map.txt\"\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/exp-config/fv3-rap-physics.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "cfe8bf85-7f11-40d3-af95-f60ddf12318f", + "metadata": {}, + "source": [ + "User- and experiment-specific values can be supplied via a third configuration file:\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d3bbf762-b4fc-49d8-90e4-e7851c9da49a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "user:\n", + " ACCOUNT: zrtrr\n", + " MACHINE: hera\n", + " PARMdir: /path/to/ufs-srweather-app/parm\n", + "workflow: \n", + " EXPTDIR: /path/to/my/output\n" + ] + } + ], + "source": [ + "%%bash\n", + "cat fixtures/exp-config/user.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "a3d1ca1f-1750-4a67-94d4-99d2c3c9db91", + "metadata": {}, + "source": [ + "Structuring the configuration as a hierarchy of increasing specificity provides a better user experience through separation of concerns: Users can see why certain values are changing, and can mix together app-supplied fragments with known-good values into larger experiment configurations.\n", + "\n", + "Here, we start by instantiating a `YAMLConfig` object from the most general base config file, which contains unrendered Jinja2 expressions and is missing certain user- and experiment-specific values:\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4bda78dc-33ee-4a23-82a8-271b40abca7b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_make_ics:\n", + " chgres_cube:\n", + " execution:\n", + " executable: execdir/chgres_cube\n", + " namelist:\n", + " update_values:\n", + " config:\n", + " cycle_day: !int '{{ cycle.strftime(''%d'') }}'\n", + " cycle_hour: !int '{{ cycle.strftime(''%H'') }}'\n", + " cycle_mon: !int '{{ cycle.strftime(''%m'') }}'\n", + " convert_atm: true\n", + " convert_nst: true\n", + " convert_sfc: true\n", + " data_dir_input_grid: '{{ task_make_ics.chgres_cube.rundir }}'\n", + " external_model: GFS\n", + " input_type: gaussian_nemsio\n", + " mosaic_file_target_grid: path/to/example_mosaic.halo.nc\n", + " tg3_from_soil: false\n", + " tracers:\n", + " - sphum\n", + " - liq_wat\n", + " tracers_input:\n", + " - spfh\n", + " - clwmr\n", + " varmap_file: '{{ user.PARMdir }}/ufs_utils/varmap_tables/GFSphys_var_map.txt'\n", + " vcoord_file_target_grid: path/to/global_hyblev.165.txt\n", + " rundir: '{{ workflow.EXPTDIR }}/make_ics'\n" + ] + } + ], + "source": [ + "experiment_config = get_yaml_config('fixtures/exp-config/base-file.yaml')\n", + "print(experiment_config)" + ] + }, + { + "cell_type": "markdown", + "id": "bc6e47f6-ed03-41ef-bacf-b1790b5bf56f", + "metadata": {}, + "source": [ + "Next, we define a list of additional config files, iterate over those, and update the base config with each, in turn. Note that, if the configs share any keys, the values from the update will override and replace existing ones. For example, the original `varmap_file:` path to file `GFSphys_var_map.txt` is updated with a path to file `GSDphys_var_map.txt`:\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e863d5ac-f727-4d91-a4bd-9bf813d35e6c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_make_ics:\n", + " chgres_cube:\n", + " execution:\n", + " executable: execdir/chgres_cube\n", + " namelist:\n", + " update_values:\n", + " config:\n", + " cycle_day: !int '{{ cycle.strftime(''%d'') }}'\n", + " cycle_hour: !int '{{ cycle.strftime(''%H'') }}'\n", + " cycle_mon: !int '{{ cycle.strftime(''%m'') }}'\n", + " convert_atm: true\n", + " convert_nst: true\n", + " convert_sfc: true\n", + " data_dir_input_grid: '{{ task_make_ics.chgres_cube.rundir }}'\n", + " external_model: GFS\n", + " input_type: gaussian_nemsio\n", + " mosaic_file_target_grid: path/to/example_mosaic.halo.nc\n", + " tg3_from_soil: false\n", + " tracers:\n", + " - sphum\n", + " - liq_wat\n", + " tracers_input:\n", + " - spfh\n", + " - clwmr\n", + " varmap_file: '{{ user.PARMdir }}/ufs_utils/varmap_tables/GSDphys_var_map.txt'\n", + " vcoord_file_target_grid: path/to/global_hyblev.165.txt\n", + " rundir: '{{ workflow.EXPTDIR }}/make_ics'\n", + "user:\n", + " ACCOUNT: zrtrr\n", + " MACHINE: hera\n", + " PARMdir: /path/to/ufs-srweather-app/parm\n", + "workflow:\n", + " EXPTDIR: /path/to/my/output\n" + ] + } + ], + "source": [ + "config_files = ['fixtures/exp-config/fv3-rap-physics.yaml',\n", + " 'fixtures/exp-config/user.yaml'\n", + "]\n", + "for config_file in config_files:\n", + " config = get_yaml_config(config_file)\n", + " experiment_config.update_from(config)\n", + "\n", + "print(experiment_config)" + ] + }, + { + "cell_type": "markdown", + "id": "aa43fe3c-6d35-49c2-b2fc-20c178ed30c3", + "metadata": {}, + "source": [ + "Once the hierarchy of configs is merged, we call the `dereference()` method to render Jinja2 expressions into final values. Keys like `varmap_file:` and `rundir:` have their values rendered using references to the `PARMdir` and `EXPTDIR` keys in the `user` and `workflow` sections, respectively. Expressions with cycle-specific references remain, and will be rendered at run time.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0fc07baf-1094-4d8c-a51e-c4e541ae4df6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_make_ics:\n", + " chgres_cube:\n", + " execution:\n", + " executable: execdir/chgres_cube\n", + " namelist:\n", + " update_values:\n", + " config:\n", + " cycle_day: !int '{{ cycle.strftime(''%d'') }}'\n", + " cycle_hour: !int '{{ cycle.strftime(''%H'') }}'\n", + " cycle_mon: !int '{{ cycle.strftime(''%m'') }}'\n", + " convert_atm: true\n", + " convert_nst: true\n", + " convert_sfc: true\n", + " data_dir_input_grid: /path/to/my/output/make_ics\n", + " external_model: GFS\n", + " input_type: gaussian_nemsio\n", + " mosaic_file_target_grid: path/to/example_mosaic.halo.nc\n", + " tg3_from_soil: false\n", + " tracers:\n", + " - sphum\n", + " - liq_wat\n", + " tracers_input:\n", + " - spfh\n", + " - clwmr\n", + " varmap_file: /path/to/ufs-srweather-app/parm/ufs_utils/varmap_tables/GSDphys_var_map.txt\n", + " vcoord_file_target_grid: path/to/global_hyblev.165.txt\n", + " rundir: /path/to/my/output/make_ics\n", + "user:\n", + " ACCOUNT: zrtrr\n", + " MACHINE: hera\n", + " PARMdir: /path/to/ufs-srweather-app/parm\n", + "workflow:\n", + " EXPTDIR: /path/to/my/output\n" + ] + } + ], + "source": [ + "experiment_config.dereference()\n", + "print(experiment_config)" + ] + }, + { + "cell_type": "markdown", + "id": "5103465d-1a64-4b66-b9cb-09910633f8e1", + "metadata": {}, + "source": [ + "To catch potential configuration errors as early as possible, the `uwtools` driver for `chgres_cube` is called to validate the config using a built-in schema. The driver requires a `cycle` parameter with a `datetime` value, and the current time is used here. As the output shows, no schema-validation errors are found\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f9f0c0df-821e-492a-9669-3ac5e43e3151", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-11-19T09:41:26] INFO Validating config against internal schema: chgres-cube\n", + "[2024-11-19T09:41:26] INFO 0 UW schema-validation errors found\n", + "[2024-11-19T09:41:26] INFO Validating config against internal schema: platform\n", + "[2024-11-19T09:41:26] INFO 0 UW schema-validation errors found\n", + "[2024-11-19T09:41:26] INFO 20241119 09Z chgres_cube valid schema: State: Ready\n" + ] + }, + { + "data": { + "text/plain": [ + "Asset(ref=None, ready=. at 0x7e1cf8123420>)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "driver = chgres_cube.ChgresCube(\n", + " config=experiment_config,\n", + " key_path=['task_make_ics'],\n", + " cycle=datetime.now()\n", + ")\n", + "driver.validate()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:DEV-uwtools] *", + "language": "python", + "name": "conda-env-DEV-uwtools-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/fixtures/exp-config/base-file.yaml b/notebooks/fixtures/exp-config/base-file.yaml new file mode 100644 index 000000000..e686a9480 --- /dev/null +++ b/notebooks/fixtures/exp-config/base-file.yaml @@ -0,0 +1,27 @@ +task_make_ics: + chgres_cube: + execution: + executable: "execdir/chgres_cube" + namelist: + update_values: + config: + cycle_day: !int "{{ cycle.strftime('%d') }}" + cycle_hour: !int "{{ cycle.strftime('%H') }}" + cycle_mon: !int "{{ cycle.strftime('%m') }}" + convert_atm: true + convert_nst: true + convert_sfc: true + data_dir_input_grid: "{{ task_make_ics.chgres_cube.rundir }}" + external_model: "GFS" + input_type: "gaussian_nemsio" + mosaic_file_target_grid: "path/to/example_mosaic.halo.nc" + tg3_from_soil: false + tracers: + - sphum + - liq_wat + tracers_input: + - spfh + - clwmr + varmap_file: "{{ user.PARMdir }}/ufs_utils/varmap_tables/GFSphys_var_map.txt" + vcoord_file_target_grid: "path/to/global_hyblev.165.txt" + rundir: '{{ workflow.EXPTDIR }}/make_ics' diff --git a/notebooks/fixtures/exp-config/fv3-rap-physics.yaml b/notebooks/fixtures/exp-config/fv3-rap-physics.yaml new file mode 100644 index 000000000..eff7ed21b --- /dev/null +++ b/notebooks/fixtures/exp-config/fv3-rap-physics.yaml @@ -0,0 +1,6 @@ +task_make_ics: + chgres_cube: + namelist: + update_values: + config: + varmap_file: "{{ user.PARMdir }}/ufs_utils/varmap_tables/GSDphys_var_map.txt" diff --git a/notebooks/fixtures/exp-config/user.yaml b/notebooks/fixtures/exp-config/user.yaml new file mode 100644 index 000000000..c8815fc35 --- /dev/null +++ b/notebooks/fixtures/exp-config/user.yaml @@ -0,0 +1,6 @@ +user: + ACCOUNT: zrtrr + MACHINE: hera + PARMdir: /path/to/ufs-srweather-app/parm +workflow: + EXPTDIR: /path/to/my/output diff --git a/notebooks/tests/test_exp_config_cb.py b/notebooks/tests/test_exp_config_cb.py new file mode 100644 index 000000000..b806948a8 --- /dev/null +++ b/notebooks/tests/test_exp_config_cb.py @@ -0,0 +1,35 @@ +from testbook import testbook +from uwtools.config.formats.yaml import YAMLConfig + + +def test_exp_config(): + with open("fixtures/exp-config/base-file.yaml", "r", encoding="utf-8") as f: + base_cfg = f.read().rstrip() + with open("fixtures/exp-config/fv3-rap-physics.yaml", "r", encoding="utf-8") as f: + fv3_rap_phys = f.read().rstrip() + with open("fixtures/exp-config/user.yaml", "r", encoding="utf-8") as f: + user_cfg = f.read().rstrip() + with testbook("exp-config-cb.ipynb", execute=True) as tb: + assert tb.cell_output_text(1) == "" + assert tb.cell_output_text(3) == base_cfg + assert tb.cell_output_text(5) == fv3_rap_phys + assert tb.cell_output_text(7) == user_cfg + assert tb.cell_output_text(9) == str(YAMLConfig("fixtures/exp-config/base-file.yaml")) + updated_cfg = ( + "cycle_day: !int '{{ cycle.strftime(''%d'') }}'", + "varmap_file: '{{ user.PARMdir }}/ufs_utils/varmap_tables/GSDphys_var_map.txt'", + "PARMdir: /path/to/ufs-srweather-app/parm", + ) + assert all(x in tb.cell_output_text(11) for x in updated_cfg) + deref_cfg = ( + "data_dir_input_grid: /path/to/my/output/make_ics", + "rundir: /path/to/my/output/make_ics", + ) + assert all(x in tb.cell_output_text(13) for x in deref_cfg) + validate_out = ( + "INFO Validating config against internal schema: chgres-cube", + "INFO 0 UW schema-validation errors found", + "INFO Validating config against internal schema: platform", + "chgres_cube valid schema: State: Ready", + ) + assert all(x in tb.cell_output_text(15) for x in validate_out) From 127b18a2cb538559e2e8696a7ff6e61103c98471 Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:01:23 -0700 Subject: [PATCH 16/24] Update notebooks branch from main (#655) --- .github/CODEOWNERS | 2 +- .../scripts/{link-check.sh => make-docs.sh} | 2 +- .github/workflows/release.yaml | 4 +- .github/workflows/test.yaml | 4 +- docs/Makefile | 8 +- docs/conf.py | 5 +- docs/deps | 18 ++ docs/environment.yml | 7 +- docs/index.rst | 11 +- docs/install-deps | 2 +- .../contributor_guide/documentation.rst | 6 +- docs/sections/user_guide/api/index.rst | 3 + docs/sections/user_guide/api/orog.rst | 6 + docs/sections/user_guide/cli/Makefile | 4 +- docs/sections/user_guide/cli/Makefile.outputs | 2 +- docs/sections/user_guide/cli/drivers/Makefile | 9 +- .../user_guide/cli/drivers/cdeps/help.rst | 1 + .../drivers/{cdeps.rst => cdeps/index.rst} | 24 +- .../user_guide/cli/drivers/cdeps/run-help.out | 4 +- .../cli/drivers/cdeps/schema-options.rst | 1 + .../cli/drivers/cdeps/show-schema.cmd | 2 +- .../cli/drivers/cdeps/show-schema.out | 21 +- .../cli/drivers/chgres_cube/help.out | 2 + .../cli/drivers/chgres_cube/help.rst | 1 + .../index.rst} | 24 +- .../cli/drivers/chgres_cube/run-help.out | 13 +- .../drivers/chgres_cube/schema-options.rst | 1 + .../cli/drivers/chgres_cube/show-schema.cmd | 2 +- .../cli/drivers/chgres_cube/show-schema.out | 21 +- .../user_guide/cli/drivers/esg_grid/help.out | 2 + .../user_guide/cli/drivers/esg_grid/help.rst | 1 + .../{esg_grid.rst => esg_grid/index.rst} | 24 +- .../cli/drivers/esg_grid/run-help.out | 5 +- .../cli/drivers/esg_grid/schema-options.rst | 1 + .../cli/drivers/esg_grid/show-schema.cmd | 2 +- .../cli/drivers/esg_grid/show-schema.out | 21 +- .../cli/drivers/filter_topo/help.out | 4 + .../cli/drivers/filter_topo/help.rst | 1 + .../index.rst} | 24 +- .../cli/drivers/filter_topo/run-help.out | 5 +- .../drivers/filter_topo/schema-options.rst | 1 + .../cli/drivers/filter_topo/show-schema.cmd | 2 +- .../cli/drivers/filter_topo/show-schema.out | 21 +- .../user_guide/cli/drivers/fv3/help.out | 2 + .../user_guide/cli/drivers/fv3/help.rst | 1 + .../cli/drivers/{fv3.rst => fv3/index.rst} | 24 +- .../user_guide/cli/drivers/fv3/run-help.out | 5 +- .../cli/drivers/fv3/schema-options.rst | 1 + .../cli/drivers/fv3/show-schema.cmd | 2 +- .../cli/drivers/fv3/show-schema.out | 19 +- .../cli/drivers/global_equiv_resol/help.out | 2 + .../cli/drivers/global_equiv_resol/help.rst | 1 + .../index.rst} | 24 +- .../drivers/global_equiv_resol/run-help.out | 6 +- .../global_equiv_resol/schema-options.rst | 1 + .../global_equiv_resol/show-schema.cmd | 2 +- .../global_equiv_resol/show-schema.out | 21 +- .../sections/user_guide/cli/drivers/index.rst | 37 +-- .../user_guide/cli/drivers/ioda/help.out | 2 + .../user_guide/cli/drivers/ioda/help.rst | 1 + .../cli/drivers/{ioda.rst => ioda/index.rst} | 24 +- .../user_guide/cli/drivers/ioda/run-help.out | 5 +- .../cli/drivers/ioda/schema-options.rst | 1 + .../cli/drivers/ioda/show-schema.cmd | 2 +- .../cli/drivers/ioda/show-schema.out | 21 +- .../user_guide/cli/drivers/jedi/help.out | 2 + .../user_guide/cli/drivers/jedi/help.rst | 1 + .../cli/drivers/{jedi.rst => jedi/index.rst} | 24 +- .../user_guide/cli/drivers/jedi/run-help.out | 5 +- .../cli/drivers/jedi/schema-options.rst | 1 + .../cli/drivers/jedi/show-schema.cmd | 2 +- .../cli/drivers/jedi/show-schema.out | 21 +- .../cli/drivers/make_hgrid/help.out | 2 + .../cli/drivers/make_hgrid/help.rst | 1 + .../{make_hgrid.rst => make_hgrid/index.rst} | 28 +- .../cli/drivers/make_hgrid/run-help.out | 5 +- .../cli/drivers/make_hgrid/schema-options.rst | 1 + .../cli/drivers/make_hgrid/show-schema.cmd | 2 +- .../cli/drivers/make_hgrid/show-schema.out | 21 +- .../cli/drivers/make_solo_mosaic/help.out | 2 + .../cli/drivers/make_solo_mosaic/help.rst | 1 + .../index.rst} | 28 +- .../cli/drivers/make_solo_mosaic/run-help.out | 5 +- .../make_solo_mosaic/schema-options.rst | 1 + .../drivers/make_solo_mosaic/show-schema.cmd | 2 +- .../drivers/make_solo_mosaic/show-schema.out | 21 +- .../user_guide/cli/drivers/mpas/help.out | 2 + .../user_guide/cli/drivers/mpas/help.rst | 1 + .../cli/drivers/{mpas.rst => mpas/index.rst} | 24 +- .../user_guide/cli/drivers/mpas/run-help.out | 5 +- .../cli/drivers/mpas/schema-options.rst | 1 + .../cli/drivers/mpas/show-schema.cmd | 2 +- .../cli/drivers/mpas/show-schema.out | 21 +- .../user_guide/cli/drivers/mpas_init/help.out | 2 + .../user_guide/cli/drivers/mpas_init/help.rst | 1 + .../{mpas_init.rst => mpas_init/index.rst} | 24 +- .../cli/drivers/mpas_init/run-help.out | 5 +- .../cli/drivers/mpas_init/schema-options.rst | 1 + .../cli/drivers/mpas_init/show-schema.cmd | 2 +- .../cli/drivers/mpas_init/show-schema.out | 21 +- .../user_guide/cli/drivers/orog/Makefile | 1 + .../user_guide/cli/drivers/orog/help.cmd | 1 + .../user_guide/cli/drivers/orog/help.out | 30 ++ .../user_guide/cli/drivers/orog/help.rst | 1 + .../user_guide/cli/drivers/orog/index.rst | 44 +++ .../user_guide/cli/drivers/orog/run-help.cmd | 1 + .../user_guide/cli/drivers/orog/run-help.out | 28 ++ .../cli/drivers/orog/schema-options.rst | 1 + .../cli/drivers/orog/show-schema.cmd | 2 + .../cli/drivers/orog/show-schema.out | 20 ++ .../user_guide/cli/drivers/orog_gsl/help.out | 4 + .../user_guide/cli/drivers/orog_gsl/help.rst | 1 + .../{orog_gsl.rst => orog_gsl/index.rst} | 24 +- .../cli/drivers/orog_gsl/run-help.out | 5 +- .../cli/drivers/orog_gsl/schema-options.rst | 1 + .../cli/drivers/orog_gsl/show-schema.cmd | 2 +- .../cli/drivers/orog_gsl/show-schema.out | 21 +- .../user_guide/cli/drivers/schism/Makefile | 1 + .../user_guide/cli/drivers/schism/help.cmd | 1 + .../user_guide/cli/drivers/schism/help.out | 20 ++ .../user_guide/cli/drivers/schism/help.rst | 1 + .../user_guide/cli/drivers/schism/index.rst | 34 +++ .../cli/drivers/schism/run-help.cmd | 1 + .../cli/drivers/schism/run-help.out | 20 ++ .../cli/drivers/schism/schema-options.rst | 1 + .../cli/drivers/schism/show-schema.cmd | 2 + .../cli/drivers/schism/show-schema.out | 20 ++ .../cli/drivers/sfc_climo_gen/help.out | 2 + .../cli/drivers/sfc_climo_gen/help.rst | 1 + .../index.rst} | 24 +- .../cli/drivers/sfc_climo_gen/run-help.out | 5 +- .../drivers/sfc_climo_gen/schema-options.rst | 1 + .../cli/drivers/sfc_climo_gen/show-schema.cmd | 2 +- .../cli/drivers/sfc_climo_gen/show-schema.out | 21 +- .../user_guide/cli/drivers/shared/help.rst | 13 + .../cli/drivers}/shared/idempotent.rst | 0 .../cli/drivers/shared/schema-options.rst | 9 + .../user_guide/cli/drivers/shave/help.out | 4 + .../user_guide/cli/drivers/shave/help.rst | 1 + .../drivers/{shave.rst => shave/index.rst} | 24 +- .../user_guide/cli/drivers/shave/run-help.out | 4 +- .../cli/drivers/shave/schema-options.rst | 1 + .../cli/drivers/shave/show-schema.cmd | 2 +- .../cli/drivers/shave/show-schema.out | 21 +- .../user_guide/cli/drivers/ungrib/help.out | 2 + .../user_guide/cli/drivers/ungrib/help.rst | 1 + .../drivers/{ungrib.rst => ungrib/index.rst} | 24 +- .../cli/drivers/ungrib/run-help.out | 5 +- .../cli/drivers/ungrib/schema-options.rst | 1 + .../cli/drivers/ungrib/show-schema.cmd | 2 +- .../cli/drivers/ungrib/show-schema.out | 21 +- .../user_guide/cli/drivers/upp/help.out | 4 + .../user_guide/cli/drivers/upp/help.rst | 1 + .../cli/drivers/{upp.rst => upp/index.rst} | 24 +- .../user_guide/cli/drivers/upp/run-help.out | 6 +- .../cli/drivers/upp/schema-options.rst | 1 + .../cli/drivers/upp/show-schema.cmd | 2 +- .../cli/drivers/upp/show-schema.out | 21 +- .../user_guide/cli/drivers/ww3/Makefile | 1 + .../user_guide/cli/drivers/ww3/help.cmd | 1 + .../user_guide/cli/drivers/ww3/help.out | 22 ++ .../user_guide/cli/drivers/ww3/help.rst | 1 + .../user_guide/cli/drivers/ww3/index.rst | 34 +++ .../user_guide/cli/drivers/ww3/run-help.cmd | 1 + .../user_guide/cli/drivers/ww3/run-help.out | 22 ++ .../cli/drivers/ww3/schema-options.rst | 1 + .../cli/drivers/ww3/show-schema.cmd | 2 + .../cli/drivers/ww3/show-schema.out | 20 ++ .../config/compare-bad-extension-fix.out | 12 +- .../cli/tools/config/compare-diff.out | 12 +- .../cli/tools/config/compare-match.out | 5 +- .../cli/tools/config/compare-verbose.out | 14 +- .../cli/tools/config/validate-fail.out | 6 +- .../cli/tools/config/validate-pass-stdin.out | 2 +- .../cli/tools/config/validate-pass.out | 2 +- .../cli/tools/config/validate-verbose.out | 41 +-- .../sections/user_guide/cli/tools/execute.rst | 2 +- .../cli/tools/execute/alt-schema.out | 14 +- .../user_guide/cli/tools/execute/execute.out | 14 +- .../tools/fs/copy-exec-no-target-dir-err.out | 6 +- .../cli/tools/fs/copy-exec-timedep.out | 20 +- .../user_guide/cli/tools/fs/copy-exec.out | 30 +- .../tools/fs/link-exec-no-target-dir-err.out | 6 +- .../cli/tools/fs/link-exec-timedep.out | 20 +- .../user_guide/cli/tools/fs/link-exec.out | 30 +- .../fs/makedirs-exec-no-target-dir-err.out | 6 +- .../cli/tools/fs/makedirs-exec-timedep.out | 30 +- .../user_guide/cli/tools/fs/makedirs-exec.out | 30 +- .../cli/tools/rocoto/realize-exec-file.out | 4 +- .../rocoto/realize-exec-stdin-stdout.out | 4 +- .../rocoto/realize-exec-stdout-verbose.out | 40 +-- .../cli/tools/rocoto/realize-exec-stdout.out | 4 +- .../cli/tools/rocoto/validate-bad-file.out | 48 ++-- .../cli/tools/rocoto/validate-good-file.out | 2 +- .../cli/tools/rocoto/validate-good-stdin.out | 2 +- .../yaml/components/filter_topo.rst | 8 + .../user_guide/yaml/components/index.rst | 1 + .../user_guide/yaml/components/orog.rst | 57 ++++ .../user_guide/yaml/components/schism.rst | 2 +- .../user_guide/yaml/components/shave.rst | 6 +- docs/sections/user_guide/yaml/rocoto.rst | 32 ++- docs/sections/user_guide/yaml/tags.rst | 20 +- docs/shared/chgres_cube.yaml | 3 +- docs/shared/filter_topo.yaml | 2 + docs/shared/mpas.yaml | 2 +- docs/shared/orog.yaml | 15 + docs/shared/schism.yaml | 2 +- docs/shared/shave.yaml | 3 +- docs/shared/upp.yaml | 3 +- docs/shared/ww3.yaml | 2 +- recipe/meta.json | 16 +- recipe/meta.yaml | 12 +- src/uwtools/api/config.py | 1 + src/uwtools/api/execute.py | 2 +- src/uwtools/api/orog.py | 28 ++ src/uwtools/api/rocoto.py | 2 +- src/uwtools/cli.py | 10 +- src/uwtools/config/formats/base.py | 72 +++-- src/uwtools/config/formats/fieldtable.py | 6 + src/uwtools/config/formats/ini.py | 6 + src/uwtools/config/formats/nml.py | 8 + src/uwtools/config/formats/sh.py | 6 + src/uwtools/config/formats/yaml.py | 6 + src/uwtools/config/jinja2.py | 7 +- src/uwtools/config/support.py | 13 +- src/uwtools/config/tools.py | 5 +- src/uwtools/config/validator.py | 43 +-- src/uwtools/drivers/chgres_cube.py | 25 +- src/uwtools/drivers/driver.py | 105 ++++--- src/uwtools/drivers/esg_grid.py | 2 +- src/uwtools/drivers/filter_topo.py | 18 +- src/uwtools/drivers/fv3.py | 2 +- src/uwtools/drivers/mpas.py | 2 +- src/uwtools/drivers/mpas_init.py | 2 +- src/uwtools/drivers/orog.py | 129 +++++++++ src/uwtools/drivers/orog_gsl.py | 31 +- src/uwtools/drivers/sfc_climo_gen.py | 2 +- src/uwtools/drivers/shave.py | 43 ++- src/uwtools/drivers/upp.py | 50 +++- src/uwtools/exceptions.py | 6 + src/uwtools/fs.py | 12 +- .../jsonschema/chgres-cube.jsonschema | 40 ++- ...onschema => execution-parallel.jsonschema} | 0 .../jsonschema/filter-topo.jsonschema | 10 +- .../resources/jsonschema/fv3.jsonschema | 2 +- .../resources/jsonschema/jedi.jsonschema | 2 +- .../resources/jsonschema/mpas-init.jsonschema | 2 +- .../resources/jsonschema/mpas.jsonschema | 2 +- .../resources/jsonschema/orog.jsonschema | 73 +++++ .../resources/jsonschema/rocoto.jsonschema | 56 ++-- .../jsonschema/sfc-climo-gen.jsonschema | 2 +- .../resources/jsonschema/shave.jsonschema | 12 +- .../resources/jsonschema/ungrib.jsonschema | 2 +- .../resources/jsonschema/upp.jsonschema | 6 +- src/uwtools/rocoto.py | 27 +- src/uwtools/strings.py | 1 + src/uwtools/tests/api/test_config.py | 7 +- src/uwtools/tests/api/test_drivers.py | 3 + src/uwtools/tests/api/test_execute.py | 4 +- src/uwtools/tests/config/formats/test_base.py | 92 ++++-- .../tests/config/formats/test_fieldtable.py | 42 +++ src/uwtools/tests/config/formats/test_ini.py | 39 ++- src/uwtools/tests/config/formats/test_nml.py | 42 ++- src/uwtools/tests/config/formats/test_sh.py | 37 ++- src/uwtools/tests/config/formats/test_yaml.py | 42 ++- src/uwtools/tests/config/test_jinja2.py | 22 +- src/uwtools/tests/config/test_support.py | 25 +- src/uwtools/tests/config/test_tools.py | 65 +++-- src/uwtools/tests/config/test_validator.py | 38 ++- src/uwtools/tests/drivers/test_cdeps.py | 17 +- src/uwtools/tests/drivers/test_chgres_cube.py | 36 ++- src/uwtools/tests/drivers/test_driver.py | 226 +++++++++------ src/uwtools/tests/drivers/test_esg_grid.py | 18 +- src/uwtools/tests/drivers/test_filter_topo.py | 33 ++- src/uwtools/tests/drivers/test_fv3.py | 18 +- .../tests/drivers/test_global_equiv_resol.py | 18 +- src/uwtools/tests/drivers/test_ioda.py | 22 +- src/uwtools/tests/drivers/test_jedi.py | 26 +- src/uwtools/tests/drivers/test_make_hgrid.py | 18 +- .../tests/drivers/test_make_solo_mosaic.py | 22 +- src/uwtools/tests/drivers/test_mpas.py | 36 ++- src/uwtools/tests/drivers/test_mpas_init.py | 18 +- src/uwtools/tests/drivers/test_orog.py | 178 ++++++++++++ src/uwtools/tests/drivers/test_orog_gsl.py | 40 ++- src/uwtools/tests/drivers/test_schism.py | 8 +- .../tests/drivers/test_sfc_climo_gen.py | 18 +- src/uwtools/tests/drivers/test_shave.py | 47 +++- src/uwtools/tests/drivers/test_support.py | 9 +- src/uwtools/tests/drivers/test_ungrib.py | 26 +- src/uwtools/tests/drivers/test_upp.py | 50 +++- src/uwtools/tests/drivers/test_ww3.py | 8 +- .../tests/fixtures/hello_workflow.yaml | 6 +- src/uwtools/tests/fixtures/include_files.ini | 2 +- src/uwtools/tests/fixtures/include_files.nml | 2 +- src/uwtools/tests/fixtures/include_files.sh | 2 +- src/uwtools/tests/fixtures/include_files.yaml | 6 +- .../fixtures/include_files_with_sect.nml | 2 +- src/uwtools/tests/fixtures/testdriver.py | 8 +- src/uwtools/tests/test_cli.py | 16 +- src/uwtools/tests/test_fs.py | 6 +- src/uwtools/tests/test_rocoto.py | 155 +++++++--- src/uwtools/tests/test_schemas.py | 265 ++++++++++++------ src/uwtools/tests/utils/test_api.py | 3 +- src/uwtools/tests/utils/test_file.py | 2 +- src/uwtools/utils/api.py | 9 + 305 files changed, 3272 insertions(+), 1494 deletions(-) rename .github/scripts/{link-check.sh => make-docs.sh} (86%) create mode 100755 docs/deps create mode 100644 docs/sections/user_guide/api/orog.rst mode change 120000 => 100644 docs/sections/user_guide/cli/drivers/Makefile create mode 120000 docs/sections/user_guide/cli/drivers/cdeps/help.rst rename docs/sections/user_guide/cli/drivers/{cdeps.rst => cdeps/index.rst} (64%) create mode 120000 docs/sections/user_guide/cli/drivers/cdeps/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/chgres_cube/help.rst rename docs/sections/user_guide/cli/drivers/{chgres_cube.rst => chgres_cube/index.rst} (76%) create mode 120000 docs/sections/user_guide/cli/drivers/chgres_cube/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/esg_grid/help.rst rename docs/sections/user_guide/cli/drivers/{esg_grid.rst => esg_grid/index.rst} (74%) create mode 120000 docs/sections/user_guide/cli/drivers/esg_grid/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/filter_topo/help.rst rename docs/sections/user_guide/cli/drivers/{filter_topo.rst => filter_topo/index.rst} (69%) create mode 120000 docs/sections/user_guide/cli/drivers/filter_topo/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/fv3/help.rst rename docs/sections/user_guide/cli/drivers/{fv3.rst => fv3/index.rst} (75%) create mode 120000 docs/sections/user_guide/cli/drivers/fv3/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/global_equiv_resol/help.rst rename docs/sections/user_guide/cli/drivers/{global_equiv_resol.rst => global_equiv_resol/index.rst} (69%) create mode 120000 docs/sections/user_guide/cli/drivers/global_equiv_resol/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/ioda/help.rst rename docs/sections/user_guide/cli/drivers/{ioda.rst => ioda/index.rst} (69%) create mode 120000 docs/sections/user_guide/cli/drivers/ioda/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/jedi/help.rst rename docs/sections/user_guide/cli/drivers/{jedi.rst => jedi/index.rst} (68%) create mode 120000 docs/sections/user_guide/cli/drivers/jedi/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/make_hgrid/help.rst rename docs/sections/user_guide/cli/drivers/{make_hgrid.rst => make_hgrid/index.rst} (67%) create mode 120000 docs/sections/user_guide/cli/drivers/make_hgrid/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.rst rename docs/sections/user_guide/cli/drivers/{make_solo_mosaic.rst => make_solo_mosaic/index.rst} (67%) create mode 120000 docs/sections/user_guide/cli/drivers/make_solo_mosaic/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/mpas/help.rst rename docs/sections/user_guide/cli/drivers/{mpas.rst => mpas/index.rst} (77%) create mode 120000 docs/sections/user_guide/cli/drivers/mpas/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/mpas_init/help.rst rename docs/sections/user_guide/cli/drivers/{mpas_init.rst => mpas_init/index.rst} (77%) create mode 120000 docs/sections/user_guide/cli/drivers/mpas_init/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/orog/Makefile create mode 100644 docs/sections/user_guide/cli/drivers/orog/help.cmd create mode 100644 docs/sections/user_guide/cli/drivers/orog/help.out create mode 120000 docs/sections/user_guide/cli/drivers/orog/help.rst create mode 100644 docs/sections/user_guide/cli/drivers/orog/index.rst create mode 100644 docs/sections/user_guide/cli/drivers/orog/run-help.cmd create mode 100644 docs/sections/user_guide/cli/drivers/orog/run-help.out create mode 120000 docs/sections/user_guide/cli/drivers/orog/schema-options.rst create mode 100644 docs/sections/user_guide/cli/drivers/orog/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/orog/show-schema.out create mode 120000 docs/sections/user_guide/cli/drivers/orog_gsl/help.rst rename docs/sections/user_guide/cli/drivers/{orog_gsl.rst => orog_gsl/index.rst} (69%) create mode 120000 docs/sections/user_guide/cli/drivers/orog_gsl/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/schism/Makefile create mode 100644 docs/sections/user_guide/cli/drivers/schism/help.cmd create mode 100644 docs/sections/user_guide/cli/drivers/schism/help.out create mode 120000 docs/sections/user_guide/cli/drivers/schism/help.rst create mode 100644 docs/sections/user_guide/cli/drivers/schism/index.rst create mode 100644 docs/sections/user_guide/cli/drivers/schism/run-help.cmd create mode 100644 docs/sections/user_guide/cli/drivers/schism/run-help.out create mode 120000 docs/sections/user_guide/cli/drivers/schism/schema-options.rst create mode 100644 docs/sections/user_guide/cli/drivers/schism/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/schism/show-schema.out create mode 120000 docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.rst rename docs/sections/user_guide/cli/drivers/{sfc_climo_gen.rst => sfc_climo_gen/index.rst} (74%) create mode 120000 docs/sections/user_guide/cli/drivers/sfc_climo_gen/schema-options.rst create mode 100644 docs/sections/user_guide/cli/drivers/shared/help.rst rename docs/{ => sections/user_guide/cli/drivers}/shared/idempotent.rst (100%) create mode 100644 docs/sections/user_guide/cli/drivers/shared/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/shave/help.rst rename docs/sections/user_guide/cli/drivers/{shave.rst => shave/index.rst} (69%) create mode 120000 docs/sections/user_guide/cli/drivers/shave/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/ungrib/help.rst rename docs/sections/user_guide/cli/drivers/{ungrib.rst => ungrib/index.rst} (75%) create mode 120000 docs/sections/user_guide/cli/drivers/ungrib/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/upp/help.rst rename docs/sections/user_guide/cli/drivers/{upp.rst => upp/index.rst} (76%) create mode 120000 docs/sections/user_guide/cli/drivers/upp/schema-options.rst create mode 120000 docs/sections/user_guide/cli/drivers/ww3/Makefile create mode 100644 docs/sections/user_guide/cli/drivers/ww3/help.cmd create mode 100644 docs/sections/user_guide/cli/drivers/ww3/help.out create mode 120000 docs/sections/user_guide/cli/drivers/ww3/help.rst create mode 100644 docs/sections/user_guide/cli/drivers/ww3/index.rst create mode 100644 docs/sections/user_guide/cli/drivers/ww3/run-help.cmd create mode 100644 docs/sections/user_guide/cli/drivers/ww3/run-help.out create mode 120000 docs/sections/user_guide/cli/drivers/ww3/schema-options.rst create mode 100644 docs/sections/user_guide/cli/drivers/ww3/show-schema.cmd create mode 100644 docs/sections/user_guide/cli/drivers/ww3/show-schema.out create mode 100644 docs/sections/user_guide/yaml/components/orog.rst create mode 100644 docs/shared/orog.yaml create mode 100644 src/uwtools/api/orog.py create mode 100644 src/uwtools/drivers/orog.py rename src/uwtools/resources/jsonschema/{execution.jsonschema => execution-parallel.jsonschema} (100%) create mode 100644 src/uwtools/resources/jsonschema/orog.jsonschema create mode 100644 src/uwtools/tests/drivers/test_orog.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1121ad809..30ce63d6a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ # See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners -* @NaureenBharwaniNOAA @christinaholtNOAA @elcarpenterNOAA @fgabelmannjr @maddenp-noaa @weirae +* @NaureenBharwaniNOAA @christinaholtNOAA @elcarpenterNOAA @fgabelmannjr @maddenp-noaa @weirae @Byrnetp diff --git a/.github/scripts/link-check.sh b/.github/scripts/make-docs.sh similarity index 86% rename from .github/scripts/link-check.sh rename to .github/scripts/make-docs.sh index c1c89f4fa..1f187e543 100755 --- a/.github/scripts/link-check.sh +++ b/.github/scripts/make-docs.sh @@ -3,4 +3,4 @@ source $(dirname ${BASH_SOURCE[0]})/common.sh ci_conda_activate cd docs source install-deps -make linkcheck +make docs diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b1e01ed1b..775e6a02e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -13,8 +13,8 @@ jobs: - uses: actions/checkout@v3 - name: Install conda run: .github/scripts/install-conda.sh - - name: Documentation Link Check - run: .github/scripts/link-check.sh + - name: Make Docs + run: .github/scripts/make-docs.sh - name: Format Check run: .github/scripts/format-check.sh - name: Make Package diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 97eeff098..7fcf90532 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -17,8 +17,8 @@ jobs: - uses: actions/checkout@v3 - name: Install conda run: .github/scripts/install-conda.sh - - name: Documentation Link Check - run: .github/scripts/link-check.sh + - name: Make Docs + run: .github/scripts/make-docs.sh - name: Format Check run: .github/scripts/format-check.sh - name: Test diff --git a/docs/Makefile b/docs/Makefile index 72693a881..a05140390 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,10 +1,9 @@ BUILDDIR = build -LINKCHECKDIR = $(BUILDDIR)/linkcheck SOURCEDIR = . SPHINXBUILD = sphinx-build SPHINXOPTS = -a -n -W --keep-going -.PHONY: help clean docs examples linkcheck +.PHONY: help clean docs examples help: $(SPHINXBUILD) -M help $(SOURCEDIR) $(BUILDDIR) $(SPHINXOPTS) @@ -13,15 +12,12 @@ clean: $(RM) -rv $(BUILDDIR) docs: - $(MAKE) linkcheck $(MAKE) html + linkchecker --check-extern --no-warnings build/html/index.html examples: COLUMNS=80 $(MAKE) -C sections -linkcheck: - $(SPHINXBUILD) -b linkcheck $(SPHINXOPTS) -c $(CURDIR) $(CURDIR) build/linkcheck - %: mkdir -pv $(BUILDDIR) $(SPHINXBUILD) -M $@ $(SOURCEDIR) $(BUILDDIR) $(SPHINXOPTS) -w $(BUILDDIR)/warnings.log diff --git a/docs/conf.py b/docs/conf.py index 0087720b5..8e0e67f94 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,13 +14,13 @@ autodoc_mock_imports = ["f90nml", "iotaa", "jsonschema", "lxml", "referencing"] autodoc_typehints = "description" copyright = str(dt.datetime.now().year) +exclude_patterns = ["**/shared/*.rst"] extensions = ["sphinx.ext.autodoc", "sphinx.ext.extlinks", "sphinx.ext.intersphinx"] extlinks_detect_hardcoded_links = True html_logo = os.path.join("static", "ufs.png") html_static_path = ["static"] html_theme = "sphinx_rtd_theme" intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} -linkcheck_ignore = [r"https://github.com/.*#.*"] nitpick_ignore_regex = [("py:class", r"^uwtools\..*"), ("py:class", "f90nml.Namelist")] numfig = True numfig_format = {"figure": "Figure %s"} @@ -53,9 +53,11 @@ "noaa": ("https://www.noaa.gov/%s", "%s"), "pylint": ("https://pylint.readthedocs.io/en/stable/%s", "%s"), "pytest": ("https://docs.pytest.org/en/7.4.x/%s", "%s"), + "python": ("https://docs.python.org/3/library/%s", "%s"), "rocoto": ("https://christopherwharrop.github.io/rocoto/%s", "%s"), "rst": ("https://www.sphinx-doc.org/en/master/usage/restructuredtext/%s", "%s"), "rtd": ("https://readthedocs.org/projects/uwtools/%s", "%s"), + "schism": ("https://schism-dev.github.io/schism/master/%s", "%s"), "sfc-climo-gen": ("https://ufs-community.github.io/UFS_UTILS/sfc_climo_gen/%s", "%s"), "shell-redirection": ("https://www.gnu.org/software/bash/manual/html_node/Redirections.html%s", "%s"), "ufs": ("https://ufs.epic.noaa.gov/%s", "%s"), @@ -63,6 +65,7 @@ "ufs-weather-model": ("https://github.com/ufs-community/ufs-weather-model/%s", "%s"), "uwtools": ("https://github.com/ufs-community/uwtools/%s", "%s"), "weather-model-io": ("https://ufs-weather-model.readthedocs.io/en/latest/InputsOutputs.html#%s", "%s"), + "ww3": ("https://polar.ncep.noaa.gov/waves/wavewatch/%s", "%s"), } def setup(app): diff --git a/docs/deps b/docs/deps new file mode 100755 index 000000000..a16afe9c8 --- /dev/null +++ b/docs/deps @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +from pathlib import Path +import re +import yaml + +path = Path(__file__).parent / "environment.yml" +with open(path, "r", encoding="utf-8") as f: + deps = yaml.safe_load(f)["dependencies"] +specs = [] +for dep in deps: + m = re.match(r"([^ ]+) *(.*)", dep) + assert m + pkg, ver = m.groups() + if re.match(r"^\d", ver): + ver = f"={ver}" + specs.append(f"'{pkg}{ver}'") +print(" ".join(sorted(specs))) diff --git a/docs/environment.yml b/docs/environment.yml index e71700ed5..86e4aa822 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -2,7 +2,8 @@ name: readthedocs channels: - conda-forge dependencies: - - python=3.12 - - sphinx_rtd_theme=2.0.* - - sphinxcontrib-bibtex=2.6.* + - linkchecker 10.4.* + - python >=3.9,<3.13 # keep in sync with meta.yaml run req + - sphinx_rtd_theme 3.0.* + - sphinxcontrib-bibtex 2.6.* - tree diff --git a/docs/index.rst b/docs/index.rst index 5f83e1004..9dffdd951 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -77,7 +77,6 @@ File/Directory Provisioning | **CLI**: ``uw fs -h`` | **API**: ``import uwtools.api.fs`` - This tool helps users define the source and destination of files to be copied or linked, or directories to be created, in the same UW YAML language used by UW drivers. | :any:`CLI documentation with examples` @@ -86,7 +85,7 @@ There is a video demonstration of the use of the ``uw fs`` tool (formerly ``uw f .. raw:: html - + Rocoto Configurability ^^^^^^^^^^^^^^^^^^^^^^ @@ -162,6 +161,12 @@ make_solo_mosaic | **CLI**: ``uw make_solo_mosaic -h`` | **API**: ``import uwtools.api.make_solo_mosaic`` +orog +"""" + +| **CLI**: ``uw orog -h`` +| **API**: ``import uwtools.api.orog`` + orog_gsl """""""" @@ -236,8 +241,6 @@ mpas | **CLI**: ``uw mpas -h`` | **API**: ``import uwtools.api.mpas`` -.. include:: /shared/binder_links.rst - ------------------ **Disclaimer** diff --git a/docs/install-deps b/docs/install-deps index e75232d68..e9239185a 100644 --- a/docs/install-deps +++ b/docs/install-deps @@ -1 +1 @@ -conda install -q -y $(python -c "import yaml; f=open('$(dirname ${BASH_SOURCE[0]})/environment.yml'); print(' '.join(yaml.safe_load(f)['dependencies']))") +conda install -q -y $($(dirname ${BASH_SOURCE[0]})/deps) diff --git a/docs/sections/contributor_guide/documentation.rst b/docs/sections/contributor_guide/documentation.rst index ee82459cf..8641910a1 100644 --- a/docs/sections/contributor_guide/documentation.rst +++ b/docs/sections/contributor_guide/documentation.rst @@ -44,7 +44,7 @@ Please follow these guidelines when contributing to the documentation: * In [[sub]sub]section titles, capitalize all "principal" words. In practice this usually means all words but articles (a, an, the), logicals (and, etc.), and prepositions (for, of, etc.). Always fully capitalize acronyms (e.g., YAML). * Never capitalize proper names when their owners do not (e.g., write `"pandas" `_, not "Pandas", even at the start of a sentence) or when referring to a software artifact (e.g., write ``numpy`` when referring to the library, and "NumPy" when referring to the project). * When referring to YAML constructs, `block` refers to an entry whose value is a nested collection of key/value pairs, while `entry` refers to a single key/value pair. -* When using the ``.. code-block::`` directive, align the actual code with the word ``code``. Also, when ``.. code-block::`` directives appear in bulleted or numberd lists, align them with the text following the space to the right of the bullet/number. For example: +* When using the ``.. code-block::`` directive, align the actual code with the word ``code``. Also, when ``.. code-block::`` directives appear in bulleted or numbered lists, align them with the text following the space to the right of the bullet/number. For example: .. code-block:: text @@ -52,7 +52,7 @@ Please follow these guidelines when contributing to the documentation: .. code-block:: python - n = 88 + n = 42 or @@ -62,4 +62,4 @@ Please follow these guidelines when contributing to the documentation: .. code-block:: python - n = 88 + n = 42 diff --git a/docs/sections/user_guide/api/index.rst b/docs/sections/user_guide/api/index.rst index cd29d7614..ba4bc9b77 100644 --- a/docs/sections/user_guide/api/index.rst +++ b/docs/sections/user_guide/api/index.rst @@ -2,6 +2,8 @@ API === .. toctree:: + :maxdepth: 1 + cdeps chgres_cube config @@ -19,6 +21,7 @@ API make_solo_mosaic mpas mpas_init + orog orog_gsl rocoto schism diff --git a/docs/sections/user_guide/api/orog.rst b/docs/sections/user_guide/api/orog.rst new file mode 100644 index 000000000..a4f9925f1 --- /dev/null +++ b/docs/sections/user_guide/api/orog.rst @@ -0,0 +1,6 @@ +``uwtools.api.orog`` +==================== + +.. automodule:: uwtools.api.orog + :inherited-members: + :members: diff --git a/docs/sections/user_guide/cli/Makefile b/docs/sections/user_guide/cli/Makefile index 0e58d33d6..38ba707b3 100644 --- a/docs/sections/user_guide/cli/Makefile +++ b/docs/sections/user_guide/cli/Makefile @@ -1,8 +1,8 @@ -SUBDIRS = $(shell find . -maxdepth 1 -mindepth 1 -type d) +SUBDIRS = $(shell find . -maxdepth 1 -mindepth 1 -type d | sort) .PHONY: all $(SUBDIRS) all: $(SUBDIRS) $(SUBDIRS): - $(MAKE) -C $@ -j + @$(MAKE) -C $@ -j diff --git a/docs/sections/user_guide/cli/Makefile.outputs b/docs/sections/user_guide/cli/Makefile.outputs index 48f78253b..0c0ff7bab 100644 --- a/docs/sections/user_guide/cli/Makefile.outputs +++ b/docs/sections/user_guide/cli/Makefile.outputs @@ -6,4 +6,4 @@ OUTPUTS = $(COMMANDS:cmd=out) all: $(OUTPUTS) $(OUTPUTS): - bash $(basename $@).cmd >$@ 2>&1 | true + @bash $(basename $@).cmd >$@ 2>&1 | true diff --git a/docs/sections/user_guide/cli/drivers/Makefile b/docs/sections/user_guide/cli/drivers/Makefile deleted file mode 120000 index d0b0e8e00..000000000 --- a/docs/sections/user_guide/cli/drivers/Makefile +++ /dev/null @@ -1 +0,0 @@ -../Makefile \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/Makefile b/docs/sections/user_guide/cli/drivers/Makefile new file mode 100644 index 000000000..b974b6377 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/Makefile @@ -0,0 +1,8 @@ +SUBDIRS = $(filter-out ./shared,$(shell find . -maxdepth 1 -mindepth 1 -type d | sort)) + +.PHONY: all $(SUBDIRS) + +all: $(SUBDIRS) + +$(SUBDIRS): + @$(MAKE) -C $@ -j diff --git a/docs/sections/user_guide/cli/drivers/cdeps/help.rst b/docs/sections/user_guide/cli/drivers/cdeps/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/cdeps/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/cdeps.rst b/docs/sections/user_guide/cli/drivers/cdeps/index.rst similarity index 64% rename from docs/sections/user_guide/cli/drivers/cdeps.rst rename to docs/sections/user_guide/cli/drivers/cdeps/index.rst index 1044ad6c6..c24fa84b4 100644 --- a/docs/sections/user_guide/cli/drivers/cdeps.rst +++ b/docs/sections/user_guide/cli/drivers/cdeps/index.rst @@ -1,23 +1,11 @@ ``cdeps`` ========= -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the :CDEPS:`cdeps<>` component. -.. literalinclude:: cdeps/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: cdeps/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: cdeps/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: cdeps/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -45,10 +33,4 @@ Its contents are described in depth in section :ref:`cdeps_yaml`. Each of the va .. include:: /shared/key_path.rst -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: cdeps/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: cdeps/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/cdeps/run-help.out b/docs/sections/user_guide/cli/drivers/cdeps/run-help.out index 7ddba5ecc..abe05b455 100644 --- a/docs/sections/user_guide/cli/drivers/cdeps/run-help.out +++ b/docs/sections/user_guide/cli/drivers/cdeps/run-help.out @@ -1,6 +1,6 @@ usage: uw cdeps atm --cycle CYCLE [-h] [--version] [--config-file PATH] [--dry-run] [--graph-file PATH] [--key-path KEY[.KEY...]] - [--quiet] [--verbose] + [--schema-file PATH] [--quiet] [--verbose] The data atmosphere configuration with all required content @@ -22,6 +22,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/cdeps/schema-options.rst b/docs/sections/user_guide/cli/drivers/cdeps/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/cdeps/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/cdeps/show-schema.cmd b/docs/sections/user_guide/cli/drivers/cdeps/show-schema.cmd index 5be54d6cd..f4f64f7ab 100644 --- a/docs/sections/user_guide/cli/drivers/cdeps/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/cdeps/show-schema.cmd @@ -1,2 +1,2 @@ uw cdeps --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/cdeps/show-schema.out b/docs/sections/user_guide/cli/drivers/cdeps/show-schema.out index b5e4a7c03..6c3911d50 100644 --- a/docs/sections/user_guide/cli/drivers/cdeps/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/cdeps/show-schema.out @@ -8,14 +8,13 @@ "additionalProperties": false, "properties": { "dtlimit": { -... - "required": [ - "rundir" - ] - } - }, - "required": [ - "cdeps" - ], - "type": "object" -} + "type": "number" + }, + "mapalgo": { + "type": "string" + }, + "readmode": { + "enum": [ + "single" + ] + }, diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/help.out b/docs/sections/user_guide/cli/drivers/chgres_cube/help.out index 7d632bb8b..2728654fa 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube/help.out +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/help.out @@ -20,5 +20,7 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/help.rst b/docs/sections/user_guide/cli/drivers/chgres_cube/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube.rst b/docs/sections/user_guide/cli/drivers/chgres_cube/index.rst similarity index 76% rename from docs/sections/user_guide/cli/drivers/chgres_cube.rst rename to docs/sections/user_guide/cli/drivers/chgres_cube/index.rst index 6d954a1c6..1c0d960ff 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube.rst +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/index.rst @@ -1,23 +1,11 @@ ``chgres_cube`` =============== -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the :ufs-utils:`chgres_cube` component. -.. literalinclude:: chgres_cube/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: chgres_cube/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: chgres_cube/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: chgres_cube/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -59,10 +47,4 @@ Its contents are described in depth in section :ref:`chgres_cube_yaml`. Each of $ uw chgres_cube provisioned_rundir --config-file config.yaml --cycle 2023-12-15T18 --batch -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: chgres_cube/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: chgres_cube/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out b/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out index e4a817eb0..3f0eca53e 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/run-help.out @@ -1,12 +1,15 @@ -usage: uw chgres_cube run --cycle CYCLE [-h] [--version] [--config-file PATH] - [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] +usage: uw chgres_cube run --cycle CYCLE --leadtime LEADTIME [-h] [--version] + [--config-file PATH] [--batch] [--dry-run] + [--graph-file PATH] [--key-path KEY[.KEY...]] + [--schema-file PATH] [--quiet] [--verbose] A run Required arguments: --cycle CYCLE - The cycle in ISO8601 format (e.g. 2024-05-23T18) + The cycle in ISO8601 format (e.g. 2024-10-31T18) + --leadtime LEADTIME + The leadtime as hours[:minutes[:seconds]] Optional arguments: -h, --help @@ -24,6 +27,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/schema-options.rst b/docs/sections/user_guide/cli/drivers/chgres_cube/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.cmd b/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.cmd index f539df7cc..d5276c539 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.cmd @@ -1,2 +1,2 @@ uw chgres_cube --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.out b/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.out index 97d8138dd..75ff6dc13 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/chgres_cube/show-schema.out @@ -8,14 +8,13 @@ "properties": { "batchargs": { "additionalProperties": true, -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "chgres_cube" - ], - "type": "object" -} + "properties": { + "cores": { + "type": "integer" + }, + "debug": { + "type": "boolean" + }, + "exclusive": { + "type": "boolean" + }, diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/help.out b/docs/sections/user_guide/cli/drivers/esg_grid/help.out index 0069553e6..0e0ae5a98 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid/help.out +++ b/docs/sections/user_guide/cli/drivers/esg_grid/help.out @@ -20,5 +20,7 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/help.rst b/docs/sections/user_guide/cli/drivers/esg_grid/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/esg_grid/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/esg_grid.rst b/docs/sections/user_guide/cli/drivers/esg_grid/index.rst similarity index 74% rename from docs/sections/user_guide/cli/drivers/esg_grid.rst rename to docs/sections/user_guide/cli/drivers/esg_grid/index.rst index e04e8af80..9bad1de34 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid.rst +++ b/docs/sections/user_guide/cli/drivers/esg_grid/index.rst @@ -1,23 +1,11 @@ ``esg_grid`` ============ -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the :ufs-utils:`regional_esg_grid` component. -.. literalinclude:: esg_grid/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: esg_grid/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: esg_grid/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: esg_grid/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -59,10 +47,4 @@ The driver creates a ``runscript.esg_grid`` file in the directory specified by ` $ uw esg_grid provisioned_rundir --config-file config.yaml --batch -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: esg_grid/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: esg_grid/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out b/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out index 27163b2ff..94eeee88b 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out +++ b/docs/sections/user_guide/cli/drivers/esg_grid/run-help.out @@ -1,6 +1,7 @@ usage: uw esg_grid run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] + [--quiet] [--verbose] A run @@ -20,6 +21,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/schema-options.rst b/docs/sections/user_guide/cli/drivers/esg_grid/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/esg_grid/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.cmd b/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.cmd index a98ee17dc..5c1be9e1a 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.cmd @@ -1,2 +1,2 @@ uw esg_grid --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.out b/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.out index 8ac2cdd16..6c5f9493d 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/esg_grid/show-schema.out @@ -8,14 +8,13 @@ "properties": { "regional_grid_nml": { "additionalProperties": false, -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "esg_grid" - ], - "type": "object" -} + "properties": { + "delx": { + "type": "number" + }, + "dely": { + "type": "number" + }, + "lx": { + "type": "number" + }, diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/help.out b/docs/sections/user_guide/cli/drivers/filter_topo/help.out index 1c426c0f4..cd71378b2 100644 --- a/docs/sections/user_guide/cli/drivers/filter_topo/help.out +++ b/docs/sections/user_guide/cli/drivers/filter_topo/help.out @@ -12,6 +12,8 @@ Optional arguments: Positional arguments: TASK + filtered_output_file + The filtered output file staged from raw input input_grid_file The input grid file namelist_file @@ -22,5 +24,7 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/help.rst b/docs/sections/user_guide/cli/drivers/filter_topo/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/filter_topo/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/filter_topo.rst b/docs/sections/user_guide/cli/drivers/filter_topo/index.rst similarity index 69% rename from docs/sections/user_guide/cli/drivers/filter_topo.rst rename to docs/sections/user_guide/cli/drivers/filter_topo/index.rst index a7a77ff5d..58eb2844f 100644 --- a/docs/sections/user_guide/cli/drivers/filter_topo.rst +++ b/docs/sections/user_guide/cli/drivers/filter_topo/index.rst @@ -1,23 +1,11 @@ ``filter_topo`` =============== -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the UFS Utils preprocessing component ``filter_topo``. Documentation for this UFS Utils component is :ufs-utils:`here `. -.. literalinclude:: filter_topo/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: filter_topo/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: filter_topo/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: filter_topo/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -53,10 +41,4 @@ Its contents are described in section :ref:`filter_topo_yaml`. .. include:: /shared/key_path.rst -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: filter_topo/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: filter_topo/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out b/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out index 0ba5caf14..95604d511 100644 --- a/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out +++ b/docs/sections/user_guide/cli/drivers/filter_topo/run-help.out @@ -1,6 +1,7 @@ usage: uw filter_topo run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] + [--quiet] [--verbose] A run @@ -20,6 +21,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/schema-options.rst b/docs/sections/user_guide/cli/drivers/filter_topo/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/filter_topo/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.cmd b/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.cmd index 43883c9fa..d1e197206 100644 --- a/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.cmd @@ -1,2 +1,2 @@ uw filter_topo --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.out b/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.out index d1ff44ffc..a0793b09b 100644 --- a/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/filter_topo/show-schema.out @@ -6,16 +6,15 @@ "config": { "additionalProperties": false, "properties": { + "filtered_orog": { + "type": "string" + }, "input_grid_file": { "type": "string" -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "filter_topo" - ], - "type": "object" -} + }, + "input_raw_orog": { + "type": "string" + } + }, + "required": [ + "filtered_orog", diff --git a/docs/sections/user_guide/cli/drivers/fv3/help.out b/docs/sections/user_guide/cli/drivers/fv3/help.out index 09420ef44..a383dbafe 100644 --- a/docs/sections/user_guide/cli/drivers/fv3/help.out +++ b/docs/sections/user_guide/cli/drivers/fv3/help.out @@ -34,5 +34,7 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/fv3/help.rst b/docs/sections/user_guide/cli/drivers/fv3/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/fv3/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/fv3.rst b/docs/sections/user_guide/cli/drivers/fv3/index.rst similarity index 75% rename from docs/sections/user_guide/cli/drivers/fv3.rst rename to docs/sections/user_guide/cli/drivers/fv3/index.rst index c601d7464..3bf2d91ad 100644 --- a/docs/sections/user_guide/cli/drivers/fv3.rst +++ b/docs/sections/user_guide/cli/drivers/fv3/index.rst @@ -1,23 +1,11 @@ ``fv3`` ======= -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running FV3. -.. literalinclude:: fv3/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: fv3/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: fv3/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: fv3/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -54,10 +42,4 @@ The examples use a configuration file named ``config.yaml``. Its contents are de $ uw fv3 provisioned_rundir --config-file config.yaml --cycle 2024-02-11T12 --batch -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: fv3/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: fv3/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/fv3/run-help.out b/docs/sections/user_guide/cli/drivers/fv3/run-help.out index cb1c1d124..e56710a4d 100644 --- a/docs/sections/user_guide/cli/drivers/fv3/run-help.out +++ b/docs/sections/user_guide/cli/drivers/fv3/run-help.out @@ -1,6 +1,7 @@ usage: uw fv3 run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] [--quiet] + [--verbose] A run @@ -24,6 +25,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/fv3/schema-options.rst b/docs/sections/user_guide/cli/drivers/fv3/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/fv3/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/fv3/show-schema.cmd b/docs/sections/user_guide/cli/drivers/fv3/show-schema.cmd index 25d915cea..019f36cab 100644 --- a/docs/sections/user_guide/cli/drivers/fv3/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/fv3/show-schema.cmd @@ -1,2 +1,2 @@ uw fv3 --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/fv3/show-schema.out b/docs/sections/user_guide/cli/drivers/fv3/show-schema.out index 6957302c1..5b706fe20 100644 --- a/docs/sections/user_guide/cli/drivers/fv3/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/fv3/show-schema.out @@ -8,14 +8,13 @@ "properties": { "domain": { "const": "regional" -... - "rundir" + } + } + }, + "then": { + "required": [ + "lateral_boundary_conditions" + ] + } + } ], - "type": "object" - } - }, - "required": [ - "fv3" - ], - "type": "object" -} diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.out b/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.out index eb3d7b505..d9d829b91 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.out +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.out @@ -20,5 +20,7 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.rst b/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst b/docs/sections/user_guide/cli/drivers/global_equiv_resol/index.rst similarity index 69% rename from docs/sections/user_guide/cli/drivers/global_equiv_resol.rst rename to docs/sections/user_guide/cli/drivers/global_equiv_resol/index.rst index f02e99760..31af8faf1 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/index.rst @@ -1,23 +1,11 @@ ``global_equiv_resol`` ====================== -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the UFS Utils preprocessing component ``global_equiv_resol``. Documentation for this UFS Utils component is :ufs-utils:`here `. -.. literalinclude:: global_equiv_resol/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: global_equiv_resol/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: global_equiv_resol/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: global_equiv_resol/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -53,10 +41,4 @@ Its contents are described in section :ref:`global_equiv_resol_yaml`. .. include:: /shared/key_path.rst -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: global_equiv_resol/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: global_equiv_resol/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out b/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out index 497807855..5e6e9a84e 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/run-help.out @@ -1,7 +1,7 @@ usage: uw global_equiv_resol run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] - [--verbose] + [--key-path KEY[.KEY...]] + [--schema-file PATH] [--quiet] [--verbose] A run @@ -21,6 +21,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/schema-options.rst b/docs/sections/user_guide/cli/drivers/global_equiv_resol/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.cmd b/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.cmd index 6d43fbfb2..241e6a930 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.cmd @@ -1,2 +1,2 @@ uw global_equiv_resol --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.out b/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.out index bfdd2e0e0..a8b78ad83 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol/show-schema.out @@ -8,14 +8,13 @@ "properties": { "batchargs": { "additionalProperties": true, -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "global_equiv_resol" - ], - "type": "object" -} + "properties": { + "cores": { + "type": "integer" + }, + "debug": { + "type": "boolean" + }, + "exclusive": { + "type": "boolean" + }, diff --git a/docs/sections/user_guide/cli/drivers/index.rst b/docs/sections/user_guide/cli/drivers/index.rst index b61b8242b..3efe9f315 100644 --- a/docs/sections/user_guide/cli/drivers/index.rst +++ b/docs/sections/user_guide/cli/drivers/index.rst @@ -6,20 +6,23 @@ Drivers .. toctree:: :maxdepth: 1 - cdeps - chgres_cube - esg_grid - filter_topo - fv3 - global_equiv_resol - ioda - jedi - make_hgrid - make_solo_mosaic - mpas - mpas_init - orog_gsl - sfc_climo_gen - shave - ungrib - upp + cdeps/index + chgres_cube/index + esg_grid/index + filter_topo/index + fv3/index + global_equiv_resol/index + ioda/index + jedi/index + make_hgrid/index + make_solo_mosaic/index + mpas/index + mpas_init/index + orog/index + orog_gsl/index + schism/index + sfc_climo_gen/index + shave/index + ungrib/index + upp/index + ww3/index diff --git a/docs/sections/user_guide/cli/drivers/ioda/help.out b/docs/sections/user_guide/cli/drivers/ioda/help.out index 692138971..23a0d65ec 100644 --- a/docs/sections/user_guide/cli/drivers/ioda/help.out +++ b/docs/sections/user_guide/cli/drivers/ioda/help.out @@ -24,5 +24,7 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/ioda/help.rst b/docs/sections/user_guide/cli/drivers/ioda/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/ioda.rst b/docs/sections/user_guide/cli/drivers/ioda/index.rst similarity index 69% rename from docs/sections/user_guide/cli/drivers/ioda.rst rename to docs/sections/user_guide/cli/drivers/ioda/index.rst index 9ea6e745c..2d33d173b 100644 --- a/docs/sections/user_guide/cli/drivers/ioda.rst +++ b/docs/sections/user_guide/cli/drivers/ioda/index.rst @@ -1,23 +1,11 @@ ``ioda`` ======== -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the IODA components of the JEDI framework. -.. literalinclude:: ioda/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: ioda/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: ioda/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: ioda/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -53,10 +41,4 @@ The driver creates a ``runscript.ioda`` file in the directory specified by ``run .. include:: /shared/key_path.rst -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: ioda/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: ioda/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/ioda/run-help.out b/docs/sections/user_guide/cli/drivers/ioda/run-help.out index fcbe6dd72..43e297ce0 100644 --- a/docs/sections/user_guide/cli/drivers/ioda/run-help.out +++ b/docs/sections/user_guide/cli/drivers/ioda/run-help.out @@ -1,6 +1,7 @@ usage: uw ioda run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] [--quiet] + [--verbose] A run @@ -24,6 +25,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/ioda/schema-options.rst b/docs/sections/user_guide/cli/drivers/ioda/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/ioda/show-schema.cmd b/docs/sections/user_guide/cli/drivers/ioda/show-schema.cmd index 1b16cbe71..f00f3d90e 100644 --- a/docs/sections/user_guide/cli/drivers/ioda/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/ioda/show-schema.cmd @@ -1,2 +1,2 @@ uw ioda --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/ioda/show-schema.out b/docs/sections/user_guide/cli/drivers/ioda/show-schema.out index f90bcf6a7..e9ae97191 100644 --- a/docs/sections/user_guide/cli/drivers/ioda/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/ioda/show-schema.out @@ -8,14 +8,13 @@ "anyOf": [ { "required": [ -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "ioda" - ], - "type": "object" -} + "base_file" + ] + }, + { + "required": [ + "update_values" + ] + } + ], + "properties": { diff --git a/docs/sections/user_guide/cli/drivers/jedi/help.out b/docs/sections/user_guide/cli/drivers/jedi/help.out index fe3d01ae0..0386272f6 100644 --- a/docs/sections/user_guide/cli/drivers/jedi/help.out +++ b/docs/sections/user_guide/cli/drivers/jedi/help.out @@ -24,6 +24,8 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config validate_only diff --git a/docs/sections/user_guide/cli/drivers/jedi/help.rst b/docs/sections/user_guide/cli/drivers/jedi/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/jedi/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/jedi.rst b/docs/sections/user_guide/cli/drivers/jedi/index.rst similarity index 68% rename from docs/sections/user_guide/cli/drivers/jedi.rst rename to docs/sections/user_guide/cli/drivers/jedi/index.rst index 464300b94..57682264a 100644 --- a/docs/sections/user_guide/cli/drivers/jedi.rst +++ b/docs/sections/user_guide/cli/drivers/jedi/index.rst @@ -1,23 +1,11 @@ ``jedi`` ======== -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the JEDI framework. -.. literalinclude:: jedi/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: jedi/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: jedi/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: jedi/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -53,10 +41,4 @@ The driver creates a ``runscript.jedi`` file in the directory specified by ``run .. include:: /shared/key_path.rst -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: jedi/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: jedi/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/jedi/run-help.out b/docs/sections/user_guide/cli/drivers/jedi/run-help.out index 242e6185c..aa75b918a 100644 --- a/docs/sections/user_guide/cli/drivers/jedi/run-help.out +++ b/docs/sections/user_guide/cli/drivers/jedi/run-help.out @@ -1,6 +1,7 @@ usage: uw jedi run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] [--quiet] + [--verbose] A run @@ -24,6 +25,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/jedi/schema-options.rst b/docs/sections/user_guide/cli/drivers/jedi/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/jedi/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/jedi/show-schema.cmd b/docs/sections/user_guide/cli/drivers/jedi/show-schema.cmd index 0567b841e..b3859a622 100644 --- a/docs/sections/user_guide/cli/drivers/jedi/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/jedi/show-schema.cmd @@ -1,2 +1,2 @@ uw jedi --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/jedi/show-schema.out b/docs/sections/user_guide/cli/drivers/jedi/show-schema.out index 2dba61813..2fe4f7830 100644 --- a/docs/sections/user_guide/cli/drivers/jedi/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/jedi/show-schema.out @@ -8,14 +8,13 @@ "anyOf": [ { "required": [ -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "jedi" - ], - "type": "object" -} + "base_file" + ] + }, + { + "required": [ + "update_values" + ] + } + ], + "properties": { diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/help.out b/docs/sections/user_guide/cli/drivers/make_hgrid/help.out index 7acf294ca..177f6ae3c 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid/help.out +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/help.out @@ -18,5 +18,7 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/help.rst b/docs/sections/user_guide/cli/drivers/make_hgrid/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid.rst b/docs/sections/user_guide/cli/drivers/make_hgrid/index.rst similarity index 67% rename from docs/sections/user_guide/cli/drivers/make_hgrid.rst rename to docs/sections/user_guide/cli/drivers/make_hgrid/index.rst index dc6d79dd3..95cb9b43f 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid.rst +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/index.rst @@ -1,23 +1,11 @@ ``make_hgrid`` ============== -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the UFS Utils preprocessing component ``make_hgrid``. Documentation for this UFS Utils component is :ufs-utils:`here `. -.. literalinclude:: make_hgrid/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: make_hgrid/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: make_hgrid/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: make_hgrid/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -39,10 +27,10 @@ Its contents are described in section :ref:`make_hgrid_yaml`. An example runscript: - .. literalinclude:: make_hgrid/runscript.cmd + .. literalinclude:: runscript.cmd :language: text :emphasize-lines: 5 - .. literalinclude:: make_hgrid/runscript.out + .. literalinclude:: runscript.out :language: text * Run ``make_hgrid`` via a batch job @@ -61,10 +49,4 @@ Its contents are described in section :ref:`make_hgrid_yaml`. .. include:: /shared/key_path.rst -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: make_hgrid/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: make_hgrid/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out b/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out index bf4c684d9..76ad2d63b 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/run-help.out @@ -1,6 +1,7 @@ usage: uw make_hgrid run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] + [--quiet] [--verbose] A run @@ -20,6 +21,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/schema-options.rst b/docs/sections/user_guide/cli/drivers/make_hgrid/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.cmd b/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.cmd index e41d2138c..122a01acc 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.cmd @@ -1,2 +1,2 @@ uw make_hgrid --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.out b/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.out index 181977dd5..73a46bb7b 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/make_hgrid/show-schema.out @@ -8,14 +8,13 @@ "allOf": [ { "if": { -... - "execution", - "rundir" - ] - } - }, - "required": [ - "make_hgrid" - ], - "type": "object" -} + "properties": { + "grid_type": { + "const": "from_file" + } + } + }, + "then": { + "required": [ + "my_grid_file" + ] diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.out b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.out index 0b899c013..4ef71e00e 100644 --- a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.out +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.out @@ -18,5 +18,7 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.rst b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic.rst b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/index.rst similarity index 67% rename from docs/sections/user_guide/cli/drivers/make_solo_mosaic.rst rename to docs/sections/user_guide/cli/drivers/make_solo_mosaic/index.rst index 7d110acad..f5fc55d25 100644 --- a/docs/sections/user_guide/cli/drivers/make_solo_mosaic.rst +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/index.rst @@ -1,23 +1,11 @@ ``make_solo_mosaic`` ==================== -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the UFS Utils preprocessing component ``make_solo_mosaic``. Documentation for this UFS Utils component is :ufs-utils:`here `. -.. literalinclude:: make_solo_mosaic/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: make_solo_mosaic/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: make_solo_mosaic/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: make_solo_mosaic/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -40,10 +28,10 @@ Its contents are described in section :ref:`make_solo_mosaic_yaml`. .. An example runscript: - .. literalinclude:: make_solo_mosaic/runscript.cmd + .. literalinclude:: runscript.cmd :language: text :emphasize-lines: 5 - .. literalinclude:: make_solo_mosaic/runscript.out + .. literalinclude:: runscript.out :language: text * Run ``make_solo_mosaic`` via a batch job @@ -62,10 +50,4 @@ Its contents are described in section :ref:`make_solo_mosaic_yaml`. .. include:: /shared/key_path.rst -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: make_solo_mosaic/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: make_solo_mosaic/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out index 843caecf8..203357701 100644 --- a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/run-help.out @@ -1,6 +1,7 @@ usage: uw make_solo_mosaic run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] + [--quiet] [--verbose] A run @@ -20,6 +21,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/schema-options.rst b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.cmd b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.cmd index 8100c4a37..9e70dc960 100644 --- a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.cmd @@ -1,2 +1,2 @@ uw make_solo_mosaic --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.out b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.out index 61d30fd75..13ed992b6 100644 --- a/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/make_solo_mosaic/show-schema.out @@ -8,14 +8,13 @@ "properties": { "dir": { "type": "string" -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "make_solo_mosaic" - ], - "type": "object" -} + }, + "mosaic_name": { + "type": "string" + }, + "num_tiles": { + "type": "integer" + }, + "periodx": { + "type": "integer" + }, diff --git a/docs/sections/user_guide/cli/drivers/mpas/help.out b/docs/sections/user_guide/cli/drivers/mpas/help.out index cb7106e5f..aeb8b2ccf 100644 --- a/docs/sections/user_guide/cli/drivers/mpas/help.out +++ b/docs/sections/user_guide/cli/drivers/mpas/help.out @@ -26,6 +26,8 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component streams_file The streams file validate diff --git a/docs/sections/user_guide/cli/drivers/mpas/help.rst b/docs/sections/user_guide/cli/drivers/mpas/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/mpas/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/mpas.rst b/docs/sections/user_guide/cli/drivers/mpas/index.rst similarity index 77% rename from docs/sections/user_guide/cli/drivers/mpas.rst rename to docs/sections/user_guide/cli/drivers/mpas/index.rst index 090d50a92..6b7a86dfa 100644 --- a/docs/sections/user_guide/cli/drivers/mpas.rst +++ b/docs/sections/user_guide/cli/drivers/mpas/index.rst @@ -1,23 +1,11 @@ ``mpas`` ========== -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the MPAS forecast model. Each listed ``TASK`` may be called to generate the runtime asset(s) it is responsible for, and will call any task it depends on as needed. A ``provisioned_rundir`` comprises everything needed for a run, and a ``run`` runs the MPAS executable. -.. literalinclude:: mpas/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: mpas/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: mpas/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: mpas/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -59,10 +47,4 @@ Its contents are described in depth in section :ref:`mpas_yaml`. $ uw mpas provisioned_rundir --config-file config.yaml --cycle 2025-02-12T12 --batch -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: mpas/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: mpas/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/mpas/run-help.out b/docs/sections/user_guide/cli/drivers/mpas/run-help.out index 5a5ffb83f..54d66f963 100644 --- a/docs/sections/user_guide/cli/drivers/mpas/run-help.out +++ b/docs/sections/user_guide/cli/drivers/mpas/run-help.out @@ -1,6 +1,7 @@ usage: uw mpas run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] [--quiet] + [--verbose] A run @@ -24,6 +25,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/mpas/schema-options.rst b/docs/sections/user_guide/cli/drivers/mpas/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/mpas/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/mpas/show-schema.cmd b/docs/sections/user_guide/cli/drivers/mpas/show-schema.cmd index d1293fe18..2eaaa3887 100644 --- a/docs/sections/user_guide/cli/drivers/mpas/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/mpas/show-schema.cmd @@ -1,2 +1,2 @@ uw mpas --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/mpas/show-schema.out b/docs/sections/user_guide/cli/drivers/mpas/show-schema.out index 561d89def..e9aba87d1 100644 --- a/docs/sections/user_guide/cli/drivers/mpas/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/mpas/show-schema.out @@ -8,14 +8,13 @@ "properties": { "batchargs": { "additionalProperties": true, -... - "streams" - ], - "type": "object" - } - }, - "required": [ - "mpas" - ], - "type": "object" -} + "properties": { + "cores": { + "type": "integer" + }, + "debug": { + "type": "boolean" + }, + "exclusive": { + "type": "boolean" + }, diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/help.out b/docs/sections/user_guide/cli/drivers/mpas_init/help.out index c2f743422..262a62bc3 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init/help.out +++ b/docs/sections/user_guide/cli/drivers/mpas_init/help.out @@ -26,6 +26,8 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component streams_file The streams file validate diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/help.rst b/docs/sections/user_guide/cli/drivers/mpas_init/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/mpas_init/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/mpas_init.rst b/docs/sections/user_guide/cli/drivers/mpas_init/index.rst similarity index 77% rename from docs/sections/user_guide/cli/drivers/mpas_init.rst rename to docs/sections/user_guide/cli/drivers/mpas_init/index.rst index b8286a929..6a2bd9517 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init.rst +++ b/docs/sections/user_guide/cli/drivers/mpas_init/index.rst @@ -1,23 +1,11 @@ ``mpas_init`` ============= -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the MPAS ``init_atmosphere`` tool. Each listed ``TASK`` may be called to generate the runtime asset(s) it is responsible for, and will call any task it depends on as needed. A ``provisioned_rundir`` comprises everything needed for a run, and a ``run`` runs the ``init_atmosphere`` executable. -.. literalinclude:: mpas_init/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: mpas_init/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: mpas_init/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: mpas_init/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -59,10 +47,4 @@ Its contents are described in depth in section :ref:`mpas_init_yaml`. $ uw mpas_init provisioned_rundir --config-file config.yaml --cycle 2023-12-18T00 --batch -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: mpas_init/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: mpas_init/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out b/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out index 53424102b..c9682821e 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out +++ b/docs/sections/user_guide/cli/drivers/mpas_init/run-help.out @@ -1,6 +1,7 @@ usage: uw mpas_init run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] + [--quiet] [--verbose] A run @@ -24,6 +25,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/schema-options.rst b/docs/sections/user_guide/cli/drivers/mpas_init/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/mpas_init/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.cmd b/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.cmd index 32253d17d..c8010053c 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.cmd @@ -1,2 +1,2 @@ uw mpas_init --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.out b/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.out index 998687f90..6a711600f 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/mpas_init/show-schema.out @@ -8,14 +8,13 @@ "properties": { "interval_hours": { "minimum": 1, -... - "streams" - ], - "type": "object" - } - }, - "required": [ - "mpas_init" - ], - "type": "object" -} + "type": "integer" + }, + "length": { + "minimum": 1, + "type": "integer" + }, + "offset": { + "minimum": 0, + "type": "integer" + }, diff --git a/docs/sections/user_guide/cli/drivers/orog/Makefile b/docs/sections/user_guide/cli/drivers/orog/Makefile new file mode 120000 index 000000000..2486334a6 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog/Makefile @@ -0,0 +1 @@ +../../Makefile.outputs \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/orog/help.cmd b/docs/sections/user_guide/cli/drivers/orog/help.cmd new file mode 100644 index 000000000..99cc5dea8 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog/help.cmd @@ -0,0 +1 @@ +uw orog --help diff --git a/docs/sections/user_guide/cli/drivers/orog/help.out b/docs/sections/user_guide/cli/drivers/orog/help.out new file mode 100644 index 000000000..b3f75f452 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog/help.out @@ -0,0 +1,30 @@ +usage: uw orog [-h] [--version] [--show-schema] TASK ... + +Execute orog tasks + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --show-schema + Show driver schema and exit + +Positional arguments: + TASK + files_linked + Files linked for run + grid_file + The input grid file + input_config_file + The input config file + provisioned_rundir + Run directory provisioned with all required content + run + A run + runscript + The runscript + show_output + Show the output to be created by this component + validate + Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/orog/help.rst b/docs/sections/user_guide/cli/drivers/orog/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/orog/index.rst b/docs/sections/user_guide/cli/drivers/orog/index.rst new file mode 100644 index 000000000..77f16009c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog/index.rst @@ -0,0 +1,44 @@ +``orog`` +======== + +.. include:: ../shared/idempotent.rst + +The ``uw`` mode for configuring and running the UFS Utils preprocessing component ``orog``. + +.. include:: help.rst + +Examples +^^^^^^^^ + +The examples use a configuration file named ``config.yaml`` with contents similar to: + +.. highlight:: yaml +.. literalinclude:: /shared/orog.yaml + +Its contents are described in section :ref:`orog_yaml`. + +* Run ``orog`` on an interactive node + + .. code-block:: text + + $ uw orog run --config-file config.yaml + + The driver creates a ``runscript.orog`` file in the directory specified by ``rundir:`` in the config and runs it, executing ``orog``. + +* Run ``orog`` via a batch job + + .. code-block:: text + + $ uw orog run --config-file config.yaml --batch + + The driver creates a ``runscript.orog`` file in the directory specified by ``rundir:`` in the config and submits it to the batch system. Running with ``--batch`` requires a correctly configured ``platform:`` block in ``config.yaml``, as well as appropriate settings in the ``execution:`` block under ``orog:``. + +* Specifying the ``--dry-run`` flag results in the driver logging messages about actions it would have taken, without actually taking any. + + .. code-block:: text + + $ uw orog run --config-file config.yaml --batch --dry-run + +.. include:: /shared/key_path.rst + +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/orog/run-help.cmd b/docs/sections/user_guide/cli/drivers/orog/run-help.cmd new file mode 100644 index 000000000..addf31211 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog/run-help.cmd @@ -0,0 +1 @@ +uw orog run --help diff --git a/docs/sections/user_guide/cli/drivers/orog/run-help.out b/docs/sections/user_guide/cli/drivers/orog/run-help.out new file mode 100644 index 000000000..3a2bd47d8 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog/run-help.out @@ -0,0 +1,28 @@ +usage: uw orog run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] + [--graph-file PATH] [--key-path KEY[.KEY...]] + [--schema-file PATH] [--quiet] [--verbose] + +A run + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) + --batch + Submit run to batch scheduler + --dry-run + Only log info, making no changes + --graph-file PATH + Path to Graphviz DOT output [experimental] + --key-path KEY[.KEY...] + Dot-separated path of keys leading through the config to the driver's + configuration block + --schema-file PATH + Path to schema file to use for validation + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages diff --git a/docs/sections/user_guide/cli/drivers/orog/schema-options.rst b/docs/sections/user_guide/cli/drivers/orog/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/orog/show-schema.cmd b/docs/sections/user_guide/cli/drivers/orog/show-schema.cmd new file mode 100644 index 000000000..3a9f2ccd5 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog/show-schema.cmd @@ -0,0 +1,2 @@ +uw orog --show-schema >schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/orog/show-schema.out b/docs/sections/user_guide/cli/drivers/orog/show-schema.out new file mode 100644 index 000000000..1f217767e --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog/show-schema.out @@ -0,0 +1,20 @@ +{ + "properties": { + "orog": { + "additionalProperties": false, + "properties": { + "execution": { + "additionalProperties": false, + "properties": { + "batchargs": { + "additionalProperties": true, + "properties": { + "cores": { + "type": "integer" + }, + "debug": { + "type": "boolean" + }, + "exclusive": { + "type": "boolean" + }, diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/help.out b/docs/sections/user_guide/cli/drivers/orog_gsl/help.out index b49285d3d..c5ee11d8a 100644 --- a/docs/sections/user_guide/cli/drivers/orog_gsl/help.out +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/help.out @@ -12,6 +12,8 @@ Optional arguments: Positional arguments: TASK + input_config_file + The input config file input_grid_file The input grid file provisioned_rundir @@ -20,6 +22,8 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component topo_data_2p5m Global topographic data on 2.5-minute lat-lon grid topo_data_30s diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/help.rst b/docs/sections/user_guide/cli/drivers/orog_gsl/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl.rst b/docs/sections/user_guide/cli/drivers/orog_gsl/index.rst similarity index 69% rename from docs/sections/user_guide/cli/drivers/orog_gsl.rst rename to docs/sections/user_guide/cli/drivers/orog_gsl/index.rst index d589651b6..086b66018 100644 --- a/docs/sections/user_guide/cli/drivers/orog_gsl.rst +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/index.rst @@ -1,23 +1,11 @@ ``orog_gsl`` ============ -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the UFS Utils preprocessing component ``orog_gsl``. Documentation for this UFS Utils component is :ufs-utils:`here `. -.. literalinclude:: orog_gsl/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: orog_gsl/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: orog_gsl/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: orog_gsl/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -53,10 +41,4 @@ Its contents are described in section :ref:`orog_gsl_yaml`. .. include:: /shared/key_path.rst -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: orog_gsl/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: orog_gsl/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out b/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out index 5601cf270..ff7a8e7fe 100644 --- a/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/run-help.out @@ -1,6 +1,7 @@ usage: uw orog_gsl run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] + [--quiet] [--verbose] A run @@ -20,6 +21,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/schema-options.rst b/docs/sections/user_guide/cli/drivers/orog_gsl/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.cmd b/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.cmd index 3bdf61aac..012ac00d4 100644 --- a/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.cmd @@ -1,2 +1,2 @@ uw orog_gsl --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.out b/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.out index 36a92dfb9..6bb443ba4 100644 --- a/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/orog_gsl/show-schema.out @@ -8,14 +8,13 @@ "properties": { "halo": { "type": "integer" -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "orog_gsl" - ], - "type": "object" -} + }, + "input_grid_file": { + "type": "string" + }, + "resolution": { + "type": "integer" + }, + "tile": { + "maximum": 7, + "minimum": 1, diff --git a/docs/sections/user_guide/cli/drivers/schism/Makefile b/docs/sections/user_guide/cli/drivers/schism/Makefile new file mode 120000 index 000000000..2486334a6 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/schism/Makefile @@ -0,0 +1 @@ +../../Makefile.outputs \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/schism/help.cmd b/docs/sections/user_guide/cli/drivers/schism/help.cmd new file mode 100644 index 000000000..c6864aec9 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/schism/help.cmd @@ -0,0 +1 @@ +uw schism --help diff --git a/docs/sections/user_guide/cli/drivers/schism/help.out b/docs/sections/user_guide/cli/drivers/schism/help.out new file mode 100644 index 000000000..e7d8e40cb --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/schism/help.out @@ -0,0 +1,20 @@ +usage: uw schism [-h] [--version] [--show-schema] TASK ... + +Execute schism tasks + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --show-schema + Show driver schema and exit + +Positional arguments: + TASK + namelist_file + Render the namelist from the template file + provisioned_rundir + Run directory provisioned with all required content + validate + Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/schism/help.rst b/docs/sections/user_guide/cli/drivers/schism/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/schism/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/schism/index.rst b/docs/sections/user_guide/cli/drivers/schism/index.rst new file mode 100644 index 000000000..28039a476 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/schism/index.rst @@ -0,0 +1,34 @@ +``schism`` +========== + +.. include:: ../shared/idempotent.rst + +The ``uw`` mode for configuring and running the :schism:`SCHISM<>` component. + +.. include:: help.rst + +Examples +^^^^^^^^ + +The examples use a configuration file named ``config.yaml`` with content similar to: + +.. highlight:: yaml +.. literalinclude:: /shared/schism.yaml + +Its contents are described in depth in section :ref:`schism_yaml`. A Python ``datetime`` object named ``cycle`` is available for use in Jinja2 variables/expressions in the config. + +* Create a provisioned run directory: + + .. code-block:: text + + $ uw schism provisioned_rundir --config-file config.yaml --cycle 2024-10-21T12 + +* Validate the config file: + + .. code-block:: text + + $ uw schism validate --config-file config.yaml --cycle 2024-10-21T12 + +.. include:: /shared/key_path.rst + +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/schism/run-help.cmd b/docs/sections/user_guide/cli/drivers/schism/run-help.cmd new file mode 100644 index 000000000..c6864aec9 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/schism/run-help.cmd @@ -0,0 +1 @@ +uw schism --help diff --git a/docs/sections/user_guide/cli/drivers/schism/run-help.out b/docs/sections/user_guide/cli/drivers/schism/run-help.out new file mode 100644 index 000000000..e7d8e40cb --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/schism/run-help.out @@ -0,0 +1,20 @@ +usage: uw schism [-h] [--version] [--show-schema] TASK ... + +Execute schism tasks + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --show-schema + Show driver schema and exit + +Positional arguments: + TASK + namelist_file + Render the namelist from the template file + provisioned_rundir + Run directory provisioned with all required content + validate + Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/schism/schema-options.rst b/docs/sections/user_guide/cli/drivers/schism/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/schism/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/schism/show-schema.cmd b/docs/sections/user_guide/cli/drivers/schism/show-schema.cmd new file mode 100644 index 000000000..357900260 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/schism/show-schema.cmd @@ -0,0 +1,2 @@ +uw schism --show-schema >schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/schism/show-schema.out b/docs/sections/user_guide/cli/drivers/schism/show-schema.out new file mode 100644 index 000000000..5884d1d7f --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/schism/show-schema.out @@ -0,0 +1,20 @@ +{ + "properties": { + "schism": { + "additionalProperties": false, + "properties": { + "namelist": { + "additionalProperties": false, + "properties": { + "template_file": { + "type": "string" + }, + "template_values": { + "minProperties": 1, + "type": "object" + } + }, + "required": [ + "template_file" + ], + "type": "object" diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.out b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.out index f6170ae46..9882b3160 100644 --- a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.out +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.out @@ -20,5 +20,7 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.rst b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/index.rst similarity index 74% rename from docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst rename to docs/sections/user_guide/cli/drivers/sfc_climo_gen/index.rst index fe8939667..4743dfa67 100644 --- a/docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/index.rst @@ -1,23 +1,11 @@ ``sfc_climo_gen`` ================= -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the :sfc-climo-gen:`sfc_climo_gen<>` component. -.. literalinclude:: sfc_climo_gen/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: sfc_climo_gen/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: sfc_climo_gen/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: sfc_climo_gen/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -59,10 +47,4 @@ Its contents are described in depth in section :ref:`sfc_climo_gen_yaml`. $ uw sfc_climo_gen provisioned_rundir --config-file config.yaml --batch -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: sfc_climo_gen/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: sfc_climo_gen/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out index 2e5f865ed..7b6b718ad 100644 --- a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/run-help.out @@ -1,6 +1,7 @@ usage: uw sfc_climo_gen run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] + [--quiet] [--verbose] A run @@ -20,6 +21,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/schema-options.rst b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.cmd b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.cmd index 918a4c80b..c04e9d963 100644 --- a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.cmd @@ -1,2 +1,2 @@ uw sfc_climo_gen --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.out b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.out index 9cafdced3..5834b61e0 100644 --- a/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen/show-schema.out @@ -8,14 +8,13 @@ "properties": { "batchargs": { "additionalProperties": true, -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "sfc_climo_gen" - ], - "type": "object" -} + "properties": { + "cores": { + "type": "integer" + }, + "debug": { + "type": "boolean" + }, + "exclusive": { + "type": "boolean" + }, diff --git a/docs/sections/user_guide/cli/drivers/shared/help.rst b/docs/sections/user_guide/cli/drivers/shared/help.rst new file mode 100644 index 000000000..640c6bbf8 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/shared/help.rst @@ -0,0 +1,13 @@ +.. literalinclude:: help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: help.out + :language: text + +All tasks take the same arguments. For example: + +.. literalinclude:: run-help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: run-help.out + :language: text diff --git a/docs/shared/idempotent.rst b/docs/sections/user_guide/cli/drivers/shared/idempotent.rst similarity index 100% rename from docs/shared/idempotent.rst rename to docs/sections/user_guide/cli/drivers/shared/idempotent.rst diff --git a/docs/sections/user_guide/cli/drivers/shared/schema-options.rst b/docs/sections/user_guide/cli/drivers/shared/schema-options.rst new file mode 100644 index 000000000..4d8705b97 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/shared/schema-options.rst @@ -0,0 +1,9 @@ +* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: + + .. literalinclude:: show-schema.cmd + :language: text + :emphasize-lines: 1 + .. literalinclude:: show-schema.out + :language: text + +* Use the ``--schema-file`` option to specify a custom :json-schema:`JSON Schema<>` file with which to validate the driver config. A custom schema could range in complexity from the simplest, most permissive schema, ``{}``, to one based on the internal schema shown by ``--show-schema``. diff --git a/docs/sections/user_guide/cli/drivers/shave/help.out b/docs/sections/user_guide/cli/drivers/shave/help.out index 7cd91374e..d5b2bfba9 100644 --- a/docs/sections/user_guide/cli/drivers/shave/help.out +++ b/docs/sections/user_guide/cli/drivers/shave/help.out @@ -12,11 +12,15 @@ Optional arguments: Positional arguments: TASK + input_config_file + The input config file provisioned_rundir Run directory provisioned with all required content run A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/shave/help.rst b/docs/sections/user_guide/cli/drivers/shave/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/shave/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/shave.rst b/docs/sections/user_guide/cli/drivers/shave/index.rst similarity index 69% rename from docs/sections/user_guide/cli/drivers/shave.rst rename to docs/sections/user_guide/cli/drivers/shave/index.rst index ff528c5ab..6035d18f9 100644 --- a/docs/sections/user_guide/cli/drivers/shave.rst +++ b/docs/sections/user_guide/cli/drivers/shave/index.rst @@ -1,23 +1,11 @@ ``shave`` ========= -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the UFS Utils preprocessing component ``shave``. Documentation for this UFS Utils component is :ufs-utils:`here `. -.. literalinclude:: shave/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: shave/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: shave/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: shave/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -53,10 +41,4 @@ Its contents are described in section :ref:`shave_yaml`. .. include:: /shared/key_path.rst -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: shave/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: shave/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/shave/run-help.out b/docs/sections/user_guide/cli/drivers/shave/run-help.out index 6f1148e6b..085279196 100644 --- a/docs/sections/user_guide/cli/drivers/shave/run-help.out +++ b/docs/sections/user_guide/cli/drivers/shave/run-help.out @@ -1,6 +1,6 @@ usage: uw shave run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] [--key-path KEY[.KEY...]] - [--quiet] [--verbose] + [--schema-file PATH] [--quiet] [--verbose] A run @@ -20,6 +20,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/shave/schema-options.rst b/docs/sections/user_guide/cli/drivers/shave/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/shave/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/shave/show-schema.cmd b/docs/sections/user_guide/cli/drivers/shave/show-schema.cmd index 783cabcfe..21262b4f5 100644 --- a/docs/sections/user_guide/cli/drivers/shave/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/shave/show-schema.cmd @@ -1,2 +1,2 @@ uw shave --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/shave/show-schema.out b/docs/sections/user_guide/cli/drivers/shave/show-schema.out index b18e04f5f..a5a738c9e 100644 --- a/docs/sections/user_guide/cli/drivers/shave/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/shave/show-schema.out @@ -8,14 +8,13 @@ "properties": { "input_grid_file": { "type": "string" -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "shave" - ], - "type": "object" -} + }, + "nhalo": { + "minimum": 0, + "type": "integer" + }, + "nx": { + "minimum": 1, + "type": "integer" + }, + "ny": { diff --git a/docs/sections/user_guide/cli/drivers/ungrib/help.out b/docs/sections/user_guide/cli/drivers/ungrib/help.out index 252272e05..1794559ad 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib/help.out +++ b/docs/sections/user_guide/cli/drivers/ungrib/help.out @@ -22,6 +22,8 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config vtable diff --git a/docs/sections/user_guide/cli/drivers/ungrib/help.rst b/docs/sections/user_guide/cli/drivers/ungrib/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ungrib/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/ungrib.rst b/docs/sections/user_guide/cli/drivers/ungrib/index.rst similarity index 75% rename from docs/sections/user_guide/cli/drivers/ungrib.rst rename to docs/sections/user_guide/cli/drivers/ungrib/index.rst index df25b4e1f..6f8f4991c 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib.rst +++ b/docs/sections/user_guide/cli/drivers/ungrib/index.rst @@ -1,23 +1,11 @@ ``ungrib`` ========== -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the WRF preprocessing component ``ungrib``. -.. literalinclude:: ungrib/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: ungrib/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: ungrib/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: ungrib/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -59,10 +47,4 @@ Its contents are described in depth in section :ref:`ungrib_yaml`. $ uw ungrib provisioned_rundir --config-file config.yaml --cycle 2021-04-01T12 --batch -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: ungrib/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: ungrib/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/ungrib/run-help.out b/docs/sections/user_guide/cli/drivers/ungrib/run-help.out index 6733f094b..d73f89464 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib/run-help.out +++ b/docs/sections/user_guide/cli/drivers/ungrib/run-help.out @@ -1,6 +1,7 @@ usage: uw ungrib run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--key-path KEY[.KEY...]] [--quiet] [--verbose] + [--key-path KEY[.KEY...]] [--schema-file PATH] [--quiet] + [--verbose] A run @@ -24,6 +25,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/ungrib/schema-options.rst b/docs/sections/user_guide/cli/drivers/ungrib/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ungrib/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/ungrib/show-schema.cmd b/docs/sections/user_guide/cli/drivers/ungrib/show-schema.cmd index 04ae66e66..c8215c629 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/ungrib/show-schema.cmd @@ -1,2 +1,2 @@ uw ungrib --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/ungrib/show-schema.out b/docs/sections/user_guide/cli/drivers/ungrib/show-schema.out index 01684d40b..f932a5be6 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/ungrib/show-schema.out @@ -8,14 +8,13 @@ "properties": { "batchargs": { "additionalProperties": true, -... - "vtable" - ], - "type": "object" - } - }, - "required": [ - "ungrib" - ], - "type": "object" -} + "properties": { + "cores": { + "type": "integer" + }, + "debug": { + "type": "boolean" + }, + "exclusive": { + "type": "boolean" + }, diff --git a/docs/sections/user_guide/cli/drivers/upp/help.out b/docs/sections/user_guide/cli/drivers/upp/help.out index 7163d3f71..155940673 100644 --- a/docs/sections/user_guide/cli/drivers/upp/help.out +++ b/docs/sections/user_guide/cli/drivers/upp/help.out @@ -12,6 +12,8 @@ Optional arguments: Positional arguments: TASK + control_file + The GRIB control file files_copied Files copied for run files_linked @@ -24,5 +26,7 @@ Positional arguments: A run runscript The runscript + show_output + Show the output to be created by this component validate Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/upp/help.rst b/docs/sections/user_guide/cli/drivers/upp/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/upp/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/upp.rst b/docs/sections/user_guide/cli/drivers/upp/index.rst similarity index 76% rename from docs/sections/user_guide/cli/drivers/upp.rst rename to docs/sections/user_guide/cli/drivers/upp/index.rst index ae8d24eb4..b1e2cef4d 100644 --- a/docs/sections/user_guide/cli/drivers/upp.rst +++ b/docs/sections/user_guide/cli/drivers/upp/index.rst @@ -1,23 +1,11 @@ ``upp`` ======= -.. include:: /shared/idempotent.rst +.. include:: ../shared/idempotent.rst The ``uw`` mode for configuring and running the `UPP `_ component. -.. literalinclude:: upp/help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: upp/help.out - :language: text - -All tasks take the same arguments. For example: - -.. literalinclude:: upp/run-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: upp/run-help.out - :language: text +.. include:: help.rst Examples ^^^^^^^^ @@ -59,10 +47,4 @@ Its contents are described in depth in section :ref:`upp_yaml`. $ uw upp provisioned_rundir --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 --batch -* Specifying the ``--show-schema`` flag, with no other options, prints the driver's schema: - -.. literalinclude:: upp/show-schema.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: upp/show-schema.out - :language: text +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/upp/run-help.out b/docs/sections/user_guide/cli/drivers/upp/run-help.out index d727fd2ad..106254c93 100644 --- a/docs/sections/user_guide/cli/drivers/upp/run-help.out +++ b/docs/sections/user_guide/cli/drivers/upp/run-help.out @@ -1,7 +1,7 @@ usage: uw upp run --cycle CYCLE --leadtime LEADTIME [-h] [--version] [--config-file PATH] [--batch] [--dry-run] - [--graph-file PATH] [--key-path KEY[.KEY...]] [--quiet] - [--verbose] + [--graph-file PATH] [--key-path KEY[.KEY...]] + [--schema-file PATH] [--quiet] [--verbose] A run @@ -27,6 +27,8 @@ Optional arguments: --key-path KEY[.KEY...] Dot-separated path of keys leading through the config to the driver's configuration block + --schema-file PATH + Path to schema file to use for validation --quiet, -q Print no logging messages --verbose, -v diff --git a/docs/sections/user_guide/cli/drivers/upp/schema-options.rst b/docs/sections/user_guide/cli/drivers/upp/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/upp/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/upp/show-schema.cmd b/docs/sections/user_guide/cli/drivers/upp/show-schema.cmd index c0a1b290d..4ae9137f1 100644 --- a/docs/sections/user_guide/cli/drivers/upp/show-schema.cmd +++ b/docs/sections/user_guide/cli/drivers/upp/show-schema.cmd @@ -1,2 +1,2 @@ uw upp --show-schema >schema -head schema && echo ... && tail schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/upp/show-schema.out b/docs/sections/user_guide/cli/drivers/upp/show-schema.out index 11be8c436..67517e149 100644 --- a/docs/sections/user_guide/cli/drivers/upp/show-schema.out +++ b/docs/sections/user_guide/cli/drivers/upp/show-schema.out @@ -3,19 +3,18 @@ "upp": { "additionalProperties": false, "properties": { + "control_file": { + "type": "string" + }, "execution": { "additionalProperties": false, "properties": { "batchargs": { "additionalProperties": true, -... - "rundir" - ], - "type": "object" - } - }, - "required": [ - "upp" - ], - "type": "object" -} + "properties": { + "cores": { + "type": "integer" + }, + "debug": { + "type": "boolean" + }, diff --git a/docs/sections/user_guide/cli/drivers/ww3/Makefile b/docs/sections/user_guide/cli/drivers/ww3/Makefile new file mode 120000 index 000000000..2486334a6 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ww3/Makefile @@ -0,0 +1 @@ +../../Makefile.outputs \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/ww3/help.cmd b/docs/sections/user_guide/cli/drivers/ww3/help.cmd new file mode 100644 index 000000000..707095cce --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ww3/help.cmd @@ -0,0 +1 @@ +uw ww3 --help diff --git a/docs/sections/user_guide/cli/drivers/ww3/help.out b/docs/sections/user_guide/cli/drivers/ww3/help.out new file mode 100644 index 000000000..01c95264d --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ww3/help.out @@ -0,0 +1,22 @@ +usage: uw ww3 [-h] [--version] [--show-schema] TASK ... + +Execute ww3 tasks + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --show-schema + Show driver schema and exit + +Positional arguments: + TASK + namelist_file + Render the namelist from the template file + provisioned_rundir + Run directory provisioned with all required content + restart_directory + The restart directory + validate + Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/ww3/help.rst b/docs/sections/user_guide/cli/drivers/ww3/help.rst new file mode 120000 index 000000000..fd986fe2c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ww3/help.rst @@ -0,0 +1 @@ +../shared/help.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/ww3/index.rst b/docs/sections/user_guide/cli/drivers/ww3/index.rst new file mode 100644 index 000000000..3d24f9fd7 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ww3/index.rst @@ -0,0 +1,34 @@ +``ww3`` +======= + +.. include:: ../shared/idempotent.rst + +The ``uw`` mode for configuring and running the :ww3:`Wave Watch III<>` component. + +.. include:: help.rst + +Examples +^^^^^^^^ + +The examples use a configuration file named ``config.yaml`` with content similar to: + +.. highlight:: yaml +.. literalinclude:: /shared/ww3.yaml + +Its contents are described in depth in section :ref:`ww3_yaml`. A Python ``datetime`` object named ``cycle`` is available for use in Jinja2 variables/expressions in the config. + +* Create a provisioned run directory: + + .. code-block:: text + + $ uw ww3 provisioned_rundir --config-file config.yaml --cycle 2024-10-21T12 + +* Validate the config file: + + .. code-block:: text + + $ uw ww3 validate --config-file config.yaml --cycle 2024-10-21T12 + +.. include:: /shared/key_path.rst + +.. include:: schema-options.rst diff --git a/docs/sections/user_guide/cli/drivers/ww3/run-help.cmd b/docs/sections/user_guide/cli/drivers/ww3/run-help.cmd new file mode 100644 index 000000000..707095cce --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ww3/run-help.cmd @@ -0,0 +1 @@ +uw ww3 --help diff --git a/docs/sections/user_guide/cli/drivers/ww3/run-help.out b/docs/sections/user_guide/cli/drivers/ww3/run-help.out new file mode 100644 index 000000000..01c95264d --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ww3/run-help.out @@ -0,0 +1,22 @@ +usage: uw ww3 [-h] [--version] [--show-schema] TASK ... + +Execute ww3 tasks + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --show-schema + Show driver schema and exit + +Positional arguments: + TASK + namelist_file + Render the namelist from the template file + provisioned_rundir + Run directory provisioned with all required content + restart_directory + The restart directory + validate + Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/ww3/schema-options.rst b/docs/sections/user_guide/cli/drivers/ww3/schema-options.rst new file mode 120000 index 000000000..0e3ca4a92 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ww3/schema-options.rst @@ -0,0 +1 @@ +../shared/schema-options.rst \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/ww3/show-schema.cmd b/docs/sections/user_guide/cli/drivers/ww3/show-schema.cmd new file mode 100644 index 000000000..bea15145a --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ww3/show-schema.cmd @@ -0,0 +1,2 @@ +uw ww3 --show-schema >schema +head -n20 schema diff --git a/docs/sections/user_guide/cli/drivers/ww3/show-schema.out b/docs/sections/user_guide/cli/drivers/ww3/show-schema.out new file mode 100644 index 000000000..59c08e178 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ww3/show-schema.out @@ -0,0 +1,20 @@ +{ + "properties": { + "ww3": { + "additionalProperties": false, + "properties": { + "namelist": { + "additionalProperties": false, + "properties": { + "template_file": { + "type": "string" + }, + "template_values": { + "minProperties": 1, + "type": "object" + } + }, + "required": [ + "template_file" + ], + "type": "object" diff --git a/docs/sections/user_guide/cli/tools/config/compare-bad-extension-fix.out b/docs/sections/user_guide/cli/tools/config/compare-bad-extension-fix.out index 867c6f0dd..bbc7505d1 100644 --- a/docs/sections/user_guide/cli/tools/config/compare-bad-extension-fix.out +++ b/docs/sections/user_guide/cli/tools/config/compare-bad-extension-fix.out @@ -1,4 +1,8 @@ -[2024-05-23T19:39:15] INFO - a.txt -[2024-05-23T19:39:15] INFO + c.nml -[2024-05-23T19:39:15] INFO --------------------------------------------------------------------- -[2024-05-23T19:39:15] INFO values: recipient: - World + None +[2024-11-14T23:27:44] INFO - a.txt +[2024-11-14T23:27:44] INFO + c.nml +[2024-11-14T23:27:44] INFO --------------------------------------------------------------------- +[2024-11-14T23:27:44] INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line +[2024-11-14T23:27:44] INFO --------------------------------------------------------------------- +[2024-11-14T23:27:44] INFO values: +[2024-11-14T23:27:44] INFO greeting: Hello +[2024-11-14T23:27:44] INFO - recipient: World diff --git a/docs/sections/user_guide/cli/tools/config/compare-diff.out b/docs/sections/user_guide/cli/tools/config/compare-diff.out index 9c33b4c5b..bd62f4793 100644 --- a/docs/sections/user_guide/cli/tools/config/compare-diff.out +++ b/docs/sections/user_guide/cli/tools/config/compare-diff.out @@ -1,4 +1,8 @@ -[2024-05-23T19:39:16] INFO - a.nml -[2024-05-23T19:39:16] INFO + c.nml -[2024-05-23T19:39:16] INFO --------------------------------------------------------------------- -[2024-05-23T19:39:16] INFO values: recipient: - World + None +[2024-11-14T23:27:44] INFO - a.nml +[2024-11-14T23:27:44] INFO + c.nml +[2024-11-14T23:27:44] INFO --------------------------------------------------------------------- +[2024-11-14T23:27:44] INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line +[2024-11-14T23:27:44] INFO --------------------------------------------------------------------- +[2024-11-14T23:27:44] INFO values: +[2024-11-14T23:27:44] INFO greeting: Hello +[2024-11-14T23:27:44] INFO - recipient: World diff --git a/docs/sections/user_guide/cli/tools/config/compare-match.out b/docs/sections/user_guide/cli/tools/config/compare-match.out index c10927611..546e36399 100644 --- a/docs/sections/user_guide/cli/tools/config/compare-match.out +++ b/docs/sections/user_guide/cli/tools/config/compare-match.out @@ -1,3 +1,2 @@ -[2024-05-23T19:39:15] INFO - a.nml -[2024-05-23T19:39:15] INFO + b.nml -[2024-05-23T19:39:15] INFO --------------------------------------------------------------------- +[2024-11-14T23:27:45] INFO - a.nml +[2024-11-14T23:27:45] INFO + b.nml diff --git a/docs/sections/user_guide/cli/tools/config/compare-verbose.out b/docs/sections/user_guide/cli/tools/config/compare-verbose.out index 1eab26de4..ae1bb4d4e 100644 --- a/docs/sections/user_guide/cli/tools/config/compare-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/compare-verbose.out @@ -1,5 +1,9 @@ -[2024-05-23T19:39:15] DEBUG Command: uw config compare --file-1-path a.nml --file-2-path c.nml --verbose -[2024-05-23T19:39:15] INFO - a.nml -[2024-05-23T19:39:15] INFO + c.nml -[2024-05-23T19:39:15] INFO --------------------------------------------------------------------- -[2024-05-23T19:39:15] INFO values: recipient: - World + None +[2024-11-14T23:27:45] DEBUG Command: uw config compare --file-1-path a.nml --file-2-path c.nml --verbose +[2024-11-14T23:27:45] INFO - a.nml +[2024-11-14T23:27:45] INFO + c.nml +[2024-11-14T23:27:45] INFO --------------------------------------------------------------------- +[2024-11-14T23:27:45] INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line +[2024-11-14T23:27:45] INFO --------------------------------------------------------------------- +[2024-11-14T23:27:45] INFO values: +[2024-11-14T23:27:45] INFO greeting: Hello +[2024-11-14T23:27:45] INFO - recipient: World diff --git a/docs/sections/user_guide/cli/tools/config/validate-fail.out b/docs/sections/user_guide/cli/tools/config/validate-fail.out index a581b8ce3..36086bd66 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-fail.out +++ b/docs/sections/user_guide/cli/tools/config/validate-fail.out @@ -1,3 +1,3 @@ -[2024-05-23T19:39:17] ERROR 1 UW schema-validation error found -[2024-05-23T19:39:17] ERROR Error at values: -[2024-05-23T19:39:17] ERROR 'recipient' is a required property +[2024-08-26T22:54:28] ERROR 1 UW schema-validation error found in config +[2024-08-26T22:54:28] ERROR Error at values: +[2024-08-26T22:54:28] ERROR 'recipient' is a required property diff --git a/docs/sections/user_guide/cli/tools/config/validate-pass-stdin.out b/docs/sections/user_guide/cli/tools/config/validate-pass-stdin.out index a760389d0..8a5717ab3 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-pass-stdin.out +++ b/docs/sections/user_guide/cli/tools/config/validate-pass-stdin.out @@ -1 +1 @@ -[2024-05-23T19:39:15] INFO 0 UW schema-validation errors found +[2024-08-26T22:54:27] INFO 0 UW schema-validation errors found in config diff --git a/docs/sections/user_guide/cli/tools/config/validate-pass.out b/docs/sections/user_guide/cli/tools/config/validate-pass.out index efd846fd7..b0b2e202e 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-pass.out +++ b/docs/sections/user_guide/cli/tools/config/validate-pass.out @@ -1 +1 @@ -[2024-05-23T19:39:17] INFO 0 UW schema-validation errors found +[2024-08-26T22:54:28] INFO 0 UW schema-validation errors found in config diff --git a/docs/sections/user_guide/cli/tools/config/validate-verbose.out b/docs/sections/user_guide/cli/tools/config/validate-verbose.out index d1ed3d613..17dc2a651 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/validate-verbose.out @@ -1,20 +1,21 @@ -[2024-05-23T19:39:16] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose -[2024-05-23T19:39:16] DEBUG Dereferencing, current value: -[2024-05-23T19:39:16] DEBUG values: -[2024-05-23T19:39:16] DEBUG greeting: Hello -[2024-05-23T19:39:16] DEBUG recipient: World -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: Hello -[2024-05-23T19:39:17] DEBUG [dereference] Rendered: Hello -[2024-05-23T19:39:17] DEBUG [dereference] Rendering: greeting -[2024-05-23T19:39:17] DEBUG [dereference] Rendered: greeting -[2024-05-23T19:39:17] DEBUG [dereference] Rendering: World -[2024-05-23T19:39:17] DEBUG [dereference] Rendered: World -[2024-05-23T19:39:17] DEBUG [dereference] Rendering: recipient -[2024-05-23T19:39:17] DEBUG [dereference] Rendered: recipient -[2024-05-23T19:39:17] DEBUG [dereference] Rendering: values -[2024-05-23T19:39:17] DEBUG [dereference] Rendered: values -[2024-05-23T19:39:17] DEBUG Dereferencing, final value: -[2024-05-23T19:39:17] DEBUG values: -[2024-05-23T19:39:17] DEBUG greeting: Hello -[2024-05-23T19:39:17] DEBUG recipient: World -[2024-05-23T19:39:17] INFO 0 UW schema-validation errors found +[2024-08-26T22:54:28] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose +[2024-08-26T22:54:28] DEBUG Using schema file: schema.jsonschema +[2024-08-26T22:54:28] DEBUG Dereferencing, current value: +[2024-08-26T22:54:28] DEBUG values: +[2024-08-26T22:54:28] DEBUG greeting: Hello +[2024-08-26T22:54:28] DEBUG recipient: World +[2024-08-26T22:54:28] DEBUG [dereference] Rendering: Hello +[2024-08-26T22:54:28] DEBUG [dereference] Rendered: Hello +[2024-08-26T22:54:28] DEBUG [dereference] Rendering: greeting +[2024-08-26T22:54:28] DEBUG [dereference] Rendered: greeting +[2024-08-26T22:54:28] DEBUG [dereference] Rendering: World +[2024-08-26T22:54:28] DEBUG [dereference] Rendered: World +[2024-08-26T22:54:28] DEBUG [dereference] Rendering: recipient +[2024-08-26T22:54:28] DEBUG [dereference] Rendered: recipient +[2024-08-26T22:54:28] DEBUG [dereference] Rendering: values +[2024-08-26T22:54:28] DEBUG [dereference] Rendered: values +[2024-08-26T22:54:28] DEBUG Dereferencing, final value: +[2024-08-26T22:54:28] DEBUG values: +[2024-08-26T22:54:28] DEBUG greeting: Hello +[2024-08-26T22:54:28] DEBUG recipient: World +[2024-08-26T22:54:29] INFO 0 UW schema-validation errors found in config diff --git a/docs/sections/user_guide/cli/tools/execute.rst b/docs/sections/user_guide/cli/tools/execute.rst index d5d6ee945..753b7cb3f 100644 --- a/docs/sections/user_guide/cli/tools/execute.rst +++ b/docs/sections/user_guide/cli/tools/execute.rst @@ -61,4 +61,4 @@ Config ``rand.yaml`` .. literalinclude:: execute/alt-schema.out :language: text -* Other arguments behave identically to the same-named arguments to built-in ``uwtools`` drivers (see :ref:`drivers`). +* Other arguments behave identically to the same-named arguments to internal ``uwtools`` drivers (see :ref:`drivers`). diff --git a/docs/sections/user_guide/cli/tools/execute/alt-schema.out b/docs/sections/user_guide/cli/tools/execute/alt-schema.out index 9691a3987..c89f32d66 100644 --- a/docs/sections/user_guide/cli/tools/execute/alt-schema.out +++ b/docs/sections/user_guide/cli/tools/execute/alt-schema.out @@ -1,7 +1,7 @@ -[2024-08-08T23:46:07] INFO 0 UW schema-validation errors found -[2024-08-08T23:46:07] INFO rand Random-integer file: Initial state: Not Ready -[2024-08-08T23:46:07] INFO rand Random-integer file: Checking requirements -[2024-08-08T23:46:07] INFO rand Random-integer file: Requirement(s) ready -[2024-08-08T23:46:07] INFO rand Random-integer file: Executing -[2024-08-08T23:46:07] INFO rand Random-integer file: Final state: Ready -Random integer is 38 +[2024-08-26T23:03:41] INFO 0 UW schema-validation errors found in rand config +[2024-08-26T23:03:41] INFO rand Random-integer file: Initial state: Not Ready +[2024-08-26T23:03:41] INFO rand Random-integer file: Checking requirements +[2024-08-26T23:03:41] INFO rand Random-integer file: Requirement(s) ready +[2024-08-26T23:03:41] INFO rand Random-integer file: Executing +[2024-08-26T23:03:41] INFO rand Random-integer file: Final state: Ready +Random integer is 14 diff --git a/docs/sections/user_guide/cli/tools/execute/execute.out b/docs/sections/user_guide/cli/tools/execute/execute.out index 3c291fc99..0851d0480 100644 --- a/docs/sections/user_guide/cli/tools/execute/execute.out +++ b/docs/sections/user_guide/cli/tools/execute/execute.out @@ -1,7 +1,7 @@ -[2024-08-08T23:46:07] INFO 0 UW schema-validation errors found -[2024-08-08T23:46:07] INFO rand Random-integer file: Initial state: Not Ready -[2024-08-08T23:46:07] INFO rand Random-integer file: Checking requirements -[2024-08-08T23:46:07] INFO rand Random-integer file: Requirement(s) ready -[2024-08-08T23:46:07] INFO rand Random-integer file: Executing -[2024-08-08T23:46:07] INFO rand Random-integer file: Final state: Ready -Random integer is 43 +[2024-08-26T23:03:40] INFO 0 UW schema-validation errors found in rand config +[2024-08-26T23:03:40] INFO rand Random-integer file: Initial state: Not Ready +[2024-08-26T23:03:40] INFO rand Random-integer file: Checking requirements +[2024-08-26T23:03:40] INFO rand Random-integer file: Requirement(s) ready +[2024-08-26T23:03:40] INFO rand Random-integer file: Executing +[2024-08-26T23:03:40] INFO rand Random-integer file: Final state: Ready +Random integer is 80 diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out index 02c57878d..71b4ca3a1 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out @@ -1,3 +1,3 @@ -[2024-08-14T15:19:59] INFO Validating config against internal schema: files-to-stage -[2024-08-14T15:19:59] INFO 0 UW schema-validation errors found -[2024-08-14T15:19:59] ERROR Relative path 'foo' requires the target directory to be specified +[2024-08-26T23:03:40] INFO Validating config against internal schema: files-to-stage +[2024-08-26T23:03:40] INFO 0 UW schema-validation errors found in fs config +[2024-08-26T23:03:40] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out index 30b4605fa..48bee1999 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out @@ -1,13 +1,13 @@ -[2024-08-02T00:43:08] INFO Validating config against internal schema: files-to-stage -[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found -[2024-08-02T00:43:08] INFO File copies: Initial state: Not Ready -[2024-08-02T00:43:08] INFO File copies: Checking requirements -[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Initial state: Not Ready -[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Checking requirements -[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Requirement(s) ready -[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Executing -[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Final state: Ready -[2024-08-02T00:43:08] INFO File copies: Final state: Ready +[2024-08-26T23:03:42] INFO Validating config against internal schema: files-to-stage +[2024-08-26T23:03:43] INFO 0 UW schema-validation errors found in fs config +[2024-08-26T23:03:43] INFO File copies: Initial state: Not Ready +[2024-08-26T23:03:43] INFO File copies: Checking requirements +[2024-08-26T23:03:43] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Initial state: Not Ready +[2024-08-26T23:03:43] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Checking requirements +[2024-08-26T23:03:43] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Requirement(s) ready +[2024-08-26T23:03:43] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Executing +[2024-08-26T23:03:43] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Final state: Ready +[2024-08-26T23:03:43] INFO File copies: Final state: Ready copy-dst-timedep └── baz-2024-05-29T18 diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-exec.out index 7c5b50333..57221e756 100644 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec.out @@ -1,18 +1,18 @@ -[2024-08-02T00:43:08] INFO Validating config against internal schema: files-to-stage -[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found -[2024-08-02T00:43:08] INFO File copies: Initial state: Not Ready -[2024-08-02T00:43:08] INFO File copies: Checking requirements -[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Initial state: Not Ready -[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Checking requirements -[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready -[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Executing -[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Final state: Ready -[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Not Ready -[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements -[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready -[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Executing -[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready -[2024-08-02T00:43:08] INFO File copies: Final state: Ready +[2024-08-26T23:03:41] INFO Validating config against internal schema: files-to-stage +[2024-08-26T23:03:41] INFO 0 UW schema-validation errors found in fs config +[2024-08-26T23:03:41] INFO File copies: Initial state: Not Ready +[2024-08-26T23:03:41] INFO File copies: Checking requirements +[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Initial state: Not Ready +[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Checking requirements +[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready +[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Executing +[2024-08-26T23:03:41] INFO Copy src/foo -> copy-dst/foo: Final state: Ready +[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Not Ready +[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements +[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready +[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Executing +[2024-08-26T23:03:41] INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready +[2024-08-26T23:03:41] INFO File copies: Final state: Ready copy-dst ├── foo diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out index 03c1247f5..dcb5593ed 100644 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out @@ -1,3 +1,3 @@ -[2024-08-14T15:19:57] INFO Validating config against internal schema: files-to-stage -[2024-08-14T15:19:57] INFO 0 UW schema-validation errors found -[2024-08-14T15:19:57] ERROR Relative path 'foo' requires the target directory to be specified +[2024-08-26T23:03:41] INFO Validating config against internal schema: files-to-stage +[2024-08-26T23:03:41] INFO 0 UW schema-validation errors found in fs config +[2024-08-26T23:03:41] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out index 6256a41df..c97fb1879 100644 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out @@ -1,13 +1,13 @@ -[2024-08-02T00:43:09] INFO Validating config against internal schema: files-to-stage -[2024-08-02T00:43:09] INFO 0 UW schema-validation errors found -[2024-08-02T00:43:09] INFO File links: Initial state: Not Ready -[2024-08-02T00:43:09] INFO File links: Checking requirements -[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Initial state: Not Ready -[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Checking requirements -[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Requirement(s) ready -[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Executing -[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Final state: Ready -[2024-08-02T00:43:09] INFO File links: Final state: Ready +[2024-08-26T23:03:41] INFO Validating config against internal schema: files-to-stage +[2024-08-26T23:03:41] INFO 0 UW schema-validation errors found in fs config +[2024-08-26T23:03:41] INFO File links: Initial state: Not Ready +[2024-08-26T23:03:41] INFO File links: Checking requirements +[2024-08-26T23:03:41] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Initial state: Not Ready +[2024-08-26T23:03:41] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Checking requirements +[2024-08-26T23:03:41] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Requirement(s) ready +[2024-08-26T23:03:41] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Executing +[2024-08-26T23:03:42] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Final state: Ready +[2024-08-26T23:03:42] INFO File links: Final state: Ready link-dst-timedep └── baz-2024-05-29T18 -> ../src/20240529/12/006/baz diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec.out b/docs/sections/user_guide/cli/tools/fs/link-exec.out index 2247f979c..ee1524967 100644 --- a/docs/sections/user_guide/cli/tools/fs/link-exec.out +++ b/docs/sections/user_guide/cli/tools/fs/link-exec.out @@ -1,18 +1,18 @@ -[2024-08-02T00:43:08] INFO Validating config against internal schema: files-to-stage -[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found -[2024-08-02T00:43:08] INFO File links: Initial state: Not Ready -[2024-08-02T00:43:08] INFO File links: Checking requirements -[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Initial state: Not Ready -[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Checking requirements -[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Requirement(s) ready -[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Executing -[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Final state: Ready -[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Initial state: Not Ready -[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Checking requirements -[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Requirement(s) ready -[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Executing -[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Final state: Ready -[2024-08-02T00:43:08] INFO File links: Final state: Ready +[2024-08-26T23:03:42] INFO Validating config against internal schema: files-to-stage +[2024-08-26T23:03:43] INFO 0 UW schema-validation errors found in fs config +[2024-08-26T23:03:43] INFO File links: Initial state: Not Ready +[2024-08-26T23:03:43] INFO File links: Checking requirements +[2024-08-26T23:03:43] INFO Link link-dst/foo -> src/foo: Initial state: Not Ready +[2024-08-26T23:03:43] INFO Link link-dst/foo -> src/foo: Checking requirements +[2024-08-26T23:03:43] INFO Link link-dst/foo -> src/foo: Requirement(s) ready +[2024-08-26T23:03:43] INFO Link link-dst/foo -> src/foo: Executing +[2024-08-26T23:03:43] INFO Link link-dst/foo -> src/foo: Final state: Ready +[2024-08-26T23:03:43] INFO Link link-dst/subdir/bar -> src/bar: Initial state: Not Ready +[2024-08-26T23:03:43] INFO Link link-dst/subdir/bar -> src/bar: Checking requirements +[2024-08-26T23:03:43] INFO Link link-dst/subdir/bar -> src/bar: Requirement(s) ready +[2024-08-26T23:03:43] INFO Link link-dst/subdir/bar -> src/bar: Executing +[2024-08-26T23:03:43] INFO Link link-dst/subdir/bar -> src/bar: Final state: Ready +[2024-08-26T23:03:43] INFO File links: Final state: Ready link-dst ├── foo -> ../src/foo diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out index ceefc693e..84c7710bf 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out @@ -1,3 +1,3 @@ -[2024-08-14T15:19:58] INFO Validating config against internal schema: makedirs -[2024-08-14T15:19:58] INFO 0 UW schema-validation errors found -[2024-08-14T15:19:58] ERROR Relative path 'foo' requires the target directory to be specified +[2024-08-26T23:03:44] INFO Validating config against internal schema: makedirs +[2024-08-26T23:03:45] INFO 0 UW schema-validation errors found in fs config +[2024-08-26T23:03:45] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out index a89e33a29..68199d440 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out @@ -1,18 +1,18 @@ -[2024-08-12T04:35:49] INFO Validating config against internal schema: makedirs -[2024-08-12T04:35:49] INFO 0 UW schema-validation errors found -[2024-08-12T04:35:49] INFO Directories: Initial state: Not Ready -[2024-08-12T04:35:49] INFO Directories: Checking requirements -[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Initial state: Not Ready -[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Checking requirements -[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Requirement(s) ready -[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Executing -[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Final state: Ready -[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Initial state: Not Ready -[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Checking requirements -[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Requirement(s) ready -[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Executing -[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Final state: Ready -[2024-08-12T04:35:49] INFO Directories: Final state: Ready +[2024-08-26T23:03:46] INFO Validating config against internal schema: makedirs +[2024-08-26T23:03:46] INFO 0 UW schema-validation errors found in fs config +[2024-08-26T23:03:46] INFO Directories: Initial state: Not Ready +[2024-08-26T23:03:46] INFO Directories: Checking requirements +[2024-08-26T23:03:46] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Initial state: Not Ready +[2024-08-26T23:03:46] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Checking requirements +[2024-08-26T23:03:46] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Requirement(s) ready +[2024-08-26T23:03:46] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Executing +[2024-08-26T23:03:46] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Final state: Ready +[2024-08-26T23:03:46] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Initial state: Not Ready +[2024-08-26T23:03:46] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Checking requirements +[2024-08-26T23:03:46] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Requirement(s) ready +[2024-08-26T23:03:46] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Executing +[2024-08-26T23:03:46] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Final state: Ready +[2024-08-26T23:03:46] INFO Directories: Final state: Ready makedirs-parent-timedep/ ├── baz/ diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out index 376518b59..92e5ae4fb 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out @@ -1,18 +1,18 @@ -[2024-08-12T04:35:49] INFO Validating config against internal schema: makedirs -[2024-08-12T04:35:49] INFO 0 UW schema-validation errors found -[2024-08-12T04:35:49] INFO Directories: Initial state: Not Ready -[2024-08-12T04:35:49] INFO Directories: Checking requirements -[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Initial state: Not Ready -[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Checking requirements -[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Requirement(s) ready -[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Executing -[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Final state: Ready -[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Initial state: Not Ready -[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Checking requirements -[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Requirement(s) ready -[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Executing -[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Final state: Ready -[2024-08-12T04:35:49] INFO Directories: Final state: Ready +[2024-08-26T23:03:46] INFO Validating config against internal schema: makedirs +[2024-08-26T23:03:46] INFO 0 UW schema-validation errors found in fs config +[2024-08-26T23:03:46] INFO Directories: Initial state: Not Ready +[2024-08-26T23:03:46] INFO Directories: Checking requirements +[2024-08-26T23:03:46] INFO Directory makedirs-parent/foo: Initial state: Not Ready +[2024-08-26T23:03:46] INFO Directory makedirs-parent/foo: Checking requirements +[2024-08-26T23:03:46] INFO Directory makedirs-parent/foo: Requirement(s) ready +[2024-08-26T23:03:46] INFO Directory makedirs-parent/foo: Executing +[2024-08-26T23:03:46] INFO Directory makedirs-parent/foo: Final state: Ready +[2024-08-26T23:03:46] INFO Directory makedirs-parent/bar: Initial state: Not Ready +[2024-08-26T23:03:46] INFO Directory makedirs-parent/bar: Checking requirements +[2024-08-26T23:03:46] INFO Directory makedirs-parent/bar: Requirement(s) ready +[2024-08-26T23:03:46] INFO Directory makedirs-parent/bar: Executing +[2024-08-26T23:03:46] INFO Directory makedirs-parent/bar: Final state: Ready +[2024-08-26T23:03:46] INFO Directories: Final state: Ready makedirs-parent/ ├── bar/ diff --git a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-file.out b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-file.out index 894f75605..86924fddc 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-file.out +++ b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-file.out @@ -1,5 +1,5 @@ -[2024-08-02T00:43:07] INFO 0 UW schema-validation errors found -[2024-08-02T00:43:07] INFO 0 Rocoto validation errors found +[2024-08-26T23:11:41] INFO 0 UW schema-validation errors found in Rocoto config +[2024-08-26T23:11:41] INFO 0 Rocoto XML validation errors found diff --git a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out index 2a055c04a..70aad1c6e 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out +++ b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out @@ -1,21 +1,21 @@ -[2024-08-02T00:43:09] DEBUG Command: uw rocoto realize --config-file rocoto.yaml --verbose -[2024-08-02T00:43:09] DEBUG Dereferencing, current value: -[2024-08-02T00:43:09] DEBUG workflow: -[2024-08-02T00:43:09] DEBUG attrs: -[2024-08-02T00:43:09] DEBUG realtime: false -[2024-08-02T00:43:09] DEBUG scheduler: slurm -[2024-08-02T00:43:09] DEBUG cycledef: -[2024-08-02T00:43:09] DEBUG - attrs: -[2024-08-02T00:43:09] DEBUG group: howdy -[2024-08-02T00:43:09] DEBUG spec: 202209290000 202209300000 06:00:00 +[2024-08-26T23:39:19] DEBUG Command: uw rocoto realize --config-file rocoto.yaml --verbose +[2024-08-26T23:39:19] DEBUG Dereferencing, current value: +[2024-08-26T23:39:19] DEBUG workflow: +[2024-08-26T23:39:19] DEBUG attrs: +[2024-08-26T23:39:19] DEBUG realtime: false +[2024-08-26T23:39:19] DEBUG scheduler: slurm +[2024-08-26T23:39:19] DEBUG cycledef: +[2024-08-26T23:39:19] DEBUG - attrs: +[2024-08-26T23:39:19] DEBUG group: howdy +[2024-08-26T23:39:19] DEBUG spec: 202209290000 202209300000 06:00:00 ... -[2024-08-02T00:43:09] DEBUG cycledefs: howdy -[2024-08-02T00:43:09] DEBUG account: '&ACCOUNT;' -[2024-08-02T00:43:09] DEBUG command: echo hello $person -[2024-08-02T00:43:09] DEBUG jobname: hello -[2024-08-02T00:43:09] DEBUG native: --reservation my_reservation -[2024-08-02T00:43:09] DEBUG nodes: 1:ppn=1 -[2024-08-02T00:43:09] DEBUG walltime: 00:01:00 -[2024-08-02T00:43:09] DEBUG envars: -[2024-08-02T00:43:09] DEBUG person: siri -[2024-08-02T00:43:09] INFO 0 Rocoto validation errors found +[2024-08-26T23:39:20] DEBUG cycledefs: howdy +[2024-08-26T23:39:20] DEBUG account: '&ACCOUNT;' +[2024-08-26T23:39:20] DEBUG command: echo hello $person +[2024-08-26T23:39:20] DEBUG jobname: hello +[2024-08-26T23:39:20] DEBUG native: --reservation my_reservation +[2024-08-26T23:39:20] DEBUG nodes: 1:ppn=1 +[2024-08-26T23:39:20] DEBUG walltime: 00:01:00 +[2024-08-26T23:39:20] DEBUG envars: +[2024-08-26T23:39:20] DEBUG person: siri +[2024-08-26T23:39:20] INFO 0 Rocoto XML validation errors found diff --git a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout.out b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout.out index 865b4367d..7f2d1a1e1 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout.out +++ b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout.out @@ -1,5 +1,5 @@ -[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found -[2024-08-02T00:43:08] INFO 0 Rocoto validation errors found +[2024-08-26T23:11:42] INFO 0 UW schema-validation errors found in Rocoto config +[2024-08-26T23:11:42] INFO 0 Rocoto XML validation errors found diff --git a/docs/sections/user_guide/cli/tools/rocoto/validate-bad-file.out b/docs/sections/user_guide/cli/tools/rocoto/validate-bad-file.out index 30cf8fe43..fd68379ad 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/validate-bad-file.out +++ b/docs/sections/user_guide/cli/tools/rocoto/validate-bad-file.out @@ -1,24 +1,24 @@ -[2024-05-23T19:39:16] ERROR 3 Rocoto validation errors found -[2024-05-23T19:39:16] ERROR :9:0:ERROR:RELAXNGV:RELAXNG_ERR_NOELEM: Expecting an element command, got nothing -[2024-05-23T19:39:16] ERROR :9:0:ERROR:RELAXNGV:RELAXNG_ERR_INTERSEQ: Invalid sequence in interleave -[2024-05-23T19:39:16] ERROR :9:0:ERROR:RELAXNGV:RELAXNG_ERR_CONTENTVALID: Element task failed to validate content -[2024-05-23T19:39:16] ERROR Invalid Rocoto XML: -[2024-05-23T19:39:16] ERROR 1 -[2024-05-23T19:39:16] ERROR 2 -[2024-05-23T19:39:16] ERROR 4 -[2024-05-23T19:39:16] ERROR 5 ]> -[2024-05-23T19:39:16] ERROR 6 -[2024-05-23T19:39:16] ERROR 7 202209290000 202209300000 06:00:00 -[2024-05-23T19:39:16] ERROR 8 /some/path/to/&FOO; -[2024-05-23T19:39:16] ERROR 9 -[2024-05-23T19:39:16] ERROR 10 &ACCOUNT; -[2024-05-23T19:39:16] ERROR 11 1:ppn=1 -[2024-05-23T19:39:16] ERROR 12 00:01:00 -[2024-05-23T19:39:16] ERROR 13 hello -[2024-05-23T19:39:16] ERROR 14 -[2024-05-23T19:39:16] ERROR 15 person -[2024-05-23T19:39:16] ERROR 16 siri -[2024-05-23T19:39:16] ERROR 17 -[2024-05-23T19:39:16] ERROR 18 -[2024-05-23T19:39:16] ERROR 19 +[2024-08-26T23:11:41] ERROR 3 Rocoto XML validation errors found +[2024-08-26T23:11:41] ERROR :9:0:ERROR:RELAXNGV:RELAXNG_ERR_NOELEM: Expecting an element command, got nothing +[2024-08-26T23:11:41] ERROR :9:0:ERROR:RELAXNGV:RELAXNG_ERR_INTERSEQ: Invalid sequence in interleave +[2024-08-26T23:11:41] ERROR :9:0:ERROR:RELAXNGV:RELAXNG_ERR_CONTENTVALID: Element task failed to validate content +[2024-08-26T23:11:41] ERROR Invalid Rocoto XML: +[2024-08-26T23:11:41] ERROR 1 +[2024-08-26T23:11:41] ERROR 2 +[2024-08-26T23:11:41] ERROR 4 +[2024-08-26T23:11:41] ERROR 5 ]> +[2024-08-26T23:11:41] ERROR 6 +[2024-08-26T23:11:41] ERROR 7 202209290000 202209300000 06:00:00 +[2024-08-26T23:11:41] ERROR 8 /some/path/to/&FOO; +[2024-08-26T23:11:41] ERROR 9 +[2024-08-26T23:11:41] ERROR 10 &ACCOUNT; +[2024-08-26T23:11:41] ERROR 11 1:ppn=1 +[2024-08-26T23:11:41] ERROR 12 00:01:00 +[2024-08-26T23:11:41] ERROR 13 hello +[2024-08-26T23:11:41] ERROR 14 +[2024-08-26T23:11:41] ERROR 15 person +[2024-08-26T23:11:41] ERROR 16 siri +[2024-08-26T23:11:41] ERROR 17 +[2024-08-26T23:11:41] ERROR 18 +[2024-08-26T23:11:41] ERROR 19 diff --git a/docs/sections/user_guide/cli/tools/rocoto/validate-good-file.out b/docs/sections/user_guide/cli/tools/rocoto/validate-good-file.out index 75a232457..2c54c1c86 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/validate-good-file.out +++ b/docs/sections/user_guide/cli/tools/rocoto/validate-good-file.out @@ -1 +1 @@ -[2024-05-23T19:39:16] INFO 0 Rocoto validation errors found +[2024-08-26T23:11:42] INFO 0 Rocoto XML validation errors found diff --git a/docs/sections/user_guide/cli/tools/rocoto/validate-good-stdin.out b/docs/sections/user_guide/cli/tools/rocoto/validate-good-stdin.out index 75a232457..2c54c1c86 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/validate-good-stdin.out +++ b/docs/sections/user_guide/cli/tools/rocoto/validate-good-stdin.out @@ -1 +1 @@ -[2024-05-23T19:39:16] INFO 0 Rocoto validation errors found +[2024-08-26T23:11:42] INFO 0 Rocoto XML validation errors found diff --git a/docs/sections/user_guide/yaml/components/filter_topo.rst b/docs/sections/user_guide/yaml/components/filter_topo.rst index 638a76c73..4932c7235 100644 --- a/docs/sections/user_guide/yaml/components/filter_topo.rst +++ b/docs/sections/user_guide/yaml/components/filter_topo.rst @@ -20,10 +20,18 @@ config: Configuration parameters for the ``orog_gsl`` component. + **filtered_orog:** + + Name of the filtered output file. + **input_grid_file:** Path to the tiled input grid file. + **input_raw_orog:** + + Path to the raw orography file. The output of the ``orog`` driver. + namelist: ^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/index.rst b/docs/sections/user_guide/yaml/components/index.rst index 919a5ac40..550d67867 100644 --- a/docs/sections/user_guide/yaml/components/index.rst +++ b/docs/sections/user_guide/yaml/components/index.rst @@ -16,6 +16,7 @@ UW YAML for Components make_solo_mosaic mpas mpas_init + orog orog_gsl schism sfc_climo_gen diff --git a/docs/sections/user_guide/yaml/components/orog.rst b/docs/sections/user_guide/yaml/components/orog.rst new file mode 100644 index 000000000..4466f80e9 --- /dev/null +++ b/docs/sections/user_guide/yaml/components/orog.rst @@ -0,0 +1,57 @@ +.. _orog_yaml: + +orog +==== + +Structured YAML to run the component ``orog`` is validated by JSON Schema and requires the ``orog:`` block, described below. If ``orog`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. + +Documentation for the UFS Utils ``orog`` program is :ufs-utils:`here `. + +Here is a prototype UW YAML ``orog:`` block, explained in detail below: + +.. highlight:: yaml +.. literalinclude:: /shared/orog.yaml + +UW YAML for the ``orog:`` Block +------------------------------- + +old_line1_items +^^^^^^^^^^^^^^^ + +Configuration parameters for the ``orog`` component corresponding to the first line entries prior to hash 57bd832 from (July 9, 2024). If using this section, a value for ``orog_file`` should also be provided. + + +execution: +^^^^^^^^^^ + +See :ref:`here ` for details. + +files_to_copy: +^^^^^^^^^^^^^^ + +See :ref:`this page ` for details. + +files_to_link: +^^^^^^^^^^^^^^ + +Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies. + +mask: +^^^^^ + +Boolean indicating whether only the land mask will be generated. Defaults to ``false``. + +merge: +^^^^^^ + +Path to an ocean merge file. + +orog_file: +^^^^^^^^^^ + +Path to an output orography file if using a version of UFS_UTILS prior to hash 57bd832 from (July 9, 2024). If using this section, values for ``old_line1_items`` should also be provided. + +rundir: +^^^^^^^ + +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/schism.rst b/docs/sections/user_guide/yaml/components/schism.rst index 070578149..b02cda41e 100644 --- a/docs/sections/user_guide/yaml/components/schism.rst +++ b/docs/sections/user_guide/yaml/components/schism.rst @@ -3,7 +3,7 @@ schism ====== -Structured YAML to configure SCHISM as part of a compiled coupled executable is validated by JSON Schema and requires the ``schism:`` block, described below. +Structured YAML to configure :schism:`SCHISM<>` as part of a compiled coupled executable is validated by JSON Schema and requires the ``schism:`` block, described below. Here is a prototype UW YAML ``schism:`` block, explained in detail below: diff --git a/docs/sections/user_guide/yaml/components/shave.rst b/docs/sections/user_guide/yaml/components/shave.rst index e29ad3920..9c3ff4c06 100644 --- a/docs/sections/user_guide/yaml/components/shave.rst +++ b/docs/sections/user_guide/yaml/components/shave.rst @@ -27,7 +27,7 @@ Describes the required parameters to run a ``shave`` configuration. Name of the grid file with extra points to be shaved. - **nh4:** + **nhalo:** The number of halo rows/columns. @@ -39,6 +39,10 @@ Describes the required parameters to run a ``shave`` configuration. The j/y dimensions of the compute domain (not including halo) + **output_grid_file:** + + The path to the output file. + rundir: ^^^^^^^ diff --git a/docs/sections/user_guide/yaml/rocoto.rst b/docs/sections/user_guide/yaml/rocoto.rst index a26c334a5..4c4844d4c 100644 --- a/docs/sections/user_guide/yaml/rocoto.rst +++ b/docs/sections/user_guide/yaml/rocoto.rst @@ -86,12 +86,30 @@ In the example, the resulting log would appear in the XML file as: .. code-block:: xml - - /some/path/to/&FOO; - + /some/path/to/&FOO; The ``attrs:`` block is optional within the ``cyclestr:`` block and can be used to specify the cycle offset. +Wherever a ``cyclestr:`` block is accepted, a YAML sequence mixing text and ``cyclestr:`` blocks may also be provided. For example, + +.. code-block:: yaml + + log: + - cyclestr: + value: "%Y%m%d%H" + - -through- + - cyclestr: + attrs: + offset: "06:00:00" + value: "%Y%m%d%H" + - .log + +would be rendered as + +.. code-block:: xml + + %Y%m%d%H-through-%Y%m%d%H.log + Tasks Section ------------- @@ -113,7 +131,7 @@ Let's dissect the following task example: walltime: 00:01:00 envars: person: siri - dependencies: + dependency: Each task is named by its UW YAML key. Blocks under ``tasks:`` prefixed with ``task_`` will be named with what follows the prefix. In the example above the task will be named ``hello`` and will appear in the XML like this: @@ -139,7 +157,7 @@ The name of the task can be any string accepted by Rocoto as a task name (includ siri -``dependencies:`` -- [Optional] Any number of dependencies accepted by Rocoto. This section is described in more detail below. +``dependency:`` -- [Optional] Any number of dependencies accepted by Rocoto. This section is described in more detail below. The other keys not specifically mentioned here follow the same conventions as described in the :rocoto:`Rocoto<>` documentation. @@ -162,7 +180,7 @@ Each of the dependencies that requires attributes (the ``key="value"`` parts ins ... task_goodbye: command: "goodbye" - dependencies: + dependency: taskdep: attrs: task: hello @@ -191,7 +209,7 @@ Because UW YAML represents a hash table (a dictionary in Python), each key at th task_hello: command: "hello world" ... - dependencies: + dependency: and: datadep_foo: value: "foo.txt" diff --git a/docs/sections/user_guide/yaml/tags.rst b/docs/sections/user_guide/yaml/tags.rst index 3a603a0e0..a3a03c5a2 100644 --- a/docs/sections/user_guide/yaml/tags.rst +++ b/docs/sections/user_guide/yaml/tags.rst @@ -23,6 +23,24 @@ Or explicit: Additionally, UW defines the following tags to support use cases not covered by standard tags: +``!datetime`` +^^^^^^^^^^^^^ + +Converts the tagged node to a Python ``datetime`` object. For example, given ``input.yaml``: + +.. code-block:: yaml + + date1: 2024-09-01 + date2: !datetime "{{ date1 }}" + +.. code-block:: text + + % uw config realize -i ../input.yaml --output-format yaml + date1: 2024-09-01 + date2: 2024-09-01 00:00:00 + +The value provided to the tag must be in :python:`ISO 8601 format` to be interpreted correctly by the ``!datetime`` tag. + ``!float`` ^^^^^^^^^^ @@ -62,7 +80,7 @@ Parse the tagged file and include its tags. For example, given ``input.yaml``: .. code-block:: yaml - values: !INCLUDE [./supplemental.yaml] + values: !include [./supplemental.yaml] and ``supplemental.yaml``: diff --git a/docs/shared/chgres_cube.yaml b/docs/shared/chgres_cube.yaml index e7028d300..d717b7395 100644 --- a/docs/shared/chgres_cube.yaml +++ b/docs/shared/chgres_cube.yaml @@ -19,7 +19,8 @@ chgres_cube: fix_dir_target_grid: /path/to/fixdir grib2_file_input_grid: a.file.gb2 mosaic_file_target_grid: /path/to/mosaic/C432.mosaic.halo4.nc - orog_files_target_grid: /path/to/orog/C432.oro_data.tile7.halo4.nc + orog_dir_target_grid: /path/to/fixdir + orog_files_target_grid: C432.oro_data.tile7.halo4.nc sfc_files_input_grid: sfc.t{{cycle.strftime('%H') }}z.nc varmap_file: /path/to/varmap_table vcoord_file_target_grid: /path/to/global_hyblev.l65.txt diff --git a/docs/shared/filter_topo.yaml b/docs/shared/filter_topo.yaml index c3be0fa05..73fb28bbb 100644 --- a/docs/shared/filter_topo.yaml +++ b/docs/shared/filter_topo.yaml @@ -1,6 +1,8 @@ filter_topo: config: + filtered_orog: C403_filtered_orog.tile7.nc input_grid_file: /path/to/C403_grid.tile7.halo6.nc + input_raw_orog: /path/to/out.oro.nc execution: batchargs: cores: 1 diff --git a/docs/shared/mpas.yaml b/docs/shared/mpas.yaml index 8ea7a102b..93ad1b0ea 100644 --- a/docs/shared/mpas.yaml +++ b/docs/shared/mpas.yaml @@ -42,7 +42,7 @@ mpas: nhyd_model: config_dt: 60 validate: true - rundir: /path/to/run/directory + rundir: /path/to/run/dir streams: input: filename_template: conus.init.nc diff --git a/docs/shared/orog.yaml b/docs/shared/orog.yaml new file mode 100644 index 000000000..9fbdef97d --- /dev/null +++ b/docs/shared/orog.yaml @@ -0,0 +1,15 @@ +orog: + execution: + batchargs: + cores: 1 + walltime: 00:05:00 + executable: /path/to/orog + files_to_link: + fort.15: /path/to/fix/thirty.second.antarctic.new.bin + landcover30.fixed: /path/to/fix/landcover30.fixed + fort.235: /path/to/fix/gmted2010.30sec.int + grid_file: /path/to/netcdf/grid/file + rundir: /path/to/run/dir +platform: + account: me + scheduler: slurm diff --git a/docs/shared/schism.yaml b/docs/shared/schism.yaml index 66d4bfbb0..b77c8ce8d 100644 --- a/docs/shared/schism.yaml +++ b/docs/shared/schism.yaml @@ -3,4 +3,4 @@ schism: template_file: /path/to/schism/param.nml.IN template_values: dt: 100 - rundir: /path/to/run/directory + rundir: /path/to/run/dir diff --git a/docs/shared/shave.yaml b/docs/shared/shave.yaml index 18b55ea46..feacb1941 100644 --- a/docs/shared/shave.yaml +++ b/docs/shared/shave.yaml @@ -1,9 +1,10 @@ shave: config: input_grid_file: /path/to/input/grid/file - nh4: 1 + nhalo: 0 nx: 214 ny: 128 + output_grid_file: /path/to/C403_oro_data.tile7.halo0.nc execution: batchargs: cores: 1 diff --git a/docs/shared/upp.yaml b/docs/shared/upp.yaml index 769212bad..96ad01083 100644 --- a/docs/shared/upp.yaml +++ b/docs/shared/upp.yaml @@ -1,4 +1,5 @@ upp: + control_file: /path/to/postxconfig-NT.txt execution: batchargs: export: NONE @@ -12,8 +13,6 @@ upp: mpiargs: - "--ntasks $SLURM_CPUS_ON_NODE" mpicmd: srun - files_to_copy: - postxconfig-NT.txt: /path/to/postxconfig-NT.txt files_to_link: eta_micro_lookup.dat: /path/to/nam_micro_lookup.dat params_grib2_tbl_new: /path/to/params_grib2_tbl_new diff --git a/docs/shared/ww3.yaml b/docs/shared/ww3.yaml index ac1cefa83..4800d7e1a 100644 --- a/docs/shared/ww3.yaml +++ b/docs/shared/ww3.yaml @@ -3,4 +3,4 @@ ww3: template_file: /path/to/ww3/ww3_shel.nml.IN template_values: input_forcing_winds: C - rundir: /path/to/run/directory + rundir: /path/to/run/dir diff --git a/recipe/meta.json b/recipe/meta.json index faa658d13..8913b93a0 100644 --- a/recipe/meta.json +++ b/recipe/meta.json @@ -4,20 +4,20 @@ "name": "uwtools", "packages": { "dev": [ - "black =24.4.*", + "black =24.8.*", "docformatter =1.7.*", "f90nml =1.4.*", "iotaa =0.8.*", "isort =5.13.*", "jinja2 =3.1.*", "jq =1.7.*", - "jsonschema =4.23.*", - "lxml =5.2.*", - "make >=3.8", - "mypy =1.10.*", + "jsonschema >=4.18,<4.24", + "lxml =5.3.*", + "make =4.4.*", + "mypy =1.11.*", "pip", "pylint =3.2.*", - "pytest =8.2.*", + "pytest =8.3.*", "pytest-cov =5.0.*", "pytest-xdist =3.6.*", "python >=3.9,<3.13", @@ -27,8 +27,8 @@ "f90nml =1.4.*", "iotaa =0.8.*", "jinja2 =3.1.*", - "jsonschema =4.23.*", - "lxml =5.2.*", + "jsonschema >=4.18,<4.24", + "lxml =5.3.*", "python >=3.9,<3.13", "pyyaml =6.0.*" ] diff --git a/recipe/meta.yaml b/recipe/meta.yaml index f25b00ca9..479b49330 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -16,20 +16,20 @@ requirements: - f90nml 1.4.* - iotaa 0.8.* - jinja2 3.1.* - - jsonschema 4.23.* - - lxml 5.2.* + - jsonschema >=4.18,<4.24 + - lxml 5.3.* - python >=3.9,<3.13 - pyyaml 6.0.* test: requires: - - black 24.4.* + - black 24.8.* - docformatter 1.7.* - isort 5.13.* - jq 1.7.* - - make >=3.8 - - mypy 1.10.* + - make 4.4.* + - mypy 1.11.* - pylint 3.2.* - - pytest 8.2.* + - pytest 8.3.* - pytest-cov 5.0.* - pytest-xdist 3.6.* about: diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index ec05a0fa5..bbdd7c517 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -179,6 +179,7 @@ def validate( try: _validate_external( schema_file=_str2path(schema_file), + desc="config", config=_ensure_data_source(_str2path(config), stdin_ok), ) except UWConfigError: diff --git a/src/uwtools/api/execute.py b/src/uwtools/api/execute.py index 24c16cee6..6ab629db0 100644 --- a/src/uwtools/api/execute.py +++ b/src/uwtools/api/execute.py @@ -41,7 +41,7 @@ def execute( :param module: Path to driver module or name of module on sys.path. :param classname: Name of driver class to instantiate. :param task: Name of driver task to execute. - :param schema_file: Path to schema file. + :param schema_file: The JSON Schema file to use for validation. :param config: Path to config file (read stdin if missing or None). :param cycle: The cycle. :param leadtime: The leadtime. diff --git a/src/uwtools/api/orog.py b/src/uwtools/api/orog.py new file mode 100644 index 000000000..5b71576db --- /dev/null +++ b/src/uwtools/api/orog.py @@ -0,0 +1,28 @@ +""" +API access to the ``uwtools`` ``orog`` driver. +""" + +from uwtools.drivers.orog import Orog +from uwtools.drivers.support import graph +from uwtools.drivers.support import tasks as _tasks +from uwtools.utils.api import make_execute as _make_execute + +_driver = Orog +execute = _make_execute(_driver) + + +def schema() -> dict: + """ + Return the driver's schema. + """ + return _driver.schema() + + +def tasks() -> dict[str, str]: + """ + Return a mapping from task names to their one-line descriptions. + """ + return _tasks(_driver) + + +__all__ = ["Orog", "execute", "graph", "schema", "tasks"] diff --git a/src/uwtools/api/rocoto.py b/src/uwtools/api/rocoto.py index 3b0406e4b..2260a4849 100644 --- a/src/uwtools/api/rocoto.py +++ b/src/uwtools/api/rocoto.py @@ -22,7 +22,7 @@ def realize( If no input file is specified, ``stdin`` is read. A ``YAMLConfig`` object may also be provided as input. If no output file is specified, ``stdout`` is written to. Both the input config and - output Rocoto XML will be validated against appropriate schcemas. + output Rocoto XML will be validated against appropriate schemas. :param config: YAML input file or ``YAMLConfig`` object (``None`` => read ``stdin``). :param output_file: XML output file path (``None`` => write to ``stdout``). diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 15f3dda36..e268a2e9b 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -81,11 +81,14 @@ def main() -> None: STR.makesolomosaic, STR.mpas, STR.mpasinit, + STR.orog, STR.oroggsl, + STR.schism, STR.sfcclimogen, STR.shave, STR.ungrib, STR.upp, + STR.ww3, ] } modes = {**tools, **drivers} @@ -1031,6 +1034,7 @@ def _add_subparser_for_driver_task( helpmsg="Dot-separated path of keys leading through the config " "to the driver's configuration block", ) + _add_arg_schema_file(optional) checks = _add_args_verbosity(optional) return checks @@ -1128,6 +1132,7 @@ def _dispatch_to_driver(name: str, args: Args) -> bool: "dry_run": args[STR.dryrun], "graph_file": args[STR.graphfile], "key_path": args[STR.keypath], + "schema_file": args[STR.schemafile], "stdin_ok": True, } for k in [STR.batch, STR.cycle, STR.leadtime]: @@ -1172,6 +1177,8 @@ def _parse_args(raw_args: list[str]) -> tuple[Args, Checks]: component: partial(_add_subparser_for_driver, component, subparsers, with_cycle=True) for component in [ STR.cdeps, + STR.schism, + STR.ww3, ] } assets_with_cycle_and_leadtime = { @@ -1188,6 +1195,7 @@ def _parse_args(raw_args: list[str]) -> tuple[Args, Checks]: STR.globalequivresol, STR.makehgrid, STR.makesolomosaic, + STR.orog, STR.oroggsl, STR.sfcclimogen, STR.shave, @@ -1198,7 +1206,6 @@ def _parse_args(raw_args: list[str]) -> tuple[Args, Checks]: _add_subparser_for_driver, component, subparsers, with_batch=True, with_cycle=True ) for component in [ - STR.chgrescube, STR.fv3, STR.ioda, STR.jedi, @@ -1217,6 +1224,7 @@ def _parse_args(raw_args: list[str]) -> tuple[Args, Checks]: with_leadtime=True, ) for component in [ + STR.chgrescube, STR.upp, ] } diff --git a/src/uwtools/config/formats/base.py b/src/uwtools/config/formats/base.py index c93209f07..436a3ea98 100644 --- a/src/uwtools/config/formats/base.py +++ b/src/uwtools/config/formats/base.py @@ -1,8 +1,11 @@ +import difflib import os import re from abc import ABC, abstractmethod from collections import UserDict from copy import deepcopy +from io import StringIO +from math import inf from pathlib import Path from typing import Optional, Union @@ -11,7 +14,7 @@ from uwtools.config import jinja2 from uwtools.config.support import INCLUDE_TAG, depth, log_and_error, yaml_to_str from uwtools.exceptions import UWConfigError -from uwtools.logging import INDENT, log +from uwtools.logging import INDENT, MSGWIDTH, log from uwtools.utils.file import str2path @@ -76,6 +79,26 @@ def _characterize_values(self, values: dict, parent: str) -> tuple[list, list]: complete.append(f"{INDENT}{parent}{key}") return complete, template + @staticmethod + def _compare_config_get_lines(d: dict) -> list[str]: + """ + Returns a line-by-line YAML representation of the given dict. + + :param d: A dict object. + """ + sio = StringIO() + yaml.safe_dump(d, stream=sio, default_flow_style=False, indent=2, width=inf) + return sio.getvalue().splitlines(keepends=True) + + @staticmethod + def _compare_config_log_header() -> None: + """ + Log a visual header and description of diff markers. + """ + log.info("-" * MSGWIDTH) + log.info("↓ ? = info | -/+ = line unique to - or + file | blank = matching line") + log.info("-" * MSGWIDTH) + @property def _depth(self) -> int: """ @@ -158,7 +181,15 @@ def _parse_include(self, ref_dict: Optional[dict] = None) -> None: # Public methods - def compare_config(self, dict1: dict, dict2: Optional[dict] = None) -> bool: + @abstractmethod + def as_dict(self) -> dict: + """ + Returns a pure dict version of the config. + """ + + def compare_config( + self, dict1: dict, dict2: Optional[dict] = None, header: Optional[bool] = True + ) -> bool: """ Compare two config dictionaries. @@ -168,33 +199,16 @@ def compare_config(self, dict1: dict, dict2: Optional[dict] = None) -> bool: :param dict2: The second dictionary (default: this config). :return: True if the configs are identical, False otherwise. """ - dict2 = self.data if dict2 is None else dict2 - diffs: dict = {} - - for sect, items in dict2.items(): - for key, val in items.items(): - if val != dict1.get(sect, {}).get(key, ""): - try: - diffs[sect][key] = f" - {val} + {dict1.get(sect, {}).get(key)}" - except KeyError: - diffs[sect] = {} - diffs[sect][key] = f" - {val} + {dict1.get(sect, {}).get(key)}" - - for sect, items in dict1.items(): - for key, val in items.items(): - if val != dict2.get(sect, {}).get(key, "") and diffs.get(sect, {}).get(key) is None: - try: - diffs[sect][key] = f" - {dict2.get(sect, {}).get(key)} + {val}" - except KeyError: - diffs[sect] = {} - diffs[sect][key] = f" - {dict2.get(sect, {}).get(key)} + {val}" - - for sect, keys in diffs.items(): - for key in keys: - msg = f"{sect}: {key:>15}: {keys[key]}" - log.info(msg) - - return not diffs + dict2 = self.as_dict() if dict2 is None else dict2 + lines1, lines2 = map(self._compare_config_get_lines, [dict1, dict2]) + difflines = list(difflib.ndiff(lines2, lines1)) + if all(line[0] == " " for line in difflines): # i.e. no +/-/? lines + return True + if header: + self._compare_config_log_header() + for diffline in difflines: + log.info(diffline.rstrip()) + return False def dereference(self, context: Optional[dict] = None) -> None: """ diff --git a/src/uwtools/config/formats/fieldtable.py b/src/uwtools/config/formats/fieldtable.py index 3e1ffd184..10686dc2c 100644 --- a/src/uwtools/config/formats/fieldtable.py +++ b/src/uwtools/config/formats/fieldtable.py @@ -52,6 +52,12 @@ def _get_format() -> str: # Public methods + def as_dict(self) -> dict: + """ + Returns a pure dict version of the config. + """ + return self.data + def dump(self, path: Optional[Path] = None) -> None: """ Dump the config in Field Table format. diff --git a/src/uwtools/config/formats/ini.py b/src/uwtools/config/formats/ini.py index 01ac4ef54..a4577c21a 100644 --- a/src/uwtools/config/formats/ini.py +++ b/src/uwtools/config/formats/ini.py @@ -72,6 +72,12 @@ def _load(self, config_file: Optional[Path]) -> dict: # Public methods + def as_dict(self) -> dict: + """ + Returns a pure dict version of the config. + """ + return self.data + def dump(self, path: Optional[Path] = None) -> None: """ Dump the config in INI format. diff --git a/src/uwtools/config/formats/nml.py b/src/uwtools/config/formats/nml.py index 93995d136..e7903b9ab 100644 --- a/src/uwtools/config/formats/nml.py +++ b/src/uwtools/config/formats/nml.py @@ -7,6 +7,7 @@ from f90nml import Namelist from uwtools.config.formats.base import Config +from uwtools.config.support import from_od from uwtools.config.tools import config_check_depths_dump from uwtools.strings import FORMAT from uwtools.utils.file import readable, writable @@ -76,6 +77,13 @@ def _load(self, config_file: Optional[Path]) -> dict: # Public methods + def as_dict(self) -> dict: + """ + Returns a pure dict version of the config. + """ + d = self.data + return from_od(d.todict()) if isinstance(d, Namelist) else d + def dump(self, path: Optional[Path]) -> None: """ Dump the config in Fortran namelist format. diff --git a/src/uwtools/config/formats/sh.py b/src/uwtools/config/formats/sh.py index 290b8f4e2..a25daef96 100644 --- a/src/uwtools/config/formats/sh.py +++ b/src/uwtools/config/formats/sh.py @@ -73,6 +73,12 @@ def _load(self, config_file: Optional[Path]) -> dict: # Public methods + def as_dict(self) -> dict: + """ + Returns a pure dict version of the config. + """ + return self.data + def dump(self, path: Optional[Path]) -> None: """ Dump the config as key=value lines. diff --git a/src/uwtools/config/formats/yaml.py b/src/uwtools/config/formats/yaml.py index c23ef6200..99c5b32a6 100644 --- a/src/uwtools/config/formats/yaml.py +++ b/src/uwtools/config/formats/yaml.py @@ -168,6 +168,12 @@ def _yaml_loader(self) -> type[yaml.SafeLoader]: # Public methods + def as_dict(self) -> dict: + """ + Returns a pure dict version of the config. + """ + return self.data + def dump(self, path: Optional[Path] = None) -> None: """ Dump the config in YAML format. diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 58683a68a..8115e248b 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -3,6 +3,7 @@ """ import os +from datetime import datetime from functools import cached_property from pathlib import Path from typing import Optional, Union @@ -14,7 +15,7 @@ from uwtools.logging import INDENT, MSGWIDTH, log from uwtools.utils.file import get_file_format, readable, writable -_ConfigVal = Union[bool, dict, float, int, list, str, UWYAMLConvert, UWYAMLRemove] +_ConfigVal = Union[bool, datetime, dict, float, int, list, str, UWYAMLConvert, UWYAMLRemove] class J2Template: @@ -109,7 +110,7 @@ def dereference( values; render strings; convert values tagged with explicit types; and return objects of other types unmodified. Rendering may fail for valid reasons -- notably a replacement value not being available in the given context object. In such cases, return the original value: Any unrendered - Jinja2 syntax it contains may may be rendered by later processing with better context. + Jinja2 syntax it contains may be rendered by later processing with better context. When rendering dict values, replacement values will be taken from, in priority order 1. The full context dict @@ -334,7 +335,7 @@ def _supplement_values( env: bool = False, ) -> dict: """ - Optionally supplement values from given source with overrides and/or environment vairables. + Optionally supplement values from given source with overrides and/or environment variables. :param values_src: Source of values to render the template. :param values_format: Format of values when sourced from file. diff --git a/src/uwtools/config/support.py b/src/uwtools/config/support.py index 96b433862..c11737a4f 100644 --- a/src/uwtools/config/support.py +++ b/src/uwtools/config/support.py @@ -2,8 +2,9 @@ import math from collections import OrderedDict +from datetime import datetime from importlib import import_module -from typing import Type, Union +from typing import Callable, Type, Union import yaml @@ -11,7 +12,7 @@ from uwtools.logging import log from uwtools.strings import FORMAT -INCLUDE_TAG = "!INCLUDE" +INCLUDE_TAG = "!include" # Public functions @@ -107,15 +108,17 @@ class UWYAMLConvert(UWYAMLTag): method. See the pyyaml documentation for details. """ - TAGS = ("!float", "!int") + TAGS = ("!datetime", "!float", "!int") - def convert(self) -> Union[float, int]: + def convert(self) -> Union[datetime, float, int]: """ Return the original YAML value converted to the specified type. Will raise an exception if the value cannot be represented as the specified type. """ - converters: dict[str, Union[type[float], type[int]]] = dict(zip(self.TAGS, [float, int])) + converters: dict[str, Union[Callable[[str], datetime], type[float], type[int]]] = dict( + zip(self.TAGS, [datetime.fromisoformat, float, int]) + ) return converters[self.tag](self.value) diff --git a/src/uwtools/config/tools.py b/src/uwtools/config/tools.py index 2d99e651d..e92e84c6d 100644 --- a/src/uwtools/config/tools.py +++ b/src/uwtools/config/tools.py @@ -9,7 +9,7 @@ from uwtools.config.jinja2 import unrendered from uwtools.config.support import depth, format_to_config, log_and_error from uwtools.exceptions import UWConfigError, UWConfigRealizeError, UWError -from uwtools.logging import MSGWIDTH, log +from uwtools.logging import log from uwtools.strings import FORMAT from uwtools.utils.file import get_file_format @@ -34,8 +34,7 @@ def compare_configs( cfg_2: Config = format_to_config(config_2_format)(config_2_path) log.info("- %s", config_1_path) log.info("+ %s", config_2_path) - log.info("-" * MSGWIDTH) - return cfg_1.compare_config(cfg_2.data) + return cfg_1.compare_config(cfg_2.as_dict()) def config_check_depths_dump(config_obj: Union[Config, dict], target_format: str) -> None: diff --git a/src/uwtools/config/validator.py b/src/uwtools/config/validator.py index 42b723ed6..8eccec18a 100644 --- a/src/uwtools/config/validator.py +++ b/src/uwtools/config/validator.py @@ -20,48 +20,56 @@ # Public functions -def bundle(schema: dict) -> dict: +def bundle(schema: dict, keys: Optional[list] = None) -> dict: """ Bundle a schema by dereferencing links to other schemas. :param schema: A JSON Schema. + :param keys: Keys leading up to this block. Internal use only, do not manually specify. :returns: The bundled schema. """ - key = "$ref" + ref = "$ref" bundled = {} for k, v in schema.items(): + newkeys = [*(keys or []), k] + key_path = ".".join(newkeys) if isinstance(v, dict): - if list(v.keys()) == [key] and v[key].startswith("urn:uwtools:"): + if list(v.keys()) == [ref] and v[ref].startswith("urn:uwtools:"): # i.e. the current key's value is of the form: {"$ref": "urn:uwtools:.*"} - bundled[k] = bundle(_registry().get_or_retrieve(v[key]).value.contents) + uri = v[ref] + log.debug("Bundling referenced schema %s at key path: %s", uri, key_path) + bundled[k] = bundle(_registry().get_or_retrieve(uri).value.contents, newkeys) else: - bundled[k] = bundle(v) + log.debug("Bundling dict value at key path: %s", key_path) + bundled[k] = bundle(v, newkeys) else: + log.debug("Bundling %s value at key path: %s", type(v).__name__, key_path) bundled[k] = v return bundled -def get_schema_file(schema_name: str) -> Path: +def internal_schema_file(schema_name: str) -> Path: """ - Return the path to the JSON Schema file for a given name. + Return the path to the internal JSON Schema file for a given driver name. :param schema_name: Name of uwtools schema to validate the config against. """ return resource_path("jsonschema") / f"{schema_name}.jsonschema" -def validate(schema: dict, config: dict) -> bool: +def validate(schema: dict, desc: str, config: dict) -> bool: """ Report any errors arising from validation of the given config against the given JSON Schema. :param schema: The JSON Schema to use for validation. + :param desc: A description of the config being validated, for logging. :param config: The config to validate. :return: Did the YAML file conform to the schema? """ errors = _validation_errors(config, schema) log_method = log.error if errors else log.info - log_msg = "%s UW schema-validation error%s found" - log_method(log_msg, len(errors), "" if len(errors) == 1 else "s") + log_msg = "%s UW schema-validation error%s found in %s" + log_method(log_msg, len(errors), "" if len(errors) == 1 else "s", desc) for error in errors: log.error("Error at %s:", " -> ".join(str(k) for k in error.path)) log.error("%s%s", INDENT, error.message) @@ -69,36 +77,37 @@ def validate(schema: dict, config: dict) -> bool: def validate_internal( - schema_name: str, config: Optional[Union[dict, YAMLConfig, Path]] = None + schema_name: str, desc: str, config: Optional[Union[dict, YAMLConfig, Path]] = None ) -> None: """ Validate a config against a uwtools-internal schema. :param schema_name: Name of uwtools schema to validate the config against. + :param desc: A description of the config being validated, for logging. :param config: The config to validate. :raises: UWConfigError if config fails validation. """ - log.info("Validating config against internal schema: %s", schema_name) - schema_file = get_schema_file(schema_name) - log.debug("Using schema file: %s", schema_file) - validate_external(config=config, schema_file=schema_file) + validate_external(config=config, schema_file=internal_schema_file(schema_name), desc=desc) def validate_external( - schema_file: Path, config: Optional[Union[dict, YAMLConfig, Path]] = None + schema_file: Path, desc: str, config: Optional[Union[dict, YAMLConfig, Path]] = None ) -> None: """ Validate a YAML config against the JSON Schema in the given schema file. :param schema_file: The JSON Schema file to use for validation. + :param desc: A description of the config being validated, for logging. :param config: The config to validate. :raises: UWConfigError if config fails validation. """ + if not str(schema_file).startswith(str(resource_path())): + log.debug("Using schema file: %s", schema_file) with open(schema_file, "r", encoding="utf-8") as f: schema = json.load(f) cfgobj = _prep_config(config) - if not validate(schema=schema, config=cfgobj.data): + if not validate(schema=schema, desc=desc, config=cfgobj.data): raise UWConfigError("YAML validation errors") diff --git a/src/uwtools/drivers/chgres_cube.py b/src/uwtools/drivers/chgres_cube.py index 669860b07..2a0acd0e4 100644 --- a/src/uwtools/drivers/chgres_cube.py +++ b/src/uwtools/drivers/chgres_cube.py @@ -7,13 +7,13 @@ from iotaa import asset, task, tasks from uwtools.config.formats.nml import NMLConfig -from uwtools.drivers.driver import DriverCycleBased +from uwtools.drivers.driver import DriverCycleLeadtimeBased from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file -class ChgresCube(DriverCycleBased): +class ChgresCube(DriverCycleLeadtimeBased): """ A driver for chgres_cube. """ @@ -35,7 +35,7 @@ def namelist_file(self): input_files.append(base_file) if update_values := namelist.get(STR.updatevalues): config_files = update_values["config"] - for k in ["mosaic_file_target_grid", "vcoord_file_target_grid"]: + for k in ["mosaic_file_target_grid", "varmap_file", "vcoord_file_target_grid"]: input_files.append(config_files[k]) for k in [ "atm_core_files_input_grid", @@ -43,23 +43,32 @@ def namelist_file(self): "atm_tracer_files_input_grid", "grib2_file_input_grid", "nst_files_input_grid", + "sfc_files_input_grid", + ]: + if k in config_files: + grid_path = Path(config_files["data_dir_input_grid"]) + v = config_files[k] + if isinstance(v, str): + input_files.append(grid_path / v) + else: + input_files.extend([grid_path / f for f in v]) + for k in [ "orog_files_input_grid", "orog_files_target_grid", - "sfc_files_input_grid", - "varmap_file", ]: if k in config_files: + grid_path = Path(config_files[k.replace("files", "dir")]) v = config_files[k] if isinstance(v, str): - input_files.append(v) + input_files.append(grid_path / v) else: - input_files += v + input_files.extend([grid_path / f for f in v]) yield [file(Path(input_file)) for input_file in input_files] self._create_user_updated_config( config_class=NMLConfig, config_values=namelist, path=path, - schema=self._namelist_schema(), + schema=self.namelist_schema(), ) @tasks diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 1a60a6d14..878e44e8b 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -21,12 +21,12 @@ from uwtools.config.tools import walk_key_path from uwtools.config.validator import ( bundle, - get_schema_file, + internal_schema_file, validate, validate_external, validate_internal, ) -from uwtools.exceptions import UWConfigError +from uwtools.exceptions import UWConfigError, UWNotImplementedError from uwtools.logging import log from uwtools.scheduler import JobScheduler from uwtools.strings import STR @@ -49,7 +49,7 @@ def __init__( dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, - controller: Optional[str] = None, + controller: Optional[list[str]] = None, ) -> None: config_input = config if isinstance(config, YAMLConfig) else YAMLConfig(config=config) config_input.dereference( @@ -65,9 +65,9 @@ def __init__( self._config: dict = self._config_intermediate[self.driver_name()] except KeyError as e: raise UWConfigError("Required '%s' block missing in config" % self.driver_name()) from e - if controller: - self._config[STR.rundir] = self._config_intermediate[controller][STR.rundir] - self._validate(schema_file) + self._delegate(controller, STR.rundir) + self.schema_file = schema_file + self._validate() dryrun(enable=dry_run) def __repr__(self) -> str: @@ -106,14 +106,14 @@ def rundir(self) -> Path: @classmethod def schema(cls) -> dict: """ - Return the driver's schema. + Return the driver's internal schema. """ - with open(get_schema_file(schema_name=cls._schema_name()), "r", encoding="utf-8") as f: + with open(internal_schema_file(schema_name=cls._schema_name()), "r", encoding="utf-8") as f: return bundle(json.load(f)) def taskname(self, suffix: str) -> str: """ - Return a common tag for graph-task log messages. + Return a common tag for task-related log messages. :param suffix: Log-string suffix. """ @@ -160,12 +160,25 @@ def _create_user_updated_config( else: config = user_values dump = partial(config_class.dump_dict, config, path) - if validate(schema=schema or {"type": "object"}, config=config): + if validate(schema=schema or {"type": "object"}, desc="user-updated config", config=config): dump() log.debug(f"Wrote config to {path}") else: log.debug(f"Failed to validate {path}") + def _delegate(self, controller: Optional[list[str]], config_key: str) -> None: + """ + Selectively delegate config to controller. + + :param controller: Key(s) leading to block in config controlling run-time values. + :param config_key: Name of config item to delegate to controller. + """ + if controller: + val = self._config_intermediate[controller[0]] + for key in controller[1:]: + val = val[key] + self._config[config_key] = val[config_key] + # Public helper methods @classmethod @@ -175,9 +188,7 @@ def driver_name(cls) -> str: The name of this driver. """ - # Private helper methods - - def _namelist_schema( + def namelist_schema( self, config_keys: Optional[list[str]] = None, schema_keys: Optional[list[str]] = None ) -> dict: """ @@ -191,7 +202,7 @@ def _namelist_schema( for config_key in config_keys or [STR.namelist]: nmlcfg = nmlcfg[config_key] if nmlcfg.get(STR.validate, True): - schema_file = get_schema_file(schema_name=self._schema_name()) + schema_file = self.schema_file or internal_schema_file(schema_name=self._schema_name()) with open(schema_file, "r", encoding="utf-8") as f: schema = json.load(f) for schema_key in schema_keys or [ @@ -205,6 +216,8 @@ def _namelist_schema( schema = schema[schema_key] return schema + # Private helper methods + @classmethod def _schema_name(cls) -> str: """ @@ -212,17 +225,20 @@ def _schema_name(cls) -> str: """ return cls.driver_name().replace("_", "-") - def _validate(self, schema_file: Optional[Path] = None) -> None: + def _validate(self) -> None: """ Perform all necessary schema validation. - :param schema_file: The JSON Schema file to use for validation. :raises: UWConfigError if config fails validation. """ - if schema_file: - validate_external(schema_file=schema_file, config=self._config_intermediate) + kwargs: dict = { + "config": self._config_intermediate, + "desc": "%s config" % self.driver_name(), + } + if self.schema_file: + validate_external(schema_file=self.schema_file, **kwargs) else: - validate_internal(schema_name=self._schema_name(), config=self._config_intermediate) + validate_internal(schema_name=self._schema_name(), **kwargs) class AssetsCycleBased(Assets): @@ -237,7 +253,7 @@ def __init__( dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, - controller: Optional[str] = None, + controller: Optional[list[str]] = None, ): super().__init__( cycle=cycle, @@ -270,7 +286,7 @@ def __init__( dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, - controller: Optional[str] = None, + controller: Optional[list[str]] = None, ): super().__init__( cycle=cycle, @@ -310,7 +326,7 @@ def __init__( dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, - controller: Optional[str] = None, + controller: Optional[list[str]] = None, ): super().__init__( config=config, @@ -335,7 +351,7 @@ def __init__( key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, - controller: Optional[str] = None, + controller: Optional[list[str]] = None, ): super().__init__( cycle=cycle, @@ -347,8 +363,7 @@ def __init__( controller=controller, ) self._batch = batch - if controller: - self._config[STR.execution] = self.config_full[controller][STR.execution] + self._delegate(controller, STR.execution) # Workflow tasks @@ -378,6 +393,18 @@ def runscript(self): yield None self._write_runscript(path) + @external + def show_output(self): + """ + Show the output to be created by this component. + """ + yield self.taskname("expected output") + try: + print(json.dumps(self.output, indent=2, sort_keys=True)) + except UWConfigError as e: + log.error(e) + yield asset(None, lambda: True) + @task def _run_via_batch_submission(self): """ @@ -401,7 +428,16 @@ def _run_via_local_execution(self): cmd = "{x} >{x}.out 2>&1".format(x=self._runscript_path) run_shell_cmd(cmd=cmd, cwd=self.rundir, log_output=True) - # Private helper methods + # Public methods + + @property + def output(self) -> Union[dict[str, str], dict[str, list[str]]]: + """ + Returns a description of the file(s) created when this component runs. + """ + raise UWNotImplementedError("The output() method is not yet implemented for this driver") + + # Private methods @property def _run_resources(self) -> dict[str, Any]: @@ -491,15 +527,16 @@ def _scheduler(self) -> JobScheduler: """ return JobScheduler.get_scheduler(self._run_resources) - def _validate(self, schema_file: Optional[Path] = None) -> None: + def _validate(self) -> None: """ Perform all necessary schema validation. - :param schema_file: The JSON Schema file to use for validation. :raises: UWConfigError if config fails validation. """ - Assets._validate(self, schema_file) - validate_internal(schema_name=STR.platform, config=self._config_intermediate) + Assets._validate(self) + validate_internal( + schema_name=STR.platform, desc="platform config", config=self._config_intermediate + ) def _write_runscript(self, path: Path, envvars: Optional[dict[str, str]] = None) -> None: """ @@ -536,7 +573,7 @@ def __init__( key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, - controller: Optional[str] = None, + controller: Optional[list[str]] = None, ): super().__init__( cycle=cycle, @@ -571,7 +608,7 @@ def __init__( key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, - controller: Optional[str] = None, + controller: Optional[list[str]] = None, ): super().__init__( cycle=cycle, @@ -613,7 +650,7 @@ def __init__( key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, - controller: Optional[str] = None, + controller: Optional[list[str]] = None, ): super().__init__( config=config, @@ -645,7 +682,7 @@ def _add_docstring(class_: type, omit: Optional[list[str]] = None) -> None: :param key_path: Keys leading through the config to the driver's configuration block. :param batch: Run component via the batch system? :param schema_file: Path to schema file to use to validate an external driver. - :param controller: Name of block in config controlling run-time values. + :param controller: Key(s) leading to block in config controlling run-time values. """ setattr( class_, diff --git a/src/uwtools/drivers/esg_grid.py b/src/uwtools/drivers/esg_grid.py index 0d77a7da8..bcc2894b4 100644 --- a/src/uwtools/drivers/esg_grid.py +++ b/src/uwtools/drivers/esg_grid.py @@ -35,7 +35,7 @@ def namelist_file(self): config_class=NMLConfig, config_values=self.config[STR.namelist], path=path, - schema=self._namelist_schema(schema_keys=["$defs", "namelist_content"]), + schema=self.namelist_schema(schema_keys=["$defs", "namelist_content"]), ) @tasks diff --git a/src/uwtools/drivers/filter_topo.py b/src/uwtools/drivers/filter_topo.py index 3cb396a5d..849942034 100644 --- a/src/uwtools/drivers/filter_topo.py +++ b/src/uwtools/drivers/filter_topo.py @@ -10,7 +10,7 @@ from uwtools.drivers.driver import DriverTimeInvariant from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR -from uwtools.utils.tasks import symlink +from uwtools.utils.tasks import filecopy, symlink class FilterTopo(DriverTimeInvariant): @@ -27,10 +27,21 @@ def input_grid_file(self): """ src = Path(self.config["config"]["input_grid_file"]) dst = Path(self.config[STR.rundir], src.name) - yield self.taskname("Input grid") + yield self.taskname(f"Input grid {str(src)}") yield asset(dst, dst.is_file) yield symlink(target=src, linkname=dst) + @task + def filtered_output_file(self): + """ + The filtered output file staged from raw input. + """ + src = Path(self.config["config"]["input_raw_orog"]) + dst = self.rundir / self.config["config"]["filtered_orog"] + yield self.taskname(f"Raw orog input {str(dst)}") + yield asset(dst, dst.is_file) + yield filecopy(src=src, dst=dst) + @task def namelist_file(self): """ @@ -45,7 +56,7 @@ def namelist_file(self): config_class=NMLConfig, config_values=self.config[STR.namelist], path=path, - schema=self._namelist_schema(), + schema=self.namelist_schema(), ) @tasks @@ -56,6 +67,7 @@ def provisioned_rundir(self): yield self.taskname("provisioned run directory") yield [ self.input_grid_file(), + self.filtered_output_file(), self.namelist_file(), self.runscript(), ] diff --git a/src/uwtools/drivers/fv3.py b/src/uwtools/drivers/fv3.py index e417f58d1..e19843aa9 100644 --- a/src/uwtools/drivers/fv3.py +++ b/src/uwtools/drivers/fv3.py @@ -124,7 +124,7 @@ def namelist_file(self): config_class=NMLConfig, config_values=self.config[STR.namelist], path=path, - schema=self._namelist_schema(), + schema=self.namelist_schema(), ) @tasks diff --git a/src/uwtools/drivers/mpas.py b/src/uwtools/drivers/mpas.py index c1953f822..96005a78f 100644 --- a/src/uwtools/drivers/mpas.py +++ b/src/uwtools/drivers/mpas.py @@ -63,7 +63,7 @@ def namelist_file(self): config_class=NMLConfig, config_values=namelist, path=path, - schema=self._namelist_schema(), + schema=self.namelist_schema(), ) # Public helper methods diff --git a/src/uwtools/drivers/mpas_init.py b/src/uwtools/drivers/mpas_init.py index 47d954fbd..be4ec4320 100644 --- a/src/uwtools/drivers/mpas_init.py +++ b/src/uwtools/drivers/mpas_init.py @@ -65,7 +65,7 @@ def namelist_file(self): config_class=NMLConfig, config_values=namelist, path=path, - schema=self._namelist_schema(), + schema=self.namelist_schema(), ) # Public helper methods diff --git a/src/uwtools/drivers/orog.py b/src/uwtools/drivers/orog.py new file mode 100644 index 000000000..6a1d58d96 --- /dev/null +++ b/src/uwtools/drivers/orog.py @@ -0,0 +1,129 @@ +""" +A driver for UFS_UTILS's orog. +""" + +from pathlib import Path + +from iotaa import asset, external, task, tasks + +from uwtools.drivers.driver import DriverTimeInvariant +from uwtools.drivers.support import set_driver_docstring +from uwtools.strings import STR +from uwtools.utils.file import writable +from uwtools.utils.tasks import symlink + + +class Orog(DriverTimeInvariant): + """ + A driver for orog. + """ + + # Workflow tasks + + @tasks + def files_linked(self): + """ + Files linked for run. + """ + yield self.taskname("files linked") + yield [ + symlink(target=Path(target), linkname=self.rundir / linkname) + for linkname, target in self.config.get("files_to_link", {}).items() + ] + + @external + def grid_file(self): + """ + The input grid file. + """ + grid_file = Path(self.config["grid_file"]) + yield self.taskname(f"Input grid file {grid_file}") + yield asset(grid_file, grid_file.is_file) if str(grid_file) != "none" else None + + @task + def input_config_file(self): + """ + The input config file. + """ + path = self._input_config_path + yield self.taskname(str(path)) + yield asset(path, path.is_file) + yield self.grid_file() + if inputs := self.config.get("old_line1_items"): + ordered_entries = [ + "mtnres", + "lonb", + "latb", + "jcap", + "nr", + "nf1", + "nf2", + "efac", + "blat", + ] + inputs = " ".join([str(inputs[i]) for i in ordered_entries]) + outgrid = "'{}'".format(self.config["grid_file"]) + if orogfile := self.config.get("orog_file"): + orogfile = "'{}'".format(orogfile) + mask_only = ".true." if self.config.get("mask") else ".false." + merge_file = self.config.get("merge", "none") # string none is intentional + content = [i for i in [inputs, outgrid, orogfile, mask_only, merge_file] if i is not None] + with writable(path) as f: + print("\n".join(content), file=f) + + @tasks + def provisioned_rundir(self): + """ + Run directory provisioned with all required content. + """ + yield self.taskname("provisioned run directory") + yield [ + self.files_linked(), + self.input_config_file(), + self.runscript(), + ] + + @task + def runscript(self): + """ + The runscript. + """ + path = self._runscript_path + yield self.taskname(path.name) + yield asset(path, path.is_file) + yield None + envvars = { + "KMP_AFFINITY": "disabled", + "OMP_NUM_THREADS": self.config.get(STR.execution, {}).get(STR.threads, 1), + "OMP_STACKSIZE": "2048m", + } + self._write_runscript(path=path, envvars=envvars) + + # Public helper methods + + @classmethod + def driver_name(cls) -> str: + """ + The name of this driver. + """ + return STR.orog + + # Private helper methods + + @property + def _input_config_path(self) -> Path: + """ + Path to the input config file. + """ + return self.rundir / "orog.cfg" + + @property + def _runcmd(self): + """ + The full command-line component invocation. + """ + executable = self.config[STR.execution][STR.executable] + return "%s < %s" % (executable, self._input_config_path.name) + + +set_driver_docstring(Orog) diff --git a/src/uwtools/drivers/orog_gsl.py b/src/uwtools/drivers/orog_gsl.py index bfde683e7..8bf41c19a 100644 --- a/src/uwtools/drivers/orog_gsl.py +++ b/src/uwtools/drivers/orog_gsl.py @@ -9,6 +9,7 @@ from uwtools.drivers.driver import DriverTimeInvariant from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR +from uwtools.utils.file import writable from uwtools.utils.tasks import symlink @@ -19,6 +20,19 @@ class OrogGSL(DriverTimeInvariant): # Workflow tasks + @task + def input_config_file(self): + """ + The input config file. + """ + path = self._input_config_path + yield self.taskname(str(path)) + yield asset(path, path.is_file) + yield None + inputs = [str(self.config["config"][k]) for k in ("tile", "resolution", "halo")] + with writable(path) as f: + print("\n".join(inputs), file=f) + @task def input_grid_file(self): """ @@ -28,7 +42,7 @@ def input_grid_file(self): self.config["config"][k] for k in ["resolution", "tile", "halo"] ) src = Path(self.config["config"]["input_grid_file"]) - dst = Path(self.config[STR.rundir], fn) + dst = self.rundir / fn yield self.taskname("Input grid") yield asset(dst, dst.is_file) yield symlink(target=src, linkname=dst) @@ -40,6 +54,7 @@ def provisioned_rundir(self): """ yield self.taskname("provisioned run directory") yield [ + self.input_config_file(), self.input_grid_file(), self.runscript(), self.topo_data_2p5m(), @@ -53,7 +68,7 @@ def topo_data_2p5m(self): """ fn = "geo_em.d01.lat-lon.2.5m.HGT_M.nc" src = Path(self.config["config"]["topo_data_2p5m"]) - dst = Path(self.config[STR.rundir], fn) + dst = self.rundir / fn yield self.taskname("Input grid") yield asset(dst, dst.is_file) yield symlink(target=src, linkname=dst) @@ -65,7 +80,7 @@ def topo_data_30s(self): """ fn = "HGT.Beljaars_filtered.lat-lon.30s_res.nc" src = Path(self.config["config"]["topo_data_30s"]) - dst = Path(self.config[STR.rundir], fn) + dst = self.rundir / fn yield self.taskname("Input grid") yield asset(dst, dst.is_file) yield symlink(target=src, linkname=dst) @@ -81,14 +96,20 @@ def driver_name(cls) -> str: # Private helper methods + @property + def _input_config_path(self) -> Path: + """ + Path to the input config file. + """ + return self.rundir / "orog_gsl.cfg" + @property def _runcmd(self): """ The full command-line component invocation. """ - inputs = [str(self.config["config"][k]) for k in ("tile", "resolution", "halo")] executable = self.config[STR.execution][STR.executable] - return "echo '%s' | %s" % ("\n".join(inputs), executable) + return "%s < %s" % (executable, self._input_config_path.name) set_driver_docstring(OrogGSL) diff --git a/src/uwtools/drivers/sfc_climo_gen.py b/src/uwtools/drivers/sfc_climo_gen.py index 035ccc07f..de78a50d8 100644 --- a/src/uwtools/drivers/sfc_climo_gen.py +++ b/src/uwtools/drivers/sfc_climo_gen.py @@ -38,7 +38,7 @@ def namelist_file(self): config_class=NMLConfig, config_values=self.config[STR.namelist], path=path, - schema=self._namelist_schema(), + schema=self.namelist_schema(), ) @tasks diff --git a/src/uwtools/drivers/shave.py b/src/uwtools/drivers/shave.py index 868d38982..1f809e714 100644 --- a/src/uwtools/drivers/shave.py +++ b/src/uwtools/drivers/shave.py @@ -2,11 +2,15 @@ A driver for shave. """ -from iotaa import tasks +from pathlib import Path + +from iotaa import asset, task, tasks from uwtools.drivers.driver import DriverTimeInvariant from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR +from uwtools.utils.file import writable +from uwtools.utils.tasks import file class Shave(DriverTimeInvariant): @@ -16,13 +20,34 @@ class Shave(DriverTimeInvariant): # Workflow tasks + @task + def input_config_file(self): + """ + The input config file. + """ + path = self._input_config_path + yield self.taskname(str(path)) + yield asset(path, path.is_file) + config = self.config["config"] + input_file = Path(config["input_grid_file"]) + yield file(path=input_file) + flags = [ + config[key] for key in ["nx", "ny", "nhalo", "input_grid_file", "output_grid_file"] + ] + content = "{} {} {} '{}' '{}'".format(*flags) + with writable(path) as f: + print(content, file=f) + @tasks def provisioned_rundir(self): """ Run directory provisioned with all required content. """ yield self.taskname("provisioned run directory") - yield self.runscript() + yield [ + self.input_config_file(), + self.runscript(), + ] # Public helper methods @@ -35,18 +60,20 @@ def driver_name(cls) -> str: # Private helper methods + @property + def _input_config_path(self) -> Path: + """ + Path to the input config file. + """ + return self.rundir / "shave.cfg" + @property def _runcmd(self): """ The full command-line component invocation. """ executable = self.config[STR.execution][STR.executable] - config = self.config["config"] - input_file = config["input_grid_file"] - output_file = input_file.replace(".nc", "_NH0.nc") - flags = [config[key] for key in ["nx", "ny", "nh4", "input_grid_file"]] - flags.append(output_file) - return f"{executable} {' '.join(str(flag) for flag in flags)}" + return "%s < %s" % (executable, self._input_config_path.name) set_driver_docstring(Shave) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index eed851355..b57ece118 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -9,6 +9,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import DriverCycleLeadtimeBased from uwtools.drivers.support import set_driver_docstring +from uwtools.exceptions import UWConfigError from uwtools.strings import STR from uwtools.utils.tasks import file, filecopy, symlink @@ -18,8 +19,24 @@ class UPP(DriverCycleLeadtimeBased): A driver for UPP. """ + # Facts specific to the supported UPP version: + + GENPROCTYPE_IDX = 8 + NFIELDS = 16 + NPARAMS = 42 + # Workflow tasks + @tasks + def control_file(self): + """ + The GRIB control file. + """ + yield self.taskname("GRIB control file") + yield filecopy( + src=Path(self.config["control_file"]), dst=self.rundir / "postxconfig-NT.txt" + ) + @tasks def files_copied(self): """ @@ -56,7 +73,7 @@ def namelist_file(self): config_class=NMLConfig, config_values=self.config[STR.namelist], path=path, - schema=self._namelist_schema(), + schema=self.namelist_schema(), ) @tasks @@ -66,6 +83,7 @@ def provisioned_rundir(self): """ yield self.taskname("provisioned run directory") yield [ + self.control_file(), self.files_copied(), self.files_linked(), self.namelist_file(), @@ -81,6 +99,36 @@ def driver_name(cls) -> str: """ return STR.upp + @property + def output(self) -> dict[str, list[str]]: + """ + Returns a description of the file(s) created when this component runs. + """ + # Read the control file into an array of lines. Get the number of blocks (one per output + # GRIB file) and the number of variables per block. For each block, construct a filename + # from the block's identifier and the suffix defined above. + cf = self.config["control_file"] + try: + with open(cf, "r", encoding="utf-8") as f: + lines = f.read().split("\n") + except (FileNotFoundError, PermissionError) as e: + raise UWConfigError(f"Could not open UPP control file {cf}") from e + suffix = ".GrbF%02d" % int(self.leadtime.total_seconds() / 3600) + nblocks, lines = int(lines[0]), lines[1:] + nvars, lines = list(map(int, lines[:nblocks])), lines[nblocks:] + paths = [] + for _ in range(nblocks): + identifier = lines[0] + paths.append(str(self.rundir / (identifier + suffix))) + fields, lines = lines[: self.NFIELDS], lines[self.NFIELDS :] + _, lines = ( + (lines[0], lines[1:]) + if fields[self.GENPROCTYPE_IDX] == "ens_fcst" + else (None, lines) + ) + lines = lines[self.NPARAMS * nvars.pop() :] + return {"gribfiles": paths} + # Private helper methods @property diff --git a/src/uwtools/exceptions.py b/src/uwtools/exceptions.py index 616c6baf1..05aac5358 100644 --- a/src/uwtools/exceptions.py +++ b/src/uwtools/exceptions.py @@ -21,6 +21,12 @@ class UWConfigRealizeError(UWConfigError): """ +class UWNotImplementedError(UWError): + """ + Exception for signaling that uwtools has not (yet) implemented something. + """ + + class UWTemplateRenderError(UWError): """ Exception for issues arising from template rendering. diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index c5bd29b68..9f9d539c4 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -35,13 +35,13 @@ def __init__( """ Stage files and directories. - :param config: YAML-file path, or dict (read stdin if missing or None). + :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``). :param target_dir: Path to target directory. - :param cycle: A datetime object to make available for use in the config. - :param leadtime: A timedelta object to make available for use in the config. + :param cycle: A ``datetime`` object to make available for use in the config. + :param leadtime: A ``timedelta`` object to make available for use in the config. :param keys: YAML keys leading to file dst/src block. :param dry_run: Do not copy files. - :raises: UWConfigError if config fails validation. + :raises: ``UWConfigError`` if config fails validation. """ dryrun(enable=dry_run) self._keys = keys or [] @@ -87,7 +87,7 @@ def _set_config_block(self) -> None: log.debug("Following config key '%s'", key) cfg = cfg[key] if not isinstance(cfg, dict): - msg = "Expected block not found at key path: %s" % " -> ".join(self._keys) + msg = "Expected block not found at key path: %s" % ".".join(self._keys) raise UWConfigError(msg) self._config = cfg @@ -111,7 +111,7 @@ def _validate(self) -> None: :raises: UWConfigError if config fails validation. """ - validate_internal(schema_name=self._schema, config=self._config) + validate_internal(schema_name=self._schema, desc="fs config", config=self._config) class FileStager(Stager): diff --git a/src/uwtools/resources/jsonschema/chgres-cube.jsonschema b/src/uwtools/resources/jsonschema/chgres-cube.jsonschema index 15df5c93c..6a584d5ea 100644 --- a/src/uwtools/resources/jsonschema/chgres-cube.jsonschema +++ b/src/uwtools/resources/jsonschema/chgres-cube.jsonschema @@ -4,7 +4,7 @@ "additionalProperties": false, "properties": { "execution": { - "$ref": "urn:uwtools:execution" + "$ref": "urn:uwtools:execution-parallel" }, "namelist": { "additionalProperties": false, @@ -27,12 +27,31 @@ "update_values": { "properties": { "config": { - "additionalProperties": { - "type": [ - "array", - "boolean", - "number", - "string" + "additionalProperties": false, + "dependencies": { + "atm_core_files_input_grid": [ + "data_dir_input_grid" + ], + "atm_files_input_grid": [ + "data_dir_input_grid" + ], + "atm_tracer_files_input_grid": [ + "data_dir_input_grid" + ], + "grib2_file_input_grid": [ + "data_dir_input_grid" + ], + "nst_files_input_grid": [ + "data_dir_input_grid" + ], + "orog_files_input_grid": [ + "orog_dir_input_grid" + ], + "orog_files_target_grid": [ + "orog_dir_target_grid" + ], + "sfc_files_input_grid": [ + "data_dir_input_grid" ] }, "properties": { @@ -91,13 +110,6 @@ "type": "string" }, "external_model": { - "enum": [ - "GFS", - "HRRR", - "NAM", - "RAP", - "UKMET" - ], "type": "string" }, "fix_dir_target_grid": { diff --git a/src/uwtools/resources/jsonschema/execution.jsonschema b/src/uwtools/resources/jsonschema/execution-parallel.jsonschema similarity index 100% rename from src/uwtools/resources/jsonschema/execution.jsonschema rename to src/uwtools/resources/jsonschema/execution-parallel.jsonschema diff --git a/src/uwtools/resources/jsonschema/filter-topo.jsonschema b/src/uwtools/resources/jsonschema/filter-topo.jsonschema index 0f88216b3..a988a1fde 100644 --- a/src/uwtools/resources/jsonschema/filter-topo.jsonschema +++ b/src/uwtools/resources/jsonschema/filter-topo.jsonschema @@ -6,12 +6,20 @@ "config": { "additionalProperties": false, "properties": { + "filtered_orog": { + "type": "string" + }, "input_grid_file": { "type": "string" + }, + "input_raw_orog": { + "type": "string" } }, "required": [ - "input_grid_file" + "filtered_orog", + "input_grid_file", + "input_raw_orog" ] }, "execution": { diff --git a/src/uwtools/resources/jsonschema/fv3.jsonschema b/src/uwtools/resources/jsonschema/fv3.jsonschema index a1f8a759a..49fbc6125 100644 --- a/src/uwtools/resources/jsonschema/fv3.jsonschema +++ b/src/uwtools/resources/jsonschema/fv3.jsonschema @@ -30,7 +30,7 @@ "type": "string" }, "execution": { - "$ref": "urn:uwtools:execution" + "$ref": "urn:uwtools:execution-parallel" }, "field_table": { "additionalProperties": false, diff --git a/src/uwtools/resources/jsonschema/jedi.jsonschema b/src/uwtools/resources/jsonschema/jedi.jsonschema index 44364a411..6b90969b2 100644 --- a/src/uwtools/resources/jsonschema/jedi.jsonschema +++ b/src/uwtools/resources/jsonschema/jedi.jsonschema @@ -29,7 +29,7 @@ "type": "object" }, "execution": { - "$ref": "urn:uwtools:execution" + "$ref": "urn:uwtools:execution-parallel" }, "files_to_copy": { "$ref": "urn:uwtools:files-to-stage" diff --git a/src/uwtools/resources/jsonschema/mpas-init.jsonschema b/src/uwtools/resources/jsonschema/mpas-init.jsonschema index a56ecf5b2..0b20bdf61 100644 --- a/src/uwtools/resources/jsonschema/mpas-init.jsonschema +++ b/src/uwtools/resources/jsonschema/mpas-init.jsonschema @@ -31,7 +31,7 @@ "type": "object" }, "execution": { - "$ref": "urn:uwtools:execution" + "$ref": "urn:uwtools:execution-parallel" }, "files_to_copy": { "$ref": "urn:uwtools:files-to-stage" diff --git a/src/uwtools/resources/jsonschema/mpas.jsonschema b/src/uwtools/resources/jsonschema/mpas.jsonschema index f6c0005c9..b3aabb4f2 100644 --- a/src/uwtools/resources/jsonschema/mpas.jsonschema +++ b/src/uwtools/resources/jsonschema/mpas.jsonschema @@ -4,7 +4,7 @@ "additionalProperties": false, "properties": { "execution": { - "$ref": "urn:uwtools:execution" + "$ref": "urn:uwtools:execution-parallel" }, "files_to_copy": { "$ref": "urn:uwtools:files-to-stage" diff --git a/src/uwtools/resources/jsonschema/orog.jsonschema b/src/uwtools/resources/jsonschema/orog.jsonschema new file mode 100644 index 000000000..e43b67c17 --- /dev/null +++ b/src/uwtools/resources/jsonschema/orog.jsonschema @@ -0,0 +1,73 @@ +{ + "properties": { + "orog": { + "additionalProperties": false, + "properties": { + "execution": { + "$ref": "urn:uwtools:execution-parallel" + }, + "files_to_link": { + "$ref": "urn:uwtools:files-to-stage" + }, + "grid_file": { + "type": "string" + }, + "mask": { + "type": "boolean" + }, + "merge": { + "type": "string" + }, + "old_line1_items": { + "additionalProperties": false, + "minProperties": 9, + "properties": { + "blat": { + "type": "integer" + }, + "efac": { + "type": "integer" + }, + "jcap": { + "type": "integer" + }, + "latb": { + "type": "integer" + }, + "lonb": { + "type": "integer" + }, + "mtnres": { + "type": "integer" + }, + "nf1": { + "type": "integer" + }, + "nf2": { + "type": "integer" + }, + "nr": { + "type": "integer" + } + } + }, + "orog_file": { + "type": "string" + }, + "rundir": { + "type": "string" + } + }, + "required": [ + "execution", + "grid_file", + "rundir" + ], + "type": "object" + } + }, + "required": [ + "orog" + ], + "type": "object" +} diff --git a/src/uwtools/resources/jsonschema/rocoto.jsonschema b/src/uwtools/resources/jsonschema/rocoto.jsonschema index 1b3bcbf0f..882ca19b7 100644 --- a/src/uwtools/resources/jsonschema/rocoto.jsonschema +++ b/src/uwtools/resources/jsonschema/rocoto.jsonschema @@ -1,44 +1,60 @@ { "$defs": { "compoundTimeString": { - "anyOf": [ + "oneOf": [ { - "type": "integer" + "$ref": "#/$defs/compoundTimeStringElement" }, { - "type": "string" + "items": { + "$ref": "#/$defs/compoundTimeStringElement" + }, + "type": "array" + } + ] + }, + "compoundTimeStringElement": { + "oneOf": [ + { + "$ref": "#/$defs/cycleString" }, { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "cycleString": { + "additionalProperties": false, + "properties": { + "cyclestr": { "additionalProperties": false, "properties": { - "cyclestr": { + "attrs": { "additionalProperties": false, "properties": { - "attrs": { - "additionalProperties": false, - "properties": { - "offset": { - "$ref": "#/$defs/time" - } - }, - "type": "object" - }, - "value": { - "type": "string" + "offset": { + "$ref": "#/$defs/time" } }, - "required": [ - "value" - ], "type": "object" + }, + "value": { + "type": "string" } }, "required": [ - "cyclestr" + "value" ], "type": "object" } - ] + }, + "required": [ + "cyclestr" + ], + "type": "object" }, "dependency": { "additionalProperties": false, diff --git a/src/uwtools/resources/jsonschema/sfc-climo-gen.jsonschema b/src/uwtools/resources/jsonschema/sfc-climo-gen.jsonschema index d4207bac8..220454615 100644 --- a/src/uwtools/resources/jsonschema/sfc-climo-gen.jsonschema +++ b/src/uwtools/resources/jsonschema/sfc-climo-gen.jsonschema @@ -4,7 +4,7 @@ "additionalProperties": false, "properties": { "execution": { - "$ref": "urn:uwtools:execution" + "$ref": "urn:uwtools:execution-parallel" }, "namelist": { "additionalProperties": false, diff --git a/src/uwtools/resources/jsonschema/shave.jsonschema b/src/uwtools/resources/jsonschema/shave.jsonschema index 7426e0b25..ad3b9f741 100644 --- a/src/uwtools/resources/jsonschema/shave.jsonschema +++ b/src/uwtools/resources/jsonschema/shave.jsonschema @@ -9,8 +9,8 @@ "input_grid_file": { "type": "string" }, - "nh4": { - "minimum": 1, + "nhalo": { + "minimum": 0, "type": "integer" }, "nx": { @@ -20,13 +20,17 @@ "ny": { "minimum": 1, "type": "integer" + }, + "output_grid_file": { + "type": "string" } }, "required": [ "input_grid_file", - "nh4", + "nhalo", "nx", - "ny" + "ny", + "output_grid_file" ] }, "execution": { diff --git a/src/uwtools/resources/jsonschema/ungrib.jsonschema b/src/uwtools/resources/jsonschema/ungrib.jsonschema index e938b5d48..9ba6bc36c 100644 --- a/src/uwtools/resources/jsonschema/ungrib.jsonschema +++ b/src/uwtools/resources/jsonschema/ungrib.jsonschema @@ -4,7 +4,7 @@ "additionalProperties": false, "properties": { "execution": { - "$ref": "urn:uwtools:execution" + "$ref": "urn:uwtools:execution-parallel" }, "gfs_files": { "additionalProperties": false, diff --git a/src/uwtools/resources/jsonschema/upp.jsonschema b/src/uwtools/resources/jsonschema/upp.jsonschema index 1ca647919..811fd8ea0 100644 --- a/src/uwtools/resources/jsonschema/upp.jsonschema +++ b/src/uwtools/resources/jsonschema/upp.jsonschema @@ -3,8 +3,11 @@ "upp": { "additionalProperties": false, "properties": { + "control_file": { + "type": "string" + }, "execution": { - "$ref": "urn:uwtools:execution" + "$ref": "urn:uwtools:execution-parallel" }, "files_to_copy": { "$ref": "urn:uwtools:files-to-stage" @@ -187,6 +190,7 @@ } }, "required": [ + "control_file", "execution", "namelist", "rundir" diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 4bfb6fb3b..180d66aaf 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -9,6 +9,7 @@ from typing import Any, Optional, Union from lxml import etree +from lxml.builder import E # type: ignore from lxml.etree import Element, SubElement, _Element from uwtools.config.formats.yaml import YAMLConfig @@ -62,7 +63,7 @@ def validate_rocoto_xml_string(xml: str) -> bool: valid: bool = schema.validate(tree) nerr = len(schema.error_log) log_method = log.info if valid else log.error - log_method("%s Rocoto validation error%s found", nerr, "" if nerr == 1 else "s") + log_method("%s Rocoto XML validation error%s found", nerr, "" if nerr == 1 else "s") for err in list(schema.error_log): log.error(err) if not valid: @@ -91,7 +92,7 @@ def __str__(self) -> str: xml = etree.tostring( self._root, pretty_print=True, encoding="utf-8", xml_declaration=True ).decode() - xml = re.sub(r"&([^;]+);", r"&\1;", xml) + xml = re.sub(r"&([^&]+;)", r"&\1", xml) xml = self._insert_doctype(xml) return xml @@ -113,16 +114,12 @@ def _add_compound_time_string(self, e: _Element, config: Any, tag: str) -> _Elem :param tag: Name of child element to add. :return: The child element. """ - e = SubElement(e, tag) - if isinstance(config, dict): - self._set_attrs(e, config) - if subconfig := config.get(STR.cyclestr, {}): - cyclestr = SubElement(e, STR.cyclestr) - cyclestr.text = subconfig[STR.value] - self._set_attrs(cyclestr, subconfig) - else: - e.text = str(config) - return e + config = config if isinstance(config, list) else [config] + cyclestr = lambda x: E.cyclestr(x["cyclestr"]["value"], **x["cyclestr"].get("attrs", {})) + items = [cyclestr(x) if isinstance(x, dict) else str(x) for x in [tag, *config]] + child: _Element = E(*items) # pylint: disable=not-callable + e.append(child) + return child def _add_metatask(self, e: _Element, config: dict, name_attr: str) -> None: """ @@ -264,7 +261,9 @@ def _add_task_dependency_strequality(self, e: _Element, config: dict, tag: str) :param config: Configuration data for the tag. :param tag: Name of new element to add. """ - self._set_attrs(SubElement(e, tag), config) + e = SubElement(e, tag) + for k, v in config.items(): + self._add_compound_time_string(e, v, k) def _add_task_dependency_taskdep(self, e: _Element, config: dict) -> None: """ @@ -359,7 +358,7 @@ def _config_validate(self, config: Union[dict, YAMLConfig, Optional[Path]]) -> N :raises: UWConfigError if config fails validation. """ schema_file = resource_path("jsonschema/rocoto.jsonschema") - validate_yaml(schema_file=schema_file, config=config) + validate_yaml(schema_file=schema_file, desc="Rocoto config", config=config) @property def _doctype(self) -> Optional[str]: diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 098724d62..234ef728a 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -111,6 +111,7 @@ class STR: mpiargs: str = "mpiargs" mpicmd: str = "mpicmd" namelist: str = "namelist" + orog: str = "orog" oroggsl: str = "orog_gsl" outfile: str = "output_file" outfmt: str = "output_format" diff --git a/src/uwtools/tests/api/test_config.py b/src/uwtools/tests/api/test_config.py index 803f35747..71796c814 100644 --- a/src/uwtools/tests/api/test_config.py +++ b/src/uwtools/tests/api/test_config.py @@ -99,7 +99,7 @@ def test_realize_update_config_from_stdin(): def test_realize_update_config_none(): - input_config = {"n": 88} + input_config = {"n": 42} output_file = Path("output.yaml") with patch.object(config, "_realize") as _realize: config.realize(input_config=input_config, output_file=output_file) @@ -126,6 +126,7 @@ def test_validate(cfg): assert config.validate(**kwargs) is False _validate_external.assert_called_with( schema_file=Path(kwargs["schema_file"]), + desc="config", config=kwargs["config"], ) @@ -138,4 +139,6 @@ def test_validate_config_file(cast, tmp_path): kwargs: dict = {"schema_file": "schema-file", "config": cast(cfg)} with patch.object(config, "_validate_external", return_value=True) as _validate_external: assert config.validate(**kwargs) - _validate_external.assert_called_once_with(schema_file=Path(kwargs["schema_file"]), config=cfg) + _validate_external.assert_called_once_with( + schema_file=Path(kwargs["schema_file"]), desc="config", config=cfg + ) diff --git a/src/uwtools/tests/api/test_drivers.py b/src/uwtools/tests/api/test_drivers.py index 70bf280b3..b45ebaeb3 100644 --- a/src/uwtools/tests/api/test_drivers.py +++ b/src/uwtools/tests/api/test_drivers.py @@ -19,6 +19,7 @@ make_solo_mosaic, mpas, mpas_init, + orog, orog_gsl, schism, sfc_climo_gen, @@ -43,6 +44,7 @@ make_solo_mosaic, mpas, mpas_init, + orog, orog_gsl, schism, sfc_climo_gen, @@ -63,6 +65,7 @@ def test_api_execute(module): "dry_run": False, "graph_file": "/some/g.dot", "key_path": None, + "schema_file": None, "stdin_ok": True, "task": "foo", } diff --git a/src/uwtools/tests/api/test_execute.py b/src/uwtools/tests/api/test_execute.py index b81b39c76..e663d82d0 100644 --- a/src/uwtools/tests/api/test_execute.py +++ b/src/uwtools/tests/api/test_execute.py @@ -25,7 +25,7 @@ def args(): config=fixture_path("testdriver.yaml"), module=fixture_path("testdriver.py"), schema_file=fixture_path("testdriver.jsonschema"), - task="eighty_eight", + task="forty_two", ) @@ -98,7 +98,7 @@ def test_tasks_fail_no_cycle(args, caplog, kwargs): @mark.parametrize("f", [Path, str]) def test_tasks_pass(args, f): tasks = execute.tasks(classname=args.classname, module=f(args.module)) - assert tasks["eighty_eight"] == "88" + assert tasks["forty_two"] == "42" def test__get_driver_class_explicit_fail_bad_class(caplog, args): diff --git a/src/uwtools/tests/config/formats/test_base.py b/src/uwtools/tests/config/formats/test_base.py index a4ba27c87..3e926eb13 100644 --- a/src/uwtools/tests/config/formats/test_base.py +++ b/src/uwtools/tests/config/formats/test_base.py @@ -5,6 +5,8 @@ import datetime as dt import logging import os +from datetime import datetime +from textwrap import dedent from unittest.mock import patch import yaml @@ -25,7 +27,7 @@ @fixture def config(tmp_path): path = tmp_path / "config.yaml" - data = {"foo": 88} + data = {"foo": 42} with open(path, "w", encoding="utf-8") as f: yaml.dump(data, f) return ConcreteConfig(config=path) @@ -55,6 +57,9 @@ def _load(self, config_file): with readable(config_file) as f: return yaml.safe_load(f.read()) + def as_dict(self): + return self.data + def dump(self, path=None): pass @@ -67,7 +72,7 @@ def dump_dict(cfg, path=None): def test__characterize_values(config): - values = {1: "", 2: None, 3: "{{ n }}", 4: {"a": 88}, 5: [{"b": 99}], 6: "string"} + values = {1: "", 2: None, 3: "{{ n }}", 4: {"a": 42}, 5: [{"b": 43}], 6: "string"} complete, template = config._characterize_values(values=values, parent="p") assert complete == [" p1", " p2", " p4", " p4.a", " pb", " p5", " p6"] assert template == [" p3: {{ n }}"] @@ -97,7 +102,7 @@ def test__parse_include(config): config.data.update( { "config": { - "salad_include": f"!INCLUDE [{include_path}]", + "salad_include": f"!include [{include_path}]", "meat": "beef", "dressing": "poppyseed", } @@ -111,7 +116,7 @@ def test__parse_include(config): assert len(config["config"]) == 2 -@mark.parametrize("fmt", [FORMAT.ini, FORMAT.nml, FORMAT.yaml]) +@mark.parametrize("fmt", [FORMAT.nml, FORMAT.yaml]) def test_compare_config(caplog, fmt, salad_base): """ Compare two config objects. @@ -128,15 +133,65 @@ def test_compare_config(caplog, fmt, salad_base): salad_base["salad"]["dressing"] = "italian" salad_base["salad"]["size"] = "large" del salad_base["salad"]["how_many"] - assert not cfgobj.compare_config(cfgobj, salad_base) assert not cfgobj.compare_config(salad_base) # Expect to see the following differences logged: - for msg in [ - "salad: how_many: - 12 + None", - "salad: dressing: - balsamic + italian", - "salad: size: - None + large", - ]: - assert logged(caplog, msg) + expected = """ + --------------------------------------------------------------------- + ↓ ? = info | -/+ = line unique to - or + file | blank = matching line + --------------------------------------------------------------------- + salad: + base: kale + - dressing: balsamic + ? ^ ^ ^^^ + + dressing: italian + ? ^^ ^ ^ + fruit: banana + - how_many: 12 + + size: large + vegetable: tomato + """ + for line in dedent(expected).strip("\n").split("\n"): + assert logged(caplog, line) + + +def test_compare_config_ini(caplog, salad_base): + """ + Compare two config objects. + """ + log.setLevel(logging.INFO) + cfgobj = tools.format_to_config("ini")(fixture_path("simple.ini")) + salad_base["salad"]["how_many"] = "12" # str "12" (not int 12) for ini + assert cfgobj.compare_config(salad_base) is True + # Expect no differences: + assert not caplog.records + caplog.clear() + # Create differences in base dict: + salad_base["salad"]["dressing"] = "italian" + salad_base["salad"]["size"] = "large" + del salad_base["salad"]["how_many"] + assert not cfgobj.compare_config(cfgobj.as_dict(), salad_base, header=False) + # Expect to see the following differences logged: + expected = """ + salad: + base: kale + - dressing: italian + ? ^^ ^^ + + dressing: balsamic + ? ^ +++ ^ + fruit: banana + - size: large + + how_many: '12' + vegetable: tomato + """ + for line in dedent(expected).strip("\n").split("\n"): + assert logged(caplog, line) + anomalous = """ + --------------------------------------------------------------------- + ↓ ? = info | -/+ = line unique to - or + file | blank = matching line + --------------------------------------------------------------------- + """ + for line in dedent(anomalous).strip("\n").split("\n"): + assert not logged(caplog, line) def test_dereference(tmp_path): @@ -153,12 +208,15 @@ def test_dereference(tmp_path): c: !int '{{ N | int + 11 }}' d: '{{ X }}' e: - - !int '88' + - !int '42' - !float '3.14' + - !datetime '{{ D }}' f: - f1: !int '88' + f1: !int '42' f2: !float '3.14' +D: 2024-10-10 00:19:00 N: "22" + """.strip() path = tmp_path / "config.yaml" with open(path, "w", encoding="utf-8") as f: @@ -166,12 +224,14 @@ def test_dereference(tmp_path): config = YAMLConfig(path) with patch.dict(os.environ, {"N": "999"}, clear=True): config.dereference() + print(config["e"]) assert config == { "a": 44, "b": {"c": 33}, "d": "{{ X }}", - "e": [88, 3.14], - "f": {"f1": 88, "f2": 3.14}, + "e": [42, 3.14, datetime.fromisoformat("2024-10-10 00:19:00")], + "f": {"f1": 42, "f2": 3.14}, + "D": datetime.fromisoformat("2024-10-10 00:19:00"), "N": "22", } @@ -206,4 +266,4 @@ def test_update_from(config): Test that a config object can be updated. """ config.data.update({"a": "11", "b": "12", "c": "13"}) - assert config == {"foo": 88, "a": "11", "b": "12", "c": "13"} + assert config == {"foo": 42, "a": "11", "b": "12", "c": "13"} diff --git a/src/uwtools/tests/config/formats/test_fieldtable.py b/src/uwtools/tests/config/formats/test_fieldtable.py index 8776f84b5..19de6d5ce 100644 --- a/src/uwtools/tests/config/formats/test_fieldtable.py +++ b/src/uwtools/tests/config/formats/test_fieldtable.py @@ -3,6 +3,8 @@ Tests for uwtools.config.formats.fieldtable module. """ +from textwrap import dedent + from pytest import fixture, mark from uwtools.config.formats.fieldtable import FieldTableConfig @@ -17,6 +19,24 @@ def config(): return fixture_path("FV3_GFS_v16.yaml") +@fixture +def dumpkit(tmp_path): + expected = """ + "TRACER", "atmos_mod", "sphum" + "longname", "specific humidity" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1e+30" / + """ + d = { + "sphum": { + "longname": "specific humidity", + "units": "kg/kg", + "profile_type": {"name": "fixed", "surface_value": 1.0e30}, + } + } + return d, dedent(expected).strip(), tmp_path / "config.fieldtable" + + @fixture(scope="module") def ref(): with open(fixture_path("field_table.FV3_GFS_v16"), "r", encoding="utf-8") as f: @@ -49,3 +69,25 @@ def test_fieldtable_simple(config, ref, tmp_path): FieldTableConfig(config=config).dump(outfile) with open(outfile, "r", encoding="utf-8") as out: assert out.read().strip() == ref + + +def test_fieldtable_as_dict(): + d1 = {"section": {"key": "value"}} + config = FieldTableConfig(d1) + d2 = config.as_dict() + assert d2 == d1 + assert isinstance(d2, dict) + + +def test_fieldtable_dump(dumpkit): + d, expected, path = dumpkit + FieldTableConfig(d).dump(path) + with open(path, "r", encoding="utf-8") as f: + assert f.read().strip() == expected + + +def test_fieldtable_dump_dict(dumpkit): + d, expected, path = dumpkit + FieldTableConfig.dump_dict(d, path=path) + with open(path, "r", encoding="utf-8") as f: + assert f.read().strip() == expected diff --git a/src/uwtools/tests/config/formats/test_ini.py b/src/uwtools/tests/config/formats/test_ini.py index c6a9f9f7f..7845e570b 100644 --- a/src/uwtools/tests/config/formats/test_ini.py +++ b/src/uwtools/tests/config/formats/test_ini.py @@ -1,17 +1,30 @@ -# pylint: disable=missing-function-docstring,protected-access +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name """ Tests for uwtools.config.formats.ini module. """ import filecmp +from textwrap import dedent -from pytest import mark, raises +from pytest import fixture, mark, raises from uwtools.config.formats.ini import INIConfig from uwtools.exceptions import UWConfigError from uwtools.tests.support import fixture_path from uwtools.utils.file import FORMAT +# Fixtures + + +@fixture +def dumpkit(tmp_path): + expected = """ + [section] + key = value + """ + return {"section": {"key": "value"}}, dedent(expected).strip(), tmp_path / "config.ini" + + # Tests @@ -64,3 +77,25 @@ def test_ini_simple(salad_base, tmp_path): cfgobj.update({"dressing": ["ranch", "italian"]}) expected["dressing"] = ["ranch", "italian"] assert cfgobj == expected + + +def test_ini_as_dict(): + d1 = {"section": {"key": "value"}} + config = INIConfig(d1) + d2 = config.as_dict() + assert d2 == d1 + assert isinstance(d2, dict) + + +def test_ini_dump(dumpkit): + d, expected, path = dumpkit + INIConfig(d).dump(path) + with open(path, "r", encoding="utf-8") as f: + assert f.read().strip() == expected + + +def test_ini_dump_dict(dumpkit): + d, expected, path = dumpkit + INIConfig.dump_dict(d, path=path) + with open(path, "r", encoding="utf-8") as f: + assert f.read().strip() == expected diff --git a/src/uwtools/tests/config/formats/test_nml.py b/src/uwtools/tests/config/formats/test_nml.py index c91c16dde..f5bf51a9e 100644 --- a/src/uwtools/tests/config/formats/test_nml.py +++ b/src/uwtools/tests/config/formats/test_nml.py @@ -21,6 +21,16 @@ def data(): return {"nml": {"key": "val"}} +@fixture +def dumpkit(tmp_path): + expected = """ + §ion + key = 'value' + / + """ + return {"section": {"key": "value"}}, dedent(expected).strip(), tmp_path / "config.nml" + + # Tests @@ -58,22 +68,22 @@ def test_nml__parse_include_mult_sect(): def test_nml_derived_type_dict(): - nml = NMLConfig(config={"nl": {"o": {"i": 77, "j": 88}}}) - assert nml["nl"]["o"] == {"i": 77, "j": 88} + nml = NMLConfig(config={"nl": {"o": {"i": 41, "j": 42}}}) + assert nml["nl"]["o"] == {"i": 41, "j": 42} def test_nml_derived_type_file(tmp_path): s = """ &nl - o%i = 77 - o%j = 88 + o%i = 41 + o%j = 42 / """ path = tmp_path / "a.nml" with open(path, "w", encoding="utf-8") as f: print(dedent(s).strip(), file=f) nml = NMLConfig(config=path) - assert nml["nl"]["o"] == {"i": 77, "j": 88} + assert nml["nl"]["o"] == {"i": 41, "j": 42} def test_nml_dump_dict_dict(data, tmp_path): @@ -112,3 +122,25 @@ def test_nml_simple(salad_base, tmp_path): cfgobj.update({"dressing": ["ranch", "italian"]}) expected["dressing"] = ["ranch", "italian"] assert cfgobj == expected + + +def test_nml_as_dict(): + d1 = {"section": {"key": "value"}} + config = NMLConfig(d1) + d2 = config.as_dict() + assert d2 == d1 + assert isinstance(d2, dict) + + +def test_nml_dump(dumpkit): + d, expected, path = dumpkit + NMLConfig(d).dump(path) + with open(path, "r", encoding="utf-8") as f: + assert f.read().strip() == expected + + +def test_nml_dump_dict(dumpkit): + d, expected, path = dumpkit + NMLConfig.dump_dict(d, path=path) + with open(path, "r", encoding="utf-8") as f: + assert f.read().strip() == expected diff --git a/src/uwtools/tests/config/formats/test_sh.py b/src/uwtools/tests/config/formats/test_sh.py index ce6a0119e..bf4ac15bb 100644 --- a/src/uwtools/tests/config/formats/test_sh.py +++ b/src/uwtools/tests/config/formats/test_sh.py @@ -1,4 +1,4 @@ -# pylint: disable=missing-function-docstring,protected-access +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name """ Tests for uwtools.config.formats.sh module. """ @@ -6,13 +6,24 @@ from textwrap import dedent from typing import Any -from pytest import mark, raises +from pytest import fixture, mark, raises from uwtools.config.formats.sh import SHConfig from uwtools.exceptions import UWConfigError from uwtools.tests.support import fixture_path from uwtools.utils.file import FORMAT +# Fixtures + + +@fixture +def dumpkit(tmp_path): + expected = """ + key=value + """ + return {"key": "value"}, dedent(expected).strip(), tmp_path / "config.yaml" + + # Tests @@ -68,3 +79,25 @@ def test_sh(salad_base): cfgobj.update({"dressing": ["ranch", "italian"]}) expected["dressing"] = ["ranch", "italian"] assert cfgobj == expected + + +def test_sh_as_dict(): + d1 = {"a": 1} + config = SHConfig(d1) + d2 = config.as_dict() + assert d2 == d1 + assert isinstance(d2, dict) + + +def test_sh_dump(dumpkit): + d, expected, path = dumpkit + SHConfig(d).dump(path) + with open(path, "r", encoding="utf-8") as f: + assert f.read().strip() == expected + + +def test_sh_dump_dict(dumpkit): + d, expected, path = dumpkit + SHConfig.dump_dict(d, path=path) + with open(path, "r", encoding="utf-8") as f: + assert f.read().strip() == expected diff --git a/src/uwtools/tests/config/formats/test_yaml.py b/src/uwtools/tests/config/formats/test_yaml.py index 98a374d59..256edc8b1 100644 --- a/src/uwtools/tests/config/formats/test_yaml.py +++ b/src/uwtools/tests/config/formats/test_yaml.py @@ -1,4 +1,4 @@ -# pylint: disable=missing-function-docstring,protected-access +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name """ Tests for uwtools.config.formats.yaml module. """ @@ -14,7 +14,7 @@ import f90nml # type: ignore import yaml -from pytest import mark, raises +from pytest import fixture, mark, raises from uwtools import exceptions from uwtools.config import support @@ -24,6 +24,18 @@ from uwtools.tests.support import fixture_path, logged from uwtools.utils.file import FORMAT, _stdinproxy +# Fixtures + + +@fixture +def dumpkit(tmp_path): + expected = """ + section: + key: value + """ + return {"section": {"key": "value"}}, dedent(expected).strip(), tmp_path / "config.yaml" + + # Tests @@ -155,7 +167,7 @@ def test_yaml_constructor_error_not_dict_from_file(tmp_path): def test_yaml_constructor_error_not_dict_from_stdin(): # Test that a useful exception is raised if the YAML stdin input is a non-dict value. - with StringIO("88") as sio, patch.object(sys, "stdin", new=sio): + with StringIO("42") as sio, patch.object(sys, "stdin", new=sio): with raises(exceptions.UWConfigError) as e: YAMLConfig() assert "Parsed an int value from stdin, expected a dict" in str(e.value) @@ -201,10 +213,32 @@ def test_yaml_stdin_plus_relpath_failure(caplog): def test_yaml_unexpected_error(tmp_path): cfgfile = tmp_path / "cfg.yaml" with open(cfgfile, "w", encoding="utf-8") as f: - print("{n: 88}", file=f) + print("{n: 42}", file=f) with patch.object(yaml, "load") as load: msg = "Unexpected error" load.side_effect = yaml.constructor.ConstructorError(note=msg) with raises(UWConfigError) as e: YAMLConfig(config=cfgfile) assert msg in str(e.value) + + +def test_yaml_as_dict(): + d1 = {"section": {"key": "value"}} + config = YAMLConfig(d1) + d2 = config.as_dict() + assert d2 == d1 + assert isinstance(d2, dict) + + +def test_yaml_dump(dumpkit): + d, expected, path = dumpkit + YAMLConfig(d).dump(path) + with open(path, "r", encoding="utf-8") as f: + assert f.read().strip() == expected + + +def test_yaml_dump_dict(dumpkit): + d, expected, path = dumpkit + YAMLConfig.dump_dict(d, path=path) + with open(path, "r", encoding="utf-8") as f: + assert f.read().strip() == expected diff --git a/src/uwtools/tests/config/test_jinja2.py b/src/uwtools/tests/config/test_jinja2.py index c1ef18611..6d430bf8b 100644 --- a/src/uwtools/tests/config/test_jinja2.py +++ b/src/uwtools/tests/config/test_jinja2.py @@ -5,6 +5,7 @@ import logging import os +from datetime import datetime from io import StringIO from textwrap import dedent from types import SimpleNamespace as ns @@ -108,7 +109,7 @@ def test_dereference_local_values(): } -@mark.parametrize("val", (True, 3.14, 88, None)) +@mark.parametrize("val", (True, 3.14, 42, None)) def test_dereference_no_op(val): # These types of values pass through dereferencing unmodified: assert jinja2.dereference(val=val, context={}) == val @@ -157,7 +158,7 @@ def test_dereference_str_filter_rendered(): def test_derefrence_str_variable_rendered_mixed(): # A mixed result remains a str. val = "{{ n }} is an {{ t }}" - assert jinja2.dereference(val=val, context={"n": 88, "t": "int"}) == "88 is an int" + assert jinja2.dereference(val=val, context={"n": 42, "t": "int"}) == "42 is an int" def test_dereference_str_variable_rendered_str(): @@ -280,7 +281,7 @@ def test_unrendered(s, status): assert jinja2.unrendered(s) is status -@mark.parametrize("tag", ["!float", "!int"]) +@mark.parametrize("tag", ["!datetime", "!float", "!int"]) def test__deref_convert_no(caplog, tag): log.setLevel(logging.DEBUG) loader = yaml.SafeLoader(os.devnull) @@ -290,7 +291,14 @@ def test__deref_convert_no(caplog, tag): assert regex_logged(caplog, "Conversion failed") -@mark.parametrize("converted,tag,value", [(3.14, "!float", "3.14"), (88, "!int", "88")]) +@mark.parametrize( + "converted,tag,value", + [ + (datetime(2024, 9, 9, 0, 0), "!datetime", "2024-09-09 00:00:00"), + (3.14, "!float", "3.14"), + (42, "!int", "42"), + ], +) def test__deref_convert_ok(caplog, converted, tag, value): log.setLevel(logging.DEBUG) loader = yaml.SafeLoader(os.devnull) @@ -340,10 +348,10 @@ def test__report(caplog): Internal arguments when rendering template: --------------------------------------------------------------------- foo: bar -longish_variable: 88 +longish_variable: 42 --------------------------------------------------------------------- """.strip() - jinja2._report(dict(foo="bar", longish_variable=88)) + jinja2._report(dict(foo="bar", longish_variable=42)) assert "\n".join(record.message for record in caplog.records) == expected @@ -541,7 +549,7 @@ def test_searchpath_stdin_explicit(self, searchpath_assets): assert J2Template(values={}, searchpath=[a.d1]).render() == "2" def test_undeclared_variables(self): - s = "{{ a }} {{ b.c }} {{ d.e.f[g] }} {{ h[i] }} {{ j[88] }} {{ k|default(l) }}" + s = "{{ a }} {{ b.c }} {{ d.e.f[g] }} {{ h[i] }} {{ j[42] }} {{ k|default(l) }}" uvs = {"a", "b", "d", "g", "h", "i", "j", "k", "l"} assert J2Template(values={}, template_source=s).undeclared_variables == uvs diff --git a/src/uwtools/tests/config/test_support.py b/src/uwtools/tests/config/test_support.py index ddbae129b..f17754697 100644 --- a/src/uwtools/tests/config/test_support.py +++ b/src/uwtools/tests/config/test_support.py @@ -5,6 +5,7 @@ import logging from collections import OrderedDict +from datetime import datetime import yaml from pytest import fixture, mark, raises @@ -21,7 +22,7 @@ from uwtools.utils.file import FORMAT -@mark.parametrize("d,n", [({1: 88}, 1), ({1: {2: 88}}, 2), ({1: {2: {3: 88}}}, 3), ({1: {}}, 2)]) +@mark.parametrize("d,n", [({1: 42}, 1), ({1: {2: 42}}, 2), ({1: {2: {3: 42}}}, 3), ({1: {}}, 2)]) def test_depth(d, n): assert support.depth(d) == n @@ -87,6 +88,18 @@ def loader(self): # demonstrate that those nodes' convert() methods return representations in type type specified # by the tag. + def test_datetime_no(self, loader): + ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!datetime", value="foo")) + with raises(ValueError): + ts.convert() + + def test_datetime_ok(self, loader): + ts = support.UWYAMLConvert( + loader, yaml.ScalarNode(tag="!datetime", value="2024-08-09 12:22:42") + ) + assert ts.convert() == datetime(2024, 8, 9, 12, 22, 42) + self.comp(ts, "!datetime '2024-08-09 12:22:42'") + def test_float_no(self, loader): ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!float", value="foo")) with raises(ValueError): @@ -103,13 +116,13 @@ def test_int_no(self, loader): ts.convert() def test_int_ok(self, loader): - ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!int", value="88")) - assert ts.convert() == 88 - self.comp(ts, "!int '88'") + ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!int", value="42")) + assert ts.convert() == 42 + self.comp(ts, "!int '42'") def test___repr__(self, loader): - ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!int", value="88")) - assert str(ts) == "!int 88" + ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!int", value="42")) + assert str(ts) == "!int 42" class Test_UWYAMLRemove: diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index c80be3d6d..69e5c77e7 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -32,7 +32,7 @@ @fixture def compare_configs_assets(tmp_path): - d = {"foo": {"bar": 88}, "baz": {"qux": 99}} + d = {"foo": {"bar": 42}, "baz": {"qux": 43}} a = tmp_path / "a" b = tmp_path / "b" with writable(a) as f: @@ -50,7 +50,7 @@ def realize_config_testobj(realize_config_yaml_input): @fixture def realize_config_yaml_input(tmp_path): path = tmp_path / "a.yaml" - d = {1: {2: {3: 88}}} # depth 3 + d = {1: {2: {3: 42}}} # depth 3 with writable(path) as f: yaml.dump(d, f) return path @@ -117,7 +117,25 @@ def test_compare_configs_changed_value(compare_configs_assets, caplog): assert not tools.compare_configs( config_1_path=a, config_1_format=FORMAT.yaml, config_2_path=b, config_2_format=FORMAT.yaml ) - assert logged(caplog, "baz: qux: - 99 + 11") + expected = """ + - %s + + %s + --------------------------------------------------------------------- + ↓ ? = info | -/+ = line unique to - or + file | blank = matching line + --------------------------------------------------------------------- + baz: + - qux: 43 + ? ^^ + + qux: 11 + ? ^^ + foo: + bar: 42 + """ % ( + str(a), + str(b), + ) + for line in dedent(expected).strip("\n").split("\n"): + assert logged(caplog, line) def test_compare_configs_missing_key(compare_configs_assets, caplog): @@ -130,7 +148,22 @@ def test_compare_configs_missing_key(compare_configs_assets, caplog): assert not tools.compare_configs( config_1_path=b, config_1_format=FORMAT.yaml, config_2_path=a, config_2_format=FORMAT.yaml ) - assert logged(caplog, "baz: qux: - None + 99") + expected = """ + - %s + + %s + --------------------------------------------------------------------- + ↓ ? = info | -/+ = line unique to - or + file | blank = matching line + --------------------------------------------------------------------- + + baz: + + qux: 43 + foo: + bar: 42 + """ % ( + str(b), + str(a), + ) + for line in dedent(expected).strip("\n").split("\n"): + assert logged(caplog, line) def test_compare_configs_bad_format(caplog): @@ -544,7 +577,7 @@ def test__ensure_format_bad_no_path_no_format(): def test__ensure_format_config_obj(): - config = NMLConfig({"nl": {"n": 88}}) + config = NMLConfig({"nl": {"n": 42}}) assert tools._ensure_format(desc="foo", config=config) == FORMAT.nml @@ -653,7 +686,7 @@ def test__realize_config_input_setup_ini_stdin(caplog): data = """ [section] foo = bar - baz = 88 + baz = 42 """ stdinproxy.cache_clear() log.setLevel(logging.DEBUG) @@ -662,7 +695,7 @@ def test__realize_config_input_setup_ini_stdin(caplog): sio.seek(0) with patch.object(sys, "stdin", new=sio): input_obj = tools._realize_config_input_setup(input_format=FORMAT.ini) - assert input_obj.data == {"section": {"foo": "bar", "baz": "88"}} # note: 88 is str, not int + assert input_obj.data == {"section": {"foo": "bar", "baz": "42"}} # note: 42 is str, not int assert logged(caplog, "Reading input from stdin") @@ -780,24 +813,24 @@ def test__realize_config_output_setup(caplog, tmp_path): def test__realize_config_update_cfgobj(realize_config_testobj): - assert realize_config_testobj[1][2][3] == 88 - update_config = YAMLConfig(config={1: {2: {3: 99}}}) + assert realize_config_testobj[1][2][3] == 42 + update_config = YAMLConfig(config={1: {2: {3: 43}}}) o = tools._realize_config_update(input_obj=realize_config_testobj, update_config=update_config) - assert o[1][2][3] == 99 + assert o[1][2][3] == 43 def test__realize_config_update_stdin(caplog, realize_config_testobj): stdinproxy.cache_clear() log.setLevel(logging.DEBUG) - assert realize_config_testobj[1][2][3] == 88 + assert realize_config_testobj[1][2][3] == 42 with StringIO() as sio: - print("{1: {2: {3: 99}}}", file=sio) + print("{1: {2: {3: 43}}}", file=sio) sio.seek(0) with patch.object(sys, "stdin", new=sio): o = tools._realize_config_update( input_obj=realize_config_testobj, update_format=FORMAT.yaml ) - assert o[1][2][3] == 99 + assert o[1][2][3] == 43 assert logged(caplog, "Reading update from stdin") @@ -806,13 +839,13 @@ def test__realize_config_update_noop(realize_config_testobj): def test__realize_config_update_file(realize_config_testobj, tmp_path): - assert realize_config_testobj[1][2][3] == 88 - values = {1: {2: {3: 99}}} + assert realize_config_testobj[1][2][3] == 42 + values = {1: {2: {3: 43}}} update_config = tmp_path / "config.yaml" with open(update_config, "w", encoding="utf-8") as f: yaml.dump(values, f) o = tools._realize_config_update(input_obj=realize_config_testobj, update_config=update_config) - assert o[1][2][3] == 99 + assert o[1][2][3] == 43 def test__realize_config_values_needed(caplog, tmp_path): diff --git a/src/uwtools/tests/config/test_validator.py b/src/uwtools/tests/config/test_validator.py index 983c7abf0..ff8ef9a29 100644 --- a/src/uwtools/tests/config/test_validator.py +++ b/src/uwtools/tests/config/test_validator.py @@ -35,7 +35,7 @@ def config(tmp_path) -> dict[str, Any]: return { "color": "blue", "dir": str(tmp_path), - "number": 88, + "number": 42, "sub": { "dir": str(tmp_path), }, @@ -59,7 +59,7 @@ def rocoto_assets(): "metatask": { "var": {"member": "foo bar baz"}, "task": { - "cores": 88, + "cores": 42, "command": "some-command", "walltime": "00:01:00", "dependency": { @@ -124,7 +124,8 @@ def write_as_json(data: dict[str, Any], path: Path) -> Path: # Test functions -def test_bundle(): +def test_bundle(caplog): + log.setLevel(logging.DEBUG) schema = {"fruit": {"$ref": "urn:uwtools:a"}, "flowers": None} with patch.object(validator, "_registry") as _registry: outer, inner = Mock(), Mock() @@ -137,21 +138,30 @@ def test_bundle(): "urn:uwtools:a", "urn:uwtools:attrs", ] - - -def test_get_schema_file(): + for msg in [ + "Bundling referenced schema urn:uwtools:a at key path: fruit", + "Bundling referenced schema urn:uwtools:attrs at key path: fruit.a", + "Bundling str value at key path: fruit.a.name", + "Bundling dict value at key path: fruit.b", + "Bundling str value at key path: fruit.b.name", + "Bundling NoneType value at key path: flowers", + ]: + assert logged(caplog, msg) + + +def test_internal_schema_file(): with patch.object(validator, "resource_path", return_value=Path("/foo/bar")): - assert validator.get_schema_file("baz") == Path("/foo/bar/baz.jsonschema") + assert validator.internal_schema_file("baz") == Path("/foo/bar/baz.jsonschema") def test_validate(config, schema): - assert validator.validate(schema=schema, config=config) + assert validator.validate(schema=schema, desc="test", config=config) def test_validate_fail_bad_enum_val(caplog, config, schema): log.setLevel(logging.INFO) config["color"] = "yellow" # invalid enum value - assert not validator.validate(schema=schema, config=config) + assert not validator.validate(schema=schema, desc="test", config=config) assert any(x for x in caplog.records if "1 UW schema-validation error found" in x.message) assert any(x for x in caplog.records if "'yellow' is not one of" in x.message) @@ -159,7 +169,7 @@ def test_validate_fail_bad_enum_val(caplog, config, schema): def test_validate_fail_bad_number_val(caplog, config, schema): log.setLevel(logging.INFO) config["number"] = "string" # invalid number value - assert not validator.validate(schema=schema, config=config) + assert not validator.validate(schema=schema, desc="test", config=config) assert any(x for x in caplog.records if "1 UW schema-validation error found" in x.message) assert any(x for x in caplog.records if "'string' is not of type 'number'" in x.message) @@ -167,7 +177,7 @@ def test_validate_fail_bad_number_val(caplog, config, schema): def test_validate_internal_no(caplog, schema_file): with patch.object(validator, "resource_path", return_value=schema_file.parent): with raises(UWConfigError) as e: - validator.validate_internal(schema_name="a", config={"color": "orange"}) + validator.validate_internal(schema_name="a", desc="test", config={"color": "orange"}) assert logged(caplog, "Error at color:") assert logged(caplog, " 'orange' is not one of ['blue', 'red']") assert str(e.value) == "YAML validation errors" @@ -175,14 +185,14 @@ def test_validate_internal_no(caplog, schema_file): def test_validate_internal_ok(schema_file): with patch.object(validator, "resource_path", return_value=schema_file.parent): - validator.validate_internal(schema_name="a", config={"color": "blue"}) + validator.validate_internal(schema_name="a", desc="test", config={"color": "blue"}) def test_validate_external(assets, config, schema): schema_file, _, cfgobj = assets with patch.object(validator, "validate") as validate: - validator.validate_external(schema_file=schema_file, config=cfgobj) - validate.assert_called_once_with(schema=schema, config=config) + validator.validate_external(schema_file=schema_file, desc="test", config=cfgobj) + validate.assert_called_once_with(schema=schema, desc="test", config=config) def test_prep_config_cfgobj(prep_config_dict): diff --git a/src/uwtools/tests/drivers/test_cdeps.py b/src/uwtools/tests/drivers/test_cdeps.py index 66ad90867..75166c5db 100644 --- a/src/uwtools/tests/drivers/test_cdeps.py +++ b/src/uwtools/tests/drivers/test_cdeps.py @@ -17,6 +17,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers import cdeps from uwtools.drivers.cdeps import CDEPS +from uwtools.drivers.driver import AssetsCycleBased from uwtools.logging import log from uwtools.tests.support import logged from uwtools.tests.test_schemas import CDEPS_CONFIG @@ -35,6 +36,14 @@ def driverobj(tmp_path): # Tests +@mark.parametrize( + "method", + ["taskname", "_validate"], +) +def test_CDEPS(method): + assert getattr(CDEPS, method) is getattr(AssetsCycleBased, method) + + def test_CDEPS_atm(driverobj): with patch.object(CDEPS, "atm_nml") as atm_nml: with patch.object(CDEPS, "atm_stream") as atm_stream: @@ -43,6 +52,10 @@ def test_CDEPS_atm(driverobj): atm_nml.assert_called_once_with() +def test_CDEPS_driver_name(driverobj): + assert driverobj.driver_name() == CDEPS.driver_name() == "cdeps" + + @mark.parametrize("group", ["atm", "ocn"]) def test_CDEPS_nml(caplog, driverobj, group): log.setLevel(logging.DEBUG) @@ -111,10 +124,6 @@ def test_CDEPS_streams(driverobj, group): assert f.read().strip() == dedent(expected).strip() -def test_CDEPS_driver_name(driverobj): - assert driverobj.driver_name() == CDEPS.driver_name() == "cdeps" - - def test_CDEPS__model_namelist_file(driverobj): group = "atm_in" path = Path("/path/to/some.nml") diff --git a/src/uwtools/tests/drivers/test_chgres_cube.py b/src/uwtools/tests/drivers/test_chgres_cube.py index de399e7fe..f8494a28e 100644 --- a/src/uwtools/tests/drivers/test_chgres_cube.py +++ b/src/uwtools/tests/drivers/test_chgres_cube.py @@ -10,10 +10,11 @@ import f90nml # type: ignore from iotaa import refs -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.chgres_cube import ChgresCube from uwtools.drivers.driver import Driver +from uwtools.exceptions import UWNotImplementedError from uwtools.logging import log from uwtools.scheduler import Slurm from uwtools.tests.support import logged, regex_logged @@ -48,7 +49,7 @@ def config(tmp_path): "update_values": { "config": { "atm_core_files_input_grid": [str(afile), str(afile)], - "atm_files_input_grid": str(afile), + "atm_files_input_grid": [str(afile), str(afile)], "atm_tracer_files_input_grid": str(afile), "atm_weight_file": str(afile), "convert_atm": True, @@ -59,6 +60,11 @@ def config(tmp_path): "grib2_file_input_grid": str(afile), "mosaic_file_input_grid": str(afile), "mosaic_file_target_grid": str(afile), + "orog_dir_input_grid": "/path/to/dir", + "orog_files_input_grid": [str(afile), str(afile)], + "orog_dir_target_grid": "/path/to/dir", + "orog_files_target_grid": str(afile), + "nst_files_input_grid": str(afile), "sfc_files_input_grid": str(afile), "varmap_file": str(afile), "vcoord_file_target_grid": str(afile), @@ -76,8 +82,13 @@ def config(tmp_path): @fixture -def driverobj(config, cycle): - return ChgresCube(config=config, cycle=cycle, batch=True) +def driverobj(config, cycle, leadtime): + return ChgresCube(config=config, cycle=cycle, leadtime=leadtime, batch=True) + + +@fixture +def leadtime(): + return dt.timedelta(hours=24) # Tests @@ -96,6 +107,7 @@ def driverobj(config, cycle): "_scheduler", "_validate", "_write_runscript", + "output", "run", ], ) @@ -103,6 +115,10 @@ def test_ChgresCube(method): assert getattr(ChgresCube, method) is getattr(Driver, method) +def test_ChgresCube_driver_name(driverobj): + assert driverobj.driver_name() == ChgresCube.driver_name() == "chgres_cube" + + def test_ChgresCube_namelist_file(caplog, driverobj): log.setLevel(logging.DEBUG) dst = driverobj.rundir / "fort.41" @@ -131,6 +147,12 @@ def test_ChgresCube_namelist_file_missing_base_file(caplog, driverobj): assert regex_logged(caplog, "missing.nml: State: Not Ready (external asset)") +def test_ChgresCube_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_ChgresCube_provisioned_rundir(driverobj): with patch.multiple(driverobj, namelist_file=D, runscript=D) as mocks: driverobj.provisioned_rundir() @@ -147,9 +169,5 @@ def test_ChgresCube_runscript(driverobj): assert [type(runscript.call_args.kwargs[x]) for x in args] == types -def test_ChgresCube_driver_name(driverobj): - assert driverobj.driver_name() == ChgresCube.driver_name() == "chgres_cube" - - def test_ChgresCube_taskname(driverobj): - assert driverobj.taskname("foo") == "20240201 18Z chgres_cube foo" + assert driverobj.taskname("foo") == "20240202 18:00:00 chgres_cube foo" diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index 9c80c7769..9f33c2961 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -19,7 +19,7 @@ from uwtools.config.formats.yaml import YAMLConfig from uwtools.drivers import driver -from uwtools.exceptions import UWConfigError +from uwtools.exceptions import UWConfigError, UWNotImplementedError from uwtools.logging import log from uwtools.scheduler import Slurm from uwtools.tests.support import regex_logged @@ -36,13 +36,13 @@ def atask(self): yield "atask" yield asset("atask", lambda: True) - def provisioned_rundir(self): - pass - @classmethod def driver_name(cls) -> str: return "concrete" + def provisioned_rundir(self): + pass + def _validate(self, schema_file: Optional[Path] = None) -> None: pass @@ -68,7 +68,11 @@ class ConcreteDriverCycleLeadtimeBased(Common, driver.DriverCycleLeadtimeBased): class ConcreteDriverTimeInvariant(Common, driver.DriverTimeInvariant): - pass + + @property + def output(self): + # The dict keys are intentionally out-of-alphabetical-order to test JSON sorting. + return {"bar": ["/path/to/bar1", "/path/to/bar2"], "foo": "/path/to/foo"} def write(path, x): @@ -195,7 +199,7 @@ def test_Assets_controller(config, controller_schema): with raises(UWConfigError): ConcreteAssetsTimeInvariant(config=config, schema_file=controller_schema) assert ConcreteAssetsTimeInvariant( - config=config, schema_file=controller_schema, controller="controller" + config=config, schema_file=controller_schema, controller=["controller"] ) @@ -243,6 +247,22 @@ def test_Assets_validate_key_path(config, controller_schema): ) +@mark.parametrize("should_pass,t", [(True, "integer"), (False, "string")]) +def test_Assets_validate_schema_file(caplog, should_pass, t, tmp_path): + path = tmp_path / "test.jsonschema" + schema = {"properties": {"concrete": {"properties": {"n": {"type": t}}}}} + with open(path, "w", encoding="utf-8") as f: + json.dump(schema, f) + test = lambda: ConcreteAssetsTimeInvariant(config={"concrete": {"n": 1}}, schema_file=path) + with patch.object(ConcreteAssetsTimeInvariant, "_validate", driver.Assets._validate): + if should_pass: + assert test() + else: + with raises(UWConfigError): + test() + assert regex_logged(caplog, "1 is not of type 'string'") + + @mark.parametrize( "base_file,update_values,expected", [ @@ -269,6 +289,13 @@ def test_Assets__create_user_updated_config_base_file( assert updated == expected +def test_Assets__delegate(driverobj): + assert "roses" not in driverobj.config + driverobj._config_intermediate["plants"] = {"flowers": {"roses": "red"}} + driverobj._delegate(["plants", "flowers"], "roses") + assert driverobj.config["roses"] == "red" + + def test_Assets__rundir(assetsobj): assert assetsobj.rundir == Path(assetsobj.config["rundir"]) @@ -284,6 +311,7 @@ def test_Assets__validate_internal(assetsobj): assetsobj._validate(assetsobj) assert validate_internal.call_args_list[0].kwargs == { "schema_name": "concrete", + "desc": "concrete config", "config": assetsobj.config_full, } @@ -295,6 +323,7 @@ def test_Assets__validate_external(config): assetsobj = ConcreteAssetsTimeInvariant(schema_file=schema_file, config=config) assert validate_external.call_args_list[0].kwargs == { "schema_file": schema_file, + "desc": "concrete config", "config": assetsobj.config_full, } @@ -324,7 +353,7 @@ def test_Driver_controller(config, controller_schema): with raises(UWConfigError): ConcreteDriverTimeInvariant(config=config, schema_file=controller_schema) assert ConcreteDriverTimeInvariant( - config=config, schema_file=controller_schema, controller="controller" + config=config, schema_file=controller_schema, controller=["controller"] ) @@ -336,6 +365,58 @@ def test_Driver_leadtime(config): assert obj.leadtime == leadtime +def test_Driver_namelist_schema_default(driverobj, tmp_path): + nmlschema = {"properties": {"n": {"type": "integer"}}, "type": "object"} + schema = { + "properties": { + "concrete": {"properties": {"namelist": {"properties": {"update_values": nmlschema}}}} + } + } + schema_path = tmp_path / "test.jsonschema" + with open(schema_path, "w", encoding="utf-8") as f: + json.dump(schema, f) + with patch.object(ConcreteDriverTimeInvariant, "config", new_callable=PropertyMock) as dc: + dc.return_value = {"namelist": {"validate": True}} + with patch.object(driver, "internal_schema_file", return_value=schema_path): + assert driverobj.namelist_schema() == nmlschema + + +def test_Driver_namelist_schema_external(driverobj, tmp_path): + nmlschema = {"properties": {"n": {"type": "integer"}}, "type": "object"} + schema = {"foo": {"bar": nmlschema}} + schema_path = tmp_path / "test.jsonschema" + with open(schema_path, "w", encoding="utf-8") as f: + json.dump(schema, f) + with patch.object(ConcreteDriverTimeInvariant, "config", new_callable=PropertyMock) as dc: + dc.return_value = {"baz": {"qux": {"validate": True}}} + driverobj.schema_file = schema_path + assert nmlschema == driverobj.namelist_schema( + config_keys=["baz", "qux"], schema_keys=["foo", "bar"] + ) + + +def test_Driver_namelist_schema_default_disable(driverobj): + with patch.object(ConcreteDriverTimeInvariant, "config", new_callable=PropertyMock) as dc: + dc.return_value = {"namelist": {"validate": False}} + assert driverobj.namelist_schema() == {"type": "object"} + + +def test_Driver_output(config): + driverobj = ConcreteDriverTimeInvariant(config) + assert driverobj.output == {"foo": "/path/to/foo", "bar": ["/path/to/bar1", "/path/to/bar2"]} + + +@mark.parametrize("cls", [ConcreteDriverCycleBased, ConcreteDriverCycleLeadtimeBased]) +def test_Driver_output_not_implemented(cls, config): + kwargs = {"config": config, "cycle": dt.datetime(2024, 10, 22, 12)} + if cls == ConcreteDriverCycleLeadtimeBased: + kwargs["leadtime"] = 6 + driverobj = cls(**kwargs) + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + @mark.parametrize("batch", [True, False]) def test_Driver_run(batch, driverobj): driverobj._batch = batch @@ -362,33 +443,25 @@ def test_Driver_runscript(arg, driverobj, type_): assert isinstance(runscript.call_args.kwargs[arg], type_) -def test_Driver__run_via_batch_submission(driverobj): - runscript = driverobj._runscript_path - executable = Path(driverobj.config["execution"]["executable"]) - executable.touch() - with patch.object(driverobj, "provisioned_rundir") as prd: - with patch.object( - ConcreteDriverTimeInvariant, "_scheduler", new_callable=PropertyMock - ) as scheduler: - driverobj._run_via_batch_submission() - scheduler().submit_job.assert_called_once_with( - runscript=runscript, submit_file=Path(f"{runscript}.submit") - ) - prd.assert_called_once_with() +def test_driver_show_output(capsys, config): + ConcreteDriverTimeInvariant(config).show_output() + expected = """ + { + "bar": [ + "/path/to/bar1", + "/path/to/bar2" + ], + "foo": "/path/to/foo" + } + """ + assert capsys.readouterr().out.strip() == dedent(expected).strip() -def test_Driver__run_via_local_execution(driverobj): - executable = Path(driverobj.config["execution"]["executable"]) - executable.touch() - with patch.object(driverobj, "provisioned_rundir") as prd: - with patch.object(driver, "run_shell_cmd") as run_shell_cmd: - driverobj._run_via_local_execution() - run_shell_cmd.assert_called_once_with( - cmd="{x} >{x}.out 2>&1".format(x=driverobj._runscript_path), - cwd=driverobj.rundir, - log_output=True, - ) - prd.assert_called_once_with() +def test_driver_show_output_fail(caplog, config): + with patch.object(ConcreteDriverTimeInvariant, "output", new_callable=PropertyMock) as output: + output.side_effect = UWConfigError("FAIL") + ConcreteDriverTimeInvariant(config).show_output() + assert "FAIL" in caplog.messages @mark.parametrize( @@ -416,41 +489,33 @@ def test_Driver__create_user_updated_config_base_file( assert updated == expected -def test_Driver__namelist_schema_custom(driverobj, tmp_path): - nmlschema = {"properties": {"n": {"type": "integer"}}, "type": "object"} - schema = {"foo": {"bar": nmlschema}} - schema_path = tmp_path / "test.jsonschema" - with open(schema_path, "w", encoding="utf-8") as f: - json.dump(schema, f) - with patch.object(ConcreteDriverTimeInvariant, "config", new_callable=PropertyMock) as dc: - dc.return_value = {"baz": {"qux": {"validate": True}}} - with patch.object(driver, "get_schema_file", return_value=schema_path): - assert ( - driverobj._namelist_schema(config_keys=["baz", "qux"], schema_keys=["foo", "bar"]) - == nmlschema +def test_Driver__run_via_batch_submission(driverobj): + runscript = driverobj._runscript_path + executable = Path(driverobj.config["execution"]["executable"]) + executable.touch() + with patch.object(driverobj, "provisioned_rundir") as prd: + with patch.object( + ConcreteDriverTimeInvariant, "_scheduler", new_callable=PropertyMock + ) as scheduler: + driverobj._run_via_batch_submission() + scheduler().submit_job.assert_called_once_with( + runscript=runscript, submit_file=Path(f"{runscript}.submit") ) + prd.assert_called_once_with() -def test_Driver__namelist_schema_default(driverobj, tmp_path): - nmlschema = {"properties": {"n": {"type": "integer"}}, "type": "object"} - schema = { - "properties": { - "concrete": {"properties": {"namelist": {"properties": {"update_values": nmlschema}}}} - } - } - schema_path = tmp_path / "test.jsonschema" - with open(schema_path, "w", encoding="utf-8") as f: - json.dump(schema, f) - with patch.object(ConcreteDriverTimeInvariant, "config", new_callable=PropertyMock) as dc: - dc.return_value = {"namelist": {"validate": True}} - with patch.object(driver, "get_schema_file", return_value=schema_path): - assert driverobj._namelist_schema() == nmlschema - - -def test_Driver__namelist_schema_default_disable(driverobj): - with patch.object(ConcreteDriverTimeInvariant, "config", new_callable=PropertyMock) as dc: - dc.return_value = {"namelist": {"validate": False}} - assert driverobj._namelist_schema() == {"type": "object"} +def test_Driver__run_via_local_execution(driverobj): + executable = Path(driverobj.config["execution"]["executable"]) + executable.touch() + with patch.object(driverobj, "provisioned_rundir") as prd: + with patch.object(driver, "run_shell_cmd") as run_shell_cmd: + driverobj._run_via_local_execution() + run_shell_cmd.assert_called_once_with( + cmd="{x} >{x}.out 2>&1".format(x=driverobj._runscript_path), + cwd=driverobj.rundir, + log_output=True, + ) + prd.assert_called_once_with() def test_Driver__run_resources_fail(driverobj): @@ -508,6 +573,10 @@ def test_Driver__runscript(driverobj): ) +def test_Driver__runscript_done_file(driverobj): + assert driverobj._runscript_done_file == "runscript.concrete.done" + + def test_Driver__runscript_execution_only(driverobj): expected = """ #!/bin/bash @@ -518,10 +587,6 @@ def test_Driver__runscript_execution_only(driverobj): assert driverobj._runscript(execution=["foo", "bar"]) == dedent(expected).strip() -def test_Driver__runscript_done_file(driverobj): - assert driverobj._runscript_done_file == "runscript.concrete.done" - - def test_Driver__runscript_path(driverobj): rundir = Path(driverobj.config["rundir"]) assert driverobj._runscript_path == rundir / "runscript.concrete" @@ -534,27 +599,30 @@ def test_Driver__scheduler(driverobj): JobScheduler.get_scheduler.assert_called_with(driverobj._run_resources) +def test_Driver__validate_external(config): + schema_file = Path("/path/to/jsonschema") + with patch.object(ConcreteAssetsTimeInvariant, "_validate", driver.Driver._validate): + with patch.object(driver, "validate_external") as validate_external: + assetsobj = ConcreteAssetsTimeInvariant(schema_file=schema_file, config=config) + assert validate_external.call_args_list[0].kwargs == { + "schema_file": schema_file, + "desc": "concrete config", + "config": assetsobj.config_full, + } + + def test_Driver__validate_internal(assetsobj): with patch.object(assetsobj, "_validate", driver.Driver._validate): with patch.object(driver, "validate_internal") as validate_internal: assetsobj._validate(assetsobj) assert validate_internal.call_args_list[0].kwargs == { "schema_name": "concrete", + "desc": "concrete config", "config": assetsobj.config_full, } assert validate_internal.call_args_list[1].kwargs == { "schema_name": "platform", - "config": assetsobj.config_full, - } - - -def test_Driver__validate_external(config): - schema_file = Path("/path/to/jsonschema") - with patch.object(ConcreteAssetsTimeInvariant, "_validate", driver.Driver._validate): - with patch.object(driver, "validate_external") as validate_external: - assetsobj = ConcreteAssetsTimeInvariant(schema_file=schema_file, config=config) - assert validate_external.call_args_list[0].kwargs == { - "schema_file": schema_file, + "desc": "platform config", "config": assetsobj.config_full, } diff --git a/src/uwtools/tests/drivers/test_esg_grid.py b/src/uwtools/tests/drivers/test_esg_grid.py index 950ce2d6d..96fa5d578 100644 --- a/src/uwtools/tests/drivers/test_esg_grid.py +++ b/src/uwtools/tests/drivers/test_esg_grid.py @@ -9,10 +9,11 @@ import f90nml # type: ignore from iotaa import refs -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.driver import Driver from uwtools.drivers.esg_grid import ESGGrid +from uwtools.exceptions import UWNotImplementedError from uwtools.logging import log from uwtools.tests.support import logged, regex_logged @@ -75,6 +76,7 @@ def driverobj(config): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", "taskname", @@ -84,6 +86,10 @@ def test_ESGGrid(method): assert getattr(ESGGrid, method) is getattr(Driver, method) +def test_ESGGrid_driver_name(driverobj): + assert driverobj.driver_name() == ESGGrid.driver_name() == "esg_grid" + + def test_ESGGrid_namelist_file(caplog, driverobj): log.setLevel(logging.DEBUG) dst = driverobj.rundir / "regional_grid.nml" @@ -112,6 +118,12 @@ def test_ESGGrid_namelist_file_missing_base_file(caplog, driverobj): assert regex_logged(caplog, "missing.nml: State: Not Ready (external asset)") +def test_ESGGrid_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_ESGGrid_provisioned_rundir(driverobj): with patch.multiple( driverobj, @@ -121,7 +133,3 @@ def test_ESGGrid_provisioned_rundir(driverobj): driverobj.provisioned_rundir() for m in mocks: mocks[m].assert_called_once_with() - - -def test_FilterTopo_driver_name(driverobj): - assert driverobj.driver_name() == ESGGrid.driver_name() == "esg_grid" diff --git a/src/uwtools/tests/drivers/test_filter_topo.py b/src/uwtools/tests/drivers/test_filter_topo.py index 54b243fdf..ef0faf543 100644 --- a/src/uwtools/tests/drivers/test_filter_topo.py +++ b/src/uwtools/tests/drivers/test_filter_topo.py @@ -8,11 +8,12 @@ import f90nml # type: ignore from iotaa import refs -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.config.support import from_od from uwtools.drivers.driver import Driver from uwtools.drivers.filter_topo import FilterTopo +from uwtools.exceptions import UWNotImplementedError # Fixtures @@ -21,10 +22,14 @@ def config(tmp_path): input_grid_file = tmp_path / "C403_grid.tile7.halo4.nc" input_grid_file.touch() + orog_output = tmp_path / "out.oro.nc" + orog_output.touch() return { "filter_topo": { "config": { + "filtered_orog": "C403_filtered_orog.tile7.nc", "input_grid_file": str(input_grid_file), + "input_raw_orog": str(orog_output), }, "execution": { "executable": "/path/to/orog_gsl", @@ -70,6 +75,7 @@ def driverobj(config): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", "taskname", @@ -79,6 +85,17 @@ def test_FilterTopo(method): assert getattr(FilterTopo, method) is getattr(Driver, method) +def test_FilterTopo_driver_name(driverobj): + assert driverobj.driver_name() == FilterTopo.driver_name() == "filter_topo" + + +def test_FilterTopo_filtered_output_file(driverobj): + path = Path(driverobj.config["rundir"], "C403_filtered_orog.tile7.nc") + assert not path.is_file() + driverobj.filtered_output_file() + assert path.is_file() + + def test_FilterTopo_input_grid_file(driverobj): path = Path(driverobj.config["rundir"], "C403_grid.tile7.halo4.nc") assert not path.is_file() @@ -93,12 +110,16 @@ def test_FilterTopo_namelist_file(driverobj): assert actual == expected +def test_FilterTopo_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_FilterTopo_provisioned_rundir(driverobj): - with patch.multiple(driverobj, input_grid_file=D, namelist_file=D, runscript=D) as mocks: + with patch.multiple( + driverobj, input_grid_file=D, filtered_output_file=D, namelist_file=D, runscript=D + ) as mocks: driverobj.provisioned_rundir() for m in mocks: mocks[m].assert_called_once_with() - - -def test_FilterTopo_driver_name(driverobj): - assert driverobj.driver_name() == FilterTopo.driver_name() == "filter_topo" diff --git a/src/uwtools/tests/drivers/test_fv3.py b/src/uwtools/tests/drivers/test_fv3.py index ebc2a08bf..80939b56a 100644 --- a/src/uwtools/tests/drivers/test_fv3.py +++ b/src/uwtools/tests/drivers/test_fv3.py @@ -10,10 +10,11 @@ import yaml from iotaa import asset, external, refs -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.driver import Driver from uwtools.drivers.fv3 import FV3 +from uwtools.exceptions import UWNotImplementedError from uwtools.logging import log from uwtools.scheduler import Slurm from uwtools.tests.support import logged, regex_logged @@ -94,6 +95,7 @@ def true(): "_scheduler", "_validate", "_write_runscript", + "output", "run", ], ) @@ -126,6 +128,10 @@ def test_FV3_diag_table_warn(caplog, driverobj): assert logged(caplog, "No 'diag_table' defined in config") +def test_FV3_driver_name(driverobj): + assert driverobj.driver_name() == FV3.driver_name() == "fv3" + + def test_FV3_field_table(driverobj): src = driverobj.rundir / "field_table.in" src.touch() @@ -207,6 +213,12 @@ def test_FV3_namelist_file_missing_base_file(caplog, driverobj): assert regex_logged(caplog, "missing.nml: State: Not Ready (external asset)") +def test_FV3_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + @mark.parametrize("domain", ("global", "regional")) def test_FV3_provisioned_rundir(domain, driverobj): driverobj._config["domain"] = domain @@ -247,9 +259,5 @@ def test_FV3_runscript(driverobj): assert [type(runscript.call_args.kwargs[x]) for x in args] == types -def test_FV3_driver_name(driverobj): - assert driverobj.driver_name() == FV3.driver_name() == "fv3" - - def test_FV3_taskname(driverobj): assert driverobj.taskname("foo") == "20240201 18Z fv3 foo" diff --git a/src/uwtools/tests/drivers/test_global_equiv_resol.py b/src/uwtools/tests/drivers/test_global_equiv_resol.py index f95201940..95159d7f9 100644 --- a/src/uwtools/tests/drivers/test_global_equiv_resol.py +++ b/src/uwtools/tests/drivers/test_global_equiv_resol.py @@ -6,10 +6,11 @@ from unittest.mock import DEFAULT as D from unittest.mock import patch -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.driver import Driver from uwtools.drivers.global_equiv_resol import GlobalEquivResol +from uwtools.exceptions import UWNotImplementedError # Fixtures @@ -55,6 +56,7 @@ def driverobj(config): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", "taskname", @@ -64,6 +66,10 @@ def test_GlobalEquivResol(method): assert getattr(GlobalEquivResol, method) is getattr(Driver, method) +def test_GlobalEquivResol_driver_name(driverobj): + assert driverobj.driver_name() == GlobalEquivResol.driver_name() == "global_equiv_resol" + + def test_GlobalEquivResol_input_file(driverobj): path = Path(driverobj.config["input_grid_file"]) assert not driverobj.input_file().ready() @@ -72,6 +78,12 @@ def test_GlobalEquivResol_input_file(driverobj): assert driverobj.input_file().ready() +def test_GlobalEquivResol_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_GlobalEquivResol_provisioned_rundir(driverobj): with patch.multiple( driverobj, @@ -83,10 +95,6 @@ def test_GlobalEquivResol_provisioned_rundir(driverobj): mocks[m].assert_called_once_with() -def test_FilterTopo_driver_name(driverobj): - assert driverobj.driver_name() == GlobalEquivResol.driver_name() == "global_equiv_resol" - - def test_GlobalEquivResol__runcmd(driverobj): cmd = driverobj._runcmd input_file_path = driverobj.config["input_grid_file"] diff --git a/src/uwtools/tests/drivers/test_ioda.py b/src/uwtools/tests/drivers/test_ioda.py index 65527f86c..b012812a8 100644 --- a/src/uwtools/tests/drivers/test_ioda.py +++ b/src/uwtools/tests/drivers/test_ioda.py @@ -6,10 +6,11 @@ from unittest.mock import DEFAULT as D from unittest.mock import patch -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.ioda import IODA from uwtools.drivers.jedi_base import JEDIBase +from uwtools.exceptions import UWNotImplementedError # Fixtures @@ -79,6 +80,7 @@ def driverobj(config, cycle): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", ], @@ -87,6 +89,16 @@ def test_IODA(method): assert getattr(IODA, method) is getattr(JEDIBase, method) +def test_IODA_driver_name(driverobj): + assert driverobj.driver_name() == IODA.driver_name() == "ioda" + + +def test_IODA_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_IODA_provisioned_rundir(driverobj): with patch.multiple( driverobj, @@ -100,8 +112,8 @@ def test_IODA_provisioned_rundir(driverobj): mocks[m].assert_called_once_with() -def test_IODA_driver_name(driverobj): - assert driverobj.driver_name() == IODA.driver_name() == "ioda" +def test_IODA_taskname(driverobj): + assert driverobj.taskname("foo") == "20240501 06Z ioda foo" def test_IODA__config_fn(driverobj): @@ -111,7 +123,3 @@ def test_IODA__config_fn(driverobj): def test_IODA__runcmd(driverobj): config = str(driverobj.rundir / driverobj._config_fn) assert driverobj._runcmd == f"/path/to/bufr2ioda.x {config}" - - -def test_IODA_taskname(driverobj): - assert driverobj.taskname("foo") == "20240501 06Z ioda foo" diff --git a/src/uwtools/tests/drivers/test_jedi.py b/src/uwtools/tests/drivers/test_jedi.py index ebd7ecfcc..21d3e4b88 100644 --- a/src/uwtools/tests/drivers/test_jedi.py +++ b/src/uwtools/tests/drivers/test_jedi.py @@ -10,12 +10,13 @@ import yaml from iotaa import asset, external -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.config.formats.yaml import YAMLConfig from uwtools.drivers import jedi, jedi_base from uwtools.drivers.jedi import JEDI from uwtools.drivers.jedi_base import JEDIBase +from uwtools.exceptions import UWNotImplementedError from uwtools.logging import log from uwtools.tests.support import regex_logged @@ -89,6 +90,7 @@ def driverobj(config, cycle): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", ], @@ -121,6 +123,10 @@ def test_JEDI_configuration_file_missing_base_file(caplog, driverobj): assert regex_logged(caplog, f"{base_file}: State: Not Ready (external asset)") +def test_JEDI_driver_name(driverobj): + assert driverobj.driver_name() == JEDI.driver_name() == "jedi" + + def test_JEDI_files_copied(driverobj): with patch.object(jedi_base, "filecopy") as filecopy: driverobj._config["rundir"] = "/path/to/run" @@ -150,6 +156,12 @@ def test_JEDI_files_linked(driverobj): ) +def test_JEDI_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_JEDI_provisioned_rundir(driverobj): with patch.multiple( driverobj, @@ -164,6 +176,10 @@ def test_JEDI_provisioned_rundir(driverobj): mocks[m].assert_called_once_with() +def test_JEDI_taskname(driverobj): + assert driverobj.taskname("foo") == "20240201 18Z jedi foo" + + def test_JEDI_validate_only(caplog, driverobj): @external @@ -188,10 +204,6 @@ def file(path: Path): assert regex_logged(caplog, "Config is valid") -def test_JEDI_driver_name(driverobj): - assert driverobj.driver_name() == JEDI.driver_name() == "jedi" - - def test_JEDI__config_fn(driverobj): assert driverobj._config_fn == "jedi.yaml" @@ -202,7 +214,3 @@ def test_JEDI__runcmd(driverobj): assert ( driverobj._runcmd == f"srun --export=ALL --ntasks $SLURM_CPUS_ON_NODE {executable} {config}" ) - - -def test_JEDI_taskname(driverobj): - assert driverobj.taskname("foo") == "20240201 18Z jedi foo" diff --git a/src/uwtools/tests/drivers/test_make_hgrid.py b/src/uwtools/tests/drivers/test_make_hgrid.py index 74874e40d..0d7c1b228 100644 --- a/src/uwtools/tests/drivers/test_make_hgrid.py +++ b/src/uwtools/tests/drivers/test_make_hgrid.py @@ -5,10 +5,11 @@ from unittest.mock import DEFAULT as D from unittest.mock import patch -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.driver import Driver from uwtools.drivers.make_hgrid import MakeHgrid +from uwtools.exceptions import UWNotImplementedError # Fixtures @@ -60,6 +61,7 @@ def driverobj(config): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", "taskname", @@ -69,6 +71,16 @@ def test_MakeHgrid(method): assert getattr(MakeHgrid, method) is getattr(Driver, method) +def test_MakeHgrid_driver_name(driverobj): + assert driverobj.driver_name() == MakeHgrid.driver_name() == "make_hgrid" + + +def test_MakeHgrid_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_MakeHgrid_provisioned_rundir(driverobj): with patch.multiple(driverobj, runscript=D) as mocks: driverobj.provisioned_rundir() @@ -76,10 +88,6 @@ def test_MakeHgrid_provisioned_rundir(driverobj): mocks[m].assert_called_once_with() -def test_MakeHgrid_driver_name(driverobj): - assert driverobj.driver_name() == MakeHgrid.driver_name() == "make_hgrid" - - def test_MakeHgrid__runcmd(driverobj): expected = [ "/path/to/make_hgrid", diff --git a/src/uwtools/tests/drivers/test_make_solo_mosaic.py b/src/uwtools/tests/drivers/test_make_solo_mosaic.py index bdd0f4573..20fa51699 100644 --- a/src/uwtools/tests/drivers/test_make_solo_mosaic.py +++ b/src/uwtools/tests/drivers/test_make_solo_mosaic.py @@ -4,10 +4,11 @@ """ from unittest.mock import patch -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.driver import Driver from uwtools.drivers.make_solo_mosaic import MakeSoloMosaic +from uwtools.exceptions import UWNotImplementedError # Fixtures @@ -56,6 +57,7 @@ def driverobj(config): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", ], @@ -64,14 +66,24 @@ def test_MakeSoloMosaic(method): assert getattr(MakeSoloMosaic, method) is getattr(Driver, method) +def test_MakeSoloMosaic_driver_name(driverobj): + assert driverobj.driver_name() == MakeSoloMosaic.driver_name() == "make_solo_mosaic" + + +def test_MakeSoloMosaic_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_MakeSoloMosaic_provisioned_rundir(driverobj): with patch.object(driverobj, "runscript") as runscript: driverobj.provisioned_rundir() runscript.assert_called_once_with() -def test_MakeSoloMosaic_driver_name(driverobj): - assert driverobj.driver_name() == MakeSoloMosaic.driver_name() == "make_solo_mosaic" +def test_MakeSoloMosaic_taskname(driverobj): + assert driverobj.taskname("foo") == "make_solo_mosaic foo" def test_MakeSoloMosaic__runcmd(driverobj): @@ -80,9 +92,5 @@ def test_MakeSoloMosaic__runcmd(driverobj): assert cmd == f"/path/to/make_solo_mosaic.exe --dir {dir_path} --num_tiles 1" -def test_MakeSoloMosaic_taskname(driverobj): - assert driverobj.taskname("foo") == "make_solo_mosaic foo" - - def test_MakeSoloMosaic__validate(driverobj): driverobj._validate() diff --git a/src/uwtools/tests/drivers/test_mpas.py b/src/uwtools/tests/drivers/test_mpas.py index b85a55588..ef225ef4b 100644 --- a/src/uwtools/tests/drivers/test_mpas.py +++ b/src/uwtools/tests/drivers/test_mpas.py @@ -12,10 +12,11 @@ import yaml from iotaa import refs from lxml import etree -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.mpas import MPAS from uwtools.drivers.mpas_base import MPASBase +from uwtools.exceptions import UWNotImplementedError from uwtools.logging import log from uwtools.tests.support import fixture_path, logged, regex_logged @@ -130,6 +131,7 @@ def driverobj(config, cycle): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", "streams_file", @@ -156,6 +158,10 @@ def test_MPAS_boundary_files(driverobj, cycle): assert all(link.is_symlink() for link in links) +def test_MPAS_driver_name(driverobj): + assert driverobj.driver_name() == MPAS.driver_name() == "mpas" + + @mark.parametrize( "key,task,test", [("files_to_copy", "files_copied", "is_file"), ("files_to_link", "files_linked", "is_symlink")], @@ -189,6 +195,15 @@ def test_MPAS_namelist_file(caplog, driverobj): assert isinstance(nml, f90nml.Namelist) +def test_MPAS_namelist_file_fails_validation(caplog, driverobj): + log.setLevel(logging.DEBUG) + driverobj._config["namelist"]["update_values"]["nhyd_model"]["foo"] = None + path = Path(refs(driverobj.namelist_file())) + assert not path.exists() + assert logged(caplog, f"Failed to validate {path}") + assert logged(caplog, " None is not of type 'array', 'boolean', 'number', 'string'") + + def test_MPAS_namelist_file_long_duration(caplog, config, cycle): log.setLevel(logging.DEBUG) config["mpas"]["length"] = 120 @@ -203,15 +218,6 @@ def test_MPAS_namelist_file_long_duration(caplog, config, cycle): assert nml["nhyd_model"]["config_run_duration"] == "5_0:00:00" -def test_MPAS_namelist_file_fails_validation(caplog, driverobj): - log.setLevel(logging.DEBUG) - driverobj._config["namelist"]["update_values"]["nhyd_model"]["foo"] = None - path = Path(refs(driverobj.namelist_file())) - assert not path.exists() - assert logged(caplog, f"Failed to validate {path}") - assert logged(caplog, " None is not of type 'array', 'boolean', 'number', 'string'") - - def test_MPAS_namelist_file_missing_base_file(caplog, driverobj): log.setLevel(logging.DEBUG) base_file = str(Path(driverobj.config["rundir"], "missing.nml")) @@ -221,6 +227,12 @@ def test_MPAS_namelist_file_missing_base_file(caplog, driverobj): assert regex_logged(caplog, "missing.nml: State: Not Ready (external asset)") +def test_MPAS_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_MPAS_provisioned_rundir(driverobj): with patch.multiple( driverobj, @@ -236,10 +248,6 @@ def test_MPAS_provisioned_rundir(driverobj): mocks[m].assert_called_once_with() -def test_MPAS_driver_name(driverobj): - assert driverobj.driver_name() == MPAS.driver_name() == "mpas" - - def test_MPAS_streams_file(config, driverobj): streams_file(config, driverobj, "mpas") diff --git a/src/uwtools/tests/drivers/test_mpas_init.py b/src/uwtools/tests/drivers/test_mpas_init.py index f9784755c..d785dd2b3 100644 --- a/src/uwtools/tests/drivers/test_mpas_init.py +++ b/src/uwtools/tests/drivers/test_mpas_init.py @@ -10,10 +10,11 @@ import f90nml # type: ignore from iotaa import refs -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.mpas_base import MPASBase from uwtools.drivers.mpas_init import MPASInit +from uwtools.exceptions import UWNotImplementedError from uwtools.logging import log from uwtools.tests.drivers.test_mpas import streams_file from uwtools.tests.support import fixture_path, logged, regex_logged @@ -112,6 +113,7 @@ def driverobj(config, cycle): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", "streams_file", @@ -137,6 +139,10 @@ def test_MPASInit_boundary_files(cycle, driverobj): assert all(link.is_symlink() for link in links) +def test_MPASInit_driver_name(driverobj): + assert driverobj.driver_name() == MPASInit.driver_name() == "mpas_init" + + @mark.parametrize( "key,task,test", [("files_to_copy", "files_copied", "is_file"), ("files_to_link", "files_linked", "is_symlink")], @@ -196,6 +202,12 @@ def test_MPASInit_namelist_file_missing_base_file(caplog, driverobj): assert regex_logged(caplog, "missing.nml: State: Not Ready (external asset)") +def test_MPASInit_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_MPASInit_provisioned_rundir(driverobj): with patch.multiple( driverobj, @@ -211,10 +223,6 @@ def test_MPASInit_provisioned_rundir(driverobj): mocks[m].assert_called_once_with() -def test_MPASInit_driver_name(driverobj): - assert driverobj.driver_name() == MPASInit.driver_name() == "mpas_init" - - def test_MPASInit_streams_file(config, driverobj): streams_file(config, driverobj, "mpas_init") diff --git a/src/uwtools/tests/drivers/test_orog.py b/src/uwtools/tests/drivers/test_orog.py new file mode 100644 index 000000000..699e4df94 --- /dev/null +++ b/src/uwtools/tests/drivers/test_orog.py @@ -0,0 +1,178 @@ +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +""" +Orog driver tests. +""" +import logging +from pathlib import Path +from unittest.mock import DEFAULT as D +from unittest.mock import patch + +from pytest import fixture, mark, raises + +from uwtools.drivers.driver import Driver +from uwtools.drivers.orog import Orog +from uwtools.exceptions import UWNotImplementedError +from uwtools.logging import log +from uwtools.scheduler import Slurm +from uwtools.tests.support import regex_logged + +# Fixtures + + +@fixture +def config(tmp_path): + afile = tmp_path / "afile" + afile.touch() + return { + "orog": { + "old_line1_items": { + "blat": 0, + "efac": 0, + "jcap": 0, + "latb": 0, + "lonb": 0, + "mtnres": 1, + "nr": 0, + "nf1": 0, + "nf2": 0, + }, + "execution": { + "batchargs": { + "walltime": "00:01:00", + }, + "executable": "/path/to/orog", + }, + "files_to_link": { + "foo": str(tmp_path / "foo"), + "bar": str(tmp_path / "bar"), + }, + "grid_file": str(tmp_path / "grid_file.in"), + "orog_file": "none", + "rundir": str(tmp_path / "run"), + }, + "platform": { + "account": "me", + "scheduler": "slurm", + }, + } + + +@fixture +def driverobj(config): + return Orog(config=config, batch=True) + + +# Tests + + +@mark.parametrize( + "method", + [ + "_run_resources", + "_run_via_batch_submission", + "_run_via_local_execution", + "_runscript", + "_runscript_done_file", + "_runscript_path", + "_scheduler", + "_validate", + "_write_runscript", + "output", + "run", + "taskname", + ], +) +def test_Orog(method): + assert getattr(Orog, method) is getattr(Driver, method) + + +def test_Orog_driver_name(driverobj): + assert driverobj.driver_name() == Orog.driver_name() == "orog" + + +def test_Orog_files_linked(driverobj): + for _, src in driverobj.config["files_to_link"].items(): + Path(src).touch() + for dst, _ in driverobj.config["files_to_link"].items(): + assert not (driverobj.rundir / dst).is_file() + driverobj.files_linked() + for dst, _ in driverobj.config["files_to_link"].items(): + assert (driverobj.rundir / dst).is_symlink() + + +@mark.parametrize("exist", [True, False]) +def test_Orog_grid_file_existence(caplog, driverobj, exist): + log.setLevel(logging.DEBUG) + grid_file = Path(driverobj.config["grid_file"]) + status = f"Input grid file {str(grid_file)}: State: Not Ready (external asset)" + if exist: + grid_file.touch() + status = f"Input grid file {str(grid_file)}: State: Ready" + driverobj.grid_file() + assert regex_logged(caplog, status) + + +def test_Orog_grid_file_nonexistence(caplog, driverobj): + log.setLevel(logging.INFO) + driverobj._config["grid_file"] = "none" + driverobj.grid_file() + assert regex_logged(caplog, "Input grid file none: State: Ready") + + +def test_Orog_input_config_file_new(driverobj): + del driverobj._config["old_line1_items"] + del driverobj._config["orog_file"] + grid_file = Path(driverobj.config["grid_file"]) + grid_file.touch() + driverobj.input_config_file() + with open(driverobj._input_config_path, "r", encoding="utf-8") as inps: + content = inps.readlines() + content = [l.strip("\n") for l in content] + assert len(content) == 3 + assert content[0] == "'{}'".format(driverobj.config["grid_file"]) + assert content[1] == ".false." + assert content[2] == "none" + + +def test_Orog_input_config_file_old(driverobj): + grid_file = Path(driverobj.config["grid_file"]) + grid_file.touch() + driverobj.input_config_file() + with open(driverobj._input_config_path, "r", encoding="utf-8") as inps: + content = inps.readlines() + content = [l.strip("\n") for l in content] + assert len(content) == 5 + assert len(content[0].split()) == 9 + assert content[1] == "'{}'".format(driverobj.config["grid_file"]) + assert content[2] == "'{}'".format(driverobj.config["orog_file"]) + assert content[3] == ".false." + assert content[4] == "none" + + +def test_Orog_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + +def test_Orog_provisioned_rundir(driverobj): + with patch.multiple(driverobj, files_linked=D, input_config_file=D, runscript=D) as mocks: + driverobj.provisioned_rundir() + for m in mocks: + mocks[m].assert_called_once_with() + + +def test_Orog_runscript(driverobj): + with patch.object(driverobj, "_runscript") as runscript: + driverobj.runscript() + runscript.assert_called_once() + args = ("envcmds", "envvars", "execution", "scheduler") + types = [list, dict, list, Slurm] + assert [type(runscript.call_args.kwargs[x]) for x in args] == types + + +def test_Orog__runcmd(driverobj): + assert driverobj._runcmd == "%s < %s" % ( + driverobj.config["execution"]["executable"], + driverobj._input_config_path.name, + ) diff --git a/src/uwtools/tests/drivers/test_orog_gsl.py b/src/uwtools/tests/drivers/test_orog_gsl.py index a942aa687..3d74e8505 100644 --- a/src/uwtools/tests/drivers/test_orog_gsl.py +++ b/src/uwtools/tests/drivers/test_orog_gsl.py @@ -6,10 +6,11 @@ from unittest.mock import DEFAULT as D from unittest.mock import patch -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.driver import Driver from uwtools.drivers.orog_gsl import OrogGSL +from uwtools.exceptions import UWNotImplementedError # Fixtures @@ -63,6 +64,7 @@ def driverobj(config): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", "taskname", @@ -72,6 +74,20 @@ def test_OrogGSL(method): assert getattr(OrogGSL, method) is getattr(Driver, method) +def test_OrogGSL_driver_name(driverobj): + assert driverobj.driver_name() == OrogGSL.driver_name() == "orog_gsl" + + +def test_OrogGSL_input_config_file(driverobj): + driverobj.input_config_file() + inputs = [str(driverobj.config["config"][k]) for k in ("tile", "resolution", "halo")] + with open(driverobj._input_config_path, "r", encoding="utf-8") as cfg_file: + content = cfg_file.readlines() + content = [l.strip("\n") for l in content] + assert len(content) == 3 + assert content == inputs + + def test_OrogGSL_input_grid_file(driverobj): path = Path(driverobj.config["rundir"], "C403_grid.tile7.halo4.nc") assert not path.is_file() @@ -79,9 +95,20 @@ def test_OrogGSL_input_grid_file(driverobj): assert path.is_symlink() +def test_OrogGSL_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_OrogGSL_provisioned_rundir(driverobj): with patch.multiple( - driverobj, input_grid_file=D, runscript=D, topo_data_2p5m=D, topo_data_30s=D + driverobj, + input_config_file=D, + input_grid_file=D, + runscript=D, + topo_data_2p5m=D, + topo_data_30s=D, ) as mocks: driverobj.provisioned_rundir() for m in mocks: @@ -102,13 +129,8 @@ def test_OrogGSL_topo_data_3os(driverobj): assert path.is_symlink() -def test_OrogGSL_driver_name(driverobj): - assert driverobj.driver_name() == OrogGSL.driver_name() == "orog_gsl" - - def test_OrogGSL__runcmd(driverobj): - inputs = [str(driverobj.config["config"][k]) for k in ("tile", "resolution", "halo")] - assert driverobj._runcmd == "echo '%s' | %s" % ( - "\n".join(inputs), + assert driverobj._runcmd == "%s < %s" % ( driverobj.config["execution"]["executable"], + driverobj._input_config_path.name, ) diff --git a/src/uwtools/tests/drivers/test_schism.py b/src/uwtools/tests/drivers/test_schism.py index 0c5fea845..8800174f1 100644 --- a/src/uwtools/tests/drivers/test_schism.py +++ b/src/uwtools/tests/drivers/test_schism.py @@ -51,6 +51,10 @@ def test_SCHISM(method): assert getattr(SCHISM, method) is getattr(AssetsCycleBased, method) +def test_SCHISM_driver_name(driverobj): + assert driverobj.driver_name() == SCHISM.driver_name() == "schism" + + def test_SCHISM_namelist_file(driverobj): src = driverobj.config["namelist"]["template_file"] with open(src, "w", encoding="utf-8") as f: @@ -69,7 +73,3 @@ def test_SCHISM_provisioned_rundir(driverobj): driverobj.provisioned_rundir() for m in mocks: mocks[m].assert_called_once_with() - - -def test_SCHISM_driver_name(driverobj): - assert driverobj.driver_name() == SCHISM.driver_name() == "schism" diff --git a/src/uwtools/tests/drivers/test_sfc_climo_gen.py b/src/uwtools/tests/drivers/test_sfc_climo_gen.py index a9355f42d..6bd7b7281 100644 --- a/src/uwtools/tests/drivers/test_sfc_climo_gen.py +++ b/src/uwtools/tests/drivers/test_sfc_climo_gen.py @@ -9,11 +9,12 @@ import f90nml # type: ignore from iotaa import asset, external, refs -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers import sfc_climo_gen from uwtools.drivers.driver import Driver from uwtools.drivers.sfc_climo_gen import SfcClimoGen +from uwtools.exceptions import UWNotImplementedError from uwtools.logging import log from uwtools.tests.support import logged @@ -97,6 +98,7 @@ def driverobj(config): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", "taskname", @@ -106,6 +108,10 @@ def test_SfcClimoGen(method): assert getattr(SfcClimoGen, method) is getattr(Driver, method) +def test_SfcClimoGen_driver_name(driverobj): + assert driverobj.driver_name() == SfcClimoGen.driver_name() == "sfc_climo_gen" + + def test_SfcClimoGen_namelist_file(caplog, driverobj): log.setLevel(logging.DEBUG) dst = driverobj.rundir / "fort.41" @@ -127,6 +133,12 @@ def test_SfcClimoGen_namelist_file_fails_validation(caplog, driverobj): assert logged(caplog, " 'string' is not of type 'integer'") +def test_SfcClimoGen_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_SfcClimoGen_provisioned_rundir(driverobj): with patch.multiple( driverobj, @@ -136,7 +148,3 @@ def test_SfcClimoGen_provisioned_rundir(driverobj): driverobj.provisioned_rundir() for m in mocks: mocks[m].assert_called_once_with() - - -def test_SfcClimoGen_driver_name(driverobj): - assert driverobj.driver_name() == SfcClimoGen.driver_name() == "sfc_climo_gen" diff --git a/src/uwtools/tests/drivers/test_shave.py b/src/uwtools/tests/drivers/test_shave.py index b6ac9c2a2..71696c416 100644 --- a/src/uwtools/tests/drivers/test_shave.py +++ b/src/uwtools/tests/drivers/test_shave.py @@ -2,13 +2,15 @@ """ Shave driver tests. """ +from pathlib import Path from unittest.mock import DEFAULT as D from unittest.mock import patch -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.driver import Driver from uwtools.drivers.shave import Shave +from uwtools.exceptions import UWNotImplementedError # Fixtures @@ -27,8 +29,9 @@ def config(tmp_path): "executable": "/path/to/shave", }, "config": { - "input_grid_file": "/path/to/input/grid/file.nc", - "nh4": 1, + "input_grid_file": str(tmp_path / "input_file.nc"), + "output_grid_file": "/path/to/input/grid/file.nc", + "nhalo": 1, "nx": 214, "ny": 128, }, @@ -61,6 +64,7 @@ def driverobj(config): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", "taskname", @@ -70,9 +74,35 @@ def test_Shave(method): assert getattr(Shave, method) is getattr(Driver, method) +def test_Shave_driver_name(driverobj): + assert driverobj.driver_name() == Shave.driver_name() == "shave" + + +def test_Shave_input_config_file(driverobj): + nx = driverobj.config["config"]["nx"] + ny = driverobj.config["config"]["ny"] + nhalo = driverobj.config["config"]["nhalo"] + input_file_path = driverobj._config["config"]["input_grid_file"] + Path(input_file_path).touch() + output_file_path = driverobj._config["config"]["output_grid_file"] + driverobj.input_config_file() + with open(driverobj._input_config_path, "r", encoding="utf-8") as cfg_file: + content = cfg_file.readlines() + content = [l.strip("\n") for l in content] + assert len(content) == 1 + assert content[0] == f"{nx} {ny} {nhalo} '{input_file_path}' '{output_file_path}'" + + +def test_Shave_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_Shave_provisioned_rundir(driverobj): with patch.multiple( driverobj, + input_config_file=D, runscript=D, ) as mocks: driverobj.provisioned_rundir() @@ -80,15 +110,6 @@ def test_Shave_provisioned_rundir(driverobj): mocks[m].assert_called_once_with() -def test_Shave_driver_name(driverobj): - assert driverobj.driver_name() == Shave.driver_name() == "shave" - - def test_Shave__runcmd(driverobj): cmd = driverobj._runcmd - nx = driverobj.config["config"]["nx"] - ny = driverobj.config["config"]["ny"] - nh4 = driverobj.config["config"]["nh4"] - input_file_path = driverobj._config["config"]["input_grid_file"] - output_file_path = input_file_path.replace(".nc", "_NH0.nc") - assert cmd == f"/path/to/shave {nx} {ny} {nh4} {input_file_path} {output_file_path}" + assert cmd == "/path/to/shave < shave.cfg" diff --git a/src/uwtools/tests/drivers/test_support.py b/src/uwtools/tests/drivers/test_support.py index d9cf9545b..474382db0 100644 --- a/src/uwtools/tests/drivers/test_support.py +++ b/src/uwtools/tests/drivers/test_support.py @@ -41,6 +41,10 @@ class Child(Parent): def test_tasks(): class SomeDriver(DriverTimeInvariant): + @classmethod + def driver_name(cls): + pass + def provisioned_rundir(self): pass @@ -59,10 +63,6 @@ def t2(self): def t3(self): "@tasks t3" - @classmethod - def driver_name(cls): - pass - @property def _resources(self): pass @@ -73,6 +73,7 @@ def _validate(self, schema_file: Optional[Path] = None): assert support.tasks(SomeDriver) == { "run": "A run.", "runscript": "The runscript.", + "show_output": "Show the output to be created by this component.", "t1": "@external t1", "t2": "@task t2", "t3": "@tasks t3", diff --git a/src/uwtools/tests/drivers/test_ungrib.py b/src/uwtools/tests/drivers/test_ungrib.py index be2a15388..0a12af8a0 100644 --- a/src/uwtools/tests/drivers/test_ungrib.py +++ b/src/uwtools/tests/drivers/test_ungrib.py @@ -7,11 +7,12 @@ from unittest.mock import patch import f90nml # type: ignore -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers import ungrib from uwtools.drivers.driver import Driver from uwtools.drivers.ungrib import Ungrib +from uwtools.exceptions import UWNotImplementedError # Fixtures @@ -69,6 +70,7 @@ def driverobj(config, cycle): "_scheduler", "_validate", "_write_runscript", + "output", "run", "runscript", ], @@ -77,6 +79,10 @@ def test_Ungrib(method): assert getattr(Ungrib, method) is getattr(Driver, method) +def test_Ungrib_driver_name(driverobj): + assert driverobj.driver_name() == Ungrib.driver_name() == "ungrib" + + def test_Ungrib_gribfiles(driverobj, tmp_path): links = [] cycle_hr = 12 @@ -102,6 +108,12 @@ def test_Ungrib_namelist_file(driverobj): assert nml["share"]["end_date"] == "2024-02-02_06:00:00" +def test_Ungrib_output(driverobj): + with raises(UWNotImplementedError) as e: + assert driverobj.output + assert str(e.value) == "The output() method is not yet implemented for this driver" + + def test_Ungrib_provisioned_rundir(driverobj): with patch.multiple( driverobj, @@ -115,6 +127,10 @@ def test_Ungrib_provisioned_rundir(driverobj): mocks[m].assert_called_once_with() +def test_Ungrib_taskname(driverobj): + assert driverobj.taskname("foo") == "20240201 18Z ungrib foo" + + def test_Ungrib_vtable(driverobj): src = driverobj.rundir / "Vtable.GFS.in" src.touch() @@ -125,10 +141,6 @@ def test_Ungrib_vtable(driverobj): assert dst.is_symlink() -def test_Ungrib_driver_name(driverobj): - assert driverobj.driver_name() == Ungrib.driver_name() == "ungrib" - - def test_Ungrib__gribfile(driverobj): src = driverobj.rundir / "GRIBFILE.AAA.in" src.touch() @@ -138,10 +150,6 @@ def test_Ungrib__gribfile(driverobj): assert dst.is_symlink() -def test_Ungrib_taskname(driverobj): - assert driverobj.taskname("foo") == "20240201 18Z ungrib foo" - - def test__ext(): assert ungrib._ext(0) == "AAA" assert ungrib._ext(26) == "ABA" diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py index 55b34cb52..12a34dde7 100644 --- a/src/uwtools/tests/drivers/test_upp.py +++ b/src/uwtools/tests/drivers/test_upp.py @@ -10,10 +10,11 @@ import f90nml # type: ignore from iotaa import refs -from pytest import fixture, mark +from pytest import fixture, mark, raises from uwtools.drivers.driver import Driver from uwtools.drivers.upp import UPP +from uwtools.exceptions import UWConfigError from uwtools.logging import log from uwtools.tests.support import logged, regex_logged @@ -24,6 +25,7 @@ def config(tmp_path): return { "upp": { + "control_file": "/path/to/postxconfig-NT.txt", "execution": { "batchargs": { "cores": 1, @@ -97,6 +99,10 @@ def test_UPP(method): assert getattr(UPP, method) is getattr(Driver, method) +def test_UPP_driver_name(driverobj): + assert driverobj.driver_name() == UPP.driver_name() == "upp" + + def test_UPP_files_copied(driverobj): for _, src in driverobj.config["files_to_copy"].items(): Path(src).touch() @@ -121,7 +127,7 @@ def test_UPP_namelist_file(caplog, driverobj): log.setLevel(logging.DEBUG) datestr = "2024-05-05_12:00:00" with open(driverobj.config["namelist"]["base_file"], "w", encoding="utf-8") as f: - print("&model_inputs datestr='%s' / &nampgb kpv=88 /" % datestr, file=f) + print("&model_inputs datestr='%s' / &nampgb kpv=42 /" % datestr, file=f) dst = driverobj.rundir / "itag" assert not dst.is_file() path = Path(refs(driverobj.namelist_file())) @@ -132,7 +138,7 @@ def test_UPP_namelist_file(caplog, driverobj): assert nml["model_inputs"]["datestr"] == datestr assert nml["model_inputs"]["grib"] == "grib2" assert nml["nampgb"]["kpo"] == 3 - assert nml["nampgb"]["kpv"] == 88 + assert nml["nampgb"]["kpv"] == 42 def test_UPP_namelist_file_fails_validation(caplog, driverobj): @@ -154,6 +160,36 @@ def test_UPP_namelist_file_missing_base_file(caplog, driverobj): assert regex_logged(caplog, "missing.nml: State: Not Ready (external asset)") +def test_UPP_output(driverobj, tmp_path): + fields = ["?"] * (UPP.NFIELDS - 1) + parameters = ["?"] * UPP.NPARAMS + # fmt: off + control_data = [ + "2", # number of blocks + "1", # number variables in 2nd block + "2", # number variables in 1st block + "FOO", # 1st block identifier + *fields, # 1st block fields + *(parameters * 2) , # 1st block variable parameters + "BAR", # 2nd block identifier + *fields, # 2nd block fields + *parameters, # 2nd block variable parameters + ] + # fmt: on + control_file = tmp_path / "postxconfig-NT.txt" + with open(control_file, "w", encoding="utf-8") as f: + print("\n".join(control_data), file=f) + driverobj._config["control_file"] = str(control_file) + expected = {"gribfiles": [str(driverobj.rundir / ("%s.GrbF24" % x)) for x in ("FOO", "BAR")]} + assert driverobj.output == expected + + +def test_UPP_output_fail(driverobj): + with raises(UWConfigError) as e: + assert driverobj.output + assert str(e.value) == "Could not open UPP control file %s" % driverobj.config["control_file"] + + def test_UPP_provisioned_rundir(driverobj): with patch.multiple( driverobj, @@ -167,8 +203,8 @@ def test_UPP_provisioned_rundir(driverobj): mocks[m].assert_called_once_with() -def test_UPP_driver_name(driverobj): - assert driverobj.driver_name() == UPP.driver_name() == "upp" +def test_UPP_taskname(driverobj): + assert driverobj.taskname("foo") == "20240507 12:00:00 upp foo" def test_UPP__namelist_path(driverobj): @@ -177,7 +213,3 @@ def test_UPP__namelist_path(driverobj): def test_UPP__runcmd(driverobj): assert driverobj._runcmd == "%s < itag" % driverobj.config["execution"]["executable"] - - -def test_UPP_taskname(driverobj): - assert driverobj.taskname("foo") == "20240507 12:00:00 upp foo" diff --git a/src/uwtools/tests/drivers/test_ww3.py b/src/uwtools/tests/drivers/test_ww3.py index 253080387..04a4aea90 100644 --- a/src/uwtools/tests/drivers/test_ww3.py +++ b/src/uwtools/tests/drivers/test_ww3.py @@ -51,6 +51,10 @@ def test_WaveWatchIII(method): assert getattr(WaveWatchIII, method) is getattr(AssetsCycleBased, method) +def test_WaveWatchIII_driver_name(driverobj): + assert driverobj.driver_name() == WaveWatchIII.driver_name() == "ww3" + + def test_WaveWatchIII_namelist_file(driverobj): src = driverobj.config["namelist"]["template_file"] with open(src, "w", encoding="utf-8") as f: @@ -77,7 +81,3 @@ def test_WaveWatchIII_restart_directory(driverobj): assert not path.is_dir() driverobj.restart_directory() assert path.is_dir() - - -def test_WaveWatchIII_driver_name(driverobj): - assert driverobj.driver_name() == WaveWatchIII.driver_name() == "ww3" diff --git a/src/uwtools/tests/fixtures/hello_workflow.yaml b/src/uwtools/tests/fixtures/hello_workflow.yaml index be9a10607..51d8691ff 100644 --- a/src/uwtools/tests/fixtures/hello_workflow.yaml +++ b/src/uwtools/tests/fixtures/hello_workflow.yaml @@ -15,9 +15,9 @@ workflow: task_hello: attrs: cycledefs: howdy - maxtries: "2" + maxtries: 2 account: "&ACCOUNT;" - command: "echo hello $person" + command: "echo account for $person is &ACCOUNT; && true" jobname: cyclestr: attrs: @@ -34,7 +34,7 @@ workflow: task_hello_#member#: attrs: cycledefs: howdy - maxtries: "1" + maxtries: 1 account: "&ACCOUNT;" command: "echo hello #member#" nodes: 1:ppn=1 diff --git a/src/uwtools/tests/fixtures/include_files.ini b/src/uwtools/tests/fixtures/include_files.ini index a096c1ac7..5d337b1d7 100644 --- a/src/uwtools/tests/fixtures/include_files.ini +++ b/src/uwtools/tests/fixtures/include_files.ini @@ -1,4 +1,4 @@ [config] -salad_include = !INCLUDE [./fruit_config.ini] +salad_include = !include [./fruit_config.ini] meat = beef dressing = poppyseed diff --git a/src/uwtools/tests/fixtures/include_files.nml b/src/uwtools/tests/fixtures/include_files.nml index 6df874c6b..fa90c6a5e 100644 --- a/src/uwtools/tests/fixtures/include_files.nml +++ b/src/uwtools/tests/fixtures/include_files.nml @@ -1,5 +1,5 @@ &config - salad_include = '!INCLUDE [./fruit_config.nml]' + salad_include = '!include [./fruit_config.nml]' meat = beef dressing = poppyseed / diff --git a/src/uwtools/tests/fixtures/include_files.sh b/src/uwtools/tests/fixtures/include_files.sh index 9b8ab8e08..d11786be6 100644 --- a/src/uwtools/tests/fixtures/include_files.sh +++ b/src/uwtools/tests/fixtures/include_files.sh @@ -1,3 +1,3 @@ -salad_include="!INCLUDE [./fruit_config.sh]" +salad_include="!include [./fruit_config.sh]" meat=beef dressing=poppyseed diff --git a/src/uwtools/tests/fixtures/include_files.yaml b/src/uwtools/tests/fixtures/include_files.yaml index 537f1b0d9..6fbc80bdc 100644 --- a/src/uwtools/tests/fixtures/include_files.yaml +++ b/src/uwtools/tests/fixtures/include_files.yaml @@ -1,3 +1,3 @@ -salad: !INCLUDE [./fruit_config.yaml] -two_files: !INCLUDE [./fruit_config.yaml, ./fruit_config_similar.yaml] -reverse_files: !INCLUDE [./fruit_config_similar.yaml, ./fruit_config.yaml] +salad: !include [./fruit_config.yaml] +two_files: !include [./fruit_config.yaml, ./fruit_config_similar.yaml] +reverse_files: !include [./fruit_config_similar.yaml, ./fruit_config.yaml] diff --git a/src/uwtools/tests/fixtures/include_files_with_sect.nml b/src/uwtools/tests/fixtures/include_files_with_sect.nml index 6bac4f1dc..d80ad8af1 100644 --- a/src/uwtools/tests/fixtures/include_files_with_sect.nml +++ b/src/uwtools/tests/fixtures/include_files_with_sect.nml @@ -1,5 +1,5 @@ &config - salad_include = '!INCLUDE [./fruit_config_mult_sect.nml]' + salad_include = '!include [./fruit_config_mult_sect.nml]' meat = beef dressing = poppyseed / diff --git a/src/uwtools/tests/fixtures/testdriver.py b/src/uwtools/tests/fixtures/testdriver.py index 025a7df55..e01754275 100644 --- a/src/uwtools/tests/fixtures/testdriver.py +++ b/src/uwtools/tests/fixtures/testdriver.py @@ -9,12 +9,12 @@ class TestDriver(AssetsCycleBased): """ @task - def eighty_eight(self): + def forty_two(self): """ - 88 + 42 """ - yield "88" - yield asset(88, lambda: True) + yield "42" + yield asset(42, lambda: True) yield None @classmethod diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index 09bab82e0..d64b458c2 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -175,7 +175,7 @@ def test__dispatch_execute(): "dry_run": False, "graph_file": None, "key_path": ["foo", "bar"], - "task": "eighty_eight", + "task": "forty_two", "stdin_ok": True, } with patch.object(cli.uwtools.api.execute, "execute") as execute: @@ -183,7 +183,7 @@ def test__dispatch_execute(): execute.assert_called_once_with( classname="TestDriver", module="testdriver", - task="eighty_eight", + task="forty_two", schema_file="/path/to/testdriver.jsonschema", key_path=["foo", "bar"], dry_run=False, @@ -373,7 +373,7 @@ def test__dispatch_config_validate_config_obj(): STR.schemafile: _dispatch_config_validate_args[STR.schemafile], STR.config: _dispatch_config_validate_args[STR.infile], } - _validate_external.assert_called_once_with(**_validate_external_args) + _validate_external.assert_called_once_with(**_validate_external_args, desc="config") @mark.parametrize( @@ -475,7 +475,7 @@ def test__dispatch_template_render_fail(valsneeded): STR.outfile: 2, STR.valsfile: 3, STR.valsfmt: 4, - STR.keyvalpairs: ["foo=88", "bar=99"], + STR.keyvalpairs: ["foo=42", "bar=43"], STR.env: 5, STR.searchpath: 6, STR.valsneeded: valsneeded, @@ -519,7 +519,7 @@ def test__dispatch_template_render_yaml(): STR.outfile: 2, STR.valsfile: 3, STR.valsfmt: 4, - STR.keyvalpairs: ["foo=88", "bar=99"], + STR.keyvalpairs: ["foo=42", "bar=43"], STR.env: 5, STR.searchpath: 6, STR.valsneeded: 7, @@ -532,7 +532,7 @@ def test__dispatch_template_render_yaml(): output_file=2, values_src=3, values_format=4, - overrides={"foo": "88", "bar": "99"}, + overrides={"foo": "42", "bar": "43"}, env=5, searchpath=6, values_needed=7, @@ -582,6 +582,7 @@ def test__dispatch_to_driver(hours): "dry_run": False, "graph_file": None, "key_path": ["foo", "bar"], + "schema_file": None, "show_schema": False, "stdin_ok": True, } @@ -596,6 +597,7 @@ def test__dispatch_to_driver(hours): dry_run=False, graph_file=None, key_path=["foo", "bar"], + schema_file=None, task="foo", stdin_ok=True, ) @@ -682,7 +684,7 @@ def test_main_fail_exception_log(): def test__parse_args(): - raw_args = ["testing", "--bar", "88"] + raw_args = ["testing", "--bar", "42"] with patch.object(cli, "Parser") as Parser: cli._parse_args(raw_args) Parser.assert_called_once() diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index b83a4fbdf..b313267bd 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -104,7 +104,7 @@ def test_Linker(assets, source): @mark.parametrize("source", ("dict", "file")) -def test_Stager__config_block_fail_bad_keypath(assets, source): +def test_Stager__config_block_fail_bad_key_path(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile with raises(UWConfigError) as e: @@ -112,10 +112,10 @@ def test_Stager__config_block_fail_bad_keypath(assets, source): assert str(e.value) == "Failed following YAML key(s): a -> x" -@mark.parametrize("val", [None, True, False, "str", 88, 3.14, [], tuple()]) +@mark.parametrize("val", [None, True, False, "str", 42, 3.14, [], tuple()]) def test_Stager__config_block_fails_bad_type(assets, val): dstdir, cfgdict, _ = assets cfgdict["a"]["b"] = val with raises(UWConfigError) as e: ConcreteStager(target_dir=dstdir, config=cfgdict, keys=["a", "b"]) - assert str(e.value) == "Expected block not found at key path: a -> b" + assert str(e.value) == "Expected block not found at key path: a.b" diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index 2c8b3fb67..94bf39327 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -6,12 +6,13 @@ from unittest.mock import DEFAULT as D from unittest.mock import PropertyMock, patch +from lxml import etree from pytest import fixture, mark, raises from uwtools import rocoto from uwtools.config.formats.yaml import YAMLConfig from uwtools.exceptions import UWConfigError, UWError -from uwtools.tests.support import fixture_path +from uwtools.tests.support import fixture_path, schema_validator # Fixtures @@ -110,50 +111,93 @@ def test_instantiate_from_cfgobj(self, assets): cfgfile, _ = assets assert rocoto._RocotoXML(config=YAMLConfig(cfgfile))._root.tag == "workflow" - def test__add_compound_time_string_basic(self, instance, root): - config = "bar" + @mark.parametrize("config", ["bar", 42]) + def test__add_compound_time_string_basic(self, config, instance, root): instance._add_compound_time_string(e=root, config=config, tag="foo") child = root[0] assert child.tag == "foo" - assert child.text == "bar" + assert child.text == str(config) def test__add_compound_time_string_cyclestr(self, instance, root): - config = {"attrs": {"bar": "88"}, "cyclestr": {"attrs": {"baz": "99"}, "value": "qux"}} + config = {"cyclestr": {"attrs": {"offset": "00:05:00"}, "value": "qux"}} + errors = schema_validator("rocoto", "$defs", "cycleString") + assert not errors(config) instance._add_compound_time_string(e=root, config=config, tag="foo") - child = root[0] - assert child.get("bar") == "88" - cyclestr = child[0] - assert cyclestr.get("baz") == "99" + cyclestr = root[0][0] + assert cyclestr.get("offset") == "00:05:00" assert cyclestr.text == "qux" + def test__add_compound_time_string_list(self, instance, root): + config = [ + "cycle-", + {"cyclestr": {"value": "%s"}}, + "-valid-", + {"cyclestr": {"value": "%s", "attrs": {"offset": "00:06:00"}}}, + ".log", + ] + errors = schema_validator("rocoto", "$defs", "compoundTimeString") + assert not errors(config) + xml = "{}".format( + "".join( + [ + "cycle-", + "%s", + "-valid-", + '%s', + ".log", + ] + ) + ) + instance._add_compound_time_string(e=root, config=config, tag="a") + assert etree.tostring(root[0]).decode("utf-8") == xml + def test__add_metatask(self, instance, root): config = { - "metatask_foo": "1", - "attrs": {"mode": "parallel", "throttle": 88}, - "task_bar": "2", + "attrs": {"mode": "parallel", "throttle": 42}, "var": {"baz": "3", "qux": "4"}, + "metatask_nest": { + "var": {"bar": "1 2 3"}, + "task_bar": { + "cores": 2, + "walltime": "00:10:00", + "command": "echo hello", + }, + }, } - taskname = "test-metatask" + errors = schema_validator("rocoto", "$defs") + metataskname = "metatask_test" + assert not errors({metataskname: config}) orig = instance._add_metatask with patch.multiple(instance, _add_metatask=D, _add_task=D) as mocks: - orig(e=root, config=config, name_attr=taskname) + orig(e=root, config=config, name_attr=metataskname) metatask = root[0] assert metatask.tag == "metatask" assert metatask.get("mode") == "parallel" - assert metatask.get("name") == taskname - assert metatask.get("throttle") == "88" + assert metatask.get("name") == metataskname + assert metatask.get("throttle") == "42" assert {e.get("name"): e.text for e in metatask.xpath("var")} == {"baz": "3", "qux": "4"} - mocks["_add_metatask"].assert_called_once_with(metatask, "1", "foo") - mocks["_add_task"].assert_called_once_with(metatask, "2", "bar") + mocks["_add_metatask"].assert_called_once_with( + metatask, + { + "var": {"bar": "1 2 3"}, + "task_bar": {"cores": 2, "walltime": "00:10:00", "command": "echo hello"}, + }, + "nest", + ) def test__add_task(self, instance, root): config = { "attrs": {"foo": "1", "bar": "2"}, "account": "baz", - "dependency": "qux", + "dependency": {"taskdep": {"attrs": {"task": "task_foo"}}}, "envars": {"A": "apple"}, + "walltime": "00:12:00", + "command": "echo hello", + "cores": 2, } - taskname = "test-task" + taskname = "task_test" + errors = schema_validator("rocoto", "$defs") + assert not errors({taskname: config}) with patch.multiple(instance, _add_task_dependency=D, _add_task_envar=D) as mocks: instance._add_task(e=root, config=config, name_attr=taskname) task = root[0] @@ -161,7 +205,9 @@ def test__add_task(self, instance, root): assert task.get("name") == taskname assert task.get("foo") == "1" assert task.get("bar") == "2" - mocks["_add_task_dependency"].assert_called_once_with(task, "qux") + mocks["_add_task_dependency"].assert_called_once_with( + task, {"taskdep": {"attrs": {"task": "task_foo"}}} + ) mocks["_add_task_envar"].assert_called_once_with(task, "A", "apple") @mark.parametrize("cores", [1, "1"]) @@ -172,6 +218,8 @@ def test__add_task_cores_int_or_str(self, cores, instance, root): def test__add_task_dependency_and(self, instance, root): config = {"and": {"or_get_obs": {"taskdep": {"attrs": {"task": "foo"}}}}} + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors(config) instance._add_task_dependency(e=root, config=config) dependency = root[0] assert dependency.tag == "dependency" @@ -187,6 +235,8 @@ def test__add_task_dependency_datadep(self, instance, root, value): age = "00:00:02:00" minsize = "1K" config = {"datadep": {"attrs": {"age": age, "minsize": minsize}, "value": value}} + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors(config) instance._add_task_dependency(e=root, config=config) dependency = root[0] assert dependency.tag == "dependency" @@ -208,6 +258,8 @@ def test__add_task_dependency_fail_bad_operand(self, instance, root): def test__add_task_dependency_metataskdep(self, instance, root): config = {"metataskdep": {"attrs": {"metatask": "foo"}}} + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors(config) instance._add_task_dependency(e=root, config=config) dependency = root[0] assert dependency.tag == "dependency" @@ -217,10 +269,12 @@ def test__add_task_dependency_metataskdep(self, instance, root): @mark.parametrize( "tag_config", - [("and", {"strneq": {"attrs": {"left": "&RUN_GSI;", "right": "YES"}}})], + [("and", {"strneq": {"left": "&RUN_GSI;", "right": "YES"}})], ) def test__add_task_dependency_operator(self, instance, root, tag_config): tag, config = tag_config + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors(config) instance._add_task_dependency_child(e=root, config=config, tag=tag) for tag, _ in config.items(): assert tag == next(iter(config)) @@ -228,6 +282,8 @@ def test__add_task_dependency_operator(self, instance, root, tag_config): def test__add_task_dependency_operator_datadep_operand(self, instance, root): value = "/some/file" config = {"value": value} + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors({"datadep": config}) instance._add_task_dependency_child(e=root, config=config, tag="datadep") e = root[0] assert e.tag == "datadep" @@ -236,6 +292,8 @@ def test__add_task_dependency_operator_datadep_operand(self, instance, root): def test__add_task_dependency_operator_task_operand(self, instance, root): taskname = "some-task" config = {"attrs": {"task": taskname}} + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors({"taskdep": config}) instance._add_task_dependency_child(e=root, config=config, tag="taskdep") e = root[0] assert e.tag == "taskdep" @@ -244,6 +302,8 @@ def test__add_task_dependency_operator_task_operand(self, instance, root): def test__add_task_dependency_operator_timedep_operand(self, instance, root): value = 20230103120000 config = value + errors = schema_validator("rocoto", "$defs", "compoundTimeString") + assert not errors(config) instance._add_task_dependency_child(e=root, config=config, tag="timedep") e = root[0] assert e.tag == "timedep" @@ -251,6 +311,8 @@ def test__add_task_dependency_operator_timedep_operand(self, instance, root): def test__add_task_dependency_sh(self, instance, root): config = {"sh_foo": {"attrs": {"runopt": "-c", "shell": "/bin/bash"}, "command": "ls"}} + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors(config) instance._add_task_dependency(e=root, config=config) dependency = root[0] assert dependency.tag == "dependency" @@ -262,31 +324,38 @@ def test__add_task_dependency_sh(self, instance, root): assert sh.text == "ls" def test__add_task_dependency_streq(self, instance, root): - config = {"streq": {"attrs": {"left": "&RUN_GSI;", "right": "YES"}}} + config = {"streq": {"left": "&RUN_GSI;", "right": "YES"}} + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors(config) instance._add_task_dependency(e=root, config=config) dependency = root[0] assert dependency.tag == "dependency" streq = dependency[0] assert streq.tag == "streq" - assert streq.get("left") == "&RUN_GSI;" + assert streq[0].text == "&RUN_GSI;" + assert streq[1].text == "YES" @mark.parametrize( "config", [ - ("streq", {"attrs": {"left": "&RUN_GSI;", "right": "YES"}}), - ("strneq", {"attrs": {"left": "&RUN_GSI;", "right": "YES"}}), + ("streq", {"left": "&RUN_GSI;", "right": "YES"}), + ("strneq", {"left": "&RUN_GSI;", "right": "YES"}), ], ) def test__add_task_dependency_strequality(self, config, instance, root): + errors = schema_validator("rocoto", "$defs", "dependency") tag, config = config + assert not errors({tag: config}) instance._add_task_dependency_strequality(e=root, config=config, tag=tag) element = root[0] assert tag == element.tag - for attr, val in config["attrs"].items(): - assert element.get(attr) == val + for idx, val in enumerate(config.values()): + assert element[idx].text == val def test__add_task_dependency_taskdep(self, instance, root): config = {"taskdep": {"attrs": {"task": "foo"}}} + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors(config) instance._add_task_dependency(e=root, config=config) dependency = root[0] assert dependency.tag == "dependency" @@ -296,6 +365,8 @@ def test__add_task_dependency_taskdep(self, instance, root): def test__add_task_dependency_taskvalid(self, instance, root): config = {"taskvalid": {"attrs": {"task": "foo"}}} + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors(config) instance._add_task_dependency(e=root, config=config) dependency = root[0] assert dependency.tag == "dependency" @@ -313,6 +384,8 @@ def test__add_task_dependency_taskvalid(self, instance, root): ) def test__add_task_dependency_timedep(self, instance, root, value): config = {"timedep": value} + errors = schema_validator("rocoto", "$defs", "dependency") + assert not errors(config) instance._add_task_dependency(e=root, config=config) dependency = root[0] assert dependency.tag == "dependency" @@ -368,29 +441,39 @@ def test__add_task_envar_compound(self, instance, root): def test__add_workflow(self, instance): config = { "workflow": { - "attrs": {"foo": "1", "bar": "2"}, - "cycledef": "3", - "log": "4", - "tasks": "5", + "attrs": {"realtime": True, "scheduler": "slurm"}, + "cycledef": [], + "log": "1", + "tasks": { + "task_foo": { + "command": "echo hello", + "cores": 1, + "walltime": "00:01:00", + }, + }, } } + errors = schema_validator("rocoto") + assert not errors(config) with patch.multiple( instance, _add_workflow_cycledef=D, _add_workflow_log=D, _add_workflow_tasks=D ) as mocks: instance._add_workflow(config=config) workflow = instance._root assert workflow.tag == "workflow" - assert workflow.get("foo") == "1" - assert workflow.get("bar") == "2" - mocks["_add_workflow_cycledef"].assert_called_once_with(workflow, "3") + assert workflow.get("realtime") == "True" + assert workflow.get("scheduler") == "slurm" + mocks["_add_workflow_cycledef"].assert_called_once_with(workflow, []) mocks["_add_workflow_log"].assert_called_once_with(workflow, config["workflow"]) - mocks["_add_workflow_tasks"].assert_called_once_with(workflow, "5") + mocks["_add_workflow_tasks"].assert_called_once_with(workflow, config["workflow"]["tasks"]) def test__add_workflow_cycledef(self, instance, root): config: list[dict] = [ {"attrs": {"group": "g1"}, "spec": "t1"}, {"attrs": {"group": "g2"}, "spec": "t2"}, ] + errors = schema_validator("rocoto", "$defs") + assert not errors({"cycledef": config}) instance._add_workflow_cycledef(e=root, config=config) for i, item in enumerate(config): assert root[i].get("group") == item["attrs"]["group"] diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index ee0dd3043..8265423c7 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -297,6 +297,29 @@ def ww3_prop(): return partial(schema_validator, "ww3", "properties", "ww3", "properties") +# Helpers + + +def non_empty_dict(errors: list[str]) -> bool: + for msg in [ + "{} does not have enough properties", # jsonschema [4.18.0,4.20.*] + "{} should be non-empty", # jsonschema [4.21.0,?] + ]: + if msg in errors: + return True + return False + + +def non_empty_list(errors: list[str]) -> bool: + for msg in [ + "[] is too short", # jsonschema [4.18.0,4.20.*] + "[] should be non-empty", # jsonschema [4.21.0,?] + ]: + if msg in errors: + return True + return False + + # batchargs @@ -308,7 +331,7 @@ def test_schema_batchargs(): # Managed properties are fine: assert not errors({"queue": "string", "walltime": "00:05:00"}) # But so are unknown ones: - assert not errors({"--foo": 88, "walltime": "00:05:00"}) + assert not errors({"--foo": 42, "walltime": "00:05:00"}) # It just has to be a map: assert "[] is not of type 'object'\n" in errors([]) # The "threads" argument is not allowed: It will be propagated, if set, from execution.threads. @@ -510,7 +533,7 @@ def test_schema_chgres_cube_namelist(chgres_cube_config, chgres_cube_prop): # Just base_file is ok: assert not errors(with_del(namelist, "update_values")) # base_file must be a string: - assert "88 is not of type 'string'\n" in errors(with_set(namelist, 88, "base_file")) + assert "42 is not of type 'string'\n" in errors(with_set(namelist, 42, "base_file")) # Just update_values is ok: assert not errors(with_del(namelist, "base_file")) # config is required with update_values: @@ -527,9 +550,8 @@ def test_schema_chgres_cube_namelist_update_values(chgres_cube_config, chgres_cu # Some entries are required: for key in ["mosaic_file_target_grid", "vcoord_file_target_grid"]: assert "is a required property" in errors(with_del(config, key)) - # Additional entries of namelist-compatible types are permitted: - for val in [[1, 2, 3], True, 88, 3.14, "bar"]: - assert not errors(with_set(config, val, "foo")) + # Additional top-level keys are not allowed: + assert "Additional properties are not allowed" in errors({**config, "foo": "bar"}) # Namelist values must be of the correct type: # boolean: for key in [ @@ -545,9 +567,6 @@ def test_schema_chgres_cube_namelist_update_values(chgres_cube_config, chgres_cu "wam_cold_start", ]: assert "not of type 'boolean'" in errors(with_set(config, None, key)) - # enum: - for key in ["external_model", "input_type"]: - assert "is not one of" in errors(with_set(config, None, key)) # integer: for key in [ "cycle_day", @@ -590,6 +609,7 @@ def test_schema_chgres_cube_namelist_update_values(chgres_cube_config, chgres_cu ]: assert "is not of type 'array', 'string'\n" in errors(with_set(config, None, key)) assert "is not of type 'string'\n" in errors(with_set(config, [1, 2, 3], key)) + assert not errors(with_set(config, ["foo", "bar", "baz"], key)) # esg-grid @@ -620,7 +640,7 @@ def test_schema_esg_grid_namelist(esg_grid_prop, esg_namelist): # Just base_file is ok: assert not errors(esg_namelist) # base_file must be a string: - assert "not valid" in errors({**esg_namelist, "base_file": 88}) + assert "not valid" in errors({**esg_namelist, "base_file": 42}) # Just update_values is ok, if it is complete: assert not errors(with_del(esg_namelist, "base_file")) # If base_file is not supplied, any missing namelist key is an error: @@ -639,13 +659,13 @@ def test_schema_esg_grid_namelist(esg_grid_prop, esg_namelist): def test_schema_esg_grid_namelist_content(key): config: dict = { "regional_grid_nml": { - "delx": 88, - "dely": 88, - "lx": 88, - "ly": 88, - "pazi": 88, - "plat": 88, - "plon": 88, + "delx": 42, + "dely": 42, + "lx": 42, + "ly": 42, + "pazi": 42, + "plat": 42, + "plon": 42, } } errors = partial(schema_validator("esg-grid", "$defs", "namelist_content")) @@ -664,18 +684,18 @@ def test_schema_esg_grid_rundir(esg_grid_prop): errors = esg_grid_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) -# execution +# execution-parallel -def test_schema_execution(): +def test_schema_parallel_execution(): config = {"executable": "fv3"} batchargs = {"batchargs": {"queue": "string", "walltime": "string"}} mpiargs = {"mpiargs": ["--flag1", "--flag2"]} threads = {"threads": 32} - errors = schema_validator("execution") + errors = schema_validator("execution-parallel") # Basic correctness: assert not errors(config) # batchargs may optionally be specified: @@ -692,26 +712,26 @@ def test_schema_execution(): ) -def test_schema_execution_executable(): - errors = schema_validator("execution", "properties", "executable") +def test_schema_parallel_execution_executable(): + errors = schema_validator("execution-parallel", "properties", "executable") # String value is ok: assert not errors("fv3.exe") # Anything else is not: - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) -def test_schema_execution_mpiargs(): - errors = schema_validator("execution", "properties", "mpiargs") +def test_schema_parallel_execution_mpiargs(): + errors = schema_validator("execution-parallel", "properties", "mpiargs") # Basic correctness: assert not errors(["string1", "string2"]) # mpiargs may be empty: assert not errors([]) # String values are expected: - assert "88 is not of type 'string'\n" in errors(["string1", 88]) + assert "42 is not of type 'string'\n" in errors(["string1", 42]) -def test_schema_execution_threads(): - errors = schema_validator("execution", "properties", "threads") +def test_schema_parallel_execution_threads(): + errors = schema_validator("execution-parallel", "properties", "threads") # threads must be non-negative, and an integer: assert not errors(1) assert not errors(4) @@ -725,7 +745,7 @@ def test_schema_execution_threads(): def test_schema_execution_serial(): config = {"executable": "fv3"} batchargs = {"batchargs": {"queue": "string", "walltime": "string"}} - errors = schema_validator("execution") + errors = schema_validator("execution-serial") # Basic correctness: assert not errors(config) # batchargs may optionally be specified: @@ -746,7 +766,7 @@ def test_schema_stage_files(): # A str -> str dict is ok: assert not errors({"file1": "/path/to/file1", "file2": "/path/to/file2"}) # An empty dict is not allowed: - assert "{} should be non-empty" in errors({}) + assert non_empty_dict(errors({})) # Non-string values are not allowed: assert "True is not of type 'string'\n" in errors({"file1": True}) @@ -758,6 +778,8 @@ def test_schema_filter_topo(): config = { "config": { "input_grid_file": "/path/to/grid/file", + "filtered_orog": "/path/to/filtered/orog/file", + "input_raw_orog": "/path/to/raw/orog/file", }, "execution": { "executable": "/path/to/filter_topo", @@ -791,7 +813,7 @@ def test_schema_filter_topo(): # Top-level rundir key requires a string value: assert "is not of type 'string'\n" in errors(with_set(config, None, "rundir")) # All config keys are requried: - for key in ["input_grid_file"]: + for key in ["filtered_orog", "input_grid_file", "input_raw_orog"]: assert f"'{key}' is a required property" in errors(with_del(config, "config", key)) # Other config keys are not allowed: assert "Additional properties are not allowed" in errors( @@ -883,7 +905,7 @@ def test_schema_fv3_diag_table(fv3_prop): # String value is ok: assert not errors("/path/to/file") # Anything else is not: - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) def test_schema_fv3_domain(fv3_prop): @@ -912,7 +934,7 @@ def test_schema_fv3_lateral_boundary_conditions(fv3_prop): assert "-1 is less than the minimum of 0" in errors(with_set(config, -1, "offset")) assert "'s' is not of type 'integer'\n" in errors(with_set(config, "s", "offset")) # path must be a string: - assert "88 is not of type 'string'\n" in errors(with_set(config, 88, "path")) + assert "42 is not of type 'string'\n" in errors(with_set(config, 42, "path")) def test_schema_fv3_length(fv3_prop): @@ -929,12 +951,12 @@ def test_schema_fv3_length(fv3_prop): def test_schema_fv3_model_configure(fv3_prop): base_file = {"base_file": "/some/path"} - update_values = {"update_values": {"foo": 88}} + update_values = {"update_values": {"foo": 42}} errors = fv3_prop("model_configure") # Just base_file is ok: assert not errors(base_file) # But base_file must be a string: - assert "88 is not of type 'string'\n" in errors({"base_file": 88}) + assert "42 is not of type 'string'\n" in errors({"base_file": 42}) # Just update_values is ok: assert not errors(update_values) # A combination of base_file and update_values is ok: @@ -946,11 +968,11 @@ def test_schema_fv3_model_configure(fv3_prop): def test_schema_fv3_model_configure_update_values(fv3_prop): errors = fv3_prop("model_configure", "properties", "update_values") # boolean, number, and string values are ok: - assert not errors({"bool": True, "int": 88, "float": 3.14, "string": "foo"}) + assert not errors({"bool": True, "int": 42, "float": 3.14, "string": "foo"}) # Other types are not, e.g.: assert "None is not of type 'boolean', 'number', 'string'\n" in errors({"null": None}) # At least one entry is required: - assert "{} should be non-empty" in errors({}) + assert non_empty_dict(errors({})) def test_schema_fv3_namelist(fv3_prop): @@ -960,7 +982,7 @@ def test_schema_fv3_namelist(fv3_prop): # Just base_file is ok: assert not errors(base_file) # base_file must be a string: - assert "88 is not of type 'string'\n" in errors({"base_file": 88}) + assert "42 is not of type 'string'\n" in errors({"base_file": 42}) # Just update_values is ok: assert not errors(update_values) # A combination of base_file and update_values is ok: @@ -973,23 +995,23 @@ def test_schema_fv3_namelist_update_values(fv3_prop): errors = fv3_prop("namelist", "properties", "update_values") # array, boolean, number, and string values are ok: assert not errors( - {"nml": {"array": [1, 2, 3], "bool": True, "int": 88, "float": 3.14, "string": "foo"}} + {"nml": {"array": [1, 2, 3], "bool": True, "int": 42, "float": 3.14, "string": "foo"}} ) # Other types are not, e.g.: assert "None is not of type 'array', 'boolean', 'number', 'string'\n" in errors( {"nml": {"null": None}} ) # At least one namelist entry is required: - assert "{} should be non-empty" in errors({}) + assert non_empty_dict(errors({})) # At least one val/var pair is required: - assert "{} should be non-empty" in errors({"nml": {}}) + assert non_empty_dict(errors({"nml": {}})) def test_schema_fv3_rundir(fv3_prop): errors = fv3_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # global-equiv-resol @@ -1016,7 +1038,7 @@ def test_schema_global_equiv_resol_paths(global_equiv_resol_prop, schema_entry): errors = global_equiv_resol_prop(schema_entry) # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # ioda @@ -1053,14 +1075,14 @@ def test_schema_ioda_configuration_file(ioda_prop): assert not errors(bf) assert not errors(uv) # update_values cannot be empty: - assert "should be non-empty" in errors({"update_values": {}}) + assert non_empty_dict(errors({"update_values": {}})) def test_schema_ioda_rundir(ioda_prop): errors = ioda_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # jedi @@ -1097,14 +1119,14 @@ def test_schema_jedi_configuration_file(jedi_prop): assert not errors(bf) assert not errors(uv) # update_values cannot be empty: - assert "should be non-empty" in errors({"update_values": {}}) + assert non_empty_dict(errors({"update_values": {}})) def test_schema_jedi_rundir(jedi_prop): errors = jedi_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # make-hgrid @@ -1166,7 +1188,7 @@ def test_schema_make_hgrid_rundir(make_hgrid_prop): errors = make_hgrid_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # make-solo-mosaic @@ -1195,7 +1217,7 @@ def test_schema_make_solo_mosaic_config(make_solo_mosaic_prop): assert f"'{key}' is a required property" in errors({}) # A string value is ok for dir: if key == "dir": - assert "not of type 'string'" in str(errors({key: 88})) + assert "not of type 'string'" in str(errors({key: 42})) # num_tiles must be an integer: else: assert "not of type 'integer'" in str(errors({key: "/path/"})) @@ -1209,7 +1231,7 @@ def test_schema_make_solo_mosaic_rundir(make_solo_mosaic_prop): errors = make_solo_mosaic_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # makedirs @@ -1222,7 +1244,7 @@ def test_schema_makedirs(): # Basic correctness: assert not errors({"makedirs": ["/path/to/dir1", "/path/to/dir2"]}) # An empty array is not allowed: - assert "[] should be non-empty" in errors({"makedirs": []}) + assert non_empty_list(errors({"makedirs": []})) # Non-string values are not allowed: assert "True is not of type 'string'\n" in errors({"makedirs": [True]}) @@ -1267,7 +1289,7 @@ def test_schema_mpas_lateral_boundary_conditions(mpas_prop): assert "-1 is less than the minimum of 0" in errors(with_set(config, -1, "offset")) assert "'s' is not of type 'integer'\n" in errors(with_set(config, "s", "offset")) # path must be a string: - assert "88 is not of type 'string'\n" in errors(with_set(config, 88, "path")) + assert "42 is not of type 'string'\n" in errors(with_set(config, 42, "path")) def test_schema_mpas_length(mpas_prop): @@ -1289,7 +1311,7 @@ def test_schema_mpas_namelist(mpas_prop): # Just base_file is ok: assert not errors(base_file) # base_file must be a string: - assert "88 is not of type 'string'\n" in errors({"base_file": 88}) + assert "42 is not of type 'string'\n" in errors({"base_file": 42}) # Just update_values is ok: assert not errors(update_values) # A combination of base_file and update_values is ok: @@ -1302,23 +1324,23 @@ def test_schema_mpas_namelist_update_values(mpas_prop): errors = mpas_prop("namelist", "properties", "update_values") # array, boolean, number, and string values are ok: assert not errors( - {"nml": {"array": [1, 2, 3], "bool": True, "int": 88, "float": 3.14, "string": "foo"}} + {"nml": {"array": [1, 2, 3], "bool": True, "int": 42, "float": 3.14, "string": "foo"}} ) # Other types are not, e.g.: assert "None is not of type 'array', 'boolean', 'number', 'string'\n" in errors( {"nml": {"null": None}} ) # At least one namelist entry is required: - assert "{} should be non-empty" in errors({}) + assert non_empty_dict(errors({})) # At least one val/var pair is required: - assert "{} should be non-empty" in errors({"nml": {}}) + assert non_empty_dict(errors({"nml": {}})) def test_schema_mpas_rundir(mpas_prop): errors = mpas_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # mpas-init @@ -1363,7 +1385,7 @@ def test_schema_mpas_init_boundary_conditions(mpas_init_prop): assert "-1 is less than the minimum of 0" in errors(with_set(config, -1, "offset")) assert "'s' is not of type 'integer'\n" in errors(with_set(config, "s", "offset")) # path must be a string: - assert "88 is not of type 'string'\n" in errors(with_set(config, 88, "path")) + assert "42 is not of type 'string'\n" in errors(with_set(config, 42, "path")) # length must be a positive int assert "0 is less than the minimum of 1" in errors(with_set(config, 0, "length")) assert "-1 is less than the minimum of 1" in errors(with_set(config, -1, "length")) @@ -1377,7 +1399,7 @@ def test_schema_mpas_init_namelist(mpas_init_prop): # Just base_file is ok: assert not errors(base_file) # base_file must be a string: - assert "88 is not of type 'string'\n" in errors({"base_file": 88}) + assert "42 is not of type 'string'\n" in errors({"base_file": 42}) # Just update_values is ok: assert not errors(update_values) # A combination of base_file and update_values is ok: @@ -1390,23 +1412,23 @@ def test_schema_mpas_init_namelist_update_values(mpas_init_prop): errors = mpas_init_prop("namelist", "properties", "update_values") # array, boolean, number, and string values are ok: assert not errors( - {"nml": {"array": [1, 2, 3], "bool": True, "int": 88, "float": 3.14, "string": "foo"}} + {"nml": {"array": [1, 2, 3], "bool": True, "int": 42, "float": 3.14, "string": "foo"}} ) # Other types are not, e.g.: assert "None is not of type 'array', 'boolean', 'number', 'string'\n" in errors( {"nml": {"null": None}} ) # At least one namelist entry is required: - assert "{} should be non-empty" in errors({}) + assert non_empty_dict(errors({})) # At least one val/var pair is required: - assert "{} should be non-empty" in errors({"nml": {}}) + assert non_empty_dict(errors({"nml": {}})) def test_schema_mpas_init_rundir(mpas_init_prop): errors = mpas_init_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # mpas-streams @@ -1478,7 +1500,7 @@ def test_schema_mpas_streams_properties_values_array(mpas_streams): for prop in ["files", "streams", "vars", "var_arrays", "var_structs"]: assert "is not of type 'array'\n" in errors({k: {**v, prop: None}}) assert "is not of type 'string'\n" in errors({k: {**v, prop: [None]}}) - assert "should be non-empty" in errors({k: {**v, prop: []}}) + assert non_empty_list(errors({k: {**v, prop: []}})) def test_schema_mpas_streams_properties_boolean(mpas_streams): @@ -1532,7 +1554,7 @@ def test_schema_namelist(): "array": [1, 2, 3], "boolean": True, "float": 3.14, - "integer": 88, + "integer": 42, "string": "foo", } } @@ -1542,15 +1564,67 @@ def test_schema_namelist(): assert errormsg % "None" in errors({"namelist": {"nonetype": None}}) assert errormsg % "{}" in errors({"namelist": {"dict": {}}}) # Needs at least one namelist value: - assert "{} should be non-empty" in errors({}) + assert non_empty_dict(errors({})) # Needs at least one name-value value: - assert "{} should be non-empty" in errors({"namelist": {}}) + assert non_empty_dict(errors({"namelist": {}})) # Namelist level must be a mapping: assert "[] is not of type 'object'\n" in errors([]) # Name-value level level must be a mapping: assert "[] is not of type 'object'\n" in errors({"namelist": []}) +# orog + + +def test_schema_orog(): + config: dict = { + "execution": { + "executable": "/path/to/orog", + }, + "grid_file": "/path/to/grid/file", + "mask": False, + "merge": "none", + "old_line1_items": { + "blat": 0, + "efac": 0, + "jcap": 0, + "latb": 0, + "lonb": 0, + "mtnres": 1, + "nr": 0, + "nf1": 0, + "nf2": 0, + }, + "orog_file": "/path/to/orog/file", + "rundir": "/path/to/run/dir", + } + + errors = schema_validator("orog", "properties", "orog") + # Basic correctness: + assert not errors(config) + # All 9 config keys are required: + assert "does not have enough properties" in errors(with_del(config, "old_line1_items", "blat")) + # Other config keys are not allowed: + assert "Additional properties are not allowed" in errors( + with_set(config, "bar", "old_line1_items", "foo") + ) + # All old_line1_items keys require integer values: + for key in config["old_line1_items"]: + assert "is not of type 'integer'\n" in errors( + with_set(config, None, "old_line1_items", key) + ) + # Some top level keys are required: + for key in ["execution", "grid_file", "rundir"]: + assert f"'{key}' is a required property" in errors(with_del(config, key)) + # Other top-level keys are not allowed: + assert "Additional properties are not allowed" in errors(with_set(config, "bar", "foo")) + # The mask key requires boolean values: + assert "is not of type 'boolean'\n" in errors({"mask": None}) + # Top-level keys require a string value: + for key in ["grid_file", "rundir", "merge", "orog_file"]: + assert "is not of type 'string'\n" in errors(with_set(config, None, "rundir")) + + # orog-gsl @@ -1664,7 +1738,7 @@ def test_schema_rocoto_metatask_attrs(): assert not errors({"mode": "serial"}) assert "'foo' is not one of ['parallel', 'serial']" in errors({"mode": "foo"}) # Positive int is ok for throttle: - assert not errors({"throttle": 88}) + assert not errors({"throttle": 42}) assert not errors({"throttle": 0}) assert "-1 is less than the minimum of 0" in errors({"throttle": -1}) assert "'foo' is not of type 'integer'\n" in errors({"throttle": "foo"}) @@ -1753,7 +1827,7 @@ def test_schema_schism_rundir(schism_prop): errors = schism_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # sfc-climo-gen @@ -1782,7 +1856,7 @@ def test_schema_sfc_climo_gen_namelist(sfc_climo_gen_prop): # Just base_file is ok: assert not errors(base_file) # base_file must be a string: - assert "88 is not of type 'string'\n" in errors({"base_file": 88}) + assert "42 is not of type 'string'\n" in errors({"base_file": 42}) # Just update_values is ok: assert not errors(update_values) # config is required with update_values: @@ -1796,7 +1870,7 @@ def test_schema_sfc_climo_gen_namelist(sfc_climo_gen_prop): def test_schema_sfc_climo_gen_namelist_update_values(sfc_climo_gen_prop): errors = sfc_climo_gen_prop("namelist", "properties", "update_values", "properties", "config") # array, boolean, number, and string values are ok: - assert not errors({"array": [1, 2, 3], "bool": True, "int": 88, "float": 3.14, "string": "foo"}) + assert not errors({"array": [1, 2, 3], "bool": True, "int": 42, "float": 3.14, "string": "foo"}) # Other types are not, e.g.: assert "None is not of type 'array', 'boolean', 'number', 'string'\n" in errors({"null": None}) # No minimum number of entries is required: @@ -1807,7 +1881,7 @@ def test_schema_sfc_climo_gen_rundir(sfc_climo_gen_prop): errors = sfc_climo_gen_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # shave @@ -1817,9 +1891,10 @@ def test_schema_shave(): config = { "config": { "input_grid_file": "/path/to/input_grid_file", - "nx": 88, - "ny": 88, - "nh4": 1, + "output_grid_file": "/path/to/output_grid_file", + "nx": 42, + "ny": 42, + "nhalo": 1, }, "execution": {"executable": "shave"}, "rundir": "/tmp", @@ -1837,16 +1912,19 @@ def test_schema_shave(): def test_schema_shave_config_properties(): # Get errors function from schema_validator errors = schema_validator("shave", "properties", "shave", "properties", "config") - for key in ("input_grid_file", "nx", "ny", "nh4"): + for key in ("input_grid_file", "nx", "ny", "nhalo"): # All config keys are required: assert f"'{key}' is a required property" in errors({}) # A string value is ok for input_grid_file: if key == "input_grid_file": - assert "not of type 'string'" in str(errors({key: 88})) - # nx, ny, and nh4 must be positive integers: - elif key in ["nx", "ny", "nh4"]: + assert "not of type 'string'" in str(errors({key: 42})) + # nx, ny, and nhalo must be integers >= their respective minimum values: + elif key in (keyvals := {"nx": 1, "ny": 1, "nhalo": 0}): + minval = keyvals[key] assert "not of type 'integer'" in str(errors({key: "/path/"})) - assert "0 is less than the minimum of 1" in str(errors({key: 0})) + assert f"{minval - 1} is less than the minimum of {minval}" in str( + errors({key: minval - 1}) + ) # It is an error for the value to be a floating-point value: assert "not of type" in str(errors({key: 3.14})) # It is an error not to supply a value: @@ -1857,7 +1935,7 @@ def test_schema_shave_rundir(shave_prop): errors = shave_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # ungrib @@ -1889,7 +1967,7 @@ def test_schema_ungrib_rundir(ungrib_prop): errors = ungrib_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # upp @@ -1897,6 +1975,7 @@ def test_schema_ungrib_rundir(ungrib_prop): def test_schema_upp(): config = { + "control_file": "/path/to/postxconfig-NT.txt", "execution": { "batchargs": { "cores": 1, @@ -1922,7 +2001,7 @@ def test_schema_upp(): # Basic correctness: assert not errors(config) # Some top-level keys are required: - for key in ("execution", "namelist", "rundir"): + for key in ("control_file", "execution", "namelist", "rundir"): assert f"'{key}' is a required property" in errors(with_del(config, key)) # Other top-level keys are optional: assert not errors({**config, "files_to_copy": {"dst": "src"}}) @@ -1931,6 +2010,12 @@ def test_schema_upp(): assert "Additional properties are not allowed" in errors({**config, "foo": "bar"}) +def test_schema_upp_control_file(upp_prop): + errors = upp_prop("control_file") + # A string value is required: + assert "is not of type 'string'" in errors(None) + + def test_schema_upp_namelist(upp_prop): maxpathlen = 256 errors = upp_prop("namelist") @@ -1959,10 +2044,10 @@ def test_schema_upp_namelist(upp_prop): assert "too long" in errors( {"update_values": {"model_inputs": {key: "c" * (maxpathlen + 1)}}} ) - assert "not of type 'string'" in errors({"update_values": {"model_inputs": {key: 88}}}) + assert "not of type 'string'" in errors({"update_values": {"model_inputs": {key: 42}}}) # model_inputs: Only one grib value is supported: assert "not one of ['grib2']" in errors({"update_values": {"model_inputs": {"grib": "grib1"}}}) - assert "not of type 'string'" in errors({"update_values": {"model_inputs": {"grib": 88}}}) + assert "not of type 'string'" in errors({"update_values": {"model_inputs": {"grib": 42}}}) # model_inputs: Only certain ioform values are supported: assert "not one of ['binarynemsio', 'netcdf']" in errors( {"update_values": {"model_inputs": {"ioform": "jpg"}}} @@ -1995,15 +2080,15 @@ def test_schema_upp_namelist(upp_prop): "write_ifi_debug_files", ]: assert not errors({"update_values": {"nampgb": {key: True}}}) - assert "not of type 'boolean'" in errors({"update_values": {"nampgb": {key: 88}}}) + assert "not of type 'boolean'" in errors({"update_values": {"nampgb": {key: 42}}}) # nampgb: String pathnames have a max length: for key in ["filenameaer"]: assert not errors({"update_values": {"nampgb": {key: "c" * maxpathlen}}}) assert "too long" in errors({"update_values": {"nampgb": {key: "c" * (maxpathlen + 1)}}}) - assert "not of type 'string'" in errors({"update_values": {"nampgb": {key: 88}}}) + assert "not of type 'string'" in errors({"update_values": {"nampgb": {key: 42}}}) # nampgb: Some integer keys are supported: for key in ["kpo", "kpv", "kth", "numx"]: - assert not errors({"update_values": {"nampgb": {key: 88}}}) + assert not errors({"update_values": {"nampgb": {key: 42}}}) assert "not of type 'integer'" in errors({"update_values": {"nampgb": {key: True}}}) # nampgb: Some arrays of numbers are supported: nitems = 70 @@ -2025,7 +2110,7 @@ def test_schema_upp_rundir(upp_prop): errors = upp_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) # ww3 @@ -2070,4 +2155,4 @@ def test_schema_ww3_rundir(ww3_prop): errors = ww3_prop("rundir") # Must be a string: assert not errors("/some/path") - assert "88 is not of type 'string'\n" in errors(88) + assert "42 is not of type 'string'\n" in errors(42) diff --git a/src/uwtools/tests/utils/test_api.py b/src/uwtools/tests/utils/test_api.py index 374cd8cf4..e41d89d3b 100644 --- a/src/uwtools/tests/utils/test_api.py +++ b/src/uwtools/tests/utils/test_api.py @@ -21,11 +21,12 @@ def execute_kwargs(): "dry_run": False, "graph_file": "/path/to/g.dot", "key_path": None, + "schema_file": None, "stdin_ok": True, } -@mark.parametrize("val", [Path("/some/path"), {"foo": 88}]) +@mark.parametrize("val", [Path("/some/path"), {"foo": 42}]) def test_ensure_data_source_passthrough(val): assert api.ensure_data_source(data_source=val, stdin_ok=False) == val diff --git a/src/uwtools/tests/utils/test_file.py b/src/uwtools/tests/utils/test_file.py index 63d9b322d..620932704 100644 --- a/src/uwtools/tests/utils/test_file.py +++ b/src/uwtools/tests/utils/test_file.py @@ -112,7 +112,7 @@ def test_resource_path(): assert file.resource_path().is_dir() -@mark.parametrize("val", [Path("/some/path"), {"foo": 88}]) +@mark.parametrize("val", [Path("/some/path"), {"foo": 42}]) def test_str2path_passthrough(val): assert file.str2path(val) == val diff --git a/src/uwtools/utils/api.py b/src/uwtools/utils/api.py index 548b3306b..38ac2a28f 100644 --- a/src/uwtools/utils/api.py +++ b/src/uwtools/utils/api.py @@ -52,6 +52,7 @@ def execute( dry_run: bool = False, graph_file: Optional[Union[Path, str]] = None, key_path: Optional[list[str]] = None, + schema_file: Optional[Union[Path, str]] = None, stdin_ok: bool = False, ) -> bool: return _execute( @@ -64,6 +65,7 @@ def execute( dry_run=dry_run, graph_file=graph_file, key_path=key_path, + schema_file=schema_file, stdin_ok=stdin_ok, ) @@ -75,6 +77,7 @@ def execute_cycle( dry_run: bool = False, graph_file: Optional[Union[Path, str]] = None, key_path: Optional[list[str]] = None, + schema_file: Optional[Union[Path, str]] = None, stdin_ok: bool = False, ) -> bool: return _execute( @@ -87,6 +90,7 @@ def execute_cycle( dry_run=dry_run, graph_file=graph_file, key_path=key_path, + schema_file=schema_file, stdin_ok=stdin_ok, ) @@ -99,6 +103,7 @@ def execute_cycle_leadtime( dry_run: bool = False, graph_file: Optional[Union[Path, str]] = None, key_path: Optional[list[str]] = None, + schema_file: Optional[Union[Path, str]] = None, stdin_ok: bool = False, ) -> bool: return _execute( @@ -111,6 +116,7 @@ def execute_cycle_leadtime( dry_run=dry_run, graph_file=graph_file, key_path=key_path, + schema_file=schema_file, stdin_ok=stdin_ok, ) @@ -146,6 +152,7 @@ def _execute( dry_run: bool = False, graph_file: Optional[Union[Path, str]] = None, key_path: Optional[list[str]] = None, + schema_file: Optional[Union[Path, str]] = None, stdin_ok: bool = False, ) -> bool: """ @@ -163,6 +170,7 @@ def _execute( :param dry_run: Do not run the executable, just report what would have been done. :param graph_file: Write Graphviz DOT output here. :param key_path: Path of keys to subsection of config file. + :param schema_file: The JSON Schema file to use for validation. :param stdin_ok: OK to read from stdin? :return: ``True`` if task completes without raising an exception. """ @@ -170,6 +178,7 @@ def _execute( config=ensure_data_source(str2path(config), stdin_ok), dry_run=dry_run, key_path=key_path, + schema_file=schema_file, ) accepted = set(getfullargspec(driver_class).args) for arg in ["batch", "cycle", "leadtime"]: From 098b25743bd9f64c471a370a199f06ebadd6e8e6 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 19 Nov 2024 21:13:30 +0000 Subject: [PATCH 17/24] Add missing newlines --- .gitignore | 2 +- notebooks/.gitignore | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 001bce703..fa050f56e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ *.egg-info *.swp .coverage -__pycache__ \ No newline at end of file +__pycache__ diff --git a/notebooks/.gitignore b/notebooks/.gitignore index 8867348fd..3d51db06f 100644 --- a/notebooks/.gitignore +++ b/notebooks/.gitignore @@ -1,2 +1,2 @@ .ipynb_checkpoints -tmp \ No newline at end of file +tmp From 6ab79eba0cbe09fd701fa98bd9fe34e82ef65a9a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 19 Nov 2024 22:07:25 +0000 Subject: [PATCH 18/24] Add exec bit to test-nb.sh --- .github/scripts/test-nb.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/scripts/test-nb.sh diff --git a/.github/scripts/test-nb.sh b/.github/scripts/test-nb.sh old mode 100644 new mode 100755 From fc00bac246b6d7c1fa7e8eb25b06c6c549433710 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 19 Nov 2024 22:32:57 +0000 Subject: [PATCH 19/24] Incorporate notebook tests into main CI tests --- .github/scripts/test-nb.sh | 7 ------- .github/scripts/test.sh | 3 ++- .github/workflows/test.yaml | 2 -- notebooks/install-deps | 2 +- 4 files changed, 3 insertions(+), 11 deletions(-) delete mode 100755 .github/scripts/test-nb.sh diff --git a/.github/scripts/test-nb.sh b/.github/scripts/test-nb.sh deleted file mode 100755 index e44216090..000000000 --- a/.github/scripts/test-nb.sh +++ /dev/null @@ -1,7 +0,0 @@ -set -ae -source $(dirname ${BASH_SOURCE[0]})/common.sh -ci_conda_activate -conda install -c ufs-community -c conda-forge --override-channels --repodata-fn repodata.json uwtools -cd notebooks -source install-deps -make test-nb diff --git a/.github/scripts/test.sh b/.github/scripts/test.sh index 048fb5240..acaedeafc 100755 --- a/.github/scripts/test.sh +++ b/.github/scripts/test.sh @@ -8,11 +8,12 @@ run_tests() { devpkgs=$(jq .packages.dev[] recipe/meta.json | tr -d ' "') conda create --yes --name $env --quiet python=$PYTHON_VERSION $devpkgs conda activate $env + . notebooks/install-deps set -x python --version git clean -dfx pip install --editable src # sets new Python version in entry-point scripts - make test + make test && make test-nb status=$? set +x conda deactivate diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7fcf90532..3351ba97d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,5 +23,3 @@ jobs: run: .github/scripts/format-check.sh - name: Test run: .github/scripts/test.sh - - name: Test Notebooks - run: .github/scripts/test-nb.sh diff --git a/notebooks/install-deps b/notebooks/install-deps index 4f1226968..1feb44160 100644 --- a/notebooks/install-deps +++ b/notebooks/install-deps @@ -1 +1 @@ -conda install -q -y jupyterlab pytest testbook +conda install -q -y --override-channels --repodata-fn repodata.json "jupyterlab<4.4" "testbook<0.5" From e7a3bffb67dbdb6ba384bd7fe3d8e958f7311b6e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 19 Nov 2024 22:39:15 +0000 Subject: [PATCH 20/24] Update notebooks/install-deps --- notebooks/install-deps | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/install-deps b/notebooks/install-deps index 1feb44160..54a214b38 100644 --- a/notebooks/install-deps +++ b/notebooks/install-deps @@ -1 +1 @@ -conda install -q -y --override-channels --repodata-fn repodata.json "jupyterlab<4.4" "testbook<0.5" +conda install -q -y --repodata-fn repodata.json "jupyterlab<4.4" "testbook<0.5" From 89672cdf07552e824f0184a49ae4d4b6f00dea0a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 19 Nov 2024 23:18:59 +0000 Subject: [PATCH 21/24] Update notebook output cells --- notebooks/config.ipynb | 71 +++-- notebooks/exp-config-cb.ipynb | 28 +- notebooks/fs.ipynb | 476 +++++++++++++++++----------------- notebooks/rocoto.ipynb | 72 ++--- notebooks/template.ipynb | 18 +- 5 files changed, 341 insertions(+), 324 deletions(-) diff --git a/notebooks/config.ipynb b/notebooks/config.ipynb index c3aca9e55..740d056a4 100644 --- a/notebooks/config.ipynb +++ b/notebooks/config.ipynb @@ -732,13 +732,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-16T15:39:13] INFO Keys that are complete:\n", - "[2024-10-16T15:39:13] INFO memo\n", - "[2024-10-16T15:39:13] INFO memo.sent\n", - "[2024-10-16T15:39:13] INFO \n", - "[2024-10-16T15:39:13] INFO Keys with unrendered Jinja2 variables/expressions:\n", - "[2024-10-16T15:39:13] INFO memo.sender_id: {{ id }}\n", - "[2024-10-16T15:39:13] INFO memo.message: {{ greeting }}, {{ recipient }}!\n" + "[2024-11-19T23:12:52] INFO Keys that are complete:\n", + "[2024-11-19T23:12:52] INFO memo\n", + "[2024-11-19T23:12:52] INFO memo.sent\n", + "[2024-11-19T23:12:52] INFO \n", + "[2024-11-19T23:12:52] INFO Keys with unrendered Jinja2 variables/expressions:\n", + "[2024-11-19T23:12:52] INFO memo.sender_id: {{ id }}\n", + "[2024-11-19T23:12:52] INFO memo.message: {{ greeting }}, {{ recipient }}!\n" ] }, { @@ -1071,10 +1071,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-16T15:39:13] INFO - fixtures/config/base-config.nml\n", - "[2024-10-16T15:39:13] INFO + fixtures/config/alt-config.nml\n", - "[2024-10-16T15:39:13] INFO ---------------------------------------------------------------------\n", - "[2024-10-16T15:39:13] INFO memo: sent: - False + True\n" + "[2024-11-19T23:12:52] INFO - fixtures/config/base-config.nml\n", + "[2024-11-19T23:12:52] INFO + fixtures/config/alt-config.nml\n", + "[2024-11-19T23:12:52] INFO ---------------------------------------------------------------------\n", + "[2024-11-19T23:12:52] INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line\n", + "[2024-11-19T23:12:52] INFO ---------------------------------------------------------------------\n", + "[2024-11-19T23:12:52] INFO memo:\n", + "[2024-11-19T23:12:52] INFO message: '{{ greeting }}, {{ recipient }}!'\n", + "[2024-11-19T23:12:52] INFO sender_id: '{{ id }}'\n", + "[2024-11-19T23:12:52] INFO - sent: false\n", + "[2024-11-19T23:12:52] INFO + sent: true\n" ] }, { @@ -1136,9 +1142,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-16T15:39:13] INFO - fixtures/config/base-config.nml\n", - "[2024-10-16T15:39:13] INFO + tmp/config-copy.nml\n", - "[2024-10-16T15:39:13] INFO ---------------------------------------------------------------------\n" + "[2024-11-19T23:12:52] INFO - fixtures/config/base-config.nml\n", + "[2024-11-19T23:12:52] INFO + tmp/config-copy.nml\n" ] }, { @@ -1178,7 +1183,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-16T15:39:13] ERROR Formats do not match: yaml vs nml\n" + "[2024-11-19T23:12:52] ERROR Formats do not match: yaml vs nml\n" ] }, { @@ -1330,7 +1335,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-16T15:39:13] INFO 0 UW schema-validation errors found\n" + "[2024-11-19T23:12:53] INFO 0 UW schema-validation errors found in config\n" ] }, { @@ -1370,9 +1375,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-16T15:39:13] ERROR 1 UW schema-validation error found\n", - "[2024-10-16T15:39:13] ERROR Error at recipient:\n", - "[2024-10-16T15:39:13] ERROR 47 is not of type 'string'\n" + "[2024-11-19T23:12:53] ERROR 1 UW schema-validation error found in config\n", + "[2024-11-19T23:12:53] ERROR Error at recipient:\n", + "[2024-11-19T23:12:53] ERROR 47 is not of type 'string'\n" ] }, { @@ -1439,6 +1444,9 @@ " | __init__(self, config: Union[dict, pathlib.Path, NoneType] = None)\n", " | :param config: Config file to load (None => read from stdin), or initial dict.\n", " |\n", + " | as_dict(self) -> dict\n", + " | Returns a pure dict version of the config.\n", + " |\n", " | dump(self, path: Optional[pathlib.Path] = None) -> None\n", " | Dump the config in INI format.\n", " |\n", @@ -1458,15 +1466,13 @@ " |\n", " | __abstractmethods__ = frozenset()\n", " |\n", - " | __annotations__ = {}\n", - " |\n", " | ----------------------------------------------------------------------\n", " | Methods inherited from uwtools.config.formats.base.Config:\n", " |\n", " | __repr__(self) -> str\n", " | Return the string representation of a Config object.\n", " |\n", - " | compare_config(self, dict1: dict, dict2: Optional[dict] = None) -> bool\n", + " | compare_config(self, dict1: dict, dict2: Optional[dict] = None, header: Optional[bool] = True) -> bool\n", " | Compare two config dictionaries.\n", " |\n", " | Assumes a section/key/value structure.\n", @@ -1535,7 +1541,7 @@ " | clear(self)\n", " | D.clear() -> None. Remove all items from D.\n", " |\n", - " | pop(self, key, default=)\n", + " | pop(self, key, default=)\n", " | D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n", " | If key is not found, d is returned if given, otherwise KeyError is raised.\n", " |\n", @@ -1687,8 +1693,17 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-16T15:39:13] INFO fruit count: grapes: - {{ grape_count }} + 8\n", - "[2024-10-16T15:39:13] INFO fruit count: kiwis: - 2 + 1\n" + "[2024-11-19T23:12:53] INFO ---------------------------------------------------------------------\n", + "[2024-11-19T23:12:53] INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line\n", + "[2024-11-19T23:12:53] INFO ---------------------------------------------------------------------\n", + "[2024-11-19T23:12:53] INFO fruit count:\n", + "[2024-11-19T23:12:53] INFO apples: '3'\n", + "[2024-11-19T23:12:53] INFO - grapes: '{{ grape_count }}'\n", + "[2024-11-19T23:12:53] INFO + grapes: '8'\n", + "[2024-11-19T23:12:53] INFO - kiwis: '2'\n", + "[2024-11-19T23:12:53] INFO ? ^\n", + "[2024-11-19T23:12:53] INFO + kiwis: '1'\n", + "[2024-11-19T23:12:53] INFO ? ^\n" ] }, { @@ -1869,9 +1884,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:DEV-uwtools] *", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-DEV-uwtools-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1883,7 +1898,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.7" } }, "nbformat": 4, diff --git a/notebooks/exp-config-cb.ipynb b/notebooks/exp-config-cb.ipynb index 0b8ab9087..823478052 100644 --- a/notebooks/exp-config-cb.ipynb +++ b/notebooks/exp-config-cb.ipynb @@ -20,7 +20,7 @@ "metadata": {}, "outputs": [], "source": [ - "from datetime import datetime\n", + "from datetime import datetime, timedelta\n", "from uwtools.api.config import get_yaml_config\n", "from uwtools.api import chgres_cube\n", "from uwtools.api.logging import use_uwtools_logger\n", @@ -260,8 +260,9 @@ } ], "source": [ - "config_files = ['fixtures/exp-config/fv3-rap-physics.yaml',\n", - " 'fixtures/exp-config/user.yaml'\n", + "config_files = [\n", + " 'fixtures/exp-config/fv3-rap-physics.yaml',\n", + " 'fixtures/exp-config/user.yaml'\n", "]\n", "for config_file in config_files:\n", " config = get_yaml_config(config_file)\n", @@ -349,17 +350,17 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T09:41:26] INFO Validating config against internal schema: chgres-cube\n", - "[2024-11-19T09:41:26] INFO 0 UW schema-validation errors found\n", - "[2024-11-19T09:41:26] INFO Validating config against internal schema: platform\n", - "[2024-11-19T09:41:26] INFO 0 UW schema-validation errors found\n", - "[2024-11-19T09:41:26] INFO 20241119 09Z chgres_cube valid schema: State: Ready\n" + "[2024-11-19T23:14:15] INFO Validating config against internal schema: chgres-cube\n", + "[2024-11-19T23:14:15] INFO 0 UW schema-validation errors found in chgres_cube config\n", + "[2024-11-19T23:14:15] INFO Validating config against internal schema: platform\n", + "[2024-11-19T23:14:15] INFO 0 UW schema-validation errors found in platform config\n", + "[2024-11-19T23:14:15] INFO 20241120 05:14:15 chgres_cube valid schema: State: Ready\n" ] }, { "data": { "text/plain": [ - "Asset(ref=None, ready=. at 0x7e1cf8123420>)" + "Asset(ref=None, ready=. at 0xffff685ad6c0>)" ] }, "execution_count": 8, @@ -371,7 +372,8 @@ "driver = chgres_cube.ChgresCube(\n", " config=experiment_config,\n", " key_path=['task_make_ics'],\n", - " cycle=datetime.now()\n", + " cycle=datetime.now(),\n", + " leadtime=timedelta(hours=6),\n", ")\n", "driver.validate()" ] @@ -379,9 +381,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:DEV-uwtools] *", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-DEV-uwtools-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -393,7 +395,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.7" } }, "nbformat": 4, diff --git a/notebooks/fs.ipynb b/notebooks/fs.ipynb index 6f42297a0..e07f078e0 100644 --- a/notebooks/fs.ipynb +++ b/notebooks/fs.ipynb @@ -129,26 +129,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:21:27] INFO Validating config against internal schema: files-to-stage\n", - "[2024-08-30T15:21:27] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:21:27] INFO File copies: Initial state: Pending\n", - "[2024-08-30T15:21:27] INFO File copies: Checking requirements\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Initial state: Pending\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Checking requirements\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Requirement(s) ready\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Executing\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Final state: Ready\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Initial state: Pending\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Checking requirements\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Requirement(s) ready\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Executing\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Final state: Ready\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Initial state: Pending\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Checking requirements\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Requirement(s) ready\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Executing\n", - "[2024-08-30T15:21:27] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Final state: Ready\n", - "[2024-08-30T15:21:27] INFO File copies: Final state: Ready\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO File copies: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO File copies: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Executing\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-target/file1-copy.nml: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Executing\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-target/data/file2-copy.txt: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Executing\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-target/data/file3-copy.csv: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO File copies: Final state: Ready\n" ] }, { @@ -190,11 +190,11 @@ "text": [ "\u001b[01;34mtmp/copy-target\u001b[0m\n", "├── \u001b[01;34mdata\u001b[0m\n", - "│   ├── file2-copy.txt\n", - "│   └── file3-copy.csv\n", - "└── file1-copy.nml\n", + "│   ├── \u001b[00mfile2-copy.txt\u001b[0m\n", + "│   └── \u001b[00mfile3-copy.csv\u001b[0m\n", + "└── \u001b[00mfile1-copy.nml\u001b[0m\n", "\n", - "2 directories, 3 files\n" + "1 directory, 3 files\n" ] } ], @@ -223,16 +223,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:21:35] INFO Validating config against internal schema: files-to-stage\n", - "[2024-08-30T15:21:35] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:21:35] INFO File copies: Initial state: Pending\n", - "[2024-08-30T15:21:35] INFO File copies: Checking requirements\n", - "[2024-08-30T15:21:35] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Initial state: Pending\n", - "[2024-08-30T15:21:35] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Checking requirements\n", - "[2024-08-30T15:21:35] WARNING File fixtures/fs/missing-file.nml: State: Pending (EXTERNAL)\n", - "[2024-08-30T15:21:35] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Requirement(s) pending\n", - "[2024-08-30T15:21:35] WARNING Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Final state: Pending\n", - "[2024-08-30T15:21:35] WARNING File copies: Final state: Pending\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO File copies: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO File copies: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Checking requirements\n", + "[2024-11-19T23:14:42] WARNING File fixtures/fs/missing-file.nml: State: Not Ready (external asset)\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Requirement(s) not ready\n", + "[2024-11-19T23:14:42] WARNING Copy fixtures/fs/missing-file.nml -> tmp/copy-target/missing-copy.nml: Final state: Not Ready\n", + "[2024-11-19T23:14:42] WARNING File copies: Final state: Not Ready\n" ] }, { @@ -273,11 +273,11 @@ "text": [ "\u001b[01;34mtmp/copy-target\u001b[0m\n", "├── \u001b[01;34mdata\u001b[0m\n", - "│   ├── file2-copy.txt\n", - "│   └── file3-copy.csv\n", - "└── file1-copy.nml\n", + "│   ├── \u001b[00mfile2-copy.txt\u001b[0m\n", + "│   └── \u001b[00mfile3-copy.csv\u001b[0m\n", + "└── \u001b[00mfile1-copy.nml\u001b[0m\n", "\n", - "2 directories, 3 files\n" + "1 directory, 3 files\n" ] } ], @@ -338,26 +338,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:21:47] INFO Validating config against internal schema: files-to-stage\n", - "[2024-08-30T15:21:47] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:21:47] INFO File copies: Initial state: Pending\n", - "[2024-08-30T15:21:47] INFO File copies: Checking requirements\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Initial state: Pending\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Checking requirements\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Requirement(s) ready\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Executing\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Final state: Ready\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Initial state: Pending\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Checking requirements\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Requirement(s) ready\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Executing\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Final state: Ready\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Initial state: Pending\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Checking requirements\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Requirement(s) ready\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Executing\n", - "[2024-08-30T15:21:47] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Final state: Ready\n", - "[2024-08-30T15:21:47] INFO File copies: Final state: Ready\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO File copies: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO File copies: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Executing\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copy-keys-target/file1-copy.nml: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Executing\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copy-keys-target/data/file2-copy.txt: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Executing\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copy-keys-target/data/file3-copy.csv: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO File copies: Final state: Ready\n" ] }, { @@ -400,11 +400,11 @@ "text": [ "\u001b[01;34mtmp/copy-keys-target\u001b[0m\n", "├── \u001b[01;34mdata\u001b[0m\n", - "│   ├── file2-copy.txt\n", - "│   └── file3-copy.csv\n", - "└── file1-copy.nml\n", + "│   ├── \u001b[00mfile2-copy.txt\u001b[0m\n", + "│   └── \u001b[00mfile3-copy.csv\u001b[0m\n", + "└── \u001b[00mfile1-copy.nml\u001b[0m\n", "\n", - "2 directories, 3 files\n" + "1 directory, 3 files\n" ] } ], @@ -449,7 +449,7 @@ " |\n", " | Methods defined here:\n", " |\n", - " | go = __iotaa_tasks__(*args, **kwargs) -> '_AssetT'\n", + " | go(self)\n", " | Copy files.\n", " |\n", " | ----------------------------------------------------------------------\n", @@ -463,13 +463,13 @@ " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", " | Stage files and directories.\n", " |\n", - " | :param config: YAML-file path, or dict (read stdin if missing or None).\n", + " | :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " | :param target_dir: Path to target directory.\n", - " | :param cycle: A datetime object to make available for use in the config.\n", - " | :param leadtime: A timedelta object to make available for use in the config.\n", + " | :param cycle: A ``datetime`` object to make available for use in the config.\n", + " | :param leadtime: A ``timedelta`` object to make available for use in the config.\n", " | :param keys: YAML keys leading to file dst/src block.\n", " | :param dry_run: Do not copy files.\n", - " | :raises: UWConfigError if config fails validation.\n", + " | :raises: ``UWConfigError`` if config fails validation.\n", " |\n", " | ----------------------------------------------------------------------\n", " | Data descriptors inherited from Stager:\n", @@ -505,26 +505,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:21:56] INFO Validating config against internal schema: files-to-stage\n", - "[2024-08-30T15:21:56] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:21:56] INFO File copies: Initial state: Pending\n", - "[2024-08-30T15:21:56] INFO File copies: Checking requirements\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Initial state: Pending\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Checking requirements\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Requirement(s) ready\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Executing\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Final state: Ready\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Initial state: Pending\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Checking requirements\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Requirement(s) ready\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Executing\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Final state: Ready\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Initial state: Pending\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Checking requirements\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Requirement(s) ready\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Executing\n", - "[2024-08-30T15:21:56] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Final state: Ready\n", - "[2024-08-30T15:21:56] INFO File copies: Final state: Ready\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO File copies: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO File copies: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Executing\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file1.nml -> tmp/copier-target/file1-copy.nml: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Executing\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file2.txt -> tmp/copier-target/data/file2-copy.txt: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Executing\n", + "[2024-11-19T23:14:42] INFO Copy fixtures/fs/file3.csv -> tmp/copier-target/data/file3-copy.csv: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO File copies: Final state: Ready\n" ] }, { @@ -569,11 +569,11 @@ "text": [ "\u001b[01;34mtmp/copier-target\u001b[0m\n", "├── \u001b[01;34mdata\u001b[0m\n", - "│   ├── file2-copy.txt\n", - "│   └── file3-copy.csv\n", - "└── file1-copy.nml\n", + "│   ├── \u001b[00mfile2-copy.txt\u001b[0m\n", + "│   └── \u001b[00mfile3-copy.csv\u001b[0m\n", + "└── \u001b[00mfile1-copy.nml\u001b[0m\n", "\n", - "2 directories, 3 files\n" + "1 directory, 3 files\n" ] } ], @@ -670,26 +670,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:22:08] INFO Validating config against internal schema: files-to-stage\n", - "[2024-08-30T15:22:08] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:22:08] INFO File links: Initial state: Pending\n", - "[2024-08-30T15:22:08] INFO File links: Checking requirements\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Pending\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Pending\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Pending\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", - "[2024-08-30T15:22:08] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", - "[2024-08-30T15:22:08] INFO File links: Final state: Ready\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO File links: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO File links: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO File links: Final state: Ready\n" ] }, { @@ -731,11 +731,11 @@ "text": [ "\u001b[01;34mtmp/link-target\u001b[0m\n", "├── \u001b[01;34mdata\u001b[0m\n", - "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> ../../../fixtures/fs/file3.csv\n", - "├── \u001b[01;36mfile1-link.nml\u001b[0m -> ../../fixtures/fs/file1.nml\n", - "└── \u001b[01;36mfile2-link.txt\u001b[0m -> ../../fixtures/fs/file2.txt\n", + "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> \u001b[00m../../../fixtures/fs/file3.csv\u001b[0m\n", + "├── \u001b[01;36mfile1-link.nml\u001b[0m -> \u001b[00m../../fixtures/fs/file1.nml\u001b[0m\n", + "└── \u001b[01;36mfile2-link.txt\u001b[0m -> \u001b[00m../../fixtures/fs/file2.txt\u001b[0m\n", "\n", - "2 directories, 3 files\n" + "1 directory, 3 files\n" ] } ], @@ -764,16 +764,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:22:11] INFO Validating config against internal schema: files-to-stage\n", - "[2024-08-30T15:22:11] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:22:11] INFO File links: Initial state: Pending\n", - "[2024-08-30T15:22:11] INFO File links: Checking requirements\n", - "[2024-08-30T15:22:11] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Initial state: Pending\n", - "[2024-08-30T15:22:11] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Checking requirements\n", - "[2024-08-30T15:22:11] WARNING Filesystem item fixtures/fs/missing-file.nml: State: Pending (EXTERNAL)\n", - "[2024-08-30T15:22:11] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Requirement(s) pending\n", - "[2024-08-30T15:22:11] WARNING Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Final state: Pending\n", - "[2024-08-30T15:22:11] WARNING File links: Final state: Pending\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO File links: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO File links: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Checking requirements\n", + "[2024-11-19T23:14:42] WARNING Filesystem item fixtures/fs/missing-file.nml: State: Not Ready (external asset)\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Requirement(s) not ready\n", + "[2024-11-19T23:14:42] WARNING Link tmp/link-target/missing-link.nml -> fixtures/fs/missing-file.nml: Final state: Not Ready\n", + "[2024-11-19T23:14:42] WARNING File links: Final state: Not Ready\n" ] }, { @@ -814,11 +814,11 @@ "text": [ "\u001b[01;34mtmp/link-target\u001b[0m\n", "├── \u001b[01;34mdata\u001b[0m\n", - "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> ../../../fixtures/fs/file3.csv\n", - "├── \u001b[01;36mfile1-link.nml\u001b[0m -> ../../fixtures/fs/file1.nml\n", - "└── \u001b[01;36mfile2-link.txt\u001b[0m -> ../../fixtures/fs/file2.txt\n", + "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> \u001b[00m../../../fixtures/fs/file3.csv\u001b[0m\n", + "├── \u001b[01;36mfile1-link.nml\u001b[0m -> \u001b[00m../../fixtures/fs/file1.nml\u001b[0m\n", + "└── \u001b[01;36mfile2-link.txt\u001b[0m -> \u001b[00m../../fixtures/fs/file2.txt\u001b[0m\n", "\n", - "2 directories, 3 files\n" + "1 directory, 3 files\n" ] } ], @@ -879,26 +879,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:22:22] INFO Validating config against internal schema: files-to-stage\n", - "[2024-08-30T15:22:22] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:22:22] INFO File links: Initial state: Pending\n", - "[2024-08-30T15:22:22] INFO File links: Checking requirements\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Pending\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Pending\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Pending\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", - "[2024-08-30T15:22:22] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", - "[2024-08-30T15:22:22] INFO File links: Final state: Ready\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO File links: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO File links: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", + "[2024-11-19T23:14:42] INFO Link tmp/link-keys-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO File links: Final state: Ready\n" ] }, { @@ -941,11 +941,11 @@ "text": [ "\u001b[01;34mtmp/link-keys-target\u001b[0m\n", "├── \u001b[01;34mdata\u001b[0m\n", - "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> ../../../fixtures/fs/file3.csv\n", - "├── \u001b[01;36mfile1-link.nml\u001b[0m -> ../../fixtures/fs/file1.nml\n", - "└── \u001b[01;36mfile2-link.txt\u001b[0m -> ../../fixtures/fs/file2.txt\n", + "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> \u001b[00m../../../fixtures/fs/file3.csv\u001b[0m\n", + "├── \u001b[01;36mfile1-link.nml\u001b[0m -> \u001b[00m../../fixtures/fs/file1.nml\u001b[0m\n", + "└── \u001b[01;36mfile2-link.txt\u001b[0m -> \u001b[00m../../fixtures/fs/file2.txt\u001b[0m\n", "\n", - "2 directories, 3 files\n" + "1 directory, 3 files\n" ] } ], @@ -990,7 +990,7 @@ " |\n", " | Methods defined here:\n", " |\n", - " | go = __iotaa_tasks__(*args, **kwargs) -> '_AssetT'\n", + " | go(self)\n", " | Link files.\n", " |\n", " | ----------------------------------------------------------------------\n", @@ -1004,13 +1004,13 @@ " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", " | Stage files and directories.\n", " |\n", - " | :param config: YAML-file path, or dict (read stdin if missing or None).\n", + " | :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " | :param target_dir: Path to target directory.\n", - " | :param cycle: A datetime object to make available for use in the config.\n", - " | :param leadtime: A timedelta object to make available for use in the config.\n", + " | :param cycle: A ``datetime`` object to make available for use in the config.\n", + " | :param leadtime: A ``timedelta`` object to make available for use in the config.\n", " | :param keys: YAML keys leading to file dst/src block.\n", " | :param dry_run: Do not copy files.\n", - " | :raises: UWConfigError if config fails validation.\n", + " | :raises: ``UWConfigError`` if config fails validation.\n", " |\n", " | ----------------------------------------------------------------------\n", " | Data descriptors inherited from Stager:\n", @@ -1046,26 +1046,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:22:28] INFO Validating config against internal schema: files-to-stage\n", - "[2024-08-30T15:22:28] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:22:28] INFO File links: Initial state: Pending\n", - "[2024-08-30T15:22:28] INFO File links: Checking requirements\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Pending\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Pending\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Pending\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", - "[2024-08-30T15:22:28] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", - "[2024-08-30T15:22:28] INFO File links: Final state: Ready\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: files-to-stage\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO File links: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO File links: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Executing\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file1-link.nml -> fixtures/fs/file1.nml: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Executing\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/file2-link.txt -> fixtures/fs/file2.txt: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Executing\n", + "[2024-11-19T23:14:42] INFO Link tmp/linker-target/data/file3-link.csv -> fixtures/fs/file3.csv: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO File links: Final state: Ready\n" ] }, { @@ -1110,11 +1110,11 @@ "text": [ "\u001b[01;34mtmp/linker-target\u001b[0m\n", "├── \u001b[01;34mdata\u001b[0m\n", - "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> ../../../fixtures/fs/file3.csv\n", - "├── \u001b[01;36mfile1-link.nml\u001b[0m -> ../../fixtures/fs/file1.nml\n", - "└── \u001b[01;36mfile2-link.txt\u001b[0m -> ../../fixtures/fs/file2.txt\n", + "│   └── \u001b[01;36mfile3-link.csv\u001b[0m -> \u001b[00m../../../fixtures/fs/file3.csv\u001b[0m\n", + "├── \u001b[01;36mfile1-link.nml\u001b[0m -> \u001b[00m../../fixtures/fs/file1.nml\u001b[0m\n", + "└── \u001b[01;36mfile2-link.txt\u001b[0m -> \u001b[00m../../fixtures/fs/file2.txt\u001b[0m\n", "\n", - "2 directories, 3 files\n" + "1 directory, 3 files\n" ] } ], @@ -1211,21 +1211,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:22:34] INFO Validating config against internal schema: makedirs\n", - "[2024-08-30T15:22:34] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:22:34] INFO Directories: Initial state: Pending\n", - "[2024-08-30T15:22:34] INFO Directories: Checking requirements\n", - "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/foo: Initial state: Pending\n", - "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/foo: Checking requirements\n", - "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/foo: Requirement(s) ready\n", - "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/foo: Executing\n", - "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/foo: Final state: Ready\n", - "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/bar/baz: Initial state: Pending\n", - "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/bar/baz: Checking requirements\n", - "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/bar/baz: Requirement(s) ready\n", - "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/bar/baz: Executing\n", - "[2024-08-30T15:22:34] INFO Directory tmp/dir-target/bar/baz: Final state: Ready\n", - "[2024-08-30T15:22:34] INFO Directories: Final state: Ready\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: makedirs\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO Directories: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Directories: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/foo: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/foo: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/foo: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/foo: Executing\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/foo: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/bar/baz: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/bar/baz: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/bar/baz: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/bar/baz: Executing\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-target/bar/baz: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Directories: Final state: Ready\n" ] }, { @@ -1270,7 +1270,7 @@ "│   └── \u001b[01;34mbaz\u001b[0m\n", "└── \u001b[01;34mfoo\u001b[0m\n", "\n", - "4 directories, 0 files\n" + "3 directories, 0 files\n" ] } ], @@ -1331,21 +1331,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:22:42] INFO Validating config against internal schema: makedirs\n", - "[2024-08-30T15:22:42] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:22:42] INFO Directories: Initial state: Pending\n", - "[2024-08-30T15:22:42] INFO Directories: Checking requirements\n", - "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/foo/bar: Initial state: Pending\n", - "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/foo/bar: Checking requirements\n", - "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/foo/bar: Requirement(s) ready\n", - "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/foo/bar: Executing\n", - "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/foo/bar: Final state: Ready\n", - "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/baz: Initial state: Pending\n", - "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/baz: Checking requirements\n", - "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/baz: Requirement(s) ready\n", - "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/baz: Executing\n", - "[2024-08-30T15:22:42] INFO Directory tmp/dir-keys-target/baz: Final state: Ready\n", - "[2024-08-30T15:22:42] INFO Directories: Final state: Ready\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: makedirs\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO Directories: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Directories: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/foo/bar: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/foo/bar: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/foo/bar: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/foo/bar: Executing\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/foo/bar: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/baz: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/baz: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/baz: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/baz: Executing\n", + "[2024-11-19T23:14:42] INFO Directory tmp/dir-keys-target/baz: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Directories: Final state: Ready\n" ] }, { @@ -1391,7 +1391,7 @@ "└── \u001b[01;34mfoo\u001b[0m\n", " └── \u001b[01;34mbar\u001b[0m\n", "\n", - "4 directories, 0 files\n" + "3 directories, 0 files\n" ] } ], @@ -1435,7 +1435,7 @@ " |\n", " | Methods defined here:\n", " |\n", - " | go = __iotaa_tasks__(*args, **kwargs) -> '_AssetT'\n", + " | go(self)\n", " | Make directories.\n", " |\n", " | ----------------------------------------------------------------------\n", @@ -1449,13 +1449,13 @@ " | __init__(self, config: Union[dict, str, pathlib.Path, NoneType] = None, target_dir: Union[str, pathlib.Path, NoneType] = None, cycle: Optional[datetime.datetime] = None, leadtime: Optional[datetime.timedelta] = None, keys: Optional[list[str]] = None, dry_run: bool = False) -> None\n", " | Stage files and directories.\n", " |\n", - " | :param config: YAML-file path, or dict (read stdin if missing or None).\n", + " | :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).\n", " | :param target_dir: Path to target directory.\n", - " | :param cycle: A datetime object to make available for use in the config.\n", - " | :param leadtime: A timedelta object to make available for use in the config.\n", + " | :param cycle: A ``datetime`` object to make available for use in the config.\n", + " | :param leadtime: A ``timedelta`` object to make available for use in the config.\n", " | :param keys: YAML keys leading to file dst/src block.\n", " | :param dry_run: Do not copy files.\n", - " | :raises: UWConfigError if config fails validation.\n", + " | :raises: ``UWConfigError`` if config fails validation.\n", " |\n", " | ----------------------------------------------------------------------\n", " | Data descriptors inherited from Stager:\n", @@ -1491,21 +1491,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-30T15:22:48] INFO Validating config against internal schema: makedirs\n", - "[2024-08-30T15:22:48] INFO 0 UW schema-validation errors found\n", - "[2024-08-30T15:22:48] INFO Directories: Initial state: Pending\n", - "[2024-08-30T15:22:48] INFO Directories: Checking requirements\n", - "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/foo: Initial state: Pending\n", - "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/foo: Checking requirements\n", - "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/foo: Requirement(s) ready\n", - "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/foo: Executing\n", - "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/foo: Final state: Ready\n", - "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/bar/baz: Initial state: Pending\n", - "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/bar/baz: Checking requirements\n", - "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/bar/baz: Requirement(s) ready\n", - "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/bar/baz: Executing\n", - "[2024-08-30T15:22:48] INFO Directory tmp/makedirs-target/bar/baz: Final state: Ready\n", - "[2024-08-30T15:22:48] INFO Directories: Final state: Ready\n" + "[2024-11-19T23:14:42] INFO Validating config against internal schema: makedirs\n", + "[2024-11-19T23:14:42] INFO 0 UW schema-validation errors found in fs config\n", + "[2024-11-19T23:14:42] INFO Directories: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Directories: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/foo: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/foo: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/foo: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/foo: Executing\n", + "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/foo: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/bar/baz: Initial state: Not Ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/bar/baz: Checking requirements\n", + "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/bar/baz: Requirement(s) ready\n", + "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/bar/baz: Executing\n", + "[2024-11-19T23:14:42] INFO Directory tmp/makedirs-target/bar/baz: Final state: Ready\n", + "[2024-11-19T23:14:42] INFO Directories: Final state: Ready\n" ] }, { @@ -1552,7 +1552,7 @@ "│   └── \u001b[01;34mbaz\u001b[0m\n", "└── \u001b[01;34mfoo\u001b[0m\n", "\n", - "4 directories, 0 files\n" + "3 directories, 0 files\n" ] } ], @@ -1564,9 +1564,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:DEV-uwtools] *", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-DEV-uwtools-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1578,7 +1578,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.7" } }, "nbformat": 4, diff --git a/notebooks/rocoto.ipynb b/notebooks/rocoto.ipynb index 984ff40d4..9b36c6d8a 100644 --- a/notebooks/rocoto.ipynb +++ b/notebooks/rocoto.ipynb @@ -62,7 +62,7 @@ "\n", " If no input file is specified, ``stdin`` is read. A ``YAMLConfig`` object may also be provided\n", " as input. If no output file is specified, ``stdout`` is written to. Both the input config and\n", - " output Rocoto XML will be validated against appropriate schcemas.\n", + " output Rocoto XML will be validated against appropriate schemas.\n", "\n", " :param config: YAML input file or ``YAMLConfig`` object (``None`` => read ``stdin``).\n", " :param output_file: XML output file path (``None`` => write to ``stdout``).\n", @@ -138,8 +138,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-31T09:34:05] INFO 0 UW schema-validation errors found\n", - "[2024-10-31T09:34:05] INFO 0 Rocoto validation errors found\n" + "[2024-11-19T23:15:43] INFO 0 UW schema-validation errors found in Rocoto config\n", + "[2024-11-19T23:15:43] INFO 0 Rocoto XML validation errors found\n" ] }, { @@ -255,13 +255,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-31T09:34:05] ERROR 3 UW schema-validation errors found\n", - "[2024-10-31T09:34:05] ERROR Error at workflow -> attrs:\n", - "[2024-10-31T09:34:05] ERROR 'realtime' is a required property\n", - "[2024-10-31T09:34:05] ERROR Error at workflow -> tasks -> task_greet:\n", - "[2024-10-31T09:34:05] ERROR 'command' is a required property\n", - "[2024-10-31T09:34:05] ERROR Error at workflow:\n", - "[2024-10-31T09:34:05] ERROR 'log' is a required property\n" + "[2024-11-19T23:15:43] ERROR 3 UW schema-validation errors found in Rocoto config\n", + "[2024-11-19T23:15:43] ERROR Error at workflow -> attrs:\n", + "[2024-11-19T23:15:43] ERROR 'realtime' is a required property\n", + "[2024-11-19T23:15:43] ERROR Error at workflow -> tasks -> task_greet:\n", + "[2024-11-19T23:15:43] ERROR 'command' is a required property\n", + "[2024-11-19T23:15:43] ERROR Error at workflow:\n", + "[2024-11-19T23:15:43] ERROR 'log' is a required property\n" ] }, { @@ -387,8 +387,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-31T09:34:05] INFO 0 UW schema-validation errors found\n", - "[2024-10-31T09:34:05] INFO 0 Rocoto validation errors found\n" + "[2024-11-19T23:15:43] INFO 0 UW schema-validation errors found in Rocoto config\n", + "[2024-11-19T23:15:43] INFO 0 Rocoto XML validation errors found\n" ] }, { @@ -576,8 +576,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-31T09:34:05] INFO 0 UW schema-validation errors found\n", - "[2024-10-31T09:34:05] INFO 0 Rocoto validation errors found\n" + "[2024-11-19T23:15:43] INFO 0 UW schema-validation errors found in Rocoto config\n", + "[2024-11-19T23:15:43] INFO 0 Rocoto XML validation errors found\n" ] }, { @@ -721,8 +721,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-31T09:34:05] INFO 0 UW schema-validation errors found\n", - "[2024-10-31T09:34:05] INFO 0 Rocoto validation errors found\n" + "[2024-11-19T23:15:43] INFO 0 UW schema-validation errors found in Rocoto config\n", + "[2024-11-19T23:15:43] INFO 0 Rocoto XML validation errors found\n" ] }, { @@ -924,7 +924,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-31T09:34:05] INFO 0 Rocoto validation errors found\n" + "[2024-11-19T23:15:43] INFO 0 Rocoto XML validation errors found\n" ] }, { @@ -1000,22 +1000,22 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-10-31T09:34:05] ERROR 4 Rocoto validation errors found\n", - "[2024-10-31T09:34:05] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_ATTRVALID: Element workflow failed to validate attributes\n", - "[2024-10-31T09:34:05] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_NOELEM: Expecting an element cycledef, got nothing\n", - "[2024-10-31T09:34:05] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_INTERSEQ: Invalid sequence in interleave\n", - "[2024-10-31T09:34:05] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_CONTENTVALID: Element workflow failed to validate content\n", - "[2024-10-31T09:34:05] ERROR Invalid Rocoto XML:\n", - "[2024-10-31T09:34:05] ERROR 1 \n", - "[2024-10-31T09:34:05] ERROR 2 \n", - "[2024-10-31T09:34:05] ERROR 3 logs/test.log\n", - "[2024-10-31T09:34:05] ERROR 4 \n", - "[2024-10-31T09:34:05] ERROR 5 1\n", - "[2024-10-31T09:34:05] ERROR 6 00:00:10\n", - "[2024-10-31T09:34:05] ERROR 7 echo Hello, World!\n", - "[2024-10-31T09:34:05] ERROR 8 greet\n", - "[2024-10-31T09:34:05] ERROR 9 \n", - "[2024-10-31T09:34:05] ERROR 10 \n" + "[2024-11-19T23:15:43] ERROR 4 Rocoto XML validation errors found\n", + "[2024-11-19T23:15:43] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_ATTRVALID: Element workflow failed to validate attributes\n", + "[2024-11-19T23:15:43] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_NOELEM: Expecting an element cycledef, got nothing\n", + "[2024-11-19T23:15:43] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_INTERSEQ: Invalid sequence in interleave\n", + "[2024-11-19T23:15:43] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_CONTENTVALID: Element workflow failed to validate content\n", + "[2024-11-19T23:15:43] ERROR Invalid Rocoto XML:\n", + "[2024-11-19T23:15:43] ERROR 1 \n", + "[2024-11-19T23:15:43] ERROR 2 \n", + "[2024-11-19T23:15:43] ERROR 3 logs/test.log\n", + "[2024-11-19T23:15:43] ERROR 4 \n", + "[2024-11-19T23:15:43] ERROR 5 1\n", + "[2024-11-19T23:15:43] ERROR 6 00:00:10\n", + "[2024-11-19T23:15:43] ERROR 7 echo Hello, World!\n", + "[2024-11-19T23:15:43] ERROR 8 greet\n", + "[2024-11-19T23:15:43] ERROR 9 \n", + "[2024-11-19T23:15:43] ERROR 10 \n" ] }, { @@ -1038,9 +1038,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:DEV-uwtools] *", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-DEV-uwtools-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1052,7 +1052,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.7" } }, "nbformat": 4, diff --git a/notebooks/template.ipynb b/notebooks/template.ipynb index 39a447f70..5e5627d05 100644 --- a/notebooks/template.ipynb +++ b/notebooks/template.ipynb @@ -45,7 +45,7 @@ "text": [ "Help on function render in module uwtools.api.template:\n", "\n", - "render(values_src: Union[dict, pathlib.Path, str, NoneType] = None, values_format: Optional[str] = None, input_file: Union[str, pathlib.Path, NoneType] = None, output_file: Union[str, pathlib.Path, NoneType] = None, overrides: Optional[dict[str, str]] = None, env: bool = False, searchpath: Optional[list[str]] = None, values_needed: bool = False, dry_run: bool = False, stdin_ok: bool = False) -> str\n", + "render(values_src: Union[dict, str, pathlib.Path, NoneType] = None, values_format: Optional[str] = None, input_file: Union[str, pathlib.Path, NoneType] = None, output_file: Union[str, pathlib.Path, NoneType] = None, overrides: Optional[dict[str, str]] = None, env: bool = False, searchpath: Optional[list[str]] = None, values_needed: bool = False, dry_run: bool = False, stdin_ok: bool = False) -> str\n", " Render a Jinja2 template to a file, based on specified values.\n", "\n", " Primary values used to render the template are taken from the specified file. The format of the\n", @@ -121,10 +121,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-08-20T08:10:58] INFO Value(s) needed to render this template are:\n", - "[2024-08-20T08:10:58] INFO first\n", - "[2024-08-20T08:10:58] INFO food\n", - "[2024-08-20T08:10:58] INFO last\n" + "[2024-11-19T23:16:37] INFO Value(s) needed to render this template are:\n", + "[2024-11-19T23:16:37] INFO first\n", + "[2024-11-19T23:16:37] INFO food\n", + "[2024-11-19T23:16:37] INFO last\n" ] }, { @@ -307,7 +307,7 @@ "text": [ "Help on function render_to_str in module uwtools.api.template:\n", "\n", - "render_to_str(values_src: Union[dict, pathlib.Path, str, NoneType] = None, values_format: Optional[str] = None, input_file: Union[str, pathlib.Path, NoneType] = None, overrides: Optional[dict[str, str]] = None, env: bool = False, searchpath: Optional[list[str]] = None, values_needed: bool = False, dry_run: bool = False) -> str\n", + "render_to_str(values_src: Union[dict, str, pathlib.Path, NoneType] = None, values_format: Optional[str] = None, input_file: Union[str, pathlib.Path, NoneType] = None, overrides: Optional[dict[str, str]] = None, env: bool = False, searchpath: Optional[list[str]] = None, values_needed: bool = False, dry_run: bool = False) -> str\n", " Render a Jinja2 template to a string, based on specified values.\n", "\n", " See ``render()`` for details on arguments, etc.\n", @@ -489,9 +489,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:DEV-uwtools] *", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-DEV-uwtools-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -503,7 +503,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.7" } }, "nbformat": 4, From 2f7cac61079f4c64be6425c5e1e46d22ae9a105a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 19 Nov 2024 23:23:38 +0000 Subject: [PATCH 22/24] Test fixes --- notebooks/tests/test_rocoto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/tests/test_rocoto.py b/notebooks/tests/test_rocoto.py index 2fe0e9adc..9f5fdac3c 100644 --- a/notebooks/tests/test_rocoto.py +++ b/notebooks/tests/test_rocoto.py @@ -77,11 +77,11 @@ def test_validate(): err_xml = f.read().rstrip() with testbook("rocoto.ipynb", execute=True) as tb: assert tb.cell_output_text(41) == simple_xml - valid_out = ("INFO 0 Rocoto validation errors found", "True") + valid_out = ("INFO 0 Rocoto XML validation errors found", "True") assert all(x in tb.cell_output_text(43) for x in valid_out) assert tb.cell_output_text(45) == err_xml err_out = ( - "ERROR 4 Rocoto validation errors found", + "ERROR 4 Rocoto XML validation errors found", "Element workflow failed to validate attributes", "Expecting an element cycledef, got nothing", "Invalid sequence in interleave", From 33a2879916f4a2b8720c51bcc582c8d04d7e01b2 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 19 Nov 2024 23:24:54 +0000 Subject: [PATCH 23/24] Test fixes --- notebooks/tests/test_rocoto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/tests/test_rocoto.py b/notebooks/tests/test_rocoto.py index 9f5fdac3c..389b8851d 100644 --- a/notebooks/tests/test_rocoto.py +++ b/notebooks/tests/test_rocoto.py @@ -12,7 +12,7 @@ def test_building_simple_workflow(): assert tb.cell_output_text(5) == simple_yaml valid_out = ( "INFO 0 UW schema-validation errors found", - "INFO 0 Rocoto validation errors found", + "INFO 0 Rocoto XML validation errors found", "True", ) assert all(x in tb.cell_output_text(7) for x in valid_out) @@ -55,7 +55,7 @@ def test_building_workflows(): assert tb.cell_output_text(17) == ent_cs_yaml valid_out = ( "INFO 0 UW schema-validation errors found", - "INFO 0 Rocoto validation errors found", + "INFO 0 Rocoto XML validation errors found", "True", ) assert all(x in tb.cell_output_text(19) for x in valid_out) From a8f250c44c4b220852eda828b9b327f4222040b4 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 19 Nov 2024 23:33:34 +0000 Subject: [PATCH 24/24] Test fixes --- notebooks/tests/test_config.py | 42 +++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/notebooks/tests/test_config.py b/notebooks/tests/test_config.py index 5e220288e..4f09b24fb 100644 --- a/notebooks/tests/test_config.py +++ b/notebooks/tests/test_config.py @@ -1,3 +1,5 @@ +from textwrap import dedent + import yaml from testbook import testbook @@ -83,13 +85,19 @@ def test_compare(): with testbook("config.ipynb", execute=True) as tb: assert base_cfg in tb.cell_output_text(57) assert alt_cfg in tb.cell_output_text(57) - diff_cmp = ( - "INFO - fixtures/config/base-config.nml", - "INFO + fixtures/config/alt-config.nml", - "INFO memo: sent: - False + True", - "False", - ) - assert all(x in tb.cell_output_text(59) for x in diff_cmp) + diff_cmp = """ + INFO - fixtures/config/base-config.nml + INFO + fixtures/config/alt-config.nml + INFO --------------------------------------------------------------------- + INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line + INFO --------------------------------------------------------------------- + INFO memo: + INFO message: '{{ greeting }}, {{ recipient }}!' + INFO sender_id: '{{ id }}' + INFO - sent: false + INFO + sent: true + """ + assert all(x in tb.cell_output_text(59) for x in dedent(diff_cmp).strip().split("\n")) assert base_cfg == cp_cfg # cell 61 creates this copy same_cmp = ("INFO - fixtures/config/base-config.nml", "INFO + tmp/config-copy.nml", "True") assert all(x in tb.cell_output_text(63) for x in same_cmp) @@ -122,12 +130,20 @@ def test_cfg_classes(): dump = f.read().rstrip() assert tb.cell_output_text(79) == cfg assert tb.cell_output_text(81) == "True" - diff_cmp = ( - "INFO fruit count: grapes: - {{ grape_count }} + 8", - "INFO fruit count: kiwis: - 2 + 1", - "False", - ) - assert all(x in tb.cell_output_text(83) for x in diff_cmp) + diff_cmp = """ + INFO --------------------------------------------------------------------- + INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line + INFO --------------------------------------------------------------------- + INFO fruit count: + INFO apples: '3' + INFO - grapes: '{{ grape_count }}' + INFO + grapes: '8' + INFO - kiwis: '2' + INFO ? ^ + INFO + kiwis: '1' + INFO ? ^ + """ + assert all(x in tb.cell_output_text(83) for x in dedent(diff_cmp).strip().split("\n")) assert "grapes = 15" in tb.cell_output_text(85) assert tb.cell_output_text(89) == dump dump_dict = ("[fruit count]", "oranges = 4", "blueberries = 9")