Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

UW 500 - Allow YAML in all conversions #414

Merged
merged 33 commits into from
Feb 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
b459871
edit config error message and test
WeirAE Feb 22, 2024
fba233a
modify error text for clarity and doc text
WeirAE Feb 22, 2024
6c5dc46
Updated to allow YAML as output always
WeirAE Feb 23, 2024
6c510c0
missed error documentation update
WeirAE Feb 23, 2024
01dc15a
Merge branch 'main' into UW-500
WeirAE Feb 23, 2024
08f27c7
modified representer for f90nml
WeirAE Feb 27, 2024
ceb9829
update to prod mode_config.rst for resolution
WeirAE Feb 27, 2024
93cafea
for some reason, a space
WeirAE Feb 27, 2024
28aee74
UW-509 docs for sfc_climo_gen driver (#412)
maddenp-noaa Feb 23, 2024
32189a1
Obtain iotaa from conda-forge (#415)
maddenp-noaa Feb 26, 2024
c8c1304
Add conda badges and update pylint (#416)
maddenp-noaa Feb 26, 2024
4b665d1
modify error text for clarity and doc text
WeirAE Feb 22, 2024
0073332
Updated to allow YAML as output always
WeirAE Feb 23, 2024
d1f64f5
missed error documentation update
WeirAE Feb 23, 2024
6307569
update to prod mode_config.rst for resolution
WeirAE Feb 27, 2024
5fd20ac
for some reason, a space
WeirAE Feb 27, 2024
fce1bb3
Merge branch 'main' into UW-500
WeirAE Feb 27, 2024
1eba589
Update error message and documentation
WeirAE Feb 27, 2024
9554890
Merge branch 'main' into UW-500
WeirAE Feb 28, 2024
173909a
Changes as follows below:
WeirAE Feb 28, 2024
c7c1208
updated recursive function within ordereddict
WeirAE Feb 28, 2024
0860739
Update src/uwtools/config/support.py
WeirAE Feb 28, 2024
6bd8411
Update src/uwtools/config/support.py
WeirAE Feb 28, 2024
0c4a8af
Update src/uwtools/config/support.py
WeirAE Feb 28, 2024
4804996
Update src/uwtools/config/support.py
WeirAE Feb 28, 2024
b259538
Update src/uwtools/config/support.py
WeirAE Feb 28, 2024
0d5f4fe
Update src/uwtools/config/support.py
WeirAE Feb 28, 2024
e0d75e2
Update src/uwtools/config/support.py
WeirAE Feb 28, 2024
4886cd5
fix f90nml calls for cleanliness
WeirAE Feb 28, 2024
f4b1ac0
consolidate representers and test
WeirAE Feb 28, 2024
cac0195
Update src/uwtools/config/support.py
WeirAE Feb 28, 2024
d6a7f59
fix organization and doc string
WeirAE Feb 28, 2024
37643d4
formatting
WeirAE Feb 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/sections/user_guide/cli/tools/mode_config.rst
Original file line number Diff line number Diff line change
Expand Up @@ -434,16 +434,16 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con
.. note:: In recognition of the different sets of value types representable in each config format, ``uw`` supports two format-combination schemes:

1. **Output matches input:** The format of the output config matches that of the input config.
2. **Input is YAML:** If the input config is YAML, any output format may be requested. In the worst case, values always have a string representation, but note that, for example, the string representation of a YAML sequence (Python ``list``) in an INI output config may not be useful.
2. **YAML:** YAML is accepted as either input or output with any other format. In the worst case, values always have a string representation, but note that, for example, the string representation of a YAML sequence (Python ``list``) in an INI output config may not be useful.

In all cases, any supplemental configs must be in the same format as the input config and must have recognized extensions.

``uw`` considers invalid combination requests errors:

.. code-block:: text

$ uw config realize --input-file b.nml --output-file a.yaml
Output format yaml must match input format nml
$ uw config realize --input-file b.nml --output-file a.ini
Accepted output formats for input format nml are nml or yaml

.. code-block:: text

Expand Down
6 changes: 3 additions & 3 deletions src/uwtools/config/formats/yaml.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import yaml

from uwtools.config.formats.base import Config
from uwtools.config.support import INCLUDE_TAG, TaggedString, log_and_error
from uwtools.config.support import INCLUDE_TAG, TaggedString, add_representers, log_and_error
from uwtools.utils.file import FORMAT, readable, writable

_MSGS = ns(
Expand Down Expand Up @@ -44,7 +44,7 @@ def __repr__(self) -> str:
"""
The string representation of a YAMLConfig object.
"""
yaml.add_representer(TaggedString, TaggedString.represent)
add_representers()
return yaml.dump(self.data, default_flow_style=False).strip()

# Private methods
Expand Down Expand Up @@ -115,7 +115,7 @@ def dump_dict(cfg: dict, path: Optional[Path] = None) -> None:
:param cfg: The in-memory config object to dump.
:param path: Path to dump config to.
"""
yaml.add_representer(TaggedString, TaggedString.represent)
add_representers()
with writable(path) as f:
yaml.dump(cfg, f, sort_keys=False)

Expand Down
43 changes: 43 additions & 0 deletions src/uwtools/config/support.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
from __future__ import annotations

from collections import OrderedDict
from importlib import import_module
from typing import Dict, Type, Union

import yaml
from f90nml import Namelist # type: ignore

from uwtools.exceptions import UWConfigError
from uwtools.logging import log
Expand All @@ -12,6 +14,16 @@
INCLUDE_TAG = "!INCLUDE"


# Public functions
def add_representers() -> None:
"""
Add representers to the YAML dumper for custom types.
"""
WeirAE marked this conversation as resolved.
Show resolved Hide resolved
yaml.add_representer(TaggedString, TaggedString.represent)
yaml.add_representer(Namelist, _represent_namelist)
yaml.add_representer(OrderedDict, _represent_ordereddict)


def depth(d: dict) -> int:
"""
The depth of a dictionary.
Expand Down Expand Up @@ -53,6 +65,37 @@ def log_and_error(msg: str) -> Exception:
return UWConfigError(msg)


# Private functions
def _represent_namelist(dumper: yaml.Dumper, data: Namelist) -> yaml.nodes.MappingNode:
"""
Convert f90nml Namelist to OrderedDict and serialize.

:param dumper: The YAML dumper.
:param data: The f90nml Namelist to serialize.
"""
# Convert the f90nml Namelist to an OrderedDict.
namelist_dict = data.todict()

# Represent the OrderedDict as a YAML mapping.
return dumper.represent_mapping("tag:yaml.org,2002:map", namelist_dict)


def _represent_ordereddict(dumper: yaml.Dumper, data: OrderedDict) -> yaml.nodes.MappingNode:
"""
Convert OrderedDict to dict and serialize.

:param dumper: The YAML dumper.
:param data: The OrderedDict to serialize.
"""

# Convert the OrderedDict to a dict.
def from_od(d: Union[OrderedDict, Dict]) -> dict:
return {key: from_od(val) if isinstance(val, dict) else val for key, val in d.items()}

# Represent the dict as a YAML mapping.
return dumper.represent_mapping("tag:yaml.org,2002:map", from_od(data))


class TaggedString:
"""
A class supporting custom YAML tags specifying type conversions.
Expand Down
6 changes: 4 additions & 2 deletions src/uwtools/config/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,8 +251,10 @@ def _validate_format_output(input_fmt: str, output_fmt: str) -> None:
:param output_fmt: Output format.
:raises: UWError if output format is incompatible.
"""
if not input_fmt in (FORMAT.yaml, output_fmt):
raise UWError("Output format %s must match input format %s" % (output_fmt, input_fmt))
if FORMAT.yaml not in (input_fmt, output_fmt) and input_fmt != output_fmt:
WeirAE marked this conversation as resolved.
Show resolved Hide resolved
raise UWError(
"Accepted output formats for input format %s are %s or yaml" % (input_fmt, input_fmt)
)


def _validate_format_supplemental(
Expand Down
22 changes: 22 additions & 0 deletions src/uwtools/tests/config/test_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
"""

import logging
from collections import OrderedDict

import pytest
import yaml
from f90nml import Namelist, reads # type: ignore
from pytest import fixture, raises

from uwtools.config import support
Expand All @@ -21,6 +23,14 @@
from uwtools.utils.file import FORMAT


def test_add_representers():
support.add_representers()
representers = yaml.Dumper.yaml_representers
assert support.TaggedString in representers
assert OrderedDict in representers
assert Namelist in representers


@pytest.mark.parametrize(
"d,n", [({1: 88}, 1), ({1: {2: 88}}, 2), ({1: {2: {3: 88}}}, 3), ({1: {}}, 2)]
)
Expand Down Expand Up @@ -56,6 +66,18 @@ def test_log_and_error(caplog):
assert logged(caplog, msg)


def test_represent_namelist():
namelist = reads("&namelist\n key = value\n/\n")
assert yaml.dump(namelist, default_flow_style=True).strip() == "{namelist: {key: value}}"


def test_represent_ordereddict():
ordereddict_values = OrderedDict([("example", OrderedDict([("key", "value")]))])
assert (
yaml.dump(ordereddict_values, default_flow_style=True).strip() == "{example: {key: value}}"
)
WeirAE marked this conversation as resolved.
Show resolved Hide resolved


class Test_TaggedString:
"""
Tests for class uwtools.config.support.TaggedString.
Expand Down
7 changes: 5 additions & 2 deletions src/uwtools/tests/config/test_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -645,12 +645,15 @@ def test__realize_config_values_needed_negative_results(caplog, tmp_path):
@pytest.mark.parametrize("output_fmt", FORMAT.extensions())
def test__validate_format_output(input_fmt, output_fmt):
call = lambda: tools._validate_format_output(input_fmt=input_fmt, output_fmt=output_fmt)
if input_fmt in (FORMAT.yaml, output_fmt):
if FORMAT.yaml in (input_fmt, output_fmt) or input_fmt == output_fmt:
WeirAE marked this conversation as resolved.
Show resolved Hide resolved
call() # no exception raised
else:
with raises(UWError) as e:
call()
assert str(e.value) == f"Output format {output_fmt} must match input format {input_fmt}"
assert (
str(e.value) == "Accepted output formats for input format "
f"{input_fmt} are {input_fmt} or yaml"
)


def test__validate_format_supplemental_fail_obj():
Expand Down
Loading