From d32c5915488041ab667a9bd32fe8fbc5f91f87d3 Mon Sep 17 00:00:00 2001 From: Fabian Date: Sun, 26 Jan 2025 11:45:39 +0100 Subject: [PATCH] add scheme to deprecate/rename config entries add check for validity of config entries --- Snakefile | 12 +++- config/deprecations.yaml | 8 +++ scripts/_helpers.py | 80 ++++++++++++++++++++++++++- test/test_config_checks.py | 109 +++++++++++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 config/deprecations.yaml create mode 100644 test/test_config_checks.py diff --git a/Snakefile b/Snakefile index e93b6e0e6..b077a475a 100644 --- a/Snakefile +++ b/Snakefile @@ -10,7 +10,14 @@ from snakemake.utils import min_version min_version("8.11") -from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir +from scripts._helpers import ( + path_provider, + copy_default_files, + get_scenarios, + get_rdir, + check_deprecated_config, + check_invalid_config, +) copy_default_files(workflow) @@ -20,6 +27,9 @@ configfile: "config/config.default.yaml" configfile: "config/config.yaml" +check_deprecated_config(config, "config/deprecations.yaml") +check_invalid_config(config, "config/config.default.yaml") + run = config["run"] scenarios = get_scenarios(run) RDIR = get_rdir(run) diff --git a/config/deprecations.yaml b/config/deprecations.yaml new file mode 100644 index 000000000..6d91fa654 --- /dev/null +++ b/config/deprecations.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# +# SPDX-License-Identifier: MIT + +# Format: list of deprecation entries with: +# - old_entry: Dot-separated path to deprecated entry (e.g. "electricity:co2_limit") +# - new_entry: [optional] New location path for renamed entries +# - message: [optional] Custom warning message diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 1251e2ab3..724e1e2de 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -6,10 +6,12 @@ import copy import hashlib import logging +import operator import os import re import time -from functools import partial, wraps +import warnings +from functools import partial, reduce, wraps from os.path import exists from pathlib import Path from shutil import copyfile @@ -28,6 +30,18 @@ REGION_COLS = ["geometry", "name", "x", "y", "country"] +class DeprecationConfigWarning(Warning): + """Warning for use of deprecated configuration entries.""" + + pass + + +class InvalidConfigWarning(Warning): + """Warning for use of invalid/unsupported configuration entries.""" + + pass + + def copy_default_files(workflow): default_files = { "config/config.default.yaml": "config/config.yaml", @@ -146,6 +160,70 @@ def path_provider(dir, rdir, shared_resources, exclude_from_shared): ) +def check_deprecated_config(config: dict, deprecations_file: str) -> None: + """Check config against deprecations and warn users""" + + with open(deprecations_file) as f: + deprecations = yaml.safe_load(f) + + def get_by_path(root, path): + try: + return reduce(operator.getitem, path.split(":"), root) + except KeyError: + return None + + def set_by_path(root, path, value): + keys = path.split(":") + for key in keys[:-1]: + root = root.setdefault(key, {}) + root[keys[-1]] = value + + for entry in deprecations: + old_entry = entry["old_entry"] + current_value = get_by_path(config, old_entry) + + if current_value is not None: + msg = f"Config entry '{old_entry}' is deprecated. " + + if "new_entry" in entry: # Rename case + new_entry = entry["new_entry"] + msg += f"Use '{new_entry}' instead." + + if get_by_path(config, new_entry) is not None: + msg += " Both keys present - remove deprecated entry." + else: + set_by_path(config, new_entry, current_value) + else: # Removal case + msg += "This entry is no longer used and should be removed." + + if "message" in entry: + msg += f" Note: {entry['message']}" + + warnings.warn(msg, DeprecationConfigWarning) + + +def check_invalid_config(config: dict, config_default_fn: str) -> None: + """Check if config contains entries that are not supported by the default config""" + + with open(config_default_fn) as f: + config_default = yaml.safe_load(f) + + def check_keys(config, config_default, path=""): + for key in config.keys(): + nested_path = f"{path}:{key}" if path else key + if key not in config_default: + warnings.warn( + f"Config entry '{nested_path}' is not supported in {config_default_fn}.", + InvalidConfigWarning, + ) + elif isinstance(config[key], dict): + # Only recurse if the key exists in both configs and is a dict in both + if isinstance(config_default[key], dict): + check_keys(config[key], config_default[key], nested_path) + + check_keys(config, config_default) + + def get_opt(opts, expr, flags=None): """ Return the first option matching the regular expression. diff --git a/test/test_config_checks.py b/test/test_config_checks.py new file mode 100644 index 000000000..f0384334d --- /dev/null +++ b/test/test_config_checks.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# +# SPDX-License-Identifier: MIT +import warnings +from io import StringIO +from unittest.mock import patch + +from scripts._helpers import ( + DeprecationConfigWarning, + InvalidConfigWarning, + check_deprecated_config, + check_invalid_config, +) + +SAMPLE_DEPRECATIONS = """ +- old_entry: "old:key" + new_entry: "new:key" + +- old_entry: "removed:key" + message: "This key is obsolete and should be deleted" + +- old_entry: "example:old_key" + new_entry: "example:new_key" + message: "Custom warning message" +""" + + +def test_config_deprecations(): + test_config = { + "old": {"key": "legacy_value"}, + "removed": {"key": "dangerous_value"}, + "example": {"old_key": "original_value", "new_key": "existing_value"}, + "unrelated": {"data": "untouched"}, + } + + with warnings.catch_warnings(record=True) as captured_warnings: + with patch("builtins.open", return_value=StringIO(SAMPLE_DEPRECATIONS)): + check_deprecated_config(test_config, "dummy_path.yaml") + + # Verify warnings + assert len(captured_warnings) == 3 + + warning_messages = [str(w.message) for w in captured_warnings] + + # Check basic rename warning + assert any( + "'old:key' is deprecated. Use 'new:key' instead" in msg + for msg in warning_messages + ) + + # Check removal warning with custom message + assert any("obsolete and should be deleted" in msg for msg in warning_messages) + + # Check custom message and conflict warning + assert any("Custom warning message" in msg for msg in warning_messages) + assert any( + "Both keys present - remove deprecated entry" in msg for msg in warning_messages + ) + + # Verify warning types + assert all( + isinstance(w.message, DeprecationConfigWarning) for w in captured_warnings + ) + + # Verify config updates + assert test_config["new"]["key"] == "legacy_value" # Renamed value + assert ( + test_config["example"]["new_key"] == "existing_value" + ) # Existing value preserved + assert "key" in test_config["removed"] # Removed key not deleted (just warned) + assert test_config["unrelated"] == {"data": "untouched"} # Unrelated data unchanged + + +def test_config_invalid_entries(): + test_config = { + "valid_section": {"nested_valid": "ok"}, + "invalid_section": {"bad_key": "value"}, + "clustering": {"invalid_option": "bad"}, + } + + default_config = """ +valid_section: + nested_valid: default + other_valid: default +clustering: + temporal: + resolution: 1 + """ + + with warnings.catch_warnings(record=True) as captured_warnings: + with patch("builtins.open", return_value=StringIO(default_config)): + check_invalid_config(test_config, "dummy_default.yaml") + + warning_messages = [str(w.message) for w in captured_warnings] + + # Check warning for invalid top-level section + assert any( + "Config entry 'invalid_section' is not supported" in msg + for msg in warning_messages + ) + + # Check warning for invalid nested option + assert any( + "Config entry 'clustering:invalid_option' is not supported" in msg + for msg in warning_messages + ) + + # Verify warning types + assert all(isinstance(w.message, InvalidConfigWarning) for w in captured_warnings)