diff --git a/data/catalogs/deltares_data.yml b/data/catalogs/deltares_data.yml index c59d8fe5a..9ebd9703c 100644 --- a/data/catalogs/deltares_data.yml +++ b/data/catalogs/deltares_data.yml @@ -761,7 +761,7 @@ ghs_pop_2015_4326_v2019a: path: socio_economic/ghs/GHS_POP_E2015_GLOBE_R2019A_4326_9ss_V1_0.tif ghs_pop_2015_54009_v2019a: - crs: 54009 + crs: "ESRI:54009" data_type: RasterDataset driver: raster kwargs: @@ -779,7 +779,7 @@ ghs_pop_2015_54009_v2019a: ghs_smod: alias: ghs_smod_2015_54009_v2019a ghs_smod_2015_54009_v2016a: - crs: 54009 + crs: "ESRI:54009" data_type: RasterDataset driver: raster kwargs: @@ -796,7 +796,7 @@ ghs_smod_2015_54009_v2016a: path: socio_economic/ghs/GHS_SMOD_POP2015_GLOBE_R2016A_54009_1k_v1_0.tif ghs_smod_2015_54009_v2019a: - crs: 54009 + crs: "ESRI:54009" data_type: RasterDataset driver: raster kwargs: diff --git a/docs/changelog.rst b/docs/changelog.rst index 0a569365a..7861ab3a3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,6 +15,7 @@ Added ----- - Export CLI now also accepts time tuples (#660) - New stats.skills VE and RSR (#666) +- Check CLI command can now validate bbox and geom regions (#664) Changed ------- diff --git a/docs/user_guide/data_overview.rst b/docs/user_guide/data_overview.rst index 871e02c66..c546d24ad 100644 --- a/docs/user_guide/data_overview.rst +++ b/docs/user_guide/data_overview.rst @@ -77,7 +77,9 @@ However, we do plan to expand its functionality over time. It can be use for exa .. code-block:: console - hydromt check grid -d /path/to/data_catalog.yml -i /path/to/model_config.yml + hydromt check grid -d /path/to/data_catalog.yml -i /path/to/model_config.yml -r '{'bbox': [-1,-1,1,1]}' + +currently only `bbox` and `geom` variants of regions are supported in validation. Also note that the geom variant will only check whether the file exists not its contents. We also plan to expand this functionality in the future. .. _get_data_python: diff --git a/hydromt/cli/main.py b/hydromt/cli/main.py index c0e247525..7d04514f1 100644 --- a/hydromt/cli/main.py +++ b/hydromt/cli/main.py @@ -16,7 +16,8 @@ from hydromt.data_catalog import DataCatalog from hydromt.validators.data_catalog import DataCatalogValidator -from hydromt.validators.model_config import HydromtModelStep +from hydromt.validators.model_config import HydromtModelSetup +from hydromt.validators.region import validate_region from .. import __version__, log from ..models import MODELS @@ -332,39 +333,51 @@ def update( @main.command( - short_help="Validate config files are correct", + short_help="Validate config / data catalog / region", ) -@click.argument( - "MODEL", +@click.option( + "-m", + "--model", type=str, + default=None, + help="Model name, e.g. wflow, sfincs, etc. to validate config file.", ) @opt_config @data_opt -@deltares_data_opt @quiet_opt @verbose_opt +@region_opt @click.pass_context def check( ctx, model, config, data, - dd, + region: Optional[Dict[Any, Any]], quiet: int, verbose: int, ): - """Verify that provided data catalog files are in the correct format. + """ + Verify that provided data catalog and config files are in the correct format. + + Additionnaly region bbox and geom can also be validated. Example usage: -------------- - hydromt check grid -d /path/to/data_catalog.yml -i /path/to/model_config.yml + Check data catalog file: + hydromt check -d /path/to/data_catalog.yml -v + + Check data catalog and grid_model config file: + hydromt check -m grid_model -d /path/to/data_catalog.yml -i /path/to/model_config.yml -v + + With region: + hydromt check -m grid_model -d /path/to/data_catalog.yml -i /path/to/model_config.yml -r '{'bbox': [-1,-1,1,1]}' -v """ # noqa: E501 # logger log_level = max(10, 30 - 10 * (verbose - quiet)) logger = log.setuplog("check", join(".", "hydromt.log"), log_level=log_level) - logger.info(f"Output dir: {export_dest_path}") try: all_exceptions = [] for cat_path in data: @@ -376,24 +389,36 @@ def check( all_exceptions.append(e) logger.info("Catalog has errors") - mod = MODELS.load(model) - try: - config_dict = cli_utils.parse_config(config) - logger.info(f"Validating config at {config}") + if region: + logger.info(f"Validating region {region}") + try: + validate_region(region) + logger.info("Region is valid!") + + except (ValidationError, ValueError, NotImplementedError) as e: + logger.info("region has errors") + all_exceptions.append(e) - HydromtModelStep.from_dict(config_dict, model=mod) - logger.info("Model config valid!") + if config: + mod = MODELS.load(model) + logger.info(f"Validating for model {model} of type {type(mod).__name__}") + try: + config_dict = cli_utils.parse_config(config) + logger.info(f"Validating config at {config}") - except (ValidationError, ValueError) as e: - logger.info("Model has errors") - all_exceptions.append(e) + HydromtModelSetup.from_dict(config_dict, model=mod) + logger.info("Model config valid!") + + except (ValidationError, ValueError) as e: + logger.info("Model has errors") + all_exceptions.append(e) if len(all_exceptions) > 0: - raise Exception(all_exceptions) + raise ValueError(all_exceptions) except Exception as e: logger.exception(e) # catch and log errors - raise + raise e finally: for handler in logger.handlers[:]: handler.close() diff --git a/hydromt/validators/__init__.py b/hydromt/validators/__init__.py index 626fa40f8..db9f0b56c 100644 --- a/hydromt/validators/__init__.py +++ b/hydromt/validators/__init__.py @@ -6,23 +6,11 @@ DataCatalogMetaData, DataCatalogValidator, ) -from .model_config import HydromtModelStep +from .model_config import HydromtModelSetup, HydromtModelStep from .region import ( - BoundingBoxBasinRegion, - BoundingBoxInterBasinRegion, BoundingBoxRegion, - BoundingBoxSubBasinRegion, - GeometryBasinRegion, - GeometryInterBasinRegion, - GeometryRegion, - GeometrySubBasinRegion, - GridRegion, - MeshRegion, - MultiPointBasinRegion, - MultiPointSubBasinRegion, - PointBasinRegion, - PointSubBasinRegion, - WGS84Point, + PathRegion, + Region, validate_region, ) @@ -31,21 +19,10 @@ "DataCatalogItemMetadata", "DataCatalogMetaData", "DataCatalogValidator", - "BoundingBoxBasinRegion", - "BoundingBoxInterBasinRegion", "BoundingBoxRegion", - "BoundingBoxSubBasinRegion", - "GeometryBasinRegion", - "GeometryInterBasinRegion", - "GeometryRegion", - "GeometrySubBasinRegion", - "GridRegion", - "MeshRegion", - "MultiPointBasinRegion", - "MultiPointSubBasinRegion", - "PointBasinRegion", - "PointSubBasinRegion", - "WGS84Point", + "PathRegion", + "Region", "validate_region", "HydromtModelStep", + "HydromtModelSetup", ] diff --git a/hydromt/validators/data_catalog.py b/hydromt/validators/data_catalog.py index c8fe5405a..a5592d314 100644 --- a/hydromt/validators/data_catalog.py +++ b/hydromt/validators/data_catalog.py @@ -1,10 +1,12 @@ """Pydantic models for the validation of Data catalogs.""" from pathlib import Path -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union -from pydantic import AnyUrl, BaseModel, ConfigDict +from pydantic import AnyUrl, BaseModel, ConfigDict, model_validator from pydantic.fields import Field from pydantic_core import Url +from pyproj import CRS +from pyproj.exceptions import CRSError from hydromt.data_catalog import _yml_from_uri_or_path from hydromt.typing import Bbox, Number, TimeRange @@ -23,6 +25,17 @@ def from_dict(input_dict): return SourceSpecDict(**input_dict) +class SourceVariant(BaseModel): + """A variant for a data source.""" + + provider: Optional[Literal["local", "aws", "gcs"]] = None + version: Optional[Union[str, Number]] = None + path: Path + rename: Optional[Dict[str, str]] = None + filesystem: Optional[Literal["local", "s3", "gcs"]] = None + storage_options: Optional[Dict[str, Any]] = None + + class Extent(BaseModel): """A validation model for describing the space and time a dataset covers.""" @@ -34,7 +47,7 @@ class DataCatalogMetaData(BaseModel): """The metadata section of a Hydromt data catalog.""" root: Optional[Path] = None - version: Optional[Union[str, int]] = None + version: Optional[Union[str, Number]] = None name: Optional[str] = None model_config: ConfigDict = ConfigDict( str_strip_whitespace=True, @@ -81,24 +94,48 @@ class DataCatalogItem(BaseModel): """A validated data source.""" name: str - data_type: str - driver: str - path: Path - crs: Optional[int] = None + data_type: Literal["RasterDataset", "GeoDataset", "GeoDataFrame", "DataFrame"] + driver: Literal[ + "csv", + "fwf", + "netcdf", + "parquet", + "raster", + "raster_tindex", + "vector", + "vector_table", + "xls", + "xlsx", + "zarr", + ] + path: Optional[Path] = None + crs: Optional[Union[int, str]] = None filesystem: Optional[str] = None kwargs: Dict[str, Any] = Field(default_factory=dict) storage_options: Dict[str, Any] = Field(default_factory=dict) + placeholders: Optional[Dict[str, Any]] = None rename: Dict[str, str] = Field(default_factory=dict) nodata: Optional[Number] = None meta: Optional[DataCatalogItemMetadata] = None unit_add: Optional[Dict[str, Number]] = None unit_mult: Optional[Dict[str, Number]] = None + variants: Optional[List[SourceVariant]] = None + version: Optional[Union[str, Number]] = None model_config: ConfigDict = ConfigDict( str_strip_whitespace=True, extra="forbid", ) + @model_validator(mode="after") + def _check_valid_crs(self) -> "DataCatalogItem": + try: + if self.crs: + _ = CRS.from_user_input(self.crs) + except CRSError as e: + raise ValueError(e) + return self + @staticmethod def from_dict(input_dict, name=None): """Convert a dictionary into a validated source item.""" diff --git a/hydromt/validators/model_config.py b/hydromt/validators/model_config.py index 0a9b19edf..39f87b5ef 100644 --- a/hydromt/validators/model_config.py +++ b/hydromt/validators/model_config.py @@ -1,6 +1,6 @@ """Pydantic models for the validation of model config files.""" from inspect import Parameter, signature -from typing import Any, Callable, Dict, Type +from typing import Any, Callable, Dict, List, Type from pydantic import BaseModel, ConfigDict, Field @@ -8,7 +8,7 @@ class HydromtModelStep(BaseModel): - """A Pydantic model for the validation of model config files.""" + """A Pydantic model for the validation of model setup functions.""" model: Type[Model] fn: Callable @@ -28,7 +28,9 @@ def from_dict(input_dict: Dict[str, Any], model: Model): try: fn = getattr(model, fn_name) except AttributeError: - raise ValueError(f"Model does not have function {fn_name}") + raise ValueError( + f"Model of type {model.__name__} does not have function {fn_name}" + ) sig = signature(fn) sig_has_var_keyword = ( @@ -54,3 +56,19 @@ def from_dict(input_dict: Dict[str, Any], model: Model): ) return HydromtModelStep(model=model, fn=fn, args=arg_dict) + + +class HydromtModelSetup(BaseModel): + """A Pydantic model for the validation of model setup files.""" + + steps: List[HydromtModelStep] + + @staticmethod + def from_dict(input_dict: Dict[str, Any], model: Model): + """Generate a validated model of a sequence steps in a model config file.""" + return HydromtModelSetup( + steps=[ + HydromtModelStep.from_dict({fn_name: fn_args}, model) + for fn_name, fn_args in input_dict.items() + ] + ) diff --git a/hydromt/validators/region.py b/hydromt/validators/region.py index 7b42113c6..ccc68620c 100644 --- a/hydromt/validators/region.py +++ b/hydromt/validators/region.py @@ -2,61 +2,30 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union -from pydantic import BaseModel, Field, ValidationError, model_validator +from pydantic import BaseModel, Field, model_validator -# bare types -class WGS84Point(BaseModel): - """A validation model for a point in WSG84 space.""" - - x: float = Field(ge=-180, le=180) - y: float = Field(ge=-90, le=90) - - @staticmethod - def from_dict(input_dict: Dict) -> "WGS84Point": - """Create a WGS84Point from a {x:,y:} dictionary.""" - return WGS84Point(**input_dict) - - @staticmethod - def from_list(input_list: List) -> "WGS84Point": - """Create a WGS84Point from a [x,y] list.""" - return WGS84Point(x=input_list[0], y=input_list[1]) - - @staticmethod - def from_xy(x: float, y: float) -> "WGS84Point": - """Create a WGS84Point from two coordinates.""" - return WGS84Point(x=x, y=y) - - -class VariableThreshold(BaseModel): - """A threshold to be applied to a region specification.""" - - name: str - threshold: float = Field(ge=0) - - -class PathlikeRegion(BaseModel): +class PathRegion(BaseModel): """A validation model for a region loaded from a file.""" path: Path - threshold: Optional[VariableThreshold] = None @staticmethod - def from_path(path: Union[Path, str]) -> "PathlikeRegion": + def from_path(path: Union[Path, str]) -> "PathRegion": """Create a region that will be loaded from a file.""" if isinstance(path, Path): - return PathlikeRegion(path=path) + return PathRegion(path=path) else: - return PathlikeRegion(path=Path(path)) + return PathRegion(path=Path(path)) @model_validator(mode="after") - def _check_path_exists(self) -> "PathlikeRegion": + def _check_path_exists(self) -> "PathRegion": if not self.path.exists(): raise ValueError(f"Path not found at {self.path}") return self -class BoundingBoxLikeRegion(BaseModel): +class BoundingBoxRegion(BaseModel): """A validation model for a region described by a bounding box in WGS84 space.""" xmin: float = Field(ge=-180) @@ -67,10 +36,10 @@ class BoundingBoxLikeRegion(BaseModel): @staticmethod def from_list( input: Union[Tuple[float, float, float, float], List[float]] - ) -> "BoundingBoxLikeRegion": + ) -> "BoundingBoxRegion": """Create a region specification from a [xmin,ymin,xmax,ymax] list.""" xmin, ymin, xmax, ymax = input - return BoundingBoxLikeRegion( + return BoundingBoxRegion( xmin=xmin, ymin=ymin, xmax=xmax, @@ -78,11 +47,11 @@ def from_list( ) @staticmethod - def from_dict(input_dict: Dict) -> "BoundingBoxLikeRegion": + def from_dict(input_dict: Dict) -> "BoundingBoxRegion": """Create a region specification from dictionary specifying values for xmin, ymin, xmax, ymax.""" xmin, ymin, xmax, ymax = input_dict["bbox"] - return BoundingBoxLikeRegion( + return BoundingBoxRegion( xmin=xmin, ymin=ymin, xmax=xmax, @@ -90,81 +59,16 @@ def from_dict(input_dict: Dict) -> "BoundingBoxLikeRegion": ) @model_validator(mode="after") - def _check_bounds_ordering(self) -> "BoundingBoxLikeRegion": + def _check_bounds_ordering(self) -> "BoundingBoxRegion": # pydantic will turn these asserion errors into validation errors for us assert self.xmin <= self.xmax assert self.ymin <= self.ymax return self -class PointLikeRegion(BaseModel): - """A validation model for a region described by a point in WGS84 space.""" - - points: List[WGS84Point] - - @staticmethod - def from_dict(input_dict: Dict) -> "PointLikeRegion": - """Create a region specification from dictionary specifying values for x and y.""" - return PointLikeRegion(points=[WGS84Point.from_dict(input_dict)]) - - @staticmethod - def from_list(input_list: Union[List, Tuple[float, float]]) -> "PointLikeRegion": - """Create a region specification from a [x,y] list or tuple.""" - if isinstance(input_list, Tuple): - return PointLikeRegion(points=[WGS84Point.from_list(list(input_list))]) - else: - return PointLikeRegion(points=[WGS84Point.from_list(input_list)]) - - @staticmethod - def from_xy(x: float, y: float) -> "PointLikeRegion": - """Create a region specification by specifying values for x and y.""" - return PointLikeRegion(points=[WGS84Point.from_xy(x=x, y=y)]) - - @staticmethod - def from_points(l: List[WGS84Point]) -> "PointLikeRegion": - """Create a region specification by specifying values for x and y.""" - return PointLikeRegion(points=l) - - @staticmethod - def from_xy_lists(xs: list[float], ys: list[float]) -> "PointLikeRegion": - """Create a region specification from lists specifying [x1,...,xn] and [y1,...,yn] respectively.""" - tups = zip(xs, ys) - - return PointLikeRegion(points=[WGS84Point.from_xy(x=x, y=y) for (x, y) in tups]) - - -GeometryRegion = PathlikeRegion -GridRegion = PathlikeRegion -MeshRegion = PathlikeRegion -GeometryBasinRegion = PathlikeRegion -GeometrySubBasinRegion = PathlikeRegion -PointSubBasinRegion = PointLikeRegion -MultiPointSubBasinRegion = PointLikeRegion -BoundingBoxSubBasinRegion = BoundingBoxLikeRegion -BoundingBoxInterBasinRegion = BoundingBoxLikeRegion -GeometryInterBasinRegion = PathlikeRegion - -BoundingBoxRegion = BoundingBoxLikeRegion -PointBasinRegion = PointLikeRegion -MultiPointBasinRegion = PointLikeRegion -BoundingBoxBasinRegion = BoundingBoxLikeRegion - - Region = Union[ BoundingBoxRegion, - GeometryRegion, - GridRegion, - MeshRegion, - PointBasinRegion, - MultiPointBasinRegion, - BoundingBoxBasinRegion, - GeometryBasinRegion, - PointSubBasinRegion, - MultiPointSubBasinRegion, - BoundingBoxSubBasinRegion, - GeometrySubBasinRegion, - BoundingBoxInterBasinRegion, - GeometryInterBasinRegion, + PathRegion, ] @@ -178,62 +82,12 @@ def validate_region(input: Dict[str, Any]) -> Optional[Region]: return BoundingBoxRegion.from_dict(val) elif "geom" in input: val = input["geom"] - return GeometryRegion.from_path(val) - elif "grid" in input: - val = input["grid"] - return GridRegion.from_path(val) - elif "mesh" in input: - val = input["mesh"] - return MeshRegion.from_path(val) - elif "basin" in input: - val = input["basin"] - if isinstance(val, list): - if isinstance(val[0], (float, int)) and len(val) == 2: # [x,y] - return PointBasinRegion.from_xy(x=val[0], y=val[1]) - elif ( - isinstance(val[0], (float, int)) and len(val) == 4 - ): # [xmin,ymin, xmaxn ymax] - return BoundingBoxBasinRegion.from_list(val) - elif isinstance(val[0], list): # [[x1,...,xn], [y1,..,yn]] - return MultiPointBasinRegion.from_xy_lists(xs=val[0], ys=val[1]) - else: - raise ValidationError(f"Unknown subbasin kind: {val}") - elif isinstance(val, str): - return GeometryBasinRegion.from_path(path=val) - else: - raise ValidationError(f"Unknown subbasin kind: {val}") - - elif "subbasin" in input: - val = input["subbasin"] - if isinstance(val, list): - if isinstance(val[0], (float, int)) and len(val) == 2: # [x,y] - return PointSubBasinRegion.from_xy(x=val[0], y=val[1]) - elif ( - isinstance(val[0], (float, int)) and len(val) == 4 - ): # [xmin, ymin, xmaxn, ymax] - return BoundingBoxSubBasinRegion.from_list(val) - elif isinstance(val[0], list): # [[x1,...,xn], [y1,..,yn]] - return MultiPointSubBasinRegion.from_xy_lists(xs=val[0], ys=val[1]) - else: - raise ValidationError(f"Unknown subbasin kind: {val}") - elif isinstance(val, str): - return GeometrySubBasinRegion.from_path(path=val) - else: - raise ValidationError(f"Unknown subbasin kind: {val}") - elif "interbasin" in input: - val = input["interbasin"] - if isinstance(val, list): - if ( - isinstance(val[0], (float, int)) and len(val) == 4 - ): # [xmin, ymin, xmaxn, ymax] - return BoundingBoxInterBasinRegion.from_list(val) - else: - raise ValidationError(f"Unknown subbasin kind: {val}") - elif isinstance(val, str): - return GeometryInterBasinRegion.from_path(path=val) - else: - raise ValidationError(f"Unknown subbasin kind: {val}") + return PathRegion.from_path(val) else: - key, val = next(iter(input.items())) - raise ValueError(f"Unknown region kind: {val}") - # model name + for region_type in ["grid", "mesh", "basin", "subbasin", "interbasin"]: + if region_type in input: + raise NotImplementedError( + f"region kind {region_type} is not supported in region validation yet, but is recognised by HydroMT." + ) + + raise NotImplementedError(f"Unknown region kind: {input}") diff --git a/tests/test_cli.py b/tests/test_cli.py index 56c9e13b8..a7fa6e5d7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,8 @@ """Tests for the cli submodule.""" +from os.path import abspath, dirname, join + import numpy as np import pytest from click.testing import CliRunner @@ -9,6 +11,8 @@ from hydromt.cli import api as hydromt_api from hydromt.cli.main import main as hydromt_cli +DATADIR = join(dirname(abspath(__file__)), "data") + def test_cli_verison(tmpdir): r = CliRunner().invoke(hydromt_cli, "--version") @@ -156,7 +160,7 @@ def test_export_cli_catalog(tmpdir): "-s", "hydro_lakes", "-d", - "tests/data/test_sources.yml", + join(DATADIR, "test_sources.yml"), ], catch_exceptions=False, ) @@ -181,7 +185,7 @@ def test_export_time_tuple(tmpdir): assert r.exit_code == 0, r.output -def test_export__multiple_sources(tmpdir): +def test_export_multiple_sources(tmpdir): r = CliRunner().invoke( hydromt_cli, [ @@ -217,6 +221,7 @@ def test_check_cli(): hydromt_cli, [ "check", + "-m", "grid_model", "-d", "tests/data/test_sources.yml", @@ -227,6 +232,90 @@ def test_check_cli(): assert r.exit_code == 0, r.output +def test_check_cli_unsupported_region(tmpdir): + with pytest.raises(Exception, match="is not supported in region validation yet"): + _ = CliRunner().invoke( + hydromt_cli, + [ + "check", + "-m", + "grid_model", + "-r", + "{'subbasin': [-7.24, 62.09], 'uparea': 50}", + "-i", + "tests/data/test_model_config.yml", + ], + catch_exceptions=False, + ) + + +def test_check_cli_known_region(tmpdir): + with pytest.raises(Exception, match="Unknown region kind"): + _ = CliRunner().invoke( + hydromt_cli, + [ + "check", + "-m", + "grid_model", + "-r", + "{'asdfasdfasdf': [-7.24, 62.09], 'uparea': 50}", + "-i", + "tests/data/test_model_config.yml", + ], + catch_exceptions=False, + ) + + +def test_check_cli_bbox_valid(tmpdir): + r = CliRunner().invoke( + hydromt_cli, + [ + "check", + "-m", + "grid_model", + "-r", + "{'bbox': [12.05,45.30,12.85,45.65]}", + "-i", + "tests/data/test_model_config.yml", + ], + ) + assert r.exit_code == 0, r.output + + +def test_check_cli_geom_valid(tmpdir): + r = CliRunner().invoke( + hydromt_cli, + [ + "check", + "-m", + "grid_model", + "-r", + "{'geom': 'tests/data/naturalearth_lowres.geojson'}", + "-i", + "tests/data/test_model_config.yml", + ], + catch_exceptions=False, + ) + assert r.exit_code == 0, r.output + + +def test_check_cli_geom_missing_file(tmpdir): + with pytest.raises(Exception, match="Path not found at asdf"): + _ = CliRunner().invoke( + hydromt_cli, + [ + "check", + "-m", + "grid_model", + "-r", + "{'geom': 'asdfasdf'}", + "-i", + "tests/data/test_model_config.yml", + ], + catch_exceptions=False, + ) + + def test_api_datasets(): # datasets assert "artifact_data" in hydromt_api.get_predifined_catalogs() diff --git a/tests/validators/test_data_catalog_validation.py b/tests/validators/test_data_catalog_validation.py index 8de246eb3..34a6e966d 100644 --- a/tests/validators/test_data_catalog_validation.py +++ b/tests/validators/test_data_catalog_validation.py @@ -7,6 +7,7 @@ from pydantic import ValidationError from pydantic_core import Url +from hydromt.data_catalog import _yml_from_uri_or_path from hydromt.validators.data_catalog import ( DataCatalogItem, DataCatalogMetaData, @@ -14,6 +15,13 @@ ) +def test_deltares_data_catalog(): + p = "data/catalogs/deltares_data.yml" + yml_dict = _yml_from_uri_or_path(p) + # whould raise error if something goes wrong + _ = DataCatalogValidator.from_dict(yml_dict) + + def test_geodataframe_entry_validation(): d = { "crs": 4326, @@ -75,6 +83,44 @@ def test_valid_catalog_with_alias(): _ = DataCatalogValidator.from_dict(d) +def test_valid_catalog_variants(): + d = { + "esa_worldcover": { + "crs": 4326, + "data_type": "RasterDataset", + "driver": "raster", + "filesystem": "local", + "kwargs": {"chunks": {"x": 36000, "y": 36000}}, + "meta": { + "category": "landuse", + "source_license": "CC BY 4.0", + "source_url": "https://doi.org/10.5281/zenodo.5571936", + }, + "variants": [ + { + "provider": "local", + "version": 2021, + "path": "landuse/esa_worldcover_2021/esa-worldcover.vrt", + }, + { + "provider": "local", + "version": 2020, + "path": "landuse/esa_worldcover/esa-worldcover.vrt", + }, + { + "provider": "aws", + "version": 2020, + "path": "s3://esa-worldcover/v100/2020/ESA_WorldCover_10m_2020_v100_Map_AWS.vrt", + "rename": {"ESA_WorldCover_10m_2020_v100_Map_AWS": "landuse"}, + "filesystem": "s3", + "storage_options": {"anon": True}, + }, + ], + } + } + _ = DataCatalogValidator.from_dict(d) + + def test_dangling_alias_catalog_entry(): d = { "chelsa": {"alias": "chelsa_v1.2"}, @@ -162,7 +208,29 @@ def test_dataset_entry_with_typo_validation(): } # 8 errors are: - # - missing crs, data_type, driver and path (4) + # - missing crs, data_type and driver (3) # - extra crs_num, datatype, diver, and filepath (5) - with pytest.raises(ValidationError, match="8 validation errors"): + with pytest.raises(ValidationError, match="7 validation errors"): + _ = DataCatalogItem.from_dict(d, name="chelsa_v1.2") + + +def test_data_type_typo(): + d = { + "crs": 4326, + "data_type": "RaserDataset", + "driver": "raster", + "path": ".", + } + with pytest.raises(ValidationError, match="1 validation error"): + _ = DataCatalogItem.from_dict(d, name="chelsa_v1.2") + + +def test_data_invalid_crs(): + d = { + "crs": 123456789, + "data_type": "RasterDataset", + "driver": "raster", + "path": ".", + } + with pytest.raises(ValidationError, match="1 validation error"): _ = DataCatalogItem.from_dict(d, name="chelsa_v1.2") diff --git a/tests/validators/test_model_config_validation.py b/tests/validators/test_model_config_validation.py index bdfa40586..bb8e9a838 100644 --- a/tests/validators/test_model_config_validation.py +++ b/tests/validators/test_model_config_validation.py @@ -4,7 +4,7 @@ import pytest from hydromt.models import GridModel, Model -from hydromt.validators.model_config import HydromtModelStep +from hydromt.validators.model_config import HydromtModelSetup, HydromtModelStep def test_base_model_build(): @@ -89,6 +89,25 @@ def test_setup_grid_from_geodataframe_validation(): HydromtModelStep.from_dict(d, model=model) +def test_setup_non_existing_grid_model_function(): + model = GridModel + d = { + "setup_config": { + "starttime": "2009-04-01T00:00:00", + "endtime": "2011-01-30T00:00:00", + "timestepsecs": 86400, + "input.path_forcing": "inmaps.nc", + }, + "setup_precip_forcing": {"precip_fn": "era5"}, + "setup_grid_from_rasterdataset": { + "raster_fn": "pet_nc", + "fill_method": "nearest", + }, + } + with pytest.raises(ValueError, match="Model does not have function"): + HydromtModelSetup.from_dict(d, model=model) + + def test_setup_grid_from_raster_reclass_validation(): model = GridModel d = { diff --git a/tests/validators/test_region_validator.py b/tests/validators/test_region_validator.py index 79a2a0225..76a5a4e7c 100644 --- a/tests/validators/test_region_validator.py +++ b/tests/validators/test_region_validator.py @@ -7,21 +7,8 @@ from pydantic_core import ValidationError from hydromt.validators.region import ( - BoundingBoxBasinRegion, - BoundingBoxInterBasinRegion, BoundingBoxRegion, - BoundingBoxSubBasinRegion, - GeometryBasinRegion, - GeometryInterBasinRegion, - GeometryRegion, - GeometrySubBasinRegion, - GridRegion, - MeshRegion, - MultiPointBasinRegion, - MultiPointSubBasinRegion, - PointBasinRegion, - PointSubBasinRegion, - WGS84Point, + PathRegion, validate_region, ) @@ -43,7 +30,7 @@ def test_invalid_bbox_point_validator(): def test_unknown_region_type_validator(): b = {"asdfasdf": [1.0, 1.0, -1.0, -1.0]} - with pytest.raises(ValueError, match="Unknown region kind"): + with pytest.raises(NotImplementedError, match="Unknown region kind"): _ = validate_region(b) @@ -51,7 +38,7 @@ def test_geom_validator(): b = {"geom": "tests/data/naturalearth_lowres.geojson"} region = validate_region(b) - assert region == GeometryRegion(path=Path("tests/data/naturalearth_lowres.geojson")) + assert region == PathRegion(path=Path("tests/data/naturalearth_lowres.geojson")) def test_geom_non_existant_path_validator(): @@ -59,111 +46,3 @@ def test_geom_non_existant_path_validator(): with pytest.raises(ValueError, match="1 validation error"): _ = validate_region(b) - - -def test_grid_validator(): - b = {"grid": "tests/data/naturalearth_lowres.geojson"} - - region = validate_region(b) - assert region == GridRegion(path=Path("tests/data/naturalearth_lowres.geojson")) - - -def test_mesh_validator(): - b = {"mesh": "tests/data/naturalearth_lowres.geojson"} - - region = validate_region(b) - assert region == MeshRegion(path=Path("tests/data/naturalearth_lowres.geojson")) - - -def test_point_sub_basin_validator(): - b = {"subbasin": [0, 0]} - - region = validate_region(b) - assert region == PointSubBasinRegion(points=[WGS84Point(x=0, y=0)]) - - -def test_multipoint_sub_basin_validator(): - b = {"subbasin": [[1, 2, 3, 4, 5], [1, 2, 3, 4, 5]]} - - region = validate_region(b) - assert region == MultiPointSubBasinRegion( - points=[ - WGS84Point(x=1, y=1), - WGS84Point(x=2, y=2), - WGS84Point(x=3, y=3), - WGS84Point(x=4, y=4), - WGS84Point(x=5, y=5), - ] - ) - - -def test_bounding_box_sub_basin_validator(): - b = {"subbasin": [-1.0, -1.0, 1.0, 1.0]} - - region = validate_region(b) - assert region == BoundingBoxSubBasinRegion(xmin=-1.0, ymin=-1.0, xmax=1.0, ymax=1.0) - - -def test_geometry_sub_basin_validator(): - b = {"subbasin": "tests/data/naturalearth_lowres.geojson"} - - region = validate_region(b) - assert region == GeometrySubBasinRegion( - path=Path("tests/data/naturalearth_lowres.geojson") - ) - - -def test_bounding_box_inter_basin_validator(): - b = {"interbasin": [-1.0, -1.0, 1.0, 1.0]} - - region = validate_region(b) - assert region == BoundingBoxInterBasinRegion( - xmin=-1.0, ymin=-1.0, xmax=1.0, ymax=1.0 - ) - - -def test_geometry_inter_basin_validator(): - b = {"interbasin": "tests/data/naturalearth_lowres.geojson"} - - region = validate_region(b) - assert region == GeometryInterBasinRegion( - path=Path("tests/data/naturalearth_lowres.geojson") - ) - - -def test_point_basin_validator(): - b = {"basin": [0, 0]} - - region = validate_region(b) - assert region == PointBasinRegion(points=[WGS84Point(x=0, y=0)]) - - -def test_multipoint_basin_validator(): - b = {"basin": [[1, 2, 3, 4, 5], [1, 2, 3, 4, 5]]} - - region = validate_region(b) - assert region == MultiPointBasinRegion( - points=[ - WGS84Point(x=1, y=1), - WGS84Point(x=2, y=2), - WGS84Point(x=3, y=3), - WGS84Point(x=4, y=4), - WGS84Point(x=5, y=5), - ] - ) - - -def test_bounding_box_basin_validator(): - b = {"basin": [-1.0, -1.0, 1.0, 1.0]} - - region = validate_region(b) - assert region == BoundingBoxBasinRegion(xmin=-1.0, ymin=-1.0, xmax=1.0, ymax=1.0) - - -def test_geometry_basin_validator(): - b = {"basin": "tests/data/naturalearth_lowres.geojson"} - - region = validate_region(b) - assert region == GeometryBasinRegion( - path=Path("tests/data/naturalearth_lowres.geojson") - )