Cleanup

pyproject.toml

@@ -34,7 +34,7 @@ python = ">=3.10, <3.13"
 click = "*"
 numpy = "^2.1.0"
 pandas = "^2.2.2"
-rasterra = "^0.6.0"
+rasterra = "^0.6.1"
 shapely = "^2.0.4"
 geopandas = "^1.0.1"
 xarray = "^2024.11.0"
@@ -48,28 +48,28 @@ gcsfs = "^2024.6.0"
 zarr = "^2.18.2"
 dask = "^2024.5.2"
 lxml = "^5.3.0"
+pydantic = "^2.10.4"
 
 
 [tool.poetry.group.dev.dependencies]
-mkdocstrings = {version = ">=0.23", extras = ["python"]}
+mkdocstrings = {version = "*", extras = ["python"]}
 mkdocs-material = "*"
 mkdocs-table-reader-plugin = "*"
-mkdocs-gen-files = "^0.5.0"
+mkdocs-gen-files = "*"
+mkdocs-literate-nav = "*"
+mkdocs-section-index = "*"
 mypy = "*"
 pre-commit = "*"
 pymdown-extensions = "*"
 pytest = "*"
 pytest-github-actions-annotate-failures = "*"
 pytest-cov = "*"
 python-kacl = "*"
-ruff = ">=0.2.0"
-pandas-stubs = "^2.2.3.241009"
-types-pyyaml = "^6.0.12.20240311"
-types-requests = "^2.31.0.20240406"
-types-tqdm = "^4.66.0.20240417"
-mkdocs-literate-nav = "^0.6.1"
-mkdocs-section-index = "^0.3.9"
-
+ruff = "*"
+pandas-stubs = "*"
+types-pyyaml = "*"
+types-requests = "*"
+types-tqdm = "*"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

scripts/gen_data_pages.py

@@ -22,7 +22,7 @@
 
 ## Data Organization
 
-The root of the climate data is located at `{cdata.root}`. There are {len([p for p in cdata.root.glob('*') if p.is_dir()])}
+The root of the climate data is located at `{cdata.root}`. There are several
 subdirectories in the root directory, but only the extracted data directory `{cdata.extracted_data.stem}` and the results
 directory `{cdata.results.stem}` are relevant to users of the database.
 
@@ -40,7 +40,6 @@
     ├── {cdata.annual_results.stem}/
     │   └── {{SCENARIO}}/
     │       └── {{ANNUAL_VARIABLE}}/
-    │           ├── {{YEAR}}.nc
     │           └── {{YEAR}}_{{DRAW}}.nc
     ├── {cdata.daily_results.stem}/
     │   └── {{SCENARIO}}/
@@ -50,7 +49,7 @@
 
 ```
 
-Specifics of the values the markers above take on can be found in the following sections.
+The file patterns will be explained in more detail in the following sections.
 """
 
 extraction_header_content = """## Extracted Data
@@ -63,7 +62,7 @@
 era5_page_content = f"""### ERA5 Data
 
 The [ECMWF Reanalysis v5 (ERA5)](https://www.ecmwf.int/en/forecasts/dataset/ecmwf-reanalysis-v5) is the
-fifth generation ECMWF atmospheric reanalysis of the global climate covering the period from January 1940 to present. ERA5 is produced
+fifth generation ECMWF atmospheric reanalysis of the global climate covering the period from January 1950 to present. ERA5 is produced
 by the Copernicus Climate Change Service (C3S) at ECMWF. There are three datasets of note:
 
   - [The Complete ERA5 global atmospheric reanalysis](https://cds.climate.copernicus.eu/datasets/reanalysis-era5-complete?tab=overview):
@@ -183,6 +182,16 @@
 We generally only generate annual results, as storing daily results for all models and all variables would be
 prohibitively expensive.
 
+### Storage and Naming Conventions
+
+    - Daily Storage Root: `{cdata.daily_results}`
+    - Naming Convention: `{{SCENARIO}}/{{DAILY_VARIABLE}}/{{YEAR}}.nc` (historical data only)
+        - `{{SCENARIO}}`: Generally, only historical data is available at the daily level, so this will be `historical`. 
+        - `{{DAILY_VARIABLE}}`: The name of the variable being stored.
+        - `{{YEAR}}`: The year of the data being stored.
+    - Annual Storage Root: `{cdata.results}`
+    - Naming Convention: `{{SCENARIO}}/{{ANNUAL_VARIABLE}}/{{YEAR}}.nc` or `{{SCENARIO}}/{{ANNUAL_VARIABLE}}/{{YEAR}}_{{DRAW}}.nc`
+
 ### Pipeline Stages
 
 The processing pipelines turn the extracted [ERA5](#era5-data) and [CMIP6](#cmip6-data) data into a coherent set of