From 5b7358c0a827b84f6cefb12bf557c61793a3c61a Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Thu, 20 Jun 2024 07:22:34 +0200 Subject: [PATCH 1/3] Switch to using xr.open_dataset to read nwp data --- pysteps_nwp_importers/importer_bom_nwp.py | 20 ++------------------ pysteps_nwp_importers/importer_knmi_nwp.py | 18 +----------------- pysteps_nwp_importers/importer_rmi_nwp.py | 21 ++------------------- 3 files changed, 5 insertions(+), 54 deletions(-) diff --git a/pysteps_nwp_importers/importer_bom_nwp.py b/pysteps_nwp_importers/importer_bom_nwp.py index ff2825d..c6dd55a 100644 --- a/pysteps_nwp_importers/importer_bom_nwp.py +++ b/pysteps_nwp_importers/importer_bom_nwp.py @@ -107,7 +107,7 @@ def import_bom_nwp(filename, **kwargs): "products but it is not installed" ) - ds = _import_bom_nwp_data_xr(filename, **kwargs) + ds = xr.open_dataset(filename) metadata = _import_bom_nwp_geodata_xr(ds, **kwargs) # rename varname_time (def: time) to t @@ -125,7 +125,7 @@ def import_bom_nwp(filename, **kwargs): print("Rainfall values are accumulated. Disaggregating by time step") accum_prcp = ds[varname] precipitation = accum_prcp - accum_prcp.shift({varname_time: 1}) - precipitation = precipitation.dropna(varname_time, "all") + precipitation = precipitation.dropna(varname_time, how="all") # update/copy attributes precipitation.name = "precipitation" # copy attributes @@ -140,22 +140,6 @@ def import_bom_nwp(filename, **kwargs): return precipitation.values, quality, metadata -def _import_bom_nwp_data_xr(filename, **kwargs): - varname_time = kwargs.get("varname_time", "time") - chunks = kwargs.get("chunks", {varname_time: 1}) - - ds_rainfall = xr.open_mfdataset( - filename, - combine="nested", - concat_dim=varname_time, - chunks=chunks, - lock=False, - parallel=True, - ) - - return ds_rainfall - - def _import_bom_nwp_geodata_xr( ds_in, **kwargs, diff --git a/pysteps_nwp_importers/importer_knmi_nwp.py b/pysteps_nwp_importers/importer_knmi_nwp.py index f41e127..fca2d21 100644 --- a/pysteps_nwp_importers/importer_knmi_nwp.py +++ b/pysteps_nwp_importers/importer_knmi_nwp.py @@ -107,7 +107,7 @@ def import_knmi_nwp(filename, **kwargs): "products but it is not installed" ) - ds = _import_knmi_nwp_data_xr(filename, **kwargs) + ds = xr.open_dataset(filename) metadata = _import_knmi_nwp_geodata_xr(ds, **kwargs) # rename varname_time (def: time) to t @@ -135,22 +135,6 @@ def import_knmi_nwp(filename, **kwargs): return precipitation.values, quality, metadata -def _import_knmi_nwp_data_xr(filename, **kwargs): - - varname_time = kwargs.get("varname_time", "time") - chunks = kwargs.get("chunks", {varname_time: 1}) - - ds_rainfall = xr.open_mfdataset( - filename, - combine="nested", - concat_dim=varname_time, - chunks=chunks, - parallel=True, - ) - - return ds_rainfall - - def _import_knmi_nwp_geodata_xr( ds_in, **kwargs, diff --git a/pysteps_nwp_importers/importer_rmi_nwp.py b/pysteps_nwp_importers/importer_rmi_nwp.py index 1fc323f..d6030ee 100644 --- a/pysteps_nwp_importers/importer_rmi_nwp.py +++ b/pysteps_nwp_importers/importer_rmi_nwp.py @@ -106,7 +106,7 @@ def import_rmi_nwp(filename, **kwargs): "products but it is not installed" ) - ds = _import_rmi_nwp_data_xr(filename, **kwargs) + ds = xr.open_dataset(filename, **kwargs) metadata = _import_rmi_nwp_geodata_xr(ds, **kwargs) # rename varname_time (def: time) to t @@ -124,7 +124,7 @@ def import_rmi_nwp(filename, **kwargs): print("Rainfall values are accumulated. Disaggregating by time step") accum_prcp = ds[varname] precipitation = accum_prcp - accum_prcp.shift({varname_time: 1}) - precipitation = precipitation.dropna(varname_time, "all") + precipitation = precipitation.dropna(varname_time, how="all") # update/copy attributes precipitation.name = "precipitation" # copy attributes @@ -139,23 +139,6 @@ def import_rmi_nwp(filename, **kwargs): return precipitation.values, quality, metadata -def _import_rmi_nwp_data_xr(filename, **kwargs): - - varname_time = kwargs.get("varname_time", "time") - chunks = kwargs.get("chunks", {varname_time: 1}) - - ds_rainfall = xr.open_mfdataset( - filename, - combine="nested", - concat_dim=varname_time, - chunks=chunks, - lock=False, - parallel=True, - ) - - return ds_rainfall - - def _import_rmi_nwp_geodata_xr( ds_in, **kwargs, From c6984be8dfe90d33b7fc7490e8f099002a98f922 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Thu, 20 Jun 2024 07:28:01 +0200 Subject: [PATCH 2/3] Update CI environ --- .github/workflows/test_importers.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_importers.yml b/.github/workflows/test_importers.yml index 26c9952..43329d1 100644 --- a/.github/workflows/test_importers.yml +++ b/.github/workflows/test_importers.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: os: [ "ubuntu-latest", "macos-latest", "windows-latest" ] - python-version: [ "3.7", "3.9" ] + python-version: ["3.9", "3.11"] max-parallel: 6 defaults: @@ -23,8 +23,8 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -39,14 +39,14 @@ jobs: if: matrix.os == 'macos-latest' working-directory: ${{github.workspace}} env: - CC: gcc-9 - CXX: g++-9 - CXX1X: g++-9 + CC: gcc-13 + CXX: g++-13 + CXX1X: g++-13 HOMEBREW_NO_INSTALL_CLEANUP: 1 run: | brew update-reset brew update - gcc-9 --version || brew install gcc@9 + gcc-13 --version || brew install gcc@13 pip install "pysteps>=1.4,<2.0" - name: Install pysteps-nwp-importers From 3b3c34f77959816ab146e394d028e2240f568140 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Sun, 23 Jun 2024 19:46:25 +0200 Subject: [PATCH 3/3] Keep original structure with private subfunctions --- pysteps_nwp_importers/importer_bom_nwp.py | 6 +++++- pysteps_nwp_importers/importer_knmi_nwp.py | 7 +++++-- pysteps_nwp_importers/importer_rmi_nwp.py | 7 +++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pysteps_nwp_importers/importer_bom_nwp.py b/pysteps_nwp_importers/importer_bom_nwp.py index c6dd55a..927db35 100644 --- a/pysteps_nwp_importers/importer_bom_nwp.py +++ b/pysteps_nwp_importers/importer_bom_nwp.py @@ -107,7 +107,7 @@ def import_bom_nwp(filename, **kwargs): "products but it is not installed" ) - ds = xr.open_dataset(filename) + ds = _import_bom_nwp_data_xr(filename) metadata = _import_bom_nwp_geodata_xr(ds, **kwargs) # rename varname_time (def: time) to t @@ -140,6 +140,10 @@ def import_bom_nwp(filename, **kwargs): return precipitation.values, quality, metadata +def _import_bom_nwp_data_xr(filename, **kwargs): + return xr.open_dataset(filename) + + def _import_bom_nwp_geodata_xr( ds_in, **kwargs, diff --git a/pysteps_nwp_importers/importer_knmi_nwp.py b/pysteps_nwp_importers/importer_knmi_nwp.py index fca2d21..401133e 100644 --- a/pysteps_nwp_importers/importer_knmi_nwp.py +++ b/pysteps_nwp_importers/importer_knmi_nwp.py @@ -107,7 +107,7 @@ def import_knmi_nwp(filename, **kwargs): "products but it is not installed" ) - ds = xr.open_dataset(filename) + ds = _import_knmi_nwp_data_xr(filename) metadata = _import_knmi_nwp_geodata_xr(ds, **kwargs) # rename varname_time (def: time) to t @@ -135,11 +135,14 @@ def import_knmi_nwp(filename, **kwargs): return precipitation.values, quality, metadata +def _import_knmi_nwp_data_xr(filename, **kwargs): + return xr.open_dataset(filename) + + def _import_knmi_nwp_geodata_xr( ds_in, **kwargs, ): - varname = kwargs.get("varname", "P_fc") varname_time = kwargs.get("varname_time", "time") # Get the projection string diff --git a/pysteps_nwp_importers/importer_rmi_nwp.py b/pysteps_nwp_importers/importer_rmi_nwp.py index d6030ee..a612c89 100644 --- a/pysteps_nwp_importers/importer_rmi_nwp.py +++ b/pysteps_nwp_importers/importer_rmi_nwp.py @@ -106,7 +106,7 @@ def import_rmi_nwp(filename, **kwargs): "products but it is not installed" ) - ds = xr.open_dataset(filename, **kwargs) + ds = _import_rmi_nwp_data_xr(filename, **kwargs) metadata = _import_rmi_nwp_geodata_xr(ds, **kwargs) # rename varname_time (def: time) to t @@ -139,11 +139,14 @@ def import_rmi_nwp(filename, **kwargs): return precipitation.values, quality, metadata +def _import_rmi_nwp_data_xr(filename, **kwargs): + return xr.open_dataset(filename) + + def _import_rmi_nwp_geodata_xr( ds_in, **kwargs, ): - varname = kwargs.get("varname", "precipitation") varname_time = kwargs.get("varname_time", "time") projdef = None