Merge pull request #4 from pySTEPS/fix/segmentation-fault-mfdataset

pySTEPS · Jun 24, 2024 · 73b3573 · 73b3573
2 parents b8e706d + 3b3c34f
commit 73b3573
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 53 deletions.
diff --git a/.github/workflows/test_importers.yml b/.github/workflows/test_importers.yml
@@ -15,16 +15,16 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "ubuntu-latest", "macos-latest", "windows-latest" ]
-        python-version: [ "3.7", "3.9" ]
+        python-version: ["3.9", "3.11"]
       max-parallel: 6
 
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -39,14 +39,14 @@ jobs:
         if: matrix.os == 'macos-latest'
         working-directory: ${{github.workspace}}
         env:
-          CC: gcc-9
-          CXX: g++-9
-          CXX1X: g++-9
+          CC: gcc-13
+          CXX: g++-13
+          CXX1X: g++-13
           HOMEBREW_NO_INSTALL_CLEANUP: 1
         run: |
           brew update-reset
           brew update
-          gcc-9 --version || brew install gcc@9
+          gcc-13 --version || brew install gcc@13
           pip install "pysteps>=1.4,<2.0"
 
       - name: Install pysteps-nwp-importers

diff --git a/pysteps_nwp_importers/importer_bom_nwp.py b/pysteps_nwp_importers/importer_bom_nwp.py
@@ -107,7 +107,7 @@ def import_bom_nwp(filename, **kwargs):
             "products but it is not installed"
         )
 
-    ds = _import_bom_nwp_data_xr(filename, **kwargs)
+    ds = _import_bom_nwp_data_xr(filename)
     metadata = _import_bom_nwp_geodata_xr(ds, **kwargs)
 
     # rename varname_time (def: time) to t
@@ -125,7 +125,7 @@ def import_bom_nwp(filename, **kwargs):
         print("Rainfall values are accumulated. Disaggregating by time step")
         accum_prcp = ds[varname]
         precipitation = accum_prcp - accum_prcp.shift({varname_time: 1})
-        precipitation = precipitation.dropna(varname_time, "all")
+        precipitation = precipitation.dropna(varname_time, how="all")
         # update/copy attributes
         precipitation.name = "precipitation"
         # copy attributes
@@ -141,19 +141,7 @@ def import_bom_nwp(filename, **kwargs):
 
 
 def _import_bom_nwp_data_xr(filename, **kwargs):
-    varname_time = kwargs.get("varname_time", "time")
-    chunks = kwargs.get("chunks", {varname_time: 1})
-
-    ds_rainfall = xr.open_mfdataset(
-        filename,
-        combine="nested",
-        concat_dim=varname_time,
-        chunks=chunks,
-        lock=False,
-        parallel=True,
-    )
-
-    return ds_rainfall
+    return xr.open_dataset(filename)
 
 
 def _import_bom_nwp_geodata_xr(

diff --git a/pysteps_nwp_importers/importer_knmi_nwp.py b/pysteps_nwp_importers/importer_knmi_nwp.py
@@ -107,7 +107,7 @@ def import_knmi_nwp(filename, **kwargs):
             "products but it is not installed"
         )
 
-    ds = _import_knmi_nwp_data_xr(filename, **kwargs)
+    ds = _import_knmi_nwp_data_xr(filename)
     metadata = _import_knmi_nwp_geodata_xr(ds, **kwargs)
 
     # rename varname_time (def: time) to t
@@ -136,26 +136,13 @@ def import_knmi_nwp(filename, **kwargs):
 
 
 def _import_knmi_nwp_data_xr(filename, **kwargs):
-
-    varname_time = kwargs.get("varname_time", "time")
-    chunks = kwargs.get("chunks", {varname_time: 1})
-
-    ds_rainfall = xr.open_mfdataset(
-        filename,
-        combine="nested",
-        concat_dim=varname_time,
-        chunks=chunks,
-        parallel=True,
-    )
-
-    return ds_rainfall
+    return xr.open_dataset(filename)
 
 
 def _import_knmi_nwp_geodata_xr(
     ds_in,
     **kwargs,
 ):
-
     varname = kwargs.get("varname", "P_fc")
     varname_time = kwargs.get("varname_time", "time")
     # Get the projection string

diff --git a/pysteps_nwp_importers/importer_rmi_nwp.py b/pysteps_nwp_importers/importer_rmi_nwp.py
@@ -124,7 +124,7 @@ def import_rmi_nwp(filename, **kwargs):
         print("Rainfall values are accumulated. Disaggregating by time step")
         accum_prcp = ds[varname]
         precipitation = accum_prcp - accum_prcp.shift({varname_time: 1})
-        precipitation = precipitation.dropna(varname_time, "all")
+        precipitation = precipitation.dropna(varname_time, how="all")
         # update/copy attributes
         precipitation.name = "precipitation"
         # copy attributes
@@ -140,27 +140,13 @@ def import_rmi_nwp(filename, **kwargs):
 
 
 def _import_rmi_nwp_data_xr(filename, **kwargs):
-
-    varname_time = kwargs.get("varname_time", "time")
-    chunks = kwargs.get("chunks", {varname_time: 1})
-
-    ds_rainfall = xr.open_mfdataset(
-        filename,
-        combine="nested",
-        concat_dim=varname_time,
-        chunks=chunks,
-        lock=False,
-        parallel=True,
-    )
-
-    return ds_rainfall
+    return xr.open_dataset(filename)
 
 
 def _import_rmi_nwp_geodata_xr(
     ds_in,
     **kwargs,
 ):
-
     varname = kwargs.get("varname", "precipitation")
     varname_time = kwargs.get("varname_time", "time")
     projdef = None