Next dev merge/shoal xarray (#129)

* Rewrote shoal mask to use dask

* Dev

* Dev

* Fixed denoising issue

* Test dataset fix

* Documentation fixes

* Documentation fixes

echopype/mask/api.py

@@ -579,11 +579,6 @@ def get_shoal_mask(
     mask: xr.DataArray
         A DataArray containing the mask for the Sv data. Regions satisfying the thresholding
         criteria are filled with ``True``, else the regions are filled with ``False``.
-    mask_: xr.DataArray
-        A DataArray containing the mask for areas in which shoals were searched.
-        Edge regions are filled with 'False', whereas the portion
-        in which shoals could be detected is 'True'
-
 
     Raises
     ------
@@ -630,10 +625,6 @@ def get_shoal_mask_multichannel(
             A DataArray containing the multichannel mask for the Sv data.
             Regions satisfying the thresholding criteria are filled with ``True``,
             else the regions are filled with ``False``.
-        mask_: xr.DataArray
-            A DataArray containing the multichannel mask for areas in which shoals were searched.
-            Edge regions are filled with 'False', whereas the portion
-            in which shoals could be detected is 'True'
 
 
     Raises

echopype/mask/shoal.py

@@ -31,17 +31,18 @@
 import pathlib
 from typing import Union
 
+import dask.array as da
 import numpy as np
 import xarray as xr
 from dask_image.ndmorph import binary_closing, binary_opening
 
 WEILL_DEFAULT_PARAMETERS = {
-    "thr": -70,
+    "thr": -55,
     "maxvgap": 5,
     "maxhgap": 5,
-    "minvlen": 0,
-    "minhlen": 0,
-    "dask_chunking": {"ping_time": 1000, "range_sample": 1000},
+    "minvlen": 5,
+    "minhlen": 5,
+    "dask_chunking": {"ping_time": 100, "range_sample": 100},
 }
 
 
@@ -97,10 +98,6 @@ def _weill(
     mask: xr.DataArray
         A DataArray containing the mask for the Sv data. Regions satisfying the thresholding
         criteria are filled with ``True``, else the regions are filled with ``False``.
-    mask_: xr.DataArray
-        A DataArray containing the mask for areas in which shoals were searched.
-        Edge regions are filled with 'False', whereas the portion in which shoals
-        could be detected is 'True'
     """
     parameter_names = ["thr", "maxvgap", "maxhgap", "minvlen", "minhlen", "dask_chunking"]
     if not all(name in parameters.keys() for name in parameter_names):
@@ -115,30 +112,45 @@ def _weill(
     maxhgap = parameters["maxhgap"]
     minvlen = parameters["minvlen"]
     minhlen = parameters["minhlen"]
+
     dask_chunking = parameters["dask_chunking"]
 
+    # Convert values to integers, handling possible NaN
+    dask_chunking = tuple(map(lambda x: int(x) if not np.isnan(x) else x, dask_chunking.values()))
+
     channel_Sv = source_Sv.sel(channel=desired_channel)
     Sv = channel_Sv["Sv"].chunk(dask_chunking)
 
-    mask = xr.where(Sv > thr, True, False).chunk(dask_chunking)
+    remove_nan = xr.where(Sv, Sv, thr - 1)  # so we have no nan values
+    mask = xr.where(remove_nan > thr, 1, 0).drop("channel").chunk(dask_chunking)
+
+    dask_mask = da.asarray(mask, allow_unknown_chunksizes=False)
+    dask_mask.compute_chunk_sizes()
 
     # close shoal gaps smaller than the specified box
-    if maxvgap > 0 & maxhgap > 0:
-        closing_array = np.ones(maxhgap, maxvgap)
-        mask = binary_closing(
-            mask,
-            structure=closing_array,
-            iterations=1,
+    if maxvgap > 0 and maxhgap > 0:
+        closing_array = da.ones(shape=(maxhgap, maxvgap), dtype=bool)
+        dask_mask = (
+            binary_closing(
+                dask_mask,
+                structure=closing_array,
+                iterations=1,
+            ).compute()
+            # .chunk(dask_chunking)
         )
+        dask_mask = da.asarray(dask_mask, allow_unknown_chunksizes=False)
 
     # drop shoals smaller than the specified box
-    if minvlen > 0 & minhlen > 0:
-        opening_array = np.ones(minhlen, minhlen)
-        mask = binary_opening(
-            mask,
+
+    if minvlen > 0 and minhlen > 0:
+        opening_array = da.ones(shape=(minhlen, minvlen), dtype=bool)
+        dask_mask = binary_opening(
+            dask_mask,
             structure=opening_array,
             iterations=1,
-        )
+        ).compute()
+        dask_mask = da.asarray(dask_mask, allow_unknown_chunksizes=False)
+
+    mask.values = dask_mask
 
-    mask = mask.drop("channel")
     return mask

echopype/tests/conftest.py

@@ -20,8 +20,8 @@ def _setup_file(file_name):
         download_ftp_file(ftp, FTP_PARTIAL_PATH, file_name, TEST_DATA_FOLDER)
     return os.path.join(TEST_DATA_FOLDER, file_name)
 
-def download_ftp_file(ftp, remote_path, file_name, local_path):
 
+def download_ftp_file(ftp, remote_path, file_name, local_path):
     # Construct the full paths
     remote_file_path = os.path.join(remote_path, file_name)
     local_file_path = os.path.join(local_path, file_name)
@@ -103,6 +103,7 @@ def setup_test_data_jr179():
     file_name = "JR179-D20080410-T150637.raw"
     return _setup_file(file_name)
 
+
 def _setup_file(file_name):
     test_data_path = os.path.join(TEST_DATA_FOLDER, file_name)
     FTP_MAIN = "ftp://ftp.bas.ac.uk"
@@ -143,8 +144,21 @@ def raw_dataset_jr179(setup_test_data_jr179):
     ed = _get_raw_dataset(setup_test_data_jr179)
     return ed
 
-def _get_sv_dataset(file_path):
-    ed = ep.open_raw(file_path, sonar_model="ek60")
-    Sv = ep.calibrate.compute_Sv(ed).compute()
-    return Sv
 
+@pytest.fixture(scope="session")
+def ed_ek_60_for_Sv():
+    bucket = "ncei-wcsd-archive"
+    base_path = "data/raw/Bell_M._Shimada/SH1707/EK60/"
+    filename = "Summer2017-D20170620-T011027.raw"
+    rawdirpath = base_path + filename
+
+    s3raw_fpath = f"s3://{bucket}/{rawdirpath}"
+    storage_opts = {"anon": True}
+    ed = ep.open_raw(s3raw_fpath, sonar_model="EK60", storage_options=storage_opts)  # type: ignore
+    return ed
+
+
+@pytest.fixture(scope="session")
+def ek60_Sv(ed_ek_60_for_Sv):
+    sv_echopype_EK60 = ep.calibrate.compute_Sv(ed_ek_60_for_Sv).compute()
+    return sv_echopype_EK60

echopype/tests/mask/test_mask_shoal.py

@@ -8,8 +8,8 @@
 
 
 @pytest.mark.parametrize(
-    "method, desired_channel,parameters,expected_tf_counts,expected_tf_counts_",
-    [("will", DESIRED_CHANNEL, WEILL_DEFAULT_PARAMETERS, (101550, 2065381))],
+    "method, desired_channel,parameters,expected_tf_counts",
+    [("will", DESIRED_CHANNEL, WEILL_DEFAULT_PARAMETERS, (4467, 2162464))],
 )
 def test_get_shoal_mask_weill(
     sv_dataset_jr161, method, desired_channel, parameters, expected_tf_counts