(Fix) Burst Average code failing with ALL bad qc flag variable - fix a…

…odn/content#453
aodn · Aug 24, 2021 · 45521ae · 45521ae
1 parent d6fe7f1
commit 45521ae
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 3 deletions.
diff --git a/aodndata/moorings/burst_average.py b/aodndata/moorings/burst_average.py
@@ -58,19 +58,20 @@ def create_burst_average_var(netcdf_file_obj):
 
 def trim_timestamps_burst_vars(burst_vars):
     """
-    Trip timestamps at the start and end of a FV02 file when all FV02 variables
+    Trim timestamps at the start and end of a FV02 file when all FV02 variables
     have a NaN value.
     In details, for every burst var created, look for the first index of non NaN
     value. The lower value will be the one kept, and the new start index of each
     burst variable including the TIME.
     """
     min_index = None
-    for var in burst_vars.keys():
+    for var in list(burst_vars.keys()):
         var_mean_burst = burst_vars[var]['var_mean']  # first non TIME product
         if not np.isnan(var_mean_burst).all():
             min_index_var = next(x for x, y in enumerate(var_mean_burst) if not isnan(y))
         else:
-            min_index_var = 0
+            del burst_vars[var]  # if var only has nan, removing it from burst average file
+            continue
 
         if min_index is None:
             min_index = min_index_var

diff --git a/...418Z_NRSYON_FV01_NRSYON-1709-SRF-ECO-PARSB-1.1_END-20180509T233213Z_C-20190308T030455Z.nc b/...418Z_NRSYON_FV01_NRSYON-1709-SRF-ECO-PARSB-1.1_END-20180509T233213Z_C-20190308T030455Z.nc
diff --git a/test_aodndata/moorings/test_burst_average.py b/test_aodndata/moorings/test_burst_average.py
@@ -15,6 +15,12 @@
     'IMOS_ANMN-NRS_BCKOSTUZ_20081120T081531Z_NRSROT_FV01_NRSROT-0811-WQM-21_END-20081120T171535Z_C-20190418T000000Z.nc'
 )
 
+INPUT_DEPTH_ALL_BAD_FLAG_FILE = os.path.join(
+    TEST_ROOT,
+    'IMOS_ANMN-NRS_FZ_20170929T003418Z_NRSYON_FV01_NRSYON-1709-SRF-ECO-PARSB-1.1_END-20180509T233213Z_C-20190308T030455Z.nc'
+)
+
+
 STATS = dict(TEMP=ma.masked_values([-999, -999, -999, -999,  4.,  5., 11., 17.], -999),
              TEMP_num_obs=ma.array([0,  0,  0,  0, 61, 61, 11, 21]),
              TEMP_burst_max=ma.masked_values([-999, -999, -999, -999,  4.,  5., 16., 27.], -999),
@@ -32,6 +38,14 @@ def test_burst_average(self):
         for var, values in STATS.items():
             self.assertTrue(np.isclose(values, ds.variables[var][:N_COMP]).all())
 
+    def test_burst_average_bad_flag(self):
+        """
+        testing burst code with input NetCDF containing all bad qc flags for the Depth variable
+        """
+        averaged_file = create_burst_average_netcdf(INPUT_DEPTH_ALL_BAD_FLAG_FILE, self.temp_dir)
+        ds = Dataset(averaged_file)
+        self.assertTrue('DEPTH' not in ds.variables)  # depth variable removed from dataset since only containing NaNs
+
 
 if __name__ == '__main__':
     unittest.main()