Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

(Fix) Burst Average code failing with ALL bad qc flag variable - fix https://github.com/aodn/content/issues/453 #279

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

(Fix) Burst Average code failing with ALL bad qc flag variable - fix https://github.com/aodn/content/issues/453 #279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions aodndata/moorings/burst_average.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,19 +58,20 @@ def create_burst_average_var(netcdf_file_obj):

def trim_timestamps_burst_vars(burst_vars):
"""
Trip timestamps at the start and end of a FV02 file when all FV02 variables
Trim timestamps at the start and end of a FV02 file when all FV02 variables
have a NaN value.
In details, for every burst var created, look for the first index of non NaN
value. The lower value will be the one kept, and the new start index of each
burst variable including the TIME.
"""
min_index = None
for var in burst_vars.keys():
for var in list(burst_vars.keys()):
var_mean_burst = burst_vars[var]['var_mean'] # first non TIME product
if not np.isnan(var_mean_burst).all():
min_index_var = next(x for x, y in enumerate(var_mean_burst) if not isnan(y))
else:
min_index_var = 0
del burst_vars[var] # if var only has nan, removing it from burst average file
continue

if min_index is None:
min_index = min_index_var
Expand Down
Binary file added ...418Z_NRSYON_FV01_NRSYON-1709-SRF-ECO-PARSB-1.1_END-20180509T233213Z_C-20190308T030455Z.nc
View file
Open in desktop
Binary file not shown.
14 changes: 14 additions & 0 deletions test_aodndata/moorings/test_burst_average.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,12 @@
'IMOS_ANMN-NRS_BCKOSTUZ_20081120T081531Z_NRSROT_FV01_NRSROT-0811-WQM-21_END-20081120T171535Z_C-20190418T000000Z.nc'
)

INPUT_DEPTH_ALL_BAD_FLAG_FILE = os.path.join(
TEST_ROOT,
'IMOS_ANMN-NRS_FZ_20170929T003418Z_NRSYON_FV01_NRSYON-1709-SRF-ECO-PARSB-1.1_END-20180509T233213Z_C-20190308T030455Z.nc'
)


STATS = dict(TEMP=ma.masked_values([-999, -999, -999, -999, 4., 5., 11., 17.], -999),
TEMP_num_obs=ma.array([0, 0, 0, 0, 61, 61, 11, 21]),
TEMP_burst_max=ma.masked_values([-999, -999, -999, -999, 4., 5., 16., 27.], -999),
Expand All @@ -32,6 +38,14 @@ def test_burst_average(self):
for var, values in STATS.items():
self.assertTrue(np.isclose(values, ds.variables[var][:N_COMP]).all())

def test_burst_average_bad_flag(self):
"""
testing burst code with input NetCDF containing all bad qc flags for the Depth variable
"""
averaged_file = create_burst_average_netcdf(INPUT_DEPTH_ALL_BAD_FLAG_FILE, self.temp_dir)
ds = Dataset(averaged_file)
self.assertTrue('DEPTH' not in ds.variables) # depth variable removed from dataset since only containing NaNs


if __name__ == '__main__':
unittest.main()