diff --git a/AutomaticQC/imosTimeSeriesSpikeQC.m b/AutomaticQC/imosTimeSeriesSpikeQC.m index 92976bdc8..58c1113c7 100644 --- a/AutomaticQC/imosTimeSeriesSpikeQC.m +++ b/AutomaticQC/imosTimeSeriesSpikeQC.m @@ -48,43 +48,29 @@ if (is_profile || time_missing), return, end -if isfield(sample_data.meta,'instrument_burst_duration') - burst_duration = sample_data.meta.instrument_burst_duration; - valid_burst_duration = ~isempty(burst_duration) && ~isnan(burst_duration) && burst_duration>0; -elseif isfield(sample_data,'instrument_burst_duration') - burst_duration = sample_data.meta.instrument_burst_duration; - valid_burst_duration = ~isempty(burst_duration) && ~isnan(burst_duration) && burst_duration>0; -end -if isfield(sample_data.meta,'instrument_burst_interval') - burst_interval = sample_data.meta.instrument_burst_interval; - valid_burst_interval = ~isempty(burst_interval) && ~isnan(burst_interval) && burst_interval>0; -elseif isfield(sample_data,'instrument_burst_interval') - burst_interval = sample_data.meta.instrument_burst_interval; - valid_burst_interval = ~isempty(burst_interval) && ~isnan(burst_interval) && burst_interval>0; -end - -has_burst = (valid_burst_duration) || (valid_burst_interval); - -if has_burst +[is_burst_sampling,is_burst_metadata_valid,burst_interval] = get_burst_metadata(sample_data); +if is_burst_metadata_valid opts_file = ['AutomaticQC' filesep 'imosTimeSeriesSpikeQCBurst.txt']; -else +elseif ~is_burst_sampling opts_file = ['AutomaticQC' filesep 'imosTimeSeriesSpikeQC.txt']; +else + warning('Invalid burst metadata...skipping'); + return end ts_variables = load_timeseries_variables(sample_data); if isempty(ts_variables), return, end +postqc_data = gen_postqc(sample_data, ts_variables, burst_interval); -postqc_data = gen_postqc(sample_data, ts_variables, has_burst); - -if has_burst && numel(postqc_data.burst_index_range) == 1 +if is_burst_sampling && numel(postqc_data.burst_index_range) == 1 warning('Burst metadata not consistent with TIME variable...skipping') return end if auto user_interaction = struct(); - spike_methods = loadSpikeClassifiers(opts_file, has_burst); + spike_methods = loadSpikeClassifiers(opts_file, is_burst_sampling); method = readProperty('auto_function',opts_file); for k = 1:length(ts_variables) @@ -93,7 +79,7 @@ end else - wobj = spikeWindow(postqc_data, ts_variables, opts_file, has_burst); + wobj = spikeWindow(postqc_data, ts_variables, opts_file, is_burst_sampling); uiwait(wobj.ui_window); user_interaction = wobj.UserData; end @@ -112,7 +98,7 @@ if isempty(user_input.spikes)%run now if preview was not triggered - if has_burst + if is_burst_sampling user_input.spikes = user_input.fun(postqc_var.valid_burst_range, postqc_var.data, user_input.args{:}); else user_input.spikes = user_input.fun(postqc_var.data, user_input.args{:}); @@ -185,7 +171,7 @@ end -function postqc = gen_postqc(sample_data, ts_variables, has_burst) +function postqc = gen_postqc(sample_data, ts_variables, burst_interval) %function postqc = gen_postqc(sample_data, ts_variables) % % Construct a procqc data structure, the input for the SpikeQC window class. @@ -195,7 +181,7 @@ % % sample_data - the toolbox data struct. % ts_variables - the time-series variables to process. -% has_burst - if the data is in burst-mode. +% burst_interval - the burst interval in seconds. Empty for non-burst data % % Outputs: % @@ -206,7 +192,7 @@ % postqc.variables{k}.time = The time values. % postqc.variables{k}.l - the first index of valid data (from continous invalid ones) % postqc.variables{k}.r - the last index of valid data (ditto) -% postqc.variables{k}.valid_burst_range = the burst index ranges [only if has_burst is true] +% postqc.variables{k}.valid_burst_range = the burst index ranges [only if burst_interval is not-empty] % % % author: hugo.oliveira@utas.edu.au @@ -221,13 +207,7 @@ postqc.variables = cell(1, n_ts_vars); postqc.dimensions = sample_data.dimensions; -if has_burst - try - burst_interval = sample_data.meta.instrument_burst_interval; - catch - burst_interval = sample_data.instrument_burst_interval; - end - +if ~isempty(burst_interval) postqc.burst_precision = precisionBounds(burst_interval / 86400); postqc.burst_index_range = findBursts(time, postqc.burst_precision); end @@ -252,7 +232,7 @@ postqc.variables{k}.l = l; postqc.variables{k}.r = r; - if has_burst + if ~isempty(burst_interval) postqc.variables{k}.valid_burst_range = findBursts(postqc.variables{k}.time, postqc.burst_precision); end @@ -263,7 +243,7 @@ postqc.variables{k}.time = time; postqc.variables{k}.l = 1; postqc.variables{k}.r = numel(pqc_var.data); - if has_burst + if ~isempty(burst_interval) postqc.variables{k}.valid_burst_range = postqc.burst_index_range; end @@ -271,3 +251,59 @@ end end + +function [is_burst_sampling,is_burst_metadata_valid,burst_interval] = get_burst_metadata(sample_data) +%function [is_burst_metadata_valid,burst_interval] = get_burst_metadata(sample_data) +% +% Discover if burst metadata is valid, by inspecting metadata at root level +% and at parser level (meta). If root level is valid, meta level is ignored. +% The burst_interval is also returned. +% +% Inputs: +% +% sample_data - the toolbox data struct. +% +% Outputs: +% +% is_burst_sampling - True if burst data is provided. +% is_burst_metadata_valid - True if both burst_duration and burst_interval are valid +% burst_interval - the burst interval value. +% +% +% author: hugo.oliveira@utas.edu.au +% + + +is_burst_sampling = false; +burst_interval = []; + +valid_burst_duration = false; +valid_burst_interval = false; + +if isfield(sample_data,'instrument_burst_duration') && ~isempty(sample_data.instrument_burst_duration) + is_burst_sampling = true; + burst_duration = sample_data.instrument_burst_duration; + valid_burst_duration = ~isnan(burst_duration) && burst_duration>0; +end + +if ~valid_burst_duration && isfield(sample_data,'meta') && isfield(sample_data.meta,'instrument_burst_duration') && ~isempty(sample_data.meta.instrument_burst_duration) + is_burst_sampling = true; + burst_duration = sample_data.meta.instrument_burst_duration; + valid_burst_duration = ~isnan(burst_duration) && burst_duration>0; +end + +if isfield(sample_data,'instrument_burst_interval') && ~isempty(sample_data.instrument_burst_interval) + is_burst_sampling = true; + burst_interval = sample_data.instrument_burst_interval; + valid_burst_interval = ~isnan(burst_interval) && burst_interval>0; +end + +if ~valid_burst_interval && isfield(sample_data,'meta') && isfield(sample_data.meta,'instrument_burst_interval') && ~isempty(sample_data.meta.instrument_burst_interval) + is_burst_sampling = true; + burst_interval = sample_data.meta.instrument_burst_interval; + valid_burst_interval = ~isempty(burst_interval) && ~isnan(burst_interval) && burst_interval>0; +end + +is_burst_metadata_valid = is_burst_sampling && valid_burst_duration && valid_burst_interval; + +end diff --git a/Java/ddb.jar b/Java/ddb.jar index 3406c509e..356607de9 100644 Binary files a/Java/ddb.jar and b/Java/ddb.jar differ diff --git a/imosToolbox.m b/imosToolbox.m index 40120ea96..0c77b9218 100644 --- a/imosToolbox.m +++ b/imosToolbox.m @@ -37,7 +37,7 @@ function imosToolbox(auto, varargin) % % Set current toolbox version -toolboxVersion = ['2.6.8 - ' computer]; +toolboxVersion = ['2.6.9 - ' computer]; if nargin == 0, auto = 'manual'; end diff --git a/imosToolbox_Linux64.bin b/imosToolbox_Linux64.bin index 1fb15c1d4..628dea2ba 100755 Binary files a/imosToolbox_Linux64.bin and b/imosToolbox_Linux64.bin differ diff --git a/imosToolbox_Win64.exe b/imosToolbox_Win64.exe index 7be70bd64..90530f186 100644 Binary files a/imosToolbox_Win64.exe and b/imosToolbox_Win64.exe differ diff --git a/test/Parser/testnetcdfParse.m b/test/Parser/testnetcdfParse.m index 84bb72c39..68c8262b9 100644 --- a/test/Parser/testnetcdfParse.m +++ b/test/Parser/testnetcdfParse.m @@ -8,22 +8,18 @@ methods (Test) - function test_no_error_reading_anmn(testCase, anmn_file) + function test_no_error_reading_anmn(~, anmn_file) data = netcdfParse({anmn_file}, ''); - assert(~isempty(data.meta.site_id)) - assert(~isempty(data.time_coverage_start)) - assert(~isempty(data.time_coverage_end)) - assert(isnan(data.meta.instrument_sample_interval) || data.meta.instrument_sample_interval > 0) - assert((isnan(data.meta.instrument_burst_interval) || data.meta.instrument_burst_interval > 0) && (isnan(data.meta.instrument_burst_duration) || data.meta.instrument_burst_duration > 0)) + assert_metadata(data); + assert_instrument_sample_interval(data); + assert_burst_sampling(data); end function test_no_error_reading_abos(~, abos_file) data = netcdfParse({abos_file}, ''); - assert(~isempty(data.meta.site_id)) - assert(~isempty(data.time_coverage_start)) - assert(~isempty(data.time_coverage_end)) - assert(isnan(data.meta.instrument_sample_interval) || data.meta.instrument_sample_interval > 0) - assert((isnan(data.meta.instrument_burst_interval) || data.meta.instrument_burst_interval > 0) && (isnan(data.meta.instrument_burst_duration) || data.meta.instrument_burst_duration > 0)) + assert_metadata(data); + assert_instrument_sample_interval(data); + assert_burst_sampling(data); end end @@ -31,9 +27,8 @@ function test_no_error_reading_abos(~, abos_file) end function ycell = only_netcdf(xcell) -ycell = {}; +ycell = cell(1,1000); c = 0; - for k = 1:length(xcell) item = xcell{k}; @@ -43,5 +38,46 @@ function test_no_error_reading_abos(~, abos_file) end end +ycell=ycell(1:c); +end +function assert_metadata(data) +got_site_id = isfield(data,'meta') && isfield(data.meta,'site_id'); +assert(got_site_id); +assert_site_id = ~isempty(data.meta.site_id); +assert(assert_site_id); + +got_time_coverage = isfield(data,'time_coverage_start') && isfield(data,'time_coverage_end'); +assert(got_time_coverage); +assert_time_coverage = ~isempty(data.time_coverage_start) && ~isempty(data.time_coverage_end); +assert(assert_time_coverage); +end + +function assert_instrument_sample_interval(data) +if isfield(data,'instrument_sample_interval') + if ~isempty(data.instrument_sample_interval) && isfield(data,'meta') && isfield(data.meta,'instrument_sample_interval') + assert(data.instrument_sample_interval == data.meta.instrument_sample_interval) + end +else + warning('%s missing instrument_sample_interval',data.toolbox_input_file) +end + +end + +function assert_burst_sampling(data) +is_burst = isfield(data,'instrument_burst_interval') || isfield(data,'instrument_burst_duration'); +if is_burst + has_burst_interval_but_not_duration = isfield(data,'instrument_burst_interval') && ~isfield(data,'instrument_burst_duration'); + has_burst_duration_but_not_interval = isfield(data,'instrument_burst_duration') && ~isfield(data,'instrument_burst_interval'); + if has_burst_interval_but_not_duration || has_burst_duration_but_not_interval + warning('%s contains incomplete burst information',data.toolbox_input_file) + else + assert_burst_interval = (~isnan(data.instrument_burst_interval)) || (data.instrument_burst_interval > 0); + assert_burst_duration = (~isnan(data.instrument_burst_duration)) || (data.instrument_burst_duration > 0); + assert(assert_burst_interval && assert_burst_duration) + if isfield(data,'meta') + assert_burst_sampling(data.meta) + end + end +end end diff --git a/test/UI/test_imosTimeSeriesSpikeQC.m b/test/UI/test_imosTimeSeriesSpikeQC.m index 1321bc7fb..14394b366 100644 --- a/test/UI/test_imosTimeSeriesSpikeQC.m +++ b/test/UI/test_imosTimeSeriesSpikeQC.m @@ -1,19 +1,99 @@ classdef test_imosTimeSeriesSpikeQC < matlab.unittest.TestCase + properties (TestParameter) + burst_file = {[toolboxRootPath() 'data/testfiles/Nortek/signature_500/v000/S100165A008_TR44Jun6.ad2cp']}; + non_burst_file = {[toolboxRootPath() 'data/testfiles/Sea_Bird_Scientific/SBE/19plus/v000/chla_aquaT3_as_aquaUV.cnv']}; + burst_file_nc = {[toolboxRootPath() 'data/testfiles/netcdf/test/UI/IMOS_ANMN-QLD_CKOSTUZ_20150523T085944Z_DARBGF_FV01_DARBGF-SURF-1505-WQM-1_END-20150714T221531Z_C-20171026T040809Z.nc']}; + non_burst_file_nc = {[toolboxRootPath() 'data/testfiles/netcdf/test/UI/IMOS_ANMN-QLD_CFKSTUZ_20150805T103003Z_DARBGF_FV01_DARBGF-1508-SBE16plus-29.3_END-20160207T225112Z_C-20171023T055701Z.nc']}; + end + + methods (Test) function test_no_ndvars_processed(~) sample_data = create_sample_data([100,1],{'2dvar','3dvar','TEMP','PITCH','ROLL'}); - dummy = sample_data.variables{1}.data; - sample_data.variables{1}.data = repmat(dummy,1,6); - sample_data.variables{1}.dimensions = 2; - sample_data.variables{2}.data = repmat(dummy,1,6,12); - sample_data.variables{2}.dimensions = 3; z = imosTimeSeriesSpikeQC(sample_data,true); procvars = fieldnames(z); assert(~inCell(procvars,'2dvar')); assert(~inCell(procvars,'3dvar')); end + function test_non_burst(~) + sample_data = create_sample_data([100,1],{'2dvar','3dvar','TEMP','PITCH','ROLL'}); + sample_data.instrument_burst_duration = ''; + sample_data.instrument_burst_interval = ''; + imosTimeSeriesSpikeQC(sample_data,true); + end + + function test_burst(~) + sample_data = create_sample_data([44,1],{'2dvar','3dvar','TEMP','PITCH','ROLL'}); + btime = (1:11)'+[1,3600,3600*2,3600*3]; + btime = btime(:); + sample_data.dimensions{1}.data = btime; + sample_data.instrument_burst_duration = 10; + sample_data.instrument_burst_interval = 3600; + imosTimeSeriesSpikeQC(sample_data,true); + end + + function test_use_meta_info(testCase) + sample_data = create_sample_data([44,1],{'2dvar','3dvar','TEMP','PITCH','ROLL'}); + btime = (1:11)'+[1,3600,3600*2,3600*3]; + btime = btime(:); + sample_data.dimensions{1}.data = btime; + sample_data.meta = struct(); + sample_data.meta.instrument_burst_duration = 10; + sample_data.meta.instrument_burst_interval = 3600; + z = imosTimeSeriesSpikeQC(sample_data,true); + assert(~isempty(z)); + sample_data.meta.instrument_burst_duration = NaN; + sample_data.meta.instrument_burst_interval = 3600; + func = @() imosTimeSeriesSpikeQC(sample_data,true); + assert(isempty(fieldnames(func()))); + last_warning_msg = lastwarn; + assert(isequal(last_warning_msg,'Invalid burst metadata...skipping')); + end + + function test_use_metadata_at_meta_level(~) + sample_data = create_sample_data([44,1],{'2dvar','3dvar','TEMP','PITCH','ROLL'}); + btime = (1:11)'+[1,3600,3600*2,3600*3]; + btime = btime(:); + sample_data.dimensions{1}.data = btime; + sample_data.instrument_burst_duration = ''; + sample_data.instrument_burst_interval = ''; + sample_data.meta = struct(); + sample_data.meta.instrument_burst_duration = 3600; + sample_data.meta.instrument_burst_interval = 10; + imosTimeSeriesSpikeQC(sample_data,true); + end + + function real_nonburst_file(~,non_burst_file) + sample_data = SBE19Parse({non_burst_file},'timeSeries'); + for k=1:length(sample_data.variables) + sample_data.variables{k}.flags = sample_data.variables{k}.data*0; + end + imosTimeSeriesSpikeQC(sample_data,true); + end + + function real_burst_file(~,burst_file) + sample_data = signatureParse({burst_file},'timeSeries'); + sample_data = sample_data{1}; + for k=1:length(sample_data.variables) + sample_data.variables{k}.flags = sample_data.variables{k}.data*0; + end + imosTimeSeriesSpikeQC(sample_data,true); + end + + function real_nonburst_file_netcdf(~,non_burst_file_nc) + sample_data = netcdfParse({non_burst_file_nc},'timeSeries'); + z = imosTimeSeriesSpikeQC(sample_data,true); + assert(~isempty(z)); + end + + function real_burst_file_netcdf(~,non_burst_file_nc) + sample_data = netcdfParse({non_burst_file_nc},'timeSeries'); + z = imosTimeSeriesSpikeQC(sample_data,true); + assert(~isempty(z)); + end + end end @@ -31,4 +111,9 @@ function test_no_ndvars_processed(~) sample_data.variables{k}.dimensions = 1; end sample_data.dimensions{1} = struct('name','TIME','data',linspace(0,length(dummy),1),'flags',zeros(asize)); +sample_data.variables{1}.data = repmat(dummy,1,6); +sample_data.variables{1}.dimensions = 2; +sample_data.variables{2}.data = repmat(dummy,1,6,12); +sample_data.variables{2}.dimensions = 3; + end