From eb1b86c833e387468ec9de7e31e71ebf0bf14099 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Tue, 28 Jul 2020 11:18:24 +1000 Subject: [PATCH 1/6] feat(fixup): imosTimeSeriesSpikeQC 2dvar list This fix up the is_vector detection for 2d vars, plus a better name for filtering the variables. --- AutomaticQC/imosTimeSeriesSpikeQC.m | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/AutomaticQC/imosTimeSeriesSpikeQC.m b/AutomaticQC/imosTimeSeriesSpikeQC.m index e367cf9e0..9444bed03 100644 --- a/AutomaticQC/imosTimeSeriesSpikeQC.m +++ b/AutomaticQC/imosTimeSeriesSpikeQC.m @@ -160,12 +160,11 @@ c = 1; for k = 1:length(ts_dims) - not_empty = ~isempty(ts_dims{k}); + no_empty_dims = ~isempty(ts_dims{k}); not_depth = ~strcmpi(varnames{k}, 'DEPTH'); var = getVar(sample_data.variables,varnames{k}); - is_vector = var && isvector(var); - - if not_empty && not_depth && is_vector + is_vector = var && isvector(sample_data.variables{var}.data); + if no_empty_dims && not_depth && is_vector ts_variables{c} = varnames{k}; c = c + 1; end From 3197ad7def3949fd0c73edcfb21f95f1169189ef Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Tue, 28 Jul 2020 11:21:47 +1000 Subject: [PATCH 2/6] feat(refactor): readability in has_burst block --- AutomaticQC/imosTimeSeriesSpikeQC.m | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/AutomaticQC/imosTimeSeriesSpikeQC.m b/AutomaticQC/imosTimeSeriesSpikeQC.m index 9444bed03..f7ca1cd1c 100644 --- a/AutomaticQC/imosTimeSeriesSpikeQC.m +++ b/AutomaticQC/imosTimeSeriesSpikeQC.m @@ -47,9 +47,17 @@ if (is_profile || time_missing), return, end -has_burst_duration = (isfield(sample_data.meta, 'instrument_burst_duration') &&~isempty(sample_data.meta.instrument_burst_duration)) || (isfield(sample_data, 'instrument_burst_duration') &&~isempty(sample_data.instrument_burst_duration)); -has_burst_interval = (isfield(sample_data.meta, 'instrument_burst_interval') &&~isempty(sample_data.meta.instrument_burst_interval)) || (isfield(sample_data, 'instrument_burst_interval') &&~isempty(sample_data.instrument_burst_interval)); - +has_burst_duration = false; +has_burst_interval = false; +try + has_burst_duration = sample_data.meta.instrument_burst_duration; + has_burst_interval = sample_data.meta.instrument_burst_interval; +catch + try + has_burst_duration = sample_data.instrument_burst_duration; + has_burst_interval = sample_data.meta.instrument_burst_interval; + end +end has_burst = has_burst_duration || has_burst_interval; if has_burst From 068c399ec42c259f2d711a660336912bc2efece1 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Tue, 28 Jul 2020 11:25:37 +1000 Subject: [PATCH 3/6] feat(bugfix): fix auto spikeQC calls. Update function name and readProperty argument order. --- AutomaticQC/imosTimeSeriesSpikeQC.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AutomaticQC/imosTimeSeriesSpikeQC.m b/AutomaticQC/imosTimeSeriesSpikeQC.m index f7ca1cd1c..083d806fd 100644 --- a/AutomaticQC/imosTimeSeriesSpikeQC.m +++ b/AutomaticQC/imosTimeSeriesSpikeQC.m @@ -79,8 +79,8 @@ if auto user_interaction = struct(); - spike_methods = load_classifiers(opts_file, has_burst); - method = readProperty(opts_file, 'auto_method'); + spike_methods = loadSpikeClassifiers(opts_file, has_burst); + method = readProperty('auto_function',opts_file); for k = 1:length(ts_variables) user_interaction.(ts_variables{k}) = spike_methods(method); From 6cee5e4afeaeeba53fe92f37429de4f2170307db Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Tue, 28 Jul 2020 11:26:47 +1000 Subject: [PATCH 4/6] feat(cleanup): delete old unused code and wspaces --- AutomaticQC/imosTimeSeriesSpikeQC.m | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/AutomaticQC/imosTimeSeriesSpikeQC.m b/AutomaticQC/imosTimeSeriesSpikeQC.m index 083d806fd..14e52b00f 100644 --- a/AutomaticQC/imosTimeSeriesSpikeQC.m +++ b/AutomaticQC/imosTimeSeriesSpikeQC.m @@ -137,7 +137,6 @@ % Construct the SpikeQC log strings for each variable % and each parameters used. % -nspikes = length(user_input.spikes); fun_name = func2str(user_input.fun); nargs = length(user_input.args); arg_string = cell(1, nargs); @@ -192,7 +191,7 @@ % sample_data - the toolbox data struct. % ts_variables - the time-series variables to process. % has_burst - if the data is in burst-mode. -% +% % Outputs: % % postqc - A data structure that is a copycat of sample_data but with clean timeSeries variables only. @@ -267,19 +266,3 @@ end end - -function [valid_burst_range, max_burst_len] = cut_burst_range(burst_index_ranges, l, r) -%function [valid_burst_range, max_burst_len] = cut_burst_range(burst_index_ranges, l, r) -% find the valid burst range and the maximum burst length -% -find_burst_ind = @(ind, x)(any(x == ind)); -find_l = @(x)(find_burst_ind(l, x)); -find_r = @(x)(find_burst_ind(r, x)); -first_valid_burst = find(cellfun(find_l, burst_index_ranges)); -last_valid_burst = find(cellfun(find_r, burst_index_ranges)); - -valid_burst_range = burst_index_ranges(first_valid_burst:last_valid_burst); - -compute_burst_len = @(x)(max(x) - min(x)); -max_burst_len = max(cellfun(compute_burst_len, valid_burst_range)); -end From a4e155cd063b5a3b99f44aab4c341e96e75da287 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Tue, 28 Jul 2020 11:28:04 +1000 Subject: [PATCH 5/6] feat(bugfix): fix auto_function str entries Also, update the comments for exact names of the tests in the UI. --- AutomaticQC/imosTimeSeriesSpikeQC.txt | 8 +++++--- AutomaticQC/imosTimeSeriesSpikeQCBurst.txt | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/AutomaticQC/imosTimeSeriesSpikeQC.txt b/AutomaticQC/imosTimeSeriesSpikeQC.txt index 4acd4626c..6e4e15c42 100644 --- a/AutomaticQC/imosTimeSeriesSpikeQC.txt +++ b/AutomaticQC/imosTimeSeriesSpikeQC.txt @@ -1,5 +1,7 @@ %If auto is on, use the default options from named method -auto_function = hampel +auto_function = Hampel + +%Hampel hampel_function = imosSpikeClassifierHampel %window of data at each side @@ -9,7 +11,7 @@ hampel_madfactor = 5 %clipping_mad hampel_lower_mad_limit = 0.0 -%SavGol Filter with OTSU Threshold - despiking2 +%OTSU-Savgol/despiking1 otsu_savgol_function = imosSpikeClassifierNonBurstSavGolOTSU %odd window filtering size @@ -21,7 +23,7 @@ otsu_savgol_nbins = 100 %scale to reduce the otsu threshold otsu_savgol_oscale = 1 -%OTSU Threhsold - despiking1 +%OTSU-Thresholding/despiking2 otsu_threshold_function = imosSpikeClassifierOTSU %number of histogram bins in otsu thresholding diff --git a/AutomaticQC/imosTimeSeriesSpikeQCBurst.txt b/AutomaticQC/imosTimeSeriesSpikeQCBurst.txt index fef70534a..9a7edc07a 100644 --- a/AutomaticQC/imosTimeSeriesSpikeQCBurst.txt +++ b/AutomaticQC/imosTimeSeriesSpikeQCBurst.txt @@ -1,7 +1,7 @@ %If auto is on, use the default options from named method -auto_function = burst_hampel +auto_function = Hampel-Burst -% burst Hampel filter +% Hampel-Burst burst_hampel_function = imosSpikeClassifierBurstHampel % boolean use windows as bursts instead of samples burst_hampel_use_burst_window = 0 @@ -15,7 +15,7 @@ burst_hampel_lower_mad_limit = 0 %detect all not only repeated burst_hampel_repeated_only = 0 -%Running Statistics +%Running-Statistics/imosBurstQC burst_runningstats_function = imosSpikeClassifierBurstRunningStats % the scale statistic function From 2acf9d548165b0d4f4eaf19239bfee72a5d75fa8 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Tue, 28 Jul 2020 11:30:22 +1000 Subject: [PATCH 6/6] feat(tests): add imosTimeSeriesSpikeQC tests --- test/UI/test_imosTimeSeriesSpikeQC.m | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 test/UI/test_imosTimeSeriesSpikeQC.m diff --git a/test/UI/test_imosTimeSeriesSpikeQC.m b/test/UI/test_imosTimeSeriesSpikeQC.m new file mode 100644 index 000000000..1321bc7fb --- /dev/null +++ b/test/UI/test_imosTimeSeriesSpikeQC.m @@ -0,0 +1,34 @@ +classdef test_imosTimeSeriesSpikeQC < matlab.unittest.TestCase + + methods (Test) + function test_no_ndvars_processed(~) + sample_data = create_sample_data([100,1],{'2dvar','3dvar','TEMP','PITCH','ROLL'}); + dummy = sample_data.variables{1}.data; + sample_data.variables{1}.data = repmat(dummy,1,6); + sample_data.variables{1}.dimensions = 2; + sample_data.variables{2}.data = repmat(dummy,1,6,12); + sample_data.variables{2}.dimensions = 3; + z = imosTimeSeriesSpikeQC(sample_data,true); + procvars = fieldnames(z); + assert(~inCell(procvars,'2dvar')); + assert(~inCell(procvars,'3dvar')); + end + + end +end + +function sample_data = create_sample_data(asize,varnames) +sample_data = struct(); +sample_data.variables = cell(1, 1); +sample_data.dimensions = cell(1, 1); +dummy = randn(asize); + +for k = 1:length(varnames) + sample_data.variables{k} = struct(); + sample_data.variables{k}.name = varnames{k}; + sample_data.variables{k}.data = dummy; + sample_data.variables{k}.flags = zeros(asize); + sample_data.variables{k}.dimensions = 1; +end +sample_data.dimensions{1} = struct('name','TIME','data',linspace(0,length(dummy),1),'flags',zeros(asize)); +end