From 4777cd633d9b9ca26aad93979ab8b68a273020a2 Mon Sep 17 00:00:00 2001 From: lbesnard Date: Tue, 27 Jun 2023 06:22:30 +0000 Subject: [PATCH 1/7] (Feat) imosQCTests function to read imos flags from config --- AutomaticQC/qcFilterMain.m | 97 ++++++++++++++++++++++- FlowManager/displayManager.m | 16 +++- FlowManager/flowManager.m | 9 +++ IMOS/imosQCTest.m | 57 ++++++++++++++ IMOS/imosQCTests.m | 64 ++++++++++++++++ IMOS/imosQCTests.txt | 36 +++++++++ NetCDF/exportNetCDF.m | 115 +++++++++++++++++++++++++++- NetCDF/template/flag_attributes.txt | 15 ++++ 8 files changed, 405 insertions(+), 4 deletions(-) create mode 100644 IMOS/imosQCTest.m create mode 100644 IMOS/imosQCTests.m create mode 100644 IMOS/imosQCTests.txt create mode 100644 NetCDF/template/flag_attributes.txt diff --git a/AutomaticQC/qcFilterMain.m b/AutomaticQC/qcFilterMain.m index b4660af7e..6121a134c 100644 --- a/AutomaticQC/qcFilterMain.m +++ b/AutomaticQC/qcFilterMain.m @@ -71,6 +71,10 @@ probBadIdx = fsam.(type{m}){k}.flags == probBadFlag; badIdx = fsam.(type{m}){k}.flags == badFlag; + % call the sub fuction to add the results of the failed tests + % into a new variable + addFailedTestsFlags() + if ~strcmpi(func2str(filter), 'imosHistoricalManualSetQC') % imosHistoricalManualSetQC can downgrade flags % otherwise we can only upgrade flags % set current flag areas @@ -84,6 +88,10 @@ probGoodIdx = canBeFlagProbGoodIdx & probGoodIdx; probBadIdx = canBeFlagProbBadIdx & probBadIdx; badIdx = canBeFlagBadIdx & badIdx; + + % call the sub fuction to add the results of the failed tests + % into a new variable + addFailedTestsFlags() end sam.(type{m}){k}.flags(goodIdx) = fsam.(type{m}){k}.flags(goodIdx); @@ -206,6 +214,10 @@ probBadIdx = f == probBadFlag; badIdx = f == badFlag; + % call the sub fuction to add the results of the failed tests + % into a new variable + addFailedTestsFlags() + % update new flags in current variable goodIdx = canBeFlagGoodIdx & goodIdx; probGoodIdx = canBeFlagProbGoodIdx & probGoodIdx; @@ -268,4 +280,87 @@ end end end -end + + function addFailedTestsFlags() + % addFailedTestsFlags inherits all the variable available in the + % workspace. + % + % we now have for the test filterName the result (good, probgood, probBad, bad) value for each variable and each datapoint. + % For most QC routines, a failed test is where data is not flagged + % as good. + % However, more information is available on + % https://github.com/aodn/imos-toolbox/wiki/QCProcedures + % + % This function aims to store, in a new variable named "flag_tests" + % the result of why a data point wasn't flagged as good, i.e. which + % QC routine was responsible for "rejecting" a data point. + % + % To do so, each failed QC routine is linked to the power of 2 of + % an integer. + % For example: + % manualQC is linked to 2^1 + % imosCorrMagVelocitySetQC is linked to 2^2 + % ... + % Any integer can be written as the sum of disting powers of 2 + % numbers (see various proofs online + % https://math.stackexchange.com/questions/176678/strong-induction-proof-every-natural-number-sum-of-distinct-powers-of-2) + % + % + % Note that QC tests can be rerun by the user multiple times. This + % however should be dealt in the FlowManager/displayManager.m + % function + % + % + % the failed tests depends on the test. It is not always the same + % for all. See table for each individual test at + % https://github.com/aodn/imos-toolbox/wiki/QCProcedures + if strcmp(filterName, 'imosTiltVelocitySetQC') % see https://github.com/aodn/imos-toolbox/wiki/QCProcedures#adcp-tilt-test---imostiltvelocitysetqc---compulsory + % For the imosTiltVelocitySetQC, users would like to know if the + % failed test is because the data was flagged as bad or as probably + % good + % this test (imosTiltVelocitySetQC) + % there is a two level flag for failed tests. We're taking this + % into account below + failedTestsIdx_probGood = probGoodIdx; + failedTestsIdx_bad = badIdx; + elseif strcmp(filterName, 'imosErrorVelocitySetQC') % see https://github.com/aodn/imos-toolbox/wiki/QCProcedures#adcp-error-velocity-test---imoserrorvelocitysetqc---optional + % a pass test for this test only is if the data is flagged good + % or probably good if ECUR is NaN + + % TODO: implement this scenario + + strcmp(sam.(type{m}){k}.name, 'ECUR') ; + failedTestsIdx = badIdx; + else + failedTestsIdx = probGoodIdx | probBadIdx | badIdx; % result matrice for all variables + end + + + if strcmp(filterName, 'imosTiltVelocitySetQC') + failedTests_probGood = zeros(size(failedTestsIdx_probGood)); + failedTests_probGood(logical(failedTestsIdx_probGood)) = imosQCTest(strcat(filterName, '_probably_good')); + + failedTests_bad = zeros(size(failedTestsIdx_bad)); + failedTests_bad(logical(failedTestsIdx_bad)) = imosQCTest(strcat(filterName, '_bad_data')); + + if ~isfield(sam.(type{m}){k}, 'flag_tests') % if the flat_tests field does not exist yet + sam.(type{m}){k}.flag_tests = failedTests_bad + failedTests_probGood; + + else + sam.(type{m}){k}.flag_tests = sam.(type{m}){k}.flag_tests + failedTests_bad + failedTests_probGood; + end + else + + failedTests = zeros(size(failedTestsIdx)); + failedTests(logical(failedTestsIdx)) = imosQCTest(filterName); + + if ~isfield(sam.(type{m}){k},'flag_tests') % if the flat_tests field does not exist yet + sam.(type{m}){k}.flag_tests = failedTests; + else + % if it already exists, we sum the previous array with the new one + sam.(type{m}){k}.flag_tests = sam.(type{m}){k}.flag_tests + failedTests; + + end + end + end +end \ No newline at end of file diff --git a/FlowManager/displayManager.m b/FlowManager/displayManager.m index de8b72647..2f1e0bb16 100644 --- a/FlowManager/displayManager.m +++ b/FlowManager/displayManager.m @@ -623,8 +623,20 @@ function resetPropQCCallback() for j=1:length(resetIdx) for i=1:length(sample_data{resetIdx(j)}.variables) - if isfield(sample_data{resetIdx(j)}.variables(i), 'ancillary_comment') - sample_data{resetIdx(j)}.variables(i) = rmfield(sample_data{resetIdx(j)}.variables(i), 'ancillary_comment'); + % previous code could not run since it was using variables(i) instead of variables{i}. + % TODO ask @ggalibert about this + if isfield(sample_data{resetIdx(j)}.variables{i}, 'ancillary_comment') + sample_data{resetIdx(j)}.variables{i} = rmfield(sample_data{resetIdx(j)}.variables{i}, 'ancillary_comment'); + end + + % delete the flag_tests previous results so that we don't + % sum up again the failed QC routines. The good thing is + % that all QC routines a reset anyway, which makes our + % life easier + if isfield(sample_data{resetIdx(j)}.variables{i}, 'flag_tests') + % we don't bother doing heavy calculation. + % We simply delete the "flag_tests" variable + sample_data{resetIdx(j)}.variables{i} = rmfield(sample_data{resetIdx(j)}.variables{i}, 'flag_tests'); end end diff --git a/FlowManager/flowManager.m b/FlowManager/flowManager.m index bd76b8c56..427e3b90a 100644 --- a/FlowManager/flowManager.m +++ b/FlowManager/flowManager.m @@ -341,6 +341,15 @@ function manualQCRequestCallback(setIdx, varIdx, dataIdx, flag, manualQcComment) % we update the flags values autoQCData{setIdx}.variables{varIdx}.flags(dataIdx) = flag; + % update the flag_tests field containing information on failed tests + % with manual QC information + if isfield(autoQCData{setIdx}.variables{varIdx}, 'flag_tests') + autoQCData{setIdx}.variables{varIdx}.flag_tests(dataIdx) = imosQCTest('userManualQC'); % if manual QC done on a point, we overwrite previous failed QC routine information + else + autoQCData{setIdx}.variables{varIdx}.flag_tests(dataIdx) = zeros(size(dataIdx)); % initialise array + autoQCData{setIdx}.variables{varIdx}.flag_tests(dataIdx) = imosQCTest('userManualQC'); + end + qcSet = str2double(readProperty('toolbox.qc_set')); rawFlag = imosQCFlag('raw', qcSet, 'flag'); diff --git a/IMOS/imosQCTest.m b/IMOS/imosQCTest.m new file mode 100644 index 000000000..c3c1265f0 --- /dev/null +++ b/IMOS/imosQCTest.m @@ -0,0 +1,57 @@ +function value = imosQCTest( testName ) +%imosQCTest Returns an appropriate QC flag_value (integer) +% given a qc test routine (String) + +% The value returned by this function is the power of 2 of the qc routine +% positional integer available in imosQCTests.txt. This is used to store +% information in a variable in the form of integers +% +% +% Inputs: +% +% testName - name of QC test +% +% Outputs: +% value - integer +% +% Author: Laurent Besnard +% +% Example: +% +% value = imosQCTest( 'userManualQC'); +% +% +% assert(value==2) +% + +narginchk(1, 1); +if ~ischar(testName), error('field must be a string'); end + +value = ''; + +% open the IMOSQCFTests.txt file - it should be +% in the same directory as this m-file +path = ''; +if ~isdeployed, [path, ~, ~] = fileparts(which('imosToolbox.m')); end +if isempty(path), path = pwd; end +path = fullfile(path, 'IMOS'); + +fidS = -1; +try + % read in the QC sets + fidS = fopen([path filesep 'imosQCTests.txt'], 'rt'); + if fidS == -1, return; end + sets = textscan(fidS, '%s%f', 'delimiter', ',', 'commentStyle', '%'); + fclose(fidS); + + [~, idx] = ismember(testName, sets{1,1}); + + value = 2^sets{1,2}(idx); % return the + + +catch e + if fidS ~= -1, fclose(fidS); end + rethrow(e); +end + + diff --git a/IMOS/imosQCTests.m b/IMOS/imosQCTests.m new file mode 100644 index 000000000..10630153f --- /dev/null +++ b/IMOS/imosQCTests.m @@ -0,0 +1,64 @@ +function values = imosQCTests() +%imosQCTests Returns a 1x2 cell array reading the info found in +%ImosQCTests.txt +% - the first one contains a list of QC routines +% - the second cell returns a positional integer for each routine +% +% Inputs: +% +% N.A. +% +% Outputs: +% 1x2 cell array +% +% Author: Laurent Besnard +% + +% +% Copyright (C) 2023, Australian Ocean Data Network (AODN) and Integrated +% Marine Observing System (IMOS). +% +% This program is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation version 3 of the License. +% +% This program is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. + +% You should have received a copy of the GNU General Public License +% along with this program. +% If not, see . +% + +% open the IMOSQCFTests file - it should be +% in the same directory as this m-file +path = ''; +if ~isdeployed, [path, ~, ~] = fileparts(which('imosToolbox.m')); end +if isempty(path), path = pwd; end +path = fullfile(path, 'IMOS'); + +fidS = -1; +try + % read in the QC sets + fidS = fopen([path filesep 'imosQCTests.txt'], 'rt'); + if fidS == -1, return; end + sets = textscan(fidS, '%s%f', 'delimiter', ',', 'commentStyle', '%'); + fclose(fidS); + + % rewrite sets with correct qc flag test values using the imosQCTest + % function + nQcFlags = length(sets{1}); + for k = 1:nQcFlags + sets{1,2}(k) = imosQCTest(sets{1,1}{k}); + end + + values = sets; + +catch e + if fidS ~= -1, fclose(fidS); end + rethrow(e); +end + + diff --git a/IMOS/imosQCTests.txt b/IMOS/imosQCTests.txt new file mode 100644 index 000000000..70d3aaea7 --- /dev/null +++ b/IMOS/imosQCTests.txt @@ -0,0 +1,36 @@ +% +% A list of all the toolbox QC routines flag values. This list is intended +% to by used by the qcFilterMain.m function to tag, for each data point, +% which QC test failed +% +% the positional integer value has to be strictly above 0 since 0 is used +% as a _FillValue +% +% QC routine, positional integer +userManualQC, 1 +imosCorrMagVelocitySetQC, 2 +imosDensityInversionSetQC, 3 +imosEchoIntensitySetQC, 4 +imosEchoIntensityVelocitySetQC, 5 +imosEchoRangeSetQC, 6 +imosErrorVelocitySetQC, 7 +imosGlobalRangeQC, 8 +imosHistoricalManualSetQC, 9 +imosHorizontalVelocitySetQC, 10 +imosImpossibleDateQC, 11 +imosImpossibleDepthQC, 12 +imosImpossibleLocationSetQC, 13 +imosInOutWaterQC, 14 +imosPercentGoodVelocitySetQC, 15 +imosRateOfChangeQC, 16 +imosRegionalRangeQC, 17 +imosSalinityFromPTQC, 18 +imosSideLobeVelocitySetQC, 19 +imosStationarityQC, 20 +imosSurfaceDetectionByDepthSetQC, 21 +imosTier2ProfileVelocitySetQC, 22 +imosTiltVelocitySetQC_probably_good,23 +imosTiltVelocitySetQC_bad_data, 24 +imosTimeSeriesSpikeQC, 25 +imosVerticalSpikeQC, 26 +imosVerticalVelocityQC, 27 diff --git a/NetCDF/exportNetCDF.m b/NetCDF/exportNetCDF.m index b3bd3bee4..d7c0637a4 100644 --- a/NetCDF/exportNetCDF.m +++ b/NetCDF/exportNetCDF.m @@ -61,6 +61,13 @@ dateFmt = readProperty('exportNetCDF.dateFormat'); qcSet = str2double(readProperty('toolbox.qc_set')); qcType = imosQCFlag('', qcSet, 'type'); + + % lbesnard: tried the following + % qcFlagType = 'NC_INT'; % as described in + % https://au.mathworks.com/help/matlab/ref/netcdf.defvar.html however it + % breaks the code in the netcdf3tomatlabtype function. TODO: ask + % @ggalibert why it was coded this way. + qcFlagType = 'int'; qcDimId = []; try @@ -153,6 +160,7 @@ dimAtts = rmfield(dimAtts, {'data', 'name'}); if isfield(dimAtts, 'typeCastFunc'), dimAtts = rmfield(dimAtts, 'typeCastFunc'); end if isfield(dimAtts, 'flags'), dimAtts = rmfield(dimAtts, 'flags'); end + if isfield(dimAtts, 'flag_tests'), dimAtts = rmfield(dimAtts, 'flag_tests'); end % QC variable is not CF for dimensions if isfield(dimAtts, 'FillValue_'), dimAtts = rmfield(dimAtts, 'FillValue_'); end % _FillValues for dimensions are not CF dimAtts = rmfield(dimAtts, 'stringlen'); @@ -299,6 +307,7 @@ varAtts = rmfield(varAtts, {'data', 'dimensions', 'stringlen', 'name'}); if isfield(varAtts, 'typeCastFunc'), varAtts = rmfield(varAtts, 'typeCastFunc'); end if isfield(varAtts, 'flags'), varAtts = rmfield(varAtts, 'flags'); end + if isfield(varAtts, 'flag_tests'), varAtts = rmfield(varAtts, 'flag_tests'); end if isfield(varAtts, 'ancillary_comment'), varAtts = rmfield(varAtts, 'ancillary_comment');end if isfield(vars{m}, 'flags') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF @@ -306,6 +315,12 @@ % to the ancillary variables attribute varAtts.ancillary_variables = [varname '_quality_control']; end + + if isfield(vars{m}, 'flag_tests') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF + % add the *_failed_tests variable (defined below) + % to the ancillary variables attribute + varAtts.ancillary_variables = [varAtts.ancillary_variables ,' ' ,varname '_failed_tests']; + end % add the attributes putAtts(fid, vid, vars{m}, varAtts, 'variable', varNetcdfType{end}, dateFmt, mode); @@ -322,10 +337,25 @@ else qcvid = NaN; end + + + if isfield(vars{m}, 'flag_tests') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF + % create the ancillary *_failed_tests variable + qcflagvid = addQCFlagsVar(... + fid, sample_data, m, [qcDimId dids], 'variables', qcFlagType, qcSet, dateFmt, mode); + + if ~isempty(dimLen) + netcdf.defVarChunking(fid, qcflagvid, 'CHUNKED', dimLen); + netcdf.defVarDeflate(fid, qcflagvid, true, true, compressionLevel); + end + else + qcflagvid = NaN; + end % save variable IDs for later reference sample_data.variables{m}.vid = vid; sample_data.variables{m}.qcvid = qcvid; + sample_data.variables{m}.qcflagvid = qcflagvid; end % we're finished defining dimensions/attributes/variables @@ -433,6 +463,18 @@ netcdf.putVar(fid, qcvid, flags); end + + if isfield(vars{m}, 'flag_tests') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF + flag_tests = vars{m}.flag_tests; + qcflagvid = vars{m}.qcflagvid; + + typeCastFunction = str2func(netcdf3ToMatlabType(qcFlagType)); + flag_tests = typeCastFunction(flag_tests); + + if nDims > 1, flag_tests = permute(flag_tests, nDims:-1:1); end + + netcdf.putVar(fid, qcflagvid, flag_tests); + end end % @@ -470,7 +512,7 @@ % switch(type) case 'dimensions' - var = sample_data.dimensions{varIdx}; + var = sample_data.dimensions{varIdx}; % when is this even triggered? (Loz, 20230624) template = 'qc_coord'; case 'variables' var = sample_data.variables{varIdx}; @@ -600,6 +642,77 @@ putAtts(fid, vid, var, qcAtts, template, netcdfType, dateFmt, ''); end + + +function vid = addQCFlagsVar(... + fid, sample_data, varIdx, dims, type, netcdfType, ~, dateFmt, ~) +%addQCFlagsVar Adds an ancillary variable, containing failed QC routines info +% for the variable with the given index. +% +% Inputs: +% fid - NetCDF file identifier +% sample_data - Struct containing entire data set +% varIdx - Index into sample_data.variables, specifying the +% variable. +% dims - Vector of NetCDF dimension identifiers. +% type - either 'dimensions' or 'variables', to differentiate between +% coordinate variables and data variables. +% netcdfType - The netCDF type in which the flags should be output. +% dateFmt - Date format in which date attributes should be output. +% mode - Toolbox processing mode. +% +% Outputs: +% vid - NetCDF variable identifier of the QC variable that was +% created. +% + switch(type) + % same as addQCVar function, it doesn't seem that the template + % "flag_coord" is ever being triggered ? + %case 'dimensions' + % var = sample_data.dimensions{varIdx}; + % template = 'flag_coord'; + case 'variables' + var = sample_data.variables{varIdx}; + template = 'flag'; + otherwise + error(['invalid type: ' type]); + end + + path = readProperty('toolbox.templateDir'); + if isempty(path) || ~exist(path, 'dir') + path = ''; + if ~isdeployed, [path, ~, ~] = fileparts(which('imosToolbox.m')); end + if isempty(path), path = pwd; end + path = fullfile(path, 'NetCDF', 'template'); + end + + varname = [var.name '_failed_tests']; + + qcAtts = parseNetCDFTemplate(... + fullfile(path, [template '_attributes.txt']), sample_data, varIdx); + + % get qc flag values + qcFlags = imosQCTests; %g('', qcSet, 'values'); + nQcFlags = length(qcFlags{1}); + qcDescs = cell(nQcFlags, 1); + + % get flag descriptions + for k = 1:nQcFlags + qcDescs{k} = qcFlags{1}{k}; + end + + qcAtts.flag_values = qcFlags{2}; + + % turn descriptions into space separated string + qcDescs = cellfun(@(x)(sprintf('%s ', x)), qcDescs, 'UniformOutput', false); + qcDescs{end}(end) = []; + qcAtts.flag_meanings = [qcDescs{:}]; + + vid = netcdf.defVar(fid, varname, netcdfType, dims); + putAtts(fid, vid, var, qcAtts, template, netcdfType, dateFmt, ''); % netcdf.getAtt(fid,vid,'flag_values') check why it's doing something funky +end + + function putAtts(fid, vid, var, template, templateFile, netcdfType, dateFmt, mode) %PUTATTS Puts all the attributes from the given template into the given NetCDF % variable. diff --git a/NetCDF/template/flag_attributes.txt b/NetCDF/template/flag_attributes.txt new file mode 100644 index 000000000..082810665 --- /dev/null +++ b/NetCDF/template/flag_attributes.txt @@ -0,0 +1,15 @@ +S, long_name = quality control failed tests for [mat imosParameters(sample_data.variables{k}.name, 'long_name')] +S, standard_name = [mat regexprep(strcat(imosParameters(sample_data.variables{k}.name, 'standard_name'), ' status_flag'), '^ .*', '')] +Q, _FillValue = 0 +N, add_offset = +N, scale_factor = +S, comment = variable to inform on which QC routines was responsible for not flagging a data point as Good. The result value is the sum of flag_values associated to each QC routine. Every number needs to be split into the sum of power 2 to match witch every flag_values/flag_meanings +S, history = +S, references = +S, quality_control_conventions = [mat imosQCFlag('', str2double(readProperty('toolbox.qc_set', 'toolboxProperties.txt')), 'set_desc')] + +% these fields are automatically populated upon NetCDF export +Q, flag_values = +S, flag_meanings = +#S, quality_control_global_conventions = Argo reference table 2a (see http://www.cmar.csiro.au/argo/dmqc/user_doc/QC_flags.html), applied on data in position only (between global attributes time_deployment_start and time_deployment_end) +Q, quality_control_global = From aa7be9fb1c8cc352413c14449c9f9269c2daac6c Mon Sep 17 00:00:00 2001 From: lbesnard Date: Wed, 28 Jun 2023 02:03:57 +0000 Subject: [PATCH 2/7] Add scenario for imosErrorVelocitySetQC --- AutomaticQC/qcFilterMain.m | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/AutomaticQC/qcFilterMain.m b/AutomaticQC/qcFilterMain.m index 6121a134c..de3c2f243 100644 --- a/AutomaticQC/qcFilterMain.m +++ b/AutomaticQC/qcFilterMain.m @@ -324,13 +324,28 @@ function addFailedTestsFlags() failedTestsIdx_probGood = probGoodIdx; failedTestsIdx_bad = badIdx; elseif strcmp(filterName, 'imosErrorVelocitySetQC') % see https://github.com/aodn/imos-toolbox/wiki/QCProcedures#adcp-error-velocity-test---imoserrorvelocitysetqc---optional - % a pass test for this test only is if the data is flagged good - % or probably good if ECUR is NaN + % for this specific QC routine, a test is considerer to be pass + % if the data is flagged good or probably good if ECUR is NaN - % TODO: implement this scenario + % find variable index for ECUR + notdone = true; + while notdone + for kk = 1:length(sam.(type{m})) + if strcmp(sam.(type{m}){kk}.name, 'ECUR') + notdone = false; + break + end + end + end + + if not(notdone) + if all(isnan(sam.(type{m}){kk}.data(:))) + failedTestsIdx = badIdx | probBadIdx; + else + failedTestsIdx = badIdx | probBadIdx | probGoodIdx ; + end + end - strcmp(sam.(type{m}){k}.name, 'ECUR') ; - failedTestsIdx = badIdx; else failedTestsIdx = probGoodIdx | probBadIdx | badIdx; % result matrice for all variables end From 3203f5b64c5a2abcbae8cdb2bc1cfae8e756ef86 Mon Sep 17 00:00:00 2001 From: lbesnard Date: Wed, 28 Jun 2023 05:03:15 +0000 Subject: [PATCH 3/7] wip --- AutomaticQC/qcFilterMain.m | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/AutomaticQC/qcFilterMain.m b/AutomaticQC/qcFilterMain.m index de3c2f243..35dc292a2 100644 --- a/AutomaticQC/qcFilterMain.m +++ b/AutomaticQC/qcFilterMain.m @@ -71,10 +71,6 @@ probBadIdx = fsam.(type{m}){k}.flags == probBadFlag; badIdx = fsam.(type{m}){k}.flags == badFlag; - % call the sub fuction to add the results of the failed tests - % into a new variable - addFailedTestsFlags() - if ~strcmpi(func2str(filter), 'imosHistoricalManualSetQC') % imosHistoricalManualSetQC can downgrade flags % otherwise we can only upgrade flags % set current flag areas @@ -87,13 +83,13 @@ goodIdx = canBeFlagGoodIdx & goodIdx; probGoodIdx = canBeFlagProbGoodIdx & probGoodIdx; probBadIdx = canBeFlagProbBadIdx & probBadIdx; - badIdx = canBeFlagBadIdx & badIdx; - - % call the sub fuction to add the results of the failed tests - % into a new variable - addFailedTestsFlags() + badIdx = canBeFlagBadIdx & badIdx; end + % call the sub fuction to add the results of the failed tests + % into a new variable + addFailedTestsFlags() + sam.(type{m}){k}.flags(goodIdx) = fsam.(type{m}){k}.flags(goodIdx); sam.(type{m}){k}.flags(probGoodIdx) = fsam.(type{m}){k}.flags(probGoodIdx); sam.(type{m}){k}.flags(probBadIdx) = fsam.(type{m}){k}.flags(probBadIdx); From 468a5d48f8482feb20567bfb878364e89430a13c Mon Sep 17 00:00:00 2001 From: lbesnard Date: Wed, 28 Jun 2023 07:33:26 +0000 Subject: [PATCH 4/7] wip --- IMOS/imosQCTests.txt | 54 ++++++++++++++--------------- NetCDF/exportNetCDF.m | 26 ++++++++++---- NetCDF/template/flag_attributes.txt | 7 ++-- 3 files changed, 49 insertions(+), 38 deletions(-) diff --git a/IMOS/imosQCTests.txt b/IMOS/imosQCTests.txt index 70d3aaea7..96b26da14 100644 --- a/IMOS/imosQCTests.txt +++ b/IMOS/imosQCTests.txt @@ -7,30 +7,30 @@ % as a _FillValue % % QC routine, positional integer -userManualQC, 1 -imosCorrMagVelocitySetQC, 2 -imosDensityInversionSetQC, 3 -imosEchoIntensitySetQC, 4 -imosEchoIntensityVelocitySetQC, 5 -imosEchoRangeSetQC, 6 -imosErrorVelocitySetQC, 7 -imosGlobalRangeQC, 8 -imosHistoricalManualSetQC, 9 -imosHorizontalVelocitySetQC, 10 -imosImpossibleDateQC, 11 -imosImpossibleDepthQC, 12 -imosImpossibleLocationSetQC, 13 -imosInOutWaterQC, 14 -imosPercentGoodVelocitySetQC, 15 -imosRateOfChangeQC, 16 -imosRegionalRangeQC, 17 -imosSalinityFromPTQC, 18 -imosSideLobeVelocitySetQC, 19 -imosStationarityQC, 20 -imosSurfaceDetectionByDepthSetQC, 21 -imosTier2ProfileVelocitySetQC, 22 -imosTiltVelocitySetQC_probably_good,23 -imosTiltVelocitySetQC_bad_data, 24 -imosTimeSeriesSpikeQC, 25 -imosVerticalSpikeQC, 26 -imosVerticalVelocityQC, 27 +userManualQC, 0 +imosCorrMagVelocitySetQC, 1 +imosDensityInversionSetQC, 2 +imosEchoIntensitySetQC, 3 +imosEchoIntensityVelocitySetQC, 4 +imosEchoRangeSetQC, 5 +imosErrorVelocitySetQC, 6 +imosGlobalRangeQC, 7 +imosHistoricalManualSetQC, 8 +imosHorizontalVelocitySetQC, 9 +imosImpossibleDateQC, 10 +imosImpossibleDepthQC, 11 +imosImpossibleLocationSetQC, 12 +imosInOutWaterQC, 13 +imosPercentGoodVelocitySetQC, 14 +imosRateOfChangeQC, 15 +imosRegionalRangeQC, 16 +imosSalinityFromPTQC, 17 +imosSideLobeVelocitySetQC, 18 +imosStationarityQC, 19 +imosSurfaceDetectionByDepthSetQC, 20 +imosTier2ProfileVelocitySetQC, 21 +imosTiltVelocitySetQC_probably_good,22 +imosTiltVelocitySetQC_bad_data, 23 +imosTimeSeriesSpikeQC, 24 +imosVerticalSpikeQC, 25 +imosVerticalVelocityQC, 26 diff --git a/NetCDF/exportNetCDF.m b/NetCDF/exportNetCDF.m index d7c0637a4..fc5467514 100644 --- a/NetCDF/exportNetCDF.m +++ b/NetCDF/exportNetCDF.m @@ -666,11 +666,6 @@ % created. % switch(type) - % same as addQCVar function, it doesn't seem that the template - % "flag_coord" is ever being triggered ? - %case 'dimensions' - % var = sample_data.dimensions{varIdx}; - % template = 'flag_coord'; case 'variables' var = sample_data.variables{varIdx}; template = 'flag'; @@ -692,7 +687,7 @@ fullfile(path, [template '_attributes.txt']), sample_data, varIdx); % get qc flag values - qcFlags = imosQCTests; %g('', qcSet, 'values'); + qcFlags = imosQCTests; nQcFlags = length(qcFlags{1}); qcDescs = cell(nQcFlags, 1); @@ -701,7 +696,24 @@ qcDescs{k} = qcFlags{1}{k}; end - qcAtts.flag_values = qcFlags{2}; + % see + % http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html#flags + % The flag_masks and flag_meanings attributes describe a number of + % independent Boolean conditions using bit field notation by setting + % unique bits in each flag_masks value. The flag_masks attribute is the + % same type as the variable to which it is attached, and contains a list + % of values matching unique bit fields. The flag_meanings attribute is + % defined as above, one for each flag_masks value. A flagged condition is + % identified by performing a bitwise AND of the variable value and each + % flag_masks value; a non-zero result indicates a true condition. Thus, + % any or all of the flagged conditions may be true, depending on the + % variable bit settings. The following example illustrates the use of + % flag_masks to express six sensor status conditions. + % + % In the case of this [variable]_failed_tests variable, we're using the + % flag_masks rather than flag_values + + qcAtts.flag_masks = qcFlags{2}; % turn descriptions into space separated string qcDescs = cellfun(@(x)(sprintf('%s ', x)), qcDescs, 'UniformOutput', false); diff --git a/NetCDF/template/flag_attributes.txt b/NetCDF/template/flag_attributes.txt index 082810665..a46fdaf73 100644 --- a/NetCDF/template/flag_attributes.txt +++ b/NetCDF/template/flag_attributes.txt @@ -3,13 +3,12 @@ S, standard_name = [mat regexprep(strcat(imosParameters(s Q, _FillValue = 0 N, add_offset = N, scale_factor = -S, comment = variable to inform on which QC routines was responsible for not flagging a data point as Good. The result value is the sum of flag_values associated to each QC routine. Every number needs to be split into the sum of power 2 to match witch every flag_values/flag_meanings +S, comment = variable to inform on which QC routine(s) was(were) responsible for not flagging a data point as Good. The result value is the sum of flag_masks associated to each QC routine. Every number needs to be split into the sum of power 2 to match witch every flag_masks/flag_meanings combination. S, history = S, references = -S, quality_control_conventions = [mat imosQCFlag('', str2double(readProperty('toolbox.qc_set', 'toolboxProperties.txt')), 'set_desc')] % these fields are automatically populated upon NetCDF export -Q, flag_values = +Q, flag_masks = S, flag_meanings = #S, quality_control_global_conventions = Argo reference table 2a (see http://www.cmar.csiro.au/argo/dmqc/user_doc/QC_flags.html), applied on data in position only (between global attributes time_deployment_start and time_deployment_end) -Q, quality_control_global = +#Q, quality_control_global = From fb8bb8e1bf3e56f4e92966063bf2e7a708fedcc7 Mon Sep 17 00:00:00 2001 From: lbesnard Date: Thu, 29 Jun 2023 07:34:51 +0000 Subject: [PATCH 5/7] wip --- AutomaticQC/qcFilterMain.m | 6 +++++- IMOS/imosQCTest.m | 2 +- NetCDF/exportNetCDF.m | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/AutomaticQC/qcFilterMain.m b/AutomaticQC/qcFilterMain.m index 35dc292a2..c75937739 100644 --- a/AutomaticQC/qcFilterMain.m +++ b/AutomaticQC/qcFilterMain.m @@ -350,9 +350,11 @@ function addFailedTestsFlags() if strcmp(filterName, 'imosTiltVelocitySetQC') failedTests_probGood = zeros(size(failedTestsIdx_probGood)); failedTests_probGood(logical(failedTestsIdx_probGood)) = imosQCTest(strcat(filterName, '_probably_good')); + failedTests_probGood = int64(failedTests_probGood); failedTests_bad = zeros(size(failedTestsIdx_bad)); failedTests_bad(logical(failedTestsIdx_bad)) = imosQCTest(strcat(filterName, '_bad_data')); + failedTests_bad = int64(failedTests_bad); if ~isfield(sam.(type{m}){k}, 'flag_tests') % if the flat_tests field does not exist yet sam.(type{m}){k}.flag_tests = failedTests_bad + failedTests_probGood; @@ -364,12 +366,14 @@ function addFailedTestsFlags() failedTests = zeros(size(failedTestsIdx)); failedTests(logical(failedTestsIdx)) = imosQCTest(filterName); + failedTests = int64(failedTests); if ~isfield(sam.(type{m}){k},'flag_tests') % if the flat_tests field does not exist yet sam.(type{m}){k}.flag_tests = failedTests; else - % if it already exists, we sum the previous array with the new one + % if it already exists, we sum the previous array with the new one sam.(type{m}){k}.flag_tests = sam.(type{m}){k}.flag_tests + failedTests; + end end diff --git a/IMOS/imosQCTest.m b/IMOS/imosQCTest.m index c3c1265f0..fc44e7afa 100644 --- a/IMOS/imosQCTest.m +++ b/IMOS/imosQCTest.m @@ -46,7 +46,7 @@ [~, idx] = ismember(testName, sets{1,1}); - value = 2^sets{1,2}(idx); % return the + value = int64(2^sets{1,2}(idx)); % return the catch e diff --git a/NetCDF/exportNetCDF.m b/NetCDF/exportNetCDF.m index fc5467514..95a17822d 100644 --- a/NetCDF/exportNetCDF.m +++ b/NetCDF/exportNetCDF.m @@ -713,7 +713,7 @@ % In the case of this [variable]_failed_tests variable, we're using the % flag_masks rather than flag_values - qcAtts.flag_masks = qcFlags{2}; + qcAtts.flag_masks = int64(qcFlags{2}); % turn descriptions into space separated string qcDescs = cellfun(@(x)(sprintf('%s ', x)), qcDescs, 'UniformOutput', false); From c43afaea5dd2359ba16b8841247d09a1fbad3290 Mon Sep 17 00:00:00 2001 From: lbesnard Date: Tue, 11 Jul 2023 01:15:34 +0000 Subject: [PATCH 6/7] replace variable flag_tests with failed_test string --- AutomaticQC/qcFilterMain.m | 22 +++++++++---------- FlowManager/displayManager.m | 8 +++---- FlowManager/flowManager.m | 10 ++++----- IMOS/imosQCTest.m | 2 +- IMOS/imosQCTests.txt | 6 ++--- NetCDF/exportNetCDF.m | 22 +++++++++---------- ...ibutes.txt => failed_tests_attributes.txt} | 8 +++---- 7 files changed, 38 insertions(+), 40 deletions(-) rename NetCDF/template/{flag_attributes.txt => failed_tests_attributes.txt} (60%) diff --git a/AutomaticQC/qcFilterMain.m b/AutomaticQC/qcFilterMain.m index c75937739..91bd6691f 100644 --- a/AutomaticQC/qcFilterMain.m +++ b/AutomaticQC/qcFilterMain.m @@ -287,7 +287,7 @@ function addFailedTestsFlags() % However, more information is available on % https://github.com/aodn/imos-toolbox/wiki/QCProcedures % - % This function aims to store, in a new variable named "flag_tests" + % This function aims to store, in a new variable named "failed_tests" % the result of why a data point wasn't flagged as good, i.e. which % QC routine was responsible for "rejecting" a data point. % @@ -350,29 +350,29 @@ function addFailedTestsFlags() if strcmp(filterName, 'imosTiltVelocitySetQC') failedTests_probGood = zeros(size(failedTestsIdx_probGood)); failedTests_probGood(logical(failedTestsIdx_probGood)) = imosQCTest(strcat(filterName, '_probably_good')); - failedTests_probGood = int64(failedTests_probGood); + failedTests_probGood = int32(failedTests_probGood); failedTests_bad = zeros(size(failedTestsIdx_bad)); - failedTests_bad(logical(failedTestsIdx_bad)) = imosQCTest(strcat(filterName, '_bad_data')); - failedTests_bad = int64(failedTests_bad); + failedTests_bad(logical(failedTestsIdx_bad)) = imosQCTest(strcat(filterName, '_bad')); + failedTests_bad = int32(failedTests_bad); - if ~isfield(sam.(type{m}){k}, 'flag_tests') % if the flat_tests field does not exist yet - sam.(type{m}){k}.flag_tests = failedTests_bad + failedTests_probGood; + if ~isfield(sam.(type{m}){k}, 'failed_tests') % if the flat_tests field does not exist yet + sam.(type{m}){k}.failed_tests = failedTests_bad + failedTests_probGood; else - sam.(type{m}){k}.flag_tests = sam.(type{m}){k}.flag_tests + failedTests_bad + failedTests_probGood; + sam.(type{m}){k}.failed_tests = sam.(type{m}){k}.failed_tests + failedTests_bad + failedTests_probGood; end else failedTests = zeros(size(failedTestsIdx)); failedTests(logical(failedTestsIdx)) = imosQCTest(filterName); - failedTests = int64(failedTests); + failedTests = int32(failedTests); - if ~isfield(sam.(type{m}){k},'flag_tests') % if the flat_tests field does not exist yet - sam.(type{m}){k}.flag_tests = failedTests; + if ~isfield(sam.(type{m}){k},'failed_tests') % if the flat_tests field does not exist yet + sam.(type{m}){k}.failed_tests = failedTests; else % if it already exists, we sum the previous array with the new one - sam.(type{m}){k}.flag_tests = sam.(type{m}){k}.flag_tests + failedTests; + sam.(type{m}){k}.failed_tests = sam.(type{m}){k}.failed_tests + failedTests; end diff --git a/FlowManager/displayManager.m b/FlowManager/displayManager.m index 2f1e0bb16..fbc9a50fd 100644 --- a/FlowManager/displayManager.m +++ b/FlowManager/displayManager.m @@ -629,14 +629,14 @@ function resetPropQCCallback() sample_data{resetIdx(j)}.variables{i} = rmfield(sample_data{resetIdx(j)}.variables{i}, 'ancillary_comment'); end - % delete the flag_tests previous results so that we don't + % delete the failed_tests previous results so that we don't % sum up again the failed QC routines. The good thing is % that all QC routines a reset anyway, which makes our % life easier - if isfield(sample_data{resetIdx(j)}.variables{i}, 'flag_tests') + if isfield(sample_data{resetIdx(j)}.variables{i}, 'failed_tests') % we don't bother doing heavy calculation. - % We simply delete the "flag_tests" variable - sample_data{resetIdx(j)}.variables{i} = rmfield(sample_data{resetIdx(j)}.variables{i}, 'flag_tests'); + % We simply delete the "failed_tests" variable + sample_data{resetIdx(j)}.variables{i} = rmfield(sample_data{resetIdx(j)}.variables{i}, 'failed_tests'); end end diff --git a/FlowManager/flowManager.m b/FlowManager/flowManager.m index 427e3b90a..f9339108b 100644 --- a/FlowManager/flowManager.m +++ b/FlowManager/flowManager.m @@ -341,13 +341,13 @@ function manualQCRequestCallback(setIdx, varIdx, dataIdx, flag, manualQcComment) % we update the flags values autoQCData{setIdx}.variables{varIdx}.flags(dataIdx) = flag; - % update the flag_tests field containing information on failed tests + % update the failed_tests field containing information on failed tests % with manual QC information - if isfield(autoQCData{setIdx}.variables{varIdx}, 'flag_tests') - autoQCData{setIdx}.variables{varIdx}.flag_tests(dataIdx) = imosQCTest('userManualQC'); % if manual QC done on a point, we overwrite previous failed QC routine information + if isfield(autoQCData{setIdx}.variables{varIdx}, 'failed_tests') + autoQCData{setIdx}.variables{varIdx}.failed_tests(dataIdx) = imosQCTest('userManualQC'); % if manual QC done on a point, we overwrite previous failed QC routine information else - autoQCData{setIdx}.variables{varIdx}.flag_tests(dataIdx) = zeros(size(dataIdx)); % initialise array - autoQCData{setIdx}.variables{varIdx}.flag_tests(dataIdx) = imosQCTest('userManualQC'); + autoQCData{setIdx}.variables{varIdx}.failed_tests(dataIdx) = zeros(size(dataIdx)); % initialise array + autoQCData{setIdx}.variables{varIdx}.failed_tests(dataIdx) = imosQCTest('userManualQC'); end qcSet = str2double(readProperty('toolbox.qc_set')); diff --git a/IMOS/imosQCTest.m b/IMOS/imosQCTest.m index fc44e7afa..f89d2baf0 100644 --- a/IMOS/imosQCTest.m +++ b/IMOS/imosQCTest.m @@ -46,7 +46,7 @@ [~, idx] = ismember(testName, sets{1,1}); - value = int64(2^sets{1,2}(idx)); % return the + value = int32(2^sets{1,2}(idx)); % return the catch e diff --git a/IMOS/imosQCTests.txt b/IMOS/imosQCTests.txt index 96b26da14..a489934c9 100644 --- a/IMOS/imosQCTests.txt +++ b/IMOS/imosQCTests.txt @@ -8,6 +8,7 @@ % % QC routine, positional integer userManualQC, 0 +imosHistoricalManualSetQC, 0 imosCorrMagVelocitySetQC, 1 imosDensityInversionSetQC, 2 imosEchoIntensitySetQC, 3 @@ -15,8 +16,7 @@ imosEchoIntensityVelocitySetQC, 4 imosEchoRangeSetQC, 5 imosErrorVelocitySetQC, 6 imosGlobalRangeQC, 7 -imosHistoricalManualSetQC, 8 -imosHorizontalVelocitySetQC, 9 +imosHorizontalVelocitySetQC, 8 imosImpossibleDateQC, 10 imosImpossibleDepthQC, 11 imosImpossibleLocationSetQC, 12 @@ -30,7 +30,7 @@ imosStationarityQC, 19 imosSurfaceDetectionByDepthSetQC, 20 imosTier2ProfileVelocitySetQC, 21 imosTiltVelocitySetQC_probably_good,22 -imosTiltVelocitySetQC_bad_data, 23 +imosTiltVelocitySetQC_bad, 23 imosTimeSeriesSpikeQC, 24 imosVerticalSpikeQC, 25 imosVerticalVelocityQC, 26 diff --git a/NetCDF/exportNetCDF.m b/NetCDF/exportNetCDF.m index 95a17822d..70bbf8741 100644 --- a/NetCDF/exportNetCDF.m +++ b/NetCDF/exportNetCDF.m @@ -160,7 +160,7 @@ dimAtts = rmfield(dimAtts, {'data', 'name'}); if isfield(dimAtts, 'typeCastFunc'), dimAtts = rmfield(dimAtts, 'typeCastFunc'); end if isfield(dimAtts, 'flags'), dimAtts = rmfield(dimAtts, 'flags'); end - if isfield(dimAtts, 'flag_tests'), dimAtts = rmfield(dimAtts, 'flag_tests'); end % QC variable is not CF for dimensions + if isfield(dimAtts, 'failed_tests'), dimAtts = rmfield(dimAtts, 'failed_tests'); end % QC variable is not CF for dimensions if isfield(dimAtts, 'FillValue_'), dimAtts = rmfield(dimAtts, 'FillValue_'); end % _FillValues for dimensions are not CF dimAtts = rmfield(dimAtts, 'stringlen'); @@ -307,7 +307,7 @@ varAtts = rmfield(varAtts, {'data', 'dimensions', 'stringlen', 'name'}); if isfield(varAtts, 'typeCastFunc'), varAtts = rmfield(varAtts, 'typeCastFunc'); end if isfield(varAtts, 'flags'), varAtts = rmfield(varAtts, 'flags'); end - if isfield(varAtts, 'flag_tests'), varAtts = rmfield(varAtts, 'flag_tests'); end + if isfield(varAtts, 'failed_tests'), varAtts = rmfield(varAtts, 'failed_tests'); end if isfield(varAtts, 'ancillary_comment'), varAtts = rmfield(varAtts, 'ancillary_comment');end if isfield(vars{m}, 'flags') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF @@ -316,7 +316,7 @@ varAtts.ancillary_variables = [varname '_quality_control']; end - if isfield(vars{m}, 'flag_tests') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF + if isfield(vars{m}, 'failed_tests') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF % add the *_failed_tests variable (defined below) % to the ancillary variables attribute varAtts.ancillary_variables = [varAtts.ancillary_variables ,' ' ,varname '_failed_tests']; @@ -339,7 +339,7 @@ end - if isfield(vars{m}, 'flag_tests') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF + if isfield(vars{m}, 'failed_tests') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF % create the ancillary *_failed_tests variable qcflagvid = addQCFlagsVar(... fid, sample_data, m, [qcDimId dids], 'variables', qcFlagType, qcSet, dateFmt, mode); @@ -464,16 +464,16 @@ netcdf.putVar(fid, qcvid, flags); end - if isfield(vars{m}, 'flag_tests') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF - flag_tests = vars{m}.flag_tests; + if isfield(vars{m}, 'failed_tests') && sample_data.meta.level > 0 && ~isempty(vars{m}.dimensions) % ancillary variables for coordinate scalar variable is not CF + failed_tests = vars{m}.failed_tests; qcflagvid = vars{m}.qcflagvid; typeCastFunction = str2func(netcdf3ToMatlabType(qcFlagType)); - flag_tests = typeCastFunction(flag_tests); + failed_tests = typeCastFunction(failed_tests); - if nDims > 1, flag_tests = permute(flag_tests, nDims:-1:1); end + if nDims > 1, failed_tests = permute(failed_tests, nDims:-1:1); end - netcdf.putVar(fid, qcflagvid, flag_tests); + netcdf.putVar(fid, qcflagvid, failed_tests); end end @@ -668,7 +668,7 @@ switch(type) case 'variables' var = sample_data.variables{varIdx}; - template = 'flag'; + template = 'failed_tests'; otherwise error(['invalid type: ' type]); end @@ -713,7 +713,7 @@ % In the case of this [variable]_failed_tests variable, we're using the % flag_masks rather than flag_values - qcAtts.flag_masks = int64(qcFlags{2}); + qcAtts.flag_masks = int32(qcFlags{2}); % turn descriptions into space separated string qcDescs = cellfun(@(x)(sprintf('%s ', x)), qcDescs, 'UniformOutput', false); diff --git a/NetCDF/template/flag_attributes.txt b/NetCDF/template/failed_tests_attributes.txt similarity index 60% rename from NetCDF/template/flag_attributes.txt rename to NetCDF/template/failed_tests_attributes.txt index a46fdaf73..8dcc38e12 100644 --- a/NetCDF/template/flag_attributes.txt +++ b/NetCDF/template/failed_tests_attributes.txt @@ -1,6 +1,6 @@ -S, long_name = quality control failed tests for [mat imosParameters(sample_data.variables{k}.name, 'long_name')] +S, long_name = failed tests for [mat imosParameters(sample_data.variables{k}.name, 'long_name')] S, standard_name = [mat regexprep(strcat(imosParameters(sample_data.variables{k}.name, 'standard_name'), ' status_flag'), '^ .*', '')] -Q, _FillValue = 0 +Q, _FillValue = [mat int32(0)] N, add_offset = N, scale_factor = S, comment = variable to inform on which QC routine(s) was(were) responsible for not flagging a data point as Good. The result value is the sum of flag_masks associated to each QC routine. Every number needs to be split into the sum of power 2 to match witch every flag_masks/flag_meanings combination. @@ -9,6 +9,4 @@ S, references = % these fields are automatically populated upon NetCDF export Q, flag_masks = -S, flag_meanings = -#S, quality_control_global_conventions = Argo reference table 2a (see http://www.cmar.csiro.au/argo/dmqc/user_doc/QC_flags.html), applied on data in position only (between global attributes time_deployment_start and time_deployment_end) -#Q, quality_control_global = +S, flag_meanings = \ No newline at end of file From 5192361f455293ddd2beda6a9b64dec523c6f8e8 Mon Sep 17 00:00:00 2001 From: lbesnard Date: Mon, 24 Jul 2023 05:18:28 +0000 Subject: [PATCH 7/7] improve comment attribute; merge manualQc and historicalQc together --- AutomaticQC/qcFilterMain.m | 6 +++++- IMOS/imosQCTests.txt | 1 - NetCDF/template/failed_tests_attributes.txt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/AutomaticQC/qcFilterMain.m b/AutomaticQC/qcFilterMain.m index 91bd6691f..35b921da4 100644 --- a/AutomaticQC/qcFilterMain.m +++ b/AutomaticQC/qcFilterMain.m @@ -365,7 +365,11 @@ function addFailedTestsFlags() else failedTests = zeros(size(failedTestsIdx)); - failedTests(logical(failedTestsIdx)) = imosQCTest(filterName); + if strcmp(filterName, 'imosHistoricalManualSetQC') + failedTests(logical(failedTestsIdx)) = imosQCTest('userManualQC'); % we replace imosHistoricalManualSetQC with userManualQC so that we only have 1 flag_value/flag_meaning combo in the NetCDF for something very similar in the end + else + failedTests(logical(failedTestsIdx)) = imosQCTest(filterName); + end failedTests = int32(failedTests); if ~isfield(sam.(type{m}){k},'failed_tests') % if the flat_tests field does not exist yet diff --git a/IMOS/imosQCTests.txt b/IMOS/imosQCTests.txt index a489934c9..38e8d2ccc 100644 --- a/IMOS/imosQCTests.txt +++ b/IMOS/imosQCTests.txt @@ -8,7 +8,6 @@ % % QC routine, positional integer userManualQC, 0 -imosHistoricalManualSetQC, 0 imosCorrMagVelocitySetQC, 1 imosDensityInversionSetQC, 2 imosEchoIntensitySetQC, 3 diff --git a/NetCDF/template/failed_tests_attributes.txt b/NetCDF/template/failed_tests_attributes.txt index 8dcc38e12..dafa47f25 100644 --- a/NetCDF/template/failed_tests_attributes.txt +++ b/NetCDF/template/failed_tests_attributes.txt @@ -3,7 +3,7 @@ S, standard_name = [mat regexprep(strcat(imosParameters(s Q, _FillValue = [mat int32(0)] N, add_offset = N, scale_factor = -S, comment = variable to inform on which QC routine(s) was(were) responsible for not flagging a data point as Good. The result value is the sum of flag_masks associated to each QC routine. Every number needs to be split into the sum of power 2 to match witch every flag_masks/flag_meanings combination. +S, comment = The flag_masks attribute is the same type as the variable to which it is attached, and contains a list of values matching unique bit fields. The flag_meanings attribute is defined as above, one for each flag_masks value. A flagged condition is identified by performing a bitwise AND of the variable value and each flag_masks value; a non-zero result indicates a true condition. Thus, any or all of the flagged conditions may be true, depending on the variable bit settings. S, history = S, references =