Merge pull request #471 from aodn/2.5.32

2.5.32

AutomaticQC/CTDSurfaceSoakQC.m

@@ -1,4 +1,4 @@
-function [data flags paramsLog] = CTDSurfaceSoakQC( sample_data, data, k, type, auto )
+function [data, flags, paramsLog] = CTDSurfaceSoakQC( sample_data, data, k, type, auto )
 %IMOSINOUTWATERQC Flags samples which were taken before and after the instrument was placed
 % in the water.
 %
@@ -92,7 +92,7 @@
 
 % only concerned here with pumped sensors or variables derived from pumped
 % observations
-pumpedVar = {'TEMP', 'CNDC', 'DOX1', 'DOX2', 'PSAL', 'DENS'};
+pumpedVar = {'TEMP', 'CNDC', 'DOX', 'DOXY', 'DOX1', 'DOX2', 'DOXS', 'PSAL', 'DENS'};
 ignoreVar = {'TIME', 'PROFILE', 'DIRECTION', 'LATITUDE', 'LONGITUDE', 'BOT_DEPTH', 'ETIME'};
 
 qcSet = str2double(readProperty('toolbox.qc_set'));
@@ -107,16 +107,16 @@
     flags = ones(lenData, 1, 'int8')*rawFlag;
     if any(iPumped)
         % initially all data is raw
-        switch find(iPumped);
-            case 1 % temperature
+        switch pumpedVar{iPumped};
+            case 'TEMP' % temperature
                 if ~isempty(iTSS)
                     flags = sample_data.variables{iTSS}.data;
                 end
-            case {2, 5, 6} % dependent on conductivity
+            case {'CNDC', 'PSAL', 'DENS'} % dependent on conductivity
                 if ~isempty(iCSS)
                     flags = sample_data.variables{iCSS}.data;
                 end
-            case {3, 4} % dependent on oxygen
+            case {'DOX', 'DOXY', 'DOX1', 'DOX2', 'DOXS'} % dependent on oxygen
                 if ~isempty(iOSS)
                     flags = sample_data.variables{iOSS}.data;
                 end

AutomaticQC/imosGlobalRangeQC.txt

@@ -6,7 +6,9 @@ DEPTH
 CPHL
 CHLU
 CHLF
+DOX
 DOXY
+DOXS
 DOX1
 DOX2
 UCUR

AutomaticQC/imosRateOfChangeQC.txt

@@ -3,7 +3,9 @@ PSAL = 2*stdDev
 PRES = 2*stdDev
 PRES_REL = 2*stdDev
 DEPTH = 2*stdDev
+DOX = 2*stdDev
 DOXY = 2*stdDev
+DOXS = 2*stdDev
 DOX1 = 2*stdDev
 DOX2 = 2*stdDev
 UCUR = 2*stdDev

AutomaticQC/imosVerticalSpikeQC.txt

@@ -6,6 +6,8 @@ DEPTH = 3
 CPHL = PABIM
 CHLU = PABIM
 CHLF = PABIM
+DOX = PABIM
 DOXY = PABIM
+DOXS = PABIM
 DOX1 = PABIM
 DOX2 = PABIM

DDB/executeQuery.m

@@ -0,0 +1,64 @@
+function result = executeQuery( table, field, value)
+%EXECUTEQUERY Wrapper around Java DDB interface, allowing queries to the DDB
+% pass query to DDB database or CSV file Query
+%
+% Inputs:
+%   table  - The table to query.
+%
+%   field  - Name of field on which to restrict query. If passed in as an 
+%            empty matrix, the entire table is returned. 
+%
+%   value  - Value of field on which to restrict query. 
+%
+% Outputs:
+%   result - Vector of structs.
+%
+% See Also executeDDBQuery and executeCSVQuery
+%
+%
+% Author:       Peter Jansen <[email protected]>
+%
+
+%
+% Copyright (c) 2016, Australian Ocean Data Network (AODN) and Integrated 
+% Marine Observing System (IMOS).
+% All rights reserved.
+% 
+% Redistribution and use in source and binary forms, with or without 
+% modification, are permitted provided that the following conditions are met:
+% 
+%     * Redistributions of source code must retain the above copyright notice, 
+%       this list of conditions and the following disclaimer.
+%     * Redistributions in binary form must reproduce the above copyright 
+%       notice, this list of conditions and the following disclaimer in the 
+%       documentation and/or other materials provided with the distribution.
+%     * Neither the name of the AODN/IMOS nor the names of its contributors 
+%       may be used to endorse or promote products derived from this software 
+%       without specific prior written permission.
+% 
+% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+% AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+% IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
+% ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
+% LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+% CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+% SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+% INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
+% CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+% ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+% POSSIBILITY OF SUCH DAMAGE.
+%
+
+isDatabaseCSV = false;
+ddb = readProperty('toolbox.ddb');
+if isdir(ddb)
+    isDatabaseCSV = true;
+end
+if isDatabaseCSV
+    result = executeCSVQuery(table, field,   value);
+else
+    result = executeDDBQuery(table, field,   value);
+end
+
+end
+

FlowManager/autoIMOSToolbox.m

@@ -158,19 +158,12 @@ function autoIMOSToolbox(toolboxVersion, fieldTrip, dataDir, ppChain, qcChain, e
 [~, sourceFolder] = fileparts(dataDir);
 fprintf('%s\n', ['Processing field trip ' fieldTrip ' from folder ' sourceFolder]);
 
-%check for CSV file import
-isCSV = false;
-ddb = readProperty('toolbox.ddb');
-if isdir(ddb)
-    isCSV = true;
-end
-
 % get infos from current field trip
 switch mode
     case 'profile'
-        [~, deps, sits, dataDir] = getCTDs(true, isCSV);
+        [~, deps, sits, dataDir] = getCTDs(true);
     case 'timeSeries'
-        [~, deps, sits, dataDir] = getDeployments(true, isCSV);
+        [~, deps, sits, dataDir] = getDeployments(true);
 end
 
 if isempty(deps)

FlowManager/importManager.m

@@ -85,7 +85,7 @@
   rawFiles    = {};
 
   if ~isempty(ddb) || (~isempty(driver) && ~isempty(connection))
-      [structs, rawFiles] = ddbImport(auto, iMooring, ddb, mode);
+      [structs, rawFiles] = ddbImport(auto, iMooring, mode);
   else
       if auto, error('manual import cannot be automated without deployment database'); end
       [structs, rawFiles] = manualImport(mode);
@@ -192,7 +192,7 @@
   end
 end
 
-function [sample_data, rawFiles] = ddbImport(auto, iMooring, ddb, mode)
+function [sample_data, rawFiles] = ddbImport(auto, iMooring, mode)
 %DDBIMPORT Imports data sets using metadata retrieved from a deployment
 % database.
 %
@@ -202,8 +202,6 @@
 %   iMooring    - Optional logical(comes with auto == true). Contains
 %                 the logical indices to extract only the deployments 
 %                 from one mooring set of deployments.
-%   ddb         - deployment database string attribute from
-%                 toolboxProperties.txt
 %   mode        - toolbox execution mode.
 %
 % Outputs:
@@ -216,18 +214,12 @@
   rawFiles    = {};
   allFiles    = {};
 
-  %check for CSV file import
-  isCSV = false;
-  if isdir(ddb)
-      isCSV = true;
-  end
-
   while true
       switch mode
           case 'profile'
-              [fieldTrip, deps, sits, dataDir] = getCTDs(auto, isCSV); % one entry is one CTD profile instrument file
+              [fieldTrip, deps, sits, dataDir] = getCTDs(auto); % one entry is one CTD profile instrument file
           case 'timeSeries'
-              [fieldTrip, deps, sits, dataDir] = getDeployments(auto, isCSV); % one entry is one moored instrument file
+              [fieldTrip, deps, sits, dataDir] = getDeployments(auto); % one entry is one moored instrument file
       end
 
       if isempty(fieldTrip), return; end
@@ -341,7 +333,7 @@
     % display status dialog to highlight any discrepancies (file not found
     % for a deployment, more than one file found for a deployment)
     if ~auto
-      [deps, allFiles] = dataFileStatusDialog(deps, allFiles, isCSV);
+      [deps, allFiles] = dataFileStatusDialog(deps, allFiles);
 
       % user cancelled file dialog
       if isempty(deps), continue; end
@@ -379,7 +371,7 @@
           waitbar(k / length(deps), progress, fileDisplay);
       end
       % import data
-      sample_data{end+1} = parse(deps(k), allFiles{k}, parsers, noParserPrompt, mode, isCSV);
+      sample_data{end+1} = parse(deps(k), allFiles{k}, parsers, noParserPrompt, mode);
       rawFiles{   end+1} = allFiles{k};
 
       if iscell(sample_data{end})
@@ -436,7 +428,7 @@
   % close progress dialog
   if ~auto, close(progress); end
 
-  function sam = parse(deployment, files, parsers, noParserPrompt, mode, isCSV)
+  function sam = parse(deployment, files, parsers, noParserPrompt, mode)
   %PARSE Parses a raw data file, returns a sample_data struct.
   %
   % Inputs:
@@ -452,7 +444,7 @@
   %   sam        - Struct containing sample data.
 
     % get the appropriate parser function
-    parser = getParserFunc(deployment, parsers, noParserPrompt, isCSV);
+    parser = getParserFunc(deployment, parsers, noParserPrompt);
     if isnumeric(parser)
       error(['no parser found for instrument ' deployment.InstrumentID]); 
     end
@@ -461,7 +453,7 @@
     sam = parser(files, mode);
   end
 
-  function parser = getParserFunc(deployment, parsers, noParserPrompt, isCSV)
+  function parser = getParserFunc(deployment, parsers, noParserPrompt)
   %GETPARSERFUNC Searches for a parser function which is able to parse data 
   % for the given deployment.
   %
@@ -477,12 +469,7 @@
   %                function wasn't found.
   %
 
-    if isCSV
-      executeQueryFunc = @executeCSVQuery;
-    else
-      executeQueryFunc = @executeDDBQuery;
-    end
-    instrument = executeQueryFunc('Instruments', 'InstrumentID', deployment.InstrumentID);
+    instrument = executeQuery('Instruments', 'InstrumentID', deployment.InstrumentID);
 
     % there should be exactly one instrument 
     if length(instrument) ~= 1

GUI/dataFileStatusDialog.m

@@ -1,4 +1,4 @@
-function [deployments files] = dataFileStatusDialog( deployments, files, isCSV )
+function [deployments files] = dataFileStatusDialog( deployments, files)
 %DATAFILESTATUSDIALOG Displays a list of deployments, and raw files for
 % each, allowing the user to verify/change which raw data files map to which
 % deployment.
@@ -16,7 +16,6 @@
 %
 %   files       - Cell array of cell arrays of strings, each containing the
 %                 list of file names corresponding to each deployment.
-%   isCSV       - Logical for whether we use a ddb in .csv format or not.
 %
 % Outputs:
 %   deployments - Same as input, potentially with some deployments removed.
@@ -56,11 +55,10 @@
 % ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 % POSSIBILITY OF SUCH DAMAGE.
 %
-  narginchk(3,3);
+  narginchk(2, 2);
 
   if ~isstruct(deployments), error('deployments must be a struct'); end
   if ~iscell  (files),       error('files must be a cell array');   end
-  if ~islogical(isCSV),      error('isCSV must be a logical');   end
 
   % copy the inputs so we can rollback if the user cancels
   origDeployments = deployments;
@@ -69,7 +67,7 @@
   % get the toolbox execution mode
   mode = readProperty('toolbox.mode');
 
-  deploymentDescs = genDepDescriptions(deployments, files, isCSV);
+  deploymentDescs = genDepDescriptions(deployments, files);
 
   % Sort data_samples
   %
@@ -297,7 +295,7 @@ function fileAddCallback(source,ev)
 
   %% Description generation
 
-  function descs = genDepDescriptions(deployments, files, isCSV)
+  function descs = genDepDescriptions(deployments, files)
   %GENDEPDESCRIPTIONS Creates a cell array of descriptions of the given
   % deployments, suitable for use in the deployments list.
   %
@@ -324,12 +322,7 @@ function fileAddCallback(source,ev)
 
 
       % get some site information if it exists
-      if isCSV
-          executeQueryFunc = @executeCSVQuery;
-      else
-          executeQueryFunc = @executeDDBQuery;
-      end
-      site = executeQueryFunc('Sites', 'Site', dep.Site);
+      site = executeQuery('Sites', 'Site', dep.Site);
 
       if ~isempty(site)
 

GUI/startDialog.m

@@ -1,4 +1,4 @@
-function [fieldTrip dataDir] = startDialog(mode, isCSV)
+function [fieldTrip dataDir] = startDialog(mode)
 %STARTDIALOG Displays a dialog prompting the user to select a Field Trip 
 % and a directory which contains raw data files.
 %
@@ -11,7 +11,6 @@
 % Input:
 %
 %   mode - String, toolox execution mode.
-%   isCSV - optional boolean (default = false). True if importing from csv files.
 %   
 % Outputs:
 %
@@ -53,11 +52,7 @@
 % ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 % POSSIBILITY OF SUCH DAMAGE.
 %
-  narginchk(1,2);
-
-  if nargin == 1
-      isCSV = false;
-  end
+  narginchk(1, 1);
 
   dateFmt     = readProperty('toolbox.dateFormat');
 
@@ -74,12 +69,7 @@
   if isnan(highDate),       highDate = now_utc; end
 
   % retrieve all field trip IDs; they are displayed as a drop down menu
-  if isCSV
-      executeQueryFunc = @executeCSVQuery;
-  else
-      executeQueryFunc = @executeDDBQuery;
-  end
-  fieldTrips = executeQueryFunc('FieldTrip', [], []);
+  fieldTrips = executeQuery('FieldTrip', [], []);
 
   if isempty(fieldTrips), error('No field trip entries in DDB'); end