diff --git a/DDB/executeCSVQuery.m b/DDB/executeCSVQuery.m index f2051a5b4..3d04fdde3 100644 --- a/DDB/executeCSVQuery.m +++ b/DDB/executeCSVQuery.m @@ -79,11 +79,8 @@ end end - %get location of csv files: - dirnm = readProperty('toolbox.ddb.connection'); - if isempty(dirnm) - dirnm = pwd; - end + %get location of csv files + dirnm = readProperty('toolbox.ddb'); % complete the file name: file = fullfile(dirnm, [file '.csv']); diff --git a/FlowManager/autoIMOSToolbox.m b/FlowManager/autoIMOSToolbox.m index aab53c3a0..7189ef7c4 100644 --- a/FlowManager/autoIMOSToolbox.m +++ b/FlowManager/autoIMOSToolbox.m @@ -158,12 +158,19 @@ function autoIMOSToolbox(toolboxVersion, fieldTrip, dataDir, ppChain, qcChain, e [~, sourceFolder] = fileparts(dataDir); fprintf('%s\n', ['Processing field trip ' fieldTrip ' from folder ' sourceFolder]); +%check for CSV file import +isCSV = false; +ddb = readProperty('toolbox.ddb'); +if isdir(ddb) + isCSV = true; +end + % get infos from current field trip switch mode case 'profile' - [~, deps, sits, dataDir] = getCTDs(true); + [~, deps, sits, dataDir] = getCTDs(true, isCSV); case 'timeSeries' - [~, deps, sits, dataDir] = getDeployments(true); + [~, deps, sits, dataDir] = getDeployments(true, isCSV); end if isempty(deps) diff --git a/FlowManager/importManager.m b/FlowManager/importManager.m index 5f8804da8..156a54563 100644 --- a/FlowManager/importManager.m +++ b/FlowManager/importManager.m @@ -217,12 +217,18 @@ rawFiles = {}; allFiles = {}; + %check for CSV file import + isCSV = false; + if isdir(ddb) + isCSV = true; + end + while true switch mode case 'profile' - [fieldTrip deps sits dataDir] = getCTDs(auto); % one entry is one CTD profile instrument file + [fieldTrip deps sits dataDir] = getCTDs(auto, isCSV); % one entry is one CTD profile instrument file case 'timeSeries' - [fieldTrip deps sits dataDir] = getDeployments(auto); % one entry is one moored instrument file + [fieldTrip deps sits dataDir] = getDeployments(auto, isCSV); % one entry is one moored instrument file end if isempty(fieldTrip), return; end @@ -324,7 +330,7 @@ % display status dialog to highlight any discrepancies (file not found % for a deployment, more than one file found for a deployment) if ~auto - [deps allFiles] = dataFileStatusDialog(deps, allFiles); + [deps allFiles] = dataFileStatusDialog(deps, allFiles, isCSV); % user cancelled file dialog if isempty(deps), continue; end @@ -362,7 +368,7 @@ waitbar(k / length(deps), progress, fileDisplay); end % import data - sample_data{end+1} = parse(deps(k), allFiles{k}, parsers, noParserPrompt, mode, ddb); + sample_data{end+1} = parse(deps(k), allFiles{k}, parsers, noParserPrompt, mode, isCSV); rawFiles{ end+1} = allFiles{k}; if iscell(sample_data{end}) @@ -419,7 +425,7 @@ % close progress dialog if ~auto, close(progress); end - function sam = parse(deployment, files, parsers, noParserPrompt, mode, ddb) + function sam = parse(deployment, files, parsers, noParserPrompt, mode, isCSV) %PARSE Parses a raw data file, returns a sample_data struct. % % Inputs: @@ -435,7 +441,7 @@ % sam - Struct containing sample data. % get the appropriate parser function - parser = getParserFunc(deployment, parsers, noParserPrompt, ddb); + parser = getParserFunc(deployment, parsers, noParserPrompt, isCSV); if isnumeric(parser) error(['no parser found for instrument ' deployment.InstrumentID]); end @@ -444,7 +450,7 @@ sam = parser(files, mode); end - function parser = getParserFunc(deployment, parsers, noParserPrompt, ddb) + function parser = getParserFunc(deployment, parsers, noParserPrompt, isCSV) %GETPARSERFUNC Searches for a parser function which is able to parse data % for the given deployment. % @@ -459,13 +465,13 @@ % parser - Function handle to the parser function, or 0 if a parser % function wasn't found. % - if strcmp('csv',ddb) - instrument = executeCSVQuery(... - 'Instruments', 'InstrumentID', deployment.InstrumentID); - else - instrument = executeDDBQuery(... - 'Instruments', 'InstrumentID', deployment.InstrumentID); - end + + if isCSV + executeQueryFunc = @executeCSVQuery; + else + executeQueryFunc = @executeDDBQuery; + end + instrument = executeQueryFunc('Instruments', 'InstrumentID', deployment.InstrumentID); % there should be exactly one instrument if length(instrument) ~= 1 @@ -501,4 +507,4 @@ % get the parser function handle parser = getParser(parser); end -end +end \ No newline at end of file diff --git a/GUI/dataFileStatusDialog.m b/GUI/dataFileStatusDialog.m index 7a2bc7c5a..7dd740e74 100644 --- a/GUI/dataFileStatusDialog.m +++ b/GUI/dataFileStatusDialog.m @@ -1,4 +1,4 @@ -function [deployments files] = dataFileStatusDialog( deployments, files ) +function [deployments files] = dataFileStatusDialog( deployments, files, isCSV ) %DATAFILESTATUSDIALOG Displays a list of deployments, and raw files for % each, allowing the user to verify/change which raw data files map to which % deployment. @@ -16,6 +16,7 @@ % % files - Cell array of cell arrays of strings, each containing the % list of file names corresponding to each deployment. +% isCSV - Logical for whether we use a ddb in .csv format or not. % % Outputs: % deployments - Same as input, potentially with some deployments removed. @@ -55,10 +56,11 @@ % ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE % POSSIBILITY OF SUCH DAMAGE. % - narginchk(2,2); + narginchk(3,3); if ~isstruct(deployments), error('deployments must be a struct'); end if ~iscell (files), error('files must be a cell array'); end + if ~islogical(isCSV), error('isCSV must be a logical'); end % copy the inputs so we can rollback if the user cancels origDeployments = deployments; @@ -67,7 +69,7 @@ % get the toolbox execution mode mode = readProperty('toolbox.mode'); - deploymentDescs = genDepDescriptions(deployments, files); + deploymentDescs = genDepDescriptions(deployments, files, isCSV); % Sort data_samples % @@ -307,12 +309,10 @@ function fileAddCallback(source,ev) %% Description generation - function descs = genDepDescriptions(deployments, files) + function descs = genDepDescriptions(deployments, files, isCSV) %GENDEPDESCRIPTIONS Creates a cell array of descriptions of the given % deployments, suitable for use in the deployments list. % - - ddb = readProperty('toolbox.ddb'); % set values for lists descs = {}; @@ -336,11 +336,12 @@ function fileAddCallback(source,ev) % get some site information if it exists - if strcmp('csv', ddb) - site = executeCSVQuery('Sites', 'Site', dep.Site); + if isCSV + executeQueryFunc = @executeCSVQuery; else - site = executeDDBQuery('Sites', 'Site', dep.Site); + executeQueryFunc = @executeDDBQuery; end + site = executeQueryFunc('Sites', 'Site', dep.Site); if ~isempty(site) diff --git a/NetCDF/makeNetCDFCompliant.m b/NetCDF/makeNetCDFCompliant.m index 4f9ac7f4c..a5ecd11a4 100644 --- a/NetCDF/makeNetCDFCompliant.m +++ b/NetCDF/makeNetCDFCompliant.m @@ -142,10 +142,16 @@ sample_data.dimensions{k} = mergeAtts(sample_data.dimensions{k}, dimAtts); end + %check for CSV file import + isCSV = false; + ddb = readProperty('toolbox.ddb'); + if isdir(ddb) + isCSV = true; + end + % % variables % - ddb = readProperty('toolbox.ddb'); for k = 1:length(sample_data.variables) @@ -167,11 +173,11 @@ if isfield(sample_data.meta, 'deployment') iTime = getVar(sample_data.dimensions, 'TIME'); sample_data.variables{k}.sensor_serial_number = ... - getSensorSerialNumber(sample_data.variables{k}.name, sample_data.meta.deployment.InstrumentID, sample_data.dimensions{iTime}.data(1), ddb); + getSensorSerialNumber(sample_data.variables{k}.name, sample_data.meta.deployment.InstrumentID, sample_data.dimensions{iTime}.data(1), isCSV); elseif isfield(sample_data.meta, 'profile') iTime = getVar(sample_data.variables, 'TIME'); sample_data.variables{k}.sensor_serial_number = ... - getSensorSerialNumber(sample_data.variables{k}.name, sample_data.meta.profile.InstrumentID, sample_data.variables{iTime}.data(1), ddb); + getSensorSerialNumber(sample_data.variables{k}.name, sample_data.meta.profile.InstrumentID, sample_data.variables{iTime}.data(1), isCSV); end end end @@ -194,7 +200,7 @@ end end -function target = getSensorSerialNumber ( IMOSParam, InstrumentID, timeFirstSample, ddb ) +function target = getSensorSerialNumber ( IMOSParam, InstrumentID, timeFirstSample, isCSV ) %GETSENSORSERIALNUMBER gets the sensor serial number associated to an IMOS %paramter for a given deployment ID % @@ -202,11 +208,12 @@ target = ''; % query the ddb for all sensor config related to this instrument ID -if strcmp('csv',ddb) - InstrumentSensorConfig = executeCSVQuery('InstrumentSensorConfig', 'InstrumentID', InstrumentID); +if isCSV + executeQueryFunc = @executeCSVQuery; else - InstrumentSensorConfig = executeDDBQuery('InstrumentSensorConfig', 'InstrumentID', InstrumentID); + executeQueryFunc = @executeDDBQuery; end +InstrumentSensorConfig = executeQueryFunc('InstrumentSensorConfig', 'InstrumentID', InstrumentID); lenConfig = length(InstrumentSensorConfig); % only consider relevant config based on timeFirstSample for i=1:lenConfig diff --git a/Parser/convertSBEcnvVar.m b/Parser/convertSBEcnvVar.m index 69d3b99b7..07adbf8df 100644 --- a/Parser/convertSBEcnvVar.m +++ b/Parser/convertSBEcnvVar.m @@ -213,8 +213,15 @@ % A/D counts to volts (sensor_analog_output 0 to 7) case {'v0', 'v1', 'v2', 'v3', 'v4', 'v5', 'v6', 'v7'} origName = name; - name = ['volt_', getVoltageName(origName, instHeader)]; - comment = getVoltageComment(origName, procHeader); + name = getVoltageName(origName, instHeader); + if ~strcmpi(name, 'not_assigned') + name = ['volt_', getVoltageName(origName, instHeader)]; + comment = getVoltageComment(origName, procHeader); + else + name = ''; + data = []; + comment = ''; + end case 'f1' if strcmpi(mode, 'profile') @@ -317,4 +324,6 @@ end end +name = strrep(name, ' ', '_'); + end \ No newline at end of file diff --git a/Parser/readSBE19cnv.m b/Parser/readSBE19cnv.m index dc1e753bf..cf8bd5169 100644 --- a/Parser/readSBE19cnv.m +++ b/Parser/readSBE19cnv.m @@ -80,7 +80,7 @@ % we deliberately overwrite it assuming the last version is the % most relevant - otherwise + case 'timeSeries' % if the same parameter appears multiple times, % don't overwrite it in the data struct - append % a number to the end of the variable name, as diff --git a/Preprocessing/timeDriftPP.m b/Preprocessing/timeDriftPP.m index da4c26c4a..560d45237 100644 --- a/Preprocessing/timeDriftPP.m +++ b/Preprocessing/timeDriftPP.m @@ -70,7 +70,12 @@ % auto logical in input to enable running under batch processing if nargin<3, auto=false; end +%check for CSV file import +isCSV = false; ddb = readProperty('toolbox.ddb'); +if isdir(ddb) + isCSV = true; +end descs = {}; nSample = length(sample_data); @@ -84,13 +89,10 @@ descs{k} = genSampleDataDesc(sample_data{k}); %check to see if the offsets are available already from the ddb - if strcmp('csv',ddb) + if isCSV if isfield(sample_data{k}.meta.deployment,'StartOffset') startOffsets(k) = sample_data{k}.meta.deployment.StartOffset; end - end - - if strcmp('csv',ddb) if isfield(sample_data{k}.meta.deployment,'EndOffset') endOffsets(k) = sample_data{k}.meta.deployment.EndOffset; end diff --git a/Util/getCTDs.m b/Util/getCTDs.m index 804fa567f..0645f8dc9 100644 --- a/Util/getCTDs.m +++ b/Util/getCTDs.m @@ -1,4 +1,4 @@ -function [fieldTrip ctds sites dataDir] = getCTDs(auto) +function [fieldTrip ctds sites dataDir] = getCTDs(auto, isCSV) %GETCTDS Prompts the user for a field trip ID and data directory. % Retrieves and returns the field trip, all ctds from the DDB that % are related to the field trip, and the selected data directory. @@ -7,6 +7,7 @@ % auto - if true, the user is not prompted to select a field % trip/directory; the values in toolboxProperties are % used. +% isCSV - If true, look for csv files rather than using database. % % Outputs: % fieldTrip - field trip struct - the field trip selected by the user. @@ -54,7 +55,7 @@ % prompt the user to select a field trip and % directory which contains raw data files if ~auto - [fieldTrip dataDir] = startDialog('profile'); + [fieldTrip dataDir] = startDialog('profile', isCSV); % if automatic, just get the defaults from toolboxProperties.txt else dataDir = readProperty('startDialog.dataDir.profile'); @@ -72,18 +73,23 @@ fId = fieldTrip.FieldTripID; % query the ddb for all ctds related to this field trip -ctds = executeDDBQuery('CTDData', 'FieldTrip', fId); +if isCSV + executeQueryFunc = @executeCSVQuery; +else + executeQueryFunc = @executeDDBQuery; +end +ctds = executeQueryFunc('CTDData', 'FieldTrip', fId); % query the ddb for all sites related to these ctds lenDep = length(ctds); for i=1:lenDep if i==1 - tempVal = executeDDBQuery('Sites', 'Site', ctds(i).Site); + tempVal = executeQueryFunc('Sites', 'Site', ctds(i).Site); % A CTDData doesn't necessarily has an associated site, % CTDData already contains some site information if ~isempty(tempVal), sites = tempVal; end else - tempVal = executeDDBQuery('Sites', 'Site', ctds(i).Site); + tempVal = executeQueryFunc('Sites', 'Site', ctds(i).Site); if ~isempty(tempVal), sites(i) = tempVal; end end end \ No newline at end of file diff --git a/Util/getDeployments.m b/Util/getDeployments.m index dfabee25e..d668826f7 100644 --- a/Util/getDeployments.m +++ b/Util/getDeployments.m @@ -7,8 +7,7 @@ % auto - if true, the user is not prompted to select a field % trip/directory; the values in toolboxProperties are % used. -% isCSV - optional [false = default]. If true, look for csv files -% rather than using database +% isCSV - If true, look for csv files rather than using database. % % Outputs: % fieldTrip - field trip struct - the field trip selected by the user. @@ -51,21 +50,11 @@ % ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE % POSSIBILITY OF SUCH DAMAGE. % -if nargin == 1 - isCSV = false; -end +narginchk(2,2); deployments = struct; sites = struct; -%check for CSV file import: -ddb = readProperty('toolbox.ddb'); -if strcmp(ddb,'csv') - isCSV = true; -else - isCSV = false; -end - % prompt the user to select a field trip and % directory which contains raw data files if ~auto @@ -86,13 +75,12 @@ fId = fieldTrip.FieldTripID; +% query the ddb/csv file for all deployments related to this field trip if isCSV executeQueryFunc = @executeCSVQuery; else executeQueryFunc = @executeDDBQuery; end - -% query the ddb/csv file for all deployments related to this field trip deployments = executeQueryFunc('DeploymentData', 'EndFieldTrip', fId); % query the ddb for all sites related to these deployments diff --git a/Util/parseAttributeValue.m b/Util/parseAttributeValue.m index 7716dd6a0..b6c9f96e5 100644 --- a/Util/parseAttributeValue.m +++ b/Util/parseAttributeValue.m @@ -167,6 +167,12 @@ connection = readProperty('toolbox.ddb.connection'); if isempty(ddb) && (isempty(driver) || isempty(connection)), return; end + %check for CSV file import + isCSV = false; + if isdir(ddb) + isCSV = true; + end + % get the relevant deployment/CTD cast if isfield(sample_data.meta, 'profile') deployment = sample_data.meta.profile; @@ -208,11 +214,13 @@ % a foreign key, so our only choice is to give up if isempty(field_value), return; end - if strcmp('csv',ddb) - result = executeCSVQuery(related_table, related_pkey, field_value); + if isCSV + executeQueryFunc = @executeCSVQuery; else - result = executeDDBQuery(related_table, related_pkey, field_value); + executeQueryFunc = @executeDDBQuery; end + result = executeQueryFunc(related_table, related_pkey, field_value); + if length(result) ~= 1, return; end value = result.(related_field); diff --git a/imosToolbox.m b/imosToolbox.m index 3a12f6bdb..db0aba0b9 100644 --- a/imosToolbox.m +++ b/imosToolbox.m @@ -73,7 +73,7 @@ function imosToolbox(auto, varargin) end % Set current toolbox version -toolboxVersion = ['2.5.10 - ' computer]; +toolboxVersion = ['2.5.11 - ' computer]; switch auto case 'auto', autoIMOSToolbox(toolboxVersion, varargin{:}); diff --git a/imosToolbox_Linux64.bin b/imosToolbox_Linux64.bin index 0397f9e15..7bec90227 100755 Binary files a/imosToolbox_Linux64.bin and b/imosToolbox_Linux64.bin differ diff --git a/imosToolbox_Win32.exe b/imosToolbox_Win32.exe index 5fc23e930..cba5d984c 100644 Binary files a/imosToolbox_Win32.exe and b/imosToolbox_Win32.exe differ diff --git a/imosToolbox_Win64.exe b/imosToolbox_Win64.exe index 56959b9b6..0f62eb18c 100644 Binary files a/imosToolbox_Win64.exe and b/imosToolbox_Win64.exe differ