Skip to content

Commit

Permalink
Merge pull request #338 from aodn/2.5.5
Browse files Browse the repository at this point in the history
2.5.5
  • Loading branch information
ggalibert committed Apr 15, 2016
2 parents 6baa8f3 + ddfd802 commit af92a16
Show file tree
Hide file tree
Showing 45 changed files with 3,832 additions and 358 deletions.
184 changes: 184 additions & 0 deletions DDB/executeCSVQuery.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
function result = executeCSVQuery( file, field, value)
%EXECUTECSVQUERY Alternative to executeDDBQuery, uses CSV files.
%
% Uses multiple csv files to obtain information equivalent to
% executeDDBQuery.
%
% Inputs:
% file - The csv file to query.
%
% field - Name of field to search for value. If passed in as an
% empty matrix, the entire table is returned.
%
% value - Value of field on which to restrict query.
%
% Outputs:
% result - Vector of structs, each entry representing one tuple of the
% query result.
%
% Author: Rebecca Cowley <[email protected]>

%
% Copyright (c) 2009, eMarine Information Infrastructure (eMII) and Integrated
% Marine Observing System (IMOS).
% All rights reserved.
%
% Redistribution and use in source and binary forms, with or without
% modification, are permitted provided that the following conditions are met:
%
% * Redistributions of source code must retain the above copyright notice,
% this list of conditions and the following disclaimer.
% * Redistributions in binary form must reproduce the above copyright
% notice, this list of conditions and the following disclaimer in the
% documentation and/or other materials provided with the distribution.
% * Neither the name of the eMII/IMOS nor the names of its contributors
% may be used to endorse or promote products derived from this software
% without specific prior written permission.
%
% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
% AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
% IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
% ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
% LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
% CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
% SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
% INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
% CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
% ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
% POSSIBILITY OF SUCH DAMAGE.
%
narginchk(3,3);

if ~ischar(file), error('file must be a string'); end
if ~isempty(field) ...
&& ~ischar(field), error('field must be a string'); end

% in order to reduce the number of queries to the ddb (slow, we store
% each result in a global structure so that we don't perform twice the
% same query.
persistent csvStruct;
if isempty(csvStruct)
csvStruct = struct;
csvStruct.table = {};
csvStruct.field = {};
csvStruct.value = {};
csvStruct.result = {};
else
iTable = strcmpi(file, csvStruct.table);
if any(iTable)
iField = strcmpi(field, csvStruct.field);
iField = iField & iTable;
if any(iField)
iValue = strcmpi(csvStruct.value, num2str(value));
iValue = iValue & iField;
if any(iValue)
result = csvStruct.result{iValue};
return;
end
end
end
end

%get location of csv files:
dirnm = readProperty('toolbox.ddb.connection');
if isempty(dirnm)
dirnm = pwd;
end

% complete the file name:
file = fullfile(dirnm, [file '.csv']);

%check the file exists, if not, prompt user to select file
if exist(file,'file') == 0
%open dialog to select a file
disp(['Deployment CSV file ' file ' not found'])
return
% Code it in....
end

% open the file
fid = fopen(file);

%figure out how many columns we have:
header1 = fgetl(fid);
ncols = length(strfind(header1,',')) + 1;
fmtHeader1 = repmat('%q', 1, ncols);
header1 = textscan(header1, fmtHeader1, 'Delimiter', ',', 'CollectOutput', 1);
header1 = header1{1};

%build the format string
header2 = fgetl(fid);
fmt = strrep(header2, ',', '');

%close and re-open the file
fclose(fid);
fid = fopen(file);

%extract all the data
data = textscan(fid, fmt, ...
'Delimiter', ',' , ...
'HeaderLines', 2);
% data = data{1};
for i=1:length(data)
if isfloat(data{i})
myData(:, i) = num2cell(data{i});
else
myData(:, i) = data{i};
end
end
fclose(fid);

result = extractdata(header1, myData, field, value);

% save result in structure
csvStruct.table{end+1} = file;
csvStruct.field{end+1} = field;
csvStruct.value{end+1} = num2str(value);
csvStruct.result{end+1} = result;
end

function result = extractdata(header, data, field, value);
% Extract the required values from the data
%headers become field names

if ~isempty(field)
ifield = strcmp(field,header);
if ~any(ifield)
disp(['Field ' field ' not found in the csv file'])
result = [];
return
end

% extract the values in field:
ivalue = strcmp(value,data(:,ifield));
data = data(ivalue,:);
end

%look for dates and convert to matlab date format:
dateTimeFmt = 'dd/mm/yy HH:MM';
idate = cellfun(@isempty,strfind(lower(header),'date'));
idate = find(~idate);
for a = idate
iempty = cellfun(@isempty, data(:,a));
data(~iempty,a) = cellfun(@(x) datenum(x,dateTimeFmt), data(~iempty,a), 'Uniform', false);
% iempty = cellfun(@isempty, data(:,a));
% cellDateStr = data{~iempty,a};
% arrayDateNum = datenum(cellDateStr, dateTimeFmt);
% cellDateNum = num2cell(arrayDateNum);
% data{~iempty,a} = cellDateNum;
end

%look for times and convert to matlab date format:
idate = cellfun(@isempty,strfind(lower(header),'time'));
idateTZ = cellfun(@isempty,strfind(lower(header),'timezone'));
idateTD = cellfun(@isempty,strfind(lower(header),'timedriftinstrument'));
idate = find(~idate & idateTZ & idateTD);
for a = idate
iempty = cellfun(@isempty, data(:,a));
data(~iempty,a) = cellfun(@(x) datenum(x,dateTimeFmt), data(~iempty,a), 'Uniform', false);
end

%make the structure:
result = cell2struct(data,header,2);

end
2 changes: 1 addition & 1 deletion FlowManager/autoIMOSToolbox.m
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ function autoIMOSToolbox(toolboxVersion, fieldTrip, dataDir, ppChain, qcChain, e
if isempty(sample_data), continue; end

raw_data = preprocessManager(sample_data, 'raw', mode, true);
qc_data = preprocessManager(sample_data, 'qc', mode, true);
qc_data = preprocessManager(raw_data, 'qc', mode, true);
clear sample_data;
qc_data = autoQCManager(qc_data, true);

Expand Down
52 changes: 37 additions & 15 deletions FlowManager/importManager.m
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@
end

% If the toolbox.ddb property has been set, assume that we have a
% deployment database. Otherwise perform a manual import
% deployment database. Or if it is designated as 'csv', use a CSV file for
% import. Otherwise perform a manual import.
ddb = readProperty('toolbox.ddb');

driver = readProperty('toolbox.ddb.driver');
Expand All @@ -81,10 +82,10 @@
rawFiles = {};

if ~isempty(ddb) || (~isempty(driver) && ~isempty(connection))
[structs rawFiles] = ddbImport(auto, iMooring);
[structs rawFiles] = ddbImport(auto, iMooring, ddb);
else
if auto, error('manual import cannot be automated without deployment database'); end
[structs rawFiles] = manualImport();
if auto, error('manual import cannot be automated without deployment database'); end
[structs rawFiles] = manualImport();
end

% user cancelled
Expand Down Expand Up @@ -190,13 +191,18 @@
end
end

function [sample_data rawFiles] = ddbImport(auto, iMooring)
function [sample_data rawFiles] = ddbImport(auto, iMooring, ddb)
%DDBIMPORT Imports data sets using metadata retrieved from a deployment
% database.
%
% Inputs:
% auto - if true, the import process is automated, with no user
% interaction.
% iMooring - Optional logical(comes with auto == true). Contains
% the logical indices to extract only the deployments
% from one mooring set of deployments.
% ddb - deployment database string attribute from
% toolboxProperties.txt
%
% Outputs:
% sample_data - cell array containig the imported data sets, or empty
Expand Down Expand Up @@ -304,8 +310,16 @@
rawFile = deps(k).FileName;

hits = fsearch(rawFile, dataDir, 'files');

allFiles{k} = hits;

% we remove any potential .pqc or .mqc files found (reserved for use
% by the toolbox)
reservedExts = {'.pqc', '.mqc'};
for l=1:length(hits)
[~, ~, ext] = fileparts(hits{l});
if all(~strcmp(ext, reservedExts))
allFiles{k}{end+1} = hits{l};
end
end
end

% display status dialog to highlight any discrepancies (file not found
Expand Down Expand Up @@ -348,9 +362,8 @@
fileDisplay = fileDisplay(3:end);
waitbar(k / length(deps), progress, fileDisplay);
end

% import data
sample_data{end+1} = parse(deps(k), allFiles{k}, parsers, noParserPrompt, mode);
sample_data{end+1} = parse(deps(k), allFiles{k}, parsers, noParserPrompt, mode, ddb);
rawFiles{ end+1} = allFiles{k};

if iscell(sample_data{end})
Expand Down Expand Up @@ -407,7 +420,7 @@
% close progress dialog
if ~auto, close(progress); end

function sam = parse(deployment, files, parsers, noParserPrompt, mode)
function sam = parse(deployment, files, parsers, noParserPrompt, mode, ddb)
%PARSE Parses a raw data file, returns a sample_data struct.
%
% Inputs:
Expand All @@ -416,12 +429,14 @@
% parsers - Cell array of strings containing all available parsers.
% noParserPrompt - Whether to prompt the user if a parser cannot be found.
% mode - Toolbox data type mode ('profile' or 'timeSeries').
% ddb - deployment database string attribute from
% toolboxProperties.txt
%
% Outputs:
% sam - Struct containing sample data.

% get the appropriate parser function
parser = getParserFunc(deployment, parsers, noParserPrompt);
parser = getParserFunc(deployment, parsers, noParserPrompt, ddb);
if isnumeric(parser)
error(['no parser found for instrument ' deployment.InstrumentID]);
end
Expand All @@ -430,21 +445,28 @@
sam = parser(files, mode);
end

function parser = getParserFunc(deployment, parsers, noParserPrompt)
function parser = getParserFunc(deployment, parsers, noParserPrompt, ddb)
%GETPARSERFUNC Searches for a parser function which is able to parse data
% for the given deployment.
%
% Inputs:
% deployment - struct containing information about the deployment.
% parsers - Cell array of strings containing all available parsers.
% noParserPrompt - Whether to prompt the user if a parser cannot be found.
% ddb - deployment database string attribute from
% toolboxProperties.txt
%
% Outputs:
% parser - Function handle to the parser function, or 0 if a parser
% function wasn't found.
%
instrument = executeDDBQuery(...
'Instruments', 'InstrumentID', deployment.InstrumentID);
if strcmp('csv',ddb)
instrument = executeCSVQuery(...
'Instruments', 'InstrumentID', deployment.InstrumentID);
else
instrument = executeDDBQuery(...
'Instruments', 'InstrumentID', deployment.InstrumentID);
end

% there should be exactly one instrument
if length(instrument) ~= 1
Expand Down Expand Up @@ -480,4 +502,4 @@
% get the parser function handle
parser = getParser(parser);
end
end
end
Loading

0 comments on commit af92a16

Please sign in to comment.