diff --git a/AutomaticQC/imosTiltVelocitySetQC.m b/AutomaticQC/imosTiltVelocitySetQC.m index 40e686ae2..9e64615f7 100644 --- a/AutomaticQC/imosTiltVelocitySetQC.m +++ b/AutomaticQC/imosTiltVelocitySetQC.m @@ -165,15 +165,16 @@ if strcmpi(paramName, 'ROLL'), idRoll = i; end end -% check if the data is compatible with the QC algorithm +% check if the data is compatible with the QC algorithm, otherwise quit +% silently idMandatory = idPitch & idRoll & (idUcur | idVcur | idWcur | idCspd | idCdir); - if ~idMandatory, return; end qcSet = str2double(readProperty('toolbox.qc_set')); goodFlag = imosQCFlag('good', qcSet, 'flag'); -% we try to find out which kind of ADCP we're dealing with +% we try to find out which kind of ADCP we're dealing with and if it is +% listed as one we should process instrument = sample_data.instrument; if isfield(sample_data, 'meta') if isfield(sample_data.meta, 'instrument_make') && isfield(sample_data.meta, 'instrument_model') @@ -184,8 +185,10 @@ [firstTiltThreshold, secondTiltThreshold, firstFlagThreshold, secondFlagThreshold] = getTiltThresholds(instrument); if isempty(firstTiltThreshold) - error(['Impossible to determine from which ADCP make/model is ' sample_data.toolbox_input_file ... - ' instrument = "' instrument '" => Fill imosTiltVelositySetQC.txt with according relevant make/model information!']); + % couldn't find this instrument so quit the test + disp(['Info: imosTiltVelositySetQC could not be performed on ' sample_data.toolbox_input_file ... + ' instrument = "' instrument '" => Fill imosTiltVelositySetQC.txt with relevant make/model information if you wish to run this test on this dataset.']); + return; end paramsLog = ['firstTiltThreshold=' num2str(firstTiltThreshold) ', secondTiltThreshold=' num2str(secondTiltThreshold)]; diff --git a/DDB/executeCSVQuery.m b/DDB/executeCSVQuery.m index f8496c325..f2051a5b4 100644 --- a/DDB/executeCSVQuery.m +++ b/DDB/executeCSVQuery.m @@ -102,13 +102,22 @@ %figure out how many columns we have: header1 = fgetl(fid); ncols = length(strfind(header1,',')) + 1; - fmtHeader1 = repmat('%q', 1, ncols); - header1 = textscan(header1, fmtHeader1, 'Delimiter', ',', 'CollectOutput', 1); + fmtHeader = repmat('%q', 1, ncols); + header1 = textscan(header1, fmtHeader, 'Delimiter', ',', 'CollectOutput', 1); header1 = header1{1}; %build the format string header2 = fgetl(fid); - fmt = strrep(header2, ',', ''); + header2 = textscan(header2, fmtHeader, 'Delimiter', ',', 'CollectOutput', 1); + header2 = header2{1}; + + iDate = cellfun(@(x) ~strncmpi(x, '%', 1), header2); + iDateFmt = header2; + iDateFmt(iDate) = header2(iDate); + + [header2{iDate}] = deal('%q'); + + fmt = cell2mat(header2); %close and re-open the file fclose(fid); @@ -118,17 +127,27 @@ data = textscan(fid, fmt, ... 'Delimiter', ',' , ... 'HeaderLines', 2); -% data = data{1}; - for i=1:length(data) + + nCols = length(data); + nRows = length(data{1}); + myData = cell(nRows, nCols); + for i=1:nCols if isfloat(data{i}) myData(:, i) = num2cell(data{i}); else - myData(:, i) = data{i}; +% if iDate(i) +% iEmpty = cellfun(@isempty, data{i}); +% if any(~iEmpty) +% myData(~iEmpty, i) = cellfun(@(x) datenum(x, iDateFmt{i}), data{i}(~iEmpty), 'Uniform', false); +% end +% else + myData(:, i) = data{i}; +% end end end fclose(fid); - result = extractdata(header1, myData, field, value); + result = extractdata(header1, myData, iDate, iDateFmt, field, value); % save result in structure csvStruct.table{end+1} = file; @@ -137,7 +156,7 @@ csvStruct.result{end+1} = result; end -function result = extractdata(header, data, field, value); +function result = extractdata(header, data, iDate, iDateFmt, field, value) % Extract the required values from the data %headers become field names @@ -154,28 +173,12 @@ data = data(ivalue,:); end -%look for dates and convert to matlab date format: -dateTimeFmt = 'dd/mm/yy HH:MM'; -idate = cellfun(@isempty,strfind(lower(header),'date')); -idate = find(~idate); -for a = idate - iempty = cellfun(@isempty, data(:,a)); - data(~iempty,a) = cellfun(@(x) datenum(x,dateTimeFmt), data(~iempty,a), 'Uniform', false); -% iempty = cellfun(@isempty, data(:,a)); -% cellDateStr = data{~iempty,a}; -% arrayDateNum = datenum(cellDateStr, dateTimeFmt); -% cellDateNum = num2cell(arrayDateNum); -% data{~iempty,a} = cellDateNum; -end - -%look for times and convert to matlab date format: -idate = cellfun(@isempty,strfind(lower(header),'time')); -idateTZ = cellfun(@isempty,strfind(lower(header),'timezone')); -idateTD = cellfun(@isempty,strfind(lower(header),'timedriftinstrument')); -idate = find(~idate & idateTZ & idateTD); -for a = idate - iempty = cellfun(@isempty, data(:,a)); - data(~iempty,a) = cellfun(@(x) datenum(x,dateTimeFmt), data(~iempty,a), 'Uniform', false); +% look for dates and convert them +for i = find(iDate) + iEmpty = cellfun(@isempty, data(:,i)); + if any(~iEmpty) + data(~iEmpty, i) = cellfun(@(x) datenum(x, iDateFmt{i}), data(~iEmpty,i), 'Uniform', false); + end end %make the structure: diff --git a/Parser/SBE37SMParse.m b/Parser/SBE37SMParse.m index 8de28833e..d79bd29b8 100644 --- a/Parser/SBE37SMParse.m +++ b/Parser/SBE37SMParse.m @@ -217,11 +217,9 @@ % there's no real structure to the header information, which % is annoying. my approach is to use various regexes to search -% for info we want, and to ignore everything else. inefficient, -% but it's the nicest way i can think of -%BDM (18/2/2011) - changed expressions to reflect newer SBE header info -% headerExpr = '^\*\s*(SBE \S+|SeacatPlus)\s+V\s+(\S+)\s+SERIAL NO.\s+(\d+)'; -headerExpr = ''; +% for info we want, and to ignore everything else. +newHeaderExpr= ''; +oldHeaderExpr= '^\*\s*(SBE\S+|SeacatPlus)\s+V\s+(\S+)\s+SERIAL NO.\s+(\d+)'; %old style header e.g. '* SBE37-SM V 2.6b SERIAL NO. 4481 10 Feb 2014 04:33:43' scanExpr = 'number of scans to average = (\d+)'; memExpr = 'samples = (\d+), free = (\d+), casts = (\d+)'; sampleExpr = ['sample interval = (\d+) (\w+), ' ... @@ -245,14 +243,14 @@ firmExpr ='(\S+)'; exprs = {... - headerExpr scanExpr ... + newHeaderExpr scanExpr ... memExpr sampleExpr ... modeExpr pressureExpr ... voltExpr outputExpr ... castExpr intervalExpr ... sbe38Expr optodeExpr ... voltCalExpr otherExpr ... - firmExpr}; + firmExpr oldHeaderExpr}; for k = 1:length(headerLines) @@ -266,11 +264,8 @@ % yes, ugly, but easiest way to figure out which regex we're on switch m - % header + % new style header case 1 - % header.instrument_model = tkns{1}{1}; - % header.instrument_firmware = tkns{1}{2}; - % header.instrument_serial_no = tkns{1}{3}; header.instrument_model = tkns{1}{1}; header.instrument_serial_no = tkns{1}{2}; @@ -344,6 +339,12 @@ %firmware version case 15 header.instrument_firmware = tkns{1}{1}; + + %old style header + case 16 + header.instrument_model = tkns{1}{1}; + header.instrument_firmware = tkns{1}{2}; + header.instrument_serial_no = tkns{1}{3}; end break; diff --git a/Parser/SBE3x.m b/Parser/SBE3x.m index 443c43144..fcaca5aa3 100644 --- a/Parser/SBE3x.m +++ b/Parser/SBE3x.m @@ -142,7 +142,7 @@ header_expr = '^[\*]\s*(SBE\S+)\s+V\s+(\S+)\s+(\d+)$'; cal_coeff_expr = '^[\*]\s*(\w+)\s*=\s*(\S+)\s*$'; sensor_cal_expr = '^[\*]\s*(\w+):\s*(.+)\s*$'; -salinity_expr = '^* output salinity.*$'; +salinity_expr = '^[\*] output salinity.*$'; pressure_cal_expr = ['^[\*]\s*pressure\s+S/N\s+(\d+)'... % serial number ',\s*range\s*=\s*(\d+)' ... % pressure range '\s+psia:?\s*(.+)\s*$']; % cal date including Seaterm v2 variation format @@ -204,6 +204,10 @@ % % * output salinity with each sample % +% as opposed to: +% * do not output salinity with each sample +% +% % The fourth type of line is a name-value pair of a particular calibration % coefficient. These lines have the format: % diff --git a/imosToolbox.m b/imosToolbox.m index adfccdd47..ff62ca405 100644 --- a/imosToolbox.m +++ b/imosToolbox.m @@ -73,7 +73,7 @@ function imosToolbox(auto, varargin) end % Set current toolbox version -toolboxVersion = ['2.5.7 - ' computer]; +toolboxVersion = ['2.5.8 - ' computer]; switch auto case 'auto', autoIMOSToolbox(toolboxVersion, varargin{:}); diff --git a/imosToolbox_Linux64.bin b/imosToolbox_Linux64.bin index e5e3e7d87..84d0841e4 100755 Binary files a/imosToolbox_Linux64.bin and b/imosToolbox_Linux64.bin differ diff --git a/imosToolbox_Win32.exe b/imosToolbox_Win32.exe index 9fbc86f76..62330125a 100644 Binary files a/imosToolbox_Win32.exe and b/imosToolbox_Win32.exe differ diff --git a/imosToolbox_Win64.exe b/imosToolbox_Win64.exe index a18b04cea..4e93dd6d0 100644 Binary files a/imosToolbox_Win64.exe and b/imosToolbox_Win64.exe differ