diff --git a/FlowManager/autoQCManager.m b/FlowManager/autoQCManager.m index 9a4f03d38..634b457d3 100644 --- a/FlowManager/autoQCManager.m +++ b/FlowManager/autoQCManager.m @@ -126,8 +126,8 @@ end end - % we reset QC flags to 0 for k = 1:length(sample_data) + % reset QC flags to 0 type{1} = 'dimensions'; type{2} = 'variables'; for m = 1:length(type) @@ -136,6 +136,9 @@ sample_data{k}.(type{m}){l}.flags(:) = 0; end end + + % reset QC results + sample_data{k}.meta.QCres = {}; end % run each data set through the chain diff --git a/Graph/TimeSeries/graphTimeSeriesGeneric.m b/Graph/TimeSeries/graphTimeSeriesGeneric.m index ff2a2b4b4..ed1bc9d4e 100644 --- a/Graph/TimeSeries/graphTimeSeriesGeneric.m +++ b/Graph/TimeSeries/graphTimeSeriesGeneric.m @@ -89,32 +89,6 @@ end end -% Set axis position so that 1D data and 2D data vertically matches on X axis -mainPanel = findobj('Tag', 'mainPanel'); -last_pos_with_colorbar = get(mainPanel, 'UserData'); -if isempty(last_pos_with_colorbar) % this is to avoid too many calls to colorbar() - cb = colorbar(); - set(get(cb, 'YLabel'), 'String', 'TEST'); - pos_with_colorbar = get(ax, 'Position'); - last_pos_with_colorbar = pos_with_colorbar; - colorbar(cb, 'off'); - set(mainPanel, 'UserData', last_pos_with_colorbar); -else - pos_with_colorbar = get(ax, 'Position'); - - if pos_with_colorbar(1) == last_pos_with_colorbar(1) - pos_with_colorbar(3) = last_pos_with_colorbar(3); - else - cb = colorbar(); - set(get(cb, 'YLabel'), 'String', 'TEST'); - pos_with_colorbar = get(ax, 'Position'); - last_pos_with_colorbar = pos_with_colorbar; - colorbar(cb, 'off'); - set(mainPanel, 'UserData', last_pos_with_colorbar); - end -end -set(ax, 'Position', pos_with_colorbar); - % set background to be grey set(ax, 'Color', [0.75 0.75 0.75]) diff --git a/Graph/TimeSeries/graphTimeSeriesTimeDepth.m b/Graph/TimeSeries/graphTimeSeriesTimeDepth.m index 085f02203..cdc9a3884 100644 --- a/Graph/TimeSeries/graphTimeSeriesTimeDepth.m +++ b/Graph/TimeSeries/graphTimeSeriesTimeDepth.m @@ -70,9 +70,15 @@ % xPcolor = [time.data(1:end-1) - diff(time.data)/2; time.data(end) - (time.data(end)-time.data(end-1))/2]; % yPcolor = [depth.data(1:end-1) - diff(depth.data)/2; depth.data(end) - (depth.data(end)-depth.data(end-1))/2]; +posWithoutCb = get(ax, 'Position'); + h = pcolor(ax, double(xPcolor), double(yPcolor), double(var.data')); set(h, 'FaceColor', 'flat', 'EdgeColor', 'none'); -cb = colorbar(); +cb = colorbar('peer', ax); + +% reset position to what it was without the colorbar so that it aligns with +% 1D datasets +set(ax, 'Position', posWithoutCb); % Attach the context menu to colorbar hMenu = setTimeSerieColorbarContextMenu(var); diff --git a/Graph/TimeSeries/graphTimeSeriesTimeFrequency.m b/Graph/TimeSeries/graphTimeSeriesTimeFrequency.m index b9e61c249..2cc22e12c 100644 --- a/Graph/TimeSeries/graphTimeSeriesTimeFrequency.m +++ b/Graph/TimeSeries/graphTimeSeriesTimeFrequency.m @@ -72,7 +72,7 @@ h = pcolor(ax, double(xPcolor), double(yPcolor), double(var.data')); set(h, 'FaceColor', 'flat', 'EdgeColor', 'none'); -cb = colorbar(); +cb = colorbar('peer',ax); % Attach the context menu to colorbar hMenu = setTimeSerieColorbarContextMenu(var); diff --git a/Graph/TimeSeries/graphTimeSeriesTimeFrequencyDirection.m b/Graph/TimeSeries/graphTimeSeriesTimeFrequencyDirection.m index 7c3144d64..3d815fd9e 100644 --- a/Graph/TimeSeries/graphTimeSeriesTimeFrequencyDirection.m +++ b/Graph/TimeSeries/graphTimeSeriesTimeFrequencyDirection.m @@ -200,7 +200,7 @@ 'Position' , posUi2(mainPanel, 50, 1, 50, 1, 0), ... 'String' ,'Time cursor'); -cb = colorbar(); +cb = colorbar('peer',ax); % Attach the context menu to colorbar hMenu = setTimeSerieColorbarContextMenu(myVar); diff --git a/Graph/graphTimeSeries.m b/Graph/graphTimeSeries.m index 5e8a6f0ce..5b44b205b 100644 --- a/Graph/graphTimeSeries.m +++ b/Graph/graphTimeSeries.m @@ -1,4 +1,4 @@ -function [graphs lines vars] = graphTimeSeries( parent, sample_data, vars ) +function [graphs, lines, vars] = graphTimeSeries( parent, sample_data, vars ) %GRAPHTIMESERIES Graphs the given data in a time series style using subplots. % % Graphs the selected variables from the given data set. Each variable is @@ -186,20 +186,46 @@ col = col(mod(vars(k),length(col))+1,:); % plot the variable - [lines(k,:) labels] = plotFunc(graphs(k), sample_data, k, col, xTickProp); + [lines(k,:), labels] = plotFunc(graphs(k), sample_data, k, col, xTickProp); if ~isempty(labels) % set x labels and ticks xlabel(graphs(k), labels{1}); % set y label and ticks - try uom = [' (' imosParameters(labels{2}, 'uom') ')']; + try uom = ['(' imosParameters(labels{2}, 'uom') ')']; catch e, uom = ''; end - yLabel = strrep([labels{2} uom], '_', ' '); - if length(yLabel) > 20, yLabel = [yLabel(1:17) '...']; end + + % decide where to cut the Y label to display it on 1 or 2 lines + % depending on the number of words obtained from the variable name + yLabel = regexp(labels{2}, '\_', 'split'); + if numel(yLabel) < 4 + nthWordToCut = min(2, numel(yLabel)); + elseif numel(yLabel) < 6 + nthWordToCut = 3; + else + nthWordToCut = 4; + end + yLabel = {strjoin(yLabel(1:nthWordToCut), ' '), ... + strjoin(yLabel(nthWordToCut+1:end), ' ')}; + yLabel = yLabel(~cellfun(@isempty, yLabel)); + + yLabel{end+1} = strrep(uom, '_', ' '); + iLength = 12; % arbitrary string cutoff length + %iLong = strlength(yLabel) > iLength; % only R2016b onwards + iLong = cellfun(@length, yLabel) > iLength; + yLabel(iLong) = cellfun(@(x) [x(1:iLength) '...'], yLabel(iLong), 'UniformOutput', false); ylabel(graphs(k), yLabel); end end - + + function str = strjoin(strCell, sep) + %STRJOIN Join strings in a cell array. + % http://stackoverflow.com/questions/5292437/how-can-i-concatenate-strings-in-a-cell-array-with-spaces-between-them-in-matlab + % R2012b onwards + nCells = numel(strCell); + strCell(1:nCells-1) = strcat(strCell(1:nCells-1), {sep}); + str = [strCell{:}]; + end end \ No newline at end of file diff --git a/IMOS/finaliseData.m b/IMOS/finaliseData.m index 521777368..c04537fcb 100644 --- a/IMOS/finaliseData.m +++ b/IMOS/finaliseData.m @@ -96,6 +96,10 @@ end end + % CF requires DEPTH/TIME coordinate variables must be monotonic (strictly increasing + % or decreasing) + sam = forceMonotonic(sam, mode); + % add empty QC flags for all variables for k = 1:length(sam.variables) @@ -181,13 +185,13 @@ % set the time coverage period from the data switch mode case 'profile' - time = getVar(sam.variables, 'TIME'); - if time ~= 0 + iTime = getVar(sam.variables, 'TIME'); + if iTime ~= 0 if isempty(sam.time_coverage_start), - sam.time_coverage_start = sam.variables{time}.data(1); + sam.time_coverage_start = sam.variables{iTime}.data(1); end if isempty(sam.time_coverage_end), - sam.time_coverage_end = sam.variables{time}.data(end); + sam.time_coverage_end = sam.variables{iTime}.data(end); end else if isempty(sam.time_coverage_start), sam.time_coverage_start = []; end @@ -195,13 +199,13 @@ end case 'timeSeries' - time = getVar(sam.dimensions, 'TIME'); - if time ~= 0 + iTime = getVar(sam.dimensions, 'TIME'); + if iTime ~= 0 if isempty(sam.time_coverage_start), - sam.time_coverage_start = sam.dimensions{time}.data(1); + sam.time_coverage_start = sam.dimensions{iTime}.data(1); end if isempty(sam.time_coverage_end), - sam.time_coverage_end = sam.dimensions{time}.data(end); + sam.time_coverage_end = sam.dimensions{iTime}.data(end); end else if isempty(sam.time_coverage_start), sam.time_coverage_start = []; end @@ -210,4 +214,87 @@ end +end + +function sam = forceMonotonic(sam, mode) + % We make sure that DEPTH/TIME coordinate variables appear + % ordered and that there is no redundant value. + switch mode + case 'profile' + if strcmpi(sam.dimensions{1}.name, 'MAXZ'), return; end % this case produces non compliant files anyway + + dimensionName = 'DEPTH'; % so far we only produce downcasts to be compliant + sortMode = 'ascend'; + sortModeStr = 'increasingly'; + + case 'timeSeries' + dimensionName = 'TIME'; + sortMode = 'ascend'; + sortModeStr = 'increasingly'; + + end + iDim = getVar(sam.dimensions, dimensionName); + + fixStr = ['Try to re-play and fix this dataset when possible using the ' ... + 'manufacturer''s software before processing it with the toolbox.']; + currentDateStr = datestr(now_utc, readProperty('exportNetCDF.dateFormat')); + + [sam.dimensions{iDim}.data, iSort] = sort(sam.dimensions{iDim}.data, sortMode); + if any(iSort ~= (1:length(sam.dimensions{iDim}.data))') + % We need to sort variables that are functions of this + % dimension accordingly + for k = 1:length(sam.variables) + iFunOfDim = sam.variables{k}.dimensions == iDim; + if any(iFunOfDim) + sam.variables{k}.data = sam.variables{k}.data(iSort,:); % data(iSort,:) works because we know that the sorted dimension is the first one! + end + end + + sortedStr = [dimensionName ' values (and their corresponding measurements) had ' ... + 'to be sorted ' sortModeStr]; + disp(['Info : ' sortedStr ' in ' sam.toolbox_input_file '. ' fixStr]); + if isfield(sam.dimensions{iDim}, 'comment') + sam.dimensions{iDim}.comment = [sam.dimensions{iDim}.comment ' ' sortedStr '.']; + else + sam.dimensions{iDim}.comment = [sortedStr '.']; + end + if isfield(sam, 'history') + sam.history = sprintf('%s\n%s - %s', sam.history, currentDateStr, [sortedStr '.']); + else + sam.history = sprintf('%s - %s', currentDateStr, [sortedStr '.']); + end + end + + iRedundantDim = [(diff(sam.dimensions{iDim}.data) == 0); false]; + if any(iRedundantDim) + sam.dimensions{iDim}.data(iRedundantDim) = []; % removing first duplicate values + + % We need to remove duplicates for variables that are functions of this + % dimension accordingly + for k = 1:length(sam.variables) + iFunOfDim = sam.variables{k}.dimensions == iDim; + if any(iFunOfDim) + extraDims = size(sam.variables{k}.data); + extraDims(iFunOfDim) = 1; + sam.variables{k}.data(repmat(iRedundantDim, extraDims)) = []; + extraDims(iFunOfDim) = length(sam.dimensions{iDim}.data); + sam.variables{k}.data = reshape(sam.variables{k}.data, extraDims); + end + end + + redundantStr = [num2str(sum(iRedundantDim)) ' ' dimensionName ' first duplicate values ' ... + '(and their corresponding measurements) had to be discarded']; + disp(['Info : ' redundantStr ' in ' sam.toolbox_input_file '. ' fixStr]); + if isfield(sam.dimensions{iDim}, 'comment') + sam.dimensions{iDim}.comment = [sam.dimensions{iDim}.comment ' ' redundantStr '.']; + else + sam.dimensions{iDim}.comment = [redundantStr '.']; + end + if isfield(sam, 'history') + sam.history = sprintf('%s\n%s - %s', sam.history, currentDateStr, [redundantStr '.']); + else + sam.history = sprintf('%s - %s', currentDateStr, [redundantStr '.']); + end + end + end \ No newline at end of file diff --git a/Parser/StarmonDSTParse.m b/Parser/StarmonDSTParse.m new file mode 100644 index 000000000..62b85c951 --- /dev/null +++ b/Parser/StarmonDSTParse.m @@ -0,0 +1,446 @@ +function sample_data = StarmonDSTParse( filename, mode ) +%STARMONMINIPARSE Parses an ASCII file from Starmon DST Tilt or CTD .DAT +%file format. Based on StarmonMiniParse.m. +% +% The files consist of two sections: +% +% - file headerContent - headerContent information with description of the data structure and content. +% These lines are suffixed with a #. +% - data - rows of data. +% +% Inputs: +% filename - cell array of files to import (only one supported). +% mode - Toolbox data type mode. +% +% Outputs: +% sample_data - Struct containing sample data. +% +% Author: Peter Jansen +% Contributor: Guillaume Galibert +% +% +% Copyright (c) 2016, Australian Ocean Data Network (AODN) and Integrated +% Marine Observing System (IMOS). +% All rights reserved. +% +% Redistribution and use in source and binary forms, with or without +% modification, are permitted provided that the following conditions are met: +% +% * Redistributions of source code must retain the above copyright notice, +% this list of conditions and the following disclaimer. +% * Redistributions in binary form must reproduce the above copyright +% notice, this list of conditions and the following disclaimer in the +% documentation and/or other materials provided with the distribution. +% * Neither the name of the AODN/IMOS nor the names of its contributors +% may be used to endorse or promote products derived from this software +% without specific prior written permission. +% +% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +% AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +% IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +% ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +% LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +% CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +% SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +% INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +% CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +% ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +% POSSIBILITY OF SUCH DAMAGE. +% +narginchk(1,2); + +if ~iscellstr(filename) + error('filename must be a cell array of strings'); +end + +% only one file supported currently +filename = filename{1}; + +% retrieve and parse the header content +header = getHeader(filename); + +% retrieve and parse the data +data = getData(filename, header); + +% create sample data struct, +% and copy all the data in +sample_data = struct; + +sample_data.toolbox_input_file = filename; +sample_data.meta.header = header; + +sample_data.meta.instrument_make = 'Star ODDI'; +if isfield(data, 'PITCH') + sample_data.meta.instrument_model = 'DST Tilt'; +else + sample_data.meta.instrument_model = 'DST CTD'; +end +sample_data.meta.instrument_sample_interval = median(diff(data.TIME.values*24*3600)); +sample_data.meta.instrument_serial_no = header.serialNo; +sample_data.meta.featureType = mode; + +sample_data.dimensions = {}; +sample_data.variables = {}; + +% generate time data from headerContent information +sample_data.dimensions{1}.name = 'TIME'; +sample_data.dimensions{1}.typeCastFunc = str2func(netcdf3ToMatlabType(imosParameters(sample_data.dimensions{1}.name, 'type'))); +sample_data.dimensions{1}.data = sample_data.dimensions{1}.typeCastFunc(data.TIME.values); + +sample_data.variables{end+1}.name = 'TIMESERIES'; +sample_data.variables{end}.typeCastFunc = str2func(netcdf3ToMatlabType(imosParameters(sample_data.variables{end}.name, 'type'))); +sample_data.variables{end}.data = sample_data.variables{end}.typeCastFunc(1); +sample_data.variables{end}.dimensions = []; +sample_data.variables{end+1}.name = 'LATITUDE'; +sample_data.variables{end}.typeCastFunc = str2func(netcdf3ToMatlabType(imosParameters(sample_data.variables{end}.name, 'type'))); +sample_data.variables{end}.data = sample_data.variables{end}.typeCastFunc(NaN); +sample_data.variables{end}.dimensions = []; +sample_data.variables{end+1}.name = 'LONGITUDE'; +sample_data.variables{end}.typeCastFunc = str2func(netcdf3ToMatlabType(imosParameters(sample_data.variables{end}.name, 'type'))); +sample_data.variables{end}.data = sample_data.variables{end}.typeCastFunc(NaN); +sample_data.variables{end}.dimensions = []; +sample_data.variables{end+1}.name = 'NOMINAL_DEPTH'; +sample_data.variables{end}.typeCastFunc = str2func(netcdf3ToMatlabType(imosParameters(sample_data.variables{end}.name, 'type'))); +sample_data.variables{end}.data = sample_data.variables{end}.typeCastFunc(NaN); +sample_data.variables{end}.dimensions = []; + +% scan through the list of parameters that were read +% from the file, and create a variable for each +vars = fieldnames(data); +coordinates = 'TIME LATITUDE LONGITUDE NOMINAL_DEPTH'; +for k = 1:length(vars) + + if strncmp('TIME', vars{k}, 4), continue; end + + % dimensions definition must stay in this order : T, Z, Y, X, others; + % to be CF compliant + sample_data.variables{end+1}.dimensions = 1; + sample_data.variables{end }.name = vars{k}; + sample_data.variables{end }.typeCastFunc = str2func(netcdf3ToMatlabType(imosParameters(sample_data.variables{end}.name, 'type'))); + sample_data.variables{end }.data = sample_data.variables{end }.typeCastFunc(data.(vars{k}).values); + sample_data.variables{end }.coordinates = coordinates; + sample_data.variables{end }.comment = data.(vars{k}).comment; + + if isfield(data.(vars{k}), 'applied_offset') + % let's document the constant pressure atmosphere offset previously + % applied by SeaBird software on the absolute presure measurement + sample_data.variables{end}.applied_offset = data.(vars{k}).applied_offset; + end +end + +end + +function header = getHeader(filename) +%GETHEADER Reads any Starmon Mini / DST tilt / DST CTD .DAT header and return usefull content in a +% structure. +% +% Each header item is contained in one line, and all header lines start with a # +% (bookmark) and a number. Then follows a description of the header item, and then 1-4 +% directives, all separated by tabs. Eventually a comment trails the directives, preceded +% by a semicolon (;). +% +header = struct; + +fileId = fopen(filename); + +headerFormat = '#%[^\r]'; +headerContent = textscan(fileId, headerFormat); + +fclose(fileId); + +decimalPointExpr = '6\tDecimal point:\t(\S+)'; +fieldSepExpr = '5\tField separation:\t(\d+)'; +recorderExpr = '#\tRecorder\t(\d+)\t([^\t]*)\t([^\t]*)'; +dateTimeExpr = '2\tDate & Time:\t(\d+)'; +axisExpr = '#\tAxis\t(\d+)\t([^\(]*)\(([^\)]*)\)'; +reconversionExpr = '11\tReconvertion:\t(\d+)'; +dateDefExpr = '7\tDate def.:\t([^\t]*)\t\/'; +recorder2Expr = '1\tRecorder:\t(\S+)'; +decimalPtExpr = '6\tDecimal point:\t(\d+)'; +timeDefExpr = '8\tTime def.:\t(\d+)'; +channelExpr = '(\d+)\tChannel (\d+):\t([^\(]*)\(([^\)]*)\)'; +dateDef2Expr = '7\tDate def.:\t(\d+)\t(\d+)'; + +exprs = {decimalPointExpr,fieldSepExpr,recorderExpr,dateTimeExpr,axisExpr,reconversionExpr,dateDefExpr,recorder2Expr,decimalPtExpr,timeDefExpr,channelExpr,dateDef2Expr}; +header.nParam = 0; + +header.nLines = length(headerContent{1}); +header.isReconverted = false; +header.dateFormat = 'dd/mm/yyyy'; +header.timeFormat = 'HH:MM:SS'; +header.isDateJoined = true; + +% try each of the expressions +for m = 1:length(exprs) + tkns = regexp(headerContent{1}, exprs{m}, 'tokens'); + matches = find(~cellfun(@isempty,tkns)); + if (~isempty(matches)) + % yes, ugly, but easiest way to figure out which regex we're on + switch exprs{m} + case decimalPointExpr + header.decimalChar = char(tkns{matches}{1}); + + case fieldSepExpr + if strcmpi(tkns{matches}{1}, '0') + header.fieldSep = '\t'; + else + header.fieldSep = ' '; + end + + case recorderExpr + header.instrument_model = tkns{matches}{1}{2}; + header.serialNo = tkns{matches}{1}{3}; + + case dateTimeExpr + if cell2mat(tkns{matches}{1}) == 0 + header.isDateJoined = false; + else + header.isDateJoined = true; + end + + case axisExpr + for x=1:length(matches) + if ~isempty(tkns{matches(x)}) + header.param(x).axis = str2double(tkns{matches(x)}{1}{1}); + header.param(x).column = genvarname(tkns{matches(x)}{1}{2}); + header.param(x).units = genvarname(tkns{matches(x)}{1}{3}); + header.nParam = max(header.nParam, header.param(x).axis); + end + end + + case reconversionExpr + if ~strcmpi(char(tkns{matches}{1}), '0') + header.isReconverted = true; + end + + case dateDefExpr + header.dateFormat = char(tkns{matches}{1}); + + case recorder2Expr + header.instrument_model = 'Starmon Mini'; + header.serialNo = tkns{matches}{1}{1}; + + case decimalPtExpr + if strcmpi(char(tkns{matches}{1}), '0') + header.decimalChar = ','; + else + header.decimalChar = '.'; + end + + case timeDefExpr + if strcmpi(char(tkns{matches}{1}), '0') + header.timeFormat = 'HH:MM:SS'; + else + header.timeFormat = 'HH.MM.SS'; + end + + case channelExpr + for x=1:length(matches) + if ~isempty(tkns{matches(x)}) + header.param(x).axis = str2double(tkns{matches(x)}{1}{2}); + header.param(x).column = genvarname(tkns{matches(x)}{1}{3}); + header.param(x).units = genvarname(tkns{matches(x)}{1}{4}); + header.nParam = max(header.nParam, header.param(x).axis); + end + end + + case dateDef2Expr + if tkns{matches}{1}{1} == '0' && tkns{matches}{1}{2} == '0' + header.dateFormat = 'dd.mm.yy'; + elseif tkns{matches}{1}{1} == '1' && tkns{matches}{1}{2} == '0' + header.dateFormat = 'mm.dd.yy'; + elseif tkns{matches}{1}{1} == '0' && tkns{matches}{1}{2} == '1' + header.dateFormat = 'dd/mm/yy'; + elseif tkns{matches}{1}{1} == '1' && tkns{matches}{1}{2} == '1' + header.dateFormat = 'mm/dd/yy'; + elseif tkns{matches}{1}{1} == '0' && tkns{matches}{1}{2} == '2' + header.dateFormat = 'dd-mm-yy'; + elseif tkns{matches}{1}{1} == '1' && tkns{matches}{1}{2} == '2' + header.dateFormat = 'mm-dd-yy'; + end + + end + end +end + +header.nCol = header.nParam+2; + +end + +function data = getData(filename, header) +%GETDATA Reads any Starmon Mini / DST Tilt / DST CTD .DAT file data based on the header content +% +% The first column is the measurement number, +% the second column the date and the time, depending on the set-up. +% The third column is the time or the first measured parameter, depending on set-up. +% The following column(s) contain the converted measured parameters with units and +% number of decimals according to set-up. +% +% Number of parameters can range from 1-3, and number of columns 3-6 accordingly. +% +data = struct; + +if strcmpi(header.fieldSep, ' ') || ~header.isDateJoined + extraColumnTime = '%s'; + nColumnTime = 2; +else + extraColumnTime = ''; + nColumnTime = 1; +end + +dataFormat = ['%*d%s' extraColumnTime]; +for i=1:header.nCol + if header.isReconverted + dataFormat = [dataFormat '%s%s']; + else + dataFormat = [dataFormat '%s']; + end +end + +fileId = fopen(filename); +DataContent = textscan(fileId, dataFormat, 'Delimiter', header.fieldSep, 'HeaderLines', header.nLines); +fclose(fileId); + +% we convert the data +if strcmpi(header.fieldSep, ' ') || ~header.isDateJoined + data.TIME.values = datenum(DataContent{1}, header.dateFormat) + datenum(DataContent{2}, header.timeFormat) - datenum(datestr(now, 'yyyy-01-01')); +else + data.TIME.values = datenum(DataContent{1}, [header.dateFormat ' ' header.timeFormat]); +end + +for i=1:header.nCol-(1+nColumnTime) % we start after the "n date time" + + if header.isReconverted && rem(i, 2) == 0 % second param column + iParam = i - 1; + else % first param column or params are not converted + iParam = i; + end + + iContent = i + nColumnTime; + + switch header.param(iParam).column + case {'Temperature'} + var = 'TEMP'; + values = strrep(DataContent{iContent}, ',', '.'); + values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double + comment = ''; + if header.isReconverted + if header.isTempCorr + if rem(i, 2) == 1 + var = [var '_1']; + else + var = [var '_2']; + comment = 'Normal temperature correction applied.'; + end + end + end + if strcmpi(header.param(iParam).units, 'x0xFFFDF') + data.(var).values = (values - 32) * 5/9; % conversion from Fahrenheit to Celsius + else + data.(var).values = values; + end + data.(var).comment = comment; + + case 'Pressure' + var = 'PRES'; + values = strrep(DataContent{iContent}, ',', '.'); + values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double + comment = ''; + applied_offset = []; + if header.isReconverted + if header.isPresCorr + comment = ''; + if rem(i, 2) == 0 + var = 'PRES_REL'; + comment = ['A zero offset of value ' header.presCorrValue 'mbar was adjusted.']; + applied_offset = header.presCorrValue/100; + end + end + end + data.(var).values = values; + data.(var).comment = comment; + data.(var).applied_offset = applied_offset; + + case 'Depth' + var = 'DEPTH'; + values = strrep(DataContent{iContent}, ',', '.'); + values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double + comment = 'depth calculated from pressure, using g=9.81 m/s/s'; + if header.isReconverted + if header.isPresCorr + if rem(i, 2) == 1 + var = [var '_1']; + else + var = [var '_2']; + comment = ['A zero offset of value ' header.presCorrValue 'mbar was adjusted to pressure.']; + end + end + end + data.(var).values = values; + data.(var).comment = comment; + + case 'Salinity' + var = 'PSAL'; + values = strrep(DataContent{iContent}, ',', '.'); + values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double + comment = ''; + if header.isReconverted + if header.isTempCorr || header.isPresCorr + if rem(i, 2) == 1 + var = [var '_1']; + else + var = [var '_2']; + if header.isTempCorr && header.isPresCorr + comment = ['Normal temperature correction applied. A zero offset of value ' ... + header.presCorrValue 'mbar was adjusted to pressure.']; + elseif header.isTempCorr + comment = 'Normal temperature correction applied.'; + elseif header.isPresCorr + comment = ['A zero offset of value ' header.presCorrValue 'mbar was adjusted to pressure.']; + end + end + end + end + data.(var).values = values; + data.(var).comment = comment; + + case 'Conductivity' + var = 'CNDC'; + values = strrep(DataContent{iContent}, ',', '.'); + values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double + comment = ''; + data.(var).values = values; + data.(var).comment = comment; + + case 'Sound Velocity' + var = 'SOUND_VEL'; + values = strrep(DataContent{iContent}, ',', '.'); + values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double + comment = ''; + data.(var).values = values; + data.(var).comment = comment; + + case 'Roll' + var = 'ROLL'; + values = strrep(DataContent{iContent}, ',', '.'); + values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double + comment = ''; + data.(var).values = values; + data.(var).comment = comment; + + case 'Pitch' + var = 'PITCH'; + values = strrep(DataContent{iContent}, ',', '.'); + values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double + comment = ''; + data.(var).values = values; + data.(var).comment = comment; + + otherwise + fprintf('%s\n', ['Warning : ' header.param(i).column ' not supported' ... + ' in ' filename '. Contact AODN.']); + + end +end + +end diff --git a/Parser/StarmonMiniParse.m b/Parser/StarmonMiniParse.m index dcf431c84..52428c4d6 100644 --- a/Parser/StarmonMiniParse.m +++ b/Parser/StarmonMiniParse.m @@ -4,9 +4,9 @@ % % The files consist of two sections: % -% - file headerContent - headerContent information with description of the data structure and content. -% These lines are suffixed with a #. -% - data - rows of data. +% - file headerContent - headerContent information with description of the data structure and content. +% These lines are suffixed with a #. +% - data - rows of data. % % Inputs: % filename - cell array of files to import (only one supported). @@ -252,13 +252,14 @@ case 14 % channel 3 info channelInfo = textscan(headerContent{3}{i}, '%s%s%d%d%*s', 'Delimiter', '\t'); header.param(3).axis = channelInfo{1}{1}; - header.param(3).column = channelInfo{2}{1}; + header.param(3).column = genvarname(channelInfo{2}{1}); header.param(3).nDec = channelInfo{3}; if channelInfo{4} == 1 header.param(3).isPositiveUp = true; else header.param(3).isPositiveUp = false; end + end end end @@ -318,7 +319,6 @@ case {'Temp0x280xB0C0x29', 'Temp0x280xFFFDC0x29'} %degrees C var = 'TEMP'; values = strrep(DataContent{iContent}, ',', '.'); -% values = cellfun(@str2double, values); values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double comment = ''; if header.isReconverted @@ -352,7 +352,7 @@ data.(var).values = (values - 32) * 5/9; data.(var).comment = comment; - case 'Pres(dbar)' + case 'Pres0x28dbar0x29' var = 'PRES'; values = strrep(DataContent{iContent}, ',', '.'); values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double @@ -372,7 +372,7 @@ data.(var).comment = comment; data.(var).applied_offset = applied_offset; - case 'Depth(m)' + case 'Depth0x28m0x29' var = 'DEPTH'; values = strrep(DataContent{iContent}, ',', '.'); values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double @@ -390,7 +390,7 @@ data.(var).values = values; data.(var).comment = comment; - case 'Sal(psu)' + case 'Sal0x28psu0x29' var = 'PSAL'; values = strrep(DataContent{iContent}, ',', '.'); values = sscanf(sprintf('%s*', values{:}), '%f*'); % ~35x faster than str2double diff --git a/Parser/readWQMdat.m b/Parser/readWQMdat.m index d61614bea..9a1086d05 100644 --- a/Parser/readWQMdat.m +++ b/Parser/readWQMdat.m @@ -221,20 +221,6 @@ end time = datenum(time, timeFormat); - % WQM instrumensts (or the .DAT conversion sofware) have a habit of - % generating erroneous data sometimes, either missing a character , or - % inserting a 0 instead of the correct in the output to .DAT files. - % This is a simple check to make sure that all of the timestamps appear - % to be correct; there's only so much we can do though. - iBadTime = (diff(time) <= 0); - iBadTime = [false; iBadTime]; - time(iBadTime) = []; - - for k = 1:length(samples) - if k == 2, continue; end % time data has already been collected and filtered - samples{k}(iBadTime) = []; - end - % Let's find each start of bursts dt = [0; diff(time)]; iBurst = [1; find(dt>(1/24/60)); length(time)+1]; diff --git a/Parser/readWQMraw.m b/Parser/readWQMraw.m index b3f93025b..bb384aa27 100644 --- a/Parser/readWQMraw.m +++ b/Parser/readWQMraw.m @@ -137,15 +137,6 @@ % convert and save the time data time = wqmdata.datenumber; - % WQM instrumensts (or the .DAT conversion sofware) have a habit of - % generating erroneous data sometimes, either missing a character , or - % inserting a 0 instead of the correct in the output to .DAT files. - % This is a simple check to make sure that all of the timestamps appear - % to be correct; there's only so much we can do though. - iBadTime = (diff(time) <= 0); - iBadTime = [false; iBadTime]; - time(iBadTime) = []; - % Let's find each start of bursts dt = [0; diff(time)]; iBurst = [1; find(dt>(1/24/60)); length(time)+1]; diff --git a/Util/populateMetadata.m b/Util/populateMetadata.m index 5f5e57eb7..4f073e0eb 100644 --- a/Util/populateMetadata.m +++ b/Util/populateMetadata.m @@ -68,8 +68,6 @@ ivDepth = 0; ivNomDepth = 0; - ivPres = 0; - ivPresRel = 0; ivLat = 0; ivLon = 0; ivBotDepth = 0; @@ -86,12 +84,6 @@ if strcmpi(sample_data.variables{i}.name, 'DEPTH') ivDepth = i; end - if strcmpi(sample_data.variables{i}.name, 'PRES') - ivPres = i; - end - if strcmpi(sample_data.variables{i}.name, 'PRES_REL') - ivPresRel = i; - end if strcmpi(sample_data.variables{i}.name, 'BOT_DEPTH') ivBotDepth = i; end diff --git a/imosToolbox.m b/imosToolbox.m index 40f4241f7..399eb7538 100644 --- a/imosToolbox.m +++ b/imosToolbox.m @@ -73,7 +73,7 @@ function imosToolbox(auto, varargin) end % Set current toolbox version -toolboxVersion = ['2.5.22 - ' computer]; +toolboxVersion = ['2.5.23 - ' computer]; switch auto case 'auto', autoIMOSToolbox(toolboxVersion, varargin{:}); diff --git a/imosToolbox_Linux64.bin b/imosToolbox_Linux64.bin index f22a14964..789c55667 100755 Binary files a/imosToolbox_Linux64.bin and b/imosToolbox_Linux64.bin differ diff --git a/imosToolbox_Win32.exe b/imosToolbox_Win32.exe index 61ad63024..d99fb9faa 100644 Binary files a/imosToolbox_Win32.exe and b/imosToolbox_Win32.exe differ diff --git a/imosToolbox_Win64.exe b/imosToolbox_Win64.exe index 4448d938b..26589abc2 100644 Binary files a/imosToolbox_Win64.exe and b/imosToolbox_Win64.exe differ