From 0ef3e440d1969b94503eee7dafce68dc6a49e333 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Wed, 28 Oct 2020 11:22:31 +1100 Subject: [PATCH 01/13] feat(Util): errormsg function A wrapper to error, but with better stack tracing when printing. --- Util/errormsg.m | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 Util/errormsg.m diff --git a/Util/errormsg.m b/Util/errormsg.m new file mode 100644 index 000000000..9b4e69da2 --- /dev/null +++ b/Util/errormsg.m @@ -0,0 +1,43 @@ +function errormsg(msg,varargin) +% function errormsg(msg,varargin) +% +% A Wrapper to error, but showing where +% the error ocurred. +% +% Inputs: +% +% msg - A message string. +% varargin - sprintf/further arguments (e.g.fields for msg). +% +% +% Example: +% +% %basic usage +% +% +% author: hugo.oliveira@utas.edu.au +% +cstack = dbstack; +if numel(cstack) == 1 + name = cstack.name; + stack = cstack.stack; +else + name = cstack(2).name; + stack = cstack(2:end); +end + +actualmsg = sprintf(msg,varargin{:}); +dotsplit = split(name,'.'); +if numel(dotsplit) == 1 + name_as_id = sprintf('%s:%s',dotsplit{1},dotsplit{1}); +else + id_msg = repmat('%s:',1,numel(dotsplit)); + id_msg = id_msg(1:end-1); + name_as_id = sprintf(id_msg,dotsplit{:}); +end +actual_exception = MException(name_as_id,actualmsg); +try + actual_exception.none +catch me + throwAsCaller(actual_exception) +end From f01917904793d004a1aa7047748283247849d805 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Thu, 29 Oct 2020 09:55:58 +1100 Subject: [PATCH 02/13] feat(bugfix): fix output size in struct2parameters --- Util/StructUtils/struct2parameters.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Util/StructUtils/struct2parameters.m b/Util/StructUtils/struct2parameters.m index 4f85f6c8a..8afd8b3f9 100644 --- a/Util/StructUtils/struct2parameters.m +++ b/Util/StructUtils/struct2parameters.m @@ -41,7 +41,7 @@ % keys = fieldnames(astruct); values = struct2cell(astruct); -pcell = cell(1, numel(keys)*numel(values)); +pcell = cell(1, numel(keys)*2); c=-1; for k = 1:length(keys) c=c+2; From 1959c8b91c07ad1be76f01b0a8637e9b72055b4c Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Tue, 27 Oct 2020 12:22:05 +1100 Subject: [PATCH 03/13] feat(bugfix): test_docstring/check_docstrings This commit fixes the too many open files problem in `test_docstring` by adding a missing `fclose` on the file id. This can happen during long sessions or repeated calls to `test_docstring`/`check_docstrings`. A small fix is also provided in `check_docstrings` where a call to an invalid/empty folder was not generating an error. --- Util/TestUtils/check_docstrings.m | 4 ++++ Util/TestUtils/test_docstring.m | 2 ++ 2 files changed, 6 insertions(+) diff --git a/Util/TestUtils/check_docstrings.m b/Util/TestUtils/check_docstrings.m index f693dc5e0..015f2975e 100644 --- a/Util/TestUtils/check_docstrings.m +++ b/Util/TestUtils/check_docstrings.m @@ -25,6 +25,10 @@ wrong_files = {}; files = rdir(folder); +if isempty(files) + errormsg('No files presnet at %s folder',folder) +end + srcfiles = cell2mat(cellfun(@is_matlab,files,'UniformOutput',false)); matlab_files = files(srcfiles); self_calling = contains(matlab_files,'check_docstrings.m'); diff --git a/Util/TestUtils/test_docstring.m b/Util/TestUtils/test_docstring.m index 398d64ec2..50ea05096 100644 --- a/Util/TestUtils/test_docstring.m +++ b/Util/TestUtils/test_docstring.m @@ -69,4 +69,6 @@ msg = [file msg]; end +fclose(fid); + end From 6ceac6ca3fa2a0952508c567bd23a7703a39af13 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Mon, 26 Oct 2020 16:35:54 +1100 Subject: [PATCH 04/13] feat(Util): squashCells and col/row vec creation `squashCells` allows any number of cell arguments to be squashed together, in order, into a bigger cell with all cell items in alternate fashion. See squashCells docstring for examples. `col` is just a simple wrapper to create a column vector. `row`, ditto, but for row vectors. --- Util/CellUtils/squashCells.m | 86 ++++++++++++++++++++++++++++++++++++ Util/col.m | 29 ++++++++++++ Util/row.m | 28 ++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 Util/CellUtils/squashCells.m create mode 100644 Util/col.m create mode 100644 Util/row.m diff --git a/Util/CellUtils/squashCells.m b/Util/CellUtils/squashCells.m new file mode 100644 index 000000000..06225a2f1 --- /dev/null +++ b/Util/CellUtils/squashCells.m @@ -0,0 +1,86 @@ +function [mcell] = squashCells(varargin) +% function [mcell] = squashCells(varargin) +% +% Squash an arbitrary number of cell arguments +% sequentially into a cell of alternate items +% such as: +% +% ∀ k Z-* mcell(m+1:k*m) = {varargin{k*m/m}{k*m/m},...} +% +% Inputs: +% +% varargin - arbitrary cell arguments +% +% Outputs: +% +% mcell - a cell with all arguments squashed into one. +% +% Example: +% +% %typical usage +% c1={'A','B','C','D','E'}; +% c2={1,2,3}; +% c3={'F','G'}; +% mcell = squashCells(c1,c2,c3); +% assert(isequal(mcell([3,6]),c3)) +% assert(isequal(mcell([2,5,8]),c2)) +% assert(isequal(mcell([1,4,7,9,10]),c1)) +% +% %short to large cells +% c1={'a'}; +% c2={'b','d','f'}; +% c3={'c','e','g','h','i','j','k'}; +% mcell = squashCells(c1,c2,c3); +% assert(isequal(mcell,num2cell('abcdefghijk'))) +% +% +% %empty is ignored. +% [mcell] = squashCells({1,2,3},{},{'A','B','C'}); +% assert(isequal(mcell,{1,'A',2,'B',3,'C'})); +% +% author: hugo.oliveira@utas.edu.au +% +if any(~cellfun(@iscell,varargin)) + error('%s: Cannot merge non cell objects',mfilename) +end + +fcells = ~cellfun(@isempty,varargin); +varargin = varargin(fcells); + +totalgroups = numel(varargin); +sizes = cellfun(@numel, varargin); + +mcell = cell(1, sum(sizes)); +inds = cell(1, totalgroups); + +for k = 1:totalgroups + inds{k} = zeros(1, sizes(k)); +end + +ngroups = totalgroups; +gsizes = col(num2cell(sizes)); +slots = gsizes; +group_index_store = num2cell(col(1:ngroups)); +k = 1; +group = find([group_index_store{:}] == k, 1); + +while ~isempty(group) + if slots{group} ~= 0 + slots{group} = slots{group} - 1; + index_in_group = gsizes{group} - slots{group}; + inds{group}(index_in_group) = k; + group_index_store{group} = group_index_store{group} + ngroups; + k = k + 1; + group = find([group_index_store{:}] == k, 1); + else + group_index_store = num2cell([group_index_store{:}] - 1); + ngroups = ngroups - 1; + group = find([group_index_store{:}] == k, 1); + end +end + +for k = 1:totalgroups + mcell(inds{k}) = varargin{k}; +end + +end diff --git a/Util/col.m b/Util/col.m new file mode 100644 index 000000000..4dc1adb70 --- /dev/null +++ b/Util/col.m @@ -0,0 +1,29 @@ +function [col] = col(array) +% function [col] = col(array) +% +% Compose a col vector from array. +% +% Inputs: +% +% array - an array. +% +% Outputs: +% +% col - a col vector. +% +% Example: +% +% assert(isequal(col([1;2;3]),[1;2;3])) +% assert(isequal(col([1,2,3]),[1;2;3])) +% assert(isequal(col(ones(3,3)),repmat(1,9,1))) +% assert(iscolumn(col(randn(1,30)))) +% +% author: hugo.oliveira@utas.edu.au +% +if iscolumn(array) + col = array; +else + col = array(:); +end + +end diff --git a/Util/row.m b/Util/row.m new file mode 100644 index 000000000..98c2ab102 --- /dev/null +++ b/Util/row.m @@ -0,0 +1,28 @@ +function [row] = row(array) +% function [row] = row(array) +% +% Compose a row vector from array. +% +% Inputs: +% +% array - an array. +% +% Outputs: +% +% row - a row vector. +% +% Example: +% +% assert(isequal(row([1,2,3]),[1,2,3])) +% assert(isequal(row(ones(3,3)),repmat(1,1,9))) +% assert(isrow(row(randn(30,1)))) +% +% author: hugo.oliveira@utas.edu.au +% +if isrow(array) + row = array; +else + row = transpose(array(:)); +end + +end From 22a9eef9b3c5855cec216ba9f78fa5da8dd38ef8 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Mon, 26 Oct 2020 16:40:11 +1100 Subject: [PATCH 05/13] feat(Util): getindex & wrappers for type validation `getindex` is a wrapper to get the index value of common matlab types. It serves the purpose to be used inside an anonymous function, and to extract a fieldname of a struct with the respective fieldname index. `isindex` evaluate if the argument is a valid array index: 1. non-zero positive integer; or 2. non-zero positive integer array; or 3. logical array with at least 1 true value; `issize` is just a a checker for a positive integer numeric array. Finally, simple wrappers to check the types within a cell are: `iscellfh` - check items are function-handles. `iscellnumeric` - ditto, but for any numeric value. `iscellstruct` - ditto, but for structs. `iscellsize` - ditto, but for positive, non-zero, and row vector only integers. --- Util/CellUtils/iscellfh.m | 36 +++++++++++++++++++++ Util/CellUtils/iscellnumeric.m | 35 +++++++++++++++++++++ Util/CellUtils/iscellsize.m | 36 +++++++++++++++++++++ Util/CellUtils/iscellstruct.m | 35 +++++++++++++++++++++ Util/getindex.m | 57 ++++++++++++++++++++++++++++++++++ Util/isindex.m | 46 +++++++++++++++++++++++++++ Util/issize.m | 40 ++++++++++++++++++++++++ 7 files changed, 285 insertions(+) create mode 100644 Util/CellUtils/iscellfh.m create mode 100644 Util/CellUtils/iscellnumeric.m create mode 100644 Util/CellUtils/iscellsize.m create mode 100644 Util/CellUtils/iscellstruct.m create mode 100644 Util/getindex.m create mode 100644 Util/isindex.m create mode 100644 Util/issize.m diff --git a/Util/CellUtils/iscellfh.m b/Util/CellUtils/iscellfh.m new file mode 100644 index 000000000..cea23f578 --- /dev/null +++ b/Util/CellUtils/iscellfh.m @@ -0,0 +1,36 @@ +function [bool] = iscellfh(acell) +% function [bool] = iscellfh(acell) +% +% Determine whether input is a cell array of +% function handles +% +% Inputs: +% +% acell [cell[array]] - a cell to be verified. +% +% Outputs: +% +% bool - True or False. +% +% Example: +% +% assert(iscellfh({@double})); +% assert(~iscellfh({@double,''})); +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) + +bool = false; + +if isempty(acell) + return +end + +try + bool = all(cellfun(@isfunctionhandle, acell)); +catch +end + +end diff --git a/Util/CellUtils/iscellnumeric.m b/Util/CellUtils/iscellnumeric.m new file mode 100644 index 000000000..b8499cbdc --- /dev/null +++ b/Util/CellUtils/iscellnumeric.m @@ -0,0 +1,35 @@ +function [bool] = iscellnumeric(acell) +% function [bool] = iscellnumeric(acell) +% +% Determine whether input is a cell array of +% numeric arrays. +% +% Inputs: +% +% acell [cell[array]] - a cell to be verified. +% +% Outputs: +% +% bool - True or False. +% +% Example: +% +% assert(iscellnumeric({1,2,3})) +% assert(~iscellnumeric({1,2,'a'})) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) +bool = false; + +if isempty(acell) + return +end + +try + bool = all(cellfun(@isnumeric, acell)); +catch +end + +end diff --git a/Util/CellUtils/iscellsize.m b/Util/CellUtils/iscellsize.m new file mode 100644 index 000000000..a240a73e5 --- /dev/null +++ b/Util/CellUtils/iscellsize.m @@ -0,0 +1,36 @@ +function [bool] = iscellsize(acell) +% function [bool] = iscellsize(acell) +% +% Determine whether input is a cell array of +% numeric size arrays. +% +% Inputs: +% +% acell [cell[array]] - a cell to be verified. +% +% Outputs: +% +% bool - True or False. +% +% Example: +% +% assert(iscellsize({[1,10]})) +% assert(~iscellsize({1,2,[1, 10],[10, 99, 33]})) +% assert(~iscellsize({1,2,'a'})) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) +bool = false; + +if isempty(acell) + return +end + +try + bool = all(cellfun(@issize, acell)); +catch +end + +end diff --git a/Util/CellUtils/iscellstruct.m b/Util/CellUtils/iscellstruct.m new file mode 100644 index 000000000..441618494 --- /dev/null +++ b/Util/CellUtils/iscellstruct.m @@ -0,0 +1,35 @@ +function [bool] = iscellstruct(acell) +% function [bool] = iscellstruct(acell) +% +% Determine whether input is a cell array of +% structs. +% +% Inputs: +% +% acell [cell[array]] - a cell to be verified. +% +% Outputs: +% +% bool - True or False. +% +% Example: +% +% assert(iscellstruct({struct()})); +% assert(~iscellstruct({struct(),'abc'})) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) +bool = false; + +if isempty(acell) + return +end + +try + bool = all(cellfun(@isstruct, acell)); +catch +end + +end diff --git a/Util/getindex.m b/Util/getindex.m new file mode 100644 index 000000000..eb2c8e4be --- /dev/null +++ b/Util/getindex.m @@ -0,0 +1,57 @@ +function [value] = getindex(arg,index) +% function [value] = getindex(arg,index) +% +% Get an index from an argument. +% +% Inputs: +% +% arg [string | char | logical | cell | struct | array] - an argument. +% +% Outputs: +% +% value - If a struct, this is or arg.(fieldnames(arg)(index)), +% If a cell, this is arg(index), +% Otherwise, this is type(arg)(arg(index)). +% +% Example: +% +% %basic usage +% assert(isequal(getindex({1,2,3},3),3)); %parenthesis access +% assert(~isequal(getindex({1,2,{3}},3),{{3}})); %double cell inconsistency +% assert(isequal(getindex(struct('one',1,'two',2),2),2)); +% assert(isequal(getindex([true,true,false],3),false)); +% +% %raise error for invalid inputs +% f=false;try;getindex(1,10);catch;f=true;end +% assert(f) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(2,2); + +aclass = class(arg); + +if isnumeric(arg) && isscalar(arg) + errormsg('Argument is a scalar.') +end +access_is_by_fieldname = isstruct(arg); +access_is_by_brackets = iscell(arg); +access_is_by_parenthesis = islogical(arg) || ischar(arg) || isstring(arg) || isnumeric(arg); + +if ~access_is_by_fieldname && ~access_is_by_brackets && ~access_is_by_parenthesis + errormsg('Index fetching for class %d is not implemented.',aclass) +end + +try + if access_is_by_fieldname + fnames = fieldnames(arg); + value = arg.(fnames{index}); + elseif access_is_by_brackets + value = arg{index}; + elseif access_is_by_parenthesis + value = arg(index); + end +catch + errormsg('Index %d out of range',index) +end diff --git a/Util/isindex.m b/Util/isindex.m new file mode 100644 index 000000000..0c30e7534 --- /dev/null +++ b/Util/isindex.m @@ -0,0 +1,46 @@ +function [bool] = isindex(item) +% function [bool] = isindex(item) +% +% Determine whether input is a valid +% array index - positive integer array +% or logical array. +% +% Inputs: +% +% item - a strictly integer positive array +% or logical array. +% +% Outputs: +% +% bool - true or false. +% +% Example: +% +% assert(isindex(1)) +% assert(isindex([1,2,3])) +% assert(isindex([0,1,0,1])) +% assert(~isindex(0)) +% assert(~isindex([0,0,0])) +% assert(~isindex([-1,2,3])) +% assert(~isindex([1.1,2.2])) +% assert(~isindex([NaN,Inf,3])) +% assert(~isindex([1,2,'a'])) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1); +bool = false; + +if ~isnumeric(item) + return +end + +try + i64 = int64(item); + if isequal(i64, item) && all(i64>-1) && any(logical(item)) + bool = true; + end +catch +end +end diff --git a/Util/issize.m b/Util/issize.m new file mode 100644 index 000000000..2ca1cc377 --- /dev/null +++ b/Util/issize.m @@ -0,0 +1,40 @@ +function [bool] = issize(carray) +% function [bool] = issize(carray) +% +% Check if carray is a valid +% and complete size array. +% +% A valid and complete size array +% is a non-scalar row vector +% with positive integers only. +% +% Inputs: +% +% carray - an array. +% +% Outputs: +% +% bool - true or false. +% +% Example: +% +% %basic +% assert(issize([1,10])) +% +% %a scalar size is incomplete +% assert(~issize(1)) +% +% %a logical size is invalid +% assert(~issize([0,1])) +% +% %a column-vector is invalid +% assert(~issize([1;1])) +% +% %a fractional double value is invalid +% assert(~issize([1.5,3.3])) +% +% +% author: hugo.oliveira@utas.edu.au +% +bool = ~isscalar(carray) && isrow(carray) && isindex(carray) && all(carray > 0); +end From 52cfa2ddb90bef9aa3ace980e55c7e207bfe330f Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Mon, 26 Oct 2020 17:01:21 +1100 Subject: [PATCH 06/13] feat(Util): remove singleton dims from size arrays `remove_singleton` is useful for size matching verification. --- Util/remove_singleton.m | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 Util/remove_singleton.m diff --git a/Util/remove_singleton.m b/Util/remove_singleton.m new file mode 100644 index 000000000..8845fbc14 --- /dev/null +++ b/Util/remove_singleton.m @@ -0,0 +1,36 @@ +function [nssize] = remove_singleton(sizearray) +% function [nssize] = remove_singleton(sizearray) +% +% Remove the singleton dimension sizes +% from a sizearray +% +% Inputs: +% +% sizearray - A size array output +% +% Outputs: +% +% nssize - A size array without singleton (1's) +% +% Example: +% +% assert(isequal(remove_singleton([1,1,10,30]),[10,30])); +% assert(isequal(remove_singleton([2,3,4]),[2,3,4])); +% assert(isempty(remove_singleton([1]))) +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1,1) +if ~isindex(sizearray) + error('%s: Not a numeric index array') +end + +nssize = sizearray; +singletons = find(nssize==1); +if singletons + nssize = num2cell(nssize); + nssize(singletons) = []; + nssize = cell2mat(nssize); +end + +end From a57cdb352c40e650a0a4cfa31cb4f6c6b9ceec67 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Mon, 26 Oct 2020 17:02:17 +1100 Subject: [PATCH 07/13] feat(Util): allrepeats, inclusive. Inclusive find of all repetition indexes in an array. --- Util/allrepeats.m | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 Util/allrepeats.m diff --git a/Util/allrepeats.m b/Util/allrepeats.m new file mode 100644 index 000000000..0b48344ba --- /dev/null +++ b/Util/allrepeats.m @@ -0,0 +1,38 @@ +function [bind] = allrepeats(array) +% function [] = allrepeats(array) +% +% Inclusive find of all repeated +% values within an array. +% +% Inputs: +% +% array - the array +% +% Outputs: +% +% bind - indexes where of repeated +% values, including the first +% item. +% +% Example: +% +% %basic usage +% bind = allrepeats([1,2,3,1,5,1]); +% assert(any(bind)); +% assert(isequal(bind,[1,4,6])); +% +% %no repeats +% assert(~any(allrepeats([1,2,3]))) +% +% author: hugo.oliveira@utas.edu.au +% +if ~isnumeric(array) + error('%s: first argument is not a numeric array',mfilename); +end +bind = []; +[uniq,~,uind] = isunique(array); +if ~uniq + aind = 1:numel(array); + repeats = array(setdiff(aind,uind)); + bind = find(ismember(array,repeats)); +end From 2fe17f75635c06c99334ba7d9c9c22272d8362d1 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Mon, 26 Oct 2020 20:05:53 +1100 Subject: [PATCH 08/13] feat(IMOS): IMOS package utilities The IMOS package allows some better access/assignment/generation patterns for the existing IMOS Toolbox data structures. In summary, this contains some type-checked utilities which allows: 1. Shorter ways to access fieldname/values from cell of structs. Some access and return patterns are now handled explicitly in one-liners with functions like `IMOS.get`, `IMOS.getitem`, `IMOS.get_data_sizes`, `IMOS.get_data_numel`. 2. Easy check/validation of variable data sizes against dimensions at cell initialisation time. See `IMOS.discover_data_dimensions`. 3. Easy composition of Dimensions and Variables cells. See `IMOS.gen_variables` and `IMOS.gen_dimensions`. 4. Name and imos type resolutions. See `IMOS.resolve.imos_type` and `IMOS.resolve.name`. 5. Better imosParameter table access and discovery via cell or struct lookup. See `IMOS.params` and `IMOS.varparams`. 6. Random generation of names, types, and data plus random IMOS parameter cherry-picking. See `IMOS.random` functions. 7. Type checking of common IMOS objects (dimensions,variables). See `IMOS.is_toolbox_` type of functions. --- Util/+IMOS/+random/get_random_param.m | 39 +++++++ Util/+IMOS/+random/imos_data.m | 67 +++++++++++ Util/+IMOS/+random/imos_name.m | 21 ++++ Util/+IMOS/+random/imos_type.m | 19 ++++ Util/+IMOS/+resolve/imos_type.m | 46 ++++++++ Util/+IMOS/+resolve/name.m | 37 ++++++ Util/+IMOS/+templates/dimensions.m | 40 +++++++ Util/+IMOS/cellfun.m | 77 +++++++++++++ Util/+IMOS/discover_data_dimensions.m | 97 ++++++++++++++++ Util/+IMOS/featuretype_variables.m | 58 ++++++++++ Util/+IMOS/gen_dimensions.m | 158 ++++++++++++++++++++++++++ Util/+IMOS/gen_variables.m | 158 ++++++++++++++++++++++++++ Util/+IMOS/get.m | 49 ++++++++ Util/+IMOS/get_data_numel.m | 33 ++++++ Util/+IMOS/get_data_sizes.m | 33 ++++++ Util/+IMOS/getitem.m | 63 ++++++++++ Util/+IMOS/has_fieldname.m | 45 ++++++++ Util/+IMOS/is_imos_name.m | 29 +++++ Util/+IMOS/is_toolbox_dim.m | 39 +++++++ Util/+IMOS/is_toolbox_dimcell.m | 27 +++++ Util/+IMOS/is_toolbox_var.m | 38 +++++++ Util/+IMOS/is_toolbox_varcell.m | 28 +++++ Util/+IMOS/params.m | 63 ++++++++++ Util/+IMOS/varparams.m | 64 +++++++++++ 24 files changed, 1328 insertions(+) create mode 100644 Util/+IMOS/+random/get_random_param.m create mode 100644 Util/+IMOS/+random/imos_data.m create mode 100644 Util/+IMOS/+random/imos_name.m create mode 100644 Util/+IMOS/+random/imos_type.m create mode 100644 Util/+IMOS/+resolve/imos_type.m create mode 100644 Util/+IMOS/+resolve/name.m create mode 100644 Util/+IMOS/+templates/dimensions.m create mode 100644 Util/+IMOS/cellfun.m create mode 100644 Util/+IMOS/discover_data_dimensions.m create mode 100644 Util/+IMOS/featuretype_variables.m create mode 100644 Util/+IMOS/gen_dimensions.m create mode 100644 Util/+IMOS/gen_variables.m create mode 100644 Util/+IMOS/get.m create mode 100644 Util/+IMOS/get_data_numel.m create mode 100644 Util/+IMOS/get_data_sizes.m create mode 100644 Util/+IMOS/getitem.m create mode 100644 Util/+IMOS/has_fieldname.m create mode 100644 Util/+IMOS/is_imos_name.m create mode 100644 Util/+IMOS/is_toolbox_dim.m create mode 100644 Util/+IMOS/is_toolbox_dimcell.m create mode 100644 Util/+IMOS/is_toolbox_var.m create mode 100644 Util/+IMOS/is_toolbox_varcell.m create mode 100644 Util/+IMOS/params.m create mode 100644 Util/+IMOS/varparams.m diff --git a/Util/+IMOS/+random/get_random_param.m b/Util/+IMOS/+random/get_random_param.m new file mode 100644 index 000000000..eaa6d09e3 --- /dev/null +++ b/Util/+IMOS/+random/get_random_param.m @@ -0,0 +1,39 @@ +function [param] = get_random_param(param_name) +% function [param] = get_random_param(param_name) +% +% Get a random IMOS parameter index. +% +% Inputs: +% +% param_name - the parameter name. See IMOS.param. +% +% Outputs: +% +% param - A random item. +% +% Example: +% +% valid_entry = @(x)(~isempty(x) && ischar(x)); +% assert(valid_entry(IMOS.random.get_random_param('name'))) +% assert(valid_entry(IMOS.random.get_random_param('long_name'))) +% assert(valid_entry(IMOS.random.get_random_param('netcdf_ctype'))) +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) + +if ~ischar(param_name) + error('First argument is not a char') +end + +imosparams = IMOS.params(); +available_names = fieldnames(imosparams); + +if ~inCell(available_names, param_name) + errormsg('IMOS parameter %s do not exist', param_name) +end + +all_values = unique(imosparams.(param_name)); +rind = random_between(1, numel(all_values), 1, 'int'); +param = all_values{rind}; +end diff --git a/Util/+IMOS/+random/imos_data.m b/Util/+IMOS/+random/imos_data.m new file mode 100644 index 000000000..06197e3a4 --- /dev/null +++ b/Util/+IMOS/+random/imos_data.m @@ -0,0 +1,67 @@ +function [rdata] = imos_data(imos_type, data_size) +% function [rdata] = imos_data(imos_type,data_size) +% +% Generate a random IMOS variable +% given a type and a size. +% A column vector is used if data_size provided +% is singleton. +% +% If no arguments, a double random data of size [100x1] +% is generated. +% +% Inputs: +% +% imos_type [function_handle] - the fh imos type. +% data_size [array] - an size array to use +% in the resolution of the data. +% +% Outputs: +% +% rdata [type] - the random data array of specific type +% +% Example: +% +% %random generation 100xN +% [rdata] = IMOS.random.imos_data(); +% assert(iscolumn(rdata) && numel(rdata)==100) +% +% %random typed +% [rdata] = IMOS.random.imos_data(@int32); +% assert(isint32(rdata) && iscolumn(rdata) && numel(rdata)==100) +% +% %random typed with defined size +% [rdata] = IMOS.random.imos_data(@int32,[6,1]); +% assert(isint32(rdata) && iscolumn(rdata) && numel(rdata)==6) +% [rdata] = IMOS.random.imos_data(@int32,[1,6]); +% assert(isint32(rdata) && isrow(rdata) && numel(rdata)==6) +% [rdata] = IMOS.random.imos_data(@int32,[6,6]); +% assert(isint32(rdata) && isnumeric(rdata) && numel(rdata)==36) +% +% %invalid type argument +% f=false;try; IMOS.random.imos_data('float');catch;f=true;end +% assert(f) +% +% %invalid size argument +% f=false;try; IMOS.random.imos_data(@double,1);catch;f=true;end +% assert(f) +% +% +% author: hugo.oliveira@utas.edu.au +% + +if nargin == 0 + rdata = randn(100, 1); + return +elseif nargin > 0 && ~isfunctionhandle(imos_type) + errormsg('First argument `imos_type` is not a function handle') +elseif nargin > 1 && ~issize(data_size) + errormsg('Second argument `data_size` is not a valid size') +end + +if nargin == 1 + rdata = imos_type(randn(100, 1)); +else + rdata = imos_type(randn(data_size)); +end + +end diff --git a/Util/+IMOS/+random/imos_name.m b/Util/+IMOS/+random/imos_name.m new file mode 100644 index 000000000..c1e12a6bf --- /dev/null +++ b/Util/+IMOS/+random/imos_name.m @@ -0,0 +1,21 @@ +function [name] = imos_name +% function [name] = imos_name +% +% Generate a random IMOS name +% +% Inputs: +% +% Outputs: +% +% name [string] - an IMOS name from imosParameters.txt +% +% Example: +% +% [name] = IMOS.random.imos_name(); +% assert(~isempty(name)) +% assert(all(isstrprop(name,'print'))) +% +% author: hugo.oliveira@utas.edu.au +% +name = IMOS.random.get_random_param('name'); +end diff --git a/Util/+IMOS/+random/imos_type.m b/Util/+IMOS/+random/imos_type.m new file mode 100644 index 000000000..602e8f822 --- /dev/null +++ b/Util/+IMOS/+random/imos_type.m @@ -0,0 +1,19 @@ +function [rtype] = imos_type +% function [rtype] = imos_type +% +% Generate a random imos type function handle +% +% Inputs: +% +% Outputs: +% +% rtype [function_handle] - the function handle of the type. +% +% Example: +% +% assert(isa(IMOS.random.imos_type,'function_handle')) +% +% author: hugo.oliveira@utas.edu.au +% +rtype = str2func(netcdf3ToMatlabType(IMOS.random.get_random_param('netcdf_ctype'))); +end diff --git a/Util/+IMOS/+resolve/imos_type.m b/Util/+IMOS/+resolve/imos_type.m new file mode 100644 index 000000000..cb295a8ae --- /dev/null +++ b/Util/+IMOS/+resolve/imos_type.m @@ -0,0 +1,46 @@ +function [itype] = imos_type(imos_name) +% function [itype] = imos_type(imos_name) +% +% Resolve an IMOS data type function handle +% based on its name. +% +% If the name is not an IMOS dimension/variable name, +% an error is raised. +% +% Inputs: +% +% imos_name [str] - an IMOS variable name to use +% in the resolution of the type. +% +% Outputs: +% +% itype [function_handle] - the resolved function handle. +% +% Example: +% +% %basic +% assert(isequal(IMOS.resolve.imos_type('TIME'),@double)); +% +% %imos_name missing +% f=false;try;IMOS.resolve.imos_type('ABC');catch;f=true;end; +% assert(f) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) + +if ~ischar(imos_name) + errormsg('First argument is not a valid string') +end + +params = IMOS.params(); +[name_exist, name_ind] = inCell(params.name, imos_name); + +if ~name_exist + errormsg('Name %s is missing within the imosParameter table.', imos_name) +else + itype = str2func(netcdf3ToMatlabType(params.netcdf_ctype{name_ind})); +end + +end diff --git a/Util/+IMOS/+resolve/name.m b/Util/+IMOS/+resolve/name.m new file mode 100644 index 000000000..73c569b95 --- /dev/null +++ b/Util/+IMOS/+resolve/name.m @@ -0,0 +1,37 @@ +function [rname] = name(cnames, rindex) +% function [rname] = name(cnames,rindex) +% +% Return cnames{rindex} if cnames is a cell +% of strings and rindex 1) + errormsg('Second argument `rindex` is not a singleton valid index.') +end + +rname = IMOS.getitem(cnames, rindex); +end diff --git a/Util/+IMOS/+templates/dimensions.m b/Util/+IMOS/+templates/dimensions.m new file mode 100644 index 000000000..48abca472 --- /dev/null +++ b/Util/+IMOS/+templates/dimensions.m @@ -0,0 +1,40 @@ +classdef dimensions + % A collection of minimal IMOS dimensions templates. + properties (Constant) + timeseries = timeseries_dims(); + profile = profile_dims(); + ad_profile = ad_profile_dims(); + adcp_raw = adcp_raw_dims(); + adcp_remapped = adcp_remapped_dims(); + end +end + +function [dimensions] = timeseries_dims() +%create basic timeseries dimensions +dimensions = {struct('name','TIME','typeCastFunc',getIMOSType('TIME'),'data',[],'comment','')}; +end + +function [dimensions] = profile_dims() +%create basic profile dimensions +dimensions = {struct('name','DEPTH','typeCastFunc',getIMOSType('DEPTH'),'data',[],'comment','','axis','Z')}; +end + +function [dimensions] = ad_profile_dims() +%create basic ad profile dimensions +dimensions = cell(1,2); +dimensions{1} = struct('name','MAXZ','typeCastFunc',getIMOSType('MAXZ'),'data',[]); +dimensions{2} = struct('name','PROFILE','typeCastFunc',getIMOSType('PROFILE'),'data',[]); +dimensions{2}.data = dimensions{2}.typeCastFunc([1,2]); +end + +function [dimensions] = adcp_raw_dims() +dimensions = timeseries_dims(); +dimensions{2} = struct('name','DIST_ALONG_BEAMS','typeCastFunc',getIMOSType('DIST_ALONG_BEAMS'),'data',[]); +end + +function [dimensions] = adcp_remapped_dims() +dimensions = timeseries_dims(); +dimensions{2} = struct('name','HEIGHT_ABOVE_SENSOR','typeCastFunc',getIMOSType('HEIGHT_ABOVE_SENSOR'),'data',[]); +end + + diff --git a/Util/+IMOS/cellfun.m b/Util/+IMOS/cellfun.m new file mode 100644 index 000000000..9f5d6f84b --- /dev/null +++ b/Util/+IMOS/cellfun.m @@ -0,0 +1,77 @@ +function [result, failed_items] = cellfun(func, icell) +% function [result, failed_items] = cellfun(func,icell) +% +% A MATLAB cellfun function that wraps errors +% in empty results. +% The items that triggered the fails may be returned. +% +% Inputs: +% +% func [function_handle] - a FH to apply to each cell member +% icell [cell[Any]] - a cell where func will be applied itemwise. +% +% Outputs: +% +% result [cell{any}] - the result of func(icell{k}) +% failed_items [cell{any}] - the icell{k} items that erroed. +% +% Example: +% +% %valid inputs +% [result] = IMOS.cellfun(@iscell,{{},{}}); +% assert(all(cell2mat(result))); +% +% %empty results for invalid evaluations +% [result,failed_items] = IMOS.cellfun(@zeros,{'a',1,'b'}); +% assert(isempty(result{1})) +% assert(isequal(result{2},0)) +% assert(isempty(result{3})) +% assert(isequal(failed_items{1},'a')) +% assert(isequal(failed_items{2},'b')) +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(2, 2) + +if ~isfunctionhandle(func) + errormsg('First argument `func` is not a function handle') +elseif ~iscell(icell) + errormsg('Second argument `icell` is not a cell') +end + +nc = numel(icell); +result = cell(1, nc); +failed_items = {}; +fcount = 0; + +try + [result] = cellfun(func, icell, 'UniformOutput', false); +catch + + for k = 1:numel(icell) + + try + result{k} = func(icell{k}); + catch + + if nargout > 1 + + if fcount == 0 + failed_items = cell(1, nc - k); + end + + fcount = fcount + 1; + failed_items{fcount} = icell{k}; + end + + end + + end + +end + +if nargout > 1 && fcount > 0 + failed_items = failed_items(1:fcount); +end + +end diff --git a/Util/+IMOS/discover_data_dimensions.m b/Util/+IMOS/discover_data_dimensions.m new file mode 100644 index 000000000..b1380b4bc --- /dev/null +++ b/Util/+IMOS/discover_data_dimensions.m @@ -0,0 +1,97 @@ +function discovered_indexes = discover_data_dimensions(tdata, tdims) +% function [discovered_indexes] = discover_data_dimensions(tdata,tdims) +% +% Discover the respective dimensional indexes +% of a data array based on available IMOS toolbox dimensions. +% The function ignores the singleton dimensions. +% +% Inputs: +% +% tdata [array] - A nd-array. +% tdims [cell[struct]] - the IMOS toolbox dimensions cell. +% +% Outputs: +% +% discovered_indexes [array] - The cell indexes of the dimensions +% in tdims which match the size of tdata. +% +% Example: +% +% %basic +% a = zeros(5,1); +% b = IMOS.gen_dimensions('timeSeries',1,{'TIME'},{@single},{[1:5]'}); +% assert(IMOS.discover_data_dimensions(a,b)==1) +% +% %extended +% a = zeros(5,10); +% b = IMOS.gen_dimensions('timeSeries',3,{},{},{[1:3]',[1:5]',[1:10]'}); +% assert(all(IMOS.discover_data_dimensions(a,b)==[2,3])) +% +% %fails for row vectors +% a=zeros(2,1); +% b={}; +% f=false;try;IMOS.discover_data_dimensions(a,b);catch;f=true;end +% assert(f) +% +% %duplicated dimensions can't be discovered. +% a = zeros(2,1); +% b = IMOS.gen_dimensions('timeSeries',2,{},{},{[1:3]',[1:3]'}); +% try;IMOS.discover_data_dimensions(a,b);catch;r=true;end +% assert(r) +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(2, 2); + +if ~isnumeric(tdata) && ~iscell(tdata) + errormsg('First argument `tdata` is not a data array or cell') +elseif ~iscellstruct(tdims) + errormsg('Second argument `tdims` is not a cell of structs') +end + +if isempty(tdata) + discovered_indexes = []; + return +end + +available_dims_names = IMOS.get(tdims, 'name'); +available_dims_len = IMOS.get_data_numel(tdims); +available_dims_len_as_array = cell2mat(available_dims_len); + +tdata_size = size(tdata); +non_singleton_var_dims_len = remove_singleton(tdata_size); +[discovered_indexes] = whereincell(available_dims_len, num2cell(non_singleton_var_dims_len)); + +try + invalid_discovery = ~isequal(numel(tdata), prod(available_dims_len_as_array(discovered_indexes))); +catch + invalid_discovery = true; +end + +if invalid_discovery + missing_dims_len = setdiff(non_singleton_var_dims_len, available_dims_len_as_array); + mfmt = repmat('%d ', 1, numel(missing_dims_len)); + errormsg(['Provided `data` contains undefined dimensions of length(s): [ ' mfmt '].'], missing_dims_len); +end + +repeats = allrepeats(available_dims_len_as_array); +discovered_are_repeats = any(ismember(repeats, discovered_indexes)); + +if discovered_are_repeats + blame_names = available_dims_names(repeats); + blame_lens = available_dims_len(repeats); + dfmt = repmat('%d,', 1, numel(tdata_size)); + dfmt = dfmt(1:end - 1); + nfmt = repmat('%s[%d], ', 1, numel(blame_names)); + nfmt = nfmt(1:end - 1); + eargs = cat(2, num2cell(tdata_size), squashCells(blame_names, blame_lens)); + errormsg(['Impossible dimensional discovery for `data` with size=[' dfmt ']. Dimensions ' nfmt ' are of the same length.'], eargs{:}) +end + +incomplete_discovery = ~isequal(numel(discovered_indexes), numel(non_singleton_var_dims_len)); + +if incomplete_discovery + discovered_indexes = []; +end + +end diff --git a/Util/+IMOS/featuretype_variables.m b/Util/+IMOS/featuretype_variables.m new file mode 100644 index 000000000..8baf53fba --- /dev/null +++ b/Util/+IMOS/featuretype_variables.m @@ -0,0 +1,58 @@ +function [variables] = featuretype_variables(featureType) +% function [variables] = featuretype_variables(featureType) +% +% Create the basic variables for a featureType. +% +% Inputs: +% +% featureType - The IMOS toolbox featureType. +% +% Outputs: +% +% variables - The basic variables required. +% +% Example: +% +% %basic timeSeries templating +% [basic_vars] = IMOS.featuretype_variables('timeSeries'); +% assert(strcmp(basic_vars{1}.name,'TIMESERIES')); +% assert(isequal(basic_vars{1}.data,1)) +% assert(strcmp(basic_vars{2}.name,'LATITUDE')) +% assert(isnan(basic_vars{2}.data)) +% assert(strcmp(basic_vars{3}.name,'LONGITUDE')) +% assert(isnan(basic_vars{3}.data)) +% assert(strcmp(basic_vars{4}.name,'NOMINAL_DEPTH')) +% assert(isnan(basic_vars{4}.data)) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) + +if ~ischar(featureType) + errormsg('First argument `featureType` must be a string') +end + +ft = lower(featureType); + +switch ft + case 'timeseries' + ts_names = {'TIMESERIES', 'LATITUDE', 'LONGITUDE', 'NOMINAL_DEPTH'}; + ts_types = cellfun(@getIMOSType, ts_names, 'UniformOutput', false); + ts_data = {1, NaN, NaN, NaN}; + variables = IMOS.gen_variables(IMOS.gen_dimensions(ft), ts_names, ts_types, ts_data, 'comments', ''); + case 'ad_profile' + p_names = {'TIME', 'DIRECTION', 'LATITUDE', 'LONGITUDE', 'BOT_DEPTH'}; + p_types = cellfun(@getIMOSType, p_names, 'UniformOutput', false); + p_data = {NaN, {'A', 'D'}, [NaN NaN], [NaN NaN], [NaN NaN]}; + variables = IMOS.gen_variables(IMOS.gen_dimensions(ft), p_names, p_types, p_data, 'comments', ''); + case 'profile' + p_names = {'PROFILE', 'TIME', 'DIRECTION', 'LATITUDE', 'LONGITUDE', 'BOT_DEPTH'}; + p_types = cellfun(@getIMOSType, p_names, 'UniformOutput', false); + p_data = {1, NaN, {'D'}, NaN, NaN, NaN}; + variables = IMOS.gen_variables(IMOS.gen_dimensions(ft), p_names, p_types, p_data, 'comments', ''); + otherwise + variables = {}; +end + +end diff --git a/Util/+IMOS/gen_dimensions.m b/Util/+IMOS/gen_dimensions.m new file mode 100644 index 000000000..5f48626b4 --- /dev/null +++ b/Util/+IMOS/gen_dimensions.m @@ -0,0 +1,158 @@ +function dimensions = gen_dimensions(mode, ndims, d_names, d_types, d_datac, varargin) +%function dimensions = gen_dimensions(mode, ndims, d_names, d_types, d_datac, varargin) +% +% Generate a toolbox dimension cell of structs. Empty or incomplete +% arguments will trigger random (names/types) or empty entries (data). +% +% Inputs: +% +% mode - `timeSeries` | `profile`. If empty, 'timeSeries' is used. +% ndims - number of dimensions [int]. If empty, 1 is used. +% d_names - dimension names [cell{str}]. if empty, randomized named. +% d_types - dimension types [cell{@function_handle}]. +% If d_names matches a toolbox variable, the variable +% type will be used, othewise randomized type is used. +% d_datac - dimension data [cell{any}]. Ditto as in d_names. +% varargin - extra parameters are cast to all structure fieldnames. +% +% Outputs: +% +% d - a cell with dimensions structs. +% +% Example: +% +% %basic construct +% dimensions = IMOS.gen_dimensions('timeSeries',1,{'TIME'},{@double},{[1:10]'},'calendar','gregorian','start_offset',10); +% tdim = dimensions{1}; +% assert(isequal(tdim.name,'TIME')) +% assert(isequal(tdim.typeCastFunc,@double)) +% assert(all(isequal(tdim.data,[1:10]'))); +% assert(strcmp(tdim.calendar,'gregorian')); +% assert(tdim.start_offset==10); +% +% %fill missing dimension spec with random stuff +% dimensions = IMOS.gen_dimensions('timeSeries',2,{'TIME'},{},{[1:100]'},'calendar','xxx'); +% tdim = dimensions{1}; +% assert(isequal(tdim.data,[1:100]')) +% newdim = dimensions{2}; +% assert(ischar(newdim.name)); +% assert(isfunctionhandle(newdim.typeCastFunc)) +% assert(isequal(newdim.data,[])); +% +% %raise error when same dimension name is used +% try;IMOS.gen_dimensions('timeSeries',3,{'A','B','A'});catch;r=true;end; +% assert(r); +% +% +% author: hugo.oliveira@utas.edu.au +% +if nargin == 0 + mode = 'timeSeries'; +elseif nargin > 1 && ~ischar(mode) + errormsg('First argument `mode` must be the toolbox mode.') +elseif nargin > 2 && ~isindex(ndims) + errormsg('Second argument `ndims` must be a valid number of dimensions.') +elseif nargin > 3 && ~iscell(d_names) + errormsg('Third argument `d_names` must be a cell.') +elseif nargin > 4 && ~iscell(d_types) + errormsg('Fourth argument `d_types` must be a cell.') +elseif nargin > 5 && ~iscell(d_datac) + errormsg('Fifth argument `d_datac` must be a cell.') +end + +try + got_any_name = numel(d_names) > 0; +catch + got_any_name = false; +end + +if got_any_name + unames = union(d_names, d_names); + names_not_unique = ~isequal(unames, sort(d_names)); + if names_not_unique + repeat_indexes = allrepeats(whereincell(unames, d_names)); + repeated_names = d_names(repeat_indexes); + repeated_names = union(repeated_names, repeated_names); + rfmt = repmat('`%s`,', 1, numel(repeated_names)); + rfmt(end) = '.'; + errormsg(['Invalid dimensions content. Dimensions not unique: ' rfmt], repeated_names{:}); + end + +end + +if nargin < 2 + + if strcmpi(mode, 'timeSeries') + dimensions = IMOS.templates.dimensions.timeseries; + return + elseif strcmpi(mode, 'profile') + dimensions = IMOS.templates.dimensions.profile; + return + elseif strcmpi(mode, 'ad_profile') + dimensions = IMOS.templates.dimensions.ad_profile; + return + else + ndims = 1; + d_names = random_names(); + d_types = random_numeric_typefun(); + typecast = d_types{1}; + d_datac = {typecast(randn(1, 100))}; + end + +end + +dimensions = cell(1, ndims); + +for k = 1:ndims + + try + name = d_names{k}; + catch + + try + dimensions{k} = IMOS.templates.dimensions.(mode){k}; + continue; + catch + name = random_names(1); + name = name{1}; + end + + end + + try + type = d_types{k}; + catch + + try + type = IMOS.resolve.imos_type(name); + catch + type = IMOS.random.imos_type(); + end + + end + + try + data = d_datac{k}; + catch + data = []; + end + + if ~isnumeric(data) + errormsg('%s dimension data is not numeric.', name) + end + + if ~isempty(data) && ~isscalar(data) + + if isrow(data) + fprintf('IMOS.%s: Transposing %s dimension at index=%d from row to column vector\n', mfilename, name, k); + data = data'; + elseif ~iscolumn(data) + errormsg('%s dimension data at index=%d is not a vector.', name, k); + end + + end + + dimensions{k} = struct('name', name, 'typeCastFunc', type, 'data', data, varargin{:}); +end + +end diff --git a/Util/+IMOS/gen_variables.m b/Util/+IMOS/gen_variables.m new file mode 100644 index 000000000..85cb8e8c5 --- /dev/null +++ b/Util/+IMOS/gen_variables.m @@ -0,0 +1,158 @@ +function [variables] = gen_variables(dimensions, v_names, v_types, v_data, varargin) +% function [variables] = gen_variables(dimensions,v_names, v_types,v_data,varargin) +% +% Generate a toolbox variable cell of structs. Empty or incomplete +% arguments will trigger random (names/types) or empty entries (data). +% +% Inputs: +% +% dimensions - a toolbox dimension cell. +% v_names - a cell of variable names. If cell is empty or +% out-of-bounds, a random entry is used. +% v_types - a cell of MATLAB function handles types. +% If name of variable is an IMOS parameter name, +% the respective imos type is used, otherwise +% a random type is used. +% v_data - a cell with the variable array values. ditto as in v_names. +% varargin - extra fieldname,fieldvalue to add to each variable. +% for example: 'comment','test' -> variables.comment = test. +% +% Outputs: +% +% variables - The variable cell. +% +% Example: +% %basic usage +% tsdims = IMOS.gen_dimensions('timeSeries'); +% tsvars = IMOS.gen_variables(tsdims); +% assert(iscellstruct(tsvars)) +% assert(strcmp(tsvars{1}.name,'TIMESERIES')) +% assert(tsvars{1}.data==1) +% assert(strcmp(tsvars{2}.name,'LATITUDE')) +% assert(strcmp(tsvars{end}.name,'NOMINAL_DEPTH')) +% +% % profile +% pdims = IMOS.gen_dimensions('profile'); +% pvars = IMOS.gen_variables(pdims); +% assert(iscellstruct(pvars)) +% assert(strcmp(pvars{1}.name,'PROFILE')) +% assert(pvars{1}.data==1) +% assert(strcmp(pvars{2}.name,'TIME')) +% assert(isequal(pvars{2}.typeCastFunc,@double)) +% assert(isempty(pvars{2}.dimensions)) %empty by design +% assert(isnan(pvars{2}.data)) %nan by design +% assert(strcmp(pvars{3}.name,'DIRECTION')) +% assert(isequal(pvars{3}.typeCastFunc,@char)) +% assert(isequal(pvars{3}.data,{'D'})) +% +% %misc usage +% mydims = IMOS.gen_dimensions('timeSeries',2,{'TIME','X'},{},{zeros(60,1),[1:10]'},'comment','123'); +% variables = IMOS.gen_variables(mydims,{'Y','Z','VALID'},{@double,@int32,@logical},{ones(10,1),zeros(60,1),1}); +% %one-to-one +% assert(strcmp(variables{1}.name,'Y')) +% assert(isequal(variables{1}.typeCastFunc,@double)) +% assert(isequal(variables{1}.data,ones(10,1))) +% assert(isequal(variables{1}.dimensions,2)) +% assert(isempty(variables{1}.comment)) +% %missing dims of variable vectors are assigned to empty and data is typecast +% assert(strcmp(variables{end}.name,'VALID')) +% assert(isequal(variables{end}.typeCastFunc,@logical)) +% assert(isequal(variables{end}.data,true)) +% assert(isempty(variables{end}.dimensions)) +% assert(isempty(variables{end}.comment)) +% +% %missing dimensions on multi-dimensional variable arrays trigger an error +% names = {'INVALID_COLUMN','INVALID_ROW','INVALID_MULTIARRAY'}; +% data = {ones(1,3),ones(3,1),ones(33,60)}; +% try;IMOS.gen_variables(mydims,names,{},data);catch;r=true;end; +% assert(r) +% +% +% author: hugo.oliveira@utas.edu.au +% +if nargin == 0 + errormsg('Missing toolbox dimensions cell argument') +end + +if nargin > 1 && ~IMOS.is_toolbox_dimcell(dimensions) + errormsg('First argument `dimensions` is not a toolbox dimensions cell') +elseif nargin > 2 && ~iscell(v_names) + errormsg('Second argument `v_names` is not a cell') +elseif nargin > 3 && ~iscell(v_types) + errormsg('Third argument `v_types` is not a cell') +elseif nargin > 4 && ~iscell(v_data) + errormsg('Fourth argument `v_data` is not acell') +end + +is_timeseries = getVar(dimensions, 'TIME') == 1; +is_ad_profile = ~is_timeseries && getVar(dimensions, 'MAXZ') && getVar(dimensions, 'PROFILE'); +is_single_profile = ~is_timeseries && ~is_ad_profile && getVar(dimensions, 'DEPTH'); + +if nargin < 2 && is_timeseries + variables = IMOS.featuretype_variables('timeSeries'); +elseif nargin < 2 && is_ad_profile + variables = IMOS.featuretype_variables('ad_profile'); +elseif nargin < 2 && is_single_profile + variables = IMOS.featuretype_variables('profile'); +else + variables = {}; +end + +if nargin < 2 + return +end + +if isempty(varargin) + varargin = {'comment', ''}; +end + +ns = numel(variables); +ndata = numel(v_names); +variables{ndata} = {}; + +for k = ns + 1:ndata + + try + name = v_names{k}; + catch + name = random_names(1); + name = name{1}; + end + + try + imos_type = v_types{k}; + catch + + try + imos_type = IMOS.resolve.imos_type(name); + catch + imos_type = IMOS.random.imos_type(); + end + + end + + try + data = v_data{k}; + catch + data = []; + end + + if isstring(data) || iscellstr(data) + is_profile = strcmp(dimensions{1}.name, 'DEPTH'); + + if is_profile + dim_indexes = []; + else + dim_indexes = 2; + end + + data = {data}; %wrap in double-cell for struct assignment. + variables{k} = struct('name', name, 'typeCastFunc', imos_type, 'dimensions', dim_indexes, 'data', data, varargin{:}); + else + dim_indexes = IMOS.discover_data_dimensions(data, dimensions); + variables{k} = struct('name', name, 'typeCastFunc', imos_type, 'dimensions', dim_indexes, 'data', imos_type(data), varargin{:}); + end + +end + +end diff --git a/Util/+IMOS/get.m b/Util/+IMOS/get.m new file mode 100644 index 000000000..cb3830bdf --- /dev/null +++ b/Util/+IMOS/get.m @@ -0,0 +1,49 @@ +function [fcell] = get(icell, fieldname) +% function [fcell] = get(icell,fieldname) +% +% Get a fieldname from a IMOS cell of structs. +% +% Inputs: +% +% icell [cell[struct]] - an IMOS cell of structs. +% fieldname - the struct fieldname. +% +% Outputs: +% +% fcell[Any] - A cell with all fieldnames. If fieldnames +% are missing, empty entries are included. +% +% Example: +% +% dimcell = {struct('name','TIME','typeCastFunc',@double,'data',[1:10],'comment','')}; +% dimcell{end+1} = struct('name','TEST'); +% [names] = IMOS.get(dimcell,'name'); +% assert(isequal(names,{'TIME','TEST'})); +% [comments] = IMOS.get(dimcell,'comment'); +% assert(ischar(comments{1}) && isempty(comments{1})) +% assert(isdouble(comments{2}) && isempty(comments{2})) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(2, 2); + +if ~iscellstruct(icell) + errormsg('First argument `icell` is not a cell of structs') +elseif ~ischar(fieldname) + errormsg('Second argument `fieldname` is not a string') +end + +get_field = @(obj)(resolve_field(obj, fieldname)); +fcell = cellfun(get_field, icell, 'UniformOutput', false); +end + +function r = resolve_field(obj, fname) + +try + r = obj.(fname); +catch + r = []; +end + +end diff --git a/Util/+IMOS/get_data_numel.m b/Util/+IMOS/get_data_numel.m new file mode 100644 index 000000000..eacb1ce65 --- /dev/null +++ b/Util/+IMOS/get_data_numel.m @@ -0,0 +1,33 @@ +function [cnumel] = get_data_numel(icell) +% function [cnumel] = get_data_numel(icell) +% +% Get the total number of elements of all +% the `data` fields in an IMOS cell of structs. +% +% If data field is missing, empty is returned. +% +% Inputs: +% +% icell [cell[struct]] - an IMOS cell of structs. +% +% Outputs: +% +% cnumel - the numel of each struct.data array members. +% +% Example: +% +% icell = {struct('name','TIME','typeCastFunc',@double,'data',[1:10])}; +% [cnumel] = IMOS.get_data_numel(icell); +% assert(isequal(cnumel{1},[10])); +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1); + +if ~iscellstruct(icell) + error('First argument is not a cell of structs') +end + +get_numel = @(obj)(numel(obj.data)); +cnumel = IMOS.cellfun(get_numel, icell); +end diff --git a/Util/+IMOS/get_data_sizes.m b/Util/+IMOS/get_data_sizes.m new file mode 100644 index 000000000..e97d59116 --- /dev/null +++ b/Util/+IMOS/get_data_sizes.m @@ -0,0 +1,33 @@ +function [csizes] = get_data_sizes(icell) +% function [csizes] = get_data_sizes(icell) +% +% Get the data size of all the `data` +% fields in an IMOS cell of structs. +% +% Inputs: +% +% icell [cell[struct]] - an IMOS cell of structs. +% +% Outputs: +% +% csizes - the sizes of struct.data array. +% +% Example: +% +% %typical use +% icell = {struct('name','TIME','typeCastFunc',@double,'data',[1:10])}; +% [csizes] = IMOS.get_data_sizes(icell); +% assert(isequal(csizes{1},[1,10])); +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1); + +if ~iscellstruct(icell) + error('First argument `icell` is not a cell of structs') +end + +get_size = @(obj)(size(obj.data)); +csizes = IMOS.cellfun(get_size, icell); +end diff --git a/Util/+IMOS/getitem.m b/Util/+IMOS/getitem.m new file mode 100644 index 000000000..2f555ff27 --- /dev/null +++ b/Util/+IMOS/getitem.m @@ -0,0 +1,63 @@ +function [item] = getitem(carray, rindex) +% function [item] = getitem(carray,rindex) +% +% Resolve access to a cell item. +% +% Empty data is returned when the function is called +% without arguments or when the requested index is out of +% bounds. +% +% If the index is not provided, selects randomly from +% the cell. +% +% Inputs: +% +% carray [cell[any]] - a cell with items. +% rindex [integer] - an integer index. Optional. +% +% Outputs: +% +% item - an array. +% +% Example: +% +% %empty cases +% [item] = IMOS.getitem(); +% assert(isempty(item)) +% [item] = IMOS.getitem({}); +% assert(isempty(item)) +% +% %out-of-bounds resolution +% [item] = IMOS.getitem({'a'},2); +% assert(isempty(item)) +% +% % errors +% try;IMOS.getitem('a');catch;r=true;end; +% assert(r) +% try;IMOS.getitem({'a'});catch;r2=true;end; +% assert(r2) +% +% %resolve in-bounds +% [item] = IMOS.getitem({[1],[2],[3]},1); +% assert(item==1) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(0,2) +if nargin==0 || (nargin == 1 && iscell(carray) && isempty(carray)) + item = []; + return +elseif ~iscell(carray) + error('First argument `carray` must be a cell') +elseif ~isindex(rindex) + error('Second argument `rindex` must be an integer/logical index') +end + +try + item = carray{rindex}; +catch + item = []; +end + +end diff --git a/Util/+IMOS/has_fieldname.m b/Util/+IMOS/has_fieldname.m new file mode 100644 index 000000000..c7cef3537 --- /dev/null +++ b/Util/+IMOS/has_fieldname.m @@ -0,0 +1,45 @@ +function [bool] = has_fieldname(scell, fieldname) +% function [bool] = has_fieldname(scell,fieldname) +% +% Check if all members in a cell of structs +% contains a certain fieldname. +% +% Inputs: +% +% scell [cell[struct]] - a cell with structs. +% fieldname - a fieldname string. +% +% Outputs: +% +% bool - a boolean/logical array [1,N]. +% +% +% Example: +% +% scell = {struct('name','123','id',1),struct('name','456')}; +% assert(IMOS.has_fieldname(scell,'name')) +% assert(~IMOS.has_fieldname(scell,'id')) +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(2, 2) + +if ~iscellstruct(scell) + error('First argument `scell` is not a cell of structs') +elseif ~ischar(fieldname) + error('Second argument `fieldname` is not a char') +end + +bool = false; + +if ~iscellstruct(scell) + return +end + +try + got_field = @(x)(isfield(x, fieldname)); + bool = all(cellfun(got_field, scell)); +catch +end + +end diff --git a/Util/+IMOS/is_imos_name.m b/Util/+IMOS/is_imos_name.m new file mode 100644 index 000000000..eb1cf2639 --- /dev/null +++ b/Util/+IMOS/is_imos_name.m @@ -0,0 +1,29 @@ +function [bool] = is_imos_name(name) +% function [bool] = is_imos_name(name) +% +% Check if a name is a bool IMOS name. +% +% Inputs: +% +% name - variable/dimension IMOS name +% +% Outputs: +% +% bool - True if a valid IMOS parameter +% +% Example: +% +% assert(IMOS.is_imos_name('TIME')); +% assert(~IMOS.is_imos_name('aBcDeFg')) +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) +IMOS.params().name; +bool = false; + +if inCell(IMOS.params().name, name) + bool = true; +end + +end diff --git a/Util/+IMOS/is_toolbox_dim.m b/Util/+IMOS/is_toolbox_dim.m new file mode 100644 index 000000000..aa738c275 --- /dev/null +++ b/Util/+IMOS/is_toolbox_dim.m @@ -0,0 +1,39 @@ +function [bool] = is_toolbox_dim(dstruct) +% function [bool] = is_toolbox_dim(dstruct) +% +% Check if a struct is a toolbox dimension struct. +% +% Inputs: +% +% dstruct[struct] - a toolbox dimension struct +% +% Outputs: +% +% bool - True or False +% +% Example: +% +% %basic +% dstruct = struct('name','TIME','typeCastFunc',@double,'data',[]); +% assert(IMOS.is_toolbox_dim(dstruct)) +% +% %false +% dstruct = struct('name','abc'); +% assert(~IMOS.is_toolbox_dim(dstruct)) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) + +try + assert(isstruct(dstruct)) + assert(ischar(dstruct.name)) + assert(isfunctionhandle(dstruct.typeCastFunc)) + dstruct.data; %TODO reinforce shape & numeric !? + bool = true; +catch + bool = false; +end + +end diff --git a/Util/+IMOS/is_toolbox_dimcell.m b/Util/+IMOS/is_toolbox_dimcell.m new file mode 100644 index 000000000..e5a982bed --- /dev/null +++ b/Util/+IMOS/is_toolbox_dimcell.m @@ -0,0 +1,27 @@ +function [bool] = is_toolbox_dimcell(tcell) +% function [bool] = is_toolbox_dimcell(tcell) +% +% Check if all items in the cell are toolbox dimensions +% +% Inputs: +% +% tcell - a cell of structs. +% +% Outputs: +% +% bool - True or False. +% +% Example: +% +% %basic usage +% dims = IMOS.gen_dimensions(); +% assert(IMOS.is_toolbox_dimcell(dims)); +% f=false;try;IMOS.is_toolbox_dimcell(dims{1});catch;f=true;end +% assert(f) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) +bool = all(cellfun(@IMOS.is_toolbox_dim, tcell)); +end diff --git a/Util/+IMOS/is_toolbox_var.m b/Util/+IMOS/is_toolbox_var.m new file mode 100644 index 000000000..b722c6b4d --- /dev/null +++ b/Util/+IMOS/is_toolbox_var.m @@ -0,0 +1,38 @@ +function [bool] = is_toolbox_var(vstruct) +% function [bool] = is_toolbox_var(vstruct) +% +% Check if a struct is a toolbox variable struct. +% +% Inputs: +% +% vstruct - A struct. +% +% Outputs: +% +% bool - True for a toolbox variable struct. +% +% Example: +% +% %true +% vstruct = struct('name','abc','typeCastFunc',@double,'dimensions',[],'data',[]); +% assert(IMOS.is_toolbox_var(vstruct)) +% +% %false +% vstruct = struct('name','abc','typeCastFunc',@double,'data',[]); +% assert(~IMOS.is_toolbox_var(vstruct)) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) + +try + assert(IMOS.is_toolbox_dim(vstruct)) + assert(isindex(vstruct.dimensions) || isempty(vstruct.dimensions))%TODO: reinforce shape + %TODO: reinforce coordinates for non featuretype values!? + bool = true; +catch + bool = false; +end + +end diff --git a/Util/+IMOS/is_toolbox_varcell.m b/Util/+IMOS/is_toolbox_varcell.m new file mode 100644 index 000000000..a02b389bc --- /dev/null +++ b/Util/+IMOS/is_toolbox_varcell.m @@ -0,0 +1,28 @@ +function [bool] = is_toolbox_varcell(tcell) +% function [bool] = is_toolbox_varcell(tcell) +% +% Check if all items in a cell of structs are +% toolbox variables. +% +% Inputs: +% +% tcell [cell[structs]] - a cell of structs. +% +% Outputs: +% +% bool - True or False. +% +% Example: +% +% %basic usage +% d = IMOS.gen_dimensions(); +% v = IMOS.gen_variables(d); +% assert(IMOS.is_toolbox_varcell(v)); +% f=false;try;IMOS.is_toolbox_varcell(v{1});catch;f=true;end +% assert(f) +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) +bool = all(cellfun(@IMOS.is_toolbox_var, tcell)); +end diff --git a/Util/+IMOS/params.m b/Util/+IMOS/params.m new file mode 100644 index 000000000..58bbf5641 --- /dev/null +++ b/Util/+IMOS/params.m @@ -0,0 +1,63 @@ +function [pdata] = params() +% function [pdata] = params() +% +% Load the entire imosParameters +% as a structure of cells. +% +% Inputs: +% +% Outputs: +% +% pdata [struct[cells]] - A struct with the of IMOS parameters +% +% Example: +% +% assert(iscell(IMOS.params().name)) +% assert(ischar(IMOS.params().name{1})) +% assert(ischar(IMOS.params().data_code{1})) +% assert(ischar(IMOS.params().netcdf_ctype{1})) +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(0, 0) +persistent params; + +if isempty(params) + param_file = [toolboxRootPath 'IMOS/imosParameters.txt']; + fid = fopen(param_file, 'rt'); + + if fid == -1 + error('Couldn''t read imosParameters.txt file') + end + + try + params = textscan(fid, '%s%d%s%s%s%s%s%f%f%f%s', ... + 'delimiter', ',', 'commentStyle', '%'); + fclose(fid); + catch e + error('Invalid entry found in imosParameters.txt') + end + +end + +pdata.name = params{1}; +pdata.is_cf_parameter = num2cell(logical(params{2})); +pdata.long_name = params{3}; + +pdata.units = params{4}; +is_percent = @(x)(strcmp(x, 'percent')); +ind_percent = find(cellfun(is_percent, params{4})); + +for k = 1:numel(ind_percent) + pdata.units{k} = '%'; +end + +pdata.direction_positive = params{5}; +pdata.reference_datum = params{6}; +pdata.data_code = params{7}; +pdata.fill_value = num2cell(params{8}); +pdata.valid_min = num2cell(params{9}); +pdata.valid_max = num2cell(params{10}); +pdata.netcdf_ctype = params{11}; + +end diff --git a/Util/+IMOS/varparams.m b/Util/+IMOS/varparams.m new file mode 100644 index 000000000..31e417a48 --- /dev/null +++ b/Util/+IMOS/varparams.m @@ -0,0 +1,64 @@ +function [vparams] = varparams() +% function [vparams] = varparams() +% +% Load a variable named structure +% containing all IMOS parameters. +% +% Inputs: +% +% Outputs: +% +% vparams [struct] - A struct with the of IMOS parameters. +% +% Example: +% +% %basic +% varinfo = IMOS.varparams(); +% assert(isequal(varinfo.TIME.name,'TIME')) +% assert(isequal(varinfo.TIME.is_cf_parameter,true)) +% assert(isequal(varinfo.TIME.long_name,'time')) +% assert(isequal(varinfo.VDIR.direction_positive,'clockwise')) +% assert(isequal(varinfo.VDIR.reference_datum,'true north')) +% assert(isequal(varinfo.VDIR.data_code,'W')) +% assert(isequal(varinfo.VDIR.fill_value,999999)) +% assert(isequal(varinfo.VDIR.valid_min,0)) +% assert(isequal(varinfo.VDIR.valid_max,360)) +% assert(isequal(varinfo.VDIR.netcdf_ctype,'float')) +% +% % type checking +% assert(isstruct(IMOS.varparams())) +% assert(isstruct(IMOS.varparams().TIME)) +% assert(ischar(IMOS.varparams().TIME.name)) +% assert(islogical(IMOS.varparams().TIME.is_cf_parameter)) +% assert(ischar(IMOS.varparams().TIME.units)) +% assert(ischar(IMOS.varparams().TIME.direction_positive)) +% assert(ischar(IMOS.varparams().TIME.reference_datum)) +% assert(ischar(IMOS.varparams().TIME.data_code)) +% assert(isnumeric(IMOS.varparams().TIME.fill_value)) +% assert(isnumeric(IMOS.varparams().TIME.valid_min)) +% assert(isnumeric(IMOS.varparams().TIME.valid_max)) +% assert(ischar(IMOS.varparams().TIME.netcdf_ctype)) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(0, 0) + +iparams = IMOS.params(); + +fvalues = struct2cell(iparams); +fnames = fieldnames(iparams); +varnames = iparams.name; + +for k = 1:length(varnames) + varname = varnames{k}; + + for kk = 1:length(fnames) + fname = fnames{kk}; + fvalue = fvalues{kk}{k}; + vparams.(varname).(fname) = fvalue; + end + +end + +end From 80a2d1f14850e2fbdd6b7eb0d80765a23eb1e0b6 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Tue, 27 Oct 2020 13:10:32 +1100 Subject: [PATCH 09/13] feat(netcdf): nc_get_var & nc_flat metadata lookup `nc_get_var` is just a wrapper to obtain the variable of a netcdf by its name. `nc_flat` flattens the `ncinfo` metadata output into a named structure, recursively. The function turns the access to all netcdf derived fields into a much easier ordeal, by using name referencing, tree pruning (e.g. Groups/Attributes), all with recursive resolution. The typical usage of `nc_flat` is to avoid index searching, and allow dictionary like access patterns (a-la python:xarray). For example, to get the size and units of the TIME variable without nc_flat you would use: ``` info = ncinfo(ncfile); vnames = {info.Variables.Name}; [has_var,var_ind] = inCell(vnames,'TIME'); vsize = info.Variables(var_ind).Size; vattrs = {info.Variables(var_ind).Attributes}; [has_units,unit_ind] = inCell(vattrs,'units'); vunits = info.Variables(var_ind).Attributes(unit_ind).Value; ``` with nc_flat: ``` info = nc_flat(ncinfo(ncfile)); vsize = info.Variables.TIME.Size; vunits = info.Variables.TIME.Attributes.units; ``` --- Util/NetCDF/nc_flat.m | 146 +++++++++++++++++++++++++++++++++++++++ Util/NetCDF/nc_get_var.m | 35 ++++++++++ 2 files changed, 181 insertions(+) create mode 100644 Util/NetCDF/nc_flat.m create mode 100644 Util/NetCDF/nc_get_var.m diff --git a/Util/NetCDF/nc_flat.m b/Util/NetCDF/nc_flat.m new file mode 100644 index 000000000..2dd1dfb37 --- /dev/null +++ b/Util/NetCDF/nc_flat.m @@ -0,0 +1,146 @@ +function [flat_struct] = nc_flat(ncstruct, keep_empty) +% function [flat_struct] = nc_flat(ncstruct, keep_empty) +% +% Flat the ncinfo structure, recursively, +% into a flattened form with named/dictionary like access. +% Prunning is also allowed. +% +% Inputs: +% +% ncinfo_struct [struct] - a ncinfo like structure +% keep_empty [bool] - flag to keep or prune empty entries. +% +% Outputs: +% +% +% Example: +% +% % basic +% ncstruct = struct('Filename','x.nc','Name','/','Dimensions',[],'Variables',[]); +% ncstruct.Attributes = struct('Name','one','Value',1); +% ncstruct.Attributes(2) = struct('Name','two','Value',2); +% ncstruct.Groups = []; +% ncstruct.Format = 'netcdf4'; +% [flat_struct] = nc_flat(ncstruct,true); +% assert(flat_struct.Attributes.one==1) +% assert(flat_struct.Attributes.two==2) +% assert(isstruct(flat_struct.Dimensions)) +% assert(isempty(flat_struct.Dimensions)) +% +% % recursion +% ncstruct.Groups = rmfield(ncstruct,{'Filename','Format'}); +% ncstruct.Groups.Name = 'Group_A'; +% ncstruct.Groups.Attributes(1).Name = 'three'; +% ncstruct.Groups.Attributes(1).Value = 3; +% ncstruct.Groups(2) = rmfield(ncstruct,{'Filename','Format'}); +% ncstruct.Groups(2).Name = 'Group_B'; +% ncstruct.Groups(2).Attributes(1).Name = 'four'; +% ncstruct.Groups(2).Attributes(1).Value = 4; +% [flat_struct] = nc_flat(ncstruct); +% assert(flat_struct.Attributes.one==1) +% assert(flat_struct.Attributes.two==2) +% assert(flat_struct.Groups.Group_A.Attributes.three==3) +% assert(flat_struct.Groups.Group_B.Attributes.four==4) +% +% % prunning +% [flat_struct] = nc_flat(ncstruct,false); +% assert(isequal(fieldnames(flat_struct),{'Filename','Attributes','Groups','Format'}')); +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 2) + +if nargin < 2 + keep_empty = true; +end + +names = {ncstruct.Name}'; +ncstruct = rmfield(ncstruct, 'Name'); +fnames = fieldnames(ncstruct); + +root_fields = {'Filename', 'Dimensions', 'Variables', 'Attributes', 'Groups', 'Format'}; +dims_fields = {'Length', 'Unlimited'}; +vars_fields = {'Dimensions', 'Size', 'Datatype', 'Attributes'}; +attrs_fields = {'Value'}; +group_fields = {'Dimensions', 'Variables', 'Attributes', 'Groups'}; + +at_root_level = all(contains(root_fields, fnames)); +at_dims_level = all(contains(dims_fields, fnames)); +at_vars_level = all(contains(vars_fields, fnames)); +at_attrs_level = all(contains(attrs_fields, fnames)); +at_group_level = all(contains(fnames, group_fields)); + +if at_attrs_level + + flat_struct = cell2struct({ncstruct.Value}, names, 2); + +elseif at_dims_level + + flat_struct = cell2struct(num2cell(ncstruct), names', 2); + +elseif at_vars_level + for k = 1:numel(names) + flat_struct.(names{k}) = clean_prune(ncstruct(k), {'Attributes', 'Dimensions'}, keep_empty); + end + +elseif at_group_level + + for k = 1:numel(names) + flat_struct.(names{k}) = clean_prune(ncstruct(k), {'Attributes', 'Dimensions', 'Variables', 'Groups'}, keep_empty); + end + +elseif at_root_level + for k = 1:numel(names) + flat_struct = clean_prune(ncstruct(k), {'Attributes', 'Dimensions', 'Variables', 'Groups'}, keep_empty); + end + +end + +end + +function [s] = clean_prune(s, fnames, keep_flag) +% +% Try to prune/flat the fieldnames of a ncinfo structure. +% +% If keep_flag is true and prune fails, +% the fieldname is kept as an empty struct. +% Otherwise, the fieldname is removed +% from the structure. +% + +narginchk(3, 3); + +total = numel(fnames); +remove_list = cell(1, total); + +c = 0; + +for n = 1:total + name = fnames{n}; + + try + s.(name) = nc_flat(s.(name), keep_flag); + catch + + if keep_flag + s.(name) = struct([]); + else + c = c + 1; + remove_list{c} = name; + end + + end + +end + +unflat_fields = c>0; +if unflat_fields + remove_list = remove_list(1:c); + need_removal = any(contains(remove_list,fieldnames(s))); + if need_removal + s = rmfield(s, remove_list); + end +end + +end diff --git a/Util/NetCDF/nc_get_var.m b/Util/NetCDF/nc_get_var.m new file mode 100644 index 000000000..8dab2c745 --- /dev/null +++ b/Util/NetCDF/nc_get_var.m @@ -0,0 +1,35 @@ +function [var] = nc_get_var(id,varname,varargin) +% function [var] = nc_get_var(id,varname,varargin) +% +% A wrapper to read a variable by its name +% with the netcdf matlab library. +% +% Inputs: +% +% id [int] - the netcdf file id. +% varname [str] - the variable name. +% varargin [int] - Other netcdf.getVar args, like +% start,count,stride. +% +% Outputs: +% +% var - the array. +% +% +% author: hugo.oliveira@utas.edu.au +% +try + varid = netcdf.inqVarID(id,varname); +catch err + err = addCause(err,MException(err.identifier,'%s: %s Variable name not found.',mfilename,varname)); + throw(err); +end + +try + var = netcdf.getVar(id,varid,varargin{:}); +catch err + err = addCause(err,MException(err.identifier,'%s: invalid getVar arguments for %s Variable',mfilename,varname)); + throw(err); +end + +end From 9830892575f6f63c38a9fcc25109bde00c5d18e7 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Thu, 29 Oct 2020 15:12:09 +1100 Subject: [PATCH 10/13] feat(StructUtils): filterfields function `filterfields` is a wrapper to filter struct fieldnames with a string, returning matched and unmatched entries. --- Util/StructUtils/filterFields.m | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 Util/StructUtils/filterFields.m diff --git a/Util/StructUtils/filterFields.m b/Util/StructUtils/filterFields.m new file mode 100644 index 000000000..2f6d555b3 --- /dev/null +++ b/Util/StructUtils/filterFields.m @@ -0,0 +1,41 @@ +function [filtered, unmatched] = filterFields(astruct, astr) +% function [filtered,unmatched] = filterFields(astruct,astr) +% +% Returns only fieldnames that partially or completely +% match a string. +% +% Inputs: +% +% astruct [struct] - a structure with fields +% astr [str] - a string to match against the fieldnames +% +% Outputs: +% +% filtered [cell{str}] - matched fieldnames +% unmatched [cell{str}] - the unmatched fieldnames. +% +% Example: +% +% %basic usage +% x = struct('a','','aa','','b','','bb','','ab','','ba',''); +% [filtered,unmatched] = filterFields(x,'a'); +% assert(numel(filtered)==4) +% assert(numel(unmatched)==2) +% assert(strcmp(unmatched{1},'b')) +% assert(strcmp(unmatched{2},'bb')) +% +% +% author: hugo.oliveira@utas.edu.au +% + +narginchk(2, 2) + +fnames = fieldnames(astruct); +where = contains(fnames, astr); +filtered = fnames(where); + +if nargout > 1 + unmatched = fnames(~where); +end + +end From df7f53adf862014dc0bea598b992e35967507778 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Thu, 29 Oct 2020 12:33:51 +1100 Subject: [PATCH 11/13] feat(Parser): OceanContour netcdf & mat parser This commit introduces the `oceanContourParser`, supporting the netCDF and matfiles from the Nortek OceanContour software. The parser was based on the `signatureParser` code and import the same metadata and variables. The `OceanContour` class contains all methods related to the parser, including variable and attribute mappings between entities, as well as the actual reader. At the moment, support is available mostly for ping average data ("Avg"). More test files are needed for other types of output. --- Parser/OceanContour/OceanContour.m | 699 +++++++++++++++++++++++++++++ Parser/oceanContourParse.m | 40 ++ 2 files changed, 739 insertions(+) create mode 100644 Parser/OceanContour/OceanContour.m create mode 100644 Parser/oceanContourParse.m diff --git a/Parser/OceanContour/OceanContour.m b/Parser/OceanContour/OceanContour.m new file mode 100644 index 000000000..20d234030 --- /dev/null +++ b/Parser/OceanContour/OceanContour.m @@ -0,0 +1,699 @@ +classdef OceanContour + %classdef OceanContour + % + % This is a class containing methods that defines several + % fields and functions related to the OceanContour Parser. + % This includes utility functions and variable/attribute + % mappings to the toolbox structures. + % + % author: hugo.oliveira@utas.edu.au + % + %TODO: Design metadata typecasting. + properties (Constant) + beam_angles = struct('Signature250', 20, 'Signature500', 25, 'Signature1000', 25); + end + + methods (Static) + + function metaname = build_meta_attr_midname(group_name) + %function metaname = build_meta_attr_midname(group_name) + % + % Generate the middle name for global attributes given + % a group name. + % + % Input: + % + % group_name - the fieldname or netcdf group name. + % + % Output: + % + % metaname - the mid/partial metadata attribute name string. + % + % Example: + % + % midname = OceanContour.build_meta_attr_midname('Avg'); + % assert(strcmp(midname,'avg')) + % midname = OceanContour.build_meta_attr_midname('burstAltimeter'); + % assert(strcmp(midname,'burstAltimeter')) + % + if ~ischar(group_name) + errormsg('first argument is not a string') + end + + metaname = [lower(group_name(1)) group_name(2:end)]; + end + + function attname = build_instrument_name(group_name, var_name) + %function attname = build_instrument_name(group_name,var_name) + % + % Generate instrument tokens for the attribute names + % for in OceanContour files. + % + % The token is a three part string: + % part1 - "Instrument" string, followed + % part2 - group/dataset name (with the first letter lower) + % part3 - the "variable" token/name. + % + % Inputs: + % + % group_name [str] - the dataset group (field) name + % var_name [str] - the variable name (last token). + % + % Output: + % attname [str] - the attribute name. + % + % Example: + % name = OceanContour.build_instrument_name('Avg', 'coordSystem'); + % assert(strcmpi(name,'Instrument_avg_coordSystem')) + % + % + % author: hugo.oliveira@utas.edu.au + % + narginchk(2, 2) + + if ~ischar(group_name) + errormsg('first argument is not a string') + elseif ~ischar(var_name) + errormsg('second argument is not a string') + end + + meta_attr_midname = OceanContour.build_meta_attr_midname(group_name); + attname = ['Instrument_' meta_attr_midname '_' var_name]; + end + + function [ucur_name, vcur_name, heading_name] = build_magnetic_variables(custom_magnetic_declination) + %function attname = build_magnetic_variables(custom_magnetic_declination) + % + % Generate VAR or VAR_MAG toolbox variable style names + % based on provided magnetic declination info. + % + narginchk(1, 1) + + if ~islogical(custom_magnetic_declination) + errormsg('build_magnetic_variables: first argument is not a logical') + end + + if custom_magnetic_declination + %TODO: This is probably unecessary + %I believe OceanContourDouble-check if OceanContour will change variable names if custom magnetic declination is used. + warning('%s: Assigning non ENU Velocities to ENU variables. Verify the magnetic declination angles.') + ucur_name = 'UCUR_MAG'; + vcur_name = 'VCUR_MAG'; + heading_name = 'HEADING_MAG'; + else + ucur_name = 'UCUR'; + vcur_name = 'VCUR'; + heading_name = 'HEADING'; + end + + end + + function verify_mat_groups(matdata) + %just raise a proper error for invalid OceanContour mat files. + try + matdata.Config; + catch + errormsg('%s do not contains the ''Config'' metadata fieldname', filename) + end + + ngroups = numel(fieldnames(matdata)); + + if ngroups < 2 + errormsg('%s do not contains any data fieldname', fielname) + end + + end + + function verify_netcdf_groups(info) + %just raise a proper error for invalid OceanContour netcdf groups. + try + assert(strcmp(info.Groups(1).Name, 'Config')) + assert(strcmp(info.Groups(2).Name, 'Data')) + catch + errormsg('contains an invalid OceanContour structure. please report this error with your data file: %s', filename) + end + + end + + function warning_failed(failed_items, filename) + %just raise a proper warning for failed variable reads + for k = 1:numel(failed_items) + warning('%s: Couldn''t read variable `%s` in %s', mfilename, failed_items{k}, filename) + end + + end + + + function [attmap] = get_attmap(ftype, group_name) + %function [attmap] = get_attmap(ftype, group_name) + % + % Generate dynamical attribute mappings based on + % the dataset group name. + % + % Inputs: + % + % ftype [str] - the file type. 'mat' or 'netcdf'; + % group_name [str] - the OceanContour dataset group name. + % + % Outputs: + % + % attmap [struct[str,str]] - mapping between imos attributes + % & OceanContour attributes. + % + % + % Example: + % + % %basic usage + % attmap = OceanContour.get_attmap('Avg'); + % fnames = fieldnames(attmap); + % assert(contains(fnames,'instrument_model')) + % original_name =attmap.instrument_model; + % assert(strcmp(original_name,'Instrument_instrumentName')); + % + % author: hugo.oliveira@utas.edu.au + % + + if ~ischar(ftype) + errormsg('First argument is not a string') + elseif ~strcmpi(ftype, 'mat') && ~strcmpi(ftype, 'netcdf') + errormsg('First argument %s is an invalid ftype. Accepted file types are ''mat'' and ''netcdf''.', ftype) + elseif ~ischar(group_name) + errormsg('Second argument is not a string') + end + + attmap = struct(); + + meta_attr_midname = OceanContour.build_meta_attr_midname(group_name); + + attmap.('instrument_model') = 'Instrument_instrumentName'; + attmap.('beam_angle') = 'DataInfo_slantAngles'; + attmap.('beam_interval') = 'DataInfo_slantAngles'; + attmap.('coordinate_system') = OceanContour.build_instrument_name(group_name, 'coordSystem'); + attmap.('nBeams') = OceanContour.build_instrument_name(group_name, 'nBeams'); + attmap.('activeBeams') = OceanContour.build_instrument_name(group_name, 'activeBeams'); %no previous name + attmap.('magDec') = 'Instrument_user_decl'; + + if strcmpi(ftype, 'mat') + attmap.('instrument_serial_no') = 'Instrument_serialNumberDoppler'; + attmap.('binSize') = OceanContour.build_instrument_name(group_name, 'cellSize'); + end + + %custom & dynamical fields + attmap.(['instrument_' meta_attr_midname '_enable']) = OceanContour.build_instrument_name(group_name, 'enable'); + + switch meta_attr_midname + case 'avg' + attmap.('instrument_avg_interval') = OceanContour.build_instrument_name(group_name, 'averagingInterval'); + attmap.('instrument_sample_interval') = OceanContour.build_instrument_name(group_name, 'measurementInterval'); + %TODO: need a more complete file to test below below + case 'burst' + attmap.('instrument_burst_interval') = OceanContour.build_instrument_name(group_name, 'burstInterval'); + case 'bursthr' + attmap.('instrument_bursthr_interval') = OceanContour.build_instrument_name(group_name, 'burstHourlyInterval'); + case 'burstAltimeter' + attmap.('instrument_burstAltimeter_interval') = OceanContour.build_instrument_name(group_name, 'burstAltimeterInterval'); + case 'burstRawAltimeter' + attmap.('instrument_burstRawAltimeter_interval') = OceanContour.build_instrument_name(group_name, 'burstRawAltimeterInterval'); + end + + end + + function [varmap] = get_varmap(ftype, group_name, nbeams, custom_magnetic_declination) + %function [varmap] = get_varmap(ftype, group_name,nbeams,custom_magnetic_declination) + % + % Generate dynamical variable mappings for a certain + % group of variables, given the number of beams and if custom + % magnetic adjustments were made. + % + % Inputs: + % + % ftype [str] - The file type. 'mat' or 'netcdf'. + % group_name [str] - the OceanContour dataset group name. + % nbeams [double] - The nbeams used on the dataset. + % custom_magnetic_declination [logical] - true for custom + % magnetic values. + % + % Outputs: + % + % vttmap [struct[str,str]] - mapping between imos variables + % & OceanContour variables. + % + % + % Example: + % + % %basic usage + % + % varmap = OceanContour.get_attmap('Avg',4,False); + % assert(strcmp(attmap.WCUR_2,'Vel_Up2')); + % + % % nbeams == 3 + % varmap = OceanContour.get_varmap('Avg',3,False); + % f=false;try;varmap.WCUR_2;catch;f=true;end + % assert(f) + % + % % custom magdec - may change with further testing + % varmap = OceanContour.get_varmap('Avg',4,True); + % assert(strcmp(varmap.UCUR_MAG,'Vel_East')) + % + % + % author: hugo.oliveira@utas.edu.au + % + narginchk(4, 4) + + if ~ischar(ftype) + errormsg('First argument is not a string') + elseif ~strcmpi(ftype, 'mat') && ~strcmpi(ftype, 'netcdf') + errormsg('First argument %s is an invalid ftype. Accepted file types are ''mat'' and ''netcdf''.', ftype) + elseif ~ischar(group_name) + errormsg('First argument is not a string') + elseif ~isscalar(nbeams) + errormsg('Second argument is not a scalar') + elseif ~islogical(custom_magnetic_declination) + errormsg('Third argument is not logical') + end + + is_netcdf = strcmpi(ftype, 'netcdf'); + [ucur_name, vcur_name, heading_name] = OceanContour.build_magnetic_variables(custom_magnetic_declination); + + varmap = struct(); + varmap.('binSize') = 'CellSize'; + varmap.('TIME') = 'MatlabTimeStamp'; + + if is_netcdf + varmap.('instrument_serial_no') = 'SerialNumber'; + %TODO: reinforce uppercase at first letter? nEed to see more files. + varmap.('HEIGHT_ABOVE_SENSOR') = [group_name 'VelocityENU_Range']; + %TODO: Handle magnetic & along beam cases. + %varmap.('DIST_ALONG_BEAMS') = [group_name 'Velocity???_Range']; + %TODO: evaluate if when magnetic declination is provided, the + %velocity fields will be corrected or not (as well as any rename/comments added). + varmap.(ucur_name) = 'Vel_East'; + varmap.(vcur_name) = 'Vel_North'; + varmap.(heading_name) = 'Heading'; + varmap.('WCUR') = 'Vel_Up1'; + varmap.('ABSI1') = 'Amp_Beam1'; + varmap.('ABSI2') = 'Amp_Beam2'; + varmap.('ABSI3') = 'Amp_Beam3'; + varmap.('CMAG1') = 'Cor_Beam1'; + varmap.('CMAG2') = 'Cor_Beam2'; + varmap.('CMAG3') = 'Cor_Beam3'; + + if nbeams > 3 + varmap.('WCUR_2') = 'Vel_Up2'; + varmap.('ABSI4') = 'Amp_Beam4'; + varmap.('CMAG4') = 'Cor_Beam4'; + end + + else + %instrument_serial_no is on metadata for matfiles. + varmap.('HEIGHT_ABOVE_SENSOR') = 'Range'; + varmap.(ucur_name) = 'VelEast'; + varmap.(vcur_name) = 'VelNorth'; + varmap.(heading_name) = 'Heading'; + varmap.('WCUR') = 'VelUp1'; + varmap.('ABSI1') = 'AmpBeam1'; + varmap.('ABSI2') = 'AmpBeam2'; + varmap.('ABSI3') = 'AmpBeam3'; + varmap.('CMAG1') = 'CorBeam1'; + varmap.('CMAG2') = 'CorBeam2'; + varmap.('CMAG3') = 'CorBeam3'; + + if nbeams > 3 + varmap.('WCUR_2') = 'VelUp2'; + varmap.('ABSI4') = 'AmpBeam4'; + varmap.('CMAG4') = 'CorBeam4'; + end + + end + + varmap.('TEMP') = 'WaterTemperature'; + varmap.('PRES_REL') = 'Pressure'; + varmap.('SSPD') = 'SpeedOfSound'; + varmap.('BAT_VOLT') = 'Battery'; + varmap.('PITCH') = 'Pitch'; + varmap.('ROLL') = 'Roll'; + varmap.('ERROR') = 'Error'; + varmap.('AMBIG_VEL') = 'Ambiguity'; + varmap.('TRANSMIT_E') = 'TransmitEnergy'; + varmap.('NOMINAL_CORR') = 'NominalCor'; + + end + + function [imap] = get_importmap(nbeams, custom_magnetic_declination) + %function [imap] = get_importmap(custom_magnetic_declination) + % + % Return default variables to import from the OceanContour files. + % + % Inputs: + % + % nbeams [scalar] - the number of ADCP beams. + % custom_magnetic_declination [logical] - true for custom + % magnetic values. + % + % Outputs: + % + % imap [struct[cell]] - Struct with different variables + % classes to import + % + % + % Example: + % + % %basic usage + % imap = OceanContour.get_importmap(False); + % assert(inCell(imap.all_variables,'PITCH')) + % assert(inCell(imap.all_variables,'ROLL')) + % + % author: hugo.oliveira@utas.edu.au + % + narginchk(2, 2) + + if ~isscalar(nbeams) + errormsg('First argument is not a scalar') + elseif ~islogical(custom_magnetic_declination) + errormsg('Second argument is not a logical') + end + + imap = struct(); + [ucur_name, vcur_name, heading_name] = OceanContour.build_magnetic_variables(custom_magnetic_declination); + + ENU = struct(); + + ENU.one_dimensional = {'TEMP', 'PRES_REL', 'SSPD', 'BAT_VOLT', 'PITCH', 'ROLL', heading_name, 'ERROR', 'AMBIG_VEL', 'TRANSMIT_E', 'NOMINAL_CORR'}; + ENU.velocity_variables = {ucur_name, vcur_name, 'WCUR'}; + ENU.beam_amplitude_variables = {'ABSI1', 'ABSI2', 'ABSI3'}; + ENU.correlation_variables = {'CMAG1', 'CMAG2', 'CMAG3'}; + + if nbeams > 3 + ENU.velocity_variables = [ENU.velocity_variables, 'WCUR_2']; + ENU.beam_amplitude_variables = [ENU.beam_amplitude_variables 'ABSI4']; + ENU.correlation_variables = [ENU.correlation_variables 'CMAG4']; + end + + ENU.two_dimensional = [ENU.velocity_variables, ENU.beam_amplitude_variables]; + ENU.all_variables = [ENU.one_dimensional, ENU.two_dimensional]; + + %TODO: Implement Non-ENU cases. + + imap.('ENU') = ENU; + + end + + function [sample_data] = readOceanContourFile(filename) + % function [sample_data] = readOceanContourFile(filename) + % + % Read an OceanContour netcdf or mat file and convert fields + % to the matlab toolbox structure. Variables are read + % as is. + % + % Supported Innstruments: Nortek ADCP Signatures. + % + % The Ocean contour software write nested netcdf4 groups: + % > root + % | + % {root_groups} + % | -> Config ["global" file metadata only] + % | -> Data [file datasets leaf] + % | + % {data_groups} + % | -> Avg [data+metadata] + % | -> ... [data+metadata] + % + % Or a flat mat file: + % > root + % | + %{data_groups} + % | -> [dataset-name] [data] + % | -> Config [metadata] + % + % + % Inputs: + % + % filename [str] - the filename. + % + % Outputs: + % + % sample_data - the toolbox structure. + % + % Example: + % + % %read from netcdf + % file = [toolboxRootPath 'data/testfiles/netcdf/Nortek/OceanContour/Signature/s500_enu_avg.nc']; + % [sample_data] = readOceanContour(file); + % assert(strcmpi(sample_data{1}.meta.instrument_model,'Signature500')) + % assert(isequal(sample_data{1}.meta.instrument_avg_interval,60)) + % assert(isequal(sample_data{1}.meta.instrument_sample_interval,600)) + % assert(strcmpi(sample_data{1}.meta.coordinate_system,'ENU')) + % assert(isequal(sample_data{1}.meta.nBeams,4)) + % assert(strcmpi(sample_data{1}.dimensions{2}.name,'HEIGHT_ABOVE_SENSOR')) + % assert(~isempty(sample_data{1}.variables{end}.data)) + % + % % read from matfile + % file = [toolboxRootPath 'data/testfiles/mat/Nortek/OceanContour/Signature/s500_enu_avg.mat']; + % [sample_data] = readOceanContour(file); + % assert(strcmpi(sample_data{1}.meta.instrument_model,'Signature500')) + % assert(isequal(sample_data{1}.meta.instrument_avg_interval,60)) + % assert(isequal(sample_data{1}.meta.instrument_sample_interval,600)) + % assert(strcmpi(sample_data{1}.meta.coordinate_system,'ENU')) + % assert(isequal(sample_data{1}.meta.nBeams,4)) + % assert(strcmpi(sample_data{1}.dimensions{2}.name,'HEIGHT_ABOVE_SENSOR')) + % assert(~isempty(sample_data{1}.variables{end}.data)) + % + % + % author: hugo.oliveira@utas.edu.au + % + narginchk(1, 1) + + try + info = ncinfo(filename); + ftype = 'netcdf'; + + catch + + try + matdata = load(filename); + ftype = 'mat'; + catch + errormsg('%s is not a mat or netcdf file', filename) + end + + end + + is_netcdf = strcmpi(ftype, 'netcdf'); + + if is_netcdf + OceanContour.verify_netcdf_groups(info); + file_metadata = nc_flat(info.Groups(1).Attributes, false); + data_metadata = nc_flat(info.Groups(2).Groups, false); + + ncid = netcdf.open(filename); + root_groups = netcdf.inqGrps(ncid); + data_group = root_groups(2); + + dataset_groups = netcdf.inqGrps(data_group); + get_group_name = @(x)(netcdf.inqGrpName(x)); + + else + OceanContour.verify_mat_groups(matdata); + file_metadata = matdata.Config; + matdata = rmfield(matdata, 'Config'); %mem optimisation. + + dataset_groups = fieldnames(matdata); + get_group_name = @(x)(getindex(split(x, '_Data'), 1)); + + end + + n_datasets = numel(dataset_groups); + sample_data = cell(1, n_datasets); + + for k = 1:n_datasets + + % start by loading preliminary information into the metadata struct, so we + % can define the variable names and variables to import. + meta = struct(); + + group_name = get_group_name(dataset_groups); + meta_attr_midname = OceanContour.build_meta_attr_midname(group_name); + + %load toolbox_attr_names:file_attr_names dict. + att_mapping = OceanContour.get_attmap(ftype, group_name); + + %access pattern - use lookup based on expected names, + get_att = @(x)(file_metadata.(att_mapping.(x))); + + nBeams = double(get_att('nBeams')); + + try + activeBeams = double(get_att('activeBeams')); + catch + activeBeams = Inf; + end + + meta.nBeams = min(nBeams, activeBeams); + + try + assert(meta.nBeams == 4); + %TODO: support variable nBeams. need more files. + catch + errormsg('Only 4 Beam ADCP are supported. %s got %d nBeams', filename, meta.nBeams) + end + + meta.magDec = get_att('magDec'); + custom_magnetic_declination = logical(meta.magDec); + + %Now that we know some preliminary info, we can load the variable + % name mappings and the list of variables to import. + var_mapping = OceanContour.get_varmap(ftype, group_name, nBeams, custom_magnetic_declination); + import_mapping = OceanContour.get_importmap(nBeams, custom_magnetic_declination); + + %subset the global metadata fields to only the respective group. + dataset_meta_id = ['_' meta_attr_midname '_']; + [~, other_datasets_meta_names] = filterFields(file_metadata, dataset_meta_id); + dataset_meta = rmfield(file_metadata, other_datasets_meta_names); + + %load extra metadata and unify the variable access pattern into + % the same function name. + if is_netcdf + meta.dim_meta = data_metadata.(group_name).Dimensions; + meta.var_meta = data_metadata.(group_name).Variables; + gid = dataset_groups(k); + get_var = @(x)(nc_get_var(gid, var_mapping.(x))); + else + fname = getindex(dataset_groups, k); + get_var = @(x)(transpose(matdata.(fname).(var_mapping.(x)))); + end + + meta.featureType = ''; + meta.instrument_make = 'Nortek'; + meta.instrument_model = get_att('instrument_model'); + + if is_netcdf + inst_serial_no = unique(get_var('instrument_serial_no')); + else + %serial no is at metadata/Config level in the mat files. + instr_serial_no = unique(get_att('instrument_serial_no')); + end + + try + assert(numel(inst_serial_no) == 1) + catch + errormsg('Multi instrument serial numbers found in %s.', filename) + end + meta.instrument_serial_no = num2str(inst_serial_no); + + try + assert(contains(meta.instrument_model, 'Signature')) + %TODO: support other models. need more files. + catch + errormsg('Only Signature ADCPs are supported.', filename) + end + + default_beam_angle = OceanContour.beam_angles.(meta.instrument_model); + instrument_beam_angles = single(get_att('beam_angle')); + try + dataset_beam_angles = instrument_beam_angles(1:meta.nBeams); + assert(isequal(unique(dataset_beam_angles), default_beam_angle)) + %TODO: workaround for inconsistent beam_angles. need more files. + catch + errormsg('Inconsistent beam angle/Instrument information in %s', filename) + end + meta.beam_angle = default_beam_angle; + + meta.('instrument_sample_interval') = single(get_att('instrument_sample_interval')); + + mode_sampling_duration_str = ['instrument_' meta_attr_midname '_interval']; + meta.(mode_sampling_duration_str) = get_att(mode_sampling_duration_str); + + time = get_var('TIME'); + + try + actual_sample_interval = single(mode(diff(time)) * 86400.); + assert(isequal(meta.('instrument_sample_interval'), actual_sample_interval)) + catch + expected = meta.('instrument_sample_interval'); + got = actual_sample_interval; + errormsg('Inconsistent instrument sampling interval in %s . Metadata is set to %d, while time variable indicates %d', filename, expected, got); + end + + meta.coordinate_system = get_att('coordinate_system'); + + try + assert(strcmp(meta.coordinate_system, 'ENU')) + %TODO: support other CS. need more files. + catch + errormsg('Unsuported coordinates. %s contains non-ENU data.', filename) + end + + z = get_var('HEIGHT_ABOVE_SENSOR'); + + try + assert(all(z > 0)); + catch + errormsg('invalid VelocityENU_Range in %s', filename) + %TODO: plan any workaround for diff ranges. files!? + end + + meta.binSize = unique(get_var('binSize')); + + try + assert(numel(meta.binSize) == 1) + catch + errormsg('Nonuniform CellSizes in %s', mfilename, filename) + %TODO: plan any workaround for nonuniform cells. need files. + end + + meta.file_meta = file_metadata; + meta.dataset_meta = dataset_meta; + + dimensions = IMOS.templates.dimensions.adcp_remapped; + dimensions{1}.data = time; + dimensions{1}.comment = 'time imported from matlabTimeStamp variable'; + dimensions{2}.data = z; + dimensions{2}.comment = 'height imported from VelocityENU_Range'; + + switch meta.coordinate_system + case 'ENU' + onedim_vnames = import_mapping.('ENU').one_dimensional; + twodim_vnames = import_mapping.('ENU').two_dimensional; + otherwise + errormsg('%s coordinates found in %s is not implemented yet', filename, adcp_data_type) + end + + onedim_vcoords = [dimensions{1}.name ' LATITUDE LONGITUDE ' 'NOMINAL_DEPTH']; %TODO: point to Pressure/Depth via CF-conventions + onedim_vtypes = IMOS.cellfun(@getIMOSType, onedim_vnames); + [onedim_vdata, failed_items] = IMOS.cellfun(get_var, onedim_vnames); + + if ~isempty(failed_items) + OceanContour.warning_failed(failed_items, filename) + end + + twodim_vcoords = [dimensions{1}.name ' LATITUDE LONGITUDE ' dimensions{2}.name]; + twodim_vtypes = IMOS.cellfun(@getIMOSType, twodim_vnames); + [twodim_vdata, failed_items] = IMOS.cellfun(get_var, twodim_vnames); + + if ~isempty(failed_items) + OceanContour.warning_failed(failed_items, filename) + end + + %TODO: Implement unit conversions monads. + variables = [... + IMOS.featuretype_variables('timeSeries'), ... + IMOS.gen_variables(dimensions, onedim_vnames, onedim_vtypes, onedim_vdata, 'coordinates', onedim_vcoords), ... + IMOS.gen_variables(dimensions, twodim_vnames, twodim_vtypes, twodim_vdata, 'coordinates', twodim_vcoords), ... + ]; + + dataset = struct(); + dataset.toolbox_input_file = filename; + dataset.toolbox_parser = mfilename; + dataset.netcdf_group_name = group_name; + dataset.meta = meta; + dataset.dimensions = dimensions; + dataset.variables = variables; + + sample_data{k} = dataset; + end + + end + + end + +end diff --git a/Parser/oceanContourParse.m b/Parser/oceanContourParse.m new file mode 100644 index 000000000..2182be68c --- /dev/null +++ b/Parser/oceanContourParse.m @@ -0,0 +1,40 @@ +function [sample_data] = oceanContourParser(filename_in_cell, toolbox_mode) +% function [data] = oceanContourParser(filename_in_cell, toolbox_mode) +% +% The OceanContour Parser for netcdf or mat files. +% +% Inputs: +% +% filename [cell[str]] - A cell containing one filename string. +% toolbox_mode [str] - the processing mode string. +% Default: 'timeSeries' +% +% Outputs: +% +% sample_data - toolbox struct containing the data. +% +% Example: +% +% [data] = OceanContourParser(file,mode) +% assert() +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 2) + +invalid_file_arg = ~iscellstr(filename_in_cell) && length(filename_in_cell) ~= 1; + +if invalid_file_arg + errormsg('First argument file isn''t a singleton cell of strings') +end + +filename = filename_in_cell{1}; +inexistent_file = isempty(dir(filename)); + +if inexistent_file + errormsg('file %s doesn''t exist', filename) +end + +sample_data = OceanContour.readOceanContourFile(filename); + +end From 3f69f0306b54fcf47db13328c1645939c901821b Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Wed, 25 Nov 2020 13:19:29 +1100 Subject: [PATCH 12/13] feat(review): rename remove_singleton and refs --- Util/+IMOS/discover_data_dimensions.m | 2 +- Util/{remove_singleton.m => removeSingleton.m} | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) rename Util/{remove_singleton.m => removeSingleton.m} (65%) diff --git a/Util/+IMOS/discover_data_dimensions.m b/Util/+IMOS/discover_data_dimensions.m index b1380b4bc..589c815a5 100644 --- a/Util/+IMOS/discover_data_dimensions.m +++ b/Util/+IMOS/discover_data_dimensions.m @@ -59,7 +59,7 @@ available_dims_len_as_array = cell2mat(available_dims_len); tdata_size = size(tdata); -non_singleton_var_dims_len = remove_singleton(tdata_size); +non_singleton_var_dims_len = removeSingleton(tdata_size); [discovered_indexes] = whereincell(available_dims_len, num2cell(non_singleton_var_dims_len)); try diff --git a/Util/remove_singleton.m b/Util/removeSingleton.m similarity index 65% rename from Util/remove_singleton.m rename to Util/removeSingleton.m index 8845fbc14..8d4d41394 100644 --- a/Util/remove_singleton.m +++ b/Util/removeSingleton.m @@ -1,5 +1,5 @@ -function [nssize] = remove_singleton(sizearray) -% function [nssize] = remove_singleton(sizearray) +function [nssize] = removeSingleton(sizearray) +% function [nssize] = removeSingleton(sizearray) % % Remove the singleton dimension sizes % from a sizearray @@ -14,9 +14,9 @@ % % Example: % -% assert(isequal(remove_singleton([1,1,10,30]),[10,30])); -% assert(isequal(remove_singleton([2,3,4]),[2,3,4])); -% assert(isempty(remove_singleton([1]))) +% assert(isequal(removeSingleton([1,1,10,30]),[10,30])); +% assert(isequal(removeSingleton([2,3,4]),[2,3,4])); +% assert(isempty(removeSingleton([1]))) % % author: hugo.oliveira@utas.edu.au % From bb7d95413228997fa768cf471256765d069e2be3 Mon Sep 17 00:00:00 2001 From: Hugo Oliveira Date: Wed, 25 Nov 2020 13:24:43 +1100 Subject: [PATCH 13/13] feat(review): typo fixes --- Util/+IMOS/+random/get_random_param.m | 2 +- Util/TestUtils/checkDocstrings.m | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Util/+IMOS/+random/get_random_param.m b/Util/+IMOS/+random/get_random_param.m index eaa6d09e3..6bc0c2216 100644 --- a/Util/+IMOS/+random/get_random_param.m +++ b/Util/+IMOS/+random/get_random_param.m @@ -30,7 +30,7 @@ available_names = fieldnames(imosparams); if ~inCell(available_names, param_name) - errormsg('IMOS parameter %s do not exist', param_name) + errormsg('IMOS parameter %s does not exist', param_name) end all_values = unique(imosparams.(param_name)); diff --git a/Util/TestUtils/checkDocstrings.m b/Util/TestUtils/checkDocstrings.m index ab85903c1..a93657314 100644 --- a/Util/TestUtils/checkDocstrings.m +++ b/Util/TestUtils/checkDocstrings.m @@ -26,7 +26,7 @@ files = rdir(folder); if isempty(files) - errormsg('No files presnet at %s folder',folder) + errormsg('No files present at %s folder',folder) end srcfiles = cell2mat(cellfun(@is_matlab,files,'UniformOutput',false));