diff --git a/_test/test_serializers/test_serialise.m b/_test/test_serializers/test_serialise.m index 9e063adb39..09c825a2de 100644 --- a/_test/test_serializers/test_serialise.m +++ b/_test/test_serializers/test_serialise.m @@ -19,6 +19,22 @@ end + % + function test_serial_arrays(~) + f1=1:10; + f2 =f1'; + f1 = repmat(f1,5,1,5); + f2 = repmat(f2,1,5,5); + bytes1 = hlp_serialise(f1); + bytes2 = hlp_serialise(f2); + [f1_rec,nbytes1] = hlp_deserialise(bytes1); + assertEqual(numel(bytes1),nbytes1) + [f2_rec,nbytes2] = hlp_deserialise(bytes2); + assertEqual(numel(bytes2),nbytes2) + % + assertEqual(f1,f1_rec); + assertEqual(f2,f2_rec); + end %------------------------------------------------------------------ @@ -683,7 +699,7 @@ function test_ser_serializeble_obj_array_level1(obj) assertEqual(size,numel(ser)); %-------------------------------------------------------------- % Serialize using C++ - + if ~obj.use_mex skipTest('Mex mode is not currently available for: test_ser_serializeble_obj_array'); end @@ -724,6 +740,7 @@ function test_ser_serializeble_obj(obj) size = hlp_serial_sise(serCl); assertEqual(size,numel(ser)); + skipTest('C++ deserializer does not work propertly; #394') if ~obj.use_mex skipTest('Mex mode is not currently available for: test_ser_serializeble_obj'); end @@ -734,7 +751,6 @@ function test_ser_serializeble_obj(obj) ser_c = c_serialise(serCl); assertEqual(ser_c,ser) - skipTest('C++ deserializer does not work propertly; #394') [serCl_rec,nbytes] = c_deserialise(ser_c); % @@ -745,7 +761,23 @@ function test_ser_serializeble_obj(obj) assertEqual(size_c,numel(ser)); end - - + function test_pack_unpack_header(~) + + type_details = hlp_serial_types.type_details; + for ntype = 1:numel(type_details) + for nElem = 1:3 + for nDims = 1:8 + for sizeV1 = 0:1 + packed_tag = hlp_serial_types.pack_tag_data(... + nElem,nDims,sizeV1,type_details(ntype)); + [type_rec, nDims_rec] = ... + hlp_serial_types.unpack_tag_data(packed_tag); + assertEqual(type_rec,type_details(ntype)); + assertEqual(nDims_rec,nDims); + end + end + end + end + end end end \ No newline at end of file diff --git a/herbert_core/utilities/misc/hlp_deserialise.m b/herbert_core/utilities/misc/hlp_deserialise.m index 4660264764..a49597eee7 100644 --- a/herbert_core/utilities/misc/hlp_deserialise.m +++ b/herbert_core/utilities/misc/hlp_deserialise.m @@ -43,7 +43,7 @@ [v,pos] = deserialise_simple_data(m,pos); case {13,14,15,16,17,18,19,20,21,22} [v,pos] = deserialise_complex_data(m,pos); - case {23} + case 23 [v,pos] = deserialise_cell(m,pos); case {24} [v,pos] = deserialise_struct(m,pos); @@ -53,6 +53,8 @@ [v,pos] = deserialise_object(m,pos); case {29, 30, 31} [v,pos] = deserialise_sparse(m,pos); +% case 32 +% [v,pos] = deserialise_themselves(m,pos); otherwise error('MATLAB:deserialise_value:unrecognised_tag', 'Cannot deserialise tag %s.', hlp_serial_types.type_details(type+1).name); end @@ -73,7 +75,9 @@ end function [v,pos] = deserialise_simple_data(m, pos) -[type, nDims, pos] = get_tag_data(m, pos); +[type, nDims] = hlp_serial_types.unpack_tag_data(m(pos)); +pos = pos+1; + switch type.name case {'logical', 'char', 'string'} @@ -110,8 +114,8 @@ function [v, pos] = deserialise_complex_data(m, pos) -[type, nDims, pos] = get_tag_data(m, pos); - +[type, nDims] = hlp_serial_types.unpack_tag_data(m(pos)); +pos = pos+1; if nDims == 0 [data, pos] = read_bytes(m, pos, type.name, 1); @@ -130,7 +134,9 @@ % Sparse data types function [v, pos] = deserialise_sparse(m, pos) -[type, ~, pos] = get_tag_data(m, pos); +[type, ~] = hlp_serial_types.unpack_tag_data(m(pos)); +pos = pos+1; + switch type.name case 'sparse_logical' @@ -162,7 +168,10 @@ end function [v, pos] = deserialise_cell(m, pos) -[~, nDims, pos] = get_tag_data(m, pos); +[~, nDims] = hlp_serial_types.unpack_tag_data(m(pos)); +pos = pos+1; + + if nDims == 0 [v, pos] = deserialise_value(m, pos); @@ -187,7 +196,10 @@ end function [v, pos] = deserialise_struct(m, pos) -[~, nDims, pos] = get_tag_data(m, pos); +[~, nDims] = hlp_serial_types.unpack_tag_data(m(pos)); +pos = pos+1; + + if nDims == 0 v = struct(); elseif nDims == 1 @@ -228,7 +240,10 @@ end function [v, pos] = deserialise_function_handle(m, pos) -[~, tag, pos] = get_tag_data(m, pos); +[~, tag] = hlp_serial_types.unpack_tag_data(m(pos)); +pos = pos+1; + + switch tag case 1 % Simple [name, pos] = deserialise_simple_data(m, pos); @@ -257,7 +272,9 @@ end function [v, pos] = deserialise_object(m, pos) -[~, nDims, pos] = get_tag_data(m, pos); +[~, nDims] = hlp_serial_types.unpack_tag_data(m(pos)); +pos = pos+1; + if nDims == 0 [class_name, pos] = deserialise_simple_data(m, pos); @@ -296,7 +313,8 @@ v(1:totalElem) = feval(class_name); for i=1:totalElem [v(i),nbytes] = v(i).deserialize(m, pos); - pos = pos+nbytes+8; + %pos = pos+nbytes+8; + pos = pos+nbytes; end case 1 % Serialise as saveobj (must have loadobj) @@ -323,11 +341,3 @@ end end -function [type, nDims, pos] = get_tag_data(m, pos) -[type, pos] = read_bytes(m, pos, 'uint8', 1); -% Take top 3 bits -nDims = bitshift(bitand(32+64+128, type), -5); -% Take bottom 5 bits -type = hlp_serial_types.type_details(bitand(31, type) + 1); - -end \ No newline at end of file diff --git a/herbert_core/utilities/misc/hlp_serial_size.m b/herbert_core/utilities/misc/hlp_serial_size.m index 9d00f6d8eb..5d6424485f 100644 --- a/herbert_core/utilities/misc/hlp_serial_size.m +++ b/herbert_core/utilities/misc/hlp_serial_size.m @@ -30,7 +30,11 @@ elseif islogical(v) sz = serial_size_logical(v); elseif isobject(v) - sz = serial_size_object(v); + if isa(v,'serializable') + sz = size_themselves(v); + else + sz = serial_size_object(v); + end elseif isjava(v) warn_once('hlp_serialize:cannot_serialize_java','Cannot properly serialize Java class %s; using a placeholder instead.',class(v)); sz = serial_size_string(['<>']); @@ -238,13 +242,15 @@ end end end - +function sz = size_themselves(v) +% serializable object calculates its size for themselves + sz = hlp_serial_types.tag_size + v.serial_size(); +end % Object / class function sz = serial_size_object(v) % can object serialize/deserizlize itself? -if isa(v,'serializable') - sz = hlp_serial_types.tag_size + v.serial_size(); -elseif ismethod(v, 'serialize') + +if ismethod(v, 'serialize') % it has to have serial_size method too sz = hlp_serial_types.tag_size + serial_size_string(class(v)) + v.serial_size(); %m = [uint8(135); serialize_string(class(v)); v.serialize()]; diff --git a/herbert_core/utilities/misc/hlp_serial_types.m b/herbert_core/utilities/misc/hlp_serial_types.m index c302441bfc..9bc0f30e7f 100644 --- a/herbert_core/utilities/misc/hlp_serial_types.m +++ b/herbert_core/utilities/misc/hlp_serial_types.m @@ -8,10 +8,10 @@ 'complex_uint8', 'complex_int16', 'complex_uint16', 'complex_int32',... 'complex_uint32', 'complex_int64', 'complex_uint64', 'cell', 'struct',... 'function_handle', 'value_object', 'handle_object_ref', 'enum',... - 'sparse_logical', 'sparse_double', 'sparse_complex_double',... - 'serializable'} + 'sparse_logical', 'sparse_double', 'sparse_complex_double'};%,... + %'serializable'} - lookup = containers.Map(hlp_serial_types.types,1:33); + lookup = containers.Map(hlp_serial_types.types,1:32); % Details associated with type type_details = struct('name',... @@ -20,15 +20,17 @@ {1, 1, 2, 8,... % 'logical', 'char', 'string', 'double',... 4, 1, 1, 2, 2, 4, 4,... % 'single', 'int8', 'uint8', 'int16', 'uint16', 'int32', 'uint32',... 8, 8, 16, 8, 2, ... % 'int64', 'uint64', 'complex_double', 'complex_single', 'complex_int8',... - 2, 4, 4, 8,... % 'complex_uint8', 'complex_int16', 'complex_uint16', 'complex_int32',... + 2, 4, 4, 8,... % 'complex_uint8', 'complex_int16', 'complex_uint16', 'complex_int32',... 8, 16, 16, 0, 0, ... % 'complex_uint32', 'complex_int64', 'complex_uint64', 'cell', 'struct',... 0, 0, 0, 0, ... % 'function_handle', 'value_object', 'handle_object_ref', 'enum',... - 1, 8, 16,... % 'sparse_logical', 'sparse_double', 'sparse_complex_double' - 0},... % 'serializable' -- object serializes itself - 'tag',... % Lookup tags for type of serialised data - cellfun(@uint8, num2cell(0:32), 'UniformOutput', false)); + 1, 8, 16},... % 'sparse_logical', 'sparse_double', 'sparse_complex_double' + 'tag',... % Lookup tags for type of serialised data, + cellfun(@uint8, num2cell(0:31), 'UniformOutput', false)); + % 0},... % 'serializable' -- object serializes itself + % - tag_size = 1; % Size of standard tag (uint8) in bytes + + tag_size = 2; % Size of standard tag (uint8) in bytes ndims_size = 1;% Size of standard num dimensions (uint8) in bytes dim_size = 4; % Size of standard dimension (uint32) in bytes dims_tag = uint8([32 64 96 128 160 192 224]); % Dims tags to set number of dimensions @@ -38,7 +40,10 @@ function details = get_details(type) details = hlp_serial_types.type_details(hlp_serial_types.lookup(type)); end - +% function tag = dims_tag(nDims) +% tag = uint8(nDims); +% end +% function size = get_size(type) size = hlp_serial_types.type_details(hlp_serial_types.lookup(type)).size; end @@ -68,7 +73,25 @@ else objID_struc = hlp_serial_types.get_details('value_object'); end - + end + % + function [type, nDims] = unpack_tag_data(head_byte) + % Take top 3 bits + nDims = bitshift(bitand(32+64+128, head_byte), -5); + % Take bottom 5 bits and retrieve the type from types map + type = hlp_serial_types.type_details(bitand(31, head_byte) + 1); + end + % + function comb_tag = pack_tag_data(nElem,nDims,sizeV1,type_struc) + if nElem == 0 + comb_tag = hlp_serial_types.dims_tag(1) + type_struc.tag; + elseif nElem == 1 + comb_tag = type_struc.tag; + elseif nDims == 2 && sizeV1 == 1 % List + comb_tag =hlp_serial_types.dims_tag(1) + type_struc.tag; + else + comb_tag =hlp_serial_types.dims_tag(nDims) + type_struc.tag; + end end end diff --git a/herbert_core/utilities/misc/hlp_serialise.m b/herbert_core/utilities/misc/hlp_serialise.m index 805956d4c0..1611610544 100644 --- a/herbert_core/utilities/misc/hlp_serialise.m +++ b/herbert_core/utilities/misc/hlp_serialise.m @@ -88,18 +88,21 @@ v = uint8(v); end +sizeV1 = size(v,1); +comb_tag = hlp_serial_types.pack_tag_data(nElem,nDims,sizeV1,type); + if nElem == 0 % Null element - m = [hlp_serial_types.dims_tag(1) + type.tag; ... + m = [comb_tag; ... typecast(uint32(0), 'uint8').']; elseif nElem == 1 % Scalar - m = [type.tag; ... + m = [comb_tag; ... typecast(v, 'uint8').']; -elseif nDims == 2 && size(v,1) == 1 % List - m = [hlp_serial_types.dims_tag(1) + type.tag; ... +elseif nDims == 2 && sizeV1 == 1 % List + m = [comb_tag; ... typecast(uint32(nElem), 'uint8').'; ... typecast(v, 'uint8').']; else % General array - m = [hlp_serial_types.dims_tag(nDims) + type.tag; ... + m = [comb_tag; ... typecast(uint32(size(v)), 'uint8').'; ... typecast(v(:).', 'uint8').']; end @@ -109,20 +112,24 @@ function m = serialise_complex_data(v, type) nElem = numel(v); nDims = uint8(ndims(v)); + +sizeV1 = size(v,1); +comb_tag = hlp_serial_types.pack_tag_data(nElem,nDims,sizeV1,type); + if nElem == 0 % Null element - m = [hlp_serial_types.dims_tag(1) + type.tag; ... + m = [comb_tag; ... typecast(uint32(0), 'uint8')]; elseif nElem == 1 % Scalar - m = [type.tag; ... + m = [comb_tag; ... typecast(real(v), 'uint8').'; ... typecast(imag(v), 'uint8').']; -elseif nDims == 2 && size(v,1) == 1 % List - m = [hlp_serial_types.dims_tag(1) + type.tag; ... +elseif nDims == 2 && sizeV1 == 1 % List + m = [comb_tag; ... typecast(uint32(nElem), 'uint8').'; ... typecast(real(v), 'uint8').'; ... typecast(imag(v), 'uint8').']; else % General array - m = [hlp_serial_types.dims_tag(nDims) + type.tag; ... + m = [comb_tag; ... typecast(uint32(size(v)), 'uint8').'; ... typecast(real(v(:)), 'uint8'); ... typecast(imag(v(:)), 'uint8')]; @@ -133,6 +140,8 @@ function m = serialise_sparse_data(v, type) [i,j,data] = find(v); +% HACK +comb_tag = hlp_serial_types.pack_tag_data(3,2,1,type); switch type.name case 'sparse_logical' @@ -143,7 +152,7 @@ dims = size(v); nElem = nnz(v); -m = [hlp_serial_types.dims_tag(2) + type.tag; ... +m = [comb_tag; ... typecast(uint32(dims), 'uint8').'; ... typecast(uint32(nElem), 'uint8').'; ... typecast(uint64(i(:)-1).', 'uint8').'; ... @@ -173,20 +182,23 @@ nElem = numel(v); nDims = uint8(ndims(v)); +sizeV1 = size(v,1); +comb_tag = hlp_serial_types.pack_tag_data(nElem,nDims,sizeV1,type); + if nElem == 0 % Null element - m = [hlp_serial_types.dims_tag(1) + type.tag; ... + m = [comb_tag; ... typecast(uint32(0), 'uint8').']; elseif nElem == 1 % Scalar - m = [type.tag; ... + m = [comb_tag; ... fnInfo; ... data]; -elseif nDims == 2 && size(v,1) == 1 % List - m = [hlp_serial_types.dims_tag(1) + type.tag; ... +elseif nDims == 2 && sizeV1 == 1 % List + m = [comb_tag; ... typecast(uint32(nElem), 'uint8').'; ... fnInfo; ... data]; else % General array - m = [hlp_serial_types.dims_tag(nDims) + type.tag; ... + m = [comb_tag; ... typecast(uint32(size(v)), 'uint8').'; ... fnInfo; ... data]; @@ -200,23 +212,28 @@ nElem = numel(v); nDims = uint8(ndims(v)); +sizeV1 = size(v,1); +comb_tag = hlp_serial_types.pack_tag_data(nElem,nDims,sizeV1,type); + + if nElem == 0 % Null element - m = [hlp_serial_types.dims_tag(1) + type.tag; ... + m = [comb_tag ; ... typecast(uint32(0), 'uint8').']; elseif nElem == 1 % Scalar m = [type.tag; data]; -elseif nDims == 2 && size(v,1) == 1 % List - m = [hlp_serial_types.dims_tag(1) + type.tag; ... +elseif nDims == 2 && sizeV1 == 1 % List + m = [comb_tag ; ... typecast(uint32(nElem), 'uint8').'; ... data]; else % General array - m = [hlp_serial_types.dims_tag(nDims) + type.tag; ... + m = [comb_tag ; ... typecast(uint32(size(v)), 'uint8').'; ... data]; end end function m = serialize_themselves(v, type) - +% serializable type serializes itself +m = [type.tag;v.serialize()]; end function m = serialise_object(v, type) @@ -225,7 +242,7 @@ class_name = serialise_simple_data(class(v), hlp_serial_types.get_details('char')); % can object serialise/deserialise itself? -if any(strcmp(methods(v), 'serialize')) +if ismethod(v, 'serialize') conts = arrayfun(@(x) (x.serialize()), v); ser_tag = uint8(0); else @@ -265,21 +282,43 @@ function m = serialise_function_handle(v, type) % get the representation rep = functions(v); + +% HACK +switch rep.type + % Tag is used to distinguish function type + case {'simple', 'classsimple'} + % simple function + nElem = 0; + nDims = 1; + sizeV1 = 2; %irrelevant + case 'anonymous' + % anonymous function + nElem = 3; + nDims = 2; + sizeV1 = 2; % not 1 + case {'scopedfunction','nested'} + % scoped function + nElem = 3; + nDims = 3; + sizeV1 = 2; % not 1 +end +comb_tag = hlp_serial_types.pack_tag_data(nElem,nDims,sizeV1,type); + switch rep.type % Tag is used to distinguish function type case {'simple', 'classsimple'} % simple function - m = [hlp_serial_types.dims_tag(1)+type.tag; ... Tag + m = [comb_tag; ... Tag serialise_simple_data(rep.function, hlp_serial_types.get_details('char'))]; % Name of function case 'anonymous' % anonymous function - m = [hlp_serial_types.dims_tag(2)+type.tag; ... Tag + m = [comb_tag; ... Tag serialise_simple_data(char(v), hlp_serial_types.get_details('char')); ... % Code serialise_struct(rep.workspace{1}, hlp_serial_types.get_details('struct'))]; % Workspace case {'scopedfunction','nested'} % scoped function - m = [hlp_serial_types.dims_tag(3)+type.tag; ... Tag + m = [comb_tag; ... Tag serialise_cell(rep.parentage, hlp_serial_types.get_details('cell'))]; % Parentage otherwise warn_once('hlp_serialise:unknown_handle_type','A function handle with unsupported type "%s" was encountered; using a placeholder instead.',rep.type);