Skip to content

Commit

Permalink
Re pace-neutrons/Herbert#365 modified serialise class, with headers i…
Browse files Browse the repository at this point in the history 
…nformation extracted into separate routine
  • Loading branch information
@abuts
abuts committed Oct 22, 2021
1 parent ce16558 commit 13d3d36
Show file tree
Hide file tree
Showing 5 changed files with 173 additions and 63 deletions.
40 changes: 36 additions & 4 deletions _test/test_serializers/test_serialise.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,22 @@


end
%
function test_serial_arrays(~)
f1=1:10;
f2 =f1';
f1 = repmat(f1,5,1,5);
f2 = repmat(f2,1,5,5);
bytes1 = hlp_serialise(f1);
bytes2 = hlp_serialise(f2);
[f1_rec,nbytes1] = hlp_deserialise(bytes1);
assertEqual(numel(bytes1),nbytes1)
[f2_rec,nbytes2] = hlp_deserialise(bytes2);
assertEqual(numel(bytes2),nbytes2)
%
assertEqual(f1,f1_rec);
assertEqual(f2,f2_rec);
end


%------------------------------------------------------------------
Expand Down Expand Up @@ -683,7 +699,7 @@ function test_ser_serializeble_obj_array_level1(obj)
assertEqual(size,numel(ser));
%--------------------------------------------------------------
% Serialize using C++

if ~obj.use_mex
skipTest('Mex mode is not currently available for: test_ser_serializeble_obj_array');
end
Expand Down Expand Up @@ -724,6 +740,7 @@ function test_ser_serializeble_obj(obj)

size = hlp_serial_sise(serCl);
assertEqual(size,numel(ser));
skipTest('C++ deserializer does not work propertly; #394')
if ~obj.use_mex
skipTest('Mex mode is not currently available for: test_ser_serializeble_obj');
end
Expand All @@ -734,7 +751,6 @@ function test_ser_serializeble_obj(obj)
ser_c = c_serialise(serCl);
assertEqual(ser_c,ser)

skipTest('C++ deserializer does not work propertly; #394')
[serCl_rec,nbytes] = c_deserialise(ser_c);

%
Expand All @@ -745,7 +761,23 @@ function test_ser_serializeble_obj(obj)

assertEqual(size_c,numel(ser));
end


function test_pack_unpack_header(~)

type_details = hlp_serial_types.type_details;
for ntype = 1:numel(type_details)
for nElem = 1:3
for nDims = 1:8
for sizeV1 = 0:1
packed_tag = hlp_serial_types.pack_tag_data(...
nElem,nDims,sizeV1,type_details(ntype));
[type_rec, nDims_rec] = ...
hlp_serial_types.unpack_tag_data(packed_tag);
assertEqual(type_rec,type_details(ntype));
assertEqual(nDims_rec,nDims);
end
end
end
end
end
end
end
46 changes: 28 additions & 18 deletions herbert_core/utilities/misc/hlp_deserialise.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
[v,pos] = deserialise_simple_data(m,pos);
case {13,14,15,16,17,18,19,20,21,22}
[v,pos] = deserialise_complex_data(m,pos);
case {23}
case 23
[v,pos] = deserialise_cell(m,pos);
case {24}
[v,pos] = deserialise_struct(m,pos);
Expand All @@ -53,6 +53,8 @@
[v,pos] = deserialise_object(m,pos);
case {29, 30, 31}
[v,pos] = deserialise_sparse(m,pos);
% case 32
% [v,pos] = deserialise_themselves(m,pos);
otherwise
error('MATLAB:deserialise_value:unrecognised_tag', 'Cannot deserialise tag %s.', hlp_serial_types.type_details(type+1).name);
end
Expand All @@ -73,7 +75,9 @@
end

function [v,pos] = deserialise_simple_data(m, pos)
[type, nDims, pos] = get_tag_data(m, pos);
[type, nDims] = hlp_serial_types.unpack_tag_data(m(pos));
pos = pos+1;


switch type.name
case {'logical', 'char', 'string'}
Expand Down Expand Up @@ -110,8 +114,8 @@

function [v, pos] = deserialise_complex_data(m, pos)

[type, nDims, pos] = get_tag_data(m, pos);

[type, nDims] = hlp_serial_types.unpack_tag_data(m(pos));
pos = pos+1;

if nDims == 0
[data, pos] = read_bytes(m, pos, type.name, 1);
Expand All @@ -130,7 +134,9 @@

% Sparse data types
function [v, pos] = deserialise_sparse(m, pos)
[type, ~, pos] = get_tag_data(m, pos);
[type, ~] = hlp_serial_types.unpack_tag_data(m(pos));
pos = pos+1;


switch type.name
case 'sparse_logical'
Expand Down Expand Up @@ -162,7 +168,10 @@
end

function [v, pos] = deserialise_cell(m, pos)
[~, nDims, pos] = get_tag_data(m, pos);
[~, nDims] = hlp_serial_types.unpack_tag_data(m(pos));
pos = pos+1;



if nDims == 0
[v, pos] = deserialise_value(m, pos);
Expand All @@ -187,7 +196,10 @@
end

function [v, pos] = deserialise_struct(m, pos)
[~, nDims, pos] = get_tag_data(m, pos);
[~, nDims] = hlp_serial_types.unpack_tag_data(m(pos));
pos = pos+1;


if nDims == 0
v = struct();
elseif nDims == 1
Expand Down Expand Up @@ -228,7 +240,10 @@
end

function [v, pos] = deserialise_function_handle(m, pos)
[~, tag, pos] = get_tag_data(m, pos);
[~, tag] = hlp_serial_types.unpack_tag_data(m(pos));
pos = pos+1;


switch tag
case 1 % Simple
[name, pos] = deserialise_simple_data(m, pos);
Expand Down Expand Up @@ -257,7 +272,9 @@
end

function [v, pos] = deserialise_object(m, pos)
[~, nDims, pos] = get_tag_data(m, pos);
[~, nDims] = hlp_serial_types.unpack_tag_data(m(pos));
pos = pos+1;


if nDims == 0
[class_name, pos] = deserialise_simple_data(m, pos);
Expand Down Expand Up @@ -296,7 +313,8 @@
v(1:totalElem) = feval(class_name);
for i=1:totalElem
[v(i),nbytes] = v(i).deserialize(m, pos);
pos = pos+nbytes+8;
%pos = pos+nbytes+8;
pos = pos+nbytes;
end
case 1 % Serialise as saveobj (must have loadobj)

Expand All @@ -323,11 +341,3 @@
end
end

function [type, nDims, pos] = get_tag_data(m, pos)
[type, pos] = read_bytes(m, pos, 'uint8', 1);
% Take top 3 bits
nDims = bitshift(bitand(32+64+128, type), -5);
% Take bottom 5 bits
type = hlp_serial_types.type_details(bitand(31, type) + 1);

end
16 changes: 11 additions & 5 deletions herbert_core/utilities/misc/hlp_serial_size.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,11 @@
elseif islogical(v)
sz = serial_size_logical(v);
elseif isobject(v)
sz = serial_size_object(v);
if isa(v,'serializable')
sz = size_themselves(v);
else
sz = serial_size_object(v);
end
elseif isjava(v)
warn_once('hlp_serialize:cannot_serialize_java','Cannot properly serialize Java class %s; using a placeholder instead.',class(v));
sz = serial_size_string(['<<hlp_serialize: ' class(v) ' unsupported>>']);
Expand Down Expand Up @@ -238,13 +242,15 @@
end
end
end

function sz = size_themselves(v)
% serializable object calculates its size for themselves
sz = hlp_serial_types.tag_size + v.serial_size();
end
% Object / class
function sz = serial_size_object(v)
% can object serialize/deserizlize itself?
if isa(v,'serializable')
sz = hlp_serial_types.tag_size + v.serial_size();
elseif ismethod(v, 'serialize')

if ismethod(v, 'serialize')
% it has to have serial_size method too
sz = hlp_serial_types.tag_size + serial_size_string(class(v)) + v.serial_size();
%m = [uint8(135); serialize_string(class(v)); v.serialize()];
Expand Down
45 changes: 34 additions & 11 deletions herbert_core/utilities/misc/hlp_serial_types.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
'complex_uint8', 'complex_int16', 'complex_uint16', 'complex_int32',...
'complex_uint32', 'complex_int64', 'complex_uint64', 'cell', 'struct',...
'function_handle', 'value_object', 'handle_object_ref', 'enum',...
'sparse_logical', 'sparse_double', 'sparse_complex_double',...
'serializable'}
'sparse_logical', 'sparse_double', 'sparse_complex_double'};%,...
%'serializable'}

lookup = containers.Map(hlp_serial_types.types,1:33);
lookup = containers.Map(hlp_serial_types.types,1:32);

% Details associated with type
type_details = struct('name',...
Expand All @@ -20,15 +20,17 @@
{1, 1, 2, 8,... % 'logical', 'char', 'string', 'double',...
4, 1, 1, 2, 2, 4, 4,... % 'single', 'int8', 'uint8', 'int16', 'uint16', 'int32', 'uint32',...
8, 8, 16, 8, 2, ... % 'int64', 'uint64', 'complex_double', 'complex_single', 'complex_int8',...
2, 4, 4, 8,... % 'complex_uint8', 'complex_int16', 'complex_uint16', 'complex_int32',...
2, 4, 4, 8,... % 'complex_uint8', 'complex_int16', 'complex_uint16', 'complex_int32',...
8, 16, 16, 0, 0, ... % 'complex_uint32', 'complex_int64', 'complex_uint64', 'cell', 'struct',...
0, 0, 0, 0, ... % 'function_handle', 'value_object', 'handle_object_ref', 'enum',...
1, 8, 16,... % 'sparse_logical', 'sparse_double', 'sparse_complex_double'
0},... % 'serializable' -- object serializes itself
'tag',... % Lookup tags for type of serialised data
cellfun(@uint8, num2cell(0:32), 'UniformOutput', false));
1, 8, 16},... % 'sparse_logical', 'sparse_double', 'sparse_complex_double'
'tag',... % Lookup tags for type of serialised data,
cellfun(@uint8, num2cell(0:31), 'UniformOutput', false));
% 0},... % 'serializable' -- object serializes itself
%

tag_size = 1; % Size of standard tag (uint8) in bytes

tag_size = 2; % Size of standard tag (uint8) in bytes
ndims_size = 1;% Size of standard num dimensions (uint8) in bytes
dim_size = 4; % Size of standard dimension (uint32) in bytes
dims_tag = uint8([32 64 96 128 160 192 224]); % Dims tags to set number of dimensions
Expand All @@ -38,7 +40,10 @@
function details = get_details(type)
details = hlp_serial_types.type_details(hlp_serial_types.lookup(type));
end

% function tag = dims_tag(nDims)
% tag = uint8(nDims);
% end
%
function size = get_size(type)
size = hlp_serial_types.type_details(hlp_serial_types.lookup(type)).size;
end
Expand Down Expand Up @@ -68,7 +73,25 @@
else
objID_struc = hlp_serial_types.get_details('value_object');
end

end
%
function [type, nDims] = unpack_tag_data(head_byte)
% Take top 3 bits
nDims = bitshift(bitand(32+64+128, head_byte), -5);
% Take bottom 5 bits and retrieve the type from types map
type = hlp_serial_types.type_details(bitand(31, head_byte) + 1);
end
%
function comb_tag = pack_tag_data(nElem,nDims,sizeV1,type_struc)
if nElem == 0
comb_tag = hlp_serial_types.dims_tag(1) + type_struc.tag;
elseif nElem == 1
comb_tag = type_struc.tag;
elseif nDims == 2 && sizeV1 == 1 % List
comb_tag =hlp_serial_types.dims_tag(1) + type_struc.tag;
else
comb_tag =hlp_serial_types.dims_tag(nDims) + type_struc.tag;
end
end
end

Expand Down
Loading

0 comments on commit 13d3d36

Please sign in to comment.