From 6577b077129f54254f9db347f9141ecfc24171c6 Mon Sep 17 00:00:00 2001 From: Duc Le Date: Mon, 21 Mar 2022 08:40:18 +0000 Subject: [PATCH] Add compiled workers #488 --- admin/CMakeLists.txt | 42 +++++++++++++ admin/compiled_worker.m | 63 +++++++++++++++++++ .../configuration/@hpc_config/hpc_config.m | 9 ++- .../private/check_worker_configured.m | 17 ----- 4 files changed, 111 insertions(+), 20 deletions(-) create mode 100644 admin/compiled_worker.m delete mode 100644 horace_core/configuration/@hpc_config/private/check_worker_configured.m diff --git a/admin/CMakeLists.txt b/admin/CMakeLists.txt index 37c749d433..06ad13867c 100644 --- a/admin/CMakeLists.txt +++ b/admin/CMakeLists.txt @@ -40,3 +40,45 @@ install( FILES "horace_install.m" ${TEMPLATE_FILES} DESTINATION "." ) + +# Builds compiled worker +find_program( + Matlab_MCC_COMPILER + "mcc" + PATHS ${Matlab_BINARIES_DIR} + NO_DEFAULT_PATH +) +if(Matlab_MCC_COMPILER) + # We need to copy horace|herbert_core to a new folder because mcc doesn't like empty or docify files + if(WIN32) + add_custom_command( + OUTPUT CTF + COMMAND powershell New-Item -ItemType Directory -Force -Path "CTF" + COMMAND powershell Copy-Item "${LOCAL_INIT_DIR}/worker_v2.m" "CTF" + COMMAND cmd /c (robocopy "${Horace_CORE}" CTF/Horace /s /nfl /ndl /nc /ns /np) ^& IF %ERRORLEVEL% LEQ 3 exit 0 + COMMAND cmd /c (robocopy "${Herbert_CORE}" CTF/Herbert /s /nfl /ndl /nc /ns /np) ^& IF %ERRORLEVEL% LEQ 3 exit 0 + COMMAND powershell -Command "& {Get-ChildItem CTF -recurse | where-object {$_.length -eq 0} | ?{Remove-Item $_.fullname} }" + COMMAND powershell -Command "& {Get-ChildItem CTF -recurse | where-object {$_.Name -match '_docify'} | ?{Remove-Item $_.fullname -recurse}}" + COMMAND powershell -Command "& {Get-ChildItem CTF -recurse | where-object {$_.Name -match '@*_old'} | ?{Remove-Item $_.fullname -recurse}}" + ) + else() + add_custom_command( + OUTPUT CTF + COMMAND sh mkdir -p CTF + COMMAND sh cp -r "${Horace_CORE}" "${Herbert_CORE}" "${LOCAL_INIT_DIR}"/worker_v2.m CTF/ + COMMAND sh find CTF -name _docify -o -name "@*_old" -o size 0 -exec rm -rf '{}' \; + ) + endif() + add_custom_command( + OUTPUT compiled_worker.exe + COMMAND ${Matlab_MCC_COMPILER} -W main:compiled_worker -a CTF "${CMAKE_CURRENT_SOURCE_DIR}/compiled_worker.m" + DEPENDS CTF accumulate_cut_c bin_pixels_c calc_projections_c combine_sqw compute_pix_sums_c + hdf_mex_reader mtimesx_mex sort_pixels_by_bins + ) + # Need to add a custom target here - build it with `cmake --build . --target build_worker` + add_custom_target(build_worker DEPENDS compiled_worker.exe) +else() + add_custom_target(build_worker + COMMAND echo "Error: Matlab compiler mcc not found" && exit 1 + ) +endif() diff --git a/admin/compiled_worker.m b/admin/compiled_worker.m new file mode 100644 index 0000000000..95933ccf61 --- /dev/null +++ b/admin/compiled_worker.m @@ -0,0 +1,63 @@ +function compiled_worker(varargin) + % Entry point function for compiled Matlab worker + [control_string, logfile] = parse_input_args(varargin); + % Use evalc to capture output and pipe to logfile if needed + out = evalc(['worker_v2( ''' control_string ''')']); + if ~isempty(logfile) + fid = fopen(logfile, 'w'); + fprintf(fid, '%s\n', out); + fclose(fid); + end +end + +function [cs, lf] = parse_input_args(args) + % Parses input arguments to extract the control string if one exists + % + % The allow arguments are: + % -logfile : Save output to a logfile + % -r : CMD is parsed to see if has control string + % -batch : CMD is parsed to see if has control string + % -nosplash : Ignored + % -nojvm : Ignored + % -nodesktop : Ignored + % control_string : The first positional argument is considered the control string + % + % The control string returned is extracted from the positional or last keyword argument + % that matches - e.g. "-r worker_v2(cs1) -batch worker(cs2)" returns cs2 + % and "-r worker(cs1) -batch c(cs2) cs3" returns cs3 + cs = ''; % Default is empty string + lf = ''; + is_seen = cellfun(@(x) x(1)=='-', args); + key_idx = find(is_seen); + for ii = 1:numel(key_idx) + next_idx = key_idx(ii) + 1; + switch(args{key_idx(ii)}) + case '-logfile' + lf = args{next_idx}; + is_seen(next_idx) = 1; + case {'-r', '-batch'} + cs = parse_cmd(args{next_idx}); + is_seen(next_idx) = 1; + case {'-nosplash', '-nojvm', '-nodesktop'} + % Does nothing + otherwise + error(['Unknown option ' args{key_idx(ii)}]); + end + end + idx_not_seen = find(~is_seen); + if ~isempty(idx_not_seen) + cs = args{idx_not_seen(1)}; + end +end + +function cs = parse_cmd(cmdstr) + % Parses a command string to extract a control string + cs = ''; % Default is empty string + if contains(cmdstr, '(') && contains(cmdstr, ')') + [token, match] = regexp(cmdstr, "[\w\d\.\\\/:]*\('([\w\d\-]*)'\).*", 'tokens', 'match'); + if ~isempty(token) + while iscell(token), token = token{1}; end + cs = char(token); + end + end +end diff --git a/horace_core/configuration/@hpc_config/hpc_config.m b/horace_core/configuration/@hpc_config/hpc_config.m index 4d54cc0a9c..198d1e5ddb 100644 --- a/horace_core/configuration/@hpc_config/hpc_config.m +++ b/horace_core/configuration/@hpc_config/hpc_config.m @@ -297,9 +297,12 @@ accum = false; end if accum - [ok,mess] = check_worker_configured(this); - if ~ok - warning('HPC_CONFIG:invalid_argument',... + wrkr = parallel_config().worker; + if isempty(fileparts(which(wrkr))) && ~exist(wrkr, 'file') + html = 'https://pace-neutrons.github.io/Horace/v3.6.2/introduction/Download_and_setup.html#troubleshooting'; + mess=['Can not find worker on a data search path; See: ' html, ... + 'for the details on how to set it up']; + warning('HPC_CONFIG:invalid_argument', ... ' Can not start accumulating in separate process as: %s',... mess); accum = false; diff --git a/horace_core/configuration/@hpc_config/private/check_worker_configured.m b/horace_core/configuration/@hpc_config/private/check_worker_configured.m deleted file mode 100644 index 30bbdb5748..0000000000 --- a/horace_core/configuration/@hpc_config/private/check_worker_configured.m +++ /dev/null @@ -1,17 +0,0 @@ -function [ok,mess] = check_worker_configured(obj) -% Check if worker configured properly to allow it Horace multisession -% processing -% -pc = parallel_config(); -wkr = pc.worker; -wrker_path = fileparts(which(wkr)); -if isempty(wrker_path) - ok=false; - mess=['Can not find worker on a data search path; ',... - 'See: http://horace.isis.rl.ac.uk/Download_and_setup#Enabling_multi-sessions_processing ',... - 'for the details on how to set it up']; -else - ok=true; - mess = ''; -end -