From 5c8b9392f34883bf1ed3bc497ced9a1f8b33c09d Mon Sep 17 00:00:00 2001 From: leondavi Date: Sat, 11 Nov 2023 01:48:44 +0200 Subject: [PATCH] [AUTOGENERATED] Add worker hrl with definitions of fields for parser --- NerlnetBuild.sh | 5 ++ .../NerlnetApp/src/worker_definitions_ag.hrl | 11 ++++ src_py/nerlPlanner/ErlHeadersExporter.py | 58 +++++++++++++++++++ src_py/nerlPlanner/ErlHeadersExporterDefs.py | 52 +++++++++++++++++ .../JsonElementWorkerDefinitions.py | 3 +- 5 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 src_erl/NerlnetApp/src/worker_definitions_ag.hrl create mode 100644 src_py/nerlPlanner/ErlHeadersExporter.py create mode 100644 src_py/nerlPlanner/ErlHeadersExporterDefs.py diff --git a/NerlnetBuild.sh b/NerlnetBuild.sh index a2e03e64..d732e0a4 100755 --- a/NerlnetBuild.sh +++ b/NerlnetBuild.sh @@ -152,9 +152,14 @@ fi if command -v python3 >/dev/null 2>&1; then echo "$NERLNET_BUILD_PREFIX Python 3 is installed" # Generate auto-generated files + set -e AUTOGENERATED_WORKER_DEFINITIONS_PATH="`pwd`/src_cpp/opennnBridge/worker_definitions_ag.h" + AUTOGENERATED_WORKER_DEFINITIONS_PATH_HRL="`pwd`/src_erl/NerlnetApp/src/worker_definitions_ag.hrl" + echo "$NERLNET_BUILD_PREFIX Generate auto-generated files" python3 src_py/nerlPlanner/CppHeadersExporter.py --output $AUTOGENERATED_WORKER_DEFINITIONS_PATH #--debug + python3 src_py/nerlPlanner/ErlHeadersExporter.py --output $AUTOGENERATED_WORKER_DEFINITIONS_PATH_HRL #--debug + set +e else echo "$NERLNET_BUILD_PREFIX Python 3 is not installed" echo "Autogenerated files will not be generated" diff --git a/src_erl/NerlnetApp/src/worker_definitions_ag.hrl b/src_erl/NerlnetApp/src/worker_definitions_ag.hrl new file mode 100644 index 00000000..d716bce7 --- /dev/null +++ b/src_erl/NerlnetApp/src/worker_definitions_ag.hrl @@ -0,0 +1,11 @@ +% This is an auto generated .hrl file +-define(NERLPLANNER_VERSION,"1.0.0"). + +-define(KEY_MODEL_TYPE,modelType). +-define(KEY_LAYER_SIZES_LIST,layersSizes). +-define(KEY_LAYER_TYPES_LIST,layerTypesList). +-define(KEY_LAYERS_FUNCTIONS,layers_functions). +-define(KEY_LOSS_METHOD,lossMethod). +-define(KEY_LEARNING_RATE,lr). +-define(KEY_EPOCHS,epochs). +-define(KEY_OPTIMIZER_TYPE,optimizer). diff --git a/src_py/nerlPlanner/ErlHeadersExporter.py b/src_py/nerlPlanner/ErlHeadersExporter.py new file mode 100644 index 00000000..3196ce98 --- /dev/null +++ b/src_py/nerlPlanner/ErlHeadersExporter.py @@ -0,0 +1,58 @@ +import argparse +import os +from ErlHeadersExporterDefs import * +from JsonElementWorkerDefinitions import * +from Definitions import VERSION as NERLPLANNER_VERSION + +EMPTY_LINE = '\n' +DEBUG = False + +def gen_erlang_exporter_logger(message : str): + if DEBUG: + print(f'[NERLPLANNER][AUTO_HEADER_GENERATOR][DEBUG] {message}') + +def gen_worker_fields_hrl(header_path : str, debug : bool = False): + global DEBUG + DEBUG = debug + + auto_generated_header = AutoGeneratedHeader() + gen_erlang_exporter_logger(auto_generated_header.generate_code()) + + nerlplanner_version = Definition('NERLPLANNER_VERSION', f'"{NERLPLANNER_VERSION}"') + gen_erlang_exporter_logger(nerlplanner_version.generate_code()) + + fields_list_vals = [KEY_MODEL_TYPE, KEY_LAYER_SIZES_LIST, + KEY_LAYER_TYPES_LIST, KEY_LAYERS_FUNCTIONS, + KEY_LOSS_METHOD, KEY_LEARNING_RATE, + KEY_EPOCHS, KEY_OPTIMIZER_TYPE] + fields_list_strs = ['KEY_MODEL_TYPE', 'KEY_LAYER_SIZES_LIST', + 'KEY_LAYER_TYPES_LIST', 'KEY_LAYERS_FUNCTIONS', + 'KEY_LOSS_METHOD', 'KEY_LEARNING_RATE', + 'KEY_EPOCHS', 'KEY_OPTIMIZER_TYPE'] + + fields_list_defs = [ Definition(fields_list_strs[idx], f'{Definition.assert_not_atom(fields_list_vals[idx])}') for idx in range(len(fields_list_vals))] + [gen_erlang_exporter_logger(x.generate_code()) for x in fields_list_defs] + + + if os.path.dirname(header_path): + os.makedirs(os.path.dirname(header_path), exist_ok=True) + + with open(header_path, 'w') as f: + f.write(auto_generated_header.generate_code()) + f.write(nerlplanner_version.generate_code()) + f.write(EMPTY_LINE) + [f.write(x.generate_code()) for x in fields_list_defs] + + + + +def main(): + parser = argparse.ArgumentParser(description='Generate C++ header file for nerlPlanner') + parser.add_argument('-o', '--output', help='output header file path', required=True) + parser.add_argument('-d', '--debug', help='debug mode', action='store_true') + args = parser.parse_args() + gen_worker_fields_hrl(args.output, args.debug) + +if __name__=="__main__": + main() + diff --git a/src_py/nerlPlanner/ErlHeadersExporterDefs.py b/src_py/nerlPlanner/ErlHeadersExporterDefs.py new file mode 100644 index 00000000..8878e55c --- /dev/null +++ b/src_py/nerlPlanner/ErlHeadersExporterDefs.py @@ -0,0 +1,52 @@ +from collections import OrderedDict + +# Generates automated Erlang code! + +class AutoGeneratedHeader: + def __init__(self) -> None: + pass + + def generate_code(self): + return f'% This is an auto generated .hrl file\n' + +class ModuleName: + def __init__(self, module_name: str = '') -> None: + self.module_name = module_name + + def generate_code(self): + return f'-module({self.module_name}).\n ' + +class Record: + def __init__(self, record_name : str, in_ordered_dict : OrderedDict, all_caps : bool = False, prefix = '') -> None: + self.record_name = record_name.lower() + self.ordered_dict = in_ordered_dict + self.prefix = prefix + self.all_caps = all_caps + + def generate_code(self): + code = f'-record({self.record_name}'+'{' + for key, value in self.ordered_dict.items(): + key = key.upper() if self.all_caps else key + last_key = list(self.ordered_dict.keys())[-1].upper() if self.all_caps else list(self.ordered_dict.keys())[-1] + code += f'{self.prefix}_{key} = {value}' + code += ',' if last_key != key else '' + return code.replace('-','_') + '}).\n' + +class Definition: + def __init__(self, macro, value : str ) -> None: + self.macro = macro + self.value = value + + def force_atom_value(value : str): + # validates that first letter is lower case which is required by atoms in Erlang + return value[0].lower() + value[1:] if value else value + + def assert_not_atom(value : str): + if value: + assert value[0].islower(), f'Value {value} is not an Erlang atom' + else: + raise 'Erlang atom cannot be None or empty string!' + return value + + def generate_code(self): + return f'-define({self.macro},{self.value}).\n' \ No newline at end of file diff --git a/src_py/nerlPlanner/JsonElementWorkerDefinitions.py b/src_py/nerlPlanner/JsonElementWorkerDefinitions.py index f7b70330..ec56d994 100644 --- a/src_py/nerlPlanner/JsonElementWorkerDefinitions.py +++ b/src_py/nerlPlanner/JsonElementWorkerDefinitions.py @@ -112,7 +112,8 @@ def doc_print_dict(d):#define d pretty_dict += f' {k}:{str(v)} |' return pretty_dict#return result - +# DC fields of worker must be suitable with erlang atoms convention! (lower case first letter) +# Please run ./NerlnetBuild.sh to test any changes of this file! KEY_DOC_PREFIX = "_doc_" KEY_MODEL_TYPE = "modelType" KEY_MODEL_TYPE_DOC = "_doc_modelType"