From ad75311a8905dbe90f6d2340b7bdc98a99886a41 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Mon, 25 Nov 2024 12:15:27 +0100 Subject: [PATCH] CMake integration of pipeline-based plan writer --- loki/batch/scheduler.py | 2 +- loki/batch/tests/test_scheduler.py | 28 +++++----- loki/tests/test_cmake.py | 46 ++++++++++------ .../build_system/tests/test_file_write.py | 10 +--- scripts/loki_transform.py | 53 +++++++++++-------- 5 files changed, 77 insertions(+), 62 deletions(-) diff --git a/loki/batch/scheduler.py b/loki/batch/scheduler.py index 5be0ad12f..80c5037bf 100644 --- a/loki/batch/scheduler.py +++ b/loki/batch/scheduler.py @@ -610,7 +610,7 @@ def callgraph(self, path, with_file_graph=False, with_legend=False): warning(f'[Loki] Failed to render filegraph due to graphviz error:\n {e}') @Timer(logger=perf, text='[Loki::Scheduler] Wrote CMake plan file in {:.2f}s') - def write_cmake_plan(self, filepath, mode, buildpath, rootpath): + def write_cmake_plan(self, filepath, rootpath=None): """ Generate the "plan file" for CMake diff --git a/loki/batch/tests/test_scheduler.py b/loki/batch/tests/test_scheduler.py index 40977b85a..5508ade26 100644 --- a/loki/batch/tests/test_scheduler.py +++ b/loki/batch/tests/test_scheduler.py @@ -74,7 +74,7 @@ nodes as ir, FindNodes, FindInlineCalls, FindVariables ) from loki.transformations import ( - DependencyTransformation, ModuleWrapTransformation + DependencyTransformation, ModuleWrapTransformation, FileWriteTransformation ) @@ -1156,35 +1156,35 @@ def test_scheduler_cmake_planner(tmp_path, testdir, frontend): proj_b = sourcedir/'projB' config = SchedulerConfig.from_dict({ - 'default': {'role': 'kernel', 'expand': True, 'strict': True, 'ignore': ('header_mod',)}, + 'default': { + 'role': 'kernel', + 'expand': True, + 'strict': True, + 'ignore': ('header_mod',), + 'mode': 'foobar' + }, 'routines': { 'driverB': {'role': 'driver'}, 'kernelB': {'ignore': ['ext_driver']}, } }) + builddir = tmp_path/'scheduler_cmake_planner_dummy_dir' + builddir.mkdir(exist_ok=True) # Populate the scheduler # (this is the same as SchedulerA in test_scheduler_dependencies_ignore, so no need to # check scheduler set-up itself) scheduler = Scheduler( paths=[proj_a, proj_b], includes=proj_a/'include', - config=config, frontend=frontend, xmods=[tmp_path] + config=config, frontend=frontend, xmods=[tmp_path], + output_dir=builddir ) # Apply the transformation - builddir = tmp_path/'scheduler_cmake_planner_dummy_dir' - builddir.mkdir(exist_ok=True) planfile = builddir/'loki_plan.cmake' - scheduler.write_cmake_plan( - filepath=planfile, mode='foobar', buildpath=builddir, rootpath=sourcedir - ) - - # Validate the generated lists - expected_files = { - proj_a/'module/driverB_mod.f90', proj_a/'module/kernelB_mod.F90', - proj_a/'module/compute_l1_mod.f90', proj_a/'module/compute_l2_mod.f90' - } + scheduler.process(FileWriteTransformation(), plan=True) + scheduler.write_cmake_plan(filepath=planfile, rootpath=sourcedir) # Validate the plan file content plan_pattern = re.compile(r'set\(\s*(\w+)\s*(.*?)\s*\)', re.DOTALL) diff --git a/loki/tests/test_cmake.py b/loki/tests/test_cmake.py index 2a9f01bfa..f313d6617 100644 --- a/loki/tests/test_cmake.py +++ b/loki/tests/test_cmake.py @@ -71,9 +71,32 @@ def fixture_config(tmp_dir): the file path """ default_config = { - 'default': {'role': 'kernel', 'expand': True, 'strict': True, 'enable_imports': True}, + 'default': { + 'role': 'kernel', + 'expand': True, + 'strict': True, + 'enable_imports': True + }, 'routines': { 'driverB': {'role': 'driver'}, + }, + 'transformations': { + 'IdemTrafo': { + 'classname': 'IdemTransformation', + 'module': 'loki.transformations.idempotence', + }, + 'FileWriteTransformation': { + 'classname': 'FileWriteTransformation', + 'module': 'loki.transformations.build_system', + 'options': { + 'include_module_var_imports': True + } + } + }, + 'pipelines': { + 'idem': { + 'transformations': ['IdemTrafo'] + } } } filepath = tmp_dir/'test_cmake_loki.config' @@ -101,30 +124,25 @@ def fixture_loki_install(here, tmp_dir, ecbuild, silent, request): Install Loki using CMake into an install directory """ builddir = tmp_dir/'loki_bootstrap' - if builddir.exists(): - shutil.rmtree(builddir) - builddir.mkdir() - cmd = ['cmake', '-DENABLE_CLAW=OFF', f'-DCMAKE_MODULE_PATH={ecbuild}/cmake', str(here.parent.parent)] + cmd = [ + 'cmake', f'-DCMAKE_MODULE_PATH={ecbuild}/cmake', + '-S', str(here.parent.parent), + '-B', str(builddir) + ] if request.param: cmd += ['-DENABLE_EDITABLE=ON'] else: cmd += ['-DENABLE_EDITABLE=OFF'] - execute(cmd, silent=silent, cwd=builddir) + execute(cmd, silent=silent, cwd=tmp_dir) lokidir = tmp_dir/'loki' - if lokidir.exists(): - shutil.rmtree(lokidir) execute( ['cmake', '--install', '.', '--prefix', str(lokidir)], silent=True, cwd=builddir ) yield builddir, lokidir - if builddir.exists(): - shutil.rmtree(builddir) - if lokidir.exists(): - shutil.rmtree(lokidir) @contextmanager @@ -137,7 +155,6 @@ def clean_builddir(builddir): shutil.rmtree(builddir) builddir.mkdir() yield builddir - shutil.rmtree(builddir) @pytest.fixture(scope='module', name='cmake_project') @@ -158,7 +175,6 @@ def fixture_cmake_project(here, config, srcdir): loki_transform_plan( MODE idem - FRONTEND fp CONFIG {config} SOURCEDIR ${{CMAKE_CURRENT_SOURCE_DIR}} CALLGRAPH ${{CMAKE_CURRENT_BINARY_DIR}}/loki_callgraph @@ -166,8 +182,6 @@ def fixture_cmake_project(here, config, srcdir): SOURCES {proj_a} {proj_b} - HEADERS - {proj_a}/module/header_mod.f90 ) """ filepath = srcdir/'CMakeLists.txt' diff --git a/loki/transformations/build_system/tests/test_file_write.py b/loki/transformations/build_system/tests/test_file_write.py index fcc94bc27..d137b4a9a 100644 --- a/loki/transformations/build_system/tests/test_file_write.py +++ b/loki/transformations/build_system/tests/test_file_write.py @@ -163,10 +163,7 @@ def test_file_write_module_imports(frontend, tmp_path, enable_imports, import_le plan_file = tmp_path/'plan.cmake' root_path = tmp_path if use_rootpath else None scheduler.process(transformation, plan=True) - scheduler.write_cmake_plan( - filepath=plan_file, mode=config.default['mode'], buildpath=out_path, - rootpath=root_path - ) + scheduler.write_cmake_plan(filepath=plan_file, rootpath=root_path) # Validate the plan file content plan_pattern = re.compile(r'set\(\s*(\w+)\s*(.*?)\s*\)', re.DOTALL) @@ -302,10 +299,7 @@ def test_file_write_replicate(tmp_path, caplog, frontend, have_non_replicate_con caplog.clear() with caplog.at_level(log_levels['WARNING']): - scheduler.write_cmake_plan( - filepath=plan_file, mode=config.default['mode'], buildpath=out_path, - rootpath=tmp_path - ) + scheduler.write_cmake_plan(filepath=plan_file, rootpath=tmp_path) if have_non_replicate_conflict: assert len(caplog.records) == 1 assert 'c.f90' in caplog.records[0].message diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index 931cce88b..690e88a04 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -13,6 +13,7 @@ """ from pathlib import Path +import sys import click from loki import ( @@ -110,6 +111,12 @@ def cli(debug): help="Recursively derive explicit shape dimension for argument arrays") @click.option('--eliminate-dead-code/--no-eliminate-dead-code', default=True, help='Perform dead code elimination, where unreachable branches are trimmed from the code.') +@click.option('--plan-file', type=click.Path(), default=None, + help='Process pipeline in planning mode and generate CMake "plan" file.') +@click.option('--callgraph', '-g', type=click.Path(), default=None, + help='Generate and display the subroutine callgraph.') +@click.option('--root', type=click.Path(), default=None, + help='Root path to which all paths are relative to.') @click.option('--log-level', '-l', default='info', envvar='LOKI_LOGGING', type=click.Choice(['debug', 'detail', 'perf', 'info', 'warning', 'error']), help='Log level to output during batch processing') @@ -118,7 +125,7 @@ def convert( data_offload, remove_openmp, assume_deviceptr, frontend, trim_vector_sections, global_var_offload, remove_derived_args, inline_members, inline_marked, resolve_sequence_association, resolve_sequence_association_inlined_calls, - derive_argument_array_shape, eliminate_dead_code, log_level + derive_argument_array_shape, eliminate_dead_code, plan_file, callgraph, root, log_level ): """ Batch-processing mode for Fortran-to-Fortran transformations that @@ -133,7 +140,12 @@ def convert( loki_config['log-level'] = log_level - info(f'[Loki] Batch-processing source files using config: {config} ') + plan = plan_file is not None + + if plan: + info(f'[Loki] Creating CMake plan file from config: {config}') + else: + info(f'[Loki] Batch-processing source files using config: {config} ') config = SchedulerConfig.from_file(config) @@ -179,7 +191,7 @@ def convert( info(f'[Loki-transform] Applying custom pipeline {mode} from config:') info(str(config.pipelines[mode])) - scheduler.process( config.pipelines[mode] ) + scheduler.process(config.pipelines[mode], plan=plan) mode = mode.replace('-', '_') # Sanitize mode string @@ -187,10 +199,21 @@ def convert( file_write_trafo = scheduler.config.transformations.get('FileWriteTransformation', None) if not file_write_trafo: file_write_trafo = FileWriteTransformation(cuf='cuf' in mode) - scheduler.process(transformation=file_write_trafo) + scheduler.process(transformation=file_write_trafo, plan=plan) + + if plan: + scheduler.write_cmake_plan(plan_file, rootpath=root) + + if callgraph: + scheduler.callgraph(callgraph) return + if plan: + msg = '[Loki] ERROR: Plan mode requires a pipeline definition in the config file.\n' + msg += '[Loki] Please provide a config file with configured transformation or pipelines instead.\n' + sys.exit(msg) + # If we do not use a custom pipeline, it should be one of the internally supported ones assert mode in [ 'idem', 'c', 'idem-stack', 'sca', 'claw', 'scc', 'scc-hoist', 'scc-stack', @@ -442,32 +465,16 @@ def convert( @click.option('--log-level', '-l', default='info', envvar='LOKI_LOGGING', type=click.Choice(['debug', 'detail', 'perf', 'info', 'warning', 'error']), help='Log level to output during batch processing') +@click.pass_context def plan( - mode, config, header, source, build, root, cpp, directive, + ctx, mode, config, header, source, build, root, cpp, directive, frontend, callgraph, plan_file, log_level ): """ Create a "plan", a schedule of files to inject and transform for a given configuration. """ - - loki_config['log-level'] = log_level - - info(f'[Loki] Creating CMake plan file from config: {config}') - config = SchedulerConfig.from_file(config) - - paths = [Path(s).resolve() for s in source] - paths += [Path(h).resolve().parent for h in header] - scheduler = Scheduler(paths=paths, config=config, frontend=frontend, full_parse=False, preprocess=cpp) - - mode = mode.replace('-', '_') # Sanitize mode string - - # Construct the transformation plan as a set of CMake lists of source files - scheduler.write_cmake_plan(filepath=plan_file, mode=mode, buildpath=build, rootpath=root) - - # Output the resulting callgraph - if callgraph: - scheduler.callgraph(callgraph) + return ctx.forward(convert) if __name__ == "__main__":