CMake integration of pipeline-based plan writer

loki/batch/scheduler.py

@@ -610,7 +610,7 @@ def callgraph(self, path, with_file_graph=False, with_legend=False):
                 warning(f'[Loki] Failed to render filegraph due to graphviz error:\n  {e}')
 
     @Timer(logger=perf, text='[Loki::Scheduler] Wrote CMake plan file in {:.2f}s')
-    def write_cmake_plan(self, filepath, mode, buildpath, rootpath):
+    def write_cmake_plan(self, filepath, rootpath=None):
         """
         Generate the "plan file" for CMake
 

loki/batch/tests/test_scheduler.py

@@ -74,7 +74,7 @@
     nodes as ir, FindNodes, FindInlineCalls, FindVariables
 )
 from loki.transformations import (
-    DependencyTransformation, ModuleWrapTransformation
+    DependencyTransformation, ModuleWrapTransformation, FileWriteTransformation
 )
 
 
@@ -1156,35 +1156,35 @@ def test_scheduler_cmake_planner(tmp_path, testdir, frontend):
     proj_b = sourcedir/'projB'
 
     config = SchedulerConfig.from_dict({
-        'default': {'role': 'kernel', 'expand': True, 'strict': True, 'ignore': ('header_mod',)},
+        'default': {
+            'role': 'kernel',
+            'expand': True,
+            'strict': True,
+            'ignore': ('header_mod',),
+            'mode': 'foobar'
+        },
         'routines': {
             'driverB': {'role': 'driver'},
             'kernelB': {'ignore': ['ext_driver']},
         }
     })
+    builddir = tmp_path/'scheduler_cmake_planner_dummy_dir'
+    builddir.mkdir(exist_ok=True)
 
     # Populate the scheduler
     # (this is the same as SchedulerA in test_scheduler_dependencies_ignore, so no need to
     # check scheduler set-up itself)
     scheduler = Scheduler(
         paths=[proj_a, proj_b], includes=proj_a/'include',
-        config=config, frontend=frontend, xmods=[tmp_path]
+        config=config, frontend=frontend, xmods=[tmp_path],
+        output_dir=builddir
     )
 
     # Apply the transformation
-    builddir = tmp_path/'scheduler_cmake_planner_dummy_dir'
-    builddir.mkdir(exist_ok=True)
     planfile = builddir/'loki_plan.cmake'
 
-    scheduler.write_cmake_plan(
-        filepath=planfile, mode='foobar', buildpath=builddir, rootpath=sourcedir
-    )
-
-    # Validate the generated lists
-    expected_files = {
-        proj_a/'module/driverB_mod.f90', proj_a/'module/kernelB_mod.F90',
-        proj_a/'module/compute_l1_mod.f90', proj_a/'module/compute_l2_mod.f90'
-    }
+    scheduler.process(FileWriteTransformation(), plan=True)
+    scheduler.write_cmake_plan(filepath=planfile, rootpath=sourcedir)
 
     # Validate the plan file content
     plan_pattern = re.compile(r'set\(\s*(\w+)\s*(.*?)\s*\)', re.DOTALL)

loki/tests/test_cmake.py

@@ -71,9 +71,32 @@ def fixture_config(tmp_dir):
     the file path
     """
     default_config = {
-        'default': {'role': 'kernel', 'expand': True, 'strict': True, 'enable_imports': True},
+        'default': {
+            'role': 'kernel',
+            'expand': True,
+            'strict': True,
+            'enable_imports': True
+        },
         'routines': {
             'driverB': {'role': 'driver'},
+        },
+        'transformations': {
+            'IdemTrafo': {
+                'classname': 'IdemTransformation',
+                'module': 'loki.transformations.idempotence',
+            },
+            'FileWriteTransformation': {
+                'classname': 'FileWriteTransformation',
+                'module': 'loki.transformations.build_system',
+                'options': {
+                    'include_module_var_imports': True
+                }
+            }
+        },
+        'pipelines': {
+            'idem': {
+                'transformations': ['IdemTrafo']
+            }
         }
     }
     filepath = tmp_dir/'test_cmake_loki.config'
@@ -101,30 +124,25 @@ def fixture_loki_install(here, tmp_dir, ecbuild, silent, request):
     Install Loki using CMake into an install directory
     """
     builddir = tmp_dir/'loki_bootstrap'
-    if builddir.exists():
-        shutil.rmtree(builddir)
-    builddir.mkdir()
-    cmd = ['cmake', '-DENABLE_CLAW=OFF', f'-DCMAKE_MODULE_PATH={ecbuild}/cmake', str(here.parent.parent)]
+    cmd = [
+        'cmake', f'-DCMAKE_MODULE_PATH={ecbuild}/cmake',
+        '-S', str(here.parent.parent),
+        '-B', str(builddir)
+    ]
     if request.param:
         cmd += ['-DENABLE_EDITABLE=ON']
     else:
         cmd += ['-DENABLE_EDITABLE=OFF']
 
-    execute(cmd, silent=silent, cwd=builddir)
+    execute(cmd, silent=silent, cwd=tmp_dir)
 
     lokidir = tmp_dir/'loki'
-    if lokidir.exists():
-        shutil.rmtree(lokidir)
     execute(
         ['cmake', '--install', '.', '--prefix', str(lokidir)],
         silent=True, cwd=builddir
     )
 
     yield builddir, lokidir
-    if builddir.exists():
-        shutil.rmtree(builddir)
-    if lokidir.exists():
-        shutil.rmtree(lokidir)
 
 
 @contextmanager
@@ -137,7 +155,6 @@ def clean_builddir(builddir):
         shutil.rmtree(builddir)
     builddir.mkdir()
     yield builddir
-    shutil.rmtree(builddir)
 
 
 @pytest.fixture(scope='module', name='cmake_project')
@@ -158,16 +175,13 @@ def fixture_cmake_project(here, config, srcdir):
 
 loki_transform_plan(
     MODE      idem
-    FRONTEND  fp
     CONFIG    {config}
     SOURCEDIR ${{CMAKE_CURRENT_SOURCE_DIR}}
     CALLGRAPH ${{CMAKE_CURRENT_BINARY_DIR}}/loki_callgraph
     PLAN      ${{CMAKE_CURRENT_BINARY_DIR}}/loki_plan.cmake
     SOURCES
         {proj_a}
         {proj_b}
-    HEADERS
-        {proj_a}/module/header_mod.f90
 )
     """
     filepath = srcdir/'CMakeLists.txt'

loki/transformations/build_system/tests/test_file_write.py

@@ -163,10 +163,7 @@ def test_file_write_module_imports(frontend, tmp_path, enable_imports, import_le
     plan_file = tmp_path/'plan.cmake'
     root_path = tmp_path if use_rootpath else None
     scheduler.process(transformation, plan=True)
-    scheduler.write_cmake_plan(
-        filepath=plan_file, mode=config.default['mode'], buildpath=out_path,
-        rootpath=root_path
-    )
+    scheduler.write_cmake_plan(filepath=plan_file, rootpath=root_path)
 
     # Validate the plan file content
     plan_pattern = re.compile(r'set\(\s*(\w+)\s*(.*?)\s*\)', re.DOTALL)
@@ -302,10 +299,7 @@ def test_file_write_replicate(tmp_path, caplog, frontend, have_non_replicate_con
 
     caplog.clear()
     with caplog.at_level(log_levels['WARNING']):
-        scheduler.write_cmake_plan(
-            filepath=plan_file, mode=config.default['mode'], buildpath=out_path,
-            rootpath=tmp_path
-        )
+        scheduler.write_cmake_plan(filepath=plan_file, rootpath=tmp_path)
         if have_non_replicate_conflict:
             assert len(caplog.records) == 1
             assert 'c.f90' in caplog.records[0].message

scripts/loki_transform.py

@@ -13,6 +13,7 @@
 """
 
 from pathlib import Path
+import sys
 import click
 
 from loki import (
@@ -110,6 +111,12 @@ def cli(debug):
               help="Recursively derive explicit shape dimension for argument arrays")
 @click.option('--eliminate-dead-code/--no-eliminate-dead-code', default=True,
               help='Perform dead code elimination, where unreachable branches are trimmed from the code.')
+@click.option('--plan-file', type=click.Path(), default=None,
+              help='Process pipeline in planning mode and generate CMake "plan" file.')
+@click.option('--callgraph', '-g', type=click.Path(), default=None,
+              help='Generate and display the subroutine callgraph.')
+@click.option('--root', type=click.Path(), default=None,
+              help='Root path to which all paths are relative to.')
 @click.option('--log-level', '-l', default='info', envvar='LOKI_LOGGING',
               type=click.Choice(['debug', 'detail', 'perf', 'info', 'warning', 'error']),
               help='Log level to output during batch processing')
@@ -118,7 +125,7 @@ def convert(
         data_offload, remove_openmp, assume_deviceptr, frontend, trim_vector_sections,
         global_var_offload, remove_derived_args, inline_members, inline_marked,
         resolve_sequence_association, resolve_sequence_association_inlined_calls,
-        derive_argument_array_shape, eliminate_dead_code, log_level
+        derive_argument_array_shape, eliminate_dead_code, plan_file, callgraph, root, log_level
 ):
     """
     Batch-processing mode for Fortran-to-Fortran transformations that
@@ -133,7 +140,12 @@ def convert(
 
     loki_config['log-level'] = log_level
 
-    info(f'[Loki] Batch-processing source files using config: {config} ')
+    plan = plan_file is not None
+
+    if plan:
+        info(f'[Loki] Creating CMake plan file from config: {config}')
+    else:
+        info(f'[Loki] Batch-processing source files using config: {config} ')
 
     config = SchedulerConfig.from_file(config)
 
@@ -179,18 +191,29 @@ def convert(
         info(f'[Loki-transform] Applying custom pipeline {mode} from config:')
         info(str(config.pipelines[mode]))
 
-        scheduler.process( config.pipelines[mode] )
+        scheduler.process(config.pipelines[mode], plan=plan)
 
         mode = mode.replace('-', '_')  # Sanitize mode string
 
         # Write out all modified source files into the build directory
         file_write_trafo = scheduler.config.transformations.get('FileWriteTransformation', None)
         if not file_write_trafo:
             file_write_trafo = FileWriteTransformation(cuf='cuf' in mode)
-        scheduler.process(transformation=file_write_trafo)
+        scheduler.process(transformation=file_write_trafo, plan=plan)
+
+        if plan:
+            scheduler.write_cmake_plan(plan_file, rootpath=root)
+
+        if callgraph:
+            scheduler.callgraph(callgraph)
 
         return
 
+    if plan:
+        msg = '[Loki] ERROR: Plan mode requires a pipeline definition in the config file.\n'
+        msg += '[Loki] Please provide a config file with configured transformation or pipelines instead.\n'
+        sys.exit(msg)
+
     # If we do not use a custom pipeline, it should be one of the internally supported ones
     assert mode in [
         'idem', 'c', 'idem-stack', 'sca', 'claw', 'scc', 'scc-hoist', 'scc-stack',
@@ -442,32 +465,16 @@ def convert(
 @click.option('--log-level', '-l', default='info', envvar='LOKI_LOGGING',
               type=click.Choice(['debug', 'detail', 'perf', 'info', 'warning', 'error']),
               help='Log level to output during batch processing')
+@click.pass_context
 def plan(
-         mode, config, header, source, build, root, cpp, directive,
+         ctx, mode, config, header, source, build, root, cpp, directive,
          frontend, callgraph, plan_file, log_level
 ):
     """
     Create a "plan", a schedule of files to inject and transform for a
     given configuration.
     """
-
-    loki_config['log-level'] = log_level
-
-    info(f'[Loki] Creating CMake plan file from config: {config}')
-    config = SchedulerConfig.from_file(config)
-
-    paths = [Path(s).resolve() for s in source]
-    paths += [Path(h).resolve().parent for h in header]
-    scheduler = Scheduler(paths=paths, config=config, frontend=frontend, full_parse=False, preprocess=cpp)
-
-    mode = mode.replace('-', '_')  # Sanitize mode string
-
-    # Construct the transformation plan as a set of CMake lists of source files
-    scheduler.write_cmake_plan(filepath=plan_file, mode=mode, buildpath=build, rootpath=root)
-
-    # Output the resulting callgraph
-    if callgraph:
-        scheduler.callgraph(callgraph)
+    return ctx.forward(convert)
 
 
 if __name__ == "__main__":