Merge branch 'devel' into 'master'

PIRA Release 0.6.0

Changelog:
- Update to use MetaCG v0.6.2 to address reported build issue
- Fixes out of memory error when building the docker container
- Improves the Python-version detection
- Allows to use clang-tidy up until clang-tidy-15

See merge request tuda-sc/projects/pira!98

.gitignore

@@ -159,3 +159,7 @@ extern/install/*
 extern/src/cxxopts
 extern/src/extrap
 extern/src/json
+extern/src/scorep-mod/
+extern/src/bear/
+extern/src/metacg/
+extern/src/mpiwrap/

.gitlab-ci.yml

@@ -28,6 +28,7 @@ stages:
 # GIT_STRATEGY: clone is used to always start with a clean checkout of the repository.
 # GIT_SUBMODULE_STRATEGY: none (default), i.e., git submodules need to be handled manually
 download:
+  resource_group: disable-concurrency
   stage: download-test
   tags:
     - pira-robo
@@ -42,6 +43,7 @@ download:
 # Also uses the GIT_STRATEGY fetch.
 # XXX should this be split into multiple, i.e., 1 job per unit test suite?
 pira:
+  resource_group: disable-concurrency
   stage: unit-test
   tags:
     - pira-robo
@@ -62,6 +64,7 @@ pira:
 
 # Run linting as part of unit-tests (early)
 python-lint:
+  resource_group: disable-concurrency
   stage: unit-test
   tags:
     - pira-robo
@@ -75,6 +78,7 @@ python-lint:
 
       # Run linting as part of unit-tests (early)
 shell-lint:
+  resource_group: disable-concurrency
   stage: unit-test
   tags:
     - pira-robo
@@ -88,6 +92,7 @@ shell-lint:
     - *lb-tear-down
 
 build-reduced-deps:
+  resource_group: disable-concurrency
   stage: dependence-build-test
   tags:
     - pira-robo
@@ -105,6 +110,7 @@ build-reduced-deps:
 
 # Runs the llvm-instrumentation unit tests
 llvm-instrumentation:
+  resource_group: disable-concurrency
   stage: deps-unit-test
   tags:
     - pira-robo
@@ -121,6 +127,7 @@ llvm-instrumentation:
 # XXX we might be able to re-use the still-available software builds from the build-dependencies stage?
 # This would *significantly* reduce the time this stage requires.
 run-prepare-integration:
+  resource_group: disable-concurrency
   stage: prepare-integration-test
   only:
     - merge_requests
@@ -139,6 +146,7 @@ run-prepare-integration:
 
 # Actually run the integration tests
 run-gameoflife-test:
+  resource_group: disable-concurrency
   stage: integration-test
   only:
     - merge_requests
@@ -157,6 +165,7 @@ run-gameoflife-test:
     - *lb-tear-down
 
 run-gameoflife-test-slurm:
+  resource_group: disable-concurrency
   stage: integration-test
   allow_failure: true
   only:
@@ -176,6 +185,7 @@ run-gameoflife-test-slurm:
     - *lb-tear-down
 
 run-amg2013-test:
+  resource_group: disable-concurrency
   stage: integration-test
   only:
     - merge_requests
@@ -194,6 +204,7 @@ run-amg2013-test:
     - *lb-tear-down
 
 run-amg2013-test-slurm:
+  resource_group: disable-concurrency
   stage: integration-test
   allow_failure: true
   only:
@@ -213,6 +224,7 @@ run-amg2013-test-slurm:
     - *lb-tear-down
 
 run-gameoflife-v1-test:
+  resource_group: disable-concurrency
   stage: integration-test
   only:
     - merge_requests
@@ -231,6 +243,7 @@ run-gameoflife-v1-test:
     - *lb-tear-down
 
 run-gameoflife-v1-test-slurm:
+  resource_group: disable-concurrency
   stage: integration-test
   allow_failure: true
   only:
@@ -250,6 +263,7 @@ run-gameoflife-v1-test-slurm:
     - *lb-tear-down
 
 run-gameoflife-hybrid-test:
+  resource_group: disable-concurrency
   stage: integration-test
   only:
     - merge_requests
@@ -268,6 +282,7 @@ run-gameoflife-hybrid-test:
     - *lb-tear-down
 
 run-gameoflife-hybrid-test-slurm:
+  resource_group: disable-concurrency
   stage: integration-test
   allow_failure: true
   only:
@@ -288,6 +303,7 @@ run-gameoflife-hybrid-test-slurm:
 
 # Disabled as it requires MetaCG Version 2
 run-gameoflife-heuristic-test:
+  resource_group: disable-concurrency
   stage: integration-test
   tags:
     - pira-robo
@@ -319,6 +335,7 @@ run-gameoflife-heuristic-test:
 #    - *lb-tear-down
 
 run-load-imbalance-detection-test:
+  resource_group: disable-concurrency
   stage: integration-test
   only:
     - merge_requests
@@ -337,6 +354,7 @@ run-load-imbalance-detection-test:
     - *lb-tear-down
 
 run-load-imbalance-detection-test-slurm:
+  resource_group: disable-concurrency
   stage: integration-test
   allow_failure: true
   only:
@@ -356,6 +374,7 @@ run-load-imbalance-detection-test-slurm:
     - *lb-tear-down
 
 run-lulesh-test:
+  resource_group: disable-concurrency
   stage: integration-test
   only:
     - merge_requests
@@ -372,6 +391,7 @@ run-lulesh-test:
     - *lb-tear-down
 
 run-lulesh-test-slurm:
+  resource_group: disable-concurrency
   stage: integration-test
   allow_failure: true
   only:
@@ -423,6 +443,7 @@ run-lulesh-test-slurm:
 
 # Do a full build of all dependencies via the convenience scripts for merge requests
 build-all-deps:
+  resource_group: disable-concurrency
   stage: full-dependence-test
   only:
     - merge_requests

README.md

@@ -152,6 +152,7 @@ In contrast, with runtime filtering, the compiler inserts instrumentation hooks
 PIRA uses source-code information for constructing an initial instrumentation and deciding which functions to add to an instrumentation during the iterative refinement.
 It provides a Clang-based call-graph tool that collects all required information and outputs the information in a `.json` file.
 You can find the `cgcollector` tool in the subdirectory `./extern/src/metacg/cgcollector`.
+PIRA requires the callgraph file to be in the MetaCG file format in version 2 (MetaCG v2).
 
 More information on the CGCollector and its components can be found in the [MetaCG](https://github.com/tudasc/MetaCG) documentation.
 
@@ -161,7 +162,7 @@ Applying the CGCollector usually happens in two steps.
 
     ~~~{.sh}
     for f in $(find ./src -type f \( -iname  "*.c" -o -iname "*.cpp" \) ); do
-        cgc $f
+        cgc --metacg-format-version=2 $f
     done
     ~~~
 2. The `.ipcg`-files created in step 1 are then merged to a general file using `cgmerge`.
@@ -174,11 +175,11 @@ Applying the CGCollector usually happens in two steps.
     ~~~
 
 The final graph needs to be placed into the directory of the callgraph-analyzer. Since **PGIS** is currently used for the CG analysis, the generated whole program file is copied into the PGIS directory.
-Currently, it is important that the file in the PGIS directory is named following the pattern `item_flavor.ipcg`. An item stands for a target application. More on the terms flavor and item in the next section.
+Currently, it is important that the file in the PGIS directory is named following the pattern `item_flavor.mcg`. An item stands for a target application. More on the terms flavor and item in the next section.
 
 ~~~{.sh}
 # Assuming $PIRA holds the top-level PIRA directory
-$> cp my-app.ipcg $PIRA/extern/install/pgis/bin/item_flavor.ipcg
+$> cp my-app.mcg $PIRA/extern/install/pgis/bin/item_flavor.mcg
 ~~~
 
 #### Configuration
@@ -317,7 +318,8 @@ Additional parameters are required for some analysis modes. Specifically, PIRA L
 ```
 
 ### Load imbalance detection (PIRA LIDe)
-For more details about the load imbalance detection feature, please refer to <a href="#ref-pira-2021">[PI21]</a>. Provide the PIRA invocation with a path to a configuration file using the `--load-imbalance-detection`-parameter. This JSON-file is required to have the following structure:
+For more details about the load imbalance detection feature, please refer to <a href="#ref-pira-2021">[PI21]</a>. 
+Provide the PIRA invocation with a path to a configuration file using the `--load-imbalance-detection`-parameter. This JSON-file is required to have the following structure:
 
 ```{.json}
 {

docker/Dockerfile

@@ -16,6 +16,8 @@ RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| apt-key add - && \
 RUN ln -s /usr/bin/clang-10 /usr/bin/clang && \
   ln -s /usr/bin/clang++-10 /usr/bin/clang++
 
+RUN python3 -m pip install -U pip
+
 RUN mkdir pira
 COPY resources pira/resources
 COPY extern pira/extern

extern/src/llvm-instrumentation/cmake/ToolchainOptions.cmake

@@ -1,4 +1,3 @@
-
 # LLVM related package
 find_package(LLVM REQUIRED CONFIG)
 message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
@@ -13,66 +12,60 @@ find_package(Clang REQUIRED CONFIG)
 function(default_compile_options target)
   cmake_parse_arguments(ARG "" "" "PRIVATE_FLAGS;PUBLIC_FLAGS" ${ARGN})
 
-  target_compile_options(${target} PRIVATE
-    -Wall -Wextra -pedantic -Wunreachable-code -Wwrite-strings
-    -Wpointer-arith -Wcast-align -Wcast-qual
-    -Werror
-  )
-
-if(ARG_PRIVATE_FLAGS)
-  target_compile_options(${target} PRIVATE
-    "${ARG_PRIVATE_FLAGS}"
-  )
-endif()
-
-if(ARG_PUBLIC_FLAGS)
-  target_compile_options(${target} PUBLIC
-    "${ARG_PUBLIC_FLAGS}"
-  )
-endif()
+  target_compile_options(
+    ${target}
+    PRIVATE -Wall
+            -Wextra
+            -pedantic
+            -Wunreachable-code
+            -Wwrite-strings
+            -Wpointer-arith
+            -Wcast-align
+            -Wcast-qual
+            -Werror)
+
+  if(ARG_PRIVATE_FLAGS)
+    target_compile_options(${target} PRIVATE "${ARG_PRIVATE_FLAGS}")
+  endif()
+
+  if(ARG_PUBLIC_FLAGS)
+    target_compile_options(${target} PUBLIC "${ARG_PUBLIC_FLAGS}")
+  endif()
 endfunction()
 
-
 # Clang tidy
-find_program(CLANG_TIDY
-  NAMES clang-tidy clang-tidy-10 clang-tidy-9 clang-tidy-8 clang-tidy-7 clang-tidy-6.0
-)
+find_program(
+  CLANG_TIDY
+  NAMES clang-tidy
+        clang-tidy-15
+        clang-tidy-14
+        clang-tidy-13
+        clang-tidy-12
+        clang-tidy-11
+        clang-tidy-10
+        clang-tidy-9
+        clang-tidy-8
+        clang-tidy-7
+        clang-tidy-6.0)
 
 function(register_to_clang_tidy target)
-  set_target_properties(${target}
-    PROPERTIES
-      CXX_CLANG_TIDY ${CLANG_TIDY}
-  )
+  set_target_properties(${target} PROPERTIES CXX_CLANG_TIDY ${CLANG_TIDY})
 endfunction()
 
 function(make_llvm_module name sources)
-cmake_parse_arguments(ARG "" "" "INCLUDE_DIRS;DEPENDS;LINK_LIBS" ${ARGN})
+  cmake_parse_arguments(ARG "" "" "INCLUDE_DIRS;DEPENDS;LINK_LIBS" ${ARGN})
 
-add_llvm_library(${name}
-  MODULE  ${LIB_SOURCES}
-)
+  add_llvm_library(${name} MODULE ${LIB_SOURCES})
 
-# Maybe this needs to be "LLVM_MAIN_INCLUDE_DIRS"
-target_include_directories(${name}
-  SYSTEM
-  PUBLIC
-  ${LLVM_INCLUDE_DIRS}
-)
+  # Maybe this needs to be "LLVM_MAIN_INCLUDE_DIRS"
+  target_include_directories(${name} SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS})
 
-if(ARG_INCLUDE_DIRS)
-  target_include_directories(${name}
-    PRIVATE
-    ${ARG_INCLUDE_DIRS}
-  )
-endif()
+  if(ARG_INCLUDE_DIRS)
+    target_include_directories(${name} PRIVATE ${ARG_INCLUDE_DIRS})
+  endif()
 
-target_compile_definitions(${name}
-  PRIVATE
-  ${LLVM_DEFINITIONS}
-)
+  target_compile_definitions(${name} PRIVATE ${LLVM_DEFINITIONS})
 
-register_to_clang_tidy(${name}
-  ${INSTRUMENTATIONLIB_SOURCES}
-)
+  register_to_clang_tidy(${name} ${INSTRUMENTATIONLIB_SOURCES})
 
 endfunction()

lib/Analyzer.py

@@ -139,7 +139,7 @@ def run_analyzer_command(command: str,
     analysis_parameters_path = InvocCfg.get_instance().get_analysis_parameters_path()
     export_str = ' '
     if export_performance_models:
-      export_str += ' --export'
+      export_str += ' --export true'
       if export_runtime_only:
         export_str += ' --runtime-only'
 
@@ -158,12 +158,12 @@ def run_analyzer_command(command: str,
               'Utility::run_analyzer_command: An analysis parameters file is required for PIRA LIDe!',
               level='error')
 
-        sh_cmd = command + export_str + ' --scorep-out -c ' + cubex_file + ' --lide 1 --parameter-file ' + analysis_parameters_path + ' --debug 1 --export ' + ipcg_file
+        sh_cmd = command + export_str + ' --scorep-out --cube ' + cubex_file + ' --lide true --parameter-file ' + analysis_parameters_path + ' --debug 1 --metacg-format 2 --export true ' + ipcg_file
 
       else:
         # vanilla PIRA version 1 runner
         L.get_logger().log('Utility::run_analyzer_command: using PIRA 1 Analyzer', level='info')
-        sh_cmd = command + export_str + ' --scorep-out ' + ipcg_file + ' -c ' + cubex_file
+        sh_cmd = command + export_str + '--metacg-format 2 --scorep-out ' + ipcg_file + ' --cube ' + cubex_file
 
       L.get_logger().log('Utility::run_analyzer_command: INSTR: Run cmd: ' + sh_cmd)
       out, _ = U.shell(sh_cmd)
@@ -181,7 +181,7 @@ def run_analyzer_command(command: str,
       L.get_logger().log(
           'Utility::run_analyzer_command: An analysis parameters file is required for Extra-P mode!',
           level='error')
-    sh_cmd = command + export_str + ' --scorep-out --debug 1 --parameter-file ' + analysis_parameters_path + ' --extrap ' + extrap_config_file + ' ' + ipcg_file
+    sh_cmd = command + export_str + ' --scorep-out --metacg-format 2 --debug 1 --parameter-file ' + analysis_parameters_path + ' --extrap ' + extrap_config_file + ' ' + ipcg_file
     L.get_logger().log('Utility::run_analyzer_command: INSTR: Run cmd: ' + sh_cmd)
     out, _ = U.shell(sh_cmd)
     L.get_logger().log('Utility::run_analyzer_command: Output of analyzer:\n' + out, level='debug')
@@ -190,11 +190,11 @@ def run_analyzer_command(command: str,
   def run_analyzer_command_no_instr(command: str, analyzer_dir: str, flavor: str,
                                     benchmark_name: str) -> None:
     ipcg_file = U.get_ipcg_file_name(analyzer_dir, benchmark_name, flavor)
-    sh_cmd = command + ' --scorep-out --static '
+    sh_cmd = command + ' --metacg-format 2 --scorep-out --static '
 
     # load imbalancee detection mode
     if InvocCfg.get_instance().is_lide_enabled():
-      sh_cmd = sh_cmd + ' --debug 1 --lide 1 ' + InvocCfg.get_instance(
+      sh_cmd = sh_cmd + ' --debug 1 --metacg-format 2 --lide true ' + InvocCfg.get_instance(
       ).get_analysis_parameters_path()
 
     sh_cmd = sh_cmd + ' ' + ipcg_file

pira.py

@@ -5,7 +5,7 @@
 Description: This is PIRA.
 """
 
-__version__ = '0.5.0'
+__version__ = '0.6.0'
 
 import argparse
 import lib.Logging as log