diff --git a/.gitignore b/.gitignore index 48f4524..881d8b4 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,7 @@ extern/install/* extern/src/cxxopts extern/src/extrap extern/src/json +extern/src/scorep-mod/ +extern/src/bear/ +extern/src/metacg/ +extern/src/mpiwrap/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e612652..ee08414 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,7 @@ stages: # GIT_STRATEGY: clone is used to always start with a clean checkout of the repository. # GIT_SUBMODULE_STRATEGY: none (default), i.e., git submodules need to be handled manually download: + resource_group: disable-concurrency stage: download-test tags: - pira-robo @@ -42,6 +43,7 @@ download: # Also uses the GIT_STRATEGY fetch. # XXX should this be split into multiple, i.e., 1 job per unit test suite? pira: + resource_group: disable-concurrency stage: unit-test tags: - pira-robo @@ -62,6 +64,7 @@ pira: # Run linting as part of unit-tests (early) python-lint: + resource_group: disable-concurrency stage: unit-test tags: - pira-robo @@ -75,6 +78,7 @@ python-lint: # Run linting as part of unit-tests (early) shell-lint: + resource_group: disable-concurrency stage: unit-test tags: - pira-robo @@ -88,6 +92,7 @@ shell-lint: - *lb-tear-down build-reduced-deps: + resource_group: disable-concurrency stage: dependence-build-test tags: - pira-robo @@ -105,6 +110,7 @@ build-reduced-deps: # Runs the llvm-instrumentation unit tests llvm-instrumentation: + resource_group: disable-concurrency stage: deps-unit-test tags: - pira-robo @@ -121,6 +127,7 @@ llvm-instrumentation: # XXX we might be able to re-use the still-available software builds from the build-dependencies stage? # This would *significantly* reduce the time this stage requires. run-prepare-integration: + resource_group: disable-concurrency stage: prepare-integration-test only: - merge_requests @@ -139,6 +146,7 @@ run-prepare-integration: # Actually run the integration tests run-gameoflife-test: + resource_group: disable-concurrency stage: integration-test only: - merge_requests @@ -157,6 +165,7 @@ run-gameoflife-test: - *lb-tear-down run-gameoflife-test-slurm: + resource_group: disable-concurrency stage: integration-test allow_failure: true only: @@ -176,6 +185,7 @@ run-gameoflife-test-slurm: - *lb-tear-down run-amg2013-test: + resource_group: disable-concurrency stage: integration-test only: - merge_requests @@ -194,6 +204,7 @@ run-amg2013-test: - *lb-tear-down run-amg2013-test-slurm: + resource_group: disable-concurrency stage: integration-test allow_failure: true only: @@ -213,6 +224,7 @@ run-amg2013-test-slurm: - *lb-tear-down run-gameoflife-v1-test: + resource_group: disable-concurrency stage: integration-test only: - merge_requests @@ -231,6 +243,7 @@ run-gameoflife-v1-test: - *lb-tear-down run-gameoflife-v1-test-slurm: + resource_group: disable-concurrency stage: integration-test allow_failure: true only: @@ -250,6 +263,7 @@ run-gameoflife-v1-test-slurm: - *lb-tear-down run-gameoflife-hybrid-test: + resource_group: disable-concurrency stage: integration-test only: - merge_requests @@ -268,6 +282,7 @@ run-gameoflife-hybrid-test: - *lb-tear-down run-gameoflife-hybrid-test-slurm: + resource_group: disable-concurrency stage: integration-test allow_failure: true only: @@ -288,6 +303,7 @@ run-gameoflife-hybrid-test-slurm: # Disabled as it requires MetaCG Version 2 run-gameoflife-heuristic-test: + resource_group: disable-concurrency stage: integration-test tags: - pira-robo @@ -319,6 +335,7 @@ run-gameoflife-heuristic-test: # - *lb-tear-down run-load-imbalance-detection-test: + resource_group: disable-concurrency stage: integration-test only: - merge_requests @@ -337,6 +354,7 @@ run-load-imbalance-detection-test: - *lb-tear-down run-load-imbalance-detection-test-slurm: + resource_group: disable-concurrency stage: integration-test allow_failure: true only: @@ -356,6 +374,7 @@ run-load-imbalance-detection-test-slurm: - *lb-tear-down run-lulesh-test: + resource_group: disable-concurrency stage: integration-test only: - merge_requests @@ -372,6 +391,7 @@ run-lulesh-test: - *lb-tear-down run-lulesh-test-slurm: + resource_group: disable-concurrency stage: integration-test allow_failure: true only: @@ -423,6 +443,7 @@ run-lulesh-test-slurm: # Do a full build of all dependencies via the convenience scripts for merge requests build-all-deps: + resource_group: disable-concurrency stage: full-dependence-test only: - merge_requests diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29..0000000 diff --git a/README.md b/README.md index 840c66e..7984fb3 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ In contrast, with runtime filtering, the compiler inserts instrumentation hooks PIRA uses source-code information for constructing an initial instrumentation and deciding which functions to add to an instrumentation during the iterative refinement. It provides a Clang-based call-graph tool that collects all required information and outputs the information in a `.json` file. You can find the `cgcollector` tool in the subdirectory `./extern/src/metacg/cgcollector`. +PIRA requires the callgraph file to be in the MetaCG file format in version 2 (MetaCG v2). More information on the CGCollector and its components can be found in the [MetaCG](https://github.com/tudasc/MetaCG) documentation. @@ -161,7 +162,7 @@ Applying the CGCollector usually happens in two steps. ~~~{.sh} for f in $(find ./src -type f \( -iname "*.c" -o -iname "*.cpp" \) ); do - cgc $f + cgc --metacg-format-version=2 $f done ~~~ 2. The `.ipcg`-files created in step 1 are then merged to a general file using `cgmerge`. @@ -174,11 +175,11 @@ Applying the CGCollector usually happens in two steps. ~~~ The final graph needs to be placed into the directory of the callgraph-analyzer. Since **PGIS** is currently used for the CG analysis, the generated whole program file is copied into the PGIS directory. -Currently, it is important that the file in the PGIS directory is named following the pattern `item_flavor.ipcg`. An item stands for a target application. More on the terms flavor and item in the next section. +Currently, it is important that the file in the PGIS directory is named following the pattern `item_flavor.mcg`. An item stands for a target application. More on the terms flavor and item in the next section. ~~~{.sh} # Assuming $PIRA holds the top-level PIRA directory -$> cp my-app.ipcg $PIRA/extern/install/pgis/bin/item_flavor.ipcg +$> cp my-app.mcg $PIRA/extern/install/pgis/bin/item_flavor.mcg ~~~ #### Configuration @@ -317,7 +318,8 @@ Additional parameters are required for some analysis modes. Specifically, PIRA L ``` ### Load imbalance detection (PIRA LIDe) -For more details about the load imbalance detection feature, please refer to [PI21]. Provide the PIRA invocation with a path to a configuration file using the `--load-imbalance-detection`-parameter. This JSON-file is required to have the following structure: +For more details about the load imbalance detection feature, please refer to [PI21]. +Provide the PIRA invocation with a path to a configuration file using the `--load-imbalance-detection`-parameter. This JSON-file is required to have the following structure: ```{.json} { diff --git a/docker/Dockerfile b/docker/Dockerfile index dd3548e..26ac4ea 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,6 +16,8 @@ RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| apt-key add - && \ RUN ln -s /usr/bin/clang-10 /usr/bin/clang && \ ln -s /usr/bin/clang++-10 /usr/bin/clang++ +RUN python3 -m pip install -U pip + RUN mkdir pira COPY resources pira/resources COPY extern pira/extern diff --git a/extern/src/llvm-instrumentation/cmake/ToolchainOptions.cmake b/extern/src/llvm-instrumentation/cmake/ToolchainOptions.cmake index 91e8fff..a58157f 100644 --- a/extern/src/llvm-instrumentation/cmake/ToolchainOptions.cmake +++ b/extern/src/llvm-instrumentation/cmake/ToolchainOptions.cmake @@ -1,4 +1,3 @@ - # LLVM related package find_package(LLVM REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") @@ -13,66 +12,60 @@ find_package(Clang REQUIRED CONFIG) function(default_compile_options target) cmake_parse_arguments(ARG "" "" "PRIVATE_FLAGS;PUBLIC_FLAGS" ${ARGN}) - target_compile_options(${target} PRIVATE - -Wall -Wextra -pedantic -Wunreachable-code -Wwrite-strings - -Wpointer-arith -Wcast-align -Wcast-qual - -Werror - ) - -if(ARG_PRIVATE_FLAGS) - target_compile_options(${target} PRIVATE - "${ARG_PRIVATE_FLAGS}" - ) -endif() - -if(ARG_PUBLIC_FLAGS) - target_compile_options(${target} PUBLIC - "${ARG_PUBLIC_FLAGS}" - ) -endif() + target_compile_options( + ${target} + PRIVATE -Wall + -Wextra + -pedantic + -Wunreachable-code + -Wwrite-strings + -Wpointer-arith + -Wcast-align + -Wcast-qual + -Werror) + + if(ARG_PRIVATE_FLAGS) + target_compile_options(${target} PRIVATE "${ARG_PRIVATE_FLAGS}") + endif() + + if(ARG_PUBLIC_FLAGS) + target_compile_options(${target} PUBLIC "${ARG_PUBLIC_FLAGS}") + endif() endfunction() - # Clang tidy -find_program(CLANG_TIDY - NAMES clang-tidy clang-tidy-10 clang-tidy-9 clang-tidy-8 clang-tidy-7 clang-tidy-6.0 -) +find_program( + CLANG_TIDY + NAMES clang-tidy + clang-tidy-15 + clang-tidy-14 + clang-tidy-13 + clang-tidy-12 + clang-tidy-11 + clang-tidy-10 + clang-tidy-9 + clang-tidy-8 + clang-tidy-7 + clang-tidy-6.0) function(register_to_clang_tidy target) - set_target_properties(${target} - PROPERTIES - CXX_CLANG_TIDY ${CLANG_TIDY} - ) + set_target_properties(${target} PROPERTIES CXX_CLANG_TIDY ${CLANG_TIDY}) endfunction() function(make_llvm_module name sources) -cmake_parse_arguments(ARG "" "" "INCLUDE_DIRS;DEPENDS;LINK_LIBS" ${ARGN}) + cmake_parse_arguments(ARG "" "" "INCLUDE_DIRS;DEPENDS;LINK_LIBS" ${ARGN}) -add_llvm_library(${name} - MODULE ${LIB_SOURCES} -) + add_llvm_library(${name} MODULE ${LIB_SOURCES}) -# Maybe this needs to be "LLVM_MAIN_INCLUDE_DIRS" -target_include_directories(${name} - SYSTEM - PUBLIC - ${LLVM_INCLUDE_DIRS} -) + # Maybe this needs to be "LLVM_MAIN_INCLUDE_DIRS" + target_include_directories(${name} SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS}) -if(ARG_INCLUDE_DIRS) - target_include_directories(${name} - PRIVATE - ${ARG_INCLUDE_DIRS} - ) -endif() + if(ARG_INCLUDE_DIRS) + target_include_directories(${name} PRIVATE ${ARG_INCLUDE_DIRS}) + endif() -target_compile_definitions(${name} - PRIVATE - ${LLVM_DEFINITIONS} -) + target_compile_definitions(${name} PRIVATE ${LLVM_DEFINITIONS}) -register_to_clang_tidy(${name} - ${INSTRUMENTATIONLIB_SOURCES} -) + register_to_clang_tidy(${name} ${INSTRUMENTATIONLIB_SOURCES}) endfunction() diff --git a/lib/Analyzer.py b/lib/Analyzer.py index e71b6a2..64aa09a 100644 --- a/lib/Analyzer.py +++ b/lib/Analyzer.py @@ -139,7 +139,7 @@ def run_analyzer_command(command: str, analysis_parameters_path = InvocCfg.get_instance().get_analysis_parameters_path() export_str = ' ' if export_performance_models: - export_str += ' --export' + export_str += ' --export true' if export_runtime_only: export_str += ' --runtime-only' @@ -158,12 +158,12 @@ def run_analyzer_command(command: str, 'Utility::run_analyzer_command: An analysis parameters file is required for PIRA LIDe!', level='error') - sh_cmd = command + export_str + ' --scorep-out -c ' + cubex_file + ' --lide 1 --parameter-file ' + analysis_parameters_path + ' --debug 1 --export ' + ipcg_file + sh_cmd = command + export_str + ' --scorep-out --cube ' + cubex_file + ' --lide true --parameter-file ' + analysis_parameters_path + ' --debug 1 --metacg-format 2 --export true ' + ipcg_file else: # vanilla PIRA version 1 runner L.get_logger().log('Utility::run_analyzer_command: using PIRA 1 Analyzer', level='info') - sh_cmd = command + export_str + ' --scorep-out ' + ipcg_file + ' -c ' + cubex_file + sh_cmd = command + export_str + '--metacg-format 2 --scorep-out ' + ipcg_file + ' --cube ' + cubex_file L.get_logger().log('Utility::run_analyzer_command: INSTR: Run cmd: ' + sh_cmd) out, _ = U.shell(sh_cmd) @@ -181,7 +181,7 @@ def run_analyzer_command(command: str, L.get_logger().log( 'Utility::run_analyzer_command: An analysis parameters file is required for Extra-P mode!', level='error') - sh_cmd = command + export_str + ' --scorep-out --debug 1 --parameter-file ' + analysis_parameters_path + ' --extrap ' + extrap_config_file + ' ' + ipcg_file + sh_cmd = command + export_str + ' --scorep-out --metacg-format 2 --debug 1 --parameter-file ' + analysis_parameters_path + ' --extrap ' + extrap_config_file + ' ' + ipcg_file L.get_logger().log('Utility::run_analyzer_command: INSTR: Run cmd: ' + sh_cmd) out, _ = U.shell(sh_cmd) L.get_logger().log('Utility::run_analyzer_command: Output of analyzer:\n' + out, level='debug') @@ -190,11 +190,11 @@ def run_analyzer_command(command: str, def run_analyzer_command_no_instr(command: str, analyzer_dir: str, flavor: str, benchmark_name: str) -> None: ipcg_file = U.get_ipcg_file_name(analyzer_dir, benchmark_name, flavor) - sh_cmd = command + ' --scorep-out --static ' + sh_cmd = command + ' --metacg-format 2 --scorep-out --static ' # load imbalancee detection mode if InvocCfg.get_instance().is_lide_enabled(): - sh_cmd = sh_cmd + ' --debug 1 --lide 1 ' + InvocCfg.get_instance( + sh_cmd = sh_cmd + ' --debug 1 --metacg-format 2 --lide true ' + InvocCfg.get_instance( ).get_analysis_parameters_path() sh_cmd = sh_cmd + ' ' + ipcg_file diff --git a/pira.py b/pira.py index d093301..3e99f2d 100644 --- a/pira.py +++ b/pira.py @@ -5,7 +5,7 @@ Description: This is PIRA. """ -__version__ = '0.5.0' +__version__ = '0.6.0' import argparse import lib.Logging as log diff --git a/resources/build_submodules.sh b/resources/build_submodules.sh index 5238c4a..cc40a10 100755 --- a/resources/build_submodules.sh +++ b/resources/build_submodules.sh @@ -6,7 +6,10 @@ # Description: Script to build / install components required by PIRA #""" -scriptdir="$( cd "$(dirname "$0")" ; pwd -P )" +scriptdir="$( + cd "$(dirname "$0")" + pwd -P +)" extsourcedir=$scriptdir/../extern/src extinstalldir=$scriptdir/../extern/install @@ -16,11 +19,12 @@ export CXX=g++ allOutputTo=$PWD/buildoutput.log - # TODO Make this actually working better! # Allow configure options (here for Score-P, bc I want to build it w/o MPI) #add_flags="$2" add_flags="" +# Flags to pass to cmake when building metacg and llvm-instrumentation +extra_cmake_flags="" # Which options do we have to provide? # - Path to cube library @@ -32,6 +36,8 @@ function print_usage { echo -e " -e\tPath to Extra-P installation" echo -e " -c\tPath to Cube installation" echo -e " -p\tParallel compile jobs" + echo -e " -o\tAdditional options to build scorep" + echo -e " -f\tAdditional options to pass to cmake" } function echo_configuring { @@ -64,61 +70,64 @@ function check_directory_or_file_exists { } # Get command line parameters -while getopts ":e:c:p:o:h" opt; do +while getopts ":e:c:p:o:f:h" opt; do case $opt in - e) - if [ ! -d $OPTARG ]; then - echo "Extra-P installation directory not valid" - exit 1 - fi - extrap_install_dir=$OPTARG - echo "Setting Extra-P installation to: $extrap_install_dir" >&2 - ;; - c) - if [ ! -d $OPTARG ]; then - echo "Cube installation directory not valid" - exit 1 - fi - cube_install_dir=$OPTARG - echo "Setting Cube installation to: $cube_install_dir" >&2 - ;; - p) - parallel_jobs=$OPTARG - echo "Setting parallel builds to: $parallel_jobs" >&2 - ;; - o) - add_flags="$OPTARG" - echo "Passing additional flags " ${additional_flags} - ;; - h) - print_usage - exit 0 - ;; - \?) - echo "Invalid option -$OPTARG" >&2 - print_usage + e) + if [ ! -d $OPTARG ]; then + echo "Extra-P installation directory not valid" exit 1 - ;; - :) - echo "Option -$OPTARG requires an argument" >&2 - print_usage + fi + extrap_install_dir=$OPTARG + echo "Setting Extra-P installation to: $extrap_install_dir" >&2 + ;; + c) + if [ ! -d $OPTARG ]; then + echo "Cube installation directory not valid" exit 1 - ;; + fi + cube_install_dir=$OPTARG + echo "Setting Cube installation to: $cube_install_dir" >&2 + ;; + p) + parallel_jobs=$OPTARG + echo "Setting parallel builds to: $parallel_jobs" >&2 + ;; + o) + add_flags="$OPTARG" + echo "Passing additional flags: ${add_flags}" >&2 + ;; + f) + extra_cmake_flags="$OPTARG" + echo "Passing additional cmake flags: ${extra_cmake_flags}" >&2 + ;; + h) + print_usage + exit 0 + ;; + \?) + echo "Invalid option -$OPTARG" >&2 + print_usage + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument" >&2 + print_usage + exit 1 + ;; esac done - # Some sanity check of things that may go wrong. # This most likely just a list of things that I get wrong all the time command -v cubelib-config >${allOutputTo} 2>&1 -if [ $? -eq 0 ] && [ -z $cube_install_dir ] ; then +if [ $? -eq 0 ] && [ -z $cube_install_dir ]; then echo "[PIRA]: No cubelib path set (-c) but cubelib-config available in PATH. Please set -c option or strip from PATH." echo "" exit 1 fi command -v extrap >${allOutputTo} 2>&1 -if [ $? -eq 0 ] && [ -z $extrap_install_dir ] ; then +if [ $? -eq 0 ] && [ -z $extrap_install_dir ]; then echo "[PIRA]: No Extra-P path set (-e) but extrap available in PATH. Please set -e option or strip from PATH." echo "" exit 1 @@ -137,7 +146,6 @@ if [ -z "$clangversion" ]; then exit 1 fi - ### ==================== ## Actual work starts ### ===================== @@ -154,7 +162,7 @@ if [ $? -ne 0 ]; then rm -rf build mkdir build && cd build || exit 1 - cmake .. >${allOutputTo} 2>&1 + cmake $extra_cmake_flags .. >${allOutputTo} 2>&1 if [ $? -ne 0 ]; then echo "[PIRA] Configuring LLVM-Instrumenter failed." exit 1 @@ -191,7 +199,7 @@ if [ $? -ne 0 ]; then bash set_instrumenter_directory.sh $extsourcedir/llvm-instrumentation/build/lib >${allOutputTo} 2>&1 # TODO We should check whether a cube / scorep install was found and if so, bail out, to know exactly which scorep/cube we get. - aclocalversion=$( aclocal --version | grep 1.16 ) >${allOutputTo} 2>&1 + aclocalversion=$(aclocal --version | grep 1.16) >${allOutputTo} 2>&1 if [ -z "$aclocalversion" ]; then echo "[PIRA][Score-P] aclocal available in wrong version. Guessing to autoreconf." autoreconf -ivf >${allOutputTo} 2>&1 @@ -203,7 +211,7 @@ if [ $? -ne 0 ]; then autoreconf -ivf >${allOutputTo} 2>&1 cd ./build-backend || exit 1 >${allOutputTo} 2>&1 autoreconf -ivf >${allOutputTo} 2>&1 - cd ../build-frontend || exit 1 >${allOutputTo} 2>&1 + cd ../build-frontend || exit 1 >${allOutputTo} 2>&1 autoreconf -ivf >${allOutputTo} 2>&1 cd $extsourcedir/scorep-mod || exit 1 fi @@ -213,7 +221,7 @@ if [ $? -ne 0 ]; then mkdir scorep-build && cd scorep-build if [ -z $cube_install_dir ]; then #../configure --prefix=$extinstalldir/scorep --enable-shared --disable-static --disable-gcc-plugin --without-shmem "$add_flags" - ../configure --prefix=$extinstalldir/scorep --enable-shared --disable-static --disable-gcc-plugin --without-shmem "$add_flags" >${allOutputTo} 2>&1 + ../configure --prefix=$extinstalldir/scorep --enable-shared --disable-static --disable-gcc-plugin --without-shmem "$add_flags" >${allOutputTo} 2>&1 cube_install_dir=$extinstalldir/scorep else #../configure --prefix=$extinstalldir/scorep --with-cubelib=$cube_install_dir --enable-shared --disable-static --disable-gcc-plugin --without-shmem "$add_flags" @@ -259,7 +267,7 @@ if [ -z $extrap_install_dir ]; then echo "[PIRA] Installting Extra-P dependency PyQt5 failed." exit 1 fi - + python3 -m pip install --user matplotlib >${allOutputTo} 2>&1 if [ $? -ne 0 ]; then echo "[PIRA] Installting Extra-P dependency matplotlib failed." @@ -281,61 +289,56 @@ if [ -z $extrap_install_dir ]; then mkdir build && cd build || exit 1 # On my Ubuntu machine, the locate command is available, on the CentOS machine it wasn't # TODO This should be done just a little less fragile - command -v locate "/Python.h" >${allOutputTo} 2>&1 - if [ $? -eq 1 ]; then - pythonheader=$(dirname $(which python3))/../include/python3.8 + pythonheader=$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))") + if [ $? -eq 0 ]; then + echo "[PIRA] Found Python.h at " $pythonheader else - pythonlocation=$(locate "/Python.h" | grep "python3.") - if [ -z $pythonlocation ]; then - pythonheader=$(dirname $(which python))/../include/python3.8 - else - pythonheader=$(dirname $pythonlocation) - fi + echo "[PIRA] Getting python path failed." + exit 1 fi - echo "[PIRA] Found Python.h at " $pythonheader - + ../configure --prefix=$extinstalldir/extrap --with-cube=$cube_install_dir CPPFLAGS=-I$pythonheader >${allOutputTo} 2>&1 #../configure --prefix=$extinstalldir/extrap --with-cube=$cube_install_dir CPPFLAGS=-I$pythonheader - + if [ $? -ne 0 ]; then echo "[PIRA] Configuring Extra-P failed." exit 1 fi - echo_building "$extrap_component_name" - make -j $parallel_jobs >${allOutputTo} 2>&1 + echo_building "$extrap_component_name" + make -j $parallel_jobs >${allOutputTo} 2>&1 if [ $? -ne 0 ]; then echo "[PIRA] Building Extra-P failed." exit 1 fi - - make install >${allOutputTo} 2>&1 + + make install >${allOutputTo} 2>&1 # Manually add the required headers for Extra-P cd .. || exit 1 mkdir -p $extinstalldir/extrap/include cp ./include/*.hpp $extinstalldir/extrap/include - - extrap_install_dir=$extinstalldir/extrap + + extrap_install_dir=$extinstalldir/extrap else + extrap_install_dir=$extinstalldir/extrap echo_already_built "$extrap_component_name" fi else echo "[PIRA] Using $extrap_install_dir" fi - # CXX Opts echo "[PIRA] Getting cxxopts library" cd $extsourcedir || exit 1 if [ ! -d "$extsourcedir/cxxopts" ]; then - git clone -b 2_1 --single-branch https://github.com/jarro2783/cxxopts cxxopts >${allOutputTo} 2>&1 + git clone -b 2_1 --single-branch https://github.com/jarro2783/cxxopts cxxopts >${allOutputTo} 2>&1 fi # JSON library echo "[PIRA] Getting json library" cd $extsourcedir || exit 1 if [ ! -d "$extsourcedir/json" ]; then - git clone -b v3.9.1 --depth 1 --single-branch https://github.com/nlohmann/json json >${allOutputTo} 2>&1 + git clone -b v3.9.1 --depth 1 --single-branch https://github.com/nlohmann/json json >${allOutputTo} 2>&1 fi metacg_component_name="MetaCG" @@ -343,7 +346,7 @@ echo_processing "$metacg_component_name" cd $extsourcedir || exit 1 check_directory_or_file_exists $extsourcedir/metacg if [ $? -ne 0 ]; then - git clone -b v0.4.0 --single-branch https://github.com/tudasc/MetaCG.git metacg + git clone -b v0.6.2 --single-branch https://github.com/tudasc/MetaCG.git metacg if [ $? -ne 0 ]; then echo "[PIRA] Error in downloading MetaCG" exit 1 @@ -355,13 +358,15 @@ cd $extsourcedir/metacg || exit 1 check_directory_or_file_exists $extsourcedir/metacg/build if [ $? -ne 0 ]; then echo_configuring "$metacg_component_name" - cmake -B build -S . -DCMAKE_BUILD_TYPE=Debug \ + cmake -B build -S . $extra_cmake_flags -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_INSTALL_PREFIX=$extinstalldir/metacg \ -DCUBE_LIB=$cube_install_dir/lib \ -DCUBE_INCLUDE=$cube_install_dir/include/cubelib \ -DEXTRAP_INCLUDE=$extrap_install_dir/include \ -DEXTRAP_LIB=$extrap_install_dir/lib \ -DSPDLOG_BUILD_SHARED=ON \ + -DMETACG_BUILD_PGIS=ON \ + -DMETACG_BUILD_CGCOLLECTOR=ON \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON >${allOutputTo} 2>&1 if [ $? -ne 0 ]; then echo "[PIRA] Configuring MetaCG failed." @@ -380,7 +385,7 @@ if [ $? -ne 0 ]; then mkdir -p ${extinstalldir}/metacg/bin/out >${allOutputTo} 2>&1 fi metacg_install_dir=$extinstalldir/metacg -else +else echo_already_built "$metacg_component_name" metacg_install_dir=$extinstalldir/metacg fi @@ -412,7 +417,6 @@ else mpiwrap_install_dir=$extinstalldir/mpiwrap fi - cd $extsourcedir || exit 1 check_directory_or_file_exists $extsourcedir/bear if [ $? -ne 0 ]; then @@ -441,7 +445,6 @@ fi cd $scriptdir || exit 1 - echo -e "\n=== PIRA Installation Summary ===" echo -e "Cube installation dir:\t\t$cube_install_dir" echo -e "Score-P installation dir:\t$scorep_install_dir" diff --git a/resources/setup_paths.sh b/resources/setup_paths.sh index f790cf9..4b8a12e 100755 --- a/resources/setup_paths.sh +++ b/resources/setup_paths.sh @@ -37,6 +37,7 @@ export PIRA_DIR="$scriptdir/.." export PATH=$cube_install_dir/bin:$scorep_install_dir/bin:$extrap_install_dir/bin:$metacg_install_dir/bin:$mpiwrap_install_dir:$bear_install_dir/bin:$PATH export LD_LIBRARY_PATH=$cube_install_dir/lib:$scorep_install_dir/lib:$extrap_install_dir/lib:$metacg_install_dir/lib:$mpiwrap_install_dir/lib:$bear_install_dir/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$cube_install_dir/lib64:$scorep_install_dir/lib64:$extrap_install_dir/lib64:$metacg_install_dir/lib64:$mpiwrap_install_dir/lib64:$bear_install_dir/lib64:$LD_LIBRARY_PATH echo -e "-- PATH --\n" $PATH echo -e "\n-- LD_LIBRARY_PATH --\n" $LD_LIBRARY_PATH diff --git a/test/integration/AMG2013/run.sh b/test/integration/AMG2013/run.sh index 9b91138..038b0c9 100755 --- a/test/integration/AMG2013/run.sh +++ b/test/integration/AMG2013/run.sh @@ -49,7 +49,7 @@ bear make CC="OMPI_CC=clang mpicc" -j # Now cgcollector can read the compile_commands.json file, to retrieve the commands required for f in $(cat compile_commands.json | jq -r 'map(.directory + "/" + .file) | .[]' | grep '\.c'); do echo "Processing $f" - cgc $f >/dev/null 2>&1 + cgc --metacg-format-version=2 $f >/dev/null 2>&1 done # Build the full whole-program call-graph echo "null" > amg.ipcg # create empty json file diff --git a/test/integration/AMG2013_Slurm/run.sh b/test/integration/AMG2013_Slurm/run.sh index fcd86f3..1823f55 100755 --- a/test/integration/AMG2013_Slurm/run.sh +++ b/test/integration/AMG2013_Slurm/run.sh @@ -49,7 +49,7 @@ bear make CC="OMPI_CC=clang mpicc" -j # Now cgcollector can read the compile_commands.json file, to retrieve the commands required for f in $(cat compile_commands.json | jq -r 'map(.directory + "/" + .file) | .[]' | grep '\.c'); do echo "Processing $f" - cgc $f >/dev/null 2>&1 + cgc --metacg-format-version=2 $f >/dev/null 2>&1 done # Build the full whole-program call-graph echo "null" > amg.ipcg # create empty json file diff --git a/test/integration/GameOfLife/run.sh b/test/integration/GameOfLife/run.sh index 3b030f6..bad78e2 100755 --- a/test/integration/GameOfLife/run.sh +++ b/test/integration/GameOfLife/run.sh @@ -34,8 +34,8 @@ mv GameOfLife-PIRA-testing gol echo -e "\n----- Build GameOfLife / build call graph -----" cd gol/serial_non_template bear make gol -j 2>&1 > /dev/null -cgc main.cpp 2>&1 > /dev/null -cgc SerialGoL.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 main.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 SerialGoL.cpp 2>&1 > /dev/null echo "null" > gol.ipcg cgmerge gol.ipcg main.ipcg SerialGoL.ipcg 2>&1 > /dev/null cp gol.ipcg $PWD/../../../../../extern/install/metacg/bin/gol_ct.mcg diff --git a/test/integration/GameOfLifePiraVersion1/run.sh b/test/integration/GameOfLifePiraVersion1/run.sh index dfe52fc..94fc60a 100755 --- a/test/integration/GameOfLifePiraVersion1/run.sh +++ b/test/integration/GameOfLifePiraVersion1/run.sh @@ -34,8 +34,8 @@ mv GameOfLife-PIRA-testing gol echo -e "\n----- Build GameOfLife / build call graph -----" cd gol/serial_non_template bear make gol 2>&1 > /dev/null -cgc main.cpp 2>&1 > /dev/null -cgc SerialGoL.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 main.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 SerialGoL.cpp 2>&1 > /dev/null echo "null" > gol.ipcg cgmerge gol.ipcg main.ipcg SerialGoL.ipcg 2>&1 > /dev/null cp gol.ipcg $PWD/../../../../../extern/install/metacg/bin/gol_ct.mcg diff --git a/test/integration/GameOfLifePiraVersion1_Slurm/run.sh b/test/integration/GameOfLifePiraVersion1_Slurm/run.sh index 3626fce..761dbc0 100755 --- a/test/integration/GameOfLifePiraVersion1_Slurm/run.sh +++ b/test/integration/GameOfLifePiraVersion1_Slurm/run.sh @@ -34,8 +34,8 @@ mv GameOfLife-PIRA-testing gol echo -e "\n----- Build GameOfLife / build call graph -----" cd gol/serial_non_template bear make gol 2>&1 > /dev/null -cgc main.cpp 2>&1 > /dev/null -cgc SerialGoL.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 main.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 SerialGoL.cpp 2>&1 > /dev/null echo "null" > gol.ipcg cgmerge gol.ipcg main.ipcg SerialGoL.ipcg 2>&1 > /dev/null cp gol.ipcg $PWD/../../../../../extern/install/metacg/bin/gol_ct.mcg diff --git a/test/integration/GameOfLife_Slurm/run.sh b/test/integration/GameOfLife_Slurm/run.sh index a9c8cff..f4d8da1 100755 --- a/test/integration/GameOfLife_Slurm/run.sh +++ b/test/integration/GameOfLife_Slurm/run.sh @@ -34,8 +34,8 @@ mv GameOfLife-PIRA-testing gol echo -e "\n----- Build GameOfLife / build call graph -----" cd gol/serial_non_template bear make gol -j 2>&1 > /dev/null -cgc main.cpp 2>&1 > /dev/null -cgc SerialGoL.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 main.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 SerialGoL.cpp 2>&1 > /dev/null echo "null" > gol.ipcg cgmerge gol.ipcg main.ipcg SerialGoL.ipcg 2>&1 > /dev/null cp gol.ipcg $PWD/../../../../../extern/install/metacg/bin/gol_ct.mcg diff --git a/test/integration/GameOfLife_heuristic_Slurm/run.sh b/test/integration/GameOfLife_heuristic_Slurm/run.sh index a57d6b4..1f4e12d 100755 --- a/test/integration/GameOfLife_heuristic_Slurm/run.sh +++ b/test/integration/GameOfLife_heuristic_Slurm/run.sh @@ -34,8 +34,8 @@ mv GameOfLife-PIRA-testing gol echo -e "\n----- Build GameOfLife / build call graph -----" cd gol/serial_non_template bear make gol -j 2>&1 > /dev/null -cgc main.cpp 2>&1 > /dev/null -cgc SerialGoL.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 main.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 SerialGoL.cpp 2>&1 > /dev/null echo "null" > gol.ipcg cgmerge gol.ipcg main.ipcg SerialGoL.ipcg 2>&1 > /dev/null cp gol.ipcg $PWD/../../../../../extern/install/metacg/bin/gol_ct.mcg diff --git a/test/integration/GameOfLife_hybrid_filter/run.sh b/test/integration/GameOfLife_hybrid_filter/run.sh index e0e3673..5386664 100755 --- a/test/integration/GameOfLife_hybrid_filter/run.sh +++ b/test/integration/GameOfLife_hybrid_filter/run.sh @@ -34,8 +34,8 @@ mv GameOfLife-PIRA-testing gol echo -e "\n----- Build GameOfLife / build call graph -----" cd gol/serial_non_template bear make gol 2>&1 > /dev/null -cgc main.cpp 2>&1 > /dev/null -cgc SerialGoL.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 main.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 SerialGoL.cpp 2>&1 > /dev/null echo "null" > gol.ipcg cgmerge gol.ipcg main.ipcg SerialGoL.ipcg 2>&1 > /dev/null cp gol.ipcg $PWD/../../../../../extern/install/metacg/bin/gol_ct.mcg diff --git a/test/integration/GameOfLife_hybrid_filter_Slurm/run.sh b/test/integration/GameOfLife_hybrid_filter_Slurm/run.sh index 16e15af..2bca20b 100755 --- a/test/integration/GameOfLife_hybrid_filter_Slurm/run.sh +++ b/test/integration/GameOfLife_hybrid_filter_Slurm/run.sh @@ -34,8 +34,8 @@ mv GameOfLife-PIRA-testing gol echo -e "\n----- Build GameOfLife / build call graph -----" cd gol/serial_non_template bear make gol 2>&1 > /dev/null -cgc main.cpp 2>&1 > /dev/null -cgc SerialGoL.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 main.cpp 2>&1 > /dev/null +cgc --metacg-format-version=2 SerialGoL.cpp 2>&1 > /dev/null echo "null" > gol.ipcg cgmerge gol.ipcg main.ipcg SerialGoL.ipcg 2>&1 > /dev/null cp gol.ipcg $PWD/../../../../../extern/install/metacg/bin/gol_ct.mcg diff --git a/test/integration/Kripke/run.sh b/test/integration/Kripke/run.sh index c1b1c55..f2f5e46 100755 --- a/test/integration/Kripke/run.sh +++ b/test/integration/Kripke/run.sh @@ -63,7 +63,7 @@ cp build/compile_commands.json . # for f in $(cat build/compile_commands.json | jq -r 'map(.directory + "/" + .file) | .[]' | grep '\.cpp'); do for f in $(find ./src -type f -iname "*.cpp" ); do echo "Processing $f" - cgc $f >/dev/null 2>&1 + cgc --metacg-format-version=2 $f >/dev/null 2>&1 done # Build the full whole-program call-graph diff --git a/test/integration/Kripke_Slurm/run.sh b/test/integration/Kripke_Slurm/run.sh index 1924ff5..a917c93 100755 --- a/test/integration/Kripke_Slurm/run.sh +++ b/test/integration/Kripke_Slurm/run.sh @@ -63,7 +63,7 @@ cp build/compile_commands.json . # for f in $(cat build/compile_commands.json | jq -r 'map(.directory + "/" + .file) | .[]' | grep '\.cpp'); do for f in $(find ./src -type f -iname "*.cpp" ); do echo "Processing $f" - cgc $f >/dev/null 2>&1 + cgc --metacg-format-version=2 $f >/dev/null 2>&1 done # Build the full whole-program call-graph diff --git a/test/integration/LULESH/expected_instrumentation.json b/test/integration/LULESH/expected_instrumentation.json index f149b57..7e24982 100644 --- a/test/integration/LULESH/expected_instrumentation.json +++ b/test/integration/LULESH/expected_instrumentation.json @@ -41,7 +41,8 @@ "_ZL16LagrangeElementsR6Domaini", "_ZL31ApplyMaterialPropertiesForElemsR6Domain", "_ZL16LagrangeLeapFrogR6Domain", - "_ZL13CalcQForElemsR6Domain" + "_ZL13CalcQForElemsR6Domain", + "_ZL13LagrangeNodalR6Domain" ], "never-expect": [] }, @@ -59,7 +60,8 @@ "_ZL22CalcMonotonicQForElemsR6Domain", "_ZL17CalcForceForNodesR6Domain", "_ZL15EvalEOSForElemsR6DomainPdiPii", - "_ZL23CalcVolumeForceForElemsR6Domain" + "_ZL23CalcVolumeForceForElemsR6Domain", + "_ZL13CalcQForElemsR6Domain" ], "never-expect": [] }, @@ -76,7 +78,8 @@ "_ZL22CalcMonotonicQForElemsR6Domain", "_ZL23IntegrateStressForElemsR6DomainPdS1_S1_S1_ii", "_ZL28CalcHourglassControlForElemsR6DomainPdd", - "_ZL28CalcMonotonicQRegionForElemsR6Domainid" + "_ZL28CalcMonotonicQRegionForElemsR6Domainid", + "_Z22CalcKinematicsForElemsR6Domaindi" ], "never-expect": [] }, diff --git a/test/integration/LULESH/run.sh b/test/integration/LULESH/run.sh index 4250966..98980fb 100755 --- a/test/integration/LULESH/run.sh +++ b/test/integration/LULESH/run.sh @@ -46,7 +46,7 @@ bear make CXX="mpicxx" CXXFLAGS="$LULESH_CXXFLAGS" -j # Now cgcollector can read the compile_commands.json file, to retrieve the commands required for f in $(cat compile_commands.json | jq -r 'map(.directory + "/" + .file) | .[]' | grep '\.cc'); do echo "Processing $f" - cgc $f >/dev/null 2>&1 + cgc --metacg-format-version=2 $f >/dev/null 2>&1 done # Build the full whole-program call-graph echo "null" > lulesh.mcg # create empty json file diff --git a/test/integration/LULESH_Slurm/run.sh b/test/integration/LULESH_Slurm/run.sh index 6ff297e..7531a8a 100755 --- a/test/integration/LULESH_Slurm/run.sh +++ b/test/integration/LULESH_Slurm/run.sh @@ -46,7 +46,7 @@ bear make CXX="mpicxx" CXXFLAGS="$LULESH_CXXFLAGS" -j # Now cgcollector can read the compile_commands.json file, to retrieve the commands required for f in $(cat compile_commands.json | jq -r 'map(.directory + "/" + .file) | .[]' | grep '\.cc'); do echo "Processing $f" - cgc $f >/dev/null 2>&1 + cgc --metacg-format-version=2 $f >/dev/null 2>&1 done # Build the full whole-program call-graph echo "null" > lulesh.mcg # create empty json file diff --git a/test/integration/LoadImbalance/run.sh b/test/integration/LoadImbalance/run.sh index 43678f6..93e2db2 100755 --- a/test/integration/LoadImbalance/run.sh +++ b/test/integration/LoadImbalance/run.sh @@ -34,9 +34,9 @@ bear make clean echo -e "\n -> Running bear make" bear make CC="OMPI_CC=clang mpicc" imbalance 2>&1 > /dev/null echo -e "\n -> Running cgcollector" -cgc main.c 2>&1 > /dev/null || exit 1 -cgc lib.c 2>&1 > /dev/null || exit 1 -cgc util.h 2>&1 > /dev/null || exit 1 +cgc --metacg-format-version=2 main.c 2>&1 > /dev/null || exit 1 +cgc --metacg-format-version=2 lib.c 2>&1 > /dev/null || exit 1 +cgc --metacg-format-version=2 util.h 2>&1 > /dev/null || exit 1 echo "null" > imbalance.ipcg cgmerge imbalance.ipcg main.ipcg lib.ipcg util.ipcg || exit 1 cp imbalance.ipcg $PWD/../../../../extern/install/metacg/bin/imbalance_ct.mcg diff --git a/test/integration/LoadImbalance_Slurm/run.sh b/test/integration/LoadImbalance_Slurm/run.sh index 28365eb..e27f955 100755 --- a/test/integration/LoadImbalance_Slurm/run.sh +++ b/test/integration/LoadImbalance_Slurm/run.sh @@ -34,9 +34,9 @@ bear make clean echo -e "\n -> Running bear make" bear make CC="OMPI_CC=clang mpicc" imbalance 2>&1 > /dev/null echo -e "\n -> Running cgcollector" -cgc main.c 2>&1 > /dev/null || exit 1 -cgc lib.c 2>&1 > /dev/null || exit 1 -cgc util.h 2>&1 > /dev/null || exit 1 +cgc --metacg-format-version=2 main.c 2>&1 > /dev/null || exit 1 +cgc --metacg-format-version=2 lib.c 2>&1 > /dev/null || exit 1 +cgc --metacg-format-version=2 util.h 2>&1 > /dev/null || exit 1 echo "null" > imbalance.ipcg cgmerge imbalance.ipcg main.ipcg lib.ipcg util.ipcg || exit 1 cp imbalance.ipcg $PWD/../../../../extern/install/metacg/bin/imbalance_ct.mcg diff --git a/test/integration/check.py b/test/integration/check.py index 4775d41..87bfd76 100755 --- a/test/integration/check.py +++ b/test/integration/check.py @@ -34,11 +34,9 @@ """ import argparse -from distutils.command.config import config import json import os import re -from xmlrpc.client import FastMarshaller def prepare_instrumentation_file(lines):