From 0ccec6f0570ffad5665f1fdbc3a16f62a9015acc Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Fri, 27 Sep 2019 14:58:29 +0900 Subject: [PATCH 1/3] Adding possibility of faster builds using ccache Ignore MSVC + Recommended change Replace [[ with [ to be compatible with sh Fix indentation disable ccache by default Add clcache support Modifications to save variables in parent scope --- .ci/azure-pipelines/build-windows.yml | 2 +- CMakeLists.txt | 5 + cmake/Modules/UseCompilerCache.cmake | 134 ++++++++++++++++++++++++++ cmake/pcl_options.cmake | 4 + 4 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 cmake/Modules/UseCompilerCache.cmake diff --git a/.ci/azure-pipelines/build-windows.yml b/.ci/azure-pipelines/build-windows.yml index 0bc2fd301fd..b55b1214724 100644 --- a/.ci/azure-pipelines/build-windows.yml +++ b/.ci/azure-pipelines/build-windows.yml @@ -26,7 +26,7 @@ jobs: displayName: 'Update System PATH' - script: | vcpkg.exe install eigen3 flann gtest qhull --triplet %PLATFORM%-windows && vcpkg.exe list - displayName: 'Install Dependencies' + displayName: 'Install c++ dependencies via vcpkg' - script: | rmdir %VCPKG_ROOT%\downloads /S /Q rmdir %VCPKG_ROOT%\packages /S /Q diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d49eef2632..0f9c0d04fb8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,11 @@ include("${PCL_SOURCE_DIR}/cmake/pcl_targets.cmake") include("${PCL_SOURCE_DIR}/cmake/pcl_options.cmake") include("${PCL_SOURCE_DIR}/cmake/clang-format.cmake") +if(${PCL_ENABLE_CCACHE}) + include (UseCompilerCache) + UseCompilerCache(ccache REQUIRED) +endif() + # Enable verbose timing display? if(CMAKE_TIMING_VERBOSE AND UNIX) set_property(GLOBAL PROPERTY RULE_MESSAGES OFF) diff --git a/cmake/Modules/UseCompilerCache.cmake b/cmake/Modules/UseCompilerCache.cmake new file mode 100644 index 00000000000..0c21d296c0a --- /dev/null +++ b/cmake/Modules/UseCompilerCache.cmake @@ -0,0 +1,134 @@ +# .rst: +# UseCompilerCache +# -------- +# +# This module provides a function to setup a compiler cache tool (default: ``ccache``) +# Main function of interest is ``UseCompilerCache`` +# +# Needs CMake 3.4 at least +# Inspired from: +# * https://crascit.com/2016/04/09/using-ccache-with-cmake/ +# * https://stackoverflow.com/a/36515503/ +# * https://gitlab.kitware.com/henryiii/cmake/blob/cache/Modules/UseCompilerCache.cmake + +# .rst +# pcl_ccache_compat_file_gen +# -- Generates a wrapper file which launches the compiler commands using ccache. +# This allows support for XCode and CCache < 3.3 +function(pcl_ccache_compat_file_gen FILE_NAME CCACHE_PROGRAM COMPILER) + message(STATUS "${FILE_NAME} for ${CCACHE_PROGRAM} with ${COMPILER}") + file(WRITE "${CMAKE_BINARY_DIR}/${FILE_NAME}" "" + "#! /usr/bin/env sh\n" + "\n" + "# Xcode generator doesn't include the compiler as the\n" + "# first argument, Ninja and Makefiles do. Handle both cases.\n" + "if [ \"$1\" = \"${COMPILER}\" ] ; then\n" + " shift\n" + "fi\n" + "\n" + "export CCACHE_CPP2=true\n" + "exec \"${CCACHE_PROGRAM}\" \"${COMPILER}\" \"$@\"\n") +endfunction() + +# .rst +# UseCompilerCache([PROGRAM ] [QUIET] [REQUIRED]) +# -- Add the compiler cache tool (default to look for ccache on the path) +# to your build through CMAKE__COMPILER_LAUNCHER variables. Also +# supports XCode. Uses a wrapper for XCode and CCache < 3.3. +# Sets the COMPILER_CACHE_VERSION variable. +function(UseCompilerCache) + set(options QUIET REQUIRED) + set(oneValueArgs CCACHE) + set(multiValueArgs) + + cmake_parse_arguments(ARGS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if(NOT ARGS_CCACHE) + set(ARGS_CCACHE ccache) + endif() + + find_program(CCACHE_PROGRAM ${ARGS_CCACHE}) + + # Quit if not found + if(NOT CCACHE_PROGRAM) + if(REQUIRED) + message(FATAL_ERROR "Failed to find ${CCACHE_PROGRAM} (REQUIRED)") + endif() + return() + endif() + + if(CMAKE_GENERATOR MATCHES "Visual") + message(FATAL_ERROR "MSVC isn't compatible with current solutions. Please rename compiler cache to cl.exe and prepend its location in env PATH variable") + return() + endif() + + # Get version number + execute_process(COMMAND "${CCACHE_PROGRAM}" --version OUTPUT_VARIABLE output) + string(REPLACE "\n" ";" output "${output}") + foreach(line ${output}) + string(TOLOWER ${line} line) + string(REGEX REPLACE "^ccache version ([\\.0-9]+)$" "\\1" version "${line}") + if(version) + set(COMPILER_CACHE_VERSION ${version} PARENT_SCOPE) + break() + endif() + endforeach() + + if(NOT QUIET) + message(STATUS "Using Compiler Cache (${CCACHE_PROGRAM}) v${version} in the C/C++ toolchain") + endif() + + set(xcode_compat FALSE) + if(CMAKE_GENERATOR STREQUAL Xcode) + set(xcode_compat TRUE) + endif() + set(ccache_compat FALSE) + if((ARGS_CCACHE STREQUAL ccache) AND (version VERSION_LESS 3.3.0)) + set(ccache_compat TRUE) + endif() + + # Indirect wrapper is needed for CCache < 3.3 or XCode + if(NOT (${xcode_compat} OR ${ccache_compat})) + # Support Unix Makefiles and Ninja + message(STATUS "Compiler cache via cmake launcher prefix") + set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}" PARENT_SCOPE) + set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}" PARENT_SCOPE) + set(CMAKE_CUDA_COMPILER_LAUNCHER "${CCACHE_PROGRAM}" PARENT_SCOPE) + return() + endif() + + message(STATUS "Generating launch helpers for compiler cache") + + pcl_ccache_compat_file_gen("launch-c" ${CCACHE_PROGRAM} ${CMAKE_C_COMPILER}) + pcl_ccache_compat_file_gen("launch-cxx" ${CCACHE_PROGRAM} ${CMAKE_CXX_COMPILER}) + execute_process(COMMAND chmod a+rx + "${CMAKE_BINARY_DIR}/launch-c" + "${CMAKE_BINARY_DIR}/launch-cxx") + + # Cuda support only added in CMake 3.10 + set(cuda_supported FALSE) + if (NOT (CMAKE_VERSION VERSION_LESS 3.10) AND CMAKE_CUDA_COMPILER) + set(cuda_supported TRUE) + endif() + if(${cuda_supported}) + pcl_ccache_compat_file_gen("launch-cuda" ${CCACHE_PROGRAM} ${CMAKE_CUDA_COMPILER}) + execute_process(COMMAND chmod a+rx + "${CMAKE_BINARY_DIR}/launch-cuda") + endif() + + if(${xcode_compat}) + # Set Xcode project attributes to route compilation and linking properly + message(STATUS "Compiler cache via launch files to support XCode") + set(CMAKE_XCODE_ATTRIBUTE_CC "${CMAKE_BINARY_DIR}/launch-c" PARENT_SCOPE) + set(CMAKE_XCODE_ATTRIBUTE_CXX "${CMAKE_BINARY_DIR}/launch-cxx" PARENT_SCOPE) + set(CMAKE_XCODE_ATTRIBUTE_LD "${CMAKE_BINARY_DIR}/launch-c" PARENT_SCOPE) + set(CMAKE_XCODE_ATTRIBUTE_LDPLUSPLUS "${CMAKE_BINARY_DIR}/launch-cxx" PARENT_SCOPE) + else() + message(STATUS "Compiler cache via launch files to support Unix Makefiles and Ninja") + set(CMAKE_C_COMPILER_LAUNCHER "${CMAKE_BINARY_DIR}/launch-c" PARENT_SCOPE) + set(CMAKE_CXX_COMPILER_LAUNCHER "${CMAKE_BINARY_DIR}/launch-cxx" PARENT_SCOPE) + if (${cuda_supported}) + set(CMAKE_CUDA_COMPILER_LAUNCHER "${CMAKE_BINARY_DIR}/launch-cuda" PARENT_SCOPE) + endif() + endif() +endfunction() diff --git a/cmake/pcl_options.cmake b/cmake/pcl_options.cmake index c2453f1b3b6..2ed09435670 100644 --- a/cmake/pcl_options.cmake +++ b/cmake/pcl_options.cmake @@ -42,6 +42,10 @@ mark_as_advanced(PCL_NO_PRECOMPILE) option(PCL_ENABLE_SSE "Enable or Disable SSE optimizations." ON) mark_as_advanced(PCL_ENABLE_SSE) +# Allow the user to disable ccache if ccache is available +option(PCL_ENABLE_CCACHE "Enable using compiler cache for compilation" OFF) +mark_as_advanced(PCL_ENABLE_CCACHE) + # Display timing information for each compiler instance on screen option(CMAKE_TIMING_VERBOSE "Enable the display of timing information for each compiler instance." OFF) mark_as_advanced(CMAKE_TIMING_VERBOSE) From 89ff9a4693de3469fe5780527c657bc757556f5b Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 29 Sep 2019 19:43:52 +0900 Subject: [PATCH 2/3] Add ccache to dev dockerfile --- .dev/docker/env/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.dev/docker/env/Dockerfile b/.dev/docker/env/Dockerfile index c54c97cd640..c58d8429ccd 100644 --- a/.dev/docker/env/Dockerfile +++ b/.dev/docker/env/Dockerfile @@ -6,6 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y \ + ccache \ cmake \ g++ \ wget \ From 6680d9c392405e9da24383cbe48cd8cda92031d9 Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 29 Sep 2019 19:44:08 +0900 Subject: [PATCH 3/3] Add ccache instructions to azure pipeline Setting a high limit on cache size Per-run stats only Removed ccache from Mac, negative performance impact --- .ci/azure-pipelines/build-ubuntu.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.ci/azure-pipelines/build-ubuntu.yml b/.ci/azure-pipelines/build-ubuntu.yml index 7c24a2fe69e..ce50d7215c3 100644 --- a/.ci/azure-pipelines/build-ubuntu.yml +++ b/.ci/azure-pipelines/build-ubuntu.yml @@ -13,11 +13,21 @@ jobs: variables: BUILD_DIR: '$(Agent.BuildDirectory)/build' CMAKE_CXX_FLAGS: '-Wall -Wextra -Wabi -O2' + CCACHE_DIR: $(Pipeline.Workspace)/ccache + CCACHE_MAXSIZE: 15G steps: + - task: CacheBeta@0 + inputs: + key: ccache | gcc | $(Agent.OS) + path: $(CCACHE_DIR) + displayName: Fetch cached files for ccache + - script: ccache -z + displayName: 'Zero ccache statistics' - script: | mkdir $BUILD_DIR && cd $BUILD_DIR cmake $(Build.SourcesDirectory) \ -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" \ + -DPCL_ENABLE_CCACHE=ON \ -DPCL_ONLY_CORE_POINT_TYPES=ON \ -DBUILD_simulation=ON \ -DBUILD_surface_on_nurbs=ON \ @@ -41,6 +51,8 @@ jobs: cmake --build . -- test_filters test_registration test_registration_api cmake --build . -- -j2 displayName: 'Build Library' + - script: ccache -s + displayName: 'Display ccache statistics' - script: | cd $BUILD_DIR/test ctest -V -T Test