Optimize dockerfile (building and caching) (#294)

* Introduce multistage dockerfile for build process (core only) * Update .dockerignore * Update README * Do not apt update at build time internet should not be required when compiling * Use apt-get instead of apt as apt does not have a stable CLI and thus unsuitable for scripting - https://unix.stackexchange.com/a/590703/213124 * Automate apt-get yes prompts for non-interactive scripting * Use placeholder stage for dependabot automation * Rename stages uniquely for locatability and mark for readability * Move ARG to appropriate scope to avoid needless busting build cache of unrelated stages * Disable internet for build time to flush out all non-determinism given offline caching is impractical otherwise * Edit apt config for caching and update once * Bootstrap and cache tools for install scripts * Add missing implicit dependency * Add pseudo code to install dependencies only in pepper stage * Add dependabot config For both docker and github actions. This should: - auto open a PR when new nvidia tags are available - as well as keep Github actions versions up to date * Simplify docker ignore config by inverting exclusion set to robustly ignore randomly named IDE files and temp folders * Exclude .git from .dockerignore Required by cmake_git_version_tracking * Separate dependencies script from compilation script * WIP - RGL extensions handling * Add ROS2 installation * Add build command argument * Add ros2 standalone libs and rgl executables to exporter * Fix pcl deps * Fix ros2 extension compilation * Fix CMakeLists for tests * Add ORIGIN rpath to test/tools executables * Do not change current working directory for extension install_deps scripts * Improve docs * Bring back support for taped test * Formatting * Adjust CI * Add dependencies installation info to README * Ignore Dockerfile in .dockerignore * Remove unused components from CI * Fix CI * Output libraries to lib folder * Output executables to bin folder * Simplify exporter stage by cache dancing * Formatting alpha sort * Output libraries to lib folder kind of hacky should probably simplify using better CMake * Simplify ROS install as timezone and LC_ALL setup is more for running and shouldn't be needed if only just building * Simplify ROS install remove unnecessary packages * Formatting * Use ROS_DISTRO ARG and set ENV if ever needed for later after docker build * Drop pinned package version * Formatting * Simplify ROS install by using minimal build dependency set * Alpha sort package sub lists * Simplify RUN directive via shell * Simplify RUN directive via shell * Revert to installing core as `radar_msgs` has a number of other msg dependencies though rosdep should eventually be used here * Formatting * Avoid hard coded ubuntu code name * Simplify RUN directives * Cache apt update * Format package use septate lines for version control * Update README.md Co-authored-by: Mateusz Szczygielski <[email protected]> * Revert "Remove Disable DNS" This partially reverts commit 74317d5. * Simplify CMake for INSTALL_DESTINATION_DIR * Revert "Revert "Remove Disable DNS"" This reverts commit 32016bb0b0683769fefd0962ab32d67ca968ae8a. * Add missing dep for ros2-standalone * Adjust CI * Fix CI: use dot instead of source * Fix CI: source sh file * Improve ROS2 check * Use new dev docker to test * Fix formatting (some packages weren't installed) * Use ROS_DISTRO instead of fixed version --------- Co-authored-by: ruffsl <[email protected]>
RobotecAI · Jun 11, 2024 · c2e5162 · c2e5162
1 parent 64d52dc
commit c2e5162
Show file tree

Hide file tree

Showing 17 changed files with 610 additions and 428 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,9 +1,19 @@
-.vs
-.vscode
-Builds/*
-Testing/*
-.idea
-cmake-build-debug
-cmake-build-release
-build
-external/vcpkg
+# Ignore everything by default
+*
+
+# First-order allow exception for select directories
+!/.clang-format
+!/.githooks
+!/.git
+!/CMakeLists.txt
+!/docs
+!/extensions
+!/extensions.repos
+!external/CMakeLists.txt
+!/include
+!/ros2_standalone
+!/setup.py
+!/install_deps.py
+!/src
+!/test
+!/tools
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    commit-message:
+      prefix: "🐳 "
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    commit-message:
+      prefix: "🛠️ "
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -24,91 +24,94 @@ jobs:
         uses: actions/checkout@v3
         with:
           clean: false
-      - name: Clean repository excluding taped test data
-        run: git clean -ffdx -e external/taped_test_data
+      - name: Clean repository excluding RGL blobs repo
+        run: git clean -ffdx -e external/rgl_blobs
       - name: Import private extensions
         run: vcs import < extensions.repos
-      - name: Install taped test dependencies
-        run: ./setup.py --install-taped-test-deps
+      - name: Install taped test runtime dependencies
+        run: ./setup.py --fetch-rgl-blobs
 
   load-env:
     uses: ./.github/workflows/load-env-subworkflow.yml
 
 ###### BUILD USING RGL DOCKER ######
   build-core:
     needs: [checkout-repository, load-env]
-    uses: ./.github/workflows/build-subworkflow.yml
-    with:
-      build-command: ./setup.py --build-dir build-core
-      self-hosted-user-id: ${{ needs.load-env.outputs.user-id }}
-      optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }}
-      docker-image: localhost:5000/rgl:latest
+    runs-on: self-hosted
+    steps:
+      - run: '
+        docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }}
+        --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1
+        --build-arg BUILD_CMD="./setup.py"
+        --target=exporter --output=bin-core .'
 
   build-pcl:
     needs: [checkout-repository, load-env]
-    uses: ./.github/workflows/build-subworkflow.yml
-    with:
-      build-command: './setup.py --with-pcl --build-taped-test --build-dir build-pcl'
-      self-hosted-user-id: ${{ needs.load-env.outputs.user-id }}
-      optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }}
-      docker-image: localhost:5000/rgl:latest
+    runs-on: self-hosted
+    steps:
+      - run: '
+        docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }}
+        --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1
+        --build-arg BUILD_CMD="./setup.py --with-pcl --build-taped-test"
+        --target=exporter --output=bin-pcl .'
 
   build-ros2:
     needs: [ checkout-repository, load-env ]
-    uses: ./.github/workflows/build-subworkflow.yml
-    with:
-      build-command: '
-        source /opt/ros/humble/setup.bash &&
-        ./setup.py --with-ros2-standalone --build-dir build-ros2'
-      self-hosted-user-id: ${{ needs.load-env.outputs.user-id }}
-      optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }}
-      docker-image: localhost:5000/rgl:latest
+    runs-on: self-hosted
+    steps:
+      - run: '
+        docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }}
+        --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1
+        --build-arg BUILD_CMD=". /opt/ros/\$ROS_DISTRO/setup.sh && ./setup.py --with-ros2-standalone"
+        --target=exporter --output=bin-ros2 .'
 
   build-udp:
     needs: [ checkout-repository, load-env ]
-    uses: ./.github/workflows/build-subworkflow.yml
-    with:
-      build-command: './setup.py --with-udp --build-dir build-udp'
-      self-hosted-user-id: ${{ needs.load-env.outputs.user-id }}
-      optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }}
-      docker-image: localhost:5000/rgl:latest
+    runs-on: self-hosted
+    steps:
+      - run: '
+        docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }}
+        --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1
+        --build-arg BUILD_CMD="./setup.py --with-udp"
+        --target=exporter --output=bin-udp .'
 
   build-snow:
     needs: [ checkout-repository, load-env ]
-    uses: ./.github/workflows/build-subworkflow.yml
-    with:
-      build-command: './setup.py --with-snow --build-dir build-snow'
-      self-hosted-user-id: ${{ needs.load-env.outputs.user-id }}
-      optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }}
-      docker-image: localhost:5000/rgl:latest
+    runs-on: self-hosted
+    steps:
+      - run: '
+        docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }}
+        --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1
+        --build-arg BUILD_CMD="./setup.py --with-snow"
+        --target=exporter --output=bin-snow .'
 
   build-all:
     needs: [ checkout-repository, load-env ]
-    uses: ./.github/workflows/build-subworkflow.yml
-    with:
-      build-command: '
-        source /opt/ros/humble/setup.bash &&
-        ./setup.py --with-pcl --with-ros2-standalone --with-udp --with-snow --build-taped-test --build-dir build-all'
-      self-hosted-user-id: ${{ needs.load-env.outputs.user-id }}
-      optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }}
-      docker-image: localhost:5000/rgl:latest
-
-###### TEST WITH RGL DOCKER IMAGE ######
+    runs-on: self-hosted
+    steps:
+      - run: '
+        docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }}
+        --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1
+        --build-arg BUILD_CMD=". /opt/ros/\$ROS_DISTRO/setup.sh && ./setup.py --with-pcl --with-ros2-standalone --with-udp --with-snow --build-taped-test"
+        --target=exporter --output=bin-all .'
+
+####### TEST WITH RGL DOCKER IMAGE ######
   test-core-dev:
     needs: [build-core]
     uses: ./.github/workflows/test-subworkflow.yml
     with:
-      test-command: 'cd build-core/test && ./RobotecGPULidar_test'
-      docker-image: localhost:5000/rgl:latest
+      test-command: '
+        cd bin-core/bin/test && ./RobotecGPULidar_test'
+      docker-image: localhost:5000/robotecgpulidar-all:latest
 
   test-pcl-dev:
     needs: [ build-pcl ]
     uses: ./.github/workflows/test-subworkflow.yml
     with:
       test-command: '
-      export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data &&
-      cd build-pcl/test && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test'
-      docker-image: localhost:5000/rgl:latest
+        export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/rgl_blobs &&
+        cd bin-pcl/bin/test && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test'
+      docker-image: localhost:5000/robotecgpulidar-all:latest
 
   test-ros2-dev:
     needs: [ build-ros2 ]
@@ -117,26 +120,28 @@ jobs:
       # Source ROS2 and radar_msgs, standalone build is tested in prod environment
       # Run tests twice, each for different RMW implementation
       test-command: '
-        source /opt/ros/humble/setup.bash &&
-        source /rgldep/radar_msgs/install/setup.bash &&
-        cd build-ros2/test &&
+        . /opt/ros/$ROS_DISTRO/setup.sh &&
+        . /opt/rgl/external/radar_msgs/install/setup.sh &&
+        cd bin-ros2/bin/test &&
         export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test &&
         export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test'
-      docker-image: localhost:5000/rgl:latest
+      docker-image: localhost:5000/robotecgpulidar-all:latest
 
   test-udp-dev:
     needs: [ build-udp ]
     uses: ./.github/workflows/test-subworkflow.yml
     with:
-      test-command: 'cd build-udp/test && ./RobotecGPULidar_test'
-      docker-image: localhost:5000/rgl:latest
+      test-command: '
+        cd bin-udp/bin/test && ./RobotecGPULidar_test'
+      docker-image: localhost:5000/robotecgpulidar-all:latest
 
   test-snow-dev:
     needs: [ build-snow ]
     uses: ./.github/workflows/test-subworkflow.yml
     with:
-      test-command: 'cd build-snow/test && ./RobotecGPULidar_test'
-      docker-image: localhost:5000/rgl:latest
+      test-command: '
+        cd bin-snow/bin/test && ./RobotecGPULidar_test'
+      docker-image: localhost:5000/robotecgpulidar-all:latest
 
   test-all-dev:
     needs: [ build-all ]
@@ -146,21 +151,22 @@ jobs:
       # Set `RGL_TEST_VLP16_CALIB_FILE` for UDP-ROS2 integration test
       # Run tests twice, each for different RMW implementation
       test-command: '
-        source /opt/ros/humble/setup.bash &&
-        source /rgldep/radar_msgs/install/setup.bash &&
+        . /opt/ros/$ROS_DISTRO/setup.sh &&
+        . /opt/rgl/external/radar_msgs/install/setup.sh &&
         export RGL_TEST_VLP16_CALIB_FILE=$(pwd)/extensions/udp/test/resources/Ros2Vlp16Calib.yaml &&
-        export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data &&
-        cd build-all/test &&
-        export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test &&
-        export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test'
-      docker-image: localhost:5000/rgl:latest
+        export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/rgl_blobs &&
+        cd bin-all/bin/test &&
+        export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test &&
+        export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test'
+      docker-image: localhost:5000/robotecgpulidar-all:latest
 
-###### TEST WITH CLEAN UBUNTU DOCKER IMAGE ######
+####### TEST WITH CLEAN UBUNTU DOCKER IMAGE ######
   test-core-prod:
     needs: [build-core]
     uses: ./.github/workflows/test-subworkflow.yml
     with:
-      test-command: 'cd build-core/test && ./RobotecGPULidar_test'
+      test-command: '
+        cd bin-core/bin/test && ./RobotecGPULidar_test'
       docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04
 
   test-pcl-prod:
@@ -170,8 +176,8 @@ jobs:
       # Additionally, install PCL extension dependent libraries for runtime
       test-command: '
         apt update && apt install -y libxcursor1 libgl1 &&
-        export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data &&
-        cd build-pcl/test && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test'
+        export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/rgl_blobs &&
+        cd bin-pcl/bin/test && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test'
       docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04
 
   test-ros2-prod:
@@ -181,8 +187,8 @@ jobs:
       # Copy ROS2 libraries to be visible for libRobotecGPULidar.so
       # Run tests twice, each for different RMW implementation
       test-command: '
-        cd build-ros2/test &&
-        cp -r ../ros2_standalone/*.so* ../ &&
+        cp -p bin-ros2/lib/ros2_standalone/* bin-ros2/lib &&
+        cd bin-ros2/bin/test &&
         export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test &&
         export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test'
       docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04
@@ -191,14 +197,16 @@ jobs:
     needs: [ build-udp ]
     uses: ./.github/workflows/test-subworkflow.yml
     with:
-      test-command: 'cd build-udp/test && ./RobotecGPULidar_test'
+      test-command: '
+        cd bin-udp/bin/test && ./RobotecGPULidar_test'
       docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04
 
   test-snow-prod:
     needs: [ build-snow ]
     uses: ./.github/workflows/test-subworkflow.yml
     with:
-      test-command: 'cd build-snow/test && ./RobotecGPULidar_test'
+      test-command: '
+        cd bin-snow/bin/test && ./RobotecGPULidar_test'
       docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04
 
   test-all-prod:
@@ -210,9 +218,9 @@ jobs:
       # Run tests twice, each for different RMW implementation
       test-command: '
         apt update && apt install -y libxcursor1 libgl1 &&
-        export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data &&
-        cd build-all/test &&
-        cp -r ../ros2_standalone/*.so* ../ &&
-        export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test &&
-        export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test'
+        export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/rgl_blobs &&
+        cp -p bin-all/lib/ros2_standalone/* bin-all/lib &&
+        cd bin-all/bin/test &&
+        export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test &&
+        export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test'
       docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04
diff --git a/.github/workflows/build-subworkflow.yml b/.github/workflows/build-subworkflow.yml
diff --git a/.github/workflows/load-env-subworkflow.yml b/.github/workflows/load-env-subworkflow.yml
@@ -5,27 +5,19 @@ on:
     outputs:
       optix-install-dir:
         value: ${{ jobs.load-env.outputs.optix-install-dir }}
-      user-id:
-        value: ${{ jobs.load-env.outputs.user-id }}
 
 jobs:
   load-env:
     runs-on: self-hosted
     outputs:
       optix-install-dir: ${{ steps.set-envs.outputs.optix-install-dir }}
-      user-id: ${{ steps.set-envs.outputs.user-id }}
     steps:
       - id: check-envs
         run: |
           if [[ -z "${OptiX_INSTALL_DIR}" ]]; then
             echo "OptiX_INSTALL_DIR env is empty"
             exit 1
           fi
-          if [[ -z "$(id -u $USER)" ]]; then
-            echo "Cannot deduce id of the USER"
-            exit 1
-          fi
       - id: set-envs
         run: |
           echo "optix-install-dir=$OptiX_INSTALL_DIR" | tee -a $GITHUB_OUTPUT
-          echo "user-id=$(id -u $USER)" | tee -a $GITHUB_OUTPUT