release-testing.yml

name: Release Testing
on:
  push:
    branches:
      - "hotfix-*"
      - "rc--*"
  workflow_dispatch:
# new commits interrupt any running workflow on the same branch
concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true
env:
  CI_COMMIT_SHA: ${{ github.sha }}
  CI_JOB_NAME: ${{ github.job }}
  CI_PROJECT_DIR: ${{ github.workspace }}
  BRANCH_NAME: ${{ github.event.workflow_run.head_branch || github.ref_name }}
  CI_RUN_ID: ${{ github.run_id }}
  RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic"
  BUILDEVENT_DATASET: "github-ci-dfinity"
jobs:
  ci-main:
    name: CI Main
    uses: ./.github/workflows/ci-main.yml
    secrets: inherit
  bazel-system-test-nightly:
    name: Bazel System Test Nightly
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 180 # 3 hours
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.workflow_run.head_branch }}
      - name: Before script
        id: before-script
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
      - name: Run Bazel System Test Nightly
        id: bazel-test-all
        uses: ./.github/actions/bazel-test-all/
        with:
          BAZEL_COMMAND: "test"
          BAZEL_TARGETS: "//rs/tests/..."
          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
          BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=system_test_nightly"
          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
      - name: Upload bazel-bep
        # runs only if previous step succeeded or failed;
        # we avoid collecting artifacts of jobs that were cancelled
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-bep
          retention-days: 14
          if-no-files-found: ignore
          compression-level: 9
          path: |
            bazel-bep.pb
            profile.json
  bazel-system-test-staging:
    name: Bazel System Test Staging
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 180 # 3 hours
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.workflow_run.head_branch }}
      - name: Before script
        id: before-script
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
      - name: Run Bazel System Test Staging
        id: bazel-test-all
        uses: ./.github/actions/bazel-test-all/
        with:
          BAZEL_COMMAND: "test"
          BAZEL_TARGETS: "//rs/tests/..."
          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
          BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=system_test_staging"
          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
      - name: Upload bazel-bep
        # runs only if previous step succeeded or failed;
        # we avoid collecting artifacts of jobs that were cancelled
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-bep
          retention-days: 14
          if-no-files-found: ignore
          compression-level: 9
          path: |
            bazel-bep.pb
            profile.json
  bazel-system-test-hotfix:
    name: Bazel System Test Hotfix
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 90
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.workflow_run.head_branch }}
      - name: Before script
        id: before-script
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
      - name: Run Bazel Test All
        id: bazel-test-all
        uses: ./.github/actions/bazel-test-all/
        with:
          BAZEL_COMMAND: "test"
          BAZEL_TARGETS: "//rs/tests/..."
          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
          BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=system_test_hotfix"
          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
      - name: Upload bazel-bep
        # runs only if previous step succeeded or failed;
        # we avoid collecting artifacts of jobs that were cancelled
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-bep
          retention-days: 14
          if-no-files-found: ignore
          compression-level: 9
          path: |
            bazel-bep.pb
            profile.json
  dependency-scan-release-cut:
    name: Dependency Scan for Release
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 60
    env:
      SHELL_WRAPPER: "/usr/bin/time"
      CI_PROJECT_PATH: ${{ github.repository }}
      CI_PIPELINE_ID: ${{ github.run_id }}
      CI_COMMIT_SHA: ${{ github.sha }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
      SLACK_PSEC_BOT_OAUTH_TOKEN: ${{ secrets.SLACK_PSEC_BOT_OAUTH_TOKEN }}
      REPO_NAME: ${{ github.repository }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.workflow_run.head_branch }}
      - name: Before script
        id: before-script
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Setup python deps
        id: setup-python-deps
        shell: bash
        run: |
          # Ignore externally-managed-environment pip error, install packages system-wide.
          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
      - name: Dependency Scan for Release
        id: dependency-scan-release-cut
        shell: bash
        run: |
          set -euo pipefail
          export PYTHONPATH=$PWD/ci/src:$PWD/ci/src/dependencies
          cd ci/src/dependencies/
          $SHELL_WRAPPER python3 job/bazel_rust_ic_scanner_release_job.py
  setup-guest-os-qualification:
    name: Setting up guest os qualification pipeline
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 180 # 3 hours
    outputs:
      matrix: ${{ steps.generate.outputs.output }}
    steps:
      - name: Sparse checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.workflow_run.head_branch }}
          sparse-checkout: ".github/scripts/determine-initial-guest-os-versions.py"
      - id: generate
        name: Fetch beginning versions for qualification
        shell: bash
        run: |
          set -euo pipefail
          OUTPUT=$(python .github/scripts/determine-initial-guest-os-versions.py)
          echo "output=$OUTPUT" >> $GITHUB_OUTPUT
  guest-os-qualification:
    name: Qualifying ${{ matrix.version }} -> ${{ github.sha }}
    needs: setup-guest-os-qualification
    strategy:
      matrix: ${{ fromJson(needs.setup-guest-os-qualification.outputs.matrix) }}
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 180 # 3 hours
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.workflow_run.head_branch }}
      - name: Before script
        id: before-script
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
      - name: Run qualification for version ${{ matrix.version }} from the tip of the branch
        uses: ./.github/actions/bazel-test-all/
        with:
          BAZEL_COMMAND: "test"
          BAZEL_TARGETS: "//rs/tests/dre:guest_os_qualification"
          BAZEL_CI_CONFIG: "--config=systest --repository_cache=/cache/bazel"
          BAZEL_EXTRA_ARGS: "--keep_going --test_timeout=7200 --test_env=OLD_VERSION=${{ matrix.version }}"
          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
      - # runs only if previous step succeeded or failed;
        # we avoid collecting artifacts of jobs that were cancelled
        if: success() || failure()
        uses: actions/upload-artifact@v4
        name: Upload bazel bep for version ${{ matrix.version }}
        with:
          retention-days: 14
          if-no-files-found: ignore
          compression-level: 9
          path: |
            bazel-bep.pb
            profile.json
          name: ${{ github.job }}-${{ matrix.version }}-bep