gpu-bench.yml

# Run final tests only when attempting to merge, shown as skipped status checks beforehand
name: GPU benchmark regression test

on:
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    branches: [dev]
  merge_group:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  # Run comparative benchmark against dev, open issue on regression
  gpu-benchmark:
    if: github.event_name != 'pull_request' || github.event.action == 'enqueued'
    name: Run benchmarks on GPU
    runs-on: [self-hosted, gpu-bench]
    steps:
      - uses: actions/checkout@v4
        with:
          repository: lurk-lab/ci-workflows
      - uses: ./.github/actions/gpu-setup
        with:
          gpu-framework: 'cuda'
      - uses: ./.github/actions/ci-env
      - uses: actions/checkout@v4
      # Install dependencies
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@v2
        with:
          tool: just@1.22
      - name: Install criterion
        run: |
          cargo install cargo-criterion
          cargo install criterion-table
      - name: Set bench output format and base SHA
        run: |
          echo "ARECIBO_BENCH_OUTPUT=commit-comment" | tee -a $GITHUB_ENV
          echo "BASE_COMMIT=${{ github.event.merge_group.base_sha }}" | tee -a $GITHUB_ENV
          GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1)
          echo "GPU_ID=$(echo $GPU_NAME | awk '{ print $NF }')" | tee -a $GITHUB_ENV
          echo "GPU_NAME=$GPU_NAME" | tee -a $GITHUB_ENV
      # Checkout base branch for comparative bench
      - uses: actions/checkout@v4
        with:
          ref: dev
          path: dev
      # Copy the script so the base can bench with the same parameters
      - name: Run GPU bench on base branch
        run: |
          # Copy justfile to dev, overwriting existing config with that of PR branch
          cp ../benches/justfile .
          # Run benchmark
          just gpu-bench-ci recursive-snark recursive-snark-supernova compressed-snark compressed-snark-supernova
          # Copy bench output to PR branch
          cp *-${{ env.BASE_COMMIT }}.json ..
        working-directory: ${{ github.workspace }}/dev
      - name: Run GPU bench on PR branch
        run: |
          just gpu-bench-ci recursive-snark recursive-snark-supernova compressed-snark compressed-snark-supernova
          cp *-${{ github.sha }}.json ..
        working-directory: ${{ github.workspace }}/benches
      - name: copy the benchmark template and prepare it with data
        run: |
          cp .github/tables.toml .
          # Get CPU model
          CPU_MODEL=$(grep '^model name' /proc/cpuinfo | head -1 | awk -F ': ' '{ print $2 }')
          # Get vCPU count
          NUM_VCPUS=$(nproc --all)
          # Get total RAM in GB
          TOTAL_RAM=$(grep MemTotal /proc/meminfo | awk '{$2=$2/(1024^2); print int($2), "GB RAM";}')
          WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          
          # Use conditionals to ensure that only non-empty variables are inserted
          [[ ! -z "${{ env.GPU_NAME }}" ]] && sed -i "/^\"\"\"$/i ${{ env.GPU_NAME }}" tables.toml
          [[ ! -z "$CPU_MODEL" ]] && sed -i "/^\"\"\"$/i $CPU_MODEL" tables.toml
          [[ ! -z "$NUM_VCPUS" ]] && sed -i "/^\"\"\"$/i $NUM_VCPUS" tables.toml
          [[ ! -z "$TOTAL_RAM" ]] && sed -i "/^\"\"\"$/i $TOTAL_RAM" tables.toml          
          sed -i "/^\"\"\"$/i Workflow run: $WORKFLOW_URL" tables.toml
          echo "WORKFLOW_URL=$WORKFLOW_URL" | tee -a $GITHUB_ENV
        working-directory: ${{ github.workspace }}
      # Create a `criterion-table` and write in commit comment
      - name: Run `criterion-table`
        run: |
          cat recursive-snark-${{ env.BASE_COMMIT }}.json recursive-snark-${{ github.sha }}.json \
          recursive-snark-supernova-${{ env.BASE_COMMIT }}.json recursive-snark-supernova- ${{ github.sha }}.json \
          compressed-snark-${{ env.BASE_COMMIT }}.json compressed-snark-${{ github.sha }}.json \
          compressed-snark-supernova-${{ env.BASE_COMMIT }}.json compressed-snark-supernova- ${{ github.sha }}.json \
          | criterion-table > BENCHMARKS.md
      - name: Write bench on commit comment
        uses: peter-evans/commit-comment@v3
        with:
          body-path: BENCHMARKS.md
      # Check for a slowdown >= `$ARECIBO_NOISE_THRESHOLD` (fallback is 5%). If so, open an issue but don't block merge
      - name: Check for perf regression
        id: regression-check
        run: |
          REGRESSIONS=$(awk -F'[*x]' '/slower/{print $12}' BENCHMARKS.md)
          echo $regressions

          if [ ! -z "${{ env.ARECIBO_NOISE_THRESHOLD}}" ]; then
            NOISE_THRESHOLD=$(echo "1+${{ env.ARECIBO_NOISE_THRESHOLD }}" | bc)
          else
            NOISE_THRESHOLD=1.05
          fi

          for r in $REGRESSIONS
          do
            if (( $(echo "$r >= $NOISE_THRESHOLD" | bc -l) ))
            then
              exit 1
            fi
          done
          echo "NOISE_THRESHOLD=$NOISE_THRESHOLD" | tee -a $GITHUB_ENV
        continue-on-error: true
      # Not possible to use ${{ github.event.number }} with the `merge_group` trigger
      - name: Get PR number from merge branch
        run: |
          echo "PR_NUMBER=$(echo ${{ github.event.merge_group.head_ref }} | sed -e 's/.*pr-\(.*\)-.*/\1/')" | tee -a $GITHUB_ENV
      - uses: JasonEtco/create-an-issue@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ env.PR_NUMBER }}
          GIT_SHA: ${{ github.sha }}
          WORKFLOW_URL: ${{ env.WORKFLOW_URL }}
          NOISE_THRESHOLD: $${{ env.NOISE_THRESHOLD }}
        with:
          filename: .github/PERF_REGRESSION.md