bench_pr.yml

name: Benchmark Check

on:
  merge_group:
  push:
    branches: [main, "release-*"]
  pull_request:
    branches: [main, "release-*"]
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

# this is needed to gain access via OIDC to the S3 bucket for caching
permissions:
  id-token: write
  contents: read

env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  RISC0_BUILD_LOCKED: 1
  RISC0_RUST_TOOLCHAIN_VERSION: 1.81.0
  RISC0_CPP_TOOLCHAIN_VERSION: 2024.01.05

jobs:
  changes:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      trigger-bench: ${{ steps.filter.outputs.trigger-bench }}
    steps:
      - uses: actions/checkout@v4
      - uses: risc0/paths-filter@4067d885736b84de7c414f582ac45897079b0a78
        id: filter
        with:
          filters: |
            trigger-bench:
              - .github/workflows/bench_pr.yml
              - risc0/**

  bench:
    runs-on: [self-hosted, prod, bench, "${{ matrix.os }}", "${{ matrix.arch }}", "${{ matrix.device }}"]
    if: needs.changes.outputs.trigger-bench == 'true'
    needs: changes
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: Linux
            arch: X64
            feature: prove
            device: cpu
          - os: Linux
            arch: X64
            feature: cuda
            device: nvidia_rtx_3090_ti
          - os: macOS
            arch: ARM64
            feature: prove
            device: apple_m2_pro

    env:
      FEATURE: ${{ matrix.feature }}
      DISABLE_S3: ${{ matrix.device == 'nvidia_rtx_3090_ti' }}
      S3_BENCH_PATH: s3://risc0-ci-cache/public/bench/${{ github.base_ref || github.ref_name }}/fib/${{ matrix.os }}-${{ matrix.device }}

    steps:
      - uses: actions/checkout@v4

      - if: matrix.feature == 'cuda'
        uses: ./.github/actions/cuda
      - uses: ./.github/actions/rustup
      - uses: ./.github/actions/sccache
        with:
          key: ${{ matrix.os }}-${{ matrix.feature }}
          disable_s3: ${{ env.DISABLE_S3 }}

      - run: cargo run --bin rzup -- --verbose install rust $RISC0_RUST_TOOLCHAIN_VERSION
      - run: cargo run --bin rzup -- --verbose install cpp $RISC0_CPP_TOOLCHAIN_VERSION

      - name: Run benchmarks
        run: cargo bench --bench fib -F $FEATURE

      - name: (New Benchmarks / Re-Baseline) Check for Existing Benchmark Data
        run: |
          aws s3 ls $S3_BENCH_PATH/external.json || NOT_EXIST=true
          if [ $NOT_EXIST ]; then
              echo "No ${{ matrix.os }}-${{ matrix.device }} Benchmark data yet... Will be uploaded..." && echo "NEW_BENCHMARK=true" >> $GITHUB_ENV
          else
              echo "Previous Benchmark data for ${{ matrix.os }}-${{ matrix.device }} exists... Continuing..."
          fi

      - name: (New Benchmarks / Re-Baseline) Upload Initial Benchmark Data
        if: env.NEW_BENCHMARK == 'true'
        run: aws s3 cp target/hotbench/fib/benchmark.json $S3_BENCH_PATH/external.json

      - name: Download previous benchmark data
        run: aws s3 cp $S3_BENCH_PATH/external.json ./cache/external.json

      - name: Analyze benchmark results
        uses: risc0/github-action-benchmark@v1.19.3
        with:
          name: "${{ matrix.os }}-${{ matrix.device }}"
          tool: 'customBiggerIsBetter'
          output-file-path: target/hotbench/fib/benchmark.json
          external-data-json-path: ./cache/external.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          alert-threshold: '120%'
          comment-on-alert: true
          fail-on-alert: true
          summary-always: true

      - name: Save current benchmark data
        if: github.event_name == 'push'
        run: aws s3 cp ./cache/external.json $S3_BENCH_PATH/external.json

  # see: https://github.com/orgs/community/discussions/26822
  bench-status-check:
    if: always()
    needs:
      - changes
      - bench
    runs-on: ubuntu-latest
    steps:
      - name: Check all job status
        # see https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
        # see https://stackoverflow.com/a/67532120/4907315
        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
        run: exit 1