diff --git a/.github/PERF_REGRESSION.md b/.github/PERF_REGRESSION.md index 9f9b04115..090be25bd 100644 --- a/.github/PERF_REGRESSION.md +++ b/.github/PERF_REGRESSION.md @@ -2,6 +2,6 @@ title: ":rotating_light: Performance regression in #{{ env.PR_NUMBER }}" labels: P-Performance, automated issue --- -Regression >= 5% found during merge of: #{{ env.PR_NUMBER }} +Regression >= {{ env.NOISE_THRESHOLD }} found during merge of: #{{ env.PR_NUMBER }} Commit: {{ env.GIT_SHA }} Triggered by: {{ env.WORKFLOW_URL }} \ No newline at end of file diff --git a/.github/workflows/gpu-bench.yml b/.github/workflows/gpu-bench.yml index ec0f9ca02..db826474f 100644 --- a/.github/workflows/gpu-bench.yml +++ b/.github/workflows/gpu-bench.yml @@ -43,44 +43,25 @@ jobs: GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1) echo "GPU_ID=$(echo $GPU_NAME | awk '{ print $NF }')" | tee -a $GITHUB_ENV echo "GPU_NAME=$GPU_NAME" | tee -a $GITHUB_ENV - # Checkout gh-pages to check for cached bench result - - name: Checkout gh-pages - uses: actions/checkout@v4 - with: - ref: gh-pages - path: gh-pages - - name: Check for cached bench result - id: cached-bench - run: | - if [ -f "${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json" ] - then - echo "cached=true" | tee -a $GITHUB_OUTPUT - cp ${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json ../${{ env.BASE_COMMIT }}.json - else - echo "cached=false" | tee -a $GITHUB_OUTPUT - fi - working-directory: ${{ github.workspace }}/gh-pages # Checkout base branch for comparative bench - uses: actions/checkout@v4 - if: steps.cached-bench.outputs.cached == 'false' with: ref: dev path: dev # Copy the script so the base can bench with the same parameters - name: Run GPU bench on base branch - if: steps.cached-bench.outputs.cached == 'false' run: | - # Copy justfile & env to dev, overwriting existing config with that of PR branch - cp ../benches/justfile ../benches/bench.env . + # Copy justfile to dev, overwriting existing config with that of PR branch + cp ../benches/justfile . # Run benchmark just gpu-bench-ci recursive-snark recursive-snark-supernova compressed-snark compressed-snark-supernova # Copy bench output to PR branch - cp ${{ env.BASE_COMMIT }}.json .. + cp *-${{ env.BASE_COMMIT }}.json .. working-directory: ${{ github.workspace }}/dev - name: Run GPU bench on PR branch run: | just gpu-bench-ci recursive-snark recursive-snark-supernova compressed-snark compressed-snark-supernova - cp ${{ github.sha }}.json .. + cp *-${{ github.sha }}.json .. working-directory: ${{ github.workspace }}/benches - name: copy the benchmark template and prepare it with data run: | @@ -103,26 +84,37 @@ jobs: working-directory: ${{ github.workspace }} # Create a `criterion-table` and write in commit comment - name: Run `criterion-table` - run: cat ${{ env.BASE_COMMIT }}.json ${{ github.sha }}.json | criterion-table > BENCHMARKS.md + run: | + cat recursive-snark-${{ env.BASE_COMMIT }}.json recursive-snark-${{ github.sha }}.json \ + recursive-snark-supernova-${{ env.BASE_COMMIT }}.json recursive-snark-supernova- ${{ github.sha }}.json \ + compressed-snark-${{ env.BASE_COMMIT }}.json compressed-snark-${{ github.sha }}.json \ + compressed-snark-supernova-${{ env.BASE_COMMIT }}.json compressed-snark-supernova- ${{ github.sha }}.json \ + | criterion-table > BENCHMARKS.md - name: Write bench on commit comment uses: peter-evans/commit-comment@v3 with: body-path: BENCHMARKS.md - # Check for a slowdown >= 10%. If so, open an issue but don't block merge + # Check for a slowdown >= `$ARECIBO_NOISE_THRESHOLD` (fallback is 5%). If so, open an issue but don't block merge - name: Check for perf regression id: regression-check run: | - regressions=$(awk -F'[*x]' '/slower/{print $12}' BENCHMARKS.md) - + REGRESSIONS=$(awk -F'[*x]' '/slower/{print $12}' BENCHMARKS.md) echo $regressions - for r in $regressions + if [ ! -z "${{ env.ARECIBO_NOISE_THRESHOLD}}" ]; then + NOISE_THRESHOLD=$(echo "1+${{ env.ARECIBO_NOISE_THRESHOLD }}" | bc) + else + NOISE_THRESHOLD=1.05 + fi + + for r in $REGRESSIONS do - if (( $(echo "$r >= 1.10" | bc -l) )) + if (( $(echo "$r >= $NOISE_THRESHOLD" | bc -l) )) then exit 1 fi done + echo "NOISE_THRESHOLD=$NOISE_THRESHOLD" | tee -a $GITHUB_ENV continue-on-error: true # Not possible to use ${{ github.event.number }} with the `merge_group` trigger - name: Get PR number from merge branch @@ -134,17 +126,6 @@ jobs: PR_NUMBER: ${{ env.PR_NUMBER }} GIT_SHA: ${{ github.sha }} WORKFLOW_URL: ${{ env.WORKFLOW_URL }} + NOISE_THRESHOLD: $${{ env.NOISE_THRESHOLD }} with: - filename: .github/PERF_REGRESSION.md - - name: Remove old dev bench - run: | - rm ${{ env.BASE_COMMIT }}.json - mv ${{ github.sha }}.json ${{ github.sha }}-${{ env.GPU_ID }}.json - working-directory: ${{ github.workspace }} - - name: Commit bench result to `gh-pages` branch if no regression - if: steps.regression-check.outcome != 'failure' - uses: stefanzweifel/git-auto-commit-action@v5 - with: - branch: gh-pages - commit_message: '[automated] GPU Benchmark from PR #${{ env.PR_NUMBER }}' - file_pattern: '${{ github.sha }}-${{ env.GPU_ID }}.json' \ No newline at end of file + filename: .github/PERF_REGRESSION.md \ No newline at end of file diff --git a/benches/common/mod.rs b/benches/common/mod.rs index 43afd0959..43ec5ac13 100644 --- a/benches/common/mod.rs +++ b/benches/common/mod.rs @@ -18,7 +18,13 @@ impl BenchParams { format!("{}-StepCircuitSize-{}", name, self.step_size), ), // TODO: refine "gh-pages" - _ => BenchmarkId::new(name, format!("StepCircuitSize-{}-{}", self.sha, self.date)), + _ => BenchmarkId::new( + name, + format!( + "StepCircuitSize-{}-{}-{}", + self.step_size, self.sha, self.date + ), + ), } } } diff --git a/benches/compressed-snark-supernova.rs b/benches/compressed-snark-supernova.rs index 7d9dd83ef..44a4607b6 100644 --- a/benches/compressed-snark-supernova.rs +++ b/benches/compressed-snark-supernova.rs @@ -167,6 +167,7 @@ fn bench_compressed_snark_internal_with_arity< let bench_params = BenchParams { step_size: num_cons, + date: env!("VERGEN_GIT_COMMIT_DATE"), sha: env!("VERGEN_GIT_SHA"), }; diff --git a/benches/compressed-snark.rs b/benches/compressed-snark.rs index f7751c45b..c3eab2a17 100644 --- a/benches/compressed-snark.rs +++ b/benches/compressed-snark.rs @@ -106,6 +106,7 @@ fn bench_compressed_snark_internal, S2: RelaxedR1C let bench_params = BenchParams { step_size: num_cons, + date: env!("VERGEN_GIT_COMMIT_DATE"), sha: env!("VERGEN_GIT_SHA"), }; diff --git a/benches/recursive-snark-supernova.rs b/benches/recursive-snark-supernova.rs index 7050a0019..23b0d4c2b 100644 --- a/benches/recursive-snark-supernova.rs +++ b/benches/recursive-snark-supernova.rs @@ -157,6 +157,7 @@ fn bench_recursive_snark_internal_with_arity( let bench_params = BenchParams { step_size: num_cons, + date: env!("VERGEN_GIT_COMMIT_DATE"), sha: env!("VERGEN_GIT_SHA"), }; diff --git a/benches/recursive-snark.rs b/benches/recursive-snark.rs index 40d5c6b8f..08d3ffac4 100644 --- a/benches/recursive-snark.rs +++ b/benches/recursive-snark.rs @@ -110,6 +110,7 @@ fn bench_recursive_snark(c: &mut Criterion) { let bench_params = BenchParams { step_size: num_cons, + date: env!("VERGEN_GIT_COMMIT_DATE"), sha: env!("VERGEN_GIT_SHA"), };