Remove caching for now and test action on pull_request

.github/PERF_REGRESSION.md

@@ -2,6 +2,6 @@
 title: ":rotating_light: Performance regression in #{{ env.PR_NUMBER }}"
 labels: P-Performance, automated issue
 ---
-Regression >= 5% found during merge of: #{{ env.PR_NUMBER }}
+Regression >= {{ env.NOISE_THRESHOLD }} found during merge of: #{{ env.PR_NUMBER }}
 Commit: {{ env.GIT_SHA }}
 Triggered by: {{ env.WORKFLOW_URL }}

.github/workflows/gpu-bench.yml

@@ -43,44 +43,25 @@ jobs:
           GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1)
           echo "GPU_ID=$(echo $GPU_NAME | awk '{ print $NF }')" | tee -a $GITHUB_ENV
           echo "GPU_NAME=$GPU_NAME" | tee -a $GITHUB_ENV
-      # Checkout gh-pages to check for cached bench result
-      - name: Checkout gh-pages
-        uses: actions/checkout@v4
-        with:
-          ref: gh-pages
-          path: gh-pages
-      - name: Check for cached bench result
-        id: cached-bench
-        run: |
-          if [ -f "${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json" ]
-          then
-            echo "cached=true" | tee -a $GITHUB_OUTPUT
-            cp ${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json ../${{ env.BASE_COMMIT }}.json
-          else
-            echo "cached=false" | tee -a $GITHUB_OUTPUT
-          fi
-        working-directory: ${{ github.workspace }}/gh-pages
       # Checkout base branch for comparative bench
       - uses: actions/checkout@v4
-        if: steps.cached-bench.outputs.cached == 'false'
         with:
           ref: dev
           path: dev
       # Copy the script so the base can bench with the same parameters
       - name: Run GPU bench on base branch
-        if: steps.cached-bench.outputs.cached == 'false'
         run: |
-          # Copy justfile & env to dev, overwriting existing config with that of PR branch
-          cp ../benches/justfile ../benches/bench.env .
+          # Copy justfile to dev, overwriting existing config with that of PR branch
+          cp ../benches/justfile .
           # Run benchmark
           just gpu-bench-ci recursive-snark recursive-snark-supernova compressed-snark compressed-snark-supernova
           # Copy bench output to PR branch
-          cp ${{ env.BASE_COMMIT }}.json ..
+          cp *-${{ env.BASE_COMMIT }}.json ..
         working-directory: ${{ github.workspace }}/dev
       - name: Run GPU bench on PR branch
         run: |
           just gpu-bench-ci recursive-snark recursive-snark-supernova compressed-snark compressed-snark-supernova
-          cp ${{ github.sha }}.json ..
+          cp *-${{ github.sha }}.json ..
         working-directory: ${{ github.workspace }}/benches
       - name: copy the benchmark template and prepare it with data
         run: |
@@ -103,26 +84,37 @@ jobs:
         working-directory: ${{ github.workspace }}
       # Create a `criterion-table` and write in commit comment
       - name: Run `criterion-table`
-        run: cat ${{ env.BASE_COMMIT }}.json ${{ github.sha }}.json | criterion-table > BENCHMARKS.md
+        run: |
+          cat recursive-snark-${{ env.BASE_COMMIT }}.json recursive-snark-${{ github.sha }}.json \
+          recursive-snark-supernova-${{ env.BASE_COMMIT }}.json recursive-snark-supernova- ${{ github.sha }}.json \
+          compressed-snark-${{ env.BASE_COMMIT }}.json compressed-snark-${{ github.sha }}.json \
+          compressed-snark-supernova-${{ env.BASE_COMMIT }}.json compressed-snark-supernova- ${{ github.sha }}.json \
+          | criterion-table > BENCHMARKS.md
       - name: Write bench on commit comment
         uses: peter-evans/commit-comment@v3
         with:
           body-path: BENCHMARKS.md
-      # Check for a slowdown >= 10%. If so, open an issue but don't block merge
+      # Check for a slowdown >= `$ARECIBO_NOISE_THRESHOLD` (fallback is 5%). If so, open an issue but don't block merge
       - name: Check for perf regression
         id: regression-check
         run: |
-          regressions=$(awk -F'[*x]' '/slower/{print $12}' BENCHMARKS.md)
-
+          REGRESSIONS=$(awk -F'[*x]' '/slower/{print $12}' BENCHMARKS.md)
           echo $regressions
 
-          for r in $regressions
+          if [ ! -z "${{ env.ARECIBO_NOISE_THRESHOLD}}" ]; then
+            NOISE_THRESHOLD=$(echo "1+${{ env.ARECIBO_NOISE_THRESHOLD }}" | bc)
+          else
+            NOISE_THRESHOLD=1.05
+          fi
+
+          for r in $REGRESSIONS
           do
-            if (( $(echo "$r >= 1.10" | bc -l) ))
+            if (( $(echo "$r >= $NOISE_THRESHOLD" | bc -l) ))
             then
               exit 1
             fi
           done
+          echo "NOISE_THRESHOLD=$NOISE_THRESHOLD" | tee -a $GITHUB_ENV
         continue-on-error: true
       # Not possible to use ${{ github.event.number }} with the `merge_group` trigger
       - name: Get PR number from merge branch
@@ -134,17 +126,6 @@ jobs:
           PR_NUMBER: ${{ env.PR_NUMBER }}
           GIT_SHA: ${{ github.sha }}
           WORKFLOW_URL: ${{ env.WORKFLOW_URL }}
+          NOISE_THRESHOLD: $${{ env.NOISE_THRESHOLD }}
         with:
-          filename: .github/PERF_REGRESSION.md
-      - name: Remove old dev bench
-        run: |
-          rm ${{ env.BASE_COMMIT }}.json
-          mv ${{ github.sha }}.json ${{ github.sha }}-${{ env.GPU_ID }}.json
-        working-directory: ${{ github.workspace }}
-      - name: Commit bench result to `gh-pages` branch if no regression
-        if: steps.regression-check.outcome != 'failure'
-        uses: stefanzweifel/git-auto-commit-action@v5
-        with:
-          branch: gh-pages
-          commit_message: '[automated] GPU Benchmark from PR #${{ env.PR_NUMBER }}'
-          file_pattern: '${{ github.sha }}-${{ env.GPU_ID }}.json'
+          filename: .github/PERF_REGRESSION.md

benches/common/mod.rs

@@ -18,7 +18,13 @@ impl BenchParams {
         format!("{}-StepCircuitSize-{}", name, self.step_size),
       ),
       // TODO: refine "gh-pages"
-      _ => BenchmarkId::new(name, format!("StepCircuitSize-{}-{}", self.sha, self.date)),
+      _ => BenchmarkId::new(
+        name,
+        format!(
+          "StepCircuitSize-{}-{}-{}",
+          self.step_size, self.sha, self.date
+        ),
+      ),
     }
   }
 }

benches/compressed-snark-supernova.rs

@@ -167,6 +167,7 @@ fn bench_compressed_snark_internal_with_arity<
 
   let bench_params = BenchParams {
     step_size: num_cons,
+    date: env!("VERGEN_GIT_COMMIT_DATE"),
     sha: env!("VERGEN_GIT_SHA"),
   };
 

benches/compressed-snark.rs

@@ -106,6 +106,7 @@ fn bench_compressed_snark_internal<S1: RelaxedR1CSSNARKTrait<E1>, S2: RelaxedR1C
 
   let bench_params = BenchParams {
     step_size: num_cons,
+    date: env!("VERGEN_GIT_COMMIT_DATE"),
     sha: env!("VERGEN_GIT_SHA"),
   };
 

benches/recursive-snark-supernova.rs

@@ -157,6 +157,7 @@ fn bench_recursive_snark_internal_with_arity(
 
   let bench_params = BenchParams {
     step_size: num_cons,
+    date: env!("VERGEN_GIT_COMMIT_DATE"),
     sha: env!("VERGEN_GIT_SHA"),
   };
 

benches/recursive-snark.rs

@@ -110,6 +110,7 @@ fn bench_recursive_snark(c: &mut Criterion) {
 
     let bench_params = BenchParams {
       step_size: num_cons,
+      date: env!("VERGEN_GIT_COMMIT_DATE"),
       sha: env!("VERGEN_GIT_SHA"),
     };