Add other benchmarks and refactor

.github/PERF_REGRESSION.md

@@ -0,0 +1,7 @@
+---
+title: ":rotating_light: Performance regression in #{{ env.PR_NUMBER }}"
+labels: P-Performance, automated issue
+---
+Regression >= 5% found during merge of: #{{ env.PR_NUMBER }}
+Commit: {{ env.GIT_SHA }}
+Triggered by: {{ env.WORKFLOW_URL }}

.github/tables.toml

@@ -0,0 +1,6 @@
+[table_comments]
+
+[top_comments]
+Overview = """
+This benchmark report shows the Arecibo GPU benchmarks.
+"""

.github/workflows/gpu-bench.yml

@@ -91,13 +91,15 @@ jobs:
           NUM_VCPUS=$(nproc --all)
           # Get total RAM in GB
           TOTAL_RAM=$(grep MemTotal /proc/meminfo | awk '{$2=$2/(1024^2); print int($2), "GB RAM";}')
+          WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           
           # Use conditionals to ensure that only non-empty variables are inserted
           [[ ! -z "${{ env.GPU_NAME }}" ]] && sed -i "/^\"\"\"$/i ${{ env.GPU_NAME }}" tables.toml
           [[ ! -z "$CPU_MODEL" ]] && sed -i "/^\"\"\"$/i $CPU_MODEL" tables.toml
           [[ ! -z "$NUM_VCPUS" ]] && sed -i "/^\"\"\"$/i $NUM_VCPUS" tables.toml
           [[ ! -z "$TOTAL_RAM" ]] && sed -i "/^\"\"\"$/i $TOTAL_RAM" tables.toml          
-          sed -i "/^\"\"\"$/i Workflow run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" tables.toml
+          sed -i "/^\"\"\"$/i Workflow run: $WORKFLOW_URL" tables.toml
+          echo "WORKFLOW_URL=$WORKFLOW_URL" | tee -a $GITHUB_ENV
         working-directory: ${{ github.workspace }}
       # Create a `criterion-table` and write in commit comment
       - name: Run `criterion-table`
@@ -126,21 +128,14 @@ jobs:
       - name: Get PR number from merge branch
         run: |
           echo "PR_NUMBER=$(echo ${{ github.event.merge_group.head_ref }} | sed -e 's/.*pr-\(.*\)-.*/\1/')" | tee -a $GITHUB_ENV
-      - name: Create file for issue
-        if: steps.regression-check.outcome == 'failure'
-        run: |
-          printf '%s\n' "Regression >= 10% found during merge for PR #${{ env.PR_NUMBER }}
-          Commit: ${{ github.sha }}
-          Workflow run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" > ./_body.md
-      - name: Open issue on regression
-        if: steps.regression-check.outcome == 'failure'
-        uses: peter-evans/create-issue-from-file@v4
+      - uses: JasonEtco/create-an-issue@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ env.PR_NUMBER }}
+          GIT_SHA: ${{ github.sha }}
+          WORKFLOW_URL: ${{ env.WORKFLOW_URL }}
         with:
-          title: ':rotating_light: Performance regression detected for PR #${{ env.PR_NUMBER }}'
-          content-filepath: ./_body.md
-          labels: |
-            P-Performance
-            automated issue
+          filename: .github/PERF_REGRESSION.md
       - name: Remove old dev bench
         run: |
           rm ${{ env.BASE_COMMIT }}.json

benches/common/mod.rs

@@ -0,0 +1,38 @@
+use anyhow::anyhow;
+use criterion::BenchmarkId;
+
+// TODO: Why Copy and &'static str over String?
+#[derive(Clone, Debug, Copy)]
+pub(crate) struct BenchParams {
+  pub step_size: usize,
+  // TODO: Is this useful?
+  //date: &'static str,
+  pub sha: &'static str,
+}
+impl BenchParams {
+  pub(crate) fn bench_id(&self, name: &str) -> BenchmarkId {
+    let output_type = bench_parameters_env().unwrap_or("stdout".into());
+    match output_type.as_ref() {
+      "pr-comment" => todo!(),
+      "commit-comment" => BenchmarkId::new(
+        format!("ref={}", self.sha),
+        format!("{}-StepCircuitSize-{}", name, self.step_size),
+      ),
+      // TODO: refine "gh-pages" and "stdout"
+      _ => todo!(),
+    }
+  }
+}
+
+fn bench_parameters_env() -> anyhow::Result<String> {
+  std::env::var("ARECIBO_BENCH_OUTPUT").map_err(|e| anyhow!("Bench output env var isn't set: {e}"))
+}
+
+pub(crate) fn noise_threshold_env() -> anyhow::Result<f64> {
+  std::env::var("ARECIBO_BENCH_NOISE_THRESHOLD")
+    .map_err(|e| anyhow!("Noise threshold env var isn't set: {e}"))
+    .and_then(|nt| {
+      nt.parse::<f64>()
+        .map_err(|e| anyhow!("Failed to parse noise threshold: {e}"))
+    })
+}

benches/compressed-snark-supernova.rs

@@ -11,14 +11,17 @@ use criterion::{measurement::WallTime, *};
 use ff::PrimeField;
 use std::time::Duration;
 
+mod common;
+use common::{noise_threshold_env, BenchParams};
+
 type E1 = arecibo::provider::PallasEngine;
 type E2 = arecibo::provider::VestaEngine;
 type EE1 = arecibo::provider::ipa_pc::EvaluationEngine<E1>;
 type EE2 = arecibo::provider::ipa_pc::EvaluationEngine<E2>;
-// SNARKs without computation commitmnets
+// SNARKs without computation commitments
 type S1 = arecibo::spartan::batched::BatchedRelaxedR1CSSNARK<E1, EE1>;
 type S2 = arecibo::spartan::snark::RelaxedR1CSSNARK<E2, EE2>;
-// SNARKs with computation commitmnets
+// SNARKs with computation commitments
 type SS1 = arecibo::spartan::batched_ppsnark::BatchedRelaxedR1CSSNARK<E1, EE1>;
 type SS2 = arecibo::spartan::ppsnark::RelaxedR1CSSNARK<E2, EE2>;
 
@@ -162,8 +165,13 @@ fn bench_compressed_snark_internal_with_arity<
 
   let (prover_key, verifier_key) = CompressedSNARK::<_, _, _, _, S1, S2>::setup(&pp).unwrap();
 
+  let bench_params = BenchParams {
+    step_size: num_cons,
+    sha: env!("VERGEN_GIT_SHA"),
+  };
+
   // Benchmark the prove time
-  group.bench_function("Prove", |b| {
+  group.bench_function(bench_params.bench_id("Prove"), |b| {
     b.iter(|| {
       assert!(CompressedSNARK::<_, _, _, _, S1, S2>::prove(
         black_box(&pp),
@@ -180,7 +188,7 @@ fn bench_compressed_snark_internal_with_arity<
   let compressed_snark = res.unwrap();
 
   // Benchmark the verification time
-  group.bench_function("Verify", |b| {
+  group.bench_function(bench_params.bench_id("Verify"), |b| {
     b.iter(|| {
       assert!(black_box(&compressed_snark)
         .verify(
@@ -211,10 +219,9 @@ fn bench_one_augmented_circuit_compressed_snark(c: &mut Criterion) {
     // number of constraints in the step circuit
     let num_cons = num_cons_in_augmented_circuit - NUM_CONS_VERIFIER_CIRCUIT_PRIMARY;
 
-    let mut group = c.benchmark_group(format!(
-      "CompressedSNARKSuperNova-1circuit-StepCircuitSize-{num_cons}"
-    ));
+    let mut group = c.benchmark_group("CompressedSNARKSuperNova-1circuit");
     group.sample_size(NUM_SAMPLES);
+    group.noise_threshold(noise_threshold_env().unwrap_or(0.05));
 
     bench_compressed_snark_internal_with_arity::<S1, S2>(&mut group, 1, num_cons);
 
@@ -239,10 +246,9 @@ fn bench_two_augmented_circuit_compressed_snark(c: &mut Criterion) {
     // number of constraints in the step circuit
     let num_cons = num_cons_in_augmented_circuit - NUM_CONS_VERIFIER_CIRCUIT_PRIMARY;
 
-    let mut group = c.benchmark_group(format!(
-      "CompressedSNARKSuperNova-2circuit-StepCircuitSize-{num_cons}"
-    ));
+    let mut group = c.benchmark_group("CompressedSNARKSuperNova-2circuit");
     group.sample_size(NUM_SAMPLES);
+    group.noise_threshold(noise_threshold_env().unwrap_or(0.05));
 
     bench_compressed_snark_internal_with_arity::<S1, S2>(&mut group, 2, num_cons);
 
@@ -267,10 +273,9 @@ fn bench_two_augmented_circuit_compressed_snark_with_computational_commitments(c
     // number of constraints in the step circuit
     let num_cons = num_cons_in_augmented_circuit - NUM_CONS_VERIFIER_CIRCUIT_PRIMARY;
 
-    let mut group = c.benchmark_group(format!(
-      "CompressedSNARKSuperNova-Commitments-2circuit-StepCircuitSize-{num_cons}"
-    ));
+    let mut group = c.benchmark_group("CompressedSNARKSuperNova-Commitments-2circuit");
     group.sample_size(NUM_SAMPLES);
+    group.noise_threshold(noise_threshold_env().unwrap_or(0.05));
 
     bench_compressed_snark_internal_with_arity::<SS1, SS2>(&mut group, 2, num_cons);
 

benches/compressed-snark.rs

@@ -14,6 +14,9 @@ use criterion::{measurement::WallTime, *};
 use ff::PrimeField;
 use std::time::Duration;
 
+mod common;
+use common::{noise_threshold_env, BenchParams};
+
 type E1 = PallasEngine;
 type E2 = VestaEngine;
 type EE1 = arecibo::provider::ipa_pc::EvaluationEngine<E1>;
@@ -101,8 +104,13 @@ fn bench_compressed_snark_internal<S1: RelaxedR1CSSNARKTrait<E1>, S2: RelaxedR1C
     assert!(res.is_ok());
   }
 
+  let bench_params = BenchParams {
+    step_size: num_cons,
+    sha: env!("VERGEN_GIT_SHA"),
+  };
+
   // Bench time to produce a compressed SNARK
-  group.bench_function("Prove", |b| {
+  group.bench_function(bench_params.bench_id("Prove"), |b| {
     b.iter(|| {
       assert!(CompressedSNARK::<_, _, _, _, S1, S2>::prove(
         black_box(&pp),
@@ -117,7 +125,7 @@ fn bench_compressed_snark_internal<S1: RelaxedR1CSSNARKTrait<E1>, S2: RelaxedR1C
   let compressed_snark = res.unwrap();
 
   // Benchmark the verification time
-  group.bench_function("Verify", |b| {
+  group.bench_function(bench_params.bench_id("Verify"), |b| {
     b.iter(|| {
       assert!(black_box(&compressed_snark)
         .verify(
@@ -148,8 +156,9 @@ fn bench_compressed_snark(c: &mut Criterion) {
     // number of constraints in the step circuit
     let num_cons = num_cons_in_augmented_circuit - NUM_CONS_VERIFIER_CIRCUIT_PRIMARY;
 
-    let mut group = c.benchmark_group(format!("CompressedSNARK-StepCircuitSize-{num_cons}"));
+    let mut group = c.benchmark_group("CompressedSNARK");
     group.sample_size(NUM_SAMPLES);
+    group.noise_threshold(noise_threshold_env().unwrap_or(0.05));
 
     bench_compressed_snark_internal::<S1, S2>(&mut group, num_cons);
 
@@ -172,12 +181,10 @@ fn bench_compressed_snark_with_computational_commitments(c: &mut Criterion) {
     // number of constraints in the step circuit
     let num_cons = num_cons_in_augmented_circuit - NUM_CONS_VERIFIER_CIRCUIT_PRIMARY;
 
-    let mut group = c.benchmark_group(format!(
-      "CompressedSNARK-Commitments-StepCircuitSize-{num_cons}"
-    ));
-    group
-      .sampling_mode(SamplingMode::Flat)
-      .sample_size(NUM_SAMPLES);
+    let mut group = c.benchmark_group("CompressedSNARK-Commitments");
+    group.sampling_mode(SamplingMode::Flat);
+    group.sample_size(NUM_SAMPLES);
+    group.noise_threshold(noise_threshold_env().unwrap_or(0.05));
 
     bench_compressed_snark_internal::<SS1, SS2>(&mut group, num_cons);
 

benches/justfile

@@ -32,5 +32,5 @@ gpu-bench +benches: gpu-env
 gpu-bench-ci +benches:
   #!/bin/sh
   for bench in {{benches}}; do
-    cargo criterion --bench $bench --features "cuda" --message-format=json > {{commit}}.json
+    cargo criterion --bench $bench --features "cuda" --message-format=json > "$bench-{{commit}}".json
   done

benches/recursive-snark-supernova.rs

@@ -12,6 +12,9 @@ use criterion::{measurement::WallTime, *};
 use ff::PrimeField;
 use std::time::Duration;
 
+mod common;
+use common::{noise_threshold_env, BenchParams};
+
 // To run these benchmarks, first download `criterion` with `cargo install cargo-criterion`.
 // Then `cargo criterion --bench recursive-snark-supernova`. The results are located in `target/criterion/data/<name-of-benchmark>`.
 // For flamegraphs, run `cargo criterion --bench recursive-snark-supernova --features flamegraph -- --profile-time <secs>`.
@@ -152,8 +155,13 @@ fn bench_recursive_snark_internal_with_arity(
   assert!(recursive_snark_option.is_some());
   let recursive_snark = recursive_snark_option.unwrap();
 
+  let bench_params = BenchParams {
+    step_size: num_cons,
+    sha: env!("VERGEN_GIT_SHA"),
+  };
+
   // Benchmark the prove time
-  group.bench_function("Prove", |b| {
+  group.bench_function(bench_params.bench_id("Prove"), |b| {
     b.iter(|| {
       // produce a recursive SNARK for a step of the recursion
       assert!(black_box(&mut recursive_snark.clone())
@@ -167,7 +175,7 @@ fn bench_recursive_snark_internal_with_arity(
   });
 
   // Benchmark the verification time
-  group.bench_function("Verify", |b| {
+  group.bench_function(bench_params.bench_id("Verify"), |b| {
     b.iter(|| {
       assert!(black_box(&mut recursive_snark.clone())
         .verify(
@@ -197,10 +205,9 @@ fn bench_one_augmented_circuit_recursive_snark(c: &mut Criterion) {
     // number of constraints in the step circuit
     let num_cons = num_cons_in_augmented_circuit - NUM_CONS_VERIFIER_CIRCUIT_PRIMARY;
 
-    let mut group = c.benchmark_group(format!(
-      "RecursiveSNARKSuperNova-1circuit-StepCircuitSize-{num_cons}"
-    ));
+    let mut group = c.benchmark_group("RecursiveSNARKSuperNova-1circuit");
     group.sample_size(NUM_SAMPLES);
+    group.noise_threshold(noise_threshold_env().unwrap_or(0.05));
 
     bench_recursive_snark_internal_with_arity(&mut group, 1, num_cons);
     group.finish();
@@ -224,10 +231,9 @@ fn bench_two_augmented_circuit_recursive_snark(c: &mut Criterion) {
     // number of constraints in the step circuit
     let num_cons = num_cons_in_augmented_circuit - NUM_CONS_VERIFIER_CIRCUIT_PRIMARY;
 
-    let mut group = c.benchmark_group(format!(
-      "RecursiveSNARKSuperNova-2circuit-StepCircuitSize-{num_cons}"
-    ));
+    let mut group = c.benchmark_group("RecursiveSNARKSuperNova-2circuit");
     group.sample_size(NUM_SAMPLES);
+    group.noise_threshold(noise_threshold_env().unwrap_or(0.05));
 
     bench_recursive_snark_internal_with_arity(&mut group, 2, num_cons);
     group.finish();