add loop variations and more gpu machines (#1)

* add g6.8xlarge to xlarge

* wip

* add loop variations

.github/workflows/adhoc-matrix.yml

@@ -4,32 +4,32 @@ on:
   workflow_dispatch:
     inputs:
       provers:
-        description: 'Provers to use (comma-separated)'
+        description: "Provers to use (comma-separated)"
         required: false
         type: string
-        default: 'sp1'
+        default: "sp1"
       programs:
-        description: 'Programs to benchmark (comma-separated)'
+        description: "Programs to benchmark (comma-separated)"
         required: false
         type: string
-        default: 'loop,fibonacci,tendermint,reth1,reth2'
+        default: "loop,fibonacci,tendermint,reth1,reth2"
       filename:
-        description: 'Filename for the benchmark'
+        description: "Filename for the benchmark"
         required: false
         type: string
-        default: 'benchmark'
+        default: "benchmark"
       trials:
-        description: 'Number of trials to run'
+        description: "Number of trials to run"
         required: false
         type: string
-        default: '1'
+        default: "1"
       sp1_ref:
-        description: 'SP1 reference (commit hash or branch name)'
+        description: "SP1 reference (commit hash or branch name)"
         required: false
         type: string
-        default: 'dev'
+        default: "dev"
       additional_params:
-        description: 'Additional parameters as JSON'
+        description: "Additional parameters as JSON"
         required: false
         type: string
         default: '{"hashfns":"poseidon","shard_sizes":"22"}'
@@ -42,15 +42,27 @@ jobs:
           - instance_type: g6.16xlarge
             enable_gpu: true
             ami_id: ami-079a6a210557ef0e4
+          - instance_type: g6.8xlarge
+            enable_gpu: true
+            ami_id: ami-079a6a210557ef0e4
+          - instance_type: g6.4xlarge
+            enable_gpu: true
+            ami_id: ami-079a6a210557ef0e4
+          - instance_type: g6.2xlarge
+            enable_gpu: true
+            ami_id: ami-079a6a210557ef0e4
+          - instance_type: g6.xlarge
+            enable_gpu: true
+            ami_id: ami-079a6a210557ef0e4
           - instance_type: r7i.16xlarge
             enable_gpu: false
             ami_id: ami-079a6a210557ef0e4
-      fail-fast: false  # This prevents the entire matrix from failing if one job fails
-    
+      fail-fast: false # This prevents the entire matrix from failing if one job fails
+
     name: Run on ${{ matrix.instance_type }}
     runs-on: ubuntu-latest
-    continue-on-error: true  # This allows the workflow to continue even if this job fails
-    
+    continue-on-error: true # This allows the workflow to continue even if this job fails
+
     steps:
       - name: Echo Workflow Inputs
         run: |
@@ -60,7 +72,7 @@ jobs:
           echo "Trials: ${{ inputs.trials }}"
           echo "SP1 Reference: ${{ inputs.sp1_ref }}"
           echo "Additional Parameters: ${{ inputs.additional_params }}"
-          
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -95,9 +107,9 @@ jobs:
             const maxAttempts = 30;
             const pollInterval = 10000; // 10 seconds
             let triggeredRunId = null;
-            
+
             console.log('Triggering benchmark workflow');
-            
+
             try {
               await github.rest.actions.createWorkflowDispatch({
                 owner: context.repo.owner,
@@ -121,7 +133,7 @@ jobs:
               console.error(`Failed to trigger workflow: ${error.message}`);
               return;
             }
-            
+
             console.log('Polling for the triggered run');
             for (let attempt = 1; attempt <= maxAttempts; attempt++) {
               await new Promise(resolve => setTimeout(resolve, pollInterval));
@@ -178,12 +190,12 @@ jobs:
                 console.log(`Error while polling: ${error.message}`);
               }
             }
-            
+
             if (!triggeredRunId) {
               console.error('Failed to find the triggered workflow run with matching job after maximum attempts');
               return;
             }
-            
+
             core.setOutput('triggered_run_id', triggeredRunId);
             core.exportVariable('TRIGGERED_RUN_ID', triggeredRunId);
             console.log(`Triggered run ID: ${triggeredRunId}`);
@@ -198,16 +210,16 @@ jobs:
               console.error('No triggered run ID found');
               return;
             }
-            
+
             // 10 hours in milliseconds
             const maxWaitTime = 36000000;
             // 3 minutes in milliseconds
             const checkInterval = 180000;
             const startTime = Date.now();
-            
+
             const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${triggeredRunId}`;
             console.log(`Waiting for benchmark job to complete. Job URL: ${runUrl}`);
-            
+
             while (true) {
               const run = await github.rest.actions.getWorkflowRun({
                 owner: context.repo.owner,
@@ -242,4 +254,4 @@ jobs:
           ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
 
     outputs:
-      triggered_run_id: ${{ steps.run-benchmarks.outputs.triggered_run_id }}
+      triggered_run_id: ${{ steps.run-benchmarks.outputs.triggered_run_id }}

README.md

@@ -28,7 +28,7 @@ This workflow sets up the benchmarking environment and triggers individual bench
 ##### Inputs
 
 - `provers`: Provers to use (comma-separated, default: 'sp1')
-- `programs`: Programs to benchmark (comma-separated, default: 'loop,fibonacci,tendermint,reth1,reth2')
+- `programs`: Programs to benchmark (comma-separated, default: 'loop10k,loop100k,loop1m,loop3m,loop10m,loop30m,loop100m,fibonacci,tendermint,reth1,reth2')
 - `filename`: Filename for the benchmark (default: 'benchmark')
 - `trials`: Number of trials to run (default: '1')
 - `sp1_ref`: SP1 reference (commit hash or branch name, default: 'dev')
@@ -37,6 +37,7 @@ This workflow sets up the benchmarking environment and triggers individual bench
 ##### Matrix Strategy
 
 The workflow runs benchmarks on two types of EC2 instances:
+
 - GPU: g6.16xlarge
 - CPU: r7i.16xlarge
 
@@ -60,13 +61,15 @@ This workflow is triggered by `adhoc-matrix.yml` and runs the actual benchmarks
    Click on "Execute ZKVM-Perf (Matrix)".
 
 3. **Configure the Run**
+
    - You can use the default settings for a quick start.
    - Customize inputs as needed.
 
 4. **Start the Benchmark**
    Click "Run workflow".
 
 5. **Monitor Progress**
+
    - The workflow will start two jobs: one for GPU and one for CPU.
    - Each job will trigger a separate `run-on-runner` workflow.
 
@@ -80,12 +83,14 @@ This workflow is triggered by `adhoc-matrix.yml` and runs the actual benchmarks
 ### Setup
 
 1. Install Rust:
+
    ```sh
    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
    rustup install nightly
    ```
 
 2. Install the [SP1 toolchain](https://docs.succinct.xyz/getting-started/install.html):
+
    ```sh
    curl -L https://sp1.succinct.xyz | bash
    source ~/.bashrc
@@ -94,6 +99,7 @@ This workflow is triggered by `adhoc-matrix.yml` and runs the actual benchmarks
    ```
 
 3. Install the [Risc0 toolchain](https://dev.risczero.com/api/zkvm/install):
+
    ```sh
    curl -L https://risczero.com/install | bash
    source ~/.bashrc
@@ -112,11 +118,13 @@ This workflow is triggered by `adhoc-matrix.yml` and runs the actual benchmarks
 ### Running a Manual Sweep
 
 To conduct a sweep of the benchmarks:
+
 ```sh
 python3 sweep.py [options]
 ```
 
 Available options:
+
 - `--filename`: Filename for the benchmark (default: "benchmark")
 - `--trials`: Number of trials to run (default: 1)
 - `--programs`: List of programs to benchmark (choices: loop, fibonacci, tendermint, reth1, reth2)
@@ -134,7 +142,8 @@ To run a single benchmark:
 ./eval.sh <program> <prover> <hashfn> <shard_size> <filename> [block_number]
 ```
 
-Examples: 
+Examples:
+
 ```
 ./eval.sh fibonacci sp1 poseidon 22 benchmark
 ./eval.sh fibonacci jolt-zkvm poseidon 22 benchmark
@@ -162,6 +171,7 @@ If you encounter issues:
 For C++ compiler and library issues:
 
 Ubuntu/Debian:
+
 ```sh
 sudo apt update && sudo apt upgrade
 sudo apt install build-essential libc6
@@ -175,7 +185,8 @@ sudo yum groupinstall "Development Tools"
 sudo yum install -y gcc-c++
 ```
 
-Fedora: 
+Fedora:
+
 ```sh
 sudo dnf update
 sudo dnf groupinstall "Development Tools"
@@ -196,4 +207,4 @@ We welcome contributions to improve ZKVM-Perf! If you encounter issues or have s
 2. If your issue is new, create a detailed bug report or feature request.
 3. For code contributions, please submit a pull request with a clear description of your changes.
 
-Happy benchmarking! 🚀
+Happy benchmarking! 🚀

eval/src/risc0.rs

@@ -1,20 +1,18 @@
 #[cfg(feature = "risc0")]
 use std::fs;
 
-#[cfg(feature = "risc0")]
-use risc0_zkvm::{
-    compute_image_id, get_prover_server, ExecutorEnv, ExecutorImpl, ProverOpts, VerifierContext,
-};
 #[cfg(feature = "risc0")]
 use crate::{
     utils::{get_elf, get_reth_input, time_operation},
     HashFnId, ProgramId,
 };
-
-use crate::{
-    EvalArgs, PerformanceReport,
+#[cfg(feature = "risc0")]
+use risc0_zkvm::{
+    compute_image_id, get_prover_server, ExecutorEnv, ExecutorImpl, ProverOpts, VerifierContext,
 };
 
+use crate::{EvalArgs, PerformanceReport};
+
 pub struct Risc0Evaluator;
 
 impl Risc0Evaluator {
@@ -48,17 +46,36 @@ impl Risc0Evaluator {
         let cycles = session.user_cycles;
 
         // Setup the prover.
-        let env = if args.program == ProgramId::Reth {
-            let input = get_reth_input(args);
-            ExecutorEnv::builder()
-                .segment_limit_po2(args.shard_size as u32)
-                .write(&input)
-                .expect("Failed to write input to executor")
-                .build()
-                .unwrap()
-        } else {
-            ExecutorEnv::builder().segment_limit_po2(args.shard_size as u32).build().unwrap()
-        };
+        let mut builder = ExecutorEnv::builder().segment_limit_po2(args.shard_size as u32);
+        match args.program {
+            ProgramId::Reth => {
+                let input = get_reth_input(args);
+                builder.write(&input).expect("Failed to write input to executor");
+            }
+            ProgramId::Loop10k => {
+                builder.write::<usize>(&2500);
+            }
+            ProgramId::Loop100k => {
+                builder.write::<usize>(&25000);
+            }
+            ProgramId::Loop1m => {
+                builder.write::<usize>(&250000);
+            }
+            ProgramId::Loop3m => {
+                builder.write::<usize>(&750000);
+            }
+            ProgramId::Loop10m => {
+                builder.write::<usize>(&2500000);
+            }
+            ProgramId::Loop30m => {
+                builder.write::<usize>(&7500000);
+            }
+            ProgramId::Loop100m => {
+                builder.write::<usize>(&25000000);
+            }
+            _ => {}
+        }
+        let env = builder.build().unwrap();
         let opts = ProverOpts::default();
         let prover = get_prover_server(&opts).unwrap();
 
@@ -123,4 +140,4 @@ impl Risc0Evaluator {
     pub fn eval(_args: &EvalArgs) -> PerformanceReport {
         panic!("RISC0 feature is not enabled. Please compile with --features risc0");
     }
-}
+}

eval/src/sp1.rs

@@ -7,8 +7,8 @@ use crate::{
 
 use sp1_core_executor::SP1Context;
 
-use sp1_prover::{components::DefaultProverComponents, utils::get_cycles, SP1Prover};
 use sp1_core_machine::io::SP1Stdin;
+use sp1_prover::{components::DefaultProverComponents, utils::get_cycles, SP1Prover};
 
 #[cfg(feature = "cuda")]
 use sp1_cuda::SP1CudaProver;
@@ -30,14 +30,35 @@ impl SP1Evaluator {
         }
 
         // Get stdin.
-        let stdin = if args.program == ProgramId::Reth {
-            let input = get_reth_input(args);
-            let mut stdin = SP1Stdin::new();
-            stdin.write(&input);
-            stdin
-        } else {
-            SP1Stdin::new()
-        };
+        let mut stdin = SP1Stdin::new();
+        match args.program {
+            ProgramId::Reth => {
+                let input = get_reth_input(args);
+                stdin.write(&input);
+            }
+            ProgramId::Loop10k => {
+                stdin.write::<usize>(&2500);
+            }
+            ProgramId::Loop100k => {
+                stdin.write::<usize>(&25000);
+            }
+            ProgramId::Loop1m => {
+                stdin.write::<usize>(&250000);
+            }
+            ProgramId::Loop3m => {
+                stdin.write::<usize>(&750000);
+            }
+            ProgramId::Loop10m => {
+                stdin.write::<usize>(&2500000);
+            }
+            ProgramId::Loop30m => {
+                stdin.write::<usize>(&7500000);
+            }
+            ProgramId::Loop100m => {
+                stdin.write::<usize>(&25000000);
+            }
+            _ => {}
+        }
 
         // Get the elf.
         let elf_path = get_elf(args);