run-on-runner.yml

name: Run Benchmarks on EC2 Runner

on:
  workflow_dispatch:
    inputs:
      runner_name:
        required: true
        type: string
      instance_type:
        required: true
        type: string
      enable_gpu:
        required: true
        type: string
      provers:
        required: false
        type: string
        default: 'sp1'
      programs:
        required: false
        type: string
        default: 'loop,fibonacci,tendermint,reth1,reth2'
      filename:
        required: false
        type: string
        default: 'benchmark'
      trials:
        required: false
        type: string
        default: '1'
      sp1_ref:
        required: false
        type: string
      additional_params:
        required: false
        type: string
        default: '{"hashfns":"poseidon","shard_sizes":"22"}'

jobs:
  run-benchmark:
    runs-on: ${{ inputs.runner_name }}
    steps:
      - name: Echo Workflow Inputs
        run: |
          echo "Runner Name: ${{ inputs.runner_name }}"
          echo "Instance Type: ${{ inputs.instance_type }}"
          echo "Enable GPU: ${{ inputs.enable_gpu }}"
          echo "Provers: ${{ inputs.provers }}"
          echo "Programs: ${{ inputs.programs }}"
          echo "Filename: ${{ inputs.filename }}"
          echo "Trials: ${{ inputs.trials }}"
          echo "SP1 Ref: ${{ inputs.sp1_ref }}"
          echo "Additional Params: ${{ inputs.additional_params }}"
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Set up Docker
        uses: docker/setup-buildx-action@v2

      - name: Parse additional parameters
        id: parse-params
        run: |
          echo 'ADDITIONAL_PARAMS<<EOF' >> $GITHUB_ENV
          echo '${{ inputs.additional_params }}' >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV

      - name: Update SP1 Pin
        env:
          SP1_REF: ${{ inputs.sp1_ref }}
          RUN_BUILD: "false"
        run: |
          chmod +x update_sp1_and_build.sh
          ./update_sp1_and_build.sh

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Cache Docker layers
        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-

      - name: Build and cache Docker image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./Dockerfile.gpu
          push: false
          load: true
          tags: zkvm-perf
          build-args: |
            SP1_REF=${{ inputs.sp1_ref }}
          platforms: linux/amd64
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max

      # This step is necessary to prevent the cache from growing indefinitely
      - name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

      # - name: Build Docker image
      #   run: |
      #     docker build -t zkvm-perf --platform linux/amd64 -f Dockerfile.gpu --build-arg SP1_REF=${{ inputs.sp1_ref }} .

      - name: Make benchmarks directory
        run: mkdir -p ${{ github.workspace }}/benchmarks

      - name: Run benchmark
        run: |
          docker run ${{ inputs.enable_gpu == 'true' && '--gpus all' || '' }} --platform linux/amd64 \
            -v /var/run/docker.sock:/var/run/docker.sock \
            -v ${{ github.workspace }}/benchmarks:/usr/src/app/benchmarks \
            -e RUST_BACKTRACE=full \
            -e INSTANCE_TYPE=${{ inputs.instance_type }} \
            --network host \
            zkvm-perf \
            "python3 sweep.py --filename ${{ inputs.filename }} --trials ${{ inputs.trials }} --programs $(echo ${{ inputs.programs }} | sed 's/,/ /g') --provers ${{ inputs.provers }} --hashfns ${{ fromJson(env.ADDITIONAL_PARAMS).hashfns }} --shard-sizes ${{ fromJson(env.ADDITIONAL_PARAMS).shard_sizes }}"

      - name: List benchmark results
        run: ls -la ${{ github.workspace }}/benchmarks

      - name: Add instance type to CSV
        run: |
          for file in ${{ github.workspace }}/benchmarks/*.csv; do
            sed -i '1s/^/instance_type,/' "$file"
            sed -i "2,\$s/^/${{ inputs.instance_type }},/" "$file"
          done

      - name: Upload benchmark results
        uses: actions/upload-artifact@v3
        with:
          name: benchmark-results-${{ inputs.instance_type }}-${{ github.run_id }}
          path: ${{ github.workspace }}/benchmarks/*.csv

      - name: Print Results
        run: |
          cat ${{ github.workspace }}/benchmarks/*.csv