ci: Improve automated benchmark report #54
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Runs benchmarks on self-hosted infra via `workflow_dispatch` and scheduled on Thursdays at 2pm UTC | ||
| # | ||
| # `workflow_dispatch` trigger | ||
| # The workflow can be run at https://github.com/argumentcomputer/zk-light-clients/actions/workflows/bench.yml | ||
| # The benchmark report can be found in the logs and as a comment on the latest commit on `dev`. | ||
| # The report can also be sent as a Zulip message to https://zulip.argument.xyz | ||
| # | ||
| # `schedule` trigger | ||
| # The workflow runs every week on Thursday at 2pm UTC | ||
| # It runs all light client benchmarks and sends them to Zulip | ||
| name: Light client benchmark | ||
| on: | ||
| schedule: | ||
| # Full report on Thursday, 2pm UTC | ||
| - cron: '0 14 * * 4' | ||
| workflow_dispatch: | ||
| inputs: | ||
| # Which light client to bench, e.g. `aptos`, `ethereum` or `kadena` | ||
| light-client: | ||
| description: 'Name of the light client to benchmark' | ||
| type: string | ||
| required: true | ||
| # Name of the `light-client` benchmark to run, e.g. | ||
| # Aptos: `inclusion` or `epoch_change` | ||
| # Ethereum: `inclusion or `committee_change` | ||
| # Kadena: `longest_chain` or `spv` | ||
| # Runs in the `light-client` directory, so it cannot benchmark `proof_server` or `programs` | ||
| bench-name: | ||
| description: 'Name of the benchmark to run' | ||
| type: string | ||
| required: true | ||
| # List of comma-separated env vars, e.g. `RUST_LOG=debug,MODE=SNARK` | ||
| # `RUSTFLAGS="-C target-cpu=native --cfg tokio_unstable -C opt-level=3"` is set by default | ||
| env: | ||
| description: 'List of comma-separated environment variables' | ||
| type: string | ||
| required: false | ||
| # Optionally send a message to the below Zulip streams | ||
| # Defaults to false | ||
| zulip: | ||
| description: 'Send the report to Zulip' | ||
| type: boolean | ||
| required: false | ||
| # User(s) to whom to send a private DM (optional) | ||
| # Comma-separated list of user ID integers, e.g. `11,12` (IDs can be found in user profiles) | ||
| # If not specified, sends to a stream/topic pair instead | ||
| private: | ||
| description: 'Send DM to given user ID(s)' | ||
| type: string | ||
| required: false | ||
| # Zulip stream in which to send the message (optional) | ||
| # Ignored if `private` input is specified | ||
| # Defaults to `light_client` stream | ||
| channel: | ||
| description: 'Send message to channel (default is `light-client`). Ignored if `private` input is specified' | ||
| type: string | ||
| required: false | ||
| # Zulip topic in which to send the message (optional) | ||
| # Ignored if `private` input is specified | ||
| # Defaults to `chat` | ||
| topic: | ||
| description: 'Send message to topic (default is `chat`). Ignored if `private` input is specified' | ||
| type: string | ||
| required: false | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | ||
| cancel-in-progress: true | ||
| jobs: | ||
| benchmark-manual: | ||
| name: LC bench (manual) | ||
| if: github.event_name == 'workflow_dispatch' | ||
| runs-on: warp-custom-r7iz-metal-32xl | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| repository: argumentcomputer/ci-workflows | ||
| - uses: ./.github/actions/ci-env | ||
| - uses: actions/checkout@v4 | ||
| - name: Setup CI | ||
| uses: ./.github/actions/setup | ||
| - name: Install extra deps | ||
| run: | | ||
| sudo apt-get update && sudo apt-get install -y python3-pip | ||
| pip3 --version | ||
| pip3 install --break-system-packages jtbl | ||
| echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile | ||
| source ~/.profile | ||
| which jtbl | ||
| - name: Set env | ||
| run: | | ||
| # Default benchmark settings optimized for light clients, can be overwritten with `env` input | ||
| echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV | ||
| echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV | ||
| echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV | ||
| echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV | ||
| echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a $GITHUB_ENV | ||
| echo "MODE=SNARK" | tee -a $GITHUB_ENV | ||
| IFS=',' read -ra ENV_VARS <<< "${{ inputs.env }}" | ||
| for VAR in "${ENV_VARS[@]}"; do | ||
| VAR_NAME="${VAR%%=*}" | ||
| VAR_VALUE="${VAR#*=}" | ||
| echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV | ||
| done | ||
| - name: Parse Zulip inputs | ||
| run: | | ||
| if [[ "${{ inputs.zulip }}" == "true" ]]; then | ||
| if [[ ! -z "${{ inputs.private }}" ]]; then | ||
| TYPE="private" | ||
| # Stream = private DM | ||
| STREAM="${{ inputs.private }}" | ||
| else | ||
| TYPE="stream" | ||
| if [[ ! -z "${{ inputs.stream }}" ]]; then | ||
| STREAM="${{ inputs.stream }}" | ||
| elif [[ -z "$STREAM" ]]; then | ||
| STREAM="light_client" | ||
| fi | ||
| if [[ ! -z "${{ inputs.topic }}" ]]; then | ||
| TOPIC="${{ inputs.topic }}" | ||
| elif [[ -z "$TOPIC" ]]; then | ||
| TOPIC="chat" | ||
| fi | ||
| fi | ||
| echo "TYPE=$TYPE" | tee -a $GITHUB_ENV | ||
| echo "STREAM=$STREAM" | tee -a $GITHUB_ENV | ||
| echo "TOPIC=$TOPIC" | tee -a $GITHUB_ENV | ||
| fi | ||
| - name: Run benchmarks | ||
| run: | | ||
| make bench-ci BENCH=${{ inputs.bench-name }} 2>&1 | tee out.txt | ||
| working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client | ||
| - name: Create report | ||
| id: run-benchmarks | ||
| run: | | ||
| grep 'cycles=' out.txt > cycles.txt | ||
| grep 'proving_time' out.txt > timings.txt | ||
| while IFS=$'\t' read -r f1 f2 | ||
| do | ||
| num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }') | ||
| timings=$(echo "$f2" | jq ' | ||
| to_entries | | ||
| map( | ||
| if .key == "proving_time" then | ||
| {key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")} | ||
| elif .key == "verifying_time" then | ||
| {key, value: ((.value / 1000 * 1000 | floor) / 1000 | tostring + "s")} | ||
| else | ||
| . | ||
| end | ||
| ) | | ||
| from_entries | ||
| ') | ||
| echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json | ||
| done < <(paste cycles.txt timings.txt) | ||
| echo '# `${{ inputs.light-client }}` Benchmark Results' | tee -a summary.md | ||
| echo '## `${{ inputs.bench-name }}` Prove' | tee -a summary.md | ||
| cat summary.json | jtbl -m | tee -a summary.md | ||
| echo "" | tee -a summary.md | ||
| echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md | ||
| echo "report<<EOF" >> $GITHUB_OUTPUT | ||
| cat summary.md >> $GITHUB_OUTPUT | ||
| echo "EOF" >> $GITHUB_OUTPUT | ||
| working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client | ||
| - name: Write bench on commit comment | ||
| uses: peter-evans/commit-comment@v3 | ||
| with: | ||
| body-path: ${{ github.workspace }}/${{ inputs.light-client }}/light-client/summary.md | ||
| - name: Send report to Zulip | ||
| if: inputs.zulip | ||
| uses: zulip/github-actions-zulip/send-message@v1 | ||
| with: | ||
| api-key: ${{ secrets.ZULIP_API_KEY }} | ||
| email: "[email protected]" | ||
| organization-url: "https://zulip.argument.xyz" | ||
| to: "${{ env.STREAM }}" | ||
| type: "${{ env.TYPE }}" | ||
| # Ignored if `type: private` | ||
| topic: "${{ env.TOPIC }}" | ||
| content: "${{ steps.run-benchmarks.outputs.report }}" | ||
| benchmark-scheduled: | ||
| name: LC bench (scheduled) | ||
| if: github.event_name == 'schedule' | ||
| runs-on: warp-custom-r7iz-metal-32xl | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| include: | ||
| - light-client: aptos | ||
| benchmark: inclusion | ||
| - light-client: aptos | ||
| benchmark: epoch_change | ||
| - light-client: ethereum | ||
| benchmark: inclusion | ||
| - light-client: ethereum | ||
| benchmark: committee_change | ||
| - light-client: kadena | ||
| benchmark: spv | ||
| - light-client: kadena | ||
| benchmark: longest_chain | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| repository: argumentcomputer/ci-workflows | ||
| - uses: ./.github/actions/ci-env | ||
| - uses: actions/checkout@v4 | ||
| - name: Setup CI | ||
| uses: ./.github/actions/setup | ||
| - name: Install extra deps | ||
| run: | | ||
| sudo apt-get update && sudo apt-get install -y python3-pip | ||
| pip3 --version | ||
| pip3 install --break-system-packages jtbl | ||
| echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile | ||
| source ~/.profile | ||
| which jtbl | ||
| - name: Set env | ||
| run: | | ||
| # Default benchmark settings optimized for light clients, can be overwritten with `env` input | ||
| echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV | ||
| echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV | ||
| echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV | ||
| echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV | ||
| echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a | ||
| $GITHUB_ENV | ||
| echo "MODE=SNARK" | tee -a $GITHUB_ENV | ||
| IFS=',' read -ra ENV_VARS <<< "ethereum" | ||
| for VAR in "${ENV_VARS[@]}"; do | ||
| VAR_NAME="${VAR%%=*}" | ||
| VAR_VALUE="${VAR#*=}" | ||
| echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV | ||
| done | ||
| - name: Set Zulip env | ||
| run: | | ||
| echo "TYPE=stream" | tee -a $GITHUB_ENV | ||
| echo "STREAM=light-client" | tee -a $GITHUB_ENV | ||
| echo "TOPIC=Benchmark Reports" | tee -a $GITHUB_ENV | ||
| - name: Run benchmarks | ||
| run: | | ||
| make bench-ci BENCH=${{ matrix.benchmark }} 2>&1 | tee out.txt | ||
| working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client | ||
| - name: Create report | ||
| run: | | ||
| grep 'cycles=' out.txt > cycles.txt | ||
| grep 'proving_time' out.txt > timings.txt | ||
| while IFS=$'\t' read -r f1 f2 | ||
| do | ||
| num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }') | ||
| timings=$(echo "$f2" | jq ' | ||
| to_entries | | ||
| map( | ||
| if .key == "proving_time" then | ||
| {key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")} | ||
| elif .key == "verifying_time" then | ||
| {key, value: ((.value / 1000 * 1000 | floor) / 1000 | tostring + "s")} | ||
| else | ||
| . | ||
| end | ||
| ) | | ||
| from_entries | ||
| ') | ||
| echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json | ||
| done < <(paste cycles.txt timings.txt) | ||
| COMMIT_SHORT=$(git rev-parse --short HEAD) | ||
| echo '# `${{ matrix.light-client }}` Benchmark Results' | tee -a summary.md | ||
| echo 'Commit: `$COMMIT_SHORT`' | tee -a summary.md | ||
| echo '## `${{ matrix.benchmark }}` Proof' | tee -a summary.md | ||
| cat summary.json | jtbl -m | tee -a summary.md | ||
| echo "" | tee -a summary.md | ||
| echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md | ||
| working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client | ||
| - name: Write bench on commit comment | ||
| uses: peter-evans/commit-comment@v3 | ||
| id: commit-comment | ||
| with: | ||
| body-path: ${{ github.workspace }}/${{ matrix.light-client }}/light-client/summary.md | ||
| - name: Prep report for Zulip | ||
| id: prep-report | ||
| run: | | ||
| COMMIT=$(git rev-parse HEAD) | ||
| ID=${{ steps.commit-comment.outputs.id }} | ||
| echo "Commit comment: https://github.com/${{ github.repository }}/commit/$COMMIT#commit-comment-$ID" | tee -a summary.md | ||
| echo "report<<EOF" >> $GITHUB_OUTPUT | ||
| cat summary.md >> $GITHUB_OUTPUT | ||
| echo "EOF" >> $GITHUB_OUTPUT | ||
| working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client | ||
| - name: Send report to Zulip | ||
| uses: zulip/github-actions-zulip/send-message@v1 | ||
| with: | ||
| api-key: ${{ secrets.ZULIP_API_KEY }} | ||
| email: "[email protected]" | ||
| organization-url: "https://zulip.argument.xyz" | ||
| to: "${{ env.STREAM }}" | ||
| type: "${{ env.TYPE }}" | ||
| # Ignored if `type: private` | ||
| topic: "${{ env.TOPIC }}" | ||
| content: "${{ steps.prep-report.outputs.report }}" | ||
