performance-and-size.yml

name: Performance and size

on:
  push:
    branches: [main]
  workflow_dispatch:

env:
  CARGO_TERM_COLOR: always

jobs:
  run-benchmarks:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

      - uses: brndnmtthws/rust-action-cargo-binstall@v1
        with:
          packages: hyperfine
      
      - name: Install valgrind
        run: sudo apt-get install valgrind

      - name: Build Ezno
        run: cargo build --release
        env:
          CARGO_PROFILE_RELEASE_DEBUG: true

      - name: Get last run
        id: get-last-run
        run: >
          # First result should come up within 10 runs from main
          id=$(gh api \
            -H "Accept: application/vnd.github+json" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            '/repos/kaleidawave/ezno/actions/runs?branch=main&per_page=10' \
            --jq '[.workflow_runs[] | select(.path == ".github/workflows/examples.yml")][0].id')
          echo "run_id=$id" >> "$GITHUB_OUTPUT"

      - name: Get base ezno
        if: github.ref_name != 'main'
        uses: actions/download-artifact@v4
        continue-on-error: true
        with:
          name: latest-checker
          path: previous-ezno
          run-id: ${{ steps.get-last-run.outputs.run_id }}

      - name: Set compilers
        id: compilers
        shell: bash
        run: |
          if [ -d "previous-ezno" ]; then
            echo "::notice::Comparing against previous"
            echo "BINARIES=./target/release/ezno,./previous-ezno/ezno" >> "$GITHUB_OUTPUT"
          else
            echo "::notice::Running singularly"
            echo "BINARIES=./target/release/ezno" >> "$GITHUB_OUTPUT"
          fi

      - name: Run checker performance
        shell: bash
        run: |
          # Generate a file which contains everything that Ezno currently implements  
          cargo run -p ezno-parser \
           --example code_blocks_to_script ./checker/specification/specification.md \
           --comment-headers \
           --out ./demo.tsx

          echo "### Checking
          \`\`\`shell
          $(hyperfine -i -L compiler ${{ steps.compilers.outputs.BINARIES }} '{compiler} check demo.tsx')
          \`\`\`" >> $GITHUB_STEP_SUMMARY

          echo "<details>
          <summary>Input</summary>

          > Code generated from specification.md. this is not meant to accurately represent a program but instead give an idea for how it scales across all the type checking features
          \`\`\`tsx
          $(cat ./demo.tsx)
          \`\`\`
          </details>
          " >> $GITHUB_STEP_SUMMARY

          echo "::info::Wrote code to summary"
          
          command_output=$(./target/release/ezno check demo.tsx --timings --max-diagnostics all 2>&1 || true)
          
          diagnostics=""; statistics=""; found_splitter=false;
          while IFS= read -r line; do
              if [[ "$line" == "---"* ]]; then found_splitter=true;
              elif [[ "$found_splitter"  == false ]]; then diagnostics+="$line"$'\n';
              else statistics+="$line"$'\n'; fi
          done <<< "$command_output"

          echo "<details>
          <summary>Diagnostics</summary>

          \`\`\`
          $diagnostics
          \`\`\`
          </details>
          " >> $GITHUB_STEP_SUMMARY

          if [ -d "previous-ezno" ]; then
            OUT=$(./previous-ezno/ezno check demo.tsx --timings --max-diagnostics all 2>&1 || true)
            $base_statistics=$(echo $OUT | rg "Diagnostics:" -A 100)
            echo "
            <details>
            <summary>Statistics</summary>
            
            \`\`\`
            $statistics
            \`\`\`
            against base
            \`\`\`
            $base_statistics
            \`\`\`
            against base
            </details>
            " >> $GITHUB_STEP_SUMMARY
          else
            echo "<details>
            <summary>Statistics</summary>

            \`\`\`
            $statistics
            \`\`\`
            </details>" >> $GITHUB_STEP_SUMMARY
          fi

      - name: Run checker performance w/staging
        shell: bash
        if: github.ref_name != 'main'
        run: |
          echo "::group::Running with staging"
          
          cat ./checker/specification/specification.md ./checker/specification/staging.md > with-staging.md
          cargo run -p ezno-parser --example code_blocks_to_script with-staging.md --comment-headers --out ./with-staging.tsx

          ./target/release/ezno check with-staging.tsx --timings || true
          ./target/release/ezno check with-staging.tsx --timings || true

          echo "::endgroup::"
            
      - name: Run checker performance on large file
        shell: bash
        run: |
          echo "::group::Running large"

          cat ./checker/specification/specification.md > main.md
          cargo run -p ezno-parser --example code_blocks_to_script main.md --comment-headers --out ./code.tsx

          for i in {1..10}; do
            cat ./code.tsx >> large.tsx
          done

          ./target/release/ezno check large.tsx --timings --max-diagnostics 0 || true

          hyperfine -i -L compiler ${{ steps.compilers.outputs.BINARIES }} '{compiler} check large.tsx'
          echo "::endgroup::"

      - name: Valgrind and callgrind
        shell: bash
        continue-on-error: true
        run: |
          IFS=',' read -ra ITEMS <<< ${{ steps.compilers.outputs.BINARIES }}

          for compiler in ${ITEMS[@]}; do
            echo "::group::Running $compiler"
            
            echo "::group::Callgrind"
            valgrind --tool=callgrind --callgrind-out-file=cpu-out $compiler check demo.tsx --max-diagnostics 0 | true
            echo "CPU usage:"
            head -n100 cpu-out
            echo "::endgroup::"
  
            echo "::group::Valgrind"
            valgrind --log-file=memory-out $compiler check demo.tsx --max-diagnostics 0 | true
            echo "Memory usage:"
            cat memory-out
            echo "::endgroup::"

            echo "::endgroup::"
          done

      - name: Run parsing & stringing (minfied) benchmarks
        shell: bash
        continue-on-error: true
        run: |
          strings=(
            "https://esm.sh/v128/react-dom@18.2.0/es2022/react-dom.mjs"
            "https://esm.sh/v135/typescript@5.3.3/es2022/typescript.mjs"
          )

          for url in "${strings[@]}"; do
            # TODO copy expression
            curl -sS $url > input.js

            echo "::group::Comparison"
            hyperfine -i \
              -L compiler ${{ steps.compilers.outputs.BINARIES }} \
              '{compiler} ast-explorer full input.js --timings'
            echo "::endgroup::"
          done

      - name: Upload checker
        if: github.ref == 'main'
        uses: actions/upload-artifact@v4
        with:
          name: latest-checker
          path: target/release/ezno
          retention-days: 90