feat: Benchmark Release Workflow (1st Pass) (deephaven#354)

stanbrub · Oct 8, 2024 · 4c998ac · 4c998ac
1 parent 756fb5d
commit 4c998ac
Show file tree

Hide file tree

Showing 9 changed files with 345 additions and 41 deletions.
diff --git a/.github/distro/benchmark.properties b/.github/distro/benchmark.properties
@@ -0,0 +1,42 @@
+# Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
+
+# Deephaven engine address (same one the UI uses)
+deephaven.addr=localhost:10000
+
+# External java client address (Barrage Java Client)
+client.redpanda.addr=localhost:9092
+
+# External ReST schema registry supporting kafka (Kafka Producer)
+client.schema.registry.addr=localhost:8081
+
+# Internal ReST schema registry (Use in query scripts for kafka consume)
+schema.registry.addr=redpanda:8081
+#schema.registry.addr=localhost:8081
+
+# Internal kafka consumer address (Use in query scripts for kafka consume)
+kafka.consumer.addr=redpanda:29092
+#kafka.consumer.addr=localhost:29092
+
+# Default timeout to complete processes (Executing queries, generating records)
+default.completion.timeout=20 minutes
+
+# Default data distribution for column data (random, ascending, descending, runlength)
+default.data.distribution=random
+
+# Slows down record generation (Used for experiments not full test runs)
+generator.pause.per.row=0 millis
+
+# Compression used for generating and storing records (ZSTD, LZ4, LZO, GZIP, SNAPPY, NONE) 
+record.compression=SNAPPY
+
+# Row count to scale tests (Tests can override but typically do not)
+scale.row.count=1000000
+
+# True: Use a timestamp for the parent directory of each test run
+# False: Overwrite previous test results for each test run
+# Blank: Overwrite if JUnit launch, timestamp if Benchmark main launch
+timestamp.test.results=
+
+# Experimental: Docker compose file (e.g. /mypath/docker-compose.yml
+docker.compose.file=./docker-compose.yml
+
diff --git a/.github/distro/benchmark.sh b/.github/distro/benchmark.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
+#
+# This script shows an example of how to run the Deephaven operational benchmarks against
+# the Docker Deephaven in this directory. Each run will purge results and generated data from
+# any previous run and produce results as a set of csv files for each iteration.
+#
+# Examples:
+# - Run Where benchmarks three times: ./benchmark.sh 1 Where 
+# - Run Where and AvgBy benchmarks three times: ./benchmark.sh 3 "Where,Avg*"
+# - Run all benchmarks one time: ./benchmark.sh 1 *
+#
+# Notes:
+# - Docker is a prerequisite, and the Deephaven image will be installed if it's missing
+# - Available benchmarks for the test class list are test class simple names (no package)
+# - Running a full set (*) at default scale will take several hours or more
+# - Use the benchmark.properties file to change scale.row.count for higher/lower scale
+# - Benchmark metrics are in the results directory
+
+if [[ $# != 2 ]]; then
+    echo "$0: Missing iteration count or test class list argument"
+    exit 1
+fi
+
+ITERATIONS=$1
+TEST_WILD=$2
+BENCH_MAIN="io.deephaven.benchmark.run.BenchmarkMain"
+TEST_PACKAGE="io.deephaven.benchmark.tests.standard"
+
+sudo docker compose down
+
+rm -rf ./results
+sudo rm -f ./data/*.def
+sudo rm -f ./data/*.parquet
+
+sudo docker compose up -d 
+
+TEST_REGEX="^.*[.]("
+for r in $(echo ${TEST_WILD} | sed 's/\s*,\s*/ /g'); do
+  TEST_REGEX="${TEST_REGEX}"$(echo "(${r}Test)|" | sed 's/\*/.*/g')
+done
+TEST_REGEX=$(echo ${TEST_REGEX} | sed -E 's/\|+$//g')
+TEST_REGEX="${TEST_REGEX})$"
+
+for i in `seq 1 ${ITERATIONS}`
+do
+
+echo "*** Starting Iteration: $i ***"
+
+java -Dbenchmark.profile=benchmark.properties -cp "libs/*" ${BENCH_MAIN} -p ${TEST_PACKAGE} -n "${TEST_REGEX}" 
+
+done
+
+sudo docker compose down
+
diff --git a/.github/distro/docker-compose.yml b/.github/distro/docker-compose.yml
@@ -0,0 +1,68 @@
+services:
+  deephaven:
+    image: ghcr.io/deephaven/server:${VERSION}
+    ports:
+      - "${DEEPHAVEN_PORT:-10000}:10000"
+    volumes:
+      - ./data:/data
+    environment:
+      - "START_OPTS=-Xmx24G -DAuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler"
+
+  redpanda:
+    command:
+    - redpanda
+    - start
+    - --smp 2
+    - --memory 2G
+    - --reserve-memory 0M
+    - --overprovisioned
+    - --node-id 0
+    - --check=false
+    - --kafka-addr
+    - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
+    - --advertise-kafka-addr
+    - PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
+    - --pandaproxy-addr 0.0.0.0:8082
+    - --advertise-pandaproxy-addr redpanda:8082
+    image: redpandadata/redpanda:v24.1.2
+    ports:
+    - 8081:8081
+    - 8082:8082
+    - 9092:9092
+    - 29092:29092
+
+  minio-server:
+    image: minio/minio:RELEASE.2024-08-26T15-33-07Z
+    command: server /minio --console-address ":9001"
+    hostname: minio
+    environment:
+      MINIO_DOMAIN: minio
+    networks:
+      default:
+        aliases:
+          - data.minio
+    expose:
+    - "9000"
+    - "9001"
+    ports:
+    - 9000:9000
+    - 9001:9001
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    volumes:
+    - ./minio:/minio
+
+  minio-bucket:
+    image: minio/mc
+    depends_on:
+      - minio-server
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set endpoint http://minio:9000 minioadmin minioadmin;
+      /usr/bin/mc mb endpoint/data;
+      /usr/bin/mc anonymous set public endpoint/data;
+      exit 0;
+      "
diff --git a/.github/scripts/base62.sh b/.github/scripts/base62.sh
diff --git a/.github/scripts/make-release-distro.sh b/.github/scripts/make-release-distro.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# Copyright (c) 2024-2024 Deephaven Data Labs and Patent Pending
+
+# Create a tar file with the given version using the git project located in the 
+# working directory 
+
+if [[ $# != 3 ]]; then
+    echo "$0: Missing release version, release commit or previous version/commit argument"
+    exit 1
+fi
+
+RELEASE_VERSION=$1
+RELEASE_COMMIT=$2
+PREVIOUS_VERSION=$3
+RELEASE_TAG="v${RELEASE_VERSION}"
+PREVIOUS_TAG="v${PREVIOUS_VERSION}"
+ARTIFACT=deephaven-benchmark-${RELEASE_VERSION}
+DISTRO=target/distro
+THIS=$(basename "$0")
+RELEASE_NOTES=target/release-notes.md
+
+# Make the Release Notes File
+echo "**What's Changed**" > ${RELEASE_NOTES}
+git log --oneline ${PREVIOUS_TAG}...${RELEASE_COMMIT} | sed -e 's/^/- /' >> ${RELEASE_NOTES}
+echo "**Full Changelog**: https://github.com/deephaven/benchmark/compare/${PREVIOUS_TAG}...${RELEASE_TAG}" >> ${RELEASE_NOTES}
+
+# Build the Distro for running standard benchmarks
+mkdir -p ${DISTRO}/libs/
+cp .github/distro/* ${DISTRO}
+cp target/dependencies/* ${DISTRO}/libs
+cp target/deephaven-benchmark-1.0-SNAPSHOT.jar ${DISTRO}/libs/${ARTIFACT}.jar
+cp target/deephaven-benchmark-1.0-SNAPSHOT-tests.jar ${DISTRO}/libs/${ARTIFACT}-tests.jar
+echo "VERSION=${RELEASE_VERSION}" > ${DISTRO}/.env
+
+cd ${DISTRO}
+tar cvzf ../${ARTIFACT}.tar * .env
+
diff --git a/.github/scripts/test-release-distro.sh b/.github/scripts/test-release-distro.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# Copyright (c) 2024-2024 Deephaven Data Labs and Patent Pending
+
+# Test the release distro created in the target directory
+# - Untar is into a new directory
+# - Run a small set of tests with the benchmark.sh script
+
+TAG=$1
+ARTIFACT=deephaven-benchmark-${TAG}
+DISTRO=target/distro
+TEST_DIR=target/test-distro
+
+mkdir -p ${TEST_DIR}
+cp target/${ARTIFACT}.tar ${TEST_DIR}
+cd ${TEST_DIR}
+tar xvf ${ARTIFACT}.tar
+
+./benchmark.sh 1 Where
+
+tar cvzf ../${ARTIFACT}-results.tar results/
diff --git a/.github/workflows/mvn-integration-test.yml b/.github/workflows/mvn-integration-test.yml
@@ -22,7 +22,6 @@ jobs:
       with:
         java-version: '21'
         distribution: 'temurin'
-        cache: maven
 
     - name: Docker Pull Deephaven and Redpanda
       run: |

diff --git a/.github/workflows/publish-benchmarks.yml b/.github/workflows/publish-benchmarks.yml
@@ -8,11 +8,105 @@ name: Publish Deephaven Benchmarks
 
 on:
   workflow_dispatch:
+    inputs:
+      version:
+        description: 'Benchmark Release Version'
+        required: true
+        default: '0.36.0'
+        type: string
+      release-commit:
+        description: 'Benchmark Release Commit Hash'
+        required: true
+        default: '849d7e99a873ad6ff1ccd6c1a5099d33f8393f1a'
+        type: string
+      previous-version:
+        description: 'Benchmark Previous Version'
+        required: true
+        default: '0.35.0'
+        type: string
+      mark-latest:
+        description: 'Mark Release as Latest'
+        required: false
+        type: boolean
+
+env:
+  VERSION: "${{inputs.version}}"
+  COMMIT: "${{inputs.release-commit}}"
+  PREV_VERSION: "${{inputs.previous-version}}"
+  LATEST_DIR: ${{github.workspace}}/latest
+  RELEASE_DIR: ${{github.workspace}}/release
+
+permissions:
+  actions: write
+  contents: write
 
 jobs:
-  echo:
+  build:
     runs-on: ubuntu-22.04
+
     steps:
-    - name: Echo Something
+    - name: Set up JDK 21
+      uses: actions/setup-java@v4
+      with:
+        java-version: '21'
+        distribution: 'temurin'
+
+    - name: Checkout Latest
+      uses: actions/checkout@v4
+      with:
+        path: latest
+
+    - name: Checkout Release
+      uses: actions/checkout@v4
+      with:
+        path: release
+        fetch-depth: 0
+        fetch-tags: true
+
+    - name: Checkout Distro Target
+      working-directory: ${{env.RELEASE_DIR}}
+      run: git checkout ${COMMIT}
+
+    - name: Build Release Artifacts
+      working-directory: ${{env.RELEASE_DIR}}
+      run: |
+        cp ${LATEST_DIR}/.github/resources/integration-docker-compose.yml docker-compose.yml
+        docker compose up -d
+
+    - name: Build Release Artifacts
+      working-directory: ${{env.RELEASE_DIR}}
+      run: | 
+        mvn -B verify --file pom.xml
+        docker compose down 
+      
+    - name: Build Release Distro and Notes
+      working-directory: ${{env.RELEASE_DIR}}
+      run: ${LATEST_DIR}/.github/scripts/make-release-distro.sh ${VERSION} ${COMMIT} ${PREV_VERSION}
+
+    - name: Test Release Distro
+      working-directory: ${{env.RELEASE_DIR}}
       run: |
-        echo "Something"
+        ${LATEST_DIR}/.github/scripts/test-release-distro.sh ${VERSION}
+        ls -l ${RELEASE_DIR}/target/
+      
+    - name: Archive Results
+      uses: actions/upload-artifact@v4
+      with:
+        name: Deephaven Benchmark Release
+        path: |
+          ${{env.RELEASE_DIR}}/target/deephaven-benchmark-${{env.VERSION}}.tar
+          ${{env.RELEASE_DIR}}/target/deephaven-benchmark-${{env.VERSION}}-results.tar
+          ${{env.RELEASE_DIR}}/target/release-notes.md
+          
+    - name: Publish Github Release
+      if: ${{ github.ref_name == 'main' }}
+      uses: ncipollo/release-action@v1
+      with:
+        tag: "v${{env.VERSION}}"
+        commit: ${{env.COMMIT}}
+        makeLatest: ${{inputs.mark-latest}}
+        allowUpdates: true
+        artifacts: "${{env.RELEASE_DIR}}/target/deephaven-benchmark-${{env.VERSION}}.tar"
+        bodyFile: "${{env.RELEASE_DIR}}/target/release-notes.md"
+
+