diff --git a/README.md b/README.md index f1fe625..1ee04eb 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +![logo](./assets/logo.png) # ozon-framer ozon-framer is the most performant grpc load generator diff --git a/assets/logo.png b/assets/logo.png new file mode 100644 index 0000000..0de479d Binary files /dev/null and b/assets/logo.png differ diff --git a/benchmarks/jmeter-java-dsl/.gitignore b/benchmarks/jmeter-java-dsl/.gitignore new file mode 100644 index 0000000..624a284 --- /dev/null +++ b/benchmarks/jmeter-java-dsl/.gitignore @@ -0,0 +1,2 @@ +report.jtl +target diff --git a/benchmarks/jmeter-java-dsl/.mvn/wrapper/maven-wrapper.jar b/benchmarks/jmeter-java-dsl/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000..cb28b0e Binary files /dev/null and b/benchmarks/jmeter-java-dsl/.mvn/wrapper/maven-wrapper.jar differ diff --git a/benchmarks/jmeter-java-dsl/.mvn/wrapper/maven-wrapper.properties b/benchmarks/jmeter-java-dsl/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..5f0536e --- /dev/null +++ b/benchmarks/jmeter-java-dsl/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,2 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.5/apache-maven-3.9.5-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/benchmarks/jmeter-java-dsl/Dockerfile b/benchmarks/jmeter-java-dsl/Dockerfile new file mode 100644 index 0000000..c599ccb --- /dev/null +++ b/benchmarks/jmeter-java-dsl/Dockerfile @@ -0,0 +1,4 @@ +FROM openjdk:20-oracle +WORKDIR /app +COPY . /app +ENTRYPOINT ["./mvnw", "verify" ] diff --git a/benchmarks/jmeter-java-dsl/Makefile b/benchmarks/jmeter-java-dsl/Makefile new file mode 100644 index 0000000..403551b --- /dev/null +++ b/benchmarks/jmeter-java-dsl/Makefile @@ -0,0 +1,11 @@ + + +.PHONY: verify +verify: + ./mvnw verify + + + +.PHONY: run-dumb +run-dumb: + cd ../dumb-server && make run \ No newline at end of file diff --git a/benchmarks/jmeter-java-dsl/REAME.md b/benchmarks/jmeter-java-dsl/REAME.md new file mode 100644 index 0000000..fd6b955 --- /dev/null +++ b/benchmarks/jmeter-java-dsl/REAME.md @@ -0,0 +1,6 @@ +How to run +- Run in docker (needs docker-compose v2): `docker compose up --abort-on-container-exit` +- Run local: + * Install JDK > 20, maven + * Run dumb server: `(cd ../dumb-server && make run)` + * Execute `./run.sh` diff --git a/benchmarks/jmeter-java-dsl/docker-compose.yaml b/benchmarks/jmeter-java-dsl/docker-compose.yaml new file mode 100644 index 0000000..e073000 --- /dev/null +++ b/benchmarks/jmeter-java-dsl/docker-compose.yaml @@ -0,0 +1,20 @@ +name: jmeter-benchmark +include: + - ../dumb-server/docker-compose.yaml +services: + jmeter: + container_name: jmeter + image: jmeter + build: + context: ./ + dockerfile: Dockerfile + network_mode: host + working_dir: /app + tty: true + volumes: + - ./:/app + deploy: + resources: + limits: + cpus: '2' + memory: 3G diff --git a/benchmarks/jmeter-java-dsl/mvnw b/benchmarks/jmeter-java-dsl/mvnw new file mode 100755 index 0000000..66df285 --- /dev/null +++ b/benchmarks/jmeter-java-dsl/mvnw @@ -0,0 +1,308 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.2.0 +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /usr/local/etc/mavenrc ] ; then + . /usr/local/etc/mavenrc + fi + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "$(uname)" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME + else + JAVA_HOME="/Library/Java/Home"; export JAVA_HOME + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=$(java-config --jre-home) + fi +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --unix "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --unix "$CLASSPATH") +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && + JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="$(which javac)" + if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=$(which readlink) + if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then + if $darwin ; then + javaHome="$(dirname "\"$javaExecutable\"")" + javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" + else + javaExecutable="$(readlink -f "\"$javaExecutable\"")" + fi + javaHome="$(dirname "\"$javaExecutable\"")" + javaHome=$(expr "$javaHome" : '\(.*\)/bin') + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=$(cd "$wdir/.." || exit 1; pwd) + fi + # end of workaround + done + printf '%s' "$(cd "$basedir" || exit 1; pwd)" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + # Remove \r in case we run on Windows within Git Bash + # and check out the repository with auto CRLF management + # enabled. Otherwise, we may read lines that are delimited with + # \r\n and produce $'-Xarg\r' rather than -Xarg due to word + # splitting rules. + tr -s '\r\n' ' ' < "$1" + fi +} + +log() { + if [ "$MVNW_VERBOSE" = true ]; then + printf '%s\n' "$1" + fi +} + +BASE_DIR=$(find_maven_basedir "$(dirname "$0")") +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR +log "$MAVEN_PROJECTBASEDIR" + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" +if [ -r "$wrapperJarPath" ]; then + log "Found $wrapperJarPath" +else + log "Couldn't find $wrapperJarPath, downloading it ..." + + if [ -n "$MVNW_REPOURL" ]; then + wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + else + wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + fi + while IFS="=" read -r key value; do + # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) + safeValue=$(echo "$value" | tr -d '\r') + case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; + esac + done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" + log "Downloading from: $wrapperUrl" + + if $cygwin; then + wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") + fi + + if command -v wget > /dev/null; then + log "Found wget ... using wget" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + else + wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + log "Found curl ... using curl" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + else + curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + fi + else + log "Falling back to using Java to download" + javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" + javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaSource=$(cygpath --path --windows "$javaSource") + javaClass=$(cygpath --path --windows "$javaClass") + fi + if [ -e "$javaSource" ]; then + if [ ! -e "$javaClass" ]; then + log " - Compiling MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/javac" "$javaSource") + fi + if [ -e "$javaClass" ]; then + log " - Running MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +# If specified, validate the SHA-256 sum of the Maven wrapper jar file +wrapperSha256Sum="" +while IFS="=" read -r key value; do + case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; + esac +done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" +if [ -n "$wrapperSha256Sum" ]; then + wrapperSha256Result=false + if command -v sha256sum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + elif command -v shasum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." + echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." + exit 1 + fi + if [ $wrapperSha256Result = false ]; then + echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 + echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 + echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 + exit 1 + fi +fi + +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --windows "$CLASSPATH") + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# shellcheck disable=SC2086 # safe args +exec "$JAVACMD" \ + $MAVEN_OPTS \ + $MAVEN_DEBUG_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/benchmarks/jmeter-java-dsl/pom.xml b/benchmarks/jmeter-java-dsl/pom.xml new file mode 100644 index 0000000..d04834e --- /dev/null +++ b/benchmarks/jmeter-java-dsl/pom.xml @@ -0,0 +1,222 @@ + + + 4.0.0 + + org.example + test-compiler + 1.0-SNAPSHOT + + + 20 + 20 + 1.62.2 + 1.6.2 + 0.6.1 + 4.26.0 + + 20 + 20 + UTF-8 + + + + + us.abstracta.jmeter + jmeter-java-dsl + 1.26 + test + + + + org.junit.jupiter + junit-jupiter-api + 5.10.2 + test + + + + com.google.protobuf + protoc + 4.26.0 + pom + + + + + io.grpc + protoc-gen-grpc-java + 1.62.2 + pom + + + + io.grpc + grpc-api + 1.62.2 + + + + com.google.protobuf + protobuf-java + ${protoc.version} + + + + io.grpc + grpc-stub + 1.62.2 + + + + io.grpc + grpc-protobuf + 1.62.2 + + + javax.annotation + javax.annotation-api + 1.3.2 + + + + + + io.grpc + grpc-netty + 1.62.2 + + + + io.grpc + grpc-core + 1.62.2 + + + + us.abstracta.jmeter + jmeter-java-dsl-dashboard + 1.26 + test + + + + + + + kr.motd.maven + os-maven-plugin + 1.6.1 + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf.plugin.version} + + com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier} + + grpc-java + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + + ${project.basedir}/src/main/resources/ + + + + + compile + compile-custom + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + test-jar + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.6.0 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory}/libs + + + + jmeter-java-dsl + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + true + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.2.5 + + + + ${project.build.directory}/${project.artifactId}-${project.version}-tests.jar + + + + + + + + + integration-test + verify + + benchmark + + + 2000 + localhost + 9090 + 100000 + 10 + 1000 + deflate + + -Xmx2g -Xms2g + + + + + + + + + \ No newline at end of file diff --git a/benchmarks/jmeter-java-dsl/run.sh b/benchmarks/jmeter-java-dsl/run.sh new file mode 100755 index 0000000..394d1c4 --- /dev/null +++ b/benchmarks/jmeter-java-dsl/run.sh @@ -0,0 +1,2 @@ +#!env sh +make verify diff --git a/benchmarks/jmeter-java-dsl/src/main/resources/api/api.proto b/benchmarks/jmeter-java-dsl/src/main/resources/api/api.proto new file mode 100644 index 0000000..ac76939 --- /dev/null +++ b/benchmarks/jmeter-java-dsl/src/main/resources/api/api.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package test.api; + +service TestApi { + rpc Test (TestRequest) returns (EmptyResponse); +} + +message TestRequest { + string field = 1; +} + +message EmptyResponse {} diff --git a/benchmarks/jmeter-java-dsl/src/test/java/PerformanceIT.java b/benchmarks/jmeter-java-dsl/src/test/java/PerformanceIT.java new file mode 100644 index 0000000..c9ede89 --- /dev/null +++ b/benchmarks/jmeter-java-dsl/src/test/java/PerformanceIT.java @@ -0,0 +1,77 @@ +import static us.abstracta.jmeter.javadsl.JmeterDsl.*; +import static us.abstracta.jmeter.javadsl.dashboard.DashboardVisualizer.dashboardVisualizer; + +import java.io.IOException; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +//import benchmark.BenchmarkGrpc; +//import benchmark.BenchmarkOuterClass; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.netty.channel.nio.NioEventLoopGroup; +import org.junit.jupiter.api.Test; +import test.api.Api; +import test.api.TestApiGrpc; +import us.abstracta.jmeter.javadsl.core.TestPlanStats; +import us.abstracta.jmeter.javadsl.core.threadgroups.RpsThreadGroup; + +public class PerformanceIT { +// BenchmarkGrpc.TestApiBlockingStub. + + @Test + public void testPerformance() throws IOException { + String host = System.getProperty("target.host"); + System.out.println(host); + String getenv = System.getProperty("target.port"); + System.out.println(getenv); + int port = Integer.parseInt(getenv); + int maxThreads = Integer.parseInt(System.getProperty("max.threads")); + int targetRPS = Integer.parseInt(System.getProperty("target.rps")); + int numClients = Integer.parseInt(System.getProperty("grpc.numclients")); + + Map stubs = new HashMap<>(); + for (int i = 0; i < numClients; i++) { + ManagedChannel client = ManagedChannelBuilder + .forAddress(host, port) + .keepAliveTime(5, TimeUnit.SECONDS) + .usePlaintext().build(); + TestApiGrpc.TestApiBlockingStub stub = TestApiGrpc.newBlockingStub(client); + stub.withCompression(System.getProperty("grpc.compression")); + stub.withExecutor(Executors.newCachedThreadPool()); + stubs.put(String.valueOf(i), stub); + } + Random generator = new Random(); + Object[] clients = stubs.values().toArray(); + + Api.TestRequest trueRequest = Api.TestRequest.newBuilder().setField("true").build(); + + RpsThreadGroup children = rpsThreadGroup() + .maxThreads(maxThreads) + .rampTo(targetRPS, Duration.ofSeconds(120)) + .rampToAndHold(targetRPS, Duration.ofSeconds(120), Duration.ofSeconds(120)) + .children( + jsr223Sampler(vars -> { + try { + TestApiGrpc.TestApiBlockingStub value = + (TestApiGrpc.TestApiBlockingStub) clients[generator.nextInt(clients.length)]; + Api.EmptyResponse resp = value.test(trueRequest); + vars.sampleResult.setSentBytes(trueRequest.getSerializedSize()); + vars.sampleResult.setBytes(resp.getSerializedSize()); + } catch (Exception e) { + vars.sampleResult.setResponseCode("500"); + } + }) + ); + TestPlanStats stats = testPlan( + children, +// dashboardVisualizer(), + + htmlReporter("./")).run(); + + + } + +} \ No newline at end of file