[INTERNAL] - Upgrade dependencies

build.gradle

@@ -7,10 +7,10 @@ plugins {
   id "com.jfrog.artifactory"
   id "idea"
   id "jacoco" // Java Code Coverage plugin
-  id "com.github.ben-manes.versions" version "0.42.0"
-  id "com.github.spotbugs" version "5.0.9" apply false
+  id "com.github.ben-manes.versions" version "0.50.0"
+  id "com.github.spotbugs" version "5.0.14" apply false
   id "checkstyle"
-  id "org.openapi.generator" version "5.3.0"
+  id "org.openapi.generator" version "5.4.0"
 }
 
 group = 'com.linkedin.cruisecontrol'
@@ -98,13 +98,13 @@ subprojects {
 
   //code quality and inspections
   checkstyle {
-    toolVersion = '10.0'
+    toolVersion = '10.3.4'
     ignoreFailures = false
     configDirectory = file("$rootDir/checkstyle")
   }
 
   spotbugs {
-    toolVersion = '4.7.1'
+    toolVersion = '4.8.3'
     excludeFilter = file("$rootDir/gradle/findbugs-exclude.xml")
     ignoreFailures = false
     jvmArgs = [ '-Xms512m' ]
@@ -170,8 +170,9 @@ project(':cruise-control-core') {
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'log4j', module: 'log4j'
     }
-    implementation "org.slf4j:slf4j-api:1.7.36"
-    implementation "org.apache.logging.log4j:log4j-slf4j-impl:2.17.2"
+    implementation "org.slf4j:slf4j-api:2.0.10"
+    implementation "org.apache.logging.log4j:log4j-slf4j2-impl:2.22.1"
+    implementation "org.apache.logging.log4j:log4j-core:2.22.1"
     implementation 'org.apache.commons:commons-math3:3.6.1'
     api "org.eclipse.jetty:jetty-servlet:${jettyVersion}"
     implementation 'com.google.code.findbugs:jsr305:3.0.2'
@@ -271,28 +272,29 @@ project(':cruise-control') {
 
     api project(':cruise-control-metrics-reporter')
     api project(':cruise-control-core')
-    implementation "org.slf4j:slf4j-api:1.7.36"
-    implementation "org.apache.logging.log4j:log4j-slf4j-impl:2.17.2"
+    implementation "org.slf4j:slf4j-api:2.0.10"
+    implementation "org.apache.logging.log4j:log4j-slf4j2-impl:2.22.1"
+    implementation "org.apache.logging.log4j:log4j-core:2.22.1"
     implementation "org.apache.zookeeper:zookeeper:${zookeeperVersion}"
     implementation "io.netty:netty-handler:${nettyVersion}"
     implementation "io.netty:netty-transport-native-epoll:${nettyVersion}"
     api "org.apache.kafka:kafka_$scalaBinaryVersion:$kafkaVersion"
     api "org.apache.kafka:kafka-clients:$kafkaVersion"
-    // Add following dependency when upgrading to Kafka 3.5
-    //api "org.apache.kafka:kafka-storage:$kafkaVersion"
+    api "org.apache.kafka:kafka-storage:$kafkaVersion"
     implementation "org.scala-lang:scala-library:$scalaVersion"
     implementation 'org.apache.commons:commons-math3:3.6.1'
-    implementation 'org.apache.httpcomponents:httpclient:4.5.13'
+    implementation 'org.apache.httpcomponents:httpclient:4.5.14'
     implementation 'commons-codec:commons-codec:1.15'
-    implementation 'com.google.code.gson:gson:2.9.0'
+    implementation 'com.google.code.gson:gson:2.10.1'
     implementation "org.eclipse.jetty:jetty-server:${jettyVersion}"
-    implementation 'io.dropwizard.metrics:metrics-jmx:4.2.9'
-    implementation 'com.nimbusds:nimbus-jose-jwt:9.24'
+    implementation 'io.dropwizard.metrics:metrics-jmx:4.2.18'
+    implementation 'com.nimbusds:nimbus-jose-jwt:9.31'
     implementation 'io.swagger.parser.v3:swagger-parser-v3:2.1.16'
-    implementation 'io.github.classgraph:classgraph:4.8.141'
+    implementation 'io.github.classgraph:classgraph:4.8.158'
     implementation 'com.google.code.findbugs:jsr305:3.0.2'
+    implementation 'com.google.guava:guava:32.1.1-jre'
     // Temporary pin for vulnerability
-    implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
+    implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.1'
     api "io.vertx:vertx-web-openapi:${vertxVersion}"
     api "io.vertx:vertx-core:${vertxVersion}"
     api "io.vertx:vertx-web:${vertxVersion}"
@@ -310,13 +312,14 @@ project(':cruise-control') {
     testImplementation 'org.easymock:easymock:4.3'
     testImplementation "org.apache.kafka:kafka_$scalaBinaryVersion:$kafkaVersion:test"
     testImplementation "org.apache.kafka:kafka-clients:$kafkaVersion:test"
-    testImplementation 'commons-io:commons-io:2.11.0'
-    testImplementation 'org.apache.httpcomponents:httpclient:4.5.13:tests'
+    testImplementation 'commons-io:commons-io:2.15.1'
+    testImplementation 'org.apache.httpcomponents:httpclient:4.5.14:tests'
     testImplementation 'org.bouncycastle:bcpkix-jdk15on:1.70'
-    testImplementation 'org.apache.kerby:kerb-simplekdc:2.0.1'
-    testImplementation 'com.jayway.jsonpath:json-path:2.7.0'
+    testImplementation 'org.apache.kerby:kerb-simplekdc:2.0.3'
+    testImplementation 'com.jayway.jsonpath:json-path:2.8.0'
     testImplementation 'org.powermock:powermock-module-junit4:2.0.9'
     testImplementation 'org.powermock:powermock-api-easymock:2.0.9'
+    testImplementation 'net.minidev:json-smart:2.4.11'
   }
 
   publishing {
@@ -340,7 +343,7 @@ project(':cruise-control') {
 
   tasks.create(name: "copyDependantLibs", type: Copy) {
     from (configurations.testRuntimeClasspath) {
-      include('log4j-slf4j-impl*')
+      include('log4j-slf4j2-impl*')
     }
     from (configurations.runtimeClasspath) {
 
@@ -422,6 +425,7 @@ project(':cruise-control') {
   }
 
   compileJava.dependsOn generateOpenApiJson
+  processResources.dependsOn generateOpenApiJson
 }
 
 project(':cruise-control-metrics-reporter') {
@@ -438,22 +442,23 @@ project(':cruise-control-metrics-reporter') {
       exclude group: 'log4j', module: 'log4j'
     }
 
-    implementation "org.slf4j:slf4j-api:1.7.36"
+    implementation "org.slf4j:slf4j-api:2.0.10"
     implementation "com.yammer.metrics:metrics-core:2.2.0"
-    implementation "org.apache.logging.log4j:log4j-slf4j-impl:2.17.2"
+    implementation "org.apache.logging.log4j:log4j-slf4j2-impl:2.22.1"
+    implementation "org.apache.logging.log4j:log4j-core:2.22.1"
     implementation "org.apache.kafka:kafka_$scalaBinaryVersion:$kafkaVersion"
     implementation "org.apache.kafka:kafka-clients:$kafkaVersion"
     implementation 'com.google.code.findbugs:jsr305:3.0.2'
     // Temporary pin for vulnerability
-    implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
+    implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.1'
 
     testImplementation 'junit:junit:4.13.2'
     testImplementation 'org.bouncycastle:bcpkix-jdk15on:1.70'
     testImplementation 'org.powermock:powermock-module-junit4:2.0.9'
     testImplementation 'org.powermock:powermock-api-easymock:2.0.9'
     testImplementation "org.apache.kafka:kafka-clients:$kafkaVersion:test"
     testImplementation "org.apache.kafka:kafka-clients:$kafkaVersion"
-    testImplementation 'commons-io:commons-io:2.11.0'
+    testImplementation 'commons-io:commons-io:2.15.1'
     testImplementation "org.apache.zookeeper:zookeeper:${zookeeperVersion}"
     testOutput sourceSets.test.output
   }
@@ -514,6 +519,6 @@ static def getScalaBinaryVersion(versionStr) {
 
 //wrapper generation task
 wrapper {
-  gradleVersion = '7.4'
+  gradleVersion = '8.2.1'
   distributionType = Wrapper.DistributionType.ALL
 }

cruise-control/src/main/java/com/linkedin/kafka/cruisecontrol/config/KafkaTopicConfigProvider.java

@@ -15,7 +15,7 @@
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
-import scala.jdk.javaapi.CollectionConverters;
+import scala.collection.JavaConverters;
 import org.apache.zookeeper.client.ZKClientConfig;
 
 /**
@@ -57,7 +57,7 @@ public Properties topicConfigs(String topic) {
       _zkSecurityEnabled,
       _zkClientConfig);
     try {
-      AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
+      AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient, null);
       return adminZkClient.fetchEntityConfig(ConfigType.Topic(), topic);
     } finally {
       KafkaCruiseControlUtils.closeKafkaZkClientWithTimeout(kafkaZkClient);
@@ -73,7 +73,7 @@ public Map<String, Properties> topicConfigs(Set<String> topics) {
                                                                               _zkClientConfig);
     Map<String, Properties> topicConfigs = new HashMap<>();
     try {
-      AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
+      AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient, null);
 
       for (String topic : topics) {
         try {
@@ -98,8 +98,9 @@ public Map<String, Properties> allTopicConfigs() {
                                                                               _zkSecurityEnabled,
                                                                               _zkClientConfig);
     try {
-      AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
-      return CollectionConverters.asJava(adminZkClient.getAllTopicConfigs());
+      AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient, null);
+      final scala.collection.Map<String, Properties> allTopicConfigs = adminZkClient.getAllTopicConfigs();
+      return JavaConverters.mapAsJavaMapConverter(allTopicConfigs).asJava();
     } finally {
       KafkaCruiseControlUtils.closeKafkaZkClientWithTimeout(kafkaZkClient);
     }

cruise-control/src/test/java/com/linkedin/kafka/cruisecontrol/monitor/LoadMonitorTest.java

@@ -41,6 +41,7 @@
 import org.apache.kafka.common.KafkaFuture;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.config.ConfigResource;
+import org.apache.kafka.common.config.TopicConfig;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.DescribeLogDirsResponse;
 import org.apache.kafka.common.utils.MockTime;
@@ -63,7 +64,6 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static kafka.log.LogConfig.CleanupPolicyProp;
 
 
 /**
@@ -566,7 +566,7 @@ private TestContext prepareContext(int numWindowToPreserve, boolean isClusterJBO
     props.put(MonitorConfig.MIN_SAMPLES_PER_PARTITION_METRICS_WINDOW_CONFIG,
               Integer.toString(MIN_SAMPLES_PER_WINDOW));
     props.put(MonitorConfig.PARTITION_METRICS_WINDOW_MS_CONFIG, Long.toString(WINDOW_MS));
-    props.put(CleanupPolicyProp(), DEFAULT_CLEANUP_POLICY);
+    props.put(TopicConfig.CLEANUP_POLICY_CONFIG, DEFAULT_CLEANUP_POLICY);
     props.put(MonitorConfig.SAMPLE_STORE_CLASS_CONFIG, NoopSampleStore.class.getName());
     props.put(MonitorConfig.SAMPLE_PARTITION_METRIC_STORE_ON_EXECUTION_CLASS_CONFIG, NoopSampleStore.class.getName());
     props.put(ExecutorConfig.ZOOKEEPER_SECURITY_ENABLED_CONFIG, "false");

cruise-control/src/test/java/com/linkedin/kafka/cruisecontrol/monitor/sampling/SamplingUtilsTest.java

@@ -26,6 +26,7 @@
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.TopicPartitionInfo;
 import org.apache.kafka.common.config.ConfigResource;
+import org.apache.kafka.common.config.TopicConfig;
 import org.easymock.EasyMock;
 import org.junit.Test;
 
@@ -34,8 +35,6 @@
 import static com.linkedin.kafka.cruisecontrol.KafkaCruiseControlUtils.maybeIncreasePartitionCount;
 import static com.linkedin.kafka.cruisecontrol.KafkaCruiseControlUtils.maybeUpdateTopicConfig;
 import static com.linkedin.kafka.cruisecontrol.KafkaCruiseControlUtils.wrapTopic;
-import static kafka.log.LogConfig.CleanupPolicyProp;
-import static kafka.log.LogConfig.RetentionMsProp;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -60,7 +59,7 @@ public void testMaybeUpdateTopicConfig() throws InterruptedException, ExecutionE
     Config topicConfig = EasyMock.createMock(Config.class);
     AlterConfigsResult alterConfigsResult = EasyMock.createMock(AlterConfigsResult.class);
     Set<AlterConfigOp> alterConfigOps = Collections.singleton(new AlterConfigOp(
-        new ConfigEntry(RetentionMsProp(), Long.toString(MOCK_DESIRED_RETENTION_MS)), AlterConfigOp.OpType.SET));
+        new ConfigEntry(TopicConfig.RETENTION_MS_CONFIG, Long.toString(MOCK_DESIRED_RETENTION_MS)), AlterConfigOp.OpType.SET));
     Map<ConfigResource, KafkaFuture<Config>> describeConfigsValues = Collections.singletonMap(MOCK_TOPIC_RESOURCE,
                                                                                               describedConfigsFuture);
     Map<ConfigResource, KafkaFuture<Void>> alterConfigsValues = Collections.singletonMap(MOCK_TOPIC_RESOURCE,
@@ -70,9 +69,9 @@ public void testMaybeUpdateTopicConfig() throws InterruptedException, ExecutionE
     EasyMock.expect(adminClient.describeConfigs(EasyMock.eq(Collections.singleton(MOCK_TOPIC_RESOURCE)))).andReturn(describeConfigsResult);
     EasyMock.expect(describeConfigsResult.values()).andReturn(describeConfigsValues);
     EasyMock.expect(describedConfigsFuture.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andReturn(topicConfig);
-    EasyMock.expect(topicConfig.get(EasyMock.eq(CleanupPolicyProp()))).andReturn(new ConfigEntry(CleanupPolicyProp(),
+    EasyMock.expect(topicConfig.get(EasyMock.eq(TopicConfig.CLEANUP_POLICY_CONFIG))).andReturn(new ConfigEntry(TopicConfig.CLEANUP_POLICY_CONFIG,
                                                                                                  DEFAULT_CLEANUP_POLICY));
-    EasyMock.expect(topicConfig.get(EasyMock.eq(RetentionMsProp()))).andReturn(new ConfigEntry(RetentionMsProp(),
+    EasyMock.expect(topicConfig.get(EasyMock.eq(TopicConfig.RETENTION_MS_CONFIG))).andReturn(new ConfigEntry(TopicConfig.RETENTION_MS_CONFIG,
                                                                                                MOCK_CURRENT_RETENTION_MS));
     EasyMock.expect(adminClient.incrementalAlterConfigs(EasyMock.eq(Collections.singletonMap(MOCK_TOPIC_RESOURCE,
                                                                                              alterConfigOps))))

cruise-control/src/test/java/com/linkedin/kafka/cruisecontrol/servlet/KafkaCruiseControlServletEndpointTest.java

@@ -22,7 +22,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import javax.servlet.http.HttpSession;
-import kafka.utils.MockTime;
+import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.Time;
 import org.easymock.EasyMock;
 import org.junit.Assert;

cruise-control/src/test/java/com/linkedin/kafka/cruisecontrol/servlet/UserTaskManagerTest.java

@@ -6,7 +6,7 @@
 
 import com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig;
 import com.linkedin.kafka.cruisecontrol.servlet.handler.async.runnable.OperationFuture;
-import kafka.utils.MockTime;
+import org.apache.kafka.common.utils.MockTime;
 import org.apache.kafka.common.utils.Time;
 import org.easymock.Capture;
 import org.easymock.EasyMock;

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM gradle:jdk17
+FROM gradle:jdk21
 
 USER root
 RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends git ca-certificates
@@ -38,22 +38,23 @@ RUN set -eux; \
 	; \
 	rm -rf /var/lib/apt/lists/*
 
-ENV JAVA_HOME /usr/local/openjdk-17
+ENV JAVA_HOME /usr/local/openjdk-21
 ENV PATH $JAVA_HOME/bin:$PATH
 
 # Default to UTF-8 file.encoding
 ENV LANG C.UTF-8
+ENV JAVA_VERSION 21
 RUN set -eux; \
 	\
 	arch="$(dpkg --print-architecture)"; \
 	case "$arch" in \
 		'amd64') \
-			downloadUrl='https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.tar.gz'; \
-			downloadSha256='https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.tar.gz.sha256'; \
+			downloadUrl='https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.tar.gz'; \
+			downloadSha256='https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.tar.gz.sha256'; \
 			;; \
 		'arm64') \
-			downloadUrl='https://download.oracle.com/java/17/latest/jdk-17_linux-aarch64_bin.tar.gz'; \
-			downloadSha256='https://download.oracle.com/java/17/latest/jdk-17_linux-aarch64_bin.tar.gz.sha256'; \
+			downloadUrl='https://download.oracle.com/java/21/latest/jdk-21_linux-aarch64_bin.tar.gz'; \
+			downloadSha256='https://download.oracle.com/java/21/latest/jdk-21_linux-aarch64_bin.tar.gz.sha256'; \
 			;; \
 		*) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \
 	esac; \

gradle.properties

@@ -3,8 +3,8 @@ org.gradle.daemon=false
 org.gradle.parallel=false
 org.gradle.jvmargs=-Xms512m -Xmx512m
 scalaVersion=2.13.10
-kafkaVersion=3.1.0
-zookeeperVersion=3.7.2
+kafkaVersion=3.6.1
+zookeeperVersion=3.6.3
 nettyVersion=4.1.100.Final
 jettyVersion=9.4.53.v20231009
-vertxVersion=4.5.0
+vertxVersion=4.5.1

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-all.zip
+networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -55,7 +55,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -80,13 +80,10 @@ do
     esac
 done
 
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-APP_NAME="Gradle"
+# This is normally unused
+# shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -143,12 +140,16 @@ fi
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
+        # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
+        # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -193,6 +194,10 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
 # Collect all arguments for the java command;
 #   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
 #     shell script including quotes and variable substitutions, so put them in
@@ -205,6 +210,12 @@ set -- \
         org.gradle.wrapper.GradleWrapperMain \
         "$@"
 
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
 # Use "xargs" to parse quoted args.
 #
 # With -n1 it outputs one arg per line, with the quotes and backslashes removed.