From 58e11918955cc9e0acbdfb319f06f11f2f72ff61 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Fri, 29 Oct 2021 00:12:18 +0300 Subject: [PATCH] [INTERNAL] - Mark MinTopicLeaderPerBrokerGoal as soft goal In our environment we've seen a case where RackAwareDistributionGoal and MinTopicLeaderPerBrokerGoal are conflicting. As both are hard goals the rebalance fails Lowering MinTopicLeaderPerBrokerGoal as a soft goal --- .circleci/config.yml | 2 ++ .../goals/MinTopicLeadersPerBrokerGoal.java | 18 +++++++++--------- .../analyzer/ReplicationFactorChangeTest.java | 9 +++------ 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index be6de9fda..dc1e6e6f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,8 @@ jobs: - checkout - restore_cache: key: dependency-cache-{{ checksum "build.gradle" }} + - run: + command: git tag 2.5.0-SNAPSHOT - run: command: ./gradlew --no-daemon clean javadoc - run: diff --git a/cruise-control/src/main/java/com/linkedin/kafka/cruisecontrol/analyzer/goals/MinTopicLeadersPerBrokerGoal.java b/cruise-control/src/main/java/com/linkedin/kafka/cruisecontrol/analyzer/goals/MinTopicLeadersPerBrokerGoal.java index 85d817575..177f42014 100644 --- a/cruise-control/src/main/java/com/linkedin/kafka/cruisecontrol/analyzer/goals/MinTopicLeadersPerBrokerGoal.java +++ b/cruise-control/src/main/java/com/linkedin/kafka/cruisecontrol/analyzer/goals/MinTopicLeadersPerBrokerGoal.java @@ -86,7 +86,7 @@ public String name() { @Override public boolean isHardGoal() { - return true; + return false; } /** @@ -292,8 +292,7 @@ protected void updateGoalState(ClusterModel clusterModel, OptimizationOptions op finish(); } - private void ensureBrokersAllHaveEnoughLeaderOfTopics(ClusterModel clusterModel, OptimizationOptions optimizationOptions) - throws OptimizationFailureException { + private void ensureBrokersAllHaveEnoughLeaderOfTopics(ClusterModel clusterModel, OptimizationOptions optimizationOptions) { if (_mustHaveTopicMinLeadersPerBroker.isEmpty()) { // Early termination to avoid some unnecessary computation return; @@ -305,9 +304,9 @@ private void ensureBrokersAllHaveEnoughLeaderOfTopics(ClusterModel clusterModel, for (String mustHaveLeaderPerBrokerTopicName : _mustHaveTopicMinLeadersPerBroker.keySet()) { int leaderCount = broker.numLeadersFor(mustHaveLeaderPerBrokerTopicName); if (leaderCount < minTopicLeadersPerBroker(mustHaveLeaderPerBrokerTopicName)) { - throw new OptimizationFailureException(String.format("[%s] Broker %d has insufficient per-broker leaders for topic %s (required: %d " - + "current: %d).", name(), broker.id(), mustHaveLeaderPerBrokerTopicName, - minTopicLeadersPerBroker(mustHaveLeaderPerBrokerTopicName), leaderCount)); + LOG.warn("[{}] Broker {} has insufficient per-broker leaders for topic {} (required: {} current: {}).", + name(), broker.id(), mustHaveLeaderPerBrokerTopicName, + minTopicLeadersPerBroker(mustHaveLeaderPerBrokerTopicName), leaderCount); } } } @@ -343,7 +342,7 @@ private void maybeMoveLeaderOfTopicToBroker(String topicMustHaveLeaderPerBroker, Broker broker, ClusterModel clusterModel, Set optimizedGoals, - OptimizationOptions optimizationOptions) throws OptimizationFailureException { + OptimizationOptions optimizationOptions) { int topicLeaderCountOnReceiverBroker = broker.numLeadersFor(topicMustHaveLeaderPerBroker); if (topicLeaderCountOnReceiverBroker >= minTopicLeadersPerBroker(topicMustHaveLeaderPerBroker)) { // This broker has enough leader replica(s) for the given topic @@ -408,9 +407,10 @@ private void maybeMoveLeaderOfTopicToBroker(String topicMustHaveLeaderPerBroker, } } } - throw new OptimizationFailureException(String.format("[%s] Cannot make broker %d have at least %d leaders from topic %s.", + LOG.warn("{} Cannot make broker {} have at least {} leaders from topic {}. Leaders on broker: {}", name(), broker.id(), - minTopicLeadersPerBroker(topicMustHaveLeaderPerBroker), topicMustHaveLeaderPerBroker)); + minTopicLeadersPerBroker(topicMustHaveLeaderPerBroker), + topicMustHaveLeaderPerBroker, topicLeaderCountOnReceiverBroker); } /** diff --git a/cruise-control/src/test/java/com/linkedin/kafka/cruisecontrol/analyzer/ReplicationFactorChangeTest.java b/cruise-control/src/test/java/com/linkedin/kafka/cruisecontrol/analyzer/ReplicationFactorChangeTest.java index bb69624ee..cf6b276bd 100644 --- a/cruise-control/src/test/java/com/linkedin/kafka/cruisecontrol/analyzer/ReplicationFactorChangeTest.java +++ b/cruise-control/src/test/java/com/linkedin/kafka/cruisecontrol/analyzer/ReplicationFactorChangeTest.java @@ -156,9 +156,8 @@ private static Class expectedExceptionClass(short replicati Class goalClass, boolean smallCluster) { if ((replicationFactor == LARGE_REPLICATION_FACTOR && goalClass == RackAwareGoal.class) - || (replicationFactor == LARGE_REPLICATION_FACTOR && goalClass == ReplicaCapacityGoal.class && !smallCluster) - || (replicationFactor == SMALL_REPLICATION_FACTOR && goalClass == MinTopicLeadersPerBrokerGoal.class) - || (replicationFactor == LARGE_REPLICATION_FACTOR && goalClass == MinTopicLeadersPerBrokerGoal.class && !smallCluster)) { + || (replicationFactor == LARGE_REPLICATION_FACTOR && goalClass == ReplicaCapacityGoal.class && !smallCluster) + ) { return OptimizationFailureException.class; } return null; @@ -188,9 +187,7 @@ private static boolean expectedToOptimize(short replicationFactor, Class