Skip to content

Commit

Permalink
feat:support default instance circuit breaker rule. (#585)
Browse files Browse the repository at this point in the history
  • Loading branch information
@SkyeBeFreeman
SkyeBeFreeman authored Feb 25, 2025
1 parent d21ee97 commit bd069d2
Show file tree
Hide file tree
Showing 7 changed files with 313 additions and 1 deletion.
10 changes: 10 additions & 0 deletions polaris-common/polaris-config-default/src/main/resources/conf/default-config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,16 @@ consumer:
#描述:熔断策略，SDK会根据策略名称加载对应的熔断器插件（已注册的熔断器插件名）
chain:
- composite
# 描述：是否启用默认熔断规则
defaultRuleEnable: true
# 描述：连续错误数熔断器默认连续错误数
defaultErrorCount: 10
# 描述：错误率熔断器默认错误率
defaultErrorPercent: 50
# 描述：错误率熔断器默认统计周期（单位：毫秒）
defaultInterval: 60000
# 描述：错误率熔断器默认最小请求数
defaultMinimumRequest: 10
#描述: 熔断插件配置
plugin:
#描述:基于周期连续错误数熔断策略配置
Expand Down
35 changes: 35 additions & 0 deletions ...is-config/src/main/java/com/tencent/polaris/api/config/consumer/CircuitBreakerConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import com.tencent.polaris.api.config.plugin.PluginConfig;
import com.tencent.polaris.api.config.verify.Verifier;

import java.util.List;

/**
Expand Down Expand Up @@ -84,5 +85,39 @@ public interface CircuitBreakerConfig extends PluginConfig, Verifier {
*/
long getCountersExpireInterval();

/**
* 是否启用默认规则
*
* @return boolean
*/
boolean isDefaultRuleEnable();

/**
* 连续错误数熔断器默认连续错误数
*
* @return 连续错误数
*/
int getDefaultErrorCount();

/**
* 错误率熔断器默认错误率
*
* @return 错误率
*/
int getDefaultErrorPercent();

/**
* 错误率熔断器默认统计周期
*
* @return 统计周期
*/
int getDefaultInterval();

/**
* 错误率熔断器默认最小请求数
*
* @return 最小请求数
*/
int getDefaultMinimumRequest();

}
89 changes: 89 additions & 0 deletions ...g/src/main/java/com/tencent/polaris/factory/config/consumer/CircuitBreakerConfigImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.tencent.polaris.factory.config.plugin.PluginConfigImpl;
import com.tencent.polaris.factory.util.ConfigUtils;
import com.tencent.polaris.factory.util.TimeStrJsonDeserializer;

import java.util.List;

/**
Expand Down Expand Up @@ -61,6 +62,21 @@ public class CircuitBreakerConfigImpl extends PluginConfigImpl implements Circui
@JsonDeserialize(using = TimeStrJsonDeserializer.class)
private Long countersExpireInterval;

@JsonProperty
private Boolean defaultRuleEnable;

@JsonProperty
private Integer defaultErrorCount;

@JsonProperty
private Integer defaultErrorPercent;

@JsonProperty
private Integer defaultInterval;

@JsonProperty
private Integer defaultMinimumRequest;

@Override
public boolean isEnable() {
if (null == enable) {
Expand Down Expand Up @@ -148,6 +164,54 @@ public void setEnableRemotePull(boolean enableRemotePull) {
this.enableRemotePull = enableRemotePull;
}

@Override
public boolean isDefaultRuleEnable() {
if (null == defaultRuleEnable) {
defaultRuleEnable = true;
}
return defaultRuleEnable;
}

public void setDefaultRuleEnable(Boolean defaultRuleEnable) {
this.defaultRuleEnable = defaultRuleEnable;
}

@Override
public int getDefaultErrorCount() {
return defaultErrorCount;
}

public void setDefaultErrorCount(Integer defaultErrorCount) {
this.defaultErrorCount = defaultErrorCount;
}

@Override
public int getDefaultErrorPercent() {
return defaultErrorPercent;
}

public void setDefaultErrorPercent(Integer defaultErrorPercent) {
this.defaultErrorPercent = defaultErrorPercent;
}

@Override
public int getDefaultInterval() {
return defaultInterval;
}

public void setDefaultInterval(Integer defaultInterval) {
this.defaultInterval = defaultInterval;
}

@Override
public int getDefaultMinimumRequest() {
return defaultMinimumRequest;
}

public void setDefaultMinimumRequest(Integer defaultMinimumRequest) {
this.defaultMinimumRequest = defaultMinimumRequest;
}

@Override
public void verify() {
ConfigUtils.validateNull(enable, "circuitBreaker.enable");
Expand All @@ -163,6 +227,11 @@ public void verify() {
ConfigUtils.validatePositive(requestCountAfterHalfOpen, "circuitBreaker.requestCountAfterHalfOpen");
ConfigUtils.validatePositive(successCountAfterHalfOpen, "circuitBreaker.successCountAfterHalfOpen");
ConfigUtils.validateNull(enableRemotePull, "circuitBreaker.enableRemotePull");
ConfigUtils.validateNull(defaultRuleEnable, "circuitBreaker.defaultRuleEnable");
ConfigUtils.validatePositive(defaultErrorCount, "circuitBreaker.defaultErrorCount");
ConfigUtils.validatePositive(defaultErrorPercent, "circuitBreaker.defaultErrorPercent");
ConfigUtils.validatePositive(defaultInterval, "circuitBreaker.defaultInterval");
ConfigUtils.validatePositive(defaultMinimumRequest, "circuitBreaker.defaultMinimumRequest");
verifyPluginConfig();
}

Expand Down Expand Up @@ -194,6 +263,21 @@ public void setDefault(Object defaultObject) {
if (null == enableRemotePull) {
setEnableRemotePull(circuitBreakerConfig.isEnableRemotePull());
}
if (null == defaultRuleEnable) {
setDefaultRuleEnable(circuitBreakerConfig.isDefaultRuleEnable());
}
if (null == defaultErrorCount) {
setDefaultErrorCount(circuitBreakerConfig.getDefaultErrorCount());
}
if (null == defaultErrorPercent) {
setDefaultErrorPercent(circuitBreakerConfig.getDefaultErrorPercent());
}
if (null == defaultInterval) {
setDefaultInterval(circuitBreakerConfig.getDefaultInterval());
}
if (null == defaultMinimumRequest) {
setDefaultMinimumRequest(circuitBreakerConfig.getDefaultMinimumRequest());
}
if (enable) {
setDefaultPluginConfig(circuitBreakerConfig);
}
Expand All @@ -211,6 +295,11 @@ public String toString() {
", successCountAfterHalfOpen=" + successCountAfterHalfOpen +
", enableRemotePull=" + enableRemotePull +
", countersExpireInterval=" + countersExpireInterval +
", defaultRuleEnable=" + defaultRuleEnable +
", defaultErrorCount=" + defaultErrorCount +
", defaultErrorPercent=" + defaultErrorPercent +
", defaultInterval=" + defaultInterval +
", defaultMinimumRequest=" + defaultMinimumRequest +
'}';
}
}
4 changes: 3 additions & 1 deletion ...main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@
import com.tencent.polaris.api.plugin.compose.Extensions;
import com.tencent.polaris.api.plugin.detect.HealthChecker;
import com.tencent.polaris.api.pojo.*;
import com.tencent.polaris.api.utils.TrieUtil;
import com.tencent.polaris.api.utils.CollectionUtils;
import com.tencent.polaris.api.utils.TrieUtil;
import com.tencent.polaris.client.flow.DefaultServiceResourceProvider;
import com.tencent.polaris.client.util.NamedThreadFactory;
import com.tencent.polaris.logging.LoggerFactory;
Expand Down Expand Up @@ -241,6 +241,8 @@ private Optional<ResourceCounters> initResourceCounter(Resource resource) throws
throw t;
}

cbSvcRule = CircuitBreakerUtils.fillDefaultCircuitBreakerRuleInNeeded(resource, cbSvcRule, circuitBreakerConfig);

// pull fd rule
ServiceEventKey fdEventKey = new ServiceEventKey(resource.getService(),
ServiceEventKey.EventType.FAULT_DETECTING);
Expand Down
101 changes: 101 additions & 0 deletions .../java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/CircuitBreakerUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,34 @@

package com.tencent.polaris.plugins.circuitbreaker.composite.utils;

import com.google.common.collect.Lists;
import com.google.protobuf.StringValue;
import com.tencent.polaris.api.config.consumer.CircuitBreakerConfig;
import com.tencent.polaris.api.plugin.circuitbreaker.ResourceStat;
import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource;
import com.tencent.polaris.api.plugin.event.FlowEventConstants;
import com.tencent.polaris.api.pojo.CircuitBreakerStatus;
import com.tencent.polaris.api.pojo.RetStatus;
import com.tencent.polaris.api.pojo.ServiceRule;
import com.tencent.polaris.api.utils.CollectionUtils;
import com.tencent.polaris.api.utils.IPAddressUtils;
import com.tencent.polaris.api.utils.RuleUtils;
import com.tencent.polaris.client.pojo.ServiceRuleByProto;
import com.tencent.polaris.logging.LoggerFactory;
import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto;
import com.tencent.polaris.specification.api.v1.model.ModelProto;
import org.slf4j.Logger;

import java.util.List;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

public class CircuitBreakerUtils {

private static final Logger LOG = LoggerFactory.getLogger(CircuitBreakerUtils.class);

public static long DEFAULT_ERROR_RATE_INTERVAL_MS = 60 * 1000;

public static long MIN_CLEANUP_INTERVAL = 60 * 1000;
Expand Down Expand Up @@ -142,4 +153,94 @@ public static String getInstanceCircuitBreakerName(String host, int port) {
public static String getServiceCircuitBreakerName(String targetNamespaceId, String serviceName) {
return targetNamespaceId + "#" + serviceName;
}

public static ServiceRule fillDefaultCircuitBreakerRuleInNeeded(Resource resource, ServiceRule serviceRule, CircuitBreakerConfig circuitBreakerConfig) {
if (serviceRule instanceof ServiceRuleByProto && serviceRule.getRule() instanceof CircuitBreakerProto.CircuitBreaker) {
List<CircuitBreakerProto.CircuitBreakerRule> rules = ((CircuitBreakerProto.CircuitBreaker) serviceRule.getRule()).getRulesList();
if (shouldFilled(circuitBreakerConfig, rules)) {
CircuitBreakerProto.CircuitBreaker.Builder newCircuitBreakerBuilder = CircuitBreakerProto.CircuitBreaker.newBuilder().mergeFrom((CircuitBreakerProto.CircuitBreaker) serviceRule.getRule());
CircuitBreakerProto.CircuitBreakerRule.Builder defaultCircuitBreakerRuleBuilder = CircuitBreakerProto.CircuitBreakerRule.newBuilder();
// set name
defaultCircuitBreakerRuleBuilder.setName("default-polaris-instance-circuit-breaker");
// set enable
defaultCircuitBreakerRuleBuilder.setEnable(true);
// set level
defaultCircuitBreakerRuleBuilder.setLevel(CircuitBreakerProto.Level.INSTANCE);
// build ruleMatcher
CircuitBreakerProto.RuleMatcher.Builder ruleMatcher = CircuitBreakerProto.RuleMatcher.newBuilder();
CircuitBreakerProto.RuleMatcher.SourceService.Builder sourceServiceBuilder = CircuitBreakerProto.RuleMatcher.SourceService.newBuilder();
sourceServiceBuilder.setNamespace(resource.getCallerService().getNamespace());
sourceServiceBuilder.setService(resource.getCallerService().getService());
ruleMatcher.setSource(sourceServiceBuilder);
CircuitBreakerProto.RuleMatcher.DestinationService.Builder destinationServiceBuilder = CircuitBreakerProto.RuleMatcher.DestinationService.newBuilder();
destinationServiceBuilder.setNamespace(resource.getService().getNamespace());
destinationServiceBuilder.setService(resource.getService().getService());
ruleMatcher.setDestination(destinationServiceBuilder);
defaultCircuitBreakerRuleBuilder.setRuleMatcher(ruleMatcher);
// build blockConfigs
List<CircuitBreakerProto.BlockConfig> blockConfigList = Lists.newArrayList();
// build failure block config
CircuitBreakerProto.BlockConfig.Builder failureBlockConfigBuilder = CircuitBreakerProto.BlockConfig.newBuilder();
failureBlockConfigBuilder.setName("failure-block-config");
// build ret code error condition
CircuitBreakerProto.ErrorCondition.Builder errorConditionBuilder = CircuitBreakerProto.ErrorCondition.newBuilder();
errorConditionBuilder.setInputType(CircuitBreakerProto.ErrorCondition.InputType.RET_CODE);
ModelProto.MatchString.Builder codeMatchStringBuilder = ModelProto.MatchString.newBuilder();
codeMatchStringBuilder.setType(ModelProto.MatchString.MatchStringType.IN);
String statusCodes = IntStream.range(500, 600)
.mapToObj(String::valueOf)
.collect(Collectors.joining(","));
codeMatchStringBuilder.setValue(StringValue.of(statusCodes));
errorConditionBuilder.setCondition(codeMatchStringBuilder);
failureBlockConfigBuilder.addErrorConditions(errorConditionBuilder);
// build failure trigger conditions
// build error rate trigger condition
CircuitBreakerProto.TriggerCondition.Builder errorRateTriggerConditionBuilder = CircuitBreakerProto.TriggerCondition.newBuilder();
errorRateTriggerConditionBuilder.setTriggerType(CircuitBreakerProto.TriggerCondition.TriggerType.ERROR_RATE);
errorRateTriggerConditionBuilder.setErrorPercent(circuitBreakerConfig.getDefaultErrorPercent());
errorRateTriggerConditionBuilder.setInterval(circuitBreakerConfig.getDefaultInterval() / 1000);
errorRateTriggerConditionBuilder.setMinimumRequest(circuitBreakerConfig.getDefaultMinimumRequest());
failureBlockConfigBuilder.addTriggerConditions(errorRateTriggerConditionBuilder);
// build consecutive error trigger condition
CircuitBreakerProto.TriggerCondition.Builder consecutiveErrorTriggerConditionBuilder = CircuitBreakerProto.TriggerCondition.newBuilder();
consecutiveErrorTriggerConditionBuilder.setTriggerType(CircuitBreakerProto.TriggerCondition.TriggerType.CONSECUTIVE_ERROR);
consecutiveErrorTriggerConditionBuilder.setErrorCount(circuitBreakerConfig.getDefaultErrorCount());
failureBlockConfigBuilder.addTriggerConditions(consecutiveErrorTriggerConditionBuilder);
blockConfigList.add(failureBlockConfigBuilder.build());
defaultCircuitBreakerRuleBuilder.addAllBlockConfigs(blockConfigList);
// build recoverCondition
CircuitBreakerProto.RecoverCondition.Builder recoverConditionBuilder = CircuitBreakerProto.RecoverCondition.newBuilder();
recoverConditionBuilder.setSleepWindow((int) circuitBreakerConfig.getSleepWindow() / 1000);
recoverConditionBuilder.setConsecutiveSuccess(circuitBreakerConfig.getSuccessCountAfterHalfOpen());
defaultCircuitBreakerRuleBuilder.setRecoverCondition(recoverConditionBuilder);
newCircuitBreakerBuilder.addRules(defaultCircuitBreakerRuleBuilder);
CircuitBreakerProto.CircuitBreaker newCircuitBreaker = newCircuitBreakerBuilder.build();
if (LOG.isDebugEnabled()) {
LOG.debug("Resource {} set default circuit breaker rule {}", resource, newCircuitBreaker);
} else {
LOG.info("Resource {} set default circuit breaker rule with DefaultErrorCount:{}, " +
"DefaultErrorPercent:{}, DefaultInterval:{}, DefaultMinimumRequest:{}",
resource, circuitBreakerConfig.getDefaultErrorCount(), circuitBreakerConfig.getDefaultErrorPercent(),
circuitBreakerConfig.getDefaultInterval(), circuitBreakerConfig.getDefaultMinimumRequest());
}
return new ServiceRuleByProto(newCircuitBreaker, serviceRule.getRevision(), ((ServiceRuleByProto) serviceRule).isLoadedFromFile(), ((ServiceRuleByProto) serviceRule).getEventType());
}
}
return serviceRule;
}

private static boolean shouldFilled(CircuitBreakerConfig circuitBreakerConfig, List<CircuitBreakerProto.CircuitBreakerRule> rules) {
if (!circuitBreakerConfig.isDefaultRuleEnable()) {
return false;
}
if (CollectionUtils.isEmpty(rules)) {
return true;
}
for (CircuitBreakerProto.CircuitBreakerRule rule : rules) {
if (rule.getEnable() && CircuitBreakerUtils.checkRule(rule)) {
return false;
}
}
return true;
}
}
Loading

0 comments on commit bd069d2

Please sign in to comment.