Skip to content

feat:support default instance circuit breaker rule. #585

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 25, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,16 @@ consumer:
#描述:熔断策略,SDK会根据策略名称加载对应的熔断器插件(已注册的熔断器插件名)
chain:
- composite
# 描述:是否启用默认熔断规则
defaultRuleEnable: true
# 描述:连续错误数熔断器默认连续错误数
defaultErrorCount: 10
# 描述:错误率熔断器默认错误率
defaultErrorPercent: 50
# 描述:错误率熔断器默认统计周期(单位:毫秒)
defaultInterval: 60000
# 描述:错误率熔断器默认最小请求数
defaultMinimumRequest: 10
#描述: 熔断插件配置
plugin:
#描述:基于周期连续错误数熔断策略配置
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import com.tencent.polaris.api.config.plugin.PluginConfig;
import com.tencent.polaris.api.config.verify.Verifier;

import java.util.List;

/**
Expand Down Expand Up @@ -84,5 +85,39 @@ public interface CircuitBreakerConfig extends PluginConfig, Verifier {
*/
long getCountersExpireInterval();

/**
* 是否启用默认规则
*
* @return boolean
*/
boolean isDefaultRuleEnable();

/**
* 连续错误数熔断器默认连续错误数
*
* @return 连续错误数
*/
int getDefaultErrorCount();

/**
* 错误率熔断器默认错误率
*
* @return 错误率
*/
int getDefaultErrorPercent();

/**
* 错误率熔断器默认统计周期
*
* @return 统计周期
*/
int getDefaultInterval();

/**
* 错误率熔断器默认最小请求数
*
* @return 最小请求数
*/
int getDefaultMinimumRequest();

}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.tencent.polaris.factory.config.plugin.PluginConfigImpl;
import com.tencent.polaris.factory.util.ConfigUtils;
import com.tencent.polaris.factory.util.TimeStrJsonDeserializer;

import java.util.List;

/**
Expand Down Expand Up @@ -61,6 +62,21 @@ public class CircuitBreakerConfigImpl extends PluginConfigImpl implements Circui
@JsonDeserialize(using = TimeStrJsonDeserializer.class)
private Long countersExpireInterval;

@JsonProperty
private Boolean defaultRuleEnable;

@JsonProperty
private Integer defaultErrorCount;

@JsonProperty
private Integer defaultErrorPercent;

@JsonProperty
private Integer defaultInterval;

@JsonProperty
private Integer defaultMinimumRequest;

@Override
public boolean isEnable() {
if (null == enable) {
Expand Down Expand Up @@ -148,6 +164,54 @@ public void setEnableRemotePull(boolean enableRemotePull) {
this.enableRemotePull = enableRemotePull;
}

@Override
public boolean isDefaultRuleEnable() {
if (null == defaultRuleEnable) {
defaultRuleEnable = true;
}
return defaultRuleEnable;
}

public void setDefaultRuleEnable(Boolean defaultRuleEnable) {
this.defaultRuleEnable = defaultRuleEnable;
}

@Override
public int getDefaultErrorCount() {
return defaultErrorCount;
}

public void setDefaultErrorCount(Integer defaultErrorCount) {
this.defaultErrorCount = defaultErrorCount;
}

@Override
public int getDefaultErrorPercent() {
return defaultErrorPercent;
}

public void setDefaultErrorPercent(Integer defaultErrorPercent) {
this.defaultErrorPercent = defaultErrorPercent;
}

@Override
public int getDefaultInterval() {
return defaultInterval;
}

public void setDefaultInterval(Integer defaultInterval) {
this.defaultInterval = defaultInterval;
}

@Override
public int getDefaultMinimumRequest() {
return defaultMinimumRequest;
}

public void setDefaultMinimumRequest(Integer defaultMinimumRequest) {
this.defaultMinimumRequest = defaultMinimumRequest;
}

@Override
public void verify() {
ConfigUtils.validateNull(enable, "circuitBreaker.enable");
Expand All @@ -163,6 +227,11 @@ public void verify() {
ConfigUtils.validatePositive(requestCountAfterHalfOpen, "circuitBreaker.requestCountAfterHalfOpen");
ConfigUtils.validatePositive(successCountAfterHalfOpen, "circuitBreaker.successCountAfterHalfOpen");
ConfigUtils.validateNull(enableRemotePull, "circuitBreaker.enableRemotePull");
ConfigUtils.validateNull(defaultRuleEnable, "circuitBreaker.defaultRuleEnable");
ConfigUtils.validatePositive(defaultErrorCount, "circuitBreaker.defaultErrorCount");
ConfigUtils.validatePositive(defaultErrorPercent, "circuitBreaker.defaultErrorPercent");
ConfigUtils.validatePositive(defaultInterval, "circuitBreaker.defaultInterval");
ConfigUtils.validatePositive(defaultMinimumRequest, "circuitBreaker.defaultMinimumRequest");
verifyPluginConfig();
}

Expand Down Expand Up @@ -194,6 +263,21 @@ public void setDefault(Object defaultObject) {
if (null == enableRemotePull) {
setEnableRemotePull(circuitBreakerConfig.isEnableRemotePull());
}
if (null == defaultRuleEnable) {
setDefaultRuleEnable(circuitBreakerConfig.isDefaultRuleEnable());
}
if (null == defaultErrorCount) {
setDefaultErrorCount(circuitBreakerConfig.getDefaultErrorCount());
}
if (null == defaultErrorPercent) {
setDefaultErrorPercent(circuitBreakerConfig.getDefaultErrorPercent());
}
if (null == defaultInterval) {
setDefaultInterval(circuitBreakerConfig.getDefaultInterval());
}
if (null == defaultMinimumRequest) {
setDefaultMinimumRequest(circuitBreakerConfig.getDefaultMinimumRequest());
}
if (enable) {
setDefaultPluginConfig(circuitBreakerConfig);
}
Expand All @@ -211,6 +295,11 @@ public String toString() {
", successCountAfterHalfOpen=" + successCountAfterHalfOpen +
", enableRemotePull=" + enableRemotePull +
", countersExpireInterval=" + countersExpireInterval +
", defaultRuleEnable=" + defaultRuleEnable +
", defaultErrorCount=" + defaultErrorCount +
", defaultErrorPercent=" + defaultErrorPercent +
", defaultInterval=" + defaultInterval +
", defaultMinimumRequest=" + defaultMinimumRequest +
'}';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@
import com.tencent.polaris.api.plugin.compose.Extensions;
import com.tencent.polaris.api.plugin.detect.HealthChecker;
import com.tencent.polaris.api.pojo.*;
import com.tencent.polaris.api.utils.TrieUtil;
import com.tencent.polaris.api.utils.CollectionUtils;
import com.tencent.polaris.api.utils.TrieUtil;
import com.tencent.polaris.client.flow.DefaultServiceResourceProvider;
import com.tencent.polaris.client.util.NamedThreadFactory;
import com.tencent.polaris.logging.LoggerFactory;
Expand Down Expand Up @@ -241,6 +241,8 @@ private Optional<ResourceCounters> initResourceCounter(Resource resource) throws
throw t;
}

cbSvcRule = CircuitBreakerUtils.fillDefaultCircuitBreakerRuleInNeeded(resource, cbSvcRule, circuitBreakerConfig);

// pull fd rule
ServiceEventKey fdEventKey = new ServiceEventKey(resource.getService(),
ServiceEventKey.EventType.FAULT_DETECTING);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,34 @@

package com.tencent.polaris.plugins.circuitbreaker.composite.utils;

import com.google.common.collect.Lists;
import com.google.protobuf.StringValue;
import com.tencent.polaris.api.config.consumer.CircuitBreakerConfig;
import com.tencent.polaris.api.plugin.circuitbreaker.ResourceStat;
import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource;
import com.tencent.polaris.api.plugin.event.FlowEventConstants;
import com.tencent.polaris.api.pojo.CircuitBreakerStatus;
import com.tencent.polaris.api.pojo.RetStatus;
import com.tencent.polaris.api.pojo.ServiceRule;
import com.tencent.polaris.api.utils.CollectionUtils;
import com.tencent.polaris.api.utils.IPAddressUtils;
import com.tencent.polaris.api.utils.RuleUtils;
import com.tencent.polaris.client.pojo.ServiceRuleByProto;
import com.tencent.polaris.logging.LoggerFactory;
import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto;
import com.tencent.polaris.specification.api.v1.model.ModelProto;
import org.slf4j.Logger;

import java.util.List;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

public class CircuitBreakerUtils {

private static final Logger LOG = LoggerFactory.getLogger(CircuitBreakerUtils.class);

public static long DEFAULT_ERROR_RATE_INTERVAL_MS = 60 * 1000;

public static long MIN_CLEANUP_INTERVAL = 60 * 1000;
Expand Down Expand Up @@ -142,4 +153,94 @@ public static String getInstanceCircuitBreakerName(String host, int port) {
public static String getServiceCircuitBreakerName(String targetNamespaceId, String serviceName) {
return targetNamespaceId + "#" + serviceName;
}

public static ServiceRule fillDefaultCircuitBreakerRuleInNeeded(Resource resource, ServiceRule serviceRule, CircuitBreakerConfig circuitBreakerConfig) {
if (serviceRule instanceof ServiceRuleByProto && serviceRule.getRule() instanceof CircuitBreakerProto.CircuitBreaker) {
List<CircuitBreakerProto.CircuitBreakerRule> rules = ((CircuitBreakerProto.CircuitBreaker) serviceRule.getRule()).getRulesList();
if (shouldFilled(circuitBreakerConfig, rules)) {
CircuitBreakerProto.CircuitBreaker.Builder newCircuitBreakerBuilder = CircuitBreakerProto.CircuitBreaker.newBuilder().mergeFrom((CircuitBreakerProto.CircuitBreaker) serviceRule.getRule());
CircuitBreakerProto.CircuitBreakerRule.Builder defaultCircuitBreakerRuleBuilder = CircuitBreakerProto.CircuitBreakerRule.newBuilder();
// set name
defaultCircuitBreakerRuleBuilder.setName("default-polaris-instance-circuit-breaker");
// set enable
defaultCircuitBreakerRuleBuilder.setEnable(true);
// set level
defaultCircuitBreakerRuleBuilder.setLevel(CircuitBreakerProto.Level.INSTANCE);
// build ruleMatcher
CircuitBreakerProto.RuleMatcher.Builder ruleMatcher = CircuitBreakerProto.RuleMatcher.newBuilder();
CircuitBreakerProto.RuleMatcher.SourceService.Builder sourceServiceBuilder = CircuitBreakerProto.RuleMatcher.SourceService.newBuilder();
sourceServiceBuilder.setNamespace(resource.getCallerService().getNamespace());
sourceServiceBuilder.setService(resource.getCallerService().getService());
ruleMatcher.setSource(sourceServiceBuilder);
CircuitBreakerProto.RuleMatcher.DestinationService.Builder destinationServiceBuilder = CircuitBreakerProto.RuleMatcher.DestinationService.newBuilder();
destinationServiceBuilder.setNamespace(resource.getService().getNamespace());
destinationServiceBuilder.setService(resource.getService().getService());
ruleMatcher.setDestination(destinationServiceBuilder);
defaultCircuitBreakerRuleBuilder.setRuleMatcher(ruleMatcher);
// build blockConfigs
List<CircuitBreakerProto.BlockConfig> blockConfigList = Lists.newArrayList();
// build failure block config
CircuitBreakerProto.BlockConfig.Builder failureBlockConfigBuilder = CircuitBreakerProto.BlockConfig.newBuilder();
failureBlockConfigBuilder.setName("failure-block-config");
// build ret code error condition
CircuitBreakerProto.ErrorCondition.Builder errorConditionBuilder = CircuitBreakerProto.ErrorCondition.newBuilder();
errorConditionBuilder.setInputType(CircuitBreakerProto.ErrorCondition.InputType.RET_CODE);
ModelProto.MatchString.Builder codeMatchStringBuilder = ModelProto.MatchString.newBuilder();
codeMatchStringBuilder.setType(ModelProto.MatchString.MatchStringType.IN);
String statusCodes = IntStream.range(500, 600)
.mapToObj(String::valueOf)
.collect(Collectors.joining(","));
codeMatchStringBuilder.setValue(StringValue.of(statusCodes));
errorConditionBuilder.setCondition(codeMatchStringBuilder);
failureBlockConfigBuilder.addErrorConditions(errorConditionBuilder);
// build failure trigger conditions
// build error rate trigger condition
CircuitBreakerProto.TriggerCondition.Builder errorRateTriggerConditionBuilder = CircuitBreakerProto.TriggerCondition.newBuilder();
errorRateTriggerConditionBuilder.setTriggerType(CircuitBreakerProto.TriggerCondition.TriggerType.ERROR_RATE);
errorRateTriggerConditionBuilder.setErrorPercent(circuitBreakerConfig.getDefaultErrorPercent());
errorRateTriggerConditionBuilder.setInterval(circuitBreakerConfig.getDefaultInterval() / 1000);
errorRateTriggerConditionBuilder.setMinimumRequest(circuitBreakerConfig.getDefaultMinimumRequest());
failureBlockConfigBuilder.addTriggerConditions(errorRateTriggerConditionBuilder);
// build consecutive error trigger condition
CircuitBreakerProto.TriggerCondition.Builder consecutiveErrorTriggerConditionBuilder = CircuitBreakerProto.TriggerCondition.newBuilder();
consecutiveErrorTriggerConditionBuilder.setTriggerType(CircuitBreakerProto.TriggerCondition.TriggerType.CONSECUTIVE_ERROR);
consecutiveErrorTriggerConditionBuilder.setErrorCount(circuitBreakerConfig.getDefaultErrorCount());
failureBlockConfigBuilder.addTriggerConditions(consecutiveErrorTriggerConditionBuilder);
blockConfigList.add(failureBlockConfigBuilder.build());
defaultCircuitBreakerRuleBuilder.addAllBlockConfigs(blockConfigList);
// build recoverCondition
CircuitBreakerProto.RecoverCondition.Builder recoverConditionBuilder = CircuitBreakerProto.RecoverCondition.newBuilder();
recoverConditionBuilder.setSleepWindow((int) circuitBreakerConfig.getSleepWindow() / 1000);
recoverConditionBuilder.setConsecutiveSuccess(circuitBreakerConfig.getSuccessCountAfterHalfOpen());
defaultCircuitBreakerRuleBuilder.setRecoverCondition(recoverConditionBuilder);
newCircuitBreakerBuilder.addRules(defaultCircuitBreakerRuleBuilder);
CircuitBreakerProto.CircuitBreaker newCircuitBreaker = newCircuitBreakerBuilder.build();
if (LOG.isDebugEnabled()) {
LOG.debug("Resource {} set default circuit breaker rule {}", resource, newCircuitBreaker);
} else {
LOG.info("Resource {} set default circuit breaker rule with DefaultErrorCount:{}, " +
"DefaultErrorPercent:{}, DefaultInterval:{}, DefaultMinimumRequest:{}",
resource, circuitBreakerConfig.getDefaultErrorCount(), circuitBreakerConfig.getDefaultErrorPercent(),
circuitBreakerConfig.getDefaultInterval(), circuitBreakerConfig.getDefaultMinimumRequest());
}
return new ServiceRuleByProto(newCircuitBreaker, serviceRule.getRevision(), ((ServiceRuleByProto) serviceRule).isLoadedFromFile(), ((ServiceRuleByProto) serviceRule).getEventType());
}
}
return serviceRule;
}

private static boolean shouldFilled(CircuitBreakerConfig circuitBreakerConfig, List<CircuitBreakerProto.CircuitBreakerRule> rules) {
if (!circuitBreakerConfig.isDefaultRuleEnable()) {
return false;
}
if (CollectionUtils.isEmpty(rules)) {
return true;
}
for (CircuitBreakerProto.CircuitBreakerRule rule : rules) {
if (rule.getEnable() && CircuitBreakerUtils.checkRule(rule)) {
return false;
}
}
return true;
}
}
Loading
Loading