From 6b6bc50a3023520b85314d395880bf008d97d6c8 Mon Sep 17 00:00:00 2001 From: andrew shan <45474304+andrewshan@users.noreply.github.com> Date: Thu, 18 Jul 2024 16:27:40 +0800 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20=E4=BC=98=E5=8C=96=E7=86=94=E6=96=AD?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E5=AE=9E=E7=8E=B0=EF=BC=8C=E8=A7=A3=E5=86=B3?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E9=80=9A=E9=85=8DAPI=E5=86=85=E5=AD=98?= =?UTF-8?q?=E5=8D=A0=E7=94=A8=E8=BF=87=E9=AB=98=E9=97=AE=E9=A2=98=20(#533)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: support trace reporting * feat: restore version to 1.15.7-SNAPSHOT * feat: add initService to AssemblyAPI * test: add test for initService * fix: TraceReporterConfig type mistake * feat: support modify loglevel over parameter & support log in tracereporter * feat: 上报监控及调用链失败不抛异常 * fix: 校验失败打印端口号 * fix: add toString method to all configration classes * fix: 修复通配API导致counter及healthChecker膨胀的问题 * fix: 去掉统一的错误统计时长的配置，放回插件化配置中 * fix: 补齐测试用例 * fix: testcase failure * fix: test case failed by git test * fix: test case failed * fix: test case failure * fix: 健康检查变更影响范围过大问题 * fix: 解决探测规则生效多个的问题，只生效1个探测规则，并进行排序 * fix: 修复用例失败问题 --- .../circuitbreak/api/CircuitBreakAPI.java | 75 +- .../factory/test/CircuitBreakerMultiTest.java | 412 +++++++++++ .../factory/test/CircuitBreakerTest.java | 43 +- .../resources/circuitBreakerMethodRule.json | 62 ++ .../circuitBreakerMethodRuleChanged.json | 59 ++ .../circuitBreakerMethodRuleNoDetect.json | 59 ++ .../test/resources/circuitBreakerRule.json | 6 +- .../resources/circuitBreakerRuleNoDetect.json | 56 ++ .../test/resources/faultDetectMethodRule.json | 31 + .../faultDetectMethodRuleChanged.json | 31 + .../src/test/resources/faultDetectRule.json | 27 + .../main/resources/conf/default-config.yml | 6 +- .../config/consumer/CircuitBreakerConfig.java | 9 +- .../consumer/CircuitBreakerConfigImpl.java | 24 +- .../client/util/NamedThreadFactory.java | 2 +- .../circuitbreaker/entity/SubsetResource.java | 91 --- .../circuitbreaker-composite/pom.xml | 11 + .../CircuitBreakerRuleContainer.java | 317 -------- .../CircuitBreakerRuleDictionary.java | 181 +++++ .../composite/CircuitBreakerRuleListener.java | 42 +- .../composite/FaultDetectRuleDictionary.java | 108 +++ .../composite/HealthCheckContainer.java | 147 ++++ .../HealthCheckInstanceProvider.java | 31 + .../circuitbreaker/composite/MatchUtils.java | 62 -- .../composite/PolarisCircuitBreaker.java | 649 +++++++++++------ .../composite/ResourceCounters.java | 677 +++++++++--------- .../composite/ResourceHealthChecker.java | 535 +++++++------- .../composite/trigger/CounterOptions.java | 63 +- .../composite/trigger/ErrRateCounter.java | 6 +- .../composite/utils/CircuitBreakerUtils.java | 54 ++ .../{ => utils}/HealthCheckUtils.java | 7 +- .../composite/utils/MatchUtils.java | 88 +++ .../CircuitBreakerRuleContainerTest.java | 427 ++++++----- .../trigger/PolarisCircuitBreakerTest.java | 62 +- .../trigger/ResourceCountersTest.java | 19 +- .../trigger/ResourceHealthCheckerTest.java | 79 +- polaris-test/polaris-test-common/pom.xml | 7 + .../polaris/test/common/MockInitContext.java | 55 ++ 38 files changed, 2982 insertions(+), 1638 deletions(-) create mode 100644 polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerMultiTest.java create mode 100644 polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRule.json create mode 100644 polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRuleChanged.json create mode 100644 polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRuleNoDetect.json create mode 100644 polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerRuleNoDetect.json create mode 100644 polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectMethodRule.json create mode 100644 polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectMethodRuleChanged.json create mode 100644 polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectRule.json delete mode 100644 polaris-plugins/polaris-plugin-api/src/main/java/com/tencent/polaris/api/plugin/circuitbreaker/entity/SubsetResource.java delete mode 100644 polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleContainer.java create mode 100644 polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleDictionary.java create mode 100644 polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/FaultDetectRuleDictionary.java create mode 100644 polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckContainer.java create mode 100644 polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckInstanceProvider.java delete mode 100644 polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/MatchUtils.java create mode 100644 polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/CircuitBreakerUtils.java rename polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/{ => utils}/HealthCheckUtils.java (87%) create mode 100644 polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/MatchUtils.java create mode 100644 polaris-test/polaris-test-common/src/main/java/com/tencent/polaris/test/common/MockInitContext.java diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-api/src/main/java/com/tencent/polaris/circuitbreak/api/CircuitBreakAPI.java b/polaris-circuitbreaker/polaris-circuitbreaker-api/src/main/java/com/tencent/polaris/circuitbreak/api/CircuitBreakAPI.java index 6a6bad9e5..fd019c25e 100644 --- a/polaris-circuitbreaker/polaris-circuitbreaker-api/src/main/java/com/tencent/polaris/circuitbreak/api/CircuitBreakAPI.java +++ b/polaris-circuitbreaker/polaris-circuitbreaker-api/src/main/java/com/tencent/polaris/circuitbreak/api/CircuitBreakAPI.java @@ -17,43 +17,54 @@ package com.tencent.polaris.circuitbreak.api; +import java.io.Closeable; + import com.tencent.polaris.api.plugin.circuitbreaker.ResourceStat; import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; import com.tencent.polaris.circuitbreak.api.pojo.CheckResult; import com.tencent.polaris.circuitbreak.api.pojo.FunctionalDecoratorRequest; import com.tencent.polaris.circuitbreak.api.pojo.InvokeContext; -public interface CircuitBreakAPI { - - /** - * check and acquire circuitbreaker - * - * @param resource - * @return pass or not, and fallback config if needed - */ - CheckResult check(Resource resource); - - /** - * report the resource invoke result - * - * @param reportStat - */ - void report(ResourceStat reportStat); - - - /** - * make the function decorator - * - * @param functionalDecoratorRequest - * @return decorator - */ - FunctionalDecorator makeFunctionalDecorator(FunctionalDecoratorRequest functionalDecoratorRequest); - - /** - * make the invoke handler - * @param requestContext - * @return InvokeHandler - */ - InvokeHandler makeInvokeHandler(InvokeContext.RequestContext requestContext); +public interface CircuitBreakAPI extends AutoCloseable, Closeable { + + /** + * check and acquire circuitbreaker + * + * @param resource + * @return pass or not, and fallback config if needed + */ + CheckResult check(Resource resource); + + /** + * report the resource invoke result + * + * @param reportStat + */ + void report(ResourceStat reportStat); + + + /** + * make the function decorator + * + * @param functionalDecoratorRequest + * @return decorator + */ + FunctionalDecorator makeFunctionalDecorator(FunctionalDecoratorRequest functionalDecoratorRequest); + + /** + * make the invoke handler + * @param requestContext + * @return InvokeHandler + */ + InvokeHandler makeInvokeHandler(InvokeContext.RequestContext requestContext); + + /** + * 清理并释放资源 + */ + void destroy(); + @Override + default void close() { + destroy(); + } } diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerMultiTest.java b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerMultiTest.java new file mode 100644 index 000000000..235599ea3 --- /dev/null +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerMultiTest.java @@ -0,0 +1,412 @@ +/* + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.circuitbreaker.factory.test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import com.google.common.cache.Cache; +import com.google.protobuf.StringValue; +import com.google.protobuf.util.JsonFormat; +import com.tencent.polaris.api.config.Configuration; +import com.tencent.polaris.api.plugin.circuitbreaker.CircuitBreaker; +import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; +import com.tencent.polaris.api.pojo.ServiceKey; +import com.tencent.polaris.api.utils.StringUtils; +import com.tencent.polaris.circuitbreak.api.CircuitBreakAPI; +import com.tencent.polaris.circuitbreak.api.FunctionalDecorator; +import com.tencent.polaris.circuitbreak.api.pojo.FunctionalDecoratorRequest; +import com.tencent.polaris.circuitbreak.client.exception.CallAbortedException; +import com.tencent.polaris.circuitbreak.factory.CircuitBreakAPIFactory; +import com.tencent.polaris.client.api.BaseEngine; +import com.tencent.polaris.client.util.Utils; +import com.tencent.polaris.factory.config.ConfigurationImpl; +import com.tencent.polaris.logging.LoggerFactory; +import com.tencent.polaris.plugins.circuitbreaker.composite.HealthCheckContainer; +import com.tencent.polaris.plugins.circuitbreaker.composite.PolarisCircuitBreaker; +import com.tencent.polaris.plugins.circuitbreaker.composite.ResourceCounters; +import com.tencent.polaris.plugins.circuitbreaker.composite.ResourceHealthChecker; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; +import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto; +import com.tencent.polaris.test.common.TestUtils; +import com.tencent.polaris.test.mock.discovery.NamingServer; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; + +import static com.tencent.polaris.test.common.Consts.NAMESPACE_TEST; +import static com.tencent.polaris.test.common.Consts.SERVICE_CIRCUIT_BREAKER; +import static com.tencent.polaris.test.common.TestUtils.SERVER_ADDRESS_ENV; + +public class CircuitBreakerMultiTest { + + private static final Logger LOG = LoggerFactory.getLogger(CircuitBreakerMultiTest.class); + + private NamingServer namingServer; + + private final ServiceKey matchMethodService = new ServiceKey("Test", "SvcCbMethod"); + + private final ServiceKey matchMethodDetectService = new ServiceKey("Test", "SvcCbMethodDetect"); + + @Before + public void before() throws IOException { + try { + namingServer = NamingServer.startNamingServer(-1); + System.setProperty(SERVER_ADDRESS_ENV, String.format("127.0.0.1:%d", namingServer.getPort())); + } + catch (IOException e) { + Assert.fail(e.getMessage()); + } + + CircuitBreakerProto.CircuitBreakerRule cbRule1 = loadCbRule("circuitBreakerMethodRuleNoDetect.json"); + CircuitBreakerProto.CircuitBreaker circuitBreaker = CircuitBreakerProto.CircuitBreaker.newBuilder() + .addRules(cbRule1).setRevision(StringValue.newBuilder().setValue("0000").build()).build(); + namingServer.getNamingService().setCircuitBreaker(matchMethodService, circuitBreaker); + + + CircuitBreakerProto.CircuitBreakerRule cbRule3 = loadCbRule("circuitBreakerMethodRule.json"); + CircuitBreakerProto.CircuitBreakerRule cbRule4 = loadCbRule("circuitBreakerRule.json"); + circuitBreaker = CircuitBreakerProto.CircuitBreaker.newBuilder() + .addRules(cbRule3).addRules(cbRule4).setRevision(StringValue.newBuilder().setValue("1111").build()).build(); + namingServer.getNamingService().setCircuitBreaker(matchMethodDetectService, circuitBreaker); + FaultDetectorProto.FaultDetectRule rule1 = loadFdRule("faultDetectRule.json"); + FaultDetectorProto.FaultDetectRule rule2 = loadFdRule("faultDetectMethodRule.json"); + FaultDetectorProto.FaultDetector faultDetector = FaultDetectorProto.FaultDetector.newBuilder() + .addRules(rule1).addRules(rule2).setRevision("2222").build(); + namingServer.getNamingService().setFaultDetector(matchMethodDetectService, faultDetector); + } + + private CircuitBreakerProto.CircuitBreakerRule loadCbRule(String fileName) throws IOException { + CircuitBreakerProto.CircuitBreakerRule.Builder circuitBreakerRuleBuilder = CircuitBreakerProto.CircuitBreakerRule + .newBuilder(); + InputStream inputStream = getClass().getClassLoader().getResourceAsStream(fileName); + Assert.assertNotNull(inputStream); + String json = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("")); + JsonFormat.parser().ignoringUnknownFields().merge(json, circuitBreakerRuleBuilder); + return circuitBreakerRuleBuilder.build(); + } + + private FaultDetectorProto.FaultDetectRule loadFdRule(String fileName) throws IOException { + FaultDetectorProto.FaultDetectRule.Builder builder = FaultDetectorProto.FaultDetectRule.newBuilder(); + InputStream inputStream = getClass().getClassLoader().getResourceAsStream(fileName); + Assert.assertNotNull(inputStream); + String json = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("")); + JsonFormat.parser().ignoringUnknownFields().merge(json, builder); + return builder.build(); + } + + @Test + public void testMultipleUrlsNoRule() { + Configuration configuration = TestUtils.configWithEnvAddress(); + ConfigurationImpl configurationImpl = (ConfigurationImpl) configuration; + configurationImpl.getConsumer().getCircuitBreaker().setCountersExpireInterval(5000); + try (CircuitBreakAPI circuitBreakAPI = CircuitBreakAPIFactory.createCircuitBreakAPIByConfig(configurationImpl)) { + for (int i = 0; i < 50; i++) { + FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( + new ServiceKey(NAMESPACE_TEST, SERVICE_CIRCUIT_BREAKER), "/test/" + i); + FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); + int finalI = i; + Consumer integerConsumer = decorator.decorateConsumer(num -> { + if (num % 2 == 0) { + throw new IllegalArgumentException("invoke failed" + finalI); + } + else { + System.out.println("invoke success" + finalI); + } + }); + integerConsumer.accept(1); + } + try { + Thread.sleep(10 * 1000); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + + BaseEngine baseEngine = (BaseEngine) circuitBreakAPI; + CircuitBreaker resourceBreaker = baseEngine.getSDKContext().getExtensions().getResourceBreaker(); + PolarisCircuitBreaker polarisCircuitBreaker = (PolarisCircuitBreaker) resourceBreaker; + Cache> methodCache = polarisCircuitBreaker.getCountersCache() + .get(CircuitBreakerProto.Level.METHOD); + Assert.assertEquals(0, methodCache.size()); + } + } + + @Test + public void testMultipleUrlsMethodRule() { + Configuration configuration = TestUtils.configWithEnvAddress(); + ConfigurationImpl configurationImpl = (ConfigurationImpl) configuration; + configurationImpl.getConsumer().getCircuitBreaker().setCountersExpireInterval(5000); + try (CircuitBreakAPI circuitBreakAPI = CircuitBreakAPIFactory.createCircuitBreakAPIByConfig(configurationImpl)) { + for (int i = 0; i < 50; i++) { + FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( + matchMethodService, "/test1/path/" + i); + FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); + int finalI = i; + Consumer integerConsumer = decorator.decorateConsumer(num -> { + if (num % 2 == 0) { + throw new IllegalArgumentException("invoke failed" + finalI); + } + else { + System.out.println("invoke success" + finalI); + } + }); + try { + integerConsumer.accept(1); + Utils.sleepUninterrupted(1000); + integerConsumer.accept(2); + Utils.sleepUninterrupted(1000); + } + catch (Exception e) { + if (!(e instanceof IllegalArgumentException)) { + throw e; + } + } + Assert.assertThrows(CallAbortedException.class, () -> integerConsumer.accept(3)); + } + Utils.sleepUninterrupted(20 * 1000); + + BaseEngine baseEngine = (BaseEngine) circuitBreakAPI; + CircuitBreaker resourceBreaker = baseEngine.getSDKContext().getExtensions().getResourceBreaker(); + PolarisCircuitBreaker polarisCircuitBreaker = (PolarisCircuitBreaker) resourceBreaker; + Cache> methodCache = polarisCircuitBreaker.getCountersCache() + .get(CircuitBreakerProto.Level.METHOD); + Assert.assertEquals(0, methodCache.size()); + } + } + + @Test + public void testFaultDetector() { + Configuration configuration = TestUtils.configWithEnvAddress(); + ConfigurationImpl configurationImpl = (ConfigurationImpl) configuration; + configurationImpl.getConsumer().getCircuitBreaker().setCountersExpireInterval(5000); + try (CircuitBreakAPI circuitBreakAPI = CircuitBreakAPIFactory.createCircuitBreakAPIByConfig(configurationImpl)) { + for (int i = 0; i < 50; i++) { + String method = ""; + if (i > 0) { + method = "/test1/path/" + i; + } + FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( + matchMethodDetectService, method); + FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); + int finalI = i; + Consumer integerConsumer = decorator.decorateConsumer(num -> { + if (num % 2 == 0) { + throw new IllegalArgumentException("invoke failed" + finalI); + } + else { + System.out.println("invoke success" + finalI); + } + }); + integerConsumer.accept(1); + } + BaseEngine baseEngine = (BaseEngine) circuitBreakAPI; + CircuitBreaker resourceBreaker = baseEngine.getSDKContext().getExtensions().getResourceBreaker(); + PolarisCircuitBreaker polarisCircuitBreaker = (PolarisCircuitBreaker) resourceBreaker; + Map healthCheckCache = polarisCircuitBreaker.getHealthCheckCache(); + Assert.assertEquals(1, healthCheckCache.size()); + HealthCheckContainer healthCheckContainer = healthCheckCache.get(matchMethodDetectService); + Assert.assertNotNull(healthCheckContainer); + Collection healthCheckerValues = healthCheckContainer.getHealthCheckerValues(); + Assert.assertEquals(2, healthCheckerValues.size()); + for (ResourceHealthChecker resourceHealthChecker : healthCheckerValues) { + if (StringUtils.equals(resourceHealthChecker.getFaultDetectRule().getId(), "fd1")) { + Assert.assertEquals(1, resourceHealthChecker.getResources().size()); + } + } + Utils.sleepUninterrupted(20 * 1000); + + Cache> methodCache = polarisCircuitBreaker.getCountersCache() + .get(CircuitBreakerProto.Level.METHOD); + Assert.assertEquals(0, methodCache.size()); + } + } + + @Test + public void testCircuitBreakerRuleChanged() throws IOException { + Configuration configuration = TestUtils.configWithEnvAddress(); + ConfigurationImpl configurationImpl = (ConfigurationImpl) configuration; + try (CircuitBreakAPI circuitBreakAPI = CircuitBreakAPIFactory.createCircuitBreakAPIByConfig(configurationImpl)) { + for (int i = 0; i < 50; i++) { + String method = ""; + if (i > 0) { + method = "/test1/path/" + i; + } + FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( + matchMethodDetectService, method); + FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); + int finalI = i; + Consumer integerConsumer = decorator.decorateConsumer(num -> { + if (num % 2 == 0) { + throw new IllegalArgumentException("invoke failed" + finalI); + } + else { + System.out.println("invoke success" + finalI); + } + }); + integerConsumer.accept(1); + } + CircuitBreakerProto.CircuitBreakerRule cbRule1 = loadCbRule("circuitBreakerMethodRuleChanged.json"); + CircuitBreakerProto.CircuitBreaker circuitBreaker = CircuitBreakerProto.CircuitBreaker.newBuilder() + .addRules(cbRule1).setRevision(StringValue.newBuilder().setValue("444441").build()).build(); + namingServer.getNamingService().setCircuitBreaker(matchMethodDetectService, circuitBreaker); + Utils.sleepUninterrupted(20 * 1000); + BaseEngine baseEngine = (BaseEngine) circuitBreakAPI; + CircuitBreaker resourceBreaker = baseEngine.getSDKContext().getExtensions().getResourceBreaker(); + PolarisCircuitBreaker polarisCircuitBreaker = (PolarisCircuitBreaker) resourceBreaker; + Cache> methodCache = polarisCircuitBreaker.getCountersCache() + .get(CircuitBreakerProto.Level.METHOD); + Assert.assertEquals(0, methodCache.size()); + + Map healthCheckCache = polarisCircuitBreaker.getHealthCheckCache(); + HealthCheckContainer healthCheckContainer = healthCheckCache.get(matchMethodDetectService); + Assert.assertNotNull(healthCheckContainer); + Collection healthCheckerValues = healthCheckContainer.getHealthCheckerValues(); + Assert.assertEquals(2, healthCheckerValues.size()); + for (int i = 0; i < 10; i++) { + String method = "/test1/path/" + i; + FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( + matchMethodDetectService, method); + FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); + int finalI = i; + Consumer integerConsumer = decorator.decorateConsumer(num -> { + if (num < 3) { + throw new IllegalArgumentException("invoke failed" + finalI); + } + else { + System.out.println("invoke success" + finalI); + } + }); + try { + integerConsumer.accept(1); + } catch (Exception e) { + if (!(e instanceof IllegalArgumentException)) { + throw e; + } + System.out.println("invoke 1 failed" + finalI); + } + Utils.sleepUninterrupted(1000); + try { + integerConsumer.accept(2); + } catch (Exception e) { + if (!(e instanceof IllegalArgumentException)) { + throw e; + } + System.out.println("invoke 2 failed" + finalI); + } + Utils.sleepUninterrupted(1000); + Assert.assertThrows(CallAbortedException.class, () -> integerConsumer.accept(3)); + } + } + } + + @Test + public void testFaultDetectRuleChanged() throws IOException { + Configuration configuration = TestUtils.configWithEnvAddress(); + ConfigurationImpl configurationImpl = (ConfigurationImpl) configuration; + try (CircuitBreakAPI circuitBreakAPI = CircuitBreakAPIFactory.createCircuitBreakAPIByConfig(configurationImpl)) { + for (int i = 0; i < 10; i++) { + String method = ""; + if (i < 9) { + method = "/test1/path/" + i; + } + FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( + matchMethodDetectService, method); + FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); + int finalI = i; + Consumer integerConsumer = decorator.decorateConsumer(num -> { + if (num % 2 == 0) { + throw new IllegalArgumentException("invoke failed" + finalI); + } + else { + System.out.println("invoke success" + finalI); + } + }); + integerConsumer.accept(1); + } + BaseEngine baseEngine = (BaseEngine) circuitBreakAPI; + CircuitBreaker resourceBreaker = baseEngine.getSDKContext().getExtensions().getResourceBreaker(); + PolarisCircuitBreaker polarisCircuitBreaker = (PolarisCircuitBreaker) resourceBreaker; + Map healthCheckCache = polarisCircuitBreaker.getHealthCheckCache(); + Assert.assertEquals(1, healthCheckCache.size()); + HealthCheckContainer healthCheckContainer = healthCheckCache.get(matchMethodDetectService); + Assert.assertNotNull(healthCheckContainer); + Collection healthCheckerValues = healthCheckContainer.getHealthCheckerValues(); + Assert.assertEquals(2, healthCheckerValues.size()); +// for (ResourceHealthChecker resourceHealthChecker : healthCheckerValues) { +// if (StringUtils.equals(resourceHealthChecker.getFaultDetectRule().getId(), "fd1")) { +// Assert.assertEquals(1, resourceHealthChecker.getResources().size()); +// } +// } + FaultDetectorProto.FaultDetectRule rule1 = loadFdRule("faultDetectMethodRuleChanged.json"); + FaultDetectorProto.FaultDetector faultDetector = FaultDetectorProto.FaultDetector.newBuilder() + .addRules(rule1).setRevision("33333").build(); + namingServer.getNamingService().setFaultDetector(matchMethodDetectService, faultDetector); + + Utils.sleepUninterrupted(20 * 1000); + healthCheckContainer = healthCheckCache.get(matchMethodDetectService); + Assert.assertNull(healthCheckContainer); + for (int i = 0; i < 3; i++) { + String method = ""; + if (i > 0) { + method = "/test1/path/" + i; + } + FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( + matchMethodDetectService, method); + FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); + int finalI = i; + Consumer integerConsumer = decorator.decorateConsumer(num -> { + if (num % 2 == 0) { + throw new IllegalArgumentException("invoke failed" + finalI); + } + else { + System.out.println("invoke success" + finalI); + } + }); + integerConsumer.accept(1); + } + healthCheckContainer = healthCheckCache.get(matchMethodDetectService); + Assert.assertNotNull(healthCheckContainer); + healthCheckerValues = healthCheckContainer.getHealthCheckerValues(); + Assert.assertEquals(1, healthCheckerValues.size()); +// for (ResourceHealthChecker resourceHealthChecker : healthCheckerValues) { +// Assert.assertEquals(2, resourceHealthChecker.getResources().size()); +// } + } + } + + @After + public void after() { + if (null != namingServer) { + namingServer.terminate(); + } + } + +} diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerTest.java b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerTest.java index a145cb6f4..e6005059a 100644 --- a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerTest.java +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerTest.java @@ -90,7 +90,7 @@ public void before() throws IOException { namingServer.getNamingService().batchAddInstances(serviceKey, 10010, MAX_COUNT, parameter); CircuitBreakerProto.CircuitBreakerRule.Builder circuitBreakerRuleBuilder = CircuitBreakerProto.CircuitBreakerRule .newBuilder(); - InputStream inputStream = getClass().getClassLoader().getResourceAsStream("circuitBreakerRule.json"); + InputStream inputStream = getClass().getClassLoader().getResourceAsStream("circuitBreakerRuleNoDetect.json"); String json = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)).lines() .collect(Collectors.joining("")); JsonFormat.parser().ignoringUnknownFields().merge(json, circuitBreakerRuleBuilder); @@ -261,28 +261,31 @@ public void testCircuitBreakByErrorRate() { @Test public void testFunctionalDecorator() { Configuration configuration = TestUtils.configWithEnvAddress(); - CircuitBreakAPI circuitBreakAPI = CircuitBreakAPIFactory.createCircuitBreakAPIByConfig(configuration); - FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( - new ServiceKey(NAMESPACE_TEST, SERVICE_CIRCUIT_BREAKER), "'"); - FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); - Consumer integerConsumer = decorator.decorateConsumer(num -> { - if (num % 2 == 0) { - throw new IllegalArgumentException("invoke failed"); - } else { - System.out.println("invoke success"); + try (CircuitBreakAPI circuitBreakAPI = CircuitBreakAPIFactory.createCircuitBreakAPIByConfig(configuration)) { + FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( + new ServiceKey(NAMESPACE_TEST, SERVICE_CIRCUIT_BREAKER), "'"); + FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); + Consumer integerConsumer = decorator.decorateConsumer(num -> { + if (num % 2 == 0) { + throw new IllegalArgumentException("invoke failed"); + } + else { + System.out.println("invoke success"); + } + }); + try { + integerConsumer.accept(1); + Utils.sleepUninterrupted(1000); + integerConsumer.accept(2); + Utils.sleepUninterrupted(1000); } - }); - try { - integerConsumer.accept(1); - Utils.sleepUninterrupted(1000); - integerConsumer.accept(2); - Utils.sleepUninterrupted(1000); - } catch (Exception e) { - if (!(e instanceof IllegalArgumentException)) { - throw e; + catch (Exception e) { + if (!(e instanceof IllegalArgumentException)) { + throw e; + } } + Assert.assertThrows(CallAbortedException.class, () -> integerConsumer.accept(3)); } - Assert.assertThrows(CallAbortedException.class, () -> integerConsumer.accept(3)); } } \ No newline at end of file diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRule.json b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRule.json new file mode 100644 index 000000000..16c6eb04b --- /dev/null +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRule.json @@ -0,0 +1,62 @@ +{ + "@type": "type.googleapis.com/v1.CircuitBreakerRule", + "id": "5f1601f01823474d9be39c0bbb26ab87", + "name": "test", + "namespace": "TestCircuitBreakerRule1", + "enable": true, + "revision": "10b120c08706429f8fdc3fb44a53224c", + "ctime": "1754-08-31 06:49:24", + "mtime": "2023-02-21 17:35:31", + "etime": "", + "description": "", + "level": "METHOD", + "ruleMatcher": { + "source": { + "service": "*", + "namespace": "*" + }, + "destination": { + "service": "SvcCbMethodDetect", + "namespace": "Test", + "method": { + "type": "REGEX", + "value": "/test1/path/.*" + } + } + }, + "errorConditions": [ + { + "inputType": "RET_CODE", + "condition": { + "type": "NOT_EQUALS", + "value": "0", + "valueType": "TEXT" + } + } + ], + "triggerCondition": [ + { + "triggerType": "CONSECUTIVE_ERROR", + "errorCount": 1, + "errorPercent": 1, + "interval": 5, + "minimumRequest": 5 + } + ], + "maxEjectionPercent": 0, + "recoverCondition": { + "sleepWindow": 60, + "consecutiveSuccess": 3 + }, + "faultDetectConfig": { + "enable": true + }, + "fallbackConfig": { + "enable": false, + "response": { + "code": 0, + "headers": [], + "body": "" + } + } +} \ No newline at end of file diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRuleChanged.json b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRuleChanged.json new file mode 100644 index 000000000..ab33526dc --- /dev/null +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRuleChanged.json @@ -0,0 +1,59 @@ +{ + "@type": "type.googleapis.com/v1.CircuitBreakerRule", + "id": "5f1601f01823474d9be39c0bb126ab87", + "name": "test", + "namespace": "TestCircuitBreakerRule1", + "enable": true, + "revision": "10b120c08706429f85dc3fb44a53224c", + "ctime": "1754-08-31 06:49:24", + "mtime": "2023-02-21 17:35:31", + "etime": "", + "description": "", + "level": "METHOD", + "ruleMatcher": { + "source": { + "service": "*", + "namespace": "*" + }, + "destination": { + "service": "SvcCbMethodDetect", + "namespace": "Test", + "method": { + "type": "REGEX", + "value": "/test1/path/.*" + } + } + }, + "errorConditions": [ + { + "inputType": "RET_CODE", + "condition": { + "type": "NOT_EQUALS", + "value": "0", + "valueType": "TEXT" + } + } + ], + "triggerCondition": [ + { + "triggerType": "CONSECUTIVE_ERROR", + "errorCount": 2 + } + ], + "maxEjectionPercent": 0, + "recoverCondition": { + "sleepWindow": 60, + "consecutiveSuccess": 3 + }, + "faultDetectConfig": { + "enable": false + }, + "fallbackConfig": { + "enable": false, + "response": { + "code": 0, + "headers": [], + "body": "" + } + } +} \ No newline at end of file diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRuleNoDetect.json b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRuleNoDetect.json new file mode 100644 index 000000000..0dbf7c49e --- /dev/null +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerMethodRuleNoDetect.json @@ -0,0 +1,59 @@ +{ + "@type": "type.googleapis.com/v1.CircuitBreakerRule", + "id": "5f1601f01823474d9be39c0bbb26ab87", + "name": "test", + "namespace": "TestCircuitBreakerRule1", + "enable": true, + "revision": "10b120c08706429f8fdc3fb44a53524c", + "ctime": "1754-08-31 06:49:24", + "mtime": "2023-02-21 17:35:31", + "etime": "", + "description": "", + "level": "METHOD", + "ruleMatcher": { + "source": { + "service": "*", + "namespace": "*" + }, + "destination": { + "service": "SvcCbMethod", + "namespace": "Test", + "method": { + "type": "REGEX", + "value": "/test1/path/.*" + } + } + }, + "errorConditions": [ + { + "inputType": "RET_CODE", + "condition": { + "type": "NOT_EQUALS", + "value": "0", + "valueType": "TEXT" + } + } + ], + "triggerCondition": [ + { + "triggerType": "CONSECUTIVE_ERROR", + "errorCount": 1 + } + ], + "maxEjectionPercent": 0, + "recoverCondition": { + "sleepWindow": 60, + "consecutiveSuccess": 3 + }, + "faultDetectConfig": { + "enable": false + }, + "fallbackConfig": { + "enable": false, + "response": { + "code": 0, + "headers": [], + "body": "" + } + } +} \ No newline at end of file diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerRule.json b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerRule.json index 0a1e97f2a..4791d0dfe 100644 --- a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerRule.json +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerRule.json @@ -16,8 +16,8 @@ "namespace": "*" }, "destination": { - "service": "*", - "namespace": "*", + "service": "SvcCbMethodDetect", + "namespace": "Test", "method": null } }, @@ -26,7 +26,7 @@ "inputType": "RET_CODE", "condition": { "type": "NOT_EQUALS", - "value": "200", + "value": "0", "valueType": "TEXT" } } diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerRuleNoDetect.json b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerRuleNoDetect.json new file mode 100644 index 000000000..1548f10ca --- /dev/null +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/circuitBreakerRuleNoDetect.json @@ -0,0 +1,56 @@ +{ + "@type": "type.googleapis.com/v1.CircuitBreakerRule", + "id": "5f1601f01823474d9be39c0bbb26ab87", + "name": "test", + "namespace": "TestCircuitBreakerRule", + "enable": true, + "revision": "10b120c08706429f8fdd3fb44a53224b", + "ctime": "1754-08-31 06:49:24", + "mtime": "2023-02-21 17:35:31", + "etime": "", + "description": "", + "level": "SERVICE", + "ruleMatcher": { + "source": { + "service": "*", + "namespace": "*" + }, + "destination": { + "service": "*", + "namespace": "*", + "method": null + } + }, + "errorConditions": [ + { + "inputType": "RET_CODE", + "condition": { + "type": "NOT_EQUALS", + "value": "0", + "valueType": "TEXT" + } + } + ], + "triggerCondition": [ + { + "triggerType": "CONSECUTIVE_ERROR", + "errorCount": 1 + } + ], + "maxEjectionPercent": 0, + "recoverCondition": { + "sleepWindow": 60, + "consecutiveSuccess": 3 + }, + "faultDetectConfig": { + "enable": false + }, + "fallbackConfig": { + "enable": false, + "response": { + "code": 0, + "headers": [], + "body": "" + } + } +} \ No newline at end of file diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectMethodRule.json b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectMethodRule.json new file mode 100644 index 000000000..3bc1701e0 --- /dev/null +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectMethodRule.json @@ -0,0 +1,31 @@ +{ + "id": "fd2", + "name": "testfd1", + "description": "ffff2", + "targetService": { + "namespace": "Test", + "service": "SvcCbMethodDetect", + "method": { + "type": "REGEX", + "value": "/test1/path/.*" + } + }, + "interval": 30, + "timeout": 60, + "port": 0, + "protocol": "HTTP", + "httpConfig": { + "method": "GET", + "url": "/health", + "headers": [] + }, + "tcpConfig": { + "send": "", + "receive": [] + }, + "udpConfig": { + "send": "", + "receive": [] + }, + "revision": "10b120c08706429f8fdc3fb44a53125c" +} \ No newline at end of file diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectMethodRuleChanged.json b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectMethodRuleChanged.json new file mode 100644 index 000000000..9a425950e --- /dev/null +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectMethodRuleChanged.json @@ -0,0 +1,31 @@ +{ + "id": "fd3", + "name": "testfd1", + "description": "ffff2", + "targetService": { + "namespace": "Test", + "service": "SvcCbMethodDetect", + "method": { + "type": "REGEX", + "value": "/test1/path/.*" + } + }, + "interval": 30, + "timeout": 60, + "port": 0, + "protocol": "HTTP", + "httpConfig": { + "method": "GET", + "url": "/health", + "headers": [] + }, + "tcpConfig": { + "send": "", + "receive": [] + }, + "udpConfig": { + "send": "", + "receive": [] + }, + "revision": "10b120c08706429f8fdc3fb44a53125c" +} \ No newline at end of file diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectRule.json b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectRule.json new file mode 100644 index 000000000..190059988 --- /dev/null +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/resources/faultDetectRule.json @@ -0,0 +1,27 @@ +{ + "id": "fd1", + "name": "testfd", + "description": "ffff1", + "targetService": { + "namespace": "Test", + "service": "SvcCbMethodDetect" + }, + "interval": 30, + "timeout": 60, + "port": 0, + "protocol": "HTTP", + "httpConfig": { + "method": "GET", + "url": "/health", + "headers": [] + }, + "tcpConfig": { + "send": "", + "receive": [] + }, + "udpConfig": { + "send": "", + "receive": [] + }, + "revision": "10b120c08706429f8fdc3fb44a53225c" +} \ No newline at end of file diff --git a/polaris-common/polaris-config-default/src/main/resources/conf/default-config.yml b/polaris-common/polaris-config-default/src/main/resources/conf/default-config.yml index 21cca1166..32b4aa3a1 100644 --- a/polaris-common/polaris-config-default/src/main/resources/conf/default-config.yml +++ b/polaris-common/polaris-config-default/src/main/resources/conf/default-config.yml @@ -213,7 +213,9 @@ consumer: enable: true #描述: 故障检测周期，根据周期内故障进行熔断 checkPeriod: 1m - #描述: 首次熔断时间，后续熔断时间=重试次数*sleepWindow + #描述：错误率统计时长 + errorRateInterval: 60s + #描述: 熔断后等待时长 sleepWindow: 30s #描述: 熔断器半开后最大允许的请求数 requestCountAfterHalfOpen: 3 @@ -221,6 +223,8 @@ consumer: successCountAfterHalfOpen: 3 #描述: 熔断规则远程拉取开关 enableRemotePull: true + #描述：熔断计数器淘汰周期（距离上一次请求上报） + countersExpireInterval: 300s #描述:熔断策略，SDK会根据策略名称加载对应的熔断器插件（已注册的熔断器插件名） chain: - composite diff --git a/polaris-common/polaris-config/src/main/java/com/tencent/polaris/api/config/consumer/CircuitBreakerConfig.java b/polaris-common/polaris-config/src/main/java/com/tencent/polaris/api/config/consumer/CircuitBreakerConfig.java index cc462489b..32b2aec09 100644 --- a/polaris-common/polaris-config/src/main/java/com/tencent/polaris/api/config/consumer/CircuitBreakerConfig.java +++ b/polaris-common/polaris-config/src/main/java/com/tencent/polaris/api/config/consumer/CircuitBreakerConfig.java @@ -71,11 +71,18 @@ public interface CircuitBreakerConfig extends PluginConfig, Verifier { */ int getSuccessCountAfterHalfOpen(); - /** * 熔断规则远程拉取开关 * * @return true if 启用远程拉取 */ boolean isEnableRemotePull(); + + /** + * 熔断计数器淘汰时长 + * @return 0 if 用不淘汰 + */ + long getCountersExpireInterval(); + + } diff --git a/polaris-common/polaris-config/src/main/java/com/tencent/polaris/factory/config/consumer/CircuitBreakerConfigImpl.java b/polaris-common/polaris-config/src/main/java/com/tencent/polaris/factory/config/consumer/CircuitBreakerConfigImpl.java index 629c6a12a..a673887af 100644 --- a/polaris-common/polaris-config/src/main/java/com/tencent/polaris/factory/config/consumer/CircuitBreakerConfigImpl.java +++ b/polaris-common/polaris-config/src/main/java/com/tencent/polaris/factory/config/consumer/CircuitBreakerConfigImpl.java @@ -57,6 +57,10 @@ public class CircuitBreakerConfigImpl extends PluginConfigImpl implements Circui @JsonProperty private Boolean enableRemotePull; + @JsonProperty + @JsonDeserialize(using = TimeStrJsonDeserializer.class) + private Long countersExpireInterval; + @Override public boolean isEnable() { if (null == enable) { @@ -128,6 +132,18 @@ public boolean isEnableRemotePull() { return enableRemotePull; } + @Override + public long getCountersExpireInterval() { + if (null == countersExpireInterval) { + return 0; + } + return countersExpireInterval; + } + + public void setCountersExpireInterval(long countersExpireInterval) { + this.countersExpireInterval = countersExpireInterval; + } + public void setEnableRemotePull(boolean enableRemotePull) { this.enableRemotePull = enableRemotePull; } @@ -143,6 +159,7 @@ public void verify() { } ConfigUtils.validateInterval(checkPeriod, "circuitBreaker.checkPeriod"); ConfigUtils.validateInterval(sleepWindow, "circuitBreaker.sleepWindow"); + ConfigUtils.validateInterval(countersExpireInterval, "circuitBreaker.countersExpireInterval"); ConfigUtils.validatePositive(requestCountAfterHalfOpen, "circuitBreaker.requestCountAfterHalfOpen"); ConfigUtils.validatePositive(successCountAfterHalfOpen, "circuitBreaker.successCountAfterHalfOpen"); ConfigUtils.validateNull(enableRemotePull, "circuitBreaker.enableRemotePull"); @@ -165,6 +182,9 @@ public void setDefault(Object defaultObject) { if (null == sleepWindow) { setSleepWindow(circuitBreakerConfig.getSleepWindow()); } + if (null == countersExpireInterval) { + setCountersExpireInterval(circuitBreakerConfig.getCountersExpireInterval()); + } if (null == requestCountAfterHalfOpen) { setRequestCountAfterHalfOpen(circuitBreakerConfig.getRequestCountAfterHalfOpen()); } @@ -181,7 +201,6 @@ public void setDefault(Object defaultObject) { } @Override - @SuppressWarnings("checkstyle:all") public String toString() { return "CircuitBreakerConfigImpl{" + "enable=" + enable + @@ -191,6 +210,7 @@ public String toString() { ", requestCountAfterHalfOpen=" + requestCountAfterHalfOpen + ", successCountAfterHalfOpen=" + successCountAfterHalfOpen + ", enableRemotePull=" + enableRemotePull + - "} " + super.toString(); + ", countersExpireInterval=" + countersExpireInterval + + '}'; } } diff --git a/polaris-common/polaris-model/src/main/java/com/tencent/polaris/client/util/NamedThreadFactory.java b/polaris-common/polaris-model/src/main/java/com/tencent/polaris/client/util/NamedThreadFactory.java index 31469896f..dc4929d3a 100644 --- a/polaris-common/polaris-model/src/main/java/com/tencent/polaris/client/util/NamedThreadFactory.java +++ b/polaris-common/polaris-model/src/main/java/com/tencent/polaris/client/util/NamedThreadFactory.java @@ -29,7 +29,7 @@ public class NamedThreadFactory implements ThreadFactory { private final AtomicInteger idx = new AtomicInteger(0); public NamedThreadFactory(String component) { - this(component, false); + this(component, true); } public NamedThreadFactory(String component, boolean daemon) { diff --git a/polaris-plugins/polaris-plugin-api/src/main/java/com/tencent/polaris/api/plugin/circuitbreaker/entity/SubsetResource.java b/polaris-plugins/polaris-plugin-api/src/main/java/com/tencent/polaris/api/plugin/circuitbreaker/entity/SubsetResource.java deleted file mode 100644 index 8c21c4aa7..000000000 --- a/polaris-plugins/polaris-plugin-api/src/main/java/com/tencent/polaris/api/plugin/circuitbreaker/entity/SubsetResource.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making Polaris available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package com.tencent.polaris.api.plugin.circuitbreaker.entity; - -import com.tencent.polaris.api.pojo.ServiceKey; -import com.tencent.polaris.client.util.CommonValidator; -import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.Level; -import com.tencent.polaris.specification.api.v1.model.ModelProto.MatchString; -import java.util.Map; -import java.util.Objects; - -public class SubsetResource extends AbstractResource { - - private final String subset; - - private final Map metadata; - - public SubsetResource(ServiceKey service, String subset, Map metadata) { - this(service, subset, metadata, null); - } - - public SubsetResource(ServiceKey service, String subset, Map metadata, - ServiceKey callerService) { - super(service, callerService); - CommonValidator.validateService(service); - CommonValidator.validateNamespaceService(service.getNamespace(), service.getService()); - CommonValidator.validateText(subset, "subset"); - this.subset = subset; - this.metadata = metadata; - } - - @Override - public Level getLevel() { - return Level.GROUP; - } - - public ServiceKey getService() { - return service; - } - - public String getSubset() { - return subset; - } - - public Map getMetadata() { - return metadata; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof SubsetResource)) { - return false; - } - if (!super.equals(o)) { - return false; - } - SubsetResource that = (SubsetResource) o; - return Objects.equals(subset, that.subset); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), subset); - } - - @Override - public String toString() { - return "SubsetResource{" + - "subset='" + subset + '\'' + - ", metadata=" + metadata + - "} " + super.toString(); - } -} diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/pom.xml b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/pom.xml index d8b85950b..cfcd51d75 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/pom.xml +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/pom.xml @@ -20,12 +20,23 @@ circuitbreaker-common ${project.version} + + com.google.guava + guava + ${guava.version} + org.slf4j slf4j-api ${slf4j.version} provided + + com.tencent.polaris + polaris-test-common + ${project.version} + test + com.tencent.polaris polaris-test-mock-discovery diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleContainer.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleContainer.java deleted file mode 100644 index f3381bf37..000000000 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleContainer.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making Polaris available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package com.tencent.polaris.plugins.circuitbreaker.composite; - -import static com.tencent.polaris.plugins.circuitbreaker.composite.MatchUtils.isWildcardMatcherSingle; -import static com.tencent.polaris.plugins.circuitbreaker.composite.MatchUtils.matchMethod; -import static com.tencent.polaris.plugins.circuitbreaker.composite.MatchUtils.matchService; - -import com.tencent.polaris.api.plugin.cache.FlowCache; -import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; -import com.tencent.polaris.api.pojo.ServiceEventKey; -import com.tencent.polaris.api.pojo.ServiceEventKey.EventType; -import com.tencent.polaris.api.pojo.ServiceKey; -import com.tencent.polaris.api.pojo.ServiceRule; -import com.tencent.polaris.api.utils.StringUtils; -import com.tencent.polaris.logging.LoggerFactory; -import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreaker; -import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreakerRule; -import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.Level; -import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.RuleMatcher; -import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.RuleMatcher.DestinationService; -import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.RuleMatcher.SourceService; -import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto.FaultDetector; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.regex.Pattern; -import org.slf4j.Logger; - -public class CircuitBreakerRuleContainer { - - private static final Logger LOG = LoggerFactory.getLogger(CircuitBreakerRuleContainer.class); - - private final Resource resource; - - private final PolarisCircuitBreaker polarisCircuitBreaker; - - private final Function regexToPattern; - - public CircuitBreakerRuleContainer(Resource resource, PolarisCircuitBreaker polarisCircuitBreaker) { - this.resource = resource; - this.polarisCircuitBreaker = polarisCircuitBreaker; - this.regexToPattern = regex -> { - FlowCache flowCache = polarisCircuitBreaker.getExtensions().getFlowCache(); - return flowCache.loadOrStoreCompiledRegex(regex); - }; - scheduleCircuitBreaker(); - } - - private class PullCircuitBreakerRuleTask implements Runnable { - - @Override - public void run() { - synchronized (resource) { - ServiceEventKey cbEventKey = new ServiceEventKey(resource.getService(), - EventType.CIRCUIT_BREAKING); - ServiceRule circuitBreakRule; - try { - circuitBreakRule = polarisCircuitBreaker.getServiceRuleProvider().getServiceRule(cbEventKey); - } catch (Throwable t) { - LOG.warn("fail to get resource for {}", cbEventKey, t); - polarisCircuitBreaker.getPullRulesExecutors() - .schedule(new PullCircuitBreakerRuleTask(), 5, TimeUnit.SECONDS); - return; - } - Map resourceResourceCounters = polarisCircuitBreaker.getCountersCache() - .get(resource.getLevel()); - CircuitBreakerRule circuitBreakerRule = selectRule(resource, circuitBreakRule, regexToPattern); - if (null != circuitBreakerRule) { - ResourceCounters resourceCounters = resourceResourceCounters.get(resource); - if (null != resourceCounters) { - CircuitBreakerRule currentActiveRule = resourceCounters.getCurrentActiveRule(); - if (StringUtils.equals(currentActiveRule.getId(), circuitBreakerRule.getId()) && StringUtils - .equals(currentActiveRule.getRevision(), circuitBreakerRule.getRevision())) { - return; - } - } - resourceResourceCounters.put(resource, new ResourceCounters(resource, circuitBreakerRule, - polarisCircuitBreaker.getStateChangeExecutors(), polarisCircuitBreaker)); - scheduleHealthCheck(); - } else { - ResourceCounters oldCounters = resourceResourceCounters.remove(resource); - if (null != oldCounters) { - LOG.info("start to destroyed counters {} for resource {}", oldCounters.getCurrentActiveRule().getName(), resource); - // remove the old health check scheduler - oldCounters.setDestroyed(true); - scheduleHealthCheck(); - } - } - } - } - } - - private class PullFaultDetectTask implements Runnable { - - @Override - public void run() { - synchronized (resource) { - LOG.info("start to pull fault detect rule for resource {}", resource); - ResourceCounters resourceCounters = polarisCircuitBreaker.getCountersCache().get(resource.getLevel()) - .get(resource); - boolean faultDetectEnabled = false; - CircuitBreakerRule currentActiveRule = null; - if (null != resourceCounters) { - currentActiveRule = resourceCounters.getCurrentActiveRule(); - } - if (null != currentActiveRule && currentActiveRule.getEnable() && currentActiveRule - .getFaultDetectConfig().getEnable()) { - ServiceEventKey fdEventKey = new ServiceEventKey(resource.getService(), - EventType.FAULT_DETECTING); - ServiceRule faultDetectRule; - try { - faultDetectRule = polarisCircuitBreaker.getServiceRuleProvider().getServiceRule(fdEventKey); - } catch (Throwable t) { - LOG.warn("fail to get resource for {}", fdEventKey, t); - polarisCircuitBreaker.getPullRulesExecutors() - .schedule(new PullFaultDetectTask(), 5, TimeUnit.SECONDS); - return; - } - FaultDetector faultDetector = selectFaultDetector(faultDetectRule); - Map healthCheckCache = polarisCircuitBreaker.getHealthCheckCache(); - Map> serviceHealthCheckCache = polarisCircuitBreaker - .getServiceHealthCheckCache(); - if (null != faultDetector) { - ResourceHealthChecker curChecker = healthCheckCache.get(resource); - if (null != curChecker) { - FaultDetector currentRule = curChecker.getFaultDetector(); - if (StringUtils.equals(faultDetector.getRevision(), currentRule.getRevision())) { - return; - } - curChecker.stop(); - } - ResourceHealthChecker newHealthChecker = new ResourceHealthChecker(resource, faultDetector, - polarisCircuitBreaker); - healthCheckCache.put(resource, newHealthChecker); - if (resource.getLevel() != Level.INSTANCE) { - ServiceKey svcKey = resource.getService(); - Map resourceHealthCheckerMap = serviceHealthCheckCache - .get(svcKey); - if (null == resourceHealthCheckerMap) { - resourceHealthCheckerMap = new ConcurrentHashMap<>(); - serviceHealthCheckCache.put(svcKey, resourceHealthCheckerMap); - } - resourceHealthCheckerMap.put(resource, newHealthChecker); - } - faultDetectEnabled = true; - } - } - if (!faultDetectEnabled) { - LOG.info("health check for resource {} is disabled, now stop the previous checker", resource); - Map healthCheckCache = polarisCircuitBreaker.getHealthCheckCache(); - ResourceHealthChecker preChecker = healthCheckCache.remove(resource); - if (null != preChecker) { - preChecker.stop(); - } - if (resource.getLevel() != Level.INSTANCE) { - ServiceKey svcKey = resource.getService(); - Map resourceHealthCheckerMap = polarisCircuitBreaker - .getServiceHealthCheckCache().get(svcKey); - if (null != resourceHealthCheckerMap) { - resourceHealthCheckerMap.remove(resource); - if (resourceHealthCheckerMap.isEmpty()) { - polarisCircuitBreaker.getServiceHealthCheckCache().remove(svcKey); - } - } - } - } - } - } - } - - private FaultDetector selectFaultDetector(ServiceRule serviceRule) { - if (null == serviceRule) { - return null; - } - Object rule = serviceRule.getRule(); - if (null == rule) { - return null; - } - return (FaultDetector) rule; - } - - private static List sortCircuitBreakerRules(List rules) { - List outRules = new ArrayList<>(rules); - outRules.sort(new Comparator() { - @Override - public int compare(CircuitBreakerRule rule1, CircuitBreakerRule rule2) { - // 1. compare destination service - RuleMatcher ruleMatcher1 = rule1.getRuleMatcher(); - String destNamespace1 = ruleMatcher1.getDestination().getNamespace(); - String destService1 = ruleMatcher1.getDestination().getService(); - String destMethod1 = ruleMatcher1.getDestination().getMethod().getValue().getValue(); - - RuleMatcher ruleMatcher2 = rule2.getRuleMatcher(); - String destNamespace2 = ruleMatcher2.getDestination().getNamespace(); - String destService2 = ruleMatcher2.getDestination().getService(); - String destMethod2 = ruleMatcher2.getDestination().getMethod().getValue().getValue(); - - int svcResult = compareService(destNamespace1, destService1, destNamespace2, destService2); - if (svcResult != 0) { - return svcResult; - } - if (rule1.getLevel() == Level.METHOD && rule1.getLevel() == rule2.getLevel()) { - int methodResult = compareSingleValue(destMethod1, destMethod2); - if (methodResult != 0) { - return methodResult; - } - } - // 2. compare source service - String srcNamespace1 = ruleMatcher1.getSource().getNamespace(); - String srcService1 = ruleMatcher1.getSource().getService(); - String srcNamespace2 = ruleMatcher2.getSource().getNamespace(); - String srcService2 = ruleMatcher2.getSource().getService(); - return compareService(srcNamespace1, srcService1, srcNamespace2, srcService2); - } - }); - return outRules; - } - - public static int compareSingleValue(String value1, String value2) { - boolean serviceWildcard1 = isWildcardMatcherSingle(value1); - boolean serviceWildcard2 = isWildcardMatcherSingle(value2); - if (serviceWildcard1 && serviceWildcard2) { - return 0; - } - if (serviceWildcard1) { - // 1 before 2 - return 1; - } - if (serviceWildcard2) { - // 1 before 2 - return -1; - } - return value1.compareTo(value2); - } - - public static int compareService(String namespace1, String service1, String namespace2, String service2) { - int nsResult = compareSingleValue(namespace1, namespace2); - if (nsResult != 0) { - return nsResult; - } - return compareSingleValue(service1, service2); - } - - public static CircuitBreakerRule selectRule(Resource resource, ServiceRule serviceRule, - Function regexToPattern) { - if (null == serviceRule) { - return null; - } - Object rule = serviceRule.getRule(); - if (null == rule) { - return null; - } - CircuitBreaker circuitBreaker = (CircuitBreaker) rule; - List rules = circuitBreaker.getRulesList(); - if (rules.isEmpty()) { - return null; - } - List sortedRules = sortCircuitBreakerRules(rules); - for (CircuitBreakerRule cbRule : sortedRules) { - if (!cbRule.getEnable()) { - continue; - } - Level level = resource.getLevel(); - if (cbRule.getLevel() != level) { - continue; - } - RuleMatcher ruleMatcher = cbRule.getRuleMatcher(); - DestinationService destination = ruleMatcher.getDestination(); - if (!matchService(resource.getService(), destination.getNamespace(), destination.getService())) { - continue; - } - SourceService source = ruleMatcher.getSource(); - if (!matchService(resource.getCallerService(), source.getNamespace(), source.getService())) { - continue; - } - boolean methodMatched = matchMethod(resource, destination.getMethod(), regexToPattern); - if (methodMatched) { - return cbRule; - } - } - return null; - } - - public void scheduleCircuitBreaker() { - polarisCircuitBreaker.getPullRulesExecutors() - .schedule(new PullCircuitBreakerRuleTask(), 50, TimeUnit.MILLISECONDS); - } - - public void scheduleHealthCheck() { - polarisCircuitBreaker.getPullRulesExecutors().schedule( - new PullFaultDetectTask(), 100, TimeUnit.MILLISECONDS); - } - - public Resource getResource() { - return resource; - } -} diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleDictionary.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleDictionary.java new file mode 100644 index 000000000..d42e58fae --- /dev/null +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleDictionary.java @@ -0,0 +1,181 @@ +/* + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.plugins.circuitbreaker.composite; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.regex.Pattern; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; +import com.tencent.polaris.api.pojo.ServiceKey; +import com.tencent.polaris.api.pojo.ServiceRule; +import com.tencent.polaris.api.utils.CollectionUtils; +import com.tencent.polaris.plugins.circuitbreaker.composite.utils.CircuitBreakerUtils; +import com.tencent.polaris.plugins.circuitbreaker.composite.utils.MatchUtils; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; + +import static com.tencent.polaris.plugins.circuitbreaker.composite.utils.MatchUtils.matchMethod; + +public class CircuitBreakerRuleDictionary { + + private final Map>> allRules = new HashMap<>(); + + private final Function regexToPattern; + + private final Object updateLock = new Object(); + + public CircuitBreakerRuleDictionary(Function regexToPattern) { + this.regexToPattern = regexToPattern; + allRules.put(CircuitBreakerProto.Level.SERVICE, CacheBuilder.newBuilder().build()); + allRules.put(CircuitBreakerProto.Level.METHOD, CacheBuilder.newBuilder().build()); + allRules.put(CircuitBreakerProto.Level.INSTANCE, CacheBuilder.newBuilder().build()); + } + + public CircuitBreakerProto.CircuitBreakerRule lookupCircuitBreakerRule(Resource resource) { + synchronized (updateLock) { + Cache> serviceKeyListCache = allRules.get(resource.getLevel()); + if (null == serviceKeyListCache) { + return null; + } + return selectRule(resource, serviceKeyListCache.getIfPresent(resource.getService()), regexToPattern); + } + } + + private static CircuitBreakerProto.CircuitBreakerRule selectRule(Resource resource, + List sortedRules, Function regexToPattern) { + if (CollectionUtils.isEmpty(sortedRules)) { + return null; + } + for (CircuitBreakerProto.CircuitBreakerRule cbRule : sortedRules) { + CircuitBreakerProto.RuleMatcher ruleMatcher = cbRule.getRuleMatcher(); + CircuitBreakerProto.RuleMatcher.DestinationService destination = ruleMatcher.getDestination(); + if (!MatchUtils.matchService(resource.getService(), destination.getNamespace(), destination.getService())) { + continue; + } + CircuitBreakerProto.RuleMatcher.SourceService source = ruleMatcher.getSource(); + if (!MatchUtils.matchService(resource.getCallerService(), source.getNamespace(), source.getService())) { + continue; + } + boolean methodMatched = matchMethod(resource, destination.getMethod(), regexToPattern); + if (methodMatched) { + return cbRule; + } + } + return null; + } + + /** + * rule on server has been changed, clear all caches to make it pull again + * @param serviceKey target service + */ + public void onServiceChanged(ServiceKey serviceKey) { + synchronized (updateLock) { + clearRules(serviceKey); + } + } + + private void clearRules(ServiceKey serviceKey) { + allRules.get(CircuitBreakerProto.Level.SERVICE).invalidate(serviceKey); + allRules.get(CircuitBreakerProto.Level.METHOD).invalidate(serviceKey); + allRules.get(CircuitBreakerProto.Level.INSTANCE).invalidate(serviceKey); + } + + public void putServiceRule(ServiceKey serviceKey, ServiceRule serviceRule) { + synchronized (updateLock) { + if (null == serviceRule || null == serviceRule.getRule()) { + clearRules(serviceKey); + return; + } + CircuitBreakerProto.CircuitBreaker circuitBreaker = (CircuitBreakerProto.CircuitBreaker) serviceRule.getRule(); + List rules = circuitBreaker.getRulesList(); + List subServiceRules = new ArrayList<>(); + List subMethodRules = new ArrayList<>(); + List subInstanceRules = new ArrayList<>(); + for (CircuitBreakerProto.CircuitBreakerRule rule : rules) { + if (!rule.getEnable() || !CircuitBreakerUtils.checkRule(rule)) { + continue; + } + CircuitBreakerProto.Level level = rule.getLevel(); + switch (level) { + case SERVICE: + subServiceRules.add(rule); + break; + case INSTANCE: + subInstanceRules.add(rule); + break; + case METHOD: + subMethodRules.add(rule); + break; + } + } + subServiceRules = sortCircuitBreakerRules(subServiceRules); + subMethodRules = sortCircuitBreakerRules(subMethodRules); + subInstanceRules = sortCircuitBreakerRules(subInstanceRules); + allRules.get(CircuitBreakerProto.Level.SERVICE).put(serviceKey, subServiceRules); + allRules.get(CircuitBreakerProto.Level.METHOD).put(serviceKey, subMethodRules); + allRules.get(CircuitBreakerProto.Level.INSTANCE).put(serviceKey, subInstanceRules); + } + } + + private static List sortCircuitBreakerRules(List rules) { + if (CollectionUtils.isEmpty(rules)) { + return rules; + } + List outRules = new ArrayList<>(rules); + outRules.sort(new Comparator() { + @Override + public int compare(CircuitBreakerProto.CircuitBreakerRule rule1, CircuitBreakerProto.CircuitBreakerRule rule2) { + // 1. compare destination service + CircuitBreakerProto.RuleMatcher ruleMatcher1 = rule1.getRuleMatcher(); + String destNamespace1 = ruleMatcher1.getDestination().getNamespace(); + String destService1 = ruleMatcher1.getDestination().getService(); + String destMethod1 = ruleMatcher1.getDestination().getMethod().getValue().getValue(); + + CircuitBreakerProto.RuleMatcher ruleMatcher2 = rule2.getRuleMatcher(); + String destNamespace2 = ruleMatcher2.getDestination().getNamespace(); + String destService2 = ruleMatcher2.getDestination().getService(); + String destMethod2 = ruleMatcher2.getDestination().getMethod().getValue().getValue(); + + int svcResult = MatchUtils.compareService(destNamespace1, destService1, destNamespace2, destService2); + if (svcResult != 0) { + return svcResult; + } + if (rule1.getLevel() == CircuitBreakerProto.Level.METHOD && rule1.getLevel() == rule2.getLevel()) { + int methodResult = MatchUtils.compareSingleValue(destMethod1, destMethod2); + if (methodResult != 0) { + return methodResult; + } + } + // 2. compare source service + String srcNamespace1 = ruleMatcher1.getSource().getNamespace(); + String srcService1 = ruleMatcher1.getSource().getService(); + String srcNamespace2 = ruleMatcher2.getSource().getNamespace(); + String srcService2 = ruleMatcher2.getSource().getService(); + return MatchUtils.compareService(srcNamespace1, srcService1, srcNamespace2, srcService2); + } + }); + return outRules; + } + +} diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleListener.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleListener.java index 4c6aba3ce..5ff950ca4 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleListener.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/CircuitBreakerRuleListener.java @@ -17,13 +17,11 @@ package com.tencent.polaris.plugins.circuitbreaker.composite; -import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; import com.tencent.polaris.api.plugin.registry.AbstractResourceEventListener; import com.tencent.polaris.api.pojo.RegistryCacheValue; import com.tencent.polaris.api.pojo.ServiceEventKey; import com.tencent.polaris.api.pojo.ServiceEventKey.EventType; import com.tencent.polaris.logging.LoggerFactory; -import java.util.Map; import org.slf4j.Logger; public class CircuitBreakerRuleListener extends AbstractResourceEventListener { @@ -44,7 +42,11 @@ public void onResourceAdd(ServiceEventKey svcEventKey, RegistryCacheValue newVal return; } LOG.info("[CircuitBreaker] onResourceAdd {}", svcEventKey); - doSchedule(svcEventKey); + if (svcEventKey.getEventType() == EventType.CIRCUIT_BREAKING) { + polarisCircuitBreaker.onCircuitBreakerRuleAdded(svcEventKey.getServiceKey()); + } else if (svcEventKey.getEventType() == EventType.FAULT_DETECTING) { + polarisCircuitBreaker.onFaultDetectRuleChanged(svcEventKey.getServiceKey(), newValue); + } } @Override @@ -55,24 +57,9 @@ public void onResourceUpdated(ServiceEventKey svcEventKey, RegistryCacheValue ol return; } LOG.info("[CircuitBreaker] onResourceUpdated {}", svcEventKey); - doSchedule(svcEventKey); - } - - private void doSchedule(ServiceEventKey svcEventKey) { - for (Map.Entry entry : polarisCircuitBreaker.getContainers() - .entrySet()) { - if (entry.getKey().getService().equals(svcEventKey.getServiceKey())) { - switch (svcEventKey.getEventType()) { - case CIRCUIT_BREAKING: - entry.getValue().scheduleCircuitBreaker(); - break; - case FAULT_DETECTING: - entry.getValue().scheduleHealthCheck(); - break; - default: - break; - } - } + onChanged(svcEventKey); + if (svcEventKey.getEventType() == EventType.FAULT_DETECTING) { + polarisCircuitBreaker.onFaultDetectRuleChanged(svcEventKey.getServiceKey(), newValue); } } @@ -83,7 +70,18 @@ public void onResourceDeleted(ServiceEventKey svcEventKey, RegistryCacheValue ol return; } LOG.info("[CircuitBreaker] onResourceDeleted {}", svcEventKey); - doSchedule(svcEventKey); + onChanged(svcEventKey); + if (svcEventKey.getEventType() == EventType.FAULT_DETECTING) { + polarisCircuitBreaker.onFaultDetectRuleDeleted(svcEventKey.getServiceKey(), oldValue); + } } + private void onChanged(ServiceEventKey svcEventKey) { + if (svcEventKey.getEventType() == EventType.CIRCUIT_BREAKING) { + polarisCircuitBreaker.onCircuitBreakerRuleChanged(svcEventKey.getServiceKey()); + } + } + + + } diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/FaultDetectRuleDictionary.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/FaultDetectRuleDictionary.java new file mode 100644 index 000000000..20ac97c15 --- /dev/null +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/FaultDetectRuleDictionary.java @@ -0,0 +1,108 @@ +/* + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.plugins.circuitbreaker.composite; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; +import com.tencent.polaris.api.pojo.ServiceKey; +import com.tencent.polaris.api.pojo.ServiceRule; +import com.tencent.polaris.api.utils.CollectionUtils; +import com.tencent.polaris.plugins.circuitbreaker.composite.utils.MatchUtils; +import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto; + +public class FaultDetectRuleDictionary { + + // key is target service, value is list FaultDetectRule + private final Map> serviceRules = new ConcurrentHashMap<>(); + + private final Object updateLock = new Object(); + + public List lookupFaultDetectRule(Resource resource) { + ServiceKey targetService = resource.getService(); + return serviceRules.get(targetService); + } + + /** + * rule on server has been changed, clear all caches to make it pull again + * @param svcKey target service + */ + public void onFaultDetectRuleChanged(ServiceKey svcKey, FaultDetectorProto.FaultDetector faultDetector) { + synchronized (updateLock) { + putServiceRule(svcKey, faultDetector); + } + } + + void onFaultDetectRuleDeleted(ServiceKey svcKey) { + synchronized (updateLock) { + serviceRules.remove(svcKey); + } + } + + public void putServiceRule(ServiceKey serviceKey, ServiceRule serviceRule) { + synchronized (updateLock) { + if (null == serviceRule) { + serviceRules.remove(serviceKey); + return; + } + putServiceRule(serviceKey, (FaultDetectorProto.FaultDetector) serviceRule.getRule()); + } + } + + public void putServiceRule(ServiceKey serviceKey, FaultDetectorProto.FaultDetector faultDetector) { + if (null == faultDetector) { + serviceRules.remove(serviceKey); + return; + } + List rules = faultDetector.getRulesList(); + if (CollectionUtils.isNotEmpty(rules)) { + rules = sortFaultDetectRules(rules); + } + serviceRules.put(serviceKey, rules); + } + + private static List sortFaultDetectRules(List rules) { + List outRules = new ArrayList<>(rules); + outRules.sort(new Comparator() { + @Override + public int compare(FaultDetectorProto.FaultDetectRule rule1, FaultDetectorProto.FaultDetectRule rule2) { + // 1. compare destination service + FaultDetectorProto.FaultDetectRule.DestinationService targetService1 = rule1.getTargetService(); + String destNamespace1 = targetService1.getNamespace(); + String destService1 = targetService1.getService(); + String destMethod1 = targetService1.getMethod().getValue().getValue(); + + FaultDetectorProto.FaultDetectRule.DestinationService targetService2 = rule2.getTargetService(); + String destNamespace2 = targetService2.getNamespace(); + String destService2 = targetService2.getService(); + String destMethod2 = targetService2.getMethod().getValue().getValue(); + + int svcResult = MatchUtils.compareService(destNamespace1, destService1, destNamespace2, destService2); + if (svcResult != 0) { + return svcResult; + } + return MatchUtils.compareSingleValue(destMethod1, destMethod2); + } + }); + return outRules; + } +} diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckContainer.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckContainer.java new file mode 100644 index 000000000..6436ffc05 --- /dev/null +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckContainer.java @@ -0,0 +1,147 @@ +/* + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.plugins.circuitbreaker.composite; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import com.tencent.polaris.api.plugin.circuitbreaker.entity.InstanceResource; +import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; +import com.tencent.polaris.api.pojo.ServiceKey; +import com.tencent.polaris.api.utils.CollectionUtils; +import com.tencent.polaris.client.pojo.Node; +import com.tencent.polaris.logging.LoggerFactory; +import com.tencent.polaris.plugins.circuitbreaker.composite.utils.HealthCheckUtils; +import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto; +import org.slf4j.Logger; + +import static com.tencent.polaris.logging.LoggingConsts.LOGGING_HEALTHCHECK_EVENT; + +public class HealthCheckContainer implements HealthCheckInstanceProvider { + + private static final Logger HC_EVENT_LOG = LoggerFactory.getLogger(LOGGING_HEALTHCHECK_EVENT); + + private static final Logger LOG = LoggerFactory.getLogger(HealthCheckContainer.class); + + private final ServiceKey serviceKey; + + private final Object updateLock = new Object(); + + // key is ruleId, value is checker + private final List healthCheckers = new ArrayList<>(); + + private final Map instances = new ConcurrentHashMap<>(); + + private final long expireIntervalMilli; + + private final ScheduledFuture future; + + public HealthCheckContainer(ServiceKey serviceKey, + List faultDetectRules, PolarisCircuitBreaker polarisCircuitBreaker) { + long checkPeriod = polarisCircuitBreaker.getCheckPeriod(); + expireIntervalMilli = polarisCircuitBreaker.getHealthCheckInstanceExpireInterval(); + this.serviceKey = serviceKey; + LOG.info("schedule expire task: service {}, interval {}", serviceKey, checkPeriod); + future = polarisCircuitBreaker.getHealthCheckExecutors().scheduleWithFixedDelay(new Runnable() { + @Override + public void run() { + cleanInstances(); + } + }, checkPeriod, checkPeriod, TimeUnit.MILLISECONDS); + if (CollectionUtils.isNotEmpty(faultDetectRules)) { + for (FaultDetectorProto.FaultDetectRule faultDetectRule : faultDetectRules) { + ResourceHealthChecker resourceHealthChecker = new ResourceHealthChecker(faultDetectRule, this, polarisCircuitBreaker); + resourceHealthChecker.start(); + healthCheckers.add(resourceHealthChecker); + } + } + } + + public void addInstance(InstanceResource instanceResource) { + ResourceHealthChecker.ProtocolInstance instance = instances.computeIfAbsent(instanceResource.getNode(), new Function() { + @Override + public ResourceHealthChecker.ProtocolInstance apply(Node node) { + HC_EVENT_LOG.info("add fault detect instance {}, service {}", instanceResource.getNode(), serviceKey); + return new ResourceHealthChecker.ProtocolInstance(HealthCheckUtils.parseProtocol(instanceResource.getProtocol()), + instanceResource); + } + }); + instance.doReport(); + } + + @Override + public Map getInstances() { + return Collections.unmodifiableMap(instances); + } + + public Collection getHealthCheckerValues() { + return Collections.unmodifiableCollection(healthCheckers); + } + + public void cleanInstances() { + long curTimeMilli = System.currentTimeMillis(); + for (Map.Entry entry : instances.entrySet()) { + ResourceHealthChecker.ProtocolInstance protocolInstance = entry.getValue(); + long lastReportMilli = protocolInstance.getLastReportMilli(); + Node node = entry.getKey(); + if (!protocolInstance.isCheckSuccess() && curTimeMilli - lastReportMilli >= expireIntervalMilli) { + instances.remove(node); + HC_EVENT_LOG + .info("clean instance from health check tasks, service {}, expired node {}, lastReportMilli {}", + serviceKey, node, lastReportMilli); + } + } + } + + public void stop() { + LOG.info("health check container for service {} has stopped", serviceKey); + synchronized (updateLock) { + for (ResourceHealthChecker resourceHealthChecker : healthCheckers) { + resourceHealthChecker.stop(); + } + } + future.cancel(true); + } + + public void addResource(Resource resource) { + synchronized (updateLock) { + for (ResourceHealthChecker resourceHealthChecker : getHealthCheckerValues()) { + if (resourceHealthChecker.matchResource(resource)) { + resourceHealthChecker.addResource(resource); + // 每个资源只匹配优先级最高的探测规则 + break; + } + } + } + } + + public void removeResource(Resource resource) { + synchronized (updateLock) { + for (ResourceHealthChecker resourceHealthChecker : getHealthCheckerValues()) { + resourceHealthChecker.removeResource(resource); + } + } + } +} diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckInstanceProvider.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckInstanceProvider.java new file mode 100644 index 000000000..c6e5eca00 --- /dev/null +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckInstanceProvider.java @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.plugins.circuitbreaker.composite; + +import java.util.Map; + +import com.tencent.polaris.client.pojo.Node; + +public interface HealthCheckInstanceProvider { + + /** + * provide instances to check resources + * @return instances + */ + Map getInstances(); +} diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/MatchUtils.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/MatchUtils.java deleted file mode 100644 index 5bdb4262a..000000000 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/MatchUtils.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making Polaris available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package com.tencent.polaris.plugins.circuitbreaker.composite; - -import com.tencent.polaris.api.plugin.circuitbreaker.entity.MethodResource; -import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; -import com.tencent.polaris.api.pojo.ServiceKey; -import com.tencent.polaris.api.utils.RuleUtils; -import com.tencent.polaris.api.utils.StringUtils; -import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.Level; -import com.tencent.polaris.specification.api.v1.model.ModelProto.MatchString; -import java.util.function.Function; -import java.util.regex.Pattern; - -public class MatchUtils { - - public static boolean matchService(ServiceKey serviceKey, String namespace, String service) { - String inputNamespace = ""; - String inputService = ""; - if (null != serviceKey) { - inputNamespace = serviceKey.getNamespace(); - inputService = serviceKey.getService(); - } - if (StringUtils.isNotBlank(namespace) && !StringUtils.equals(namespace, RuleUtils.MATCH_ALL) && !StringUtils - .equals(inputNamespace, namespace)) { - return false; - } - if (StringUtils.isNotBlank(service) && !StringUtils.equals(service, RuleUtils.MATCH_ALL) && !StringUtils - .equals(inputService, service)) { - return false; - } - return true; - } - - public static boolean matchMethod(Resource resource, MatchString matchString, - Function regexToPattern) { - if (resource.getLevel() != Level.METHOD) { - return true; - } - String method = ((MethodResource) resource).getMethod(); - return RuleUtils.matchStringValue(matchString, method, regexToPattern); - } - - public static boolean isWildcardMatcherSingle(String name) { - return name.equals(RuleUtils.MATCH_ALL) || StringUtils.isBlank(name); - } -} diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java index 501761773..1f7801814 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java @@ -17,6 +17,25 @@ package com.tencent.polaris.plugins.circuitbreaker.composite; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Function; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; +import com.tencent.polaris.api.config.consumer.CircuitBreakerConfig; import com.tencent.polaris.api.config.plugin.DefaultPlugins; import com.tencent.polaris.api.control.Destroyable; import com.tencent.polaris.api.exception.PolarisException; @@ -30,215 +49,441 @@ import com.tencent.polaris.api.plugin.compose.Extensions; import com.tencent.polaris.api.plugin.detect.HealthChecker; import com.tencent.polaris.api.pojo.CircuitBreakerStatus; +import com.tencent.polaris.api.pojo.RegistryCacheValue; import com.tencent.polaris.api.pojo.RetStatus; +import com.tencent.polaris.api.pojo.ServiceEventKey; import com.tencent.polaris.api.pojo.ServiceKey; import com.tencent.polaris.api.pojo.ServiceResourceProvider; +import com.tencent.polaris.api.pojo.ServiceRule; +import com.tencent.polaris.api.utils.CollectionUtils; import com.tencent.polaris.client.flow.DefaultServiceResourceProvider; import com.tencent.polaris.client.util.NamedThreadFactory; +import com.tencent.polaris.logging.LoggerFactory; +import com.tencent.polaris.plugins.circuitbreaker.composite.utils.CircuitBreakerUtils; +import com.tencent.polaris.plugins.circuitbreaker.composite.utils.HealthCheckUtils; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.Level; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.function.Function; +import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto; +import org.slf4j.Logger; public class PolarisCircuitBreaker extends Destroyable implements CircuitBreaker { - private final Map> countersCache = new HashMap<>(); - - private final Map healthCheckCache = new ConcurrentHashMap<>(); - - // service and method resource health checkers - private final Map> serviceHealthCheckCache = new ConcurrentHashMap<>(); - - private final ScheduledExecutorService stateChangeExecutors = new ScheduledThreadPoolExecutor(1, - new NamedThreadFactory("circuitbreaker-state-worker")); - - private final ScheduledExecutorService pullRulesExecutors = new ScheduledThreadPoolExecutor(1, - new NamedThreadFactory("circuitbreaker-pull-rules-worker")); - - private final ScheduledExecutorService healthCheckExecutors = new ScheduledThreadPoolExecutor(4, - new NamedThreadFactory("circuitbreaker-health-check-worker")); - - - private final Map containers = new ConcurrentHashMap<>(); - - private Extensions extensions; - - private ServiceResourceProvider serviceResourceProvider; - - private Map healthCheckers = Collections.emptyMap(); - - private long healthCheckInstanceExpireInterval; - - private long checkPeriod; - - @Override - public CircuitBreakerStatus checkResource(Resource resource) { - ResourceCounters resourceCounters = getResourceCounters(resource); - if (null == resourceCounters) { - return null; - } - return resourceCounters.getCircuitBreakerStatus(); - } - - private ResourceCounters getResourceCounters(Resource resource) { - Map resourceResourceCountersMap = countersCache.get(resource.getLevel()); - return resourceResourceCountersMap.get(resource); - } - - @Override - public void report(ResourceStat resourceStat) { - doReport(resourceStat, true); - } - - void doReport(ResourceStat resourceStat, boolean record) { - Resource resource = resourceStat.getResource(); - if (resource.getLevel() == Level.UNKNOWN) { - return; - } - RetStatus retStatus = resourceStat.getRetStatus(); - if (retStatus == RetStatus.RetReject || retStatus == RetStatus.RetFlowControl) { - return; - } - ResourceCounters resourceCounters = getResourceCounters(resource); - if (null == resourceCounters) { - containers.computeIfAbsent(resource, new Function() { - @Override - public CircuitBreakerRuleContainer apply(Resource resource) { - return new CircuitBreakerRuleContainer(resource, PolarisCircuitBreaker.this); - } - }); - } else { - resourceCounters.report(resourceStat); - } - addInstanceForHealthCheck(resourceStat.getResource(), record); - } - - private void addInstanceForHealthCheck(Resource resource, boolean record) { - if (!(resource instanceof InstanceResource)) { - return; - } - InstanceResource instanceResource = (InstanceResource) resource; - Map resourceResourceHealthCheckerMap = serviceHealthCheckCache - .get(instanceResource.getService()); - if (null == resourceResourceHealthCheckerMap) { - return; - } - for (ResourceHealthChecker resourceHealthChecker : resourceResourceHealthCheckerMap.values()) { - resourceHealthChecker.addInstance(instanceResource, record); - } - } - - @Override - public PluginType getType() { - return PluginTypes.CIRCUIT_BREAKER.getBaseType(); - } - - @Override - public void init(InitContext ctx) throws PolarisException { - countersCache.put(Level.SERVICE, new ConcurrentHashMap<>()); - countersCache.put(Level.METHOD, new ConcurrentHashMap<>()); - countersCache.put(Level.GROUP, new ConcurrentHashMap<>()); - countersCache.put(Level.INSTANCE, new ConcurrentHashMap<>()); - checkPeriod = 0; - if (null != ctx) { - checkPeriod = ctx.getConfig().getConsumer().getCircuitBreaker().getCheckPeriod(); - } - if (checkPeriod == 0) { - checkPeriod = HealthCheckUtils.DEFAULT_CHECK_INTERVAL; - } - healthCheckInstanceExpireInterval = HealthCheckUtils.CHECK_PERIOD_MULTIPLE * checkPeriod; - } - - @Override - public void postContextInit(Extensions extensions) throws PolarisException { - this.extensions = extensions; - serviceResourceProvider = new DefaultServiceResourceProvider(extensions); - extensions.getLocalRegistry().registerResourceListener(new CircuitBreakerRuleListener(this)); - healthCheckers = extensions.getAllHealthCheckers(); - } - - // for test - public void setServiceRuleProvider(ServiceResourceProvider serviceResourceProvider) { - this.serviceResourceProvider = serviceResourceProvider; - } - - public long getHealthCheckInstanceExpireInterval() { - return healthCheckInstanceExpireInterval; - } - - // for test - public void setHealthCheckInstanceExpireInterval(long healthCheckInstanceExpireInterval) { - this.healthCheckInstanceExpireInterval = healthCheckInstanceExpireInterval; - } - - public long getCheckPeriod() { - return checkPeriod; - } - - // for test - public void setCheckPeriod(long checkPeriod) { - this.checkPeriod = checkPeriod; - } - - @Override - protected void doDestroy() { - stateChangeExecutors.shutdown(); - pullRulesExecutors.shutdown(); - healthCheckExecutors.shutdown(); - } - - Map> getCountersCache() { - return countersCache; - } - - Map getHealthCheckCache() { - return healthCheckCache; - } - - Map> getServiceHealthCheckCache() { - return serviceHealthCheckCache; - } - - Extensions getExtensions() { - return extensions; - } - - ScheduledExecutorService getPullRulesExecutors() { - return pullRulesExecutors; - } - - ScheduledExecutorService getStateChangeExecutors() { - return stateChangeExecutors; - } - - ScheduledExecutorService getHealthCheckExecutors() { - return healthCheckExecutors; - } - - Map getContainers() { - return containers; - } - - public ServiceResourceProvider getServiceRuleProvider() { - return serviceResourceProvider; - } - - public Map getHealthCheckers() { - return healthCheckers; - } - - public void setHealthCheckers( - Map healthCheckers) { - this.healthCheckers = healthCheckers; - } - - @Override - public String getName() { - return DefaultPlugins.CIRCUIT_BREAKER_COMPOSITE; - } - + private static final Logger LOG = LoggerFactory.getLogger(PolarisCircuitBreaker.class); + + private final Map>> countersCache = new HashMap<>(); + + private final Map healthCheckCache = new ConcurrentHashMap<>(); + + private final ScheduledExecutorService stateChangeExecutors = new ScheduledThreadPoolExecutor(1, + new NamedThreadFactory("circuitbreaker-state-worker")); + + private final ScheduledExecutorService healthCheckExecutors = new ScheduledThreadPoolExecutor(4, + new NamedThreadFactory("circuitbreaker-health-check-worker")); + + private final ScheduledExecutorService expiredCleanupExecutors = new ScheduledThreadPoolExecutor(1, + new NamedThreadFactory("circuitbreaker-expired-cleanup-worker")); + + private Extensions extensions; + + private ServiceResourceProvider serviceResourceProvider; + + private Map healthCheckers = Collections.emptyMap(); + + private long healthCheckInstanceExpireInterval; + + private long checkPeriod; + + private CircuitBreakerRuleDictionary circuitBreakerRuleDictionary; + + private FaultDetectRuleDictionary faultDetectRuleDictionary; + + private CircuitBreakerConfig circuitBreakerConfig; + + @Override + public CircuitBreakerStatus checkResource(Resource resource) { + Optional resourceCounters = getResourceCounters(resource); + if (null == resourceCounters || !resourceCounters.isPresent()) { + return null; + } + return resourceCounters.get().getCircuitBreakerStatus(); + } + + private Optional getResourceCounters(Resource resource) { + Cache> resourceOptionalCache = countersCache.get(resource.getLevel()); + return resourceOptionalCache.getIfPresent(resource); + } + + @Override + public void report(ResourceStat resourceStat) { + doReport(resourceStat, true); + } + + private ResourceCounters getOrInitResourceCounters(Resource resource) throws ExecutionException { + Optional resourceCounters = getResourceCounters(resource); + boolean reloadFaultDetect = false; + if (null == resourceCounters) { + synchronized (countersCache) { + resourceCounters = getResourceCounters(resource); + if (null == resourceCounters) { + resourceCounters = initResourceCounter(resource); + reloadFaultDetect = true; + } + } + } + if (!reloadFaultDetect) { + if (null != resourceCounters && resourceCounters.isPresent() && resourceCounters.get().checkReloadFaultDetect()) { + reloadFaultDetect = true; + } + } + if (reloadFaultDetect) { + reloadFaultDetector(resource, resourceCounters.orElse(null)); + } + return resourceCounters.orElse(null); + } + + void doReport(ResourceStat resourceStat, boolean isNormalRequest) { + Resource resource = resourceStat.getResource(); + if (!CircuitBreakerUtils.checkLevel(resource.getLevel())) { + return; + } + RetStatus retStatus = resourceStat.getRetStatus(); + if (retStatus == RetStatus.RetReject || retStatus == RetStatus.RetFlowControl) { + return; + } + try { + ResourceCounters resourceCounters = getOrInitResourceCounters(resource); + if (null != resourceCounters) { + resourceCounters.report(resourceStat); + } + if (isNormalRequest) { + addInstanceForFaultDetect(resourceStat.getResource()); + } + } + catch (Throwable t) { + LOG.warn("error occur when report stat with {}", resource); + } + } + + private void reloadFaultDetector(Resource resource, ResourceCounters resourceCounters) { + boolean removeResource = false; + if (null == resourceCounters) { + removeResource = true; + } + else { + CircuitBreakerProto.CircuitBreakerRule circuitBreakerRule = resourceCounters.getCurrentActiveRule(); + if (!circuitBreakerRule.hasFaultDetectConfig() || !circuitBreakerRule.getFaultDetectConfig().getEnable()) { + removeResource = true; + } + } + + HealthCheckContainer healthCheckContainer = healthCheckCache.get(resource.getService()); + if (removeResource) { + if (null == healthCheckContainer) { + return; + } + healthCheckContainer.removeResource(resource); + } + else { + if (null == healthCheckContainer) { + List faultDetectRules = faultDetectRuleDictionary.lookupFaultDetectRule(resource); + if (CollectionUtils.isNotEmpty(faultDetectRules)) { + healthCheckContainer = healthCheckCache.computeIfAbsent(resource.getService(), new Function() { + @Override + public HealthCheckContainer apply(ServiceKey serviceKey) { + return new HealthCheckContainer(serviceKey, faultDetectRules, PolarisCircuitBreaker.this); + } + }); + } + } + if (null != healthCheckContainer) { + healthCheckContainer.addResource(resource); + } + } + } + + private Optional initResourceCounter(Resource resource) throws ExecutionException { + CircuitBreakerProto.CircuitBreakerRule circuitBreakerRule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + if (null == circuitBreakerRule) { + // pull cb rule + ServiceEventKey cbEventKey = new ServiceEventKey(resource.getService(), + ServiceEventKey.EventType.CIRCUIT_BREAKING); + ServiceRule cbSvcRule; + try { + cbSvcRule = getServiceRuleProvider().getServiceRule(cbEventKey); + } + catch (Throwable t) { + LOG.warn("fail to get circuitBreaker rule resource for {}", cbEventKey, t); + throw t; + } + + // pull fd rule + ServiceEventKey fdEventKey = new ServiceEventKey(resource.getService(), + ServiceEventKey.EventType.FAULT_DETECTING); + ServiceRule faultDetectRule; + try { + faultDetectRule = getServiceRuleProvider().getServiceRule(fdEventKey); + } + catch (Throwable t) { + LOG.warn("fail to get faultDetect rule for {}", fdEventKey, t); + throw t; + } + + circuitBreakerRuleDictionary.putServiceRule(resource.getService(), cbSvcRule); + faultDetectRuleDictionary.putServiceRule(resource.getService(), faultDetectRule); + + // 查询适合的熔断规则 + circuitBreakerRule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + } + Cache> resourceOptionalCache = countersCache.get(resource.getLevel()); + CircuitBreakerProto.CircuitBreakerRule finalCircuitBreakerRule = circuitBreakerRule; + return resourceOptionalCache.get(resource, new Callable>() { + @Override + public Optional call() { + if (null == finalCircuitBreakerRule) { + return Optional.empty(); + } + return Optional.of(new ResourceCounters(resource, finalCircuitBreakerRule, + getStateChangeExecutors(), PolarisCircuitBreaker.this)); + } + }); + } + + private void addInstanceForFaultDetect(Resource resource) { + if (!(resource instanceof InstanceResource)) { + return; + } + InstanceResource instanceResource = (InstanceResource) resource; + HealthCheckContainer healthCheckContainer = healthCheckCache + .get(instanceResource.getService()); + if (null == healthCheckContainer) { + return; + } + healthCheckContainer.addInstance(instanceResource); + } + + @Override + public PluginType getType() { + return PluginTypes.CIRCUIT_BREAKER.getBaseType(); + } + + private class CounterRemoveListener implements RemovalListener> { + + @Override + public void onRemoval(RemovalNotification> removalNotification) { + Optional value = removalNotification.getValue(); + if (null == value) { + return; + } + value.ifPresent(resourceCounters -> resourceCounters.setDestroyed(true)); + Resource resource = removalNotification.getKey(); + if (null == resource) { + return; + } + HealthCheckContainer healthCheckContainer = PolarisCircuitBreaker.this.healthCheckCache.get(resource.getService()); + if (null != healthCheckContainer) { + healthCheckContainer.removeResource(resource); + } + } + } + + @Override + public void init(InitContext ctx) throws PolarisException { + long expireIntervalMilli = ctx.getConfig().getConsumer().getCircuitBreaker().getCountersExpireInterval(); + countersCache.put(Level.SERVICE, CacheBuilder.newBuilder().expireAfterAccess( + expireIntervalMilli, TimeUnit.MILLISECONDS).removalListener(new CounterRemoveListener()).build()); + countersCache.put(Level.METHOD, CacheBuilder.newBuilder().expireAfterAccess( + expireIntervalMilli, TimeUnit.MILLISECONDS).removalListener(new CounterRemoveListener()).build()); + countersCache.put(Level.INSTANCE, CacheBuilder.newBuilder().expireAfterAccess( + expireIntervalMilli, TimeUnit.MILLISECONDS).removalListener(new CounterRemoveListener()).build()); + checkPeriod = ctx.getConfig().getConsumer().getCircuitBreaker().getCheckPeriod(); + circuitBreakerConfig = ctx.getConfig().getConsumer().getCircuitBreaker(); + healthCheckInstanceExpireInterval = HealthCheckUtils.CHECK_PERIOD_MULTIPLE * checkPeriod; + } + + @Override + public void postContextInit(Extensions extensions) throws PolarisException { + this.extensions = extensions; + circuitBreakerRuleDictionary = new CircuitBreakerRuleDictionary(extensions.getFlowCache()::loadOrStoreCompiledRegex); + faultDetectRuleDictionary = new FaultDetectRuleDictionary(); + serviceResourceProvider = new DefaultServiceResourceProvider(extensions); + extensions.getLocalRegistry().registerResourceListener(new CircuitBreakerRuleListener(this)); + healthCheckers = extensions.getAllHealthCheckers(); + long expireIntervalMilli = extensions.getConfiguration().getConsumer().getCircuitBreaker() + .getCountersExpireInterval(); + expiredCleanupExecutors.scheduleWithFixedDelay(new Runnable() { + @Override + public void run() { + for (Map.Entry>> entry : countersCache.entrySet()) { + Cache> value = entry.getValue(); + value.cleanUp(); + } + } + }, expireIntervalMilli, expireIntervalMilli, TimeUnit.MILLISECONDS); + } + + // for test + public void setServiceRuleProvider(ServiceResourceProvider serviceResourceProvider) { + this.serviceResourceProvider = serviceResourceProvider; + } + + public long getHealthCheckInstanceExpireInterval() { + return healthCheckInstanceExpireInterval; + } + + // for test + public void setHealthCheckInstanceExpireInterval(long healthCheckInstanceExpireInterval) { + this.healthCheckInstanceExpireInterval = healthCheckInstanceExpireInterval; + } + + // for test + public void setCircuitBreakerRuleDictionary(CircuitBreakerRuleDictionary circuitBreakerRuleDictionary) { + this.circuitBreakerRuleDictionary = circuitBreakerRuleDictionary; + } + + public void setFaultDetectRuleDictionary(FaultDetectRuleDictionary faultDetectRuleDictionary) { + this.faultDetectRuleDictionary = faultDetectRuleDictionary; + } + + public long getCheckPeriod() { + return checkPeriod; + } + + // for test + public void setCheckPeriod(long checkPeriod) { + this.checkPeriod = checkPeriod; + } + + @Override + protected void doDestroy() { + stateChangeExecutors.shutdown(); + healthCheckExecutors.shutdown(); + expiredCleanupExecutors.shutdown(); + } + + Extensions getExtensions() { + return extensions; + } + + ScheduledExecutorService getStateChangeExecutors() { + return stateChangeExecutors; + } + + ScheduledExecutorService getHealthCheckExecutors() { + return healthCheckExecutors; + } + + public ServiceResourceProvider getServiceRuleProvider() { + return serviceResourceProvider; + } + + public Map getHealthCheckers() { + return healthCheckers; + } + + public Map>> getCountersCache() { + return Collections.unmodifiableMap(countersCache); + } + + public Map getHealthCheckCache() { + return healthCheckCache; + } + + CircuitBreakerConfig getCircuitBreakerConfig() { + return circuitBreakerConfig; + } + + //for test + public void setCircuitBreakerConfig(CircuitBreakerConfig circuitBreakerConfig) { + this.circuitBreakerConfig = circuitBreakerConfig; + } + + @Override + public String getName() { + return DefaultPlugins.CIRCUIT_BREAKER_COMPOSITE; + } + + void onCircuitBreakerRuleChanged(ServiceKey serviceKey) { + circuitBreakerRuleDictionary.onServiceChanged(serviceKey); + synchronized (countersCache) { + LOG.info("onCircuitBreakerRuleChanged: clear service {} from ResourceCounters", serviceKey); + for (Map.Entry>> entry : countersCache.entrySet()) { + Cache> cacheValue = entry.getValue(); + for (Resource resource : cacheValue.asMap().keySet()) { + if (resource.getService().equals(serviceKey)) { + cacheValue.invalidate(resource); + } + HealthCheckContainer healthCheckContainer = healthCheckCache.get(serviceKey); + if (null != healthCheckContainer) { + LOG.info("onCircuitBreakerRuleChanged: clear resource {} from healthCheckContainer", resource); + healthCheckContainer.removeResource(resource); + } + } + } + } + } + + void onCircuitBreakerRuleAdded(ServiceKey serviceKey) { + circuitBreakerRuleDictionary.onServiceChanged(serviceKey); + synchronized (countersCache) { + LOG.info("onCircuitBreakerRuleChanged: clear service {} from ResourceCounters", serviceKey); + for (Map.Entry>> entry : countersCache.entrySet()) { + Cache> cacheValue = entry.getValue(); + for (Map.Entry> entryCache: cacheValue.asMap().entrySet()) { + Resource resource = entryCache.getKey(); + if (resource.getService().equals(serviceKey) && !entryCache.getValue().isPresent()) { + cacheValue.invalidate(resource); + } + } + } + } + } + + void onFaultDetectRuleChanged(ServiceKey svcKey, RegistryCacheValue newValue) { + ServiceRule serviceRule = (ServiceRule) newValue; + if (null == serviceRule.getRule()) { + return; + } + FaultDetectorProto.FaultDetector faultDetector = (FaultDetectorProto.FaultDetector) serviceRule.getRule(); + faultDetectRuleDictionary.onFaultDetectRuleChanged(svcKey, faultDetector); + healthCheckCache.computeIfPresent(svcKey, new BiFunction() { + @Override + public HealthCheckContainer apply(ServiceKey serviceKey, HealthCheckContainer healthCheckContainer) { + LOG.info("onFaultDetectRuleChanged: clear healthCheckContainer for service: {}", svcKey); + healthCheckContainer.stop(); + return null; + } + }); + synchronized (countersCache) { + for (Map.Entry>> entry : countersCache.entrySet()) { + Cache> cacheValue = entry.getValue(); + for (Map.Entry> entryCache : cacheValue.asMap().entrySet()) { + Resource resource = entryCache.getKey(); + if (resource.getService().equals(svcKey)) { + if (entryCache.getValue().isPresent()) { + LOG.info("onFaultDetectRuleChanged: ResourceCounters {} setReloadFaultDetect true", svcKey); + ResourceCounters resourceCounters = entryCache.getValue().get(); + resourceCounters.setReloadFaultDetect(true); + } + + } + } + } + } + } + + void onFaultDetectRuleDeleted(ServiceKey svcKey, RegistryCacheValue newValue) { + ServiceRule serviceRule = (ServiceRule) newValue; + if (null == serviceRule.getRule()) { + return; + } + faultDetectRuleDictionary.onFaultDetectRuleDeleted(svcKey); + healthCheckCache.computeIfPresent(svcKey, new BiFunction() { + @Override + public HealthCheckContainer apply(ServiceKey serviceKey, HealthCheckContainer healthCheckContainer) { + LOG.info("onFaultDetectRuleDeleted: clear healthCheckContainer for service: {}", svcKey); + healthCheckContainer.stop(); + return null; + } + }); + } } diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceCounters.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceCounters.java index 84f553ea3..bd177db61 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceCounters.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceCounters.java @@ -17,13 +17,26 @@ package com.tencent.polaris.plugins.circuitbreaker.composite; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.regex.Pattern; + +import com.tencent.polaris.api.config.consumer.CircuitBreakerConfig; import com.tencent.polaris.api.plugin.Plugin; import com.tencent.polaris.api.plugin.cache.FlowCache; import com.tencent.polaris.api.plugin.circuitbreaker.ResourceStat; import com.tencent.polaris.api.plugin.circuitbreaker.entity.InstanceResource; import com.tencent.polaris.api.plugin.circuitbreaker.entity.MethodResource; import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; -import com.tencent.polaris.api.plugin.circuitbreaker.entity.ServiceResource; import com.tencent.polaris.api.plugin.common.PluginTypes; import com.tencent.polaris.api.plugin.compose.Extensions; import com.tencent.polaris.api.plugin.stat.DefaultCircuitBreakResult; @@ -41,338 +54,350 @@ import com.tencent.polaris.plugins.circuitbreaker.composite.trigger.CounterOptions; import com.tencent.polaris.plugins.circuitbreaker.composite.trigger.ErrRateCounter; import com.tencent.polaris.plugins.circuitbreaker.composite.trigger.TriggerCounter; +import com.tencent.polaris.plugins.circuitbreaker.composite.utils.CircuitBreakerUtils; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; -import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.*; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreakerRule; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.ErrorCondition; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.FallbackConfig; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.FallbackResponse; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.FallbackResponse.MessageHeader; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.Level; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.TriggerCondition; import com.tencent.polaris.specification.api.v1.model.ModelProto.MatchString; import org.slf4j.Logger; -import java.util.*; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; -import java.util.regex.Pattern; - import static com.tencent.polaris.logging.LoggingConsts.LOGGING_CIRCUITBREAKER_EVENT; +/** + * resource counter corresponding to resource, has expire_duration: + * - for consecutive counter: no expire + * - for error rate counter: 2 * (interval + sleepWindow) + */ public class ResourceCounters implements StatusChangeHandler { - private static final Logger CB_EVENT_LOG = LoggerFactory.getLogger(LOGGING_CIRCUITBREAKER_EVENT); - - private static final Logger LOG = LoggerFactory.getLogger(ErrRateCounter.class); - - private final CircuitBreakerProto.CircuitBreakerRule currentActiveRule; - - private final List counters = new ArrayList<>(); - - private final Resource resource; - - private final ScheduledExecutorService stateChangeExecutors; - - private final AtomicReference circuitBreakerStatusReference = new AtomicReference<>(); - - private final FallbackInfo fallbackInfo; - - private final Function regexFunction; - - private Extensions extensions; - - private final AtomicBoolean destroyed = new AtomicBoolean(false); - - public ResourceCounters(Resource resource, CircuitBreakerRule currentActiveRule, - ScheduledExecutorService stateChangeExecutors, PolarisCircuitBreaker polarisCircuitBreaker) { - this.currentActiveRule = currentActiveRule; - this.resource = resource; - this.stateChangeExecutors = stateChangeExecutors; - this.regexFunction = regex -> { - if (null == polarisCircuitBreaker) { - return Pattern.compile(regex); - } - FlowCache flowCache = polarisCircuitBreaker.getExtensions().getFlowCache(); - return flowCache.loadOrStoreCompiledRegex(regex); - }; - circuitBreakerStatusReference - .set(new CircuitBreakerStatus(currentActiveRule.getName(), Status.CLOSE, System.currentTimeMillis())); - fallbackInfo = buildFallbackInfo(currentActiveRule); - if (Objects.nonNull(polarisCircuitBreaker)) { - this.extensions = polarisCircuitBreaker.getExtensions(); - } - init(); - } - - private void init() { - List triggerConditionList = currentActiveRule.getTriggerConditionList(); - for (TriggerCondition triggerCondition : triggerConditionList) { - CounterOptions counterOptions = new CounterOptions(); - counterOptions.setResource(resource); - counterOptions.setTriggerCondition(triggerCondition); - counterOptions.setStatusChangeHandler(this); - counterOptions.setExecutorService(stateChangeExecutors); - switch (triggerCondition.getTriggerType()) { - case ERROR_RATE: - counters.add(new ErrRateCounter(currentActiveRule.getName(), counterOptions)); - break; - case CONSECUTIVE_ERROR: - counters.add(new ConsecutiveCounter(currentActiveRule.getName(), counterOptions)); - break; - default: - break; - } - } - } - - private static FallbackInfo buildFallbackInfo(CircuitBreakerRule currentActiveRule) { - if (null == currentActiveRule) { - return null; - } - if (currentActiveRule.getLevel() != Level.METHOD && currentActiveRule.getLevel() != Level.SERVICE) { - return null; - } - FallbackConfig fallbackConfig = currentActiveRule.getFallbackConfig(); - if (null == fallbackConfig || !fallbackConfig.getEnable()) { - return null; - } - FallbackResponse response = fallbackConfig.getResponse(); - if (null == response) { - return null; - } - Map headers = new HashMap<>(); - for (MessageHeader messageHeader : response.getHeadersList()) { - headers.put(messageHeader.getKey(), messageHeader.getValue()); - } - return new FallbackInfo(response.getCode(), headers, response.getBody()); - } - - public CircuitBreakerRule getCurrentActiveRule() { - return currentActiveRule; - } - - @Override - public void closeToOpen(String circuitBreaker) { - synchronized (this) { - if (destroyed.get()) { - LOG.info("counters {} for resource {} is destroyed, closeToOpen skipped", currentActiveRule.getName(), resource); - return; - } - CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); - if (circuitBreakerStatus.getStatus() == Status.CLOSE) { - toOpen(circuitBreakerStatus, circuitBreaker); - } - } - } - - private void toOpen(CircuitBreakerStatus preStatus, String circuitBreaker) { - CircuitBreakerStatus newStatus = new CircuitBreakerStatus(circuitBreaker, Status.OPEN, - System.currentTimeMillis(), fallbackInfo); - circuitBreakerStatusReference.set(newStatus); - CB_EVENT_LOG.info("previous status {}, current status {}, resource {}, rule {}", preStatus.getStatus(), - newStatus.getStatus(), resource, circuitBreaker); - reportCircuitStatus(); - int sleepWindow = currentActiveRule.getRecoverCondition().getSleepWindow(); - // add callback after timeout - stateChangeExecutors.schedule(new Runnable() { - @Override - public void run() { - openToHalfOpen(); - } - }, sleepWindow, TimeUnit.SECONDS); - } - - @Override - public void openToHalfOpen() { - synchronized (this) { - if (destroyed.get()) { - LOG.info("counters {} for resource {} is destroyed, openToHalfOpen skipped", currentActiveRule.getName(), resource); - return; - } - CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); - if (circuitBreakerStatus.getStatus() != Status.OPEN) { - return; - } - int consecutiveSuccess = currentActiveRule.getRecoverCondition().getConsecutiveSuccess(); - HalfOpenStatus halfOpenStatus = new HalfOpenStatus( - circuitBreakerStatus.getCircuitBreaker(), System.currentTimeMillis(), consecutiveSuccess); - CB_EVENT_LOG.info("previous status {}, current status {}, resource {}, rule {}", - circuitBreakerStatus.getStatus(), - halfOpenStatus.getStatus(), resource, circuitBreakerStatus.getCircuitBreaker()); - circuitBreakerStatusReference.set(halfOpenStatus); - reportCircuitStatus(); - } - } - - @Override - public void halfOpenToClose() { - synchronized (this) { - if (destroyed.get()) { - LOG.info("counters {} for resource {} is destroyed, halfOpenToClose skipped", currentActiveRule.getName(), resource); - return; - } - CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); - if (circuitBreakerStatus.getStatus() == Status.HALF_OPEN) { - CircuitBreakerStatus newStatus = new CircuitBreakerStatus(circuitBreakerStatus.getCircuitBreaker(), - Status.CLOSE, System.currentTimeMillis()); - circuitBreakerStatusReference.set(newStatus); - CB_EVENT_LOG.info("previous status {}, current status {}, resource {}, rule {}", - circuitBreakerStatus.getStatus(), - newStatus.getStatus(), resource, circuitBreakerStatus.getCircuitBreaker()); - for (TriggerCounter triggerCounter : counters) { - triggerCounter.resume(); - } - reportCircuitStatus(); - } - } - } - - @Override - public void halfOpenToOpen() { - synchronized (this) { - if (destroyed.get()) { - LOG.info("counters {} for resource {} is destroyed, halfOpenToOpen skipped", currentActiveRule.getName(), resource); - return; - } - CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); - if (circuitBreakerStatus.getStatus() == Status.HALF_OPEN) { - toOpen(circuitBreakerStatus, circuitBreakerStatus.getCircuitBreaker()); - } - } - } - - public RetStatus parseRetStatus(ResourceStat resourceStat) { - List errorConditionsList = currentActiveRule.getErrorConditionsList(); - if (CollectionUtils.isEmpty(errorConditionsList)) { - return resourceStat.getRetStatus(); - } - for (ErrorCondition errorCondition : errorConditionsList) { - MatchString condition = errorCondition.getCondition(); - switch (errorCondition.getInputType()) { - case RET_CODE: - boolean codeMatched = RuleUtils - .matchStringValue(condition, String.valueOf(resourceStat.getRetCode()), regexFunction); - if (codeMatched) { - return RetStatus.RetFail; - } - break; - case DELAY: - String value = condition.getValue().getValue(); - int delayValue = Integer.parseInt(value); - if (resourceStat.getDelay() >= delayValue) { - return RetStatus.RetTimeout; - } - break; - default: - break; - } - } - return RetStatus.RetSuccess; - } - - public void report(ResourceStat resourceStat) { - RetStatus retStatus = parseRetStatus(resourceStat); - boolean success = retStatus != RetStatus.RetFail && retStatus != RetStatus.RetTimeout; - CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); - LOG.debug("[CircuitBreaker] report resource stat {}", resourceStat); - if (null != circuitBreakerStatus && circuitBreakerStatus.getStatus() == Status.HALF_OPEN) { - HalfOpenStatus halfOpenStatus = (HalfOpenStatus) circuitBreakerStatus; - boolean checked = halfOpenStatus.report(success); - LOG.debug("[CircuitBreaker] report resource halfOpen stat {}, checked {}", resourceStat.getResource(), - checked); - if (checked) { - Status nextStatus = halfOpenStatus.calNextStatus(); - switch (nextStatus) { - case CLOSE: - stateChangeExecutors.execute(new Runnable() { - @Override - public void run() { - halfOpenToClose(); - } - }); - break; - case OPEN: - stateChangeExecutors.execute(new Runnable() { - @Override - public void run() { - halfOpenToOpen(); - } - }); - break; - default: - break; - } - } - } else { - LOG.debug("[CircuitBreaker] report resource stat to counter {}", resourceStat.getResource()); - for (TriggerCounter counter : counters) { - counter.report(success); - } - } - } - - public CircuitBreakerStatus getCircuitBreakerStatus() { - return circuitBreakerStatusReference.get(); - } - - - public void reportCircuitStatus() { - if (Objects.isNull(extensions)) { - return; - } - Collection statPlugins = extensions.getPlugins().getPlugins(PluginTypes.STAT_REPORTER.getBaseType()); - if (null != statPlugins) { - try { - for (Plugin statPlugin : statPlugins) { - if (statPlugin instanceof StatReporter) { - DefaultCircuitBreakResult result = new DefaultCircuitBreakResult(); - result.setCallerService(resource.getCallerService()); - result.setCircuitBreakStatus(getCircuitBreakerStatus()); - result.setService(resource.getService().getService()); - result.setNamespace(resource.getService().getNamespace()); - result.setLevel(resource.getLevel().name()); - result.setRuleName(currentActiveRule.getName()); - switch (resource.getLevel()) { - case SERVICE: - ServiceResource serviceResource = (ServiceResource) resource; - break; - case METHOD: - MethodResource methodResource = (MethodResource) resource; - result.setMethod(methodResource.getMethod()); - break; - case INSTANCE: - InstanceResource instanceResource= (InstanceResource) resource; - result.setHost(instanceResource.getHost()); - result.setPort(instanceResource.getPort()); - break; - } - - StatInfo info = new StatInfo(); - info.setCircuitBreakGauge(result); - ((StatReporter) statPlugin).reportStat(info); - } - } - } catch (Exception ex) { - LOG.info("circuit breaker report encountered exception, e: {}", ex.getMessage()); - } - } - } - - public void setDestroyed(boolean value) { - destroyed.set(value); - toDestroy(); - } - - private void toDestroy() { - synchronized (this) { - CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); - circuitBreakerStatus.setDestroy(true); - circuitBreakerStatusReference.set(circuitBreakerStatus); - CB_EVENT_LOG.info("previous status {}, current status {}, resource {}, rule {}", - circuitBreakerStatus.getStatus(), - Status.DESTROY, resource, circuitBreakerStatus.getCircuitBreaker()); - for (TriggerCounter triggerCounter : counters) { - triggerCounter.resume(); - } - reportCircuitStatus(); - } - } + private static final Logger CB_EVENT_LOG = LoggerFactory.getLogger(LOGGING_CIRCUITBREAKER_EVENT); + + private static final Logger LOG = LoggerFactory.getLogger(ErrRateCounter.class); + + private final CircuitBreakerProto.CircuitBreakerRule currentActiveRule; + + private final List counters = new ArrayList<>(); + + private final Resource resource; + + private final ScheduledExecutorService stateChangeExecutors; + + private final AtomicReference circuitBreakerStatusReference = new AtomicReference<>(); + + private final FallbackInfo fallbackInfo; + + private final Function regexFunction; + + private Extensions extensions; + + private final AtomicBoolean destroyed = new AtomicBoolean(false); + + private final CircuitBreakerConfig circuitBreakerConfig; + + private AtomicBoolean reloadFaultDetect = new AtomicBoolean(false); + + public ResourceCounters(Resource resource, CircuitBreakerRule currentActiveRule, + ScheduledExecutorService stateChangeExecutors, PolarisCircuitBreaker polarisCircuitBreaker) { + this.currentActiveRule = currentActiveRule; + this.resource = resource; + this.stateChangeExecutors = stateChangeExecutors; + this.regexFunction = regex -> { + if (null == polarisCircuitBreaker.getExtensions()) { + return Pattern.compile(regex); + } + FlowCache flowCache = polarisCircuitBreaker.getExtensions().getFlowCache(); + return flowCache.loadOrStoreCompiledRegex(regex); + }; + circuitBreakerStatusReference + .set(new CircuitBreakerStatus(currentActiveRule.getName(), Status.CLOSE, System.currentTimeMillis())); + fallbackInfo = buildFallbackInfo(currentActiveRule); + extensions = polarisCircuitBreaker.getExtensions(); + circuitBreakerConfig = polarisCircuitBreaker.getCircuitBreakerConfig(); + init(); + } + + private void init() { + List triggerConditionList = currentActiveRule.getTriggerConditionList(); + for (TriggerCondition triggerCondition : triggerConditionList) { + CounterOptions counterOptions = new CounterOptions(); + counterOptions.setResource(resource); + counterOptions.setTriggerCondition(triggerCondition); + counterOptions.setStatusChangeHandler(this); + counterOptions.setExecutorService(stateChangeExecutors); + switch (triggerCondition.getTriggerType()) { + case ERROR_RATE: + counters.add(new ErrRateCounter(currentActiveRule.getName(), counterOptions)); + break; + case CONSECUTIVE_ERROR: + counters.add(new ConsecutiveCounter(currentActiveRule.getName(), counterOptions)); + break; + default: + break; + } + } + } + + private static FallbackInfo buildFallbackInfo(CircuitBreakerRule currentActiveRule) { + if (null == currentActiveRule) { + return null; + } + if (currentActiveRule.getLevel() != Level.METHOD && currentActiveRule.getLevel() != Level.SERVICE) { + return null; + } + FallbackConfig fallbackConfig = currentActiveRule.getFallbackConfig(); + if (!fallbackConfig.getEnable()) { + return null; + } + FallbackResponse response = fallbackConfig.getResponse(); + Map headers = new HashMap<>(); + for (MessageHeader messageHeader : response.getHeadersList()) { + headers.put(messageHeader.getKey(), messageHeader.getValue()); + } + return new FallbackInfo(response.getCode(), headers, response.getBody()); + } + + public CircuitBreakerRule getCurrentActiveRule() { + return currentActiveRule; + } + + @Override + public void closeToOpen(String circuitBreaker) { + synchronized (this) { + if (destroyed.get()) { + LOG.info("counters {} for resource {} is destroyed, closeToOpen skipped", currentActiveRule.getName(), resource); + return; + } + CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); + if (circuitBreakerStatus.getStatus() == Status.CLOSE) { + toOpen(circuitBreakerStatus, circuitBreaker); + } + } + } + + private void toOpen(CircuitBreakerStatus preStatus, String circuitBreaker) { + CircuitBreakerStatus newStatus = new CircuitBreakerStatus(circuitBreaker, Status.OPEN, + System.currentTimeMillis(), fallbackInfo); + circuitBreakerStatusReference.set(newStatus); + CB_EVENT_LOG.info("previous status {}, current status {}, resource {}, rule {}", preStatus.getStatus(), + newStatus.getStatus(), resource, circuitBreaker); + reportCircuitStatus(); + long sleepWindow = CircuitBreakerUtils.getSleepWindowMilli(currentActiveRule, circuitBreakerConfig); + // add callback after timeout + stateChangeExecutors.schedule(new Runnable() { + @Override + public void run() { + openToHalfOpen(); + } + }, sleepWindow, TimeUnit.MILLISECONDS); + } + + @Override + public void openToHalfOpen() { + synchronized (this) { + if (destroyed.get()) { + LOG.info("counters {} for resource {} is destroyed, openToHalfOpen skipped", currentActiveRule.getName(), resource); + return; + } + CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); + if (circuitBreakerStatus.getStatus() != Status.OPEN) { + return; + } + int consecutiveSuccess = currentActiveRule.getRecoverCondition().getConsecutiveSuccess(); + HalfOpenStatus halfOpenStatus = new HalfOpenStatus( + circuitBreakerStatus.getCircuitBreaker(), System.currentTimeMillis(), consecutiveSuccess); + CB_EVENT_LOG.info("previous status {}, current status {}, resource {}, rule {}", + circuitBreakerStatus.getStatus(), + halfOpenStatus.getStatus(), resource, circuitBreakerStatus.getCircuitBreaker()); + circuitBreakerStatusReference.set(halfOpenStatus); + reportCircuitStatus(); + } + } + + @Override + public void halfOpenToClose() { + synchronized (this) { + if (destroyed.get()) { + LOG.info("counters {} for resource {} is destroyed, halfOpenToClose skipped", currentActiveRule.getName(), resource); + return; + } + CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); + if (circuitBreakerStatus.getStatus() == Status.HALF_OPEN) { + CircuitBreakerStatus newStatus = new CircuitBreakerStatus(circuitBreakerStatus.getCircuitBreaker(), + Status.CLOSE, System.currentTimeMillis()); + circuitBreakerStatusReference.set(newStatus); + CB_EVENT_LOG.info("previous status {}, current status {}, resource {}, rule {}", + circuitBreakerStatus.getStatus(), + newStatus.getStatus(), resource, circuitBreakerStatus.getCircuitBreaker()); + for (TriggerCounter triggerCounter : counters) { + triggerCounter.resume(); + } + reportCircuitStatus(); + } + } + } + + @Override + public void halfOpenToOpen() { + synchronized (this) { + if (destroyed.get()) { + LOG.info("counters {} for resource {} is destroyed, halfOpenToOpen skipped", currentActiveRule.getName(), resource); + return; + } + CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); + if (circuitBreakerStatus.getStatus() == Status.HALF_OPEN) { + toOpen(circuitBreakerStatus, circuitBreakerStatus.getCircuitBreaker()); + } + } + } + + public RetStatus parseRetStatus(ResourceStat resourceStat) { + List errorConditionsList = currentActiveRule.getErrorConditionsList(); + if (CollectionUtils.isEmpty(errorConditionsList)) { + return resourceStat.getRetStatus(); + } + for (ErrorCondition errorCondition : errorConditionsList) { + MatchString condition = errorCondition.getCondition(); + switch (errorCondition.getInputType()) { + case RET_CODE: + boolean codeMatched = RuleUtils + .matchStringValue(condition, String.valueOf(resourceStat.getRetCode()), regexFunction); + if (codeMatched) { + return RetStatus.RetFail; + } + break; + case DELAY: + String value = condition.getValue().getValue(); + int delayValue = Integer.parseInt(value); + if (resourceStat.getDelay() >= delayValue) { + return RetStatus.RetTimeout; + } + break; + default: + break; + } + } + return RetStatus.RetSuccess; + } + + public void report(ResourceStat resourceStat) { + RetStatus retStatus = parseRetStatus(resourceStat); + boolean success = retStatus != RetStatus.RetFail && retStatus != RetStatus.RetTimeout; + CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); + LOG.debug("[CircuitBreaker] report resource stat {}", resourceStat); + if (null != circuitBreakerStatus && circuitBreakerStatus.getStatus() == Status.HALF_OPEN) { + HalfOpenStatus halfOpenStatus = (HalfOpenStatus) circuitBreakerStatus; + boolean checked = halfOpenStatus.report(success); + LOG.debug("[CircuitBreaker] report resource halfOpen stat {}, checked {}", resourceStat.getResource(), + checked); + if (checked) { + Status nextStatus = halfOpenStatus.calNextStatus(); + switch (nextStatus) { + case CLOSE: + stateChangeExecutors.execute(new Runnable() { + @Override + public void run() { + halfOpenToClose(); + } + }); + break; + case OPEN: + stateChangeExecutors.execute(new Runnable() { + @Override + public void run() { + halfOpenToOpen(); + } + }); + break; + default: + break; + } + } + } + else { + LOG.debug("[CircuitBreaker] report resource stat to counter {}", resourceStat.getResource()); + for (TriggerCounter counter : counters) { + counter.report(success); + } + } + } + + public CircuitBreakerStatus getCircuitBreakerStatus() { + return circuitBreakerStatusReference.get(); + } + + + public void reportCircuitStatus() { + if (Objects.isNull(extensions)) { + return; + } + Collection statPlugins = extensions.getPlugins().getPlugins(PluginTypes.STAT_REPORTER.getBaseType()); + if (null != statPlugins) { + try { + for (Plugin statPlugin : statPlugins) { + if (statPlugin instanceof StatReporter) { + DefaultCircuitBreakResult result = new DefaultCircuitBreakResult(); + result.setCallerService(resource.getCallerService()); + result.setCircuitBreakStatus(getCircuitBreakerStatus()); + result.setService(resource.getService().getService()); + result.setNamespace(resource.getService().getNamespace()); + result.setLevel(resource.getLevel().name()); + result.setRuleName(currentActiveRule.getName()); + switch (resource.getLevel()) { + case SERVICE: + break; + case METHOD: + MethodResource methodResource = (MethodResource) resource; + result.setMethod(methodResource.getMethod()); + break; + case INSTANCE: + InstanceResource instanceResource = (InstanceResource) resource; + result.setHost(instanceResource.getHost()); + result.setPort(instanceResource.getPort()); + break; + } + + StatInfo info = new StatInfo(); + info.setCircuitBreakGauge(result); + ((StatReporter) statPlugin).reportStat(info); + } + } + } + catch (Exception ex) { + LOG.info("circuit breaker report encountered exception, e: {}", ex.getMessage()); + } + } + } + + public void setReloadFaultDetect(boolean param) { + reloadFaultDetect.set(param); + } + + public boolean checkReloadFaultDetect() { + return reloadFaultDetect.compareAndSet(true, false); + } + + public void setDestroyed(boolean value) { + destroyed.set(value); + toDestroy(); + } + + private void toDestroy() { + synchronized (this) { + CircuitBreakerStatus circuitBreakerStatus = circuitBreakerStatusReference.get(); + circuitBreakerStatus.setDestroy(true); + circuitBreakerStatusReference.set(circuitBreakerStatus); + CB_EVENT_LOG.info("previous status {}, current status {}, resource {}, rule {}", + circuitBreakerStatus.getStatus(), + Status.DESTROY, resource, circuitBreakerStatus.getCircuitBreaker()); + for (TriggerCounter triggerCounter : counters) { + triggerCounter.resume(); + } + reportCircuitStatus(); + } + } } diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceHealthChecker.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceHealthChecker.java index 5bda7aeaf..8872c35bc 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceHealthChecker.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceHealthChecker.java @@ -17,13 +17,20 @@ package com.tencent.polaris.plugins.circuitbreaker.composite; -import static com.tencent.polaris.logging.LoggingConsts.LOGGING_HEALTHCHECK_EVENT; -import static com.tencent.polaris.plugins.circuitbreaker.composite.CircuitBreakerRuleContainer.compareService; -import static com.tencent.polaris.plugins.circuitbreaker.composite.CircuitBreakerRuleContainer.compareSingleValue; -import static com.tencent.polaris.plugins.circuitbreaker.composite.MatchUtils.matchMethod; -import static com.tencent.polaris.plugins.circuitbreaker.composite.MatchUtils.matchService; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import java.util.regex.Pattern; -import com.tencent.polaris.api.plugin.cache.FlowCache; import com.tencent.polaris.api.plugin.circuitbreaker.ResourceStat; import com.tencent.polaris.api.plugin.circuitbreaker.entity.InstanceResource; import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; @@ -32,289 +39,253 @@ import com.tencent.polaris.api.pojo.DetectResult; import com.tencent.polaris.api.pojo.Instance; import com.tencent.polaris.api.pojo.RetStatus; +import com.tencent.polaris.api.utils.CollectionUtils; import com.tencent.polaris.api.utils.RuleUtils; +import com.tencent.polaris.api.utils.StringUtils; import com.tencent.polaris.client.pojo.Node; import com.tencent.polaris.logging.LoggerFactory; -import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.Level; +import com.tencent.polaris.plugins.circuitbreaker.composite.utils.MatchUtils; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; +import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto; import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto.FaultDetectRule; -import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto.FaultDetectRule.DestinationService; import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto.FaultDetectRule.Protocol; -import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto.FaultDetector; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Function; -import java.util.regex.Pattern; import org.slf4j.Logger; -public class ResourceHealthChecker { +import static com.tencent.polaris.logging.LoggingConsts.LOGGING_HEALTHCHECK_EVENT; - private static final Logger HC_EVENT_LOG = LoggerFactory.getLogger(LOGGING_HEALTHCHECK_EVENT); - - private static final Logger LOG = LoggerFactory.getLogger(ResourceHealthChecker.class); - - private static final int DEFAULT_CHECK_INTERVAL = 10; - - private final Resource resource; - - private final FaultDetector faultDetector; - - private final ScheduledExecutorService checkScheduler; - - private final AtomicBoolean stopped = new AtomicBoolean(false); - - private final Map healthCheckers; - - private final PolarisCircuitBreaker polarisCircuitBreaker; - - private final Function regexToPattern; - - private final List> futures = new ArrayList<>(); - - private final Map instances = new ConcurrentHashMap<>(); - - public ResourceHealthChecker(Resource resource, FaultDetector faultDetector, - PolarisCircuitBreaker polarisCircuitBreaker) { - this.resource = resource; - this.faultDetector = faultDetector; - this.regexToPattern = regex -> { - FlowCache flowCache = polarisCircuitBreaker.getExtensions().getFlowCache(); - return flowCache.loadOrStoreCompiledRegex(regex); - }; - this.checkScheduler = polarisCircuitBreaker.getHealthCheckExecutors(); - this.healthCheckers = polarisCircuitBreaker.getHealthCheckers(); - this.polarisCircuitBreaker = polarisCircuitBreaker; - if (resource instanceof InstanceResource) { - addInstance((InstanceResource) resource, false); - } - start(); - } - - public void addInstance(InstanceResource instanceResource, boolean record) { - ProtocolInstance protocolInstance = instances.get(instanceResource.getNode()); - if (null == protocolInstance) { - instances.put(instanceResource.getNode(), - new ProtocolInstance(HealthCheckUtils.parseProtocol(instanceResource.getProtocol()), - instanceResource)); - return; - } - if (record) { - protocolInstance.doReport(); - } - } - - private static List sortFaultDetectRules(List rules) { - List outRules = new ArrayList<>(rules); - outRules.sort(new Comparator() { - @Override - public int compare(FaultDetectRule rule1, FaultDetectRule rule2) { - // 1. compare destination service - DestinationService targetService1 = rule1.getTargetService(); - String destNamespace1 = targetService1.getNamespace(); - String destService1 = targetService1.getService(); - String destMethod1 = targetService1.getMethod().getValue().getValue(); - - DestinationService targetService2 = rule2.getTargetService(); - String destNamespace2 = targetService2.getNamespace(); - String destService2 = targetService2.getService(); - String destMethod2 = targetService2.getMethod().getValue().getValue(); - - int svcResult = compareService(destNamespace1, destService1, destNamespace2, destService2); - if (svcResult != 0) { - return svcResult; - } - return compareSingleValue(destMethod1, destMethod2); - } - }); - return outRules; - } - - public static Map selectFaultDetectRules(Resource resource, - FaultDetector faultDetector, Function regexToPattern) { - List sortedRules = sortFaultDetectRules(faultDetector.getRulesList()); - Map out = new HashMap<>(); - for (FaultDetectRule sortedRule : sortedRules) { - DestinationService targetService = sortedRule.getTargetService(); - if (!matchService(resource.getService(), targetService.getNamespace(), targetService.getService())) { - continue; - } - if (resource.getLevel() == Level.METHOD) { - if (!matchMethod(resource, targetService.getMethod(), regexToPattern)) { - continue; - } - } else { - // only match empty method rules - if (!RuleUtils.isMatchAllValue(targetService.getMethod())) { - continue; - } - } - if (!out.containsKey(sortedRule.getProtocol().name())) { - out.put(sortedRule.getProtocol().name(), sortedRule); - } - } - return out; - } - - private Instance createDefaultInstance(String host, int port) { - DefaultInstance instance = new DefaultInstance(); - instance.setHost(host); - instance.setPort(port); - return instance; - } - - private Runnable createCheckTask(Protocol protocol, FaultDetectRule faultDetectRule) { - return () -> { - if (stopped.get()) { - return; - } - checkResource(protocol, faultDetectRule); - }; - } - - private void checkResource(Protocol protocol, FaultDetectRule faultDetectRule) { - int port = faultDetectRule.getPort(); - if (port > 0) { - Set hosts = new HashSet<>(); - for (Map.Entry entry : instances.entrySet()) { - Node instance = entry.getKey(); - if (!hosts.contains(instance.getHost())) { - hosts.add(instance.getHost()); - boolean success = doCheck(createDefaultInstance(instance.getHost(), port), protocol, faultDetectRule); - entry.getValue().checkSuccess.set(success); - } - } - } else { - for (Map.Entry entry : instances.entrySet()) { - Protocol currentProtocol = entry.getValue().getProtocol(); - if (currentProtocol == Protocol.UNKNOWN || protocol == currentProtocol) { - InstanceResource instance = entry.getValue().getInstanceResource(); - boolean success = doCheck( - createDefaultInstance(instance.getHost(), instance.getPort()), protocol, faultDetectRule); - entry.getValue().checkSuccess.set(success); - } - } - } - } - - private void start() { - Map protocol2Rules = selectFaultDetectRules(resource, faultDetector, regexToPattern); - for (Map.Entry entry : protocol2Rules.entrySet()) { - FaultDetectRule faultDetectRule = entry.getValue(); - Runnable checkTask = createCheckTask(Protocol.valueOf(entry.getKey()), entry.getValue()); - int interval = DEFAULT_CHECK_INTERVAL; - if (faultDetectRule.getInterval() > 0) { - interval = faultDetectRule.getInterval(); - } - LOG.info("schedule task: resource {}, protocol {}, interval {}, rule {}", resource, entry.getKey(), - interval, faultDetectRule.getName()); - ScheduledFuture future = checkScheduler - .scheduleWithFixedDelay(checkTask, interval, interval, TimeUnit.SECONDS); - futures.add(future); - } - if (resource.getLevel() != Level.INSTANCE) { - long checkPeriod = polarisCircuitBreaker.getCheckPeriod(); - LOG.info("schedule expire task: resource {}, interval {}", resource, checkPeriod); - ScheduledFuture future = checkScheduler.scheduleWithFixedDelay(new Runnable() { - @Override - public void run() { - cleanInstances(); - } - }, checkPeriod, checkPeriod, TimeUnit.MILLISECONDS); - futures.add(future); - } - } - - private boolean doCheck(Instance instance, Protocol protocol, FaultDetectRule faultDetectRule) { - HealthChecker healthChecker = healthCheckers.get(protocol.name().toLowerCase()); - if (null == healthChecker) { - HC_EVENT_LOG - .info("plugin not found, skip health check for instance {}:{}, resource {}, protocol {}", - instance.getHost(), instance.getPort(), resource, protocol); - return false; - } - DetectResult detectResult = healthChecker.detectInstance(instance, faultDetectRule); - ResourceStat resourceStat = new ResourceStat(resource, detectResult.getStatusCode(), detectResult.getDelay(), - detectResult.getRetStatus()); - HC_EVENT_LOG - .info("health check for instance {}:{}, resource {}, protocol {}, result: code {}, delay {}ms, status {}", - instance.getHost(), instance.getPort(), resource, protocol, detectResult.getStatusCode(), - detectResult.getDelay(), detectResult.getRetStatus()); - polarisCircuitBreaker.doReport(resourceStat, false); - return resourceStat.getRetStatus() == RetStatus.RetSuccess; - } - - public void cleanInstances() { - long curTimeMilli = System.currentTimeMillis(); - long expireIntervalMilli = polarisCircuitBreaker.getHealthCheckInstanceExpireInterval(); - for (Map.Entry entry : instances.entrySet()) { - ProtocolInstance protocolInstance = entry.getValue(); - long lastReportMilli = protocolInstance.getLastReportMilli(); - Node node = entry.getKey(); - if (!protocolInstance.isCheckSuccess() && curTimeMilli - lastReportMilli >= expireIntervalMilli) { - instances.remove(node); - HC_EVENT_LOG - .info("clean instance from health check tasks, resource {}, expired node {}, lastReportMilli {}", - resource, node, lastReportMilli); - } - } - } - - public void stop() { - LOG.info("health checker for resource {} has stopped", resource); - stopped.set(true); - for (ScheduledFuture future : futures) { - future.cancel(true); - } - } - - public FaultDetector getFaultDetector() { - return faultDetector; - } - - private static class ProtocolInstance { - - final Protocol protocol; - - final InstanceResource instanceResource; - - final AtomicLong lastReportMilli = new AtomicLong(0); - - final AtomicBoolean checkSuccess = new AtomicBoolean(true); - - ProtocolInstance( - Protocol protocol, InstanceResource instanceResource) { - this.protocol = protocol; - this.instanceResource = instanceResource; - lastReportMilli.set(System.currentTimeMillis()); - } - - Protocol getProtocol() { - return protocol; - } - - InstanceResource getInstanceResource() { - return instanceResource; - } - - public long getLastReportMilli() { - return lastReportMilli.get(); - } - - void doReport() { - lastReportMilli.set(System.currentTimeMillis()); - } - - boolean isCheckSuccess() {return checkSuccess.get();} - } +public class ResourceHealthChecker { + private static final Logger HC_EVENT_LOG = LoggerFactory.getLogger(LOGGING_HEALTHCHECK_EVENT); + + private static final Logger LOG = LoggerFactory.getLogger(ResourceHealthChecker.class); + + private static final Object PLACE_HOLDER_RESOURCE = new Object(); + + private static final int DEFAULT_CHECK_INTERVAL = 30; + + private final ScheduledExecutorService checkScheduler; + + private final AtomicBoolean started = new AtomicBoolean(false); + + private final AtomicBoolean stopped = new AtomicBoolean(false); + + private final Map healthCheckers; + + private final PolarisCircuitBreaker polarisCircuitBreaker; + + private ScheduledFuture future; + + private final FaultDetectRule faultDetectRule; + + private final HealthCheckInstanceProvider healthCheckInstanceProvider; + + private final AtomicLong lastCheckTimeMilli; + + private final Function regexToPattern; + + private final Map resources = new ConcurrentHashMap<>(); + + public ResourceHealthChecker(FaultDetectRule faultDetectRule, + HealthCheckInstanceProvider healthCheckInstanceProvider, PolarisCircuitBreaker polarisCircuitBreaker) { + this.checkScheduler = polarisCircuitBreaker.getHealthCheckExecutors(); + this.healthCheckers = polarisCircuitBreaker.getHealthCheckers(); + this.polarisCircuitBreaker = polarisCircuitBreaker; + this.faultDetectRule = faultDetectRule; + this.healthCheckInstanceProvider = healthCheckInstanceProvider; + lastCheckTimeMilli = new AtomicLong(System.currentTimeMillis()); + if (null != polarisCircuitBreaker.getExtensions()) { + this.regexToPattern = polarisCircuitBreaker.getExtensions().getFlowCache()::loadOrStoreCompiledRegex; + } else { + this.regexToPattern = Pattern::compile; + } + } + + private Instance createDefaultInstance(String host, int port) { + DefaultInstance instance = new DefaultInstance(); + instance.setHost(host); + instance.setPort(port); + return instance; + } + + private Runnable createCheckTask() { + return () -> { + if (stopped.get()) { + return; + } + FaultDetectRule faultDetectRule = getFaultDetectRule(); + int interval = DEFAULT_CHECK_INTERVAL; + if (faultDetectRule.getInterval() > 0) { + interval = faultDetectRule.getInterval(); + } + if (System.currentTimeMillis() - lastCheckTimeMilli.get() >= interval) { + try { + checkResource(faultDetectRule); + } + finally { + lastCheckTimeMilli.set(System.currentTimeMillis()); + } + } + + }; + } + + private void checkResource(FaultDetectRule faultDetectRule) { + Map instances = healthCheckInstanceProvider.getInstances(); + if (CollectionUtils.isEmpty(instances) || CollectionUtils.isEmpty(resources)) { + return; + } + int port = faultDetectRule.getPort(); + Protocol protocol = faultDetectRule.getProtocol(); + if (port > 0) { + Set hosts = new HashSet<>(); + for (Map.Entry entry : instances.entrySet()) { + Node instance = entry.getKey(); + if (!hosts.contains(instance.getHost())) { + hosts.add(instance.getHost()); + boolean success = doCheck(createDefaultInstance(instance.getHost(), port), protocol, faultDetectRule); + entry.getValue().checkSuccess.set(success); + } + } + } + else { + for (Map.Entry entry : instances.entrySet()) { + Protocol currentProtocol = entry.getValue().getProtocol(); + if (currentProtocol == Protocol.UNKNOWN || protocol == currentProtocol) { + InstanceResource instance = entry.getValue().getInstanceResource(); + boolean success = doCheck( + createDefaultInstance(instance.getHost(), instance.getPort()), protocol, faultDetectRule); + entry.getValue().checkSuccess.set(success); + } + } + } + } + + public void start() { + if (started.compareAndSet(false, true)) { + Runnable checkTask = createCheckTask(); + FaultDetectRule faultDetectRule = getFaultDetectRule(); + HC_EVENT_LOG.info("schedule health check task: protocol {}, interval {}, rule {}", faultDetectRule.getProtocol(), + faultDetectRule.getInterval(), faultDetectRule.getName()); + this.future = checkScheduler + .scheduleWithFixedDelay(checkTask, DEFAULT_CHECK_INTERVAL, DEFAULT_CHECK_INTERVAL, TimeUnit.SECONDS); + } + } + + private boolean matchInstanceToResource(Instance instance, Resource resource) { + if (resource.getLevel() != CircuitBreakerProto.Level.INSTANCE) { + return true; + } + InstanceResource instanceResource = (InstanceResource) resource; + return StringUtils.equals(instance.getHost(), instanceResource.getHost()) && instance.getPort() == instanceResource.getPort(); + } + + private boolean doCheck(Instance instance, Protocol protocol, FaultDetectRule faultDetectRule) { + HealthChecker healthChecker = healthCheckers.get(protocol.name().toLowerCase()); + if (null == healthChecker) { + LOG.info("plugin not found, skip health check for instance {}:{}, protocol {}", + instance.getHost(), instance.getPort(), protocol); + return false; + } + DetectResult detectResult = healthChecker.detectInstance(instance, faultDetectRule); + HC_EVENT_LOG.info("health check for instance {}:{}, protocol {}, result: code {}, delay {}ms, status {}, rule {}", + instance.getHost(), instance.getPort(), protocol, detectResult.getStatusCode(), + detectResult.getDelay(), detectResult.getRetStatus(), faultDetectRule.getName()); + Set copiedResources = new HashSet<>(resources.keySet()); + for (Resource resource : copiedResources) { + if (!matchInstanceToResource(instance, resource)) { + continue; + } + ResourceStat resourceStat = new ResourceStat( + resource, detectResult.getStatusCode(), detectResult.getDelay(), detectResult.getRetStatus()); + HC_EVENT_LOG.info("report health check to resource {}, status code {}, delay {}", resource, + detectResult.getStatusCode(), detectResult.getDelay()); + polarisCircuitBreaker.doReport(resourceStat, false); + } + return detectResult.getRetStatus() == RetStatus.RetSuccess; + } + + public void stop() { + HC_EVENT_LOG.info("health checker has stopped, rule {}", faultDetectRule.getName()); + stopped.set(true); + if (null != future) { + future.cancel(true); + } + } + + public FaultDetectRule getFaultDetectRule() { + return faultDetectRule; + } + + public static class ProtocolInstance { + + final Protocol protocol; + + final InstanceResource instanceResource; + + final AtomicLong lastReportMilli = new AtomicLong(0); + + final AtomicBoolean checkSuccess = new AtomicBoolean(true); + + ProtocolInstance( + Protocol protocol, InstanceResource instanceResource) { + this.protocol = protocol; + this.instanceResource = instanceResource; + lastReportMilli.set(System.currentTimeMillis()); + } + + Protocol getProtocol() { + return protocol; + } + + InstanceResource getInstanceResource() { + return instanceResource; + } + + public long getLastReportMilli() { + return lastReportMilli.get(); + } + + void doReport() { + lastReportMilli.set(System.currentTimeMillis()); + } + + boolean isCheckSuccess() { + return checkSuccess.get(); + } + } + + public boolean matchResource(Resource resource) { + FaultDetectRule faultDetectRule = getFaultDetectRule(); + FaultDetectorProto.FaultDetectRule.DestinationService targetService = faultDetectRule.getTargetService(); + if (!MatchUtils.matchService(resource.getService(), targetService.getNamespace(), targetService.getService())) { + return false; + } + if (resource.getLevel() == CircuitBreakerProto.Level.METHOD) { + if (!targetService.hasMethod() || StringUtils.isBlank(targetService.getMethod().getValue().getValue())) { + return false; + } + return MatchUtils.matchMethod(resource, targetService.getMethod(), regexToPattern); + } + else { + // only match empty method rules + return RuleUtils.isMatchAllValue(targetService.getMethod()); + } + } + + public void addResource(Resource resource) { + if (null == resources.putIfAbsent(resource, PLACE_HOLDER_RESOURCE)) { + HC_EVENT_LOG.info("add fault detect resource {}, rule {}", resource, faultDetectRule.getName()); + } + } + + public void removeResource(Resource resource) { + if (null != resources.remove(resource)) { + HC_EVENT_LOG.info("remove fault detect resource {}, rule {}", resource, faultDetectRule.getName()); + + } + } + + public Collection getResources() { + return Collections.unmodifiableCollection(resources.keySet()); + } } diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/CounterOptions.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/CounterOptions.java index 534b86e7d..b3697c6ee 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/CounterOptions.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/CounterOptions.java @@ -17,52 +17,53 @@ package com.tencent.polaris.plugins.circuitbreaker.composite.trigger; +import java.util.concurrent.ScheduledExecutorService; + import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; import com.tencent.polaris.plugins.circuitbreaker.composite.StatusChangeHandler; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.TriggerCondition; -import java.util.concurrent.ScheduledExecutorService; public class CounterOptions { - private Resource resource; + private Resource resource; - private TriggerCondition triggerCondition; + private TriggerCondition triggerCondition; - private ScheduledExecutorService executorService; + private ScheduledExecutorService executorService; - private StatusChangeHandler statusChangeHandler; + private StatusChangeHandler statusChangeHandler; - public Resource getResource() { - return resource; - } + public Resource getResource() { + return resource; + } - public void setResource(Resource resource) { - this.resource = resource; - } + public void setResource(Resource resource) { + this.resource = resource; + } - public TriggerCondition getTriggerCondition() { - return triggerCondition; - } + public TriggerCondition getTriggerCondition() { + return triggerCondition; + } - public void setTriggerCondition( - TriggerCondition triggerCondition) { - this.triggerCondition = triggerCondition; - } + public void setTriggerCondition( + TriggerCondition triggerCondition) { + this.triggerCondition = triggerCondition; + } - public ScheduledExecutorService getExecutorService() { - return executorService; - } + public ScheduledExecutorService getExecutorService() { + return executorService; + } - public void setExecutorService(ScheduledExecutorService executorService) { - this.executorService = executorService; - } + public void setExecutorService(ScheduledExecutorService executorService) { + this.executorService = executorService; + } - public StatusChangeHandler getStatusChangeHandler() { - return statusChangeHandler; - } + public StatusChangeHandler getStatusChangeHandler() { + return statusChangeHandler; + } - public void setStatusChangeHandler( - StatusChangeHandler statusChangeHandler) { - this.statusChangeHandler = statusChangeHandler; - } + public void setStatusChangeHandler( + StatusChangeHandler statusChangeHandler) { + this.statusChangeHandler = statusChangeHandler; + } } diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ErrRateCounter.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ErrRateCounter.java index 00d9c4d4f..5f2016c35 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ErrRateCounter.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ErrRateCounter.java @@ -23,6 +23,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; + +import com.tencent.polaris.plugins.circuitbreaker.composite.utils.CircuitBreakerUtils; import org.slf4j.Logger; public class ErrRateCounter extends TriggerCounter { @@ -51,7 +53,7 @@ public ErrRateCounter(String ruleName, CounterOptions counterOptions) { @Override protected void init() { LOG.info("[CircuitBreaker][Counter] errRateCounter {} initialized, resource {}", ruleName, resource); - int interval = triggerCondition.getInterval(); + long interval = CircuitBreakerUtils.getErrorRateIntervalSec(triggerCondition); metricWindowMs = interval * 1000L; errorPercent = triggerCondition.getErrorPercent(); minimumRequest = triggerCondition.getMinimumRequest(); @@ -79,7 +81,7 @@ public void report(boolean success) { } } - private static long getBucketIntervalMs(int interval) { + private static long getBucketIntervalMs(long interval) { long metricWindowMs = interval * 1000L; double bucketIntervalMs = (double) metricWindowMs / (double) BUCKET_COUNT; return (long) Math.ceil(bucketIntervalMs); diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/CircuitBreakerUtils.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/CircuitBreakerUtils.java new file mode 100644 index 000000000..3b14d01ec --- /dev/null +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/CircuitBreakerUtils.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.plugins.circuitbreaker.composite.utils; + +import com.tencent.polaris.api.config.consumer.CircuitBreakerConfig; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; + +public class CircuitBreakerUtils { + + public static long DEFAULT_ERROR_RATE_INTERVAL_MS = 60 * 1000; + + public static boolean checkRule(CircuitBreakerProto.CircuitBreakerRule rule) { + return checkLevel(rule.getLevel()); + } + + public static boolean checkLevel(CircuitBreakerProto.Level level) { + return level == CircuitBreakerProto.Level.SERVICE + || level == CircuitBreakerProto.Level.METHOD + || level == CircuitBreakerProto.Level.INSTANCE; + } + + public static long getSleepWindowMilli(CircuitBreakerProto.CircuitBreakerRule currentActiveRule, + CircuitBreakerConfig circuitBreakerConfig) { + long sleepWindow = currentActiveRule.getRecoverCondition().getSleepWindow() * 1000L; + if (sleepWindow == 0) { + sleepWindow = circuitBreakerConfig.getSleepWindow(); + } + return sleepWindow; + } + + public static long getErrorRateIntervalSec(CircuitBreakerProto.TriggerCondition triggerCondition) { + long interval = triggerCondition.getInterval(); + if (interval == 0) { + interval = DEFAULT_ERROR_RATE_INTERVAL_MS / 1000; + } + return interval; + } + +} diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckUtils.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/HealthCheckUtils.java similarity index 87% rename from polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckUtils.java rename to polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/HealthCheckUtils.java index bd2c6e599..10df01736 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/HealthCheckUtils.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/HealthCheckUtils.java @@ -15,7 +15,7 @@ * specific language governing permissions and limitations under the License. */ -package com.tencent.polaris.plugins.circuitbreaker.composite; +package com.tencent.polaris.plugins.circuitbreaker.composite.utils; import com.tencent.polaris.api.utils.StringUtils; import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto.FaultDetectRule.Protocol; @@ -27,11 +27,6 @@ public class HealthCheckUtils { */ public static int CHECK_PERIOD_MULTIPLE = 20; - /** - * default check expire period - */ - public static int DEFAULT_CHECK_INTERVAL = 60 * 1000; - /** * parse protocol string to enum * diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/MatchUtils.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/MatchUtils.java new file mode 100644 index 000000000..9f60a73cb --- /dev/null +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/MatchUtils.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.plugins.circuitbreaker.composite.utils; + +import java.util.function.Function; +import java.util.regex.Pattern; + +import com.tencent.polaris.api.plugin.circuitbreaker.entity.MethodResource; +import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; +import com.tencent.polaris.api.pojo.ServiceKey; +import com.tencent.polaris.api.utils.RuleUtils; +import com.tencent.polaris.api.utils.StringUtils; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.Level; +import com.tencent.polaris.specification.api.v1.model.ModelProto.MatchString; + +public class MatchUtils { + + public static boolean matchService(ServiceKey serviceKey, String namespace, String service) { + String inputNamespace = ""; + String inputService = ""; + if (null != serviceKey) { + inputNamespace = serviceKey.getNamespace(); + inputService = serviceKey.getService(); + } + if (StringUtils.isNotBlank(namespace) && !StringUtils.equals(namespace, RuleUtils.MATCH_ALL) && !StringUtils + .equals(inputNamespace, namespace)) { + return false; + } + if (StringUtils.isNotBlank(service) && !StringUtils.equals(service, RuleUtils.MATCH_ALL) && !StringUtils + .equals(inputService, service)) { + return false; + } + return true; + } + + public static boolean matchMethod(Resource resource, MatchString matchString, + Function regexToPattern) { + if (resource.getLevel() != Level.METHOD) { + return true; + } + String method = ((MethodResource) resource).getMethod(); + return RuleUtils.matchStringValue(matchString, method, regexToPattern); + } + + public static boolean isWildcardMatcherSingle(String name) { + return name.equals(RuleUtils.MATCH_ALL) || StringUtils.isBlank(name); + } + + public static int compareSingleValue(String value1, String value2) { + boolean serviceWildcard1 = isWildcardMatcherSingle(value1); + boolean serviceWildcard2 = isWildcardMatcherSingle(value2); + if (serviceWildcard1 && serviceWildcard2) { + return 0; + } + if (serviceWildcard1) { + // 1 before 2 + return 1; + } + if (serviceWildcard2) { + // 1 before 2 + return -1; + } + return value1.compareTo(value2); + } + + public static int compareService(String namespace1, String service1, String namespace2, String service2) { + int nsResult = compareSingleValue(namespace1, namespace2); + if (nsResult != 0) { + return nsResult; + } + return compareSingleValue(service1, service2); + } +} diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/CircuitBreakerRuleContainerTest.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/CircuitBreakerRuleContainerTest.java index a344bd707..3dfb3e210 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/CircuitBreakerRuleContainerTest.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/CircuitBreakerRuleContainerTest.java @@ -17,6 +17,9 @@ package com.tencent.polaris.plugins.circuitbraker.composite.trigger; +import java.util.function.Function; +import java.util.regex.Pattern; + import com.google.protobuf.StringValue; import com.tencent.polaris.api.plugin.circuitbreaker.entity.MethodResource; import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; @@ -25,7 +28,7 @@ import com.tencent.polaris.api.pojo.ServiceKey; import com.tencent.polaris.api.pojo.ServiceRule; import com.tencent.polaris.client.pojo.ServiceRuleByProto; -import com.tencent.polaris.plugins.circuitbreaker.composite.CircuitBreakerRuleContainer; +import com.tencent.polaris.plugins.circuitbreaker.composite.CircuitBreakerRuleDictionary; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreaker; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreakerRule; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.Level; @@ -33,195 +36,245 @@ import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.RuleMatcher.DestinationService; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.RuleMatcher.SourceService; import com.tencent.polaris.specification.api.v1.model.ModelProto.MatchString; -import java.util.function.Function; -import java.util.regex.Pattern; import org.junit.Assert; import org.junit.Test; public class CircuitBreakerRuleContainerTest { - @Test - public void testSelectRuleService() { - CircuitBreaker.Builder cbBuilder = CircuitBreaker.newBuilder(); - // match one service rules - CircuitBreakerRule.Builder builder = CircuitBreakerRule.newBuilder(); - builder.setName("test_cb_default_svc1"); - builder.setEnable(true); - builder.setLevel(Level.SERVICE); - RuleMatcher.Builder rmBuilder = RuleMatcher.newBuilder(); - rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("svc1").build()); - rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); - builder.setRuleMatcher(rmBuilder.build()); - cbBuilder.addRules(builder); - - builder = CircuitBreakerRule.newBuilder(); - builder.setName("test_cb_all_ns_all_svc"); - builder.setEnable(true); - builder.setLevel(Level.SERVICE); - rmBuilder = RuleMatcher.newBuilder(); - rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("*").setService("*").build()); - rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); - builder.setRuleMatcher(rmBuilder.build()); - cbBuilder.addRules(builder); - - builder = CircuitBreakerRule.newBuilder(); - builder.setName("test_cb_default_svc2"); - builder.setEnable(true); - builder.setLevel(Level.SERVICE); - rmBuilder = RuleMatcher.newBuilder(); - rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("svc2").build()); - rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); - builder.setRuleMatcher(rmBuilder.build()); - cbBuilder.addRules(builder); - - builder = CircuitBreakerRule.newBuilder(); - builder.setName("test_cb_default_all_svc"); - builder.setEnable(true); - builder.setLevel(Level.SERVICE); - rmBuilder = RuleMatcher.newBuilder(); - rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("*").build()); - rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); - builder.setRuleMatcher(rmBuilder.build()); - cbBuilder.addRules(builder); - - builder = CircuitBreakerRule.newBuilder(); - builder.setName("test_cb_all_ns_svc2"); - builder.setEnable(true); - builder.setLevel(Level.SERVICE); - rmBuilder = RuleMatcher.newBuilder(); - rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("*").setService("svc2").build()); - rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); - builder.setRuleMatcher(rmBuilder.build()); - cbBuilder.addRules(builder); - - builder = CircuitBreakerRule.newBuilder(); - builder.setName("test_cb_all_ns_all_svc_default_svc3"); - builder.setEnable(true); - builder.setLevel(Level.SERVICE); - rmBuilder = RuleMatcher.newBuilder(); - rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("*").setService("*").build()); - rmBuilder.setSource(SourceService.newBuilder().setNamespace("default").setService("svc3").build()); - builder.setRuleMatcher(rmBuilder.build()); - cbBuilder.addRules(builder); - - builder = CircuitBreakerRule.newBuilder(); - builder.setName("test_cb_default_svc2_default_svc3"); - builder.setEnable(true); - builder.setLevel(Level.SERVICE); - rmBuilder = RuleMatcher.newBuilder(); - rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("svc2").build()); - rmBuilder.setSource(SourceService.newBuilder().setNamespace("default").setService("svc3").build()); - builder.setRuleMatcher(rmBuilder.build()); - cbBuilder.addRules(builder); - - cbBuilder.setRevision(StringValue.newBuilder().setValue("xxxxxyyyyyy").build()); - - CircuitBreaker allRules = cbBuilder.build(); - ServiceRule svcRule = new ServiceRuleByProto(allRules, allRules.getRevision().getValue(), false, - EventType.CIRCUIT_BREAKING); - Function regexToPattern = new Function() { - @Override - public Pattern apply(String s) { - return Pattern.compile(s); - } - }; - Resource resource = new ServiceResource(new ServiceKey("default", "svc2")); - CircuitBreakerRule rule = CircuitBreakerRuleContainer.selectRule(resource, svcRule, regexToPattern); - Assert.assertNotNull(rule); - Assert.assertEquals("test_cb_default_svc2", rule.getName()); - - resource = new ServiceResource(new ServiceKey("default", "svc1")); - rule = CircuitBreakerRuleContainer.selectRule(resource, svcRule, regexToPattern); - Assert.assertNotNull(rule); - Assert.assertEquals("test_cb_default_svc1", rule.getName()); - - resource = new ServiceResource(new ServiceKey("default", "svc2"), - new ServiceKey("default", "svc3")); - rule = CircuitBreakerRuleContainer.selectRule(resource, svcRule, regexToPattern); - Assert.assertNotNull(rule); - Assert.assertEquals("test_cb_default_svc2_default_svc3", rule.getName()); - - resource = new ServiceResource(new ServiceKey("default1", "svc4"), - new ServiceKey("default", "svc3")); - rule = CircuitBreakerRuleContainer.selectRule(resource, svcRule, regexToPattern); - Assert.assertNotNull(rule); - Assert.assertEquals("test_cb_all_ns_all_svc_default_svc3", rule.getName()); - - resource = new ServiceResource(new ServiceKey("default", "svc1"), - new ServiceKey("default", "svc4")); - rule = CircuitBreakerRuleContainer.selectRule(resource, svcRule, regexToPattern); - Assert.assertNotNull(rule); - Assert.assertEquals("test_cb_default_svc1", rule.getName()); - - resource = new ServiceResource(new ServiceKey("default", "svc4"), - new ServiceKey("default", "svc3")); - rule = CircuitBreakerRuleContainer.selectRule(resource, svcRule, regexToPattern); - Assert.assertNotNull(rule); - Assert.assertEquals("test_cb_default_all_svc", rule.getName()); - } - - @Test - public void testSelectRuleMethod() { - CircuitBreaker.Builder cbBuilder = CircuitBreaker.newBuilder(); - // match one service rules - CircuitBreakerRule.Builder builder = CircuitBreakerRule.newBuilder(); - builder.setName("test_cb_default_svc1_foo"); - builder.setEnable(true); - builder.setLevel(Level.METHOD); - RuleMatcher.Builder rmBuilder = RuleMatcher.newBuilder(); - rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("svc1").setMethod( - MatchString.newBuilder().setValue(StringValue.newBuilder().setValue("foo").build()).build()).build()); - rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); - builder.setRuleMatcher(rmBuilder.build()); - cbBuilder.addRules(builder); - - builder = CircuitBreakerRule.newBuilder(); - builder.setName("test_cb_default_all_svc_foo1"); - builder.setEnable(true); - builder.setLevel(Level.METHOD); - rmBuilder = RuleMatcher.newBuilder(); - rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("*").setMethod( - MatchString.newBuilder().setValue(StringValue.newBuilder().setValue("foo1").build()).build()).build()); - rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); - builder.setRuleMatcher(rmBuilder.build()); - cbBuilder.addRules(builder); - - builder = CircuitBreakerRule.newBuilder(); - builder.setName("test_cb_default_svc2_all"); - builder.setEnable(true); - builder.setLevel(Level.METHOD); - rmBuilder = RuleMatcher.newBuilder(); - rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("svc2").setMethod( - MatchString.newBuilder().setValue(StringValue.newBuilder().setValue("*").build()).build()).build()); - rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); - builder.setRuleMatcher(rmBuilder.build()); - cbBuilder.addRules(builder); - - cbBuilder.setRevision(StringValue.newBuilder().setValue("xxxxxyyyyyy").build()); - - CircuitBreaker allRules = cbBuilder.build(); - ServiceRule svcRule = new ServiceRuleByProto(allRules, allRules.getRevision().getValue(), false, - EventType.CIRCUIT_BREAKING); - Function regexToPattern = new Function() { - @Override - public Pattern apply(String s) { - return Pattern.compile(s); - } - }; - - Resource resource = new MethodResource(new ServiceKey("default", "svc1"), "foo"); - CircuitBreakerRule rule = CircuitBreakerRuleContainer.selectRule(resource, svcRule, regexToPattern); - Assert.assertNotNull(rule); - Assert.assertEquals("test_cb_default_svc1_foo", rule.getName()); - - resource = new MethodResource(new ServiceKey("default", "svc1"), "foo1"); - rule = CircuitBreakerRuleContainer.selectRule(resource, svcRule, regexToPattern); - Assert.assertNotNull(rule); - Assert.assertEquals("test_cb_default_all_svc_foo1", rule.getName()); - - resource = new MethodResource(new ServiceKey("default", "svc2"), "foo2"); - rule = CircuitBreakerRuleContainer.selectRule(resource, svcRule, regexToPattern); - Assert.assertNotNull(rule); - Assert.assertEquals("test_cb_default_svc2_all", rule.getName()); - } + @Test + public void testSelectRuleService() { + + // match one service rules + CircuitBreakerRule.Builder builderDefaultSvc1 = CircuitBreakerRule.newBuilder(); + builderDefaultSvc1.setName("test_cb_default_svc1"); + builderDefaultSvc1.setEnable(true); + builderDefaultSvc1.setLevel(Level.SERVICE); + RuleMatcher.Builder rmBuilder = RuleMatcher.newBuilder(); + rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("svc1").build()); + rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); + builderDefaultSvc1.setRuleMatcher(rmBuilder.build()); + + CircuitBreakerRule.Builder builderAllNsAllSvc = CircuitBreakerRule.newBuilder(); + builderAllNsAllSvc.setName("test_cb_all_ns_all_svc"); + builderAllNsAllSvc.setEnable(true); + builderAllNsAllSvc.setLevel(Level.SERVICE); + rmBuilder = RuleMatcher.newBuilder(); + rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("*").setService("*").build()); + rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); + builderAllNsAllSvc.setRuleMatcher(rmBuilder.build()); + + CircuitBreakerRule.Builder builderDefaultSvc2 = CircuitBreakerRule.newBuilder(); + builderDefaultSvc2.setName("test_cb_default_svc2"); + builderDefaultSvc2.setEnable(true); + builderDefaultSvc2.setLevel(Level.SERVICE); + rmBuilder = RuleMatcher.newBuilder(); + rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("svc2").build()); + rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); + builderDefaultSvc2.setRuleMatcher(rmBuilder.build()); + + CircuitBreakerRule.Builder builderDefaultAllSvc = CircuitBreakerRule.newBuilder(); + builderDefaultAllSvc.setName("test_cb_default_all_svc"); + builderDefaultAllSvc.setEnable(true); + builderDefaultAllSvc.setLevel(Level.SERVICE); + rmBuilder = RuleMatcher.newBuilder(); + rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("*").build()); + rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); + builderDefaultAllSvc.setRuleMatcher(rmBuilder.build()); + + CircuitBreakerRule.Builder builderAllNsSvc2 = CircuitBreakerRule.newBuilder(); + builderAllNsSvc2.setName("test_cb_all_ns_svc2"); + builderAllNsSvc2.setEnable(true); + builderAllNsSvc2.setLevel(Level.SERVICE); + rmBuilder = RuleMatcher.newBuilder(); + rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("*").setService("svc2").build()); + rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); + builderAllNsSvc2.setRuleMatcher(rmBuilder.build()); + + CircuitBreakerRule.Builder builderAllSvcDefaultSvc3 = CircuitBreakerRule.newBuilder(); + builderAllSvcDefaultSvc3.setName("test_cb_all_ns_all_svc_default_svc3"); + builderAllSvcDefaultSvc3.setEnable(true); + builderAllSvcDefaultSvc3.setLevel(Level.SERVICE); + rmBuilder = RuleMatcher.newBuilder(); + rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("*").setService("*").build()); + rmBuilder.setSource(SourceService.newBuilder().setNamespace("default").setService("svc3").build()); + builderAllSvcDefaultSvc3.setRuleMatcher(rmBuilder.build()); + + CircuitBreakerRule.Builder builderDefaultSvc2DefaultSvc3 = CircuitBreakerRule.newBuilder(); + builderDefaultSvc2DefaultSvc3.setName("test_cb_default_svc2_default_svc3"); + builderDefaultSvc2DefaultSvc3.setEnable(true); + builderDefaultSvc2DefaultSvc3.setLevel(Level.SERVICE); + rmBuilder = RuleMatcher.newBuilder(); + rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("svc2").build()); + rmBuilder.setSource(SourceService.newBuilder().setNamespace("default").setService("svc3").build()); + builderDefaultSvc2DefaultSvc3.setRuleMatcher(rmBuilder.build()); + + Function regexToPattern = new Function() { + @Override + public Pattern apply(String s) { + return Pattern.compile(s); + } + }; + + CircuitBreakerRuleDictionary circuitBreakerRuleDictionary = new CircuitBreakerRuleDictionary(regexToPattern); + + ServiceKey svc1 = new ServiceKey("default", "svc1"); + ServiceKey svc2 = new ServiceKey("default", "svc2"); + ServiceKey svc4 = new ServiceKey("default", "svc4"); + ServiceKey svc14 = new ServiceKey("default1", "svc4"); + + CircuitBreaker.Builder cbBuilderSvc1 = CircuitBreaker.newBuilder(); + cbBuilderSvc1.addRules(builderDefaultSvc1); + cbBuilderSvc1.addRules(builderAllNsAllSvc); + cbBuilderSvc1.addRules(builderDefaultAllSvc); + cbBuilderSvc1.addRules(builderAllSvcDefaultSvc3); + cbBuilderSvc1.setRevision(StringValue.newBuilder().setValue("xxxxxyyyyyy-svc1").build()); + CircuitBreaker svc1Rules = cbBuilderSvc1.build(); + ServiceRule svcRule1 = new ServiceRuleByProto(svc1Rules, svc1Rules.getRevision().getValue(), false, + EventType.CIRCUIT_BREAKING); + circuitBreakerRuleDictionary.putServiceRule(svc1, svcRule1); + + CircuitBreaker.Builder cbBuilderSvc2 = CircuitBreaker.newBuilder(); + cbBuilderSvc2.addRules(builderAllNsAllSvc); + cbBuilderSvc2.addRules(builderDefaultSvc2); + cbBuilderSvc2.addRules(builderDefaultAllSvc); + cbBuilderSvc2.addRules(builderAllNsSvc2); + cbBuilderSvc2.addRules(builderAllSvcDefaultSvc3); + cbBuilderSvc2.addRules(builderDefaultSvc2DefaultSvc3); + cbBuilderSvc2.setRevision(StringValue.newBuilder().setValue("xxxxxyyyyyy-svc2").build()); + CircuitBreaker svc2Rules = cbBuilderSvc2.build(); + ServiceRule svcRule2 = new ServiceRuleByProto(svc2Rules, svc2Rules.getRevision().getValue(), false, + EventType.CIRCUIT_BREAKING); + circuitBreakerRuleDictionary.putServiceRule(svc2, svcRule2); + + CircuitBreaker.Builder cbBuilderSvc4 = CircuitBreaker.newBuilder(); + cbBuilderSvc4.addRules(builderAllNsAllSvc); + cbBuilderSvc4.addRules(builderDefaultAllSvc); + cbBuilderSvc4.addRules(builderAllSvcDefaultSvc3); + cbBuilderSvc4.setRevision(StringValue.newBuilder().setValue("xxxxxyyyyyy-svc4").build()); + CircuitBreaker svc4Rules = cbBuilderSvc4.build(); + ServiceRule svcRule4 = new ServiceRuleByProto(svc4Rules, svc4Rules.getRevision().getValue(), false, + EventType.CIRCUIT_BREAKING); + circuitBreakerRuleDictionary.putServiceRule(svc4, svcRule4); + + CircuitBreaker.Builder cbBuilderSvc14 = CircuitBreaker.newBuilder(); + cbBuilderSvc14.addRules(builderAllNsAllSvc); + cbBuilderSvc14.addRules(builderAllSvcDefaultSvc3); + cbBuilderSvc14.setRevision(StringValue.newBuilder().setValue("xxxxxyyyyyy-svc14").build()); + CircuitBreaker svc14Rules = cbBuilderSvc2.build(); + ServiceRule svcRule14 = new ServiceRuleByProto(svc14Rules, svc14Rules.getRevision().getValue(), false, + EventType.CIRCUIT_BREAKING); + circuitBreakerRuleDictionary.putServiceRule(svc14, svcRule14); + + Resource resource = new ServiceResource(new ServiceKey("default", "svc1")); + CircuitBreakerRule rule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + Assert.assertNotNull(rule); + Assert.assertEquals("test_cb_default_svc1", rule.getName()); + + resource = new ServiceResource(new ServiceKey("default", "svc1"), + new ServiceKey("default", "svc4")); + rule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + Assert.assertNotNull(rule); + Assert.assertEquals("test_cb_default_svc1", rule.getName()); + + resource = new ServiceResource(new ServiceKey("default", "svc2")); + rule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + Assert.assertNotNull(rule); + Assert.assertEquals("test_cb_default_svc2", rule.getName()); + + resource = new ServiceResource(new ServiceKey("default", "svc2"), + new ServiceKey("default", "svc3")); + rule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + Assert.assertNotNull(rule); + Assert.assertEquals("test_cb_default_svc2_default_svc3", rule.getName()); + + resource = new ServiceResource(new ServiceKey("default1", "svc4"), + new ServiceKey("default", "svc3")); + rule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + Assert.assertNotNull(rule); + Assert.assertEquals("test_cb_all_ns_all_svc_default_svc3", rule.getName()); + + resource = new ServiceResource(new ServiceKey("default", "svc4"), + new ServiceKey("default", "svc3")); + rule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + Assert.assertNotNull(rule); + Assert.assertEquals("test_cb_default_all_svc", rule.getName()); + } + + @Test + public void testSelectRuleMethod() { + // match one service rules + CircuitBreakerRule.Builder builderDefaultSvc1 = CircuitBreakerRule.newBuilder(); + builderDefaultSvc1.setName("test_cb_default_svc1_foo"); + builderDefaultSvc1.setEnable(true); + builderDefaultSvc1.setLevel(Level.METHOD); + RuleMatcher.Builder rmBuilder = RuleMatcher.newBuilder(); + rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("svc1").setMethod( + MatchString.newBuilder().setValue(StringValue.newBuilder().setValue("foo").build()).build()).build()); + rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); + builderDefaultSvc1.setRuleMatcher(rmBuilder.build()); + + CircuitBreakerRule.Builder builderDefaultAllSvcFoo1 = CircuitBreakerRule.newBuilder(); + builderDefaultAllSvcFoo1.setName("test_cb_default_all_svc_foo1"); + builderDefaultAllSvcFoo1.setEnable(true); + builderDefaultAllSvcFoo1.setLevel(Level.METHOD); + rmBuilder = RuleMatcher.newBuilder(); + rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("*").setMethod( + MatchString.newBuilder().setValue(StringValue.newBuilder().setValue("foo1").build()).build()).build()); + rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); + builderDefaultAllSvcFoo1.setRuleMatcher(rmBuilder.build()); + + CircuitBreakerRule.Builder builderDefaultSvc2 = CircuitBreakerRule.newBuilder(); + builderDefaultSvc2.setName("test_cb_default_svc2_all"); + builderDefaultSvc2.setEnable(true); + builderDefaultSvc2.setLevel(Level.METHOD); + rmBuilder = RuleMatcher.newBuilder(); + rmBuilder.setDestination(DestinationService.newBuilder().setNamespace("default").setService("svc2").setMethod( + MatchString.newBuilder().setValue(StringValue.newBuilder().setValue("*").build()).build()).build()); + rmBuilder.setSource(SourceService.newBuilder().setNamespace("*").setService("*").build()); + builderDefaultSvc2.setRuleMatcher(rmBuilder.build()); + + ServiceKey svc1 = new ServiceKey("default", "svc1"); + ServiceKey svc2 = new ServiceKey("default", "svc2"); + + Function regexToPattern = new Function() { + @Override + public Pattern apply(String s) { + return Pattern.compile(s); + } + }; + CircuitBreakerRuleDictionary circuitBreakerRuleDictionary = new CircuitBreakerRuleDictionary(regexToPattern); + + CircuitBreaker.Builder cbBuilderSvc1 = CircuitBreaker.newBuilder(); + cbBuilderSvc1.addRules(builderDefaultSvc1); + cbBuilderSvc1.addRules(builderDefaultAllSvcFoo1); + cbBuilderSvc1.setRevision(StringValue.newBuilder().setValue("xxxxxyyyyyy-svc1").build()); + CircuitBreaker svc1Rules = cbBuilderSvc1.build(); + ServiceRule svcRule1 = new ServiceRuleByProto(svc1Rules, svc1Rules.getRevision().getValue(), false, + EventType.CIRCUIT_BREAKING); + circuitBreakerRuleDictionary.putServiceRule(svc1, svcRule1); + + CircuitBreaker.Builder cbBuilderSvc2 = CircuitBreaker.newBuilder(); + cbBuilderSvc2.addRules(builderDefaultAllSvcFoo1); + cbBuilderSvc2.addRules(builderDefaultSvc2); + cbBuilderSvc2.setRevision(StringValue.newBuilder().setValue("xxxxxyyyyyy-svc2").build()); + CircuitBreaker svc2Rules = cbBuilderSvc2.build(); + ServiceRule svcRule2 = new ServiceRuleByProto(svc2Rules, svc2Rules.getRevision().getValue(), false, + EventType.CIRCUIT_BREAKING); + circuitBreakerRuleDictionary.putServiceRule(svc2, svcRule2); + + Resource resource = new MethodResource(new ServiceKey("default", "svc1"), "foo"); + CircuitBreakerRule rule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + Assert.assertNotNull(rule); + Assert.assertEquals("test_cb_default_svc1_foo", rule.getName()); + + resource = new MethodResource(new ServiceKey("default", "svc1"), "foo1"); + rule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + Assert.assertNotNull(rule); + Assert.assertEquals("test_cb_default_all_svc_foo1", rule.getName()); + + resource = new MethodResource(new ServiceKey("default", "svc2"), "foo2"); + rule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + Assert.assertNotNull(rule); + Assert.assertEquals("test_cb_default_svc2_all", rule.getName()); + } } diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/PolarisCircuitBreakerTest.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/PolarisCircuitBreakerTest.java index 6d79796ef..181a84425 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/PolarisCircuitBreakerTest.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/PolarisCircuitBreakerTest.java @@ -17,10 +17,17 @@ package com.tencent.polaris.plugins.circuitbraker.composite.trigger; +import java.util.Optional; +import java.util.regex.Pattern; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.protobuf.StringValue; +import com.tencent.polaris.api.config.Configuration; import com.tencent.polaris.api.plugin.circuitbreaker.ResourceStat; import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; import com.tencent.polaris.api.plugin.circuitbreaker.entity.ServiceResource; +import com.tencent.polaris.api.plugin.common.InitContext; import com.tencent.polaris.api.pojo.CircuitBreakerStatus; import com.tencent.polaris.api.pojo.CircuitBreakerStatus.Status; import com.tencent.polaris.api.pojo.ServiceEventKey; @@ -28,8 +35,12 @@ import com.tencent.polaris.api.pojo.ServiceKey; import com.tencent.polaris.api.pojo.ServiceRule; import com.tencent.polaris.client.pojo.ServiceRuleByProto; +import com.tencent.polaris.factory.ConfigAPIFactory; +import com.tencent.polaris.plugins.circuitbreaker.composite.CircuitBreakerRuleDictionary; import com.tencent.polaris.plugins.circuitbreaker.composite.CircuitBreakerRuleListener; +import com.tencent.polaris.plugins.circuitbreaker.composite.FaultDetectRuleDictionary; import com.tencent.polaris.plugins.circuitbreaker.composite.PolarisCircuitBreaker; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreaker; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreakerRule; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.ErrorCondition; @@ -40,6 +51,7 @@ import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.TriggerCondition.TriggerType; import com.tencent.polaris.specification.api.v1.model.ModelProto.MatchString; import com.tencent.polaris.specification.api.v1.model.ModelProto.MatchString.MatchStringType; +import com.tencent.polaris.test.common.MockInitContext; import com.tencent.polaris.test.mock.discovery.MockServiceResourceProvider; import org.junit.Assert; import org.junit.Test; @@ -86,8 +98,11 @@ public CircuitBreaker buildAnotherRules() { public void testCheckResource() { MockServiceResourceProvider mockServiceRuleProvider = new MockServiceResourceProvider(); PolarisCircuitBreaker polarisCircuitBreaker = new PolarisCircuitBreaker(); - polarisCircuitBreaker.init(null); + Configuration configuration = ConfigAPIFactory.defaultConfig(); + InitContext initContext = new MockInitContext(configuration); + polarisCircuitBreaker.init(initContext); polarisCircuitBreaker.setServiceRuleProvider(mockServiceRuleProvider); + polarisCircuitBreaker.setCircuitBreakerRuleDictionary(new CircuitBreakerRuleDictionary(Pattern::compile)); ServiceKey serviceKey = new ServiceKey("Test", "testSvc"); ServiceEventKey serviceEventKey = new ServiceEventKey(serviceKey, EventType.CIRCUIT_BREAKING); CircuitBreaker circuitBreaker = buildRules(); @@ -105,6 +120,7 @@ public void testCheckResource() { } } CircuitBreakerStatus circuitBreakerStatus = polarisCircuitBreaker.checkResource(svcResource); + Assert.assertNotNull(circuitBreakerStatus); Status status = circuitBreakerStatus.getStatus(); Assert.assertEquals(Status.OPEN, status); } @@ -113,8 +129,12 @@ public void testCheckResource() { public void testRuleChanged() { MockServiceResourceProvider mockServiceRuleProvider = new MockServiceResourceProvider(); PolarisCircuitBreaker polarisCircuitBreaker = new PolarisCircuitBreaker(); - polarisCircuitBreaker.init(null); + Configuration configuration = ConfigAPIFactory.defaultConfig(); + InitContext initContext = new MockInitContext(configuration); + polarisCircuitBreaker.init(initContext); polarisCircuitBreaker.setServiceRuleProvider(mockServiceRuleProvider); + polarisCircuitBreaker.setCircuitBreakerRuleDictionary(new CircuitBreakerRuleDictionary(Pattern::compile)); + polarisCircuitBreaker.setFaultDetectRuleDictionary(new FaultDetectRuleDictionary()); ServiceKey serviceKey = new ServiceKey("Test", "testSvc"); ServiceEventKey serviceEventKey = new ServiceEventKey(serviceKey, EventType.CIRCUIT_BREAKING); CircuitBreaker circuitBreaker = buildRules(); @@ -156,4 +176,42 @@ public void testRuleChanged() { Status status = circuitBreakerStatus.getStatus(); Assert.assertEquals(Status.OPEN, status); } + + @Test + public void testCache() { + Cache> cache = CacheBuilder.newBuilder().build(); + cache.put("1111", Optional.of("ssss")); + Optional value = cache.getIfPresent("2222"); + System.out.println(value); + } + + public CircuitBreaker buildMethodRules() { + CircuitBreakerRule.Builder builder = CircuitBreakerRule.newBuilder(); + builder.setName("test_cb_method_rule"); + builder.setEnable(true); + builder.setLevel(Level.METHOD); + builder.setRuleMatcher( + CircuitBreakerProto.RuleMatcher.newBuilder(). + setSource(CircuitBreakerProto.RuleMatcher.SourceService.newBuilder().setNamespace("*").setService("*").build()). + setDestination(CircuitBreakerProto.RuleMatcher.DestinationService.newBuilder(). + setNamespace("*").setService("*").setMethod(MatchString.newBuilder(). + setValue(StringValue.newBuilder().setValue(".*")). + setType(MatchStringType.REGEX)).build())); + builder.addTriggerCondition( + TriggerCondition.newBuilder().setTriggerType(TriggerType.CONSECUTIVE_ERROR).setErrorCount(10).build()); + builder.addErrorConditions(ErrorCondition.newBuilder().setInputType(InputType.RET_CODE).setCondition( + MatchString.newBuilder().setType(MatchStringType.EXACT) + .setValue(StringValue.newBuilder().setValue("500").build()).build()).build()); + builder.setRecoverCondition(RecoverCondition.newBuilder().setConsecutiveSuccess(2).setSleepWindow(5).build()); + builder.setRevision("222"); + CircuitBreaker.Builder cbBuilder = CircuitBreaker.newBuilder(); + cbBuilder.addRules(builder); + cbBuilder.setRevision(StringValue.newBuilder().setValue("xxxxxyyyyyx").build()); + return cbBuilder.build(); + } + + @Test + public void testCounterExpire() { + + } } diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceCountersTest.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceCountersTest.java index 20038a94d..b00ef7b48 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceCountersTest.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceCountersTest.java @@ -26,6 +26,8 @@ import com.tencent.polaris.api.pojo.CircuitBreakerStatus.Status; import com.tencent.polaris.api.pojo.RetStatus; import com.tencent.polaris.api.pojo.ServiceKey; +import com.tencent.polaris.factory.ConfigAPIFactory; +import com.tencent.polaris.plugins.circuitbreaker.composite.PolarisCircuitBreaker; import com.tencent.polaris.plugins.circuitbreaker.composite.ResourceCounters; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreakerRule; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.ErrorCondition; @@ -64,8 +66,9 @@ public void testStatusChanged() throws InterruptedException { builder.setRecoverCondition(RecoverCondition.newBuilder().setConsecutiveSuccess(2).setSleepWindow(5).build()); ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); Resource resource = new MethodResource(new ServiceKey("test", "TestSvc"), "foo"); - ResourceCounters resourceCounters = new ResourceCounters(resource, builder.build(), scheduledExecutorService, - null); + PolarisCircuitBreaker polarisCircuitBreaker = new PolarisCircuitBreaker(); + polarisCircuitBreaker.setCircuitBreakerConfig(ConfigAPIFactory.defaultConfig().getConsumer().getCircuitBreaker()); + ResourceCounters resourceCounters = new ResourceCounters(resource, builder.build(), scheduledExecutorService, polarisCircuitBreaker); CheckSet checkSet = new CheckSet(); Resource methodResource = new MethodResource(new ServiceKey("test", "TestSvc"), "foo"); for (int i = 0; i < 5; i++) { @@ -118,7 +121,10 @@ public void testParseStatus() { builder.setRecoverCondition(RecoverCondition.newBuilder().setConsecutiveSuccess(2).setSleepWindow(5).build()); Resource resource = new InstanceResource( new ServiceKey("test", "TestSvc"), "127.0.0.1", 8088, null, "http"); - ResourceCounters resourceCounters = new ResourceCounters(resource, builder.build(), null, null); + + PolarisCircuitBreaker polarisCircuitBreaker = new PolarisCircuitBreaker(); + polarisCircuitBreaker.setCircuitBreakerConfig(ConfigAPIFactory.defaultConfig().getConsumer().getCircuitBreaker()); + ResourceCounters resourceCounters = new ResourceCounters(resource, builder.build(), null, polarisCircuitBreaker); ResourceStat stat1 = new ResourceStat(resource, 500, 100, RetStatus.RetUnknown); RetStatus nextStatus = resourceCounters.parseRetStatus(stat1); Assert.assertEquals(RetStatus.RetFail, nextStatus); @@ -137,7 +143,7 @@ public void testParseStatus() { TriggerCondition.newBuilder().setTriggerType(TriggerType.CONSECUTIVE_ERROR).setErrorCount(5).build()); builder.setRecoverCondition(RecoverCondition.newBuilder().setConsecutiveSuccess(2).setSleepWindow(5).build()); - resourceCounters = new ResourceCounters(resource, builder.build(), null, null); + resourceCounters = new ResourceCounters(resource, builder.build(), null, polarisCircuitBreaker); nextStatus = resourceCounters.parseRetStatus(stat3); Assert.assertEquals(RetStatus.RetSuccess, nextStatus); } @@ -154,7 +160,10 @@ public void testDestroy() { TriggerCondition.newBuilder().setTriggerType(TriggerType.CONSECUTIVE_ERROR).setErrorCount(5).build()); builder.setRecoverCondition(RecoverCondition.newBuilder().setConsecutiveSuccess(1).setSleepWindow(5).build()); Resource resource = new MethodResource(new ServiceKey("test", "TestSvc"), "foo"); - ResourceCounters resourceCounters = new ResourceCounters(resource, builder.build(), null, null); + + PolarisCircuitBreaker polarisCircuitBreaker = new PolarisCircuitBreaker(); + polarisCircuitBreaker.setCircuitBreakerConfig(ConfigAPIFactory.defaultConfig().getConsumer().getCircuitBreaker()); + ResourceCounters resourceCounters = new ResourceCounters(resource, builder.build(), null, polarisCircuitBreaker); resourceCounters.setDestroyed(true); resourceCounters.report(new ResourceStat(resource, 500, 1000)); resourceCounters.closeToOpen("cb_rule_status"); diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceHealthCheckerTest.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceHealthCheckerTest.java index d7d6dba7f..49aa2624a 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceHealthCheckerTest.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceHealthCheckerTest.java @@ -20,63 +20,56 @@ import com.google.protobuf.StringValue; import com.tencent.polaris.api.plugin.circuitbreaker.entity.ServiceResource; import com.tencent.polaris.api.pojo.ServiceKey; +import com.tencent.polaris.plugins.circuitbreaker.composite.HealthCheckInstanceProvider; +import com.tencent.polaris.plugins.circuitbreaker.composite.PolarisCircuitBreaker; import com.tencent.polaris.plugins.circuitbreaker.composite.ResourceHealthChecker; import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto.FaultDetectRule; import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto.FaultDetectRule.DestinationService; import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto.FaultDetectRule.Protocol; -import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto.FaultDetector; import com.tencent.polaris.specification.api.v1.model.ModelProto.MatchString; -import java.util.Map; -import java.util.function.Function; -import java.util.regex.Pattern; import org.junit.Assert; import org.junit.Test; public class ResourceHealthCheckerTest { - @Test - public void testSelectFdRule() { - FaultDetector.Builder cbBuilder = FaultDetector.newBuilder(); - // match one service rules - FaultDetectRule.Builder builder = FaultDetectRule.newBuilder(); - builder.setName("test_cb_default_svc1"); - builder.setTargetService( - FaultDetectRule.DestinationService.newBuilder().setNamespace("default").setService("svc1").build()); - builder.setProtocol(Protocol.HTTP); - cbBuilder.addRules(builder.build()); + @Test + public void testSelectFdRule() { + // match one service rules + FaultDetectRule.Builder builderDefaultSvc1 = FaultDetectRule.newBuilder(); + builderDefaultSvc1.setId("test_cb_default_svc1"); + builderDefaultSvc1.setName("test_cb_default_svc1"); + builderDefaultSvc1.setTargetService( + FaultDetectRule.DestinationService.newBuilder().setNamespace("default").setService("svc1").build()); + builderDefaultSvc1.setProtocol(Protocol.HTTP); - builder = FaultDetectRule.newBuilder(); - builder.setName("test_cb_default_svc_foo1"); - builder.setTargetService( - DestinationService.newBuilder().setNamespace("default").setService("svc1").setMethod( - MatchString.newBuilder().setValue(StringValue.newBuilder().setValue("foo1").build()).build()) - .build()); - builder.setProtocol(Protocol.TCP); - cbBuilder.addRules(builder.build()); + FaultDetectRule.Builder builderDefaultSvcFoo1 = FaultDetectRule.newBuilder(); + builderDefaultSvcFoo1.setId("test_cb_default_svc_foo1"); + builderDefaultSvcFoo1.setName("test_cb_default_svc_foo1"); + builderDefaultSvcFoo1.setTargetService( + DestinationService.newBuilder().setNamespace("default").setService("svc1").setMethod( + MatchString.newBuilder().setValue(StringValue.newBuilder().setValue("foo1").build()).build()) + .build()); + builderDefaultSvcFoo1.setProtocol(Protocol.TCP); - builder = FaultDetectRule.newBuilder(); - builder.setName("test_cb_all_ns_all_svc"); - builder.setTargetService( - FaultDetectRule.DestinationService.newBuilder().setNamespace("*").setService("*").build()); - builder.setProtocol(Protocol.HTTP); - cbBuilder.addRules(builder.build()); + FaultDetectRule.Builder builderAllNsAllSvc = FaultDetectRule.newBuilder(); + builderAllNsAllSvc.setName("test_cb_all_ns_all_svc"); + builderAllNsAllSvc.setTargetService( + FaultDetectRule.DestinationService.newBuilder().setNamespace("*").setService("*").build()); + builderAllNsAllSvc.setProtocol(Protocol.HTTP); - cbBuilder.setRevision("xxxxxyyyyyy"); + HealthCheckInstanceProvider healthCheckInstanceProvider = () -> null; - FaultDetector allRules = cbBuilder.build(); + ResourceHealthChecker resourceHealthCheckerDefaultSvc1 = new ResourceHealthChecker(builderDefaultSvc1.build(), + healthCheckInstanceProvider, new PolarisCircuitBreaker()); + ResourceHealthChecker resourceHealthCheckerDefaultSvcFoo1 = new ResourceHealthChecker(builderDefaultSvcFoo1.build(), + healthCheckInstanceProvider, new PolarisCircuitBreaker()); + ResourceHealthChecker resourceHealthCheckerAllNsAllSvc = new ResourceHealthChecker(builderAllNsAllSvc.build(), + healthCheckInstanceProvider, new PolarisCircuitBreaker()); - ServiceResource svcResource = new ServiceResource(new ServiceKey("default", "svc1")); - Function regexToPattern = new Function() { - @Override - public Pattern apply(String s) { - return Pattern.compile(s); - } - }; - Map protocolFaultDetectRuleMap = ResourceHealthChecker - .selectFaultDetectRules(svcResource, allRules, regexToPattern); - Assert.assertEquals(1, protocolFaultDetectRuleMap.size()); - FaultDetectRule faultDetectRule = protocolFaultDetectRuleMap.get(Protocol.HTTP.name()); - Assert.assertEquals("test_cb_default_svc1", faultDetectRule.getName()); + ServiceResource svcResource = new ServiceResource(new ServiceKey("default", "svc1")); + Assert.assertTrue(resourceHealthCheckerDefaultSvc1.matchResource(svcResource)); + Assert.assertFalse(resourceHealthCheckerDefaultSvcFoo1.matchResource(svcResource)); + Assert.assertTrue(resourceHealthCheckerAllNsAllSvc.matchResource(svcResource)); + } - } } diff --git a/polaris-test/polaris-test-common/pom.xml b/polaris-test/polaris-test-common/pom.xml index 04343de7b..74f034478 100644 --- a/polaris-test/polaris-test-common/pom.xml +++ b/polaris-test/polaris-test-common/pom.xml @@ -14,4 +14,11 @@ Polaris Test Common Polaris Test Common JAR + + + com.tencent.polaris + polaris-plugin-api + ${project.version} + + \ No newline at end of file diff --git a/polaris-test/polaris-test-common/src/main/java/com/tencent/polaris/test/common/MockInitContext.java b/polaris-test/polaris-test-common/src/main/java/com/tencent/polaris/test/common/MockInitContext.java new file mode 100644 index 000000000..e7331f32b --- /dev/null +++ b/polaris-test/polaris-test-common/src/main/java/com/tencent/polaris/test/common/MockInitContext.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.test.common; + +import java.util.Collection; + +import com.tencent.polaris.api.config.Configuration; +import com.tencent.polaris.api.plugin.Supplier; +import com.tencent.polaris.api.plugin.common.InitContext; +import com.tencent.polaris.api.plugin.common.ValueContext; +import com.tencent.polaris.api.plugin.compose.ServerServiceInfo; + +public class MockInitContext implements InitContext { + + private final Configuration configuration; + + public MockInitContext(Configuration configuration) { + this.configuration = configuration; + } + + @Override + public Configuration getConfig() { + return configuration; + } + + @Override + public Supplier getPlugins() { + return null; + } + + @Override + public ValueContext getValueContext() { + return null; + } + + @Override + public Collection getServerServices() { + return null; + } +} From 2b7438d9368a3783d7b40eeb78d341b5c41a59c6 Mon Sep 17 00:00:00 2001 From: andrew shan <45474304+andrewshan@users.noreply.github.com> Date: Wed, 24 Jul 2024 10:06:26 +0800 Subject: [PATCH 2/4] begin 1.15.9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e4a2d26a2..ee2fdccc3 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,7 @@ - 1.15.8 + 1.15.9-SNAPSHOT ${maven.build.timestamp} false From f154e1957793a5bff214136fb8cc8a1e257d5787 Mon Sep 17 00:00:00 2001 From: andrew shan <45474304+andrewshan@users.noreply.github.com> Date: Fri, 9 Aug 2024 15:57:33 +0800 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E9=80=9A?= =?UTF-8?q?=E9=85=8DAPI=E7=BB=9F=E4=B8=80=E8=AE=A1=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../factory/test/CbTestUtils.java | 54 +++++ .../factory/test/CircuitBreakerMultiTest.java | 225 +++--------------- .../factory/test/FaultDetectorTest.java | 160 +++++++++++++ .../composite/PolarisCircuitBreaker.java | 105 ++++++-- .../composite/ResourceHealthChecker.java | 10 +- .../composite/utils/CircuitBreakerUtils.java | 2 + .../composite}/PolarisCircuitBreakerTest.java | 59 ++++- .../CircuitBreakerRuleContainerTest.java | 2 +- .../trigger/ConsecutiveCounterTest.java | 2 +- .../composite/trigger/ErrRateCounterTest.java | 2 +- .../trigger/ResourceCountersTest.java | 2 +- .../trigger/ResourceHealthCheckerTest.java | 2 +- pom.xml | 2 +- 13 files changed, 401 insertions(+), 226 deletions(-) create mode 100644 polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CbTestUtils.java create mode 100644 polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/FaultDetectorTest.java rename polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/{circuitbraker/composite/trigger => circuitbreaker/composite}/PolarisCircuitBreakerTest.java (77%) rename polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/{circuitbraker => circuitbreaker}/composite/trigger/CircuitBreakerRuleContainerTest.java (97%) rename polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/{circuitbraker => circuitbreaker}/composite/trigger/ConsecutiveCounterTest.java (94%) rename polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/{circuitbraker => circuitbreaker}/composite/trigger/ErrRateCounterTest.java (95%) rename polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/{circuitbraker => circuitbreaker}/composite/trigger/ResourceCountersTest.java (97%) rename polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/{circuitbraker => circuitbreaker}/composite/trigger/ResourceHealthCheckerTest.java (96%) diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CbTestUtils.java b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CbTestUtils.java new file mode 100644 index 000000000..014e67e57 --- /dev/null +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CbTestUtils.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.circuitbreaker.factory.test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +import com.google.protobuf.util.JsonFormat; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; +import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto; +import org.junit.Assert; + +public class CbTestUtils { + + public static CircuitBreakerProto.CircuitBreakerRule loadCbRule(String fileName) throws IOException { + CircuitBreakerProto.CircuitBreakerRule.Builder circuitBreakerRuleBuilder = CircuitBreakerProto.CircuitBreakerRule + .newBuilder(); + InputStream inputStream = CbTestUtils.class.getClassLoader().getResourceAsStream(fileName); + Assert.assertNotNull(inputStream); + String json = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("")); + JsonFormat.parser().ignoringUnknownFields().merge(json, circuitBreakerRuleBuilder); + return circuitBreakerRuleBuilder.build(); + } + + public static FaultDetectorProto.FaultDetectRule loadFdRule(String fileName) throws IOException { + FaultDetectorProto.FaultDetectRule.Builder builder = FaultDetectorProto.FaultDetectRule.newBuilder(); + InputStream inputStream = CbTestUtils.class.getClassLoader().getResourceAsStream(fileName); + Assert.assertNotNull(inputStream); + String json = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("")); + JsonFormat.parser().ignoringUnknownFields().merge(json, builder); + return builder.build(); + } +} diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerMultiTest.java b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerMultiTest.java index 235599ea3..792597372 100644 --- a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerMultiTest.java +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/CircuitBreakerMultiTest.java @@ -17,25 +17,18 @@ package com.tencent.polaris.circuitbreaker.factory.test; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.Map; import java.util.Optional; import java.util.function.Consumer; -import java.util.stream.Collectors; import com.google.common.cache.Cache; import com.google.protobuf.StringValue; -import com.google.protobuf.util.JsonFormat; import com.tencent.polaris.api.config.Configuration; import com.tencent.polaris.api.plugin.circuitbreaker.CircuitBreaker; import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; import com.tencent.polaris.api.pojo.ServiceKey; -import com.tencent.polaris.api.utils.StringUtils; import com.tencent.polaris.circuitbreak.api.CircuitBreakAPI; import com.tencent.polaris.circuitbreak.api.FunctionalDecorator; import com.tencent.polaris.circuitbreak.api.pojo.FunctionalDecoratorRequest; @@ -83,45 +76,24 @@ public void before() throws IOException { Assert.fail(e.getMessage()); } - CircuitBreakerProto.CircuitBreakerRule cbRule1 = loadCbRule("circuitBreakerMethodRuleNoDetect.json"); + CircuitBreakerProto.CircuitBreakerRule cbRule1 = CbTestUtils.loadCbRule("circuitBreakerMethodRuleNoDetect.json"); CircuitBreakerProto.CircuitBreaker circuitBreaker = CircuitBreakerProto.CircuitBreaker.newBuilder() .addRules(cbRule1).setRevision(StringValue.newBuilder().setValue("0000").build()).build(); namingServer.getNamingService().setCircuitBreaker(matchMethodService, circuitBreaker); - CircuitBreakerProto.CircuitBreakerRule cbRule3 = loadCbRule("circuitBreakerMethodRule.json"); - CircuitBreakerProto.CircuitBreakerRule cbRule4 = loadCbRule("circuitBreakerRule.json"); + CircuitBreakerProto.CircuitBreakerRule cbRule3 = CbTestUtils.loadCbRule("circuitBreakerMethodRule.json"); + CircuitBreakerProto.CircuitBreakerRule cbRule4 = CbTestUtils.loadCbRule("circuitBreakerRule.json"); circuitBreaker = CircuitBreakerProto.CircuitBreaker.newBuilder() .addRules(cbRule3).addRules(cbRule4).setRevision(StringValue.newBuilder().setValue("1111").build()).build(); namingServer.getNamingService().setCircuitBreaker(matchMethodDetectService, circuitBreaker); - FaultDetectorProto.FaultDetectRule rule1 = loadFdRule("faultDetectRule.json"); - FaultDetectorProto.FaultDetectRule rule2 = loadFdRule("faultDetectMethodRule.json"); + FaultDetectorProto.FaultDetectRule rule1 = CbTestUtils.loadFdRule("faultDetectRule.json"); + FaultDetectorProto.FaultDetectRule rule2 = CbTestUtils.loadFdRule("faultDetectMethodRule.json"); FaultDetectorProto.FaultDetector faultDetector = FaultDetectorProto.FaultDetector.newBuilder() .addRules(rule1).addRules(rule2).setRevision("2222").build(); namingServer.getNamingService().setFaultDetector(matchMethodDetectService, faultDetector); } - private CircuitBreakerProto.CircuitBreakerRule loadCbRule(String fileName) throws IOException { - CircuitBreakerProto.CircuitBreakerRule.Builder circuitBreakerRuleBuilder = CircuitBreakerProto.CircuitBreakerRule - .newBuilder(); - InputStream inputStream = getClass().getClassLoader().getResourceAsStream(fileName); - Assert.assertNotNull(inputStream); - String json = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)).lines() - .collect(Collectors.joining("")); - JsonFormat.parser().ignoringUnknownFields().merge(json, circuitBreakerRuleBuilder); - return circuitBreakerRuleBuilder.build(); - } - - private FaultDetectorProto.FaultDetectRule loadFdRule(String fileName) throws IOException { - FaultDetectorProto.FaultDetectRule.Builder builder = FaultDetectorProto.FaultDetectRule.newBuilder(); - InputStream inputStream = getClass().getClassLoader().getResourceAsStream(fileName); - Assert.assertNotNull(inputStream); - String json = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)).lines() - .collect(Collectors.joining("")); - JsonFormat.parser().ignoringUnknownFields().merge(json, builder); - return builder.build(); - } - @Test public void testMultipleUrlsNoRule() { Configuration configuration = TestUtils.configWithEnvAddress(); @@ -143,18 +115,13 @@ public void testMultipleUrlsNoRule() { }); integerConsumer.accept(1); } - try { - Thread.sleep(10 * 1000); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - + Utils.sleepUninterrupted(10 * 1000); BaseEngine baseEngine = (BaseEngine) circuitBreakAPI; CircuitBreaker resourceBreaker = baseEngine.getSDKContext().getExtensions().getResourceBreaker(); PolarisCircuitBreaker polarisCircuitBreaker = (PolarisCircuitBreaker) resourceBreaker; Cache> methodCache = polarisCircuitBreaker.getCountersCache() .get(CircuitBreakerProto.Level.METHOD); + polarisCircuitBreaker.cleanupExpiredResources(); Assert.assertEquals(0, methodCache.size()); } } @@ -178,74 +145,26 @@ public void testMultipleUrlsMethodRule() { System.out.println("invoke success" + finalI); } }); - try { + if (i == 0) { integerConsumer.accept(1); - Utils.sleepUninterrupted(1000); - integerConsumer.accept(2); - Utils.sleepUninterrupted(1000); - } - catch (Exception e) { - if (!(e instanceof IllegalArgumentException)) { - throw e; + try { + integerConsumer.accept(2); + } catch (Exception e) { + if (!(e instanceof IllegalArgumentException)) { + throw e; + } } + Assert.assertThrows(CallAbortedException.class, () -> integerConsumer.accept(3)); + } else { + Assert.assertThrows(CallAbortedException.class, () -> integerConsumer.accept(1)); } - Assert.assertThrows(CallAbortedException.class, () -> integerConsumer.accept(3)); - } - Utils.sleepUninterrupted(20 * 1000); - - BaseEngine baseEngine = (BaseEngine) circuitBreakAPI; - CircuitBreaker resourceBreaker = baseEngine.getSDKContext().getExtensions().getResourceBreaker(); - PolarisCircuitBreaker polarisCircuitBreaker = (PolarisCircuitBreaker) resourceBreaker; - Cache> methodCache = polarisCircuitBreaker.getCountersCache() - .get(CircuitBreakerProto.Level.METHOD); - Assert.assertEquals(0, methodCache.size()); - } - } - - @Test - public void testFaultDetector() { - Configuration configuration = TestUtils.configWithEnvAddress(); - ConfigurationImpl configurationImpl = (ConfigurationImpl) configuration; - configurationImpl.getConsumer().getCircuitBreaker().setCountersExpireInterval(5000); - try (CircuitBreakAPI circuitBreakAPI = CircuitBreakAPIFactory.createCircuitBreakAPIByConfig(configurationImpl)) { - for (int i = 0; i < 50; i++) { - String method = ""; - if (i > 0) { - method = "/test1/path/" + i; - } - FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( - matchMethodDetectService, method); - FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); - int finalI = i; - Consumer integerConsumer = decorator.decorateConsumer(num -> { - if (num % 2 == 0) { - throw new IllegalArgumentException("invoke failed" + finalI); - } - else { - System.out.println("invoke success" + finalI); - } - }); - integerConsumer.accept(1); } BaseEngine baseEngine = (BaseEngine) circuitBreakAPI; CircuitBreaker resourceBreaker = baseEngine.getSDKContext().getExtensions().getResourceBreaker(); PolarisCircuitBreaker polarisCircuitBreaker = (PolarisCircuitBreaker) resourceBreaker; - Map healthCheckCache = polarisCircuitBreaker.getHealthCheckCache(); - Assert.assertEquals(1, healthCheckCache.size()); - HealthCheckContainer healthCheckContainer = healthCheckCache.get(matchMethodDetectService); - Assert.assertNotNull(healthCheckContainer); - Collection healthCheckerValues = healthCheckContainer.getHealthCheckerValues(); - Assert.assertEquals(2, healthCheckerValues.size()); - for (ResourceHealthChecker resourceHealthChecker : healthCheckerValues) { - if (StringUtils.equals(resourceHealthChecker.getFaultDetectRule().getId(), "fd1")) { - Assert.assertEquals(1, resourceHealthChecker.getResources().size()); - } - } - Utils.sleepUninterrupted(20 * 1000); - Cache> methodCache = polarisCircuitBreaker.getCountersCache() .get(CircuitBreakerProto.Level.METHOD); - Assert.assertEquals(0, methodCache.size()); + Assert.assertEquals(1, methodCache.size()); } } @@ -273,7 +192,7 @@ public void testCircuitBreakerRuleChanged() throws IOException { }); integerConsumer.accept(1); } - CircuitBreakerProto.CircuitBreakerRule cbRule1 = loadCbRule("circuitBreakerMethodRuleChanged.json"); + CircuitBreakerProto.CircuitBreakerRule cbRule1 = CbTestUtils.loadCbRule("circuitBreakerMethodRuleChanged.json"); CircuitBreakerProto.CircuitBreaker circuitBreaker = CircuitBreakerProto.CircuitBreaker.newBuilder() .addRules(cbRule1).setRevision(StringValue.newBuilder().setValue("444441").build()).build(); namingServer.getNamingService().setCircuitBreaker(matchMethodDetectService, circuitBreaker); @@ -304,101 +223,29 @@ public void testCircuitBreakerRuleChanged() throws IOException { System.out.println("invoke success" + finalI); } }); - try { - integerConsumer.accept(1); - } catch (Exception e) { - if (!(e instanceof IllegalArgumentException)) { - throw e; + if (i == 0) { + try { + integerConsumer.accept(1); } - System.out.println("invoke 1 failed" + finalI); - } - Utils.sleepUninterrupted(1000); - try { - integerConsumer.accept(2); - } catch (Exception e) { - if (!(e instanceof IllegalArgumentException)) { - throw e; + catch (Exception e) { + if (!(e instanceof IllegalArgumentException)) { + throw e; + } } - System.out.println("invoke 2 failed" + finalI); - } - Utils.sleepUninterrupted(1000); - Assert.assertThrows(CallAbortedException.class, () -> integerConsumer.accept(3)); - } - } - } - - @Test - public void testFaultDetectRuleChanged() throws IOException { - Configuration configuration = TestUtils.configWithEnvAddress(); - ConfigurationImpl configurationImpl = (ConfigurationImpl) configuration; - try (CircuitBreakAPI circuitBreakAPI = CircuitBreakAPIFactory.createCircuitBreakAPIByConfig(configurationImpl)) { - for (int i = 0; i < 10; i++) { - String method = ""; - if (i < 9) { - method = "/test1/path/" + i; - } - FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( - matchMethodDetectService, method); - FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); - int finalI = i; - Consumer integerConsumer = decorator.decorateConsumer(num -> { - if (num % 2 == 0) { - throw new IllegalArgumentException("invoke failed" + finalI); + try { + integerConsumer.accept(2); } - else { - System.out.println("invoke success" + finalI); + catch (Exception e) { + if (!(e instanceof IllegalArgumentException)) { + throw e; + } } - }); - integerConsumer.accept(1); - } - BaseEngine baseEngine = (BaseEngine) circuitBreakAPI; - CircuitBreaker resourceBreaker = baseEngine.getSDKContext().getExtensions().getResourceBreaker(); - PolarisCircuitBreaker polarisCircuitBreaker = (PolarisCircuitBreaker) resourceBreaker; - Map healthCheckCache = polarisCircuitBreaker.getHealthCheckCache(); - Assert.assertEquals(1, healthCheckCache.size()); - HealthCheckContainer healthCheckContainer = healthCheckCache.get(matchMethodDetectService); - Assert.assertNotNull(healthCheckContainer); - Collection healthCheckerValues = healthCheckContainer.getHealthCheckerValues(); - Assert.assertEquals(2, healthCheckerValues.size()); -// for (ResourceHealthChecker resourceHealthChecker : healthCheckerValues) { -// if (StringUtils.equals(resourceHealthChecker.getFaultDetectRule().getId(), "fd1")) { -// Assert.assertEquals(1, resourceHealthChecker.getResources().size()); -// } -// } - FaultDetectorProto.FaultDetectRule rule1 = loadFdRule("faultDetectMethodRuleChanged.json"); - FaultDetectorProto.FaultDetector faultDetector = FaultDetectorProto.FaultDetector.newBuilder() - .addRules(rule1).setRevision("33333").build(); - namingServer.getNamingService().setFaultDetector(matchMethodDetectService, faultDetector); - - Utils.sleepUninterrupted(20 * 1000); - healthCheckContainer = healthCheckCache.get(matchMethodDetectService); - Assert.assertNull(healthCheckContainer); - for (int i = 0; i < 3; i++) { - String method = ""; - if (i > 0) { - method = "/test1/path/" + i; + Assert.assertThrows(CallAbortedException.class, () -> integerConsumer.accept(3)); + } + else { + Assert.assertThrows(CallAbortedException.class, () -> integerConsumer.accept(1)); } - FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( - matchMethodDetectService, method); - FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); - int finalI = i; - Consumer integerConsumer = decorator.decorateConsumer(num -> { - if (num % 2 == 0) { - throw new IllegalArgumentException("invoke failed" + finalI); - } - else { - System.out.println("invoke success" + finalI); - } - }); - integerConsumer.accept(1); } - healthCheckContainer = healthCheckCache.get(matchMethodDetectService); - Assert.assertNotNull(healthCheckContainer); - healthCheckerValues = healthCheckContainer.getHealthCheckerValues(); - Assert.assertEquals(1, healthCheckerValues.size()); -// for (ResourceHealthChecker resourceHealthChecker : healthCheckerValues) { -// Assert.assertEquals(2, resourceHealthChecker.getResources().size()); -// } } } diff --git a/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/FaultDetectorTest.java b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/FaultDetectorTest.java new file mode 100644 index 000000000..012fe9314 --- /dev/null +++ b/polaris-circuitbreaker/polaris-circuitbreaker-factory/src/test/java/com/tencent/polaris/circuitbreaker/factory/test/FaultDetectorTest.java @@ -0,0 +1,160 @@ +/* + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.circuitbreaker.factory.test; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.function.Consumer; + +import com.google.protobuf.StringValue; +import com.tencent.polaris.api.config.Configuration; +import com.tencent.polaris.api.plugin.circuitbreaker.CircuitBreaker; +import com.tencent.polaris.api.pojo.ServiceKey; +import com.tencent.polaris.circuitbreak.api.CircuitBreakAPI; +import com.tencent.polaris.circuitbreak.api.FunctionalDecorator; +import com.tencent.polaris.circuitbreak.api.pojo.FunctionalDecoratorRequest; +import com.tencent.polaris.circuitbreak.factory.CircuitBreakAPIFactory; +import com.tencent.polaris.client.api.BaseEngine; +import com.tencent.polaris.client.util.Utils; +import com.tencent.polaris.factory.config.ConfigurationImpl; +import com.tencent.polaris.plugins.circuitbreaker.composite.HealthCheckContainer; +import com.tencent.polaris.plugins.circuitbreaker.composite.PolarisCircuitBreaker; +import com.tencent.polaris.plugins.circuitbreaker.composite.ResourceHealthChecker; +import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; +import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto; +import com.tencent.polaris.test.common.TestUtils; +import com.tencent.polaris.test.mock.discovery.NamingServer; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static com.tencent.polaris.test.common.TestUtils.SERVER_ADDRESS_ENV; + +public class FaultDetectorTest { + + private NamingServer namingServer; + + private final ServiceKey matchMethodService = new ServiceKey("Test", "SvcCbMethod"); + + private final ServiceKey matchMethodDetectService = new ServiceKey("Test", "SvcCbMethodDetect"); + + @Before + public void before() throws IOException { + try { + namingServer = NamingServer.startNamingServer(-1); + System.setProperty(SERVER_ADDRESS_ENV, String.format("127.0.0.1:%d", namingServer.getPort())); + } + catch (IOException e) { + Assert.fail(e.getMessage()); + } + + CircuitBreakerProto.CircuitBreakerRule cbRule1 = CbTestUtils.loadCbRule("circuitBreakerMethodRuleNoDetect.json"); + CircuitBreakerProto.CircuitBreaker circuitBreaker = CircuitBreakerProto.CircuitBreaker.newBuilder() + .addRules(cbRule1).setRevision(StringValue.newBuilder().setValue("0000").build()).build(); + namingServer.getNamingService().setCircuitBreaker(matchMethodService, circuitBreaker); + + + CircuitBreakerProto.CircuitBreakerRule cbRule3 = CbTestUtils.loadCbRule("circuitBreakerMethodRule.json"); + CircuitBreakerProto.CircuitBreakerRule cbRule4 = CbTestUtils.loadCbRule("circuitBreakerRule.json"); + circuitBreaker = CircuitBreakerProto.CircuitBreaker.newBuilder() + .addRules(cbRule3).addRules(cbRule4).setRevision(StringValue.newBuilder().setValue("1111").build()).build(); + namingServer.getNamingService().setCircuitBreaker(matchMethodDetectService, circuitBreaker); + FaultDetectorProto.FaultDetectRule rule1 = CbTestUtils.loadFdRule("faultDetectRule.json"); + FaultDetectorProto.FaultDetectRule rule2 = CbTestUtils.loadFdRule("faultDetectMethodRule.json"); + FaultDetectorProto.FaultDetector faultDetector = FaultDetectorProto.FaultDetector.newBuilder() + .addRules(rule1).addRules(rule2).setRevision("2222").build(); + namingServer.getNamingService().setFaultDetector(matchMethodDetectService, faultDetector); + } + + @Test + public void testFaultDetectRuleChanged() throws IOException { + Configuration configuration = TestUtils.configWithEnvAddress(); + ConfigurationImpl configurationImpl = (ConfigurationImpl) configuration; + try (CircuitBreakAPI circuitBreakAPI = CircuitBreakAPIFactory.createCircuitBreakAPIByConfig(configurationImpl)) { + for (int i = 0; i < 10; i++) { + String method = ""; + if (i < 9) { + method = "/test1/path/" + i; + } + FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( + matchMethodDetectService, method); + FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); + int finalI = i; + Consumer integerConsumer = decorator.decorateConsumer(num -> { + if (num % 2 == 0) { + throw new IllegalArgumentException("invoke failed" + finalI); + } + else { + System.out.println("invoke success" + finalI); + } + }); + integerConsumer.accept(1); + } + BaseEngine baseEngine = (BaseEngine) circuitBreakAPI; + CircuitBreaker resourceBreaker = baseEngine.getSDKContext().getExtensions().getResourceBreaker(); + PolarisCircuitBreaker polarisCircuitBreaker = (PolarisCircuitBreaker) resourceBreaker; + Map healthCheckCache = polarisCircuitBreaker.getHealthCheckCache(); + Assert.assertEquals(1, healthCheckCache.size()); + HealthCheckContainer healthCheckContainer = healthCheckCache.get(matchMethodDetectService); + Assert.assertNotNull(healthCheckContainer); + Collection healthCheckerValues = healthCheckContainer.getHealthCheckerValues(); + Assert.assertEquals(2, healthCheckerValues.size()); + + FaultDetectorProto.FaultDetectRule rule1 = CbTestUtils.loadFdRule("faultDetectMethodRuleChanged.json"); + FaultDetectorProto.FaultDetector faultDetector = FaultDetectorProto.FaultDetector.newBuilder() + .addRules(rule1).setRevision("33333").build(); + namingServer.getNamingService().setFaultDetector(matchMethodDetectService, faultDetector); + + Utils.sleepUninterrupted(20 * 1000); + healthCheckContainer = healthCheckCache.get(matchMethodDetectService); + Assert.assertNull(healthCheckContainer); + for (int i = 0; i < 3; i++) { + String method = ""; + if (i > 0) { + method = "/test1/path/" + i; + } + FunctionalDecoratorRequest makeDecoratorRequest = new FunctionalDecoratorRequest( + matchMethodDetectService, method); + FunctionalDecorator decorator = circuitBreakAPI.makeFunctionalDecorator(makeDecoratorRequest); + int finalI = i; + Consumer integerConsumer = decorator.decorateConsumer(num -> { + if (num % 2 == 0) { + throw new IllegalArgumentException("invoke failed" + finalI); + } + else { + System.out.println("invoke success" + finalI); + } + }); + integerConsumer.accept(1); + } + healthCheckContainer = healthCheckCache.get(matchMethodDetectService); + Assert.assertNotNull(healthCheckContainer); + healthCheckerValues = healthCheckContainer.getHealthCheckerValues(); + Assert.assertEquals(1, healthCheckerValues.size()); + } + } + + @After + public void after() { + if (null != namingServer) { + namingServer.terminate(); + } + } +} diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java index 1f7801814..2f7c31f60 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; @@ -28,6 +29,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; @@ -43,6 +45,7 @@ import com.tencent.polaris.api.plugin.circuitbreaker.CircuitBreaker; import com.tencent.polaris.api.plugin.circuitbreaker.ResourceStat; import com.tencent.polaris.api.plugin.circuitbreaker.entity.InstanceResource; +import com.tencent.polaris.api.plugin.circuitbreaker.entity.MethodResource; import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; import com.tencent.polaris.api.plugin.common.InitContext; import com.tencent.polaris.api.plugin.common.PluginTypes; @@ -56,6 +59,7 @@ import com.tencent.polaris.api.pojo.ServiceResourceProvider; import com.tencent.polaris.api.pojo.ServiceRule; import com.tencent.polaris.api.utils.CollectionUtils; +import com.tencent.polaris.api.utils.RuleUtils; import com.tencent.polaris.client.flow.DefaultServiceResourceProvider; import com.tencent.polaris.client.util.NamedThreadFactory; import com.tencent.polaris.logging.LoggerFactory; @@ -64,6 +68,7 @@ import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.Level; import com.tencent.polaris.specification.api.v1.fault.tolerance.FaultDetectorProto; +import com.tencent.polaris.specification.api.v1.model.ModelProto; import org.slf4j.Logger; public class PolarisCircuitBreaker extends Destroyable implements CircuitBreaker { @@ -83,6 +88,10 @@ public class PolarisCircuitBreaker extends Destroyable implements CircuitBreaker private final ScheduledExecutorService expiredCleanupExecutors = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("circuitbreaker-expired-cleanup-worker")); + // map the wildcard resource to rule specific resource, + // eg. /path/wildcard/123 => /path/wildcard/.+ + private Cache resourceMapping; + private Extensions extensions; private ServiceResourceProvider serviceResourceProvider; @@ -101,7 +110,19 @@ public class PolarisCircuitBreaker extends Destroyable implements CircuitBreaker @Override public CircuitBreakerStatus checkResource(Resource resource) { - Optional resourceCounters = getResourceCounters(resource); + Resource ruleResource = getActualResource(resource); + Optional resourceCounters = getResourceCounters(ruleResource); + if (null == resourceCounters) { + if (resource.getLevel() == Level.METHOD && ruleResource == resource) { + // 可能是被淘汰了，需要重新计算RuleResource + CircuitBreakerProto.CircuitBreakerRule circuitBreakerRule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); + ruleResource = computeResourceByRule(resource, circuitBreakerRule); + if (!Objects.equals(ruleResource, resource)) { + resourceMapping.put(resource, ruleResource); + resourceCounters = getResourceCounters(ruleResource); + } + } + } if (null == resourceCounters || !resourceCounters.isPresent()) { return null; } @@ -118,12 +139,21 @@ public void report(ResourceStat resourceStat) { doReport(resourceStat, true); } + public Resource getActualResource(Resource resource) { + Resource ruleResource = resourceMapping.getIfPresent(resource); + if (null == ruleResource) { + ruleResource = resource; + } + return ruleResource; + } + private ResourceCounters getOrInitResourceCounters(Resource resource) throws ExecutionException { - Optional resourceCounters = getResourceCounters(resource); + Resource ruleResource = getActualResource(resource); + Optional resourceCounters = getResourceCounters(ruleResource); boolean reloadFaultDetect = false; if (null == resourceCounters) { synchronized (countersCache) { - resourceCounters = getResourceCounters(resource); + resourceCounters = getResourceCounters(ruleResource); if (null == resourceCounters) { resourceCounters = initResourceCounter(resource); reloadFaultDetect = true; @@ -181,7 +211,7 @@ private void reloadFaultDetector(Resource resource, ResourceCounters resourceCou if (null == healthCheckContainer) { return; } - healthCheckContainer.removeResource(resource); + healthCheckContainer.removeResource(getActualResource(resource)); } else { if (null == healthCheckContainer) { @@ -190,6 +220,7 @@ private void reloadFaultDetector(Resource resource, ResourceCounters resourceCou healthCheckContainer = healthCheckCache.computeIfAbsent(resource.getService(), new Function() { @Override public HealthCheckContainer apply(ServiceKey serviceKey) { + LOG.info("[CIRCUIT_BREAKER] init health check cache for service {}", serviceKey); return new HealthCheckContainer(serviceKey, faultDetectRules, PolarisCircuitBreaker.this); } }); @@ -236,18 +267,35 @@ private Optional initResourceCounter(Resource resource) throws } Cache> resourceOptionalCache = countersCache.get(resource.getLevel()); CircuitBreakerProto.CircuitBreakerRule finalCircuitBreakerRule = circuitBreakerRule; - return resourceOptionalCache.get(resource, new Callable>() { + Resource ruleResource = computeResourceByRule(resource, circuitBreakerRule); + return resourceOptionalCache.get(ruleResource, new Callable>() { @Override public Optional call() { + resourceMapping.put(resource, ruleResource); if (null == finalCircuitBreakerRule) { return Optional.empty(); } - return Optional.of(new ResourceCounters(resource, finalCircuitBreakerRule, + return Optional.of(new ResourceCounters(ruleResource, finalCircuitBreakerRule, getStateChangeExecutors(), PolarisCircuitBreaker.this)); } }); } + private Resource computeResourceByRule(Resource resource, CircuitBreakerProto.CircuitBreakerRule circuitBreakerRule) { + if (null == circuitBreakerRule || resource.getLevel() != Level.METHOD) { + return resource; + } + ModelProto.MatchString method = circuitBreakerRule.getRuleMatcher().getDestination().getMethod(); + if (method.getType() == ModelProto.MatchString.MatchStringType.EXACT && !RuleUtils.isMatchAllValue(method.getValue().getValue())) { + return resource; + } + //new path = matchPath + ":" + matchType + String newPath = method.getValue().getValue() + ":" + method.getType().name(); + MethodResource originalResource = (MethodResource) resource; + return new MethodResource(originalResource.getService(), newPath, originalResource.getCallerService()); + + } + private void addInstanceForFaultDetect(Resource resource) { if (!(resource instanceof InstanceResource)) { return; @@ -289,12 +337,10 @@ public void onRemoval(RemovalNotification> @Override public void init(InitContext ctx) throws PolarisException { long expireIntervalMilli = ctx.getConfig().getConsumer().getCircuitBreaker().getCountersExpireInterval(); - countersCache.put(Level.SERVICE, CacheBuilder.newBuilder().expireAfterAccess( - expireIntervalMilli, TimeUnit.MILLISECONDS).removalListener(new CounterRemoveListener()).build()); - countersCache.put(Level.METHOD, CacheBuilder.newBuilder().expireAfterAccess( - expireIntervalMilli, TimeUnit.MILLISECONDS).removalListener(new CounterRemoveListener()).build()); - countersCache.put(Level.INSTANCE, CacheBuilder.newBuilder().expireAfterAccess( - expireIntervalMilli, TimeUnit.MILLISECONDS).removalListener(new CounterRemoveListener()).build()); + resourceMapping = CacheBuilder.newBuilder().expireAfterAccess(expireIntervalMilli, TimeUnit.MILLISECONDS).build(); + countersCache.put(Level.SERVICE, CacheBuilder.newBuilder().removalListener(new CounterRemoveListener()).build()); + countersCache.put(Level.METHOD, CacheBuilder.newBuilder().removalListener(new CounterRemoveListener()).build()); + countersCache.put(Level.INSTANCE, CacheBuilder.newBuilder().removalListener(new CounterRemoveListener()).build()); checkPeriod = ctx.getConfig().getConsumer().getCircuitBreaker().getCheckPeriod(); circuitBreakerConfig = ctx.getConfig().getConsumer().getCircuitBreaker(); healthCheckInstanceExpireInterval = HealthCheckUtils.CHECK_PERIOD_MULTIPLE * checkPeriod; @@ -310,15 +356,30 @@ public void postContextInit(Extensions extensions) throws PolarisException { healthCheckers = extensions.getAllHealthCheckers(); long expireIntervalMilli = extensions.getConfiguration().getConsumer().getCircuitBreaker() .getCountersExpireInterval(); + long cleanupIntervalMilli = Math.max(expireIntervalMilli, CircuitBreakerUtils.MIN_CLEANUP_INTERVAL); expiredCleanupExecutors.scheduleWithFixedDelay(new Runnable() { @Override public void run() { - for (Map.Entry>> entry : countersCache.entrySet()) { - Cache> value = entry.getValue(); - value.cleanUp(); - } + cleanupExpiredResources(); } - }, expireIntervalMilli, expireIntervalMilli, TimeUnit.MILLISECONDS); + }, cleanupIntervalMilli, cleanupIntervalMilli, TimeUnit.MILLISECONDS); + } + + public void cleanupExpiredResources() { + resourceMapping.cleanUp(); + for (Map.Entry>> entry : countersCache.entrySet()) { + Cache> values = entry.getValue(); + values.asMap().forEach(new BiConsumer>() { + @Override + public void accept(Resource resource, Optional resourceCounters) { + // 每隔一段时间清理占位的缓存数据，避免没规则的情况下，counters无法收敛 + if (!resourceCounters.isPresent()) { + values.invalidate(resource); + } + } + }); + values.cleanUp(); + } } // for test @@ -397,6 +458,10 @@ public void setCircuitBreakerConfig(CircuitBreakerConfig circuitBreakerConfig) { this.circuitBreakerConfig = circuitBreakerConfig; } + Cache getResourceMapping() { + return resourceMapping; + } + @Override public String getName() { return DefaultPlugins.CIRCUIT_BREAKER_COMPOSITE; @@ -409,7 +474,7 @@ void onCircuitBreakerRuleChanged(ServiceKey serviceKey) { for (Map.Entry>> entry : countersCache.entrySet()) { Cache> cacheValue = entry.getValue(); for (Resource resource : cacheValue.asMap().keySet()) { - if (resource.getService().equals(serviceKey)) { + if (Objects.equals(resource.getService(), serviceKey)) { cacheValue.invalidate(resource); } HealthCheckContainer healthCheckContainer = healthCheckCache.get(serviceKey); @@ -430,7 +495,7 @@ void onCircuitBreakerRuleAdded(ServiceKey serviceKey) { Cache> cacheValue = entry.getValue(); for (Map.Entry> entryCache: cacheValue.asMap().entrySet()) { Resource resource = entryCache.getKey(); - if (resource.getService().equals(serviceKey) && !entryCache.getValue().isPresent()) { + if (Objects.equals(resource.getService(), serviceKey) && !entryCache.getValue().isPresent()) { cacheValue.invalidate(resource); } } @@ -458,7 +523,7 @@ public HealthCheckContainer apply(ServiceKey serviceKey, HealthCheckContainer he Cache> cacheValue = entry.getValue(); for (Map.Entry> entryCache : cacheValue.asMap().entrySet()) { Resource resource = entryCache.getKey(); - if (resource.getService().equals(svcKey)) { + if (Objects.equals(resource.getService(), svcKey)) { if (entryCache.getValue().isPresent()) { LOG.info("onFaultDetectRuleChanged: ResourceCounters {} setReloadFaultDetect true", svcKey); ResourceCounters resourceCounters = entryCache.getValue().get(); diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceHealthChecker.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceHealthChecker.java index 8872c35bc..ab7f7fc1d 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceHealthChecker.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/ResourceHealthChecker.java @@ -191,13 +191,19 @@ private boolean doCheck(Instance instance, Protocol protocol, FaultDetectRule fa instance.getHost(), instance.getPort(), protocol, detectResult.getStatusCode(), detectResult.getDelay(), detectResult.getRetStatus(), faultDetectRule.getName()); Set copiedResources = new HashSet<>(resources.keySet()); + Set reportedResources = new HashSet<>(); for (Resource resource : copiedResources) { if (!matchInstanceToResource(instance, resource)) { continue; } + Resource actualResource = polarisCircuitBreaker.getActualResource(resource); + if (reportedResources.contains(actualResource)) { + continue; + } + reportedResources.add(actualResource); ResourceStat resourceStat = new ResourceStat( - resource, detectResult.getStatusCode(), detectResult.getDelay(), detectResult.getRetStatus()); - HC_EVENT_LOG.info("report health check to resource {}, status code {}, delay {}", resource, + actualResource, detectResult.getStatusCode(), detectResult.getDelay(), detectResult.getRetStatus()); + HC_EVENT_LOG.info("report health check to resource {}, status code {}, delay {}", actualResource, detectResult.getStatusCode(), detectResult.getDelay()); polarisCircuitBreaker.doReport(resourceStat, false); } diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/CircuitBreakerUtils.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/CircuitBreakerUtils.java index 3b14d01ec..de6cee97c 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/CircuitBreakerUtils.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/utils/CircuitBreakerUtils.java @@ -24,6 +24,8 @@ public class CircuitBreakerUtils { public static long DEFAULT_ERROR_RATE_INTERVAL_MS = 60 * 1000; + public static long MIN_CLEANUP_INTERVAL = 60 * 1000; + public static boolean checkRule(CircuitBreakerProto.CircuitBreakerRule rule) { return checkLevel(rule.getLevel()); } diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/PolarisCircuitBreakerTest.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreakerTest.java similarity index 77% rename from polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/PolarisCircuitBreakerTest.java rename to polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreakerTest.java index 181a84425..f2a0a0373 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/PolarisCircuitBreakerTest.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreakerTest.java @@ -15,7 +15,7 @@ * specific language governing permissions and limitations under the License. */ -package com.tencent.polaris.plugins.circuitbraker.composite.trigger; +package com.tencent.polaris.plugins.circuitbreaker.composite; import java.util.Optional; import java.util.regex.Pattern; @@ -25,6 +25,7 @@ import com.google.protobuf.StringValue; import com.tencent.polaris.api.config.Configuration; import com.tencent.polaris.api.plugin.circuitbreaker.ResourceStat; +import com.tencent.polaris.api.plugin.circuitbreaker.entity.MethodResource; import com.tencent.polaris.api.plugin.circuitbreaker.entity.Resource; import com.tencent.polaris.api.plugin.circuitbreaker.entity.ServiceResource; import com.tencent.polaris.api.plugin.common.InitContext; @@ -36,10 +37,7 @@ import com.tencent.polaris.api.pojo.ServiceRule; import com.tencent.polaris.client.pojo.ServiceRuleByProto; import com.tencent.polaris.factory.ConfigAPIFactory; -import com.tencent.polaris.plugins.circuitbreaker.composite.CircuitBreakerRuleDictionary; -import com.tencent.polaris.plugins.circuitbreaker.composite.CircuitBreakerRuleListener; -import com.tencent.polaris.plugins.circuitbreaker.composite.FaultDetectRuleDictionary; -import com.tencent.polaris.plugins.circuitbreaker.composite.PolarisCircuitBreaker; +import com.tencent.polaris.factory.config.ConfigurationImpl; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreaker; import com.tencent.polaris.specification.api.v1.fault.tolerance.CircuitBreakerProto.CircuitBreakerRule; @@ -194,8 +192,8 @@ public CircuitBreaker buildMethodRules() { CircuitBreakerProto.RuleMatcher.newBuilder(). setSource(CircuitBreakerProto.RuleMatcher.SourceService.newBuilder().setNamespace("*").setService("*").build()). setDestination(CircuitBreakerProto.RuleMatcher.DestinationService.newBuilder(). - setNamespace("*").setService("*").setMethod(MatchString.newBuilder(). - setValue(StringValue.newBuilder().setValue(".*")). + setNamespace("Test").setService("testMethodSvc").setMethod(MatchString.newBuilder(). + setValue(StringValue.newBuilder().setValue("^/d/customers/base/.+$")). setType(MatchStringType.REGEX)).build())); builder.addTriggerCondition( TriggerCondition.newBuilder().setTriggerType(TriggerType.CONSECUTIVE_ERROR).setErrorCount(10).build()); @@ -206,12 +204,55 @@ public CircuitBreaker buildMethodRules() { builder.setRevision("222"); CircuitBreaker.Builder cbBuilder = CircuitBreaker.newBuilder(); cbBuilder.addRules(builder); - cbBuilder.setRevision(StringValue.newBuilder().setValue("xxxxxyyyyyx").build()); + cbBuilder.setRevision(StringValue.newBuilder().setValue("xxxxxy1yyyx").build()); return cbBuilder.build(); } @Test - public void testCounterExpire() { + public void testCircuitBreakerMethodRules() { + MockServiceResourceProvider mockServiceRuleProvider = new MockServiceResourceProvider(); + PolarisCircuitBreaker polarisCircuitBreaker = new PolarisCircuitBreaker(); + Configuration configuration = ConfigAPIFactory.defaultConfig(); + ConfigurationImpl configurationImpl = (ConfigurationImpl) configuration; + configurationImpl.getConsumer().getCircuitBreaker().setCountersExpireInterval(5000); + InitContext initContext = new MockInitContext(configuration); + polarisCircuitBreaker.init(initContext); + polarisCircuitBreaker.setServiceRuleProvider(mockServiceRuleProvider); + polarisCircuitBreaker.setCircuitBreakerRuleDictionary(new CircuitBreakerRuleDictionary(Pattern::compile)); + polarisCircuitBreaker.setFaultDetectRuleDictionary(new FaultDetectRuleDictionary()); + ServiceKey serviceKey = new ServiceKey("Test", "testMethodSvc"); + ServiceEventKey serviceEventKey = new ServiceEventKey(serviceKey, EventType.CIRCUIT_BREAKING); + CircuitBreaker circuitBreaker = buildMethodRules(); + ServiceRuleByProto serviceRule = new ServiceRuleByProto(circuitBreaker, circuitBreaker.getRevision().getValue(), + false, + EventType.CIRCUIT_BREAKING); + mockServiceRuleProvider.putServiceRule(serviceEventKey, serviceRule); + for (int i = 0; i < 1000; i++) { + Resource methodResource = new MethodResource(serviceKey, String.format("/d/customers/base/%d", i)); + ResourceStat resourceStat = new ResourceStat(methodResource, 500, 1000); + polarisCircuitBreaker.report(resourceStat); + } + try { + Thread.sleep(200); + } catch (InterruptedException e) { + e.printStackTrace(); + } + for (int i = 0; i < 1000; i++) { + Resource methodResource = new MethodResource(serviceKey, String.format("/d/customers/base/%d", i)); + CircuitBreakerStatus circuitBreakerStatus = polarisCircuitBreaker.checkResource(methodResource); + Assert.assertEquals(Status.OPEN, circuitBreakerStatus.getStatus()); + } + Assert.assertEquals(1, polarisCircuitBreaker.getCountersCache().get(Level.METHOD).size()); + Assert.assertEquals(1000, polarisCircuitBreaker.getResourceMapping().size()); + + //check cleanup + try { + Thread.sleep(6000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + polarisCircuitBreaker.cleanupExpiredResources(); + Assert.assertEquals(0, polarisCircuitBreaker.getResourceMapping().size()); } } diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/CircuitBreakerRuleContainerTest.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/CircuitBreakerRuleContainerTest.java similarity index 97% rename from polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/CircuitBreakerRuleContainerTest.java rename to polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/CircuitBreakerRuleContainerTest.java index 3dfb3e210..3e2201025 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/CircuitBreakerRuleContainerTest.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/CircuitBreakerRuleContainerTest.java @@ -15,7 +15,7 @@ * specific language governing permissions and limitations under the License. */ -package com.tencent.polaris.plugins.circuitbraker.composite.trigger; +package com.tencent.polaris.plugins.circuitbreaker.composite.trigger; import java.util.function.Function; import java.util.regex.Pattern; diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ConsecutiveCounterTest.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ConsecutiveCounterTest.java similarity index 94% rename from polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ConsecutiveCounterTest.java rename to polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ConsecutiveCounterTest.java index 0f7e93ab6..388ec47d3 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ConsecutiveCounterTest.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ConsecutiveCounterTest.java @@ -15,7 +15,7 @@ * specific language governing permissions and limitations under the License. */ -package com.tencent.polaris.plugins.circuitbraker.composite.trigger; +package com.tencent.polaris.plugins.circuitbreaker.composite.trigger; import com.tencent.polaris.api.plugin.circuitbreaker.entity.MethodResource; import com.tencent.polaris.api.pojo.ServiceKey; diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ErrRateCounterTest.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ErrRateCounterTest.java similarity index 95% rename from polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ErrRateCounterTest.java rename to polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ErrRateCounterTest.java index ab2240cd7..d9451f287 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ErrRateCounterTest.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ErrRateCounterTest.java @@ -15,7 +15,7 @@ * specific language governing permissions and limitations under the License. */ -package com.tencent.polaris.plugins.circuitbraker.composite.trigger; +package com.tencent.polaris.plugins.circuitbreaker.composite.trigger; import com.tencent.polaris.api.plugin.circuitbreaker.entity.MethodResource; import com.tencent.polaris.api.pojo.ServiceKey; diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceCountersTest.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ResourceCountersTest.java similarity index 97% rename from polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceCountersTest.java rename to polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ResourceCountersTest.java index b00ef7b48..3a1bddfa7 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceCountersTest.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ResourceCountersTest.java @@ -15,7 +15,7 @@ * specific language governing permissions and limitations under the License. */ -package com.tencent.polaris.plugins.circuitbraker.composite.trigger; +package com.tencent.polaris.plugins.circuitbreaker.composite.trigger; import com.google.protobuf.StringValue; import com.tencent.polaris.api.plugin.circuitbreaker.ResourceStat; diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceHealthCheckerTest.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ResourceHealthCheckerTest.java similarity index 96% rename from polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceHealthCheckerTest.java rename to polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ResourceHealthCheckerTest.java index 49aa2624a..ec194112a 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbraker/composite/trigger/ResourceHealthCheckerTest.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/test/java/com/tencent/polaris/plugins/circuitbreaker/composite/trigger/ResourceHealthCheckerTest.java @@ -15,7 +15,7 @@ * specific language governing permissions and limitations under the License. */ -package com.tencent.polaris.plugins.circuitbraker.composite.trigger; +package com.tencent.polaris.plugins.circuitbreaker.composite.trigger; import com.google.protobuf.StringValue; import com.tencent.polaris.api.plugin.circuitbreaker.entity.ServiceResource; diff --git a/pom.xml b/pom.xml index ba0f8fc43..377597eb8 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,7 @@ - 1.15.9 + 1.15.10-SNAPSHOT ${maven.build.timestamp} false From d19261f7b8e45af770f9ef244aa18fe03142cd39 Mon Sep 17 00:00:00 2001 From: andrew shan <45474304+andrewshan@users.noreply.github.com> Date: Sat, 10 Aug 2024 18:32:02 +0800 Subject: [PATCH 4/4] fix: use Objects.equals to compare objects --- .../plugins/circuitbreaker/composite/PolarisCircuitBreaker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java index 2f7c31f60..1b2ac668a 100644 --- a/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java +++ b/polaris-plugins/polaris-plugins-circuitbreaker/circuitbreaker-composite/src/main/java/com/tencent/polaris/plugins/circuitbreaker/composite/PolarisCircuitBreaker.java @@ -113,7 +113,7 @@ public CircuitBreakerStatus checkResource(Resource resource) { Resource ruleResource = getActualResource(resource); Optional resourceCounters = getResourceCounters(ruleResource); if (null == resourceCounters) { - if (resource.getLevel() == Level.METHOD && ruleResource == resource) { + if (resource.getLevel() == Level.METHOD && Objects.equals(ruleResource, resource)) { // 可能是被淘汰了，需要重新计算RuleResource CircuitBreakerProto.CircuitBreakerRule circuitBreakerRule = circuitBreakerRuleDictionary.lookupCircuitBreakerRule(resource); ruleResource = computeResourceByRule(resource, circuitBreakerRule);