From 45af26126f8f59a6210de520fdca363922d388de Mon Sep 17 00:00:00 2001 From: Elvis Souza Date: Wed, 6 Nov 2024 13:46:38 -0300 Subject: [PATCH] Canary Rate Threshold Circuit Breaker Strategy: All circuits open at start (#588) * implementing circuit breaker factory * fixmes * testing * testing * setup strategy mapping * fixing validation * must call hotload method once * adjusting logs * add server address as circuit breaker name * fixing compiling errors * refactoring * refactoring * fixing test * fixing test * fixing test * fixing test * fixing test * fixing test * fixing test * messages formatting and refactoring * refactoring * disabling metrics * heating circuit breakers at app startup * todos * adjusts * adjusts * fixing tests * fixing tests * fixing tests * testing * item tick * clearing todos * [Gradle Release Plugin] - new version commit: '3.31.1-snapshot'. * release notes --- RELEASE-NOTES.md | 3 + TODO.md | 0 gradle.properties | 2 +- .../java/com/mageddo/dns/utils/Messages.java | 16 +++--- .../dnsproxyserver/di/module/ModuleEager.java | 7 ++- .../dnsproxyserver/solver/SimpleResolver.java | 7 +++ .../failsafe/CircuitBreakerFactory.java | 6 +- .../application/CircuitBreakerDelegate.java | 11 +++- .../CircuitBreakerDelegateNonResilient.java | 7 ++- ...reakerDelegateStaticThresholdFailsafe.java | 11 ++-- .../application/DnsServerHealthChecker.java | 40 ++++++++++++++ .../application/HealthCheckerStatic.java | 15 +++++ ...uitBreakerDelegateCanaryRateThreshold.java | 31 ++++++++++- .../CircuitBreakerDelegateSelfObservable.java | 23 +++++--- .../CircuitBreakerFactory.java | 36 +++++++++--- .../CircuitExecutionsAsHealthChecker.java | 55 ------------------- .../entrypoint/CircuitBreakerHeater.java | 30 ++++++++++ .../FailSafeStateTransitor.java | 22 ++++++++ .../statetransitor/NopStateTransitor.java | 13 +++++ .../Resilience4jStateTransitor.java | 22 ++++++++ .../statetransitor/StateTransitor.java | 6 ++ .../failsafe/CircuitBreakerFactoryTest.java | 26 ++++++++- .../CircuitExecutionsAsHealthCheckerTest.java | 39 ------------- ...reakerDelegateCanaryRateThresholdTest.java | 2 +- ...cuitBreakerDelegateSelfObservableTest.java | 14 ++++- .../CircuitBreakerFactoryCompTest.java | 32 ++++++++--- .../CircuitBreakerConfigTemplates.java | 1 + .../CircuitBreakerDelegateTemplates.java | 14 +++++ 28 files changed, 346 insertions(+), 145 deletions(-) create mode 100644 TODO.md create mode 100644 src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/DnsServerHealthChecker.java create mode 100644 src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/HealthCheckerStatic.java delete mode 100644 src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitExecutionsAsHealthChecker.java create mode 100644 src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/entrypoint/CircuitBreakerHeater.java create mode 100644 src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/FailSafeStateTransitor.java create mode 100644 src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/NopStateTransitor.java create mode 100644 src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/Resilience4jStateTransitor.java create mode 100644 src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/StateTransitor.java delete mode 100644 src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitExecutionsAsHealthCheckerTest.java create mode 100644 src/test/java/testing/templates/solver/remote/CircuitBreakerDelegateTemplates.java diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 95b00bffd..6eb319043 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,6 @@ +## 3.31.1 +* Adjusting Canary Rate Threshold Circuit Breaker to mark all circuits as open at the start. #533 + ## 3.31.0 * Canary Rate Threshold Circuit Breaker. #533 diff --git a/TODO.md b/TODO.md new file mode 100644 index 000000000..e69de29bb diff --git a/gradle.properties b/gradle.properties index 536a17c03..84a45b516 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=3.31.0-snapshot +version=3.31.1-snapshot diff --git a/src/main/java/com/mageddo/dns/utils/Messages.java b/src/main/java/com/mageddo/dns/utils/Messages.java index e4827453d..eaadfb4a9 100644 --- a/src/main/java/com/mageddo/dns/utils/Messages.java +++ b/src/main/java/com/mageddo/dns/utils/Messages.java @@ -38,24 +38,24 @@ public static String simplePrint(Response res) { return simplePrint(res.getMessage()); } - public static String simplePrint(Message message) { - if (message == null) { + public static String simplePrint(Message reqOrRes) { + if (reqOrRes == null) { return null; } try { - final var answer = findFirstAnswerRecord(message); - final var rcode = message.getRcode(); + final var answer = findFirstAnswerRecord(reqOrRes); + final var rcode = reqOrRes.getRcode(); if (answer != null) { return String.format("rc=%d, res=%s", rcode, simplePrint(answer)); } - final var question = message.getQuestion(); + final var question = reqOrRes.getQuestion(); final var type = Objects.useItOrDefault( Objects.toString(Entry.Type.of(question.getType())), () -> String.valueOf(question.getType()) ); final var hostname = question.getName().toString(true); final var sb = new StringBuilder(); - if (Messages.hasFlag(message, Flags.QR)) { + if (Messages.hasFlag(reqOrRes, Flags.QR)) { sb.append("rc=") .append(rcode) .append(", ") @@ -64,8 +64,8 @@ public static String simplePrint(Message message) { sb.append(String.format("query=%s:%s", type, hostname)); return sb.toString(); } catch (Throwable e) { - log.warn("status=failedToSimplePrint, msg={}", message, e); - return String.valueOf(message); + log.warn("status=failedToSimplePrint, msg={}", reqOrRes, e); + return String.valueOf(reqOrRes); } } diff --git a/src/main/java/com/mageddo/dnsproxyserver/di/module/ModuleEager.java b/src/main/java/com/mageddo/dnsproxyserver/di/module/ModuleEager.java index b7326a34f..cd9ee23a4 100644 --- a/src/main/java/com/mageddo/dnsproxyserver/di/module/ModuleEager.java +++ b/src/main/java/com/mageddo/dnsproxyserver/di/module/ModuleEager.java @@ -1,6 +1,7 @@ package com.mageddo.dnsproxyserver.di.module; import com.mageddo.di.Eager; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.entrypoint.CircuitBreakerHeater; import dagger.Module; import dagger.Provides; import dagger.multibindings.ElementsIntoSet; @@ -13,7 +14,9 @@ public class ModuleEager { @Provides @Singleton @ElementsIntoSet - Set beans() { - return Set.of(); + Set beans(CircuitBreakerHeater a) { + return Set.of( + a + ); } } diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/SimpleResolver.java b/src/main/java/com/mageddo/dnsproxyserver/solver/SimpleResolver.java index 65a5ff1d9..ef837cac0 100644 --- a/src/main/java/com/mageddo/dnsproxyserver/solver/SimpleResolver.java +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/SimpleResolver.java @@ -1,5 +1,8 @@ package com.mageddo.dnsproxyserver.solver; +import com.mageddo.net.IpAddr; +import com.mageddo.net.IpAddrs; + import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -20,4 +23,8 @@ public SimpleResolver(InetSocketAddress addr) { public SimpleResolver(InetAddress host) { super(host); } + + public SimpleResolver(IpAddr addr) { + super(IpAddrs.toInetSocketAddress(addr)); + } } diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/application/failsafe/CircuitBreakerFactory.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/application/failsafe/CircuitBreakerFactory.java index 795ea5c71..0b8974630 100644 --- a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/application/failsafe/CircuitBreakerFactory.java +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/application/failsafe/CircuitBreakerFactory.java @@ -1,7 +1,6 @@ package com.mageddo.dnsproxyserver.solver.remote.application.failsafe; import com.mageddo.commons.lang.tuple.Pair; -import com.mageddo.dnsproxyserver.config.CanaryRateThresholdCircuitBreakerStrategyConfig; import com.mageddo.dnsproxyserver.config.CircuitBreakerStrategyConfig; import com.mageddo.dnsproxyserver.config.StaticThresholdCircuitBreakerStrategyConfig; import com.mageddo.dnsproxyserver.config.application.ConfigService; @@ -14,6 +13,7 @@ import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegateStaticThresholdFailsafe; import com.mageddo.dnsproxyserver.solver.remote.mapper.ResolverMapper; import com.mageddo.net.IpAddr; +import com.mageddo.net.IpAddrs; import lombok.RequiredArgsConstructor; import lombok.Value; import lombok.extern.slf4j.Slf4j; @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; @Slf4j @Singleton @@ -67,8 +68,7 @@ CircuitBreakerDelegate findCircuitBreakerHotLoad(InetSocketAddress address) { } CircuitBreakerDelegate buildCanaryRateThreshold(CircuitBreakerStrategyConfig config, InetSocketAddress address) { -// return this.canaryThresholdFactory.build(config, IpAddrs.from(address)); - return this.canaryThresholdFactory.build((CanaryRateThresholdCircuitBreakerStrategyConfig) config); + return this.canaryThresholdFactory.build(config, IpAddrs.from(address)); } private CircuitBreakerDelegateStaticThresholdFailsafe buildStaticThresholdFailSafeCircuitBreaker( diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegate.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegate.java index 1e1590724..9dd21e867 100644 --- a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegate.java +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegate.java @@ -2,6 +2,7 @@ import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus; import com.mageddo.dnsproxyserver.solver.remote.Result; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor; import java.util.function.Supplier; @@ -11,5 +12,13 @@ public interface CircuitBreakerDelegate { CircuitStatus findStatus(); - void transitionToHalfOpenState(); + StateTransitor stateTransitor(); + + default void transitionToHalfOpenState(){ + this.stateTransitor().halfOpen(); + } + + default void transitionToClosedState(){ + this.stateTransitor().closed(); + } } diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegateNonResilient.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegateNonResilient.java index c3adae278..c5ccf52db 100644 --- a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegateNonResilient.java +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegateNonResilient.java @@ -2,6 +2,8 @@ import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus; import com.mageddo.dnsproxyserver.solver.remote.Result; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.NopStateTransitor; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor; import java.util.function.Supplier; @@ -18,7 +20,8 @@ public CircuitStatus findStatus() { } @Override - public void transitionToHalfOpenState() { - + public StateTransitor stateTransitor() { + return new NopStateTransitor(); } + } diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegateStaticThresholdFailsafe.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegateStaticThresholdFailsafe.java index 2381615a4..a38e2106e 100644 --- a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegateStaticThresholdFailsafe.java +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegateStaticThresholdFailsafe.java @@ -1,9 +1,11 @@ package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application; import com.mageddo.circuitbreaker.failsafe.CircuitStatusRefresh; +import com.mageddo.commons.circuitbreaker.CircuitIsOpenException; import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus; import com.mageddo.dnsproxyserver.solver.remote.Result; -import com.mageddo.commons.circuitbreaker.CircuitIsOpenException; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.FailSafeStateTransitor; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor; import com.mageddo.dnsproxyserver.solver.remote.mapper.CircuitBreakerStateMapper; import dev.failsafe.CircuitBreaker; import dev.failsafe.CircuitBreakerOpenException; @@ -25,7 +27,7 @@ public Result execute(Supplier sup) { return Failsafe .with(this.circuitBreaker) .get((ctx) -> sup.get()); - } catch (CircuitBreakerOpenException e){ + } catch (CircuitBreakerOpenException e) { throw new CircuitIsOpenException(e); } } @@ -37,7 +39,8 @@ public CircuitStatus findStatus() { } @Override - public void transitionToHalfOpenState() { - this.circuitBreaker.halfOpen(); + public StateTransitor stateTransitor() { + return new FailSafeStateTransitor(circuitBreaker); } + } diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/DnsServerHealthChecker.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/DnsServerHealthChecker.java new file mode 100644 index 000000000..d1f35cb0f --- /dev/null +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/DnsServerHealthChecker.java @@ -0,0 +1,40 @@ +package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application; + +import com.mageddo.dns.utils.Messages; +import com.mageddo.dnsproxyserver.solver.SimpleResolver; +import com.mageddo.net.IpAddr; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ClassUtils; + +import java.io.IOException; + +@Slf4j +public class DnsServerHealthChecker implements HealthChecker { + + private final SimpleResolver resolver; + + public DnsServerHealthChecker(IpAddr addr) { + this.resolver = new SimpleResolver(addr); + } + + @Override + public boolean isHealthy() { + final var req = Messages.aQuestion("dps.dns.test"); + try { + final var res = this.resolver.send(req); + log.debug("status=done, server={}, res={}", this.resolver.getAddress(), Messages.simplePrint(res)); + return Messages.findQuestionTypeCode(res) != null; + } catch (IOException e) { + log.debug( + "status=failed, server={}, res={}, clazz={}", + this.resolver.getAddress(), e.getMessage(), ClassUtils.getSimpleName(e) + ); + return false; + } + } + + @Override + public String toString() { + return String.valueOf(this.resolver.getAddress()); + } +} diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/HealthCheckerStatic.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/HealthCheckerStatic.java new file mode 100644 index 000000000..efbbd360f --- /dev/null +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/HealthCheckerStatic.java @@ -0,0 +1,15 @@ +package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application; + +public class HealthCheckerStatic implements HealthChecker { + + private final boolean healthy; + + public HealthCheckerStatic(boolean healthy) { + this.healthy = healthy; + } + + @Override + public boolean isHealthy() { + return this.healthy; + } +} diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateCanaryRateThreshold.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateCanaryRateThreshold.java index f78c69705..bdc044d82 100644 --- a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateCanaryRateThreshold.java +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateCanaryRateThreshold.java @@ -4,10 +4,14 @@ import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus; import com.mageddo.dnsproxyserver.solver.remote.Result; import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegate; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.Resilience4jStateTransitor; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor; import com.mageddo.dnsproxyserver.solver.remote.mapper.Resilience4jStatusMapper; +import com.mageddo.json.JsonUtils; import io.github.resilience4j.circuitbreaker.CallNotPermittedException; import io.github.resilience4j.circuitbreaker.CircuitBreaker; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import java.util.function.Supplier; @@ -15,9 +19,11 @@ public class CircuitBreakerDelegateCanaryRateThreshold implements CircuitBreakerDelegate { private final CircuitBreaker circuitBreaker; + private final String name; - public CircuitBreakerDelegateCanaryRateThreshold(CircuitBreaker circuitBreaker) { + public CircuitBreakerDelegateCanaryRateThreshold(CircuitBreaker circuitBreaker, String name) { this.circuitBreaker = circuitBreaker; + this.name = name; } @Override @@ -31,11 +37,32 @@ public Result execute(Supplier sup) { @Override public CircuitStatus findStatus() { - return Resilience4jStatusMapper.toCircuitStatus(this.circuitBreaker.getState()); + final var status = Resilience4jStatusMapper.toCircuitStatus(this.circuitBreaker.getState()); + if (log.isTraceEnabled()) { + log.trace("circuit={}, status={}, metrics={}", this, status, formatMetrics()); + } + return status; + } + + @Override + public StateTransitor stateTransitor() { + return new Resilience4jStateTransitor(this.circuitBreaker); + } + + private String formatMetrics() { + if (Boolean.getBoolean("mg.solverRemote.circuitBreaker.canaryRateThreshold.detailedMetrics")) { + return JsonUtils.prettyWriteValueAsString(this.circuitBreaker.getMetrics()); + } + return ""; } @Override public void transitionToHalfOpenState() { this.circuitBreaker.transitionToHalfOpenState(); } + + @Override + public String toString() { + return StringUtils.firstNonBlank(this.name, this.getClass().getSimpleName()); + } } diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateSelfObservable.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateSelfObservable.java index e95707484..93feae553 100644 --- a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateSelfObservable.java +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateSelfObservable.java @@ -6,6 +6,7 @@ import com.mageddo.dnsproxyserver.solver.remote.Result; import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegate; import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.HealthChecker; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor; import lombok.extern.slf4j.Slf4j; import java.time.Duration; @@ -45,8 +46,8 @@ public CircuitStatus findStatus() { } @Override - public void transitionToHalfOpenState() { - this.delegate.transitionToHalfOpenState(); + public StateTransitor stateTransitor() { + return this.delegate.stateTransitor(); } private void startOpenCircuitHealthCheckWorker() { @@ -66,15 +67,16 @@ private boolean shouldRun() { private void healthCheckWhenInOpenState() { final var status = this.findStatus(); - log.trace("status=checking, status={}", status); - if (!CircuitStatus.isOpen(status)) { - log.trace("status=notOpenStatus, status={}", status); + final var notInOpenStatus = !CircuitStatus.isOpen(status); + log.trace("status=checking, statusBefore={}, notInOpenStatus={}, circuit={}", status, notInOpenStatus, this); + if (notInOpenStatus) { return; } - final var success = this.isHealthy(); - if (success) { + final var healthy = this.isHealthy(); + log.trace("healthy={}, circuit={}", healthy, this); + if (healthy) { this.transitionToHalfOpenState(); - log.debug("status=halfOpenStatus, circuitBreaker={}", this); + log.debug("status=halfOpenStatus, circuit={}", this); } } @@ -87,4 +89,9 @@ public void close() throws Exception { this.open = false; } + @Override + public String toString() { + return this.delegate.toString(); + } + } diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerFactory.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerFactory.java index 3e3b5671d..71e20685c 100644 --- a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerFactory.java +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerFactory.java @@ -1,9 +1,17 @@ package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.canaryratethreshold; import com.mageddo.dnsproxyserver.config.CanaryRateThresholdCircuitBreakerStrategyConfig; +import com.mageddo.dnsproxyserver.config.CircuitBreakerStrategyConfig; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegate; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.DnsServerHealthChecker; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.HealthChecker; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.HealthCheckerStatic; +import com.mageddo.net.IpAddr; import io.github.resilience4j.circuitbreaker.CircuitBreaker; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ClassUtils; +import org.apache.commons.lang3.Validate; import javax.inject.Inject; import javax.inject.Singleton; @@ -13,17 +21,31 @@ @RequiredArgsConstructor(onConstructor = @__({@Inject})) public class CircuitBreakerFactory { - public CircuitBreakerDelegateSelfObservable build(CanaryRateThresholdCircuitBreakerStrategyConfig config){ - final var circuitBreakerDelegate = new CircuitBreakerDelegateCanaryRateThreshold( - this.createResilienceCircuitBreakerFrom(config) + CircuitBreakerDelegateSelfObservable buildWithoutHealthCheck(CanaryRateThresholdCircuitBreakerStrategyConfig config) { + return build(config, new HealthCheckerStatic(true)); + } + + public CircuitBreakerDelegateSelfObservable build( + CanaryRateThresholdCircuitBreakerStrategyConfig config, HealthChecker healthChecker + ) { + final var canaryRateThresholdCircuitBreaker = new CircuitBreakerDelegateCanaryRateThreshold( + this.createResilienceCircuitBreakerFrom(config), healthChecker.toString() ); - final var healthChecker = new CircuitExecutionsAsHealthChecker(circuitBreakerDelegate); - return new CircuitBreakerDelegateSelfObservable( - healthChecker, healthChecker + return new CircuitBreakerDelegateSelfObservable(canaryRateThresholdCircuitBreaker, healthChecker); + } + + public CircuitBreakerDelegate build(CircuitBreakerStrategyConfig config, IpAddr addr) { + Validate.isTrue( + config.name() == CircuitBreakerStrategyConfig.Name.CANARY_RATE_THRESHOLD, + "Not the expected config: " + ClassUtils.getSimpleName(config) ); + return this.build((CanaryRateThresholdCircuitBreakerStrategyConfig) config, new DnsServerHealthChecker(addr)); } private CircuitBreaker createResilienceCircuitBreakerFrom(CanaryRateThresholdCircuitBreakerStrategyConfig config) { - return Resilience4jMapper.from(config); + final var circuitBreaker = Resilience4jMapper.from(config); + circuitBreaker.transitionToOpenState(); + return circuitBreaker; } + } diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitExecutionsAsHealthChecker.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitExecutionsAsHealthChecker.java deleted file mode 100644 index b0aff80db..000000000 --- a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitExecutionsAsHealthChecker.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.canaryratethreshold; - -import com.mageddo.commons.circuitbreaker.CircuitCheckException; -import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus; -import com.mageddo.dnsproxyserver.solver.remote.Result; -import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegate; -import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.HealthChecker; -import lombok.extern.slf4j.Slf4j; - -import java.util.function.Supplier; - -@Slf4j -public class CircuitExecutionsAsHealthChecker implements HealthChecker, CircuitBreakerDelegate { - - private final CircuitBreakerDelegate delegate; - private final boolean healthWhenNoCallTemplateToDo; - private Supplier lastCall = null; - - public CircuitExecutionsAsHealthChecker(CircuitBreakerDelegate delegate) { - this.delegate = delegate; - this.healthWhenNoCallTemplateToDo = false; - } - - @Override - public boolean isHealthy() { - try { - if (this.lastCall == null) { - log.trace("status=noLastCall, answer={}", this.healthWhenNoCallTemplateToDo); - return this.healthWhenNoCallTemplateToDo; - } - final var res = this.lastCall.get(); - log.trace("status=delegateToLastCall, answer={}", res); - return true; - } catch (CircuitCheckException e) { - log.trace("status=callFailed, answer=false, msg={}", e.getMessage()); - return false; - } - } - - @Override - public Result execute(Supplier sup) { - this.lastCall = sup; - return this.delegate.execute(sup); - } - - @Override - public CircuitStatus findStatus() { - return this.delegate.findStatus(); - } - - @Override - public void transitionToHalfOpenState() { - this.delegate.transitionToHalfOpenState(); - } -} diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/entrypoint/CircuitBreakerHeater.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/entrypoint/CircuitBreakerHeater.java new file mode 100644 index 000000000..17e2c3117 --- /dev/null +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/entrypoint/CircuitBreakerHeater.java @@ -0,0 +1,30 @@ +package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.entrypoint; + +import com.mageddo.di.Eager; +import com.mageddo.dnsproxyserver.config.application.ConfigService; +import com.mageddo.dnsproxyserver.solver.remote.application.failsafe.CircuitBreakerFactory; +import com.mageddo.net.IpAddr; +import lombok.AllArgsConstructor; + +import javax.ejb.Singleton; +import javax.inject.Inject; +import java.util.List; + +@Singleton +@AllArgsConstructor(onConstructor = @__({@Inject})) +public class CircuitBreakerHeater implements Eager { + + private final CircuitBreakerFactory circuitBreakerFactory; + private final ConfigService configService; + + @Override + public void run() { + this.findRemoteServers() + .forEach(this.circuitBreakerFactory::findCircuitBreaker) + ; + } + + private List findRemoteServers() { + return this.configService.findCurrentConfig().getRemoteDnsServers(); + } +} diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/FailSafeStateTransitor.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/FailSafeStateTransitor.java new file mode 100644 index 000000000..930816d0d --- /dev/null +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/FailSafeStateTransitor.java @@ -0,0 +1,22 @@ +package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor; + +import dev.failsafe.CircuitBreaker; + +public class FailSafeStateTransitor implements StateTransitor { + + private final CircuitBreaker circuitBreaker; + + public FailSafeStateTransitor(CircuitBreaker circuitBreaker) { + this.circuitBreaker = circuitBreaker; + } + + @Override + public void closed() { + this.circuitBreaker.close(); + } + + @Override + public void halfOpen() { + this.circuitBreaker.halfOpen(); + } +} diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/NopStateTransitor.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/NopStateTransitor.java new file mode 100644 index 000000000..f04cc1c96 --- /dev/null +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/NopStateTransitor.java @@ -0,0 +1,13 @@ +package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor; + +public class NopStateTransitor implements StateTransitor { + @Override + public void closed() { + + } + + @Override + public void halfOpen() { + + } +} diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/Resilience4jStateTransitor.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/Resilience4jStateTransitor.java new file mode 100644 index 000000000..17a8b89fc --- /dev/null +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/Resilience4jStateTransitor.java @@ -0,0 +1,22 @@ +package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor; + +import io.github.resilience4j.circuitbreaker.CircuitBreaker; + +public class Resilience4jStateTransitor implements StateTransitor { + + private final CircuitBreaker circuitBreaker; + + public Resilience4jStateTransitor(CircuitBreaker circuitBreaker) { + this.circuitBreaker = circuitBreaker; + } + + @Override + public void closed() { + this.circuitBreaker.transitionToClosedState(); + } + + @Override + public void halfOpen() { + this.circuitBreaker.transitionToHalfOpenState(); + } +} diff --git a/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/StateTransitor.java b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/StateTransitor.java new file mode 100644 index 000000000..b39d183cd --- /dev/null +++ b/src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/statetransitor/StateTransitor.java @@ -0,0 +1,6 @@ +package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor; + +public interface StateTransitor { + void closed(); + void halfOpen(); +} diff --git a/src/test/java/com/mageddo/dnsproxyserver/solver/remote/application/failsafe/CircuitBreakerFactoryTest.java b/src/test/java/com/mageddo/dnsproxyserver/solver/remote/application/failsafe/CircuitBreakerFactoryTest.java index 504c4000d..07def9ad3 100644 --- a/src/test/java/com/mageddo/dnsproxyserver/solver/remote/application/failsafe/CircuitBreakerFactoryTest.java +++ b/src/test/java/com/mageddo/dnsproxyserver/solver/remote/application/failsafe/CircuitBreakerFactoryTest.java @@ -2,6 +2,7 @@ import com.mageddo.dnsproxyserver.solver.remote.application.FailsafeCircuitBreakerFactory; import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegateNonResilient; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.canaryratethreshold.CircuitBreakerDelegateSelfObservable; import dev.failsafe.CircuitBreaker; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -11,6 +12,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import testing.templates.CircuitBreakerConfigTemplates; import testing.templates.InetSocketAddressTemplates; +import testing.templates.solver.remote.CircuitBreakerDelegateTemplates; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -18,12 +20,14 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; @ExtendWith(MockitoExtension.class) class CircuitBreakerFactoryTest { - @InjectMocks @Spy + @InjectMocks CircuitBreakerFactory factory; @Mock @@ -71,6 +75,7 @@ void mustReuseCircuitBreakerInstanceWhenSameKeyIsUsed(){ // assert assertEquals(a, b); assertEquals(a.hashCode(), b.hashCode()); + verify(this.factory, times(1)).findCircuitBreakerHotLoad(any()); } @Test @@ -141,4 +146,23 @@ void mustReturnNullWhenNoStatusIsFound(){ assertNull(status); } + @Test + void mustBuildCanaryRateThresholdCircuitBreaker(){ + // arrange + final var addr = InetSocketAddressTemplates._8_8_8_8(); + doReturn(CircuitBreakerConfigTemplates.fastCanaryRateThreshold()) + .when(this.factory) + .findCircuitBreakerConfig(); + + doReturn(CircuitBreakerDelegateTemplates.buildCanaryRateThreshold()) + .when(this.factory) + .buildCanaryRateThreshold(any(), any()); + + // act + final var circuitBreaker = this.factory.findCircuitBreaker(addr); + + // assert + assertEquals(CircuitBreakerDelegateSelfObservable.class, circuitBreaker.getClass()); + } + } diff --git a/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitExecutionsAsHealthCheckerTest.java b/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitExecutionsAsHealthCheckerTest.java deleted file mode 100644 index 736a1a0fe..000000000 --- a/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitExecutionsAsHealthCheckerTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application; - -import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.canaryratethreshold.CircuitExecutionsAsHealthChecker; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import testing.templates.solver.remote.ResultSupplierTemplates; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; - -@ExtendWith(MockitoExtension.class) -class CircuitExecutionsAsHealthCheckerTest { - - @Mock - CircuitBreakerDelegate circuitBreaker; - - @InjectMocks - CircuitExecutionsAsHealthChecker obj; - - @Test - void mustUseTheLastCallAsHealthCheck() { - - final var sup = ResultSupplierTemplates.withCallsCounterNullRes(); - - this.obj.execute(sup); - this.obj.isHealthy(); - - assertEquals(1, sup.getCalls()); - } - - @Test - void mustAnswerHealthWhenExecuteNeverCalled(){ - final var healthy = this.obj.isHealthy(); - assertFalse(healthy); - } -} diff --git a/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateCanaryRateThresholdTest.java b/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateCanaryRateThresholdTest.java index 4cca0a01b..e266d7c16 100644 --- a/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateCanaryRateThresholdTest.java +++ b/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateCanaryRateThresholdTest.java @@ -27,7 +27,7 @@ class CircuitBreakerDelegateCanaryRateThresholdTest { @BeforeEach void beforeEach() { - this.delegate = new CircuitBreakerDelegateCanaryRateThreshold(this.circuitBreaker); + this.delegate = new CircuitBreakerDelegateCanaryRateThreshold(this.circuitBreaker, "Test"); } @Test diff --git a/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateSelfObservableTest.java b/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateSelfObservableTest.java index 0c6687d2a..1f10327a8 100644 --- a/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateSelfObservableTest.java +++ b/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateSelfObservableTest.java @@ -3,6 +3,7 @@ import com.mageddo.commons.concurrent.Threads; import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus; import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.HealthChecker; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -14,6 +15,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; @ExtendWith(MockitoExtension.class) @@ -29,11 +32,11 @@ class CircuitBreakerDelegateSelfObservableTest { @BeforeEach void beforeEach() { - this.strategy = new CircuitBreakerDelegateSelfObservable( + this.strategy = spy(new CircuitBreakerDelegateSelfObservable( this.delegate, Duration.ofMillis(1000 / 30), this.healthChecker - ); + )); } @Test @@ -60,11 +63,16 @@ void mustHalfOpenCircuitAfterConfiguredTimeAndSatisfyHealthCheck() { .isHealthy() ; + final var stateTransitor = mock(StateTransitor.class); + doReturn(stateTransitor) + .when(this.delegate) + .stateTransitor(); + // act Threads.sleep(300); // assert - verify(this.delegate, atLeastOnce()).transitionToHalfOpenState(); + verify(stateTransitor, atLeastOnce()).halfOpen(); } diff --git a/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerFactoryCompTest.java b/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerFactoryCompTest.java index bf6e880c5..a9fdffce7 100644 --- a/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerFactoryCompTest.java +++ b/src/test/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerFactoryCompTest.java @@ -1,8 +1,8 @@ package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.canaryratethreshold; -import com.mageddo.commons.circuitbreaker.CircuitCheckException; import com.mageddo.commons.circuitbreaker.CircuitIsOpenException; import com.mageddo.commons.concurrent.Threads; +import com.mageddo.dnsproxyserver.config.CanaryRateThresholdCircuitBreakerStrategyConfig; import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus; import dagger.sheath.junit.DaggerTest; import org.junit.jupiter.api.Test; @@ -12,7 +12,6 @@ import testing.templates.solver.remote.ResultSupplierTemplates; import javax.inject.Inject; - import java.time.Duration; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -31,7 +30,7 @@ void mustExecuteHappyPath() { final var sup = ResultSupplierTemplates.withCallsCounterNullRes(); final var config = CircuitBreakerConfigTemplates.fastCanaryRateThreshold(); - final var circuitBreaker = this.factory.build(config); + final var circuitBreaker = buildTransitToClosedState(config); final var result = circuitBreaker.execute(sup); assertEquals(1, sup.getCalls()); @@ -39,24 +38,41 @@ void mustExecuteHappyPath() { } + private CircuitBreakerDelegateSelfObservable buildTransitToClosedState(CanaryRateThresholdCircuitBreakerStrategyConfig config) { + final var circuitBreaker = this.factory.buildWithoutHealthCheck(config); + circuitBreaker.transitionToClosedState(); + return circuitBreaker; + + } + @Test void mustTestCircuitWhenItIsOpen() { final var sup = ResultSupplierTemplates.withCallsCounterNullRes(); final var config = CircuitBreakerConfigTemplates.fastCanaryRateThreshold(); - final var circuitBreaker = this.factory.build(config); - assertEquals(CircuitStatus.CLOSED, circuitBreaker.findStatus()); + final var circuitBreaker = this.factory.buildWithoutHealthCheck(config); - assertThrows(CircuitCheckException.class, () -> circuitBreaker.execute(ResultSupplierTemplates.alwaysFail())); assertEquals(CircuitStatus.OPEN, circuitBreaker.findStatus()); - assertThrows(CircuitIsOpenException.class, () -> circuitBreaker.execute(sup)); Threads.sleep(Duration.ofMillis(1200)); - assertEquals(CircuitStatus.HALF_OPEN, circuitBreaker.findStatus()); + + assertNull(circuitBreaker.execute(sup)); assertEquals(1, sup.getCalls()); } + + + @Test + void circuitMustBeCreatedOpen() { + + final var config = CircuitBreakerConfigTemplates.fastCanaryRateThreshold(); + + final var circuitBreaker = this.factory.buildWithoutHealthCheck(config); + + assertEquals(CircuitStatus.OPEN, circuitBreaker.findStatus()); + + } } diff --git a/src/test/java/testing/templates/CircuitBreakerConfigTemplates.java b/src/test/java/testing/templates/CircuitBreakerConfigTemplates.java index 4588227f8..30a7c9ad9 100644 --- a/src/test/java/testing/templates/CircuitBreakerConfigTemplates.java +++ b/src/test/java/testing/templates/CircuitBreakerConfigTemplates.java @@ -35,4 +35,5 @@ public static CanaryRateThresholdCircuitBreakerStrategyConfig fastCanaryRateThre .failureRateThreshold(1) .build(); } + } diff --git a/src/test/java/testing/templates/solver/remote/CircuitBreakerDelegateTemplates.java b/src/test/java/testing/templates/solver/remote/CircuitBreakerDelegateTemplates.java new file mode 100644 index 000000000..d57157898 --- /dev/null +++ b/src/test/java/testing/templates/solver/remote/CircuitBreakerDelegateTemplates.java @@ -0,0 +1,14 @@ +package testing.templates.solver.remote; + +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegate; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegateNonResilient; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.HealthCheckerStatic; +import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.canaryratethreshold.CircuitBreakerDelegateSelfObservable; + +public class CircuitBreakerDelegateTemplates { + public static CircuitBreakerDelegate buildCanaryRateThreshold() { + return new CircuitBreakerDelegateSelfObservable( + new CircuitBreakerDelegateNonResilient(), new HealthCheckerStatic(true) + ); + } +}