Canary Rate Threshold Circuit Breaker Strategy: All circuits open at …

…start (#588)

* implementing circuit breaker factory

* fixmes

* testing

* testing

* setup strategy mapping

* fixing validation

* must call hotload method once

* adjusting logs

* add server address as circuit breaker name

* fixing compiling errors

* refactoring

* refactoring

* fixing test

* fixing test

* fixing test

* fixing test

* fixing test

* fixing test

* fixing test

* messages formatting and refactoring

* refactoring

* disabling metrics

* heating circuit breakers at app startup

* todos

* adjusts

* adjusts

* fixing tests

* fixing tests

* fixing tests

* testing

* item tick

* clearing todos

* [Gradle Release Plugin] - new version commit:  '3.31.1-snapshot'.

* release notes

RELEASE-NOTES.md

@@ -1,3 +1,6 @@
+## 3.31.1
+* Adjusting Canary Rate Threshold Circuit Breaker to mark all circuits as open at the start. #533
+
 ## 3.31.0
 * Canary Rate Threshold Circuit Breaker. #533
 

gradle.properties

@@ -1 +1 @@
-version=3.31.0-snapshot
+version=3.31.1-snapshot

src/main/java/com/mageddo/dns/utils/Messages.java

@@ -38,24 +38,24 @@ public static String simplePrint(Response res) {
     return simplePrint(res.getMessage());
   }
 
-  public static String simplePrint(Message message) {
-    if (message == null) {
+  public static String simplePrint(Message reqOrRes) {
+    if (reqOrRes == null) {
       return null;
     }
     try {
-      final var answer = findFirstAnswerRecord(message);
-      final var rcode = message.getRcode();
+      final var answer = findFirstAnswerRecord(reqOrRes);
+      final var rcode = reqOrRes.getRcode();
       if (answer != null) {
         return String.format("rc=%d, res=%s", rcode, simplePrint(answer));
       }
-      final var question = message.getQuestion();
+      final var question = reqOrRes.getQuestion();
       final var type = Objects.useItOrDefault(
         Objects.toString(Entry.Type.of(question.getType())),
         () -> String.valueOf(question.getType())
       );
       final var hostname = question.getName().toString(true);
       final var sb = new StringBuilder();
-      if (Messages.hasFlag(message, Flags.QR)) {
+      if (Messages.hasFlag(reqOrRes, Flags.QR)) {
         sb.append("rc=")
           .append(rcode)
           .append(", ")
@@ -64,8 +64,8 @@ public static String simplePrint(Message message) {
       sb.append(String.format("query=%s:%s", type, hostname));
       return sb.toString();
     } catch (Throwable e) {
-      log.warn("status=failedToSimplePrint, msg={}", message, e);
-      return String.valueOf(message);
+      log.warn("status=failedToSimplePrint, msg={}", reqOrRes, e);
+      return String.valueOf(reqOrRes);
     }
   }
 

src/main/java/com/mageddo/dnsproxyserver/di/module/ModuleEager.java

@@ -1,6 +1,7 @@
 package com.mageddo.dnsproxyserver.di.module;
 
 import com.mageddo.di.Eager;
+import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.entrypoint.CircuitBreakerHeater;
 import dagger.Module;
 import dagger.Provides;
 import dagger.multibindings.ElementsIntoSet;
@@ -13,7 +14,9 @@ public class ModuleEager {
   @Provides
   @Singleton
   @ElementsIntoSet
-  Set<Eager> beans() {
-    return Set.of();
+  Set<Eager> beans(CircuitBreakerHeater a) {
+    return Set.of(
+      a
+    );
   }
 }

src/main/java/com/mageddo/dnsproxyserver/solver/SimpleResolver.java

@@ -1,5 +1,8 @@
 package com.mageddo.dnsproxyserver.solver;
 
+import com.mageddo.net.IpAddr;
+import com.mageddo.net.IpAddrs;
+
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
@@ -20,4 +23,8 @@ public SimpleResolver(InetSocketAddress addr) {
   public SimpleResolver(InetAddress host) {
     super(host);
   }
+
+  public SimpleResolver(IpAddr addr) {
+    super(IpAddrs.toInetSocketAddress(addr));
+  }
 }

src/main/java/com/mageddo/dnsproxyserver/solver/remote/application/failsafe/CircuitBreakerFactory.java

@@ -1,7 +1,6 @@
 package com.mageddo.dnsproxyserver.solver.remote.application.failsafe;
 
 import com.mageddo.commons.lang.tuple.Pair;
-import com.mageddo.dnsproxyserver.config.CanaryRateThresholdCircuitBreakerStrategyConfig;
 import com.mageddo.dnsproxyserver.config.CircuitBreakerStrategyConfig;
 import com.mageddo.dnsproxyserver.config.StaticThresholdCircuitBreakerStrategyConfig;
 import com.mageddo.dnsproxyserver.config.application.ConfigService;
@@ -14,6 +13,7 @@
 import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegateStaticThresholdFailsafe;
 import com.mageddo.dnsproxyserver.solver.remote.mapper.ResolverMapper;
 import com.mageddo.net.IpAddr;
+import com.mageddo.net.IpAddrs;
 import lombok.RequiredArgsConstructor;
 import lombok.Value;
 import lombok.extern.slf4j.Slf4j;
@@ -25,6 +25,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
 
 @Slf4j
 @Singleton
@@ -67,8 +68,7 @@ CircuitBreakerDelegate findCircuitBreakerHotLoad(InetSocketAddress address) {
   }
 
   CircuitBreakerDelegate buildCanaryRateThreshold(CircuitBreakerStrategyConfig config, InetSocketAddress address) {
-//    return this.canaryThresholdFactory.build(config, IpAddrs.from(address));
-    return this.canaryThresholdFactory.build((CanaryRateThresholdCircuitBreakerStrategyConfig) config);
+    return this.canaryThresholdFactory.build(config, IpAddrs.from(address));
   }
 
   private CircuitBreakerDelegateStaticThresholdFailsafe buildStaticThresholdFailSafeCircuitBreaker(

src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegate.java

@@ -2,6 +2,7 @@
 
 import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus;
 import com.mageddo.dnsproxyserver.solver.remote.Result;
+import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor;
 
 import java.util.function.Supplier;
 
@@ -11,5 +12,13 @@ public interface CircuitBreakerDelegate {
 
   CircuitStatus findStatus();
 
-  void transitionToHalfOpenState();
+  StateTransitor stateTransitor();
+
+  default void transitionToHalfOpenState(){
+    this.stateTransitor().halfOpen();
+  }
+
+  default void transitionToClosedState(){
+    this.stateTransitor().closed();
+  }
 }

src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegateNonResilient.java

@@ -2,6 +2,8 @@
 
 import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus;
 import com.mageddo.dnsproxyserver.solver.remote.Result;
+import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.NopStateTransitor;
+import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor;
 
 import java.util.function.Supplier;
 
@@ -18,7 +20,8 @@ public CircuitStatus findStatus() {
   }
 
   @Override
-  public void transitionToHalfOpenState() {
-
+  public StateTransitor stateTransitor() {
+    return new NopStateTransitor();
   }
+
 }

src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/CircuitBreakerDelegateStaticThresholdFailsafe.java

@@ -1,9 +1,11 @@
 package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application;
 
 import com.mageddo.circuitbreaker.failsafe.CircuitStatusRefresh;
+import com.mageddo.commons.circuitbreaker.CircuitIsOpenException;
 import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus;
 import com.mageddo.dnsproxyserver.solver.remote.Result;
-import com.mageddo.commons.circuitbreaker.CircuitIsOpenException;
+import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.FailSafeStateTransitor;
+import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor;
 import com.mageddo.dnsproxyserver.solver.remote.mapper.CircuitBreakerStateMapper;
 import dev.failsafe.CircuitBreaker;
 import dev.failsafe.CircuitBreakerOpenException;
@@ -25,7 +27,7 @@ public Result execute(Supplier<Result> sup) {
       return Failsafe
         .with(this.circuitBreaker)
         .get((ctx) -> sup.get());
-    } catch (CircuitBreakerOpenException e){
+    } catch (CircuitBreakerOpenException e) {
       throw new CircuitIsOpenException(e);
     }
   }
@@ -37,7 +39,8 @@ public CircuitStatus findStatus() {
   }
 
   @Override
-  public void transitionToHalfOpenState() {
-    this.circuitBreaker.halfOpen();
+  public StateTransitor stateTransitor() {
+    return new FailSafeStateTransitor(circuitBreaker);
   }
+
 }

src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/DnsServerHealthChecker.java

@@ -0,0 +1,40 @@
+package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application;
+
+import com.mageddo.dns.utils.Messages;
+import com.mageddo.dnsproxyserver.solver.SimpleResolver;
+import com.mageddo.net.IpAddr;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.ClassUtils;
+
+import java.io.IOException;
+
+@Slf4j
+public class DnsServerHealthChecker implements HealthChecker {
+
+  private final SimpleResolver resolver;
+
+  public DnsServerHealthChecker(IpAddr addr) {
+    this.resolver = new SimpleResolver(addr);
+  }
+
+  @Override
+  public boolean isHealthy() {
+    final var req = Messages.aQuestion("dps.dns.test");
+    try {
+      final var res = this.resolver.send(req);
+      log.debug("status=done, server={}, res={}", this.resolver.getAddress(), Messages.simplePrint(res));
+      return Messages.findQuestionTypeCode(res) != null;
+    } catch (IOException e) {
+      log.debug(
+        "status=failed, server={}, res={}, clazz={}",
+        this.resolver.getAddress(), e.getMessage(), ClassUtils.getSimpleName(e)
+      );
+      return false;
+    }
+  }
+
+  @Override
+  public String toString() {
+    return String.valueOf(this.resolver.getAddress());
+  }
+}

src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/application/HealthCheckerStatic.java

@@ -0,0 +1,15 @@
+package com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application;
+
+public class HealthCheckerStatic implements HealthChecker {
+
+  private final boolean healthy;
+
+  public HealthCheckerStatic(boolean healthy) {
+    this.healthy = healthy;
+  }
+
+  @Override
+  public boolean isHealthy() {
+    return this.healthy;
+  }
+}

src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateCanaryRateThreshold.java

@@ -4,20 +4,26 @@
 import com.mageddo.dnsproxyserver.solver.remote.CircuitStatus;
 import com.mageddo.dnsproxyserver.solver.remote.Result;
 import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegate;
+import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.Resilience4jStateTransitor;
+import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor;
 import com.mageddo.dnsproxyserver.solver.remote.mapper.Resilience4jStatusMapper;
+import com.mageddo.json.JsonUtils;
 import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
 import io.github.resilience4j.circuitbreaker.CircuitBreaker;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 
 import java.util.function.Supplier;
 
 @Slf4j
 public class CircuitBreakerDelegateCanaryRateThreshold implements CircuitBreakerDelegate {
 
   private final CircuitBreaker circuitBreaker;
+  private final String name;
 
-  public CircuitBreakerDelegateCanaryRateThreshold(CircuitBreaker circuitBreaker) {
+  public CircuitBreakerDelegateCanaryRateThreshold(CircuitBreaker circuitBreaker, String name) {
     this.circuitBreaker = circuitBreaker;
+    this.name = name;
   }
 
   @Override
@@ -31,11 +37,32 @@ public Result execute(Supplier<Result> sup) {
 
   @Override
   public CircuitStatus findStatus() {
-    return Resilience4jStatusMapper.toCircuitStatus(this.circuitBreaker.getState());
+    final var status = Resilience4jStatusMapper.toCircuitStatus(this.circuitBreaker.getState());
+    if (log.isTraceEnabled()) {
+      log.trace("circuit={}, status={}, metrics={}", this, status, formatMetrics());
+    }
+    return status;
+  }
+
+  @Override
+  public StateTransitor stateTransitor() {
+    return new Resilience4jStateTransitor(this.circuitBreaker);
+  }
+
+  private String formatMetrics() {
+    if (Boolean.getBoolean("mg.solverRemote.circuitBreaker.canaryRateThreshold.detailedMetrics")) {
+      return JsonUtils.prettyWriteValueAsString(this.circuitBreaker.getMetrics());
+    }
+    return "";
   }
 
   @Override
   public void transitionToHalfOpenState() {
     this.circuitBreaker.transitionToHalfOpenState();
   }
+
+  @Override
+  public String toString() {
+    return StringUtils.firstNonBlank(this.name, this.getClass().getSimpleName());
+  }
 }

src/main/java/com/mageddo/dnsproxyserver/solver/remote/circuitbreaker/canaryratethreshold/CircuitBreakerDelegateSelfObservable.java

@@ -6,6 +6,7 @@
 import com.mageddo.dnsproxyserver.solver.remote.Result;
 import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.CircuitBreakerDelegate;
 import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.application.HealthChecker;
+import com.mageddo.dnsproxyserver.solver.remote.circuitbreaker.statetransitor.StateTransitor;
 import lombok.extern.slf4j.Slf4j;
 
 import java.time.Duration;
@@ -45,8 +46,8 @@ public CircuitStatus findStatus() {
   }
 
   @Override
-  public void transitionToHalfOpenState() {
-    this.delegate.transitionToHalfOpenState();
+  public StateTransitor stateTransitor() {
+    return this.delegate.stateTransitor();
   }
 
   private void startOpenCircuitHealthCheckWorker() {
@@ -66,15 +67,16 @@ private boolean shouldRun() {
 
   private void healthCheckWhenInOpenState() {
     final var status = this.findStatus();
-    log.trace("status=checking, status={}", status);
-    if (!CircuitStatus.isOpen(status)) {
-      log.trace("status=notOpenStatus, status={}", status);
+    final var notInOpenStatus = !CircuitStatus.isOpen(status);
+    log.trace("status=checking, statusBefore={}, notInOpenStatus={}, circuit={}", status, notInOpenStatus, this);
+    if (notInOpenStatus) {
       return;
     }
-    final var success = this.isHealthy();
-    if (success) {
+    final var healthy = this.isHealthy();
+    log.trace("healthy={}, circuit={}", healthy, this);
+    if (healthy) {
       this.transitionToHalfOpenState();
-      log.debug("status=halfOpenStatus, circuitBreaker={}", this);
+      log.debug("status=halfOpenStatus, circuit={}", this);
     }
   }
 
@@ -87,4 +89,9 @@ public void close() throws Exception {
     this.open = false;
   }
 
+  @Override
+  public String toString() {
+    return this.delegate.toString();
+  }
+
 }