diff --git a/circuitbreaker/circuitbreaker.go b/circuitbreaker/circuitbreaker.go index 12f945a..d43d054 100644 --- a/circuitbreaker/circuitbreaker.go +++ b/circuitbreaker/circuitbreaker.go @@ -10,14 +10,30 @@ import ( "github.com/slok/goresilience/metrics" ) -type state string +type state uint const ( - stateOpen state = "open" - stateHalfOpen state = "halfopen" - stateClosed state = "closed" + stateNew state = iota + stateOpen + stateHalfOpen + stateClosed ) +var stateStrings = []string { + "new", + "open", + "halfopen", + "closed", +} + +func (st state) label () string { + return stateStrings[st] +} + +func (st state) condition () int { + return int(st) +} + // Config is the configuration of the circuit breaker. type Config struct { // ErrorPercentThresholdToOpen is the error percent based on total execution requests @@ -124,7 +140,7 @@ func NewMiddleware(cfg Config) goresilience.Middleware { return func(next goresilience.Runner) goresilience.Runner { return &circuitbreaker{ - state: stateClosed, + state: stateNew, recorder: newBucketWindow(cfg.MetricsSlidingWindowBucketQuantity, cfg.MetricsBucketDuration), stateStarted: time.Now(), cfg: cfg, @@ -157,6 +173,9 @@ func (c *circuitbreaker) Run(ctx context.Context, f goresilience.Func) error { func (c *circuitbreaker) preDecideState(metricsRec metrics.Recorder) { state := c.getState() switch state { + case stateNew: + // Close the breaker as this is the first time through and generate a statistic. + c.moveState(stateClosed, metricsRec) case stateOpen: // Check if the circuit has been the required time in closed. If yes then // we move to half open state. @@ -223,7 +242,8 @@ func (c *circuitbreaker) moveState(state state, metricsRec metrics.Recorder) { // Only change if the state changed. if c.state != state { - metricsRec.IncCircuitbreakerState(string(state)) + metricsRec.IncCircuitbreakerState(state.label()) + metricsRec.SetCircuitbreakerCurrentState(state.condition()) c.state = state c.stateStarted = time.Now() diff --git a/metrics/dummy.go b/metrics/dummy.go index bbd3811..263cf5c 100644 --- a/metrics/dummy.go +++ b/metrics/dummy.go @@ -15,6 +15,7 @@ func (dummy) IncBulkheadQueued() {} func (dummy) IncBulkheadProcessed() {} func (dummy) IncBulkheadTimeout() {} func (dummy) IncCircuitbreakerState(state string) {} +func (dummy) SetCircuitbreakerCurrentState(condition int) {} func (dummy) IncChaosInjectedFailure(kind string) {} func (dummy) SetConcurrencyLimitInflightExecutions(q int) {} func (dummy) SetConcurrencyLimitExecutingExecutions(q int) {} diff --git a/metrics/metrics.go b/metrics/metrics.go index c3baddc..7015acb 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -22,6 +22,8 @@ type Recorder interface { IncBulkheadTimeout() // IncCircuitbreakerState increments the number of state change. IncCircuitbreakerState(state string) + // SetCircuitbreakerCurrentState records the state of ciruit breaker. + SetCircuitbreakerCurrentState(condition int) // IncChaosInjectedFailure increments the number of times injected failure. IncChaosInjectedFailure(kind string) // SetConcurrencyLimitInflightExecutions sets the number of queued and executions at a given moment. diff --git a/metrics/prometheus.go b/metrics/prometheus.go index b98c851..947f1a4 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -7,7 +7,7 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -var ( +const ( promNamespace = "goresilience" promCommandSubsystem = "command" @@ -17,8 +17,11 @@ var ( promCBSubsystem = "circuitbreaker" promChaosSubsystem = "chaos" promConcurrencyLimitSubsystem = "concurrencylimit" + ) +var DefaultPrometheusRecorder = NewPrometheusRecorder(prometheus.DefaultRegisterer) + type prometheusRec struct { // Metrics. cmdExecutionDuration *prometheus.HistogramVec @@ -28,6 +31,7 @@ type prometheusRec struct { bulkProcessed *prometheus.CounterVec bulkTimeouts *prometheus.CounterVec cbStateChanges *prometheus.CounterVec + cbCurrentState *prometheus.GaugeVec chaosFailureInjections *prometheus.CounterVec concurrencyLimitInflights *prometheus.GaugeVec concurrencyLimitExecuting *prometheus.GaugeVec @@ -59,6 +63,7 @@ func (p prometheusRec) WithID(id string) Recorder { bulkProcessed: p.bulkProcessed, bulkTimeouts: p.bulkTimeouts, cbStateChanges: p.cbStateChanges, + cbCurrentState: p.cbCurrentState, chaosFailureInjections: p.chaosFailureInjections, concurrencyLimitInflights: p.concurrencyLimitInflights, concurrencyLimitExecuting: p.concurrencyLimitExecuting, @@ -122,6 +127,13 @@ func (p *prometheusRec) registerMetrics() { Help: "Total number of state changes made by the circuit breaker runner.", }, []string{"id", "state"}) + p.cbCurrentState = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: promNamespace, + Subsystem: promCBSubsystem, + Name: "current_state", + Help: "The current state of the circuit breaker runner.", + }, []string{"id"}) + p.chaosFailureInjections = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: promNamespace, Subsystem: promChaosSubsystem, @@ -172,6 +184,7 @@ func (p *prometheusRec) registerMetrics() { p.bulkProcessed, p.bulkTimeouts, p.cbStateChanges, + p.cbCurrentState, p.chaosFailureInjections, p.concurrencyLimitInflights, p.concurrencyLimitExecuting, @@ -210,6 +223,10 @@ func (p prometheusRec) IncCircuitbreakerState(state string) { p.cbStateChanges.WithLabelValues(p.id, state).Inc() } +func (p prometheusRec) SetCircuitbreakerCurrentState(condition int) { + p.cbCurrentState.WithLabelValues(p.id).Set(float64(condition)) +} + func (p prometheusRec) IncChaosInjectedFailure(kind string) { p.chaosFailureInjections.WithLabelValues(p.id, kind).Inc() } diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go index 82f1c70..022362b 100644 --- a/metrics/prometheus_test.go +++ b/metrics/prometheus_test.go @@ -138,6 +138,21 @@ func TestPrometheus(t *testing.T) { `goresilience_circuitbreaker_state_changes_total{id="test2",state="close"} 1`, }, }, + { + name: "Recording circuitbreaker circuit breaker state should expose the state.", + recordMetrics: func(m metrics.Recorder) { + m1 := m.WithID("test") + m2 := m.WithID("test2") + m1.SetCircuitbreakerCurrentState(0) // new + m1.SetCircuitbreakerCurrentState(3) // open + m1.SetCircuitbreakerCurrentState(1) // close + m2.SetCircuitbreakerCurrentState(2) // half-open + }, + expMetrics: []string{ + `goresilience_circuitbreaker_current_state{id="test"} 1`, + `goresilience_circuitbreaker_current_state{id="test2"} 2`, + }, + }, { name: "Recording chaos metrics should expose the metrics.", recordMetrics: func(m metrics.Recorder) {