Add metrics for budget

internal/controller/applicationdisruptionbudget_controller.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"reflect"
+	"strconv"
 
 	"k8s.io/apimachinery/pkg/api/errors"
 
@@ -40,6 +41,7 @@ import (
 
 	nodedisruptionv1alpha1 "github.com/criteo/node-disruption-controller/api/v1alpha1"
 	"github.com/criteo/node-disruption-controller/pkg/resolver"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // ApplicationDisruptionBudgetReconciler reconciles a ApplicationDisruptionBudget object
@@ -67,16 +69,23 @@ func (r *ApplicationDisruptionBudgetReconciler) Reconcile(ctx context.Context, r
 	logger := log.FromContext(ctx)
 	adb := &nodedisruptionv1alpha1.ApplicationDisruptionBudget{}
 	err := r.Client.Get(ctx, req.NamespacedName, adb)
+	ref := nodedisruptionv1alpha1.NamespacedName{
+		Namespace: req.Namespace,
+		Name:      req.Name,
+		Kind:      "ApplicationDisruptionBudget",
+	}
 
 	if err != nil {
 		if errors.IsNotFound(err) {
 			// If the resource was not found, nothing has to be done
+			PruneADBMetrics(ref)
 			return ctrl.Result{}, nil
 		}
 		return ctrl.Result{}, err
 	}
 
-	logger.Info("Start reconcile of ADB", "version", adb.ResourceVersion)
+	UpdateADBMetrics(ref, adb)
+	logger.Info("Start reconcile of adb", "version", adb.ResourceVersion)
 
 	resolver := ApplicationDisruptionBudgetResolver{
 		ApplicationDisruptionBudget: adb.DeepCopy(),
@@ -97,6 +106,18 @@ func (r *ApplicationDisruptionBudgetReconciler) Reconcile(ctx context.Context, r
 	return ctrl.Result{}, err
 }
 
+// PruneNodeDisruptionMetric remove metrics for an ADB that don't exist anymore
+func PruneADBMetrics(ref nodedisruptionv1alpha1.NamespacedName) {
+	DisruptionBudgetMaxDisruptions.DeletePartialMatch(prometheus.Labels{"budget_disruption_namespace": ref.Namespace, "budget_disruption_name": ref.Name, "budget_disruption_kind": ref.Kind})
+	PruneBudgetStatusMetrics(ref)
+}
+
+// UpdateADBMetrics update metrics for an ADB
+func UpdateADBMetrics(ref nodedisruptionv1alpha1.NamespacedName, adb *nodedisruptionv1alpha1.ApplicationDisruptionBudget) {
+	DisruptionBudgetMaxDisruptions.WithLabelValues(ref.Namespace, ref.Name, ref.Kind).Set(float64(adb.Spec.MaxDisruptions))
+	UpdateBudgetStatusMetrics(ref, adb.Status)
+}
+
 // MapFuncBuilder returns a MapFunc that is used to dispatch reconcile requests to
 // budgets when an event is triggered by one of their matching object
 func (r *ApplicationDisruptionBudgetReconciler) MapFuncBuilder() handler.MapFunc {
@@ -107,7 +128,7 @@ func (r *ApplicationDisruptionBudgetReconciler) MapFuncBuilder() handler.MapFunc
 		if err != nil {
 			// We cannot return an error so at least it should be logged
 			logger := log.FromContext(context.Background())
-			logger.Error(err, "Could not list ADBs in watch function")
+			logger.Error(err, "Could not list adbs in watch function")
 			return requests
 		}
 
@@ -231,6 +252,7 @@ func (r *ApplicationDisruptionBudgetResolver) CallHealthHook(ctx context.Context
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.ApplicationDisruptionBudget.Spec.HealthHook.URL, bytes.NewReader(data))
 	if err != nil {
+		DisruptionBudgetCheckHealthHookErrorTotal.WithLabelValues(nd.Namespace, nd.Name, nd.Kind).Inc()
 		return err
 	}
 
@@ -240,14 +262,18 @@ func (r *ApplicationDisruptionBudgetResolver) CallHealthHook(ctx context.Context
 
 	resp, err := client.Do(req)
 	if err != nil {
+		DisruptionBudgetCheckHealthHookErrorTotal.WithLabelValues(nd.Namespace, nd.Name, nd.Kind).Inc()
 		return err
 	}
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
+		DisruptionBudgetCheckHealthHookErrorTotal.WithLabelValues(nd.Namespace, nd.Name, nd.Kind).Inc()
 		return err
 	}
 
+	DisruptionBudgetCheckHealthHookStatusCodeTotal.WithLabelValues(nd.Namespace, nd.Name, nd.Kind, strconv.Itoa(resp.StatusCode)).Inc()
+
 	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
 		return nil
 	}

internal/controller/budget.go

@@ -5,6 +5,7 @@ import (
 
 	nodedisruptionv1alpha1 "github.com/criteo/node-disruption-controller/api/v1alpha1"
 	"github.com/criteo/node-disruption-controller/pkg/resolver"
+	"github.com/prometheus/client_golang/prometheus"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -25,6 +26,39 @@ type Budget interface {
 	GetNamespacedName() nodedisruptionv1alpha1.NamespacedName
 }
 
+// PruneBudgetMetrics remove metrics for a Disruption Budget that doesn't exist anymore
+func PruneBudgetStatusMetrics(ref nodedisruptionv1alpha1.NamespacedName) {
+	DisruptionBudgetDisruptions.DeletePartialMatch(prometheus.Labels{"budget_disruption_namespace": ref.Namespace, "budget_disruption_name": ref.Name, "budget_disruption_kind": ref.Kind})
+	DisruptionBudgetWatchedNodes.DeletePartialMatch(prometheus.Labels{"budget_disruption_namespace": ref.Namespace, "budget_disruption_name": ref.Name, "budget_disruption_kind": ref.Kind})
+	DisruptionBudgetDisruptionsAllowed.DeletePartialMatch(prometheus.Labels{"budget_disruption_namespace": ref.Namespace, "budget_disruption_name": ref.Name, "budget_disruption_kind": ref.Kind})
+	DisruptionBudgetCurrentDisruptions.DeletePartialMatch(prometheus.Labels{"budget_disruption_namespace": ref.Namespace, "budget_disruption_name": ref.Name, "budget_disruption_kind": ref.Kind})
+
+	DisruptionBudgetRejectedTotal.DeletePartialMatch(prometheus.Labels{"budget_disruption_namespace": ref.Namespace, "budget_disruption_name": ref.Name, "budget_disruption_kind": ref.Kind})
+	DisruptionBudgetGrantedTotal.DeletePartialMatch(prometheus.Labels{"budget_disruption_namespace": ref.Namespace, "budget_disruption_name": ref.Name, "budget_disruption_kind": ref.Kind})
+	DisruptionBudgetCheckHealthHookStatusCodeTotal.DeletePartialMatch(prometheus.Labels{"budget_disruption_namespace": ref.Namespace, "budget_disruption_name": ref.Name, "budget_disruption_kind": ref.Kind})
+	DisruptionBudgetCheckHealthHookErrorTotal.DeletePartialMatch(prometheus.Labels{"budget_disruption_namespace": ref.Namespace, "budget_disruption_name": ref.Name, "budget_disruption_kind": ref.Kind})
+}
+
+func UpdateBudgetStatusMetrics(ref nodedisruptionv1alpha1.NamespacedName, status nodedisruptionv1alpha1.DisruptionBudgetStatus) {
+	for _, node_name := range status.WatchedNodes {
+		DisruptionBudgetWatchedNodes.WithLabelValues(ref.Namespace, ref.Name, ref.Kind, node_name).Set(1)
+	}
+	for _, disruption := range status.Disruptions {
+		nd_state := 0
+		state := nodedisruptionv1alpha1.NodeDisruptionState(disruption.State)
+		if state == nodedisruptionv1alpha1.Pending {
+			nd_state = 0
+		} else if state == nodedisruptionv1alpha1.Rejected {
+			nd_state = -1
+		} else if state == nodedisruptionv1alpha1.Granted {
+			nd_state = 1
+		}
+		DisruptionBudgetDisruptions.WithLabelValues(ref.Namespace, ref.Name, ref.Kind, disruption.Name).Set(float64(nd_state))
+	}
+	DisruptionBudgetDisruptionsAllowed.WithLabelValues(ref.Namespace, ref.Name, ref.Kind).Set(float64(status.DisruptionsAllowed))
+	DisruptionBudgetCurrentDisruptions.WithLabelValues(ref.Namespace, ref.Name, ref.Kind).Set(float64(status.CurrentDisruptions))
+}
+
 // GetAllBudgetsInSync fetch all the budgets from Kubernetes and synchronise them
 func GetAllBudgetsInSync(ctx context.Context, k8sClient client.Client) ([]Budget, error) {
 	opts := []client.ListOption{}

internal/controller/metrics.go

@@ -0,0 +1,131 @@
+package controller
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const (
+	METIC_PREFIX = "node_disruption_controller_"
+)
+
+var (
+	// NODE DISRUPTION METRICS
+	NodeDisruptionGrantedTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: METIC_PREFIX + "node_disruption_granted_total",
+			Help: "Total number of granted node disruptions",
+		},
+		[]string{},
+	)
+	NodeDisruptionRejectedTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: METIC_PREFIX + "node_disruption_rejected_total",
+			Help: "Total number of rejected node disruptions",
+		},
+		[]string{},
+	)
+	NodeDisruptionState = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "node_disruption_state",
+			Help: "State of node disruption: pending=0, rejected=-1, accepted=1",
+		},
+		[]string{"node_disruption_name"},
+	)
+	NodeDisruptionCreated = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "node_disruption_created",
+			Help: "Date of create of the node disruption",
+		},
+		[]string{"node_disruption_name"},
+	)
+	NodeDisruptionDeadline = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "node_disruption_deadline",
+			Help: "Date of the deadline of the node disruption (0 if unset)",
+		},
+		[]string{"node_disruption_name"},
+	)
+	NodeDisruptionImpactedNodes = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "node_disruption_impacted_node",
+			Help: "high cardinality: create a metric for each node impacted by a given node disruption",
+		},
+		[]string{"node_disruption_name", "node_name"},
+	)
+	// DISRUPTION BUDGET METRICS
+	DisruptionBudgetCheckHealthHookStatusCodeTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: METIC_PREFIX + "disruption_budget_health_hook_status_code_total",
+			Help: "Total number of request by HTTP status code",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind", "status_code"},
+	)
+	DisruptionBudgetCheckHealthHookErrorTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: METIC_PREFIX + "disruption_budget_health_hook_error_total",
+			Help: "Total number of connection/response errors while requesting health hook",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind"},
+	)
+	DisruptionBudgetRejectedTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: METIC_PREFIX + "disruption_budget_rejected_total",
+			Help: "Total number of rejected node disruption by the disruption budget",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind"},
+	)
+	DisruptionBudgetGrantedTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: METIC_PREFIX + "disruption_budget_granted_total",
+			Help: "Total number of granted node disruption by the disruption budget",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind"},
+	)
+	DisruptionBudgetMaxDisruptions = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "disruption_budget_max_disruptions",
+			Help: "Reflect the MaxDisruptions fields from budget Spec",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind"},
+	)
+	DisruptionBudgetCurrentDisruptions = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "disruption_budget_current_disruptions",
+			Help: "Reflect the CurrentDisruptions fields from budget Status",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind"},
+	)
+	DisruptionBudgetDisruptionsAllowed = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "disruption_budget_disruptions_allowed",
+			Help: "Reflect the DisruptionsAllowed fields from budget Status",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind"},
+	)
+	DisruptionBudgetMaxDisruptedNodes = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "disruption_budget_max_disrupted_nodes",
+			Help: "Reflect the MaxDisruptedNodes fields from budget Spec",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind"},
+	)
+	DisruptionBudgetMinUndisruptedNodes = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "disruption_budget_min_undisrupted_nodes",
+			Help: "Reflect the MinUndisruptedNodes fields from budget Spec",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind"},
+	)
+	DisruptionBudgetWatchedNodes = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "node_disruption_watched_nodes",
+			Help: "high cardinality: create a metric for each node watched by a budget",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind", "node_name"},
+	)
+	DisruptionBudgetDisruptions = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: METIC_PREFIX + "budget_disruption_disruptions",
+			Help: "high cardinality: create a metric for each disruption by a budget",
+		},
+		[]string{"disruption_budget_namespace", "disruption_budget_name", "disruption_budget_kind", "node_disruption_name"},
+	)
+)