add custom metrics

controllers/tenant_controller.go

@@ -13,6 +13,7 @@ import (
 	extract "github.com/cybozu-go/cattage/pkg/client"
 	"github.com/cybozu-go/cattage/pkg/config"
 	"github.com/cybozu-go/cattage/pkg/constants"
+	"github.com/cybozu-go/cattage/pkg/metrics"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -89,6 +90,7 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 		if err := r.finalize(ctx, tenant); err != nil {
 			return ctrl.Result{}, fmt.Errorf("failed to finalize: %w", err)
 		}
+		r.removeMetrics(tenant)
 		return ctrl.Result{}, nil
 	}
 
@@ -100,6 +102,7 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 				err = err2
 			}
 		}
+		r.setMetrics(tenant)
 	}(tenant.Status)
 
 	err = r.reconcileNamespaces(ctx, tenant)
@@ -691,6 +694,22 @@ func (r *TenantReconciler) updateAllTenantNamespacesConfigMap(ctx context.Contex
 	return nil
 }
 
+func (r *TenantReconciler) setMetrics(tenant *cattagev1beta1.Tenant) {
+	switch tenant.Status.Health {
+	case cattagev1beta1.TenantHealthy:
+		metrics.HealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(1)
+		metrics.UnhealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(0)
+	case cattagev1beta1.TenantUnhealthy:
+		metrics.HealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(0)
+		metrics.UnhealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(1)
+	}
+}
+
+func (r *TenantReconciler) removeMetrics(tenant *cattagev1beta1.Tenant) {
+	metrics.HealthyVec.DeleteLabelValues(tenant.Name, tenant.Namespace)
+	metrics.UnhealthyVec.DeleteLabelValues(tenant.Name, tenant.Namespace)
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *TenantReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	tenantHandler := func(ctx context.Context, o client.Object) []reconcile.Request {

controllers/tenant_controller_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/cybozu-go/cattage/pkg/argocd"
 	tenantconfig "github.com/cybozu-go/cattage/pkg/config"
 	"github.com/cybozu-go/cattage/pkg/constants"
+	"github.com/cybozu-go/cattage/pkg/metrics"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gstruct"
@@ -22,6 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 )
 
@@ -68,6 +70,7 @@ var _ = Describe("Tenant controller", Ordered, func() {
 			},
 		}
 		tr := NewTenantReconciler(mgr.GetClient(), config)
+		metrics.Register(k8smetrics.Registry)
 		err = tr.SetupWithManager(mgr)
 		Expect(err).ToNot(HaveOccurred())
 		err = SetupIndexForNamespace(ctx, mgr)

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/onsi/ginkgo/v2 v2.16.0
 	github.com/onsi/gomega v1.31.1
+	github.com/prometheus/client_golang v1.18.0
 	github.com/spf13/cobra v1.8.0
 	k8s.io/api v0.29.2
 	k8s.io/apimachinery v0.29.2
@@ -49,7 +50,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect

pkg/metrics/metrics.go

@@ -0,0 +1,31 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+const (
+	metricsNameSpace = "cattage"
+	tenantSubsystem = "tenant"
+)
+
+var (
+	HealthyVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: metricsNameSpace,
+		Subsystem: tenantSubsystem,
+		Name:      "healthy",
+		Help:      "The tenant status about healthy condition",
+	}, []string{"name", "namespace"})
+
+	UnhealthyVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: metricsNameSpace,
+		Subsystem: tenantSubsystem,
+		Name:      "unhealthy",
+		Help:      "The tenant status about unhealthy condition",
+	}, []string{"name", "namespace"})
+)
+
+func init() {
+	k8smetrics.Registry.MustRegister(HealthyVec, UnhealthyVec)
+}