Merge pull request #53 from cybozu-go/add-custom-metrics

Add custom metrics

.github/workflows/ci.yaml

@@ -116,6 +116,6 @@ jobs:
         uses: goreleaser/goreleaser-action@f82d6c1c344bcacabba2c841718984797f664a6b # v4.2.0
         with:
           version: latest
-          args: --snapshot --skip-publish --clean
+          args: --snapshot --skip=publish --clean
       - name: Test built containers
         run: make container-structure-test

controllers/tenant_controller.go

@@ -13,6 +13,7 @@ import (
 	extract "github.com/cybozu-go/cattage/pkg/client"
 	"github.com/cybozu-go/cattage/pkg/config"
 	"github.com/cybozu-go/cattage/pkg/constants"
+	"github.com/cybozu-go/cattage/pkg/metrics"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -100,6 +101,7 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 				err = err2
 			}
 		}
+		r.setMetrics(tenant)
 	}(tenant.Status)
 
 	err = r.reconcileNamespaces(ctx, tenant)
@@ -284,6 +286,7 @@ func (r *TenantReconciler) finalize(ctx context.Context, tenant *cattagev1beta1.
 		return err
 	}
 
+	r.removeMetrics(tenant)
 	controllerutil.RemoveFinalizer(tenant, constants.Finalizer)
 	err = r.client.Update(ctx, tenant)
 	if err != nil {
@@ -691,6 +694,22 @@ func (r *TenantReconciler) updateAllTenantNamespacesConfigMap(ctx context.Contex
 	return nil
 }
 
+func (r *TenantReconciler) setMetrics(tenant *cattagev1beta1.Tenant) {
+	switch tenant.Status.Health {
+	case cattagev1beta1.TenantHealthy:
+		metrics.HealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(1)
+		metrics.UnhealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(0)
+	case cattagev1beta1.TenantUnhealthy:
+		metrics.HealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(0)
+		metrics.UnhealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(1)
+	}
+}
+
+func (r *TenantReconciler) removeMetrics(tenant *cattagev1beta1.Tenant) {
+	metrics.HealthyVec.DeleteLabelValues(tenant.Name, tenant.Namespace)
+	metrics.UnhealthyVec.DeleteLabelValues(tenant.Name, tenant.Namespace)
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *TenantReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	tenantHandler := func(ctx context.Context, o client.Object) []reconcile.Request {

controllers/tenant_controller_test.go

@@ -15,13 +15,15 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gstruct"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 )
 
@@ -600,6 +602,142 @@ var _ = Describe("Tenant controller", Ordered, func() {
 		}).Should(Succeed())
 	})
 
+	It("should expose custom metrics", func() {
+		customMetricsNames := []string{"cattage_tenant_healthy", "cattage_tenant_unhealthy"}
+		tenant := &cattagev1beta1.Tenant{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:       "m-team",
+				Finalizers: []string{constants.Finalizer},
+			},
+			Spec: cattagev1beta1.TenantSpec{
+				RootNamespaces: []cattagev1beta1.RootNamespaceSpec{
+					{Name: "app-m"},
+				},
+				ArgoCD: cattagev1beta1.ArgoCDSpec{},
+			},
+		}
+		err := k8sClient.Create(ctx, tenant)
+		Expect(err).ToNot(HaveOccurred())
+
+		Eventually(func() error {
+			expected := `
+			# HELP cattage_tenant_healthy The tenant status about healthy condition
+			# TYPE cattage_tenant_healthy gauge
+			cattage_tenant_healthy{name="a-team",namespace=""} 1
+			cattage_tenant_healthy{name="c-team",namespace=""} 1
+			cattage_tenant_healthy{name="m-team",namespace=""} 1
+			cattage_tenant_healthy{name="x-team",namespace=""} 1
+			cattage_tenant_healthy{name="y-team",namespace=""} 1
+			# HELP cattage_tenant_unhealthy The tenant status about unhealthy condition
+			# TYPE cattage_tenant_unhealthy gauge
+			cattage_tenant_unhealthy{name="a-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="c-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="m-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="x-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="y-team",namespace=""} 0
+			`
+			expectedReader := strings.NewReader(expected)
+			if err := testutil.GatherAndCompare(k8smetrics.Registry, expectedReader, customMetricsNames...); err != nil {
+				return err
+			}
+
+			return nil
+		}).Should(Succeed())
+
+		By("injecting invalid delegates config")
+		err = k8sClient.Get(ctx, client.ObjectKey{Name: tenant.Name}, tenant)
+		Expect(err).ToNot(HaveOccurred())
+		tenant.Spec.Delegates = []cattagev1beta1.DelegateSpec{
+			{
+				Name: "team-does-not-exist",
+				Roles: []string{
+					"admin",
+				},
+			},
+		}
+		err = k8sClient.Update(ctx, tenant)
+		Expect(err).ToNot(HaveOccurred())
+		Eventually(func() error {
+			expected := `
+			# HELP cattage_tenant_healthy The tenant status about healthy condition
+			# TYPE cattage_tenant_healthy gauge
+			cattage_tenant_healthy{name="a-team",namespace=""} 1
+			cattage_tenant_healthy{name="c-team",namespace=""} 1
+			cattage_tenant_healthy{name="m-team",namespace=""} 0
+			cattage_tenant_healthy{name="x-team",namespace=""} 1
+			cattage_tenant_healthy{name="y-team",namespace=""} 1
+			# HELP cattage_tenant_unhealthy The tenant status about unhealthy condition
+			# TYPE cattage_tenant_unhealthy gauge
+			cattage_tenant_unhealthy{name="a-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="c-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="m-team",namespace=""} 1
+			cattage_tenant_unhealthy{name="x-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="y-team",namespace=""} 0
+			`
+			expectedReader := strings.NewReader(expected)
+			if err := testutil.GatherAndCompare(k8smetrics.Registry, expectedReader, customMetricsNames...); err != nil {
+				return err
+			}
+			return nil
+		}).Should(Succeed())
+
+		By("removing invalid delegates config")
+		err = k8sClient.Get(ctx, client.ObjectKey{Name: tenant.Name}, tenant)
+		Expect(err).ToNot(HaveOccurred())
+		tenant.Spec.Delegates = []cattagev1beta1.DelegateSpec{}
+		err = k8sClient.Update(ctx, tenant)
+		Expect(err).ToNot(HaveOccurred())
+		Eventually(func() error {
+			expected := `
+			# HELP cattage_tenant_healthy The tenant status about healthy condition
+			# TYPE cattage_tenant_healthy gauge
+			cattage_tenant_healthy{name="a-team",namespace=""} 1
+			cattage_tenant_healthy{name="c-team",namespace=""} 1
+			cattage_tenant_healthy{name="m-team",namespace=""} 1
+			cattage_tenant_healthy{name="x-team",namespace=""} 1
+			cattage_tenant_healthy{name="y-team",namespace=""} 1
+			# HELP cattage_tenant_unhealthy The tenant status about unhealthy condition
+			# TYPE cattage_tenant_unhealthy gauge
+			cattage_tenant_unhealthy{name="a-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="c-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="m-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="x-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="y-team",namespace=""} 0
+			`
+			expectedReader := strings.NewReader(expected)
+			if err := testutil.GatherAndCompare(k8smetrics.Registry, expectedReader, customMetricsNames...); err != nil {
+				return err
+			}
+			return nil
+		}).Should(Succeed())
+
+		By("removing tenant")
+		err = k8sClient.Delete(ctx, tenant)
+		Expect(err).ToNot(HaveOccurred())
+
+		Eventually(func() error {
+			expected := `
+			# HELP cattage_tenant_healthy The tenant status about healthy condition
+			# TYPE cattage_tenant_healthy gauge
+			cattage_tenant_healthy{name="a-team",namespace=""} 1
+			cattage_tenant_healthy{name="c-team",namespace=""} 1
+			cattage_tenant_healthy{name="x-team",namespace=""} 1
+			cattage_tenant_healthy{name="y-team",namespace=""} 1
+			# HELP cattage_tenant_unhealthy The tenant status about unhealthy condition
+			# TYPE cattage_tenant_unhealthy gauge
+			cattage_tenant_unhealthy{name="a-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="c-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="x-team",namespace=""} 0
+			cattage_tenant_unhealthy{name="y-team",namespace=""} 0
+			`
+			expectedReader := strings.NewReader(expected)
+			if err := testutil.GatherAndCompare(k8smetrics.Registry, expectedReader, customMetricsNames...); err != nil {
+				return err
+			}
+			return nil
+		}).Should(Succeed())
+	})
+
 	Context("Migration to Argo CD 2.5", func() {
 		It("should remove old applications", func() {
 			oldApp, err := fillApplication("app", config.ArgoCD.Namespace, "a-team")

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/onsi/ginkgo/v2 v2.16.0
 	github.com/onsi/gomega v1.31.1
+	github.com/prometheus/client_golang v1.18.0
 	github.com/spf13/cobra v1.8.0
 	k8s.io/api v0.29.2
 	k8s.io/apimachinery v0.29.2
@@ -49,7 +50,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect

pkg/metrics/metrics.go

@@ -0,0 +1,31 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+const (
+	metricsNameSpace = "cattage"
+	tenantSubsystem  = "tenant"
+)
+
+var (
+	HealthyVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: metricsNameSpace,
+		Subsystem: tenantSubsystem,
+		Name:      "healthy",
+		Help:      "The tenant status about healthy condition",
+	}, []string{"name", "namespace"})
+
+	UnhealthyVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: metricsNameSpace,
+		Subsystem: tenantSubsystem,
+		Name:      "unhealthy",
+		Help:      "The tenant status about unhealthy condition",
+	}, []string{"name", "namespace"})
+)
+
+func init() {
+	k8smetrics.Registry.MustRegister(HealthyVec, UnhealthyVec)
+}