From 1da8ac1fade53b44ae26374e078eafcd8ee4af24 Mon Sep 17 00:00:00 2001 From: Ryota Kawamitsu Date: Wed, 19 Jun 2024 00:48:58 +0000 Subject: [PATCH 1/4] add custom metrics --- controllers/tenant_controller.go | 19 ++++++++++++++++ controllers/tenant_controller_test.go | 3 +++ go.mod | 2 +- pkg/metrics/metrics.go | 31 +++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 pkg/metrics/metrics.go diff --git a/controllers/tenant_controller.go b/controllers/tenant_controller.go index b9382bf..054bf84 100644 --- a/controllers/tenant_controller.go +++ b/controllers/tenant_controller.go @@ -13,6 +13,7 @@ import ( extract "github.com/cybozu-go/cattage/pkg/client" "github.com/cybozu-go/cattage/pkg/config" "github.com/cybozu-go/cattage/pkg/constants" + "github.com/cybozu-go/cattage/pkg/metrics" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -89,6 +90,7 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res if err := r.finalize(ctx, tenant); err != nil { return ctrl.Result{}, fmt.Errorf("failed to finalize: %w", err) } + r.removeMetrics(tenant) return ctrl.Result{}, nil } @@ -100,6 +102,7 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res err = err2 } } + r.setMetrics(tenant) }(tenant.Status) err = r.reconcileNamespaces(ctx, tenant) @@ -691,6 +694,22 @@ func (r *TenantReconciler) updateAllTenantNamespacesConfigMap(ctx context.Contex return nil } +func (r *TenantReconciler) setMetrics(tenant *cattagev1beta1.Tenant) { + switch tenant.Status.Health { + case cattagev1beta1.TenantHealthy: + metrics.HealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(1) + metrics.UnhealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(0) + case cattagev1beta1.TenantUnhealthy: + metrics.HealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(0) + metrics.UnhealthyVec.WithLabelValues(tenant.Name, tenant.Namespace).Set(1) + } +} + +func (r *TenantReconciler) removeMetrics(tenant *cattagev1beta1.Tenant) { + metrics.HealthyVec.DeleteLabelValues(tenant.Name, tenant.Namespace) + metrics.UnhealthyVec.DeleteLabelValues(tenant.Name, tenant.Namespace) +} + // SetupWithManager sets up the controller with the Manager. func (r *TenantReconciler) SetupWithManager(mgr ctrl.Manager) error { tenantHandler := func(ctx context.Context, o client.Object) []reconcile.Request { diff --git a/controllers/tenant_controller_test.go b/controllers/tenant_controller_test.go index 5b0a651..15388fa 100644 --- a/controllers/tenant_controller_test.go +++ b/controllers/tenant_controller_test.go @@ -12,6 +12,7 @@ import ( "github.com/cybozu-go/cattage/pkg/argocd" tenantconfig "github.com/cybozu-go/cattage/pkg/config" "github.com/cybozu-go/cattage/pkg/constants" + "github.com/cybozu-go/cattage/pkg/metrics" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" @@ -22,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) @@ -68,6 +70,7 @@ var _ = Describe("Tenant controller", Ordered, func() { }, } tr := NewTenantReconciler(mgr.GetClient(), config) + metrics.Register(k8smetrics.Registry) err = tr.SetupWithManager(mgr) Expect(err).ToNot(HaveOccurred()) err = SetupIndexForNamespace(ctx, mgr) diff --git a/go.mod b/go.mod index b94cd4e..de5a3ea 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/onsi/ginkgo/v2 v2.16.0 github.com/onsi/gomega v1.31.1 + github.com/prometheus/client_golang v1.18.0 github.com/spf13/cobra v1.8.0 k8s.io/api v0.29.2 k8s.io/apimachinery v0.29.2 @@ -49,7 +50,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..2bfd0fb --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,31 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + metricsNameSpace = "cattage" + tenantSubsystem = "tenant" +) + +var ( + HealthyVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricsNameSpace, + Subsystem: tenantSubsystem, + Name: "healthy", + Help: "The tenant status about healthy condition", + }, []string{"name", "namespace"}) + + UnhealthyVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricsNameSpace, + Subsystem: tenantSubsystem, + Name: "unhealthy", + Help: "The tenant status about unhealthy condition", + }, []string{"name", "namespace"}) +) + +func init() { + k8smetrics.Registry.MustRegister(HealthyVec, UnhealthyVec) +} From bf053b0afe90ae5baae363bc3e07840c5a99cdd2 Mon Sep 17 00:00:00 2001 From: Ryota Kawamitsu Date: Wed, 19 Jun 2024 08:32:37 +0000 Subject: [PATCH 2/4] add tests for custom metrics --- controllers/tenant_controller_test.go | 139 +++++++++++++++++++++++++- 1 file changed, 137 insertions(+), 2 deletions(-) diff --git a/controllers/tenant_controller_test.go b/controllers/tenant_controller_test.go index 15388fa..a80c6dc 100644 --- a/controllers/tenant_controller_test.go +++ b/controllers/tenant_controller_test.go @@ -12,10 +12,10 @@ import ( "github.com/cybozu-go/cattage/pkg/argocd" tenantconfig "github.com/cybozu-go/cattage/pkg/config" "github.com/cybozu-go/cattage/pkg/constants" - "github.com/cybozu-go/cattage/pkg/metrics" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" + "github.com/prometheus/client_golang/prometheus/testutil" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -70,7 +70,6 @@ var _ = Describe("Tenant controller", Ordered, func() { }, } tr := NewTenantReconciler(mgr.GetClient(), config) - metrics.Register(k8smetrics.Registry) err = tr.SetupWithManager(mgr) Expect(err).ToNot(HaveOccurred()) err = SetupIndexForNamespace(ctx, mgr) @@ -603,6 +602,142 @@ var _ = Describe("Tenant controller", Ordered, func() { }).Should(Succeed()) }) + It("should expose custom metrics", func() { + customMetricsNames := []string{"cattage_tenant_healthy", "cattage_tenant_unhealthy"} + tenant := &cattagev1beta1.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "m-team", + Finalizers: []string{constants.Finalizer}, + }, + Spec: cattagev1beta1.TenantSpec{ + RootNamespaces: []cattagev1beta1.RootNamespaceSpec{ + {Name: "app-m"}, + }, + ArgoCD: cattagev1beta1.ArgoCDSpec{}, + }, + } + err := k8sClient.Create(ctx, tenant) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() error { + expected := ` + # HELP cattage_tenant_healthy The tenant status about healthy condition + # TYPE cattage_tenant_healthy gauge + cattage_tenant_healthy{name="a-team",namespace=""} 1 + cattage_tenant_healthy{name="c-team",namespace=""} 1 + cattage_tenant_healthy{name="m-team",namespace=""} 1 + cattage_tenant_healthy{name="x-team",namespace=""} 1 + cattage_tenant_healthy{name="y-team",namespace=""} 1 + # HELP cattage_tenant_unhealthy The tenant status about unhealthy condition + # TYPE cattage_tenant_unhealthy gauge + cattage_tenant_unhealthy{name="a-team",namespace=""} 0 + cattage_tenant_unhealthy{name="c-team",namespace=""} 0 + cattage_tenant_unhealthy{name="m-team",namespace=""} 0 + cattage_tenant_unhealthy{name="x-team",namespace=""} 0 + cattage_tenant_unhealthy{name="y-team",namespace=""} 0 + ` + expectedReader := strings.NewReader(expected) + if err := testutil.GatherAndCompare(k8smetrics.Registry, expectedReader, customMetricsNames...); err != nil { + return err + } + + return nil + }).Should(Succeed()) + + By("injecting invalid delegates config") + err = k8sClient.Get(ctx, client.ObjectKey{Name: tenant.Name}, tenant) + Expect(err).ToNot(HaveOccurred()) + tenant.Spec.Delegates = []cattagev1beta1.DelegateSpec{ + { + Name: "team-does-not-exist", + Roles: []string{ + "admin", + }, + }, + } + err = k8sClient.Update(ctx, tenant) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { + expected := ` + # HELP cattage_tenant_healthy The tenant status about healthy condition + # TYPE cattage_tenant_healthy gauge + cattage_tenant_healthy{name="a-team",namespace=""} 1 + cattage_tenant_healthy{name="c-team",namespace=""} 1 + cattage_tenant_healthy{name="m-team",namespace=""} 0 + cattage_tenant_healthy{name="x-team",namespace=""} 1 + cattage_tenant_healthy{name="y-team",namespace=""} 1 + # HELP cattage_tenant_unhealthy The tenant status about unhealthy condition + # TYPE cattage_tenant_unhealthy gauge + cattage_tenant_unhealthy{name="a-team",namespace=""} 0 + cattage_tenant_unhealthy{name="c-team",namespace=""} 0 + cattage_tenant_unhealthy{name="m-team",namespace=""} 1 + cattage_tenant_unhealthy{name="x-team",namespace=""} 0 + cattage_tenant_unhealthy{name="y-team",namespace=""} 0 + ` + expectedReader := strings.NewReader(expected) + if err := testutil.GatherAndCompare(k8smetrics.Registry, expectedReader, customMetricsNames...); err != nil { + return err + } + return nil + }).Should(Succeed()) + + By("removing invalid delegates config") + err = k8sClient.Get(ctx, client.ObjectKey{Name: tenant.Name}, tenant) + Expect(err).ToNot(HaveOccurred()) + tenant.Spec.Delegates = []cattagev1beta1.DelegateSpec{} + err = k8sClient.Update(ctx, tenant) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { + expected := ` + # HELP cattage_tenant_healthy The tenant status about healthy condition + # TYPE cattage_tenant_healthy gauge + cattage_tenant_healthy{name="a-team",namespace=""} 1 + cattage_tenant_healthy{name="c-team",namespace=""} 1 + cattage_tenant_healthy{name="m-team",namespace=""} 1 + cattage_tenant_healthy{name="x-team",namespace=""} 1 + cattage_tenant_healthy{name="y-team",namespace=""} 1 + # HELP cattage_tenant_unhealthy The tenant status about unhealthy condition + # TYPE cattage_tenant_unhealthy gauge + cattage_tenant_unhealthy{name="a-team",namespace=""} 0 + cattage_tenant_unhealthy{name="c-team",namespace=""} 0 + cattage_tenant_unhealthy{name="m-team",namespace=""} 0 + cattage_tenant_unhealthy{name="x-team",namespace=""} 0 + cattage_tenant_unhealthy{name="y-team",namespace=""} 0 + ` + expectedReader := strings.NewReader(expected) + if err := testutil.GatherAndCompare(k8smetrics.Registry, expectedReader, customMetricsNames...); err != nil { + return err + } + return nil + }).Should(Succeed()) + + By("removing tenant") + err = k8sClient.Delete(ctx, tenant) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() error { + expected := ` + # HELP cattage_tenant_healthy The tenant status about healthy condition + # TYPE cattage_tenant_healthy gauge + cattage_tenant_healthy{name="a-team",namespace=""} 1 + cattage_tenant_healthy{name="c-team",namespace=""} 1 + cattage_tenant_healthy{name="x-team",namespace=""} 1 + cattage_tenant_healthy{name="y-team",namespace=""} 1 + # HELP cattage_tenant_unhealthy The tenant status about unhealthy condition + # TYPE cattage_tenant_unhealthy gauge + cattage_tenant_unhealthy{name="a-team",namespace=""} 0 + cattage_tenant_unhealthy{name="c-team",namespace=""} 0 + cattage_tenant_unhealthy{name="x-team",namespace=""} 0 + cattage_tenant_unhealthy{name="y-team",namespace=""} 0 + ` + expectedReader := strings.NewReader(expected) + if err := testutil.GatherAndCompare(k8smetrics.Registry, expectedReader, customMetricsNames...); err != nil { + return err + } + return nil + }).Should(Succeed()) + }) + Context("Migration to Argo CD 2.5", func() { It("should remove old applications", func() { oldApp, err := fillApplication("app", config.ArgoCD.Namespace, "a-team") From 04d2f26e1cbf89209d407cf79b4d944bbba509d6 Mon Sep 17 00:00:00 2001 From: Ryota Kawamitsu Date: Thu, 20 Jun 2024 02:26:24 +0000 Subject: [PATCH 3/4] replace deprecated --skip-publish flag of goreleaser --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9df1ce8..2b3ad33 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -116,6 +116,6 @@ jobs: uses: goreleaser/goreleaser-action@f82d6c1c344bcacabba2c841718984797f664a6b # v4.2.0 with: version: latest - args: --snapshot --skip-publish --clean + args: --snapshot --skip=publish --clean - name: Test built containers run: make container-structure-test From f75bb97079f456284c82d5dc2eed2028ee4fd611 Mon Sep 17 00:00:00 2001 From: Ryota Kawamitsu Date: Wed, 26 Jun 2024 04:53:25 +0000 Subject: [PATCH 4/4] Move removeMetrics just before RemoveFinalizer --- controllers/tenant_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/tenant_controller.go b/controllers/tenant_controller.go index 054bf84..3cec08b 100644 --- a/controllers/tenant_controller.go +++ b/controllers/tenant_controller.go @@ -90,7 +90,6 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res if err := r.finalize(ctx, tenant); err != nil { return ctrl.Result{}, fmt.Errorf("failed to finalize: %w", err) } - r.removeMetrics(tenant) return ctrl.Result{}, nil } @@ -287,6 +286,7 @@ func (r *TenantReconciler) finalize(ctx context.Context, tenant *cattagev1beta1. return err } + r.removeMetrics(tenant) controllerutil.RemoveFinalizer(tenant, constants.Finalizer) err = r.client.Update(ctx, tenant) if err != nil {