From 7352c590656f79e514b08f16673ddfd69ff7f562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Tue, 28 Nov 2023 08:18:55 +0100 Subject: [PATCH 1/9] WiP metrics --- internal/controller/metrics/metrics.go | 46 ++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 internal/controller/metrics/metrics.go diff --git a/internal/controller/metrics/metrics.go b/internal/controller/metrics/metrics.go new file mode 100644 index 00000000..84719aa8 --- /dev/null +++ b/internal/controller/metrics/metrics.go @@ -0,0 +1,46 @@ +package metrics + +import ( + v1 "github.com/kyma-project/infrastructure-manager/api/v1" + "github.com/prometheus/client_golang/prometheus" + ctrlMetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + shootName = "shootName" + globalAccountId = "globalAccountId" + runtimeId = "runtimeId" + state = "state" +) + +var ( + + //TODO: test custom metric, remove when done with https://github.com/kyma-project/infrastructure-manager/issues/11 + totalReconciliationLoopsStarted = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "im_reconciliation_loops_started_total", + Help: "Number of times reconciliation loop was started", + }, + ) + + metricGardenerClustersState = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ //nolint:gochecknoglobals + Subsystem: "infrastructure_manager", + Name: "im_gardener_clusters_state", + Help: "Indicates the Status.state for GardenerCluster CRs", + }, []string{shootName, state}) +) + +func init() { + ctrlMetrics.Registry.MustRegister(totalReconciliationLoopsStarted, metricGardenerClustersState) +} + +func IncrementReconciliationLoopsStarted() { + totalReconciliationLoopsStarted.Inc() +} + +func SetGardenerClusterStates(cluster v1.GardenerCluster) { + // Increase a value using compact (but order-sensitive!) WithLabelValues(). + //metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, "READY").Add(1) + metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, cluster.Spec.Shoot.Name).Add(1) +} From 96bf120b642f47d9f880fe2ad72f6c2fe3c7ef1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 4 Dec 2023 08:16:45 +0100 Subject: [PATCH 2/9] WiP metrics --- .../controller/gardener_cluster_controller.go | 22 ++++++++++++------- internal/controller/metrics/metrics.go | 21 +++++++++++------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/internal/controller/gardener_cluster_controller.go b/internal/controller/gardener_cluster_controller.go index 112d7593..a01ac4c0 100644 --- a/internal/controller/gardener_cluster_controller.go +++ b/internal/controller/gardener_cluster_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "github.com/kyma-project/infrastructure-manager/internal/controller/metrics" "time" "github.com/go-logr/logr" @@ -79,10 +80,12 @@ type KubeconfigProvider interface { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.15.0/pkg/reconcile func (controller *GardenerClusterController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { //nolint:revive - controller.log.Info("Starting reconciliation.", loggingContext(req)...) + controller.log.Info("Starting reconciliation. 16:51", loggingContext(req)...) var cluster imv1.GardenerCluster + metrics.IncrementReconciliationLoopsStarted() + err := controller.Get(ctx, req.NamespacedName, &cluster) if err != nil { if k8serrors.IsNotFound(err) { @@ -93,7 +96,7 @@ func (controller *GardenerClusterController) Reconcile(ctx context.Context, req controller.log.Info("Secret has been deleted.", loggingContext(req)...) } - return controller.resultWithoutRequeue(), err + return controller.resultWithoutRequeue(&cluster), err } lastSyncTime := time.Now() @@ -105,25 +108,25 @@ func (controller *GardenerClusterController) Reconcile(ctx context.Context, req if k8serrors.IsNotFound(err) { err = nil } - return controller.resultWithoutRequeue(), err + return controller.resultWithoutRequeue(&cluster), err } // there was a request to rotate the kubeconfig if kubeconfigStatus == ksRotated { err = controller.removeForceRotationAnnotation(ctx, &cluster) if err != nil { - return controller.resultWithoutRequeue(), err + return controller.resultWithoutRequeue(&cluster), err } } if kubeconfigStatus == ksCreated || kubeconfigStatus == ksModified { err = controller.persistStatusChange(ctx, &cluster) if err != nil { - return controller.resultWithoutRequeue(), err + return controller.resultWithoutRequeue(nil), err } } - return controller.resultWithRequeue(), nil + return controller.resultWithRequeue(&cluster), nil } func loggingContextFromCluster(cluster *imv1.GardenerCluster) []any { @@ -134,14 +137,17 @@ func loggingContext(req ctrl.Request) []any { return []any{"GardenerCluster", req.Name, "Namespace", req.Namespace} } -func (controller *GardenerClusterController) resultWithRequeue() ctrl.Result { +func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.GardenerCluster) ctrl.Result { + metrics.IncGardenerClusterStates(*cluster, controller.log) + return ctrl.Result{ Requeue: true, RequeueAfter: controller.rotationPeriod, } } -func (controller *GardenerClusterController) resultWithoutRequeue() ctrl.Result { +func (controller *GardenerClusterController) resultWithoutRequeue(cluster *imv1.GardenerCluster) ctrl.Result { + metrics.DecGardenerClusterStates(*cluster, controller.log) return ctrl.Result{} } diff --git a/internal/controller/metrics/metrics.go b/internal/controller/metrics/metrics.go index 84719aa8..795a8271 100644 --- a/internal/controller/metrics/metrics.go +++ b/internal/controller/metrics/metrics.go @@ -1,6 +1,7 @@ package metrics import ( + "github.com/go-logr/logr" v1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/prometheus/client_golang/prometheus" ctrlMetrics "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -16,9 +17,9 @@ const ( var ( //TODO: test custom metric, remove when done with https://github.com/kyma-project/infrastructure-manager/issues/11 - totalReconciliationLoopsStarted = prometheus.NewCounter( + playground_totalReconciliationLoopsStarted = prometheus.NewCounter( prometheus.CounterOpts{ - Name: "im_reconciliation_loops_started_total", + Name: "im_playground_reconciliation_loops_started_total", Help: "Number of times reconciliation loop was started", }, ) @@ -32,15 +33,19 @@ var ( ) func init() { - ctrlMetrics.Registry.MustRegister(totalReconciliationLoopsStarted, metricGardenerClustersState) + ctrlMetrics.Registry.MustRegister(playground_totalReconciliationLoopsStarted, metricGardenerClustersState) } func IncrementReconciliationLoopsStarted() { - totalReconciliationLoopsStarted.Inc() + playground_totalReconciliationLoopsStarted.Inc() } -func SetGardenerClusterStates(cluster v1.GardenerCluster) { - // Increase a value using compact (but order-sensitive!) WithLabelValues(). - //metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, "READY").Add(1) - metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, cluster.Spec.Shoot.Name).Add(1) +func IncGardenerClusterStates(cluster v1.GardenerCluster, log logr.Logger) { + metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, string(cluster.Status.State)).Inc() + log.Info("IncMetrics(): cluster.Spec.Shoot.Name: %s, cluster.Status.State: %s ", cluster.Spec.Shoot.Name, cluster.Status.State) +} + +func DecGardenerClusterStates(cluster v1.GardenerCluster, log logr.Logger) { + metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, string(cluster.Status.State)).Dec() + log.Info("DecMetrics(): cluster.Spec.Shoot.Name: %s, cluster.Status.State: %s ", cluster.Spec.Shoot.Name, cluster.Status.State) } From cfa449feabd09617f82a051a21b53b7118b38c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 4 Dec 2023 10:35:18 +0100 Subject: [PATCH 3/9] WiP metrics --- internal/controller/gardener_cluster_controller.go | 4 ++-- internal/controller/metrics/metrics.go | 9 ++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/internal/controller/gardener_cluster_controller.go b/internal/controller/gardener_cluster_controller.go index a01ac4c0..ea9fb6c7 100644 --- a/internal/controller/gardener_cluster_controller.go +++ b/internal/controller/gardener_cluster_controller.go @@ -138,7 +138,7 @@ func loggingContext(req ctrl.Request) []any { } func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.GardenerCluster) ctrl.Result { - metrics.IncGardenerClusterStates(*cluster, controller.log) + metrics.SetGardenerClusterStates(*cluster, controller.log) return ctrl.Result{ Requeue: true, @@ -147,7 +147,7 @@ func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.Gar } func (controller *GardenerClusterController) resultWithoutRequeue(cluster *imv1.GardenerCluster) ctrl.Result { - metrics.DecGardenerClusterStates(*cluster, controller.log) + metrics.SetGardenerClusterStates(*cluster, controller.log) return ctrl.Result{} } diff --git a/internal/controller/metrics/metrics.go b/internal/controller/metrics/metrics.go index 795a8271..6882e899 100644 --- a/internal/controller/metrics/metrics.go +++ b/internal/controller/metrics/metrics.go @@ -40,12 +40,7 @@ func IncrementReconciliationLoopsStarted() { playground_totalReconciliationLoopsStarted.Inc() } -func IncGardenerClusterStates(cluster v1.GardenerCluster, log logr.Logger) { - metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, string(cluster.Status.State)).Inc() +func SetGardenerClusterStates(cluster v1.GardenerCluster, log logr.Logger) { + metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, string(cluster.Status.State)).Set(1) log.Info("IncMetrics(): cluster.Spec.Shoot.Name: %s, cluster.Status.State: %s ", cluster.Spec.Shoot.Name, cluster.Status.State) } - -func DecGardenerClusterStates(cluster v1.GardenerCluster, log logr.Logger) { - metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, string(cluster.Status.State)).Dec() - log.Info("DecMetrics(): cluster.Spec.Shoot.Name: %s, cluster.Status.State: %s ", cluster.Spec.Shoot.Name, cluster.Status.State) -} From 4109a0f21945d9225228c1e49d255eb466bbb56c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 4 Dec 2023 10:37:35 +0100 Subject: [PATCH 4/9] clean-up reconciliation log --- internal/controller/gardener_cluster_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/gardener_cluster_controller.go b/internal/controller/gardener_cluster_controller.go index ea9fb6c7..0276bed9 100644 --- a/internal/controller/gardener_cluster_controller.go +++ b/internal/controller/gardener_cluster_controller.go @@ -80,7 +80,7 @@ type KubeconfigProvider interface { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.15.0/pkg/reconcile func (controller *GardenerClusterController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { //nolint:revive - controller.log.Info("Starting reconciliation. 16:51", loggingContext(req)...) + controller.log.Info("Starting reconciliation.", loggingContext(req)...) var cluster imv1.GardenerCluster From c79c2f7805298ccd00b85fb7757113831b13ad6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 4 Dec 2023 10:38:36 +0100 Subject: [PATCH 5/9] WiP metrics --- internal/controller/gardener_cluster_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/gardener_cluster_controller.go b/internal/controller/gardener_cluster_controller.go index 0276bed9..5796443b 100644 --- a/internal/controller/gardener_cluster_controller.go +++ b/internal/controller/gardener_cluster_controller.go @@ -122,7 +122,7 @@ func (controller *GardenerClusterController) Reconcile(ctx context.Context, req if kubeconfigStatus == ksCreated || kubeconfigStatus == ksModified { err = controller.persistStatusChange(ctx, &cluster) if err != nil { - return controller.resultWithoutRequeue(nil), err + return controller.resultWithoutRequeue(&cluster), err } } From 7a73b40ee0f4f1fdb46cf675a042ddd362d55d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 4 Dec 2023 11:44:25 +0100 Subject: [PATCH 6/9] fixes imports order --- internal/controller/gardener_cluster_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/gardener_cluster_controller.go b/internal/controller/gardener_cluster_controller.go index 5796443b..b96f8a84 100644 --- a/internal/controller/gardener_cluster_controller.go +++ b/internal/controller/gardener_cluster_controller.go @@ -19,11 +19,11 @@ package controller import ( "context" "fmt" - "github.com/kyma-project/infrastructure-manager/internal/controller/metrics" "time" "github.com/go-logr/logr" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" + "github.com/kyma-project/infrastructure-manager/internal/controller/metrics" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" From 2296310186e0f30e17b98e3a25943c75fe2a20af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 4 Dec 2023 11:49:07 +0100 Subject: [PATCH 7/9] fixes godox linter (todo comment) --- internal/controller/metrics/metrics.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/controller/metrics/metrics.go b/internal/controller/metrics/metrics.go index 6882e899..2bef80f6 100644 --- a/internal/controller/metrics/metrics.go +++ b/internal/controller/metrics/metrics.go @@ -16,7 +16,7 @@ const ( var ( - //TODO: test custom metric, remove when done with https://github.com/kyma-project/infrastructure-manager/issues/11 + //TODO: test custom metric, remove when done with https://github.com/kyma-project/infrastructure-manager/issues/11 //nolint:all playground_totalReconciliationLoopsStarted = prometheus.NewCounter( prometheus.CounterOpts{ Name: "im_playground_reconciliation_loops_started_total", @@ -42,5 +42,4 @@ func IncrementReconciliationLoopsStarted() { func SetGardenerClusterStates(cluster v1.GardenerCluster, log logr.Logger) { metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, string(cluster.Status.State)).Set(1) - log.Info("IncMetrics(): cluster.Spec.Shoot.Name: %s, cluster.Status.State: %s ", cluster.Spec.Shoot.Name, cluster.Status.State) } From 2ed34c1cc1e795e3177aa60341adeb51f363e113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 4 Dec 2023 12:09:54 +0100 Subject: [PATCH 8/9] fixes/mutes lint errors --- .../controller/gardener_cluster_controller.go | 2 +- internal/controller/metrics/metrics.go | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/controller/gardener_cluster_controller.go b/internal/controller/gardener_cluster_controller.go index faa2a527..94257a3b 100644 --- a/internal/controller/gardener_cluster_controller.go +++ b/internal/controller/gardener_cluster_controller.go @@ -145,7 +145,7 @@ func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.Gar } } -func (controller *GardenerClusterController) resultWithoutRequeue(cluster *imv1.GardenerCluster) ctrl.Result { +func (controller *GardenerClusterController) resultWithoutRequeue(cluster *imv1.GardenerCluster) ctrl.Result { //nolint:unparam metrics.SetGardenerClusterStates(*cluster, controller.log) return ctrl.Result{} } diff --git a/internal/controller/metrics/metrics.go b/internal/controller/metrics/metrics.go index 2bef80f6..b71ff452 100644 --- a/internal/controller/metrics/metrics.go +++ b/internal/controller/metrics/metrics.go @@ -8,23 +8,21 @@ import ( ) const ( - shootName = "shootName" - globalAccountId = "globalAccountId" - runtimeId = "runtimeId" - state = "state" + shootName = "shootName" + state = "state" ) var ( - //TODO: test custom metric, remove when done with https://github.com/kyma-project/infrastructure-manager/issues/11 //nolint:all - playground_totalReconciliationLoopsStarted = prometheus.NewCounter( + //nolint:godox //TODO: test custom metric, remove when done with https://github.com/kyma-project/infrastructure-manager/issues/11 + playgroundTotalReconciliationLoopsStarted = prometheus.NewCounter( //nolint:gochecknoglobals prometheus.CounterOpts{ Name: "im_playground_reconciliation_loops_started_total", Help: "Number of times reconciliation loop was started", }, ) - metricGardenerClustersState = prometheus.NewGaugeVec( + metricGardenerClustersState = prometheus.NewGaugeVec( //nolint:gochecknoglobals prometheus.GaugeOpts{ //nolint:gochecknoglobals Subsystem: "infrastructure_manager", Name: "im_gardener_clusters_state", @@ -33,11 +31,11 @@ var ( ) func init() { - ctrlMetrics.Registry.MustRegister(playground_totalReconciliationLoopsStarted, metricGardenerClustersState) + ctrlMetrics.Registry.MustRegister(playgroundTotalReconciliationLoopsStarted, metricGardenerClustersState) } func IncrementReconciliationLoopsStarted() { - playground_totalReconciliationLoopsStarted.Inc() + playgroundTotalReconciliationLoopsStarted.Inc() } func SetGardenerClusterStates(cluster v1.GardenerCluster, log logr.Logger) { From eec2f697c3c37219a707d23a15e4d6088b6d3625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 4 Dec 2023 12:11:01 +0100 Subject: [PATCH 9/9] removes unused logger --- internal/controller/gardener_cluster_controller.go | 4 ++-- internal/controller/metrics/metrics.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/controller/gardener_cluster_controller.go b/internal/controller/gardener_cluster_controller.go index 94257a3b..e337cc22 100644 --- a/internal/controller/gardener_cluster_controller.go +++ b/internal/controller/gardener_cluster_controller.go @@ -137,7 +137,7 @@ func loggingContext(req ctrl.Request) []any { } func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.GardenerCluster) ctrl.Result { - metrics.SetGardenerClusterStates(*cluster, controller.log) + metrics.SetGardenerClusterStates(*cluster) return ctrl.Result{ Requeue: true, @@ -146,7 +146,7 @@ func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.Gar } func (controller *GardenerClusterController) resultWithoutRequeue(cluster *imv1.GardenerCluster) ctrl.Result { //nolint:unparam - metrics.SetGardenerClusterStates(*cluster, controller.log) + metrics.SetGardenerClusterStates(*cluster) return ctrl.Result{} } diff --git a/internal/controller/metrics/metrics.go b/internal/controller/metrics/metrics.go index b71ff452..c6f7a99a 100644 --- a/internal/controller/metrics/metrics.go +++ b/internal/controller/metrics/metrics.go @@ -1,7 +1,6 @@ package metrics import ( - "github.com/go-logr/logr" v1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/prometheus/client_golang/prometheus" ctrlMetrics "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -38,6 +37,6 @@ func IncrementReconciliationLoopsStarted() { playgroundTotalReconciliationLoopsStarted.Inc() } -func SetGardenerClusterStates(cluster v1.GardenerCluster, log logr.Logger) { +func SetGardenerClusterStates(cluster v1.GardenerCluster) { metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, string(cluster.Status.State)).Set(1) }