Update runtime state metrics on application startup

cmd/main.go

@@ -28,6 +28,7 @@ import (
 	"github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	gardener_apis "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1"
 	gardener_oidc "github.com/gardener/oidc-webhook-authenticator/apis/authentication/v1alpha1"
+	"github.com/go-logr/logr"
 	validator "github.com/go-playground/validator/v10"
 	infrastructuremanagerv1 "github.com/kyma-project/infrastructure-manager/api/v1"
 	"github.com/kyma-project/infrastructure-manager/internal/auditlogging"
@@ -44,6 +45,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -106,7 +108,9 @@ func main() {
 	logger := zap.New(zap.UseFlagOptions(&opts))
 	ctrl.SetLogger(logger)
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	restConfig := ctrl.GetConfigOrDie()
+
+	mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
 		Metrics: metricsserver.Options{
 			BindAddress: metricsAddr,
 		},
@@ -183,15 +187,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	// refresh runtime metrics
-	metrics.ResetRuntimeMetrics()
-	var runtimeList infrastructuremanagerv1.RuntimeList
-	if err = mgr.GetClient().List(context.TODO(), &runtimeList); err != nil {
-		for _, rt := range runtimeList.Items {
-			metrics.SetRuntimeStates(rt)
-		}
-	}
-
 	cfg := fsm.RCCfg{
 		GardenerRequeueDuration:     defaultGardenerRequeueDuration,
 		ControlPlaneRequeueDuration: defaultControlPlaneRequeueDuration,
@@ -229,6 +224,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	refreshRuntimeMetrics(restConfig, logger, metrics)
+
 	setupLog.Info("Starting Manager", "kubeconfigExpirationTime", expirationTime, "kubeconfigRotationPeriod", rotationPeriod)
 
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
@@ -313,3 +310,29 @@ func validateAuditLogDataMap(data map[string]map[string]auditlogging.AuditLogDat
 
 	return nil
 }
+
+func refreshRuntimeMetrics(restConfig *rest.Config, logger logr.Logger, metrics metrics.Metrics) {
+	k8sClient, err := client.New(restConfig, client.Options{})
+	if err != nil {
+		setupLog.Error(err, "Unable to set up client for refreshing runtime CR metrics")
+		os.Exit(1)
+	}
+
+	err = infrastructuremanagerv1.AddToScheme(k8sClient.Scheme())
+	if err != nil {
+		setupLog.Error(err, "unable to set up client")
+		os.Exit(1)
+	}
+
+	logger.Info("Refreshing runtime CR metrics")
+	metrics.ResetRuntimeMetrics()
+	rl := infrastructuremanagerv1.RuntimeList{}
+	if err = k8sClient.List(context.Background(), &rl, &client.ListOptions{Namespace: "kcp-system"}); err != nil {
+		setupLog.Error(err, "error while listing unable to list runtimes")
+		os.Exit(1)
+	}
+
+	for _, rt := range rl.Items {
+		metrics.SetRuntimeStates(rt)
+	}
+}

internal/controller/metrics/metrics.go

@@ -12,6 +12,7 @@ import (
 
 const (
 	runtimeIDKeyName               = "runtimeId"
+	runtimeNameKeyName             = "runtimeName"
 	shootNameIDKeyName             = "shootName"
 	rotationDuration               = "rotationDuration"
 	expirationDuration             = "expirationDuration"
@@ -33,7 +34,7 @@ const (
 //go:generate mockery --name=Metrics
 type Metrics interface {
 	SetRuntimeStates(runtime v1.Runtime)
-	CleanUpRuntimeGauge(runtimeID string)
+	CleanUpRuntimeGauge(runtimeID, runtimeName string)
 	ResetRuntimeMetrics()
 	IncRuntimeFSMStopCounter()
 	SetGardenerClusterStates(cluster v1.GardenerCluster)
@@ -68,7 +69,7 @@ func NewMetrics() Metrics {
 				Subsystem: componentName,
 				Name:      RuntimeStateMetricName,
 				Help:      "Exposes current Status.state for Runtime CRs",
-			}, []string{runtimeIDKeyName, shootNameIDKeyName, provider, state, message}),
+			}, []string{runtimeIDKeyName, runtimeNameKeyName, shootNameIDKeyName, provider, state, message}),
 		runtimeFSMUnexpectedStopsCnt: prometheus.NewCounter(
 			prometheus.CounterOpts{
 				Name: RuntimeFSMStopMetricName,
@@ -90,14 +91,15 @@ func (m metricsImpl) SetRuntimeStates(runtime v1.Runtime) {
 			reason = runtime.Status.Conditions[size-1].Message
 		}
 
-		m.CleanUpRuntimeGauge(runtimeID)
-		m.runtimeStateGauge.WithLabelValues(runtimeID, runtime.Spec.Shoot.Name, runtime.Spec.Shoot.Provider.Type, string(runtime.Status.State), reason).Set(1)
+		m.CleanUpRuntimeGauge(runtimeID, runtime.Name)
+		m.runtimeStateGauge.WithLabelValues(runtimeID, runtime.Name, runtime.Spec.Shoot.Name, runtime.Spec.Shoot.Provider.Type, string(runtime.Status.State), reason).Set(1)
 	}
 }
 
-func (m metricsImpl) CleanUpRuntimeGauge(runtimeID string) {
+func (m metricsImpl) CleanUpRuntimeGauge(runtimeID, runtimeName string) {
 	m.runtimeStateGauge.DeletePartialMatch(prometheus.Labels{
-		runtimeIDKeyName: runtimeID,
+		runtimeIDKeyName:   runtimeID,
+		runtimeNameKeyName: runtimeName,
 	})
 }
 

internal/controller/runtime/fsm/runtime_fsm_apply_crb_test.go

@@ -25,7 +25,7 @@ var _ = Describe(`runtime_fsm_apply_crb`, Label("applyCRB"), func() {
 	withMockedMetrics := func() fakeFSMOpt {
 		m := &mocks.Metrics{}
 		m.On("SetRuntimeStates", mock.Anything).Return()
-		m.On("CleanUpRuntimeGauge", mock.Anything).Return()
+		m.On("CleanUpRuntimeGauge", mock.Anything, mock.Anything).Return()
 		m.On("IncRuntimeFSMStopCounter").Return()
 		return withMetrics(m)
 	}

internal/controller/runtime/fsm/runtime_fsm_create_kubeconfig_test.go

@@ -39,7 +39,7 @@ var _ = Describe("KIM sFnCreateKubeconfig", func() {
 	withMockedMetrics := func() fakeFSMOpt {
 		m := &mocks.Metrics{}
 		m.On("SetRuntimeStates", mock.Anything).Return()
-		m.On("CleanUpRuntimeGauge", mock.Anything).Return()
+		m.On("CleanUpRuntimeGauge", mock.Anything, mock.Anything).Return()
 		m.On("IncRuntimeFSMStopCounter").Return()
 		return withMetrics(m)
 	}

internal/controller/runtime/fsm/runtime_fsm_initialise.go

@@ -100,6 +100,6 @@ func removeFinalizerAndStop(ctx context.Context, m *fsm, s *systemState) (stateF
 	}
 
 	// remove from metrics
-	m.Metrics.CleanUpRuntimeGauge(runtimeID)
+	m.Metrics.CleanUpRuntimeGauge(runtimeID, s.instance.Name)
 	return stop()
 }

internal/controller/runtime/fsm/runtime_fsm_initialise_test.go

@@ -40,7 +40,7 @@ var _ = Describe("KIM sFnInitialise", func() {
 	withMockedMetrics := func() fakeFSMOpt {
 		m := &mocks.Metrics{}
 		m.On("SetRuntimeStates", mock.Anything).Return()
-		m.On("CleanUpRuntimeGauge", mock.Anything).Return()
+		m.On("CleanUpRuntimeGauge", mock.Anything, mock.Anything).Return()
 		m.On("IncRuntimeFSMStopCounter").Return()
 		return withMetrics(m)
 	}

internal/controller/runtime/fsm/runtime_fsm_persist_shoot_test.go

@@ -28,7 +28,7 @@ var _ = Describe("KIM sFnPersist", func() {
 	withMockedMetrics := func() fakeFSMOpt {
 		m := &mocks.Metrics{}
 		m.On("SetRuntimeStates", mock.Anything).Return()
-		m.On("CleanUpRuntimeGauge", mock.Anything).Return()
+		m.On("CleanUpRuntimeGauge", mock.Anything, mock.Anything).Return()
 		m.On("IncRuntimeFSMStopCounter").Return()
 		return withMetrics(m)
 	}

internal/controller/runtime/suite_test.go

@@ -119,7 +119,7 @@ var _ = BeforeSuite(func() {
 	mm := &mocks.Metrics{}
 	mm.On("SetRuntimeStates", mock.Anything).Return()
 	mm.On("IncRuntimeFSMStopCounter").Return()
-	mm.On("CleanUpRuntimeGauge", mock.Anything).Return()
+	mm.On("CleanUpRuntimeGauge", mock.Anything, mock.Anything).Return()
 
 	fsmCfg := fsm.RCCfg{
 		Finalizer:                   infrastructuremanagerv1.Finalizer,