Adding metrics for Runtime CR state

cmd/main.go

@@ -65,8 +65,9 @@ func init() {
 
 const defaultMinimalRotationTimeRatio = 0.6
 const defaultExpirationTime = 24 * time.Hour
-const defaultRuntimeReconcilerEnabled = true
 const defaultGardenerRequestTimeout = 60 * time.Second
+const defaultControlPlaneRequeueDuration = 10 * time.Second
+const defaultGardenerRequeueDuration = 15 * time.Second
 
 func main() {
 	var metricsAddr string
@@ -77,7 +78,6 @@ func main() {
 	var minimalRotationTimeRatio float64
 	var expirationTime time.Duration
 	var gardenerRequestTimeout time.Duration
-	var enableRuntimeReconciler bool
 	var converterConfigFilepath string
 	var shootSpecDumpEnabled bool
 	var auditLogMandatory bool
@@ -92,7 +92,6 @@ func main() {
 	flag.Float64Var(&minimalRotationTimeRatio, "minimal-rotation-time", defaultMinimalRotationTimeRatio, "The ratio determines what is the minimal time that needs to pass to rotate certificate.")
 	flag.DurationVar(&expirationTime, "kubeconfig-expiration-time", defaultExpirationTime, "Dynamic kubeconfig expiration time")
 	flag.DurationVar(&gardenerRequestTimeout, "gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for requests to Gardener")
-	flag.BoolVar(&enableRuntimeReconciler, "runtime-reconciler-enabled", defaultRuntimeReconcilerEnabled, "Feature flag for all runtime reconciler functionalities")
 	flag.StringVar(&converterConfigFilepath, "converter-config-filepath", "/converter-config/converter_config.json", "A file path to the gardener shoot converter configuration.")
 	flag.BoolVar(&shootSpecDumpEnabled, "shoot-spec-dump-enabled", false, "Feature flag to allow persisting specs of created shoots")
 	flag.BoolVar(&auditLogMandatory, "audit-log-mandatory", true, "Feature flag to enable strict mode for audit log configuration")
@@ -165,46 +164,48 @@ func main() {
 	getReader := func() (io.Reader, error) {
 		return os.Open(converterConfigFilepath)
 	}
-	var converterConfig config.Config
-	if err = converterConfig.Load(getReader); err != nil {
+	var config config.Config
+	if err = config.Load(getReader); err != nil {
 		setupLog.Error(err, "unable to load converter configuration")
 		os.Exit(1)
 	}
 
 	validate := validator.New(validator.WithRequiredStructEnabled())
-	if err = validate.Struct(converterConfig); err != nil {
+	if err = validate.Struct(config); err != nil {
 		setupLog.Error(err, "invalid converter configuration")
 		os.Exit(1)
 	}
 
-	err = validateAuditLogConfiguration(converterConfig.ConverterConfig.AuditLog.TenantConfigPath)
+	err = validateAuditLogConfiguration(config.ConverterConfig.AuditLog.TenantConfigPath)
 	if err != nil {
 		setupLog.Error(err, "invalid Audit Log configuration")
 		os.Exit(1)
 	}
 
 	cfg := fsm.RCCfg{
-		Finalizer:         infrastructuremanagerv1.Finalizer,
-		ShootNamesapace:   gardenerNamespace,
-		Config:            converterConfig,
-		AuditLogMandatory: auditLogMandatory,
+		GardenerRequeueDuration:     defaultGardenerRequeueDuration,
+		ControlPlaneRequeueDuration: defaultControlPlaneRequeueDuration,
+		Finalizer:                   infrastructuremanagerv1.Finalizer,
+		ShootNamesapace:             gardenerNamespace,
+		Config:                      config,
+		AuditLogMandatory:           auditLogMandatory,
+		Metrics:                     metrics,
+		AuditLogging:                auditlogging.NewAuditLogging(config.ConverterConfig.AuditLog.TenantConfigPath, config.ConverterConfig.AuditLog.PolicyConfigMapName, gardenerClient),
 	}
 	if shootSpecDumpEnabled {
 		cfg.PVCPath = "/testdata/kim"
 	}
 
-	if enableRuntimeReconciler {
-		runtimeReconciler := runtime_controller.NewRuntimeReconciler(
-			mgr,
-			gardenerClient,
-			logger,
-			cfg,
-		)
-
-		if err = runtimeReconciler.SetupWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to setup controller with Manager", "controller", "Runtime")
-			os.Exit(1)
-		}
+	runtimeReconciler := runtime_controller.NewRuntimeReconciler(
+		mgr,
+		gardenerClient,
+		logger,
+		cfg,
+	)
+
+	if err = runtimeReconciler.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to setup controller with Manager", "controller", "Runtime")
+		os.Exit(1)
 	}
 
 	//+kubebuilder:scaffold:builder
@@ -218,7 +219,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	setupLog.Info("Starting Manager", "kubeconfigExpirationTime", expirationTime, "kubeconfigRotationPeriod", rotationPeriod, "enableRuntimeReconciler", enableRuntimeReconciler)
+	setupLog.Info("Starting Manager", "kubeconfigExpirationTime", expirationTime, "kubeconfigRotationPeriod", rotationPeriod)
 
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")

config/manager/converter_config.yaml

@@ -22,6 +22,7 @@ data:
         "projectName": "kyma-dev"
       },
       "machineImage": {
+        "defaultName": "gardenlinux",
         "defaultVersion": "1312.3.0"
       }
     }

internal/auditlogging/auditlogging.go

@@ -25,6 +25,7 @@ var ErrMissingMapping = errors.New("missing mapping for selected region in provi
 //go:generate mockery --name=AuditLogging
 type AuditLogging interface {
 	Enable(ctx context.Context, shoot *gardener.Shoot) (bool, error)
+	UpdateShootClient(client client.Client)
 }
 
 //go:generate mockery --name=AuditLogConfigurator
@@ -34,6 +35,7 @@ type AuditLogConfigurator interface {
 	GetSeedObj(ctx context.Context, seedKey types.NamespacedName) (gardener.Seed, error)
 	UpdateShoot(ctx context.Context, shoot *gardener.Shoot) error
 	GetConfigFromFile() (data map[string]map[string]AuditLogData, err error)
+	UpdateClient(client client.Client)
 }
 
 type AuditLog struct {
@@ -95,6 +97,14 @@ func (a *auditLogConfig) GetSeedObj(ctx context.Context, seedKey types.Namespace
 	return seed, nil
 }
 
+func (a *auditLogConfig) UpdateClient(client client.Client) {
+	a.client = client
+}
+
+func (al *AuditLog) UpdateShootClient(client client.Client) {
+	al.UpdateClient(client)
+}
+
 func (al *AuditLog) Enable(ctx context.Context, shoot *gardener.Shoot) (bool, error) {
 	seedName := getSeedName(*shoot)
 

internal/controller/metrics/metrics.go

@@ -19,20 +19,37 @@ const (
 	RuntimeIDLabel                 = "kyma-project.io/runtime-id"
 	ShootNameLabel                 = "kyma-project.io/shoot-name"
 	GardenerClusterStateMetricName = "im_gardener_clusters_state"
+	RuntimeStateMetricName         = "im_runtime_state"
+	RuntimeFSMStopMetricName       = "unexpected_stops_total"
+	provider                       = "provider"
 	state                          = "state"
 	reason                         = "reason"
+	message                        = "message"
 	KubeconfigExpirationMetricName = "im_kubeconfig_expiration"
 	expires                        = "expires"
 	lastSyncAnnotation             = "operator.kyma-project.io/last-sync"
 )
 
-type Metrics struct {
+//go:generate mockery --name=Metrics
+type Metrics interface {
+	SetRuntimeStates(runtime v1.Runtime)
+	CleanUpRuntimeGauge(runtimeID string)
+	IncRuntimeFSMStopCounter()
+	SetGardenerClusterStates(cluster v1.GardenerCluster)
+	CleanUpGardenerClusterGauge(runtimeID string)
+	CleanUpKubeconfigExpiration(runtimeID string)
+	SetKubeconfigExpiration(secret corev1.Secret, rotationPeriod time.Duration, minimalRotationTimeRatio float64)
+}
+
+type metricsImpl struct {
 	gardenerClustersStateGaugeVec *prometheus.GaugeVec
 	kubeconfigExpirationGauge     *prometheus.GaugeVec
+	runtimeStateGauge             *prometheus.GaugeVec
+	runtimeFSMUnexpectedStopsCnt  prometheus.Counter
 }
 
 func NewMetrics() Metrics {
-	m := Metrics{
+	m := &metricsImpl{
 		gardenerClustersStateGaugeVec: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Subsystem: componentName,
@@ -45,12 +62,49 @@ func NewMetrics() Metrics {
 				Name:      KubeconfigExpirationMetricName,
 				Help:      "Exposes current kubeconfig expiration value in epoch timestamp value format",
 			}, []string{runtimeIDKeyName, shootNameIDKeyName, expires, rotationDuration, expirationDuration}),
+		runtimeStateGauge: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Subsystem: componentName,
+				Name:      RuntimeStateMetricName,
+				Help:      "Exposes current Status.state for Runtime CRs",
+			}, []string{runtimeIDKeyName, shootNameIDKeyName, provider, state, message}),
+		runtimeFSMUnexpectedStopsCnt: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: RuntimeFSMStopMetricName,
+				Help: "Exposes the number of unexpected state machine stop events",
+			}),
 	}
-	ctrlMetrics.Registry.MustRegister(m.gardenerClustersStateGaugeVec, m.kubeconfigExpirationGauge)
+	ctrlMetrics.Registry.MustRegister(m.gardenerClustersStateGaugeVec, m.kubeconfigExpirationGauge, m.runtimeStateGauge, m.runtimeFSMUnexpectedStopsCnt)
 	return m
 }
 
-func (m Metrics) SetGardenerClusterStates(cluster v1.GardenerCluster) {
+func (m metricsImpl) SetRuntimeStates(runtime v1.Runtime) {
+	runtimeID := runtime.GetLabels()[RuntimeIDLabel]
+
+	if runtimeID != "" {
+		size := len(runtime.Status.Conditions)
+
+		var reason = "No value"
+		if size > 0 {
+			reason = runtime.Status.Conditions[size-1].Message
+		}
+
+		m.CleanUpRuntimeGauge(runtimeID)
+		m.runtimeStateGauge.WithLabelValues(runtimeID, runtime.Spec.Shoot.Name, runtime.Spec.Shoot.Provider.Type, string(runtime.Status.State), reason).Set(1)
+	}
+}
+
+func (m metricsImpl) CleanUpRuntimeGauge(runtimeID string) {
+	m.runtimeStateGauge.DeletePartialMatch(prometheus.Labels{
+		runtimeIDKeyName: runtimeID,
+	})
+}
+
+func (m metricsImpl) IncRuntimeFSMStopCounter() {
+	m.runtimeFSMUnexpectedStopsCnt.Inc()
+}
+
+func (m metricsImpl) SetGardenerClusterStates(cluster v1.GardenerCluster) {
 	var runtimeID = cluster.GetLabels()[RuntimeIDLabel]
 	var shootName = cluster.GetLabels()[ShootNameLabel]
 
@@ -65,13 +119,13 @@ func (m Metrics) SetGardenerClusterStates(cluster v1.GardenerCluster) {
 	}
 }
 
-func (m Metrics) CleanUpGardenerClusterGauge(runtimeID string) {
+func (m metricsImpl) CleanUpGardenerClusterGauge(runtimeID string) {
 	m.gardenerClustersStateGaugeVec.DeletePartialMatch(prometheus.Labels{
 		runtimeIDKeyName: runtimeID,
 	})
 }
 
-func (m Metrics) CleanUpKubeconfigExpiration(runtimeID string) {
+func (m metricsImpl) CleanUpKubeconfigExpiration(runtimeID string) {
 	m.kubeconfigExpirationGauge.DeletePartialMatch(prometheus.Labels{
 		runtimeIDKeyName: runtimeID,
 	})
@@ -81,7 +135,7 @@ func computeExpirationInSeconds(rotationPeriod time.Duration, minimalRotationTim
 	return rotationPeriod.Seconds() / minimalRotationTimeRatio
 }
 
-func (m Metrics) SetKubeconfigExpiration(secret corev1.Secret, rotationPeriod time.Duration, minimalRotationTimeRatio float64) {
+func (m metricsImpl) SetKubeconfigExpiration(secret corev1.Secret, rotationPeriod time.Duration, minimalRotationTimeRatio float64) {
 	var runtimeID = secret.GetLabels()[RuntimeIDLabel]
 	var shootName = secret.GetLabels()[ShootNameLabel]