Merge branch 'main' into migrator-prevent-runtime-recreation

cmd/main.go

@@ -40,13 +40,17 @@ import (
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig"
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
 	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	k8slabels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/util/flowcontrol"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -66,11 +70,20 @@ func init() {
 	//+kubebuilder:scaffold:scheme
 }
 
-const defaultMinimalRotationTimeRatio = 0.6
-const defaultExpirationTime = 24 * time.Hour
-const defaultGardenerRequestTimeout = 60 * time.Second
-const defaultControlPlaneRequeueDuration = 10 * time.Second
-const defaultGardenerRequeueDuration = 15 * time.Second
+// Default values for the Runtime controller configuration
+const (
+	defaultControlPlaneRequeueDuration   = 10 * time.Second
+	defaultGardenerRequestTimeout        = 3 * time.Second
+	defaultGardenerRateLimiterQPS        = 5
+	defaultGardenerRateLimiterBurst      = 5
+	defaultMinimalRotationTimeRatio      = 0.6
+	defaultExpirationTime                = 24 * time.Hour
+	defaultGardenerReconciliationTimeout = 60 * time.Second
+	defaultGardenerRequeueDuration       = 15 * time.Second
+	defaultShootCreateRequeueDuration    = 60 * time.Second
+	defaultShootDeleteRequeueDuration    = 90 * time.Second
+	defaultShootReconcileRequeueDuration = 30 * time.Second
+)
 
 func main() {
 	var metricsAddr string
@@ -80,7 +93,10 @@ func main() {
 	var gardenerProjectName string
 	var minimalRotationTimeRatio float64
 	var expirationTime time.Duration
-	var gardenerRequestTimeout time.Duration
+	var gardenerCtrlReconciliationTimeout time.Duration
+	var runtimeCtrlGardenerRequestTimeout time.Duration
+	var runtimeCtrlGardenerRateLimiterQPS int
+	var runtimeCtrlGardenerRateLimiterBurst int
 	var converterConfigFilepath string
 	var shootSpecDumpEnabled bool
 	var auditLogMandatory bool
@@ -94,14 +110,15 @@ func main() {
 	flag.StringVar(&gardenerProjectName, "gardener-project-name", "gardener-project", "Name of the Gardener project")
 	flag.Float64Var(&minimalRotationTimeRatio, "minimal-rotation-time", defaultMinimalRotationTimeRatio, "The ratio determines what is the minimal time that needs to pass to rotate certificate.")
 	flag.DurationVar(&expirationTime, "kubeconfig-expiration-time", defaultExpirationTime, "Dynamic kubeconfig expiration time")
-	flag.DurationVar(&gardenerRequestTimeout, "gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for requests to Gardener")
+	flag.DurationVar(&gardenerCtrlReconciliationTimeout, "gardener-ctrl-reconcilation-timeout", defaultGardenerReconciliationTimeout, "Timeout duration for reconlication for Gardener Cluster Controller")
+	flag.DurationVar(&runtimeCtrlGardenerRequestTimeout, "gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for Gardener client for Runtime Controller")
+	flag.IntVar(&runtimeCtrlGardenerRateLimiterQPS, "gardener-ratelimiter-qps", defaultGardenerRateLimiterQPS, "Gardener client rate limiter QPS for Runtime Controller")
+	flag.IntVar(&runtimeCtrlGardenerRateLimiterBurst, "gardener-ratelimiter-burst", defaultGardenerRateLimiterBurst, "Gardener client rate limiter burst for Runtime Controller")
 	flag.StringVar(&converterConfigFilepath, "converter-config-filepath", "/converter-config/converter_config.json", "A file path to the gardener shoot converter configuration.")
 	flag.BoolVar(&shootSpecDumpEnabled, "shoot-spec-dump-enabled", false, "Feature flag to allow persisting specs of created shoots")
 	flag.BoolVar(&auditLogMandatory, "audit-log-mandatory", true, "Feature flag to enable strict mode for audit log configuration")
 
-	opts := zap.Options{
-		Development: true,
-	}
+	opts := zap.Options{}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
@@ -119,6 +136,7 @@ func main() {
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "f1c68560.kyma-project.io",
+		Cache:                  restrictWatchedNamespace(),
 		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
 		// when the Manager ends. This requires the binary to immediately end when the
 		// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
@@ -137,7 +155,7 @@ func main() {
 	}
 
 	gardenerNamespace := fmt.Sprintf("garden-%s", gardenerProjectName)
-	gardenerClient, shootClient, dynamicKubeconfigClient, err := initGardenerClients(gardenerKubeconfigPath, gardenerNamespace)
+	gardenerClient, shootClient, dynamicKubeconfigClient, err := initGardenerClients(gardenerKubeconfigPath, gardenerNamespace, runtimeCtrlGardenerRequestTimeout, runtimeCtrlGardenerRateLimiterQPS, runtimeCtrlGardenerRateLimiterBurst)
 
 	if err != nil {
 		setupLog.Error(err, "unable to initialize gardener clients", "controller", "GardenerCluster")
@@ -158,7 +176,7 @@ func main() {
 		logger,
 		rotationPeriod,
 		minimalRotationTimeRatio,
-		gardenerRequestTimeout,
+		gardenerCtrlReconciliationTimeout,
 		metrics,
 	).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "GardenerCluster")
@@ -188,14 +206,17 @@ func main() {
 	}
 
 	cfg := fsm.RCCfg{
-		GardenerRequeueDuration:     defaultGardenerRequeueDuration,
-		ControlPlaneRequeueDuration: defaultControlPlaneRequeueDuration,
-		Finalizer:                   infrastructuremanagerv1.Finalizer,
-		ShootNamesapace:             gardenerNamespace,
-		Config:                      config,
-		AuditLogMandatory:           auditLogMandatory,
-		Metrics:                     metrics,
-		AuditLogging:                auditLogDataMap,
+		GardenerRequeueDuration:       defaultGardenerRequeueDuration,
+		RequeueDurationShootCreate:    defaultShootCreateRequeueDuration,
+		RequeueDurationShootDelete:    defaultShootDeleteRequeueDuration,
+		RequeueDurationShootReconcile: defaultShootReconcileRequeueDuration,
+		ControlPlaneRequeueDuration:   defaultControlPlaneRequeueDuration,
+		Finalizer:                     infrastructuremanagerv1.Finalizer,
+		ShootNamesapace:               gardenerNamespace,
+		Config:                        config,
+		AuditLogMandatory:             auditLogMandatory,
+		Metrics:                       metrics,
+		AuditLogging:                  auditLogDataMap,
 	}
 	if shootSpecDumpEnabled {
 		cfg.PVCPath = "/testdata/kim"
@@ -234,12 +255,15 @@ func main() {
 	}
 }
 
-func initGardenerClients(kubeconfigPath string, namespace string) (client.Client, gardener_apis.ShootInterface, client.SubResourceClient, error) {
+func initGardenerClients(kubeconfigPath string, namespace string, timeout time.Duration, rlQPS, rlBurst int) (client.Client, gardener_apis.ShootInterface, client.SubResourceClient, error) {
 	restConfig, err := gardener.NewRestConfigFromFile(kubeconfigPath)
 	if err != nil {
 		return nil, nil, nil, err
 	}
 
+	restConfig.Timeout = timeout
+	restConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(float32(rlQPS), rlBurst)
+
 	gardenerClientSet, err := gardener_apis.NewForConfig(restConfig)
 	if err != nil {
 		return nil, nil, nil, err
@@ -314,3 +338,26 @@ func refreshRuntimeMetrics(restConfig *rest.Config, logger logr.Logger, metrics
 		metrics.SetRuntimeStates(rt)
 	}
 }
+
+func restrictWatchedNamespace() cache.Options {
+	return cache.Options{
+		ByObject: map[client.Object]cache.ByObject{
+			&corev1.Secret{}: {
+				Label: k8slabels.Everything(),
+				Namespaces: map[string]cache.Config{
+					"kcp-system": {},
+				},
+			},
+			&infrastructuremanagerv1.Runtime{}: {
+				Namespaces: map[string]cache.Config{
+					"kcp-system": {},
+				},
+			},
+			&infrastructuremanagerv1.GardenerCluster{}: {
+				Namespaces: map[string]cache.Config{
+					"kcp-system": {},
+				},
+			},
+		},
+	}
+}

hack/runtime-migrator/cmd/main.go

@@ -11,18 +11,17 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-playground/validator/v10"
-	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
-	v12 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
-
 	"github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	gardener_types "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1"
+	"github.com/go-playground/validator/v10"
 	"github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/config"
 	kimConfig "github.com/kyma-project/infrastructure-manager/pkg/config"
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener"
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig"
+	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
 	"github.com/pkg/errors"
+	v12 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"

hack/runtime-migrator/cmd/migration.go

@@ -16,6 +16,7 @@ import (
 	"github.com/kyma-project/infrastructure-manager/pkg/config"
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig"
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
+	"github.com/pkg/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )

hack/runtime-migrator/internal/config/config.go

@@ -3,11 +3,12 @@ package config
 import (
 	"flag"
 	"fmt"
+	"log"
+
 	v1 "github.com/kyma-project/infrastructure-manager/api/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/clientcmd"
-	"log"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 

hack/runtime-migrator/internal/migration/output.go

@@ -3,12 +3,13 @@ package migration
 import (
 	"encoding/json"
 	"fmt"
-	v1 "github.com/kyma-project/infrastructure-manager/api/v1"
-	"github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/runtime"
 	"os"
 	"path"
-	"sigs.k8s.io/yaml"
 	"time"
+
+	v1 "github.com/kyma-project/infrastructure-manager/api/v1"
+	"github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/runtime"
+	"sigs.k8s.io/yaml"
 )
 
 type OutputWriter struct {

hack/runtime-migrator/internal/runtime/migrator.go

@@ -3,6 +3,7 @@ package runtime
 import (
 	"context"
 	"fmt"
+
 	"github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	v1 "github.com/kyma-project/infrastructure-manager/api/v1"
 	migrator "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/config"

hack/runtime-migrator/internal/runtime/verifier.go

@@ -1,14 +1,15 @@
 package runtime
 
 import (
+	"slices"
+
 	"github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	v1 "github.com/kyma-project/infrastructure-manager/api/v1"
 	"github.com/kyma-project/infrastructure-manager/hack/shoot-comparator/pkg/shoot"
 	"github.com/kyma-project/infrastructure-manager/pkg/config"
 	gardener_shoot "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot"
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
 	"k8s.io/utils/ptr"
-	"slices"
 )
 
 type Verifier struct {

internal/controller/kubeconfig/gardener_cluster_controller.go

@@ -34,6 +34,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	pkgctrl "sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
 
@@ -43,6 +44,7 @@ const (
 	clusterCRNameLabel                = "operator.kyma-project.io/cluster-name"
 
 	rotationPeriodRatio = 0.95
+	numberOfWorkers     = 25
 )
 
 // GardenerClusterController reconciles a GardenerCluster object
@@ -436,5 +438,6 @@ func (controller *GardenerClusterController) SetupWithManager(mgr ctrl.Manager)
 			predicate.AnnotationChangedPredicate{},
 			predicate.GenerationChangedPredicate{}),
 		)).
+		WithOptions(pkgctrl.Options{MaxConcurrentReconciles: numberOfWorkers}).
 		Complete(controller)
 }

internal/controller/runtime/fsm/runtime_fsm.go

@@ -26,14 +26,17 @@ type writerGetter = func(filePath string) (io.Writer, error)
 
 // runtime reconciler specific configuration
 type RCCfg struct {
-	GardenerRequeueDuration     time.Duration
-	ControlPlaneRequeueDuration time.Duration
-	Finalizer                   string
-	PVCPath                     string
-	ShootNamesapace             string
-	AuditLogMandatory           bool
-	Metrics                     metrics.Metrics
-	AuditLogging                auditlogs.Configuration
+	GardenerRequeueDuration       time.Duration
+	RequeueDurationShootCreate    time.Duration
+	RequeueDurationShootDelete    time.Duration
+	RequeueDurationShootReconcile time.Duration
+	ControlPlaneRequeueDuration   time.Duration
+	Finalizer                     string
+	PVCPath                       string
+	ShootNamesapace               string
+	AuditLogMandatory             bool
+	Metrics                       metrics.Metrics
+	AuditLogging                  auditlogs.Configuration
 	config.Config
 }
 

internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go

@@ -2,14 +2,15 @@ package fsm
 
 import (
 	"context"
+	"fmt"
 	"slices"
 
-	authenticationv1alpha1 "github.com/gardener/gardener/pkg/apis/authentication/v1alpha1"
-	gardener_api "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -27,9 +28,7 @@ var (
 )
 
 func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
-	// prepare subresource client to request admin kubeconfig
-	srscClient := m.ShootClient.SubResource("adminkubeconfig")
-	shootAdminClient, err := GetShootClient(ctx, srscClient, s.shoot)
+	shootAdminClient, err := GetShootClient(ctx, m.Client, s.instance)
 	if err != nil {
 		updateCRBApplyFailed(&s.instance)
 		return updateStatusAndStopWithError(err)
@@ -66,15 +65,15 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s
 }
 
 //nolint:gochecknoglobals
-var GetShootClient = func(ctx context.Context,
-	adminKubeconfigClient client.SubResourceClient, shoot *gardener_api.Shoot) (client.Client, error) {
-	// request for admin kubeconfig with low expiration timeout
-	var req authenticationv1alpha1.AdminKubeconfigRequest
-	if err := adminKubeconfigClient.Create(ctx, shoot, &req); err != nil {
+var GetShootClient = func(ctx context.Context, cnt client.Client, runtime imv1.Runtime) (client.Client, error) {
+	runtimeID := runtime.Labels[imv1.LabelKymaRuntimeID]
+
+	secret, err := getKubeconfigSecret(ctx, cnt, runtimeID, runtime.Namespace)
+	if err != nil {
 		return nil, err
 	}
 
-	restConfig, err := clientcmd.RESTConfigFromKubeConfig(req.Status.Kubeconfig)
+	restConfig, err := clientcmd.RESTConfigFromKubeConfig(secret.Data[kubeconfigSecretKey])
 	if err != nil {
 		return nil, err
 	}
@@ -87,6 +86,24 @@ var GetShootClient = func(ctx context.Context,
 	return shootClientWithAdmin, nil
 }
 
+func getKubeconfigSecret(ctx context.Context, cnt client.Client, runtimeID, namespace string) (corev1.Secret, error) {
+	secretName := fmt.Sprintf("kubeconfig-%s", runtimeID)
+
+	var kubeconfigSecret corev1.Secret
+	secretKey := types.NamespacedName{Name: secretName, Namespace: namespace}
+
+	err := cnt.Get(ctx, secretKey, &kubeconfigSecret)
+
+	if err != nil {
+		return corev1.Secret{}, err
+	}
+
+	if kubeconfigSecret.Data == nil {
+		return corev1.Secret{}, fmt.Errorf("kubeconfig secret `%s` does not contain kubeconfig data", kubeconfigSecret.Name)
+	}
+	return kubeconfigSecret, nil
+}
+
 func isRBACUserKindOneOf(names []string) func(rbacv1.Subject) bool {
 	return func(s rbacv1.Subject) bool {
 		return s.Kind == rbacv1.UserKind &&