Merge pull request #475 from koala7659/kim-add-multiple-workers

Set the number of concurrent reconciles for the Runtime controller to 25

cmd/main.go

@@ -46,6 +46,7 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/util/flowcontrol"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -66,11 +67,20 @@ func init() {
 	//+kubebuilder:scaffold:scheme
 }
 
-const defaultMinimalRotationTimeRatio = 0.6
-const defaultExpirationTime = 24 * time.Hour
-const defaultGardenerRequestTimeout = 60 * time.Second
-const defaultControlPlaneRequeueDuration = 10 * time.Second
-const defaultGardenerRequeueDuration = 15 * time.Second
+// Default values for the Runtime controller configuration
+const (
+	defaultControlPlaneRequeueDuration   = 10 * time.Second
+	defaultGardenerRequestTimeout        = 3 * time.Second
+	defaultGardenerRateLimiterQPS        = 5
+	defaultGardenerRateLimiterBurst      = 5
+	defaultMinimalRotationTimeRatio      = 0.6
+	defaultExpirationTime                = 24 * time.Hour
+	defaultGardenerReconciliationTimeout = 60 * time.Second
+	defaultGardenerRequeueDuration       = 15 * time.Second
+	defaultShootCreateRequeueDuration    = 60 * time.Second
+	defaultShootDeleteRequeueDuration    = 90 * time.Second
+	defaultShootReconcileRequeueDuration = 30 * time.Second
+)
 
 func main() {
 	var metricsAddr string
@@ -80,7 +90,10 @@ func main() {
 	var gardenerProjectName string
 	var minimalRotationTimeRatio float64
 	var expirationTime time.Duration
-	var gardenerRequestTimeout time.Duration
+	var gardenerCtrlReconciliationTimeout time.Duration
+	var runtimeCtrlGardenerRequestTimeout time.Duration
+	var runtimeCtrlGardenerRateLimiterQPS int
+	var runtimeCtrlGardenerRateLimiterBurst int
 	var converterConfigFilepath string
 	var shootSpecDumpEnabled bool
 	var auditLogMandatory bool
@@ -94,7 +107,10 @@ func main() {
 	flag.StringVar(&gardenerProjectName, "gardener-project-name", "gardener-project", "Name of the Gardener project")
 	flag.Float64Var(&minimalRotationTimeRatio, "minimal-rotation-time", defaultMinimalRotationTimeRatio, "The ratio determines what is the minimal time that needs to pass to rotate certificate.")
 	flag.DurationVar(&expirationTime, "kubeconfig-expiration-time", defaultExpirationTime, "Dynamic kubeconfig expiration time")
-	flag.DurationVar(&gardenerRequestTimeout, "gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for requests to Gardener")
+	flag.DurationVar(&gardenerCtrlReconciliationTimeout, "gardener-ctrl-reconcilation-timeout", defaultGardenerReconciliationTimeout, "Timeout duration for reconlication for Gardener Cluster Controller")
+	flag.DurationVar(&runtimeCtrlGardenerRequestTimeout, "gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for Gardener client for Runtime Controller")
+	flag.IntVar(&runtimeCtrlGardenerRateLimiterQPS, "gardener-ratelimiter-qps", defaultGardenerRateLimiterQPS, "Gardener client rate limiter QPS for Runtime Controller")
+	flag.IntVar(&runtimeCtrlGardenerRateLimiterBurst, "gardener-ratelimiter-burst", defaultGardenerRateLimiterBurst, "Gardener client rate limiter burst for Runtime Controller")
 	flag.StringVar(&converterConfigFilepath, "converter-config-filepath", "/converter-config/converter_config.json", "A file path to the gardener shoot converter configuration.")
 	flag.BoolVar(&shootSpecDumpEnabled, "shoot-spec-dump-enabled", false, "Feature flag to allow persisting specs of created shoots")
 	flag.BoolVar(&auditLogMandatory, "audit-log-mandatory", true, "Feature flag to enable strict mode for audit log configuration")
@@ -137,7 +153,7 @@ func main() {
 	}
 
 	gardenerNamespace := fmt.Sprintf("garden-%s", gardenerProjectName)
-	gardenerClient, shootClient, dynamicKubeconfigClient, err := initGardenerClients(gardenerKubeconfigPath, gardenerNamespace)
+	gardenerClient, shootClient, dynamicKubeconfigClient, err := initGardenerClients(gardenerKubeconfigPath, gardenerNamespace, runtimeCtrlGardenerRequestTimeout, runtimeCtrlGardenerRateLimiterQPS, runtimeCtrlGardenerRateLimiterBurst)
 
 	if err != nil {
 		setupLog.Error(err, "unable to initialize gardener clients", "controller", "GardenerCluster")
@@ -158,7 +174,7 @@ func main() {
 		logger,
 		rotationPeriod,
 		minimalRotationTimeRatio,
-		gardenerRequestTimeout,
+		gardenerCtrlReconciliationTimeout,
 		metrics,
 	).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "GardenerCluster")
@@ -188,14 +204,17 @@ func main() {
 	}
 
 	cfg := fsm.RCCfg{
-		GardenerRequeueDuration:     defaultGardenerRequeueDuration,
-		ControlPlaneRequeueDuration: defaultControlPlaneRequeueDuration,
-		Finalizer:                   infrastructuremanagerv1.Finalizer,
-		ShootNamesapace:             gardenerNamespace,
-		Config:                      config,
-		AuditLogMandatory:           auditLogMandatory,
-		Metrics:                     metrics,
-		AuditLogging:                auditLogDataMap,
+		GardenerRequeueDuration:       defaultGardenerRequeueDuration,
+		RequeueDurationShootCreate:    defaultShootCreateRequeueDuration,
+		RequeueDurationShootDelete:    defaultShootDeleteRequeueDuration,
+		RequeueDurationShootReconcile: defaultShootReconcileRequeueDuration,
+		ControlPlaneRequeueDuration:   defaultControlPlaneRequeueDuration,
+		Finalizer:                     infrastructuremanagerv1.Finalizer,
+		ShootNamesapace:               gardenerNamespace,
+		Config:                        config,
+		AuditLogMandatory:             auditLogMandatory,
+		Metrics:                       metrics,
+		AuditLogging:                  auditLogDataMap,
 	}
 	if shootSpecDumpEnabled {
 		cfg.PVCPath = "/testdata/kim"
@@ -234,12 +253,15 @@ func main() {
 	}
 }
 
-func initGardenerClients(kubeconfigPath string, namespace string) (client.Client, gardener_apis.ShootInterface, client.SubResourceClient, error) {
+func initGardenerClients(kubeconfigPath string, namespace string, timeout time.Duration, rlQPS, rlBurst int) (client.Client, gardener_apis.ShootInterface, client.SubResourceClient, error) {
 	restConfig, err := gardener.NewRestConfigFromFile(kubeconfigPath)
 	if err != nil {
 		return nil, nil, nil, err
 	}
 
+	restConfig.Timeout = timeout
+	restConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(float32(rlQPS), rlBurst)
+
 	gardenerClientSet, err := gardener_apis.NewForConfig(restConfig)
 	if err != nil {
 		return nil, nil, nil, err

internal/controller/kubeconfig/gardener_cluster_controller.go

@@ -34,6 +34,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	pkgctrl "sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
 
@@ -43,6 +44,7 @@ const (
 	clusterCRNameLabel                = "operator.kyma-project.io/cluster-name"
 
 	rotationPeriodRatio = 0.95
+	numberOfWorkers     = 25
 )
 
 // GardenerClusterController reconciles a GardenerCluster object
@@ -436,5 +438,6 @@ func (controller *GardenerClusterController) SetupWithManager(mgr ctrl.Manager)
 			predicate.AnnotationChangedPredicate{},
 			predicate.GenerationChangedPredicate{}),
 		)).
+		WithOptions(pkgctrl.Options{MaxConcurrentReconciles: numberOfWorkers}).
 		Complete(controller)
 }

internal/controller/runtime/fsm/runtime_fsm.go

@@ -26,14 +26,17 @@ type writerGetter = func(filePath string) (io.Writer, error)
 
 // runtime reconciler specific configuration
 type RCCfg struct {
-	GardenerRequeueDuration     time.Duration
-	ControlPlaneRequeueDuration time.Duration
-	Finalizer                   string
-	PVCPath                     string
-	ShootNamesapace             string
-	AuditLogMandatory           bool
-	Metrics                     metrics.Metrics
-	AuditLogging                auditlogs.Configuration
+	GardenerRequeueDuration       time.Duration
+	RequeueDurationShootCreate    time.Duration
+	RequeueDurationShootDelete    time.Duration
+	RequeueDurationShootReconcile time.Duration
+	ControlPlaneRequeueDuration   time.Duration
+	Finalizer                     string
+	PVCPath                       string
+	ShootNamesapace               string
+	AuditLogMandatory             bool
+	Metrics                       metrics.Metrics
+	AuditLogging                  auditlogs.Configuration
 	config.Config
 }
 

internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go

@@ -2,14 +2,15 @@ package fsm
 
 import (
 	"context"
+	"fmt"
 	"slices"
 
-	authenticationv1alpha1 "github.com/gardener/gardener/pkg/apis/authentication/v1alpha1"
-	gardener_api "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -27,9 +28,7 @@ var (
 )
 
 func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
-	// prepare subresource client to request admin kubeconfig
-	srscClient := m.ShootClient.SubResource("adminkubeconfig")
-	shootAdminClient, err := GetShootClient(ctx, srscClient, s.shoot)
+	shootAdminClient, err := GetShootClient(ctx, m.Client, s.instance)
 	if err != nil {
 		updateCRBApplyFailed(&s.instance)
 		return updateStatusAndStopWithError(err)
@@ -66,15 +65,15 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s
 }
 
 //nolint:gochecknoglobals
-var GetShootClient = func(ctx context.Context,
-	adminKubeconfigClient client.SubResourceClient, shoot *gardener_api.Shoot) (client.Client, error) {
-	// request for admin kubeconfig with low expiration timeout
-	var req authenticationv1alpha1.AdminKubeconfigRequest
-	if err := adminKubeconfigClient.Create(ctx, shoot, &req); err != nil {
+var GetShootClient = func(ctx context.Context, cnt client.Client, runtime imv1.Runtime) (client.Client, error) {
+	runtimeID := runtime.Labels[imv1.LabelKymaRuntimeID]
+
+	secret, err := getKubeconfigSecret(ctx, cnt, runtimeID, runtime.Namespace)
+	if err != nil {
 		return nil, err
 	}
 
-	restConfig, err := clientcmd.RESTConfigFromKubeConfig(req.Status.Kubeconfig)
+	restConfig, err := clientcmd.RESTConfigFromKubeConfig(secret.Data[kubeconfigSecretKey])
 	if err != nil {
 		return nil, err
 	}
@@ -87,6 +86,24 @@ var GetShootClient = func(ctx context.Context,
 	return shootClientWithAdmin, nil
 }
 
+func getKubeconfigSecret(ctx context.Context, cnt client.Client, runtimeID, namespace string) (corev1.Secret, error) {
+	secretName := fmt.Sprintf("kubeconfig-%s", runtimeID)
+
+	var kubeconfigSecret corev1.Secret
+	secretKey := types.NamespacedName{Name: secretName, Namespace: namespace}
+
+	err := cnt.Get(ctx, secretKey, &kubeconfigSecret)
+
+	if err != nil {
+		return corev1.Secret{}, err
+	}
+
+	if kubeconfigSecret.Data == nil {
+		return corev1.Secret{}, fmt.Errorf("kubeconfig secret `%s` does not contain kubeconfig data", kubeconfigSecret.Name)
+	}
+	return kubeconfigSecret, nil
+}
+
 func isRBACUserKindOneOf(names []string) func(rbacv1.Subject) bool {
 	return func(s rbacv1.Subject) bool {
 		return s.Kind == rbacv1.UserKind &&

internal/controller/runtime/fsm/runtime_fsm_apply_crb_test.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"time"
 
-	gardener_api "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
 	"github.com/kyma-project/infrastructure-manager/internal/controller/metrics/mocks"
 	. "github.com/onsi/ginkgo/v2"
@@ -154,8 +153,8 @@ var _ = Describe(`runtime_fsm_apply_crb`, Label("applyCRB"), func() {
 	defaultSetup := func(f *fsm) error {
 		GetShootClient = func(
 			_ context.Context,
-			_ client.SubResourceClient,
-			_ *gardener_api.Shoot) (client.Client, error) {
+			_ client.Client,
+			_ imv1.Runtime) (client.Client, error) {
 			return f.Client, nil
 		}
 		return nil
@@ -229,8 +228,8 @@ var _ = Describe(`runtime_fsm_apply_crb`, Label("applyCRB"), func() {
 			setup: func(f *fsm) error {
 				GetShootClient = func(
 					_ context.Context,
-					_ client.SubResourceClient,
-					_ *gardener_api.Shoot) (client.Client, error) {
+					_ client.Client,
+					_ imv1.Runtime) (client.Client, error) {
 					return nil, testErr
 				}
 				return nil

internal/controller/runtime/fsm/runtime_fsm_configure_oidc.go

@@ -72,8 +72,7 @@ func createDefaultOIDCConfig(defaultSharedIASTenant config.OidcProvider) gardene
 }
 
 func recreateOpenIDConnectResources(ctx context.Context, m *fsm, s *systemState) error {
-	srscClient := m.ShootClient.SubResource("adminkubeconfig")
-	shootAdminClient, shootClientError := GetShootClient(ctx, srscClient, s.shoot)
+	shootAdminClient, shootClientError := GetShootClient(ctx, m.Client, s.instance)
 	if shootClientError != nil {
 		return shootClientError
 	}

internal/controller/runtime/fsm/runtime_fsm_configure_oidc_test.go

@@ -63,7 +63,7 @@ func TestOidcState(t *testing.T) {
 		var fakeClient = fake.NewClientBuilder().
 			WithScheme(scheme).
 			Build()
-		fsm := &fsm{K8s: K8s{
+		testFsm := &fsm{K8s: K8s{
 			ShootClient: fakeClient,
 			Client:      fakeClient,
 		},
@@ -77,8 +77,8 @@ func TestOidcState(t *testing.T) {
 		}
 		GetShootClient = func(
 			_ context.Context,
-			_ client.SubResourceClient,
-			_ *gardener.Shoot) (client.Client, error) {
+			_ client.Client,
+			_ imv1.Runtime) (client.Client, error) {
 			return fakeClient, nil
 		}
 		// end of fake client setup
@@ -106,7 +106,7 @@ func TestOidcState(t *testing.T) {
 		}
 
 		// when
-		stateFn, _, _ := sFnConfigureOidc(ctx, fsm, systemState)
+		stateFn, _, _ := sFnConfigureOidc(ctx, testFsm, systemState)
 
 		// then
 		require.Contains(t, stateFn.name(), "sFnApplyClusterRoleBindings")
@@ -131,14 +131,14 @@ func TestOidcState(t *testing.T) {
 		var fakeClient = fake.NewClientBuilder().
 			WithScheme(scheme).
 			Build()
-		fsm := &fsm{K8s: K8s{
+		testFsm := &fsm{K8s: K8s{
 			ShootClient: fakeClient,
 			Client:      fakeClient,
 		}}
 		GetShootClient = func(
 			_ context.Context,
-			_ client.SubResourceClient,
-			_ *gardener.Shoot) (client.Client, error) {
+			_ client.Client,
+			_ imv1.Runtime) (client.Client, error) {
 			return fakeClient, nil
 		}
 		// end of fake client setup
@@ -171,7 +171,7 @@ func TestOidcState(t *testing.T) {
 		}
 
 		// when
-		stateFn, _, _ := sFnConfigureOidc(ctx, fsm, systemState)
+		stateFn, _, _ := sFnConfigureOidc(ctx, testFsm, systemState)
 
 		// then
 		require.Contains(t, stateFn.name(), "sFnApplyClusterRoleBindings")
@@ -198,14 +198,14 @@ func TestOidcState(t *testing.T) {
 		var fakeClient = fake.NewClientBuilder().
 			WithScheme(scheme).
 			Build()
-		fsm := &fsm{K8s: K8s{
+		testFSM := &fsm{K8s: K8s{
 			ShootClient: fakeClient,
 			Client:      fakeClient,
 		}}
 		GetShootClient = func(
 			_ context.Context,
-			_ client.SubResourceClient,
-			_ *gardener.Shoot) (client.Client, error) {
+			_ client.Client,
+			_ imv1.Runtime) (client.Client, error) {
 			return fakeClient, nil
 		}
 		// end of fake client setup
@@ -241,7 +241,7 @@ func TestOidcState(t *testing.T) {
 		}
 
 		// when
-		stateFn, _, _ := sFnConfigureOidc(ctx, fsm, systemState)
+		stateFn, _, _ := sFnConfigureOidc(ctx, testFSM, systemState)
 
 		// then
 		require.Contains(t, stateFn.name(), "sFnApplyClusterRoleBindings")

internal/controller/runtime/fsm/runtime_fsm_create_kubeconfig.go

@@ -12,6 +12,10 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
+const (
+	kubeconfigSecretKey = "config"
+)
+
 func sFnCreateKubeconfig(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
 	m.log.Info("Create Gardener Cluster CR state")
 
@@ -110,7 +114,7 @@ func makeGardenerClusterForRuntime(runtime imv1.Runtime, shoot *gardener.Shoot)
 				Secret: imv1.Secret{
 					Name:      fmt.Sprintf("kubeconfig-%s", runtime.Labels[imv1.LabelKymaRuntimeID]),
 					Namespace: runtime.Namespace,
-					Key:       "config",
+					Key:       kubeconfigSecretKey,
 				},
 			},
 		},