From 93358bddd0c303fd3ca1f684af6a5e8b0fc61e89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Thu, 31 Oct 2024 18:07:14 +0100 Subject: [PATCH 01/30] Set the max number of concurrent reconciles for the runtime controller to 10 --- internal/controller/runtime/runtime_controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index d0293824..6aba9a78 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -19,6 +19,7 @@ package runtime import ( "context" "fmt" + "sigs.k8s.io/controller-runtime/pkg/controller" "github.com/go-logr/logr" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" @@ -91,6 +92,7 @@ func (r *RuntimeReconciler) UpdateShootClient(client client.Client) { func (r *RuntimeReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&imv1.Runtime{}). + WithOptions(controller.Options{MaxConcurrentReconciles: 10}). WithEventFilter(predicate.Or( predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}, From 986c9a1cdbd4badc6d25c1c2d46cdf41d69f41aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Fri, 15 Nov 2024 12:41:08 +0100 Subject: [PATCH 02/30] Change number of worker to 25 --- internal/controller/runtime/runtime_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index e065d650..5a3dcac9 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -87,7 +87,7 @@ func NewRuntimeReconciler(mgr ctrl.Manager, shootClient client.Client, logger lo func (r *RuntimeReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&imv1.Runtime{}). - WithOptions(controller.Options{MaxConcurrentReconciles: 10}). + WithOptions(controller.Options{MaxConcurrentReconciles: 25}). WithEventFilter(predicate.Or( predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}, From 5067486e8041f29ab99cbb38783de7adbecf7667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Tue, 19 Nov 2024 18:12:39 +0100 Subject: [PATCH 03/30] Reading shoot kubeconfing data from secret --- .../runtime_fsm_apply_clusterrolebindings.go | 41 ++++++++++++++----- .../runtime/fsm/runtime_fsm_apply_crb_test.go | 9 ++-- .../runtime/fsm/runtime_fsm_configure_oidc.go | 4 +- .../fsm/runtime_fsm_configure_oidc_test.go | 24 +++++------ .../fsm/runtime_fsm_create_kubeconfig.go | 6 ++- internal/controller/runtime/suite_test.go | 10 ++--- 6 files changed, 58 insertions(+), 36 deletions(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go index ce92b36d..e0407345 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go +++ b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go @@ -2,10 +2,12 @@ package fsm import ( "context" + "fmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "slices" + "time" - authenticationv1alpha1 "github.com/gardener/gardener/pkg/apis/authentication/v1alpha1" - gardener_api "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -24,9 +26,7 @@ var ( ) func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { - // prepare subresource client to request admin kubeconfig - srscClient := m.ShootClient.SubResource("adminkubeconfig") - shootAdminClient, err := GetShootClient(ctx, srscClient, s.shoot) + shootAdminClient, err := GetShootClient(ctx, m.Client, s.instance) if err != nil { updateCRBApplyFailed(&s.instance) return updateStatusAndStopWithError(err) @@ -61,19 +61,20 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s } //nolint:gochecknoglobals -var GetShootClient = func(ctx context.Context, - adminKubeconfigClient client.SubResourceClient, shoot *gardener_api.Shoot) (client.Client, error) { - // request for admin kubeconfig with low expiration timeout - var req authenticationv1alpha1.AdminKubeconfigRequest - if err := adminKubeconfigClient.Create(ctx, shoot, &req); err != nil { +var GetShootClient = func(ctx context.Context, cnt client.Client, runtime imv1.Runtime) (client.Client, error) { + runtimeID := runtime.Labels[imv1.LabelKymaRuntimeID] + + secret, err := getKubeconfigSecret(ctx, cnt, runtimeID, runtime.Namespace) + if err != nil { return nil, err } - restConfig, err := clientcmd.RESTConfigFromKubeConfig(req.Status.Kubeconfig) + restConfig, err := clientcmd.RESTConfigFromKubeConfig(secret.Data[kubeconfigSecretKey]) if err != nil { return nil, err } + restConfig.Timeout = 250 * time.Millisecond shootClientWithAdmin, err := client.New(restConfig, client.Options{}) if err != nil { return nil, err @@ -82,6 +83,24 @@ var GetShootClient = func(ctx context.Context, return shootClientWithAdmin, nil } +func getKubeconfigSecret(ctx context.Context, cnt client.Client, runtimeID, namespace string) (corev1.Secret, error) { + secretName := fmt.Sprintf("kubeconfig-%s", runtimeID) + + var kubeconfigSecret corev1.Secret + secretKey := types.NamespacedName{Name: secretName, Namespace: namespace} + + err := cnt.Get(ctx, secretKey, &kubeconfigSecret) + + if err != nil { + return corev1.Secret{}, err + } + + if kubeconfigSecret.Data == nil { + return corev1.Secret{}, fmt.Errorf("kubeconfig secret `%s` does not contain kubeconfig data", kubeconfigSecret.Name) + } + return kubeconfigSecret, nil +} + func isRBACUserKindOneOf(names []string) func(rbacv1.Subject) bool { return func(s rbacv1.Subject) bool { return s.Kind == rbacv1.UserKind && diff --git a/internal/controller/runtime/fsm/runtime_fsm_apply_crb_test.go b/internal/controller/runtime/fsm/runtime_fsm_apply_crb_test.go index c8b7eb38..7b4af3a7 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_apply_crb_test.go +++ b/internal/controller/runtime/fsm/runtime_fsm_apply_crb_test.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - gardener_api "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/infrastructure-manager/internal/controller/metrics/mocks" . "github.com/onsi/ginkgo/v2" @@ -113,8 +112,8 @@ var _ = Describe(`runtime_fsm_apply_crb`, Label("applyCRB"), func() { defaultSetup := func(f *fsm) error { GetShootClient = func( _ context.Context, - _ client.SubResourceClient, - _ *gardener_api.Shoot) (client.Client, error) { + _ client.Client, + _ imv1.Runtime) (client.Client, error) { return f.Client, nil } return nil @@ -188,8 +187,8 @@ var _ = Describe(`runtime_fsm_apply_crb`, Label("applyCRB"), func() { setup: func(f *fsm) error { GetShootClient = func( _ context.Context, - _ client.SubResourceClient, - _ *gardener_api.Shoot) (client.Client, error) { + _ client.Client, + _ imv1.Runtime) (client.Client, error) { return nil, testErr } return nil diff --git a/internal/controller/runtime/fsm/runtime_fsm_configure_oidc.go b/internal/controller/runtime/fsm/runtime_fsm_configure_oidc.go index 58c8ce0d..3c0fc057 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_configure_oidc.go +++ b/internal/controller/runtime/fsm/runtime_fsm_configure_oidc.go @@ -72,8 +72,8 @@ func createDefaultOIDCConfig(defaultSharedIASTenant config.OidcProvider) gardene } func recreateOpenIDConnectResources(ctx context.Context, m *fsm, s *systemState) error { - srscClient := m.ShootClient.SubResource("adminkubeconfig") - shootAdminClient, shootClientError := GetShootClient(ctx, srscClient, s.shoot) + + shootAdminClient, shootClientError := GetShootClient(ctx, m.Client, s.instance) if shootClientError != nil { return shootClientError } diff --git a/internal/controller/runtime/fsm/runtime_fsm_configure_oidc_test.go b/internal/controller/runtime/fsm/runtime_fsm_configure_oidc_test.go index c3a6beb7..e74ba1ab 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_configure_oidc_test.go +++ b/internal/controller/runtime/fsm/runtime_fsm_configure_oidc_test.go @@ -63,7 +63,7 @@ func TestOidcState(t *testing.T) { var fakeClient = fake.NewClientBuilder(). WithScheme(scheme). Build() - fsm := &fsm{K8s: K8s{ + testFsm := &fsm{K8s: K8s{ ShootClient: fakeClient, Client: fakeClient, }, @@ -77,8 +77,8 @@ func TestOidcState(t *testing.T) { } GetShootClient = func( _ context.Context, - _ client.SubResourceClient, - _ *gardener.Shoot) (client.Client, error) { + _ client.Client, + _ imv1.Runtime) (client.Client, error) { return fakeClient, nil } // end of fake client setup @@ -106,7 +106,7 @@ func TestOidcState(t *testing.T) { } // when - stateFn, _, _ := sFnConfigureOidc(ctx, fsm, systemState) + stateFn, _, _ := sFnConfigureOidc(ctx, testFsm, systemState) // then require.Contains(t, stateFn.name(), "sFnApplyClusterRoleBindings") @@ -131,14 +131,14 @@ func TestOidcState(t *testing.T) { var fakeClient = fake.NewClientBuilder(). WithScheme(scheme). Build() - fsm := &fsm{K8s: K8s{ + testFsm := &fsm{K8s: K8s{ ShootClient: fakeClient, Client: fakeClient, }} GetShootClient = func( _ context.Context, - _ client.SubResourceClient, - _ *gardener.Shoot) (client.Client, error) { + _ client.Client, + _ imv1.Runtime) (client.Client, error) { return fakeClient, nil } // end of fake client setup @@ -171,7 +171,7 @@ func TestOidcState(t *testing.T) { } // when - stateFn, _, _ := sFnConfigureOidc(ctx, fsm, systemState) + stateFn, _, _ := sFnConfigureOidc(ctx, testFsm, systemState) // then require.Contains(t, stateFn.name(), "sFnApplyClusterRoleBindings") @@ -198,14 +198,14 @@ func TestOidcState(t *testing.T) { var fakeClient = fake.NewClientBuilder(). WithScheme(scheme). Build() - fsm := &fsm{K8s: K8s{ + testFSM := &fsm{K8s: K8s{ ShootClient: fakeClient, Client: fakeClient, }} GetShootClient = func( _ context.Context, - _ client.SubResourceClient, - _ *gardener.Shoot) (client.Client, error) { + _ client.Client, + _ imv1.Runtime) (client.Client, error) { return fakeClient, nil } // end of fake client setup @@ -241,7 +241,7 @@ func TestOidcState(t *testing.T) { } // when - stateFn, _, _ := sFnConfigureOidc(ctx, fsm, systemState) + stateFn, _, _ := sFnConfigureOidc(ctx, testFSM, systemState) // then require.Contains(t, stateFn.name(), "sFnApplyClusterRoleBindings") diff --git a/internal/controller/runtime/fsm/runtime_fsm_create_kubeconfig.go b/internal/controller/runtime/fsm/runtime_fsm_create_kubeconfig.go index 72155b6d..a52c79e5 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_create_kubeconfig.go +++ b/internal/controller/runtime/fsm/runtime_fsm_create_kubeconfig.go @@ -12,6 +12,10 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ) +const ( + kubeconfigSecretKey = "config" +) + func sFnCreateKubeconfig(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { m.log.Info("Create Gardener Cluster CR state") @@ -110,7 +114,7 @@ func makeGardenerClusterForRuntime(runtime imv1.Runtime, shoot *gardener.Shoot) Secret: imv1.Secret{ Name: fmt.Sprintf("kubeconfig-%s", runtime.Labels[imv1.LabelKymaRuntimeID]), Namespace: runtime.Namespace, - Key: "config", + Key: kubeconfigSecretKey, }, }, }, diff --git a/internal/controller/runtime/suite_test.go b/internal/controller/runtime/suite_test.go index 4392baea..06023d99 100644 --- a/internal/controller/runtime/suite_test.go +++ b/internal/controller/runtime/suite_test.go @@ -25,7 +25,7 @@ import ( gardener_api "github.com/gardener/gardener/pkg/apis/core/v1beta1" gardener_oidc "github.com/gardener/oidc-webhook-authenticator/apis/authentication/v1alpha1" - infrastructuremanagerv1 "github.com/kyma-project/infrastructure-manager/api/v1" + imv1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/infrastructure-manager/internal/controller/metrics/mocks" "github.com/kyma-project/infrastructure-manager/internal/controller/runtime/fsm" "github.com/kyma-project/infrastructure-manager/pkg/config" @@ -95,7 +95,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - err = infrastructuremanagerv1.AddToScheme(scheme.Scheme) + err = imv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) mgr, err := ctrl.NewManager(cfg, ctrl.Options{ @@ -107,7 +107,7 @@ var _ = BeforeSuite(func() { clientScheme := runtime.NewScheme() _ = gardener_api.AddToScheme(clientScheme) - _ = infrastructuremanagerv1.AddToScheme(clientScheme) + _ = imv1.AddToScheme(clientScheme) // tracker will be updated with different shoot sequence for each test case tracker := clienttesting.NewObjectTracker(clientScheme, serializer.NewCodecFactory(clientScheme).UniversalDecoder()) @@ -122,7 +122,7 @@ var _ = BeforeSuite(func() { mm.On("CleanUpRuntimeGauge", mock.Anything, mock.Anything).Return() fsmCfg := fsm.RCCfg{ - Finalizer: infrastructuremanagerv1.Finalizer, + Finalizer: imv1.Finalizer, Config: convConfig, Metrics: mm, AuditLogging: map[string]map[string]auditlogs.AuditLogData{}, @@ -145,7 +145,7 @@ var _ = BeforeSuite(func() { err = gardener_oidc.AddToScheme(shootClientScheme) k8sFakeClientRoleBindings = fake.NewClientBuilder().WithScheme(shootClientScheme).Build() - fsm.GetShootClient = func(_ context.Context, _ client.SubResourceClient, _ *gardener_api.Shoot) (client.Client, error) { + fsm.GetShootClient = func(_ context.Context, _ client.Client, _ imv1.Runtime) (client.Client, error) { return k8sFakeClientRoleBindings, nil } From 506e072d38f92e9d2654f04a7f5075e59f3cd4b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Tue, 19 Nov 2024 18:16:31 +0100 Subject: [PATCH 04/30] Running linter on the source code --- .../runtime/fsm/runtime_fsm_apply_clusterrolebindings.go | 4 ++-- internal/controller/runtime/fsm/runtime_fsm_configure_oidc.go | 1 - internal/controller/runtime/runtime_controller.go | 2 +- internal/controller/runtime/suite_test.go | 1 - pkg/gardener/shoot/extender/provider_test.go | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go index e0407345..dc12dc2c 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go +++ b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go @@ -3,15 +3,15 @@ package fsm import ( "context" "fmt" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" "slices" "time" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/controller/runtime/fsm/runtime_fsm_configure_oidc.go b/internal/controller/runtime/fsm/runtime_fsm_configure_oidc.go index 3c0fc057..fa7b140a 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_configure_oidc.go +++ b/internal/controller/runtime/fsm/runtime_fsm_configure_oidc.go @@ -72,7 +72,6 @@ func createDefaultOIDCConfig(defaultSharedIASTenant config.OidcProvider) gardene } func recreateOpenIDConnectResources(ctx context.Context, m *fsm, s *systemState) error { - shootAdminClient, shootClientError := GetShootClient(ctx, m.Client, s.instance) if shootClientError != nil { return shootClientError diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index 5a3dcac9..ed2dc768 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -19,7 +19,6 @@ package runtime import ( "context" "fmt" - "sigs.k8s.io/controller-runtime/pkg/controller" "github.com/go-logr/logr" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" @@ -28,6 +27,7 @@ import ( "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/predicate" ) diff --git a/internal/controller/runtime/suite_test.go b/internal/controller/runtime/suite_test.go index 06023d99..473b8948 100644 --- a/internal/controller/runtime/suite_test.go +++ b/internal/controller/runtime/suite_test.go @@ -39,7 +39,6 @@ import ( v12 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - //nolint:revive "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" diff --git a/pkg/gardener/shoot/extender/provider_test.go b/pkg/gardener/shoot/extender/provider_test.go index 0d96a8a4..48ac307a 100644 --- a/pkg/gardener/shoot/extender/provider_test.go +++ b/pkg/gardener/shoot/extender/provider_test.go @@ -202,7 +202,7 @@ func TestProviderExtender(t *testing.T) { CurrentZonesConfig: []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"}, TestForPatch: true, }, - //"Patch option different image name - override image name and version with current image name and version": {}, + // "Patch option different image name - override image name and version with current image name and version": {}, } { t.Run(tname, func(t *testing.T) { // given From 5b3b29556797293df37ed6dae421f40e546bb8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Wed, 20 Nov 2024 09:08:17 +0100 Subject: [PATCH 05/30] remove timeout from shoot client create function --- .../runtime/fsm/runtime_fsm_apply_clusterrolebindings.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go index dc12dc2c..dec0ca1c 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go +++ b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go @@ -3,9 +3,6 @@ package fsm import ( "context" "fmt" - "slices" - "time" - imv1 "github.com/kyma-project/infrastructure-manager/api/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -15,6 +12,7 @@ import ( "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "slices" ) var ( @@ -74,7 +72,6 @@ var GetShootClient = func(ctx context.Context, cnt client.Client, runtime imv1.R return nil, err } - restConfig.Timeout = 250 * time.Millisecond shootClientWithAdmin, err := client.New(restConfig, client.Options{}) if err != nil { return nil, err From 78dd23834fc668e2ab0ce75e09ba44bdbd8833b5 Mon Sep 17 00:00:00 2001 From: m00g3n Date: Wed, 20 Nov 2024 17:29:50 +0100 Subject: [PATCH 06/30] add experimental rate limiter to gardener client --- cmd/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/main.go b/cmd/main.go index cbe7554f..c30a6b08 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -46,6 +46,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/rest" + "k8s.io/client-go/util/flowcontrol" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -240,6 +241,8 @@ func initGardenerClients(kubeconfigPath string, namespace string) (client.Client return nil, nil, nil, err } + restConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(5, 5) + gardenerClientSet, err := gardener_apis.NewForConfig(restConfig) if err != nil { return nil, nil, nil, err From a25de9f49c6674be70346f64a0cd9bfca5f4e5a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Thu, 21 Nov 2024 13:07:45 +0100 Subject: [PATCH 07/30] increase gardener time requeue time --- .../controller/runtime/fsm/runtime_fsm_delete_shoot.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go index 37f4a33d..c94ab734 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go @@ -2,6 +2,7 @@ package fsm import ( "context" + "time" gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" @@ -10,13 +11,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + deletionRequeueTime = 2 * time.Minute +) + func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { m.log.Info("delete shoot state") // wait section if !s.shoot.GetDeletionTimestamp().IsZero() { m.log.Info("Waiting for shoot to be deleted", "Name", s.shoot.Name, "Namespace", s.shoot.Namespace) - return requeueAfter(m.RCCfg.GardenerRequeueDuration) + return requeueAfter(deletionRequeueTime) } // action section @@ -58,7 +63,7 @@ func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl } // out section - return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) + return updateStatusAndRequeueAfter(deletionRequeueTime) } // workaround From 2bd723b370e33d64cfab321e2fdf46603f1e2304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Fri, 22 Nov 2024 13:56:41 +0100 Subject: [PATCH 08/30] adding params to configure rate limiter for Gardener clients --- cmd/main.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index c30a6b08..68751965 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -71,7 +71,8 @@ const defaultMinimalRotationTimeRatio = 0.6 const defaultExpirationTime = 24 * time.Hour const defaultGardenerRequestTimeout = 60 * time.Second const defaultControlPlaneRequeueDuration = 10 * time.Second -const defaultGardenerRequeueDuration = 15 * time.Second +const defaultGardenerRateLimiterQps = 5 +const defaultGardenerRateLimiterBurst = 5 func main() { var metricsAddr string @@ -82,6 +83,9 @@ func main() { var minimalRotationTimeRatio float64 var expirationTime time.Duration var gardenerRequestTimeout time.Duration + var runtimeControllerGardenerRequestTimeout time.Duration + var runtimeControllerGardenerRateLimiterQps int + var runtimeControllerGardenerRateLimiterBurst int var converterConfigFilepath string var shootSpecDumpEnabled bool var auditLogMandatory bool @@ -96,6 +100,9 @@ func main() { flag.Float64Var(&minimalRotationTimeRatio, "minimal-rotation-time", defaultMinimalRotationTimeRatio, "The ratio determines what is the minimal time that needs to pass to rotate certificate.") flag.DurationVar(&expirationTime, "kubeconfig-expiration-time", defaultExpirationTime, "Dynamic kubeconfig expiration time") flag.DurationVar(&gardenerRequestTimeout, "gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for requests to Gardener") + flag.DurationVar(&runtimeControllerGardenerRequestTimeout, "rt-ctrl-gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for requests from Runtime Controller to Gardener") + flag.IntVar(&runtimeControllerGardenerRateLimiterQps, "rt-ctrl-gardener-ratelimiter-qps", defaultGardenerRateLimiterQps, "Timeout duration for requests to Gardener") + flag.IntVar(&runtimeControllerGardenerRateLimiterBurst, "rt-ctrl-gardener-ratelimiter-burst", defaultGardenerRateLimiterBurst, "Timeout duration for requests to Gardener") flag.StringVar(&converterConfigFilepath, "converter-config-filepath", "/converter-config/converter_config.json", "A file path to the gardener shoot converter configuration.") flag.BoolVar(&shootSpecDumpEnabled, "shoot-spec-dump-enabled", false, "Feature flag to allow persisting specs of created shoots") flag.BoolVar(&auditLogMandatory, "audit-log-mandatory", true, "Feature flag to enable strict mode for audit log configuration") From 312fba0cd48079cc4f00d14f83781a7555fc26e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Sat, 23 Nov 2024 22:01:14 +0100 Subject: [PATCH 09/30] configurable gardener client parameters for Runtime controller --- cmd/main.go | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 68751965..fa18fbda 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -67,12 +67,21 @@ func init() { //+kubebuilder:scaffold:scheme } -const defaultMinimalRotationTimeRatio = 0.6 -const defaultExpirationTime = 24 * time.Hour -const defaultGardenerRequestTimeout = 60 * time.Second -const defaultControlPlaneRequeueDuration = 10 * time.Second -const defaultGardenerRateLimiterQps = 5 -const defaultGardenerRateLimiterBurst = 5 +// Default values for the Runtime controller configuration +const ( + defaultControlPlaneRequeueDuration = 10 * time.Second + defaultRuntimeCtrlGardenerRequestTimeout = 1 * time.Second + defaultRuntimeCtrlGardenerRateLimiterQps = 5 + defaultRuntimeCtrlGardenerRateLimiterBurst = 5 +) + +// Default values for the Gardener Cluster controller configuration +const ( + defaultMinimalRotationTimeRatio = 0.6 + defaultExpirationTime = 24 * time.Hour + defaultGardenerReconciliationTimeout = 60 * time.Second + defaultGardenerRequeueDuration = 15 * time.Second +) func main() { var metricsAddr string @@ -82,10 +91,10 @@ func main() { var gardenerProjectName string var minimalRotationTimeRatio float64 var expirationTime time.Duration - var gardenerRequestTimeout time.Duration - var runtimeControllerGardenerRequestTimeout time.Duration - var runtimeControllerGardenerRateLimiterQps int - var runtimeControllerGardenerRateLimiterBurst int + var gardenerCtrlReconciliationTimeout time.Duration + var runtimeCtrlGardenerRequestTimeout time.Duration + var runtimeCtrlGardenerRateLimiterQps int + var runtimeCtrlGardenerRateLimiterBurst int var converterConfigFilepath string var shootSpecDumpEnabled bool var auditLogMandatory bool @@ -99,10 +108,10 @@ func main() { flag.StringVar(&gardenerProjectName, "gardener-project-name", "gardener-project", "Name of the Gardener project") flag.Float64Var(&minimalRotationTimeRatio, "minimal-rotation-time", defaultMinimalRotationTimeRatio, "The ratio determines what is the minimal time that needs to pass to rotate certificate.") flag.DurationVar(&expirationTime, "kubeconfig-expiration-time", defaultExpirationTime, "Dynamic kubeconfig expiration time") - flag.DurationVar(&gardenerRequestTimeout, "gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for requests to Gardener") - flag.DurationVar(&runtimeControllerGardenerRequestTimeout, "rt-ctrl-gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for requests from Runtime Controller to Gardener") - flag.IntVar(&runtimeControllerGardenerRateLimiterQps, "rt-ctrl-gardener-ratelimiter-qps", defaultGardenerRateLimiterQps, "Timeout duration for requests to Gardener") - flag.IntVar(&runtimeControllerGardenerRateLimiterBurst, "rt-ctrl-gardener-ratelimiter-burst", defaultGardenerRateLimiterBurst, "Timeout duration for requests to Gardener") + flag.DurationVar(&gardenerCtrlReconciliationTimeout, "gardener-ctrl-reconcilation-timeout", defaultGardenerReconciliationTimeout, "Timeout duration for reconlication for Gardener Cluster Controller") + flag.DurationVar(&runtimeCtrlGardenerRequestTimeout, "rt-ctrl-gardener-request-timeout", defaultRuntimeCtrlGardenerRequestTimeout, "Timeout duration for Gardener client for Runtime Controller") + flag.IntVar(&runtimeCtrlGardenerRateLimiterQps, "runtime-ctrl-gardener-ratelimiter-qps", defaultRuntimeCtrlGardenerRateLimiterQps, "Gardener client rate limiter QPS for Runtime Controller") + flag.IntVar(&runtimeCtrlGardenerRateLimiterBurst, "runtime-ctrl-gardener-ratelimiter-burst", defaultRuntimeCtrlGardenerRateLimiterBurst, "Gardener client rate limiter burst for Runtime Controller") flag.StringVar(&converterConfigFilepath, "converter-config-filepath", "/converter-config/converter_config.json", "A file path to the gardener shoot converter configuration.") flag.BoolVar(&shootSpecDumpEnabled, "shoot-spec-dump-enabled", false, "Feature flag to allow persisting specs of created shoots") flag.BoolVar(&auditLogMandatory, "audit-log-mandatory", true, "Feature flag to enable strict mode for audit log configuration") @@ -145,7 +154,7 @@ func main() { } gardenerNamespace := fmt.Sprintf("garden-%s", gardenerProjectName) - gardenerClient, shootClient, dynamicKubeconfigClient, err := initGardenerClients(gardenerKubeconfigPath, gardenerNamespace) + gardenerClient, shootClient, dynamicKubeconfigClient, err := initGardenerClients(gardenerKubeconfigPath, gardenerNamespace, runtimeCtrlGardenerRequestTimeout, runtimeCtrlGardenerRateLimiterQps, runtimeCtrlGardenerRateLimiterBurst) if err != nil { setupLog.Error(err, "unable to initialize gardener clients", "controller", "GardenerCluster") @@ -166,7 +175,7 @@ func main() { logger, rotationPeriod, minimalRotationTimeRatio, - gardenerRequestTimeout, + gardenerCtrlReconciliationTimeout, metrics, ).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GardenerCluster") @@ -242,13 +251,14 @@ func main() { } } -func initGardenerClients(kubeconfigPath string, namespace string) (client.Client, gardener_apis.ShootInterface, client.SubResourceClient, error) { +func initGardenerClients(kubeconfigPath string, namespace string, timeout time.Duration, rlQPS, rlBurst int) (client.Client, gardener_apis.ShootInterface, client.SubResourceClient, error) { restConfig, err := gardener.NewRestConfigFromFile(kubeconfigPath) if err != nil { return nil, nil, nil, err } - restConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(5, 5) + restConfig.Timeout = timeout + restConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(float32(rlQPS), rlBurst) gardenerClientSet, err := gardener_apis.NewForConfig(restConfig) if err != nil { From 06bfa9adc69ff7f923064c6d6ede0fdaaca7aab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Sat, 23 Nov 2024 22:02:24 +0100 Subject: [PATCH 10/30] change parameter name to make it more uniform with other ones --- cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index fa18fbda..8e26be57 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -109,7 +109,7 @@ func main() { flag.Float64Var(&minimalRotationTimeRatio, "minimal-rotation-time", defaultMinimalRotationTimeRatio, "The ratio determines what is the minimal time that needs to pass to rotate certificate.") flag.DurationVar(&expirationTime, "kubeconfig-expiration-time", defaultExpirationTime, "Dynamic kubeconfig expiration time") flag.DurationVar(&gardenerCtrlReconciliationTimeout, "gardener-ctrl-reconcilation-timeout", defaultGardenerReconciliationTimeout, "Timeout duration for reconlication for Gardener Cluster Controller") - flag.DurationVar(&runtimeCtrlGardenerRequestTimeout, "rt-ctrl-gardener-request-timeout", defaultRuntimeCtrlGardenerRequestTimeout, "Timeout duration for Gardener client for Runtime Controller") + flag.DurationVar(&runtimeCtrlGardenerRequestTimeout, "runtime-ctrl-gardener-request-timeout", defaultRuntimeCtrlGardenerRequestTimeout, "Timeout duration for Gardener client for Runtime Controller") flag.IntVar(&runtimeCtrlGardenerRateLimiterQps, "runtime-ctrl-gardener-ratelimiter-qps", defaultRuntimeCtrlGardenerRateLimiterQps, "Gardener client rate limiter QPS for Runtime Controller") flag.IntVar(&runtimeCtrlGardenerRateLimiterBurst, "runtime-ctrl-gardener-ratelimiter-burst", defaultRuntimeCtrlGardenerRateLimiterBurst, "Gardener client rate limiter burst for Runtime Controller") flag.StringVar(&converterConfigFilepath, "converter-config-filepath", "/converter-config/converter_config.json", "A file path to the gardener shoot converter configuration.") From 620b488429eca6a825faeabecc2607948adf5ad5 Mon Sep 17 00:00:00 2001 From: Rafal Foks Date: Mon, 25 Nov 2024 12:09:58 +0100 Subject: [PATCH 11/30] Restrict the controller to watch only kcp-system namespace --- cmd/main.go | 27 +++++++++++++++++++ config/rbac/cluster_editor_role.yaml | 5 ++-- config/rbac/cluster_viewer_role.yaml | 3 ++- config/rbac/role.yaml | 26 +++--------------- config/rbac/role_binding.yaml | 8 +++--- config/rbac/runtime_editor_role.yaml | 3 ++- config/rbac/runtime_viewer_role.yaml | 3 ++- config/rbac/service_account.yaml | 2 +- .../kubeconfig/gardener_cluster_controller.go | 8 +++--- .../controller/runtime/runtime_controller.go | 6 ++--- 10 files changed, 51 insertions(+), 40 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index cbe7554f..8208221b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -22,7 +22,9 @@ import ( "flag" "fmt" "io" + corev1 "k8s.io/api/core/v1" "os" + "sigs.k8s.io/controller-runtime/pkg/cache" "time" "github.com/gardener/gardener/pkg/apis/core/v1beta1" @@ -41,6 +43,7 @@ import ( "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" "github.com/pkg/errors" rbacv1 "k8s.io/api/rbac/v1" + k8slabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -119,6 +122,7 @@ func main() { HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "f1c68560.kyma-project.io", + Cache: restrictWatchedNamespace(), // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily // when the Manager ends. This requires the binary to immediately end when the // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly @@ -314,3 +318,26 @@ func refreshRuntimeMetrics(restConfig *rest.Config, logger logr.Logger, metrics metrics.SetRuntimeStates(rt) } } + +func restrictWatchedNamespace() cache.Options { + return cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Label: k8slabels.Everything(), + Namespaces: map[string]cache.Config{ + "kcp-system": {}, + }, + }, + &infrastructuremanagerv1.Runtime{}: { + Namespaces: map[string]cache.Config{ + "kcp-system": {}, + }, + }, + &infrastructuremanagerv1.GardenerCluster{}: { + Namespaces: map[string]cache.Config{ + "kcp-system": {}, + }, + }, + }, + } +} diff --git a/config/rbac/cluster_editor_role.yaml b/config/rbac/cluster_editor_role.yaml index 64abe8ce..84dda500 100644 --- a/config/rbac/cluster_editor_role.yaml +++ b/config/rbac/cluster_editor_role.yaml @@ -1,15 +1,16 @@ # permissions for end users to edit clusters. apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: labels: - app.kubernetes.io/name: clusterrole + app.kubernetes.io/name: 1errole app.kubernetes.io/instance: cluster-editor-role app.kubernetes.io/component: rbac app.kubernetes.io/created-by: infrastructure-manager app.kubernetes.io/part-of: infrastructure-manager app.kubernetes.io/managed-by: kustomize name: cluster-editor-role + namespace: kcp-system rules: - apiGroups: - infrastructuremanager.kyma-project.io diff --git a/config/rbac/cluster_viewer_role.yaml b/config/rbac/cluster_viewer_role.yaml index d183607b..43df4e6d 100644 --- a/config/rbac/cluster_viewer_role.yaml +++ b/config/rbac/cluster_viewer_role.yaml @@ -1,6 +1,6 @@ # permissions for end users to view clusters. apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: labels: app.kubernetes.io/name: clusterrole @@ -10,6 +10,7 @@ metadata: app.kubernetes.io/part-of: infrastructure-manager app.kubernetes.io/managed-by: kustomize name: cluster-viewer-role + namespace: kcp-system rules: - apiGroups: - infrastructuremanager.kyma-project.io diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7681c444..6b06da55 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -1,8 +1,9 @@ --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: name: infrastructure-manager-role + namespace: kcp-system rules: - apiGroups: - "" @@ -19,6 +20,7 @@ rules: - infrastructuremanager.kyma-project.io resources: - gardenerclusters + - runtimes verbs: - create - delete @@ -31,29 +33,7 @@ rules: - infrastructuremanager.kyma-project.io resources: - gardenerclusters/finalizers - verbs: - - update -- apiGroups: - - infrastructuremanager.kyma-project.io - resources: - gardenerclusters/status - verbs: - - update -- apiGroups: - - infrastructuremanager.kyma-project.io - resources: - - runtimes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - infrastructuremanager.kyma-project.io - resources: - runtimes/finalizers verbs: - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 60f28ad3..6cd75b05 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -1,8 +1,8 @@ apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: RoleBinding metadata: labels: - app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/name: rolebinding app.kubernetes.io/instance: infrastructure-manager-rolebinding app.kubernetes.io/component: rbac app.kubernetes.io/created-by: infrastructure-manager @@ -11,9 +11,9 @@ metadata: name: infrastructure-manager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io - kind: ClusterRole + kind: Role name: infrastructure-manager-role subjects: - kind: ServiceAccount name: infrastructure-manager - namespace: system + namespace: kcp-system diff --git a/config/rbac/runtime_editor_role.yaml b/config/rbac/runtime_editor_role.yaml index 014838b7..ceb3ba03 100644 --- a/config/rbac/runtime_editor_role.yaml +++ b/config/rbac/runtime_editor_role.yaml @@ -1,11 +1,12 @@ # permissions for end users to edit runtimes. apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: labels: app.kubernetes.io/name: infrastructure-manager app.kubernetes.io/managed-by: kustomize name: runtime-editor-role + namespace: kcp-system rules: - apiGroups: - infrastructuremanager.kyma-project.io diff --git a/config/rbac/runtime_viewer_role.yaml b/config/rbac/runtime_viewer_role.yaml index d9d0024e..0c5ac175 100644 --- a/config/rbac/runtime_viewer_role.yaml +++ b/config/rbac/runtime_viewer_role.yaml @@ -1,11 +1,12 @@ # permissions for end users to view runtimes. apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: labels: app.kubernetes.io/name: infrastructure-manager app.kubernetes.io/managed-by: kustomize name: runtime-viewer-role + namespace: kcp-system rules: - apiGroups: - infrastructuremanager.kyma-project.io diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index 5c3330c2..3ddd25a9 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -9,4 +9,4 @@ metadata: app.kubernetes.io/part-of: infrastructure-manager app.kubernetes.io/managed-by: kustomize name: infrastructure-manager - namespace: system + namespace: kcp-system diff --git a/internal/controller/kubeconfig/gardener_cluster_controller.go b/internal/controller/kubeconfig/gardener_cluster_controller.go index c532e203..87cba31e 100644 --- a/internal/controller/kubeconfig/gardener_cluster_controller.go +++ b/internal/controller/kubeconfig/gardener_cluster_controller.go @@ -77,10 +77,10 @@ type KubeconfigProvider interface { Fetch(ctx context.Context, shootName string) (string, error) } -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;delete -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters/finalizers,verbs=update -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters/status,verbs=update +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters,verbs=get;list;watch;create;update;patch;delete,namespace=kcp-system +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;delete,namespace=kcp-system +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters/finalizers,verbs=update,namespace=kcp-system +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters/status,verbs=update,namespace=kcp-system // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index 1f8eb574..5aa8f530 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -41,9 +41,9 @@ type RuntimeReconciler struct { EventRecorder record.EventRecorder } -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes/finalizers,verbs=update +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes,verbs=get;list;watch;create;update;patch;delete,namespace=kcp-system +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes/status,verbs=get;update;patch,namespace=kcp-system +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes/finalizers,verbs=update,namespace=kcp-system var requCounter = 0 // nolint:gochecknoglobals From d9f0b2882a226ec78eaddd57304097f997beb328 Mon Sep 17 00:00:00 2001 From: Rafal Foks Date: Mon, 25 Nov 2024 12:43:10 +0100 Subject: [PATCH 12/30] Fix imports --- cmd/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 8208221b..8b74982c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -22,11 +22,12 @@ import ( "flag" "fmt" "io" - corev1 "k8s.io/api/core/v1" "os" - "sigs.k8s.io/controller-runtime/pkg/cache" "time" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/cache" + "github.com/gardener/gardener/pkg/apis/core/v1beta1" gardener_apis "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" gardener_oidc "github.com/gardener/oidc-webhook-authenticator/apis/authentication/v1alpha1" From 12cd16658859d724a5dcd1068dee6d28e10513ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Mon, 25 Nov 2024 13:06:40 +0100 Subject: [PATCH 13/30] different increased requeue times for different Gardener operation --- cmd/main.go | 40 ++++++++++--------- .../controller/runtime/fsm/runtime_fsm.go | 19 +++++---- .../runtime/fsm/runtime_fsm_delete_shoot.go | 10 +---- ...runtime_fsm_waiting_for_shoot_reconcile.go | 4 +- .../fsm/runtime_fsm_waiting_shoot_creation.go | 4 +- .../controller/runtime/fsm/utilz_for_test.go | 2 + .../controller/runtime/runtime_controller.go | 2 + internal/controller/runtime/suite_test.go | 15 ++++--- 8 files changed, 51 insertions(+), 45 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 8e26be57..429329c5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -69,18 +69,17 @@ func init() { // Default values for the Runtime controller configuration const ( - defaultControlPlaneRequeueDuration = 10 * time.Second - defaultRuntimeCtrlGardenerRequestTimeout = 1 * time.Second - defaultRuntimeCtrlGardenerRateLimiterQps = 5 - defaultRuntimeCtrlGardenerRateLimiterBurst = 5 -) - -// Default values for the Gardener Cluster controller configuration -const ( + defaultControlPlaneRequeueDuration = 10 * time.Second + defaultGardenerRequestTimeout = 3 * time.Second + defaultGardenerRateLimiterQps = 5 + defaultGardenerRateLimiterBurst = 5 defaultMinimalRotationTimeRatio = 0.6 defaultExpirationTime = 24 * time.Hour defaultGardenerReconciliationTimeout = 60 * time.Second defaultGardenerRequeueDuration = 15 * time.Second + defaultShootCreateRequeueDuration = 60 * time.Second + defaultShootDeleteRequeueDuration = 90 * time.Second + defaultShootReconcileRequeueDuration = 30 * time.Second ) func main() { @@ -109,9 +108,9 @@ func main() { flag.Float64Var(&minimalRotationTimeRatio, "minimal-rotation-time", defaultMinimalRotationTimeRatio, "The ratio determines what is the minimal time that needs to pass to rotate certificate.") flag.DurationVar(&expirationTime, "kubeconfig-expiration-time", defaultExpirationTime, "Dynamic kubeconfig expiration time") flag.DurationVar(&gardenerCtrlReconciliationTimeout, "gardener-ctrl-reconcilation-timeout", defaultGardenerReconciliationTimeout, "Timeout duration for reconlication for Gardener Cluster Controller") - flag.DurationVar(&runtimeCtrlGardenerRequestTimeout, "runtime-ctrl-gardener-request-timeout", defaultRuntimeCtrlGardenerRequestTimeout, "Timeout duration for Gardener client for Runtime Controller") - flag.IntVar(&runtimeCtrlGardenerRateLimiterQps, "runtime-ctrl-gardener-ratelimiter-qps", defaultRuntimeCtrlGardenerRateLimiterQps, "Gardener client rate limiter QPS for Runtime Controller") - flag.IntVar(&runtimeCtrlGardenerRateLimiterBurst, "runtime-ctrl-gardener-ratelimiter-burst", defaultRuntimeCtrlGardenerRateLimiterBurst, "Gardener client rate limiter burst for Runtime Controller") + flag.DurationVar(&runtimeCtrlGardenerRequestTimeout, "gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for Gardener client for Runtime Controller") + flag.IntVar(&runtimeCtrlGardenerRateLimiterQps, "gardener-ratelimiter-qps", defaultGardenerRateLimiterQps, "Gardener client rate limiter QPS for Runtime Controller") + flag.IntVar(&runtimeCtrlGardenerRateLimiterBurst, "gardener-ratelimiter-burst", defaultGardenerRateLimiterBurst, "Gardener client rate limiter burst for Runtime Controller") flag.StringVar(&converterConfigFilepath, "converter-config-filepath", "/converter-config/converter_config.json", "A file path to the gardener shoot converter configuration.") flag.BoolVar(&shootSpecDumpEnabled, "shoot-spec-dump-enabled", false, "Feature flag to allow persisting specs of created shoots") flag.BoolVar(&auditLogMandatory, "audit-log-mandatory", true, "Feature flag to enable strict mode for audit log configuration") @@ -205,14 +204,17 @@ func main() { } cfg := fsm.RCCfg{ - GardenerRequeueDuration: defaultGardenerRequeueDuration, - ControlPlaneRequeueDuration: defaultControlPlaneRequeueDuration, - Finalizer: infrastructuremanagerv1.Finalizer, - ShootNamesapace: gardenerNamespace, - Config: config, - AuditLogMandatory: auditLogMandatory, - Metrics: metrics, - AuditLogging: auditLogDataMap, + GardenerRequeueDuration: defaultGardenerRequeueDuration, + RequeueDurationShootCreate: defaultShootCreateRequeueDuration, + RequeueDurationShootDelete: defaultShootDeleteRequeueDuration, + RequeueDurationShootReconcile: defaultShootReconcileRequeueDuration, + ControlPlaneRequeueDuration: defaultControlPlaneRequeueDuration, + Finalizer: infrastructuremanagerv1.Finalizer, + ShootNamesapace: gardenerNamespace, + Config: config, + AuditLogMandatory: auditLogMandatory, + Metrics: metrics, + AuditLogging: auditLogDataMap, } if shootSpecDumpEnabled { cfg.PVCPath = "/testdata/kim" diff --git a/internal/controller/runtime/fsm/runtime_fsm.go b/internal/controller/runtime/fsm/runtime_fsm.go index 0a5391d1..5d79d549 100644 --- a/internal/controller/runtime/fsm/runtime_fsm.go +++ b/internal/controller/runtime/fsm/runtime_fsm.go @@ -26,14 +26,17 @@ type writerGetter = func(filePath string) (io.Writer, error) // runtime reconciler specific configuration type RCCfg struct { - GardenerRequeueDuration time.Duration - ControlPlaneRequeueDuration time.Duration - Finalizer string - PVCPath string - ShootNamesapace string - AuditLogMandatory bool - Metrics metrics.Metrics - AuditLogging auditlogs.Configuration + GardenerRequeueDuration time.Duration + RequeueDurationShootCreate time.Duration + RequeueDurationShootDelete time.Duration + RequeueDurationShootReconcile time.Duration + ControlPlaneRequeueDuration time.Duration + Finalizer string + PVCPath string + ShootNamesapace string + AuditLogMandatory bool + Metrics metrics.Metrics + AuditLogging auditlogs.Configuration config.Config } diff --git a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go index c94ab734..2cdfe46d 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go @@ -2,8 +2,6 @@ package fsm import ( "context" - "time" - gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" "k8s.io/utils/ptr" @@ -11,17 +9,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - deletionRequeueTime = 2 * time.Minute -) - func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { m.log.Info("delete shoot state") // wait section if !s.shoot.GetDeletionTimestamp().IsZero() { m.log.Info("Waiting for shoot to be deleted", "Name", s.shoot.Name, "Namespace", s.shoot.Namespace) - return requeueAfter(deletionRequeueTime) + return requeueAfter(m.RCCfg.RequeueDurationShootDelete) } // action section @@ -63,7 +57,7 @@ func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl } // out section - return updateStatusAndRequeueAfter(deletionRequeueTime) + return updateStatusAndRequeueAfter(m.RCCfg.RequeueDurationShootDelete) } // workaround diff --git a/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go b/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go index 788e60d1..1b6f2855 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go +++ b/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go @@ -23,7 +23,7 @@ func sFnWaitForShootReconcile(_ context.Context, m *fsm, s *systemState) (stateF "Unknown", "Shoot update is in progress") - return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) + return updateStatusAndRequeueAfter(m.RCCfg.RequeueDurationShootReconcile) case gardener.LastOperationStateFailed: lastErrors := s.shoot.Status.LastErrors @@ -36,7 +36,7 @@ func sFnWaitForShootReconcile(_ context.Context, m *fsm, s *systemState) (stateF imv1.ConditionReasonShootCreationPending, "Unknown", "Retryable gardener errors during cluster reconcile") - return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) + return updateStatusAndRequeueAfter(m.RCCfg.RequeueDurationShootReconcile) } msg := fmt.Sprintf("error during cluster processing: reconcilation failed for shoot %s, reason: %s, exiting with no retry", s.shoot.Name, reason) diff --git a/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go b/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go index 2c0036fc..d0322379 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go +++ b/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go @@ -39,7 +39,7 @@ func sFnWaitForShootCreation(_ context.Context, m *fsm, s *systemState) (stateFn "Unknown", "Shoot creation in progress") - return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) + return updateStatusAndRequeueAfter(m.RCCfg.RequeueDurationShootCreate) case gardener.LastOperationStateFailed: lastErrors := s.shoot.Status.LastErrors @@ -52,7 +52,7 @@ func sFnWaitForShootCreation(_ context.Context, m *fsm, s *systemState) (stateFn imv1.ConditionReasonShootCreationPending, "Unknown", "Retryable gardener errors during cluster provisioning") - return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) + return updateStatusAndRequeueAfter(m.RCCfg.RequeueDurationShootCreate) } msg := fmt.Sprintf("Provisioning failed for shoot: %s ! Last state: %s, Description: %s", s.shoot.Name, s.shoot.Status.LastOperation.State, s.shoot.Status.LastOperation.Description) diff --git a/internal/controller/runtime/fsm/utilz_for_test.go b/internal/controller/runtime/fsm/utilz_for_test.go index f5134dd7..d0c8801c 100644 --- a/internal/controller/runtime/fsm/utilz_for_test.go +++ b/internal/controller/runtime/fsm/utilz_for_test.go @@ -24,6 +24,8 @@ type fakeFSMOpt func(*fsm) error const defaultControlPlaneRequeueDuration = 10 * time.Second const defaultGardenerRequeueDuration = 15 * time.Second +const defaultRequeueDurationShootCreate = 15 * time.Second +const defaultRequeueDurationShootDelete = 15 * time.Second var ( errFailedToCreateFakeFSM = fmt.Errorf("failed to create fake FSM") diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index ed2dc768..ea65147f 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -51,6 +51,8 @@ var requCounter = 0 // nolint:gochecknoglobals func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { r.Log.Info(request.String()) + // n\] + var runtime imv1.Runtime if err := r.Get(ctx, request.NamespacedName, &runtime); err != nil { return ctrl.Result{ diff --git a/internal/controller/runtime/suite_test.go b/internal/controller/runtime/suite_test.go index 473b8948..8eb94681 100644 --- a/internal/controller/runtime/suite_test.go +++ b/internal/controller/runtime/suite_test.go @@ -121,12 +121,15 @@ var _ = BeforeSuite(func() { mm.On("CleanUpRuntimeGauge", mock.Anything, mock.Anything).Return() fsmCfg := fsm.RCCfg{ - Finalizer: imv1.Finalizer, - Config: convConfig, - Metrics: mm, - AuditLogging: map[string]map[string]auditlogs.AuditLogData{}, - GardenerRequeueDuration: 3 * time.Second, - ControlPlaneRequeueDuration: 3 * time.Second, + Finalizer: imv1.Finalizer, + Config: convConfig, + Metrics: mm, + AuditLogging: map[string]map[string]auditlogs.AuditLogData{}, + GardenerRequeueDuration: 3 * time.Second, + ControlPlaneRequeueDuration: 3 * time.Second, + RequeueDurationShootReconcile: 3 * time.Second, + RequeueDurationShootCreate: 3 * time.Second, + RequeueDurationShootDelete: 3 * time.Second, } runtimeReconciler = NewRuntimeReconciler(mgr, gardenerTestClient, logger, fsmCfg) From 96ecad7408e6dbb08376bd0bdcde74891cd81eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Mon, 25 Nov 2024 13:18:48 +0100 Subject: [PATCH 14/30] linter fix --- .../runtime/fsm/runtime_fsm_apply_clusterrolebindings.go | 3 ++- internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go index 7072709c..87613dba 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go +++ b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go @@ -3,6 +3,8 @@ package fsm import ( "context" "fmt" + "slices" + imv1 "github.com/kyma-project/infrastructure-manager/api/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -12,7 +14,6 @@ import ( "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "slices" ) var ( diff --git a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go index 2cdfe46d..0875cbb9 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go @@ -2,6 +2,7 @@ package fsm import ( "context" + gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" "k8s.io/utils/ptr" From 04ee03602ba2a8ccea88b4fff6877398a14050c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Mon, 25 Nov 2024 14:18:36 +0100 Subject: [PATCH 15/30] making linter happy fix --- cmd/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 429329c5..a1d15634 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -71,7 +71,7 @@ func init() { const ( defaultControlPlaneRequeueDuration = 10 * time.Second defaultGardenerRequestTimeout = 3 * time.Second - defaultGardenerRateLimiterQps = 5 + defaultGardenerRateLimiterQPS = 5 defaultGardenerRateLimiterBurst = 5 defaultMinimalRotationTimeRatio = 0.6 defaultExpirationTime = 24 * time.Hour @@ -92,7 +92,7 @@ func main() { var expirationTime time.Duration var gardenerCtrlReconciliationTimeout time.Duration var runtimeCtrlGardenerRequestTimeout time.Duration - var runtimeCtrlGardenerRateLimiterQps int + var runtimeCtrlGardenerRateLimiterQPS int var runtimeCtrlGardenerRateLimiterBurst int var converterConfigFilepath string var shootSpecDumpEnabled bool @@ -109,7 +109,7 @@ func main() { flag.DurationVar(&expirationTime, "kubeconfig-expiration-time", defaultExpirationTime, "Dynamic kubeconfig expiration time") flag.DurationVar(&gardenerCtrlReconciliationTimeout, "gardener-ctrl-reconcilation-timeout", defaultGardenerReconciliationTimeout, "Timeout duration for reconlication for Gardener Cluster Controller") flag.DurationVar(&runtimeCtrlGardenerRequestTimeout, "gardener-request-timeout", defaultGardenerRequestTimeout, "Timeout duration for Gardener client for Runtime Controller") - flag.IntVar(&runtimeCtrlGardenerRateLimiterQps, "gardener-ratelimiter-qps", defaultGardenerRateLimiterQps, "Gardener client rate limiter QPS for Runtime Controller") + flag.IntVar(&runtimeCtrlGardenerRateLimiterQPS, "gardener-ratelimiter-qps", defaultGardenerRateLimiterQPS, "Gardener client rate limiter QPS for Runtime Controller") flag.IntVar(&runtimeCtrlGardenerRateLimiterBurst, "gardener-ratelimiter-burst", defaultGardenerRateLimiterBurst, "Gardener client rate limiter burst for Runtime Controller") flag.StringVar(&converterConfigFilepath, "converter-config-filepath", "/converter-config/converter_config.json", "A file path to the gardener shoot converter configuration.") flag.BoolVar(&shootSpecDumpEnabled, "shoot-spec-dump-enabled", false, "Feature flag to allow persisting specs of created shoots") @@ -153,7 +153,7 @@ func main() { } gardenerNamespace := fmt.Sprintf("garden-%s", gardenerProjectName) - gardenerClient, shootClient, dynamicKubeconfigClient, err := initGardenerClients(gardenerKubeconfigPath, gardenerNamespace, runtimeCtrlGardenerRequestTimeout, runtimeCtrlGardenerRateLimiterQps, runtimeCtrlGardenerRateLimiterBurst) + gardenerClient, shootClient, dynamicKubeconfigClient, err := initGardenerClients(gardenerKubeconfigPath, gardenerNamespace, runtimeCtrlGardenerRequestTimeout, runtimeCtrlGardenerRateLimiterQPS, runtimeCtrlGardenerRateLimiterBurst) if err != nil { setupLog.Error(err, "unable to initialize gardener clients", "controller", "GardenerCluster") From e1e75a10d512b5994ceb085b8689cbc52c8b0ff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Mon, 25 Nov 2024 14:33:01 +0100 Subject: [PATCH 16/30] remove magic number to satisfy linter --- internal/controller/runtime/runtime_controller.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index ea65147f..03829ed3 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -31,6 +31,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ) +const ( + numberOfWorkers = 25 +) + // RuntimeReconciler reconciles a Runtime object // nolint:revive type RuntimeReconciler struct { @@ -51,8 +55,6 @@ var requCounter = 0 // nolint:gochecknoglobals func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { r.Log.Info(request.String()) - // n\] - var runtime imv1.Runtime if err := r.Get(ctx, request.NamespacedName, &runtime); err != nil { return ctrl.Result{ @@ -89,7 +91,7 @@ func NewRuntimeReconciler(mgr ctrl.Manager, shootClient client.Client, logger lo func (r *RuntimeReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&imv1.Runtime{}). - WithOptions(controller.Options{MaxConcurrentReconciles: 25}). + WithOptions(controller.Options{MaxConcurrentReconciles: numberOfWorkers}). WithEventFilter(predicate.Or( predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}, From 946289830b57b6fbb822d1517a2a63666d61897a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Golicz?= Date: Mon, 25 Nov 2024 15:21:36 +0100 Subject: [PATCH 17/30] Increase number of workers for GardenerCluster controller to 25 --- internal/controller/kubeconfig/gardener_cluster_controller.go | 3 +++ internal/controller/runtime/runtime_controller.go | 4 +--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/controller/kubeconfig/gardener_cluster_controller.go b/internal/controller/kubeconfig/gardener_cluster_controller.go index c532e203..67aa9f5b 100644 --- a/internal/controller/kubeconfig/gardener_cluster_controller.go +++ b/internal/controller/kubeconfig/gardener_cluster_controller.go @@ -34,6 +34,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + pkgctrl "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -43,6 +44,7 @@ const ( clusterCRNameLabel = "operator.kyma-project.io/cluster-name" rotationPeriodRatio = 0.95 + numberOfWorkers = 25 ) // GardenerClusterController reconciles a GardenerCluster object @@ -436,5 +438,6 @@ func (controller *GardenerClusterController) SetupWithManager(mgr ctrl.Manager) predicate.AnnotationChangedPredicate{}, predicate.GenerationChangedPredicate{}), )). + WithOptions(pkgctrl.Options{MaxConcurrentReconciles: numberOfWorkers}). Complete(controller) } diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index 03829ed3..f621ae40 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -31,9 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ) -const ( - numberOfWorkers = 25 -) +const numberOfWorkers = 25 // RuntimeReconciler reconciles a Runtime object // nolint:revive From b208dfe25590ee7d16351ec61028795d6be9cba2 Mon Sep 17 00:00:00 2001 From: VOID404 Date: Fri, 22 Nov 2024 08:48:33 +0100 Subject: [PATCH 18/30] Add runtime + shoot to FSM logs --- internal/controller/runtime/runtime_controller.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index f621ae40..4e84dbbe 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -62,8 +62,18 @@ func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) r.Log.Info("Reconciling Runtime", "Name", runtime.Name, "Namespace", runtime.Namespace) + runtimeID, ok := runtime.Labels["kyma-project.io/runtime-id"] + if !ok { + runtimeID = runtime.Name + } + + shootName, ok := runtime.Labels["kyma-project.io/shoot-name"] + if !ok { + shootName = "N/D" + } + stateFSM := fsm.NewFsm( - r.Log.WithName(fmt.Sprintf("reqID %d", requCounter)), + r.Log.WithName(fmt.Sprintf("reqID %d", requCounter)).WithValues("runtime", runtimeID, "shoot", shootName), r.Cfg, fsm.K8s{ Client: r.Client, From d5c39b1d3df3096ddeee9458d7c24fb432f443b8 Mon Sep 17 00:00:00 2001 From: VOID404 Date: Fri, 22 Nov 2024 08:56:39 +0100 Subject: [PATCH 19/30] Reorder fields and logs --- internal/controller/runtime/runtime_controller.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index 4e84dbbe..4a71edd2 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -60,8 +60,6 @@ func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) }, client.IgnoreNotFound(err) } - r.Log.Info("Reconciling Runtime", "Name", runtime.Name, "Namespace", runtime.Namespace) - runtimeID, ok := runtime.Labels["kyma-project.io/runtime-id"] if !ok { runtimeID = runtime.Name @@ -72,8 +70,11 @@ func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) shootName = "N/D" } + log := r.Log.WithValues("runtime", runtimeID, "shoot", shootName) + log.Info("Reconciling Runtime", "Name", runtime.Name, "Namespace", runtime.Namespace) + stateFSM := fsm.NewFsm( - r.Log.WithName(fmt.Sprintf("reqID %d", requCounter)).WithValues("runtime", runtimeID, "shoot", shootName), + log.WithName(fmt.Sprintf("reqID %d", requCounter)), r.Cfg, fsm.K8s{ Client: r.Client, From be42c93dd40a78ef2ae1097431c34a1f82e33c85 Mon Sep 17 00:00:00 2001 From: VOID404 Date: Mon, 25 Nov 2024 08:50:39 +0100 Subject: [PATCH 20/30] Improve log field names --- internal/controller/runtime/runtime_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index 4a71edd2..abe33778 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -70,7 +70,7 @@ func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) shootName = "N/D" } - log := r.Log.WithValues("runtime", runtimeID, "shoot", shootName) + log := r.Log.WithValues("runtimeID", runtimeID, "shootID", shootName) log.Info("Reconciling Runtime", "Name", runtime.Name, "Namespace", runtime.Namespace) stateFSM := fsm.NewFsm( From 240ef20165d02c564a7a4fe3c6c1eb8e6bb07816 Mon Sep 17 00:00:00 2001 From: VOID404 Date: Mon, 25 Nov 2024 08:53:44 +0100 Subject: [PATCH 21/30] Log patch shoot error --- internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go index 506fd052..b435f8a9 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go @@ -67,7 +67,7 @@ func sFnPatchExistingShoot(ctx context.Context, m *fsm, s *systemState) (stateFn m.log.Error(err, "Failed to patch shoot object, exiting with no retry") m.Metrics.IncRuntimeFSMStopCounter() - return updateStatePendingWithErrorAndStop(&s.instance, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonProcessingErr, "Shoot patch error") + return updateStatePendingWithErrorAndStop(&s.instance, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonProcessingErr, fmt.Sprintf("Gardener API shoot patch error: %v", err)) } if updatedShoot.Generation == s.shoot.Generation { From 36eee00c9f28470b54f00cc2e88250757b664d11 Mon Sep 17 00:00:00 2001 From: VOID404 Date: Mon, 25 Nov 2024 09:01:57 +0100 Subject: [PATCH 22/30] Add missing import --- internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go index b435f8a9..f45df6c9 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go @@ -3,6 +3,7 @@ package fsm import ( "context" "slices" + "fmt" gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" From 43bd6793daa43226a9260ab4429cf6b5006aac2c Mon Sep 17 00:00:00 2001 From: VOID404 Date: Mon, 25 Nov 2024 10:48:06 +0100 Subject: [PATCH 23/30] Sort imports --- hack/runtime-migrator/cmd/main.go | 9 ++++----- hack/runtime-migrator/cmd/migration.go | 2 +- hack/runtime-migrator/internal/config/config.go | 3 ++- hack/runtime-migrator/internal/migration/output.go | 7 ++++--- hack/runtime-migrator/internal/runtime/migrator.go | 1 + hack/runtime-migrator/internal/runtime/verifier.go | 3 ++- internal/controller/metrics/mocks/Metrics.go | 6 ++---- .../controller/runtime/fsm/runtime_fsm_patch_shoot.go | 2 +- 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/hack/runtime-migrator/cmd/main.go b/hack/runtime-migrator/cmd/main.go index eb00d060..71e5c25b 100644 --- a/hack/runtime-migrator/cmd/main.go +++ b/hack/runtime-migrator/cmd/main.go @@ -11,18 +11,17 @@ import ( "strings" "time" - "github.com/go-playground/validator/v10" - "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" - v12 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "github.com/gardener/gardener/pkg/apis/core/v1beta1" gardener_types "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" + "github.com/go-playground/validator/v10" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/config" kimConfig "github.com/kyma-project/infrastructure-manager/pkg/config" "github.com/kyma-project/infrastructure-manager/pkg/gardener" "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" + "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" "github.com/pkg/errors" + v12 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" diff --git a/hack/runtime-migrator/cmd/migration.go b/hack/runtime-migrator/cmd/migration.go index 263d38ef..a496fe96 100644 --- a/hack/runtime-migrator/cmd/migration.go +++ b/hack/runtime-migrator/cmd/migration.go @@ -3,7 +3,6 @@ package main import ( "context" "fmt" - "github.com/pkg/errors" "log/slog" "github.com/gardener/gardener/pkg/apis/core/v1beta1" @@ -15,6 +14,7 @@ import ( "github.com/kyma-project/infrastructure-manager/pkg/config" "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" + "github.com/pkg/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/hack/runtime-migrator/internal/config/config.go b/hack/runtime-migrator/internal/config/config.go index 451968a9..2f1a60ac 100644 --- a/hack/runtime-migrator/internal/config/config.go +++ b/hack/runtime-migrator/internal/config/config.go @@ -3,11 +3,12 @@ package config import ( "flag" "fmt" + "log" + v1 "github.com/kyma-project/infrastructure-manager/api/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/clientcmd" - "log" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/hack/runtime-migrator/internal/migration/output.go b/hack/runtime-migrator/internal/migration/output.go index e34d13bc..24d4fff0 100644 --- a/hack/runtime-migrator/internal/migration/output.go +++ b/hack/runtime-migrator/internal/migration/output.go @@ -3,12 +3,13 @@ package migration import ( "encoding/json" "fmt" - v1 "github.com/kyma-project/infrastructure-manager/api/v1" - "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/runtime" "os" "path" - "sigs.k8s.io/yaml" "time" + + v1 "github.com/kyma-project/infrastructure-manager/api/v1" + "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/runtime" + "sigs.k8s.io/yaml" ) type OutputWriter struct { diff --git a/hack/runtime-migrator/internal/runtime/migrator.go b/hack/runtime-migrator/internal/runtime/migrator.go index d63aa723..45574acd 100644 --- a/hack/runtime-migrator/internal/runtime/migrator.go +++ b/hack/runtime-migrator/internal/runtime/migrator.go @@ -3,6 +3,7 @@ package runtime import ( "context" "fmt" + "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1 "github.com/kyma-project/infrastructure-manager/api/v1" migrator "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/config" diff --git a/hack/runtime-migrator/internal/runtime/verifier.go b/hack/runtime-migrator/internal/runtime/verifier.go index 87451a54..af3e40b0 100644 --- a/hack/runtime-migrator/internal/runtime/verifier.go +++ b/hack/runtime-migrator/internal/runtime/verifier.go @@ -1,6 +1,8 @@ package runtime import ( + "slices" + "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/infrastructure-manager/hack/shoot-comparator/pkg/shoot" @@ -8,7 +10,6 @@ import ( gardener_shoot "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" "k8s.io/utils/ptr" - "slices" ) type Verifier struct { diff --git a/internal/controller/metrics/mocks/Metrics.go b/internal/controller/metrics/mocks/Metrics.go index ffd12e7d..737e7d73 100644 --- a/internal/controller/metrics/mocks/Metrics.go +++ b/internal/controller/metrics/mocks/Metrics.go @@ -3,13 +3,11 @@ package mocks import ( - corev1 "k8s.io/api/core/v1" - - mock "github.com/stretchr/testify/mock" - time "time" v1 "github.com/kyma-project/infrastructure-manager/api/v1" + mock "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" ) // Metrics is an autogenerated mock type for the Metrics type diff --git a/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go index f45df6c9..664109ab 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go @@ -2,8 +2,8 @@ package fsm import ( "context" - "slices" "fmt" + "slices" gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" From 71979e44785662558adab000ed245be9c602086c Mon Sep 17 00:00:00 2001 From: VOID404 Date: Mon, 25 Nov 2024 14:59:18 +0100 Subject: [PATCH 24/30] fixup! Improve log field names --- internal/controller/runtime/runtime_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index abe33778..bf898fda 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -70,7 +70,7 @@ func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) shootName = "N/D" } - log := r.Log.WithValues("runtimeID", runtimeID, "shootID", shootName) + log := r.Log.WithValues("runtimeID", runtimeID, "shootName", shootName) log.Info("Reconciling Runtime", "Name", runtime.Name, "Namespace", runtime.Namespace) stateFSM := fsm.NewFsm( From 83990ce1d82cb95ccd959570badd6fcd5547a308 Mon Sep 17 00:00:00 2001 From: VOID404 Date: Mon, 25 Nov 2024 15:00:30 +0100 Subject: [PATCH 25/30] Add requestID to logs --- internal/controller/runtime/runtime_controller.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index bf898fda..9d39a760 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -19,6 +19,7 @@ package runtime import ( "context" "fmt" + "sync/atomic" "github.com/go-logr/logr" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" @@ -42,6 +43,7 @@ type RuntimeReconciler struct { Log logr.Logger Cfg fsm.RCCfg EventRecorder record.EventRecorder + RequestID atomic.Uint64 } //+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes,verbs=get;list;watch;create;update;patch;delete @@ -70,11 +72,11 @@ func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) shootName = "N/D" } - log := r.Log.WithValues("runtimeID", runtimeID, "shootName", shootName) + log := r.Log.WithValues("runtimeID", runtimeID, "shootName", shootName, "requestID", r.RequestID.Add(1)) log.Info("Reconciling Runtime", "Name", runtime.Name, "Namespace", runtime.Namespace) stateFSM := fsm.NewFsm( - log.WithName(fmt.Sprintf("reqID %d", requCounter)), + log, r.Cfg, fsm.K8s{ Client: r.Client, From ca3135136a0df7695a801c6c82a3498366fd6196 Mon Sep 17 00:00:00 2001 From: VOID404 Date: Mon, 25 Nov 2024 15:01:39 +0100 Subject: [PATCH 26/30] Correct imports --- internal/controller/runtime/runtime_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index 9d39a760..c4af0468 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -18,7 +18,6 @@ package runtime import ( "context" - "fmt" "sync/atomic" "github.com/go-logr/logr" From 83d2e2876acb8a8c0b3b74b915551c3cafe717b5 Mon Sep 17 00:00:00 2001 From: VOID404 Date: Tue, 26 Nov 2024 09:17:59 +0100 Subject: [PATCH 27/30] Disable debug logger --- cmd/main.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index a1d15634..8d616bee 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -115,9 +115,7 @@ func main() { flag.BoolVar(&shootSpecDumpEnabled, "shoot-spec-dump-enabled", false, "Feature flag to allow persisting specs of created shoots") flag.BoolVar(&auditLogMandatory, "audit-log-mandatory", true, "Feature flag to enable strict mode for audit log configuration") - opts := zap.Options{ - Development: true, - } + opts := zap.Options{} opts.BindFlags(flag.CommandLine) flag.Parse() From 90d203d67f7924a7d5281337924c2a28d9fdbc16 Mon Sep 17 00:00:00 2001 From: Rafal Foks Date: Tue, 26 Nov 2024 10:24:14 +0100 Subject: [PATCH 28/30] Revert RBAC changes in favor of different PR --- config/rbac/cluster_editor_role.yaml | 5 ++-- config/rbac/cluster_viewer_role.yaml | 3 +-- config/rbac/role.yaml | 26 ++++++++++++++++--- config/rbac/role_binding.yaml | 8 +++--- config/rbac/runtime_editor_role.yaml | 3 +-- config/rbac/runtime_viewer_role.yaml | 3 +-- config/rbac/service_account.yaml | 2 +- .../kubeconfig/gardener_cluster_controller.go | 8 +++--- .../controller/runtime/runtime_controller.go | 6 ++--- 9 files changed, 40 insertions(+), 24 deletions(-) diff --git a/config/rbac/cluster_editor_role.yaml b/config/rbac/cluster_editor_role.yaml index 84dda500..64abe8ce 100644 --- a/config/rbac/cluster_editor_role.yaml +++ b/config/rbac/cluster_editor_role.yaml @@ -1,16 +1,15 @@ # permissions for end users to edit clusters. apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: labels: - app.kubernetes.io/name: 1errole + app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: cluster-editor-role app.kubernetes.io/component: rbac app.kubernetes.io/created-by: infrastructure-manager app.kubernetes.io/part-of: infrastructure-manager app.kubernetes.io/managed-by: kustomize name: cluster-editor-role - namespace: kcp-system rules: - apiGroups: - infrastructuremanager.kyma-project.io diff --git a/config/rbac/cluster_viewer_role.yaml b/config/rbac/cluster_viewer_role.yaml index 43df4e6d..d183607b 100644 --- a/config/rbac/cluster_viewer_role.yaml +++ b/config/rbac/cluster_viewer_role.yaml @@ -1,6 +1,6 @@ # permissions for end users to view clusters. apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole @@ -10,7 +10,6 @@ metadata: app.kubernetes.io/part-of: infrastructure-manager app.kubernetes.io/managed-by: kustomize name: cluster-viewer-role - namespace: kcp-system rules: - apiGroups: - infrastructuremanager.kyma-project.io diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6b06da55..7681c444 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -1,9 +1,8 @@ --- apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: name: infrastructure-manager-role - namespace: kcp-system rules: - apiGroups: - "" @@ -20,7 +19,6 @@ rules: - infrastructuremanager.kyma-project.io resources: - gardenerclusters - - runtimes verbs: - create - delete @@ -33,7 +31,29 @@ rules: - infrastructuremanager.kyma-project.io resources: - gardenerclusters/finalizers + verbs: + - update +- apiGroups: + - infrastructuremanager.kyma-project.io + resources: - gardenerclusters/status + verbs: + - update +- apiGroups: + - infrastructuremanager.kyma-project.io + resources: + - runtimes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructuremanager.kyma-project.io + resources: - runtimes/finalizers verbs: - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 6cd75b05..60f28ad3 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -1,8 +1,8 @@ apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding +kind: ClusterRoleBinding metadata: labels: - app.kubernetes.io/name: rolebinding + app.kubernetes.io/name: clusterrolebinding app.kubernetes.io/instance: infrastructure-manager-rolebinding app.kubernetes.io/component: rbac app.kubernetes.io/created-by: infrastructure-manager @@ -11,9 +11,9 @@ metadata: name: infrastructure-manager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io - kind: Role + kind: ClusterRole name: infrastructure-manager-role subjects: - kind: ServiceAccount name: infrastructure-manager - namespace: kcp-system + namespace: system diff --git a/config/rbac/runtime_editor_role.yaml b/config/rbac/runtime_editor_role.yaml index ceb3ba03..014838b7 100644 --- a/config/rbac/runtime_editor_role.yaml +++ b/config/rbac/runtime_editor_role.yaml @@ -1,12 +1,11 @@ # permissions for end users to edit runtimes. apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: labels: app.kubernetes.io/name: infrastructure-manager app.kubernetes.io/managed-by: kustomize name: runtime-editor-role - namespace: kcp-system rules: - apiGroups: - infrastructuremanager.kyma-project.io diff --git a/config/rbac/runtime_viewer_role.yaml b/config/rbac/runtime_viewer_role.yaml index 0c5ac175..d9d0024e 100644 --- a/config/rbac/runtime_viewer_role.yaml +++ b/config/rbac/runtime_viewer_role.yaml @@ -1,12 +1,11 @@ # permissions for end users to view runtimes. apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: labels: app.kubernetes.io/name: infrastructure-manager app.kubernetes.io/managed-by: kustomize name: runtime-viewer-role - namespace: kcp-system rules: - apiGroups: - infrastructuremanager.kyma-project.io diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index 3ddd25a9..5c3330c2 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -9,4 +9,4 @@ metadata: app.kubernetes.io/part-of: infrastructure-manager app.kubernetes.io/managed-by: kustomize name: infrastructure-manager - namespace: kcp-system + namespace: system diff --git a/internal/controller/kubeconfig/gardener_cluster_controller.go b/internal/controller/kubeconfig/gardener_cluster_controller.go index 87cba31e..c532e203 100644 --- a/internal/controller/kubeconfig/gardener_cluster_controller.go +++ b/internal/controller/kubeconfig/gardener_cluster_controller.go @@ -77,10 +77,10 @@ type KubeconfigProvider interface { Fetch(ctx context.Context, shootName string) (string, error) } -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters,verbs=get;list;watch;create;update;patch;delete,namespace=kcp-system -//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;delete,namespace=kcp-system -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters/finalizers,verbs=update,namespace=kcp-system -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters/status,verbs=update,namespace=kcp-system +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;delete +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters/finalizers,verbs=update +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=gardenerclusters/status,verbs=update // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index 5aa8f530..1f8eb574 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -41,9 +41,9 @@ type RuntimeReconciler struct { EventRecorder record.EventRecorder } -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes,verbs=get;list;watch;create;update;patch;delete,namespace=kcp-system -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes/status,verbs=get;update;patch,namespace=kcp-system -//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes/finalizers,verbs=update,namespace=kcp-system +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes/finalizers,verbs=update var requCounter = 0 // nolint:gochecknoglobals From 11bf66f912fccdb36a53686a2b5abc1b13f3fab6 Mon Sep 17 00:00:00 2001 From: VOID404 Date: Tue, 26 Nov 2024 10:32:25 +0100 Subject: [PATCH 29/30] Remove unused global --- internal/controller/runtime/runtime_controller.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index c4af0468..20f181fb 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -49,8 +49,6 @@ type RuntimeReconciler struct { //+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes/status,verbs=get;update;patch //+kubebuilder:rbac:groups=infrastructuremanager.kyma-project.io,resources=runtimes/finalizers,verbs=update -var requCounter = 0 // nolint:gochecknoglobals - func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { r.Log.Info(request.String()) From 6b0f498f81f55349a0620a6b0227329e0a8d26f9 Mon Sep 17 00:00:00 2001 From: Rafal Foks Date: Tue, 26 Nov 2024 10:50:17 +0100 Subject: [PATCH 30/30] Fix linter --- cmd/main.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index f84901bb..3806bdac 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -25,9 +25,6 @@ import ( "os" "time" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/cache" - "github.com/gardener/gardener/pkg/apis/core/v1beta1" gardener_apis "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" gardener_oidc "github.com/gardener/oidc-webhook-authenticator/apis/authentication/v1alpha1" @@ -43,6 +40,7 @@ import ( "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" k8slabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -52,6 +50,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/util/flowcontrol" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap"