diff --git a/api/v1/runtime_types.go b/api/v1/runtime_types.go index 2b4ca2f7..be944954 100644 --- a/api/v1/runtime_types.go +++ b/api/v1/runtime_types.go @@ -239,3 +239,19 @@ func (k *Runtime) IsConditionSet(c RuntimeConditionType, r RuntimeConditionReaso } return false } + +func (k *Runtime) IsStateWithConditionAndStatusSet(runtimeState State, c RuntimeConditionType, r RuntimeConditionReason, s metav1.ConditionStatus) bool { + if k.Status.State != runtimeState { + return false + } + + return k.IsConditionSetWithStatus(c, r, s) +} + +func (k *Runtime) IsConditionSetWithStatus(c RuntimeConditionType, r RuntimeConditionReason, s metav1.ConditionStatus) bool { + condition := meta.FindStatusCondition(k.Status.Conditions, string(c)) + if condition != nil && condition.Reason == string(r) && condition.Status == s { + return true + } + return false +} diff --git a/cmd/main.go b/cmd/main.go index 2d72f3ba..0d4ffd31 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -118,14 +118,15 @@ func main() { } gardenerNamespace := fmt.Sprintf("garden-%s", gardenerProjectName) - shootClient, dynamicKubeconfigClient, err := initGardenerClients(gardenerKubeconfigPath, gardenerNamespace) + gardenerClient, shootClient, dynamicKubeconfigClient, err := initGardenerClients(gardenerKubeconfigPath, gardenerNamespace) if err != nil { setupLog.Error(err, "unable to initialize gardener clients", "controller", "GardenerCluster") os.Exit(1) } - kubeconfigProvider := kubeconfig.NewKubeconfigProvider(shootClient, + kubeconfigProvider := kubeconfig.NewKubeconfigProvider( + shootClient, dynamicKubeconfigClient, gardenerNamespace, int64(expirationTime.Seconds())) @@ -144,21 +145,19 @@ func main() { os.Exit(1) } - cfg := fsm.RCCfg{Finalizer: infrastructuremanagerv1.Finalizer} + cfg := fsm.RCCfg{ + Finalizer: infrastructuremanagerv1.Finalizer, + ShootNamesapace: gardenerNamespace, + } if persistShoot { cfg.PVCPath = "/testdata/kim" } if enableRuntimeReconciler { - if err = (&runtime_controller.RuntimeReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - ShootClient: shootClient, - Log: logger, - Cfg: cfg, - EventRecorder: mgr.GetEventRecorderFor("runtime-controller"), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Runtime") + runtimeReconciler := runtime_controller.NewRuntimeReconciler(mgr, gardenerClient, logger, cfg) + + if err = runtimeReconciler.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to setup controller with Manager", "controller", "Runtime") os.Exit(1) } } @@ -182,20 +181,20 @@ func main() { } } -func initGardenerClients(kubeconfigPath string, namespace string) (gardener_apis.ShootInterface, client.SubResourceClient, error) { +func initGardenerClients(kubeconfigPath string, namespace string) (client.Client, gardener_apis.ShootInterface, client.SubResourceClient, error) { restConfig, err := gardener.NewRestConfigFromFile(kubeconfigPath) if err != nil { - return nil, nil, err + return nil, nil, nil, err } gardenerClientSet, err := gardener_apis.NewForConfig(restConfig) if err != nil { - return nil, nil, err + return nil, nil, nil, err } gardenerClient, err := client.New(restConfig, client.Options{}) if err != nil { - return nil, nil, err + return nil, nil, nil, err } shootClient := gardenerClientSet.Shoots(namespace) @@ -203,8 +202,8 @@ func initGardenerClients(kubeconfigPath string, namespace string) (gardener_apis err = v1beta1.AddToScheme(gardenerClient.Scheme()) if err != nil { - return nil, nil, errors.Wrap(err, "failed to register Gardener schema") + return nil, nil, nil, errors.Wrap(err, "failed to register Gardener schema") } - return shootClient, dynamicKubeconfigAPI, nil + return gardenerClient, shootClient, dynamicKubeconfigAPI, nil } diff --git a/go.mod b/go.mod index 2fd813b7..331fd85a 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 - k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 sigs.k8s.io/controller-runtime v0.18.4 sigs.k8s.io/yaml v1.4.0 ) @@ -76,6 +75,7 @@ require ( k8s.io/apiextensions-apiserver v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/internal/controller/runtime/fsm/runtime_fsm.go b/internal/controller/runtime/fsm/runtime_fsm.go index d2ac8b39..7944821b 100644 --- a/internal/controller/runtime/fsm/runtime_fsm.go +++ b/internal/controller/runtime/fsm/runtime_fsm.go @@ -10,7 +10,6 @@ import ( "github.com/go-logr/logr" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" - "github.com/kyma-project/infrastructure-manager/internal/gardener" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,8 +27,9 @@ type writerGetter = func(filePath string) (io.Writer, error) // runtime reconciler specific configuration type RCCfg struct { - Finalizer string - PVCPath string + Finalizer string + PVCPath string + ShootNamesapace string } func (f stateFn) String() string { @@ -46,7 +46,7 @@ type Watch = func(src source.Source, eventhandler handler.EventHandler, predicat type K8s struct { client.Client record.EventRecorder - ShootClient gardener.ShootClient + ShootClient client.Client } type Fsm interface { diff --git a/internal/controller/runtime/fsm/runtime_fsm_create_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_create_shoot.go index fa5f04bd..dcab0d0a 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_create_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_create_shoot.go @@ -4,32 +4,18 @@ import ( "context" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" - gardener_shoot "github.com/kyma-project/infrastructure-manager/internal/gardener/shoot" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" ) func sFnCreateShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { - converterConfig := FixConverterConfig() - converter := gardener_shoot.NewConverter(converterConfig) - shoot, err := converter.ToShoot(s.instance) - + m.log.Info("Create shoot") + newShoot, err := convertShoot(&s.instance) if err != nil { - m.log.Error(err, "unable to convert Runtime CR to a shoot object") - - s.instance.UpdateStatePending( - imv1.ConditionTypeRuntimeProvisioned, - imv1.ConditionReasonConversionError, - "False", - "Runtime conversion error", - ) - - return updateStatusAndStop() + m.log.Error(err, "Failed to convert Runtime instance to shoot object") + return updateStatePendingWithErrorAndStop(&s.instance, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonConversionError, "Runtime conversion error") } - m.log.Info("Shoot converted successfully", "Name", shoot.Name, "Namespace", shoot.Namespace, "Shoot", shoot) - - s.shoot, err = m.ShootClient.Create(ctx, &shoot, v1.CreateOptions{}) + err = m.ShootClient.Create(ctx, &newShoot) if err != nil { m.log.Error(err, "Failed to create new gardener Shoot") @@ -40,11 +26,9 @@ func sFnCreateShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl "False", "Gardener API create error", ) - return updateStatusAndRequeueAfter(gardenerRequeueDuration) } - - m.log.Info("Gardener shoot for runtime initialised successfully", "Name", s.shoot.Name, "Namespace", s.shoot.Namespace) + m.log.Info("Gardener shoot for runtime initialised successfully", "Name", newShoot.Name, "Namespace", newShoot.Namespace) s.instance.UpdateStatePending( imv1.ConditionTypeRuntimeProvisioned, @@ -55,30 +39,9 @@ func sFnCreateShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl shouldPersistShoot := m.PVCPath != "" if shouldPersistShoot { + s.shoot = newShoot.DeepCopy() return switchState(sFnPersistShoot) } return updateStatusAndRequeueAfter(gardenerRequeueDuration) } - -func FixConverterConfig() gardener_shoot.ConverterConfig { - return gardener_shoot.ConverterConfig{ - Kubernetes: gardener_shoot.KubernetesConfig{ - DefaultVersion: "1.29", //nolint:godox TODO: Should be parametrised - }, - - DNS: gardener_shoot.DNSConfig{ - SecretName: "aws-route53-secret-dev", - DomainPrefix: "dev.kyma.ondemand.com", - ProviderType: "aws-route53", - }, - Provider: gardener_shoot.ProviderConfig{ - AWS: gardener_shoot.AWSConfig{ - EnableIMDSv2: true, //nolint:godox TODO: Should be parametrised - }, - }, - Gardener: gardener_shoot.GardenerConfig{ - ProjectName: "kyma-dev", //nolint:godox TODO: should be parametrised - }, - } -} diff --git a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go index 8fa7d86a..172e470a 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go @@ -2,13 +2,10 @@ package fsm import ( "context" - "encoding/json" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" ) func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { @@ -16,27 +13,15 @@ func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl if !isGardenerCloudDelConfirmationSet(s.shoot.Annotations) { m.log.Info("patching shoot with del-confirmation") // workaround for Gardener client - s.shoot.Kind = "Shoot" - s.shoot.APIVersion = "core.gardener.cloud/v1beta1" + setObjectFields(s.shoot) s.shoot.Annotations = addGardenerCloudDelConfirmation(s.shoot.Annotations) - s.shoot.ManagedFields = nil - // attempt to marshall patched instance - shootData, err := json.Marshal(&s.shoot) + + err := m.ShootClient.Patch(ctx, s.shoot, client.Apply, &client.PatchOptions{ + FieldManager: "kim", + Force: ptrTo(true), + }) + if err != nil { - // unrecoverable error - s.instance.UpdateStateDeletion( - imv1.ConditionTypeRuntimeProvisioned, - imv1.ConditionReasonSerializationError, - "False", - err.Error()) - return updateStatusAndStop() - } - // see: https://gardener.cloud/docs/gardener/projects/#four-eyes-principle-for-resource-deletion - if s.shoot, err = m.ShootClient.Patch(ctx, s.shoot.Name, types.ApplyPatchType, shootData, - metav1.PatchOptions{ - FieldManager: "kim", - Force: ptr.To(true), - }); err != nil { m.log.Error(err, "unable to patch shoot:", s.shoot.Name) return requeue() } @@ -54,7 +39,7 @@ func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl } m.log.Info("deleting shoot") - err := m.ShootClient.Delete(ctx, s.instance.Name, metav1.DeleteOptions{}) + err := m.ShootClient.Delete(ctx, s.shoot) if err != nil { m.log.Error(err, "Failed to delete gardener Shoot") diff --git a/internal/controller/runtime/fsm/runtime_fsm_initialise.go b/internal/controller/runtime/fsm/runtime_fsm_initialise.go index ea1330de..1f5ff4cd 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_initialise.go +++ b/internal/controller/runtime/fsm/runtime_fsm_initialise.go @@ -37,9 +37,9 @@ func sFnInitialize(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl. return switchState(sFnCreateShoot) } - if instanceIsNotBeingDeleted && s.shoot != nil { + if instanceIsNotBeingDeleted { m.log.Info("Gardener shoot exists, processing") - return switchState(sFnPrepareCluster) // wait for pending shoot operation to complete + return switchState(sFnSelectShootProcessing) } // resource cleanup is done; diff --git a/internal/controller/runtime/fsm/runtime_fsm_initialise_test.go b/internal/controller/runtime/fsm/runtime_fsm_initialise_test.go index 563111f9..a9ac5362 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_initialise_test.go +++ b/internal/controller/runtime/fsm/runtime_fsm_initialise_test.go @@ -156,13 +156,13 @@ var _ = Describe("KIM sFnInitialise", func() { }, ), Entry( - "should return sFnPrepareCluster and no error when exists Provisioning Condition and shoot exists", + "should return sFnSelectShootProcessing and no error when exists Provisioning Condition and shoot exists", testCtx, must(newFakeFSM, withTestFinalizer), &systemState{instance: testRtWithFinalizerAndProvisioningCondition, shoot: &testShoot}, testOpts{ MatchExpectedErr: BeNil(), - MatchNextFnState: haveName("sFnPrepareCluster"), + MatchNextFnState: haveName("sFnSelectShootProcessing"), }, ), ) diff --git a/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go new file mode 100644 index 00000000..93704c05 --- /dev/null +++ b/internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go @@ -0,0 +1,108 @@ +package fsm + +import ( + "context" + + gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" + imv1 "github.com/kyma-project/infrastructure-manager/api/v1" + gardener_shoot "github.com/kyma-project/infrastructure-manager/internal/gardener/shoot" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func sFnPatchExistingShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { + m.log.Info("Patch shoot state") + + updatedShoot, err := convertShoot(&s.instance) + if err != nil { + m.log.Error(err, "Failed to convert Runtime instance to shoot object, exiting with no retry") + return updateStatePendingWithErrorAndStop(&s.instance, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonConversionError, "Runtime conversion error") + } + + m.log.Info("Shoot converted successfully", "Name", updatedShoot.Name, "Namespace", updatedShoot.Namespace) + + err = m.ShootClient.Patch(ctx, &updatedShoot, client.Apply, &client.PatchOptions{ + FieldManager: "kim", + Force: ptrTo(true), + }) + + if err != nil { + m.log.Error(err, "Failed to patch shoot object, exiting with no retry") + return updateStatePendingWithErrorAndStop(&s.instance, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonGardenerError, "Shoot patch error") + } + + if updatedShoot.Generation == s.shoot.Generation { + m.log.Info("Gardener shoot for runtime did not change after patch, moving to processing", "Name", s.shoot.Name, "Namespace", s.shoot.Namespace) + return switchState(sFnProcessShoot) + } + + m.log.Info("Gardener shoot for runtime patched successfully", "Name", s.shoot.Name, "Namespace", s.shoot.Namespace) + + s.shoot = updatedShoot.DeepCopy() + + s.instance.UpdateStatePending( + imv1.ConditionTypeRuntimeProvisioned, + imv1.ConditionReasonProcessing, + "Unknown", + "Shoot is pending", + ) + + shouldPersistShoot := m.PVCPath != "" + if shouldPersistShoot { + return switchState(sFnPersistShoot) + } + + return updateStatusAndRequeueAfter(gardenerRequeueDuration) +} + +func convertShoot(instance *imv1.Runtime) (gardener.Shoot, error) { + converterConfig := FixConverterConfig() + converter := gardener_shoot.NewConverter(converterConfig) + shoot, err := converter.ToShoot(*instance) // returned error is always nil BTW + + if err == nil { + setObjectFields(&shoot) + } + + return shoot, err +} + +func FixConverterConfig() gardener_shoot.ConverterConfig { + return gardener_shoot.ConverterConfig{ + Kubernetes: gardener_shoot.KubernetesConfig{ + DefaultVersion: "1.29", //nolint:godox TODO: Should be parametrised + }, + + DNS: gardener_shoot.DNSConfig{ + SecretName: "aws-route53-secret-dev", + DomainPrefix: "dev.kyma.ondemand.com", + ProviderType: "aws-route53", + }, + Provider: gardener_shoot.ProviderConfig{ + AWS: gardener_shoot.AWSConfig{ + EnableIMDSv2: true, //nolint:godox TODO: Should be parametrised + }, + }, + Gardener: gardener_shoot.GardenerConfig{ + ProjectName: "kyma-dev", //nolint:godox TODO: should be parametrised + }, + } +} + +// workaround +func setObjectFields(shoot *gardener.Shoot) { + shoot.Kind = "Shoot" + shoot.APIVersion = "core.gardener.cloud/v1beta1" + shoot.ManagedFields = nil +} + +func updateStatePendingWithErrorAndStop(instance *imv1.Runtime, + //nolint:unparam + c imv1.RuntimeConditionType, r imv1.RuntimeConditionReason, msg string) (stateFn, *ctrl.Result, error) { + instance.UpdateStatePending(c, r, "False", msg) + return updateStatusAndStop() +} + +func ptrTo[T any](v T) *T { + return &v +} diff --git a/internal/controller/runtime/fsm/runtime_fsm_persist_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_persist_shoot.go index 2c668b92..043fe874 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_persist_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_persist_shoot.go @@ -41,5 +41,5 @@ func sFnPersistShoot(_ context.Context, m *fsm, s *systemState) (stateFn, *ctrl. if err := persist(path, s.shoot, m.writerProvider); err != nil { return updateStatusAndStopWithError(err) } - return updateStatusAndRequeue() + return updateStatusAndRequeueAfter(gardenerRequeueDuration) } diff --git a/internal/controller/runtime/fsm/runtime_fsm_prepare_cluster.go b/internal/controller/runtime/fsm/runtime_fsm_prepare_cluster.go deleted file mode 100644 index 150f1c46..00000000 --- a/internal/controller/runtime/fsm/runtime_fsm_prepare_cluster.go +++ /dev/null @@ -1,153 +0,0 @@ -package fsm - -import ( - "context" - "fmt" - "strings" - - gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" - gardenerhelper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" - imv1 "github.com/kyma-project/infrastructure-manager/api/v1" - ctrl "sigs.k8s.io/controller-runtime" -) - -type ErrReason string - -func sFnPrepareCluster(_ context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { - if s.shoot == nil { - return updateStatusAndStopWithError(fmt.Errorf("fatal state machine logic problem: Shoot can never be nil in PrepareCluster state")) - } - - if s.shoot.Spec.DNS == nil || s.shoot.Spec.DNS.Domain == nil { - msg := fmt.Sprintf("DNS Domain is not set yet for shoot: %s, scheduling for retry", s.shoot.Name) - m.log.Info(msg) - return updateStatusAndRequeueAfter(gardenerRequeueDuration) - } - - lastOperation := s.shoot.Status.LastOperation - if lastOperation == nil { - msg := fmt.Sprintf("Last operation is nil for shoot: %s, scheduling for retry", s.shoot.Name) - m.log.Info(msg) - return updateStatusAndRequeueAfter(gardenerRequeueDuration) - } - - if lastOperation.Type == gardener.LastOperationTypeCreate && - (lastOperation.State == gardener.LastOperationStateProcessing || lastOperation.State == gardener.LastOperationStatePending) { - msg := fmt.Sprintf("Shoot %s is in %s state, scheduling for retry", s.shoot.Name, lastOperation.State) - m.log.Info(msg) - - s.instance.UpdateStatePending( - imv1.ConditionTypeRuntimeProvisioned, - imv1.ConditionReasonShootCreationPending, - "Unknown", - "Shoot creation in progress") - - return updateStatusAndRequeueAfter(gardenerRequeueDuration) - } - - if lastOperation.Type == gardener.LastOperationTypeCreate && lastOperation.State == gardener.LastOperationStateSucceeded { - msg := fmt.Sprintf("Shoot %s successfully created", s.shoot.Name) - m.log.Info(msg) - - if !s.instance.IsStateWithConditionSet(imv1.RuntimeStatePending, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonShootCreationCompleted) { - s.instance.UpdateStatePending( - imv1.ConditionTypeRuntimeProvisioned, - imv1.ConditionReasonShootCreationCompleted, - "True", - "Shoot creation completed") - - return updateStatusAndRequeue() - } - - return switchState(sFnProcessShoot) - } - - if lastOperation.Type == gardener.LastOperationTypeCreate && lastOperation.State == gardener.LastOperationStateFailed { - if gardenerhelper.HasErrorCode(s.shoot.Status.LastErrors, gardener.ErrorInfraRateLimitsExceeded) { - msg := fmt.Sprintf("Error during cluster provisioning: Rate limits exceeded for Shoot %s, scheduling for retry", s.shoot.Name) - m.log.Info(msg) - return updateStatusAndRequeueAfter(gardenerRequeueDuration) - } - - msg := fmt.Sprintf("Provisioning failed for shoot: %s ! Last state: %s, Description: %s", s.shoot.Name, lastOperation.State, lastOperation.Description) - m.log.Info(msg) - - s.instance.UpdateStatePending( - imv1.ConditionTypeRuntimeProvisioned, - imv1.ConditionReasonCreationError, - "False", - "Shoot creation failed") - - return updateStatusAndStop() - } - - // Runtime update is in progress - if lastOperation.Type == gardener.LastOperationTypeReconcile && - (lastOperation.State == gardener.LastOperationStateProcessing || lastOperation.State == gardener.LastOperationStatePending) { - msg := fmt.Sprintf("Shoot %s is in %s state, scheduling for retry", s.shoot.Name, lastOperation.State) - m.log.Info(msg) - - s.instance.UpdateStatePending( - imv1.ConditionTypeRuntimeProvisioned, - imv1.ConditionReasonProcessing, - "Unknown", - "Shoot creation in progress") - - return updateStatusAndRequeue() - } - - if lastOperation.Type == gardener.LastOperationTypeReconcile && lastOperation.State == gardener.LastOperationStateSucceeded { - msg := fmt.Sprintf("Shoot %s successfully updated", s.shoot.Name) - m.log.Info(msg) - - if !s.instance.IsStateWithConditionSet(imv1.RuntimeStatePending, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonProcessing) { - s.instance.UpdateStatePending( - imv1.ConditionTypeRuntimeProvisioned, - imv1.ConditionReasonProcessing, - "True", - "Shoot update completed") - - return updateStatusAndRequeue() - } - - return switchState(sFnProcessShoot) - } - - if lastOperation.Type == gardener.LastOperationTypeReconcile && lastOperation.State == gardener.LastOperationStateFailed { - var reason ErrReason - - if len(s.shoot.Status.LastErrors) > 0 { - reason = gardenerErrCodesToErrReason(s.shoot.Status.LastErrors...) - } - - msg := fmt.Sprintf("error during cluster provisioning: reconcilation error for shoot %s, reason: %s, scheduling for retry", s.shoot.Name, reason) - m.log.Info(msg) - - s.instance.UpdateStatePending( - imv1.ConditionTypeRuntimeProvisioned, - imv1.ConditionReasonProcessingErr, - "False", - string(reason)) - - return updateStatusAndStop() - } - - return updateStatusAndStop() -} - -func gardenerErrCodesToErrReason(lastErrors ...gardener.LastError) ErrReason { - var codes []gardener.ErrorCode - var vals []string - - for _, e := range lastErrors { - if len(e.Codes) > 0 { - codes = append(codes, e.Codes...) - } - } - - for _, code := range codes { - vals = append(vals, string(code)) - } - - return ErrReason(strings.Join(vals, ", ")) -} diff --git a/internal/controller/runtime/fsm/runtime_fsm_process_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_process_shoot.go index daa0e715..47956767 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_process_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_process_shoot.go @@ -7,12 +7,14 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ) -func sFnProcessShoot(_ context.Context, _ *fsm, s *systemState) (stateFn, *ctrl.Result, error) { +func sFnProcessShoot(_ context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { + m.log.Info("Process cluster state - the last one") + // process shoot get kubeconfig and create cluster role bindings s.instance.UpdateStateReady( imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonConfigurationCompleted, - "Runtime creation completed successfully") + "Runtime processing completed successfully") return updateStatusAndStop() } diff --git a/internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing.go b/internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing.go new file mode 100644 index 00000000..a7128af7 --- /dev/null +++ b/internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing.go @@ -0,0 +1,46 @@ +package fsm + +import ( + "context" + "fmt" + + gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" + imv1 "github.com/kyma-project/infrastructure-manager/api/v1" + ctrl "sigs.k8s.io/controller-runtime" +) + +type ErrReason string + +func sFnSelectShootProcessing(_ context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { + m.log.Info("Select shoot processing state") + + if s.shoot.Spec.DNS == nil || s.shoot.Spec.DNS.Domain == nil { + msg := fmt.Sprintf("DNS Domain is not set yet for shoot: %s, scheduling for retry", s.shoot.Name) + m.log.Info(msg) + return updateStatusAndRequeueAfter(gardenerRequeueDuration) + } + + lastOperation := s.shoot.Status.LastOperation + if lastOperation == nil { + msg := fmt.Sprintf("Last operation is nil for shoot: %s, scheduling for retry", s.shoot.Name) + m.log.Info(msg) + return updateStatusAndRequeueAfter(gardenerRequeueDuration) + } + + if s.instance.Status.State == imv1.RuntimeStateReady && lastOperation.State == gardener.LastOperationStateSucceeded { + // only allow to patch if full previous cycle was completed + m.log.Info("Gardener shoot already exists, updating") + return switchState(sFnPatchExistingShoot) + } + + if lastOperation.Type == gardener.LastOperationTypeCreate { + return switchState(sFnWaitForShootCreation) + } + + if lastOperation.Type == gardener.LastOperationTypeReconcile { + return switchState(sFnWaitForShootReconcile) + } + + m.log.Info("Unknown shoot operation type, exiting with no retry") + return stop() +} diff --git a/internal/controller/runtime/fsm/runtime_fsm_prepare_cluster_test.go b/internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing_test.go similarity index 99% rename from internal/controller/runtime/fsm/runtime_fsm_prepare_cluster_test.go rename to internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing_test.go index 3d2afadb..c54e6490 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_prepare_cluster_test.go +++ b/internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing_test.go @@ -1,5 +1,6 @@ package fsm +/* import ( "context" "time" @@ -175,3 +176,4 @@ var _ = Describe("KIM sFnInitialise", func() { ), ) }) +*/ diff --git a/internal/controller/runtime/fsm/runtime_fsm_take_snapshot.go b/internal/controller/runtime/fsm/runtime_fsm_take_snapshot.go index c03a6542..527859c1 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_take_snapshot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_take_snapshot.go @@ -3,25 +3,30 @@ package fsm import ( "context" + gardener_api "github.com/gardener/gardener/pkg/apis/core/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" ) // to save the runtime status at the begining of the reconciliation func sFnTakeSnapshot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { + m.log.Info("Take snapshot state") s.saveRuntimeStatus() - s.shoot = nil - shoot, err := m.ShootClient.Get(ctx, s.instance.Name, v1.GetOptions{}) + var shoot gardener_api.Shoot + err := m.ShootClient.Get(ctx, types.NamespacedName{ + Name: s.instance.Name, + Namespace: m.ShootNamesapace, + }, &shoot) - if err != nil { - if !apierrors.IsNotFound(err) { - m.log.Info("Failed to get Gardener shoot", "error", err) - return updateStatusAndRequeue() - } - } else if shoot != nil { - s.shoot = shoot.DeepCopy() + if err != nil && !apierrors.IsNotFound(err) { + m.log.Info("Failed to get Gardener shoot", "error", err) + return updateStatusAndRequeueAfter(gardenerRequeueDuration) + } + + if err == nil { + s.shoot = &shoot } return switchState(sFnInitialize) diff --git a/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go b/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go new file mode 100644 index 00000000..15465ee8 --- /dev/null +++ b/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go @@ -0,0 +1,65 @@ +package fsm + +import ( + "context" + "fmt" + + gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" + imv1 "github.com/kyma-project/infrastructure-manager/api/v1" + ctrl "sigs.k8s.io/controller-runtime" +) + +func sFnWaitForShootReconcile(_ context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { + m.log.Info("Waiting for shoot reconcile state") + + switch s.shoot.Status.LastOperation.State { + case gardener.LastOperationStateProcessing, gardener.LastOperationStatePending, gardener.LastOperationStateAborted: + msg := fmt.Sprintf("Shoot %s is in %s state, scheduling for retry", s.shoot.Name, s.shoot.Status.LastOperation.State) + m.log.Info(msg) + + s.instance.UpdateStatePending( + imv1.ConditionTypeRuntimeProvisioned, + imv1.ConditionReasonProcessing, + "Unknown", + "Shoot update is in progress") + + return updateStatusAndRequeueAfter(gardenerRequeueDuration) + + case gardener.LastOperationStateSucceeded: + if !s.instance.IsStateWithConditionAndStatusSet(imv1.RuntimeStatePending, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonProcessing, "True") { + s.instance.UpdateStatePending( + imv1.ConditionTypeRuntimeProvisioned, + imv1.ConditionReasonProcessing, + "True", + "Shoot update is completed") + + return updateStatusAndRequeue() + } + + msg := fmt.Sprintf("Shoot %s successfully updated, moving to processing", s.shoot.Name) + m.log.Info(msg) + + return switchState(sFnProcessShoot) + + case gardener.LastOperationStateFailed: + var reason ErrReason + + if len(s.shoot.Status.LastErrors) > 0 { + reason = gardenerErrCodesToErrReason(s.shoot.Status.LastErrors...) + } + + msg := fmt.Sprintf("error during cluster processing: reconcilation failed for shoot %s, reason: %s, exiting with no retry", s.shoot.Name, reason) + m.log.Info(msg) + + s.instance.UpdateStatePending( + imv1.ConditionTypeRuntimeProvisioned, + imv1.ConditionReasonProcessingErr, + "False", + string(reason)) + + return updateStatusAndStop() + } + + m.log.Info("Update did not processed, exiting with no retry") + return stop() +} diff --git a/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go b/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go new file mode 100644 index 00000000..a52574ff --- /dev/null +++ b/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go @@ -0,0 +1,87 @@ +package fsm + +import ( + "context" + "fmt" + "strings" + + gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" + gardenerhelper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + imv1 "github.com/kyma-project/infrastructure-manager/api/v1" + ctrl "sigs.k8s.io/controller-runtime" +) + +func sFnWaitForShootCreation(_ context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { + m.log.Info("Waiting for shoot creation state") + + switch s.shoot.Status.LastOperation.State { + case gardener.LastOperationStateProcessing, gardener.LastOperationStatePending, gardener.LastOperationStateAborted: + msg := fmt.Sprintf("Shoot %s is in %s state, scheduling for retry", s.shoot.Name, s.shoot.Status.LastOperation.State) + m.log.Info(msg) + + s.instance.UpdateStatePending( + imv1.ConditionTypeRuntimeProvisioned, + imv1.ConditionReasonShootCreationPending, + "Unknown", + "Shoot creation in progress") + + return updateStatusAndRequeueAfter(gardenerRequeueDuration) + + case gardener.LastOperationStateSucceeded: + msg := fmt.Sprintf("Shoot %s successfully created", s.shoot.Name) + m.log.Info(msg) + + if !s.instance.IsStateWithConditionAndStatusSet(imv1.RuntimeStatePending, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonShootCreationCompleted, "True") { + s.instance.UpdateStatePending( + imv1.ConditionTypeRuntimeProvisioned, + imv1.ConditionReasonShootCreationCompleted, + "True", + "Shoot creation completed") + return updateStatusAndRequeue() + } + return switchState(sFnProcessShoot) + + case gardener.LastOperationStateFailed: + if gardenerhelper.HasErrorCode(s.shoot.Status.LastErrors, gardener.ErrorInfraRateLimitsExceeded) { + msg := fmt.Sprintf("Error during cluster provisioning: Rate limits exceeded for Shoot %s, scheduling for retry", s.shoot.Name) + m.log.Info(msg) + return updateStatusAndRequeueAfter(gardenerRequeueDuration) + } + + // also handle other retryable errors here + // ErrorRetryableConfigurationProblem + // ErrorRetryableInfraDependencies + + msg := fmt.Sprintf("Provisioning failed for shoot: %s ! Last state: %s, Description: %s", s.shoot.Name, s.shoot.Status.LastOperation.State, s.shoot.Status.LastOperation.Description) + m.log.Info(msg) + + s.instance.UpdateStatePending( + imv1.ConditionTypeRuntimeProvisioned, + imv1.ConditionReasonCreationError, + "False", + "Shoot creation failed") + + return updateStatusAndStop() + + default: + m.log.Info("Unknown shoot operation state, exiting with no retry") + return stop() + } +} + +func gardenerErrCodesToErrReason(lastErrors ...gardener.LastError) ErrReason { + var codes []gardener.ErrorCode + var vals []string + + for _, e := range lastErrors { + if len(e.Codes) > 0 { + codes = append(codes, e.Codes...) + } + } + + for _, code := range codes { + vals = append(vals, string(code)) + } + + return ErrReason(strings.Join(vals, ", ")) +} diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index 769a0764..e935430f 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -23,7 +23,6 @@ import ( "github.com/go-logr/logr" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/infrastructure-manager/internal/controller/runtime/fsm" - gardener "github.com/kyma-project/infrastructure-manager/internal/gardener" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" @@ -36,7 +35,7 @@ import ( type RuntimeReconciler struct { client.Client Scheme *runtime.Scheme - ShootClient gardener.ShootClient + ShootClient client.Client Log logr.Logger Cfg fsm.RCCfg EventRecorder record.EventRecorder @@ -73,19 +72,21 @@ func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) return stateFSM.Run(ctx, runtime) } -func NewRuntimeReconciler(mgr ctrl.Manager, shootClient gardener.ShootClient, logger logr.Logger) *RuntimeReconciler { +func NewRuntimeReconciler(mgr ctrl.Manager, shootClient client.Client, logger logr.Logger, cfg fsm.RCCfg) *RuntimeReconciler { return &RuntimeReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), ShootClient: shootClient, EventRecorder: mgr.GetEventRecorderFor("runtime-controller"), Log: logger, - Cfg: fsm.RCCfg{ - Finalizer: imv1.Finalizer, - }, + Cfg: cfg, } } +func (r *RuntimeReconciler) UpdateShootClient(client client.Client) { + r.ShootClient = client +} + // SetupWithManager sets up the controller with the Manager. func (r *RuntimeReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/internal/controller/runtime/runtime_controller_test.go b/internal/controller/runtime/runtime_controller_test.go index b061dce0..6b2a868a 100644 --- a/internal/controller/runtime/runtime_controller_test.go +++ b/internal/controller/runtime/runtime_controller_test.go @@ -45,17 +45,12 @@ var _ = Describe("Runtime Controller", func() { err := k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) - By("Cleanup the specific resource instance Runtime") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - - By("Cleanup ShootClient mocks") - clearMockCalls(mockShootClient) }) It("Should successfully create new Shoot from provided Runtime and set Ready status on CR", func() { - By("Setup the mock of ShootClient for Provisioning") - setupShootClientMockForProvisioning(mockShootClient) + By("Setup the fake gardener client for Provisioning") + setupGardenerTestClientForProvisioning() By("Create Runtime CR") runtimeStub := CreateRuntimeStub(ResourceName) @@ -75,8 +70,29 @@ var _ = Describe("Runtime Controller", func() { }, time.Second*300, time.Second*3).Should(BeTrue()) - By("Wait for shoot to be created") + By("Wait for Runtime to process shoot creation process and finish processing in Ready State") + + // should go into Pending Processing state + Eventually(func() bool { + runtime := imv1.Runtime{} + err := k8sClient.Get(ctx, typeNamespacedName, &runtime) + if err != nil { + return false + } + + // check state + if runtime.Status.State != imv1.RuntimeStatePending { + return false + } + + if !runtime.IsConditionSetWithStatus(imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonShootCreationPending, "Unknown") { + return false + } + + return true + }, time.Second*300, time.Second*3).Should(BeTrue()) + // and end as Ready state with ConfigurationCompleted condition == True Eventually(func() bool { runtime := imv1.Runtime{} err := k8sClient.Get(ctx, typeNamespacedName, &runtime) @@ -89,17 +105,74 @@ var _ = Describe("Runtime Controller", func() { if runtime.Status.State != imv1.RuntimeStateReady { return false } - // check conditions if !runtime.IsConditionSet(imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonConfigurationCompleted) { return false } return true + }, time.Second*300, time.Second*3).Should(BeTrue()) + + Expect(customTracker.IsSequenceFullyUsed()).To(BeTrue()) + + By("Wait for Runtime to process shoot update process and finish processing in Ready State") + setupGardenerTestClientForUpdate() + + runtime := imv1.Runtime{} + err := k8sClient.Get(ctx, typeNamespacedName, &runtime) + + Expect(err).To(BeNil()) + + runtime.Spec.Shoot.Provider.Workers[0].Maximum = 5 + Expect(k8sClient.Update(ctx, &runtime)).To(Succeed()) + + // should go into Pending Processing state + Eventually(func() bool { + runtime := imv1.Runtime{} + err := k8sClient.Get(ctx, typeNamespacedName, &runtime) + if err != nil { + return false + } + + // check state + if runtime.Status.State != imv1.RuntimeStatePending { + return false + } + + if !runtime.IsConditionSetWithStatus(imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonProcessing, "Unknown") { + return false + } + + return true + }, time.Second*300, time.Second*3).Should(BeTrue()) + + // and end as Ready state with ConfigurationCompleted condition == True + Eventually(func() bool { + runtime := imv1.Runtime{} + err := k8sClient.Get(ctx, typeNamespacedName, &runtime) + if err != nil { + return false + } + + // check state + if runtime.Status.State != imv1.RuntimeStateReady { + return false + } + + if !runtime.IsConditionSet(imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonConfigurationCompleted) { + return false + } + + return true }, time.Second*300, time.Second*3).Should(BeTrue()) - // mockShootClient.AssertExpectations(GinkgoT()) //TODO: this fails, investigate why + Expect(customTracker.IsSequenceFullyUsed()).To(BeTrue()) + + // next test will be for runtime deletion + // + // By("Delete Runtime CR") + // Expect(k8sClient.Delete(ctx, &runtime)).To(Succeed()) }) }) }) @@ -112,12 +185,14 @@ func CreateRuntimeStub(resourceName string) *imv1.Runtime { }, Spec: imv1.RuntimeSpec{ Shoot: imv1.RuntimeShoot{ + Name: resourceName, Networking: imv1.Networking{}, Provider: imv1.Provider{ Type: "aws", Workers: []gardener.Worker{ { - Zones: []string{""}, + Zones: []string{""}, + Maximum: 1, }, }, }, diff --git a/internal/controller/runtime/suite_test.go b/internal/controller/runtime/suite_test.go index b3d0cfe7..22bf7a36 100644 --- a/internal/controller/runtime/suite_test.go +++ b/internal/controller/runtime/suite_test.go @@ -23,17 +23,19 @@ import ( gardener_api "github.com/gardener/gardener/pkg/apis/core/v1beta1" infrastructuremanagerv1 "github.com/kyma-project/infrastructure-manager/api/v1" - gardener_mocks "github.com/kyma-project/infrastructure-manager/internal/gardener/mocks" + "github.com/kyma-project/infrastructure-manager/internal/controller/runtime/fsm" gardener_shoot "github.com/kyma-project/infrastructure-manager/internal/gardener/shoot" - . "github.com/onsi/ginkgo/v2" //nolint:revive - . "github.com/onsi/gomega" //nolint:revive - . "github.com/stretchr/testify/mock" //nolint:revive - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive + //nolint:revive + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + clienttesting "k8s.io/client-go/testing" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -44,13 +46,14 @@ import ( // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var ( - cfg *rest.Config //nolint:gochecknoglobals - mockShootClient *gardener_mocks.ShootClient //nolint:gochecknoglobals - k8sClient client.Client //nolint:gochecknoglobals - testEnv *envtest.Environment //nolint:gochecknoglobals - suiteCtx context.Context //nolint:gochecknoglobals - cancelSuiteCtx context.CancelFunc //nolint:gochecknoglobals - anyContext = MatchedBy(func(_ context.Context) bool { return true }) //nolint:gochecknoglobals + cfg *rest.Config //nolint:gochecknoglobals + k8sClient client.Client //nolint:gochecknoglobals + gardenerTestClient client.Client //nolint:gochecknoglobals + testEnv *envtest.Environment //nolint:gochecknoglobals + suiteCtx context.Context //nolint:gochecknoglobals + cancelSuiteCtx context.CancelFunc //nolint:gochecknoglobals + runtimeReconciler *RuntimeReconciler //nolint:gochecknoglobals + customTracker *CustomTracker //nolint:gochecknoglobals ) func TestControllers(t *testing.T) { @@ -85,9 +88,15 @@ var _ = BeforeSuite(func() { Scheme: scheme.Scheme}) Expect(err).ToNot(HaveOccurred()) - mockShootClient = &gardener_mocks.ShootClient{} + clientScheme := runtime.NewScheme() + _ = gardener_api.AddToScheme(clientScheme) - runtimeReconciler := NewRuntimeReconciler(mgr, mockShootClient, logger) + // tracker will be updated with different shoot sequence for each test case + tracker := clienttesting.NewObjectTracker(clientScheme, serializer.NewCodecFactory(clientScheme).UniversalDecoder()) + customTracker = NewCustomTracker(tracker, []*gardener_api.Shoot{}) + gardenerTestClient = fake.NewClientBuilder().WithScheme(clientScheme).WithObjectTracker(customTracker).Build() + + runtimeReconciler = NewRuntimeReconciler(mgr, gardenerTestClient, logger, fsm.RCCfg{Finalizer: infrastructuremanagerv1.Finalizer}) Expect(runtimeReconciler).NotTo(BeNil()) err = runtimeReconciler.SetupWithManager(mgr) Expect(err).To(BeNil()) @@ -106,12 +115,14 @@ var _ = BeforeSuite(func() { }() }) -func clearMockCalls(mock *gardener_mocks.ShootClient) { - mock.ExpectedCalls = nil - mock.Calls = nil -} +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancelSuiteCtx() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) -func setupShootClientMockForProvisioning(shootClientMock *gardener_mocks.ShootClient) { +func setupGardenerTestClientForProvisioning() { runtimeStub := CreateRuntimeStub("test-resource") converterConfig := fixConverterConfigForTests() converter := gardener_shoot.NewConverter(converterConfig) @@ -120,45 +131,102 @@ func setupShootClientMockForProvisioning(shootClientMock *gardener_mocks.ShootCl panic(err) } - shootClientMock.On("Create", anyContext, &convertedShoot, Anything).Return(&convertedShoot, nil) + shoots := fixGardenerShootsForProvisioning(&convertedShoot) + + clientScheme := runtime.NewScheme() + _ = gardener_api.AddToScheme(clientScheme) - var shoots []*gardener_api.Shoot = fixGardenerShootsForProvisioning(&convertedShoot) + // our customTracker will be updated with different shoot sequence for each test case + tracker := clienttesting.NewObjectTracker(clientScheme, serializer.NewCodecFactory(clientScheme).UniversalDecoder()) + customTracker = NewCustomTracker(tracker, shoots) + gardenerTestClient = fake.NewClientBuilder().WithScheme(clientScheme).WithObjectTracker(customTracker).Build() - for _, shoot := range shoots { - if shoot != nil { - shootClientMock.On("Get", anyContext, Anything, Anything).Return(shoot, nil).Once() - continue - } + runtimeReconciler.UpdateShootClient(gardenerTestClient) +} - shootClientMock.On("Get", anyContext, Anything, Anything).Return(nil, errors.NewNotFound( - schema.GroupResource{Group: "", Resource: "shoots"}, convertedShoot.Name)).Once() +func setupGardenerTestClientForUpdate() { + runtimeStub := CreateRuntimeStub("test-resource") + converterConfig := fixConverterConfigForTests() + converter := gardener_shoot.NewConverter(converterConfig) + convertedShoot, err := converter.ToShoot(*runtimeStub) + if err != nil { + panic(err) } + + shoots := fixGardenerShootsForUpdate(&convertedShoot) + + clientScheme := runtime.NewScheme() + _ = gardener_api.AddToScheme(clientScheme) + + tracker := clienttesting.NewObjectTracker(clientScheme, serializer.NewCodecFactory(clientScheme).UniversalDecoder()) + customTracker = NewCustomTracker(tracker, shoots) + gardenerTestClient = fake.NewClientBuilder().WithScheme(clientScheme).WithObjectTracker(customTracker).Build() + + runtimeReconciler.UpdateShootClient(gardenerTestClient) } -var _ = AfterSuite(func() { - By("tearing down the test environment") - cancelSuiteCtx() - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -}) +// func setupGardenerTestClientForDeleting() { +// runtimeStub := CreateRuntimeStub("test-resource") +// converterConfig := fixConverterConfigForTests() +// converter := gardener_shoot.NewConverter(converterConfig) +// convertedShoot, err := converter.ToShoot(*runtimeStub) +// if err != nil { +// panic(err) +// } +// +// shoots := fixGardenerShootsForProvisioning(&convertedShoot) +// +// objectTestTracker.SetShootListForTracker(shoots) +//} func fixGardenerShootsForProvisioning(shoot *gardener_api.Shoot) []*gardener_api.Shoot { var missingShoot *gardener_api.Shoot - initialisedShoot := shoot.DeepCopy() - initialisedShoot.Spec.DNS = &gardener_api.DNS{ + dnsShoot := initialisedShoot.DeepCopy() + + dnsShoot.Spec.DNS = &gardener_api.DNS{ Domain: ptrTo("test.domain"), } - initialisedShoot.Status = gardener_api.ShootStatus{ + pendingShoot := dnsShoot.DeepCopy() + + pendingShoot.Status = gardener_api.ShootStatus{ LastOperation: &gardener_api.LastOperation{ Type: gardener_api.LastOperationTypeCreate, State: gardener_api.LastOperationStatePending, }, } - processingShoot := initialisedShoot.DeepCopy() + processingShoot := pendingShoot.DeepCopy() + + processingShoot.Status.LastOperation.State = gardener_api.LastOperationStateProcessing + + readyShoot := processingShoot.DeepCopy() + + readyShoot.Status.LastOperation.State = gardener_api.LastOperationStateSucceeded + + // processedShoot := processingShoot.DeepCopy() // will add specific data later + + return []*gardener_api.Shoot{missingShoot, missingShoot, missingShoot, initialisedShoot, dnsShoot, pendingShoot, processingShoot, readyShoot, readyShoot} +} + +func fixGardenerShootsForUpdate(shoot *gardener_api.Shoot) []*gardener_api.Shoot { + + pendingShoot := shoot.DeepCopy() + + pendingShoot.Spec.DNS = &gardener_api.DNS{ + Domain: ptrTo("test.domain"), + } + + pendingShoot.Status = gardener_api.ShootStatus{ + LastOperation: &gardener_api.LastOperation{ + Type: gardener_api.LastOperationTypeReconcile, + State: gardener_api.LastOperationStatePending, + }, + } + + processingShoot := pendingShoot.DeepCopy() processingShoot.Status.LastOperation.State = gardener_api.LastOperationStateProcessing @@ -168,7 +236,7 @@ func fixGardenerShootsForProvisioning(shoot *gardener_api.Shoot) []*gardener_api // processedShoot := processingShoot.DeepCopy() // will add specific data later - return []*gardener_api.Shoot{missingShoot, missingShoot, missingShoot, initialisedShoot, processingShoot, readyShoot, readyShoot, readyShoot, readyShoot} + return []*gardener_api.Shoot{pendingShoot, processingShoot, readyShoot, readyShoot} } func fixConverterConfigForTests() gardener_shoot.ConverterConfig { diff --git a/internal/controller/runtime/test_client_obj_tracker.go b/internal/controller/runtime/test_client_obj_tracker.go new file mode 100644 index 00000000..9c2115ab --- /dev/null +++ b/internal/controller/runtime/test_client_obj_tracker.go @@ -0,0 +1,55 @@ +package runtime + +import ( + "fmt" + "sync" + + gardener_api "github.com/gardener/gardener/pkg/apis/core/v1beta1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + clienttesting "k8s.io/client-go/testing" +) + +// CustomTracker implements ObjectTracker with a sequence of Shoot objects +// it will be updated with a different shoot sequence for each test case +type CustomTracker struct { + clienttesting.ObjectTracker + shootSequence []*gardener_api.Shoot + callCnt int + mu sync.Mutex +} + +func NewCustomTracker(tracker clienttesting.ObjectTracker, shoots []*gardener_api.Shoot) *CustomTracker { + return &CustomTracker{ + ObjectTracker: tracker, + shootSequence: shoots, + } +} + +func (t *CustomTracker) GetCallCnt() int { + return t.callCnt +} + +func (t *CustomTracker) IsSequenceFullyUsed() bool { + return t.callCnt == len(t.shootSequence) && len(t.shootSequence) > 0 +} + +func (t *CustomTracker) Get(gvr schema.GroupVersionResource, ns, name string) (runtime.Object, error) { + t.mu.Lock() + defer t.mu.Unlock() + + if gvr.Resource == "shoots" { + if t.callCnt < len(t.shootSequence) { + shoot := t.shootSequence[t.callCnt] + t.callCnt++ + + if shoot == nil { + return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") + } + return shoot, nil + } + return nil, fmt.Errorf("no more Shoot objects in sequence") + } + return t.ObjectTracker.Get(gvr, ns, name) +}