Full implementation of timeout based on annotation with cleanup

internal/controller/runtime/fsm/runtime_fsm_create_shoot.go

@@ -2,9 +2,9 @@ package fsm
 
 import (
 	"context"
-
 	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"time"
 )
 
 func sFnCreateShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
@@ -16,6 +16,16 @@ func sFnCreateShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl
 		return updateStatePendingWithErrorAndStop(&s.instance, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonConversionError, "Runtime conversion error")
 	}
 
+	if s.instance.Annotations == nil {
+		s.instance.Annotations = make(map[string]string)
+	}
+
+	if _, found := s.instance.Annotations[imv1.AnnotationRuntimeOperationStarted]; !found || s.instance.Annotations[imv1.AnnotationRuntimeOperationStarted] == "" {
+		s.instance.Annotations[imv1.AnnotationRuntimeOperationStarted] = time.Now().UTC().Format(time.RFC3339)
+		m.Update(ctx, &s.instance)
+		return requeue()
+	}
+
 	err = m.ShootClient.Create(ctx, &newShoot)
 
 	if err != nil {

internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go

@@ -2,6 +2,8 @@ package fsm
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
 	"k8s.io/utils/ptr"
@@ -14,10 +16,25 @@ func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl
 
 	// wait section
 	if !s.shoot.GetDeletionTimestamp().IsZero() {
+		if s.instance.HasTimeoutElapsed(m.DeprovisionTimeout) {
+			m.log.Info(fmt.Sprintf("Runtime deprovision timeout for %s", s.shoot.Name))
+			s.instance.UpdateStateDeletion(imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonShootDeletionTimeout, "False", "Runtime deprovisioning timeout")
+			return updateStatusAndRequeue() // Requeue to clear annotation
+		}
+
 		m.log.Info("Waiting for shoot to be deleted", "Name", s.shoot.Name, "Namespace", s.shoot.Namespace)
 		return requeueAfter(gardenerRequeueDuration)
 	}
 
+	if s.instance.Annotations == nil {
+		s.instance.Annotations = make(map[string]string)
+	}
+	if _, found := s.instance.Annotations[imv1.AnnotationRuntimeOperationStarted]; !found {
+		s.instance.Annotations[imv1.AnnotationRuntimeOperationStarted] = time.Now().UTC().Format(time.RFC3339)
+		m.Update(ctx, &s.instance)
+		return requeue()
+	}
+
 	// action section
 	if !isGardenerCloudDelConfirmationSet(s.shoot.Annotations) {
 		m.log.Info("patching shoot with del-confirmation")

internal/controller/runtime/fsm/runtime_fsm_initialise.go

@@ -33,19 +33,26 @@ func sFnInitialize(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.
 		return updateStatusAndRequeue()
 	}
 
+	if s.instance.Status.State == imv1.RuntimeStateFailed || s.instance.Status.State == imv1.RuntimeStateReady {
+		if s.instance.Annotations != nil {
+			if val, found := s.instance.Annotations[imv1.AnnotationRuntimeOperationStarted]; found {
+				if val != "" {
+					s.instance.Annotations[imv1.AnnotationRuntimeOperationStarted] = ""
+					m.Update(ctx, &s.instance)
+					return stop()
+				}
+			}
+		}
+	}
+
 	if instanceIsNotBeingDeleted && s.shoot == nil {
 		m.log.Info("Gardener shoot does not exist, creating new one")
 		if !dryRunMode {
-			return switchState(sFnCreateShoot)
+			return switchState(sFnCreateShoot) // create shoot and add annotation
 		}
 		return switchState(sFnCreateShootDryRun)
 	}
 
-	if instanceIsNotBeingDeleted && !dryRunMode {
-		m.log.Info("Gardener shoot exists, processing")
-		return switchState(sFnSelectShootProcessing)
-	}
-
 	// instance is being deleted
 	if !instanceIsNotBeingDeleted && instanceHasFinalizer {
 		if s.shoot != nil && !dryRunMode {
@@ -55,6 +62,11 @@ func sFnInitialize(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.
 		return removeFinalizerAndStop(ctx, m, s) // resource cleanup completed
 	}
 
+	if instanceIsNotBeingDeleted && !dryRunMode {
+		m.log.Info("Gardener shoot exists, processing")
+		return switchState(sFnSelectShootProcessing)
+	}
+
 	m.log.Info("noting to reconcile, stopping sfm")
 	return stop()
 }

internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go

@@ -2,6 +2,7 @@ package fsm
 
 import (
 	"context"
+	"time"
 
 	gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
@@ -23,6 +24,15 @@ func sFnPatchExistingShoot(ctx context.Context, m *fsm, s *systemState) (stateFn
 
 	m.log.Info("Shoot converted successfully", "Name", updatedShoot.Name, "Namespace", updatedShoot.Namespace)
 
+	if s.instance.Annotations == nil {
+		s.instance.Annotations = make(map[string]string)
+	}
+	if _, found := s.instance.Annotations[imv1.AnnotationRuntimeOperationStarted]; !found {
+		s.instance.Annotations[imv1.AnnotationRuntimeOperationStarted] = time.Now().UTC().Format(time.RFC3339)
+		m.Update(ctx, &s.instance)
+		return requeue()
+	}
+
 	err = m.ShootClient.Patch(ctx, &updatedShoot, client.Apply, &client.PatchOptions{
 		FieldManager: "kim",
 		Force:        ptr.To(true),

internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing.go

@@ -27,7 +27,8 @@ func sFnSelectShootProcessing(_ context.Context, m *fsm, s *systemState) (stateF
 		return updateStatusAndRequeueAfter(gardenerRequeueDuration)
 	}
 
-	if s.instance.Status.State == imv1.RuntimeStateReady && lastOperation.State == gardener.LastOperationStateSucceeded {
+	if s.instance.Status.State == imv1.RuntimeStateReady && lastOperation.State == gardener.LastOperationStateSucceeded ||
+		s.instance.Status.State == imv1.RuntimeStateFailed { // to make possible to recover from timeout during previous operation
 		// only allow to patch if full previous cycle was completed
 		m.log.Info("Gardener shoot already exists, updating")
 		return switchState(sFnPatchExistingShoot)

internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go

@@ -3,19 +3,18 @@ package fsm
 import (
 	"context"
 	"fmt"
-
 	gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
-func sFnWaitForShootReconcile(_ context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
+func sFnWaitForShootReconcile(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
 	m.log.Info("Waiting for shoot reconcile state")
 
 	if s.instance.HasTimeoutElapsed(m.UpdateTimeout) {
 		m.log.Info(fmt.Sprintf("Shoot creation timeout for %s", s.shoot.Name))
 		s.instance.UpdateStatePending(imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonShootProcessingTimeout, "False", "Shoot reconcile timeout")
-		return updateStatusAndStop()
+		return updateStatusAndRequeue() // Requeue to clear annotation
 	}
 
 	switch s.shoot.Status.LastOperation.State {

internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go

@@ -42,7 +42,7 @@ func sFnWaitForShootCreation(_ context.Context, m *fsm, s *systemState) (stateFn
 	if s.instance.HasTimeoutElapsed(m.ProvisionTimeout) {
 		m.log.Info(fmt.Sprintf("Shoot creation timeout for %s", s.shoot.Name))
 		s.instance.UpdateStatePending(imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonShootCreationTimeout, "False", "Shoot creation timeout")
-		return updateStatusAndStop()
+		return updateStatusAndRequeue() // Requeue to clear annotation
 	}
 
 	switch s.shoot.Status.LastOperation.State {

internal/controller/runtime/suite_test.go

@@ -101,7 +101,14 @@ var _ = BeforeSuite(func() {
 	customTracker = NewCustomTracker(tracker, []*gardener_api.Shoot{})
 	gardenerTestClient = fake.NewClientBuilder().WithScheme(clientScheme).WithObjectTracker(customTracker).Build()
 
-	runtimeReconciler = NewRuntimeReconciler(mgr, gardenerTestClient, logger, fsm.RCCfg{Finalizer: infrastructuremanagerv1.Finalizer})
+	testConfig := fsm.RCCfg{
+		Finalizer:          infrastructuremanagerv1.Finalizer,
+		ProvisionTimeout:   5 * time.Minute,
+		DeprovisionTimeout: 5 * time.Minute,
+		UpdateTimeout:      5 * time.Minute,
+	}
+
+	runtimeReconciler = NewRuntimeReconciler(mgr, gardenerTestClient, logger, testConfig)
 	Expect(runtimeReconciler).NotTo(BeNil())
 	err = runtimeReconciler.SetupWithManager(mgr)
 	Expect(err).To(BeNil())
@@ -196,7 +203,7 @@ func fixShootsSequenceForProvisioning(shoot *gardener_api.Shoot) []*gardener_api
 
 	// processedShoot := processingShoot.DeepCopy() // will add specific data later
 
-	return []*gardener_api.Shoot{missingShoot, missingShoot, missingShoot, initialisedShoot, dnsShoot, pendingShoot, processingShoot, readyShoot, readyShoot, readyShoot, readyShoot}
+	return []*gardener_api.Shoot{missingShoot, missingShoot, missingShoot, missingShoot, initialisedShoot, dnsShoot, pendingShoot, processingShoot, readyShoot, readyShoot, readyShoot, readyShoot}
 }
 
 func fixShootsSequenceForUpdate(shoot *gardener_api.Shoot) []*gardener_api.Shoot {
@@ -223,7 +230,7 @@ func fixShootsSequenceForUpdate(shoot *gardener_api.Shoot) []*gardener_api.Shoot
 
 	// processedShoot := processingShoot.DeepCopy() // will add specific data later
 
-	return []*gardener_api.Shoot{pendingShoot, processingShoot, readyShoot, readyShoot}
+	return []*gardener_api.Shoot{pendingShoot, pendingShoot, processingShoot, readyShoot, readyShoot}
 }
 
 func fixShootsSequenceForDelete(shoot *gardener_api.Shoot) []*gardener_api.Shoot {
@@ -252,7 +259,7 @@ func fixShootsSequenceForDelete(shoot *gardener_api.Shoot) []*gardener_api.Shoot
 	pendingDeleteShoot.Status.LastOperation.Type = gardener_api.LastOperationTypeDelete
 	pendingDeleteShoot.Status.LastOperation.State = gardener_api.LastOperationStatePending
 
-	return []*gardener_api.Shoot{currentShoot, currentShoot, currentShoot, pendingDeleteShoot, nil}
+	return []*gardener_api.Shoot{currentShoot, currentShoot, currentShoot, currentShoot, pendingDeleteShoot, nil}
 }
 
 func fixConverterConfigForTests() gardener_shoot.ConverterConfig {