diff --git a/api/v1/runtime_types.go b/api/v1/runtime_types.go index 72e53f70..2b4ca2f7 100644 --- a/api/v1/runtime_types.go +++ b/api/v1/runtime_types.go @@ -28,7 +28,10 @@ import ( //+kubebuilder:printcolumn:name="SHOOT-NAME",type=string,JSONPath=`.metadata.labels.kyma-project\.io/shoot-name` //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" -const Finalizer = "runtime-controller.infrastructure-manager.kyma-project.io/deletion-hook" +const ( + Finalizer = "runtime-controller.infrastructure-manager.kyma-project.io/deletion-hook" + AnnotationGardenerCloudDelConfirmation = "confirmation.gardener.cloud/deletion" +) const ( RuntimeStateReady = "Ready" @@ -60,12 +63,13 @@ const ( ConditionReasonConfigurationCompleted = RuntimeConditionReason("ConfigurationCompleted") ConditionReasonConfigurationErr = RuntimeConditionReason("ConfigurationError") - ConditionReasonDeletion = RuntimeConditionReason("Deletion") - ConditionReasonDeletionErr = RuntimeConditionReason("DeletionErr") - ConditionReasonConversionError = RuntimeConditionReason("ConversionErr") - ConditionReasonCreationError = RuntimeConditionReason("CreationErr") - ConditionReasonGardenerError = RuntimeConditionReason("GardenerErr") - ConditionReasonDeleted = RuntimeConditionReason("Deleted") + ConditionReasonDeletion = RuntimeConditionReason("Deletion") + ConditionReasonDeletionErr = RuntimeConditionReason("DeletionErr") + ConditionReasonConversionError = RuntimeConditionReason("ConversionErr") + ConditionReasonCreationError = RuntimeConditionReason("CreationErr") + ConditionReasonGardenerError = RuntimeConditionReason("GardenerErr") + ConditionReasonSerializationError = RuntimeConditionReason("SerializationErr") + ConditionReasonDeleted = RuntimeConditionReason("Deleted") ) //+kubebuilder:object:root=true diff --git a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go index 7e6a8f79..acdb9bd1 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go +++ b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot.go @@ -2,14 +2,47 @@ package fsm import ( "context" + "encoding/json" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" ) func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { + if !isGardenerCloudDelConfirmationSet(s.shoot.Annotations) { + m.log.Info("patching shoot with del-confirmation") + // workaround for Gardener client + s.shoot.Kind = "Shoot" + s.shoot.APIVersion = "core.gardener.cloud/v1beta1" + s.shoot.Annotations = addGardenerCloudDelConfirmation(s.shoot.Annotations) + s.shoot.ManagedFields = nil + // attempt to marshall patched instance + shootData, err := json.Marshal(&s.shoot) + if err != nil { + // unrecoverable error + s.instance.UpdateStateDeletion( + imv1.ConditionTypeRuntimeProvisioned, + imv1.ConditionReasonSerializationError, + "False", + err.Error()) + return updateStatusAndStop() + } + // see: https://gardener.cloud/docs/gardener/projects/#four-eyes-principle-for-resource-deletion + if s.shoot, err = m.ShootClient.Patch(ctx, s.shoot.Name, types.ApplyPatchType, shootData, + metav1.PatchOptions{ + FieldManager: "kim", + Force: ptr.To(true), + }); err != nil { + m.log.Error(err, "unable to patch shoot:") + return requeue() + } + } + if !s.instance.IsStateWithConditionSet(imv1.RuntimeStateTerminating, imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonDeletion) { + m.log.Info("setting state to in deletion") s.instance.UpdateStateDeletion( imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonDeletion, @@ -19,8 +52,8 @@ func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl return updateStatusAndRequeue() } - err := m.ShootClient.Delete(ctx, s.instance.Name, v1.DeleteOptions{}) - + m.log.Info("deleting shoot") + err := m.ShootClient.Delete(ctx, s.instance.Name, metav1.DeleteOptions{}) if err != nil { m.log.Error(err, "Failed to delete gardener Shoot") @@ -32,5 +65,22 @@ func sFnDeleteShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl ) return updateStatusAndRequeue() } - return updateStatusAndStop() + + return updateStatusAndRequeue() +} + +func isGardenerCloudDelConfirmationSet(a map[string]string) bool { + if len(a) == 0 { + return false + } + val, found := a[imv1.AnnotationGardenerCloudDelConfirmation] + return found && (val == "true") +} + +func addGardenerCloudDelConfirmation(a map[string]string) map[string]string { + if len(a) == 0 { + a = map[string]string{} + } + a[imv1.AnnotationGardenerCloudDelConfirmation] = "true" + return a } diff --git a/internal/controller/runtime/fsm/runtime_fsm_delete_shoot_test.go b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot_test.go new file mode 100644 index 00000000..267eea9e --- /dev/null +++ b/internal/controller/runtime/fsm/runtime_fsm_delete_shoot_test.go @@ -0,0 +1,64 @@ +package fsm + +import ( + "fmt" + "testing" + + v1 "github.com/kyma-project/infrastructure-manager/api/v1" +) + +func Test_addGardenerCloudDelConfirmation(t *testing.T) { + instance := v1.Runtime{} + actual := addGardenerCloudDelConfirmation(instance.Annotations) + if _, found := actual[v1.AnnotationGardenerCloudDelConfirmation]; !found { + t.Errorf("actual map should contain '%s' annotation", v1.AnnotationGardenerCloudDelConfirmation) + } +} + +func Test_IsGardenerCloudDelConfirmation(t *testing.T) { + var cases = []struct { + annotations map[string]string + expected bool + }{ + { + expected: false, + }, + { + annotations: (map[string]string{}), + expected: false, + }, + { + annotations: map[string]string{"test": "me"}, + expected: false, + }, + { + annotations: map[string]string{ + v1.AnnotationGardenerCloudDelConfirmation: "true", + "test": "me", + }, + expected: true, + }, + { + annotations: map[string]string{ + v1.AnnotationGardenerCloudDelConfirmation: "anything", + "test": "me", + }, + expected: false, + }, + { + annotations: map[string]string{ + v1.AnnotationGardenerCloudDelConfirmation: "", + }, + expected: false, + }, + } + + for i, tt := range cases { + t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { + if actual := isGardenerCloudDelConfirmationSet(tt.annotations); actual != tt.expected { + t.Errorf("expected IsGardenerCloudDelConfirmation == %t", + tt.expected) + } + }) + } +} diff --git a/internal/controller/runtime/fsm/runtime_fsm_initialise.go b/internal/controller/runtime/fsm/runtime_fsm_initialise.go index c0bd9c5d..9a37a186 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_initialise.go +++ b/internal/controller/runtime/fsm/runtime_fsm_initialise.go @@ -42,15 +42,18 @@ func sFnInitialize(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl. return switchState(sFnPrepareCluster) // wait for pending shoot operation to complete } - if !instanceIsNotBeingDeleted && instanceHasFinalizer && s.shoot != nil { - m.log.Info("Instance is being deleted") + shootInOrAfterDeleting := s.shoot == nil || !s.shoot.GetDeletionTimestamp().IsZero() + + if !instanceIsNotBeingDeleted && instanceHasFinalizer && !shootInOrAfterDeleting { + m.log.Info("Delete instance resources") return switchState(sFnDeleteShoot) } - if !instanceIsNotBeingDeleted && instanceHasFinalizer && s.shoot == nil { + if !instanceIsNotBeingDeleted && instanceHasFinalizer && shootInOrAfterDeleting { return removeFinalizerAndStop(ctx, m, s) } + m.log.Info("noting to reconcile, stopping sfm") return stop() } diff --git a/internal/controller/runtime/runtime_controller.go b/internal/controller/runtime/runtime_controller.go index 5415cd53..769a0764 100644 --- a/internal/controller/runtime/runtime_controller.go +++ b/internal/controller/runtime/runtime_controller.go @@ -55,7 +55,7 @@ func (r *RuntimeReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.Get(ctx, request.NamespacedName, &runtime); err != nil { return ctrl.Result{ - Requeue: true, + Requeue: false, }, client.IgnoreNotFound(err) } @@ -90,6 +90,6 @@ func NewRuntimeReconciler(mgr ctrl.Manager, shootClient gardener.ShootClient, lo func (r *RuntimeReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&imv1.Runtime{}). - WithEventFilter(predicate.And(predicate.GenerationChangedPredicate{})). + WithEventFilter(predicate.GenerationChangedPredicate{}). Complete(r) } diff --git a/internal/gardener/client.go b/internal/gardener/client.go index 4a543318..9f5e075f 100644 --- a/internal/gardener/client.go +++ b/internal/gardener/client.go @@ -7,6 +7,7 @@ import ( gardener_api "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -16,6 +17,7 @@ type ShootClient interface { Create(ctx context.Context, shoot *gardener_api.Shoot, opts v1.CreateOptions) (*gardener_api.Shoot, error) Get(ctx context.Context, name string, opts v1.GetOptions) (*gardener_api.Shoot, error) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *gardener_api.Shoot, err error) // List(ctx context.Context, opts v1.ListOptions) (*gardener.ShootList, error) } diff --git a/internal/gardener/mocks/ShootClient.go b/internal/gardener/mocks/ShootClient.go index cd0cc706..ac582f90 100644 --- a/internal/gardener/mocks/ShootClient.go +++ b/internal/gardener/mocks/ShootClient.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.36.1. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks @@ -7,6 +7,8 @@ import ( mock "github.com/stretchr/testify/mock" + types "k8s.io/apimachinery/pkg/types" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" @@ -21,6 +23,10 @@ type ShootClient struct { func (_m *ShootClient) Create(ctx context.Context, shoot *v1beta1.Shoot, opts v1.CreateOptions) (*v1beta1.Shoot, error) { ret := _m.Called(ctx, shoot, opts) + if len(ret) == 0 { + panic("no return value specified for Create") + } + var r0 *v1beta1.Shoot var r1 error if rf, ok := ret.Get(0).(func(context.Context, *v1beta1.Shoot, v1.CreateOptions) (*v1beta1.Shoot, error)); ok { @@ -47,6 +53,10 @@ func (_m *ShootClient) Create(ctx context.Context, shoot *v1beta1.Shoot, opts v1 func (_m *ShootClient) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { ret := _m.Called(ctx, name, opts) + if len(ret) == 0 { + panic("no return value specified for Delete") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string, v1.DeleteOptions) error); ok { r0 = rf(ctx, name, opts) @@ -61,6 +71,10 @@ func (_m *ShootClient) Delete(ctx context.Context, name string, opts v1.DeleteOp func (_m *ShootClient) Get(ctx context.Context, name string, opts v1.GetOptions) (*v1beta1.Shoot, error) { ret := _m.Called(ctx, name, opts) + if len(ret) == 0 { + panic("no return value specified for Get") + } + var r0 *v1beta1.Shoot var r1 error if rf, ok := ret.Get(0).(func(context.Context, string, v1.GetOptions) (*v1beta1.Shoot, error)); ok { @@ -83,6 +97,43 @@ func (_m *ShootClient) Get(ctx context.Context, name string, opts v1.GetOptions) return r0, r1 } +// Patch provides a mock function with given fields: ctx, name, pt, data, opts, subresources +func (_m *ShootClient) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (*v1beta1.Shoot, error) { + _va := make([]interface{}, len(subresources)) + for _i := range subresources { + _va[_i] = subresources[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, name, pt, data, opts) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for Patch") + } + + var r0 *v1beta1.Shoot + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, types.PatchType, []byte, v1.PatchOptions, ...string) (*v1beta1.Shoot, error)); ok { + return rf(ctx, name, pt, data, opts, subresources...) + } + if rf, ok := ret.Get(0).(func(context.Context, string, types.PatchType, []byte, v1.PatchOptions, ...string) *v1beta1.Shoot); ok { + r0 = rf(ctx, name, pt, data, opts, subresources...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1beta1.Shoot) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, types.PatchType, []byte, v1.PatchOptions, ...string) error); ok { + r1 = rf(ctx, name, pt, data, opts, subresources...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // NewShootClient creates a new instance of ShootClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewShootClient(t interface {