draft implementation of externaly deleted resource handler

apis/clusters/v1beta1/cassandra_types.go

@@ -491,6 +491,10 @@ func (c *CassandraSpec) validateResizeSettings(nodeNumber int) error {
 	return nil
 }
 
+func (c *Cassandra) SetState(state string) {
+	c.Status.State = state
+}
+
 func init() {
 	SchemeBuilder.Register(&Cassandra{}, &CassandraList{})
 }

controllers/clusters/cassandra_controller.go

@@ -892,37 +892,67 @@ func (r *CassandraReconciler) newWatchStatusJob(cassandra *v1beta1.Cassandra) sc
 				}
 
 				if !isClusterActive(cassandra.Status.ID, activeClusters) {
-					l.Info("Cluster is not found in Instaclustr. Deleting resource.",
-						"cluster ID", cassandra.Status.ClusterStatus.ID,
-						"cluster name", cassandra.Spec.Name,
-					)
-
-					patch := cassandra.NewPatch()
-					cassandra.Annotations[models.ClusterDeletionAnnotation] = ""
-					cassandra.Annotations[models.ResourceStateAnnotation] = models.DeletingEvent
-					err = r.Patch(context.TODO(), cassandra, patch)
-					if err != nil {
-						l.Error(err, "Cannot patch Cassandra cluster resource",
-							"cluster ID", cassandra.Status.ID,
-							"cluster name", cassandra.Spec.Name,
-							"resource name", cassandra.Name,
-						)
-
-						return err
-					}
-
-					err = r.Delete(context.TODO(), cassandra)
+					//l.Info("Cluster is not found in Instaclustr. Deleting resource.",
+					//	"cluster ID", cassandra.Status.ClusterStatus.ID,
+					//	"cluster name", cassandra.Spec.Name,
+					//)
+
+					//l.Info("Cluster is not found in Instaclustr. Changing resource state to DELETED...")
+					//
+					//patch := cassandra.NewPatch()
+					//cassandra.Status.State = models.ResourceDeletedStatus
+					//err = r.Status().Patch(context.TODO(), cassandra, patch)
+					//if err != nil {
+					//	l.Error(err, "Cannot patch Cassandra cluster status",
+					//		"cluster ID", cassandra.Status.ID,
+					//		"cluster name", cassandra.Spec.Name,
+					//	)
+					//
+					//	return err
+					//}
+					//
+					//r.EventRecorder.Event(cassandra, models.Normal, models.ExternalDeleted,
+					//	"Cluster is not found in Instaclustr. Resource is marked as deleted.",
+					//)
+
+					err := ResourceDeleted(context.TODO(), r.Client, r.EventRecorder, cassandra)
 					if err != nil {
-						l.Error(err, "Cannot delete Cassandra cluster resource",
-							"cluster ID", cassandra.Status.ID,
-							"cluster name", cassandra.Spec.Name,
-							"resource name", cassandra.Name,
+						l.Error(err, "Cannot patch Cassandra cluster status")
+						r.EventRecorder.Eventf(cassandra, models.Warning, models.PatchFailed,
+							"Cannot patch Cassandra cluster status. Reason: %v", err,
 						)
-
 						return err
 					}
 
-					return nil
+					r.Scheduler.RemoveJob(cassandra.GetJobID(scheduler.BackupsChecker))
+					r.Scheduler.RemoveJob(cassandra.GetJobID(scheduler.UserCreator))
+					r.Scheduler.RemoveJob(cassandra.GetJobID(scheduler.StatusChecker))
+
+					//cassandra.Annotations[models.ClusterDeletionAnnotation] = ""
+					//	cassandra.Annotations[models.ResourceStateAnnotation] = models.DeletingEvent
+					//	err = r.Patch(context.TODO(), cassandra, patch)
+					//	if err != nil {
+					//		l.Error(err, "Cannot patch Cassandra cluster resource",
+					//			"cluster ID", cassandra.Status.ID,
+					//			"cluster name", cassandra.Spec.Name,
+					//			"resource name", cassandra.Name,
+					//		)
+					//
+					//		return err
+					//	}
+					//
+					//	err = r.Delete(context.TODO(), cassandra)
+					//	if err != nil {
+					//		l.Error(err, "Cannot delete Cassandra cluster resource",
+					//			"cluster ID", cassandra.Status.ID,
+					//			"cluster name", cassandra.Spec.Name,
+					//			"resource name", cassandra.Name,
+					//		)
+					//
+					//		return err
+					//	}
+					//
+					//	return nil
 				}
 			}
 

controllers/clusters/helpers.go

@@ -17,8 +17,11 @@ limitations under the License.
 package clusters
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sort"
 
 	"github.com/hashicorp/go-version"
@@ -182,6 +185,36 @@ func createSpecDifferenceMessage(k8sSpec, iSpec any) (string, error) {
 	return msg + specDifference, nil
 }
 
+type Object interface {
+	client.Object
+	SetState(state string)
+	NewPatch() client.Patch
+}
+
+// ResourceDeleted sets the state of the resource to models.ExternalDeleted
+func ResourceDeleted(
+	ctx context.Context,
+	client client.Client,
+	recorder record.EventRecorder,
+	obj Object,
+) error {
+	l := log.FromContext(ctx)
+
+	patch := obj.NewPatch()
+
+	obj.SetState(models.ResourceDeletedStatus)
+
+	err := client.Status().Patch(ctx, obj, patch)
+	if err != nil {
+		return err
+	}
+
+	l.Info(msgInstaclustrResourceNotFound)
+	recorder.Eventf(obj, models.Warning, models.ExternalDeleted, msgInstaclustrResourceNotFound)
+
+	return nil
+}
+
 var msgDeleteClusterWithTwoFactorDelete = "Please confirm cluster deletion via email or phone. " +
 	"If you have canceled a cluster deletion and want to put the cluster on deletion again, " +
 	"remove \"triggered\" from Instaclustr.com/clusterDeletion annotation."
@@ -191,3 +224,5 @@ var msgExternalChanges = "The k8s specification is different from Instaclustr Co
 	"so that it would corresponds to the data from Instaclustr."
 
 var msgSpecStillNoMatch = "k8s resource specification still doesn't match with data on the Instaclustr Console. Double check the difference."
+
+const msgInstaclustrResourceNotFound = "The resource is not found on Instaclustr, marking it as deleted"

pkg/models/apiv1.go

@@ -17,8 +17,9 @@ limitations under the License.
 package models
 
 const (
-	RunningStatus = "RUNNING"
-	Disabled      = "DISABLED"
+	RunningStatus         = "RUNNING"
+	Disabled              = "DISABLED"
+	ResourceDeletedStatus = "DELETED"
 )
 
 type ClusterProviderV1 struct {

pkg/models/operator.go

@@ -139,6 +139,7 @@ const (
 	DeletionStarted  = "DeletionStarted"
 	DeletionFailed   = "DeletionFailed"
 	Deleted          = "Deleted"
+	ExternalDeleted  = "ExternalDeleted"
 )
 
 const (