Skip to content

Commit

Permalink
issue-572, handling of the resource external deletion was added
Browse files Browse the repository at this point in the history
  • Loading branch information
Bohdan Siryk authored and Bohdan Siryk committed Sep 28, 2023
1 parent 3723b60 commit 3ff0d92
Show file tree
Hide file tree
Showing 28 changed files with 901 additions and 628 deletions.
83 changes: 33 additions & 50 deletions controllers/clusters/cadence_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,20 +222,22 @@ func (r *CadenceReconciler) HandleCreateCluster(
"Cluster creation request is sent. Cluster ID: %s", id)
}

err := r.startClusterStatusJob(cadence)
if err != nil {
logger.Error(err, "Cannot start cluster status job",
"cadence cluster ID", cadence.Status.ID,
)
if cadence.Status.State != models.DeletedStatus {
err := r.startClusterStatusJob(cadence)
if err != nil {
logger.Error(err, "Cannot start cluster status job",
"cadence cluster ID", cadence.Status.ID,
)

r.EventRecorder.Eventf(cadence, models.Warning, models.CreationFailed,
"Cluster status check job is failed. Reason: %v", err)
r.EventRecorder.Eventf(cadence, models.Warning, models.CreationFailed,
"Cluster status check job is failed. Reason: %v", err)

return models.ReconcileRequeue
}
return models.ReconcileRequeue
}

r.EventRecorder.Event(cadence, models.Normal, models.Created,
"Cluster status check job is started")
r.EventRecorder.Event(cadence, models.Normal, models.Created,
"Cluster status check job is started")
}

return models.ExitReconcile
}
Expand Down Expand Up @@ -790,45 +792,7 @@ func (r *CadenceReconciler) newWatchStatusJob(cadence *v1beta1.Cadence) schedule
iData, err := r.API.GetCadence(cadence.Status.ID)
if err != nil {
if errors.Is(err, instaclustr.NotFound) {
activeClusters, err := r.API.ListClusters()
if err != nil {
l.Error(err, "Cannot list account active clusters")
return err
}

if !isClusterActive(cadence.Status.ID, activeClusters) {
l.Info("Cluster is not found in Instaclustr. Deleting resource.",
"cluster ID", cadence.Status.ClusterStatus.ID,
"cluster name", cadence.Spec.Name,
)

patch := cadence.NewPatch()
cadence.Annotations[models.ClusterDeletionAnnotation] = ""
cadence.Annotations[models.ResourceStateAnnotation] = models.DeletingEvent
err = r.Patch(context.TODO(), cadence, patch)
if err != nil {
l.Error(err, "Cannot patch Cadence cluster resource",
"cluster ID", cadence.Status.ID,
"cluster name", cadence.Spec.Name,
"resource name", cadence.Name,
)

return err
}

err = r.Delete(context.TODO(), cadence)
if err != nil {
l.Error(err, "Cannot delete Cadence cluster resource",
"cluster ID", cadence.Status.ID,
"cluster name", cadence.Spec.Name,
"resource name", cadence.Name,
)

return err
}

return nil
}
return r.handleExternalDelete(context.Background(), cadence)
}

l.Error(err, "Cannot get Cadence cluster from the Instaclustr API",
Expand Down Expand Up @@ -1281,3 +1245,22 @@ func (r *CadenceReconciler) reconcileMaintenanceEvents(ctx context.Context, c *v

return nil
}

func (r *CadenceReconciler) handleExternalDelete(ctx context.Context, c *v1beta1.Cadence) error {
l := log.FromContext(ctx)

patch := c.NewPatch()
c.Status.State = models.DeletedStatus
err := r.Status().Patch(ctx, c, patch)
if err != nil {
return err
}

l.Info(instaclustr.MsgInstaclustrResourceNotFound)
r.EventRecorder.Eventf(c, models.Warning, models.ExternalDeleted, instaclustr.MsgInstaclustrResourceNotFound)

r.Scheduler.RemoveJob(c.GetJobID(scheduler.BackupsChecker))
r.Scheduler.RemoveJob(c.GetJobID(scheduler.StatusChecker))

return nil
}
136 changes: 60 additions & 76 deletions controllers/clusters/cassandra_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,54 +221,56 @@ func (r *CassandraReconciler) handleCreateCluster(
)
}

err = r.startClusterStatusJob(cassandra)
if err != nil {
l.Error(err, "Cannot start cluster status job",
"cassandra cluster ID", cassandra.Status.ID)
if cassandra.Status.State != models.DeletedStatus {
err = r.startClusterStatusJob(cassandra)
if err != nil {
l.Error(err, "Cannot start cluster status job",
"cassandra cluster ID", cassandra.Status.ID)

r.EventRecorder.Eventf(
cassandra, models.Warning, models.CreationFailed,
"Cluster status check job is failed. Reason: %v",
err,
)
return models.ReconcileRequeue
}

r.EventRecorder.Eventf(
cassandra, models.Warning, models.CreationFailed,
"Cluster status check job is failed. Reason: %v",
err,
cassandra, models.Normal, models.Created,
"Cluster status check job is started",
)
return models.ReconcileRequeue
}

r.EventRecorder.Eventf(
cassandra, models.Normal, models.Created,
"Cluster status check job is started",
)
err = r.startClusterBackupsJob(cassandra)
if err != nil {
l.Error(err, "Cannot start cluster backups check job",
"cluster ID", cassandra.Status.ID,
)

err = r.startClusterBackupsJob(cassandra)
if err != nil {
l.Error(err, "Cannot start cluster backups check job",
"cluster ID", cassandra.Status.ID,
)
r.EventRecorder.Eventf(
cassandra, models.Warning, models.CreationFailed,
"Cluster backups check job is failed. Reason: %v",
err,
)
return models.ReconcileRequeue
}

r.EventRecorder.Eventf(
cassandra, models.Warning, models.CreationFailed,
"Cluster backups check job is failed. Reason: %v",
err,
cassandra, models.Normal, models.Created,
"Cluster backups check job is started",
)
return models.ReconcileRequeue
}

r.EventRecorder.Eventf(
cassandra, models.Normal, models.Created,
"Cluster backups check job is started",
)
if cassandra.Spec.UserRefs != nil {
err = r.startUsersCreationJob(cassandra)
if err != nil {
l.Error(err, "Failed to start user creation job")
r.EventRecorder.Eventf(cassandra, models.Warning, models.CreationFailed,
"User creation job is failed. Reason: %v", err)
return models.ReconcileRequeue
}

if cassandra.Spec.UserRefs != nil {
err = r.startUsersCreationJob(cassandra)
if err != nil {
l.Error(err, "Failed to start user creation job")
r.EventRecorder.Eventf(cassandra, models.Warning, models.CreationFailed,
"User creation job is failed. Reason: %v", err)
return models.ReconcileRequeue
r.EventRecorder.Event(cassandra, models.Normal, models.Created,
"Cluster user creation job is started")
}

r.EventRecorder.Event(cassandra, models.Normal, models.Created,
"Cluster user creation job is started")
}

return models.ExitReconcile
Expand Down Expand Up @@ -885,45 +887,7 @@ func (r *CassandraReconciler) newWatchStatusJob(cassandra *v1beta1.Cassandra) sc
iData, err := r.API.GetCassandra(cassandra.Status.ID)
if err != nil {
if errors.Is(err, instaclustr.NotFound) {
activeClusters, err := r.API.ListClusters()
if err != nil {
l.Error(err, "Cannot list account active clusters")
return err
}

if !isClusterActive(cassandra.Status.ID, activeClusters) {
l.Info("Cluster is not found in Instaclustr. Deleting resource.",
"cluster ID", cassandra.Status.ClusterStatus.ID,
"cluster name", cassandra.Spec.Name,
)

patch := cassandra.NewPatch()
cassandra.Annotations[models.ClusterDeletionAnnotation] = ""
cassandra.Annotations[models.ResourceStateAnnotation] = models.DeletingEvent
err = r.Patch(context.TODO(), cassandra, patch)
if err != nil {
l.Error(err, "Cannot patch Cassandra cluster resource",
"cluster ID", cassandra.Status.ID,
"cluster name", cassandra.Spec.Name,
"resource name", cassandra.Name,
)

return err
}

err = r.Delete(context.TODO(), cassandra)
if err != nil {
l.Error(err, "Cannot delete Cassandra cluster resource",
"cluster ID", cassandra.Status.ID,
"cluster name", cassandra.Spec.Name,
"resource name", cassandra.Name,
)

return err
}

return nil
}
return r.handleExternalDelete(context.Background(), cassandra)
}

l.Error(err, "Cannot get cluster from the Instaclustr API",
Expand Down Expand Up @@ -1262,6 +1226,26 @@ func (r *CassandraReconciler) reconcileMaintenanceEvents(ctx context.Context, c
return nil
}

func (r *CassandraReconciler) handleExternalDelete(ctx context.Context, c *v1beta1.Cassandra) error {
l := log.FromContext(ctx)

patch := c.NewPatch()
c.Status.State = models.DeletedStatus
err := r.Status().Patch(ctx, c, patch)
if err != nil {
return err
}

l.Info(instaclustr.MsgInstaclustrResourceNotFound)
r.EventRecorder.Eventf(c, models.Warning, models.ExternalDeleted, instaclustr.MsgInstaclustrResourceNotFound)

r.Scheduler.RemoveJob(c.GetJobID(scheduler.BackupsChecker))
r.Scheduler.RemoveJob(c.GetJobID(scheduler.UserCreator))
r.Scheduler.RemoveJob(c.GetJobID(scheduler.StatusChecker))

return nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *CassandraReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
Expand Down
52 changes: 52 additions & 0 deletions controllers/clusters/cassandra_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package clusters

import (
"context"
"fmt"
"os"

. "github.com/onsi/ginkgo/v2"
Expand All @@ -28,7 +29,9 @@ import (

"github.com/instaclustr/operator/apis/clusters/v1beta1"
"github.com/instaclustr/operator/controllers/tests"
"github.com/instaclustr/operator/pkg/instaclustr"
openapi "github.com/instaclustr/operator/pkg/instaclustr/mock/server/go"
"github.com/instaclustr/operator/pkg/models"
)

const newCassandraNodeSize = "CAS-DEV-t4g.small-30"
Expand Down Expand Up @@ -106,4 +109,53 @@ var _ = Describe("Cassandra Controller", func() {
}, timeout, interval).Should(BeTrue())
})
})

When("Deleting the Cassandra resource by avoiding operator", func() {
It("should try to get the cluster details and receive StatusNotFound", func() {
cassandraManifest2 := cassandraManifest.DeepCopy()
cassandraManifest2.Name += "-test-external-delete"
cassandraManifest2.ResourceVersion = ""

cassandra2 := v1beta1.Cassandra{}
cassandra2NamespacedName := types.NamespacedName{
Namespace: cassandraManifest2.Namespace,
Name: cassandraManifest2.Name,
}

Expect(k8sClient.Create(ctx, cassandraManifest2)).Should(Succeed())

doneCh := tests.NewChannelWithTimeout(timeout)

Eventually(func() bool {
if err := k8sClient.Get(ctx, cassandra2NamespacedName, &cassandra2); err != nil {
return false
}

if cassandra2.Status.State != models.RunningStatus {
return false
}

doneCh <- struct{}{}

return true
}, timeout, interval).Should(BeTrue())

<-doneCh

By("testing by deleting the cluster by Instaclustr API client")
Expect(instaClient.DeleteCluster(cassandra2.Status.ID, instaclustr.CassandraEndpoint)).Should(Succeed())

Eventually(func() bool {
err := k8sClient.Get(ctx, cassandra2NamespacedName, &cassandra2)
if err != nil {
return false
}

fmt.Println("cassandra2.Status.State", cassandra2.Status.State)

return cassandra2.Status.State == models.DeletedStatus
}, timeout, interval).Should(BeTrue())
})
})

})
5 changes: 2 additions & 3 deletions controllers/clusters/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,10 @@ import (
"sort"

"github.com/hashicorp/go-version"
"k8s.io/utils/strings/slices"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/instaclustr/operator/apis/clusters/v1beta1"
"github.com/instaclustr/operator/pkg/models"
"k8s.io/utils/strings/slices"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// confirmDeletion confirms if resource is deleting and set appropriate annotation.
Expand Down
Loading

0 comments on commit 3ff0d92

Please sign in to comment.