diff --git a/controllers/alphacontrollers/recordingrulegroupset_controller.go b/controllers/alphacontrollers/recordingrulegroupset_controller.go
index e8a6a3a..b969aad 100644
--- a/controllers/alphacontrollers/recordingrulegroupset_controller.go
+++ b/controllers/alphacontrollers/recordingrulegroupset_controller.go
@@ -106,8 +106,13 @@ func (r *RecordingRuleGroupSetReconciler) Reconcile(ctx context.Context, req ctr
 			log.V(1).Info("Deleting RecordingRuleGroupSet", "recordingRuleGroup ID", id)
 			if _, err := rRGClient.DeleteRecordingRuleGroupSet(ctx, deleteRRGReq); err != nil {
 				// if fail to delete the external dependency here, return with error
-				// so that it can be retried
+				// so that it can be retried unless it is deleted manually.
 				log.Error(err, "Received an error while Deleting a RecordingRuleGroupSet", "recordingRuleGroup ID", id)
+				if status.Code(err) == codes.NotFound {
+					controllerutil.RemoveFinalizer(ruleGroupSetCRD, myFinalizerName)
+					err := r.Update(ctx, ruleGroupSetCRD)
+					return ctrl.Result{}, err
+				}
 				return ctrl.Result{}, err
 			}
 
diff --git a/controllers/alphacontrollers/rulegroup_controller.go b/controllers/alphacontrollers/rulegroup_controller.go
index 2a3b3c1..56b12ef 100644
--- a/controllers/alphacontrollers/rulegroup_controller.go
+++ b/controllers/alphacontrollers/rulegroup_controller.go
@@ -107,8 +107,13 @@ func (r *RuleGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 			log.V(1).Info("Deleting Rule-Group", "Rule-Group ID", ruleGroupId)
 			if _, err := rulesGroupsClient.DeleteRuleGroup(ctx, deleteRuleGroupReq); err != nil {
 				// if fail to delete the external dependency here, return with error
-				// so that it can be retried
+				// so that it can be retried unless it is deleted manually.
 				log.Error(err, "Received an error while Deleting a Rule-Group", "Rule-Group ID", ruleGroupId)
+				if status.Code(err) == codes.NotFound {
+					controllerutil.RemoveFinalizer(ruleGroupCRD, myFinalizerName)
+					err := r.Update(ctx, ruleGroupCRD)
+					return ctrl.Result{}, err
+				}
 				return ctrl.Result{}, err
 			}