From f81648b50b6ee16c17bf37438321b628aa32b66f Mon Sep 17 00:00:00 2001
From: Timofei Larkin <lllamnyp@gmail.com>
Date: Tue, 25 Jun 2024 18:27:31 +0300
Subject: [PATCH] Add splitbrain condition, make tests terser

---
 api/v1alpha1/etcdcluster_types.go             |  3 ++
 internal/controller/etcdcluster_controller.go |  8 +++-
 test/e2e/e2e_test.go                          | 39 +++----------------
 3 files changed, 16 insertions(+), 34 deletions(-)

diff --git a/api/v1alpha1/etcdcluster_types.go b/api/v1alpha1/etcdcluster_types.go
index 71cf65a6..bd7ef6d7 100644
--- a/api/v1alpha1/etcdcluster_types.go
+++ b/api/v1alpha1/etcdcluster_types.go
@@ -55,6 +55,7 @@ type EtcdClusterSpec struct {
 const (
 	EtcdConditionInitialized = "Initialized"
 	EtcdConditionReady       = "Ready"
+	EtcdConditionError       = "Error"
 )
 
 type EtcdCondType string
@@ -66,6 +67,7 @@ const (
 	EtcdCondTypeWaitingForFirstQuorum EtcdCondType = "WaitingForFirstQuorum"
 	EtcdCondTypeStatefulSetReady      EtcdCondType = "StatefulSetReady"
 	EtcdCondTypeStatefulSetNotReady   EtcdCondType = "StatefulSetNotReady"
+	EtcdCondTypeSplitbrain            EtcdCondType = "Splitbrain"
 )
 
 const (
@@ -74,6 +76,7 @@ const (
 	EtcdReadyCondNegMessage          EtcdCondMessage = "Cluster StatefulSet is not Ready"
 	EtcdReadyCondPosMessage          EtcdCondMessage = "Cluster StatefulSet is Ready"
 	EtcdReadyCondNegWaitingForQuorum EtcdCondMessage = "Waiting for first quorum to be established"
+	EtcdErrorCondSplitbrainMessage   EtcdCondMessage = "Etcd endpoints reporting more than one unique cluster ID"
 )
 
 // EtcdClusterStatus defines the observed state of EtcdCluster
diff --git a/internal/controller/etcdcluster_controller.go b/internal/controller/etcdcluster_controller.go
index 981a3c3b..87aeab69 100644
--- a/internal/controller/etcdcluster_controller.go
+++ b/internal/controller/etcdcluster_controller.go
@@ -133,7 +133,13 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	state.setClusterID()
 	if state.inSplitbrain() {
 		log.Error(ctx, fmt.Errorf("etcd cluster in splitbrain"), "etcd cluster in splitbrain, dropping from reconciliation queue")
-		return ctrl.Result{}, nil
+		factory.SetCondition(instance, factory.NewCondition(etcdaenixiov1alpha1.EtcdConditionError).
+			WithStatus(true).
+			WithReason(string(etcdaenixiov1alpha1.EtcdCondTypeSplitbrain)).
+			WithMessage(string(etcdaenixiov1alpha1.EtcdErrorCondSplitbrainMessage)).
+			Complete(),
+		)
+		return r.updateStatus(ctx, instance)
 	}
 	// fill conditions
 	if len(instance.Status.Conditions) == 0 {
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
index 9b18844f..8e5578f6 100644
--- a/test/e2e/e2e_test.go
+++ b/test/e2e/e2e_test.go
@@ -97,15 +97,6 @@ var _ = Describe("etcd-operator", Ordered, func() {
 			})
 
 			Eventually(func() error {
-				cmd := exec.Command("kubectl", "get",
-					"statefulset/test",
-					"--namespace", namespace,
-				)
-				_, err = utils.Run(cmd)
-				return err
-			}, time.Second*20, time.Second*2).Should(Succeed())
-
-			By("wait for statefulset is ready", func() {
 				cmd := exec.Command("kubectl", "wait",
 					"statefulset/test",
 					"--for", "jsonpath={.status.readyReplicas}=3",
@@ -113,8 +104,8 @@ var _ = Describe("etcd-operator", Ordered, func() {
 					"--timeout", "5m",
 				)
 				_, err = utils.Run(cmd)
-				ExpectWithOffset(1, err).NotTo(HaveOccurred())
-			})
+				return err
+			}, time.Second*20, time.Second*2).Should(Succeed(), "wait for statefulset is ready")
 
 			client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"})
 			Expect(err).NotTo(HaveOccurred())
@@ -155,15 +146,6 @@ var _ = Describe("etcd-operator", Ordered, func() {
 			})
 
 			Eventually(func() error {
-				cmd := exec.Command("kubectl", "get",
-					"statefulset/test",
-					"--namespace", namespace,
-				)
-				_, err = utils.Run(cmd)
-				return err
-			}, time.Second*20, time.Second*2).Should(Succeed())
-
-			By("wait for statefulset is ready", func() {
 				cmd := exec.Command("kubectl", "wait",
 					"statefulset/test",
 					"--for", "jsonpath={.status.readyReplicas}=3",
@@ -171,8 +153,8 @@ var _ = Describe("etcd-operator", Ordered, func() {
 					"--timeout", "5m",
 				)
 				_, err = utils.Run(cmd)
-				ExpectWithOffset(1, err).NotTo(HaveOccurred())
-			})
+				return err
+			}, time.Second*20, time.Second*2).Should(Succeed(), "wait for statefulset is ready")
 
 			client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"})
 			Expect(err).NotTo(HaveOccurred())
@@ -212,15 +194,6 @@ var _ = Describe("etcd-operator", Ordered, func() {
 			})
 
 			Eventually(func() error {
-				cmd := exec.Command("kubectl", "get",
-					"statefulset/test",
-					"--namespace", namespace,
-				)
-				_, err = utils.Run(cmd)
-				return err
-			}, time.Second*20, time.Second*2).Should(Succeed())
-
-			By("wait for statefulset is ready", func() {
 				cmd := exec.Command("kubectl", "wait",
 					"statefulset/test",
 					"--for", "jsonpath={.status.availableReplicas}=3",
@@ -228,8 +201,8 @@ var _ = Describe("etcd-operator", Ordered, func() {
 					"--timeout", "5m",
 				)
 				_, err = utils.Run(cmd)
-				ExpectWithOffset(1, err).NotTo(HaveOccurred())
-			})
+				return err
+			}, time.Second*20, time.Second*2).Should(Succeed(), "wait for statefulset is ready")
 
 			client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"})
 			Expect(err).NotTo(HaveOccurred())