Add splitbrain condition, make tests terser

api/v1alpha1/etcdcluster_types.go

@@ -55,6 +55,7 @@ type EtcdClusterSpec struct {
 const (
  EtcdConditionInitialized = "Initialized"
  EtcdConditionReady = "Ready"
+ EtcdConditionError = "Error"
 )
 
 type EtcdCondType string
@@ -66,6 +67,7 @@ const (
  EtcdCondTypeWaitingForFirstQuorum EtcdCondType = "WaitingForFirstQuorum"
  EtcdCondTypeStatefulSetReady EtcdCondType = "StatefulSetReady"
  EtcdCondTypeStatefulSetNotReady EtcdCondType = "StatefulSetNotReady"
+ EtcdCondTypeSplitbrain EtcdCondType = "Splitbrain"
 )
 
 const (
@@ -74,6 +76,7 @@ const (
  EtcdReadyCondNegMessage EtcdCondMessage = "Cluster StatefulSet is not Ready"
  EtcdReadyCondPosMessage EtcdCondMessage = "Cluster StatefulSet is Ready"
  EtcdReadyCondNegWaitingForQuorum EtcdCondMessage = "Waiting for first quorum to be established"
+ EtcdErrorCondSplitbrainMessage EtcdCondMessage = "Etcd endpoints reporting more than one unique cluster ID"
 )
 
 // EtcdClusterStatus defines the observed state of EtcdCluster

internal/controller/etcdcluster_controller.go

@@ -133,7 +133,13 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
  state.setClusterID()
  if state.inSplitbrain() {
  log.Error(ctx, fmt.Errorf("etcd cluster in splitbrain"), "etcd cluster in splitbrain, dropping from reconciliation queue")
- return ctrl.Result{}, nil
+ factory.SetCondition(instance, factory.NewCondition(etcdaenixiov1alpha1.EtcdConditionError).
+ WithStatus(true).
+ WithReason(string(etcdaenixiov1alpha1.EtcdCondTypeSplitbrain)).
+ WithMessage(string(etcdaenixiov1alpha1.EtcdErrorCondSplitbrainMessage)).
+ Complete(),
+ )
+ return r.updateStatus(ctx, instance)
  }
  // fill conditions
  if len(instance.Status.Conditions) == 0 {

test/e2e/e2e_test.go

@@ -97,24 +97,15 @@ var _ = Describe("etcd-operator", Ordered, func() {
  })
 
  Eventually(func() error {
- cmd := exec.Command("kubectl", "get",
- "statefulset/test",
- "--namespace", namespace,
- )
- _, err = utils.Run(cmd)
- return err
- }, time.Second*20, time.Second*2).Should(Succeed())
-
- By("wait for statefulset is ready", func() {
  cmd := exec.Command("kubectl", "wait",
  "statefulset/test",
  "--for", "jsonpath={.status.readyReplicas}=3",
  "--namespace", namespace,
  "--timeout", "5m",
  )
  _, err = utils.Run(cmd)
- ExpectWithOffset(1, err).NotTo(HaveOccurred())
- })
+ return err
+ }, time.Second*20, time.Second*2).Should(Succeed(), "wait for statefulset is ready")
 
  client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"})
  Expect(err).NotTo(HaveOccurred())
@@ -155,24 +146,15 @@ var _ = Describe("etcd-operator", Ordered, func() {
  })
 
  Eventually(func() error {
- cmd := exec.Command("kubectl", "get",
- "statefulset/test",
- "--namespace", namespace,
- )
- _, err = utils.Run(cmd)
- return err
- }, time.Second*20, time.Second*2).Should(Succeed())
-
- By("wait for statefulset is ready", func() {
  cmd := exec.Command("kubectl", "wait",
  "statefulset/test",
  "--for", "jsonpath={.status.readyReplicas}=3",
  "--namespace", namespace,
  "--timeout", "5m",
  )
  _, err = utils.Run(cmd)
- ExpectWithOffset(1, err).NotTo(HaveOccurred())
- })
+ return err
+ }, time.Second*20, time.Second*2).Should(Succeed(), "wait for statefulset is ready")
 
  client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"})
  Expect(err).NotTo(HaveOccurred())
@@ -212,24 +194,15 @@ var _ = Describe("etcd-operator", Ordered, func() {
  })
 
  Eventually(func() error {
- cmd := exec.Command("kubectl", "get",
- "statefulset/test",
- "--namespace", namespace,
- )
- _, err = utils.Run(cmd)
- return err
- }, time.Second*20, time.Second*2).Should(Succeed())
-
- By("wait for statefulset is ready", func() {
  cmd := exec.Command("kubectl", "wait",
  "statefulset/test",
  "--for", "jsonpath={.status.availableReplicas}=3",
  "--namespace", namespace,
  "--timeout", "5m",
  )
  _, err = utils.Run(cmd)
- ExpectWithOffset(1, err).NotTo(HaveOccurred())
- })
+ return err
+ }, time.Second*20, time.Second*2).Should(Succeed(), "wait for statefulset is ready")
 
  client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"})
  Expect(err).NotTo(HaveOccurred())