Add validation for taints for API E2E tests (#5880)

* add taint validation for api e2e tests

* revert cluster.go

config/crd/bases/anywhere.eks.amazonaws.com_nutanixmachineconfigs.yaml

@@ -191,8 +191,8 @@ spec:
                       description: The machine address.
                       type: string
                     type:
-                      description: Machine address type, one of Hostname, ExternalIP
-                        or InternalIP.
+                      description: Machine address type, one of Hostname, ExternalIP,
+                        InternalIP, ExternalDNS or InternalDNS.
                       type: string
                   required:
                   - address

internal/pkg/api/taints.go

@@ -0,0 +1,85 @@
+package api
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+)
+
+// MasterTaint will be deprecated from kubernetes version 1.25 onwards.
+func MasterTaint() corev1.Taint {
+	return corev1.Taint{
+		Key:    "node-role.kubernetes.io/master",
+		Effect: corev1.TaintEffectNoSchedule,
+	}
+}
+
+// ControlPlaneTaint has been added from 1.24 onwards.
+func ControlPlaneTaint() corev1.Taint {
+	return corev1.Taint{
+		Key:    "node-role.kubernetes.io/control-plane",
+		Effect: corev1.TaintEffectNoSchedule,
+	}
+}
+
+// ValidateControlPlaneTaints will validate that a controlPlane node has the expected taints.
+func ValidateControlPlaneTaints(controlPlane v1alpha1.ControlPlaneConfiguration, node corev1.Node) (err error) {
+	cpTaints := controlPlane.Taints
+
+	// if no taints are specified, kubeadm defaults it to a well-known control plane taint.
+	// so, we make sure to check for that well-known taint if no taints are provided in the spec.
+	var taintsValid bool
+	if cpTaints == nil {
+		taintsValid = validateDefaultControlPlaneTaints(node)
+	} else {
+		taintsValid = v1alpha1.TaintsSliceEqual(cpTaints, node.Spec.Taints)
+	}
+
+	if !taintsValid {
+		return fmt.Errorf("taints on control plane node %v and corresponding control plane configuration do not match; configured taints: %v; node taints: %v",
+			node.Name, cpTaints, node.Spec.Taints)
+	}
+	return nil
+}
+
+// ValidateControlPlaneNoTaints will validate that a controlPlane has no taints, for example in the case of a single node cluster.
+func ValidateControlPlaneNoTaints(controlPlane v1alpha1.ControlPlaneConfiguration, node corev1.Node) (err error) {
+	valid := len(controlPlane.Taints) == 0 && len(node.Spec.Taints) == 0
+	if !valid {
+		return fmt.Errorf("taints on control plane node %v or corresponding control plane configuration found; configured taints: %v; node taints: %v",
+			node.Name, controlPlane.Taints, node.Spec.Taints)
+	}
+	return nil
+}
+
+// ValidateWorkerNodeTaints will validate that a worker node has the expected taints in the worker node group configuration.
+func ValidateWorkerNodeTaints(w v1alpha1.WorkerNodeGroupConfiguration, node corev1.Node) (err error) {
+	valid := v1alpha1.TaintsSliceEqual(node.Spec.Taints, w.Taints)
+	if !valid {
+		return fmt.Errorf("taints on node %v and corresponding worker node group configuration %v do not match", node.Name, w.Name)
+	}
+	return nil
+}
+
+func validateDefaultControlPlaneTaints(node corev1.Node) bool {
+	// Due to the transition from "master" to "control-plane", CP nodes can have one or both
+	// of these taints, depending on the k8s version. So checking that the node has at least one
+	// of them.
+
+	masterTaint := MasterTaint()
+	cpTaint := ControlPlaneTaint()
+
+	for _, v := range node.Spec.Taints {
+		if taintEqual(v, masterTaint) || taintEqual(v, cpTaint) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func taintEqual(a, b corev1.Taint) bool {
+	return a.Key == b.Key && a.Effect == b.Effect && a.Value == b.Value
+}

pkg/api/v1alpha1/cluster_types.go

@@ -1143,6 +1143,12 @@ func (c *Cluster) ManagementClusterEqual(s2 *Cluster) bool {
 	return c.IsSelfManaged() && s2.IsSelfManaged() || c.Spec.ManagementCluster.Equal(s2.Spec.ManagementCluster)
 }
 
+// IsSingleNode checks if the cluster has only a single node specified between the controlplane and worker nodes.
+func (c *Cluster) IsSingleNode() bool {
+	return c.Spec.ControlPlaneConfiguration.Count == 1 &&
+		len(c.Spec.WorkerNodeGroupConfigurations) <= 0
+}
+
 func (c *Cluster) MachineConfigRefs() []Ref {
 	machineConfigRefMap := make(refSet, 1)
 

pkg/api/v1alpha1/cluster_types_test.go

@@ -2686,3 +2686,56 @@ func TestKubeVersionToValidSemver(t *testing.T) {
 		})
 	}
 }
+
+func TestClusterIsSingleNode(t *testing.T) {
+	testCases := []struct {
+		testName string
+		cluster  *v1alpha1.Cluster
+		want     bool
+	}{
+		{
+			testName: "cluster with single node",
+			cluster: &v1alpha1.Cluster{
+				Spec: v1alpha1.ClusterSpec{
+					ControlPlaneConfiguration: v1alpha1.ControlPlaneConfiguration{
+						Count: 1,
+					},
+				},
+			},
+			want: true,
+		},
+		{
+			testName: "cluster with cp and worker",
+			cluster: &v1alpha1.Cluster{
+				Spec: v1alpha1.ClusterSpec{
+					ControlPlaneConfiguration: v1alpha1.ControlPlaneConfiguration{
+						Count: 1,
+					},
+					WorkerNodeGroupConfigurations: []v1alpha1.WorkerNodeGroupConfiguration{
+						{
+							Count: ptr.Int(3),
+						},
+					},
+				},
+			},
+			want: false,
+		},
+		{
+			testName: "cluster with multiple cp",
+			cluster: &v1alpha1.Cluster{
+				Spec: v1alpha1.ClusterSpec{
+					ControlPlaneConfiguration: v1alpha1.ControlPlaneConfiguration{
+						Count: 3,
+					},
+				},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.testName, func(t *testing.T) {
+			g := NewWithT(t)
+			g.Expect(tt.cluster.IsSingleNode()).To(Equal(tt.want))
+		})
+	}
+}

test/framework/cluster/validations/cluster.go

@@ -14,6 +14,7 @@ import (
 	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/aws/eks-anywhere/internal/pkg/api"
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	"github.com/aws/eks-anywhere/pkg/clusterapi"
 	"github.com/aws/eks-anywhere/pkg/constants"
@@ -73,7 +74,10 @@ func ValidateControlPlaneNodes(ctx context.Context, vc clusterf.StateValidationC
 	errorList := make([]error, 0)
 	for _, node := range cpNodes.Items {
 		if err := validateNodeReady(node, clus.Spec.KubernetesVersion); err != nil {
-			errorList = append(errorList, fmt.Errorf("failed to validate controlplane %s", err))
+			errorList = append(errorList, fmt.Errorf("failed to validate controlplane node ready: %v", err))
+		}
+		if err := validateControlPlaneTaints(clus, node); err != nil {
+			errorList = append(errorList, fmt.Errorf("failed to validate controlplane node taints: %v", err))
 		}
 	}
 	if len(errorList) > 0 {
@@ -104,11 +108,15 @@ func ValidateWorkerNodes(ctx context.Context, vc clusterf.StateValidationConfig)
 			if err := validateNodeReady(node, vc.ClusterSpec.Cluster.Spec.KubernetesVersion); err != nil {
 				errorList = append(errorList, fmt.Errorf("failed to validate worker node ready %v", err))
 			}
+			if err := api.ValidateWorkerNodeTaints(w, node); err != nil {
+				errorList = append(errorList, fmt.Errorf("failed to validate worker node taints %v", err))
+			}
 		}
 		if workerGroupCount != *w.Count {
 			errorList = append(errorList, fmt.Errorf("worker node group %s count does not match expected: %d of %d", w.Name, workerGroupCount, *w.Count))
 		}
 	}
+
 	if len(errorList) > 0 {
 		return apierrors.NewAggregate(errorList)
 	}
@@ -166,6 +174,14 @@ func validateNodeReady(node corev1.Node, kubeVersion v1alpha1.KubernetesVersion)
 	return nil
 }
 
+func validateControlPlaneTaints(cluster *v1alpha1.Cluster, node corev1.Node) error {
+	if cluster.IsSingleNode() {
+		return api.ValidateControlPlaneNoTaints(cluster.Spec.ControlPlaneConfiguration, node)
+	}
+
+	return api.ValidateControlPlaneTaints(cluster.Spec.ControlPlaneConfiguration, node)
+}
+
 func filterWorkerNodes(nodes []corev1.Node, ms []v1beta1.MachineSet, w v1alpha1.WorkerNodeGroupConfiguration) []corev1.Node {
 	wNodes := make([]corev1.Node, 0)
 	for _, node := range nodes {

test/framework/cluster/validations/cluster_test.go

@@ -14,6 +14,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
+	"github.com/aws/eks-anywhere/internal/pkg/api"
 	"github.com/aws/eks-anywhere/internal/test"
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	"github.com/aws/eks-anywhere/pkg/cluster"
@@ -319,7 +320,90 @@ func TestValidateControlPlanes(t *testing.T) {
 					}
 				}),
 			},
-			wantErr: "controlplane node test-node-2 not ready yet.",
+
+			wantErr: "node test-node-2 not ready yet.",
+		},
+		{
+			name: "control plane node with taint match ",
+			spec: test.NewClusterSpec(func(s *cluster.Spec) {
+				s.Cluster = testCluster()
+				s.Cluster.Spec.ControlPlaneConfiguration = v1alpha1.ControlPlaneConfiguration{
+					Count:  2,
+					Taints: []corev1.Taint{api.ControlPlaneTaint()},
+				}
+			}),
+			nodes: []*corev1.Node{
+				controlPlaneNode(func(node *corev1.Node) {
+					node.Name = "test-node-1"
+					node.Status.Conditions = []corev1.NodeCondition{
+						{
+							Type:   corev1.NodeReady,
+							Status: corev1.ConditionTrue,
+						},
+					}
+				}),
+				controlPlaneNode(func(node *corev1.Node) {
+					node.Name = "test-node-2"
+					node.Status.Conditions = []corev1.NodeCondition{
+						{
+							Type:   corev1.NodeReady,
+							Status: corev1.ConditionTrue,
+						},
+					}
+				}),
+			},
+			wantErr: "",
+		},
+		{
+			name: "control plane single node with taints ",
+			spec: test.NewClusterSpec(func(s *cluster.Spec) {
+				s.Cluster = testCluster()
+				s.Cluster.Spec.ControlPlaneConfiguration = v1alpha1.ControlPlaneConfiguration{
+					Count:  1,
+					Taints: []corev1.Taint{api.ControlPlaneTaint()},
+				}
+			}),
+			nodes: []*corev1.Node{
+				controlPlaneNode(func(node *corev1.Node) {
+					node.Name = "test-node-1"
+					node.Status.Conditions = []corev1.NodeCondition{
+						{
+							Type:   corev1.NodeReady,
+							Status: corev1.ConditionTrue,
+						},
+					}
+					node.Spec.Taints = append(node.Spec.Taints, api.ControlPlaneTaint())
+				}),
+			},
+			wantErr: "taints on control plane node test-node-1 or corresponding control plane configuration found",
+		},
+		{
+			name: "control plane node with taint does not match ",
+			spec: test.NewClusterSpec(func(s *cluster.Spec) {
+				s.Cluster = testCluster()
+				s.Cluster.Spec.ControlPlaneConfiguration = v1alpha1.ControlPlaneConfiguration{
+					Count: 1,
+					Taints: []corev1.Taint{
+						{
+							Key:    "key1",
+							Value:  "value1",
+							Effect: corev1.TaintEffectNoExecute,
+						},
+					},
+				}
+			}),
+			nodes: []*corev1.Node{
+				controlPlaneNode(func(node *corev1.Node) {
+					node.Name = "test-node-1"
+					node.Status.Conditions = []corev1.NodeCondition{
+						{
+							Type:   corev1.NodeReady,
+							Status: corev1.ConditionTrue,
+						},
+					}
+				}),
+			},
+			wantErr: "failed to validate controlplane node taints: taints on control plane node test-node-1 or corresponding control plane configuration found",
 		},
 	}
 
@@ -480,7 +564,11 @@ func controlPlaneNode(opts ...controlPlaneNodeOpt) *corev1.Node {
 				"node-role.kubernetes.io/control-plane": "",
 			},
 		},
-		Spec: corev1.NodeSpec{},
+		Spec: corev1.NodeSpec{
+			Taints: []corev1.Taint{
+				api.ControlPlaneTaint(),
+			},
+		},
 		Status: corev1.NodeStatus{
 			Conditions: []corev1.NodeCondition{
 				{