E2e/modular upgrades (#6868)

* E2E tests for modular upgrades for tinkerbell provider

Signed-off-by: Rahul Ganesh <rahulgab@amazon.com>

* Add E2E tests for modular upgrade for bare-metal provider

Signed-off-by: Rahul Ganesh <rahulgab@amazon.com>

* Update modular upgrades tests to Hardware count file

Signed-off-by: Rahul Ganesh <rahulgab@amazon.com>

---------

Signed-off-by: Rahul Ganesh <rahulgab@amazon.com>
Co-authored-by: Rahul Ganesh <rahulgab@amazon.com>

internal/pkg/api/cluster.go

@@ -129,6 +129,33 @@ func WithServiceCidr(svcCidr string) ClusterFiller {
 	}
 }
 
+// WithWorkerKubernetesVersion sets the kubernetes version field for the given worker group.
+func WithWorkerKubernetesVersion(name string, version *anywherev1.KubernetesVersion) ClusterFiller {
+	return func(c *anywherev1.Cluster) {
+		pos := -1
+		for i, wng := range c.Spec.WorkerNodeGroupConfigurations {
+			if wng.Name == name {
+				wng.KubernetesVersion = version
+				pos = i
+				c.Spec.WorkerNodeGroupConfigurations[pos] = wng
+				break
+			}
+		}
+		// Append the worker node group if not already found in existing configuration
+		if pos == -1 {
+			c.Spec.WorkerNodeGroupConfigurations = append(c.Spec.WorkerNodeGroupConfigurations, workerNodeWithKubernetesVersion(name, version))
+		}
+	}
+}
+
+func workerNodeWithKubernetesVersion(name string, version *anywherev1.KubernetesVersion) anywherev1.WorkerNodeGroupConfiguration {
+	return anywherev1.WorkerNodeGroupConfiguration{
+		Name:              name,
+		Count:             ptr.Int(1),
+		KubernetesVersion: version,
+	}
+}
+
 func WithWorkerNodeCount(r int) ClusterFiller {
 	return func(c *anywherev1.Cluster) {
 		if len(c.Spec.WorkerNodeGroupConfigurations) == 0 {

internal/pkg/api/tinkerbell.go

@@ -54,6 +54,29 @@ func WithTinkerbellOSImageURL(value string) TinkerbellFiller {
 	}
 }
 
+// WithTinkerbellCPMachineConfigOSImageURL sets the OSImageURL & OSFamily for control-plane machine config.
+func WithTinkerbellCPMachineConfigOSImageURL(imageURL string, OSFamily anywherev1.OSFamily) TinkerbellFiller {
+	return func(config TinkerbellConfig) {
+		clusterName := config.clusterName
+		cpName := providers.GetControlPlaneNodeName(clusterName)
+		cpMachineConfig := config.machineConfigs[cpName]
+		cpMachineConfig.Spec.OSImageURL = imageURL
+		cpMachineConfig.Spec.OSFamily = OSFamily
+		config.machineConfigs[cpName] = cpMachineConfig
+	}
+}
+
+// WithTinkerbellWorkerMachineConfigOSImageURL sets the OSImageURL & OSFamily for worker machine config.
+func WithTinkerbellWorkerMachineConfigOSImageURL(imageURL string, OSFamily anywherev1.OSFamily) TinkerbellFiller {
+	return func(config TinkerbellConfig) {
+		clusterName := config.clusterName
+		workerMachineConfig := config.machineConfigs[clusterName]
+		workerMachineConfig.Spec.OSImageURL = imageURL
+		workerMachineConfig.Spec.OSFamily = OSFamily
+		config.machineConfigs[clusterName] = workerMachineConfig
+	}
+}
+
 // WithHookImagesURLPath modify HookImagesURL, it's useful for airgapped testing.
 func WithHookImagesURLPath(value string) TinkerbellFiller {
 	return func(config TinkerbellConfig) {

test/e2e/TINKERBELL_HARDWARE_COUNT.yaml

@@ -127,4 +127,6 @@ TestTinkerbellAirgappedKubernetes128BottleRocketRegistryMirror: 2
 TestTinkerbellAirgappedKubernetes128BottlerocketProxyConfigFlow: 2
 TestTinkerbellAirgappedKubernetes128UbuntuProxyConfigFlow: 2
 TestTinkerbellKubernetes128UbuntuOOB: 2
-TestTinkerbellK8sUpgrade127to128WithUbuntuOOB: 4
+TestTinkerbellK8sUpgrade127to128WithUbuntuOOB: 4
+TestTinkerbellKubernetes127UbuntuTo128UpgradeCPOnly: 3
+TestTinkerbellKubernetes127UbuntuTo128UpgradeWorkerOnly: 3

test/e2e/tinkerbell_test.go

@@ -119,6 +119,54 @@ func TestTinkerbellKubernetes127UbuntuTo128Upgrade(t *testing.T) {
 	)
 }
 
+func TestTinkerbellKubernetes127UbuntuTo128UpgradeCPOnly(t *testing.T) {
+	provider := framework.NewTinkerbell(t)
+	kube127 := v1alpha1.Kube127
+	test := framework.NewClusterE2ETest(
+		t,
+		provider,
+		framework.WithClusterFiller(api.WithKubernetesVersion(kube127)),
+		framework.WithClusterFiller(api.WithControlPlaneCount(1)),
+		framework.WithClusterFiller(api.WithWorkerNodeCount(1)),
+		framework.WithClusterFiller(api.WithWorkerKubernetesVersion(nodeGroupLabel1, &kube127)),
+		framework.WithControlPlaneHardware(2),
+		framework.WithWorkerHardware(1),
+	).WithClusterConfig(
+		provider.WithCPKubeVersionAndOS(v1alpha1.Kube127, framework.Ubuntu2004),
+		provider.WithWorkerKubeVersionAndOS(v1alpha1.Kube127, framework.Ubuntu2004),
+	)
+	runSimpleUpgradeFlowWorkerNodeVersionForBareMetal(
+		test,
+		framework.WithClusterUpgrade(api.WithKubernetesVersion(v1alpha1.Kube128)),
+		provider.WithProviderUpgrade(framework.Ubuntu128ImageForCP()),
+	)
+}
+
+func TestTinkerbellKubernetes127UbuntuTo128UpgradeWorkerOnly(t *testing.T) {
+	provider := framework.NewTinkerbell(t)
+	kube127 := v1alpha1.Kube127
+	kube128 := v1alpha1.Kube128
+	test := framework.NewClusterE2ETest(
+		t,
+		provider,
+		framework.WithClusterFiller(),
+		framework.WithClusterFiller(api.WithKubernetesVersion(kube128)),
+		framework.WithClusterFiller(api.WithControlPlaneCount(1)),
+		framework.WithClusterFiller(api.WithWorkerNodeCount(1)),
+		framework.WithClusterFiller(api.WithWorkerKubernetesVersion(nodeGroupLabel1, &kube127)),
+		framework.WithControlPlaneHardware(1),
+		framework.WithWorkerHardware(2),
+	).WithClusterConfig(
+		provider.WithCPKubeVersionAndOS(v1alpha1.Kube128, framework.Ubuntu2004),
+		provider.WithWorkerKubeVersionAndOS(v1alpha1.Kube127, framework.Ubuntu2004),
+	)
+	runSimpleUpgradeFlowWorkerNodeVersionForBareMetal(
+		test,
+		framework.WithClusterUpgrade(api.WithWorkerKubernetesVersion(nodeGroupLabel1, &kube128)),
+		provider.WithProviderUpgrade(framework.Ubuntu128ImageForWorker()),
+	)
+}
+
 func TestTinkerbellKubernetes125To126Ubuntu2204Upgrade(t *testing.T) {
 	provider := framework.NewTinkerbell(t)
 	test := framework.NewClusterE2ETest(

test/e2e/upgrade.go

@@ -51,6 +51,17 @@ func runSimpleUpgradeFlowForBareMetal(test *framework.ClusterE2ETest, updateVers
 	test.ValidateHardwareDecommissioned()
 }
 
+func runSimpleUpgradeFlowWorkerNodeVersionForBareMetal(test *framework.ClusterE2ETest, clusterOpts ...framework.ClusterE2ETestOpt) {
+	test.GenerateHardwareConfig()
+	test.PowerOffHardware()
+	test.CreateCluster(framework.WithControlPlaneWaitTimeout("20m"))
+	test.UpgradeClusterWithNewConfig(clusterOpts)
+	test.ValidateClusterState()
+	test.StopIfFailed()
+	test.DeleteCluster()
+	test.ValidateHardwareDecommissioned()
+}
+
 // runSimpleUpgradeFlowForBaremetalWithoutClusterConfigGeneration runs the Create, Upgrade and Delete cluster flows
 // for Baremetal that use the cluster config generated by the WithClusterConfig method when the test object is created,
 // and avoids regenerating a cluster config with defaults.

test/framework/cluster/validations/cluster.go

@@ -98,14 +98,18 @@ func ValidateWorkerNodes(ctx context.Context, vc clusterf.StateValidationConfig)
 	// deduce the worker node group configuration to node mapping via the machine deployment and machine set
 	for _, w := range wn {
 		workerGroupCount := 0
+		k8sVersion := vc.ClusterSpec.Cluster.Spec.KubernetesVersion
+		if w.KubernetesVersion != nil {
+			k8sVersion = *w.KubernetesVersion
+		}
 		ms, err := getWorkerNodeMachineSets(ctx, vc, w)
 		if err != nil {
 			return fmt.Errorf("failed to get machine sets when validating worker node: %v", err)
 		}
 		workerNodes := filterWorkerNodes(nodes.Items, ms, w)
 		workerGroupCount += len(workerNodes)
 		for _, node := range workerNodes {
-			if err := validateNodeReady(node, vc.ClusterSpec.Cluster.Spec.KubernetesVersion); err != nil {
+			if err := validateNodeReady(node, k8sVersion); err != nil {
 				errorList = append(errorList, fmt.Errorf("failed to validate worker node ready %v", err))
 			}
 			if err := api.ValidateWorkerNodeTaints(w, node); err != nil {

test/framework/tinkerbell.go

@@ -34,6 +34,8 @@ const (
 	tinkerbellInventoryCsvFilePathEnvVar         = "T_TINKERBELL_INVENTORY_CSV"
 	tinkerbellSSHAuthorizedKey                   = "T_TINKERBELL_SSH_AUTHORIZED_KEY"
 	TinkerbellCIEnvironment                      = "T_TINKERBELL_CI_ENVIRONMENT"
+	controlPlaneIdentifier                       = "cp"
+	workerIdentifier                             = "worker"
 )
 
 var requiredTinkerbellEnvVars = []string{
@@ -148,12 +150,32 @@ func (t *Tinkerbell) WithKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersi
 	return api.JoinClusterConfigFillers(
 		api.ClusterToConfigFiller(api.WithKubernetesVersion(kubeVersion)),
 		api.TinkerbellToConfigFiller(
-			imageForKubeVersionAndOS(kubeVersion, os),
+			imageForKubeVersionAndOS(kubeVersion, os, ""),
 			api.WithOsFamilyForAllTinkerbellMachines(osFamiliesForOS[os]),
 		),
 	)
 }
 
+// WithCPKubeVersionAndOS returns a cluster config filler that sets the cluster kube version and the right image for CP
+// tinkerbell machine configs.
+func (t *Tinkerbell) WithCPKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS) api.ClusterConfigFiller {
+	return api.JoinClusterConfigFillers(
+		api.TinkerbellToConfigFiller(
+			imageForKubeVersionAndOS(kubeVersion, os, controlPlaneIdentifier),
+		),
+	)
+}
+
+// WithWorkerKubeVersionAndOS returns a cluster config filler that sets the cluster kube version and the right image for all
+// Worker tinkerbell machine configs.
+func (t *Tinkerbell) WithWorkerKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS) api.ClusterConfigFiller {
+	return api.JoinClusterConfigFillers(
+		api.TinkerbellToConfigFiller(
+			imageForKubeVersionAndOS(kubeVersion, os, workerIdentifier),
+		),
+	)
+}
+
 // WithNewWorkerNodeGroup returns an api.ClusterFiller that adds a new workerNodeGroupConfiguration and
 // a corresponding TinkerbellMachineConfig to the cluster config.
 func (t *Tinkerbell) WithNewWorkerNodeGroup(name string, workerNodeGroup *WorkerNodeGroup) api.ClusterConfigFiller {
@@ -168,63 +190,70 @@ func envVarForImage(os OS, kubeVersion anywherev1.KubernetesVersion) string {
 
 // withKubeVersionAndOS returns a cluster config filler that sets the cluster kube version and the right image for all
 // tinkerbell machine configs.
-func withKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS, release *releasev1.EksARelease) TinkerbellOpt {
+func withKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS, machineConfigType string, release *releasev1.EksARelease) TinkerbellOpt {
+	if machineConfigType == controlPlaneIdentifier || machineConfigType == workerIdentifier {
+		return func(t *Tinkerbell) {
+			t.fillers = append(t.fillers,
+				imageForKubeVersionAndOS(kubeVersion, os, machineConfigType),
+			)
+		}
+	}
 	return func(t *Tinkerbell) {
 		t.fillers = append(t.fillers,
-			imageForKubeVersionAndOS(kubeVersion, os),
+			imageForKubeVersionAndOS(kubeVersion, os, ""),
 			api.WithOsFamilyForAllTinkerbellMachines(osFamiliesForOS[os]),
 		)
 	}
 }
 
 // WithUbuntu124Tinkerbell tink test with ubuntu 1.24.
 func WithUbuntu124Tinkerbell() TinkerbellOpt {
-	return withKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004, nil)
+	return withKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004, "", nil)
 }
 
 // WithUbuntu125Tinkerbell tink test with ubuntu 1.25.
 func WithUbuntu125Tinkerbell() TinkerbellOpt {
-	return withKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004, nil)
+	return withKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004, "", nil)
 }
 
 // WithUbuntu126Tinkerbell tink test with ubuntu 1.26.
 func WithUbuntu126Tinkerbell() TinkerbellOpt {
-	return withKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, nil)
+	return withKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, "", nil)
 }
 
 // WithUbuntu127Tinkerbell tink test with ubuntu 1.27.
 func WithUbuntu127Tinkerbell() TinkerbellOpt {
-	return withKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, nil)
+	return withKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, "", nil)
 }
 
 // WithUbuntu128Tinkerbell tink test with ubuntu 1.28.
 func WithUbuntu128Tinkerbell() TinkerbellOpt {
-	return withKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, nil)
+	return withKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, "", nil)
 }
 
 // WithRedHat124Tinkerbell tink test with redhat 1.24.
 func WithRedHat124Tinkerbell() TinkerbellOpt {
-	return withKubeVersionAndOS(anywherev1.Kube124, RedHat8, nil)
+	return withKubeVersionAndOS(anywherev1.Kube124, RedHat8, "", nil)
 }
 
 // WithRedHat125Tinkerbell tink test with redhat 1.25.
 func WithRedHat125Tinkerbell() TinkerbellOpt {
-	return withKubeVersionAndOS(anywherev1.Kube125, RedHat8, nil)
+	return withKubeVersionAndOS(anywherev1.Kube125, RedHat8, "", nil)
 }
 
 // WithRedHat126Tinkerbell tink test with redhat 1.26.
 func WithRedHat126Tinkerbell() TinkerbellOpt {
-	return withKubeVersionAndOS(anywherev1.Kube126, RedHat8, nil)
+	return withKubeVersionAndOS(anywherev1.Kube126, RedHat8, "", nil)
 }
 
 // WithRedHat127Tinkerbell tink test with redhat 1.27.
 func WithRedHat127Tinkerbell() TinkerbellOpt {
-	return withKubeVersionAndOS(anywherev1.Kube127, RedHat8, nil)
+	return withKubeVersionAndOS(anywherev1.Kube127, RedHat8, "", nil)
 }
 
 // WithRedHat128Tinkerbell tink test with redhat 1.27.
 func WithRedHat128Tinkerbell() TinkerbellOpt {
-	return withKubeVersionAndOS(anywherev1.Kube128, RedHat8, nil)
+	return withKubeVersionAndOS(anywherev1.Kube128, RedHat8, "", nil)
 }
 
 func WithBottleRocketTinkerbell() TinkerbellOpt {
@@ -271,46 +300,85 @@ func WithHookImagesURLPath(url string) TinkerbellOpt {
 	}
 }
 
-func imageForKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, operatingSystem OS) api.TinkerbellFiller {
-	return api.WithTinkerbellOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion)))
+// imageForKubeVersionAndOS sets osImageURL on the appropriate field in the Machine Config based on the machineConfigType string provided else sets it at Data Center config.
+func imageForKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, operatingSystem OS, machineConfigType string) api.TinkerbellFiller {
+	var tinkerbellFiller api.TinkerbellFiller
+	if machineConfigType == workerIdentifier {
+		tinkerbellFiller = api.WithTinkerbellWorkerMachineConfigOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion)), osFamiliesForOS[operatingSystem])
+	} else if machineConfigType == controlPlaneIdentifier {
+		tinkerbellFiller = api.WithTinkerbellCPMachineConfigOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion)), osFamiliesForOS[operatingSystem])
+	} else {
+		tinkerbellFiller = api.WithTinkerbellOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion)))
+	}
+	return tinkerbellFiller
 }
 
 // Ubuntu124Image represents an Ubuntu raw image corresponding to Kubernetes 1.24.
 func Ubuntu124Image() api.TinkerbellFiller {
-	return imageForKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004)
+	return imageForKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004, "")
 }
 
 // Ubuntu125Image represents an Ubuntu raw image corresponding to Kubernetes 1.25.
 func Ubuntu125Image() api.TinkerbellFiller {
-	return imageForKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004)
+	return imageForKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004, "")
 }
 
 // Ubuntu126Image represents an Ubuntu raw image corresponding to Kubernetes 1.26.
 func Ubuntu126Image() api.TinkerbellFiller {
-	return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004)
+	return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, "")
 }
 
 // Ubuntu127Image represents an Ubuntu raw image corresponding to Kubernetes 1.27.
 func Ubuntu127Image() api.TinkerbellFiller {
-	return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004)
+	return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, "")
 }
 
 // Ubuntu128Image represents an Ubuntu raw image corresponding to Kubernetes 1.28.
 func Ubuntu128Image() api.TinkerbellFiller {
-	return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004)
+	return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, "")
+}
+
+// Ubuntu126ImageForCP represents an Ubuntu raw image corresponding to Kubernetes 1.28 and is set for CP machine config.
+func Ubuntu126ImageForCP() api.TinkerbellFiller {
+	return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, controlPlaneIdentifier)
+}
+
+// Ubuntu127ImageForCP represents an Ubuntu raw image corresponding to Kubernetes 1.27 and is set for CP machine config.
+func Ubuntu127ImageForCP() api.TinkerbellFiller {
+	return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, controlPlaneIdentifier)
+}
+
+// Ubuntu128ImageForCP represents an Ubuntu raw image corresponding to Kubernetes 1.28 and is set for CP machine config.
+func Ubuntu128ImageForCP() api.TinkerbellFiller {
+	return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, controlPlaneIdentifier)
+}
+
+// Ubuntu126ImageForWorker represents an Ubuntu raw image corresponding to Kubernetes 1.28 and is set for worker machine config.
+func Ubuntu126ImageForWorker() api.TinkerbellFiller {
+	return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, workerIdentifier)
+}
+
+// Ubuntu127ImageForWorker represents an Ubuntu raw image corresponding to Kubernetes 1.27 and is set for worker machine config.
+func Ubuntu127ImageForWorker() api.TinkerbellFiller {
+	return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, workerIdentifier)
+}
+
+// Ubuntu128ImageForWorker represents an Ubuntu raw image corresponding to Kubernetes 1.28 and is set for worker machine config.
+func Ubuntu128ImageForWorker() api.TinkerbellFiller {
+	return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, workerIdentifier)
 }
 
 // Ubuntu2204Kubernetes126Image represents an Ubuntu 22.04 raw image corresponding to Kubernetes 1.26.
 func Ubuntu2204Kubernetes126Image() api.TinkerbellFiller {
-	return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2204)
+	return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2204, "")
 }
 
 // Ubuntu2204Kubernetes127Image represents an Ubuntu 22.04 raw image corresponding to Kubernetes 1.27.
 func Ubuntu2204Kubernetes127Image() api.TinkerbellFiller {
-	return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2204)
+	return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2204, "")
 }
 
 // Ubuntu2204Kubernetes128Image represents an Ubuntu 22.04 raw image corresponding to Kubernetes 1.28.
 func Ubuntu2204Kubernetes128Image() api.TinkerbellFiller {
-	return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2204)
+	return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2204, "")
 }