Add controlplane spec field CPUpgrade (#7377)

config/crd/bases/anywhere.eks.amazonaws.com_controlplaneupgrades.yaml

@@ -74,6 +74,14 @@ spec:
                     description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
                     type: string
                 type: object
+              controlPlaneSpecData:
+                description: ControlPlaneSpecData contains base64 encoded KCP spec
+                  that's used to update the statuses of CAPI objects once the control
+                  plane upgrade is done. This field is needed so that we have a static
+                  copy of the control plane spec in case it gets modified after the
+                  ControlPlaneUpgrade was created, as ControlPlane is a reference
+                  to the object in real time.
+                type: string
               etcdVersion:
                 description: EtcdVersion refers to the version of ETCD to upgrade
                   to.
@@ -148,6 +156,7 @@ spec:
                 type: array
             required:
             - controlPlane
+            - controlPlaneSpecData
             - etcdVersion
             - kubernetesVersion
             - machinesRequireUpgrade

config/manifest/eksa-components.yaml

@@ -4452,6 +4452,14 @@ spec:
                     description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
                     type: string
                 type: object
+              controlPlaneSpecData:
+                description: ControlPlaneSpecData contains base64 encoded KCP spec
+                  that's used to update the statuses of CAPI objects once the control
+                  plane upgrade is done. This field is needed so that we have a static
+                  copy of the control plane spec in case it gets modified after the
+                  ControlPlaneUpgrade was created, as ControlPlane is a reference
+                  to the object in real time.
+                type: string
               etcdVersion:
                 description: EtcdVersion refers to the version of ETCD to upgrade
                   to.
@@ -4526,6 +4534,7 @@ spec:
                 type: array
             required:
             - controlPlane
+            - controlPlaneSpecData
             - etcdVersion
             - kubernetesVersion
             - machinesRequireUpgrade

controllers/controlplaneupgrade_controller.go

@@ -18,6 +18,8 @@ package controllers
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"time"
 
@@ -27,6 +29,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/patch"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -39,7 +43,8 @@ import (
 
 // controlPlaneUpgradeFinalizerName is the finalizer added to NodeUpgrade objects to handle deletion.
 const (
-	controlPlaneUpgradeFinalizerName = "controlplaneupgrades.anywhere.eks.amazonaws.com/finalizer"
+	controlPlaneUpgradeFinalizerName      = "controlplaneupgrades.anywhere.eks.amazonaws.com/finalizer"
+	kubeadmClusterConfigurationAnnotation = "controlplane.cluster.x-k8s.io/kubeadm-cluster-configuration"
 )
 
 // ControlPlaneUpgradeReconciler reconciles a ControlPlaneUpgradeReconciler object.
@@ -215,19 +220,9 @@ func (r *ControlPlaneUpgradeReconciler) updateStatus(ctx context.Context, log lo
 			return fmt.Errorf("getting node upgrader for machine %s: %v", machineRef.Name, err)
 		}
 		if nodeUpgrade.Status.Completed {
-			machine, err := getCapiMachine(ctx, r.client, nodeUpgrade)
-			if err != nil {
+			if err := r.updateMachine(ctx, log, cpUpgrade, nodeUpgrade); err != nil {
 				return err
 			}
-			machinePatchHelper, err := patch.NewHelper(machine, r.client)
-			if err != nil {
-				return err
-			}
-			log.Info("Updating K8s version in  machine", "Machine", machine.Name)
-			machine.Spec.Version = &nodeUpgrade.Spec.KubernetesVersion
-			if err := machinePatchHelper.Patch(ctx, machine); err != nil {
-				return fmt.Errorf("updating spec for machine %s: %v", machine.Name, err)
-			}
 			nodesUpgradeCompleted++
 			nodesUpgradeRequired--
 		}
@@ -239,6 +234,49 @@ func (r *ControlPlaneUpgradeReconciler) updateStatus(ctx context.Context, log lo
 	return nil
 }
 
+func (r *ControlPlaneUpgradeReconciler) updateMachine(ctx context.Context, log logr.Logger, cpUpgrade *anywherev1.ControlPlaneUpgrade, nodeUpgrade *anywherev1.NodeUpgrade) error {
+	machine, err := getCapiMachine(ctx, r.client, nodeUpgrade)
+	if err != nil {
+		return err
+	}
+	machinePatchHelper, err := patch.NewHelper(machine, r.client)
+	if err != nil {
+		return err
+	}
+	log.Info("Updating K8s version and kubeadmClusterConfiguration annotation in machine", "Machine", machine.Name)
+	// Update the machine k8s version
+	machine.Spec.Version = &nodeUpgrade.Spec.KubernetesVersion
+
+	// Update the machine kubeadmClusterConfiguration annotation
+	kcc, err := getKubeadmClusterConfig(cpUpgrade)
+	if err != nil {
+		return err
+	}
+	annotations.AddAnnotations(machine, map[string]string{kubeadmClusterConfigurationAnnotation: kcc})
+
+	if err := machinePatchHelper.Patch(ctx, machine); err != nil {
+		return fmt.Errorf("updating spec for machine %s: %v", machine.Name, err)
+	}
+
+	return nil
+}
+
+func getKubeadmClusterConfig(cpUpgrade *anywherev1.ControlPlaneUpgrade) (string, error) {
+	kcpSpec := &controlplanev1.KubeadmControlPlaneSpec{}
+	decodedKcpSpec, err := base64.StdEncoding.DecodeString(cpUpgrade.Spec.ControlPlaneSpecData)
+	if err != nil {
+		return "", fmt.Errorf("decoding cpUpgrade.Spec.ControlPlaneSpec: %v", err)
+	}
+	if err := json.Unmarshal(decodedKcpSpec, kcpSpec); err != nil {
+		return "", fmt.Errorf("unmarshaling cpUpgrade.Spec.ControlPlaneSpec: %v", err)
+	}
+	clusterConfig, err := json.Marshal(kcpSpec.KubeadmConfigSpec.ClusterConfiguration)
+	if err != nil {
+		return "", fmt.Errorf("marshaling KCP cluster configuration: %v", err)
+	}
+	return string(clusterConfig), nil
+}
+
 func getCapiMachine(ctx context.Context, client client.Client, nodeUpgrade *anywherev1.NodeUpgrade) (*clusterv1.Machine, error) {
 	machine := &clusterv1.Machine{}
 	if err := client.Get(ctx, GetNamespacedNameType(nodeUpgrade.Spec.Machine.Name, nodeUpgrade.Spec.Machine.Namespace), machine); err != nil {

controllers/controlplaneupgrade_controller_test.go

@@ -2,6 +2,8 @@ package controllers_test
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"strings"
 	"testing"
@@ -12,6 +14,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
@@ -122,6 +125,46 @@ func TestCPUpgradeReconcileNodeUpgraderCreate(t *testing.T) {
 	g.Expect(err).ToNot(HaveOccurred())
 }
 
+func TestCPUpgradeReconcileNodeUpgraderInvalidKCPSpec(t *testing.T) {
+	g := NewWithT(t)
+	ctx := context.Background()
+	cluster, machines, nodes, cpUpgrade, nodeUpgrades := getObjectsForCPUpgradeTest()
+	for i := range nodeUpgrades {
+		nodeUpgrades[i].Name = fmt.Sprintf("%s-node-upgrader", machines[i].Name)
+		nodeUpgrades[i].Status = anywherev1.NodeUpgradeStatus{
+			Completed: true,
+		}
+	}
+
+	for _, test := range []struct {
+		name    string
+		kcpSpec string
+		errMsg  string
+	}{
+		{
+			name:    "invalid base64",
+			kcpSpec: "not-a-valid-base-64",
+			errMsg:  "decoding cpUpgrade.Spec.ControlPlaneSpec",
+		},
+		{
+			name:    "invalid json",
+			kcpSpec: "aW52YWxpZC1qc29uCg==",
+			errMsg:  "unmarshaling cpUpgrade.Spec.ControlPlaneSpec",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			cpUpgrade.Spec.ControlPlaneSpecData = test.kcpSpec
+			objs := []runtime.Object{cluster, machines[0], machines[1], nodes[0], nodes[1], cpUpgrade, nodeUpgrades[0]}
+			client := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build()
+			r := controllers.NewControlPlaneUpgradeReconciler(client)
+			req := cpUpgradeRequest(cpUpgrade)
+			_, err := r.Reconcile(ctx, req)
+			g.Expect(err).To(HaveOccurred())
+			g.Expect(err.Error()).To(ContainSubstring(test.errMsg))
+		})
+	}
+}
+
 func TestCPUpgradeReconcileNodesNotReadyYet(t *testing.T) {
 	g := NewWithT(t)
 	ctx := context.Background()
@@ -247,7 +290,7 @@ func cpUpgradeRequest(cpUpgrade *anywherev1.ControlPlaneUpgrade) reconcile.Reque
 }
 
 func generateCPUpgrade(machine []*clusterv1.Machine) *anywherev1.ControlPlaneUpgrade {
-	etcdVersion := "v3.5.9-eks-1-28-9"
+	kcpSpec, _ := json.Marshal(&controlplanev1.KubeadmControlPlaneSpec{})
 	return &anywherev1.ControlPlaneUpgrade{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "cp-upgrade-request",
@@ -271,8 +314,9 @@ func generateCPUpgrade(machine []*clusterv1.Machine) *anywherev1.ControlPlaneUpg
 					Namespace: machine[1].Namespace,
 				},
 			},
-			KubernetesVersion: "v1.28.3-eks-1-28-9",
-			EtcdVersion:       etcdVersion,
+			KubernetesVersion:    "v1.28.3-eks-1-28-9",
+			EtcdVersion:          "v3.5.9-eks-1-28-9",
+			ControlPlaneSpecData: base64.StdEncoding.EncodeToString(kcpSpec),
 		},
 	}
 }

controllers/kubeadmcontrolplane_controller.go

@@ -18,6 +18,8 @@ package controllers
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
@@ -127,7 +129,11 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, log logr.
 			if err != nil {
 				return ctrl.Result{}, fmt.Errorf("retrieving list of control plane machines: %v", err)
 			}
-			if err := r.client.Create(ctx, controlPlaneUpgrade(kcp, machines)); client.IgnoreAlreadyExists(err) != nil {
+			cpUpgrade, err := controlPlaneUpgrade(kcp, machines)
+			if err != nil {
+				return ctrl.Result{}, fmt.Errorf("generating ControlPlaneUpgrade: %v", err)
+			}
+			if err := r.client.Create(ctx, cpUpgrade); client.IgnoreAlreadyExists(err) != nil {
 				return ctrl.Result{}, fmt.Errorf("failed to create control plane upgrade for KubeadmControlPlane %s:  %v", kcp.Name, err)
 			}
 			return ctrl.Result{}, nil
@@ -183,7 +189,12 @@ func (r *KubeadmControlPlaneReconciler) validateStackedEtcd(kcp *controlplanev1.
 	return nil
 }
 
-func controlPlaneUpgrade(kcp *controlplanev1.KubeadmControlPlane, machines []corev1.ObjectReference) *anywherev1.ControlPlaneUpgrade {
+func controlPlaneUpgrade(kcp *controlplanev1.KubeadmControlPlane, machines []corev1.ObjectReference) (*anywherev1.ControlPlaneUpgrade, error) {
+	kcpSpec, err := json.Marshal(kcp.Spec)
+	if err != nil {
+		return nil, fmt.Errorf("marshaling KCP spec: %v", err)
+	}
+
 	return &anywherev1.ControlPlaneUpgrade{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      cpUpgradeName(kcp.Name),
@@ -204,8 +215,9 @@ func controlPlaneUpgrade(kcp *controlplanev1.KubeadmControlPlane, machines []cor
 			KubernetesVersion:      kcp.Spec.Version,
 			EtcdVersion:            kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local.ImageTag,
 			MachinesRequireUpgrade: machines,
+			ControlPlaneSpecData:   base64.StdEncoding.EncodeToString(kcpSpec),
 		},
-	}
+	}, nil
 }
 
 func cpUpgradeName(kcpName string) string {

controllers/kubeadmcontrolplane_controller_test.go

@@ -2,6 +2,8 @@ package controllers_test
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"testing"
 
 	. "github.com/onsi/gomega"
@@ -108,6 +110,9 @@ func TestKCPReconcileCreateControlPlaneUpgrade(t *testing.T) {
 	g.Expect(len(cpu.Spec.MachinesRequireUpgrade)).To(BeEquivalentTo(len(kcpObjs.cpUpgrade.Spec.MachinesRequireUpgrade)))
 	g.Expect(cpu.Spec.EtcdVersion).To(BeEquivalentTo(kcpObjs.cpUpgrade.Spec.EtcdVersion))
 	g.Expect(cpu.Spec.KubernetesVersion).To(BeEquivalentTo(kcpObjs.cpUpgrade.Spec.KubernetesVersion))
+	kcpSpec, err := json.Marshal(kcpObjs.kcp.Spec)
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(cpu.Spec.ControlPlaneSpecData).To(BeEquivalentTo(base64.StdEncoding.EncodeToString(kcpSpec)))
 }
 
 func TestKCPReconcileControlPlaneUpgradeReady(t *testing.T) {

pkg/api/v1alpha1/controlplaneupgrade_types.go

@@ -21,6 +21,13 @@ type ControlPlaneUpgradeSpec struct {
 
 	// EtcdVersion refers to the version of ETCD to upgrade to.
 	EtcdVersion string `json:"etcdVersion"`
+
+	// ControlPlaneSpecData contains base64 encoded KCP spec that's used to update
+	// the statuses of CAPI objects once the control plane upgrade is done.
+	// This field is needed so that we have a static copy of the control plane spec
+	// in case it gets modified after the ControlPlaneUpgrade was created,
+	// as ControlPlane is a reference to the object in real time.
+	ControlPlaneSpecData string `json:"controlPlaneSpecData"`
 }
 
 // ControlPlaneUpgradeStatus defines the observed state of ControlPlaneUpgrade.