From 45b80b4e267c30896bbbeb9da2a7524c30d92aed Mon Sep 17 00:00:00 2001 From: ahreehong <46465244+ahreehong@users.noreply.github.com> Date: Tue, 5 Dec 2023 09:39:39 -0800 Subject: [PATCH] Add support for etcd scaling --- pkg/api/v1alpha1/cluster_webhook.go | 13 +- .../upgradevalidations/immutable_fields.go | 6 +- .../preflightvalidations_test.go | 32 ---- test/e2e/cloudstack_test.go | 87 +++++++++ test/e2e/docker_test.go | 85 +++++++++ test/e2e/vsphere_test.go | 173 ++++++++++++++++++ 6 files changed, 352 insertions(+), 44 deletions(-) diff --git a/pkg/api/v1alpha1/cluster_webhook.go b/pkg/api/v1alpha1/cluster_webhook.go index 9b3f717aab2d8..47da227f138be 100644 --- a/pkg/api/v1alpha1/cluster_webhook.go +++ b/pkg/api/v1alpha1/cluster_webhook.go @@ -328,13 +328,12 @@ func validateImmutableFieldsCluster(new, old *Cluster) field.ErrorList { field.Forbidden(specPath.Child("externalEtcdConfiguration"), "cannot switch from local to external etcd topology"), ) } - if new.Spec.ExternalEtcdConfiguration != nil && old.Spec.ExternalEtcdConfiguration != nil { - if old.Spec.ExternalEtcdConfiguration.Count != new.Spec.ExternalEtcdConfiguration.Count { - allErrs = append( - allErrs, - field.Forbidden(specPath.Child("externalEtcdConfiguration.count"), fmt.Sprintf("field is immutable %v", new.Spec.ExternalEtcdConfiguration.Count)), - ) - } + + if new.Spec.ExternalEtcdConfiguration == nil && old.Spec.ExternalEtcdConfiguration != nil { + allErrs = append( + allErrs, + field.Forbidden(specPath.Child("externalEtcdConfiguration"), "cannot switch from external to local etcd topology"), + ) } if !new.Spec.GitOpsRef.Equal(old.Spec.GitOpsRef) && !old.IsSelfManaged() { diff --git a/pkg/validations/upgradevalidations/immutable_fields.go b/pkg/validations/upgradevalidations/immutable_fields.go index 4959cf53d8b51..ec2a242d3d20f 100644 --- a/pkg/validations/upgradevalidations/immutable_fields.go +++ b/pkg/validations/upgradevalidations/immutable_fields.go @@ -72,11 +72,7 @@ func ValidateImmutableFields(ctx context.Context, k validations.KubectlClient, c oldETCD := oSpec.ExternalEtcdConfiguration newETCD := nSpec.ExternalEtcdConfiguration - if oldETCD != nil && newETCD != nil { - if oldETCD.Count != newETCD.Count { - return errors.New("spec.externalEtcdConfiguration.count is immutable") - } - } else if oldETCD != newETCD { + if oldETCD != nil && newETCD == nil || oldETCD == nil && newETCD != nil { return errors.New("adding or removing external etcd during upgrade is not supported") } diff --git a/pkg/validations/upgradevalidations/preflightvalidations_test.go b/pkg/validations/upgradevalidations/preflightvalidations_test.go index 5830b72dc7fbe..8e0ac8882caca 100644 --- a/pkg/validations/upgradevalidations/preflightvalidations_test.go +++ b/pkg/validations/upgradevalidations/preflightvalidations_test.go @@ -534,20 +534,6 @@ func TestPreflightValidationsVsphere(t *testing.T) { crdResponse: errors.New("error getting clusters crd: crd not found"), wantErr: explodingClusterError, }, - { - name: "ValidationEtcdImmutable", - clusterVersion: "v1.19.16-eks-1-19-4", - upgradeVersion: "1.19", - getClusterResponse: goodClusterResponse, - cpResponse: nil, - workerResponse: nil, - nodeResponse: nil, - crdResponse: nil, - wantErr: composeError("spec.externalEtcdConfiguration.count is immutable"), - modifyExistingSpecFunc: func(s *cluster.Spec) { - s.Cluster.Spec.ExternalEtcdConfiguration.Count++ - }, - }, { name: "ValidationControlPlaneImmutable", clusterVersion: "v1.19.16-eks-1-19-4", @@ -903,24 +889,6 @@ func TestPreflightValidationsVsphere(t *testing.T) { } }, }, - { - name: "ValidationEtcdConfigReplicasImmutable", - clusterVersion: "v1.19.16-eks-1-19-4", - upgradeVersion: "1.19", - getClusterResponse: goodClusterResponse, - cpResponse: nil, - workerResponse: nil, - nodeResponse: nil, - crdResponse: nil, - wantErr: composeError("spec.externalEtcdConfiguration.count is immutable"), - modifyExistingSpecFunc: func(s *cluster.Spec) { - s.Cluster.Spec.ExternalEtcdConfiguration.Count += 1 - s.Cluster.Spec.DatacenterRef = anywherev1.Ref{ - Kind: anywherev1.VSphereDatacenterKind, - Name: "vsphere test", - } - }, - }, { name: "ValidationEtcdConfigPreviousSpecEmpty", clusterVersion: "v1.19.16-eks-1-19-4", diff --git a/test/e2e/cloudstack_test.go b/test/e2e/cloudstack_test.go index 13eea48b43ba3..7616cb95d6a68 100644 --- a/test/e2e/cloudstack_test.go +++ b/test/e2e/cloudstack_test.go @@ -4422,3 +4422,90 @@ func TestCloudStackKubernetes128ValidateDomainFourLevelsSimpleFlow(t *testing.T) ) runSimpleFlow(test) } + +// etcd scale tests +func TestCloudstackKubernetes128EtcdScaleUp(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithExternalEtcdTopology(3), + ), + ) +} + +func TestCloudstackKubernetes128EtcdScaleDown(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(3), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithExternalEtcdTopology(1), + ), + ) +} + +func TestCloudstackKubernetes127to128EtcdScaleUp(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube127), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(3), + ), + provider.WithProviderUpgrade(provider.Redhat128Template()), + ) +} + +func TestCloudstackKubernetes127to128EtcdScaleDown(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube127), + api.WithExternalEtcdTopology(3), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + ), + provider.WithProviderUpgrade(provider.Redhat128Template()), + ) +} diff --git a/test/e2e/docker_test.go b/test/e2e/docker_test.go index 8a4df11cef923..a9f6628938757 100644 --- a/test/e2e/docker_test.go +++ b/test/e2e/docker_test.go @@ -1346,3 +1346,88 @@ func TestDockerKubernetesUpgradeManagementComponents(t *testing.T) { test.RunEKSA([]string{"upgrade", "management-components", "-f", test.ClusterConfigLocation, "-v", "99"}) test.DeleteCluster() } + +// etcd scale tests +func TestDockerKubernetes128EtcdScaleUp(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewDocker(t), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithExternalEtcdTopology(3), + ), + ) +} + +func TestDockerKubernetes128EtcdScaleDown(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewDocker(t), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(3), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithExternalEtcdTopology(1), + ), + ) +} + +func TestDockerKubernetes127to128EtcdScaleUp(t *testing.T) { + provider := framework.NewDocker(t) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube127), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(3), + ), + ) +} + +func TestDockerKubernetes127to128EtcdScaleDown(t *testing.T) { + provider := framework.NewDocker(t) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube127), + api.WithExternalEtcdTopology(3), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + ), + ) +} diff --git a/test/e2e/vsphere_test.go b/test/e2e/vsphere_test.go index 3677c53e6c16e..4fd219e5f1ed4 100644 --- a/test/e2e/vsphere_test.go +++ b/test/e2e/vsphere_test.go @@ -3348,3 +3348,176 @@ func TestVSphereKubernetes127To128UbuntuManagementCPUpgradeAPI(t *testing.T) { provider.WithUbuntu128(), ) } + +// etcd scale tests +func TestVSphereKubernetesBottlerocket128EtcdScaleUp(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithBottleRocket128()), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithExternalEtcdTopology(3), + ), + ) +} + +func TestVSphereKubernetesBottlerocket128EtcdScaleDown(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithBottleRocket128()), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(3), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithExternalEtcdTopology(1), + ), + ) +} + +func TestVSphereKubernetesBottlerocket127to128EtcdScaleUp(t *testing.T) { + provider := framework.NewVSphere(t, framework.WithBottleRocket127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube127), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(3), + ), + provider.WithProviderUpgrade(provider.Bottlerocket128Template()), + ) +} + +func TestVSphereKubernetesBottlerocket127to128EtcdScaleDown(t *testing.T) { + provider := framework.NewVSphere(t, framework.WithBottleRocket127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube127), + api.WithExternalEtcdTopology(3), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + ), + provider.WithProviderUpgrade(provider.Bottlerocket128Template()), + ) +} + +func TestVSphereKubernetesUbuntu128EtcdScaleUp(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithUbuntu128()), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithExternalEtcdTopology(3), + ), + ) +} + +func TestVSphereKubernetesUbuntu128EtcdScaleDown(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithUbuntu128()), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(3), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithExternalEtcdTopology(1), + ), + ) +} + +func TestVSphereKubernetesUbuntu127to128EtcdScaleUp(t *testing.T) { + provider := framework.NewVSphere(t, framework.WithUbuntu127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube127), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(3), + ), + provider.WithProviderUpgrade(provider.Ubuntu128Template()), + ) +} + +func TestVSphereKubernetesUbuntu127to128EtcdScaleDown(t *testing.T) { + provider := framework.NewVSphere(t, framework.WithUbuntu127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube127), + api.WithExternalEtcdTopology(3), + api.WithControlPlaneCount(1), + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + ), + provider.WithProviderUpgrade(provider.Ubuntu128Template()), + ) +}