From f1aba9a6701e9cdd88c3604615299a6e3ae2b15c Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Wed, 25 Sep 2024 00:35:13 +0200 Subject: [PATCH] Update E2E tests to run Kubernetes 1.31 (#482) Update default e2e os images to rocky9 --- Makefile | 6 +- test/e2e/clusterctl_upgrade_test.go | 41 +++++----- test/e2e/config/nutanix.yaml | 74 +++++++++---------- .../{v1.3.5 => v1.5.2}/base/cni-patch.yaml | 0 .../base/controlplane-vip.yaml | 2 +- .../{v1.3.5 => v1.5.2}/base/crs.yaml | 0 .../cluster-template/kustomization.yaml | 2 +- test/e2e/data/shared/metadata.yaml | 3 + 8 files changed, 64 insertions(+), 64 deletions(-) rename test/e2e/data/infrastructure-nutanix/{v1.3.5 => v1.5.2}/base/cni-patch.yaml (100%) rename test/e2e/data/infrastructure-nutanix/{v1.3.5 => v1.5.2}/base/controlplane-vip.yaml (97%) rename test/e2e/data/infrastructure-nutanix/{v1.3.5 => v1.5.2}/base/crs.yaml (100%) rename test/e2e/data/infrastructure-nutanix/{v1.3.5 => v1.5.2}/cluster-template/kustomization.yaml (81%) diff --git a/Makefile b/Makefile index 67a598e2fc..64f1d0f0b7 100644 --- a/Makefile +++ b/Makefile @@ -256,10 +256,10 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi ##@ Templates .PHONY: cluster-e2e-templates -cluster-e2e-templates: cluster-e2e-templates-v1beta1 cluster-e2e-templates-v135 ## Generate cluster templates for all versions +cluster-e2e-templates: cluster-e2e-templates-v1beta1 cluster-e2e-templates-v152 ## Generate cluster templates for all versions -cluster-e2e-templates-v135: ## Generate cluster templates for CAPX v1.3.0 - kustomize build $(NUTANIX_E2E_TEMPLATES)/v1.3.5/cluster-template --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1.3.5/cluster-template.yaml +cluster-e2e-templates-v152: ## Generate cluster templates for CAPX v1.5.2 + kustomize build $(NUTANIX_E2E_TEMPLATES)/v1.5.2/cluster-template --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1.5.2/cluster-template.yaml cluster-e2e-templates-v1beta1: ## Generate cluster templates for v1beta1 kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template.yaml diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index 41af09cc89..35c1601444 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -26,20 +26,19 @@ import ( "path/filepath" "time" + "github.com/blang/semver/v4" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "github.com/blang/semver/v4" - "github.com/nutanix-cloud-native/cluster-api-provider-nutanix/test/e2e/log" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" yaml "sigs.k8s.io/cluster-api/cmd/clusterctl/client/yamlprocessor" - capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + capie2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-provider-nutanix/test/e2e/log" ) var kubernetesVersion = getKubernetesVersion() @@ -58,9 +57,9 @@ func getKubernetesVersion() string { return "undefined" } -var _ = Describe("[clusterctl-Upgrade] Upgrade CAPX (v1.3.5 => current) K8S "+kubernetesVersion, Label("clusterctl-upgrade"), func() { - preWaitForCluster := createPreWaitForClusterFunc(func() capi_e2e.ClusterctlUpgradeSpecInput { - return capi_e2e.ClusterctlUpgradeSpecInput{ +var _ = Describe("[clusterctl-Upgrade] Upgrade CAPX (v1.5.2 => current) K8S "+kubernetesVersion, Label("clusterctl-upgrade"), func() { + preWaitForCluster := createPreWaitForClusterFunc(func() capie2e.ClusterctlUpgradeSpecInput { + return capie2e.ClusterctlUpgradeSpecInput{ E2EConfig: e2eConfig, ClusterctlConfigPath: clusterctlConfigPath, BootstrapClusterProxy: bootstrapClusterProxy, @@ -68,8 +67,8 @@ var _ = Describe("[clusterctl-Upgrade] Upgrade CAPX (v1.3.5 => current) K8S "+ku } }) - postUpgradeFunc := createPostUpgradeFunc(func() capi_e2e.ClusterctlUpgradeSpecInput { - return capi_e2e.ClusterctlUpgradeSpecInput{ + postUpgradeFunc := createPostUpgradeFunc(func() capie2e.ClusterctlUpgradeSpecInput { + return capie2e.ClusterctlUpgradeSpecInput{ E2EConfig: e2eConfig, ClusterctlConfigPath: clusterctlConfigPath, BootstrapClusterProxy: bootstrapClusterProxy, @@ -77,26 +76,26 @@ var _ = Describe("[clusterctl-Upgrade] Upgrade CAPX (v1.3.5 => current) K8S "+ku } }) - capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput { - return capi_e2e.ClusterctlUpgradeSpecInput{ + capie2e.ClusterctlUpgradeSpec(ctx, func() capie2e.ClusterctlUpgradeSpecInput { + return capie2e.ClusterctlUpgradeSpecInput{ E2EConfig: e2eConfig, ClusterctlConfigPath: clusterctlConfigPath, BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/clusterctl-{OS}-{ARCH}", + InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.6/clusterctl-{OS}-{ARCH}", InitWithKubernetesVersion: e2eConfig.GetVariable("KUBERNETES_VERSION"), - InitWithCoreProvider: "cluster-api:v1.6.2", - InitWithBootstrapProviders: []string{"kubeadm:v1.6.2"}, - InitWithControlPlaneProviders: []string{"kubeadm:v1.6.2"}, - InitWithInfrastructureProviders: []string{"nutanix:v1.3.5"}, + InitWithCoreProvider: "cluster-api:v1.7.6", + InitWithBootstrapProviders: []string{"kubeadm:v1.7.6"}, + InitWithControlPlaneProviders: []string{"kubeadm:v1.7.6"}, + InitWithInfrastructureProviders: []string{"nutanix:v1.5.2"}, PreWaitForCluster: preWaitForCluster, PostUpgrade: postUpgradeFunc, } }) }) -func createPreWaitForClusterFunc(testInputFunc func() capi_e2e.ClusterctlUpgradeSpecInput) func(framework.ClusterProxy, string, string) { +func createPreWaitForClusterFunc(testInputFunc func() capie2e.ClusterctlUpgradeSpecInput) func(framework.ClusterProxy, string, string) { return func(managementClusterProxy framework.ClusterProxy, mgmtClusterNamespace, mgmtClusterName string) { testInput := testInputFunc() Expect(testInput.E2EConfig).NotTo(BeNil(), "Invalid argument. testInput.E2EConfig can't be nil when calling createPreWaitForClusterFunc") @@ -105,7 +104,7 @@ func createPreWaitForClusterFunc(testInputFunc func() capi_e2e.ClusterctlUpgrade By("Get latest version of CAPX provider") - latestVersionString := "v1.3.5" + latestVersionString := "v1.5.2" latestVersion, err := semver.ParseTolerant(latestVersionString) Expect(err).NotTo(HaveOccurred()) @@ -153,7 +152,7 @@ func createPreWaitForClusterFunc(testInputFunc func() capi_e2e.ClusterctlUpgrade } } -func createPostUpgradeFunc(testInputFunc func() capi_e2e.ClusterctlUpgradeSpecInput) func(framework.ClusterProxy, string, string) { +func createPostUpgradeFunc(testInputFunc func() capie2e.ClusterctlUpgradeSpecInput) func(framework.ClusterProxy, string, string) { return func(managementClusterProxy framework.ClusterProxy, clusterNamespace string, clusterName string) { testInput := testInputFunc() Expect(testInput.E2EConfig).NotTo(BeNil(), "Invalid argument. testInput.E2EConfig can't be nil when calling createPostUpgradeFunc") @@ -164,7 +163,7 @@ func createPostUpgradeFunc(testInputFunc func() capi_e2e.ClusterctlUpgradeSpecIn yamlProc := yaml.NewSimpleProcessor() - latestVersionString := "v1.3.5" + latestVersionString := "v1.5.2" latestVersion, err := semver.ParseTolerant(latestVersionString) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/config/nutanix.yaml b/test/e2e/config/nutanix.yaml index 51441edc57..c65b7e6863 100644 --- a/test/e2e/config/nutanix.yaml +++ b/test/e2e/config/nutanix.yaml @@ -13,26 +13,26 @@ images: loadBehavior: mustLoad # ## PLEASE KEEP THESE UP TO DATE WITH THE COMPONENTS # Cluster API v1beta1 Preloads - - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.6.2 + - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.7.6 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.6.2 + - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.7.6 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.6.2 + - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.7.6 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.7.3 + - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.8.3 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.7.3 + - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.8.3 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.7.3 + - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.8.3 loadBehavior: tryLoad providers: - name: cluster-api type: CoreProvider versions: - - name: v1.6.2 + - name: v1.7.6 # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/core-components.yaml" + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.6/core-components.yaml" type: "url" contract: v1beta1 files: @@ -40,9 +40,9 @@ providers: replacements: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" - - name: v1.7.3 + - name: v1.8.3 # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.3/core-components.yaml" + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.3/core-components.yaml" type: "url" contract: v1beta1 files: @@ -53,9 +53,9 @@ providers: - name: kubeadm type: BootstrapProvider versions: - - name: v1.6.2 + - name: v1.7.6 # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/bootstrap-components.yaml" + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.6/bootstrap-components.yaml" type: "url" contract: "v1beta1" files: @@ -63,9 +63,9 @@ providers: replacements: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" - - name: v1.7.3 + - name: v1.8.3 # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.3/bootstrap-components.yaml" + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.3/bootstrap-components.yaml" type: "url" contract: "v1beta1" files: @@ -76,9 +76,9 @@ providers: - name: kubeadm type: ControlPlaneProvider versions: - - name: v1.6.2 + - name: v1.7.6 # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/control-plane-components.yaml" + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.6/control-plane-components.yaml" type: "url" contract: v1beta1 files: @@ -86,9 +86,9 @@ providers: replacements: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" - - name: v1.7.3 + - name: v1.8.3 # Use manifest from source files - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.3/control-plane-components.yaml" + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.3/control-plane-components.yaml" type: "url" contract: v1beta1 files: @@ -99,13 +99,13 @@ providers: - name: nutanix type: InfrastructureProvider versions: - - name: v1.3.5 + - name: v1.5.2 type: url - value: https://github.com/nutanix-cloud-native/cluster-api-provider-nutanix/releases/download/v1.3.5/infrastructure-components.yaml + value: https://github.com/nutanix-cloud-native/cluster-api-provider-nutanix/releases/download/v1.5.2/infrastructure-components.yaml contract: v1beta1 files: - sourcePath: "../../../metadata.yaml" - - sourcePath: "../data/infrastructure-nutanix/v1.3.5/cluster-template.yaml" + - sourcePath: "../data/infrastructure-nutanix/v1.5.2/cluster-template.yaml" - sourcePath: "../data/infrastructure-nutanix/ccm-update.yaml" - name: ${LOCAL_PROVIDER_VERSION} type: kustomize @@ -141,7 +141,7 @@ variables: # allowing the same e2e config file to be re-used in different Prow jobs e.g. each one with a K8s version permutation. # The following Kubernetes versions should be the latest versions with already published kindest/node images. # This avoids building node images in the default case which improves the test duration significantly. - KUBERNETES_VERSION_MANAGEMENT: "v1.30.2" + KUBERNETES_VERSION_MANAGEMENT: "v1.31.0" IP_FAMILY: "IPv4" CLUSTERCTL_LOG_LEVEL: 10 CLUSTER_TOPOLOGY: "true" @@ -153,26 +153,24 @@ variables: NUTANIX_PORT: "9440" NUTANIX_INSECURE: true NUTANIX_ADDITIONAL_TRUST_BUNDLE: "" - KUBERNETES_VERSION: "v1.30.2" - KUBERNETES_VERSION_v1_27: "v1.27.13" - KUBERNETES_VERSION_v1_28: "v1.28.9" - KUBERNETES_VERSION_v1_29: "v1.29.6" - KUBERNETES_VERSION_v1_30: "v1.30.2" + KUBERNETES_VERSION: "v1.31.0" + KUBERNETES_VERSION_v1_28: "v1.28.13" + KUBERNETES_VERSION_v1_29: "v1.29.8" + KUBERNETES_VERSION_v1_30: "v1.30.4" NUTANIX_SSH_AUTHORIZED_KEY: "" CONTROL_PLANE_ENDPOINT_IP: "" CONTROL_PLANE_ENDPOINT_IP_V124: "" CONTROL_PLANE_MACHINE_COUNT: 1 WORKER_MACHINE_COUNT: 1 NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: "" - NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: "ubuntu-2204-kube-v1.30.2.qcow2" - NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27: "ubuntu-2204-kube-v1.27.13.qcow2" - NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28: "ubuntu-2204-kube-v1.28.9.qcow2" - NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29: "ubuntu-2204-kube-v1.29.6.qcow2" - NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_30: "ubuntu-2204-kube-v1.30.2.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: "rockylinux-9-kube-v1.31.0.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28: "rockylinux-9-kube-v1.28.13.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29: "rockylinux-9-kube-v1.29.8.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_30: "rockylinux-9-kube-v1.30.4.qcow2" CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO: "cluster-upgrade-conformance" WORKERS_MACHINE_TEMPLATE_UPGRADE_TO: "cluster-upgrade-conformance" - NUTANIX_MACHINE_TEMPLATE_IMAGE_UPGRADE_TO: "ubuntu-2204-kube-v1.30.2.qcow2" - NUTANIX_MACHINE_TEMPLATE_IMAGE_UPGRADE_FROM: "ubuntu-2204-kube-v1.29.6.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_UPGRADE_TO: "rockylinux-9-kube-v1.31.0.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_UPGRADE_FROM: "rockylinux-9-kube-v1.30.4.qcow2" NUTANIX_SUBNET_NAME: "" # NOTE: 'NUTANIX_ADDITIONAL_SUBNET_NAME' is required for multi network interface e2e tests NUTANIX_ADDITIONAL_SUBNET_NAME: "" @@ -186,8 +184,8 @@ variables: EXP_CLUSTER_RESOURCE_SET: "true" EXP_CLUSTER_TOPOLOGY: "true" DOCKER_POD_CIDRS: "192.168.0.0/16" - KUBERNETES_VERSION_UPGRADE_TO: "v1.30.2" - KUBERNETES_VERSION_UPGRADE_FROM: "v1.29.6" + KUBERNETES_VERSION_UPGRADE_TO: "v1.31.0" + KUBERNETES_VERSION_UPGRADE_FROM: "v1.30.4" ETCD_VERSION_UPGRADE_TO: "3.5.12-0" COREDNS_VERSION_UPGRADE_TO: "v1.11.1" KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml" @@ -212,8 +210,8 @@ variables: # NOTE: INIT_WITH_BINARY and INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize # the management cluster to be upgraded. # NOTE: We test the latest release with a previous contract. - INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/clusterctl-{OS}-{ARCH}" - INIT_WITH_KUBERNETES_VERSION: "v1.29.6" + INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.6/clusterctl-{OS}-{ARCH}" + INIT_WITH_KUBERNETES_VERSION: "v1.30.4" intervals: default/wait-controllers: ["3m", "10s"] diff --git a/test/e2e/data/infrastructure-nutanix/v1.3.5/base/cni-patch.yaml b/test/e2e/data/infrastructure-nutanix/v1.5.2/base/cni-patch.yaml similarity index 100% rename from test/e2e/data/infrastructure-nutanix/v1.3.5/base/cni-patch.yaml rename to test/e2e/data/infrastructure-nutanix/v1.5.2/base/cni-patch.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1.3.5/base/controlplane-vip.yaml b/test/e2e/data/infrastructure-nutanix/v1.5.2/base/controlplane-vip.yaml similarity index 97% rename from test/e2e/data/infrastructure-nutanix/v1.3.5/base/controlplane-vip.yaml rename to test/e2e/data/infrastructure-nutanix/v1.5.2/base/controlplane-vip.yaml index adc4dd7429..1243e6677c 100644 --- a/test/e2e/data/infrastructure-nutanix/v1.3.5/base/controlplane-vip.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1.5.2/base/controlplane-vip.yaml @@ -24,7 +24,7 @@ spec: spec: containers: - name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.6.4 + image: ghcr.io/kube-vip/kube-vip:v0.8.2 imagePullPolicy: IfNotPresent args: - manager diff --git a/test/e2e/data/infrastructure-nutanix/v1.3.5/base/crs.yaml b/test/e2e/data/infrastructure-nutanix/v1.5.2/base/crs.yaml similarity index 100% rename from test/e2e/data/infrastructure-nutanix/v1.3.5/base/crs.yaml rename to test/e2e/data/infrastructure-nutanix/v1.5.2/base/crs.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1.3.5/cluster-template/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1.5.2/cluster-template/kustomization.yaml similarity index 81% rename from test/e2e/data/infrastructure-nutanix/v1.3.5/cluster-template/kustomization.yaml rename to test/e2e/data/infrastructure-nutanix/v1.5.2/cluster-template/kustomization.yaml index 5e3c46e5e2..9ff2d41ac9 100644 --- a/test/e2e/data/infrastructure-nutanix/v1.3.5/cluster-template/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1.5.2/cluster-template/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- https://github.com/nutanix-cloud-native/cluster-api-provider-nutanix/releases/download/v1.3.5/cluster-template.yaml +- https://github.com/nutanix-cloud-native/cluster-api-provider-nutanix/releases/download/v1.5.2/cluster-template.yaml - ../base/crs.yaml patches: - path: ../base/cni-patch.yaml diff --git a/test/e2e/data/shared/metadata.yaml b/test/e2e/data/shared/metadata.yaml index 6c960c2b1a..8c679c4488 100644 --- a/test/e2e/data/shared/metadata.yaml +++ b/test/e2e/data/shared/metadata.yaml @@ -25,3 +25,6 @@ releaseSeries: - major: 1 minor: 7 contract: v1beta1 + - major: 1 + minor: 8 + contract: v1beta1