✨ Upgrade to Kubernetes 1.28

.envrc.sample

@@ -6,7 +6,7 @@ export HCLOUD_SSH_KEY=test
 export HCLOUD_REGION=fsn1
 export CONTROL_PLANE_MACHINE_COUNT=1
 export WORKER_MACHINE_COUNT=1
-export KUBERNETES_VERSION=v1.25.2
+export KUBERNETES_VERSION=v1.28.4
 export HCLOUD_IMAGE_NAME=1.23.4-ubuntu-20.04-containerd
 export HCLOUD_CONTROL_PLANE_MACHINE_TYPE=cpx31
 export HCLOUD_WORKER_MACHINE_TYPE=cpx31

Makefile

@@ -108,23 +108,23 @@ $(ENVSUBST): # Build envsubst from tools folder.
 SETUP_ENVTEST := $(abspath $(TOOLS_BIN_DIR)/setup-envtest)
 setup-envtest: $(SETUP_ENVTEST) ## Build a local copy of setup-envtest
 $(SETUP_ENVTEST): # Build setup-envtest from tools folder.
-	go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20230620070423-a784ee78d04b
+	go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20231206145619-1ea2be573f78
 
 CTLPTL := $(abspath $(TOOLS_BIN_DIR)/ctlptl)
 ctlptl: $(CTLPTL) ## Build a local copy of ctlptl
 $(CTLPTL):
-	go install github.com/tilt-dev/ctlptl/cmd/ctlptl@v0.8.20
+	go install github.com/tilt-dev/ctlptl/cmd/ctlptl@v0.8.25
 
 CLUSTERCTL := $(abspath $(TOOLS_BIN_DIR)/clusterctl)
 clusterctl: $(CLUSTERCTL) ## Build a local copy of clusterctl
 $(CLUSTERCTL):
-	curl -sSLf https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.3/clusterctl-$$(go env GOOS)-$$(go env GOARCH) -o $(CLUSTERCTL)
+	curl -sSLf https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/clusterctl-$$(go env GOOS)-$$(go env GOARCH) -o $(CLUSTERCTL)
 	chmod a+rx $(CLUSTERCTL)
 
 HELM := $(abspath $(TOOLS_BIN_DIR)/helm)
 helm: $(HELM) ## Build a local copy of helm
 $(HELM):
-	curl -sSL https://get.helm.sh/helm-v3.13.1-linux-amd64.tar.gz | tar xz -C $(TOOLS_BIN_DIR) --strip-components=1 linux-amd64/helm
+	curl -sSL https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz | tar xz -C $(TOOLS_BIN_DIR) --strip-components=1 linux-amd64/helm
 	chmod a+rx $(HELM)
 
 KIND := $(abspath $(TOOLS_BIN_DIR)/kind)
@@ -135,7 +135,7 @@ $(KIND):
 KUBECTL := $(abspath $(TOOLS_BIN_DIR)/kubectl)
 kubectl: $(KUBECTL) ## Build a local copy of kubectl
 $(KUBECTL):
-	curl -fsSL "https://dl.k8s.io/release/v1.27.3/bin/$$(go env GOOS)/$$(go env GOARCH)/kubectl" -o $(KUBECTL)
+	curl -fsSL "https://dl.k8s.io/release/v1.28.4/bin/$$(go env GOOS)/$$(go env GOARCH)/kubectl" -o $(KUBECTL)
 	chmod a+rx $(KUBECTL)
 
 go-binsize-treemap := $(abspath $(TOOLS_BIN_DIR)/go-binsize-treemap)
@@ -151,7 +151,7 @@ $(go-cover-treemap):
 GOTESTSUM := $(abspath $(TOOLS_BIN_DIR)/gotestsum)
 gotestsum: $(GOTESTSUM) # Build gotestsum from tools folder.
 $(GOTESTSUM):
-	go install gotest.tools/gotestsum@v1.10.0
+	go install gotest.tools/gotestsum@v1.11.0
 
 all-tools: $(GOTESTSUM) $(go-cover-treemap) $(go-binsize-treemap) $(KIND) $(KUBECTL) $(CLUSTERCTL) $(CTLPTL) $(SETUP_ENVTEST) $(ENVSUBST) $(KUSTOMIZE) $(CONTROLLER_GEN) $(HELM)
 	echo 'done'
@@ -181,18 +181,19 @@ install-essentials: ## This gets the secret and installs a CNI and the CCM. Usag
 
 wait-and-get-secret:
 	# Wait for the kubeconfig to become available.
-	${TIMEOUT} 5m bash -c "while ! $(KUBECTL) get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
+	rm -f $(WORKER_CLUSTER_KUBECONFIG)
+	${TIMEOUT} --foreground 5m bash -c "while ! $(KUBECTL) get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
 	# Get kubeconfig and store it locally.
-	$(KUBECTL) get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > $(WORKER_CLUSTER_KUBECONFIG)
-	${TIMEOUT} 15m bash -c "while ! $(KUBECTL) --kubeconfig=$(WORKER_CLUSTER_KUBECONFIG) get nodes | grep control-plane; do sleep 1; done"
+	./hack/get-kubeconfig-of-workload-cluster.sh
+	${TIMEOUT} --foreground 15m bash -c "while ! $(KUBECTL) --kubeconfig=$(WORKER_CLUSTER_KUBECONFIG) get nodes | grep control-plane; do sleep 1; done"
 
 install-cilium-in-wl-cluster: $(HELM)
 	# Deploy cilium
 	$(HELM) repo add cilium https://helm.cilium.io/
 	$(HELM) repo update cilium
 	KUBECONFIG=$(WORKER_CLUSTER_KUBECONFIG) $(HELM) upgrade --install cilium cilium/cilium --version 1.12.2 \
-  	--namespace kube-system \
-	-f templates/cilium/cilium.yaml
+  		--namespace kube-system \
+		-f templates/cilium/cilium.yaml
 
 install-ccm-in-wl-cluster:
 	$(HELM) repo add syself https://charts.syself.com
@@ -300,7 +301,7 @@ delete-workload-cluster: ## Deletes the example workload Kubernetes cluster
 	@echo 'Your workload cluster will now be deleted, this can take up to 20 minutes'
 	$(KUBECTL) patch cluster $(CLUSTER_NAME) --type=merge -p '{"spec":{"paused": false}}'
 	$(KUBECTL) delete cluster $(CLUSTER_NAME)
-	${TIMEOUT} 15m bash -c "while $(KUBECTL) get cluster | grep $(NAME); do sleep 1; done"
+	${TIMEOUT} --foreground 15m bash -c "while $(KUBECTL) get cluster | grep $(NAME); do sleep 1; done"
 	@echo 'Cluster deleted'
 
 create-mgt-cluster: $(CLUSTERCTL) $(KUBECTL) cluster ## Start a mgt-cluster with the latest version of all capi components and the infra provider.
@@ -494,7 +495,7 @@ test-e2e-feature: $(E2E_CONF_FILE) $(if $(SKIP_IMAGE_BUILD),,e2e-image) $(ARTIFA
 
 .PHONY: test-e2e-feature-packer
 test-e2e-feature-packer: $(if $(SKIP_IMAGE_BUILD),,e2e-image) $(ARTIFACTS)
-	GINKGO_FOKUS="'\[Feature Packer\]'" GINKGO_NODES=1 PACKER_IMAGE_NAME=templates/node-image/1.25.2-ubuntu-22-04-containerd ./hack/ci-e2e-capi.sh
+	GINKGO_FOKUS="'\[Feature Packer\]'" GINKGO_NODES=1 PACKER_IMAGE_NAME=templates/node-image/1.28.4-ubuntu-22-04-containerd ./hack/ci-e2e-capi.sh
 
 .PHONY: test-e2e-lifecycle
 test-e2e-lifecycle: $(E2E_CONF_FILE) $(if $(SKIP_IMAGE_BUILD),,e2e-image) $(ARTIFACTS)
@@ -506,7 +507,7 @@ test-e2e-upgrade-$(INFRA_SHORT): $(E2E_CONF_FILE) $(if $(SKIP_IMAGE_BUILD),,e2e-
 
 .PHONY: test-e2e-upgrade-kubernetes
 test-e2e-upgrade-kubernetes: $(if $(SKIP_IMAGE_BUILD),,e2e-image) $(ARTIFACTS)
-	GINKGO_FOKUS="'\[Upgrade Kubernetes\]'" GINKGO_NODES=2 PACKER_KUBERNETES_UPGRADE_FROM=templates/node-image/1.24.1-ubuntu-20-04-containerd PACKER_KUBERNETES_UPGRADE_TO=templates/node-image/1.25.2-ubuntu-22-04-containerd ./hack/ci-e2e-capi.sh
+	GINKGO_FOKUS="'\[Upgrade Kubernetes\]'" GINKGO_NODES=2 PACKER_KUBERNETES_UPGRADE_FROM=templates/node-image/1.27.8-ubuntu-22-04-containerd PACKER_KUBERNETES_UPGRADE_TO=templates/node-image/1.28.4-ubuntu-22-04-containerd ./hack/ci-e2e-capi.sh
 
 .PHONY: test-e2e-conformance
 test-e2e-conformance: $(E2E_CONF_FILE) $(if $(SKIP_IMAGE_BUILD),,e2e-image) $(ARTIFACTS)
@@ -689,7 +690,7 @@ ifeq ($(BUILD_IN_CONTAINER),true)
 else
 	go version
 	golangci-lint version
-	golangci-lint run -v --out-format=github-actions
+	golangci-lint run --out-format=github-actions
 endif
 
 .PHONY: lint-yaml

README.md

@@ -112,6 +112,7 @@ This provider's versions can install and manage the following versions of Kubern
 | Kubernetes 1.25.x | ✓ |
 | Kubernetes 1.26.x | ✓ |
 | Kubernetes 1.27.x | ✓ |
+| Kubernetes 1.28.x | ✓ |
 
 Test status:
 

config/crd/bases/infrastructure.cluster.x-k8s.io_hcloudmachines.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hcloudmachines.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

config/crd/bases/infrastructure.cluster.x-k8s.io_hcloudmachinetemplates.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hcloudmachinetemplates.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

config/crd/bases/infrastructure.cluster.x-k8s.io_hcloudremediations.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hcloudremediations.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

config/crd/bases/infrastructure.cluster.x-k8s.io_hcloudremediationtemplates.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hcloudremediationtemplates.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

config/crd/bases/infrastructure.cluster.x-k8s.io_hetznerbaremetalhosts.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hetznerbaremetalhosts.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

config/crd/bases/infrastructure.cluster.x-k8s.io_hetznerbaremetalmachines.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hetznerbaremetalmachines.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

config/crd/bases/infrastructure.cluster.x-k8s.io_hetznerbaremetalmachinetemplates.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hetznerbaremetalmachinetemplates.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

config/crd/bases/infrastructure.cluster.x-k8s.io_hetznerbaremetalremediations.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hetznerbaremetalremediations.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

config/crd/bases/infrastructure.cluster.x-k8s.io_hetznerbaremetalremediationtemplates.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hetznerbaremetalremediationtemplates.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

config/crd/bases/infrastructure.cluster.x-k8s.io_hetznerclusters.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hetznerclusters.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

config/crd/bases/infrastructure.cluster.x-k8s.io_hetznerclustertemplates.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: hetznerclustertemplates.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io

controllers/hcloudmachine_controller.go

@@ -180,7 +180,7 @@ func (r *HCloudMachineReconciler) reconcileDelete(ctx context.Context, machineSc
 	// Delete servers.
 	result, err := server.NewService(machineScope).Delete(ctx)
 	if err != nil {
-		return result, fmt.Errorf("failed to delete servers for HCloudMachine %s/%s: %w", hcloudMachine.Namespace, hcloudMachine.Name, err)
+		return reconcile.Result{}, fmt.Errorf("failed to delete servers for HCloudMachine %s/%s: %w", hcloudMachine.Namespace, hcloudMachine.Name, err)
 	}
 	emptyResult := reconcile.Result{}
 	if result != emptyResult {
@@ -206,7 +206,7 @@ func (r *HCloudMachineReconciler) reconcileNormal(ctx context.Context, machineSc
 	// reconcile server
 	result, err := server.NewService(machineScope).Reconcile(ctx)
 	if err != nil {
-		return result, fmt.Errorf("failed to reconcile server for HCloudMachine %s/%s: %w",
+		return reconcile.Result{}, fmt.Errorf("failed to reconcile server for HCloudMachine %s/%s: %w",
 			hcloudMachine.Namespace, hcloudMachine.Name, err)
 	}
 

controllers/hcloudremediation_controller.go

@@ -189,7 +189,7 @@ func (r *HCloudRemediationReconciler) reconcileNormal(ctx context.Context, remed
 	// reconcile hcloud remediation
 	result, err := hcloudremediation.NewService(remediationScope).Reconcile(ctx)
 	if err != nil {
-		return result, fmt.Errorf("failed to reconcile server for HCloudRemediation %s/%s: %w",
+		return reconcile.Result{}, fmt.Errorf("failed to reconcile server for HCloudRemediation %s/%s: %w",
 			hcloudRemediation.Namespace, hcloudRemediation.Name, err)
 	}
 

controllers/hetznerbaremetalhost_controller.go

@@ -71,9 +71,9 @@ func (r *HetznerBareMetalHostReconciler) Reconcile(ctx context.Context, req ctrl
 	err := r.Get(ctx, req.NamespacedName, bmHost)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
-			return ctrl.Result{}, nil
+			return reconcile.Result{}, nil
 		}
-		return ctrl.Result{}, err
+		return reconcile.Result{}, err
 	}
 
 	log = log.WithValues("HetznerBareMetalHost", klog.KObj(bmHost))
@@ -84,17 +84,20 @@ func (r *HetznerBareMetalHostReconciler) Reconcile(ctx context.Context, req ctrl
 			infrav1.BareMetalHostFinalizer)
 		err := r.Update(ctx, bmHost)
 		if err != nil {
-			return ctrl.Result{}, fmt.Errorf("failed to add finalizer: %w", err)
+			return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err)
 		}
 		return ctrl.Result{Requeue: true}, nil
 	}
 
 	// Certain cases need to be handled here and not later in the host state machine.
 	// If res != nil, then we should return, otherwise not.
 	res, err = r.reconcileSelectedStates(ctx, bmHost)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
 	emptyResult := reconcile.Result{}
-	if res != emptyResult || err != nil {
-		return res, err
+	if res != emptyResult {
+		return res, nil
 	}
 
 	hetznerCluster := &infrav1.HetznerCluster{}
@@ -104,7 +107,7 @@ func (r *HetznerBareMetalHostReconciler) Reconcile(ctx context.Context, req ctrl
 		Name:      bmHost.Spec.Status.HetznerClusterRef,
 	}
 	if err := r.Client.Get(ctx, hetznerClusterName, hetznerCluster); err != nil {
-		return ctrl.Result{}, errors.New("HetznerCluster not found")
+		return reconcile.Result{}, errors.New("HetznerCluster not found")
 	}
 
 	log = log.WithValues("HetznerCluster", klog.KObj(hetznerCluster))
@@ -119,8 +122,11 @@ func (r *HetznerBareMetalHostReconciler) Reconcile(ctx context.Context, req ctrl
 
 	// Get secrets. Return when result != nil.
 	osSSHSecret, rescueSSHSecret, res, err := r.getSecrets(ctx, *secretManager, bmHost, hetznerCluster)
-	if err != nil || res != emptyResult {
-		return res, err
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+	if res != emptyResult {
+		return res, nil
 	}
 	// Create the scope.
 	hostScope, err := scope.NewBareMetalHostScope(scope.BareMetalHostScopeParams{
@@ -173,7 +179,7 @@ func (r *HetznerBareMetalHostReconciler) reconcileSelectedStates(ctx context.Con
 		if needsUpdate {
 			err := r.Update(ctx, bmHost)
 			if err != nil {
-				return res, fmt.Errorf("failed to add finalizer: %w", err)
+				return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err)
 			}
 		}
 
@@ -187,7 +193,7 @@ func (r *HetznerBareMetalHostReconciler) reconcileSelectedStates(ctx context.Con
 
 		bmHost.Finalizers = utils.FilterStringFromList(bmHost.Finalizers, infrav1.BareMetalHostFinalizer)
 		if err := r.Update(context.Background(), bmHost); err != nil {
-			return res, fmt.Errorf("failed to remove finalizer: %w", err)
+			return reconcile.Result{}, fmt.Errorf("failed to remove finalizer: %w", err)
 		}
 		return res, nil
 	}
@@ -325,9 +331,12 @@ func hetznerSecretErrorResult(
 		record.Warnf(bmHost, infrav1.HetznerSecretUnreachableReason, fmt.Sprintf("%s: %s", infrav1.ErrorMessageMissingHetznerSecret, err.Error()))
 		conditions.SetSummary(bmHost)
 		result, err := host.SaveHostAndReturn(ctx, client, bmHost)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
 		emptyResult := reconcile.Result{}
-		if result != emptyResult || err != nil {
-			return result, err
+		if result != emptyResult {
+			return result, nil
 		}
 
 		// No need to reconcile again, as it will be triggered as soon as the secret is updated.
@@ -347,7 +356,7 @@ func hetznerSecretErrorResult(
 		conditions.SetSummary(bmHost)
 		return host.SaveHostAndReturn(ctx, client, bmHost)
 	}
-	return ctrl.Result{}, fmt.Errorf("hetznerSecretErrorResult: an unhandled failure occurred: %T %w", err, err)
+	return reconcile.Result{}, fmt.Errorf("hetznerSecretErrorResult: an unhandled failure occurred: %T %w", err, err)
 }
 
 func hostHasFinalizer(host *infrav1.HetznerBareMetalHost) bool {

controllers/hetznerbaremetalhost_controller_test.go

@@ -243,7 +243,7 @@ var _ = Describe("HetznerBareMetalHostReconciler", func() {
 			By("making sure the it has been deleted")
 			Eventually(func() bool {
 				return apierrors.IsNotFound(testEnv.Get(ctx, key, host))
-			}, timeout, time.Second).Should(BeTrue())
+			}, 10*time.Second, time.Second).Should(BeTrue())
 		})
 	})
 
@@ -336,13 +336,16 @@ var _ = Describe("HetznerBareMetalHostReconciler", func() {
 				}, timeout, time.Second).Should(BeNil())
 
 				Eventually(func() bool {
-					// TODO: Add logging to trace flaky unit-test.
 					if err := testEnv.Get(ctx, key, host); err != nil {
+						testEnv.GetLogger().Info("......... reaches the state image installing. Get failed", "err", err)
 						return false
 					}
 					if host.Spec.Status.ProvisioningState == infrav1.StateImageInstalling {
 						return true
 					}
+					testEnv.GetLogger().Info("......... reaches the state image installing. State",
+						"is-state", host.Spec.Status.ProvisioningState,
+						"should-state", infrav1.StateImageInstalling)
 					return false
 				}, 10*time.Second).Should(BeTrue())
 			})