From ccc723e71f0044bd81abbc235adca4a7a9954066 Mon Sep 17 00:00:00 2001 From: acejilam Date: Tue, 13 Aug 2024 10:16:58 +0800 Subject: [PATCH] feature: support kubeadm.k8s.io/v1beta4 Signed-off-by: acejilam --- .../create/actions/config/config_test.go | 89 ++++++++++ pkg/cluster/internal/kubeadm/config.go | 160 +++++++++++++++++- 2 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 pkg/cluster/internal/create/actions/config/config_test.go diff --git a/pkg/cluster/internal/create/actions/config/config_test.go b/pkg/cluster/internal/create/actions/config/config_test.go new file mode 100644 index 0000000000..01305bb2c4 --- /dev/null +++ b/pkg/cluster/internal/create/actions/config/config_test.go @@ -0,0 +1,89 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package config implements the kubeadm config action +package config + +import ( + "log" + "sigs.k8s.io/kind/pkg/cluster/internal/kubeadm" + internalencoding "sigs.k8s.io/kind/pkg/internal/apis/config/encoding" + "sigs.k8s.io/kind/pkg/internal/patch" + "strings" + "testing" +) + +func TestConfig(t *testing.T) { + cases := []struct { + Name string + Input string + SubString string + }{ + { + Name: "Check if extraEnvs takes effect", + Input: `kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 6443 + hostPort: 6443 + protocol: TCP + - role: worker +kubeadmConfigPatches: + - | + kind: ClusterConfiguration + apiServer: + certSANs: + - localhost + extraEnvs: + - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + value: http://10.230.205.190:5080/api/default/traces +`, SubString: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + }, + } + + for _, tc := range cases { + tc := tc // capture tc + t.Run(tc.Name, func(t *testing.T) { + t.Parallel() + + cfg, err := internalencoding.Parse([]byte(tc.Input)) + data := kubeadm.ConfigData{ + ClusterName: "test", + KubernetesVersion: "v1.31.0", + } + cf, err := kubeadm.Config(data) + if err != nil { + log.Fatalln(err) + } + clusterPatches, clusterJSONPatches := allPatchesFromConfig(cfg) + // apply cluster-level patches first + patchedConfig, err := patch.KubeYAML(cf, clusterPatches, clusterJSONPatches) + if err != nil { + log.Fatalln(err) + } + // if needed, apply current node's patches + if len(cfg.KubeadmConfigPatches) > 0 || len(cfg.KubeadmConfigPatchesJSON6902) > 0 { + patchedConfig, _ = patch.KubeYAML(patchedConfig, cfg.KubeadmConfigPatches, cfg.KubeadmConfigPatchesJSON6902) + } + if !strings.Contains(patchedConfig, tc.SubString) { + log.Fatalln(tc.Name, "invalid") + } + }) + } + +}g diff --git a/pkg/cluster/internal/kubeadm/config.go b/pkg/cluster/internal/kubeadm/config.go index 5cf3b23cfc..e44d19a0c4 100644 --- a/pkg/cluster/internal/kubeadm/config.go +++ b/pkg/cluster/internal/kubeadm/config.go @@ -480,6 +480,161 @@ conntrack: {{end}}{{end}} ` +// ConfigTemplateBetaV4 is the kubeadm config template for API version v1beta4 +const ConfigTemplateBetaV4 = `# config generated by kind +apiVersion: kubeadm.k8s.io/v1beta4 +kind: ClusterConfiguration +metadata: + name: config +kubernetesVersion: {{.KubernetesVersion}} +clusterName: "{{.ClusterName}}" +{{ if .KubeadmFeatureGates}}featureGates: +{{ range $key, $value := .KubeadmFeatureGates }} + "{{ (StructuralData $key) }}": {{ $value }} +{{end}}{{end}} +controlPlaneEndpoint: "{{ .ControlPlaneEndpoint }}" +# on docker for mac we have to expose the api server via port forward, +# so we need to ensure the cert is valid for localhost so we can talk +# to the cluster after rewriting the kubeconfig to point to localhost +apiServer: + certSANs: [localhost, "{{.APIServerAddress}}"] + extraArgs: + "runtime-config": "{{ .RuntimeConfigString }}" +{{ if .FeatureGates }} + "feature-gates": "{{ .FeatureGatesString }}" +{{ end}} +controllerManager: + extraArgs: +{{ if .FeatureGates }} + "feature-gates": "{{ .FeatureGatesString }}" +{{ end }} + enable-hostpath-provisioner: "true" + # configure ipv6 default addresses for IPv6 clusters + {{ if .IPv6 -}} + bind-address: "::" + {{- end }} +scheduler: + extraArgs: +{{ if .FeatureGates }} + "feature-gates": "{{ .FeatureGatesString }}" +{{ end }} + # configure ipv6 default addresses for IPv6 clusters + {{ if .IPv6 -}} + bind-address: "::1" + {{- end }} +networking: + podSubnet: "{{ .PodSubnet }}" + serviceSubnet: "{{ .ServiceSubnet }}" +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: InitConfiguration +metadata: + name: config +# we use a well know token for TLS bootstrap +bootstrapTokens: +- token: "{{ .Token }}" +# we use a well know port for making the API server discoverable inside docker network. +# from the host machine such port will be accessible via a random local port instead. +localAPIEndpoint: + advertiseAddress: "{{ .AdvertiseAddress }}" + bindPort: {{.APIBindPort}} +nodeRegistration: + criSocket: "unix:///run/containerd/containerd.sock" + kubeletExtraArgs: + node-ip: "{{ .NodeAddress }}" + provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}" + node-labels: "{{ .NodeLabels }}" +{{ if .InitSkipPhases -}} +skipPhases: + {{ range $phase := .InitSkipPhases -}} + - "{{ $phase }}" + {{- end }} +{{- end }} +--- +# no-op entry that exists solely so it can be patched +apiVersion: kubeadm.k8s.io/v1beta4 +kind: JoinConfiguration +metadata: + name: config +{{ if .ControlPlane -}} +controlPlane: + localAPIEndpoint: + advertiseAddress: "{{ .AdvertiseAddress }}" + bindPort: {{.APIBindPort}} +{{- end }} +nodeRegistration: + criSocket: "unix:///run/containerd/containerd.sock" + kubeletExtraArgs: + node-ip: "{{ .NodeAddress }}" + provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}" + node-labels: "{{ .NodeLabels }}" +discovery: + bootstrapToken: + apiServerEndpoint: "{{ .ControlPlaneEndpoint }}" + token: "{{ .Token }}" + unsafeSkipCAVerification: true +{{ if .JoinSkipPhases -}} +skipPhases: + {{ range $phase := .JoinSkipPhases -}} + - "{{ $phase }}" + {{- end }} +{{- end }} +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +metadata: + name: config +cgroupDriver: {{ .CgroupDriver }} +cgroupRoot: /kubelet +failSwapOn: false +# configure ipv6 addresses in IPv6 mode +{{ if .IPv6 -}} +address: "::" +healthzBindAddress: "::" +{{- end }} +# disable disk resource management by default +# kubelet will see the host disk that the inner container runtime +# is ultimately backed by and attempt to recover disk space. we don't want that. +imageGCHighThresholdPercent: 100 +evictionHard: + nodefs.available: "0%" + nodefs.inodesFree: "0%" + imagefs.available: "0%" +{{if .FeatureGates}}featureGates: +{{ range $index, $gate := .SortedFeatureGates }} + "{{ (StructuralData $gate.Name) }}": {{ $gate.Value }} +{{end}}{{end}} +{{if ne .KubeProxyMode "none"}} +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +metadata: + name: config +mode: "{{ .KubeProxyMode }}" +{{if .FeatureGates}}featureGates: +{{ range $index, $gate := .SortedFeatureGates }} + "{{ (StructuralData $gate.Name) }}": {{ $gate.Value }} +{{end}}{{end}} +iptables: + minSyncPeriod: 1s +conntrack: +# Skip setting sysctl value "net.netfilter.nf_conntrack_max" +# It is a global variable that affects other namespaces + maxPerCore: 0 +# Set sysctl value "net.netfilter.nf_conntrack_tcp_be_liberal" +# for nftables proxy (theoretically for kernels older than 6.1) +# xref: https://github.com/kubernetes/kubernetes/issues/117924 +{{if and (eq .KubeProxyMode "nftables") (not .RootlessProvider)}} + tcpBeLiberal: true +{{end}} +{{if .RootlessProvider}} +# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established" + tcpEstablishedTimeout: 0s +# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close" + tcpCloseWaitTimeout: 0s +{{end}}{{end}} +` + // Config returns a kubeadm config generated from config data, in particular // the kubernetes version func Config(data ConfigData) (config string, err error) { @@ -503,7 +658,10 @@ func Config(data ConfigData) (config string, err error) { } // assume the latest API version, then fallback if the k8s version is too low - templateSource := ConfigTemplateBetaV3 + templateSource := ConfigTemplateBetaV4 + if ver.LessThan(version.MustParseSemantic("v1.31.0")) { + templateSource = ConfigTemplateBetaV3 + } if ver.LessThan(version.MustParseSemantic("v1.23.0")) { templateSource = ConfigTemplateBetaV2 }