From db4f2ed35deb78138d33a7b6a63bce162aae9fc2 Mon Sep 17 00:00:00 2001 From: Mitali Paygude Date: Fri, 24 May 2024 16:50:19 -0700 Subject: [PATCH] Kubelet Configuration for cp and wn Tinkerbell --- pkg/clusterapi/workers.go | 4 + pkg/clusterapi/workers_test.go | 43 +++++++++ .../tinkerbell/config/template-cp.yaml | 20 ++++ .../tinkerbell/config/template-md.yaml | 16 +++- pkg/providers/tinkerbell/template.go | 24 ++++- pkg/providers/tinkerbell/template_test.go | 94 ++++++++++++++++++- 6 files changed, 196 insertions(+), 5 deletions(-) diff --git a/pkg/clusterapi/workers.go b/pkg/clusterapi/workers.go index 4155010958384..188eb264efc8a 100644 --- a/pkg/clusterapi/workers.go +++ b/pkg/clusterapi/workers.go @@ -2,6 +2,7 @@ package clusterapi import ( "context" + "reflect" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/api/equality" @@ -124,7 +125,10 @@ func GetKubeadmConfigTemplate(ctx context.Context, client kubernetes.Client, nam func KubeadmConfigTemplateEqual(new, old *kubeadmv1.KubeadmConfigTemplate) bool { // DeepDerivative treats empty map (length == 0) as unset field. We need to manually compare certain fields // such as taints, so that setting it to empty will trigger machine recreate + // The file check with deep equal has been added since the introduction of kubelet configuration in case users + // want to get rid of the files with that context. return kubeadmConfigTemplateTaintsEqual(new, old) && kubeadmConfigTemplateExtraArgsEqual(new, old) && + reflect.DeepEqual(new.Spec.Template.Spec.Files, old.Spec.Template.Spec.Files) && equality.Semantic.DeepDerivative(new.Spec, old.Spec) } diff --git a/pkg/clusterapi/workers_test.go b/pkg/clusterapi/workers_test.go index 746e90626b431..e1771472adb24 100644 --- a/pkg/clusterapi/workers_test.go +++ b/pkg/clusterapi/workers_test.go @@ -463,6 +463,49 @@ func TestKubeadmConfigTemplateEqual(t *testing.T) { }, want: false, }, + { + name: "diff spec files", + new: &kubeadmv1.KubeadmConfigTemplate{ + Spec: kubeadmv1.KubeadmConfigTemplateSpec{ + Template: kubeadmv1.KubeadmConfigTemplateResource{ + Spec: kubeadmv1.KubeadmConfigSpec{ + JoinConfiguration: &kubeadmv1.JoinConfiguration{ + NodeRegistration: kubeadmv1.NodeRegistrationOptions{ + Taints: []corev1.Taint{ + { + Key: "key", + }, + }, + }, + }, + Files: []kubeadmv1.File{ + { + Owner: "me", + }, + }, + }, + }, + }, + }, + old: &kubeadmv1.KubeadmConfigTemplate{ + Spec: kubeadmv1.KubeadmConfigTemplateSpec{ + Template: kubeadmv1.KubeadmConfigTemplateResource{ + Spec: kubeadmv1.KubeadmConfigSpec{ + JoinConfiguration: &kubeadmv1.JoinConfiguration{ + NodeRegistration: kubeadmv1.NodeRegistrationOptions{ + Taints: []corev1.Taint{ + { + Key: "key", + }, + }, + }, + }, + }, + }, + }, + }, + want: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/providers/tinkerbell/config/template-cp.yaml b/pkg/providers/tinkerbell/config/template-cp.yaml index 838deae8b8631..245267297951a 100644 --- a/pkg/providers/tinkerbell/config/template-cp.yaml +++ b/pkg/providers/tinkerbell/config/template-cp.yaml @@ -161,7 +161,12 @@ spec: certificatesDir: /var/lib/kubeadm/pki {{- end }} initConfiguration: +{{- if .kubeletConfiguration }} + patches: + directory: /etc/kubernetes/patches +{{- end }} nodeRegistration: +{{- if not .kubeletConfiguration }} kubeletExtraArgs: provider-id: PROVIDER_ID read-only-port: "0" @@ -169,6 +174,7 @@ spec: {{- if .kubeletExtraArgs }} {{ .kubeletExtraArgs.ToYaml | indent 10 }} {{- end }} +{{- end }} {{- if not .workerNodeGroupConfigurations }} taints: [] {{- end }} @@ -184,6 +190,10 @@ spec: {{- end }} {{- end }} joinConfiguration: +{{- if .kubeletConfiguration }} + patches: + directory: /etc/kubernetes/patches +{{- end }} {{- if (eq .format "bottlerocket") }} pause: imageRepository: {{.pauseRepository}} @@ -221,6 +231,7 @@ spec: nodeRegistration: ignorePreflightErrors: - DirAvailable--etc-kubernetes-manifests +{{- if not .kubeletConfiguration }} kubeletExtraArgs: provider-id: PROVIDER_ID read-only-port: "0" @@ -228,6 +239,7 @@ spec: {{- if .kubeletExtraArgs }} {{ .kubeletExtraArgs.ToYaml | indent 10 }} {{- end }} +{{- end }} {{- if not .workerNodeGroupConfigurations }} taints: [] {{- end }} @@ -243,6 +255,14 @@ spec: {{- end }} {{- end }} files: +{{- if .kubeletConfiguration }} + - content: | +{{ .kubeletConfiguration | indent 10 -}} + providerID: PROVIDER_ID + owner: root:root + permissions: "0644" + path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.yaml +{{- end }} {{- if not .cpSkipLoadBalancerDeployment }} - content: | apiVersion: v1 diff --git a/pkg/providers/tinkerbell/config/template-md.yaml b/pkg/providers/tinkerbell/config/template-md.yaml index 83e39ed128df3..b848f6a7dbed4 100644 --- a/pkg/providers/tinkerbell/config/template-md.yaml +++ b/pkg/providers/tinkerbell/config/template-md.yaml @@ -75,6 +75,10 @@ spec: template: spec: joinConfiguration: +{{- if .kubeletConfiguration }} + patches: + directory: /etc/kubernetes/patches +{{- end }} {{- if (eq .format "bottlerocket") }} pause: imageRepository: {{.pauseRepository}} @@ -121,6 +125,7 @@ spec: {{- end }} {{- end }} {{- end }} +{{- if not .kubeletConfiguration }} kubeletExtraArgs: provider-id: PROVIDER_ID read-only-port: "0" @@ -128,9 +133,18 @@ spec: {{- if .kubeletExtraArgs }} {{ .kubeletExtraArgs.ToYaml | indent 12 }} {{- end }} -{{- if and (ne .format "bottlerocket") (or .proxyConfig .registryMirrorMap) }} +{{- end }} +{{- if or (and (ne .format "bottlerocket") (or .proxyConfig .registryMirrorMap)) .kubeletConfiguration }} files: {{- end }} +{{- if .kubeletConfiguration }} + - content: | +{{ .kubeletConfiguration | indent 12 -}} + providerID: PROVIDER_ID + owner: root:root + permissions: "0644" + path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.yaml +{{- end }} {{- if and .proxyConfig (ne .format "bottlerocket") }} - content: | [Service] diff --git a/pkg/providers/tinkerbell/template.go b/pkg/providers/tinkerbell/template.go index 1bae93348efe3..a2245c99ae205 100644 --- a/pkg/providers/tinkerbell/template.go +++ b/pkg/providers/tinkerbell/template.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" yamlutil "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/yaml" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/cluster" @@ -66,9 +67,6 @@ func NewTemplateBuilder(datacenterSpec *v1alpha1.TinkerbellDatacenterConfigSpec, func (tb *TemplateBuilder) GenerateCAPISpecControlPlane(clusterSpec *cluster.Spec, buildOptions ...providers.BuildMapOption) (content []byte, err error) { cpTemplateConfig := clusterSpec.TinkerbellTemplateConfigs[tb.controlPlaneMachineSpec.TemplateRef.Name] bundle := clusterSpec.RootVersionsBundle() - if err != nil { - return nil, err - } var OSImageURL string if tinkerbellIP := clusterSpec.Cluster.HasTinkerbellIPAnnotation(); tinkerbellIP != "" { @@ -510,6 +508,16 @@ func buildTemplateMapCP( values["bottlerocketSettings"] = brSettings } + if clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration != nil { + cpKubeletConfig := clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration.Object + kcString, err := yaml.Marshal(cpKubeletConfig) + if err != nil { + return nil, fmt.Errorf("marshaling control plane node Kubelet Configuration while building CAPI template %v", err) + } + + values["kubeletConfiguration"] = string(kcString) + } + return values, nil } @@ -586,6 +594,16 @@ func buildTemplateMapMD( values["bottlerocketSettings"] = brSettings } + if workerNodeGroupConfiguration.KubeletConfiguration != nil { + wnKubeletConfig := workerNodeGroupConfiguration.KubeletConfiguration.Object + kcString, err := yaml.Marshal(wnKubeletConfig) + if err != nil { + return nil, fmt.Errorf("marshaling Kubelet Configuration for worker node %s: %v", workerNodeGroupConfiguration.Name, err) + } + + values["kubeletConfiguration"] = string(kcString) + } + return values, nil } diff --git a/pkg/providers/tinkerbell/template_test.go b/pkg/providers/tinkerbell/template_test.go index 58dc084b40fac..7ca8737a39dd6 100644 --- a/pkg/providers/tinkerbell/template_test.go +++ b/pkg/providers/tinkerbell/template_test.go @@ -5,9 +5,12 @@ import ( "time" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/aws/eks-anywhere/internal/test" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + "github.com/aws/eks-anywhere/pkg/clusterapi" + "github.com/aws/eks-anywhere/pkg/utils/ptr" ) func TestGenerateTemplateBuilder(t *testing.T) { @@ -159,7 +162,6 @@ func TestTemplateBuilder_CertSANs(t *testing.T) { data, err := bldr.GenerateCAPISpecControlPlane(clusterSpec) g.Expect(err).ToNot(HaveOccurred()) - test.AssertContentToFile(t, string(data), tc.Output) } @@ -195,3 +197,93 @@ func TestTemplateBuilder(t *testing.T) { } } + +func TestTemplateBuilderCPKubeletConfig(t *testing.T) { + for _, tc := range []struct { + Input string + Output string + }{ + { + Input: "testdata/cluster_tinkerbell_api_server_cert_san_ip.yaml", + Output: "testdata/expected_cluster_tinkerbell_api_server_cert_san_ip.yaml", + }, + } { + g := NewWithT(t) + clusterSpec := test.NewFullClusterSpec(t, tc.Input) + + cpMachineCfg, _ := getControlPlaneMachineSpec(clusterSpec) + wngMachineCfgs, _ := getWorkerNodeGroupMachineSpec(clusterSpec) + + tinkIPBefore := "0.0.0.0" + bldr := NewTemplateBuilder(&clusterSpec.TinkerbellDatacenter.Spec, cpMachineCfg, nil, wngMachineCfgs, tinkIPBefore, time.Now) + + clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration = &unstructured.Unstructured{ + Object: map[string]interface{}{ + "maxPods": 20, + "apiVersion": "kubelet.config.k8s.io/v1beta1", + "kind": "KubeletConfiguration", + }, + } + + data, err := bldr.GenerateCAPISpecControlPlane(clusterSpec) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(data).To(ContainSubstring("maxPods")) + g.Expect(data).To(ContainSubstring("providerID")) + } +} + +func TestTemplateBuilderWNKubeletConfig(t *testing.T) { + for _, tc := range []struct { + Input string + Output string + }{ + { + Input: "testdata/cluster_tinkerbell_api_server_cert_san_ip.yaml", + Output: "testdata/expected_cluster_tinkerbell_api_server_cert_san_ip.yaml", + }, + } { + g := NewWithT(t) + clusterSpec := test.NewFullClusterSpec(t, tc.Input) + clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations = []v1alpha1.WorkerNodeGroupConfiguration{ + { + Name: "test", + Count: ptr.Int(1), + KubeletConfiguration: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "maxPods": 20, + "apiVersion": "kubelet.config.k8s.io/v1beta1", + "kind": "KubeletConfiguration", + }, + }, + MachineGroupRef: &v1alpha1.Ref{ + Name: "wn-ref", + Kind: v1alpha1.TinkerbellMachineConfigKind, + }, + }, + } + clusterSpec.TinkerbellMachineConfigs = map[string]*v1alpha1.TinkerbellMachineConfig{ + "wn-ref": { + Spec: v1alpha1.TinkerbellMachineConfigSpec{ + Users: []v1alpha1.UserConfiguration{ + { + SshAuthorizedKeys: []string{"ssh abcdef..."}, + Name: "user", + }, + }, + }, + }, + } + + cpMachineCfg, _ := getControlPlaneMachineSpec(clusterSpec) + wngMachineCfgs, _ := getWorkerNodeGroupMachineSpec(clusterSpec) + + tinkIPBefore := "0.0.0.0" + bldr := NewTemplateBuilder(&clusterSpec.TinkerbellDatacenter.Spec, cpMachineCfg, nil, wngMachineCfgs, tinkIPBefore, time.Now) + workerTemplateNames, kubeadmTemplateNames := clusterapi.InitialTemplateNamesForWorkers(clusterSpec) + data, err := bldr.GenerateCAPISpecWorkers(clusterSpec, workerTemplateNames, kubeadmTemplateNames) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(data).To(ContainSubstring("maxPods")) + g.Expect(data).To(ContainSubstring("providerID")) + } +}