Kubelet Configuration for cp and wn Tinkerbell

pkg/clusterapi/workers.go

@@ -2,6 +2,7 @@ package clusterapi
 
 import (
 	"context"
+	"reflect"
 
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -124,7 +125,10 @@ func GetKubeadmConfigTemplate(ctx context.Context, client kubernetes.Client, nam
 func KubeadmConfigTemplateEqual(new, old *kubeadmv1.KubeadmConfigTemplate) bool {
 	// DeepDerivative treats empty map (length == 0) as unset field. We need to manually compare certain fields
 	// such as taints, so that setting it to empty will trigger machine recreate
+	// The file check with deep equal has been added since the introduction of kubelet configuration in case users
+	// want to get rid of the files with that context.
 	return kubeadmConfigTemplateTaintsEqual(new, old) && kubeadmConfigTemplateExtraArgsEqual(new, old) &&
+		reflect.DeepEqual(new.Spec.Template.Spec.Files, old.Spec.Template.Spec.Files) &&
 		equality.Semantic.DeepDerivative(new.Spec, old.Spec)
 }
 

pkg/clusterapi/workers_test.go

@@ -463,6 +463,49 @@ func TestKubeadmConfigTemplateEqual(t *testing.T) {
 			},
 			want: false,
 		},
+		{
+			name: "diff spec files",
+			new: &kubeadmv1.KubeadmConfigTemplate{
+				Spec: kubeadmv1.KubeadmConfigTemplateSpec{
+					Template: kubeadmv1.KubeadmConfigTemplateResource{
+						Spec: kubeadmv1.KubeadmConfigSpec{
+							JoinConfiguration: &kubeadmv1.JoinConfiguration{
+								NodeRegistration: kubeadmv1.NodeRegistrationOptions{
+									Taints: []corev1.Taint{
+										{
+											Key: "key",
+										},
+									},
+								},
+							},
+							Files: []kubeadmv1.File{
+								{
+									Owner: "me",
+								},
+							},
+						},
+					},
+				},
+			},
+			old: &kubeadmv1.KubeadmConfigTemplate{
+				Spec: kubeadmv1.KubeadmConfigTemplateSpec{
+					Template: kubeadmv1.KubeadmConfigTemplateResource{
+						Spec: kubeadmv1.KubeadmConfigSpec{
+							JoinConfiguration: &kubeadmv1.JoinConfiguration{
+								NodeRegistration: kubeadmv1.NodeRegistrationOptions{
+									Taints: []corev1.Taint{
+										{
+											Key: "key",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			want: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/providers/tinkerbell/config/template-cp.yaml

@@ -161,14 +161,20 @@ spec:
       certificatesDir: /var/lib/kubeadm/pki
 {{- end }}
     initConfiguration:
+{{- if .kubeletConfiguration }}
+      patches: 
+        directory: /etc/kubernetes/patches
+{{- end }}
       nodeRegistration:
+{{- if not .kubeletConfiguration }}
         kubeletExtraArgs:
           provider-id: PROVIDER_ID
           read-only-port: "0"
           anonymous-auth: "false"
 {{- if .kubeletExtraArgs }}
 {{ .kubeletExtraArgs.ToYaml | indent 10 }}
 {{- end }}
+{{- end }}
 {{- if not .workerNodeGroupConfigurations }}
         taints: []
 {{- end }}
@@ -184,6 +190,10 @@ spec:
 {{- end }}
 {{- end }}
     joinConfiguration:
+{{- if .kubeletConfiguration }}
+      patches: 
+        directory: /etc/kubernetes/patches
+{{- end }}
 {{- if (eq .format "bottlerocket") }}
       pause:
         imageRepository: {{.pauseRepository}}
@@ -221,13 +231,15 @@ spec:
       nodeRegistration:
         ignorePreflightErrors:
         - DirAvailable--etc-kubernetes-manifests
+{{- if not .kubeletConfiguration }}
         kubeletExtraArgs:
           provider-id: PROVIDER_ID
           read-only-port: "0"
           anonymous-auth: "false"
 {{- if .kubeletExtraArgs }}
 {{ .kubeletExtraArgs.ToYaml | indent 10 }}
 {{- end }}
+{{- end }}
 {{- if not .workerNodeGroupConfigurations }}
         taints: []
 {{- end }}
@@ -243,6 +255,14 @@ spec:
 {{- end }}
 {{- end }}
     files:
+{{- if .kubeletConfiguration }}
+      - content: |
+{{ .kubeletConfiguration | indent 10 -}}
+          providerID: PROVIDER_ID
+        owner: root:root
+        permissions: "0644"
+        path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.yaml
+{{- end }}
 {{- if not .cpSkipLoadBalancerDeployment }}
       - content: |
           apiVersion: v1

pkg/providers/tinkerbell/config/template-md.yaml

@@ -75,6 +75,10 @@ spec:
   template:
     spec:
       joinConfiguration:
+{{- if .kubeletConfiguration }}
+        patches: 
+          directory: /etc/kubernetes/patches
+{{- end }}
 {{- if (eq .format "bottlerocket") }}
         pause:
           imageRepository: {{.pauseRepository}}
@@ -121,16 +125,26 @@ spec:
 {{- end }}
 {{- end }}
 {{- end }}
+{{- if not .kubeletConfiguration }}
           kubeletExtraArgs:
             provider-id: PROVIDER_ID
             read-only-port: "0"
             anonymous-auth: "false"
 {{- if .kubeletExtraArgs }}
 {{ .kubeletExtraArgs.ToYaml | indent 12 }}
 {{- end }}
-{{- if and (ne .format "bottlerocket") (or .proxyConfig .registryMirrorMap) }}
+{{- end }}
+{{- if or (and (ne .format "bottlerocket") (or .proxyConfig .registryMirrorMap)) .kubeletConfiguration }}
       files:
 {{- end }}
+{{- if .kubeletConfiguration }}
+        - content: |
+{{ .kubeletConfiguration | indent 12 -}}
+            providerID: PROVIDER_ID
+          owner: root:root
+          permissions: "0644"
+          path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.yaml
+{{- end }}
 {{- if and .proxyConfig (ne .format "bottlerocket") }}
         - content: |
             [Service]

pkg/providers/tinkerbell/template.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
+	"sigs.k8s.io/yaml"
 
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	"github.com/aws/eks-anywhere/pkg/cluster"
@@ -66,9 +67,6 @@ func NewTemplateBuilder(datacenterSpec *v1alpha1.TinkerbellDatacenterConfigSpec,
 func (tb *TemplateBuilder) GenerateCAPISpecControlPlane(clusterSpec *cluster.Spec, buildOptions ...providers.BuildMapOption) (content []byte, err error) {
 	cpTemplateConfig := clusterSpec.TinkerbellTemplateConfigs[tb.controlPlaneMachineSpec.TemplateRef.Name]
 	bundle := clusterSpec.RootVersionsBundle()
-	if err != nil {
-		return nil, err
-	}
 	var OSImageURL string
 
 	if tinkerbellIP := clusterSpec.Cluster.HasTinkerbellIPAnnotation(); tinkerbellIP != "" {
@@ -510,6 +508,16 @@ func buildTemplateMapCP(
 		values["bottlerocketSettings"] = brSettings
 	}
 
+	if clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration != nil {
+		cpKubeletConfig := clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration.Object
+		kcString, err := yaml.Marshal(cpKubeletConfig)
+		if err != nil {
+			return nil, fmt.Errorf("marshaling control plane node Kubelet Configuration while building CAPI template %v", err)
+		}
+
+		values["kubeletConfiguration"] = string(kcString)
+	}
+
 	return values, nil
 }
 
@@ -586,6 +594,16 @@ func buildTemplateMapMD(
 		values["bottlerocketSettings"] = brSettings
 	}
 
+	if workerNodeGroupConfiguration.KubeletConfiguration != nil {
+		wnKubeletConfig := workerNodeGroupConfiguration.KubeletConfiguration.Object
+		kcString, err := yaml.Marshal(wnKubeletConfig)
+		if err != nil {
+			return nil, fmt.Errorf("marshaling Kubelet Configuration for worker node %s: %v", workerNodeGroupConfiguration.Name, err)
+		}
+
+		values["kubeletConfiguration"] = string(kcString)
+	}
+
 	return values, nil
 }
 

pkg/providers/tinkerbell/template_test.go

@@ -5,9 +5,12 @@ import (
 	"time"
 
 	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	"github.com/aws/eks-anywhere/internal/test"
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/clusterapi"
+	"github.com/aws/eks-anywhere/pkg/utils/ptr"
 )
 
 func TestGenerateTemplateBuilder(t *testing.T) {
@@ -159,7 +162,6 @@ func TestTemplateBuilder_CertSANs(t *testing.T) {
 
 		data, err := bldr.GenerateCAPISpecControlPlane(clusterSpec)
 		g.Expect(err).ToNot(HaveOccurred())
-
 		test.AssertContentToFile(t, string(data), tc.Output)
 
 	}
@@ -195,3 +197,93 @@ func TestTemplateBuilder(t *testing.T) {
 
 	}
 }
+
+func TestTemplateBuilderCPKubeletConfig(t *testing.T) {
+	for _, tc := range []struct {
+		Input  string
+		Output string
+	}{
+		{
+			Input:  "testdata/cluster_tinkerbell_api_server_cert_san_ip.yaml",
+			Output: "testdata/expected_cluster_tinkerbell_api_server_cert_san_ip.yaml",
+		},
+	} {
+		g := NewWithT(t)
+		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
+
+		cpMachineCfg, _ := getControlPlaneMachineSpec(clusterSpec)
+		wngMachineCfgs, _ := getWorkerNodeGroupMachineSpec(clusterSpec)
+
+		tinkIPBefore := "0.0.0.0"
+		bldr := NewTemplateBuilder(&clusterSpec.TinkerbellDatacenter.Spec, cpMachineCfg, nil, wngMachineCfgs, tinkIPBefore, time.Now)
+
+		clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration = &unstructured.Unstructured{
+			Object: map[string]interface{}{
+				"maxPods":    20,
+				"apiVersion": "kubelet.config.k8s.io/v1beta1",
+				"kind":       "KubeletConfiguration",
+			},
+		}
+
+		data, err := bldr.GenerateCAPISpecControlPlane(clusterSpec)
+		g.Expect(err).ToNot(HaveOccurred())
+
+		g.Expect(data).To(ContainSubstring("maxPods"))
+		g.Expect(data).To(ContainSubstring("providerID"))
+	}
+}
+
+func TestTemplateBuilderWNKubeletConfig(t *testing.T) {
+	for _, tc := range []struct {
+		Input  string
+		Output string
+	}{
+		{
+			Input:  "testdata/cluster_tinkerbell_api_server_cert_san_ip.yaml",
+			Output: "testdata/expected_cluster_tinkerbell_api_server_cert_san_ip.yaml",
+		},
+	} {
+		g := NewWithT(t)
+		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
+		clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations = []v1alpha1.WorkerNodeGroupConfiguration{
+			{
+				Name:  "test",
+				Count: ptr.Int(1),
+				KubeletConfiguration: &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"maxPods":    20,
+						"apiVersion": "kubelet.config.k8s.io/v1beta1",
+						"kind":       "KubeletConfiguration",
+					},
+				},
+				MachineGroupRef: &v1alpha1.Ref{
+					Name: "wn-ref",
+					Kind: v1alpha1.TinkerbellMachineConfigKind,
+				},
+			},
+		}
+		clusterSpec.TinkerbellMachineConfigs = map[string]*v1alpha1.TinkerbellMachineConfig{
+			"wn-ref": {
+				Spec: v1alpha1.TinkerbellMachineConfigSpec{
+					Users: []v1alpha1.UserConfiguration{
+						{
+							SshAuthorizedKeys: []string{"ssh abcdef..."},
+							Name:              "user",
+						},
+					},
+				},
+			},
+		}
+
+		cpMachineCfg, _ := getControlPlaneMachineSpec(clusterSpec)
+		wngMachineCfgs, _ := getWorkerNodeGroupMachineSpec(clusterSpec)
+
+		tinkIPBefore := "0.0.0.0"
+		bldr := NewTemplateBuilder(&clusterSpec.TinkerbellDatacenter.Spec, cpMachineCfg, nil, wngMachineCfgs, tinkIPBefore, time.Now)
+		workerTemplateNames, kubeadmTemplateNames := clusterapi.InitialTemplateNamesForWorkers(clusterSpec)
+		data, err := bldr.GenerateCAPISpecWorkers(clusterSpec, workerTemplateNames, kubeadmTemplateNames)
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(data).To(ContainSubstring("maxPods"))
+		g.Expect(data).To(ContainSubstring("providerID"))
+	}
+}