Kubelet Configuration for cp and wn Tinkerbell

pkg/clusterapi/workers.go

@@ -2,6 +2,7 @@ package clusterapi
 
 import (
 	"context"
+	"reflect"
 
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -124,7 +125,10 @@ func GetKubeadmConfigTemplate(ctx context.Context, client kubernetes.Client, nam
 func KubeadmConfigTemplateEqual(new, old *kubeadmv1.KubeadmConfigTemplate) bool {
 	// DeepDerivative treats empty map (length == 0) as unset field. We need to manually compare certain fields
 	// such as taints, so that setting it to empty will trigger machine recreate
+	// The file check with deep equal has been added since the introduction of kubelet configuration in case users
+	// want to get rid of the files with that context.
 	return kubeadmConfigTemplateTaintsEqual(new, old) && kubeadmConfigTemplateExtraArgsEqual(new, old) &&
+		reflect.DeepEqual(new.Spec.Template.Spec.Files, old.Spec.Template.Spec.Files) &&
 		equality.Semantic.DeepDerivative(new.Spec, old.Spec)
 }
 

pkg/clusterapi/workers_test.go

@@ -463,6 +463,49 @@ func TestKubeadmConfigTemplateEqual(t *testing.T) {
 			},
 			want: false,
 		},
+		{
+			name: "diff spec files",
+			new: &kubeadmv1.KubeadmConfigTemplate{
+				Spec: kubeadmv1.KubeadmConfigTemplateSpec{
+					Template: kubeadmv1.KubeadmConfigTemplateResource{
+						Spec: kubeadmv1.KubeadmConfigSpec{
+							JoinConfiguration: &kubeadmv1.JoinConfiguration{
+								NodeRegistration: kubeadmv1.NodeRegistrationOptions{
+									Taints: []corev1.Taint{
+										{
+											Key: "key",
+										},
+									},
+								},
+							},
+							Files: []kubeadmv1.File{
+								{
+									Owner: "me",
+								},
+							},
+						},
+					},
+				},
+			},
+			old: &kubeadmv1.KubeadmConfigTemplate{
+				Spec: kubeadmv1.KubeadmConfigTemplateSpec{
+					Template: kubeadmv1.KubeadmConfigTemplateResource{
+						Spec: kubeadmv1.KubeadmConfigSpec{
+							JoinConfiguration: &kubeadmv1.JoinConfiguration{
+								NodeRegistration: kubeadmv1.NodeRegistrationOptions{
+									Taints: []corev1.Taint{
+										{
+											Key: "key",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			want: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/providers/tinkerbell/config/template-cp.yaml

@@ -161,6 +161,10 @@ spec:
       certificatesDir: /var/lib/kubeadm/pki
 {{- end }}
     initConfiguration:
+{{- if .kubeletConfiguration }}
+      patches: 
+        directory: /etc/kubernetes/patches
+{{- end }}
       nodeRegistration:
         kubeletExtraArgs:
           provider-id: PROVIDER_ID
@@ -184,6 +188,10 @@ spec:
 {{- end }}
 {{- end }}
     joinConfiguration:
+{{- if .kubeletConfiguration }}
+      patches: 
+        directory: /etc/kubernetes/patches
+{{- end }}
 {{- if (eq .format "bottlerocket") }}
       pause:
         imageRepository: {{.pauseRepository}}
@@ -243,6 +251,13 @@ spec:
 {{- end }}
 {{- end }}
     files:
+{{- if .kubeletConfiguration }}
+      - content: |
+{{ .kubeletConfiguration | indent 10 }}
+        owner: root:root
+        permissions: "0644"
+        path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.yaml
+{{- end }}
 {{- if not .cpSkipLoadBalancerDeployment }}
       - content: |
           apiVersion: v1

pkg/providers/tinkerbell/config/template-md.yaml

@@ -75,6 +75,10 @@ spec:
   template:
     spec:
       joinConfiguration:
+{{- if .kubeletConfiguration }}
+        patches: 
+          directory: /etc/kubernetes/patches
+{{- end }}
 {{- if (eq .format "bottlerocket") }}
         pause:
           imageRepository: {{.pauseRepository}}
@@ -128,9 +132,16 @@ spec:
 {{- if .kubeletExtraArgs }}
 {{ .kubeletExtraArgs.ToYaml | indent 12 }}
 {{- end }}
-{{- if and (ne .format "bottlerocket") (or .proxyConfig .registryMirrorMap) }}
+{{- if or (and (ne .format "bottlerocket") (or .proxyConfig .registryMirrorMap)) .kubeletConfiguration }}
       files:
 {{- end }}
+{{- if .kubeletConfiguration }}
+        - content: |
+{{ .kubeletConfiguration | indent 12 }}
+          owner: root:root
+          permissions: "0644"
+          path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.yaml
+{{- end }}
 {{- if and .proxyConfig (ne .format "bottlerocket") }}
         - content: |
             [Service]

pkg/providers/tinkerbell/template.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
+	"sigs.k8s.io/yaml"
 
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	"github.com/aws/eks-anywhere/pkg/cluster"
@@ -510,6 +511,16 @@ func buildTemplateMapCP(
 		values["bottlerocketSettings"] = brSettings
 	}
 
+	if clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration != nil {
+		cpKubeletConfig := clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration.Object
+		kcString, err := yaml.Marshal(cpKubeletConfig)
+		if err != nil {
+			return nil, fmt.Errorf("marshaling control plane node Kubelet Configuration while building CAPI template %v", err)
+		}
+
+		values["kubeletConfiguration"] = string(kcString)
+	}
+
 	return values, nil
 }
 
@@ -586,6 +597,16 @@ func buildTemplateMapMD(
 		values["bottlerocketSettings"] = brSettings
 	}
 
+	if workerNodeGroupConfiguration.KubeletConfiguration != nil {
+		wnKubeletConfig := workerNodeGroupConfiguration.KubeletConfiguration.Object
+		kcString, err := yaml.Marshal(wnKubeletConfig)
+		if err != nil {
+			return nil, fmt.Errorf("marshaling Kubelet Configuration for worker node %s: %v", workerNodeGroupConfiguration.Name, err)
+		}
+
+		values["kubeletConfiguration"] = string(kcString)
+	}
+
 	return values, nil
 }
 

pkg/providers/tinkerbell/template_test.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	"github.com/aws/eks-anywhere/internal/test"
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
@@ -195,3 +196,38 @@ func TestTemplateBuilder(t *testing.T) {
 
 	}
 }
+
+func TestTemplateBuilderCPKubeletConfig(t *testing.T) {
+	for _, tc := range []struct {
+		Input  string
+		Output string
+	}{
+		{
+			Input:  "testdata/cluster_tinkerbell_api_server_cert_san_ip.yaml",
+			Output: "testdata/expected_cluster_tinkerbell_api_server_cert_san_ip.yaml",
+		},
+	} {
+		g := NewWithT(t)
+		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
+
+		cpMachineCfg, _ := getControlPlaneMachineSpec(clusterSpec)
+		wngMachineCfgs, _ := getWorkerNodeGroupMachineSpec(clusterSpec)
+
+		tinkIPBefore := "0.0.0.0"
+		bldr := NewTemplateBuilder(&clusterSpec.TinkerbellDatacenter.Spec, cpMachineCfg, nil, wngMachineCfgs, tinkIPBefore, time.Now)
+
+		clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration = &unstructured.Unstructured{
+			Object: map[string]interface{}{
+				"maxPods":    20,
+				"apiVersion": "kubelet.config.k8s.io/v1beta1",
+				"kind":       "KubeletConfiguration",
+			},
+		}
+
+		data, err := bldr.GenerateCAPISpecControlPlane(clusterSpec)
+		g.Expect(err).ToNot(HaveOccurred())
+
+		g.Expect(data).To(ContainSubstring("maxPods"))
+
+	}
+}

pkg/providers/tinkerbell/workers.go

@@ -40,6 +40,7 @@ func WorkersSpec(ctx context.Context, logger logr.Logger, client kubernetes.Clie
 		return nil, err
 	}
 
+	logger.Info("Workers", string(workersYaml))
 	if err = parser.Parse(workersYaml, builder); err != nil {
 		return nil, errors.Wrap(err, "parsing Tinkerbell CAPI workers yaml")
 	}