Kubelet Configuration for cp and wn Tinkerbell

pkg/clusterapi/workers.go

@@ -2,6 +2,7 @@ package clusterapi
 
 import (
 	"context"
+	"reflect"
 
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -124,7 +125,10 @@ func GetKubeadmConfigTemplate(ctx context.Context, client kubernetes.Client, nam
 func KubeadmConfigTemplateEqual(new, old *kubeadmv1.KubeadmConfigTemplate) bool {
 	// DeepDerivative treats empty map (length == 0) as unset field. We need to manually compare certain fields
 	// such as taints, so that setting it to empty will trigger machine recreate
+	// The file check with deep equal has been added since the introduction of kubelet configuration in case users
+	// want to get rid of the files with that context.
 	return kubeadmConfigTemplateTaintsEqual(new, old) && kubeadmConfigTemplateExtraArgsEqual(new, old) &&
+		reflect.DeepEqual(new.Spec.Template.Spec.Files, old.Spec.Template.Spec.Files) &&
 		equality.Semantic.DeepDerivative(new.Spec, old.Spec)
 }
 

pkg/clusterapi/workers_test.go

@@ -463,6 +463,49 @@ func TestKubeadmConfigTemplateEqual(t *testing.T) {
 			},
 			want: false,
 		},
+		{
+			name: "diff spec files",
+			new: &kubeadmv1.KubeadmConfigTemplate{
+				Spec: kubeadmv1.KubeadmConfigTemplateSpec{
+					Template: kubeadmv1.KubeadmConfigTemplateResource{
+						Spec: kubeadmv1.KubeadmConfigSpec{
+							JoinConfiguration: &kubeadmv1.JoinConfiguration{
+								NodeRegistration: kubeadmv1.NodeRegistrationOptions{
+									Taints: []corev1.Taint{
+										{
+											Key: "key",
+										},
+									},
+								},
+							},
+							Files: []kubeadmv1.File{
+								{
+									Owner: "me",
+								},
+							},
+						},
+					},
+				},
+			},
+			old: &kubeadmv1.KubeadmConfigTemplate{
+				Spec: kubeadmv1.KubeadmConfigTemplateSpec{
+					Template: kubeadmv1.KubeadmConfigTemplateResource{
+						Spec: kubeadmv1.KubeadmConfigSpec{
+							JoinConfiguration: &kubeadmv1.JoinConfiguration{
+								NodeRegistration: kubeadmv1.NodeRegistrationOptions{
+									Taints: []corev1.Taint{
+										{
+											Key: "key",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			want: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/providers/tinkerbell/config/template-cp.yaml

@@ -161,14 +161,20 @@ spec:
       certificatesDir: /var/lib/kubeadm/pki
 {{- end }}
     initConfiguration:
+{{- if .kubeletConfiguration }}
+      patches: 
+        directory: /etc/kubernetes/patches
+{{- end }}
       nodeRegistration:
         kubeletExtraArgs:
           provider-id: PROVIDER_ID
+{{- if not .kubeletConfiguration }}
           read-only-port: "0"
           anonymous-auth: "false"
 {{- if .kubeletExtraArgs }}
 {{ .kubeletExtraArgs.ToYaml | indent 10 }}
 {{- end }}
+{{- end }}
 {{- if not .workerNodeGroupConfigurations }}
         taints: []
 {{- end }}
@@ -184,6 +190,10 @@ spec:
 {{- end }}
 {{- end }}
     joinConfiguration:
+{{- if .kubeletConfiguration }}
+      patches: 
+        directory: /etc/kubernetes/patches
+{{- end }}
 {{- if (eq .format "bottlerocket") }}
       pause:
         imageRepository: {{.pauseRepository}}
@@ -223,11 +233,13 @@ spec:
         - DirAvailable--etc-kubernetes-manifests
         kubeletExtraArgs:
           provider-id: PROVIDER_ID
+{{- if not .kubeletConfiguration }}
           read-only-port: "0"
           anonymous-auth: "false"
 {{- if .kubeletExtraArgs }}
 {{ .kubeletExtraArgs.ToYaml | indent 10 }}
 {{- end }}
+{{- end }}
 {{- if not .workerNodeGroupConfigurations }}
         taints: []
 {{- end }}
@@ -243,6 +255,13 @@ spec:
 {{- end }}
 {{- end }}
     files:
+{{- if .kubeletConfiguration }}
+      - content: |
+{{ .kubeletConfiguration | indent 10 }}
+        owner: root:root
+        permissions: "0644"
+        path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.yaml
+{{- end }}
 {{- if not .cpSkipLoadBalancerDeployment }}
       - content: |
           apiVersion: v1

pkg/providers/tinkerbell/config/template-md.yaml

@@ -75,6 +75,10 @@ spec:
   template:
     spec:
       joinConfiguration:
+{{- if .kubeletConfiguration }}
+        patches: 
+          directory: /etc/kubernetes/patches
+{{- end }}
 {{- if (eq .format "bottlerocket") }}
         pause:
           imageRepository: {{.pauseRepository}}
@@ -123,14 +127,23 @@ spec:
 {{- end }}
           kubeletExtraArgs:
             provider-id: PROVIDER_ID
+{{- if not .kubeletConfiguration }}
             read-only-port: "0"
             anonymous-auth: "false"
 {{- if .kubeletExtraArgs }}
 {{ .kubeletExtraArgs.ToYaml | indent 12 }}
 {{- end }}
-{{- if and (ne .format "bottlerocket") (or .proxyConfig .registryMirrorMap) }}
+{{- end }}
+{{- if or (and (ne .format "bottlerocket") (or .proxyConfig .registryMirrorMap)) .kubeletConfiguration }}
       files:
 {{- end }}
+{{- if .kubeletConfiguration }}
+        - content: |
+{{ .kubeletConfiguration | indent 12 }}
+          owner: root:root
+          permissions: "0644"
+          path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.yaml
+{{- end }}
 {{- if and .proxyConfig (ne .format "bottlerocket") }}
         - content: |
             [Service]

pkg/providers/tinkerbell/template.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
+	"sigs.k8s.io/yaml"
 
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	"github.com/aws/eks-anywhere/pkg/cluster"
@@ -66,9 +67,6 @@ func NewTemplateBuilder(datacenterSpec *v1alpha1.TinkerbellDatacenterConfigSpec,
 func (tb *TemplateBuilder) GenerateCAPISpecControlPlane(clusterSpec *cluster.Spec, buildOptions ...providers.BuildMapOption) (content []byte, err error) {
 	cpTemplateConfig := clusterSpec.TinkerbellTemplateConfigs[tb.controlPlaneMachineSpec.TemplateRef.Name]
 	bundle := clusterSpec.RootVersionsBundle()
-	if err != nil {
-		return nil, err
-	}
 	var OSImageURL string
 
 	if tinkerbellIP := clusterSpec.Cluster.HasTinkerbellIPAnnotation(); tinkerbellIP != "" {
@@ -401,9 +399,6 @@ func buildTemplateMapCP(
 		Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)).
 		Append(clusterapi.APIServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.APIServerExtraArgs))
 	clusterapi.SetPodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig, apiServerExtraArgs)
-	kubeletExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
-		Append(clusterapi.ResolvConfExtraArgs(clusterSpec.Cluster.Spec.ClusterNetwork.DNS.ResolvConf)).
-		Append(clusterapi.ControlPlaneNodeLabelsExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration))
 
 	values := map[string]interface{}{
 		"auditPolicy":                   auditPolicy,
@@ -430,7 +425,6 @@ func buildTemplateMapCP(
 		"etcdImageTag":                  versionsBundle.KubeDistro.Etcd.Tag,
 		"externalEtcdVersion":           versionsBundle.KubeDistro.EtcdVersion,
 		"etcdCipherSuites":              crypto.SecureCipherSuitesString(),
-		"kubeletExtraArgs":              kubeletExtraArgs.ToPartialYaml(),
 		"hardwareSelector":              controlPlaneMachineSpec.HardwareSelector,
 		"controlPlaneTaints":            clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Taints,
 		"workerNodeGroupConfigurations": clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations,
@@ -510,6 +504,22 @@ func buildTemplateMapCP(
 		values["bottlerocketSettings"] = brSettings
 	}
 
+	if clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration != nil {
+		cpKubeletConfig := clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration.Object
+		kcString, err := yaml.Marshal(cpKubeletConfig)
+		if err != nil {
+			return nil, fmt.Errorf("marshaling control plane node Kubelet Configuration while building CAPI template %v", err)
+		}
+
+		values["kubeletConfiguration"] = string(kcString)
+	} else {
+		kubeletExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
+			Append(clusterapi.ResolvConfExtraArgs(clusterSpec.Cluster.Spec.ClusterNetwork.DNS.ResolvConf)).
+			Append(clusterapi.ControlPlaneNodeLabelsExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration))
+
+		values["kubeletExtraArgs"] = kubeletExtraArgs.ToPartialYaml()
+	}
+
 	return values, nil
 }
 
@@ -523,14 +533,9 @@ func buildTemplateMapMD(
 	versionsBundle := clusterSpec.WorkerNodeGroupVersionsBundle(workerNodeGroupConfiguration)
 	format := "cloud-config"
 
-	kubeletExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
-		Append(clusterapi.WorkerNodeLabelsExtraArgs(workerNodeGroupConfiguration)).
-		Append(clusterapi.ResolvConfExtraArgs(clusterSpec.Cluster.Spec.ClusterNetwork.DNS.ResolvConf))
-
 	values := map[string]interface{}{
 		"clusterName":            clusterSpec.Cluster.Name,
 		"eksaSystemNamespace":    constants.EksaSystemNamespace,
-		"kubeletExtraArgs":       kubeletExtraArgs.ToPartialYaml(),
 		"format":                 format,
 		"kubernetesVersion":      versionsBundle.KubeDistro.Kubernetes.Tag,
 		"workerNodeGroupName":    workerNodeGroupConfiguration.Name,
@@ -586,6 +591,21 @@ func buildTemplateMapMD(
 		values["bottlerocketSettings"] = brSettings
 	}
 
+	if workerNodeGroupConfiguration.KubeletConfiguration != nil {
+		wnKubeletConfig := workerNodeGroupConfiguration.KubeletConfiguration.Object
+		kcString, err := yaml.Marshal(wnKubeletConfig)
+		if err != nil {
+			return nil, fmt.Errorf("marshaling Kubelet Configuration for worker node %s: %v", workerNodeGroupConfiguration.Name, err)
+		}
+
+		values["kubeletConfiguration"] = string(kcString)
+	} else {
+		kubeletExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
+			Append(clusterapi.WorkerNodeLabelsExtraArgs(workerNodeGroupConfiguration)).
+			Append(clusterapi.ResolvConfExtraArgs(clusterSpec.Cluster.Spec.ClusterNetwork.DNS.ResolvConf))
+		values["kubeletExtraArgs"] = kubeletExtraArgs.ToPartialYaml()
+	}
+
 	return values, nil
 }
 

pkg/providers/tinkerbell/template_test.go

@@ -5,9 +5,12 @@ import (
 	"time"
 
 	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	"github.com/aws/eks-anywhere/internal/test"
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/clusterapi"
+	"github.com/aws/eks-anywhere/pkg/utils/ptr"
 )
 
 func TestGenerateTemplateBuilder(t *testing.T) {
@@ -159,7 +162,6 @@ func TestTemplateBuilder_CertSANs(t *testing.T) {
 
 		data, err := bldr.GenerateCAPISpecControlPlane(clusterSpec)
 		g.Expect(err).ToNot(HaveOccurred())
-
 		test.AssertContentToFile(t, string(data), tc.Output)
 
 	}
@@ -195,3 +197,87 @@ func TestTemplateBuilder(t *testing.T) {
 
 	}
 }
+
+func TestTemplateBuilderCPKubeletConfig(t *testing.T) {
+	for _, tc := range []struct {
+		Input  string
+		Output string
+	}{
+		{
+			Input:  "testdata/cluster_tinkerbell_api_server_cert_san_ip.yaml",
+			Output: "testdata/expected_cluster_tinkerbell_api_server_cert_san_ip.yaml",
+		},
+	} {
+		g := NewWithT(t)
+		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
+		cpMachineCfg, _ := getControlPlaneMachineSpec(clusterSpec)
+		wngMachineCfgs, _ := getWorkerNodeGroupMachineSpec(clusterSpec)
+		tinkIPBefore := "0.0.0.0"
+		bldr := NewTemplateBuilder(&clusterSpec.TinkerbellDatacenter.Spec, cpMachineCfg, nil, wngMachineCfgs, tinkIPBefore, time.Now)
+
+		clusterSpec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration = &unstructured.Unstructured{
+			Object: map[string]interface{}{
+				"maxPods":    20,
+				"apiVersion": "kubelet.config.k8s.io/v1beta1",
+				"kind":       "KubeletConfiguration",
+			},
+		}
+
+		data, err := bldr.GenerateCAPISpecControlPlane(clusterSpec)
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(data).To(ContainSubstring("maxPods"))
+	}
+}
+
+func TestTemplateBuilderWNKubeletConfig(t *testing.T) {
+	for _, tc := range []struct {
+		Input  string
+		Output string
+	}{
+		{
+			Input:  "testdata/cluster_tinkerbell_api_server_cert_san_ip.yaml",
+			Output: "testdata/expected_cluster_tinkerbell_api_server_cert_san_ip.yaml",
+		},
+	} {
+		g := NewWithT(t)
+		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
+		clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations = []v1alpha1.WorkerNodeGroupConfiguration{
+			{
+				Name:  "test",
+				Count: ptr.Int(1),
+				KubeletConfiguration: &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"maxPods":    20,
+						"apiVersion": "kubelet.config.k8s.io/v1beta1",
+						"kind":       "KubeletConfiguration",
+					},
+				},
+				MachineGroupRef: &v1alpha1.Ref{
+					Name: "wn-ref",
+					Kind: v1alpha1.TinkerbellMachineConfigKind,
+				},
+			},
+		}
+		clusterSpec.TinkerbellMachineConfigs = map[string]*v1alpha1.TinkerbellMachineConfig{
+			"wn-ref": {
+				Spec: v1alpha1.TinkerbellMachineConfigSpec{
+					Users: []v1alpha1.UserConfiguration{
+						{
+							SshAuthorizedKeys: []string{"ssh abcdef..."},
+							Name:              "user",
+						},
+					},
+				},
+			},
+		}
+
+		cpMachineCfg, _ := getControlPlaneMachineSpec(clusterSpec)
+		wngMachineCfgs, _ := getWorkerNodeGroupMachineSpec(clusterSpec)
+		tinkIPBefore := "0.0.0.0"
+		bldr := NewTemplateBuilder(&clusterSpec.TinkerbellDatacenter.Spec, cpMachineCfg, nil, wngMachineCfgs, tinkIPBefore, time.Now)
+		workerTemplateNames, kubeadmTemplateNames := clusterapi.InitialTemplateNamesForWorkers(clusterSpec)
+		data, err := bldr.GenerateCAPISpecWorkers(clusterSpec, workerTemplateNames, kubeadmTemplateNames)
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(data).To(ContainSubstring("maxPods"))
+	}
+}