Kubelet configuration customization Docker cp and wn (#8187)

config/crd/bases/anywhere.eks.amazonaws.com_clusters.yaml

@@ -178,6 +178,11 @@ spec:
                     required:
                     - host
                     type: object
+                  kubeletConfiguration:
+                    description: KubeletConfiguration is a struct that exposes the
+                      Kubelet settings for the user to set on control plane nodes.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
                   labels:
                     additionalProperties:
                       type: string
@@ -572,8 +577,13 @@ spec:
                       description: Count defines the number of desired worker nodes.
                         Defaults to 1.
                       type: integer
+                    kubeletConfiguration:
+                      description: KubeletConfiguration is a struct that exposes the
+                        Kubelet settings for the user to set on worker nodes.
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     kubernetesVersion:
-                      description: KuberenetesVersion defines the version for worker
+                      description: KubernetesVersion defines the version for worker
                         nodes. If not set, the top level spec kubernetesVersion will
                         be used.
                       type: string

config/manifest/eksa-components.yaml

@@ -3881,6 +3881,11 @@ spec:
                     required:
                     - host
                     type: object
+                  kubeletConfiguration:
+                    description: KubeletConfiguration is a struct that exposes the
+                      Kubelet settings for the user to set on control plane nodes.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
                   labels:
                     additionalProperties:
                       type: string
@@ -4275,8 +4280,13 @@ spec:
                       description: Count defines the number of desired worker nodes.
                         Defaults to 1.
                       type: integer
+                    kubeletConfiguration:
+                      description: KubeletConfiguration is a struct that exposes the
+                        Kubelet settings for the user to set on worker nodes.
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     kubernetesVersion:
-                      description: KuberenetesVersion defines the version for worker
+                      description: KubernetesVersion defines the version for worker
                         nodes. If not set, the top level spec kubernetesVersion will
                         be used.
                       type: string

go.mod

@@ -196,6 +196,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.29.1 // indirect
 	k8s.io/cluster-bootstrap v0.28.5 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/kubelet v0.29.3
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

go.sum

@@ -1507,6 +1507,8 @@ k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKf
 k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
+k8s.io/kubelet v0.29.3 h1:X9h0ZHzc+eUeNTaksbN0ItHyvGhQ7Z0HPjnQD2oHdwU=
+k8s.io/kubelet v0.29.3/go.mod h1:jDiGuTkFOUynyBKzOoC1xRSWlgAZ9UPcTYeFyjr6vas=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=

pkg/api/v1alpha1/cluster.go

@@ -15,10 +15,12 @@ import (
 	"strings"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/kubelet/config/v1beta1"
 	"sigs.k8s.io/yaml"
 
 	"github.com/aws/eks-anywhere/pkg/constants"
@@ -192,6 +194,8 @@ var clusterConfigValidations = []func(*Cluster) error{
 	validateControlPlaneCertSANs,
 	validateControlPlaneAPIServerExtraArgs,
 	validateControlPlaneAPIServerOIDCExtraArgs,
+	validateControlPlaneKubeletConfiguration,
+	validateWorkerNodeKubeletConfiguration,
 }
 
 // GetClusterConfig parses a Cluster object from a multiobject yaml file in disk
@@ -530,6 +534,51 @@ func validateControlPlaneAPIServerOIDCExtraArgs(clusterConfig *Cluster) error {
 	return nil
 }
 
+func validateControlPlaneKubeletConfiguration(clusterConfig *Cluster) error {
+	cpKubeletConfig := clusterConfig.Spec.ControlPlaneConfiguration.KubeletConfiguration
+
+	return validateKubeletConfiguration(cpKubeletConfig)
+}
+
+func validateWorkerNodeKubeletConfiguration(clusterConfig *Cluster) error {
+	workerNodeGroupConfigs := clusterConfig.Spec.WorkerNodeGroupConfigurations
+
+	for _, workerNodeGroupConfig := range workerNodeGroupConfigs {
+		wnKubeletConfig := workerNodeGroupConfig.KubeletConfiguration
+
+		if err := validateKubeletConfiguration(wnKubeletConfig); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func validateKubeletConfiguration(eksakubeconfig *unstructured.Unstructured) error {
+	if eksakubeconfig == nil {
+		return nil
+	}
+
+	var kubeletConfig v1beta1.KubeletConfiguration
+
+	kcString, err := yaml.Marshal(eksakubeconfig)
+	if err != nil {
+		return err
+	}
+
+	_, err = yaml.YAMLToJSONStrict([]byte(kcString))
+	if err != nil {
+		return fmt.Errorf("unmarshaling the yaml, malformed yaml %v", err)
+	}
+
+	err = yaml.UnmarshalStrict(kcString, &kubeletConfig)
+	if err != nil {
+		return fmt.Errorf("unmarshaling KubeletConfiguration for %v", err)
+	}
+
+	return nil
+}
+
 func validateWorkerNodeGroups(clusterConfig *Cluster) error {
 	workerNodeGroupConfigs := clusterConfig.Spec.WorkerNodeGroupConfigurations
 	if len(workerNodeGroupConfigs) <= 0 {

pkg/api/v1alpha1/cluster_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"sigs.k8s.io/yaml"
 
 	"github.com/aws/eks-anywhere/pkg/features"
@@ -1096,6 +1097,138 @@ func TestGetAndValidateClusterConfig(t *testing.T) {
 	}
 }
 
+type clusterOpt func(c *Cluster)
+
+func baseCluster(opts ...clusterOpt) *Cluster {
+	c := &Cluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-cluster",
+		},
+		Spec: ClusterSpec{
+			ControlPlaneConfiguration: ControlPlaneConfiguration{
+				Count: 1,
+				Endpoint: &Endpoint{
+					Host: "1.1.1.1",
+				},
+				MachineGroupRef: &Ref{},
+			},
+			WorkerNodeGroupConfigurations: []WorkerNodeGroupConfiguration{
+				{
+					Count: ptr.Int(3),
+					MachineGroupRef: &Ref{
+						Kind: VSphereMachineConfigKind,
+						Name: "eksa-unit-test-1",
+					},
+					Name: "wn-1",
+				},
+			},
+			KubernetesVersion: Kube129,
+			ExternalEtcdConfiguration: &ExternalEtcdConfiguration{
+				MachineGroupRef: &Ref{
+					Kind: VSphereMachineConfigKind,
+					Name: "eksa-unit-test-etcd",
+				},
+				Count: 1,
+			},
+			DatacenterRef: Ref{
+				Kind: VSphereDatacenterKind,
+				Name: "eksa-unit-test",
+			},
+			ClusterNetwork: ClusterNetwork{
+				CNIConfig: &CNIConfig{Cilium: &CiliumConfig{}},
+				Pods: Pods{
+					CidrBlocks: []string{"192.168.0.0/16"},
+				},
+				Services: Services{
+					CidrBlocks: []string{"10.96.0.0/12"},
+				},
+			},
+		},
+	}
+
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	return c
+}
+
+func TestValidateClusterConfigContent(t *testing.T) {
+	tests := []struct {
+		testName string
+		cluster  *Cluster
+		wantErr  bool
+		err      string
+	}{
+		{
+			testName: "valid cluster without kubelet",
+			cluster:  baseCluster(),
+			wantErr:  false,
+		},
+		{
+			testName: "valid cluster with kubelet config for cp and wn",
+			cluster: baseCluster(func(c *Cluster) {
+				c.Spec.ControlPlaneConfiguration.KubeletConfiguration = &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"maxPods":    20,
+						"apiVersion": "kubelet.config.k8s.io/v1beta1",
+						"kind":       "KubeletConfiguration",
+					},
+				}
+				c.Spec.WorkerNodeGroupConfigurations[0].KubeletConfiguration = &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"maxPods":    20,
+						"apiVersion": "kubelet.config.k8s.io/v1beta1",
+						"kind":       "KubeletConfiguration",
+					},
+				}
+			}),
+			wantErr: false,
+		},
+		{
+			testName: "invalid cluster with kubelet config for cp",
+			cluster: baseCluster(func(c *Cluster) {
+				c.Spec.ControlPlaneConfiguration.KubeletConfiguration = &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"maxPodss":   20,
+						"apiVersion": "kubelet.config.k8s.io/v1beta1",
+						"kind":       "KubeletConfiguration",
+					},
+				}
+			}),
+			wantErr: true,
+			err:     "unknown field",
+		},
+		{
+			testName: "invalid cluster with kubelet config for wn",
+			cluster: baseCluster(func(c *Cluster) {
+				c.Spec.WorkerNodeGroupConfigurations[0].KubeletConfiguration = &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"maxPodss":   20,
+						"apiVersion": "kubelet.config.k8s.io/v1beta1",
+						"kind":       "KubeletConfiguration",
+					},
+				}
+			}),
+			wantErr: true,
+			err:     "unknown field",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.testName, func(t *testing.T) {
+			err := ValidateClusterConfigContent(tt.cluster)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("ValidateClusterConfigContent() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if len(tt.err) > 0 && !strings.Contains(err.Error(), tt.err) {
+				t.Fatalf("ValidateClusterConfigContent() error = %s, wantErr %s", err.Error(), tt.err)
+			}
+		})
+	}
+}
+
 func TestGetClusterConfig(t *testing.T) {
 	tests := []struct {
 		testName    string

pkg/api/v1alpha1/cluster_types.go

@@ -8,6 +8,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
@@ -309,6 +310,9 @@ type ControlPlaneConfiguration struct {
 	MachineHealthCheck *MachineHealthCheck `json:"machineHealthCheck,omitempty"`
 	// APIServerExtraArgs defines the flags to configure for the API server.
 	APIServerExtraArgs map[string]string `json:"apiServerExtraArgs,omitempty"`
+	// KubeletConfiguration is a struct that exposes the Kubelet settings for the user to set on control plane nodes.
+	// +kubebuilder:pruning:PreserveUnknownFields
+	KubeletConfiguration *unstructured.Unstructured `json:"kubeletConfiguration,omitempty"`
 }
 
 // MachineHealthCheck allows to configure timeouts for machine health checks. Machine Health Checks are responsible for remediating unhealthy Machines.
@@ -453,10 +457,13 @@ type WorkerNodeGroupConfiguration struct {
 	// UpgradeRolloutStrategy determines the rollout strategy to use for rolling upgrades
 	// and related parameters/knobs
 	UpgradeRolloutStrategy *WorkerNodesUpgradeRolloutStrategy `json:"upgradeRolloutStrategy,omitempty"`
-	// KuberenetesVersion defines the version for worker nodes. If not set, the top level spec kubernetesVersion will be used.
+	// KubernetesVersion defines the version for worker nodes. If not set, the top level spec kubernetesVersion will be used.
 	KubernetesVersion *KubernetesVersion `json:"kubernetesVersion,omitempty"`
 	// MachineHealthCheck is a worker node level override for the timeouts and maxUnhealthy specified in the top-level MHC configuration. If not configured, the defaults in the top-level MHC configuration are used.
 	MachineHealthCheck *MachineHealthCheck `json:"machineHealthCheck,omitempty"`
+	// KubeletConfiguration is a struct that exposes the Kubelet settings for the user to set on worker nodes.
+	// +kubebuilder:pruning:PreserveUnknownFields
+	KubeletConfiguration *unstructured.Unstructured `json:"kubeletConfiguration,omitempty"`
 }
 
 // Equal compares two WorkerNodeGroupConfigurations.

pkg/providers/docker/config/template-cp.yaml

@@ -137,6 +137,13 @@ spec:
 {{ .schedulerExtraArgs.ToYaml | indent 10 }}
 {{- end }}
     files:
+{{- if .kubeletConfiguration }}
+    - content: |
+{{ .kubeletConfiguration | indent 8}}
+      owner: root:root
+      permissions: "0644"
+      path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.yaml
+{{- end }}
     - content: |
 {{ .auditPolicy | indent 8 }}
       owner: root:root
@@ -209,6 +216,10 @@ spec:
       path: /var/lib/kubeadm/aws-iam-authenticator/pki/key.pem
 {{- end}}
     initConfiguration:
+{{- if .kubeletConfiguration }}
+      patches: 
+        directory: /etc/kubernetes/patches
+{{- end }}
       nodeRegistration:
         criSocket: /var/run/containerd/containerd.sock
         kubeletExtraArgs:
@@ -230,6 +241,10 @@ spec:
         {{- end }}
 {{- end }}
     joinConfiguration:
+{{- if .kubeletConfiguration }}
+      patches: 
+        directory: /etc/kubernetes/patches
+{{- end }}
       nodeRegistration:
         criSocket: /var/run/containerd/containerd.sock
         kubeletExtraArgs: