From 28d621215acd0d5a34d760899b1028a55703b062 Mon Sep 17 00:00:00 2001 From: Mitali Paygude Date: Tue, 11 Jun 2024 16:01:40 -0700 Subject: [PATCH] BR limited settings validations for Kubelet configuration (#8268) --- pkg/api/v1alpha1/cluster.go | 12 +- pkg/validations/cluster.go | 68 ++++++++++ pkg/validations/cluster_test.go | 120 ++++++++++++++++++ .../createvalidations/preflightvalidations.go | 31 +++++ .../preflightvalidations_test.go | 42 +++++- .../preflightvalidations.go | 30 +++++ .../preflightvalidations_test.go | 49 +++++++ 7 files changed, 341 insertions(+), 11 deletions(-) diff --git a/pkg/api/v1alpha1/cluster.go b/pkg/api/v1alpha1/cluster.go index 18696fabb266..84bc48d96470 100644 --- a/pkg/api/v1alpha1/cluster.go +++ b/pkg/api/v1alpha1/cluster.go @@ -554,14 +554,14 @@ func validateWorkerNodeKubeletConfiguration(clusterConfig *Cluster) error { return nil } -func validateKubeletConfiguration(eksakubeconfig *unstructured.Unstructured) error { - if eksakubeconfig == nil { +func validateKubeletConfiguration(kubeletConfig *unstructured.Unstructured) error { + if kubeletConfig == nil { return nil } - var kubeletConfig v1beta1.KubeletConfiguration + var kubeletConfiguration v1beta1.KubeletConfiguration - kcString, err := yaml.Marshal(eksakubeconfig) + kcString, err := yaml.Marshal(kubeletConfig) if err != nil { return err } @@ -571,12 +571,12 @@ func validateKubeletConfiguration(eksakubeconfig *unstructured.Unstructured) err return fmt.Errorf("unmarshaling the yaml, malformed yaml %v", err) } - err = yaml.UnmarshalStrict(kcString, &kubeletConfig) + err = yaml.UnmarshalStrict(kcString, &kubeletConfiguration) if err != nil { return fmt.Errorf("unmarshaling KubeletConfiguration for %v", err) } - if _, ok := eksakubeconfig.Object["providerID"]; ok { + if _, ok := kubeletConfig.Object["providerID"]; ok { return errors.New("can not override providerID or cloudProvider (set by EKS Anywhere)") } diff --git a/pkg/validations/cluster.go b/pkg/validations/cluster.go index e4a75f656155..eef1df9ff174 100644 --- a/pkg/validations/cluster.go +++ b/pkg/validations/cluster.go @@ -5,6 +5,10 @@ import ( "errors" "fmt" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/yaml" + "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/clients/kubernetes" "github.com/aws/eks-anywhere/pkg/cluster" @@ -267,3 +271,67 @@ func ValidateManagementComponentsVersionSkew(ctx context.Context, k KubectlClien } return nil } + +// ValidateBottlerocketKubeletConfig validates bottlerocket settings for Kubelet Configuration. +func ValidateBottlerocketKubeletConfig(spec *cluster.Spec) error { + cpKubeletConfig := spec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration + if err := validateKubeletConfiguration(cpKubeletConfig); err != nil { + return err + } + + workerNodeGroupConfigs := spec.Cluster.Spec.WorkerNodeGroupConfigurations + for _, workerNodeGroupConfig := range workerNodeGroupConfigs { + wnKubeletConfig := workerNodeGroupConfig.KubeletConfiguration + if err := validateKubeletConfiguration(wnKubeletConfig); err != nil { + return err + } + } + + return nil +} + +func validateKubeletConfiguration(kubeletConfig *unstructured.Unstructured) error { + if kubeletConfig == nil { + return nil + } + kubeletConfigCopy, err := copyObject(kubeletConfig) + if err != nil { + return err + } + + delete(kubeletConfigCopy.Object, "kind") + delete(kubeletConfigCopy.Object, "apiVersion") + kcString, err := yaml.Marshal(kubeletConfigCopy) + if err != nil { + return err + } + + _, err = yaml.YAMLToJSONStrict([]byte(kcString)) + if err != nil { + return fmt.Errorf("unmarshaling the yaml, malformed yaml %v", err) + } + + var bottlerocketKC *v1beta1.BottlerocketKubernetesSettings + err = yaml.UnmarshalStrict(kcString, &bottlerocketKC) + if err != nil { + return fmt.Errorf("unmarshaling KubeletConfiguration for %v", err) + } + + return nil +} + +func copyObject(kubeletConfig *unstructured.Unstructured) (*unstructured.Unstructured, error) { + var kubeletConfigBackup *unstructured.Unstructured + + kcString, err := yaml.Marshal(kubeletConfig) + if err != nil { + return nil, err + } + + err = yaml.UnmarshalStrict(kcString, &kubeletConfigBackup) + if err != nil { + return nil, fmt.Errorf("unmarshaling KubeletConfiguration for %v", err) + } + + return kubeletConfigBackup, nil +} diff --git a/pkg/validations/cluster_test.go b/pkg/validations/cluster_test.go index 553b598f13a7..e5d1e4a9b7a6 100644 --- a/pkg/validations/cluster_test.go +++ b/pkg/validations/cluster_test.go @@ -10,6 +10,7 @@ import ( "github.com/golang/mock/gomock" . "github.com/onsi/gomega" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/eks-anywhere/internal/test" @@ -742,3 +743,122 @@ func TestValidateManagementComponentsVersionSkew(t *testing.T) { }) } } + +func TestValidateBottlerocketKC(t *testing.T) { + tests := []struct { + name string + spec *cluster.Spec + subErr error + }{ + { + name: "cp config", + spec: &cluster.Spec{ + Config: &cluster.Config{ + Cluster: &anywherev1.Cluster{ + Spec: anywherev1.ClusterSpec{ + ControlPlaneConfiguration: anywherev1.ControlPlaneConfiguration{ + KubeletConfiguration: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "KubeletConfiguration", + "maxPods": 50, + }, + }, + }, + }, + }, + }, + }, + subErr: nil, + }, + { + name: "worker config", + spec: &cluster.Spec{ + Config: &cluster.Config{ + Cluster: &anywherev1.Cluster{ + Spec: anywherev1.ClusterSpec{ + WorkerNodeGroupConfigurations: []anywherev1.WorkerNodeGroupConfiguration{ + { + KubeletConfiguration: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "maxPods": 50, + "kind": "KubeletConfiguration", + }, + }, + }, + }, + }, + }, + }, + }, + subErr: nil, + }, + { + name: "nil kc config", + spec: &cluster.Spec{ + Config: &cluster.Config{ + Cluster: &anywherev1.Cluster{ + Spec: anywherev1.ClusterSpec{ + ControlPlaneConfiguration: anywherev1.ControlPlaneConfiguration{}, + }, + }, + }, + }, + subErr: nil, + }, + { + name: "invalid cp config", + spec: &cluster.Spec{ + Config: &cluster.Config{ + Cluster: &anywherev1.Cluster{ + Spec: anywherev1.ClusterSpec{ + ControlPlaneConfiguration: anywherev1.ControlPlaneConfiguration{ + KubeletConfiguration: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "maxPodss": 50, + "kind": "KubeletConfiguration", + }, + }, + }, + }, + }, + }, + }, + subErr: errors.New("unknown field \"maxPodss\""), + }, + { + name: "invalid worker config", + spec: &cluster.Spec{ + Config: &cluster.Config{ + Cluster: &anywherev1.Cluster{ + Spec: anywherev1.ClusterSpec{ + WorkerNodeGroupConfigurations: []anywherev1.WorkerNodeGroupConfiguration{ + { + KubeletConfiguration: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "maxPodss": 50, + "kind": "KubeletConfiguration", + }, + }, + }, + }, + }, + }, + }, + }, + subErr: errors.New("unknown field \"maxPodss\""), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tt := newTest(t, withKubectl()) + + err := validations.ValidateBottlerocketKubeletConfig(tc.spec) + if err != nil { + t.Log(err.Error()) + tt.Expect(err.Error()).To(ContainSubstring(tc.subErr.Error())) + } else { + tt.Expect(tc.subErr).To(BeNil()) + } + }) + } +} diff --git a/pkg/validations/createvalidations/preflightvalidations.go b/pkg/validations/createvalidations/preflightvalidations.go index 76fa21e31a6c..53002e389b0b 100644 --- a/pkg/validations/createvalidations/preflightvalidations.go +++ b/pkg/validations/createvalidations/preflightvalidations.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/aws/eks-anywhere/pkg/api/v1alpha1" anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/config" "github.com/aws/eks-anywhere/pkg/constants" @@ -51,6 +52,36 @@ func (v *CreateValidations) PreflightValidations(ctx context.Context) []validati }, } + if len(v.Opts.Spec.VSphereMachineConfigs) != 0 { + cpRef := v.Opts.Spec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name + if v.Opts.Spec.VSphereMachineConfigs[cpRef].Spec.OSFamily == v1alpha1.Bottlerocket { + createValidations = append(createValidations, + func() *validations.ValidationResult { + return &validations.ValidationResult{ + Name: "validate cluster's kubelet configuration for Bottlerocket OS", + Remediation: "ensure that the settings configured for Kubelet Configuration are supported by Bottlerocket", + Err: validations.ValidateBottlerocketKubeletConfig(v.Opts.Spec), + } + }) + } + + wnConfigs := v.Opts.Spec.Cluster.Spec.WorkerNodeGroupConfigurations + for i := range wnConfigs { + workerNodeRef := wnConfigs[i].MachineGroupRef.Name + + if v.Opts.Spec.VSphereMachineConfigs[workerNodeRef].Spec.OSFamily == anywherev1.Bottlerocket { + createValidations = append(createValidations, + func() *validations.ValidationResult { + return &validations.ValidationResult{ + Name: "validate cluster's worker node kubelet configuration for Bottlerocket OS", + Remediation: "ensure that the settings configured for Kubelet Configuration are supported by Bottlerocket", + Err: validations.ValidateBottlerocketKubeletConfig(v.Opts.Spec), + } + }) + } + } + } + if v.Opts.Spec.Cluster.IsManaged() { createValidations = append( createValidations, diff --git a/pkg/validations/createvalidations/preflightvalidations_test.go b/pkg/validations/createvalidations/preflightvalidations_test.go index e5a3a7fe9973..3b775a7939aa 100644 --- a/pkg/validations/createvalidations/preflightvalidations_test.go +++ b/pkg/validations/createvalidations/preflightvalidations_test.go @@ -7,10 +7,10 @@ import ( "github.com/golang/mock/gomock" . "github.com/onsi/gomega" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/eks-anywhere/internal/test" - "github.com/aws/eks-anywhere/pkg/api/v1alpha1" anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/cluster" "github.com/aws/eks-anywhere/pkg/constants" @@ -34,7 +34,7 @@ func newPreflightValidationsTest(t *testing.T) *preflightValidationsTest { KubeconfigFile: "kubeconfig", } clusterSpec := test.NewClusterSpec(func(s *cluster.Spec) { - s.Cluster.Spec.GitOpsRef = &v1alpha1.Ref{ + s.Cluster.Spec.GitOpsRef = &anywherev1.Ref{ Name: "gitops", } }) @@ -69,12 +69,12 @@ func TestPreFlightValidationsWorkloadCluster(t *testing.T) { tt.c.Opts.ManagementCluster.Name = mgmtClusterName version := test.DevEksaVersion() - mgmt := &v1alpha1.Cluster{ + mgmt := &anywherev1.Cluster{ ObjectMeta: v1.ObjectMeta{ Name: "mgmt-cluster", }, - Spec: v1alpha1.ClusterSpec{ - ManagementCluster: v1alpha1.ManagementCluster{ + Spec: anywherev1.ClusterSpec{ + ManagementCluster: anywherev1.ManagementCluster{ Name: "mgmt-cluster", }, BundlesRef: &anywherev1.BundlesRef{ @@ -82,6 +82,38 @@ func TestPreFlightValidationsWorkloadCluster(t *testing.T) { Namespace: constants.EksaSystemNamespace, }, EksaVersion: &version, + ControlPlaneConfiguration: anywherev1.ControlPlaneConfiguration{ + KubeletConfiguration: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "staticPodPath": "path", + }, + }, + }, + }, + } + + tt.c.Opts.Spec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef = &anywherev1.Ref{ + Name: "cpRef", + } + tt.c.Opts.Spec.VSphereMachineConfigs = map[string]*anywherev1.VSphereMachineConfig{ + "cpRef": { + Spec: anywherev1.VSphereMachineConfigSpec{ + OSFamily: anywherev1.Bottlerocket, + }, + }, + } + + tt.c.Opts.Spec.Cluster.Spec.WorkerNodeGroupConfigurations = []anywherev1.WorkerNodeGroupConfiguration{ + { + MachineGroupRef: &anywherev1.Ref{ + Name: "wnRef", + }, + }, + } + + tt.c.Opts.Spec.VSphereMachineConfigs["wnRef"] = &anywherev1.VSphereMachineConfig{ + Spec: anywherev1.VSphereMachineConfigSpec{ + OSFamily: anywherev1.Bottlerocket, }, } diff --git a/pkg/validations/upgradevalidations/preflightvalidations.go b/pkg/validations/upgradevalidations/preflightvalidations.go index 650bd9f941e2..46dad6c2e6f6 100644 --- a/pkg/validations/upgradevalidations/preflightvalidations.go +++ b/pkg/validations/upgradevalidations/preflightvalidations.go @@ -124,6 +124,36 @@ func (u *UpgradeValidations) PreflightValidations(ctx context.Context) []validat }, } + if len(u.Opts.Spec.VSphereMachineConfigs) != 0 { + cpRef := u.Opts.Spec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name + if u.Opts.Spec.VSphereMachineConfigs[cpRef].Spec.OSFamily == anywherev1.Bottlerocket { + upgradeValidations = append(upgradeValidations, + func() *validations.ValidationResult { + return &validations.ValidationResult{ + Name: "validate cluster's control plane kubelet configuration for Bottlerocket OS", + Remediation: "ensure that the settings configured for Kubelet Configuration are supported by Bottlerocket", + Err: validations.ValidateBottlerocketKubeletConfig(u.Opts.Spec), + } + }) + } + + wnConfigs := u.Opts.Spec.Cluster.Spec.WorkerNodeGroupConfigurations + for i := range wnConfigs { + workerNodeRef := wnConfigs[i].MachineGroupRef.Name + + if u.Opts.Spec.VSphereMachineConfigs[workerNodeRef].Spec.OSFamily == anywherev1.Bottlerocket { + upgradeValidations = append(upgradeValidations, + func() *validations.ValidationResult { + return &validations.ValidationResult{ + Name: "validate cluster's worker node kubelet configuration for Bottlerocket OS", + Remediation: "ensure that the settings configured for Kubelet Configuration are supported by Bottlerocket", + Err: validations.ValidateBottlerocketKubeletConfig(u.Opts.Spec), + } + }) + } + } + } + if u.Opts.Spec.Cluster.IsManaged() { upgradeValidations = append( upgradeValidations, diff --git a/pkg/validations/upgradevalidations/preflightvalidations_test.go b/pkg/validations/upgradevalidations/preflightvalidations_test.go index eeb61f618726..9aba8e239b49 100644 --- a/pkg/validations/upgradevalidations/preflightvalidations_test.go +++ b/pkg/validations/upgradevalidations/preflightvalidations_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/golang/mock/gomock" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/eks-anywhere/internal/test" @@ -446,6 +447,54 @@ func TestPreflightValidationsVsphere(t *testing.T) { modifyDefaultSpecFunc func(s *cluster.Spec) additionalKubectlMocks func(k *mocks.MockKubectlClient) }{ + { + name: "ValidationBottlerocketKC", + clusterVersion: "v1.19.16-eks-1-19-4", + upgradeVersion: string(anywherev1.Kube119), + getClusterResponse: goodClusterResponse, + cpResponse: nil, + workerResponse: nil, + nodeResponse: nil, + crdResponse: nil, + wantErr: nil, + modifyDefaultSpecFunc: func(s *cluster.Spec) { + s.VSphereMachineConfigs = map[string]*anywherev1.VSphereMachineConfig{ + "test": { + Spec: anywherev1.VSphereMachineConfigSpec{ + OSFamily: anywherev1.Bottlerocket, + Users: []anywherev1.UserConfiguration{{ + Name: "mySshUsername", + SshAuthorizedKeys: []string{"mySshAuthorizedKey"}, + }}, + }, + }, + "wnRef": { + Spec: anywherev1.VSphereMachineConfigSpec{ + OSFamily: anywherev1.Bottlerocket, + Users: []anywherev1.UserConfiguration{{ + Name: "mySshUsername", + SshAuthorizedKeys: []string{"mySshAuthorizedKey"}, + }}, + }, + }, + } + s.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration = &unstructured.Unstructured{ + Object: map[string]interface{}{ + "clusterDNSIPs": []string{"ip1"}, + "kind": "KubeletConfiguration", + }, + } + s.Cluster.Spec.WorkerNodeGroupConfigurations[0].KubeletConfiguration = &unstructured.Unstructured{ + Object: map[string]interface{}{ + "clusterDNSIPs": []string{"ip1"}, + "kind": "KubeletConfiguration", + }, + } + s.Cluster.Spec.WorkerNodeGroupConfigurations[0].MachineGroupRef = &anywherev1.Ref{ + Name: "wnRef", + } + }, + }, { name: "ValidationSucceeds", clusterVersion: "v1.19.16-eks-1-19-4",