Skip to content

Commit

Permalink
BR limited settings validations for Kubelet configuration (#8268)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mitalipaygude
mitalipaygude authored Jun 11, 2024
1 parent adddcf0 commit 28d6212
Show file tree
Hide file tree
Showing 7 changed files with 341 additions and 11 deletions.
12 changes: 6 additions & 6 deletions pkg/api/v1alpha1/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,14 +554,14 @@ func validateWorkerNodeKubeletConfiguration(clusterConfig *Cluster) error {
return nil
}

func validateKubeletConfiguration(eksakubeconfig *unstructured.Unstructured) error {
if eksakubeconfig == nil {
func validateKubeletConfiguration(kubeletConfig *unstructured.Unstructured) error {
if kubeletConfig == nil {
return nil
}

var kubeletConfig v1beta1.KubeletConfiguration
var kubeletConfiguration v1beta1.KubeletConfiguration

kcString, err := yaml.Marshal(eksakubeconfig)
kcString, err := yaml.Marshal(kubeletConfig)
if err != nil {
return err
}
Expand All @@ -571,12 +571,12 @@ func validateKubeletConfiguration(eksakubeconfig *unstructured.Unstructured) err
return fmt.Errorf("unmarshaling the yaml, malformed yaml %v", err)
}

err = yaml.UnmarshalStrict(kcString, &kubeletConfig)
err = yaml.UnmarshalStrict(kcString, &kubeletConfiguration)
if err != nil {
return fmt.Errorf("unmarshaling KubeletConfiguration for %v", err)
}

if _, ok := eksakubeconfig.Object["providerID"]; ok {
if _, ok := kubeletConfig.Object["providerID"]; ok {
return errors.New("can not override providerID or cloudProvider (set by EKS Anywhere)")
}

Expand Down
68 changes: 68 additions & 0 deletions pkg/validations/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ import (
"errors"
"fmt"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
"sigs.k8s.io/yaml"

"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
"github.com/aws/eks-anywhere/pkg/clients/kubernetes"
"github.com/aws/eks-anywhere/pkg/cluster"
Expand Down Expand Up @@ -267,3 +271,67 @@ func ValidateManagementComponentsVersionSkew(ctx context.Context, k KubectlClien
}
return nil
}

// ValidateBottlerocketKubeletConfig validates bottlerocket settings for Kubelet Configuration.
func ValidateBottlerocketKubeletConfig(spec *cluster.Spec) error {
cpKubeletConfig := spec.Cluster.Spec.ControlPlaneConfiguration.KubeletConfiguration
if err := validateKubeletConfiguration(cpKubeletConfig); err != nil {
return err
}

workerNodeGroupConfigs := spec.Cluster.Spec.WorkerNodeGroupConfigurations
for _, workerNodeGroupConfig := range workerNodeGroupConfigs {
wnKubeletConfig := workerNodeGroupConfig.KubeletConfiguration
if err := validateKubeletConfiguration(wnKubeletConfig); err != nil {
return err
}
}

return nil
}

func validateKubeletConfiguration(kubeletConfig *unstructured.Unstructured) error {
if kubeletConfig == nil {
return nil
}
kubeletConfigCopy, err := copyObject(kubeletConfig)
if err != nil {
return err
}

delete(kubeletConfigCopy.Object, "kind")
delete(kubeletConfigCopy.Object, "apiVersion")
kcString, err := yaml.Marshal(kubeletConfigCopy)
if err != nil {
return err
}

_, err = yaml.YAMLToJSONStrict([]byte(kcString))
if err != nil {
return fmt.Errorf("unmarshaling the yaml, malformed yaml %v", err)
}

var bottlerocketKC *v1beta1.BottlerocketKubernetesSettings
err = yaml.UnmarshalStrict(kcString, &bottlerocketKC)
if err != nil {
return fmt.Errorf("unmarshaling KubeletConfiguration for %v", err)
}

return nil
}

func copyObject(kubeletConfig *unstructured.Unstructured) (*unstructured.Unstructured, error) {
var kubeletConfigBackup *unstructured.Unstructured

kcString, err := yaml.Marshal(kubeletConfig)
if err != nil {
return nil, err
}

err = yaml.UnmarshalStrict(kcString, &kubeletConfigBackup)
if err != nil {
return nil, fmt.Errorf("unmarshaling KubeletConfiguration for %v", err)
}

return kubeletConfigBackup, nil
}
120 changes: 120 additions & 0 deletions pkg/validations/cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/golang/mock/gomock"
. "github.com/onsi/gomega"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/aws/eks-anywhere/internal/test"
Expand Down Expand Up @@ -742,3 +743,122 @@ func TestValidateManagementComponentsVersionSkew(t *testing.T) {
})
}
}

func TestValidateBottlerocketKC(t *testing.T) {
tests := []struct {
name string
spec *cluster.Spec
subErr error
}{
{
name: "cp config",
spec: &cluster.Spec{
Config: &cluster.Config{
Cluster: &anywherev1.Cluster{
Spec: anywherev1.ClusterSpec{
ControlPlaneConfiguration: anywherev1.ControlPlaneConfiguration{
KubeletConfiguration: &unstructured.Unstructured{
Object: map[string]interface{}{
"kind": "KubeletConfiguration",
"maxPods": 50,
},
},
},
},
},
},
},
subErr: nil,
},
{
name: "worker config",
spec: &cluster.Spec{
Config: &cluster.Config{
Cluster: &anywherev1.Cluster{
Spec: anywherev1.ClusterSpec{
WorkerNodeGroupConfigurations: []anywherev1.WorkerNodeGroupConfiguration{
{
KubeletConfiguration: &unstructured.Unstructured{
Object: map[string]interface{}{
"maxPods": 50,
"kind": "KubeletConfiguration",
},
},
},
},
},
},
},
},
subErr: nil,
},
{
name: "nil kc config",
spec: &cluster.Spec{
Config: &cluster.Config{
Cluster: &anywherev1.Cluster{
Spec: anywherev1.ClusterSpec{
ControlPlaneConfiguration: anywherev1.ControlPlaneConfiguration{},
},
},
},
},
subErr: nil,
},
{
name: "invalid cp config",
spec: &cluster.Spec{
Config: &cluster.Config{
Cluster: &anywherev1.Cluster{
Spec: anywherev1.ClusterSpec{
ControlPlaneConfiguration: anywherev1.ControlPlaneConfiguration{
KubeletConfiguration: &unstructured.Unstructured{
Object: map[string]interface{}{
"maxPodss": 50,
"kind": "KubeletConfiguration",
},
},
},
},
},
},
},
subErr: errors.New("unknown field \"maxPodss\""),
},
{
name: "invalid worker config",
spec: &cluster.Spec{
Config: &cluster.Config{
Cluster: &anywherev1.Cluster{
Spec: anywherev1.ClusterSpec{
WorkerNodeGroupConfigurations: []anywherev1.WorkerNodeGroupConfiguration{
{
KubeletConfiguration: &unstructured.Unstructured{
Object: map[string]interface{}{
"maxPodss": 50,
"kind": "KubeletConfiguration",
},
},
},
},
},
},
},
},
subErr: errors.New("unknown field \"maxPodss\""),
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
tt := newTest(t, withKubectl())

err := validations.ValidateBottlerocketKubeletConfig(tc.spec)
if err != nil {
t.Log(err.Error())
tt.Expect(err.Error()).To(ContainSubstring(tc.subErr.Error()))
} else {
tt.Expect(tc.subErr).To(BeNil())
}
})
}
}
31 changes: 31 additions & 0 deletions pkg/validations/createvalidations/preflightvalidations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"

"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
"github.com/aws/eks-anywhere/pkg/config"
"github.com/aws/eks-anywhere/pkg/constants"
Expand Down Expand Up @@ -51,6 +52,36 @@ func (v *CreateValidations) PreflightValidations(ctx context.Context) []validati
},
}

if len(v.Opts.Spec.VSphereMachineConfigs) != 0 {
cpRef := v.Opts.Spec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name
if v.Opts.Spec.VSphereMachineConfigs[cpRef].Spec.OSFamily == v1alpha1.Bottlerocket {
createValidations = append(createValidations,
func() *validations.ValidationResult {
return &validations.ValidationResult{
Name: "validate cluster's kubelet configuration for Bottlerocket OS",
Remediation: "ensure that the settings configured for Kubelet Configuration are supported by Bottlerocket",
Err: validations.ValidateBottlerocketKubeletConfig(v.Opts.Spec),
}
})
}

wnConfigs := v.Opts.Spec.Cluster.Spec.WorkerNodeGroupConfigurations
for i := range wnConfigs {
workerNodeRef := wnConfigs[i].MachineGroupRef.Name

if v.Opts.Spec.VSphereMachineConfigs[workerNodeRef].Spec.OSFamily == anywherev1.Bottlerocket {
createValidations = append(createValidations,
func() *validations.ValidationResult {
return &validations.ValidationResult{
Name: "validate cluster's worker node kubelet configuration for Bottlerocket OS",
Remediation: "ensure that the settings configured for Kubelet Configuration are supported by Bottlerocket",
Err: validations.ValidateBottlerocketKubeletConfig(v.Opts.Spec),
}
})
}
}
}

if v.Opts.Spec.Cluster.IsManaged() {
createValidations = append(
createValidations,
Expand Down
42 changes: 37 additions & 5 deletions pkg/validations/createvalidations/preflightvalidations_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import (
"github.com/golang/mock/gomock"
. "github.com/onsi/gomega"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/aws/eks-anywhere/internal/test"
"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
"github.com/aws/eks-anywhere/pkg/cluster"
"github.com/aws/eks-anywhere/pkg/constants"
Expand All @@ -34,7 +34,7 @@ func newPreflightValidationsTest(t *testing.T) *preflightValidationsTest {
KubeconfigFile: "kubeconfig",
}
clusterSpec := test.NewClusterSpec(func(s *cluster.Spec) {
s.Cluster.Spec.GitOpsRef = &v1alpha1.Ref{
s.Cluster.Spec.GitOpsRef = &anywherev1.Ref{
Name: "gitops",
}
})
Expand Down Expand Up @@ -69,19 +69,51 @@ func TestPreFlightValidationsWorkloadCluster(t *testing.T) {
tt.c.Opts.ManagementCluster.Name = mgmtClusterName
version := test.DevEksaVersion()

mgmt := &v1alpha1.Cluster{
mgmt := &anywherev1.Cluster{
ObjectMeta: v1.ObjectMeta{
Name: "mgmt-cluster",
},
Spec: v1alpha1.ClusterSpec{
ManagementCluster: v1alpha1.ManagementCluster{
Spec: anywherev1.ClusterSpec{
ManagementCluster: anywherev1.ManagementCluster{
Name: "mgmt-cluster",
},
BundlesRef: &anywherev1.BundlesRef{
Name: "bundles-29",
Namespace: constants.EksaSystemNamespace,
},
EksaVersion: &version,
ControlPlaneConfiguration: anywherev1.ControlPlaneConfiguration{
KubeletConfiguration: &unstructured.Unstructured{
Object: map[string]interface{}{
"staticPodPath": "path",
},
},
},
},
}

tt.c.Opts.Spec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef = &anywherev1.Ref{
Name: "cpRef",
}
tt.c.Opts.Spec.VSphereMachineConfigs = map[string]*anywherev1.VSphereMachineConfig{
"cpRef": {
Spec: anywherev1.VSphereMachineConfigSpec{
OSFamily: anywherev1.Bottlerocket,
},
},
}

tt.c.Opts.Spec.Cluster.Spec.WorkerNodeGroupConfigurations = []anywherev1.WorkerNodeGroupConfiguration{
{
MachineGroupRef: &anywherev1.Ref{
Name: "wnRef",
},
},
}

tt.c.Opts.Spec.VSphereMachineConfigs["wnRef"] = &anywherev1.VSphereMachineConfig{
Spec: anywherev1.VSphereMachineConfigSpec{
OSFamily: anywherev1.Bottlerocket,
},
}

Expand Down
Loading

0 comments on commit 28d6212

Please sign in to comment.