Fix bug when installer does not create CCM secret for Nutanix workloa…

…d cluster (#8191) * Fix bug when installer do not in create CCM secret for Nutanix worker cluster - fixed templates - fixed reconciler - improved tests * Fix linter errors
aws · May 24, 2024 · 230d5f0 · 230d5f0
1 parent e49451a
commit 230d5f0
Show file tree

Hide file tree

Showing 20 changed files with 483 additions and 2 deletions.
diff --git a/pkg/providers/nutanix/config/cp-template.yaml b/pkg/providers/nutanix/config/cp-template.yaml
@@ -781,3 +781,31 @@ spec:
     name: user-ca-bundle
 {{- end }}
   strategy: Reconcile
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "{{.clusterName}}-nutanix-ccm-secret"
+  namespace: "{{.eksaSystemNamespace}}"
+stringData:
+  nutanix-ccm-secret.yaml: |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: nutanix-creds
+      namespace: kube-system
+    stringData:
+      credentials: |-
+        [
+          {        
+            "type": "basic_auth",
+            "data": {
+              "prismCentral": {
+                "username": "{{ .nutanixPCUsername }}",
+                "password": "{{ .nutanixPCPassword }}"
+              },
+              "prismElements": null
+            }
+          }
+        ]
+type: addons.cluster.x-k8s.io/resource-set
diff --git a/pkg/providers/nutanix/controlplane.go b/pkg/providers/nutanix/controlplane.go
@@ -27,13 +27,15 @@ type ControlPlane struct {
 	BaseControlPlane
 	ConfigMaps          []*corev1.ConfigMap
 	ClusterResourceSets []*addonsv1.ClusterResourceSet
+	Secrets             []*corev1.Secret
 }
 
 // Objects returns the control plane objects associated with the Nutanix cluster.
 func (p ControlPlane) Objects() []kubernetes.Object {
 	o := p.BaseControlPlane.Objects()
 	o = appendKubeObjects[*corev1.ConfigMap](o, p.ConfigMaps)
 	o = appendKubeObjects[*addonsv1.ClusterResourceSet](o, p.ClusterResourceSets)
+	o = appendKubeObjects[*corev1.Secret](o, p.Secrets)
 
 	return o
 }
@@ -154,6 +156,12 @@ func newControlPlaneParser(logger logr.Logger) (*yamlutil.Parser, *ControlPlaneB
 				return &addonsv1.ClusterResourceSet{}
 			},
 		),
+		yamlutil.NewMapping(
+			constants.SecretKind,
+			func() yamlutil.APIObject {
+				return &corev1.Secret{}
+			},
+		),
 	)
 
 	if err != nil {
@@ -183,6 +191,8 @@ func buildObjects(cp *ControlPlane, lookup yamlutil.ObjectLookup) {
 			cp.ConfigMaps = append(cp.ConfigMaps, obj.(*corev1.ConfigMap))
 		case constants.ClusterResourceSetKind:
 			cp.ClusterResourceSets = append(cp.ClusterResourceSets, obj.(*addonsv1.ClusterResourceSet))
+		case constants.SecretKind:
+			cp.Secrets = append(cp.Secrets, obj.(*corev1.Secret))
 		}
 	}
 }
diff --git a/pkg/providers/nutanix/reconciler/reconciler.go b/pkg/providers/nutanix/reconciler/reconciler.go
@@ -3,6 +3,7 @@ package reconciler
 import (
 	"context"
 	"fmt"
+	"os"
 	"reflect"
 
 	"github.com/go-logr/logr"
@@ -137,6 +138,7 @@ func (r *Reconciler) reconcileClusterSecret(ctx context.Context, log logr.Logger
 // Reconcile reconciles the cluster to the desired state.
 func (r *Reconciler) Reconcile(ctx context.Context, log logr.Logger, c *anywherev1.Cluster) (controller.Result, error) {
 	log = log.WithValues("provider", "nutanix")
+
 	clusterSpec, err := cluster.BuildSpec(ctx, clientutil.NewKubeClient(r.client), c)
 	if err != nil {
 		return controller.Result{}, err
@@ -182,6 +184,9 @@ func (r *Reconciler) ValidateClusterSpec(ctx context.Context, log logr.Logger, c
 		return controller.ResultWithReturn(), nil
 	}
 
+	os.Setenv(constants.EksaNutanixUsernameKey, creds.PrismCentral.Username)
+	os.Setenv(constants.EksaNutanixPasswordKey, creds.PrismCentral.Password)
+
 	return controller.Result{}, nil
 }
 
@@ -198,13 +203,16 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger,
 }
 
 func toClientControlPlane(cp *nutanix.ControlPlane) *clusters.ControlPlane {
-	other := make([]client.Object, 0, len(cp.ConfigMaps)+len(cp.ClusterResourceSets)+1)
+	other := make([]client.Object, 0, len(cp.ConfigMaps)+len(cp.ClusterResourceSets)+len(cp.Secrets)+1)
 	for _, o := range cp.ClusterResourceSets {
 		other = append(other, o)
 	}
 	for _, o := range cp.ConfigMaps {
 		other = append(other, o)
 	}
+	for _, o := range cp.Secrets {
+		other = append(other, o)
+	}
 
 	return &clusters.ControlPlane{
 		Cluster:                     cp.Cluster,

diff --git a/pkg/providers/nutanix/template.go b/pkg/providers/nutanix/template.go
@@ -59,7 +59,7 @@ func (ntb *TemplateBuilder) GenerateCAPISpecControlPlane(clusterSpec *cluster.Sp
 		etcdMachineSpec = *ntb.etcdMachineSpec
 	}
 
-	values, err := buildTemplateMapCP(ntb.datacenterSpec, clusterSpec, *ntb.controlPlaneMachineSpec, etcdMachineSpec)
+	values, err := buildTemplateMapCP(ntb.datacenterSpec, clusterSpec, *ntb.controlPlaneMachineSpec, etcdMachineSpec, ntb.creds)
 	if err != nil {
 		return nil, err
 	}
@@ -156,6 +156,7 @@ func buildTemplateMapCP(
 	clusterSpec *cluster.Spec,
 	controlPlaneMachineSpec v1alpha1.NutanixMachineConfigSpec,
 	etcdMachineSpec v1alpha1.NutanixMachineConfigSpec,
+	creds credentials.BasicAuthCredential,
 ) (map[string]interface{}, error) {
 	versionsBundle := clusterSpec.RootVersionsBundle()
 	format := "cloud-config"
@@ -218,6 +219,8 @@ func buildTemplateMapCP(
 		"subnetName":                   controlPlaneMachineSpec.Subnet.Name,
 		"subnetUUID":                   controlPlaneMachineSpec.Subnet.UUID,
 		"apiServerCertSANs":            clusterSpec.Cluster.Spec.ControlPlaneConfiguration.CertSANs,
+		"nutanixPCUsername":            creds.PrismCentral.BasicAuth.Username,
+		"nutanixPCPassword":            creds.PrismCentral.BasicAuth.Password,
 	}
 
 	if controlPlaneMachineSpec.Project != nil {

diff --git a/pkg/providers/nutanix/template_test.go b/pkg/providers/nutanix/template_test.go
@@ -549,6 +549,9 @@ func TestTemplateBuilder_CertSANs(t *testing.T) {
 		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
 
 		machineCfg := clusterSpec.NutanixMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)
+
+		t.Setenv(constants.EksaNutanixUsernameKey, "admin")
+		t.Setenv(constants.EksaNutanixPasswordKey, "password")
 		creds := GetCredsFromEnv()
 
 		bldr := NewNutanixTemplateBuilder(&clusterSpec.NutanixDatacenter.Spec, &machineCfg.Spec, nil,
@@ -574,6 +577,9 @@ func TestTemplateBuilder_additionalTrustBundle(t *testing.T) {
 		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
 
 		machineCfg := clusterSpec.NutanixMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)
+
+		t.Setenv(constants.EksaNutanixUsernameKey, "admin")
+		t.Setenv(constants.EksaNutanixPasswordKey, "password")
 		creds := GetCredsFromEnv()
 
 		bldr := NewNutanixTemplateBuilder(&clusterSpec.NutanixDatacenter.Spec, &machineCfg.Spec, nil,
@@ -599,6 +605,9 @@ func TestTemplateBuilderEtcdEncryption(t *testing.T) {
 		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
 
 		machineCfg := clusterSpec.NutanixMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)
+
+		t.Setenv(constants.EksaNutanixUsernameKey, "admin")
+		t.Setenv(constants.EksaNutanixPasswordKey, "password")
 		creds := GetCredsFromEnv()
 
 		bldr := NewNutanixTemplateBuilder(&clusterSpec.NutanixDatacenter.Spec, &machineCfg.Spec, nil,
@@ -624,6 +633,9 @@ func TestTemplateBuilderEtcdEncryptionKubernetes129(t *testing.T) {
 		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
 
 		machineCfg := clusterSpec.NutanixMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)
+
+		t.Setenv(constants.EksaNutanixUsernameKey, "admin")
+		t.Setenv(constants.EksaNutanixPasswordKey, "password")
 		creds := GetCredsFromEnv()
 
 		bldr := NewNutanixTemplateBuilder(&clusterSpec.NutanixDatacenter.Spec, &machineCfg.Spec, nil,

diff --git a/pkg/providers/nutanix/testdata/expected_cluster_api_additional_trust_bundle.yaml b/pkg/providers/nutanix/testdata/expected_cluster_api_additional_trust_bundle.yaml
@@ -641,3 +641,31 @@ spec:
   - kind: ConfigMap
     name: user-ca-bundle
   strategy: Reconcile
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "eksa-unit-test-nutanix-ccm-secret"
+  namespace: "eksa-system"
+stringData:
+  nutanix-ccm-secret.yaml: |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: nutanix-creds
+      namespace: kube-system
+    stringData:
+      credentials: |-
+        [
+          {        
+            "type": "basic_auth",
+            "data": {
+              "prismCentral": {
+                "username": "admin",
+                "password": "password"
+              },
+              "prismElements": null
+            }
+          }
+        ]
+type: addons.cluster.x-k8s.io/resource-set
diff --git a/pkg/providers/nutanix/testdata/expected_cluster_api_server_cert_san_domain_name.yaml b/pkg/providers/nutanix/testdata/expected_cluster_api_server_cert_san_domain_name.yaml
@@ -582,3 +582,31 @@ spec:
   - kind: Secret
     name: test-nutanix-ccm-secret
   strategy: Reconcile
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "test-nutanix-ccm-secret"
+  namespace: "eksa-system"
+stringData:
+  nutanix-ccm-secret.yaml: |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: nutanix-creds
+      namespace: kube-system
+    stringData:
+      credentials: |-
+        [
+          {        
+            "type": "basic_auth",
+            "data": {
+              "prismCentral": {
+                "username": "admin",
+                "password": "password"
+              },
+              "prismElements": null
+            }
+          }
+        ]
+type: addons.cluster.x-k8s.io/resource-set
diff --git a/pkg/providers/nutanix/testdata/expected_cluster_api_server_cert_san_ip.yaml b/pkg/providers/nutanix/testdata/expected_cluster_api_server_cert_san_ip.yaml
@@ -582,3 +582,31 @@ spec:
   - kind: Secret
     name: test-nutanix-ccm-secret
   strategy: Reconcile
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "test-nutanix-ccm-secret"
+  namespace: "eksa-system"
+stringData:
+  nutanix-ccm-secret.yaml: |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: nutanix-creds
+      namespace: kube-system
+    stringData:
+      credentials: |-
+        [
+          {        
+            "type": "basic_auth",
+            "data": {
+              "prismCentral": {
+                "username": "admin",
+                "password": "password"
+              },
+              "prismElements": null
+            }
+          }
+        ]
+type: addons.cluster.x-k8s.io/resource-set
diff --git a/pkg/providers/nutanix/testdata/expected_results_additional_categories.yaml b/pkg/providers/nutanix/testdata/expected_results_additional_categories.yaml
@@ -586,3 +586,31 @@ spec:
   - kind: Secret
     name: eksa-unit-test-nutanix-ccm-secret
   strategy: Reconcile
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "eksa-unit-test-nutanix-ccm-secret"
+  namespace: "eksa-system"
+stringData:
+  nutanix-ccm-secret.yaml: |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: nutanix-creds
+      namespace: kube-system
+    stringData:
+      credentials: |-
+        [
+          {        
+            "type": "basic_auth",
+            "data": {
+              "prismCentral": {
+                "username": "admin",
+                "password": "password"
+              },
+              "prismElements": null
+            }
+          }
+        ]
+type: addons.cluster.x-k8s.io/resource-set
diff --git a/pkg/providers/nutanix/testdata/expected_results_etcd_encryption.yaml b/pkg/providers/nutanix/testdata/expected_results_etcd_encryption.yaml
@@ -631,3 +631,31 @@ spec:
   - kind: Secret
     name: test-nutanix-ccm-secret
   strategy: Reconcile
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "test-nutanix-ccm-secret"
+  namespace: "eksa-system"
+stringData:
+  nutanix-ccm-secret.yaml: |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: nutanix-creds
+      namespace: kube-system
+    stringData:
+      credentials: |-
+        [
+          {        
+            "type": "basic_auth",
+            "data": {
+              "prismCentral": {
+                "username": "admin",
+                "password": "password"
+              },
+              "prismElements": null
+            }
+          }
+        ]
+type: addons.cluster.x-k8s.io/resource-set
diff --git a/pkg/providers/nutanix/testdata/expected_results_etcd_encryption_1_29.yaml b/pkg/providers/nutanix/testdata/expected_results_etcd_encryption_1_29.yaml
@@ -661,3 +661,31 @@ spec:
   - kind: Secret
     name: test-nutanix-ccm-secret
   strategy: Reconcile
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "test-nutanix-ccm-secret"
+  namespace: "eksa-system"
+stringData:
+  nutanix-ccm-secret.yaml: |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: nutanix-creds
+      namespace: kube-system
+    stringData:
+      credentials: |-
+        [
+          {        
+            "type": "basic_auth",
+            "data": {
+              "prismCentral": {
+                "username": "admin",
+                "password": "password"
+              },
+              "prismElements": null
+            }
+          }
+        ]
+type: addons.cluster.x-k8s.io/resource-set