diff --git a/internal/testing/mock/mock_client.go b/internal/testing/mock/mock_client.go index bc8664076..e3ec63a01 100644 --- a/internal/testing/mock/mock_client.go +++ b/internal/testing/mock/mock_client.go @@ -18,18 +18,17 @@ package mock import ( "k8s.io/client-go/informers" - coreinformers "k8s.io/client-go/informers/core/v1" "github.com/nutanix-cloud-native/cloud-provider-nutanix/pkg/provider/interfaces" ) +// MockClient is a mock implementation of the interfaces.Client interface type MockClient struct { - mockPrism MockPrism - sharedInformers informers.SharedInformerFactory - secretInformer coreinformers.SecretInformer - configMapInformer coreinformers.ConfigMapInformer + mockPrism MockPrism + sharedInformers informers.SharedInformerFactory } +// CreateMockClient creates a new MockClient func CreateMockClient(mockEnvironment MockEnvironment) *MockClient { return &MockClient{ mockPrism: MockPrism{ @@ -38,10 +37,12 @@ func CreateMockClient(mockEnvironment MockEnvironment) *MockClient { } } +// Get returns the mockPrism func (mc *MockClient) Get() (interfaces.Prism, error) { return &mc.mockPrism, nil } +// SetInformers sets the sharedInformers func (mc *MockClient) SetInformers(sharedInformers informers.SharedInformerFactory) { mc.sharedInformers = sharedInformers } diff --git a/pkg/provider/client.go b/pkg/provider/client.go index 39ede6393..536d455f5 100644 --- a/pkg/provider/client.go +++ b/pkg/provider/client.go @@ -20,17 +20,17 @@ import ( "context" "fmt" - prismgoclient "github.com/nutanix-cloud-native/prism-go-client" "github.com/nutanix-cloud-native/prism-go-client/environment" - credentialTypes "github.com/nutanix-cloud-native/prism-go-client/environment/credentials" - kubernetesEnv "github.com/nutanix-cloud-native/prism-go-client/environment/providers/kubernetes" - envTypes "github.com/nutanix-cloud-native/prism-go-client/environment/types" - prismClientV3 "github.com/nutanix-cloud-native/prism-go-client/v3" + credentialtypes "github.com/nutanix-cloud-native/prism-go-client/environment/credentials" + kubernetesenv "github.com/nutanix-cloud-native/prism-go-client/environment/providers/kubernetes" + envtypes "github.com/nutanix-cloud-native/prism-go-client/environment/types" + prismclientv3 "github.com/nutanix-cloud-native/prism-go-client/v3" "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + "github.com/nutanix-cloud-native/cloud-provider-nutanix/internal/constants" "github.com/nutanix-cloud-native/cloud-provider-nutanix/pkg/provider/config" "github.com/nutanix-cloud-native/cloud-provider-nutanix/pkg/provider/interfaces" ) @@ -38,72 +38,80 @@ import ( const errEnvironmentNotReady = "environment not initialized or ready yet" type nutanixClient struct { - env *envTypes.Environment + env envtypes.Environment config config.Config secretInformer coreinformers.SecretInformer sharedInformers informers.SharedInformerFactory configMapInformer coreinformers.ConfigMapInformer + clientCache *prismclientv3.ClientCache } -func (n *nutanixClient) Get() (interfaces.Prism, error) { - if err := n.setupEnvironment(); err != nil { - return nil, fmt.Errorf("%s: %v", errEnvironmentNotReady, err) +// Key returns the constant client name +// This implements the CachedClientParams interface of prism-go-client +func (n *nutanixClient) Key() string { + return constants.ClientName +} + +// ManagementEndpoint returns the management endpoint of the Nutanix cluster +// This implements the CachedClientParams interface of prism-go-client +func (n *nutanixClient) ManagementEndpoint() envtypes.ManagementEndpoint { + if n.env == nil { + klog.Error("environment not initialized") + return envtypes.ManagementEndpoint{} } - env := *n.env - me, err := env.GetManagementEndpoint(envTypes.Topology{}) + + mgmtEndpoint, err := n.env.GetManagementEndpoint(envtypes.Topology{}) if err != nil { - return nil, err - } - creds := &prismgoclient.Credentials{ - URL: me.Address.Host, // Not really an URL - Endpoint: me.Address.Host, - Insecure: me.Insecure, - Username: me.ApiCredentials.Username, - Password: me.ApiCredentials.Password, + klog.Errorf("failed to get management endpoint: %s", err.Error()) + return envtypes.ManagementEndpoint{} } - clientOpts := make([]prismClientV3.ClientOption, 0) - if me.AdditionalTrustBundle != "" { - clientOpts = append(clientOpts, prismClientV3.WithPEMEncodedCertBundle([]byte(me.AdditionalTrustBundle))) + return *mgmtEndpoint +} + +func (n *nutanixClient) Get() (interfaces.Prism, error) { + if err := n.setupEnvironment(); err != nil { + return nil, fmt.Errorf("%s: %w", errEnvironmentNotReady, err) } - nutanixClient, err := prismClientV3.NewV3Client(*creds, clientOpts...) - if err != nil { - return nil, err + if n.clientCache == nil { + return nil, fmt.Errorf("%s: client cache not initialized", errEnvironmentNotReady) } - _, err = nutanixClient.V3.GetCurrentLoggedInUser(context.Background()) + client, err := n.clientCache.GetOrCreate(n) if err != nil { return nil, err } - return nutanixClient.V3, nil + return client.V3, nil } func (n *nutanixClient) setupEnvironment() error { if n.env != nil { return nil } + ccmNamespace, err := GetCCMNamespace() if err != nil { return err } + pc := n.config.PrismCentral if pc.CredentialRef != nil { if pc.CredentialRef.Namespace == "" { pc.CredentialRef.Namespace = ccmNamespace } } + additionalTrustBundleRef := pc.AdditionalTrustBundle if additionalTrustBundleRef != nil && - additionalTrustBundleRef.Kind == credentialTypes.NutanixTrustBundleKindConfigMap && + additionalTrustBundleRef.Kind == credentialtypes.NutanixTrustBundleKindConfigMap && additionalTrustBundleRef.Namespace == "" { additionalTrustBundleRef.Namespace = ccmNamespace } - env := environment.NewEnvironment(kubernetesEnv.NewProvider(pc, - n.secretInformer, n.configMapInformer)) - n.env = &env + n.env = environment.NewEnvironment(kubernetesenv.NewProvider(pc, n.secretInformer, n.configMapInformer)) + return nil } diff --git a/pkg/provider/client_test.go b/pkg/provider/client_test.go index cb0bddfdd..9856be129 100644 --- a/pkg/provider/client_test.go +++ b/pkg/provider/client_test.go @@ -17,13 +17,18 @@ limitations under the License. package provider import ( + "os" "time" + "github.com/nutanix-cloud-native/prism-go-client/environment/credentials" + "github.com/nutanix-cloud-native/prism-go-client/environment/providers/local" + prismclientv3 "github.com/nutanix-cloud-native/prism-go-client/v3" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" + "github.com/nutanix-cloud-native/cloud-provider-nutanix/internal/constants" "github.com/nutanix-cloud-native/cloud-provider-nutanix/internal/testing/mock" "github.com/nutanix-cloud-native/cloud-provider-nutanix/pkg/provider/config" ) @@ -51,4 +56,145 @@ var _ = Describe("Test Client", func() { Expect(nClient.sharedInformers).ToNot(BeNil()) }) }) + + Context("Test Key", func() { + It("should return the client name", func() { + Expect(nClient.Key()).To(Equal(constants.ClientName)) + }) + }) + + Context("Test ManagementEndpoint", func() { + BeforeEach(func() { + nClient = nutanixClient{ + config: config, + } + }) + + It("should return the empty management endpoint if env is uninitialized", func() { + Expect(nClient.ManagementEndpoint()).To(BeZero()) + }) + + It("should return the empty management endpoint if env isn't properly initialized", func() { + p := local.NewProvider() + nClient.env = p + Expect(nClient.ManagementEndpoint()).To(BeZero()) + }) + + It("should return the management endpoint", func() { + p := local.NewProvider() + err := os.Setenv("NUTANIX_ENDPOINT", "prism.nutanix.com") + defer os.Unsetenv("NUTANIX_ENDPOINT") + Expect(err).To(BeNil()) + nClient.env = p + Expect(nClient.ManagementEndpoint()).ToNot(BeZero()) + }) + }) + + Context("Test Get", func() { + BeforeEach(func() { + nClient = nutanixClient{ + config: config, + } + }) + + It("should return error if env is uninitialized", func() { + client, err := nClient.Get() + Expect(err).ToNot(BeNil()) + Expect(client).To(BeNil()) + }) + + It("should return error if clientCache is uninitialized", func() { + p := local.NewProvider() + err := os.Setenv("NUTANIX_ENDPOINT", "prism.nutanix.com") + Expect(err).To(BeNil()) + defer os.Unsetenv("NUTANIX_ENDPOINT") + nClient.env = p + client, err := nClient.Get() + Expect(err).ToNot(BeNil()) + Expect(client).To(BeNil()) + }) + + It("should return an error when client creation fails", func() { + p := local.NewProvider() + err := os.Setenv("NUTANIX_ENDPOINT", "prism.nutanix.com") + Expect(err).To(BeNil()) + defer os.Unsetenv("NUTANIX_ENDPOINT") + nClient.env = p + nClient.clientCache = prismclientv3.NewClientCache(prismclientv3.WithSessionAuth(false)) + client, err := nClient.Get() + Expect(err).ToNot(BeNil()) + Expect(client).To(BeNil()) + }) + + It("should return a client when client creation succeeds", func() { + p := local.NewProvider() + err := os.Setenv("NUTANIX_ENDPOINT", "prism.nutanix.com") + Expect(err).To(BeNil()) + defer os.Unsetenv("NUTANIX_ENDPOINT") + + err = os.Setenv("NUTANIX_USERNAME", "username") + Expect(err).To(BeNil()) + defer os.Unsetenv("NUTANIX_USERNAME") + + err = os.Setenv("NUTANIX_PASSWORD", "password") + Expect(err).To(BeNil()) + defer os.Unsetenv("NUTANIX_PASSWORD") + + nClient.env = p + nClient.clientCache = prismclientv3.NewClientCache(prismclientv3.WithSessionAuth(false)) + client, err := nClient.Get() + Expect(err).To(BeNil()) + Expect(client).ToNot(BeNil()) + }) + }) + + Context("Test setupEnvironment", func() { + It("should return nil if env is already initialized", func() { + nClient.env = local.NewProvider() + Expect(nClient.setupEnvironment()).To(BeNil()) + }) + + It("should return error if CCM namespace is not set", func() { + err := os.Setenv(constants.CCMNamespaceKey, "") + Expect(err).To(BeNil()) + defer os.Unsetenv(constants.CCMNamespaceKey) + + Expect(nClient.setupEnvironment()).ToNot(BeNil()) + }) + + It("should set the namespace for credential ref if not set", func() { + err := os.Setenv(constants.CCMNamespaceKey, "kube-system") + Expect(err).To(BeNil()) + defer os.Unsetenv(constants.CCMNamespaceKey) + + nClient.config.PrismCentral.CredentialRef.Namespace = "" + defer func() { + nClient.config = mock.GenerateMockConfig() + }() + + err = nClient.setupEnvironment() + Expect(err).To(BeNil()) + Expect(nClient.config.PrismCentral.CredentialRef.Namespace).To(Equal("kube-system")) + }) + + It("should set the namespace for additional trust bundle if not set", func() { + err := os.Setenv(constants.CCMNamespaceKey, "kube-system") + Expect(err).To(BeNil()) + defer os.Unsetenv(constants.CCMNamespaceKey) + + nClient.config = mock.GenerateMockConfig() + nClient.config.PrismCentral.AdditionalTrustBundle = &credentials.NutanixTrustBundleReference{ + Kind: credentials.NutanixTrustBundleKindConfigMap, + Name: "nutanix-trust-bundle", + } + + defer func() { + nClient.config = mock.GenerateMockConfig() + }() + + err = nClient.setupEnvironment() + Expect(err).To(BeNil()) + Expect(nClient.config.PrismCentral.AdditionalTrustBundle.Namespace).To(Equal("kube-system")) + }) + }) }) diff --git a/pkg/provider/manager.go b/pkg/provider/manager.go index 36bb6c18c..c008d67c2 100644 --- a/pkg/provider/manager.go +++ b/pkg/provider/manager.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - prismClientV3 "github.com/nutanix-cloud-native/prism-go-client/v3" + prismclientv3 "github.com/nutanix-cloud-native/prism-go-client/v3" v1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" @@ -45,7 +45,8 @@ func newNutanixManager(config config.Config) (*nutanixManager, error) { m := &nutanixManager{ config: config, nutanixClient: &nutanixClient{ - config: config, + config: config, + clientCache: prismclientv3.NewClientCache(prismclientv3.WithSessionAuth(true)), }, } return m, nil @@ -218,7 +219,7 @@ func (n *nutanixManager) isNodeShutdown(ctx context.Context, node *v1.Node) (boo return false, nil } -func (n *nutanixManager) isVMShutdown(vm *prismClientV3.VMIntentResponse) bool { +func (n *nutanixManager) isVMShutdown(vm *prismclientv3.VMIntentResponse) bool { return *vm.Spec.Resources.PowerState == constants.PoweredOffState } @@ -261,7 +262,7 @@ func (n *nutanixManager) generateProviderID(ctx context.Context, vmUUID string) return fmt.Sprintf("%s://%s", constants.ProviderName, strings.ToLower(vmUUID)), nil } -func (n *nutanixManager) getNodeAddresses(ctx context.Context, vm *prismClientV3.VMIntentResponse) ([]v1.NodeAddress, error) { +func (n *nutanixManager) getNodeAddresses(ctx context.Context, vm *prismclientv3.VMIntentResponse) ([]v1.NodeAddress, error) { if vm == nil { return nil, fmt.Errorf("vm cannot be nil when getting node addresses") } @@ -292,7 +293,7 @@ func (n *nutanixManager) stripNutanixIDFromProviderID(providerID string) string return strings.TrimPrefix(providerID, fmt.Sprintf("%s://", constants.ProviderName)) } -func (n *nutanixManager) getTopologyInfo(ctx context.Context, nutanixClient interfaces.Prism, vm *prismClientV3.VMIntentResponse) (config.TopologyInfo, error) { +func (n *nutanixManager) getTopologyInfo(ctx context.Context, nutanixClient interfaces.Prism, vm *prismclientv3.VMIntentResponse) (config.TopologyInfo, error) { topologyDiscovery := n.config.TopologyDiscovery switch topologyDiscovery.Type { @@ -304,7 +305,7 @@ func (n *nutanixManager) getTopologyInfo(ctx context.Context, nutanixClient inte return config.TopologyInfo{}, fmt.Errorf("unsupported topology discovery type: %s", topologyDiscovery.Type) } -func (n *nutanixManager) getTopologyInfoUsingPrism(ctx context.Context, nClient interfaces.Prism, vm *prismClientV3.VMIntentResponse) (config.TopologyInfo, error) { +func (n *nutanixManager) getTopologyInfoUsingPrism(ctx context.Context, nClient interfaces.Prism, vm *prismclientv3.VMIntentResponse) (config.TopologyInfo, error) { ti := config.TopologyInfo{} if nClient == nil { return ti, fmt.Errorf("nutanix client cannot be nil when searching for Prism topology info") @@ -326,7 +327,7 @@ func (n *nutanixManager) getTopologyInfoUsingPrism(ctx context.Context, nClient return ti, nil } -func (n *nutanixManager) getTopologyInfoUsingCategories(ctx context.Context, nutanixClient interfaces.Prism, vm *prismClientV3.VMIntentResponse) (config.TopologyInfo, error) { +func (n *nutanixManager) getTopologyInfoUsingCategories(ctx context.Context, nutanixClient interfaces.Prism, vm *prismclientv3.VMIntentResponse) (config.TopologyInfo, error) { tc := &config.TopologyInfo{} if vm == nil { return *tc, fmt.Errorf("vm cannot be nil while getting topology info") @@ -369,7 +370,7 @@ func (n *nutanixManager) getZoneInfoFromCategories(categories map[string]string, return nil } -func (n *nutanixManager) getTopologyInfoFromCluster(ctx context.Context, nClient interfaces.Prism, vm *prismClientV3.VMIntentResponse, ti *config.TopologyInfo) error { +func (n *nutanixManager) getTopologyInfoFromCluster(ctx context.Context, nClient interfaces.Prism, vm *prismclientv3.VMIntentResponse, ti *config.TopologyInfo) error { if nClient == nil { return fmt.Errorf("nutanix client cannot be nil when searching for topology info") } @@ -390,7 +391,7 @@ func (n *nutanixManager) getTopologyInfoFromCluster(ctx context.Context, nClient return nil } -func (n *nutanixManager) getTopologyInfoFromVM(vm *prismClientV3.VMIntentResponse, ti *config.TopologyInfo) error { +func (n *nutanixManager) getTopologyInfoFromVM(vm *prismclientv3.VMIntentResponse, ti *config.TopologyInfo) error { if vm == nil { return fmt.Errorf("vm cannot be nil when searching for topology info") } @@ -413,7 +414,7 @@ func (n *nutanixManager) hasEmptyTopologyInfo(ti config.TopologyInfo) bool { return false } -func (n *nutanixManager) getPrismCentralCluster(ctx context.Context, nClient interfaces.Prism) (*prismClientV3.ClusterIntentResponse, error) { +func (n *nutanixManager) getPrismCentralCluster(ctx context.Context, nClient interfaces.Prism) (*prismclientv3.ClusterIntentResponse, error) { const filter = "" if nClient == nil { return nil, fmt.Errorf("nutanix client cannot be nil when getting prism central cluster") @@ -423,7 +424,7 @@ func (n *nutanixManager) getPrismCentralCluster(ctx context.Context, nClient int return nil, err } - foundPCs := make([]*prismClientV3.ClusterIntentResponse, 0) + foundPCs := make([]*prismclientv3.ClusterIntentResponse, 0) for _, s := range responsePEs.Entities { if n.hasPEClusterServiceEnabled(s, constants.PrismCentralService) { foundPCs = append(foundPCs, s) @@ -439,7 +440,7 @@ func (n *nutanixManager) getPrismCentralCluster(ctx context.Context, nClient int return nil, fmt.Errorf("more than one Prism Central cluster ") } -func (n *nutanixManager) hasPEClusterServiceEnabled(peCluster *prismClientV3.ClusterIntentResponse, serviceName string) bool { +func (n *nutanixManager) hasPEClusterServiceEnabled(peCluster *prismclientv3.ClusterIntentResponse, serviceName string) bool { if peCluster.Status == nil || peCluster.Status.Resources == nil || peCluster.Status.Resources.Config == nil { diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 629ebf1d4..d0e59e91d 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -19,7 +19,6 @@ package provider import ( "fmt" "io" - "io/ioutil" clientset "k8s.io/client-go/kubernetes" cloudprovider "k8s.io/cloud-provider" @@ -46,7 +45,7 @@ func init() { } func newNtnxCloud(configReader io.Reader) (cloudprovider.Interface, error) { - bytes, err := ioutil.ReadAll(configReader) + bytes, err := io.ReadAll(configReader) if err != nil { klog.Infof("Error in initializing %s cloudprovid config %q\n", constants.ProviderName, err) return nil, err