Skip to content

Commit

Permalink
Fix credentials delete logic (#301)
Browse files Browse the repository at this point in the history
- check if no delete attempt occured
 - check if no secret present in cluster
 - add integration test case in e2e suite
 - change Makefile test-e2e target to make able to run selected
   tests using GINKGO_FOCUS env variable
 - code grooming
  • Loading branch information
adiantum authored and thunderboltsid committed Apr 30, 2024
1 parent 3534352 commit 3b7d552
Show file tree
Hide file tree
Showing 4 changed files with 255 additions and 3 deletions.
7 changes: 7 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ ifneq ($(LABEL_FILTERS),)
endif
JUNIT_REPORT_FILE ?= "junit.e2e_suite.1.xml"
GINKGO_SKIP ?= "clusterctl-Upgrade"
GINKGO_FOCUS ?= ""
GINKGO_NODES ?= 1
E2E_CONF_FILE ?= ${E2E_DIR}/config/nutanix.yaml
ARTIFACTS ?= ${REPO_ROOT}/_artifacts
Expand All @@ -161,6 +162,11 @@ FLAVOR ?= e2e
TEST_NAMESPACE=capx-test-ns
TEST_CLUSTER_NAME=mycluster

# set ginkgo focus flags, if any
ifneq ($(strip $(GINKGO_FOCUS)),)
_FOCUS_ARGS := $(foreach arg,$(strip $(GINKGO_FOCUS)),--focus="$(arg)")
endif

# to set multiple ginkgo skip flags, if any
ifneq ($(strip $(GINKGO_SKIP)),)
_SKIP_ARGS := $(foreach arg,$(strip $(GINKGO_SKIP)),--skip="$(arg)")
Expand Down Expand Up @@ -376,6 +382,7 @@ test-e2e: docker-build-e2e $(GINKGO_BIN) cluster-e2e-templates cluster-templates
--tags=e2e \
--label-filter=$(LABEL_FILTER_ARGS) \
$(_SKIP_ARGS) \
$(_FOCUS_ARGS) \
--nodes=$(GINKGO_NODES) \
--no-color=$(GINKGO_NOCOLOR) \
--output-dir="$(ARTIFACTS)" \
Expand Down
14 changes: 11 additions & 3 deletions controllers/nutanixcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -316,17 +316,25 @@ func (r *NutanixClusterReconciler) reconcileCredentialRefDelete(ctx context.Cont
}
err = r.Client.Get(ctx, secretKey, secret)
if err != nil {
if errors.IsNotFound(err) {
log.V(1).Info(fmt.Sprintf("Secret %s in namespace %s for cluster %s not found. Ignoring since object must be deleted", secret.Name, secret.Namespace, nutanixCluster.Name))
return nil
}
return err
}
ctrlutil.RemoveFinalizer(secret, infrav1.NutanixClusterCredentialFinalizer)
log.V(1).Info(fmt.Sprintf("removing finalizers from secret %s in namespace %s for cluster %s", secret.Name, secret.Namespace, nutanixCluster.Name))
if err := r.Client.Update(ctx, secret); err != nil {
return err
}
log.Info(fmt.Sprintf("removing secret %s in namespace %s for cluster %s", secret.Name, secret.Namespace, nutanixCluster.Name))
if err := r.Client.Delete(ctx, secret); err != nil {
return err

if secret.DeletionTimestamp.IsZero() {
log.Info(fmt.Sprintf("removing secret %s in namespace %s for cluster %s", secret.Name, secret.Namespace, nutanixCluster.Name))
if err := r.Client.Delete(ctx, secret); err != nil {
return err
}
}

return nil
}

Expand Down
215 changes: 215 additions & 0 deletions test/e2e/capx_regression.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
//go:build e2e
// +build e2e

/*
Copyright 2022 Nutanix
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/test/framework"
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
"sigs.k8s.io/controller-runtime/pkg/client"

infrav1 "github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1beta1"
)

var _ = Describe("Nutanix regression tests", Label("capx-regression-test", "regression", "slow", "network"), func() {
const (
specName = "capx-regression"

controlplaneEndpointIPKey = envVarControlPlaneEndpointIP
controlplaneEndpointPortKey = envVarControlPlaneEndpointPort
defaultControlPlaneEndpointPort = 6443
)

var (
namespace *corev1.Namespace
clusterName string
clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
cancelWatches context.CancelFunc
testHelper testHelperInterface
)

BeforeEach(func() {
Expect(e2eConfig).NotTo(BeNil(), "E2EConfig can't be nil")

testHelper = newTestHelper(e2eConfig)
clusterName = testHelper.generateTestClusterName(specName)
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)

Expect(bootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy can't be nil")
Expect(artifactFolder).NotTo(BeEmpty(), "ArtifactFolder can't be empty")

namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder)

Expect(clusterctlConfigPath).NotTo(BeEmpty(), "ClusterctlConfigPath can't be empty")
})

AfterEach(func() {
bcpClient := bootstrapClusterProxy.GetClient()
myCluster := &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: clusterName,
Namespace: namespace.Name,
},
}

// Delete the cluster
// Note: We are not using the framework.DeleteClusterAndWait function here because we want to
// succeed even if the cluster is already deleted.
// This is because the cluster may have been deleted by the test itself, and we don't want to
// fail the test in that case.
Byf("Deleting cluster %s/%s", namespace.Name, clusterName)
Expect(bcpClient.Delete(ctx, myCluster)).To(SatisfyAny(Succeed(), MatchError(ContainSubstring("not found"))))

Byf("Waiting for cluster %s/%s to be deleted", namespace.Name, clusterName)
framework.WaitForClusterDeleted(ctx, framework.WaitForClusterDeletedInput{
Getter: bcpClient,
Cluster: myCluster,
}, e2eConfig.GetIntervals(specName, "wait-delete-cluster")...)

Byf("Deleting namespace used for hosting the %q test spec", specName)
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
Deleter: bcpClient,
Name: myCluster.Namespace,
})

cancelWatches()
})

It("[Test regression: credentials-delete-race-regression] Create a cluster and delete the credentials secret before delete the cluster", func() {
Expect(namespace).ToNot(BeNil(), "Namespace can't be nil")
Expect(clusterName).ToNot(BeNil(), "ClusterName can't be nil")

By("Create and deploy secret using e2e credentials", func() {
up := getBaseAuthCredentials(*e2eConfig)
testHelper.createSecret(createSecretParams{
username: up.username,
password: up.password,
namespace: namespace,
clusterName: clusterName,
})
})

By("Create and deploy Nutanix cluster", func() {
flavor := clusterctl.DefaultFlavor

deployParams := deployClusterParams{
clusterName: clusterName,
namespace: namespace,
flavor: flavor,
clusterctlConfigPath: clusterctlConfigPath,
artifactFolder: artifactFolder,
bootstrapClusterProxy: bootstrapClusterProxy,
}

testHelper.deployClusterAndWait(deployParams, clusterResources)
})

By("Checking cluster credential condition is true", func() {
testHelper.verifyConditionOnNutanixCluster(verifyConditionParams{
clusterName: clusterName,
namespace: namespace,
bootstrapClusterProxy: bootstrapClusterProxy,
expectedCondition: clusterv1.Condition{
Type: infrav1.CredentialRefSecretOwnerSetCondition,
Status: corev1.ConditionTrue,
},
})
})

By("Checking cluster prism client init condition is true", func() {
testHelper.verifyConditionOnNutanixCluster(verifyConditionParams{
clusterName: clusterName,
namespace: namespace,
bootstrapClusterProxy: bootstrapClusterProxy,
expectedCondition: clusterv1.Condition{
Type: infrav1.PrismCentralClientCondition,
Status: corev1.ConditionTrue,
},
})
})

By("Dumping cluster resources", func() {
dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, clusterResources.Cluster, e2eConfig.GetIntervals, true)
})

By("Deleting the secret", func() {
testHelper.deleteSecret(deleteSecretParams{
namespace: namespace,
clusterName: clusterName,
})
})

By("Deleting the cluster", func() {
Byf("Deleting cluster %s/%s", namespace.Name, clusterName)
framework.DeleteCluster(ctx, framework.DeleteClusterInput{
Deleter: bootstrapClusterProxy.GetClient(),
Cluster: &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: clusterName,
Namespace: namespace.Name,
},
},
})

Byf("Waiting for cluster %s/%s to be deleted", namespace.Name, clusterName)
framework.WaitForClusterDeleted(ctx, framework.WaitForClusterDeletedInput{
Getter: bootstrapClusterProxy.GetClient(),
Cluster: &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: clusterName,
Namespace: namespace.Name,
},
},
}, e2eConfig.GetIntervals(specName, "wait-delete-cluster")...)
})

//Check if secret is deleted
By("Checking if secret is deleted", func() {
err := bootstrapClusterProxy.GetClient().Get(ctx,
client.ObjectKey{
Name: clusterName,
Namespace: namespace.Name,
}, &corev1.Secret{})
Expect(err).To(HaveOccurred())
Expect(apierrors.IsNotFound(err)).To(BeTrue())
})

//Check if cluster is deleted
By("Checking if cluster is deleted", func() {
err := bootstrapClusterProxy.GetClient().Get(ctx,
client.ObjectKey{
Name: clusterName,
Namespace: namespace.Name,
}, &clusterv1.Cluster{})
Expect(err).To(HaveOccurred())
Expect(apierrors.IsNotFound(err)).To(BeTrue())
})

By("PASSED!")
})
})
22 changes: 22 additions & 0 deletions test/e2e/test_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ const (
clusterVarKey = "NUTANIX_PRISM_ELEMENT_CLUSTER_NAME"
subnetVarKey = "NUTANIX_SUBNET_NAME"

envVarControlPlaneEndpointIP = "CONTROL_PLANE_ENDPOINT_IP"
envVarControlPlaneEndpointPort = "CONTROL_PLANE_ENDPOINT_PORT"

nameType = "name"
)

Expand All @@ -80,6 +83,7 @@ type testHelperInterface interface {
createUUIDProjectNMT(ctx context.Context, clusterName, namespace string) *infrav1.NutanixMachineTemplate
deployCluster(params deployClusterParams, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult)
deployClusterAndWait(params deployClusterParams, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult)
deleteSecret(params deleteSecretParams)
findGPU(ctx context.Context, gpuName string) *prismGoClientV3.GPU
generateNMTName(clusterName string) string
generateNMTProviderID(clusterName string) string
Expand Down Expand Up @@ -669,3 +673,21 @@ func (t testHelper) verifyGPUNutanixMachines(ctx context.Context, params verifyG
)))
}
}

type deleteSecretParams struct {
namespace *corev1.Namespace
clusterName string
}

func (t testHelper) deleteSecret(params deleteSecretParams) {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: params.clusterName,
Namespace: params.namespace.Name,
},
}

Eventually(func() error {
return bootstrapClusterProxy.GetClient().Delete(ctx, secret)
}, time.Second*5, defaultInterval).Should(Succeed())
}

0 comments on commit 3b7d552

Please sign in to comment.