From e0a8c77d0e66debad87c2212324bd4b12a2b8df2 Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Fri, 20 Sep 2024 20:51:26 +0800 Subject: [PATCH] e2e: DuplicatePods: build a descheduler image and run the descheduler as a pod --- test/e2e/e2e_duplicatepods_test.go | 226 ++++++++++++++++++----------- 1 file changed, 140 insertions(+), 86 deletions(-) diff --git a/test/e2e/e2e_duplicatepods_test.go b/test/e2e/e2e_duplicatepods_test.go index 92672de188..a080410fb8 100644 --- a/test/e2e/e2e_duplicatepods_test.go +++ b/test/e2e/e2e_duplicatepods_test.go @@ -21,30 +21,68 @@ import ( "os" "strings" "testing" + "time" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" componentbaseconfig "k8s.io/component-base/config" utilptr "k8s.io/utils/ptr" "sigs.k8s.io/descheduler/pkg/api" + apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2" "sigs.k8s.io/descheduler/pkg/descheduler/client" - eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils" "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/framework/plugins/removeduplicates" - frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing" - frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" ) +func removeDuplicatesPolicy(removeDuplicatesArgs *removeduplicates.RemoveDuplicatesArgs, evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy { + return &apiv1alpha2.DeschedulerPolicy{ + Profiles: []apiv1alpha2.DeschedulerProfile{ + { + Name: removeduplicates.PluginName + "Profile", + PluginConfigs: []apiv1alpha2.PluginConfig{ + { + Name: removeduplicates.PluginName, + Args: runtime.RawExtension{ + Object: removeDuplicatesArgs, + }, + }, + { + Name: defaultevictor.PluginName, + Args: runtime.RawExtension{ + Object: evictorArgs, + }, + }, + }, + Plugins: apiv1alpha2.Plugins{ + Filter: apiv1alpha2.PluginSet{ + Enabled: []string{ + defaultevictor.PluginName, + }, + }, + Balance: apiv1alpha2.PluginSet{ + Enabled: []string{ + removeduplicates.PluginName, + }, + }, + }, + }, + }, + } +} + func TestRemoveDuplicates(t *testing.T) { ctx := context.Background() clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "") if err != nil { - t.Errorf("Error during client creation with %v", err) + t.Errorf("Error during kubernetes client creation with %v", err) } nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) @@ -62,67 +100,33 @@ func TestRemoveDuplicates(t *testing.T) { defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) t.Log("Creating duplicates pods") - - deploymentObj := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "duplicate-pod", - Namespace: testNamespace.Name, - Labels: map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, - }, - Spec: v1.PodSpec{ - SecurityContext: &v1.PodSecurityContext{ - RunAsNonRoot: utilptr.To(true), - RunAsUser: utilptr.To[int64](1000), - RunAsGroup: utilptr.To[int64](1000), - SeccompProfile: &v1.SeccompProfile{ - Type: v1.SeccompProfileTypeRuntimeDefault, - }, - }, - Containers: []v1.Container{{ - Name: "pause", - ImagePullPolicy: "Always", - Image: "registry.k8s.io/pause", - Ports: []v1.ContainerPort{{ContainerPort: 80}}, - SecurityContext: &v1.SecurityContext{ - AllowPrivilegeEscalation: utilptr.To(false), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{ - "ALL", - }, - }, - }, - }}, - }, - }, - }, - } + testLabel := map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"} + deploymentObj := buildTestDeployment("duplicate-pod", testNamespace.Name, 0, testLabel, nil) tests := []struct { - description string + name string replicasNum int beforeFunc func(deployment *appsv1.Deployment) - expectedEvictedPodCount uint - minReplicas uint + expectedEvictedPodCount int + removeDuplicatesArgs *removeduplicates.RemoveDuplicatesArgs + evictorArgs *defaultevictor.DefaultEvictorArgs }{ { - description: "Evict Pod even Pods schedule to specific node", + name: "Evict Pod even Pods schedule to specific node", replicasNum: 4, beforeFunc: func(deployment *appsv1.Deployment) { deployment.Spec.Replicas = utilptr.To[int32](4) deployment.Spec.Template.Spec.NodeName = workerNodes[0].Name }, expectedEvictedPodCount: 2, + removeDuplicatesArgs: &removeduplicates.RemoveDuplicatesArgs{}, + evictorArgs: &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + MinReplicas: 3, + }, }, { - description: "Evict Pod even Pods with local storage", + name: "Evict Pod even Pods with local storage", replicasNum: 5, beforeFunc: func(deployment *appsv1.Deployment) { deployment.Spec.Replicas = utilptr.To[int32](5) @@ -138,19 +142,28 @@ func TestRemoveDuplicates(t *testing.T) { } }, expectedEvictedPodCount: 2, + removeDuplicatesArgs: &removeduplicates.RemoveDuplicatesArgs{}, + evictorArgs: &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + MinReplicas: 3, + }, }, { - description: "Ignores eviction with minReplicas of 4", + name: "Ignores eviction with minReplicas of 4", replicasNum: 3, beforeFunc: func(deployment *appsv1.Deployment) { deployment.Spec.Replicas = utilptr.To[int32](3) }, expectedEvictedPodCount: 0, - minReplicas: 4, + removeDuplicatesArgs: &removeduplicates.RemoveDuplicatesArgs{}, + evictorArgs: &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + MinReplicas: 4, + }, }, } for _, tc := range tests { - t.Run(tc.description, func(t *testing.T) { + t.Run(tc.name, func(t *testing.T) { t.Logf("Creating deployment %v in %v namespace", deploymentObj.Name, deploymentObj.Namespace) tc.beforeFunc(deploymentObj) @@ -158,52 +171,93 @@ func TestRemoveDuplicates(t *testing.T) { if err != nil { t.Logf("Error creating deployment: %v", err) if err = clientSet.AppsV1().Deployments(deploymentObj.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"})).String(), + LabelSelector: labels.SelectorFromSet(deploymentObj.Labels).String(), }); err != nil { t.Fatalf("Unable to delete deployment: %v", err) } return } - defer clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{}) - waitForPodsRunning(ctx, t, clientSet, map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, tc.replicasNum, testNamespace.Name) + defer func() { + clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{}) + waitForPodsToDisappear(ctx, t, clientSet, deploymentObj.Labels, deploymentObj.Namespace) + }() + waitForPodsRunning(ctx, t, clientSet, deploymentObj.Labels, tc.replicasNum, deploymentObj.Namespace) - // Run removeduplicates plugin - evictionPolicyGroupVersion, err := eutils.SupportEviction(clientSet) - if err != nil || len(evictionPolicyGroupVersion) == 0 { - t.Fatalf("Error creating eviction policy group %v", err) + preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...) + + // Deploy the descheduler with the configured policy + tc.removeDuplicatesArgs.Namespaces = &api.Namespaces{ + Include: []string{testNamespace.Name}, + } + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs)) + if err != nil { + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - handle, podEvictor, err := frameworktesting.InitFrameworkHandle( - ctx, - clientSet, - nil, - defaultevictor.DefaultEvictorArgs{ - EvictLocalStoragePods: true, - MinReplicas: tc.minReplicas, - }, - nil, - ) + t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name) + _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) if err != nil { - t.Fatalf("Unable to initialize a framework handle: %v", err) + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - plugin, err := removeduplicates.New(&removeduplicates.RemoveDuplicatesArgs{ - Namespaces: &api.Namespaces{ - Include: []string{testNamespace.Name}, - }, - }, - handle, - ) + defer func() { + t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) + err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } + }() + + deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) + t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) + _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { - t.Fatalf("Unable to initialize the plugin: %v", err) + t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) + } + + deschedulerPodName := "" + defer func() { + if deschedulerPodName != "" { + printPodLogs(ctx, t, clientSet, deschedulerPodName) + } + + t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) + err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) + } + + waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) + }() + + t.Logf("Waiting for the descheduler pod running") + deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) + if len(deschedulerPods) != 0 { + deschedulerPodName = deschedulerPods[0].Name + } + + // Run RemoveDuplicates strategy + var meetsExpectations bool + var actualEvictedPodCount int + if err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + currentRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...) + actualEvictedPod := preRunNames.Difference(currentRunNames) + actualEvictedPodCount = actualEvictedPod.Len() + t.Logf("preRunNames: %v, currentRunNames: %v, actualEvictedPodCount: %v\n", preRunNames.List(), currentRunNames.List(), actualEvictedPodCount) + if actualEvictedPodCount != tc.expectedEvictedPodCount { + t.Logf("Expecting %v number of pods evicted, got %v instead", tc.expectedEvictedPodCount, actualEvictedPodCount) + return false, nil + } + meetsExpectations = true + return true, nil + }); err != nil { + t.Errorf("Error waiting for descheduler running: %v", err) } - t.Log("Running removeduplicates plugin") - plugin.(frameworktypes.BalancePlugin).Balance(ctx, workerNodes) - waitForTerminatingPodsToDisappear(ctx, t, clientSet, testNamespace.Name) - actualEvictedPodCount := podEvictor.TotalEvicted() - if actualEvictedPodCount != tc.expectedEvictedPodCount { - t.Errorf("Test error for description: %s. Unexpected number of pods have been evicted, got %v, expected %v", tc.description, actualEvictedPodCount, tc.expectedEvictedPodCount) + if !meetsExpectations { + t.Errorf("Unexpected number of pods have been evicted, got %v, expected %v", actualEvictedPodCount, tc.expectedEvictedPodCount) + } else { + t.Logf("Total of %d Pods were evicted for %s", actualEvictedPodCount, tc.name) } }) }