From 995ae73c56c2e45a865caab9acdfc8198136b6ea Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Thu, 1 Aug 2024 11:20:47 +0200 Subject: [PATCH] Add OCP specific CSI certification tests We use testsuite "openshift/csi" to run test against 3rd party CSI drivers as a way to certify them in OpenShift. So far we used upstream tests there. Add OCP specific tests to the suite, based on a new OCP specific YAML manifest provided by the CSI driver vendor. Usage: TEST_OCP_DRIVER_FILES=azure-disk-ocp.yaml TEST_CSI_DRIVER_FILES=azure-disk-manifest.yaml ./openshift-tests run openshift/csi --- pkg/clioptions/clusterdiscovery/csi.go | 66 ++++++++++++++++++++- test/extended/storage/csi/csi.go | 81 ++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 test/extended/storage/csi/csi.go diff --git a/pkg/clioptions/clusterdiscovery/csi.go b/pkg/clioptions/clusterdiscovery/csi.go index 834e6a2e8b71..970607abf3b0 100644 --- a/pkg/clioptions/clusterdiscovery/csi.go +++ b/pkg/clioptions/clusterdiscovery/csi.go @@ -6,23 +6,51 @@ import ( "path/filepath" "strings" + "github.com/openshift/origin/test/extended/storage/csi" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework/testfiles" "k8s.io/kubernetes/test/e2e/storage/external" + "sigs.k8s.io/yaml" ) const ( CSIManifestEnvVar = "TEST_CSI_DRIVER_FILES" + OCPManifestEnvVar = "TEST_OCP_DRIVER_FILES" ) // Initialize openshift/csi suite, i.e. define CSI tests from TEST_CSI_DRIVER_FILES. func initCSITests(dryRun bool) error { - manifestList := os.Getenv(CSIManifestEnvVar) - if manifestList != "" { - manifests := strings.Split(manifestList, ",") + ocpDrivers := sets.New[string]() + upstreamDrivers := sets.New[string]() + + // Load OCP specific tests first, because AddOpenShiftCSITests() modifies global list of + // testsuites.CSISuites used by AddDriverDefinition() below. + ocpManifestList := os.Getenv(OCPManifestEnvVar) + if ocpManifestList != "" { + manifests := strings.Split(ocpManifestList, ",") + for _, manifest := range manifests { + fmt.Printf("Loading OCP test manifest from %q\n", manifest) + csiDriver, err := csi.AddOpenShiftCSITests(manifest) + if err != nil { + return fmt.Errorf("failed to load OCP manifest from %q: %s", manifest, err) + } + ocpDrivers.Insert(csiDriver) + } + } + + upstreamManifestList := os.Getenv(CSIManifestEnvVar) + if upstreamManifestList != "" { + manifests := strings.Split(upstreamManifestList, ",") for _, manifest := range manifests { if err := external.AddDriverDefinition(manifest); err != nil { return fmt.Errorf("failed to load manifest from %q: %s", manifest, err) } + csiDriver, err := parseDriverName(manifest) + if err != nil { + return fmt.Errorf("failed to parse CSI driver name from manifest %q: %s", manifest, err) + } + upstreamDrivers.Insert(csiDriver) + // Register the base dir of the manifest file as a file source. // With this we can reference the CSI driver's storageClass // in the manifest file (FromFile field). @@ -32,5 +60,37 @@ func initCSITests(dryRun bool) error { } } + // We allow missing OCP specific manifest for CI jobs that do not have it defined yet, + // but all OCP specific manifest must have a corresponding upstream manifest. + if ocpDrivers.Difference(upstreamDrivers).Len() > 0 { + return fmt.Errorf("env. var %s must describe the same CSI drivers as %s: %v vs. %v", OCPManifestEnvVar, CSIManifestEnvVar, ocpDrivers.UnsortedList(), upstreamDrivers.UnsortedList()) + } + return nil } + +func parseDriverName(filename string) (string, error) { + bytes, err := os.ReadFile(filename) + if err != nil { + return "", err + } + + // Minimal chunk of the upstream CSI driver manifest to extract the driver name. + // See vendor/k8s.io/kubernetes/test/e2e/storage/external/external.go for the full definition. + // It's private in that file, so we can't import it here. + type upstreamManifest struct { + DriverInfo struct { + Name string + } + } + manifest := &upstreamManifest{} + err = yaml.Unmarshal(bytes, manifest) + if err != nil { + return "", err + } + if manifest.DriverInfo.Name == "" { + return "", fmt.Errorf("missing driver name in manifest %q", filename) + } + return manifest.DriverInfo.Name, nil + +} diff --git a/test/extended/storage/csi/csi.go b/test/extended/storage/csi/csi.go new file mode 100644 index 000000000000..23c056969f48 --- /dev/null +++ b/test/extended/storage/csi/csi.go @@ -0,0 +1,81 @@ +package csi + +import ( + "fmt" + "os" + "time" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/kubernetes/test/e2e/storage/testsuites" +) + +const ( + // The defaul timeout for the LUN stress test. + DefaultLUNStressTestTimeout = "40m" + // The default nr. of Pods to run in the LUN stress test. + DefaultLUNStressTestPodsTotal = 260 +) + +// OpenShiftCSIDriverConfig holds definition test parameters of OpenShift specific CSI test +type OpenShiftCSIDriverConfig struct { + // Name of the CSI driver. + Driver string + // Configuration of the LUN stress test. If nil, the test is skipped. + LUNStressTest *LUNStressTestConfig +} + +// Definition of the LUN stress test parameters. +// The test runs PodsTotal Pods with a single volume each, targeting the same node. +// All Pods are created at once. We expect that the CSI driver reports correct +// attach capacity and that the scheduler respects it*. +// Each pod does something very simple, like `ls /the/volume` and exits quickly. +// *) We know the scheduler does not respect the attach limit, see +// https://github.com/kubernetes/kubernetes/issues/126502 +type LUNStressTestConfig struct { + // How many Pods with one volume each to run in total. Set to 0 to disable the test. + PodsTotal int + // How long to wait for all Pods to start. 40 minutes by default. + Timeout time.Duration +} + +// runtime.DecodeInto needs a runtime.Object but doesn't do any +// deserialization of it and therefore none of the methods below need +// an implementation. +var _ runtime.Object = &OpenShiftCSIDriverConfig{} + +func (d *OpenShiftCSIDriverConfig) DeepCopyObject() runtime.Object { + return nil +} + +func (d *OpenShiftCSIDriverConfig) GetObjectKind() schema.ObjectKind { + return nil +} + +// Register all OCP specific CSI tests into upstream testsuites.CSISuites. +func AddOpenShiftCSITests(filename string) (string, error) { + bytes, err := os.ReadFile(filename) + if err != nil { + return "", err + } + + // Select sane defaults + cfg := &OpenShiftCSIDriverConfig{ + LUNStressTest: &LUNStressTestConfig{ + PodsTotal: DefaultLUNStressTestPodsTotal, + Timeout: DefaultLUNStressTestTimeout, + }, + } + // No validation here, just like upstream. + if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), bytes, cfg); err != nil { + return "", fmt.Errorf("%s: %w", filename, err) + } + + // Register this OCP specific test suite in the upstream test framework. + // In the end, the test suite will be executed as any other upstream storage test. + // Note: this must be done before external.AddDriverDefinition which actually goes through + // the registered testsuites and generates ginkgo tests for them. + testsuites.CSISuites = append(testsuites.CSISuites, initSCSILUNOverflowCSISuite(cfg.LUNStressTest)) + return cfg.Driver, nil +}