diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 03827cebc93f..10d81377692a 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -97,7 +97,13 @@ jobs: env: KIND_NODE_TAG: ${{ matrix.version }} IMAGE_PULL_POLICY: IfNotPresent - run: make conformance + run: make experimental-conformance + + - name: Upload Conformance Report + uses: actions/upload-artifact@v3 + with: + name: conformance-report-k8s-${{ matrix.version }} + path: ./conformance-report.yaml e2e-test: runs-on: ubuntu-latest diff --git a/test/conformance/experimental_conformance_test.go b/test/conformance/experimental_conformance_test.go new file mode 100644 index 000000000000..a9f91fd5a398 --- /dev/null +++ b/test/conformance/experimental_conformance_test.go @@ -0,0 +1,138 @@ +//go:build experimental +// +build experimental + +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package conformance + +import ( + "os" + "testing" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/yaml" + + "sigs.k8s.io/gateway-api/apis/v1alpha2" + "sigs.k8s.io/gateway-api/apis/v1beta1" + confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1" + "sigs.k8s.io/gateway-api/conformance/tests" + "sigs.k8s.io/gateway-api/conformance/utils/flags" + "sigs.k8s.io/gateway-api/conformance/utils/suite" +) + +var ( + cfg *rest.Config + k8sClientset *kubernetes.Clientset + mgrClient client.Client + supportedFeatures sets.Set[suite.SupportedFeature] + exemptFeatures sets.Set[suite.SupportedFeature] + namespaceLabels map[string]string + implementation *confv1a1.Implementation + conformanceProfiles sets.Set[suite.ConformanceProfileName] + skipTests []string +) + +func TestExperimentalConformance(t *testing.T) { + var err error + cfg, err = config.GetConfig() + if err != nil { + t.Fatalf("Error loading Kubernetes config: %v", err) + } + mgrClient, err = client.New(cfg, client.Options{}) + if err != nil { + t.Fatalf("Error initializing Kubernetes client: %v", err) + } + k8sClientset, err = kubernetes.NewForConfig(cfg) + if err != nil { + t.Fatalf("Error initializing Kubernetes REST client: %v", err) + } + + v1alpha2.AddToScheme(mgrClient.Scheme()) + v1beta1.AddToScheme(mgrClient.Scheme()) + + // standard conformance flags + supportedFeatures = suite.AllFeatures + exemptFeatures = suite.MeshCoreFeatures + for feature := range exemptFeatures { + supportedFeatures.Delete(feature) + } + + skipTests = []string{ + tests.HTTPRouteRedirectPortAndScheme.ShortName, + } + + // experimental conformance flags + conformanceProfiles = suite.ParseConformanceProfiles(*flags.ConformanceProfiles) + + // if some conformance profiles have been set, run the experimental conformance suite... + implementation, err = suite.ParseImplementation( + *flags.ImplementationOrganization, + *flags.ImplementationProject, + *flags.ImplementationUrl, + *flags.ImplementationVersion, + *flags.ImplementationContact, + ) + if err != nil { + t.Fatalf("Error parsing implementation's details: %v", err) + } + + testExperimentalConformance(t) +} + +func testExperimentalConformance(t *testing.T) { + t.Logf("Running experimental conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]\n conformance profiles: [%v]", + *flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, supportedFeatures, exemptFeatures, conformanceProfiles) + + cSuite, err := suite.NewExperimentalConformanceTestSuite( + suite.ExperimentalConformanceOptions{ + Options: suite.Options{ + Client: mgrClient, + RestConfig: cfg, + // This clientset is needed in addition to the client only because + // controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452). + Clientset: k8sClientset, + GatewayClassName: *flags.GatewayClassName, + Debug: *flags.ShowDebug, + CleanupBaseResources: *flags.CleanupBaseResources, + SupportedFeatures: supportedFeatures, + ExemptFeatures: exemptFeatures, + SkipTests: skipTests, + }, + Implementation: *implementation, + ConformanceProfiles: conformanceProfiles, + }) + if err != nil { + t.Fatalf("error creating experimental conformance test suite: %v", err) + } + + cSuite.Setup(t) + cSuite.Run(t, tests.ConformanceTests) + report, err := cSuite.Report() + if err != nil { + t.Fatalf("error generating conformance profile report: %v", err) + } + writeReport(t.Logf, *report, *flags.ReportOutput) +} + +func writeReport(logf func(string, ...any), report confv1a1.ConformanceReport, output string) error { + rawReport, err := yaml.Marshal(report) + if err != nil { + return err + } + + if output != "" { + if err = os.WriteFile(output, rawReport, 0644); err != nil { + return err + } + } + logf("Conformance report:\n%s", string(rawReport)) + + return nil +} diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 64752e3757e4..beca4b9506d3 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -96,6 +96,9 @@ kube-demo-undeploy: ## Uninstall the Kubernetes resources installed from the `ma .PHONY: conformance conformance: create-cluster kube-install-image kube-deploy run-conformance delete-cluster ## Create a kind cluster, deploy EG into it, run Gateway API conformance, and clean up. +.PHONY: experimental-conformance ## Create a kind cluster, deploy EG into it, run Gateway API experimental conformance, and clean up. +experimental-conformance: create-cluster kube-install-image kube-deploy run-experimental-conformance delete-cluster ## Create a kind cluster, deploy EG into it, run Gateway API conformance, and clean up. + .PHONY: e2e e2e: create-cluster kube-install-image kube-deploy install-ratelimit run-e2e delete-cluster @@ -173,6 +176,15 @@ run-conformance: ## Run Gateway API conformance. kubectl apply -f test/config/gatewayclass.yaml go test -v -tags conformance ./test/conformance --gateway-class=envoy-gateway --debug=true +.PHONY: run-experimental-conformance +run-experimental-conformance: ## Run Experimental Gateway API conformance. + @$(LOG_TARGET) + kubectl wait --timeout=$(WAIT_TIMEOUT) -n gateway-system deployment/gateway-api-admission-server --for=condition=Available + kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available + kubectl wait --timeout=$(WAIT_TIMEOUT) -n gateway-system job/gateway-api-admission --for=condition=Complete + kubectl apply -f test/config/gatewayclass.yaml + go test -v -tags experimental ./test/conformance -run TestExperimentalConformance --gateway-class=envoy-gateway --debug=true --organization=envoyproxy --conformance-profiles="HTTP,TLS" --project=envoy-gateway --url=https://github.com/envoyproxy/gateway --version=latest --report-output="conformance-report.yaml" --contact=https://github.com/envoyproxy/gateway/blob/main/GOVERNANCE.md + .PHONY: delete-cluster delete-cluster: $(tools/kind) ## Delete kind cluster. @$(LOG_TARGET)