diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 2028f3ec659..a7c4e64cebf 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -93,7 +93,7 @@ jobs: chmod +x bin/linux/arm64/envoy-gateway # conformance - - name: Run Conformance Tests + - name: Run Standard Conformance Tests env: KIND_NODE_TAG: ${{ matrix.version }} IMAGE_PULL_POLICY: IfNotPresent diff --git a/.github/workflows/experimental_conformance.yaml b/.github/workflows/experimental_conformance.yaml new file mode 100644 index 00000000000..8ed829e416c --- /dev/null +++ b/.github/workflows/experimental_conformance.yaml @@ -0,0 +1,35 @@ +name: Experimental Conformance Test +on: + push: + paths: + - 'charts/gateway-helm/crds/gatewayapi-crds.yaml' + pull_request: + paths: + - 'charts/gateway-helm/crds/gatewayapi-crds.yaml' + - 'test/conformance/*.go' + # Add workflow_dispatch to trigger this workflow manually by maintainers. + workflow_dispatch: + +jobs: + experimental-conformance-test: + runs-on: ubuntu-latest + strategy: + matrix: + version: [ v1.26.6, v1.27.3, v1.28.0 ] + steps: + - uses: actions/checkout@v3 + - uses: ./tools/github-actions/setup-deps + + # gateway api experimental conformance + - name: Run Experimental Conformance Tests + env: + CONFORMANCE_REPORT_PATH: conformance-report-k8s-${{ matrix.version }}.yaml + KIND_NODE_TAG: ${{ matrix.version }} + IMAGE_PULL_POLICY: IfNotPresent + run: make experimental-conformance + + - name: Upload Conformance Report + uses: actions/upload-artifact@v3 + with: + name: conformance-report-k8s-${{ matrix.version }} + path: ./test/conformance/conformance-report-k8s-${{ matrix.version }}.yaml diff --git a/test/conformance/experimental_conformance_test.go b/test/conformance/experimental_conformance_test.go new file mode 100644 index 00000000000..ef28ce7551e --- /dev/null +++ b/test/conformance/experimental_conformance_test.go @@ -0,0 +1,134 @@ +//go:build experimental +// +build experimental + +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package conformance + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/yaml" + + "sigs.k8s.io/gateway-api/apis/v1alpha2" + "sigs.k8s.io/gateway-api/apis/v1beta1" + confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1" + "sigs.k8s.io/gateway-api/conformance/tests" + "sigs.k8s.io/gateway-api/conformance/utils/flags" + "sigs.k8s.io/gateway-api/conformance/utils/suite" +) + +var ( + cfg *rest.Config + k8sClientset *kubernetes.Clientset + mgrClient client.Client + implementation *confv1a1.Implementation + conformanceProfiles sets.Set[suite.ConformanceProfileName] + skipTests []string +) + +func TestExperimentalConformance(t *testing.T) { + var err error + cfg, err = config.GetConfig() + if err != nil { + t.Fatalf("Error loading Kubernetes config: %v", err) + } + mgrClient, err = client.New(cfg, client.Options{}) + if err != nil { + t.Fatalf("Error initializing Kubernetes client: %v", err) + } + k8sClientset, err = kubernetes.NewForConfig(cfg) + if err != nil { + t.Fatalf("Error initializing Kubernetes REST client: %v", err) + } + + err = v1alpha2.AddToScheme(mgrClient.Scheme()) + assert.NoError(t, err) + err = v1beta1.AddToScheme(mgrClient.Scheme()) + assert.NoError(t, err) + + // experimental conformance flags + conformanceProfiles = sets.New( + suite.HTTPConformanceProfileName, + suite.TLSConformanceProfileName, + ) + + // if some conformance profiles have been set, run the experimental conformance suite... + implementation, err = suite.ParseImplementation( + *flags.ImplementationOrganization, + *flags.ImplementationProject, + *flags.ImplementationURL, + *flags.ImplementationVersion, + *flags.ImplementationContact, + ) + if err != nil { + t.Fatalf("Error parsing implementation's details: %v", err) + } + + experimentalConformance(t) +} + +func experimentalConformance(t *testing.T) { + t.Logf("Running experimental conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n enable all features: %t \n conformance profiles: [%v]", + *flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, conformanceProfiles) + + cSuite, err := suite.NewExperimentalConformanceTestSuite( + suite.ExperimentalConformanceOptions{ + Options: suite.Options{ + Client: mgrClient, + RestConfig: cfg, + // This clientset is needed in addition to the client only because + // controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452). + Clientset: k8sClientset, + GatewayClassName: *flags.GatewayClassName, + Debug: *flags.ShowDebug, + CleanupBaseResources: *flags.CleanupBaseResources, + SkipTests: skipTests, + SupportedFeatures: sets.Set[suite.SupportedFeature]{}.Insert(suite.HTTPRouteExtendedFeatures.UnsortedList()...), + }, + Implementation: *implementation, + ConformanceProfiles: conformanceProfiles, + }) + if err != nil { + t.Fatalf("error creating experimental conformance test suite: %v", err) + } + + cSuite.Setup(t) + err = cSuite.Run(t, tests.ConformanceTests) + if err != nil { + t.Fatalf("error running conformance profile report: %v", err) + } + report, err := cSuite.Report() + if err != nil { + t.Fatalf("error generating conformance profile report: %v", err) + } + + err = experimentalConformanceReport(t.Logf, *report, *flags.ReportOutput) + assert.NoError(t, err) +} + +func experimentalConformanceReport(logf func(string, ...any), report confv1a1.ConformanceReport, output string) error { + rawReport, err := yaml.Marshal(report) + if err != nil { + return err + } + + if output != "" { + if err = os.WriteFile(output, rawReport, 0600); err != nil { + return err + } + } + logf("Conformance report:\n%s", string(rawReport)) + + return nil +} diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 64752e3757e..1f71612fecb 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -96,6 +96,9 @@ kube-demo-undeploy: ## Uninstall the Kubernetes resources installed from the `ma .PHONY: conformance conformance: create-cluster kube-install-image kube-deploy run-conformance delete-cluster ## Create a kind cluster, deploy EG into it, run Gateway API conformance, and clean up. +.PHONY: experimental-conformance ## Create a kind cluster, deploy EG into it, run Gateway API experimental conformance, and clean up. +experimental-conformance: create-cluster kube-install-image kube-deploy run-experimental-conformance delete-cluster ## Create a kind cluster, deploy EG into it, run Gateway API conformance, and clean up. + .PHONY: e2e e2e: create-cluster kube-install-image kube-deploy install-ratelimit run-e2e delete-cluster @@ -173,6 +176,17 @@ run-conformance: ## Run Gateway API conformance. kubectl apply -f test/config/gatewayclass.yaml go test -v -tags conformance ./test/conformance --gateway-class=envoy-gateway --debug=true +CONFORMANCE_REPORT_PATH ?= + +.PHONY: run-experimental-conformance +run-experimental-conformance: ## Run Experimental Gateway API conformance. + @$(LOG_TARGET) + kubectl wait --timeout=$(WAIT_TIMEOUT) -n gateway-system deployment/gateway-api-admission-server --for=condition=Available + kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available + kubectl wait --timeout=$(WAIT_TIMEOUT) -n gateway-system job/gateway-api-admission --for=condition=Complete + kubectl apply -f test/config/gatewayclass.yaml + go test -v -tags experimental ./test/conformance -run TestExperimentalConformance --gateway-class=envoy-gateway --debug=true --organization=envoyproxy --project=envoy-gateway --url=https://github.com/envoyproxy/gateway --version=latest --report-output="$(CONFORMANCE_REPORT_PATH)" --contact=https://github.com/envoyproxy/gateway/blob/main/GOVERNANCE.md + .PHONY: delete-cluster delete-cluster: $(tools/kind) ## Delete kind cluster. @$(LOG_TARGET)