From cd51ad3624d076293e921d02b613f703b6aed809 Mon Sep 17 00:00:00 2001 From: Guy Daich Date: Sun, 14 Jul 2024 09:21:40 -0500 Subject: [PATCH] fix: enable upgrade test (#3764) adapt upgrade test to v1.1 Signed-off-by: Guy Daich Co-authored-by: zirain --- internal/utils/helm/package.go | 5 + internal/utils/helm/package_test.go | 106 ++++++++++ test/e2e/base/manifests.yaml | 70 ------- test/e2e/embed.go | 3 + test/e2e/tests/eg_upgrade.go | 314 ++++++++++++++++++++++------ test/e2e/tests/envoy_shutdown.go | 4 - test/e2e/upgrade/eg_upgrade_test.go | 17 +- test/e2e/upgrade/manifests.yaml | 125 +++++++++++ tools/make/kube.mk | 14 +- 9 files changed, 501 insertions(+), 157 deletions(-) create mode 100644 internal/utils/helm/package_test.go create mode 100644 test/e2e/upgrade/manifests.yaml diff --git a/internal/utils/helm/package.go b/internal/utils/helm/package.go index 4cbf81eff4c..68d769a4bfa 100644 --- a/internal/utils/helm/package.go +++ b/internal/utils/helm/package.go @@ -93,6 +93,10 @@ func (pt *PackageTool) Setup() error { return err } + if pt.logger == nil { + pt.logger = NewPrinterForWriter(os.Stdout, false) + } + if err = pt.actionConfig.Init( kubectlFactory, ns, @@ -350,6 +354,7 @@ func (pt *PackageTool) setUninstallOptions(opts *PackageOptions) { pt.actionUninstall.KeepHistory = false pt.actionUninstall.DryRun = opts.DryRun + pt.actionUninstall.Timeout = opts.Timeout if opts.Wait { pt.actionUninstall.Wait = opts.Wait diff --git a/internal/utils/helm/package_test.go b/internal/utils/helm/package_test.go new file mode 100644 index 00000000000..ad5cf138ea8 --- /dev/null +++ b/internal/utils/helm/package_test.go @@ -0,0 +1,106 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package helm + +import ( + "testing" + "time" + + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/cli/values" +) + +func TestPackageTool_Setup(t *testing.T) { + type fields struct { + chartName string + envSettings *cli.EnvSettings + actionConfig *action.Configuration + actionInstall *action.Install + actionUninstall *action.Uninstall + valuesOpts *values.Options + logger Printer + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "shouldSetup", + fields: fields{ + chartName: "mychart", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pt := &PackageTool{ + chartName: tt.fields.chartName, + envSettings: tt.fields.envSettings, + actionConfig: tt.fields.actionConfig, + actionInstall: tt.fields.actionInstall, + actionUninstall: tt.fields.actionUninstall, + valuesOpts: tt.fields.valuesOpts, + logger: tt.fields.logger, + } + if err := pt.Setup(); (err != nil) != tt.wantErr { + t.Errorf("Setup() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestPackageTool_setInstallOptions(t *testing.T) { + type fields struct { + chartName string + envSettings *cli.EnvSettings + actionConfig *action.Configuration + actionInstall *action.Install + actionUninstall *action.Uninstall + valuesOpts *values.Options + logger Printer + } + type args struct { + opts *PackageOptions + } + tests := []struct { + name string + fields fields + args args + }{ + { + name: "shouldSetInstallOptions", + fields: fields{ + chartName: "mychart", + }, + args: args{ + opts: &PackageOptions{ + Version: "1.0.2", + Timeout: 1 * time.Second, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pt := &PackageTool{ + chartName: tt.fields.chartName, + envSettings: tt.fields.envSettings, + actionConfig: tt.fields.actionConfig, + actionInstall: tt.fields.actionInstall, + actionUninstall: tt.fields.actionUninstall, + valuesOpts: tt.fields.valuesOpts, + logger: tt.fields.logger, + } + if err := pt.Setup(); err != nil { + t.Errorf("Setup() error = %v", err) + } + pt.setInstallOptions(tt.args.opts) + }) + } +} diff --git a/test/e2e/base/manifests.yaml b/test/e2e/base/manifests.yaml index 076240d60ef..2881fde3ebb 100644 --- a/test/e2e/base/manifests.yaml +++ b/test/e2e/base/manifests.yaml @@ -664,26 +664,6 @@ spec: targetPort: 8000 --- apiVersion: v1 -kind: Namespace -metadata: - name: gateway-upgrade-infra ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: ha-gateway - namespace: gateway-upgrade-infra -spec: - gatewayClassName: upgrade - listeners: - - allowedRoutes: - namespaces: - from: Same - name: http1 - port: 80 - protocol: HTTP ---- -apiVersion: v1 data: tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURPVENDQWlHZ0F3SUJBZ0lVUWNxbnZtQXlkRUtuOEdqWTdjZzVDb3A2QWp3d0RRWUpLb1pJaHZjTkFRRUwKQlFBd1JURUxNQWtHQTFVRUJoTUNRVlV4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJRmRwWkdkcGRITWdVSFI1SUV4MFpEQWVGdzB5TkRBMU1USXhOakF3TlROYUZ3MHlOVEExCk1USXhOakF3TlROYU1FVXhDekFKQmdOVkJBWVRBa0ZWTVJNd0VRWURWUVFJREFwVGIyMWxMVk4wWVhSbE1TRXcKSHdZRFZRUUtEQmhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQgpBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRQ2kzUis1WGx3SnlYSTNidTRVQ3E0NXgwSkdWQVBTVXRFTFlLUkxpOEo2CnlxOStySE1hVUtubDhsdldLaHlCNDk4WkJBdVVGS0RpcGhkS1A2eU0rRGl1azVIa2UrK0NmeGxkUDFiSGZiNlkKSGFWczh2cFMyUThneUF6NEZqc3NnNThMV1NKWTdEeEhSOWJibUVWelhSUjNWOEtDeDVaYVlkZ3RxU0NZTGJMTwozaGtGRGQramZxSzM3RHdiT253d21OQ2R0QmpRSTF1TmF2dm1QZzB0c3pwd29TQUtPRitPR0pHcTZHcDdNY0NtClFHZ3dYNkV0YzMwd3hJQTd6c3RnTWwzT293a3p4NHNMcFdJamdCSDVlVk9oYnB6NXROLzB2VFZ3Z3hlbTlOVisKQURjSTFBcnY5M1ZsaFB6VEFmZUNDUlljeFFiNlp4dnBuMWlRbVIrZkVpT0JBZ01CQUFHaklUQWZNQjBHQTFVZApEZ1FXQkJTMGRnRHNtQ3AyU0pZVzNPa3pkNDZtbFNndHZ6QU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFab0NCCnE0M2taV1RZT21QR3JYMU5RMllIVTQ2Y0pzRGxsN2JFL0ZIRUo1eEJEcWRGaUdhWkZBcGRkK3Mra2tkUUw5NUUKcU1SVk9nYS83TUFIL042dlRmb2tXcnVKUUFqaStpLzhGSllWb1VZTWMyeUxqYXp3ZS9ZMHlzTDRWRTNGUlZybApmVHRCTC9nVkhjNk9ZOFBpVFh4eitqdy9FN2kxQkRxZkdSK29sYmt4ZkVmWnhHN0tEZUVtQnVva0dxbDlYQXhSCjMzbnhSbFZuODdxSnJrdUlzdWl2ZzczaVVNMVpGUE1CRVp0OEJjU05MaWhxZEx0b29FVy9mcGZ1am9oaC9yTjUKOFA1ajJpWm9KOGpBS0t4YW5SaWhXTklSNzJtYnJ1R2hYOFRIQkxzczFvZlpLdHBXMzlUOTBTM2hnWkFwSmNZYQp2aGVwSnRtbm9jcHNnYUJiL0E9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQ2kzUis1WGx3SnlYSTMKYnU0VUNxNDV4MEpHVkFQU1V0RUxZS1JMaThKNnlxOStySE1hVUtubDhsdldLaHlCNDk4WkJBdVVGS0RpcGhkSwpQNnlNK0RpdWs1SGtlKytDZnhsZFAxYkhmYjZZSGFWczh2cFMyUThneUF6NEZqc3NnNThMV1NKWTdEeEhSOWJiCm1FVnpYUlIzVjhLQ3g1WmFZZGd0cVNDWUxiTE8zaGtGRGQramZxSzM3RHdiT253d21OQ2R0QmpRSTF1TmF2dm0KUGcwdHN6cHdvU0FLT0YrT0dKR3E2R3A3TWNDbVFHZ3dYNkV0YzMwd3hJQTd6c3RnTWwzT293a3p4NHNMcFdJagpnQkg1ZVZPaGJwejV0Ti8wdlRWd2d4ZW05TlYrQURjSTFBcnY5M1ZsaFB6VEFmZUNDUlljeFFiNlp4dnBuMWlRCm1SK2ZFaU9CQWdNQkFBRUNnZ0VBQkZlOUxUbXNMb3VBWGZTWmdRZStnT0pZbU1pTDZpcG91aWI0Wlk1dFUvM3kKYVZoRXlOVHhuVlRadkoyT2lWc0JzWVNLWUo5TzBaRmFSOUhJSnBHR1BTYzRvOGNYWGpkb2RqUmFrbjdtbDQ2NwprT3JQR0lsQ3ZFb0NVTVdTdkExNXpzMGdmNTZUdmthMjFxL2VHdmtPZ3c3SVBEbVJSNEZpT05nNGt3ODlwRU1FClhNMVROK1NZSGR1eU50RG1wSy90bjFFcUxtOUJUVy9XSk8rMHhLaG1EVStDZnN6Y2hmcS82QitQSVNteEhXUTUKV2JVL3BBNVlvRlM3TWZYbjhMcXVuSG55RGcvNERaQ2NXQkpZZTdZNWtqczVVd2c2MnJBanZxTUZPaWk2ZEgvWgpSQWFWbzlUeEEwZVB4TkZIY2w5M2xuQzRvSGpFV0pvajIzOUdTb3pMWVFLQmdRRGJaNDhNOUpNdkJ2Q2JvSWlXCi9jc3U2ZFRNNmlRTzd3dVJEekhETlpGN0pxWmxOVUhkMU1hZXVYOGhYaE1YeWZpN2FFcGhKMGFZOXNwZ29SdWYKamIyeWVhb3JBbnlOL1VIZ2NjdG5ESWdheEJ0K1JVREZUeE1uTnhrRUVnTnF5WDJCZllOdlNyMmxOVm9PbGhUWQo4VzV6UGJyekNDbUgzSXd6bkhmcW54QVZJUUtCZ1FDK0IwOEVpRERZd1ZKRU9uNyt3eDZyZWlQQklmanBFNUNICjM4ekpYVVBUaWRLZWpORnU4aUlGZmkrdzEvY0VBTTJXcE1xVEtnM2RCSTlzUGZnWXZFSUlvMW5adUFNVnhaY04Kb0k2QXdUckdWMldHSjdQNlNNbjhyWTJhUGRQcDl1ZXE2MFEwZ2p2NVQ1eUliekdPaElmeUpxSXJHYlFvbGdkagp3cXg4K2ZQaVlRS0JnRWJSdklqd0FQb3pBVU1hcER3b200Yi9EeU05aUhvUml1Zzl3VkJEWUR3aUU1K2pleWxCClh3TW8yUEpLVFZ0bVpCVUo2c2hGUnpKa3BwcGVKbTV2OEFWRjVEbVJ5ZWFERXRxQm9LZ1lrVzRpVXNXRlVRemYKSTAyTEtWWDVBb1ZibUZsTnpEa0dKUVRJbmRNTGVwczBBdlRMdmlab1FnK0tqdTZ4Mkxzd3NKNUJBb0dCQUxFcApDUzcxZFd5dkZ1NUxCdGltdWpJdDVhV0o4WkFDVUcyTVpWU1o0Y0VXcmNocENsdi8yMTM1bmFhbVFVRjNLalEyCm9ER0JOSG1JWmRvSkVBS25pSHliSmdwSGRvRFd2SlBVeXVZWXY1M29IdHRxcW0wOWJTcG45eXNFVjB1NWg1UWUKVUhFUHRiQWgyNUtLNzgycG0wQlRhajc2Y0s2aDZIUEdLNTg4UEhZaEFvR0FILzJMS044WnJ3c1R6Nmx6T2c3ZApzdUFuaDVFTUp0TEhTSDJHeFJ5aFcrYVFHdGNxdDZYK1dkNDZnd1BMQjRjd2QzL01nQkFvcFhGazhYV3pTVUlhCnI5SG9SQzZJT2tzQ0lOallCd2h2TjArcm5oN3JzTm5XZVd5Z2tWQ2tERDN5NlNTa2RTZjliOUZzWUJtOHY0VkcKYzFqdmVjWVF6S243QzFRU2FtUnAzRUk9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K @@ -692,56 +672,6 @@ metadata: name: my-tls-secret namespace: envoy-gateway-system type: kubernetes.io/tls ---- -apiVersion: v1 -kind: Service -metadata: - name: infra-backend - namespace: gateway-upgrade-infra -spec: - selector: - app: infra-backend - ports: - - protocol: TCP - port: 8080 - targetPort: 3000 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: infra-backend - namespace: gateway-upgrade-infra - labels: - app: infra-backend -spec: - replicas: 2 - selector: - matchLabels: - app: infra-backend - template: - metadata: - labels: - app: infra-backend - spec: - containers: - - name: infra-backend - # From https://github.com/kubernetes-sigs/gateway-api/blob/main/conformance/echo-basic/echo-basic.go - image: gcr.io/k8s-staging-gateway-api/echo-basic:v20231214-v1.0.0-140-gf544a46e - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SERVICE_NAME - value: infra-backend - resources: - requests: - cpu: 10m - --- apiVersion: gateway.networking.k8s.io/v1beta1 kind: Gateway diff --git a/test/e2e/embed.go b/test/e2e/embed.go index 6b5a891a43f..982f5d24836 100644 --- a/test/e2e/embed.go +++ b/test/e2e/embed.go @@ -25,6 +25,9 @@ import ( //go:embed testdata/*.yaml base/* var Manifests embed.FS +//go:embed testdata/*.yaml upgrade/* +var UpgradeManifests embed.FS + func CheckInstallScheme(t *testing.T, c client.Client) { require.NoError(t, gwapiv1a3.Install(c.Scheme())) require.NoError(t, gwapiv1a2.Install(c.Scheme())) diff --git a/test/e2e/tests/eg_upgrade.go b/test/e2e/tests/eg_upgrade.go index d4128102e73..672cabcc596 100644 --- a/test/e2e/tests/eg_upgrade.go +++ b/test/e2e/tests/eg_upgrade.go @@ -9,43 +9,116 @@ package tests import ( - "net/url" + "bytes" + "context" + "fmt" "os" "testing" "time" "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/registry" + "helm.sh/helm/v3/pkg/kube" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/gateway-api/conformance/utils/http" "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" "sigs.k8s.io/gateway-api/conformance/utils/suite" -) + "sigs.k8s.io/gateway-api/conformance/utils/tlog" + "sigs.k8s.io/gateway-api/pkg/consts" -func init() { - UpgradeTests = append(UpgradeTests, EGUpgradeTest) -} + "github.com/envoyproxy/gateway/internal/cmd/options" + "github.com/envoyproxy/gateway/internal/utils/helm" +) var EGUpgradeTest = suite.ConformanceTest{ ShortName: "EGUpgrade", Description: "Upgrading from the last eg version should not lead to failures", - Manifests: []string{"testdata/eg-upgrade.yaml"}, + Manifests: []string{}, Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { t.Run("Upgrade from an older eg release should succeed", func(t *testing.T) { + chartPath := "../../../charts/gateway-helm" relName := "eg" depNS := "envoy-gateway-system" lastVersionTag := os.Getenv("last_version_tag") if lastVersionTag == "" { - lastVersionTag = "v1.0.0" // Default version tag if not specified + lastVersionTag = "v1.0.2" // Default version tag if not specified + } + + // Uninstall the current version of EG + relNamespace := "envoy-gateway-system" + options.DefaultConfigFlags.Namespace = ptr.To(relNamespace) + + ht := helm.NewPackageTool() + if err := ht.Setup(); err != nil { + t.Errorf("failed to setup of packageTool: %v", err) + } + + // cleanup some resources to avoid finalizer deadlocks when deleting CRDs + t.Log("cleanup test data") + cleanUpResources(suite.Client, t) + + // Uninstall the version deployed for e2e test from the branch + // Make sure to remove all CRDs and CRs, as these may be incompatible with a latestVersion + t.Log("start uninstall envoy gateway resources") + if err := ht.RunUninstall(&helm.PackageOptions{ + ReleaseName: relName, + Wait: true, + Timeout: suite.TimeoutConfig.NamespacesMustBeReady, + }); err != nil { + t.Fatalf("failed to uninstall envoy-gateway: %v", err) + } + + err := deleteChartCRDsFromPath(depNS, chartPath, t, suite.TimeoutConfig.NamespacesMustBeReady) + if err != nil { + t.Fatalf("Failed to delete chart CRDs: %s", err.Error()) + } + + t.Log("success to uninstall envoy gateway resources") + + // Install latest version + if err := ht.RunInstall(&helm.PackageOptions{ + Version: lastVersionTag, + ReleaseName: relName, + ReleaseNamespace: relNamespace, + Wait: true, + Timeout: suite.TimeoutConfig.NamespacesMustBeReady, + }); err != nil { + t.Fatalf("failed to install envoy-gateway: %v", err) + } + + // Apply base and test manifests deleted during uninstall phase; Manifests in current branch must be compatible with the latestVersion + // Since we're applying the GWC now, we must set the controller name, as the suite cannot identify it from the GWC status + suite.ControllerName = "gateway.envoyproxy.io/gatewayclass-controller" + suite.Applier.ControllerName = "gateway.envoyproxy.io/gatewayclass-controller" + suite.Applier.GatewayClass = suite.GatewayClassName + suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, suite.BaseManifests, suite.Cleanup) + + for _, manifestLocation := range []string{"testdata/eg-upgrade.yaml"} { + tlog.Logf(t, "Applying %s", manifestLocation) + suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, true) } + // wait for everything to startup + kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{depNS}) + + // verify latestVersion is working ns := "gateway-upgrade-infra" routeNN := types.NamespacedName{Name: "http-backend-eg-upgrade", Namespace: ns} gwNN := types.NamespacedName{Name: "ha-gateway", Namespace: ns} gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) - reqURL := url.URL{Scheme: "http", Host: http.CalculateHost(t, gwAddr, "http"), Path: "/eg-upgrade"} + kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{depNS}) expectOkResp := http.ExpectedResponse{ Request: http.Request{ @@ -64,32 +137,18 @@ var EGUpgradeTest = suite.ConformanceTest{ t.Errorf("failed to get expected response for the first three requests: %v", err) } - t.Log("Validate route to backend is functional", reqURL.String()) - - // Uninstall the current version of EG - err := helmUninstall(relName, depNS, t) - if err != nil { - t.Fatalf("Failed to upgrade the release: %s", err.Error()) - } - - t.Log("Install the last version tag") - err = helmInstall(relName, depNS, lastVersionTag, suite.TimeoutConfig.NamespacesMustBeReady, t) - if err != nil { - t.Fatalf("Failed to upgrade the release: %s", err.Error()) - } - - // wait for everything to startup - kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{depNS}) - + // Perform helm upgrade of EG t.Log("Attempting to upgrade to current version of eg deployment") - err = helmUpgradeChartFromPath(relName, depNS, "../../../charts/gateway-helm", suite.TimeoutConfig.NamespacesMustBeReady, t) + // TODO: when helm tool supports upgrade-from-source action, use it + err = upgradeChartFromPath(relName, depNS, chartPath, suite.TimeoutConfig.NamespacesMustBeReady, t) if err != nil { t.Fatalf("Failed to upgrade the release: %s", err.Error()) } + // wait for installation kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{depNS}) - t.Log("Confirm routing works after upgrade the eg with current main version") + t.Log("Confirm routing works after upgrading Envoy Gateway with current main version") http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectOkResp) // fire the rest of requests if err := GotExactExpectedResponse(t, 5, suite.RoundTripper, expectOkReq, expectOkResp); err != nil { @@ -99,7 +158,52 @@ var EGUpgradeTest = suite.ConformanceTest{ }, } -func helmUpgradeChartFromPath(relName, relNamespace, chartPath string, timeout time.Duration, t *testing.T) error { +func cleanUpResources(c client.Client, t *testing.T) { + gvks := []schema.GroupVersionKind{ + {Group: "gateway.networking.k8s.io", Version: "v1", Kind: "HTTPRoute"}, + {Group: "gateway.networking.k8s.io", Version: "v1", Kind: "Gateway"}, + {Group: "gateway.networking.k8s.io", Version: "v1", Kind: "GatewayClass"}, + } + for _, gvk := range gvks { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(gvk) + + if err := c.List(context.Background(), list); err != nil { + t.Fatalf("failed fetching %s: %v", obj.GetObjectKind(), err) + } + + for _, o := range list.Items { + t.Logf("deleting %s: %s/%s", o.GetObjectKind(), o.GetNamespace(), o.GetName()) + if err := c.Delete(context.Background(), &o); err != nil { + if !kerrors.IsNotFound(err) { + t.Fatalf("error deleting %s: %s/%s : %v", o.GetObjectKind(), o.GetNamespace(), o.GetName(), err) + } + } + } + + if err := wait.PollUntilContextTimeout(context.TODO(), time.Second, time.Minute, true, + func(ctx context.Context) (bool, error) { + err := c.List(ctx, list) + if err != nil { + return false, nil + } + + if len(list.Items) > 0 { + t.Logf("Waiting for deletion of %d %s", len(list.Items), gvk.String()) + return false, nil + } + + return true, nil + }); err != nil { + t.Fatalf("failed to wait for %s deletion: %v", gvk.String(), err) + } + } +} + +func upgradeChartFromPath(relName, relNamespace, chartPath string, timeout time.Duration, t *testing.T) error { actionConfig := new(action.Configuration) if err := actionConfig.Init(cli.New().RESTClientGetter(), relNamespace, "secret", t.Logf); err != nil { return err @@ -112,73 +216,157 @@ func helmUpgradeChartFromPath(relName, relNamespace, chartPath string, timeout t upgrade.Timeout = timeout // Load the chart from a local directory. - chart, err := loader.Load(chartPath) + gatewayChart, err := loader.Load(chartPath) if err != nil { return err } - // Run the installation. - values := map[string]interface{}{ - "deployment": map[string]interface{}{ - "envoyGateway": map[string]interface{}{ - "imagePullPolicy": "IfNotPresent", - }, - }, + err = migrateChartCRDs(actionConfig, gatewayChart, timeout) + if err != nil { + return err + } + + err = updateChartCRDs(actionConfig, gatewayChart, timeout) + if err != nil { + return err } - _, err = upgrade.Run(relName, chart, values) + + _, err = upgrade.Run(relName, gatewayChart, map[string]interface{}{}) if err != nil { return err } return nil } -func helmInstall(relName, relNamespace string, tag string, timeout time.Duration, t *testing.T) error { - actionConfig := new(action.Configuration) - if err := actionConfig.Init(cli.New().RESTClientGetter(), relNamespace, "secret", t.Logf); err != nil { +func updateChartCRDs(actionConfig *action.Configuration, gatewayChart *chart.Chart, timeout time.Duration) error { + crds, err := extractCRDs(actionConfig, gatewayChart) + if err != nil { return err } - // Set installation options. - install := action.NewInstall(actionConfig) - install.ReleaseName = relName - install.Namespace = relNamespace - install.CreateNamespace = true - install.Version = tag - install.WaitForJobs = true - install.Timeout = timeout - - registryClient, err := registry.NewClient() + // Create all CRDs in the envoy gateway chart + result, err := actionConfig.KubeClient.Update(crds, crds, false) if err != nil { - return err + return fmt.Errorf("failed to create CRDs: %w", err) } - install.SetRegistryClient(registryClient) - // todo we need to explicitly reinstall the CRDs - chartPath, err := install.LocateChart("oci://docker.io/envoyproxy/gateway-helm", cli.New()) + + // We invalidate the cache and let it rebuild the cache, + // at which point no more updated CRDs exist + client, err := actionConfig.RESTClientGetter.ToDiscoveryClient() if err != nil { return err } - // Load the chart from a local directory. - chart, err := loader.Load(chartPath) - if err != nil { + client.Invalidate() + + // Wait the specified amount of time for the resource to be recognized by the cluster + if err := actionConfig.KubeClient.Wait(result.Created, timeout); err != nil { return err } - // Run the installation. - _, err = install.Run(chart, nil) + _, err = client.ServerGroups() + return err +} + +// TODO: proper migration framework required +func migrateChartCRDs(actionConfig *action.Configuration, gatewayChart *chart.Chart, timeout time.Duration) error { + crds, err := extractCRDs(actionConfig, gatewayChart) if err != nil { return err } + + for _, crd := range crds { + if crd.Name == "backendtlspolicies.gateway.networking.k8s.io" { + newVersion, err := getGWAPIVersion(crd.Object) + if err != nil { + return err + } + // https://gateway-api.sigs.k8s.io/guides/?h=upgrade#v11-upgrade-notes + if newVersion == "v1.1.0" { + helper := resource.NewHelper(crd.Client, crd.Mapping) + existingCRD, err := helper.Get(crd.Namespace, crd.Name) + if kerrors.IsNotFound(err) { + continue + } + + // previous version exists + existingVersion, err := getGWAPIVersion(existingCRD) + if err != nil { + return err + } + + if existingVersion == "v1.0.0" { + // Delete the existing instance of the BTLS CRD + _, errs := actionConfig.KubeClient.Delete([]*resource.Info{crd}) + if errs != nil { + return fmt.Errorf("failed to delete backendtlspolicies: %s", util.MultipleErrors("", errs)) + } + + if kubeClient, ok := actionConfig.KubeClient.(kube.InterfaceExt); ok { + if err := kubeClient.WaitForDelete([]*resource.Info{crd}, timeout); err != nil { + return fmt.Errorf("failed to wait for backendtlspolicies deletion: %s", err.Error()) + } + } + } + } + } + } + return nil } -func helmUninstall(relName, relNamespace string, t *testing.T) error { +func deleteChartCRDsFromPath(relNamespace, chartPath string, t *testing.T, timeout time.Duration) error { actionConfig := new(action.Configuration) if err := actionConfig.Init(cli.New().RESTClientGetter(), relNamespace, "secret", t.Logf); err != nil { return err } - uninstall := action.NewUninstall(actionConfig) - _, err := uninstall.Run(relName) // nil can be replaced with any values you wish to override + + // Load the chart from a local directory. + gatewayChart, err := loader.Load(chartPath) + if err != nil { + return err + } + + crds, err := extractCRDs(actionConfig, gatewayChart) if err != nil { return err } + + if _, errors := actionConfig.KubeClient.Delete(crds); len(errors) != 0 { + return fmt.Errorf("failed to delete CRDs error: %s", util.MultipleErrors("", errors)) + } + + if kubeClient, ok := actionConfig.KubeClient.(kube.InterfaceExt); ok { + if err := kubeClient.WaitForDelete(crds, timeout); err != nil { + return fmt.Errorf("failed to wait for crds deletion: %s", err.Error()) + } + } + return nil } + +func getGWAPIVersion(object runtime.Object) (string, error) { + accessor, err := meta.Accessor(object) + if err != nil { + return "", err + } + annotations := accessor.GetAnnotations() + newVersion, ok := annotations[consts.BundleVersionAnnotation] + if ok { + return newVersion, nil + } + return "", fmt.Errorf("failed to determine Gateway API CRD version") +} + +// extractCRDs Extract the CRDs part of the chart +func extractCRDs(config *action.Configuration, ch *chart.Chart) ([]*resource.Info, error) { + crdResInfo := make([]*resource.Info, 0, len(ch.CRDObjects())) + + for _, crd := range ch.CRDObjects() { + resInfo, err := config.KubeClient.Build(bytes.NewBufferString(string(crd.File.Data)), false) + if err != nil { + return nil, err + } + crdResInfo = append(crdResInfo, resInfo...) + } + + return crdResInfo, nil +} diff --git a/test/e2e/tests/envoy_shutdown.go b/test/e2e/tests/envoy_shutdown.go index 7634a29ae94..6b5a35f490a 100644 --- a/test/e2e/tests/envoy_shutdown.go +++ b/test/e2e/tests/envoy_shutdown.go @@ -34,10 +34,6 @@ import ( "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" ) -func init() { - UpgradeTests = append(UpgradeTests, EnvoyShutdownTest) -} - var EnvoyShutdownTest = suite.ConformanceTest{ ShortName: "EnvoyShutdown", Description: "Deleting envoy pod should not lead to failures", diff --git a/test/e2e/upgrade/eg_upgrade_test.go b/test/e2e/upgrade/eg_upgrade_test.go index ec043318915..1db732c5a43 100644 --- a/test/e2e/upgrade/eg_upgrade_test.go +++ b/test/e2e/upgrade/eg_upgrade_test.go @@ -50,19 +50,22 @@ func TestEGUpgrade(t *testing.T) { GatewayClassName: *flags.GatewayClassName, Debug: *flags.ShowDebug, CleanupBaseResources: *flags.CleanupBaseResources, - ManifestFS: []fs.FS{e2e.Manifests}, + ManifestFS: []fs.FS{e2e.UpgradeManifests}, RunTest: *flags.RunTest, - // SupportedFeatures cannot be empty, so we set it to SupportGateway - // All e2e tests should leave Features empty. - SupportedFeatures: sets.New[features.SupportedFeature](features.SupportGateway), - SkipTests: []string{ - tests.EGUpgradeTest.ShortName, // https://github.com/envoyproxy/gateway/issues/3311 - }, + BaseManifests: "upgrade/manifests.yaml", + SupportedFeatures: sets.New[features.SupportedFeature](features.SupportGateway), + SkipTests: []string{}, }) if err != nil { t.Fatalf("Failed to create test suite: %v", err) } + // upgrade tests should be executed in a specific order + tests.UpgradeTests = []suite.ConformanceTest{ + tests.EnvoyShutdownTest, + tests.EGUpgradeTest, + } + t.Logf("Running %d Upgrade tests", len(tests.UpgradeTests)) cSuite.Setup(t, tests.UpgradeTests) diff --git a/test/e2e/upgrade/manifests.yaml b/test/e2e/upgrade/manifests.yaml new file mode 100644 index 00000000000..363ede65779 --- /dev/null +++ b/test/e2e/upgrade/manifests.yaml @@ -0,0 +1,125 @@ +# Adapted from https://github.com/kubernetes-sigs/gateway-api/blob/main/conformance/base/manifests.yaml +# This file contains limited base resources that are required for upgrade tests +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: upgrade +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: upgrade-config + namespace: envoy-gateway-system +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: upgrade-config + namespace: envoy-gateway-system +spec: + provider: + type: Kubernetes + kubernetes: + envoyDeployment: + replicas: 2 + patch: + type: StrategicMerge + value: + spec: + template: + spec: + containers: + - name: envoy + readinessProbe: + initialDelaySeconds: 5 +--- +apiVersion: v1 +kind: Namespace +metadata: + name: gateway-conformance-infra + labels: + gateway-conformance: infra +--- +apiVersion: v1 +kind: Namespace +metadata: + name: gateway-upgrade-infra +--- +apiVersion: v1 +kind: Namespace +metadata: + name: gateway-conformance-app-backend + labels: + gateway-conformance: backend +--- +apiVersion: v1 +kind: Namespace +metadata: + name: gateway-conformance-web-backend + labels: + gateway-conformance: backend +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: ha-gateway + namespace: gateway-upgrade-infra +spec: + gatewayClassName: upgrade + listeners: + - allowedRoutes: + namespaces: + from: Same + name: http1 + port: 80 + protocol: HTTP +--- +apiVersion: v1 +kind: Service +metadata: + name: infra-backend + namespace: gateway-upgrade-infra +spec: + selector: + app: infra-backend + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: infra-backend + namespace: gateway-upgrade-infra + labels: + app: infra-backend +spec: + replicas: 2 + selector: + matchLabels: + app: infra-backend + template: + metadata: + labels: + app: infra-backend + spec: + containers: + - name: infra-backend + # From https://github.com/kubernetes-sigs/gateway-api/blob/main/conformance/echo-basic/echo-basic.go + image: gcr.io/k8s-staging-gateway-api/echo-basic:v20231214-v1.0.0-140-gf544a46e + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: infra-backend + resources: + requests: + cpu: 10m diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 89a11058800..cca4fa89190 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -141,20 +141,8 @@ run-e2e: ## Run e2e tests ifeq ($(E2E_RUN_TEST),) go test -v -tags e2e ./test/e2e --gateway-class=envoy-gateway --debug=true --cleanup-base-resources=false go test -v -tags e2e ./test/e2e/merge_gateways --gateway-class=merge-gateways --debug=true --cleanup-base-resources=false - go test -v -tags e2e ./test/e2e/multiple_gc --debug=true --cleanup-base-resources=false + go test -v -tags e2e ./test/e2e/multiple_gc --debug=true --cleanup-base-resources=true go test -v -tags e2e ./test/e2e/upgrade --gateway-class=upgrade --debug=true --cleanup-base-resources=$(E2E_CLEANUP) -else -ifeq ($(E2E_RUN_EG_UPGRADE_TESTS),false) - go test -v -tags e2e ./test/e2e/merge_gateways --gateway-class=merge-gateways --debug=true --cleanup-base-resources=false \ - --run-test $(E2E_RUN_TEST) - go test -v -tags e2e ./test/e2e --gateway-class=envoy-gateway --debug=true --cleanup-base-resources=$(E2E_CLEANUP) \ - --run-test $(E2E_RUN_TEST) - go test -v -tags e2e ./test/e2e/multiple_gc --debug=true --cleanup-base-resources=$(E2E_CLEANUP) \ - --run-test $(E2E_RUN_TEST) -else - go test -v -tags e2e ./test/e2e/upgrade --gateway-class=upgrade --debug=true --cleanup-base-resources=$(E2E_CLEANUP) \ - --run-test $(E2E_RUN_TEST) -endif endif .PHONY: run-benchmark