diff --git a/api/v1alpha1/envoyextensionypolicy_types.go b/api/v1alpha1/envoyextensionypolicy_types.go index 76e863ce00d..81015567934 100644 --- a/api/v1alpha1/envoyextensionypolicy_types.go +++ b/api/v1alpha1/envoyextensionypolicy_types.go @@ -48,12 +48,14 @@ type EnvoyExtensionPolicySpec struct { // Order matters, as the extensions will be loaded in the order they are // defined in this list. // + // +kubebuilder:validation:MaxItems=16 // +optional Wasm []Wasm `json:"wasm,omitempty"` // ExtProc is an ordered list of external processing filters // that should added to the envoy filter chain // + // +kubebuilder:validation:MaxItems=16 // +optional ExtProc []ExtProc `json:"extProc,omitempty"` } diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index bcec1ff5837..efd04149dbe 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -89,6 +89,7 @@ type EnvoyProxySpec struct { Shutdown *ShutdownConfig `json:"shutdown,omitempty"` // FilterOrder defines the order of filters in the Envoy proxy's HTTP filter chain. + // The FilterPosition in the list will be applied in the order they are defined. // If unspecified, the default filter order is applied. // Default filter order is: // @@ -138,7 +139,7 @@ type BackendTLSConfig struct { // +kubebuilder:validation:XValidation:rule="(has(self.before) && !has(self.after)) || (!has(self.before) && has(self.after))",message="only one of before or after can be specified" type FilterPosition struct { // Name of the filter. - Name EnvoyFilter `json:"filter"` + Name EnvoyFilter `json:"name"` // Before defines the filter that should come before the filter. // Only one of Before or After must be set. diff --git a/api/v1alpha1/validation/envoyproxy_validate.go b/api/v1alpha1/validation/envoyproxy_validate.go index 0e4f7e22221..9c8eee5731a 100644 --- a/api/v1alpha1/validation/envoyproxy_validate.go +++ b/api/v1alpha1/validation/envoyproxy_validate.go @@ -11,6 +11,7 @@ import ( "net" "net/netip" + "github.com/dominikbraun/graph" bootstrapv3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" "github.com/google/go-cmp/cmp" @@ -62,6 +63,13 @@ func validateEnvoyProxySpec(spec *egv1a1.EnvoyProxySpec) error { errs = append(errs, validateProxyTelemetryErrs...) } + // validate filter order + if spec != nil && spec.FilterOrder != nil { + if err := validateFilterOrder(spec.FilterOrder); err != nil { + errs = append(errs, err) + } + } + return utilerrors.NewAggregate(errs) } @@ -269,3 +277,36 @@ func validateProxyAccessLog(accessLog *egv1a1.ProxyAccessLog) []error { return errs } + +func validateFilterOrder(filterOrder []egv1a1.FilterPosition) error { + g := graph.New(graph.StringHash, graph.Directed(), graph.PreventCycles()) + + for _, filter := range filterOrder { + // Ignore the error since the same filter can be added multiple times + _ = g.AddVertex(string(filter.Name)) + if filter.Before != nil { + _ = g.AddVertex(string(*filter.Before)) + } + if filter.After != nil { + _ = g.AddVertex(string(*filter.After)) + } + } + + for _, filter := range filterOrder { + var from, to string + if filter.Before != nil { + from = string(filter.Name) + to = string(*filter.Before) + } else { + from = string(*filter.After) + to = string(filter.Name) + } + if err := g.AddEdge(from, to); err != nil { + if errors.Is(err, graph.ErrEdgeCreatesCycle) { + return fmt.Errorf("there is a cycle in the filter order: %s -> %s", from, to) + } + } + } + + return nil +} diff --git a/api/v1alpha1/validation/envoyproxy_validate_test.go b/api/v1alpha1/validation/envoyproxy_validate_test.go index 6619888bdc8..6e1321eeee7 100644 --- a/api/v1alpha1/validation/envoyproxy_validate_test.go +++ b/api/v1alpha1/validation/envoyproxy_validate_test.go @@ -613,6 +613,54 @@ func TestValidateEnvoyProxy(t *testing.T) { }, expected: true, }, + { + name: "valid filter order", + proxy: &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "test", + }, + Spec: egv1a1.EnvoyProxySpec{ + FilterOrder: []egv1a1.FilterPosition{ + { + Name: egv1a1.EnvoyFilterOAuth2, + Before: ptr.To(egv1a1.EnvoyFilterJWTAuthn), + }, + { + Name: egv1a1.EnvoyFilterExtProc, + After: ptr.To(egv1a1.EnvoyFilterJWTAuthn), + }, + }, + }, + }, + expected: true, + }, + { + name: "invalid filter order with circular dependency", + proxy: &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "test", + }, + Spec: egv1a1.EnvoyProxySpec{ + FilterOrder: []egv1a1.FilterPosition{ + { + Name: egv1a1.EnvoyFilterOAuth2, + Before: ptr.To(egv1a1.EnvoyFilterJWTAuthn), + }, + { + Name: egv1a1.EnvoyFilterJWTAuthn, + Before: ptr.To(egv1a1.EnvoyFilterExtProc), + }, + { + Name: egv1a1.EnvoyFilterExtProc, + Before: ptr.To(egv1a1.EnvoyFilterOAuth2), + }, + }, + }, + }, + expected: false, + }, } for i := range testCases { diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml index f8c0135411b..6e8c04b96b5 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml @@ -189,6 +189,7 @@ spec: required: - backendRefs type: object + maxItems: 16 type: array targetRef: description: |- @@ -398,6 +399,7 @@ spec: - code - name type: object + maxItems: 16 type: array required: - targetRef diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 79328d5b80d..0282aed70d0 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -224,6 +224,7 @@ spec: filterOrder: description: |- FilterOrder defines the order of filters in the Envoy proxy's HTTP filter chain. + The FilterPosition in the list will be applied in the order they are defined. If unspecified, the default filter order is applied. Default filter order is: @@ -295,7 +296,7 @@ spec: - envoy.filters.http.wasm - envoy.filters.http.ext_proc type: string - filter: + name: description: Name of the filter. enum: - envoy.filters.http.cors @@ -310,7 +311,7 @@ spec: - envoy.filters.http.ext_proc type: string required: - - filter + - name type: object x-kubernetes-validations: - message: one of before or after must be specified diff --git a/go.mod b/go.mod index bd953f83fe5..67628322cf9 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b github.com/davecgh/go-spew v1.1.1 + github.com/dominikbraun/graph v0.23.0 github.com/envoyproxy/go-control-plane v0.12.1-0.20240425230418-212e93054f1a github.com/envoyproxy/ratelimit v1.4.1-0.20230427142404-e2a87f41d3a7 github.com/evanphx/json-patch/v5 v5.9.0 diff --git a/go.sum b/go.sum index e831080cc91..baaa24aa925 100644 --- a/go.sum +++ b/go.sum @@ -161,6 +161,8 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arX github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo= +github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= diff --git a/internal/cmd/egctl/translate.go b/internal/cmd/egctl/translate.go index 9d21b20105e..056dc47ac54 100644 --- a/internal/cmd/egctl/translate.go +++ b/internal/cmd/egctl/translate.go @@ -362,6 +362,9 @@ func translateGatewayAPIToXds(dnsDomain string, resourceType string, resources * ServiceURL: ratelimit.GetServiceURL("envoy-gateway", dnsDomain), }, } + if resources.EnvoyProxy != nil { + xTranslator.FilterOrder = resources.EnvoyProxy.Spec.FilterOrder + } xRes, err := xTranslator.Translate(val) if err != nil { return nil, fmt.Errorf("failed to translate xds ir for key %s value %+v, error:%w", key, val, err) diff --git a/internal/gatewayapi/testdata/custom-filter-order.in.yaml b/internal/gatewayapi/testdata/custom-filter-order.in.yaml new file mode 100644 index 00000000000..d0a4bedc2e1 --- /dev/null +++ b/internal/gatewayapi/testdata/custom-filter-order.in.yaml @@ -0,0 +1,130 @@ +secrets: + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: users-secret1 + data: + .htpasswd: "dXNlcjE6e1NIQX10RVNzQm1FL3lOWTNsYjZhMEw2dlZRRVpOcXc9CnVzZXIyOntTSEF9RUo5TFBGRFhzTjl5blNtYnh2anA3NUJtbHg4PQo=" +envoyproxy: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: custom-proxy-config + namespace: envoy-gateway-system + spec: + filterOrder: + - name: envoy.filters.http.wasm + before: envoy.filters.http.jwt_authn + - name: envoy.filters.http.cors + after: envoy.filters.http.basic_authn +gateways: + - apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: envoy-gateway + name: httproute-1 + spec: + hostnames: + - www.example.com + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - matches: + - path: + value: "/foo" + backendRefs: + - name: service-1 + port: 8080 +securityPolicies: + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: SecurityPolicy + metadata: + namespace: envoy-gateway + name: policy-for-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + cors: + allowOrigins: + - "https://*.test.com:8080" + - "https://www.test.org:8080" + allowMethods: + - GET + - POST + basicAuth: + users: + name: "users-secret1" + jwt: + providers: + - name: example1 + issuer: https://one.example.com + audiences: + - one.foo.com + remoteJWKS: + uri: https://one.example.com/jwt/public-key/jwks.json + claimToHeaders: + - header: one-route-example-key + claim: claim1 + - name: example2 + issuer: http://two.example.com + audiences: + - two.foo.com + remoteJWKS: + uri: http://two.example.com/jwt/public-key/jwks.json + claimToHeaders: + - header: two-route-example-key + claim: claim2 +envoyextensionpolicies: + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyExtensionPolicy + metadata: + namespace: envoy-gateway + name: policy-for-gateway # This policy should attach httproute-2 + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + wasm: + - name: wasm-filter-1 + code: + type: HTTP + http: + url: https://www.example.com/wasm-filter-1.wasm + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 + config: + parameter1: + key1: value1 + key2: value2 + parameter2: value3 + - name: wasm-filter-2 + code: + type: HTTP + http: + url: https://www.example.com/wasm-filter-2.wasm + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 + config: + parameter1: value1 + parameter2: value2 diff --git a/internal/gatewayapi/testdata/custom-filter-order.out.yaml b/internal/gatewayapi/testdata/custom-filter-order.out.yaml new file mode 100755 index 00000000000..eea919c09fe --- /dev/null +++ b/internal/gatewayapi/testdata/custom-filter-order.out.yaml @@ -0,0 +1,307 @@ +envoyExtensionPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyExtensionPolicy + metadata: + creationTimestamp: null + name: policy-for-gateway + namespace: envoy-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + wasm: + - code: + http: + url: https://www.example.com/wasm-filter-1.wasm + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 + type: HTTP + config: + parameter1: + key1: value1 + key2: value2 + parameter2: value3 + name: wasm-filter-1 + - code: + http: + url: https://www.example.com/wasm-filter-2.wasm + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 + type: HTTP + config: + parameter1: value1 + parameter2: value2 + name: wasm-filter-2 + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: Policy has been accepted. + reason: Accepted + status: "True" + type: Accepted + controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: envoy-gateway + spec: + hostnames: + - www.example.com + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + matches: + - path: + value: /foo + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Service envoy-gateway/service-1 not found + reason: BackendNotFound + status: "False" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + config: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + creationTimestamp: null + name: custom-proxy-config + namespace: envoy-gateway-system + spec: + filterOrder: + - before: envoy.filters.http.jwt_authn + name: envoy.filters.http.wasm + - after: envoy.filters.http.basic_authn + name: envoy.filters.http.cors + logging: {} + status: {} + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +securityPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: SecurityPolicy + metadata: + creationTimestamp: null + name: policy-for-gateway + namespace: envoy-gateway + spec: + basicAuth: + users: + group: null + kind: null + name: users-secret1 + cors: + allowMethods: + - GET + - POST + allowOrigins: + - https://*.test.com:8080 + - https://www.test.org:8080 + jwt: + providers: + - audiences: + - one.foo.com + claimToHeaders: + - claim: claim1 + header: one-route-example-key + issuer: https://one.example.com + name: example1 + remoteJWKS: + uri: https://one.example.com/jwt/public-key/jwks.json + - audiences: + - two.foo.com + claimToHeaders: + - claim: claim2 + header: two-route-example-key + issuer: http://two.example.com + name: example2 + remoteJWKS: + uri: http://two.example.com/jwt/public-key/jwks.json + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: Policy has been accepted. + reason: Accepted + status: "True" + type: Accepted + controllerName: gateway.envoyproxy.io/gatewayclass-controller +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + filterOrder: + - before: envoy.filters.http.jwt_authn + name: envoy.filters.http.wasm + - after: envoy.filters.http.basic_authn + name: envoy.filters.http.cors + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - backendWeights: + invalid: 1 + valid: 0 + directResponse: + statusCode: 500 + hostname: www.example.com + isHTTP2: false + name: httproute/envoy-gateway/httproute-1/rule/0/match/0/www_example_com + pathMatch: + distinct: false + name: "" + prefix: /foo + security: + basicAuth: + name: securitypolicy/envoy-gateway/policy-for-gateway + users: dXNlcjE6e1NIQX10RVNzQm1FL3lOWTNsYjZhMEw2dlZRRVpOcXc9CnVzZXIyOntTSEF9RUo5TFBGRFhzTjl5blNtYnh2anA3NUJtbHg4PQo= + cors: + allowMethods: + - GET + - POST + allowOrigins: + - distinct: false + name: "" + safeRegex: https://.*\.test\.com:8080 + - distinct: false + exact: https://www.test.org:8080 + name: "" + jwt: + providers: + - audiences: + - one.foo.com + claimToHeaders: + - claim: claim1 + header: one-route-example-key + issuer: https://one.example.com + name: example1 + remoteJWKS: + uri: https://one.example.com/jwt/public-key/jwks.json + - audiences: + - two.foo.com + claimToHeaders: + - claim: claim2 + header: two-route-example-key + issuer: http://two.example.com + name: example2 + remoteJWKS: + uri: http://two.example.com/jwt/public-key/jwks.json + wasm: + - config: + parameter1: + key1: value1 + key2: value2 + parameter2: value3 + failOpen: false + httpWasmCode: + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 + url: https://www.example.com/wasm-filter-1.wasm + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + wasmName: wasm-filter-1 + - config: + parameter1: value1 + parameter2: value2 + failOpen: false + httpWasmCode: + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 + url: https://www.example.com/wasm-filter-2.wasm + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + wasmName: wasm-filter-2 diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 1827d962d57..c0764e583ab 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -217,6 +217,15 @@ func (t *Translator) Translate(resources *Resources) *TranslateResult { // Sort xdsIR based on the Gateway API spec sortXdsIRMap(xdsIR) + // Set custom filter order if EnvoyProxy is set + // The custom filter order will be applied when generating the HTTP filter chain. + if resources.EnvoyProxy != nil { + for _, gateway := range gateways { + irKey := t.getIRKey(gateway.Gateway) + xdsIR[irKey].FilterOrder = resources.EnvoyProxy.Spec.FilterOrder + } + } + return newTranslateResult(gateways, httpRoutes, grpcRoutes, tlsRoutes, tcpRoutes, udpRoutes, clientTrafficPolicies, backendTrafficPolicies, securityPolicies, resources.BackendTLSPolicies, envoyExtensionPolicies, diff --git a/internal/ir/xds.go b/internal/ir/xds.go index 337720f7fb0..a874b773da4 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -84,6 +84,8 @@ type Xds struct { UDP []*UDPListener `json:"udp,omitempty" yaml:"udp,omitempty"` // EnvoyPatchPolicies is the intermediate representation of the EnvoyPatchPolicy resource EnvoyPatchPolicies []*EnvoyPatchPolicy `json:"envoyPatchPolicies,omitempty" yaml:"envoyPatchPolicies,omitempty"` + // FilterOrder holds the custom order of the HTTP filters + FilterOrder []egv1a1.FilterPosition `json:"filterOrder,omitempty" yaml:"filterOrder,omitempty"` } // Equal implements the Comparable interface used by watchable.DeepEqual to skip unnecessary updates. diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index 5430ef9ce6b..9f911844bcf 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -2554,6 +2554,13 @@ func (in *Xds) DeepCopyInto(out *Xds) { } } } + if in.FilterOrder != nil { + in, out := &in.FilterOrder, &out.FilterOrder + *out = make([]v1alpha1.FilterPosition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Xds. diff --git a/internal/xds/translator/httpfilters.go b/internal/xds/translator/httpfilters.go index 0b8f6dbeff9..5738d621eb8 100644 --- a/internal/xds/translator/httpfilters.go +++ b/internal/xds/translator/httpfilters.go @@ -6,7 +6,10 @@ package translator import ( + "container/list" + "fmt" "sort" + "strconv" "strings" routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" @@ -14,6 +17,7 @@ import ( "github.com/envoyproxy/go-control-plane/pkg/wellknown" "k8s.io/utils/ptr" + "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/xds/filters" "github.com/envoyproxy/gateway/internal/xds/types" @@ -91,28 +95,28 @@ func newOrderedHTTPFilter(filter *hcmv3.HttpFilter) *OrderedHTTPFilter { // When the fault filter is configured to be at the first, the computation of // the remaining filters is skipped when rejected early switch { - case filter.Name == wellknown.Fault: + case isFilterType(filter, wellknown.Fault): order = 1 - case filter.Name == wellknown.CORS: + case isFilterType(filter, wellknown.CORS): order = 2 case isFilterType(filter, extAuthFilter): order = 3 - case filter.Name == basicAuthFilter: + case isFilterType(filter, basicAuthFilter): order = 4 case isFilterType(filter, oauth2Filter): order = 5 - case filter.Name == jwtAuthn: + case isFilterType(filter, jwtAuthn): order = 6 case isFilterType(filter, extProcFilter): - order = 7 + order = 7 + mustGetFilterIndex(filter.Name) case isFilterType(filter, wasmFilter): - order = 8 - case filter.Name == localRateLimitFilter: - order = 9 - case filter.Name == wellknown.HTTPRateLimit: - order = 10 - case filter.Name == wellknown.Router: - order = 100 + order = 100 + mustGetFilterIndex(filter.Name) + case isFilterType(filter, localRateLimitFilter): + order = 201 + case isFilterType(filter, wellknown.HTTPRateLimit): + order = 202 + case isFilterType(filter, wellknown.Router): + order = 203 } return &OrderedHTTPFilter{ @@ -140,16 +144,87 @@ func (o OrderedHTTPFilters) Swap(i, j int) { // For example, the cors filter should be put at the first to avoid unnecessary // processing of other filters for unauthorized cross-region access. // The router filter must be the last one since it's a terminal filter. -func sortHTTPFilters(filters []*hcmv3.HttpFilter) []*hcmv3.HttpFilter { +func sortHTTPFilters(filters []*hcmv3.HttpFilter, filterOrder []v1alpha1.FilterPosition) []*hcmv3.HttpFilter { + // Sort the filters in the default order. orderedFilters := make(OrderedHTTPFilters, len(filters)) for i := 0; i < len(filters); i++ { orderedFilters[i] = newOrderedHTTPFilter(filters[i]) } sort.Sort(orderedFilters) - for i := 0; i < len(filters); i++ { - filters[i] = orderedFilters[i].filter + // Use a linked list to sort the filters in the custom order. + l := list.New() + for i := 0; i < len(orderedFilters); i++ { + l.PushBack(orderedFilters[i].filter) } + + // Sort the filters in the custom order. + for i := 0; i < len(filterOrder); i++ { + var ( + // The filter name in the filterOrder is the filter type. + // For example, "envoy.filters.http.oauth2". + filterType = string(filterOrder[i].Name) + // currentFilters holds all the filters of the specified filter type + // in the custom FilterOrder that we are currently processing. + // + // We need an array to store the filters because there may be multiple + // filters of the same filter type for a specific HTTPRoute. + // For example, there may be multiple wasm filters or extProc filters, for + // different custom extensions. + currentFilters []*list.Element + ) + + // Find all the filters for the current filter type in the custom FilterOrder. + // + // The real filter name is a generated name prefixed with the filter type, + // for example,"envoy.filters.http.oauth2/securitypolicy/default/policy-for-http-route-1". + for element := l.Front(); element != nil; element = element.Next() { + if isFilterType(element.Value.(*hcmv3.HttpFilter), filterType) { + currentFilters = append(currentFilters, element) + } + } + + // Skip if there are no filters found for the filter type in a custom order. + if len(currentFilters) == 0 { + continue + } + + switch { + // Move all the current filters before the first filter of the filter type + // specified in the `FilterOrder.Before` field. + case filterOrder[i].Before != nil: + for element := l.Front(); element != nil; element = element.Next() { + if isFilterType(element.Value.(*hcmv3.HttpFilter), string(*filterOrder[i].Before)) { + for _, filter := range currentFilters { + l.MoveBefore(filter, element) + } + break + } + } + // Move all the current filters after the last filter of the filter type + // specified in the `FilterOrder.After` field. + case filterOrder[i].After != nil: + var afterFilter *list.Element + for element := l.Front(); element != nil; element = element.Next() { + if isFilterType(element.Value.(*hcmv3.HttpFilter), string(*filterOrder[i].After)) { + afterFilter = element + } + } + if afterFilter != nil { + for i := range currentFilters { + l.MoveAfter(currentFilters[len(currentFilters)-1-i], afterFilter) + } + } + } + } + + // Collect the sorted filters. + i := 0 + for element := l.Front(); element != nil; element = element.Next() { + filters[i] = element.Value.(*hcmv3.HttpFilter) + i++ + } + return filters } @@ -190,7 +265,7 @@ func (t *Translator) patchHCMWithFilters( } // Sort the filters in the correct order. - mgr.HttpFilters = sortHTTPFilters(mgr.HttpFilters) + mgr.HttpFilters = sortHTTPFilters(mgr.HttpFilters, t.FilterOrder) return nil } @@ -223,6 +298,16 @@ func isFilterType(filter *hcmv3.HttpFilter, filterType string) bool { return strings.HasPrefix(filter.Name, filterType) } +// mustGetFilterIndex returns the index of the filter in its filter type. +func mustGetFilterIndex(filterName string) int { + a := strings.Split(filterName, "/") + index, err := strconv.Atoi(a[len(a)-1]) + if err != nil { + panic(fmt.Errorf("cannot get filter index from %s :%w", filterName, err)) + } + return index +} + // patchResources adds all the other needed resources referenced by this // filter to the resource version table. // for example: diff --git a/internal/xds/translator/httpfilters_test.go b/internal/xds/translator/httpfilters_test.go index 90773ce6a36..2f277aa969a 100644 --- a/internal/xds/translator/httpfilters_test.go +++ b/internal/xds/translator/httpfilters_test.go @@ -11,13 +11,17 @@ import ( hcmv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" "github.com/envoyproxy/go-control-plane/pkg/wellknown" "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" + + "github.com/envoyproxy/gateway/api/v1alpha1" ) func Test_sortHTTPFilters(t *testing.T) { tests := []struct { - name string - filters []*hcmv3.HttpFilter - want []*hcmv3.HttpFilter + name string + filters []*hcmv3.HttpFilter + filterOrder []v1alpha1.FilterPosition + want []*hcmv3.HttpFilter }{ { name: "sort filters", @@ -25,33 +29,384 @@ func Test_sortHTTPFilters(t *testing.T) { httpFilterForTest(wellknown.Router), httpFilterForTest(wellknown.CORS), httpFilterForTest(jwtAuthn), - httpFilterForTest(oauth2Filter + "-route1"), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wellknown.Fault), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + }, + want: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Fault), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(jwtAuthn), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(wellknown.Router), + }, + }, + { + name: "custom filter order-singleton filter", + filters: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Router), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(jwtAuthn), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), httpFilterForTest(basicAuthFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), httpFilterForTest(wellknown.Fault), - httpFilterForTest(extAuthFilter + "-route1"), - httpFilterForTest(wasmFilter + "-route1"), - httpFilterForTest(extProcFilter + "-route1"), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + }, + filterOrder: []v1alpha1.FilterPosition{ + { + Name: v1alpha1.EnvoyFilterFault, + After: ptr.To(v1alpha1.EnvoyFilterCORS), + }, + { + Name: v1alpha1.EnvoyFilterRateLimit, + Before: ptr.To(v1alpha1.EnvoyFilterJWTAuthn), + }, + }, + want: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.CORS), + httpFilterForTest(wellknown.Fault), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(jwtAuthn), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wellknown.Router), + }, + }, + { + name: "custom filter order-singleton-before-multipleton", + filters: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Router), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(jwtAuthn), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wellknown.Fault), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + }, + filterOrder: []v1alpha1.FilterPosition{ + { + Name: v1alpha1.EnvoyFilterRateLimit, + Before: ptr.To(v1alpha1.EnvoyFilterWasm), + }, + }, + want: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Fault), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(jwtAuthn), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wellknown.Router), + }, + }, + { + name: "custom filter order-singleton-after-multipleton", + filters: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Router), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(jwtAuthn), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wellknown.Fault), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + }, + filterOrder: []v1alpha1.FilterPosition{ + { + Name: v1alpha1.EnvoyFilterJWTAuthn, + After: ptr.To(v1alpha1.EnvoyFilterWasm), + }, + }, + want: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Fault), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(jwtAuthn), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(wellknown.Router), + }, + }, + { + name: "custom filter order-multipleton-before-singleton", + filters: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Router), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(jwtAuthn), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wellknown.Fault), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + }, + filterOrder: []v1alpha1.FilterPosition{ + { + Name: v1alpha1.EnvoyFilterWasm, + Before: ptr.To(v1alpha1.EnvoyFilterJWTAuthn), + }, }, want: []*hcmv3.HttpFilter{ httpFilterForTest(wellknown.Fault), httpFilterForTest(wellknown.CORS), - httpFilterForTest(extAuthFilter + "-route1"), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), httpFilterForTest(basicAuthFilter), - httpFilterForTest(oauth2Filter + "-route1"), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), httpFilterForTest(jwtAuthn), - httpFilterForTest(extProcFilter + "-route1"), - httpFilterForTest(wasmFilter + "-route1"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), httpFilterForTest(localRateLimitFilter), httpFilterForTest(wellknown.HTTPRateLimit), httpFilterForTest(wellknown.Router), }, }, + { + name: "custom filter order-multipleton-after-singleton", + filters: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Router), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(jwtAuthn), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wellknown.Fault), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + }, + filterOrder: []v1alpha1.FilterPosition{ + { + Name: v1alpha1.EnvoyFilterWasm, + After: ptr.To(v1alpha1.EnvoyFilterRateLimit), + }, + }, + want: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Fault), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(jwtAuthn), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(wellknown.Router), + }, + }, + { + name: "custom filter order-multipleton-before-multipleton", + filters: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Router), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(jwtAuthn), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wellknown.Fault), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + }, + filterOrder: []v1alpha1.FilterPosition{ + { + Name: v1alpha1.EnvoyFilterWasm, + Before: ptr.To(v1alpha1.EnvoyFilterExtProc), + }, + }, + want: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Fault), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(jwtAuthn), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(wellknown.Router), + }, + }, + { + name: "custom filter order-multipleton-after-multipleton", + filters: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Router), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(jwtAuthn), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wellknown.Fault), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + }, + filterOrder: []v1alpha1.FilterPosition{ + { + Name: v1alpha1.EnvoyFilterExtProc, + After: ptr.To(v1alpha1.EnvoyFilterWasm), + }, + }, + want: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Fault), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(jwtAuthn), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(wellknown.Router), + }, + }, + { + name: "custom filter order-complex-ordering", + filters: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Router), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(jwtAuthn), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wellknown.Fault), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + }, + filterOrder: []v1alpha1.FilterPosition{ + { + Name: v1alpha1.EnvoyFilterLocalRateLimit, + Before: ptr.To(v1alpha1.EnvoyFilterJWTAuthn), + }, + { + Name: v1alpha1.EnvoyFilterLocalRateLimit, + After: ptr.To(v1alpha1.EnvoyFilterCORS), + }, + { + Name: v1alpha1.EnvoyFilterWasm, + Before: ptr.To(v1alpha1.EnvoyFilterOAuth2), + }, + { + Name: v1alpha1.EnvoyFilterExtProc, + Before: ptr.To(v1alpha1.EnvoyFilterWasm), + }, + }, + want: []*hcmv3.HttpFilter{ + httpFilterForTest(wellknown.Fault), + httpFilterForTest(wellknown.CORS), + httpFilterForTest(localRateLimitFilter), + httpFilterForTest(extAuthFilter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(basicAuthFilter), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(extProcFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/0"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/1"), + httpFilterForTest(wasmFilter + "/envoyextensionpolicy/default/policy-for-http-route-1/2"), + httpFilterForTest(oauth2Filter + "/securitypolicy/default/policy-for-http-route-1"), + httpFilterForTest(jwtAuthn), + httpFilterForTest(wellknown.HTTPRateLimit), + httpFilterForTest(wellknown.Router), + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, sortHTTPFilters(tt.filters), "sortHTTPFilters(%v)", tt.filters) + result := sortHTTPFilters(tt.filters, tt.filterOrder) + assert.Equalf(t, tt.want, result, "sortHTTPFilters(%v)", tt.filters) }) } } diff --git a/internal/xds/translator/runner/runner.go b/internal/xds/translator/runner/runner.go index 573afc38228..6e2c8ba7880 100644 --- a/internal/xds/translator/runner/runner.go +++ b/internal/xds/translator/runner/runner.go @@ -60,7 +60,9 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { r.Xds.Delete(key) } else { // Translate to xds resources - t := &translator.Translator{} + t := &translator.Translator{ + FilterOrder: val.FilterOrder, + } // Set the extension manager if an extension is loaded if r.ExtensionManager != nil { diff --git a/internal/xds/translator/testdata/in/xds-ir/custom-filter-order.yaml b/internal/xds/translator/testdata/in/xds-ir/custom-filter-order.yaml new file mode 100644 index 00000000000..8dcefc9c880 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/custom-filter-order.yaml @@ -0,0 +1,84 @@ +filterOrder: +- before: envoy.filters.http.jwt_authn + name: envoy.filters.http.wasm +- after: envoy.filters.http.basic_authn + name: envoy.filters.http.cors +http: +- address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - backendWeights: + invalid: 1 + valid: 0 + directResponse: + statusCode: 500 + hostname: www.example.com + isHTTP2: false + name: httproute/envoy-gateway/httproute-1/rule/0/match/0/www_example_com + pathMatch: + distinct: false + name: "" + prefix: /foo + security: + basicAuth: + name: securitypolicy/envoy-gateway/policy-for-gateway + users: dXNlcjE6e1NIQX10RVNzQm1FL3lOWTNsYjZhMEw2dlZRRVpOcXc9CnVzZXIyOntTSEF9RUo5TFBGRFhzTjl5blNtYnh2anA3NUJtbHg4PQo= + cors: + allowMethods: + - GET + - POST + allowOrigins: + - distinct: false + name: "" + safeRegex: https://.*\.test\.com:8080 + - distinct: false + exact: https://www.test.org:8080 + name: "" + jwt: + providers: + - audiences: + - one.foo.com + claimToHeaders: + - claim: claim1 + header: one-route-example-key + issuer: https://one.example.com + name: example1 + remoteJWKS: + uri: https://one.example.com/jwt/public-key/jwks.json + - audiences: + - two.foo.com + claimToHeaders: + - claim: claim2 + header: two-route-example-key + issuer: http://two.example.com + name: example2 + remoteJWKS: + uri: http://two.example.com/jwt/public-key/jwks.json + wasm: + - config: + parameter1: + key1: value1 + key2: value2 + parameter2: value3 + failOpen: false + httpWasmCode: + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 + url: https://www.example.com/wasm-filter-1.wasm + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + wasmName: wasm-filter-1 + - config: + parameter1: value1 + parameter2: value2 + failOpen: false + httpWasmCode: + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 + url: https://www.example.com/wasm-filter-2.wasm + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + wasmName: wasm-filter-2 diff --git a/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.clusters.yaml new file mode 100644 index 00000000000..ae5259499fd --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.clusters.yaml @@ -0,0 +1,99 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + dnsRefreshRate: 30s + lbPolicy: LEAST_REQUEST + loadAssignment: + clusterName: one_example_com_443 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: one.example.com + portValue: 443 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: one_example_com_443/backend/0 + name: one_example_com_443 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + respectDnsTtl: true + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + validationContext: + trustedCa: + filename: /etc/ssl/certs/ca-certificates.crt + sni: one.example.com + type: STRICT_DNS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + dnsRefreshRate: 30s + lbPolicy: LEAST_REQUEST + loadAssignment: + clusterName: two_example_com_80 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: two.example.com + portValue: 80 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: two_example_com_80/backend/0 + name: two_example_com_80 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + respectDnsTtl: true + type: STRICT_DNS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + dnsRefreshRate: 30s + lbPolicy: LEAST_REQUEST + loadAssignment: + clusterName: www_example_com_443 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: www.example.com + portValue: 443 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: www_example_com_443/backend/0 + name: www_example_com_443 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + respectDnsTtl: true + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + validationContext: + trustedCa: + filename: /etc/ssl/certs/ca-certificates.crt + sni: www.example.com + type: STRICT_DNS diff --git a/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.endpoints.yaml new file mode 100644 index 00000000000..fe51488c706 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.endpoints.yaml @@ -0,0 +1 @@ +[] diff --git a/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.listeners.yaml new file mode 100644 index 00000000000..750690af34b --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.listeners.yaml @@ -0,0 +1,125 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.cors + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors + - disabled: true + name: envoy.filters.http.basic_auth + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.basic_auth.v3.BasicAuth + users: + inlineBytes: dXNlcjE6e1NIQX10RVNzQm1FL3lOWTNsYjZhMEw2dlZRRVpOcXc9CnVzZXIyOntTSEF9RUo5TFBGRFhzTjl5blNtYnh2anA3NUJtbHg4PQo= + - disabled: true + name: envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm + config: + configuration: + '@type': type.googleapis.com/google.protobuf.StringValue + value: '{"parameter1":{"key1":"value1","key2":"value2"},"parameter2":"value3"}' + name: wasm-filter-1 + vmConfig: + code: + remote: + httpUri: + cluster: www_example_com_443 + timeout: 10s + uri: https://www.example.com/wasm-filter-1.wasm + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 + runtime: envoy.wasm.runtime.v8 + vmId: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + - disabled: true + name: envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm + config: + configuration: + '@type': type.googleapis.com/google.protobuf.StringValue + value: '{"parameter1":"value1","parameter2":"value2"}' + name: wasm-filter-2 + vmConfig: + code: + remote: + httpUri: + cluster: www_example_com_443 + timeout: 10s + uri: https://www.example.com/wasm-filter-2.wasm + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 + runtime: envoy.wasm.runtime.v8 + vmId: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + - name: envoy.filters.http.jwt_authn + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.JwtAuthentication + providers: + httproute/envoy-gateway/httproute-1/rule/0/match/0/www_example_com/example1: + audiences: + - one.foo.com + claimToHeaders: + - claimName: claim1 + headerName: one-route-example-key + forward: true + issuer: https://one.example.com + payloadInMetadata: https://one.example.com + remoteJwks: + asyncFetch: {} + cacheDuration: 300s + httpUri: + cluster: one_example_com_443 + timeout: 10s + uri: https://one.example.com/jwt/public-key/jwks.json + retryPolicy: {} + httproute/envoy-gateway/httproute-1/rule/0/match/0/www_example_com/example2: + audiences: + - two.foo.com + claimToHeaders: + - claimName: claim2 + headerName: two-route-example-key + forward: true + issuer: http://two.example.com + payloadInMetadata: http://two.example.com + remoteJwks: + asyncFetch: {} + cacheDuration: 300s + httpUri: + cluster: two_example_com_80 + timeout: 10s + uri: http://two.example.com/jwt/public-key/jwks.json + retryPolicy: {} + requirementMap: + httproute/envoy-gateway/httproute-1/rule/0/match/0/www_example_com: + requiresAny: + requirements: + - providerName: httproute/envoy-gateway/httproute-1/rule/0/match/0/www_example_com/example1 + - providerName: httproute/envoy-gateway/httproute-1/rule/0/match/0/www_example_com/example2 + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: envoy-gateway/gateway-1/http + serverHeaderTransformation: PASS_THROUGH + statPrefix: http + useRemoteAddress: true + drainType: MODIFY_ONLY + name: envoy-gateway/gateway-1/http + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.routes.yaml new file mode 100644 index 00000000000..5299f2ff4fb --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.routes.yaml @@ -0,0 +1,35 @@ +- ignorePortInHostMatching: true + name: envoy-gateway/gateway-1/http + virtualHosts: + - domains: + - www.example.com + name: envoy-gateway/gateway-1/http/www_example_com + routes: + - directResponse: + status: 500 + match: + pathSeparatedPrefix: /foo + name: httproute/envoy-gateway/httproute-1/rule/0/match/0/www_example_com + typedPerFilterConfig: + envoy.filters.http.basic_auth: + '@type': type.googleapis.com/envoy.extensions.filters.http.basic_auth.v3.BasicAuthPerRoute + users: + inlineBytes: dXNlcjE6e1NIQX10RVNzQm1FL3lOWTNsYjZhMEw2dlZRRVpOcXc9CnVzZXIyOntTSEF9RUo5TFBGRFhzTjl5blNtYnh2anA3NUJtbHg4PQo= + envoy.filters.http.cors: + '@type': type.googleapis.com/envoy.extensions.filters.http.cors.v3.CorsPolicy + allowCredentials: false + allowMethods: GET, POST + allowOriginStringMatch: + - safeRegex: + regex: https://.*\.test\.com:8080 + - exact: https://www.test.org:8080 + forwardNotMatchingPreflights: false + envoy.filters.http.jwt_authn: + '@type': type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.PerRouteConfig + requirementName: httproute/envoy-gateway/httproute-1/rule/0/match/0/www_example_com + envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/0: + '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig + config: {} + envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/1: + '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig + config: {} diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index eaef03e97b5..228c2d77418 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -26,6 +26,7 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" "k8s.io/utils/ptr" + "github.com/envoyproxy/gateway/api/v1alpha1" extensionTypes "github.com/envoyproxy/gateway/internal/extension/types" "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/utils/protocov" @@ -48,6 +49,9 @@ type Translator struct { // ExtensionManager holds the config for interacting with extensions when generating xDS // resources. Only required during xds translation. ExtensionManager *extensionTypes.Manager + + // FilterOrder holds the custom order of the HTTP filters + FilterOrder []v1alpha1.FilterPosition } type GlobalRateLimitSettings struct { diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index 28868500b68..60dcf8636c8 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -122,6 +122,7 @@ func TestTranslateXds(t *testing.T) { GlobalRateLimit: &GlobalRateLimitSettings{ ServiceURL: ratelimit.GetServiceURL("envoy-gateway-system", dnsDomain), }, + FilterOrder: x.FilterOrder, } tCtx, err := tr.Translate(x) diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index d485100eab3..50adb24c983 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -1397,7 +1397,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `filter` | _[EnvoyFilter](#envoyfilter)_ | true | Name of the filter. | +| `name` | _[EnvoyFilter](#envoyfilter)_ | true | Name of the filter. | | `before` | _[EnvoyFilter](#envoyfilter)_ | true | Before defines the filter that should come before the filter.
Only one of Before or After must be set. | | `after` | _[EnvoyFilter](#envoyfilter)_ | true | After defines the filter that should come after the filter.
Only one of Before or After must be set. |