feat: support custom HTTP filter ordering (#3273)

* Support custom HTTP filter ordering. Signed-off-by: huabing zhao <zhaohuabing@gmail.com> * address comments Signed-off-by: huabing zhao <zhaohuabing@gmail.com> * address comments Signed-off-by: huabing zhao <zhaohuabing@gmail.com> * add comments Signed-off-by: huabing zhao <zhaohuabing@gmail.com> * add comments Signed-off-by: huabing zhao <zhaohuabing@gmail.com> * minor wording Signed-off-by: huabing zhao <zhaohuabing@gmail.com> --------- Signed-off-by: huabing zhao <zhaohuabing@gmail.com>
envoyproxy · May 7, 2024 · d048a9f · d048a9f
1 parent 4c52f10
commit d048a9f
Show file tree

Hide file tree

Showing 25 changed files with 1,380 additions and 33 deletions.
diff --git a/api/v1alpha1/envoyextensionypolicy_types.go b/api/v1alpha1/envoyextensionypolicy_types.go
@@ -48,12 +48,14 @@ type EnvoyExtensionPolicySpec struct {
 	// Order matters, as the extensions will be loaded in the order they are
 	// defined in this list.
 	//
+	// +kubebuilder:validation:MaxItems=16
 	// +optional
 	Wasm []Wasm `json:"wasm,omitempty"`
 
 	// ExtProc is an ordered list of external processing filters
 	// that should added to the envoy filter chain
 	//
+	// +kubebuilder:validation:MaxItems=16
 	// +optional
 	ExtProc []ExtProc `json:"extProc,omitempty"`
 }

diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go
@@ -89,6 +89,7 @@ type EnvoyProxySpec struct {
 	Shutdown *ShutdownConfig `json:"shutdown,omitempty"`
 
 	// FilterOrder defines the order of filters in the Envoy proxy's HTTP filter chain.
+	// The FilterPosition in the list will be applied in the order they are defined.
 	// If unspecified, the default filter order is applied.
 	// Default filter order is:
 	//
@@ -138,7 +139,7 @@ type BackendTLSConfig struct {
 // +kubebuilder:validation:XValidation:rule="(has(self.before) && !has(self.after)) || (!has(self.before) && has(self.after))",message="only one of before or after can be specified"
 type FilterPosition struct {
 	// Name of the filter.
-	Name EnvoyFilter `json:"filter"`
+	Name EnvoyFilter `json:"name"`
 
 	// Before defines the filter that should come before the filter.
 	// Only one of Before or After must be set.

diff --git a/api/v1alpha1/validation/envoyproxy_validate.go b/api/v1alpha1/validation/envoyproxy_validate.go
@@ -11,6 +11,7 @@ import (
 	"net"
 	"net/netip"
 
+	"github.com/dominikbraun/graph"
 	bootstrapv3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
 	clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	"github.com/google/go-cmp/cmp"
@@ -62,6 +63,13 @@ func validateEnvoyProxySpec(spec *egv1a1.EnvoyProxySpec) error {
 		errs = append(errs, validateProxyTelemetryErrs...)
 	}
 
+	// validate filter order
+	if spec != nil && spec.FilterOrder != nil {
+		if err := validateFilterOrder(spec.FilterOrder); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
 	return utilerrors.NewAggregate(errs)
 }
 
@@ -269,3 +277,36 @@ func validateProxyAccessLog(accessLog *egv1a1.ProxyAccessLog) []error {
 
 	return errs
 }
+
+func validateFilterOrder(filterOrder []egv1a1.FilterPosition) error {
+	g := graph.New(graph.StringHash, graph.Directed(), graph.PreventCycles())
+
+	for _, filter := range filterOrder {
+		// Ignore the error since the same filter can be added multiple times
+		_ = g.AddVertex(string(filter.Name))
+		if filter.Before != nil {
+			_ = g.AddVertex(string(*filter.Before))
+		}
+		if filter.After != nil {
+			_ = g.AddVertex(string(*filter.After))
+		}
+	}
+
+	for _, filter := range filterOrder {
+		var from, to string
+		if filter.Before != nil {
+			from = string(filter.Name)
+			to = string(*filter.Before)
+		} else {
+			from = string(*filter.After)
+			to = string(filter.Name)
+		}
+		if err := g.AddEdge(from, to); err != nil {
+			if errors.Is(err, graph.ErrEdgeCreatesCycle) {
+				return fmt.Errorf("there is a cycle in the filter order: %s -> %s", from, to)
+			}
+		}
+	}
+
+	return nil
+}
diff --git a/api/v1alpha1/validation/envoyproxy_validate_test.go b/api/v1alpha1/validation/envoyproxy_validate_test.go
@@ -613,6 +613,54 @@ func TestValidateEnvoyProxy(t *testing.T) {
 			},
 			expected: true,
 		},
+		{
+			name: "valid filter order",
+			proxy: &egv1a1.EnvoyProxy{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test",
+					Name:      "test",
+				},
+				Spec: egv1a1.EnvoyProxySpec{
+					FilterOrder: []egv1a1.FilterPosition{
+						{
+							Name:   egv1a1.EnvoyFilterOAuth2,
+							Before: ptr.To(egv1a1.EnvoyFilterJWTAuthn),
+						},
+						{
+							Name:  egv1a1.EnvoyFilterExtProc,
+							After: ptr.To(egv1a1.EnvoyFilterJWTAuthn),
+						},
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "invalid filter order with circular dependency",
+			proxy: &egv1a1.EnvoyProxy{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test",
+					Name:      "test",
+				},
+				Spec: egv1a1.EnvoyProxySpec{
+					FilterOrder: []egv1a1.FilterPosition{
+						{
+							Name:   egv1a1.EnvoyFilterOAuth2,
+							Before: ptr.To(egv1a1.EnvoyFilterJWTAuthn),
+						},
+						{
+							Name:   egv1a1.EnvoyFilterJWTAuthn,
+							Before: ptr.To(egv1a1.EnvoyFilterExtProc),
+						},
+						{
+							Name:   egv1a1.EnvoyFilterExtProc,
+							Before: ptr.To(egv1a1.EnvoyFilterOAuth2),
+						},
+					},
+				},
+			},
+			expected: false,
+		},
 	}
 
 	for i := range testCases {

diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml
@@ -189,6 +189,7 @@ spec:
                   required:
                   - backendRefs
                   type: object
+                maxItems: 16
                 type: array
               targetRef:
                 description: |-
@@ -398,6 +399,7 @@ spec:
                   - code
                   - name
                   type: object
+                maxItems: 16
                 type: array
             required:
             - targetRef

diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml
@@ -224,6 +224,7 @@ spec:
               filterOrder:
                 description: |-
                   FilterOrder defines the order of filters in the Envoy proxy's HTTP filter chain.
+                  The FilterPosition in the list will be applied in the order they are defined.
                   If unspecified, the default filter order is applied.
                   Default filter order is:
 
@@ -295,7 +296,7 @@ spec:
                       - envoy.filters.http.wasm
                       - envoy.filters.http.ext_proc
                       type: string
-                    filter:
+                    name:
                       description: Name of the filter.
                       enum:
                       - envoy.filters.http.cors
@@ -310,7 +311,7 @@ spec:
                       - envoy.filters.http.ext_proc
                       type: string
                   required:
-                  - filter
+                  - name
                   type: object
                   x-kubernetes-validations:
                   - message: one of before or after must be specified

diff --git a/go.mod b/go.mod
@@ -8,6 +8,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b
 	github.com/davecgh/go-spew v1.1.1
+	github.com/dominikbraun/graph v0.23.0
 	github.com/envoyproxy/go-control-plane v0.12.1-0.20240425230418-212e93054f1a
 	github.com/envoyproxy/ratelimit v1.4.1-0.20230427142404-e2a87f41d3a7
 	github.com/evanphx/json-patch/v5 v5.9.0

diff --git a/go.sum b/go.sum
@@ -161,6 +161,8 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arX
 github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
+github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=

diff --git a/internal/cmd/egctl/translate.go b/internal/cmd/egctl/translate.go
@@ -362,6 +362,9 @@ func translateGatewayAPIToXds(dnsDomain string, resourceType string, resources *
 				ServiceURL: ratelimit.GetServiceURL("envoy-gateway", dnsDomain),
 			},
 		}
+		if resources.EnvoyProxy != nil {
+			xTranslator.FilterOrder = resources.EnvoyProxy.Spec.FilterOrder
+		}
 		xRes, err := xTranslator.Translate(val)
 		if err != nil {
 			return nil, fmt.Errorf("failed to translate xds ir for key %s value %+v, error:%w", key, val, err)

diff --git a/internal/gatewayapi/testdata/custom-filter-order.in.yaml b/internal/gatewayapi/testdata/custom-filter-order.in.yaml
@@ -0,0 +1,130 @@
+secrets:
+  - apiVersion: v1
+    kind: Secret
+    metadata:
+      namespace: envoy-gateway
+      name: users-secret1
+    data:
+      .htpasswd: "dXNlcjE6e1NIQX10RVNzQm1FL3lOWTNsYjZhMEw2dlZRRVpOcXc9CnVzZXIyOntTSEF9RUo5TFBGRFhzTjl5blNtYnh2anA3NUJtbHg4PQo="
+envoyproxy:
+  apiVersion: gateway.envoyproxy.io/v1alpha1
+  kind: EnvoyProxy
+  metadata:
+    name: custom-proxy-config
+    namespace: envoy-gateway-system
+  spec:
+    filterOrder:
+      - name: envoy.filters.http.wasm
+        before: envoy.filters.http.jwt_authn
+      - name: envoy.filters.http.cors
+        after: envoy.filters.http.basic_authn
+gateways:
+  - apiVersion: gateway.networking.k8s.io/v1
+    kind: Gateway
+    metadata:
+      namespace: envoy-gateway
+      name: gateway-1
+    spec:
+      gatewayClassName: envoy-gateway-class
+      listeners:
+        - name: http
+          protocol: HTTP
+          port: 80
+          allowedRoutes:
+            namespaces:
+              from: All
+httpRoutes:
+  - apiVersion: gateway.networking.k8s.io/v1
+    kind: HTTPRoute
+    metadata:
+      namespace: envoy-gateway
+      name: httproute-1
+    spec:
+      hostnames:
+        - www.example.com
+      parentRefs:
+        - namespace: envoy-gateway
+          name: gateway-1
+          sectionName: http
+      rules:
+        - matches:
+            - path:
+                value: "/foo"
+          backendRefs:
+            - name: service-1
+              port: 8080
+securityPolicies:
+  - apiVersion: gateway.envoyproxy.io/v1alpha1
+    kind: SecurityPolicy
+    metadata:
+      namespace: envoy-gateway
+      name: policy-for-gateway
+    spec:
+      targetRef:
+        group: gateway.networking.k8s.io
+        kind: Gateway
+        name: gateway-1
+        namespace: envoy-gateway
+      cors:
+        allowOrigins:
+          - "https://*.test.com:8080"
+          - "https://www.test.org:8080"
+        allowMethods:
+          - GET
+          - POST
+      basicAuth:
+        users:
+          name: "users-secret1"
+      jwt:
+        providers:
+          - name: example1
+            issuer: https://one.example.com
+            audiences:
+              - one.foo.com
+            remoteJWKS:
+              uri: https://one.example.com/jwt/public-key/jwks.json
+            claimToHeaders:
+              - header: one-route-example-key
+                claim: claim1
+          - name: example2
+            issuer: http://two.example.com
+            audiences:
+              - two.foo.com
+            remoteJWKS:
+              uri: http://two.example.com/jwt/public-key/jwks.json
+            claimToHeaders:
+              - header: two-route-example-key
+                claim: claim2
+envoyextensionpolicies:
+  - apiVersion: gateway.envoyproxy.io/v1alpha1
+    kind: EnvoyExtensionPolicy
+    metadata:
+      namespace: envoy-gateway
+      name: policy-for-gateway  # This policy should attach httproute-2
+    spec:
+      targetRef:
+        group: gateway.networking.k8s.io
+        kind: Gateway
+        name: gateway-1
+        namespace: envoy-gateway
+      wasm:
+        - name: wasm-filter-1
+          code:
+            type: HTTP
+            http:
+              url: https://www.example.com/wasm-filter-1.wasm
+            sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5
+          config:
+            parameter1:
+              key1: value1
+              key2: value2
+            parameter2: value3
+        - name: wasm-filter-2
+          code:
+            type: HTTP
+            http:
+              url: https://www.example.com/wasm-filter-2.wasm
+            sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980
+          config:
+            parameter1: value1
+            parameter2: value2