feat(translator): http2 upstream settings (#3682)

* implement BTP HTTP2 Signed-off-by: Guy Daich <guy.daich@sap.com> * fix lint Signed-off-by: Guy Daich <guy.daich@sap.com> * api change Signed-off-by: Guy Daich <guy.daich@sap.com> * fix gen Signed-off-by: Guy Daich <guy.daich@sap.com> * change naming Signed-off-by: Guy Daich <guy.daich@sap.com> * fix api doc Signed-off-by: Guy Daich <guy.daich@sap.com> * make connection termination an enum Signed-off-by: Guy Daich <guy.daich@sap.com> * fix gen Signed-off-by: Guy Daich <guy.daich@sap.com> --------- Signed-off-by: Guy Daich <guy.daich@sap.com>
envoyproxy · Aug 5, 2024 · 0f75173 · 0f75173
1 parent b77f6a4
commit 0f75173
Show file tree

Hide file tree

Showing 45 changed files with 1,006 additions and 68 deletions.
diff --git a/api/v1alpha1/backendtrafficpolicy_types.go b/api/v1alpha1/backendtrafficpolicy_types.go
@@ -114,6 +114,11 @@ type BackendTrafficPolicySpec struct {
 	//
 	// +optional
 	DNS *DNS `json:"dns,omitempty"`
+
+	// HTTP2 provides HTTP/2 configuration for backend connections.
+	//
+	// +optional
+	HTTP2 *HTTP2Settings `json:"http2,omitempty"`
 }
 
 // +kubebuilder:object:root=true

diff --git a/api/v1alpha1/clienttrafficpolicy_types.go b/api/v1alpha1/clienttrafficpolicy_types.go
@@ -6,7 +6,6 @@
 package v1alpha1
 
 import (
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 )
@@ -289,30 +288,6 @@ type HTTP10Settings struct {
 	UseDefaultHost *bool `json:"useDefaultHost,omitempty"`
 }
 
-// HTTP2Settings provides HTTP/2 configuration on the listener.
-type HTTP2Settings struct {
-	// InitialStreamWindowSize sets the initial window size for HTTP/2 streams.
-	// If not set, the default value is 64 KiB(64*1024).
-	//
-	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="initialStreamWindowSize must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
-	// +optional
-	InitialStreamWindowSize *resource.Quantity `json:"initialStreamWindowSize,omitempty"`
-
-	// InitialConnectionWindowSize sets the initial window size for HTTP/2 connections.
-	// If not set, the default value is 1 MiB.
-	//
-	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="initialConnectionWindowSize must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
-	// +optional
-	InitialConnectionWindowSize *resource.Quantity `json:"initialConnectionWindowSize,omitempty"`
-
-	// MaxConcurrentStreams sets the maximum number of concurrent streams allowed per connection.
-	// If not set, the default value is 100.
-	// +kubebuilder:validation:Minimum=1
-	// +kubebuilder:validation:Maximum=2147483647
-	// +optional
-	MaxConcurrentStreams *uint32 `json:"maxConcurrentStreams,omitempty"`
-}
-
 // HealthCheckSettings provides HealthCheck configuration on the HTTP/HTTPS listener.
 type HealthCheckSettings struct {
 	// Path specifies the HTTP path to match on for health check requests.

diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go
@@ -10,6 +10,7 @@ import (
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
@@ -478,3 +479,41 @@ type BackendRef struct {
 // A CIDR can be an IPv4 address range such as "192.168.1.0/24" or an IPv6 address range such as "2001:0db8:11a3:09d7::/64".
 // +kubebuilder:validation:Pattern=`((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/([0-9]+))|((([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\/([0-9]+))`
 type CIDR string
+
+type InvalidMessageAction string
+
+const (
+	InvalidMessageActionTerminateConnection InvalidMessageAction = "TerminateConnection"
+	InvalidMessageActionTerminateStream     InvalidMessageAction = "TerminateStream"
+)
+
+// HTTP2Settings provides HTTP/2 configuration for listeners and backends.
+type HTTP2Settings struct {
+	// InitialStreamWindowSize sets the initial window size for HTTP/2 streams.
+	// If not set, the default value is 64 KiB(64*1024).
+	//
+	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="initialStreamWindowSize must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
+	// +optional
+	InitialStreamWindowSize *resource.Quantity `json:"initialStreamWindowSize,omitempty"`
+
+	// InitialConnectionWindowSize sets the initial window size for HTTP/2 connections.
+	// If not set, the default value is 1 MiB.
+	//
+	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="initialConnectionWindowSize must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
+	// +optional
+	InitialConnectionWindowSize *resource.Quantity `json:"initialConnectionWindowSize,omitempty"`
+
+	// MaxConcurrentStreams sets the maximum number of concurrent streams allowed per connection.
+	// If not set, the default value is 100.
+	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:validation:Maximum=2147483647
+	// +optional
+	MaxConcurrentStreams *uint32 `json:"maxConcurrentStreams,omitempty"`
+
+	// OnInvalidMessage determines if Envoy will terminate the connection or just the offending stream in the event of HTTP messaging error
+	// It's recommended for L2 Envoy deployments to set this value to TerminateStream.
+	// https://www.envoyproxy.io/docs/envoy/latest/configuration/best_practices/level_two
+	// Default: TerminateConnection
+	// +optional
+	OnInvalidMessage *InvalidMessageAction `json:"onInvalidMessage,omitempty"`
+}
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml
@@ -436,6 +436,51 @@ spec:
                         type: boolean
                     type: object
                 type: object
+              http2:
+                description: HTTP2 provides HTTP/2 configuration for backend connections.
+                properties:
+                  initialConnectionWindowSize:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: |-
+                      InitialConnectionWindowSize sets the initial window size for HTTP/2 connections.
+                      If not set, the default value is 1 MiB.
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
+                    x-kubernetes-validations:
+                    - message: initialConnectionWindowSize must be of the format "^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
+                      rule: 'type(self) == string ? self.matches(r"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$")
+                        : type(self) == int'
+                  initialStreamWindowSize:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: |-
+                      InitialStreamWindowSize sets the initial window size for HTTP/2 streams.
+                      If not set, the default value is 64 KiB(64*1024).
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
+                    x-kubernetes-validations:
+                    - message: initialStreamWindowSize must be of the format "^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
+                      rule: 'type(self) == string ? self.matches(r"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$")
+                        : type(self) == int'
+                  maxConcurrentStreams:
+                    description: |-
+                      MaxConcurrentStreams sets the maximum number of concurrent streams allowed per connection.
+                      If not set, the default value is 100.
+                    format: int32
+                    maximum: 2147483647
+                    minimum: 1
+                    type: integer
+                  onInvalidMessage:
+                    description: |-
+                      OnInvalidMessage determines if Envoy will terminate the connection or just the offending stream in the event of HTTP messaging error
+                      It's recommended for L2 Envoy deployments to set this value to TerminateStream.
+                      https://www.envoyproxy.io/docs/envoy/latest/configuration/best_practices/level_two
+                      Default: TerminateConnection
+                    type: string
+                type: object
               loadBalancer:
                 description: |-
                   LoadBalancer policy to apply when routing traffic from the gateway to

diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_clienttrafficpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_clienttrafficpolicies.yaml
@@ -300,6 +300,13 @@ spec:
                     maximum: 2147483647
                     minimum: 1
                     type: integer
+                  onInvalidMessage:
+                    description: |-
+                      OnInvalidMessage determines if Envoy will terminate the connection or just the offending stream in the event of HTTP messaging error
+                      It's recommended for L2 Envoy deployments to set this value to TerminateStream.
+                      https://www.envoyproxy.io/docs/envoy/latest/configuration/best_practices/level_two
+                      Default: TerminateConnection
+                    type: string
                 type: object
               http3:
                 description: HTTP3 provides HTTP/3 configuration on the listener.

diff --git a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml
@@ -804,7 +804,9 @@ xds:
             envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
               '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
               explicitHttpConfig:
-                http2ProtocolOptions: {}
+                http2ProtocolOptions:
+                  initialConnectionWindowSize: 1048576
+                  initialStreamWindowSize: 65536
       - cluster:
           '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster
           circuitBreakers:

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json
@@ -504,7 +504,10 @@
                   "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
                     "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
                     "explicitHttpConfig": {
-                      "http2ProtocolOptions": {}
+                      "http2ProtocolOptions": {
+                        "initialConnectionWindowSize": 1048576,
+                        "initialStreamWindowSize": 65536
+                      }
                     }
                   }
                 }

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml
@@ -285,7 +285,9 @@ xds:
             envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
               '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
               explicitHttpConfig:
-                http2ProtocolOptions: {}
+                http2ProtocolOptions:
+                  initialConnectionWindowSize: 1048576
+                  initialStreamWindowSize: 65536
       - cluster:
           '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster
           circuitBreakers:

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.cluster.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.cluster.yaml
@@ -44,7 +44,9 @@ xds:
           envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
             '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
             explicitHttpConfig:
-              http2ProtocolOptions: {}
+              http2ProtocolOptions:
+                initialConnectionWindowSize: 1048576
+                initialStreamWindowSize: 65536
     - cluster:
         '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster
         circuitBreakers:

diff --git a/internal/gatewayapi/backendtrafficpolicy.go b/internal/gatewayapi/backendtrafficpolicy.go
@@ -296,6 +296,7 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen
 		rt        *ir.Retry
 		bc        *ir.BackendConnection
 		ds        *ir.DNS
+		h2        *ir.HTTP2Settings
 		err, errs error
 	)
 
@@ -350,6 +351,13 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen
 		}
 	}
 
+	if policy.Spec.HTTP2 != nil {
+		if h2, err = buildIRHTTP2Settings(policy.Spec.HTTP2); err != nil {
+			err = perr.WithMessage(err, "HTTP2")
+			errs = errors.Join(errs, err)
+		}
+	}
+
 	if policy.Spec.DNS != nil {
 		ds = t.translateDNS(policy)
 	}
@@ -405,6 +413,7 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen
 						TCPKeepalive:      ka,
 						Retry:             rt,
 						BackendConnection: bc,
+						HTTP2:             h2,
 					}
 
 					r.DNS = ds
@@ -441,6 +450,7 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back
 		ka        *ir.TCPKeepalive
 		rt        *ir.Retry
 		ds        *ir.DNS
+		h2        *ir.HTTP2Settings
 		err, errs error
 	)
 
@@ -487,6 +497,12 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back
 			errs = errors.Join(errs, err)
 		}
 	}
+	if policy.Spec.HTTP2 != nil {
+		if h2, err = buildIRHTTP2Settings(policy.Spec.HTTP2); err != nil {
+			err = perr.WithMessage(err, "HTTP2")
+			errs = errors.Join(errs, err)
+		}
+	}
 
 	if policy.Spec.DNS != nil {
 		ds = t.translateDNS(policy)
@@ -587,6 +603,7 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back
 				FaultInjection: fi,
 				TCPKeepalive:   ka,
 				Retry:          rt,
+				HTTP2:          h2,
 			}
 
 			if r.DNS == nil {

diff --git a/internal/gatewayapi/clienttrafficpolicy.go b/internal/gatewayapi/clienttrafficpolicy.go
@@ -28,11 +28,7 @@ import (
 
 const (
 	// Use an invalid string to represent all sections (listeners) within a Gateway
-	AllSections                         = "/"
-	MinHTTP2InitialStreamWindowSize     = 65535      // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-field-config-core-v3-http2protocoloptions-initial-stream-window-size
-	MaxHTTP2InitialStreamWindowSize     = 2147483647 // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-field-config-core-v3-http2protocoloptions-initial-stream-window-size
-	MinHTTP2InitialConnectionWindowSize = MinHTTP2InitialStreamWindowSize
-	MaxHTTP2InitialConnectionWindowSize = MaxHTTP2InitialStreamWindowSize
+	AllSections = "/"
 )
 
 func hasSectionName(target *gwapiv1a2.LocalPolicyTargetReferenceWithSectionName) bool {

diff --git a/internal/gatewayapi/http.go b/internal/gatewayapi/http.go
@@ -0,0 +1,73 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package gatewayapi
+
+import (
+	"errors"
+	"fmt"
+
+	"k8s.io/utils/ptr"
+
+	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
+	"github.com/envoyproxy/gateway/internal/ir"
+)
+
+const (
+	MinHTTP2InitialStreamWindowSize     = 65535      // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-field-config-core-v3-http2protocoloptions-initial-stream-window-size
+	MaxHTTP2InitialStreamWindowSize     = 2147483647 // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-field-config-core-v3-http2protocoloptions-initial-stream-window-size
+	MinHTTP2InitialConnectionWindowSize = MinHTTP2InitialStreamWindowSize
+	MaxHTTP2InitialConnectionWindowSize = MaxHTTP2InitialStreamWindowSize
+)
+
+func buildIRHTTP2Settings(http2Settings *egv1a1.HTTP2Settings) (*ir.HTTP2Settings, error) {
+	var (
+		http2 = &ir.HTTP2Settings{}
+		errs  error
+	)
+
+	if http2Settings.InitialStreamWindowSize != nil {
+		initialStreamWindowSize, ok := http2Settings.InitialStreamWindowSize.AsInt64()
+		switch {
+		case !ok:
+			errs = errors.Join(errs, fmt.Errorf("invalid InitialStreamWindowSize value %s", http2Settings.InitialStreamWindowSize.String()))
+		case initialStreamWindowSize < MinHTTP2InitialStreamWindowSize || initialStreamWindowSize > MaxHTTP2InitialStreamWindowSize:
+			errs = errors.Join(errs, fmt.Errorf("InitialStreamWindowSize value %s is out of range, must be between %d and %d",
+				http2Settings.InitialStreamWindowSize.String(),
+				MinHTTP2InitialStreamWindowSize,
+				MaxHTTP2InitialStreamWindowSize))
+		default:
+			http2.InitialStreamWindowSize = ptr.To(uint32(initialStreamWindowSize))
+		}
+	}
+
+	if http2Settings.InitialConnectionWindowSize != nil {
+		initialConnectionWindowSize, ok := http2Settings.InitialConnectionWindowSize.AsInt64()
+		switch {
+		case !ok:
+			errs = errors.Join(errs, fmt.Errorf("invalid InitialConnectionWindowSize value %s", http2Settings.InitialConnectionWindowSize.String()))
+		case initialConnectionWindowSize < MinHTTP2InitialConnectionWindowSize || initialConnectionWindowSize > MaxHTTP2InitialConnectionWindowSize:
+			errs = errors.Join(errs, fmt.Errorf("InitialConnectionWindowSize value %s is out of range, must be between %d and %d",
+				http2Settings.InitialConnectionWindowSize.String(),
+				MinHTTP2InitialConnectionWindowSize,
+				MaxHTTP2InitialConnectionWindowSize))
+		default:
+			http2.InitialConnectionWindowSize = ptr.To(uint32(initialConnectionWindowSize))
+		}
+	}
+
+	http2.MaxConcurrentStreams = http2Settings.MaxConcurrentStreams
+
+	if http2Settings.OnInvalidMessage != nil {
+		switch *http2Settings.OnInvalidMessage {
+		case egv1a1.InvalidMessageActionTerminateStream:
+			http2.ResetStreamOnError = ptr.To(true)
+		case egv1a1.InvalidMessageActionTerminateConnection:
+			http2.ResetStreamOnError = ptr.To(false)
+		}
+	}
+
+	return http2, errs
+}