From f1a0a42b4ded71feb157979299f0905d22f45bc5 Mon Sep 17 00:00:00 2001 From: Guy Daich Date: Mon, 29 Jan 2024 11:02:19 -0600 Subject: [PATCH] feat(translator): Implement BTP Timeout API (#2454) Implement BTP Timeout API Signed-off-by: Guy Daich --- internal/gatewayapi/backendtrafficpolicy.go | 104 +++++- internal/gatewayapi/route.go | 34 +- ...ndtrafficpolicy-with-timeout-error.in.yaml | 49 +++ ...dtrafficpolicy-with-timeout-error.out.yaml | 148 +++++++++ .../backendtrafficpolicy-with-timeout.in.yaml | 99 ++++++ ...backendtrafficpolicy-with-timeout.out.yaml | 311 +++++++++++++++++ ...ndtrafficpolicy-with-timeout-error.in.yaml | 54 +++ ...dtrafficpolicy-with-timeout-error.out.yaml | 160 +++++++++ ...-backendtrafficpolicy-with-timeout.in.yaml | 100 ++++++ ...backendtrafficpolicy-with-timeout.out.yaml | 312 ++++++++++++++++++ ...httproute-backend-request-timeout.out.yaml | 4 +- .../httproute-request-timeout.out.yaml | 4 +- internal/ir/xds.go | 32 +- internal/ir/zz_generated.deepcopy.go | 85 ++++- internal/xds/translator/cluster.go | 92 ++++-- internal/xds/translator/route.go | 5 +- .../in/xds-ir/http-route-timeout.yaml | 4 +- .../testdata/in/xds-ir/timeout.yaml | 24 ++ .../testdata/out/xds-ir/timeout.clusters.yaml | 22 ++ .../out/xds-ir/timeout.endpoints.yaml | 12 + .../out/xds-ir/timeout.listeners.yaml | 33 ++ .../testdata/out/xds-ir/timeout.routes.yaml | 12 + internal/xds/translator/translator.go | 1 + internal/xds/translator/translator_test.go | 3 + 24 files changed, 1660 insertions(+), 44 deletions(-) create mode 100644 internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout-error.in.yaml create mode 100755 internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout-error.out.yaml create mode 100644 internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout.in.yaml create mode 100755 internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout.out.yaml create mode 100644 internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout-error.in.yaml create mode 100755 internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout-error.out.yaml create mode 100644 internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout.in.yaml create mode 100755 internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout.out.yaml create mode 100644 internal/xds/translator/testdata/in/xds-ir/timeout.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/timeout.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/timeout.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/timeout.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/timeout.routes.yaml diff --git a/internal/gatewayapi/backendtrafficpolicy.go b/internal/gatewayapi/backendtrafficpolicy.go index e7975c308ec..9d64daf63f2 100644 --- a/internal/gatewayapi/backendtrafficpolicy.go +++ b/internal/gatewayapi/backendtrafficpolicy.go @@ -12,6 +12,7 @@ import ( "net" "sort" "strings" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -248,6 +249,7 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen hc *ir.HealthCheck cb *ir.CircuitBreaker fi *ir.FaultInjection + to *ir.Timeout ) // Build IR @@ -283,6 +285,12 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen r.HealthCheck = hc r.CircuitBreaker = cb r.FaultInjection = fi + + // some timeout setting originate from the route + if policy.Spec.Timeout != nil { + to = t.buildTimeout(policy, r) + r.Timeout = to + } } } } @@ -298,6 +306,7 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back hc *ir.HealthCheck cb *ir.CircuitBreaker fi *ir.FaultInjection + ct *ir.Timeout ) // Build IR @@ -348,6 +357,13 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back if r.FaultInjection == nil { r.FaultInjection = fi } + + if policy.Spec.Timeout != nil { + ct = t.buildTimeout(policy, r) + if r.Timeout == nil { + r.Timeout = ct + } + } } } @@ -776,7 +792,7 @@ func (t *Translator) buildCircuitBreaker(policy *egv1a1.BackendTrafficPolicy) *i if ui32, ok := int64ToUint32(*pcb.MaxConnections); ok { cb.MaxConnections = &ui32 } else { - setCircuitBreakerPolicyErrorCondition(policy, fmt.Sprintf("invalid MaxConnections value %d", *pcb.MaxConnections)) + setBackendTrafficPolicyTranslationErrorCondition(policy, "Circuit Breaker", fmt.Sprintf("invalid MaxConnections value %d", *pcb.MaxConnections)) return nil } } @@ -785,7 +801,7 @@ func (t *Translator) buildCircuitBreaker(policy *egv1a1.BackendTrafficPolicy) *i if ui32, ok := int64ToUint32(*pcb.MaxParallelRequests); ok { cb.MaxParallelRequests = &ui32 } else { - setCircuitBreakerPolicyErrorCondition(policy, fmt.Sprintf("invalid MaxParallelRequests value %d", *pcb.MaxParallelRequests)) + setBackendTrafficPolicyTranslationErrorCondition(policy, "Circuit Breaker", fmt.Sprintf("invalid MaxParallelRequests value %d", *pcb.MaxParallelRequests)) return nil } } @@ -794,7 +810,7 @@ func (t *Translator) buildCircuitBreaker(policy *egv1a1.BackendTrafficPolicy) *i if ui32, ok := int64ToUint32(*pcb.MaxPendingRequests); ok { cb.MaxPendingRequests = &ui32 } else { - setCircuitBreakerPolicyErrorCondition(policy, fmt.Sprintf("invalid MaxPendingRequests value %d", *pcb.MaxPendingRequests)) + setBackendTrafficPolicyTranslationErrorCondition(policy, "Circuit Breaker", fmt.Sprintf("invalid MaxPendingRequests value %d", *pcb.MaxPendingRequests)) return nil } } @@ -803,8 +819,86 @@ func (t *Translator) buildCircuitBreaker(policy *egv1a1.BackendTrafficPolicy) *i return cb } -func setCircuitBreakerPolicyErrorCondition(policy *egv1a1.BackendTrafficPolicy, errMsg string) { - message := fmt.Sprintf("Unable to translate Circuit Breaker: %s", errMsg) +func (t *Translator) buildTimeout(policy *egv1a1.BackendTrafficPolicy, r *ir.HTTPRoute) *ir.Timeout { + var tto *ir.TCPTimeout + var hto *ir.HTTPTimeout + terr := false + + pto := policy.Spec.Timeout + + if pto.TCP != nil && pto.TCP.ConnectTimeout != nil { + d, err := time.ParseDuration(string(*pto.TCP.ConnectTimeout)) + if err != nil { + setBackendTrafficPolicyTranslationErrorCondition(policy, "TCP Timeout", fmt.Sprintf("invalid ConnectTimeout value %s", *pto.TCP.ConnectTimeout)) + terr = true + } else { + tto = &ir.TCPTimeout{ + ConnectTimeout: ptr.To(metav1.Duration{Duration: d}), + } + } + } + + if pto.HTTP != nil { + var cit *metav1.Duration + var mcd *metav1.Duration + + if pto.HTTP.ConnectionIdleTimeout != nil { + d, err := time.ParseDuration(string(*pto.HTTP.ConnectionIdleTimeout)) + if err != nil { + setBackendTrafficPolicyTranslationErrorCondition(policy, "HTTP Timeout", fmt.Sprintf("invalid ConnectionIdleTimeout value %s", *pto.HTTP.ConnectionIdleTimeout)) + terr = true + } else { + cit = ptr.To(metav1.Duration{Duration: d}) + } + } + + if pto.HTTP.MaxConnectionDuration != nil { + d, err := time.ParseDuration(string(*pto.HTTP.MaxConnectionDuration)) + if err != nil { + setBackendTrafficPolicyTranslationErrorCondition(policy, "HTTP Timeout", fmt.Sprintf("invalid MaxConnectionDuration value %s", *pto.HTTP.MaxConnectionDuration)) + terr = true + } else { + mcd = ptr.To(metav1.Duration{Duration: d}) + } + } + + hto = &ir.HTTPTimeout{ + ConnectionIdleTimeout: cit, + MaxConnectionDuration: mcd, + } + } + + // if backendtrafficpolicy is not translatable, return the original timeout if it's initialized + if terr { + if r.Timeout != nil { + return r.Timeout.DeepCopy() + } + } else { + // http request timeout is translated during the gateway-api route resource translation + // merge route timeout setting with backendtrafficpolicy timeout settings + if r.Timeout != nil && r.Timeout.HTTP != nil && r.Timeout.HTTP.RequestTimeout != nil { + if hto == nil { + hto = &ir.HTTPTimeout{ + RequestTimeout: r.Timeout.HTTP.RequestTimeout, + } + } else { + hto.RequestTimeout = r.Timeout.HTTP.RequestTimeout + } + } + + if hto != nil || tto != nil { + return &ir.Timeout{ + TCP: tto, + HTTP: hto, + } + } + } + + return nil +} + +func setBackendTrafficPolicyTranslationErrorCondition(policy *egv1a1.BackendTrafficPolicy, field, errMsg string) { + message := fmt.Sprintf("Unable to translate %s: %s", field, errMsg) status.SetBackendTrafficPolicyCondition(policy, gwv1a2.PolicyConditionAccepted, metav1.ConditionFalse, diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index 926a0739c1d..b7e082c849d 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -227,12 +227,21 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe func processTimeout(irRoute *ir.HTTPRoute, rule gwapiv1.HTTPRouteRule) { if rule.Timeouts != nil { + var rto *ir.Timeout + + // Timeout is translated from multiple resources and may already be partially set + if irRoute.Timeout != nil { + rto = irRoute.Timeout.DeepCopy() + } else { + rto = &ir.Timeout{} + } + if rule.Timeouts.Request != nil { d, err := time.ParseDuration(string(*rule.Timeouts.Request)) if err != nil { d, _ = time.ParseDuration(HTTPRequestTimeout) } - irRoute.Timeout = ptr.To(metav1.Duration{Duration: d}) + setRequestTimeout(rto, metav1.Duration{Duration: d}) } // Also set the IR Route Timeout to the backend request timeout @@ -242,8 +251,27 @@ func processTimeout(irRoute *ir.HTTPRoute, rule gwapiv1.HTTPRouteRule) { if err != nil { d, _ = time.ParseDuration(HTTPRequestTimeout) } - irRoute.Timeout = ptr.To(metav1.Duration{Duration: d}) + setRequestTimeout(rto, metav1.Duration{Duration: d}) } + + irRoute.Timeout = rto + } +} + +func setRequestTimeout(irTimeout *ir.Timeout, d metav1.Duration) { + switch { + case irTimeout == nil: + irTimeout = &ir.Timeout{ + HTTP: &ir.HTTPTimeout{ + RequestTimeout: ptr.To(d), + }, + } + case irTimeout.HTTP == nil: + irTimeout.HTTP = &ir.HTTPTimeout{ + RequestTimeout: ptr.To(d), + } + default: + irTimeout.HTTP.RequestTimeout = ptr.To(d) } } @@ -627,8 +655,8 @@ func (t *Translator) processHTTPRouteParentRefListener(route RouteContext, route DirectResponse: routeRoute.DirectResponse, URLRewrite: routeRoute.URLRewrite, Mirrors: routeRoute.Mirrors, - Timeout: routeRoute.Timeout, ExtensionRefs: routeRoute.ExtensionRefs, + Timeout: routeRoute.Timeout, } // Don't bother copying over the weights unless the route has invalid backends. if routeRoute.BackendWeights.Invalid > 0 { diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout-error.in.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout-error.in.yaml new file mode 100644 index 00000000000..8025e82f1d8 --- /dev/null +++ b/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout-error.in.yaml @@ -0,0 +1,49 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + namespace: default + name: grpcroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: envoy-gateway + name: policy-for-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + timeout: + http: + connectionIdleTimeout: 21s + maxConnectionDuration: 22mib + tcp: + connectTimeout: 20s + diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout-error.out.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout-error.out.yaml new file mode 100755 index 00000000000..6feff83ca6b --- /dev/null +++ b/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout-error.out.yaml @@ -0,0 +1,148 @@ +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-gateway + namespace: envoy-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + timeout: + http: + connectionIdleTimeout: 21s + maxConnectionDuration: 22mib + tcp: + connectTimeout: 20s + status: + conditions: + - lastTransitionTime: null + message: 'Unable to translate HTTP Timeout: invalid MaxConnectionDuration value + 22mib' + reason: Invalid + status: "False" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + creationTimestamp: null + name: grpcroute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: true + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: GRPC + weight: 1 + hostname: '*' + name: grpcroute/default/grpcroute-1/rule/0/match/-1/* diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout.in.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout.in.yaml new file mode 100644 index 00000000000..c73a4504d4c --- /dev/null +++ b/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout.in.yaml @@ -0,0 +1,99 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-2 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + namespace: default + name: grpcroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-2 + sectionName: http + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: envoy-gateway + name: policy-for-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + timeout: + tcp: + connectTimeout: 15s + http: + connectionIdleTimeout: 16s + maxConnectionDuration: 17s +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: default + name: policy-for-route + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + namespace: default + timeout: + tcp: + connectTimeout: 20s + http: + connectionIdleTimeout: 21s + maxConnectionDuration: 22s diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout.out.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout.out.yaml new file mode 100755 index 00000000000..9c5fe2dca53 --- /dev/null +++ b/internal/gatewayapi/testdata/backendtrafficpolicy-with-timeout.out.yaml @@ -0,0 +1,311 @@ +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-route + namespace: default + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + namespace: default + timeout: + http: + connectionIdleTimeout: 21s + maxConnectionDuration: 22s + tcp: + connectTimeout: 20s + status: + conditions: + - lastTransitionTime: null + message: BackendTrafficPolicy has been accepted. + reason: Accepted + status: "True" + type: Accepted +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-gateway + namespace: envoy-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + timeout: + http: + connectionIdleTimeout: 16s + maxConnectionDuration: 17s + tcp: + connectTimeout: 15s + status: + conditions: + - lastTransitionTime: null + message: BackendTrafficPolicy has been accepted. + reason: Accepted + status: "True" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-2 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + creationTimestamp: null + name: grpcroute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-2 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + matches: + - path: + value: / + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-2 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 + envoy-gateway/gateway-2: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-2/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-2 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-2 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: true + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: GRPC + weight: 1 + hostname: '*' + name: grpcroute/default/grpcroute-1/rule/0/match/-1/* + timeout: + http: + connectionIdleTimeout: 16s + maxConnectionDuration: 17s + tcp: + connectTimeout: 15s + envoy-gateway/gateway-2: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-2/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: HTTP + weight: 1 + hostname: gateway.envoyproxy.io + name: httproute/default/httproute-1/rule/0/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: / + timeout: + http: + connectionIdleTimeout: 21s + maxConnectionDuration: 22s + tcp: + connectTimeout: 20s diff --git a/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout-error.in.yaml b/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout-error.in.yaml new file mode 100644 index 00000000000..c95fd909515 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout-error.in.yaml @@ -0,0 +1,54 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-2 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-2 + sectionName: http + rules: + - matches: + - path: + value: "/" + timeouts: + request: 1s + backendRefs: + - name: service-1 + port: 8080 +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: default + name: policy-for-route + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + namespace: default + timeout: + tcp: + connectTimeout: 20kib + http: + maxConnectionDuration: 22s diff --git a/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout-error.out.yaml b/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout-error.out.yaml new file mode 100755 index 00000000000..7087de39768 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout-error.out.yaml @@ -0,0 +1,160 @@ +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-route + namespace: default + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + namespace: default + timeout: + http: + maxConnectionDuration: 22s + tcp: + connectTimeout: 20kib + status: + conditions: + - lastTransitionTime: null + message: 'Unable to translate TCP Timeout: invalid ConnectTimeout value 20kib' + reason: Invalid + status: "False" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-2 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-2 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + matches: + - path: + value: / + timeouts: + request: 1s + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-2 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-2: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-2/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-2 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-2 +xdsIR: + envoy-gateway/gateway-2: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-2/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: HTTP + weight: 1 + hostname: gateway.envoyproxy.io + name: httproute/default/httproute-1/rule/0/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: / + timeout: + http: + requestTimeout: 1s diff --git a/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout.in.yaml b/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout.in.yaml new file mode 100644 index 00000000000..90157f5e80f --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout.in.yaml @@ -0,0 +1,100 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-2 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + namespace: default + name: grpcroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-2 + sectionName: http + rules: + - matches: + - path: + value: "/" + timeouts: + request: 1s + backendRefs: + - name: service-1 + port: 8080 +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: envoy-gateway + name: policy-for-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + timeout: + tcp: + connectTimeout: 15s + http: + connectionIdleTimeout: 16s + maxConnectionDuration: 17s +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: default + name: policy-for-route + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + namespace: default + timeout: + tcp: + connectTimeout: 20s + http: + maxConnectionDuration: 22s diff --git a/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout.out.yaml b/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout.out.yaml new file mode 100755 index 00000000000..4f34e9e2b46 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-and-backendtrafficpolicy-with-timeout.out.yaml @@ -0,0 +1,312 @@ +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-route + namespace: default + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + namespace: default + timeout: + http: + maxConnectionDuration: 22s + tcp: + connectTimeout: 20s + status: + conditions: + - lastTransitionTime: null + message: BackendTrafficPolicy has been accepted. + reason: Accepted + status: "True" + type: Accepted +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-gateway + namespace: envoy-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + timeout: + http: + connectionIdleTimeout: 16s + maxConnectionDuration: 17s + tcp: + connectTimeout: 15s + status: + conditions: + - lastTransitionTime: null + message: BackendTrafficPolicy has been accepted. + reason: Accepted + status: "True" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-2 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + creationTimestamp: null + name: grpcroute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-2 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + matches: + - path: + value: / + timeouts: + request: 1s + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-2 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 + envoy-gateway/gateway-2: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-2/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-2 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-2 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: true + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: GRPC + weight: 1 + hostname: '*' + name: grpcroute/default/grpcroute-1/rule/0/match/-1/* + timeout: + http: + connectionIdleTimeout: 16s + maxConnectionDuration: 17s + tcp: + connectTimeout: 15s + envoy-gateway/gateway-2: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-2/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: HTTP + weight: 1 + hostname: gateway.envoyproxy.io + name: httproute/default/httproute-1/rule/0/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: / + timeout: + http: + maxConnectionDuration: 22s + requestTimeout: 1s + tcp: + connectTimeout: 20s diff --git a/internal/gatewayapi/testdata/httproute-backend-request-timeout.out.yaml b/internal/gatewayapi/testdata/httproute-backend-request-timeout.out.yaml index a9dc0004735..d30ba8e8970 100644 --- a/internal/gatewayapi/testdata/httproute-backend-request-timeout.out.yaml +++ b/internal/gatewayapi/testdata/httproute-backend-request-timeout.out.yaml @@ -128,4 +128,6 @@ xdsIR: distinct: false name: "" prefix: / - timeout: 1s + timeout: + http: + requestTimeout: 1s diff --git a/internal/gatewayapi/testdata/httproute-request-timeout.out.yaml b/internal/gatewayapi/testdata/httproute-request-timeout.out.yaml index c25eae8d45e..6001280e097 100644 --- a/internal/gatewayapi/testdata/httproute-request-timeout.out.yaml +++ b/internal/gatewayapi/testdata/httproute-request-timeout.out.yaml @@ -128,4 +128,6 @@ xdsIR: distinct: false name: "" prefix: / - timeout: 5s + timeout: + http: + requestTimeout: 5s diff --git a/internal/ir/xds.go b/internal/ir/xds.go index a659524003a..65bad668b15 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -389,8 +389,6 @@ type HTTPRoute struct { // RateLimit defines the more specific match conditions as well as limits for ratelimiting // the requests on this route. RateLimit *RateLimit `json:"rateLimit,omitempty" yaml:"rateLimit,omitempty"` - // Timeout is the time until which entire response is received from the upstream. - Timeout *metav1.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"` // load balancer policy to use when routing to the backend endpoints. LoadBalancer *LoadBalancer `json:"loadBalancer,omitempty" yaml:"loadBalancer,omitempty"` // CORS policy for the route. @@ -411,6 +409,8 @@ type HTTPRoute struct { ExtensionRefs []*UnstructuredRef `json:"extensionRefs,omitempty" yaml:"extensionRefs,omitempty"` // Circuit Breaker Settings CircuitBreaker *CircuitBreaker `json:"circuitBreaker,omitempty" yaml:"circuitBreaker,omitempty"` + // Request and connection timeout settings + Timeout *Timeout `json:"timeout,omitempty" yaml:"timeout,omitempty"` } // UnstructuredRef holds unstructured data for an arbitrary k8s resource introduced by an extension @@ -1481,3 +1481,31 @@ func (p *HealthCheckPayload) Validate() error { } return errs } + +// Backend connection timeout settings +// +k8s:deepcopy-gen=true +type Timeout struct { + // Timeout settings for TCP. + TCP *TCPTimeout `json:"tcp,omitempty" yaml:"tcp,omitempty"` + + // Timeout settings for HTTP. + HTTP *HTTPTimeout `json:"http,omitempty" yaml:"tcp,omitempty"` +} + +// +k8s:deepcopy-gen=true +type TCPTimeout struct { + // The timeout for network connection establishment, including TCP and TLS handshakes. + ConnectTimeout *metav1.Duration `json:"connectTimeout,omitempty" yaml:"connectTimeout,omitempty"` +} + +// +k8s:deepcopy-gen=true +type HTTPTimeout struct { + // RequestTimeout is the time until which entire response is received from the upstream. + RequestTimeout *metav1.Duration `json:"requestTimeout,omitempty" yaml:"requestTimeout,omitempty"` + + // The idle timeout for an HTTP connection. Idle time is defined as a period in which there are no active requests in the connection. + ConnectionIdleTimeout *metav1.Duration `json:"connectionIdleTimeout,omitempty" yaml:"connectionIdleTimeout,omitempty"` + + // The maximum duration of an HTTP connection. + MaxConnectionDuration *metav1.Duration `json:"maxConnectionDuration,omitempty" yaml:"maxConnectionDuration,omitempty"` +} diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index 79aef2dc38f..f6b57c02a88 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -626,11 +626,6 @@ func (in *HTTPRoute) DeepCopyInto(out *HTTPRoute) { *out = new(RateLimit) (*in).DeepCopyInto(*out) } - if in.Timeout != nil { - in, out := &in.Timeout, &out.Timeout - *out = new(v1.Duration) - **out = **in - } if in.LoadBalancer != nil { in, out := &in.LoadBalancer, &out.LoadBalancer *out = new(LoadBalancer) @@ -687,6 +682,11 @@ func (in *HTTPRoute) DeepCopyInto(out *HTTPRoute) { *out = new(CircuitBreaker) (*in).DeepCopyInto(*out) } + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(Timeout) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPRoute. @@ -699,6 +699,36 @@ func (in *HTTPRoute) DeepCopy() *HTTPRoute { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPTimeout) DeepCopyInto(out *HTTPTimeout) { + *out = *in + if in.RequestTimeout != nil { + in, out := &in.RequestTimeout, &out.RequestTimeout + *out = new(v1.Duration) + **out = **in + } + if in.ConnectionIdleTimeout != nil { + in, out := &in.ConnectionIdleTimeout, &out.ConnectionIdleTimeout + *out = new(v1.Duration) + **out = **in + } + if in.MaxConnectionDuration != nil { + in, out := &in.MaxConnectionDuration, &out.MaxConnectionDuration + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPTimeout. +func (in *HTTPTimeout) DeepCopy() *HTTPTimeout { + if in == nil { + return nil + } + out := new(HTTPTimeout) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HealthCheck) DeepCopyInto(out *HealthCheck) { *out = *in @@ -1485,6 +1515,26 @@ func (in *TCPListener) DeepCopy() *TCPListener { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TCPTimeout) DeepCopyInto(out *TCPTimeout) { + *out = *in + if in.ConnectTimeout != nil { + in, out := &in.ConnectTimeout, &out.ConnectTimeout + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TCPTimeout. +func (in *TCPTimeout) DeepCopy() *TCPTimeout { + if in == nil { + return nil + } + out := new(TCPTimeout) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLS) DeepCopyInto(out *TLS) { *out = *in @@ -1627,6 +1677,31 @@ func (in *TextAccessLog) DeepCopy() *TextAccessLog { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Timeout) DeepCopyInto(out *Timeout) { + *out = *in + if in.TCP != nil { + in, out := &in.TCP, &out.TCP + *out = new(TCPTimeout) + (*in).DeepCopyInto(*out) + } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(HTTPTimeout) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Timeout. +func (in *Timeout) DeepCopy() *Timeout { + if in == nil { + return nil + } + out := new(Timeout) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Tracing) DeepCopyInto(out *Tracing) { *out = *in diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index 74cb01c9101..fbbdb0029f8 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -24,7 +24,6 @@ import ( "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" - "k8s.io/utils/ptr" "github.com/envoyproxy/gateway/internal/ir" ) @@ -33,6 +32,7 @@ const ( extensionOptionsKey = "envoy.extensions.upstreams.http.v3.HttpProtocolOptions" // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-field-config-cluster-v3-cluster-per-connection-buffer-limit-bytes tcpClusterPerConnectionBufferLimitBytes = 32768 + tcpClusterPerConnectTimeout = 10 * time.Second ) type xdsClusterArgs struct { @@ -45,6 +45,7 @@ type xdsClusterArgs struct { circuitBreaker *ir.CircuitBreaker healthCheck *ir.HealthCheck http1Settings *ir.HTTP1Settings + timeout *ir.Timeout } type EndpointType int @@ -57,7 +58,6 @@ const ( func buildXdsCluster(args *xdsClusterArgs) *clusterv3.Cluster { cluster := &clusterv3.Cluster{ Name: args.name, - ConnectTimeout: durationpb.New(10 * time.Second), DnsLookupFamily: clusterv3.Cluster_V4_ONLY, CommonLbConfig: &clusterv3.Cluster_CommonLbConfig{ LocalityConfigSpecifier: &clusterv3.Cluster_CommonLbConfig_LocalityWeightedLbConfig_{ @@ -66,6 +66,8 @@ func buildXdsCluster(args *xdsClusterArgs) *clusterv3.Cluster { PerConnectionBufferLimitBytes: wrapperspb.UInt32(tcpClusterPerConnectionBufferLimitBytes), } + cluster.ConnectTimeout = buildConnectTimeout(args.timeout) + // Set Proxy Protocol if args.proxyProtocol != nil { cluster.TransportSocket = buildProxyProtocolSocket(args.proxyProtocol, args.tSocket) @@ -90,16 +92,11 @@ func buildXdsCluster(args *xdsClusterArgs) *clusterv3.Cluster { cluster.RespectDnsTtl = true } - isHTTP2 := false - for _, ds := range args.settings { - if ds.Protocol == ir.GRPC || - ds.Protocol == ir.HTTP2 { - isHTTP2 = true - break - } + // build common, HTTP/1 and HTTP/2 protocol options for cluster + epo := buildTypedExtensionProtocolOptions(args) + if epo != nil { + cluster.TypedExtensionProtocolOptions = epo } - cluster.TypedExtensionProtocolOptions = buildTypedExtensionProtocolOptions(isHTTP2, - ptr.Deref(args.http1Settings, ir.HTTP1Settings{})) // Set Load Balancer policy //nolint:gocritic @@ -317,26 +314,61 @@ func buildXdsClusterLoadAssignment(clusterName string, destSettings []*ir.Destin return &endpointv3.ClusterLoadAssignment{ClusterName: clusterName, Endpoints: localities} } -func buildTypedExtensionProtocolOptions(http2 bool, http1Opts ir.HTTP1Settings) map[string]*anypb.Any { - if !http2 && !http1Opts.EnableTrailers && !http1Opts.PreserveHeaderCase { +func buildTypedExtensionProtocolOptions(args *xdsClusterArgs) map[string]*anypb.Any { + requiresHTTP2Options := false + for _, ds := range args.settings { + if ds.Protocol == ir.GRPC || + ds.Protocol == ir.HTTP2 { + requiresHTTP2Options = true + break + } + } + + requiresCommonHTTPOptions := args.timeout != nil && args.timeout.HTTP != nil && + (args.timeout.HTTP.MaxConnectionDuration != nil || args.timeout.HTTP.ConnectionIdleTimeout != nil) + + requiresHTTP1Options := args.http1Settings != nil && (args.http1Settings.EnableTrailers || args.http1Settings.PreserveHeaderCase) + + if !(requiresCommonHTTPOptions || requiresHTTP1Options || requiresHTTP2Options) { return nil } - var anyProtocolOptions *anypb.Any protocolOptions := httpv3.HttpProtocolOptions{} - if http2 { + + if requiresCommonHTTPOptions { + protocolOptions.CommonHttpProtocolOptions = &corev3.HttpProtocolOptions{} + + if args.timeout.HTTP.ConnectionIdleTimeout != nil { + protocolOptions.CommonHttpProtocolOptions.IdleTimeout = + durationpb.New(args.timeout.HTTP.ConnectionIdleTimeout.Duration) + } + + if args.timeout.HTTP.MaxConnectionDuration != nil { + protocolOptions.CommonHttpProtocolOptions.MaxConnectionDuration = + durationpb.New(args.timeout.HTTP.MaxConnectionDuration.Duration) + } + + } + + // When setting any Typed Extension Protocol Options, UpstreamProtocolOptions are mandatory + // If translation requires HTTP2 enablement or HTTP1 trailers, set appropriate setting + // Default to http1 otherwise + // TODO: If the cluster is TLS enabled, use AutoHTTPConfig instead of ExplicitHttpConfig + // so that when ALPN is supported enabling trailers doesn't force HTTP/1.1 + switch { + case requiresHTTP2Options: protocolOptions.UpstreamProtocolOptions = &httpv3.HttpProtocolOptions_ExplicitHttpConfig_{ ExplicitHttpConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig{ ProtocolConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{}, }, } - } else if http1Opts.EnableTrailers || http1Opts.PreserveHeaderCase { - opts := &corev3.Http1ProtocolOptions{ - EnableTrailers: http1Opts.EnableTrailers, + case requiresHTTP1Options: + http1opts := &corev3.Http1ProtocolOptions{ + EnableTrailers: args.http1Settings.EnableTrailers, } - if http1Opts.PreserveHeaderCase { + if args.http1Settings.PreserveHeaderCase { preservecaseAny, _ := anypb.New(&preservecasev3.PreserveCaseFormatterConfig{}) - opts.HeaderKeyFormat = &corev3.Http1ProtocolOptions_HeaderKeyFormat{ + http1opts.HeaderKeyFormat = &corev3.Http1ProtocolOptions_HeaderKeyFormat{ HeaderFormat: &corev3.Http1ProtocolOptions_HeaderKeyFormat_StatefulFormatter{ StatefulFormatter: &corev3.TypedExtensionConfig{ Name: "preserve_case", @@ -345,17 +377,22 @@ func buildTypedExtensionProtocolOptions(http2 bool, http1Opts ir.HTTP1Settings) }, } } - // TODO: If the cluster is TLS enabled, use AutoHTTPConfig instead of ExplicitHttpConfig - // so that when ALPN is supported setting HTTP/1.1 options doesn't force HTTP/1.1 protocolOptions.UpstreamProtocolOptions = &httpv3.HttpProtocolOptions_ExplicitHttpConfig_{ ExplicitHttpConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig{ ProtocolConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{ - HttpProtocolOptions: opts, + HttpProtocolOptions: http1opts, }, }, } + default: + protocolOptions.UpstreamProtocolOptions = &httpv3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig{ + ProtocolConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{}, + }, + } } - anyProtocolOptions, _ = anypb.New(&protocolOptions) + + anyProtocolOptions, _ := anypb.New(&protocolOptions) extensionOptions := map[string]*anypb.Any{ extensionOptionsKey: anyProtocolOptions, @@ -420,3 +457,10 @@ func buildProxyProtocolSocket(proxyProtocol *ir.ProxyProtocol, tSocket *corev3.T }, } } + +func buildConnectTimeout(to *ir.Timeout) *durationpb.Duration { + if to != nil && to.TCP != nil && to.TCP.ConnectTimeout != nil { + return durationpb.New(to.TCP.ConnectTimeout.Duration) + } + return durationpb.New(tcpClusterPerConnectTimeout) +} diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index b5dd1d61a12..5e5fda0bf93 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -72,8 +72,9 @@ func buildXdsRoute(httpRoute *ir.HTTPRoute) (*routev3.Route, error) { } // Timeouts - if router.GetRoute() != nil && httpRoute.Timeout != nil { - router.GetRoute().Timeout = durationpb.New(httpRoute.Timeout.Duration) + if router.GetRoute() != nil && httpRoute.Timeout != nil && httpRoute.Timeout.HTTP != nil && + httpRoute.Timeout.HTTP.RequestTimeout != nil { + router.GetRoute().Timeout = durationpb.New(httpRoute.Timeout.HTTP.RequestTimeout.Duration) } // Add per route filter configs to the route, if needed. diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-timeout.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-timeout.yaml index 355f07279d4..da1a4243aa3 100644 --- a/internal/xds/translator/testdata/in/xds-ir/http-route-timeout.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-timeout.yaml @@ -10,7 +10,9 @@ http: routes: - name: "first-route" hostname: "*" - timeout: 5s + timeout: + http: + requestTimeout: 5s headerMatches: - name: user stringMatch: diff --git a/internal/xds/translator/testdata/in/xds-ir/timeout.yaml b/internal/xds/translator/testdata/in/xds-ir/timeout.yaml new file mode 100644 index 00000000000..daee154f055 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/timeout.yaml @@ -0,0 +1,24 @@ +http: +- name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "first-route" + hostname: "*" + timeout: + tcp: + connectTimeout: "31s" + http: + connectionIdleTimeout: "32s" + maxConnectionDuration: "33s" + destination: + name: "first-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 diff --git a/internal/xds/translator/testdata/out/xds-ir/timeout.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/timeout.clusters.yaml new file mode 100644 index 00000000000..7515ac9e7d6 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/timeout.clusters.yaml @@ -0,0 +1,22 @@ +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 31s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: first-route-dest + lbPolicy: LEAST_REQUEST + name: first-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + commonHttpProtocolOptions: + idleTimeout: 32s + maxConnectionDuration: 33s + explicitHttpConfig: + httpProtocolOptions: {} diff --git a/internal/xds/translator/testdata/out/xds-ir/timeout.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/timeout.endpoints.yaml new file mode 100644 index 00000000000..3b3f2d09076 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/timeout.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: first-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: first-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/timeout.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/timeout.listeners.yaml new file mode 100644 index 00000000000..73ee1b42ef6 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/timeout.listeners.yaml @@ -0,0 +1,33 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + statPrefix: http + upgradeConfigs: + - upgradeType: websocket + useRemoteAddress: true + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/timeout.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/timeout.routes.yaml new file mode 100644 index 00000000000..2734c7cc42a --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/timeout.routes.yaml @@ -0,0 +1,12 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + prefix: / + name: first-route + route: + cluster: first-route-dest diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index 3eb87eb6b6c..be6a7ff66db 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -506,6 +506,7 @@ func processXdsCluster(tCtx *types.ResourceVersionTable, httpRoute *ir.HTTPRoute circuitBreaker: httpRoute.CircuitBreaker, healthCheck: httpRoute.HealthCheck, http1Settings: http1Settings, + timeout: httpRoute.Timeout, }); err != nil && !errors.Is(err, ErrXdsClusterExists) { return err } diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index 7d534d456ae..7e200d848cd 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -243,6 +243,9 @@ func TestTranslateXds(t *testing.T) { { name: "http1-preserve-case", }, + { + name: "timeout", + }, } for _, tc := range testCases {