From c14229e8e4bca95f489fad48d1222bfa7b9eef27 Mon Sep 17 00:00:00 2001 From: Dingkang Li Date: Wed, 5 Jun 2024 16:21:29 +0800 Subject: [PATCH] Set zone for endpoint and arrange endpoints by zone in EDS Signed-off-by: Dingkang Li --- internal/gatewayapi/route.go | 4 + ...er-with-core-backendrefs-diff-zone.in.yaml | 201 ++++++++++++++++ ...r-with-core-backendrefs-diff-zone.out.yaml | 214 ++++++++++++++++++ internal/ir/xds.go | 2 + internal/ir/zz_generated.deepcopy.go | 5 + internal/xds/translator/cluster.go | 49 ++-- .../in/xds-ir/http-route-with-zone.yaml | 32 +++ .../out/xds-ir/ext-proc.endpoints.yaml | 16 -- ...te-weighted-invalid-backend.endpoints.yaml | 3 - .../xds-ir/http-route-with-zone.clusters.yaml | 17 ++ .../http-route-with-zone.endpoints.yaml | 24 ++ .../http-route-with-zone.listeners.yaml | 35 +++ .../xds-ir/http-route-with-zone.routes.yaml | 25 ++ 13 files changed, 594 insertions(+), 33 deletions(-) create mode 100644 internal/gatewayapi/testdata/httproute-attaching-to-listener-with-core-backendrefs-diff-zone.in.yaml create mode 100755 internal/gatewayapi/testdata/httproute-attaching-to-listener-with-core-backendrefs-diff-zone.out.yaml create mode 100644 internal/xds/translator/testdata/in/xds-ir/http-route-with-zone.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.routes.yaml diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index 21b0485ef8f..1b32ea30911 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -1488,6 +1488,10 @@ func getIREndpointsFromEndpointSlice(endpointSlice *discoveryv1.EndpointSlice, p ep := ir.NewDestEndpoint( address, uint32(*endpointPort.Port)) + + // EndpointSlice may set zone for individual endpoint, use it anyway. + ep.Zone = endpoint.Zone + endpoints = append(endpoints, ep) } } diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-listener-with-core-backendrefs-diff-zone.in.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-listener-with-core-backendrefs-diff-zone.in.yaml new file mode 100644 index 00000000000..baaf399600c --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-attaching-to-listener-with-core-backendrefs-diff-zone.in.yaml @@ -0,0 +1,201 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-static + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - matches: + - path: + value: "/1" + backendRefs: + - name: service-ip + port: 8080 + - name: service-import-ip + group: multicluster.x-k8s.io + kind: ServiceImport + port: 8081 + - apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-fqdn + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - matches: + - path: + value: "/2" + backendRefs: + - name: service-fqdn + port: 8080 + - name: service-import-fqdn + group: multicluster.x-k8s.io + kind: ServiceImport + port: 8081 +services: + - apiVersion: v1 + kind: Service + metadata: + namespace: default + name: service-fqdn + spec: + clusterIP: 1.1.1.1 + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + - apiVersion: v1 + kind: Service + metadata: + namespace: default + name: service-ip + spec: + clusterIP: 2.2.2.2 + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 +serviceImports: + - apiVersion: multicluster.x-k8s.io/v1alpha1 + kind: ServiceImport + metadata: + namespace: default + name: service-import-fqdn + spec: + ips: + - 7.7.7.7 + ports: + - port: 8081 + name: http + protocol: TCP + - apiVersion: multicluster.x-k8s.io/v1alpha1 + kind: ServiceImport + metadata: + namespace: default + name: service-import-ip + spec: + ips: + - 8.8.8.8 + ports: + - port: 8081 + name: http + protocol: TCP +endpointSlices: + - apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: endpointslice-service-fqdn + namespace: default + labels: + kubernetes.io/service-name: service-fqdn + addressType: FQDN + ports: + - name: http + protocol: TCP + port: 8080 + endpoints: + - addresses: + - "bar.foo" + zone: "a" + conditions: + ready: true + - addresses: + - "abc.xyz" + zone: "b" + conditions: + ready: true + - apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: endpointslice-service-ip + namespace: default + labels: + kubernetes.io/service-name: service-ip + addressType: IP + ports: + - name: http + protocol: TCP + port: 8080 + endpoints: + - addresses: + - "4.3.2.1" + zone: "a" + conditions: + ready: true + - addresses: + - "8.7.6.5" + zone: "b" + conditions: + ready: true + - apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: service-import-fqdn + namespace: default + labels: + multicluster.kubernetes.io/service-name: service-import-fqdn + addressType: FQDN + ports: + - name: http + protocol: TCP + port: 8081 + endpoints: + - addresses: + - "foo.bar" + zone: "a" + conditions: + ready: true + - addresses: + - "xyz.abc" + zone: "b" + conditions: + ready: true + - apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: service-import-ip + namespace: default + labels: + multicluster.kubernetes.io/service-name: service-import-ip + addressType: IPv4 + ports: + - name: http + protocol: TCP + port: 8081 + endpoints: + - addresses: + - "1.2.3.4" + zone: "a" + conditions: + ready: true + - addresses: + - "5.6.7.8" + zone: "b" + conditions: + ready: true diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-listener-with-core-backendrefs-diff-zone.out.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-listener-with-core-backendrefs-diff-zone.out.yaml new file mode 100755 index 00000000000..af5a0bb20af --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-attaching-to-listener-with-core-backendrefs-diff-zone.out.yaml @@ -0,0 +1,214 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 2 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-static + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-ip + port: 8080 + - group: multicluster.x-k8s.io + kind: ServiceImport + name: service-import-ip + port: 8081 + matches: + - path: + value: /1 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-fqdn + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-fqdn + port: 8080 + - group: multicluster.x-k8s.io + kind: ServiceImport + name: service-import-fqdn + port: 8081 + matches: + - path: + value: /2 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + name: httproute/default/httproute-static/rule/0 + settings: + - addressType: IP + endpoints: + - host: 4.3.2.1 + port: 8080 + zone: a + - host: 8.7.6.5 + port: 8080 + zone: b + protocol: HTTP + weight: 1 + - addressType: IP + endpoints: + - host: 1.2.3.4 + port: 8081 + zone: a + - host: 5.6.7.8 + port: 8081 + zone: b + protocol: HTTP + weight: 1 + hostname: '*' + isHTTP2: false + name: httproute/default/httproute-static/rule/0/match/0/* + pathMatch: + distinct: false + name: "" + prefix: /1 + - destination: + name: httproute/default/httproute-fqdn/rule/0 + settings: + - addressType: FQDN + endpoints: + - host: bar.foo + port: 8080 + zone: a + - host: abc.xyz + port: 8080 + zone: b + protocol: HTTP + weight: 1 + - addressType: FQDN + endpoints: + - host: foo.bar + port: 8081 + zone: a + - host: xyz.abc + port: 8081 + zone: b + protocol: HTTP + weight: 1 + hostname: '*' + isHTTP2: false + name: httproute/default/httproute-fqdn/rule/0/match/0/* + pathMatch: + distinct: false + name: "" + prefix: /2 diff --git a/internal/ir/xds.go b/internal/ir/xds.go index 6fec4597268..c90e39f5153 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -1093,6 +1093,8 @@ type DestinationEndpoint struct { Port uint32 `json:"port" yaml:"port"` // Path refers to the Unix Domain Socket Path *string `json:"path,omitempty" yaml:"path,omitempty"` + // Zone refers to the zone of the endpoint, maybe unset. + Zone *string `json:"zone,omitempty" yaml:"zone,omitempty"` } // Validate the fields within the DestinationEndpoint structure diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index c760c66055a..5d53a86b509 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -455,6 +455,11 @@ func (in *DestinationEndpoint) DeepCopyInto(out *DestinationEndpoint) { *out = new(string) **out = **in } + if in.Zone != nil { + in, out := &in.Zone, &out.Zone + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DestinationEndpoint. diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index e646f410944..d1d81d47b78 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -378,7 +378,8 @@ func buildXdsClusterLoadAssignment(clusterName string, destSettings []*ir.Destin localities := make([]*endpointv3.LocalityLbEndpoints, 0, len(destSettings)) for i, ds := range destSettings { - endpoints := make([]*endpointv3.LbEndpoint, 0, len(ds.Endpoints)) + // Arrange endpoints by zone. + endpointsByZone := make(map[string][]*endpointv3.LbEndpoint) var metadata *corev3.Metadata if ds.TLS != nil { @@ -404,18 +405,18 @@ func buildXdsClusterLoadAssignment(clusterName string, destSettings []*ir.Destin } // Set default weight of 1 for all endpoints. lbEndpoint.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 1} - endpoints = append(endpoints, lbEndpoint) - } - // Envoy requires a distinct region to be set for each LocalityLbEndpoints. - // If we don't do this, Envoy will merge all LocalityLbEndpoints into one. - // We use the name of the backendRef as a pseudo region name. - locality := &endpointv3.LocalityLbEndpoints{ - Locality: &corev3.Locality{ - Region: fmt.Sprintf("%s/backend/%d", clusterName, i), - }, - LbEndpoints: endpoints, - Priority: 0, + // If irEp has unset zone, leave it empty. + var zone string + if irEp.Zone != nil { + zone = *irEp.Zone + } + + if _, ok := endpointsByZone[zone]; !ok { + endpointsByZone[zone] = make([]*endpointv3.LbEndpoint, 0) + } + + endpointsByZone[zone] = append(endpointsByZone[zone], lbEndpoint) } // Set locality weight @@ -425,9 +426,29 @@ func buildXdsClusterLoadAssignment(clusterName string, destSettings []*ir.Destin } else { weight = 1 } - locality.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: weight} - localities = append(localities, locality) + for zone := range endpointsByZone { + endpoints := endpointsByZone[zone] + + // Envoy requires a distinct region to be set for each LocalityLbEndpoints. + // If we don't do this, Envoy will merge all LocalityLbEndpoints into one. + // We use the name of the backendRef as a pseudo region name. + locality := &endpointv3.LocalityLbEndpoints{ + Locality: &corev3.Locality{ + Region: fmt.Sprintf("%s/backend/%d", clusterName, i), + }, + LbEndpoints: endpoints, + Priority: 0, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: weight}, + } + + // If endpoints belong to a non-empty zone, set it as the zone name. + if len(zone) != 0 { + locality.Locality.Zone = zone + } + + localities = append(localities, locality) + } } return &endpointv3.ClusterLoadAssignment{ClusterName: clusterName, Endpoints: localities} } diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-with-zone.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-with-zone.yaml new file mode 100644 index 00000000000..327d127c7d1 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-with-zone.yaml @@ -0,0 +1,32 @@ +http: +- name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "first-route" + hostname: "*" + headerMatches: + - name: user + stringMatch: + exact: "jason" + - name: test + stringMatch: + suffix: "end" + queryParamMatches: + - name: "debug" + exact: "yes" + destination: + name: "first-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + zone: "a" + - host: "5.6.7.8" + port: 50000 + zone: "b" diff --git a/internal/xds/translator/testdata/out/xds-ir/ext-proc.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/ext-proc.endpoints.yaml index 4ec680ce7fd..04382f25a4d 100755 --- a/internal/xds/translator/testdata/out/xds-ir/ext-proc.endpoints.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/ext-proc.endpoints.yaml @@ -23,22 +23,6 @@ locality: region: httproute/default/httproute-2/rule/0/backend/0 - clusterName: envoyextensionpolicy/default/policy-for-route-2/0/grpc-backend-4 - endpoints: - - loadBalancingWeight: 1 - locality: - region: envoyextensionpolicy/default/policy-for-route-2/0/grpc-backend-4/backend/0 - clusterName: envoyextensionpolicy/default/policy-for-route-1/0/grpc-backend-2 - endpoints: - - loadBalancingWeight: 1 - locality: - region: envoyextensionpolicy/default/policy-for-route-1/0/grpc-backend-2/backend/0 - clusterName: envoyextensionpolicy/envoy-gateway/policy-for-gateway-2/0/grpc-backend-3 - endpoints: - - loadBalancingWeight: 1 - locality: - region: envoyextensionpolicy/envoy-gateway/policy-for-gateway-2/0/grpc-backend-3/backend/0 - clusterName: envoyextensionpolicy/envoy-gateway/policy-for-gateway-1/0/grpc-backend - endpoints: - - loadBalancingWeight: 1 - locality: - region: envoyextensionpolicy/envoy-gateway/policy-for-gateway-1/0/grpc-backend/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml index 7f8a0281325..3b3f2d09076 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml @@ -10,6 +10,3 @@ loadBalancingWeight: 1 locality: region: first-route-dest/backend/0 - - loadBalancingWeight: 1 - locality: - region: first-route-dest/backend/1 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.clusters.yaml new file mode 100644 index 00000000000..d53a7a1b2ce --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.clusters.yaml @@ -0,0 +1,17 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: first-route-dest + lbPolicy: LEAST_REQUEST + name: first-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.endpoints.yaml new file mode 100644 index 00000000000..b9e3985acdb --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.endpoints.yaml @@ -0,0 +1,24 @@ +- clusterName: first-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: first-route-dest/backend/0 + zone: a + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 5.6.7.8 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: first-route-dest/backend/0 + zone: b diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.listeners.yaml new file mode 100644 index 00000000000..67922c7444f --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.listeners.yaml @@ -0,0 +1,35 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http + useRemoteAddress: true + name: first-listener + drainType: MODIFY_ONLY + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.routes.yaml new file mode 100644 index 00000000000..7030f6f4cd7 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-with-zone.routes.yaml @@ -0,0 +1,25 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + headers: + - name: user + stringMatch: + exact: jason + - name: test + stringMatch: + suffix: end + prefix: / + queryParameters: + - name: debug + stringMatch: + exact: "yes" + name: first-route + route: + cluster: first-route-dest + upgradeConfigs: + - upgradeType: websocket