feat: support FQDN address type for EndpointSlice (#2138)

* initial implementation of supporting FQDN address type for EndpointSlice Signed-off-by: sh2 <shawnhxh@outlook.com> * add support to endpointslice fqdn address type with simple approach Signed-off-by: sh2 <shawnhxh@outlook.com> * fix ci unit test Signed-off-by: sh2 <shawnhxh@outlook.com> * resolve comments Signed-off-by: sh2 <shawnhxh@outlook.com> * add addrtype check and refactor condition message Signed-off-by: sh2 <shawnhxh@outlook.com> --------- Signed-off-by: sh2 <shawnhxh@outlook.com> Co-authored-by: zirain <zirain2009@gmail.com>
envoyproxy · Dec 6, 2023 · a89d95e · a89d95e
1 parent ebc18d8
commit a89d95e
Show file tree

Hide file tree

Showing 111 changed files with 1,451 additions and 190 deletions.
diff --git a/internal/gatewayapi/resource.go b/internal/gatewayapi/resource.go
@@ -108,7 +108,7 @@ func (r *Resources) GetSecret(namespace, name string) *v1.Secret {
 }
 
 func (r *Resources) GetEndpointSlicesForBackend(svcNamespace, svcName string, backendKind string) []*discoveryv1.EndpointSlice {
-	endpointSlices := []*discoveryv1.EndpointSlice{}
+	var endpointSlices []*discoveryv1.EndpointSlice
 	for _, endpointSlice := range r.EndpointSlices {
 		var backendSelectorLabel string
 		switch backendKind {

diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go
@@ -149,8 +149,14 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe
 		// a unique Xds IR HTTPRoute per match.
 		var ruleRoutes = t.processHTTPRouteRule(httpRoute, ruleIdx, httpFiltersContext, rule)
 
+		dstAddrTypeMap := make(map[ir.DestinationAddressType]int)
+
 		for _, backendRef := range rule.BackendRefs {
 			ds, backendWeight := t.processDestination(backendRef.BackendRef, parentRef, httpRoute, resources)
+			if !t.EndpointRoutingDisabled && ds != nil && len(ds.Endpoints) > 0 && ds.AddressType != nil {
+				dstAddrTypeMap[*ds.AddressType]++
+			}
+
 			for _, route := range ruleRoutes {
 				// If the route already has a direct response or redirect configured, then it was from a filter so skip
 				// processing any destinations for this route.
@@ -163,14 +169,22 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe
 						}
 						route.Destination.Settings = append(route.Destination.Settings, ds)
 						route.BackendWeights.Valid += backendWeight
-
 					} else {
 						route.BackendWeights.Invalid += backendWeight
 					}
 				}
 			}
 		}
 
+		// TODO: support mixed endpointslice address type between backendRefs
+		if !t.EndpointRoutingDisabled && len(dstAddrTypeMap) > 1 {
+			parentRef.SetCondition(httpRoute,
+				gwapiv1.RouteConditionResolvedRefs,
+				metav1.ConditionFalse,
+				gwapiv1a1.RouteReasonResolvedRefs,
+				"Mixed endpointslice address type between backendRefs is not supported")
+		}
+
 		// If the route has no valid backends then just use a direct response and don't fuss with weighted responses
 		for _, ruleRoute := range ruleRoutes {
 			if ruleRoute.BackendWeights.Invalid > 0 && ruleRoute.Destination == nil {
@@ -980,7 +994,10 @@ func (t *Translator) processDestination(backendRef gwapiv1.BackendRef,
 		return nil, weight
 	}
 
-	var endpoints []*ir.DestinationEndpoint
+	var (
+		endpoints []*ir.DestinationEndpoint
+		addrType  *ir.DestinationAddressType
+	)
 	protocol := inspectAppProtocolByRouteKind(routeType)
 	switch KindDerefOr(backendRef.Kind, KindService) {
 	case KindServiceImport:
@@ -992,9 +1009,10 @@ func (t *Translator) processDestination(backendRef gwapiv1.BackendRef,
 				break
 			}
 		}
+
 		if !t.EndpointRoutingDisabled {
 			endpointSlices := resources.GetEndpointSlicesForBackend(backendNamespace, string(backendRef.Name), KindDerefOr(backendRef.Kind, KindService))
-			endpoints = getIREndpointsFromEndpointSlice(endpointSlices, servicePort.Name, servicePort.Protocol)
+			endpoints, addrType = getIREndpointsFromEndpointSlices(endpointSlices, servicePort.Name, servicePort.Protocol)
 		} else {
 			backendIps := resources.GetServiceImport(backendNamespace, string(backendRef.Name)).Spec.IPs
 			for _, ip := range backendIps {
@@ -1023,7 +1041,7 @@ func (t *Translator) processDestination(backendRef gwapiv1.BackendRef,
 		// Route to endpoints by default
 		if !t.EndpointRoutingDisabled {
 			endpointSlices := resources.GetEndpointSlicesForBackend(backendNamespace, string(backendRef.Name), KindDerefOr(backendRef.Kind, KindService))
-			endpoints = getIREndpointsFromEndpointSlice(endpointSlices, servicePort.Name, servicePort.Protocol)
+			endpoints, addrType = getIREndpointsFromEndpointSlices(endpointSlices, servicePort.Name, servicePort.Protocol)
 		} else {
 			// Fall back to Service ClusterIP routing
 			ep := ir.NewDestEndpoint(
@@ -1033,10 +1051,20 @@ func (t *Translator) processDestination(backendRef gwapiv1.BackendRef,
 		}
 	}
 
+	// TODO: support mixed endpointslice address type for the same backendRef
+	if !t.EndpointRoutingDisabled && addrType != nil && *addrType == ir.MIXED {
+		parentRef.SetCondition(route,
+			gwapiv1.RouteConditionResolvedRefs,
+			metav1.ConditionFalse,
+			gwapiv1a1.RouteReasonResolvedRefs,
+			"Mixed endpointslice address type for the same backendRef is not supported")
+	}
+
 	ds = &ir.DestinationSetting{
-		Weight:    &weight,
-		Protocol:  protocol,
-		Endpoints: endpoints,
+		Weight:      &weight,
+		Protocol:    protocol,
+		Endpoints:   endpoints,
+		AddressType: addrType,
 	}
 	return ds, weight
 }
@@ -1137,22 +1165,51 @@ func (t *Translator) processAllowedListenersForParentRefs(routeContext RouteCont
 	return relevantRoute
 }
 
-func getIREndpointsFromEndpointSlice(endpointSlices []*discoveryv1.EndpointSlice, portName string, portProtocol corev1.Protocol) []*ir.DestinationEndpoint {
-	endpoints := []*ir.DestinationEndpoint{}
+func getIREndpointsFromEndpointSlices(endpointSlices []*discoveryv1.EndpointSlice, portName string, portProtocol corev1.Protocol) ([]*ir.DestinationEndpoint, *ir.DestinationAddressType) {
+	var (
+		dstEndpoints []*ir.DestinationEndpoint
+		dstAddrType  *ir.DestinationAddressType
+	)
+
+	addrTypeMap := make(map[ir.DestinationAddressType]int)
 	for _, endpointSlice := range endpointSlices {
-		for _, endpoint := range endpointSlice.Endpoints {
-			for _, endpointPort := range endpointSlice.Ports {
-				// Check if the endpoint port matches the service port
-				// and if endpoint is Ready
-				if *endpointPort.Name == portName &&
-					*endpointPort.Protocol == portProtocol &&
-					*endpoint.Conditions.Ready {
-					for _, address := range endpoint.Addresses {
-						ep := ir.NewDestEndpoint(
-							address,
-							uint32(*endpointPort.Port))
-						endpoints = append(endpoints, ep)
-					}
+		if endpointSlice.AddressType == discoveryv1.AddressTypeFQDN {
+			addrTypeMap[ir.FQDN]++
+		} else {
+			addrTypeMap[ir.IP]++
+		}
+		endpoints := getIREndpointsFromEndpointSlice(endpointSlice, portName, portProtocol)
+		dstEndpoints = append(dstEndpoints, endpoints...)
+	}
+
+	for addrTypeState, addrTypeCounts := range addrTypeMap {
+		if addrTypeCounts == len(endpointSlices) {
+			dstAddrType = ptr.To(addrTypeState)
+			break
+		}
+	}
+
+	if len(addrTypeMap) > 0 && dstAddrType == nil {
+		dstAddrType = ptr.To(ir.MIXED)
+	}
+
+	return dstEndpoints, dstAddrType
+}
+
+func getIREndpointsFromEndpointSlice(endpointSlice *discoveryv1.EndpointSlice, portName string, portProtocol corev1.Protocol) []*ir.DestinationEndpoint {
+	var endpoints []*ir.DestinationEndpoint
+	for _, endpoint := range endpointSlice.Endpoints {
+		for _, endpointPort := range endpointSlice.Ports {
+			// Check if the endpoint port matches the service port
+			// and if endpoint is Ready
+			if *endpointPort.Name == portName &&
+				*endpointPort.Protocol == portProtocol &&
+				*endpoint.Conditions.Ready {
+				for _, address := range endpoint.Addresses {
+					ep := ir.NewDestEndpoint(
+						address,
+						uint32(*endpointPort.Port))
+					endpoints = append(endpoints, ep)
 				}
 			}
 		}

diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-loadbalancer.out.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-loadbalancer.out.yaml
@@ -329,7 +329,8 @@ xdsIR:
         destination:
           name: grpcroute/default/grpcroute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: GRPC
@@ -356,7 +357,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-2/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP
@@ -377,7 +379,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-proxyprotocol.out.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-proxyprotocol.out.yaml
@@ -243,7 +243,8 @@ xdsIR:
         destination:
           name: grpcroute/default/grpcroute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: GRPC
@@ -270,7 +271,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-ratelimit.out.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-ratelimit.out.yaml
@@ -263,7 +263,8 @@ xdsIR:
         destination:
           name: grpcroute/default/grpcroute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: GRPC
@@ -300,7 +301,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/internal/gatewayapi/testdata/extensions/httproute-with-valid-extension-filter.out.yaml b/internal/gatewayapi/testdata/extensions/httproute-with-valid-extension-filter.out.yaml
@@ -119,7 +119,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/internal/gatewayapi/testdata/gateway-allows-same-namespace-with-allowed-httproute.out.yaml b/internal/gatewayapi/testdata/gateway-allows-same-namespace-with-allowed-httproute.out.yaml
@@ -108,7 +108,8 @@ xdsIR:
         destination:
           name: httproute/envoy-gateway/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/...ata/gateway-with-listener-with-tls-secret-in-other-namespace-allowed-by-refgrant.out.yaml b/...ata/gateway-with-listener-with-tls-secret-in-other-namespace-allowed-by-refgrant.out.yaml
@@ -115,7 +115,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/...nal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml b/...nal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml
@@ -181,7 +181,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP
@@ -201,7 +202,8 @@ xdsIR:
       destination:
         name: tlsroute/default/tlsroute-1/rule/-1
         settings:
-        - endpoints:
+        - addressType: IP
+          endpoints:
           - host: 7.7.7.7
             port: 8080
           protocol: HTTPS

diff --git a/...istener-with-valid-multiple-tls-configuration-with-same-algorithm-different-fqdn.out.yaml b/...istener-with-valid-multiple-tls-configuration-with-same-algorithm-different-fqdn.out.yaml
@@ -117,7 +117,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/.../gatewayapi/testdata/gateway-with-listener-with-valid-multiple-tls-configuration.out.yaml b/.../gatewayapi/testdata/gateway-with-listener-with-valid-multiple-tls-configuration.out.yaml
@@ -117,7 +117,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-valid-tls-configuration.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-valid-tls-configuration.out.yaml
@@ -114,7 +114,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/internal/gatewayapi/testdata/gateway-with-preexisting-status-condition.out.yaml b/internal/gatewayapi/testdata/gateway-with-preexisting-status-condition.out.yaml
@@ -108,7 +108,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/internal/gatewayapi/testdata/gateway-with-single-listener-with-multiple-tcproutes.out.yaml b/internal/gatewayapi/testdata/gateway-with-single-listener-with-multiple-tcproutes.out.yaml
@@ -125,7 +125,8 @@ xdsIR:
       destination:
         name: tcproute/default/tcproute-1/rule/-1
         settings:
-        - endpoints:
+        - addressType: IP
+          endpoints:
           - host: 7.7.7.7
             port: 8163
           protocol: TCP

diff --git a/internal/gatewayapi/testdata/gateway-with-single-listener-with-multiple-udproutes.out.yaml b/internal/gatewayapi/testdata/gateway-with-single-listener-with-multiple-udproutes.out.yaml
@@ -125,7 +125,8 @@ xdsIR:
       destination:
         name: udproute/default/udproute-1/rule/-1
         settings:
-        - endpoints:
+        - addressType: IP
+          endpoints:
           - host: 7.7.7.7
             port: 8162
           protocol: UDP

diff --git a/internal/gatewayapi/testdata/gateway-with-stale-status-condition.out.yaml b/internal/gatewayapi/testdata/gateway-with-stale-status-condition.out.yaml
@@ -114,7 +114,8 @@ xdsIR:
         destination:
           name: httproute/default/httproute-1/rule/0
           settings:
-          - endpoints:
+          - addressType: IP
+            endpoints:
             - host: 7.7.7.7
               port: 8080
             protocol: HTTP

diff --git a/internal/gatewayapi/testdata/gateway-with-two-listeners-on-same-tcp-or-tls-port.out.yaml b/internal/gatewayapi/testdata/gateway-with-two-listeners-on-same-tcp-or-tls-port.out.yaml
@@ -123,7 +123,8 @@ xdsIR:
       destination:
         name: tcproute/default/tcproute-1/rule/-1
         settings:
-        - endpoints:
+        - addressType: IP
+          endpoints:
           - host: 7.7.7.7
             port: 8163
           protocol: TCP

diff --git a/internal/gatewayapi/testdata/gateway-with-two-listeners-on-same-udp-port.out.yaml b/internal/gatewayapi/testdata/gateway-with-two-listeners-on-same-udp-port.out.yaml
@@ -121,7 +121,8 @@ xdsIR:
       destination:
         name: udproute/default/udproute-1/rule/-1
         settings:
-        - endpoints:
+        - addressType: IP
+          endpoints:
           - host: 7.7.7.7
             port: 8162
           protocol: UDP