Fix weighted route (envoyproxy#2187)

* add a pseudo region name to locality Signed-off-by: huabing zhao <zhaohuabing@gmail.com> * add ds name to locality Signed-off-by: huabing zhao <zhaohuabing@gmail.com> * Revert "add ds name to locality" This reverts commit e051644. Signed-off-by: huabing zhao <zhaohuabing@gmail.com> * fix test Signed-off-by: huabing zhao <zhaohuabing@gmail.com> * fix test Signed-off-by: huabing zhao <zhaohuabing@gmail.com>
arkodg · Nov 14, 2023 · 6ba9ea4 · 6ba9ea4
1 parent f92da90
commit 6ba9ea4
Show file tree

Hide file tree

Showing 63 changed files with 258 additions and 124 deletions.
diff --git a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml
@@ -598,7 +598,8 @@ xds:
                     portValue: 8000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: httproute/default/backend/rule/0/backend/0
       - endpointConfig:
           '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
           clusterName: grpcroute/default/backend/rule/0
@@ -611,7 +612,8 @@ xds:
                     portValue: 9000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: grpcroute/default/backend/rule/0/backend/0
       - endpointConfig:
           '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
           clusterName: tlsroute/default/backend/rule/-1
@@ -624,7 +626,8 @@ xds:
                     portValue: 3000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: tlsroute/default/backend/rule/-1/backend/0
       - endpointConfig:
           '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
           clusterName: tcproute/default/backend/rule/-1
@@ -637,7 +640,8 @@ xds:
                     portValue: 3000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: tcproute/default/backend/rule/-1/backend/0
       - endpointConfig:
           '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
           clusterName: udproute/default/backend/rule/-1
@@ -650,7 +654,8 @@ xds:
                     portValue: 3000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: udproute/default/backend/rule/-1/backend/0
     - '@type': type.googleapis.com/envoy.admin.v3.ClustersConfigDump
       dynamicActiveClusters:
       - cluster:

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json
@@ -235,7 +235,9 @@
                       }
                     ],
                     "loadBalancingWeight": 1,
-                    "locality": {}
+                    "locality": {
+                      "region": "httproute/default/backend/rule/0/backend/0"
+                    }
                   }
                 ]
               }
@@ -260,7 +262,9 @@
                       }
                     ],
                     "loadBalancingWeight": 1,
-                    "locality": {}
+                    "locality": {
+                      "region": "grpcroute/default/backend/rule/0/backend/0"
+                    }
                   }
                 ]
               }
@@ -285,7 +289,9 @@
                       }
                     ],
                     "loadBalancingWeight": 1,
-                    "locality": {}
+                    "locality": {
+                      "region": "tlsroute/default/backend/rule/-1/backend/0"
+                    }
                   }
                 ]
               }
@@ -310,7 +316,9 @@
                       }
                     ],
                     "loadBalancingWeight": 1,
-                    "locality": {}
+                    "locality": {
+                      "region": "tcproute/default/backend/rule/-1/backend/0"
+                    }
                   }
                 ]
               }
@@ -335,7 +343,9 @@
                       }
                     ],
                     "loadBalancingWeight": 1,
-                    "locality": {}
+                    "locality": {
+                      "region": "udproute/default/backend/rule/-1/backend/0"
+                    }
                   }
                 ]
               }

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml
@@ -131,7 +131,8 @@ xds:
                     portValue: 3000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: httproute/default/backend/rule/0/backend/0
       - endpointConfig:
           '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
           clusterName: grpcroute/default/backend/rule/0
@@ -144,7 +145,8 @@ xds:
                     portValue: 9000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: grpcroute/default/backend/rule/0/backend/0
       - endpointConfig:
           '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
           clusterName: tlsroute/default/backend/rule/-1
@@ -157,7 +159,8 @@ xds:
                     portValue: 3000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: tlsroute/default/backend/rule/-1/backend/0
       - endpointConfig:
           '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
           clusterName: tcproute/default/backend/rule/-1
@@ -170,7 +173,8 @@ xds:
                     portValue: 3000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: tcproute/default/backend/rule/-1/backend/0
       - endpointConfig:
           '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
           clusterName: udproute/default/backend/rule/-1
@@ -183,7 +187,8 @@ xds:
                     portValue: 3000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: udproute/default/backend/rule/-1/backend/0
     - '@type': type.googleapis.com/envoy.admin.v3.ClustersConfigDump
       dynamicActiveClusters:
       - cluster:

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.endpoint.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.endpoint.yaml
@@ -14,7 +14,8 @@ xds:
                   portValue: 3000
             loadBalancingWeight: 1
           loadBalancingWeight: 1
-          locality: {}
+          locality:
+            region: httproute/default/backend/rule/0/backend/0
     - endpointConfig:
         '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
         clusterName: grpcroute/default/backend/rule/0
@@ -27,7 +28,8 @@ xds:
                   portValue: 9000
             loadBalancingWeight: 1
           loadBalancingWeight: 1
-          locality: {}
+          locality:
+            region: grpcroute/default/backend/rule/0/backend/0
     - endpointConfig:
         '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
         clusterName: tlsroute/default/backend/rule/-1
@@ -40,7 +42,8 @@ xds:
                   portValue: 3000
             loadBalancingWeight: 1
           loadBalancingWeight: 1
-          locality: {}
+          locality:
+            region: tlsroute/default/backend/rule/-1/backend/0
     - endpointConfig:
         '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
         clusterName: tcproute/default/backend/rule/-1
@@ -53,7 +56,8 @@ xds:
                   portValue: 3000
             loadBalancingWeight: 1
           loadBalancingWeight: 1
-          locality: {}
+          locality:
+            region: tcproute/default/backend/rule/-1/backend/0
     - endpointConfig:
         '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
         clusterName: udproute/default/backend/rule/-1
@@ -66,4 +70,5 @@ xds:
                   portValue: 3000
             loadBalancingWeight: 1
           loadBalancingWeight: 1
-          locality: {}
+          locality:
+            region: udproute/default/backend/rule/-1/backend/0
diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json
@@ -235,7 +235,9 @@
                       }
                     ],
                     "loadBalancingWeight": 1,
-                    "locality": {}
+                    "locality": {
+                      "region": "httproute/envoy-gateway-system/backend/rule/0/backend/0"
+                    }
                   }
                 ]
               }
@@ -295,7 +297,9 @@
                         }
                       ],
                       "loadBalancingWeight": 1,
-                      "locality": {}
+                      "locality": {
+                        "region": "raw_githubusercontent_com_443/backend/0"
+                      }
                     }
                   ]
                 },

diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml
@@ -131,7 +131,8 @@ xds:
                     portValue: 3000
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: httproute/envoy-gateway-system/backend/rule/0/backend/0
     - '@type': type.googleapis.com/envoy.admin.v3.ClustersConfigDump
       dynamicActiveClusters:
       - cluster:
@@ -169,7 +170,8 @@ xds:
                       portValue: 443
                 loadBalancingWeight: 1
               loadBalancingWeight: 1
-              locality: {}
+              locality:
+                region: raw_githubusercontent_com_443/backend/0
           name: raw_githubusercontent_com_443
           outlierDetection: {}
           perConnectionBufferLimitBytes: 32768

diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.cluster.yaml b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.cluster.yaml
@@ -37,7 +37,8 @@ xds:
                     portValue: 443
               loadBalancingWeight: 1
             loadBalancingWeight: 1
-            locality: {}
+            locality:
+              region: raw_githubusercontent_com_443/backend/0
         name: raw_githubusercontent_com_443
         outlierDetection: {}
         perConnectionBufferLimitBytes: 32768

diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go
@@ -91,7 +91,7 @@ func buildXdsCluster(args *xdsClusterArgs) *clusterv3.Cluster {
 
 func buildXdsClusterLoadAssignment(clusterName string, destSettings []*ir.DestinationSetting) *endpointv3.ClusterLoadAssignment {
 	localities := make([]*endpointv3.LocalityLbEndpoints, 0, len(destSettings))
-	for _, ds := range destSettings {
+	for i, ds := range destSettings {
 
 		endpoints := make([]*endpointv3.LbEndpoint, 0, len(ds.Endpoints))
 
@@ -118,8 +118,13 @@ func buildXdsClusterLoadAssignment(clusterName string, destSettings []*ir.Destin
 			endpoints = append(endpoints, lbEndpoint)
 		}
 
+		// Envoy requires a distinct region to be set for each LocalityLbEndpoints.
+		// If we don't do this, Envoy will merge all LocalityLbEndpoints into one.
+		// We use the name of the backendRef as a pseudo region name.
 		locality := &endpointv3.LocalityLbEndpoints{
-			Locality:    &corev3.Locality{},
+			Locality: &corev3.Locality{
+				Region: fmt.Sprintf("%s/backend/%d", clusterName, i),
+			},
 			LbEndpoints: endpoints,
 			Priority:    0,
 		}

diff --git a/...l/xds/translator/testdata/out/extension-xds-ir/http-route-extension-filter.endpoints.yaml b/...l/xds/translator/testdata/out/extension-xds-ir/http-route-extension-filter.endpoints.yaml
@@ -8,4 +8,5 @@
             portValue: 50000
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: first-route-dest/backend/0
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route.endpoints.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route.endpoints.yaml
@@ -8,4 +8,5 @@
             portValue: 50000
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: first-route-dest/backend/0
diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml
@@ -29,7 +29,8 @@
               portValue: 4317
         loadBalancingWeight: 1
       loadBalancingWeight: 1
-      locality: {}
+      locality:
+        region: accesslog|otel-collector.default.svc.cluster.local|4317/backend/0
   name: accesslog|otel-collector.default.svc.cluster.local|4317
   outlierDetection: {}
   perConnectionBufferLimitBytes: 32768

diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml
@@ -8,4 +8,5 @@
             portValue: 50000
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: direct-route-dest/backend/0
diff --git a/internal/xds/translator/testdata/out/xds-ir/cors.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/cors.endpoints.yaml
@@ -8,4 +8,5 @@
             portValue: 50000
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: first-route-dest/backend/0
diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.endpoints.yaml
@@ -8,4 +8,5 @@
             portValue: 50000
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: direct-route-dest/backend/0
diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-mirror.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-mirror.endpoints.yaml
@@ -8,4 +8,5 @@
             portValue: 50000
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: route-dest/backend/0
diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-matches.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-matches.endpoints.yaml
@@ -8,7 +8,8 @@
             portValue: 8080
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: first-route-dest/backend/0
 - clusterName: second-route-dest
   endpoints:
   - lbEndpoints:
@@ -19,7 +20,8 @@
             portValue: 8080
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: second-route-dest/backend/0
 - clusterName: third-route-dest
   endpoints:
   - lbEndpoints:
@@ -30,7 +32,8 @@
             portValue: 8080
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: third-route-dest/backend/0
 - clusterName: fourth-route-dest
   endpoints:
   - lbEndpoints:
@@ -41,7 +44,8 @@
             portValue: 8080
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: fourth-route-dest/backend/0
 - clusterName: fifth-route-dest
   endpoints:
   - lbEndpoints:
@@ -52,7 +56,8 @@
             portValue: 8080
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: fifth-route-dest/backend/0
 - clusterName: sixth-route-dest
   endpoints:
   - lbEndpoints:
@@ -63,7 +68,8 @@
             portValue: 8080
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: sixth-route-dest/backend/0
 - clusterName: seventh-route-dest
   endpoints:
   - lbEndpoints:
@@ -74,4 +80,5 @@
             portValue: 8080
       loadBalancingWeight: 1
     loadBalancingWeight: 1
-    locality: {}
+    locality:
+      region: seventh-route-dest/backend/0