feat:support slow start mode (envoyproxy#2219)

* feat:support slow start mode Signed-off-by: zhaonan <zhaonan06@corp.netease.com> * fix api and add test Signed-off-by: zhaonan <zhaonan06@corp.netease.com> * fix cel validation and test Signed-off-by: zhaonan <zhaonan06@corp.netease.com> * fix gen-check Signed-off-by: zhaonan <zhaonan06@corp.netease.com> * fix cel validation Signed-off-by: zhaonan <zhaonan06@corp.netease.com> * update slow start api Signed-off-by: zhaonan <zhaonan06@corp.netease.com> * update api Signed-off-by: zhaonan <zhaonan06@corp.netease.com> * optimize some code Signed-off-by: zhaonan <zhaonan06@corp.netease.com> * remove unused code Signed-off-by: zhaonan <zhaonan06@corp.netease.com> --------- Signed-off-by: zhaonan <zhaonan06@corp.netease.com>
tmsnan · Nov 24, 2023 · 73fbd62 · 73fbd62
1 parent 571f8bf
commit 73fbd62
Show file tree

Hide file tree

Showing 15 changed files with 516 additions and 8 deletions.
diff --git a/api/v1alpha1/loadbalancer_types.go b/api/v1alpha1/loadbalancer_types.go
@@ -5,10 +5,13 @@
 
 package v1alpha1
 
+import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 // LoadBalancer defines the load balancer policy to be applied.
 // +union
 //
 // +kubebuilder:validation:XValidation:rule="self.type == 'ConsistentHash' ? has(self.consistentHash) : !has(self.consistentHash)",message="If LoadBalancer type is consistentHash, consistentHash field needs to be set."
+// +kubebuilder:validation:XValidation:rule="self.type in ['Random', 'ConsistentHash'] ? !has(self.slowStart) : true ",message="Currently SlowStart is only supported for RoundRobin and LeastRequest load balancers."
 type LoadBalancer struct {
 	// Type decides the type of Load Balancer policy.
 	// Valid LoadBalancerType values are
@@ -24,6 +27,13 @@ type LoadBalancer struct {
 	//
 	// +optional
 	ConsistentHash *ConsistentHash `json:"consistentHash,omitempty"`
+
+	// SlowStart defines the configuration related to the slow start load balancer policy.
+	// If set, during slow start window, traffic sent to the newly added hosts will gradually increase.
+	// Currently this is only supported for RoundRobin and LeastRequest load balancers
+	//
+	// +optional
+	SlowStart *SlowStart `json:"slowStart,omitempty"`
 }
 
 // LoadBalancerType specifies the types of LoadBalancer.
@@ -55,3 +65,14 @@ const (
 	// SourceIPConsistentHashType hashes based on the source IP address.
 	SourceIPConsistentHashType ConsistentHashType = "SourceIP"
 )
+
+// SlowStart defines the configuration related to the slow start load balancer policy.
+type SlowStart struct {
+	// Window defines the duration of the warm up period for newly added host.
+	// During slow start window, traffic sent to the newly added hosts will gradually increase.
+	// Currently only supports linear growth of traffic. For additional details,
+	// see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#config-cluster-v3-cluster-slowstartconfig
+	// +kubebuilder:validation:Required
+	Window *metav1.Duration `json:"window"`
+	// TODO: Add support for non-linear traffic increases based on user usage.
+}
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml
@@ -61,6 +61,23 @@ spec:
                     required:
                     - type
                     type: object
+                  slowStart:
+                    description: SlowStart defines the configuration related to the
+                      slow start load balancer policy. If set, during slow start window,
+                      traffic sent to the newly added hosts will gradually increase.
+                      Currently this is only supported for RoundRobin and LeastRequest
+                      load balancers
+                    properties:
+                      window:
+                        description: Window defines the duration of the warm up period
+                          for newly added host. During slow start window, traffic
+                          sent to the newly added hosts will gradually increase. Currently
+                          only supports linear growth of traffic. For additional details,
+                          see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#config-cluster-v3-cluster-slowstartconfig
+                        type: string
+                    required:
+                    - window
+                    type: object
                   type:
                     description: Type decides the type of Load Balancer policy. Valid
                       LoadBalancerType values are "ConsistentHash", "LeastRequest",
@@ -79,6 +96,10 @@ spec:
                     field needs to be set.
                   rule: 'self.type == ''ConsistentHash'' ? has(self.consistentHash)
                     : !has(self.consistentHash)'
+                - message: Currently SlowStart is only supported for RoundRobin and
+                    LeastRequest load balancers.
+                  rule: 'self.type in [''Random'', ''ConsistentHash''] ? !has(self.slowStart)
+                    : true '
               proxyProtocol:
                 description: ProxyProtocol enables the Proxy Protocol when communicating
                   with the backend.

diff --git a/internal/gatewayapi/backendtrafficpolicy.go b/internal/gatewayapi/backendtrafficpolicy.go
@@ -433,16 +433,34 @@ func (t *Translator) buildLoadBalancer(policy *egv1a1.BackendTrafficPolicy) *ir.
 			lb.ConsistentHash.SourceIP = ptr.To(true)
 		}
 	case egv1a1.LeastRequestLoadBalancerType:
-		lb = &ir.LoadBalancer{
-			LeastRequest: &ir.LeastRequest{},
+		lb = &ir.LoadBalancer{}
+		if policy.Spec.LoadBalancer.SlowStart != nil {
+			if policy.Spec.LoadBalancer.SlowStart.Window != nil {
+				lb.LeastRequest = &ir.LeastRequest{
+					SlowStart: &ir.SlowStart{
+						Window: policy.Spec.LoadBalancer.SlowStart.Window,
+					},
+				}
+			}
 		}
 	case egv1a1.RandomLoadBalancerType:
 		lb = &ir.LoadBalancer{
 			Random: &ir.Random{},
 		}
 	case egv1a1.RoundRobinLoadBalancerType:
 		lb = &ir.LoadBalancer{
-			RoundRobin: &ir.RoundRobin{},
+			RoundRobin: &ir.RoundRobin{
+				SlowStart: &ir.SlowStart{},
+			},
+		}
+		if policy.Spec.LoadBalancer.SlowStart != nil {
+			if policy.Spec.LoadBalancer.SlowStart.Window != nil {
+				lb.RoundRobin = &ir.RoundRobin{
+					SlowStart: &ir.SlowStart{
+						Window: policy.Spec.LoadBalancer.SlowStart.Window,
+					},
+				}
+			}
 		}
 	}
 

diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-loadbalancer.in.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-loadbalancer.in.yaml
@@ -62,6 +62,25 @@ httpRoutes:
       backendRefs:
       - name: service-1
         port: 8080
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: HTTPRoute
+  metadata:
+    namespace: default
+    name: httproute-2
+  spec:
+    hostnames:
+    - gateway.envoyproxy.io
+    parentRefs:
+    - namespace: envoy-gateway
+      name: gateway-2
+      sectionName: http
+    rules:
+    - matches:
+      - path:
+          value: "/test2"
+      backendRefs:
+      - name: service-2
+        port: 8080
 backendTrafficPolicies:
 - apiVersion: gateway.envoyproxy.io/v1alpha1
   kind: BackendTrafficPolicy
@@ -91,3 +110,33 @@ backendTrafficPolicies:
       type: ConsistentHash
       consistentHash:
         type: SourceIP
+- apiVersion: gateway.envoyproxy.io/v1alpha1
+  kind: BackendTrafficPolicy
+  metadata:
+    namespace: envoy-gateway
+    name: policy-for-gateway2
+  spec:
+    targetRef:
+      group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gateway-2
+      namespace: envoy-gateway
+    loadBalancer:
+      type: RoundRobin
+      slowStart:
+        window: 300s
+- apiVersion: gateway.envoyproxy.io/v1alpha1
+  kind: BackendTrafficPolicy
+  metadata:
+    namespace: default
+    name: policy-for-route2
+  spec:
+    targetRef:
+      group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: httproute-2
+      namespace: default
+    loadBalancer:
+      type: LeastRequest
+      slowStart:
+        window: 300s
diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-loadbalancer.out.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-loadbalancer.out.yaml
@@ -22,6 +22,29 @@ backendTrafficPolicies:
       reason: Accepted
       status: "True"
       type: Accepted
+- apiVersion: gateway.envoyproxy.io/v1alpha1
+  kind: BackendTrafficPolicy
+  metadata:
+    creationTimestamp: null
+    name: policy-for-route2
+    namespace: default
+  spec:
+    loadBalancer:
+      slowStart:
+        window: 5m0s
+      type: LeastRequest
+    targetRef:
+      group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: httproute-2
+      namespace: default
+  status:
+    conditions:
+    - lastTransitionTime: null
+      message: BackendTrafficPolicy has been accepted.
+      reason: Accepted
+      status: "True"
+      type: Accepted
 - apiVersion: gateway.envoyproxy.io/v1alpha1
   kind: BackendTrafficPolicy
   metadata:
@@ -43,6 +66,29 @@ backendTrafficPolicies:
       reason: Accepted
       status: "True"
       type: Accepted
+- apiVersion: gateway.envoyproxy.io/v1alpha1
+  kind: BackendTrafficPolicy
+  metadata:
+    creationTimestamp: null
+    name: policy-for-gateway2
+    namespace: envoy-gateway
+  spec:
+    loadBalancer:
+      slowStart:
+        window: 5m0s
+      type: RoundRobin
+    targetRef:
+      group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gateway-2
+      namespace: envoy-gateway
+  status:
+    conditions:
+    - lastTransitionTime: null
+      message: BackendTrafficPolicy has been accepted.
+      reason: Accepted
+      status: "True"
+      type: Accepted
 gateways:
 - apiVersion: gateway.networking.k8s.io/v1
   kind: Gateway
@@ -101,7 +147,7 @@ gateways:
       protocol: HTTP
   status:
     listeners:
-    - attachedRoutes: 1
+    - attachedRoutes: 2
       conditions:
       - lastTransitionTime: null
         message: Sending translated listener configuration to the data plane
@@ -197,6 +243,44 @@ httpRoutes:
         name: gateway-2
         namespace: envoy-gateway
         sectionName: http
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: HTTPRoute
+  metadata:
+    creationTimestamp: null
+    name: httproute-2
+    namespace: default
+  spec:
+    hostnames:
+    - gateway.envoyproxy.io
+    parentRefs:
+    - name: gateway-2
+      namespace: envoy-gateway
+      sectionName: http
+    rules:
+    - backendRefs:
+      - name: service-2
+        port: 8080
+      matches:
+      - path:
+          value: /test2
+  status:
+    parents:
+    - conditions:
+      - lastTransitionTime: null
+        message: Route is accepted
+        reason: Accepted
+        status: "True"
+        type: Accepted
+      - lastTransitionTime: null
+        message: Resolved all the Object references for the Route
+        reason: ResolvedRefs
+        status: "True"
+        type: ResolvedRefs
+      controllerName: gateway.envoyproxy.io/gatewayclass-controller
+      parentRef:
+        name: gateway-2
+        namespace: envoy-gateway
+        sectionName: http
 infraIR:
   envoy-gateway/gateway-1:
     proxy:
@@ -266,6 +350,27 @@ xdsIR:
       name: envoy-gateway/gateway-2/http
       port: 10080
       routes:
+      - backendWeights:
+          invalid: 0
+          valid: 0
+        destination:
+          name: httproute/default/httproute-2/rule/0
+          settings:
+          - endpoints:
+            - host: 7.7.7.7
+              port: 8080
+            protocol: HTTP
+            weight: 1
+        hostname: gateway.envoyproxy.io
+        loadBalancer:
+          leastRequest:
+            slowStart:
+              window: 5m0s
+        name: httproute/default/httproute-2/rule/0/match/0/gateway_envoyproxy_io
+        pathMatch:
+          distinct: false
+          name: ""
+          prefix: /test2
       - backendWeights:
           invalid: 0
           valid: 0

diff --git a/internal/ir/xds.go b/internal/ir/xds.go
@@ -1048,11 +1048,19 @@ func (l *LoadBalancer) Validate() error {
 
 // RoundRobin load balancer settings
 // +k8s:deepcopy-gen=true
-type RoundRobin struct{}
+type RoundRobin struct {
+	// SlowStart defines the slow start configuration.
+	// If set, slow start mode is enabled for newly added hosts in the cluster.
+	SlowStart *SlowStart `json:"slowStart,omitempty" yaml:"slowStart,omitempty"`
+}
 
 // LeastRequest load balancer settings
 // +k8s:deepcopy-gen=true
-type LeastRequest struct{}
+type LeastRequest struct {
+	// SlowStart defines the slow start configuration.
+	// If set, slow start mode is enabled for newly added hosts in the cluster.
+	SlowStart *SlowStart `json:"slowStart,omitempty" yaml:"slowStart,omitempty"`
+}
 
 // Random load balancer settings
 // +k8s:deepcopy-gen=true
@@ -1080,3 +1088,10 @@ type ProxyProtocol struct {
 	// Version of proxy protocol to use
 	Version ProxyProtocolVersion `json:"version,omitempty" yaml:"version,omitempty"`
 }
+
+// SlowStart defines the slow start configuration.
+// +k8s:deepcopy-gen=true
+type SlowStart struct {
+	// Window defines the duration of the warm up period for newly added host.
+	Window *metav1.Duration `json:"window" yaml:"window"`
+}