feat: add enable stats for peer endpoint of envoyproxy (#3145)

* feat: add enable stats for peer endpoint of envoyproxy

Signed-off-by: ShyunnY <1147212064@qq.com>

* fix: add warning comment

Signed-off-by: ShyunnY <1147212064@qq.com>

* fix: update doc

Signed-off-by: ShyunnY <1147212064@qq.com>

* fix: gen-check

Signed-off-by: ShyunnY <1147212064@qq.com>

---------

Signed-off-by: ShyunnY <1147212064@qq.com>
Co-authored-by: zirain <zirain2009@gmail.com>

api/v1alpha1/envoyproxy_metric_types.go

@@ -27,6 +27,10 @@ type ProxyMetrics struct {
 
 	// EnableVirtualHostStats enables envoy stat metrics for virtual hosts.
 	EnableVirtualHostStats bool `json:"enableVirtualHostStats,omitempty"`
+
+	// EnablePerEndpointStats enables per endpoint envoy stats metrics.
+	// Please use with caution.
+	EnablePerEndpointStats bool `json:"enablePerEndpointStats,omitempty"`
 }
 
 // ProxyMetricSink defines the sink of metrics.

charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml

@@ -9905,6 +9905,11 @@ spec:
                     description: Metrics defines metrics configuration for managed
                       proxies.
                     properties:
+                      enablePerEndpointStats:
+                        description: |-
+                          EnablePerEndpointStats enables per endpoint envoy stats metrics.
+                          Please use with caution.
+                        type: boolean
                       enableVirtualHostStats:
                         description: EnableVirtualHostStats enables envoy stat metrics
                           for virtual hosts.

internal/gatewayapi/listener.go

@@ -290,5 +290,6 @@ func processMetrics(envoyproxy *egv1a1.EnvoyProxy) *ir.Metrics {
 	}
 	return &ir.Metrics{
 		EnableVirtualHostStats: envoyproxy.Spec.Telemetry.Metrics.EnableVirtualHostStats,
+		EnablePerEndpointStats: envoyproxy.Spec.Telemetry.Metrics.EnablePerEndpointStats,
 	}
 }

internal/gatewayapi/listener_test.go

@@ -204,6 +204,21 @@ func TestProcessMetrics(t *testing.T) {
 				EnableVirtualHostStats: true,
 			},
 		},
+		{
+			name: "peer endpoint stats enabled",
+			proxy: &egcfgv1a1.EnvoyProxy{
+				Spec: egcfgv1a1.EnvoyProxySpec{
+					Telemetry: &egcfgv1a1.ProxyTelemetry{
+						Metrics: &egcfgv1a1.ProxyMetrics{
+							EnablePerEndpointStats: true,
+						},
+					},
+				},
+			},
+			expected: &ir.Metrics{
+				EnablePerEndpointStats: true,
+			},
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {

internal/ir/xds.go

@@ -1440,6 +1440,7 @@ type Tracing struct {
 // +k8s:deepcopy-gen=true
 type Metrics struct {
 	EnableVirtualHostStats bool `json:"enableVirtualHostStats" yaml:"enableVirtualHostStats"`
+	EnablePerEndpointStats bool `json:"enablePerEndpointStats" yaml:"enablePerEndpointStats"`
 }
 
 // TCPKeepalive define the TCP Keepalive configuration.

internal/xds/translator/accesslog.go

@@ -233,7 +233,7 @@ func convertToKeyValueList(attributes map[string]string, additionalLabels bool)
 	return keyValueList
 }
 
-func processClusterForAccessLog(tCtx *types.ResourceVersionTable, al *ir.AccessLog) error {
+func processClusterForAccessLog(tCtx *types.ResourceVersionTable, al *ir.AccessLog, metrics *ir.Metrics) error {
 	if al == nil {
 		return nil
 	}
@@ -251,6 +251,7 @@ func processClusterForAccessLog(tCtx *types.ResourceVersionTable, al *ir.AccessL
 			settings:     []*ir.DestinationSetting{ds},
 			tSocket:      nil,
 			endpointType: EndpointTypeDNS,
+			metrics:      metrics,
 		}); err != nil && !errors.Is(err, ErrXdsClusterExists) {
 			return err
 		}

internal/xds/translator/cluster.go

@@ -49,6 +49,7 @@ type xdsClusterArgs struct {
 	http1Settings  *ir.HTTP1Settings
 	timeout        *ir.Timeout
 	tcpkeepalive   *ir.TCPKeepalive
+	metrics        *ir.Metrics
 }
 
 type EndpointType int
@@ -87,6 +88,13 @@ func buildXdsCluster(args *xdsClusterArgs) *clusterv3.Cluster {
 
 	cluster.ConnectTimeout = buildConnectTimeout(args.timeout)
 
+	// set peer endpoint stats
+	if args.metrics != nil && args.metrics.EnablePerEndpointStats {
+		cluster.TrackClusterStats = &clusterv3.TrackClusterStats{
+			PerEndpointStats: args.metrics.EnablePerEndpointStats,
+		}
+	}
+
 	// Set Proxy Protocol
 	if args.proxyProtocol != nil {
 		cluster.TransportSocket = buildProxyProtocolSocket(args.proxyProtocol, args.tSocket)

internal/xds/translator/ratelimit.go

@@ -462,7 +462,7 @@ func buildRateLimitTLSocket() (*corev3.TransportSocket, error) {
 	}, nil
 }
 
-func (t *Translator) createRateLimitServiceCluster(tCtx *types.ResourceVersionTable, irListener *ir.HTTPListener) error {
+func (t *Translator) createRateLimitServiceCluster(tCtx *types.ResourceVersionTable, irListener *ir.HTTPListener, metrics *ir.Metrics) error {
 	// Return early if rate limits don't exist.
 	if !t.isRateLimitPresent(irListener) {
 		return nil
@@ -486,6 +486,7 @@ func (t *Translator) createRateLimitServiceCluster(tCtx *types.ResourceVersionTa
 		settings:     []*ir.DestinationSetting{ds},
 		tSocket:      tSocket,
 		endpointType: EndpointTypeDNS,
+		metrics:      metrics,
 	}); err != nil && !errors.Is(err, ErrXdsClusterExists) {
 		return err
 	}

internal/xds/translator/testdata/in/xds-ir/accesslog-endpoint-stats.yaml

@@ -0,0 +1,46 @@
+name: "accesslog"
+metrics:
+  enablePerEndpointStats: true
+accesslog:
+  text:
+    - path: "/dev/stdout"
+      format: |
+        [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"
+  json:
+    - path: "/dev/stdout"
+      json:
+        start_time: "%START_TIME%"
+        method: "%REQ(:METHOD)%"
+        path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"
+        protocol: "%PROTOCOL%"
+        response_code: "%RESPONSE_CODE%"
+  openTelemetry:
+    - text: |
+        [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"
+      attributes:
+        "response_code": "%RESPONSE_CODE%"
+      resources:
+        "cluster_name": "cluster1"
+      host: otel-collector.default.svc.cluster.local
+      port: 4317
+http:
+  - name: "first-listener"
+    address: "0.0.0.0"
+    port: 10080
+    hostnames:
+      - "*"
+    path:
+      mergeSlashes: true
+      escapedSlashesAction: UnescapeAndRedirect
+    routes:
+      - name: "direct-route"
+        hostname: "*"
+        destination:
+          name: "direct-route-dest"
+          settings:
+            - endpoints:
+                - host: "1.2.3.4"
+                  port: 50000
+        directResponse:
+          body: "Unknown custom filter type: UnsupportedType"
+          statusCode: 500

internal/xds/translator/testdata/in/xds-ir/http-endpoint-stats.yaml

@@ -0,0 +1,21 @@
+name: "metrics-endpoint-stats"
+metrics:
+  enablePerEndpointStats: true
+http:
+  - name: "listener-enable-endpoint-stats"
+    address: "0.0.0.0"
+    port: 10080
+    hostnames:
+      - "*"
+    path:
+      mergeSlashes: true
+      escapedSlashesAction: UnescapeAndRedirect
+    routes:
+      - name: "first-route"
+        hostname: "*"
+        destination:
+          name: "first-route-dest"
+          settings:
+            - endpoints:
+                - host: "1.2.3.4"
+                  port: 50000

internal/xds/translator/testdata/in/xds-ir/ratelimit-endpoint-stats.yaml

@@ -0,0 +1,66 @@
+metrics:
+  enablePerEndpointStats: true
+http:
+  - name: "first-listener"
+    address: "0.0.0.0"
+    port: 10080
+    hostnames:
+      - "*"
+    path:
+      mergeSlashes: true
+      escapedSlashesAction: UnescapeAndRedirect
+    routes:
+      - name: "first-route"
+        hostname: "*"
+        rateLimit:
+          global:
+            rules:
+              - headerMatches:
+                  - name: "x-user-id"
+                    exact: "one"
+                limit:
+                  requests: 5
+                  unit: second
+        pathMatch:
+          exact: "foo/bar"
+        destination:
+          name: "first-route-dest"
+          settings:
+            - endpoints:
+                - host: "1.2.3.4"
+                  port: 50000
+      - name: "second-route"
+        hostname: "*"
+        rateLimit:
+          global:
+            rules:
+              - headerMatches:
+                  - name: "x-user-id"
+                    distinct: true
+                limit:
+                  requests: 5
+                  unit: second
+        pathMatch:
+          exact: "example"
+        destination:
+          name: "second-route-dest"
+          settings:
+            - endpoints:
+                - host: "1.2.3.4"
+                  port: 50000
+      - name: "third-route"
+        hostname: "*"
+        rateLimit:
+          global:
+            rules:
+              - limit:
+                  requests: 5
+                  unit: second
+        pathMatch:
+          exact: "test"
+        destination:
+          name: "third-route-dest"
+          settings:
+            - endpoints:
+                - host: "1.2.3.4"
+                  port: 50000

internal/xds/translator/testdata/in/xds-ir/tcp-endpoint-stats.yaml

@@ -0,0 +1,15 @@
+name: "metrics-endpoint-stats"
+metrics:
+  enablePerEndpointStats: true
+tcp:
+- name: "tcp-route-enable-endpoint-stats"
+  address: "0.0.0.0"
+  port: 10080
+  destination:
+    name: "tcp-route-simple-dest"
+    settings:
+    - endpoints:
+      - host: "1.2.3.4"
+        port: 50000
+      - host: "5.6.7.8"
+        port: 50001

internal/xds/translator/testdata/in/xds-ir/tracing-endpoint-stats.yaml

@@ -0,0 +1,44 @@
+name: "tracing"
+metrics:
+  enablePerEndpointStats: true
+tracing:
+  serviceName: "fake-name.fake-ns"
+  samplingRate: 90
+  customTags:
+    "literal1":
+      type: Literal
+      literal:
+        value: "value1"
+    "env1":
+      type: Environment
+      environment:
+        name: "env1"
+        defaultValue: "-"
+    "req1":
+      type: RequestHeader
+      requestHeader:
+        name: "X-Request-Id"
+        defaultValue: "-"
+  host: otel-collector.monitoring.svc.cluster.local
+  port: 4317
+http:
+  - name: "first-listener"
+    address: "0.0.0.0"
+    port: 10080
+    hostnames:
+      - "*"
+    path:
+      mergeSlashes: true
+      escapedSlashesAction: UnescapeAndRedirect
+    routes:
+      - name: "direct-route"
+        hostname: "*"
+        destination:
+          name: "direct-route-dest"
+          settings:
+            - endpoints:
+                - host: "1.2.3.4"
+                  port: 50000
+        directResponse:
+          body: "Unknown custom filter type: UnsupportedType"
+          statusCode: 500

internal/xds/translator/testdata/in/xds-ir/udp-endpoint-stats.yaml

@@ -0,0 +1,15 @@
+name: "metrics-endpoint-stats"
+metrics:
+  enablePerEndpointStats: true
+udp:
+- name: "udp-route-enable-endpoint-stats"
+  address: "0.0.0.0"
+  port: 10080
+  destination:
+    name: "udp-route-dest"
+    settings:
+    - endpoints:
+      - host: "1.2.3.4"
+        port: 50000
+      - host: "5.6.7.8"
+        port: 50001

internal/xds/translator/testdata/out/xds-ir/accesslog-endpoint-stats.clusters.yaml

@@ -0,0 +1,53 @@
+- circuitBreakers:
+    thresholds:
+    - maxRetries: 1024
+  commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_ONLY
+  edsClusterConfig:
+    edsConfig:
+      ads: {}
+      resourceApiVersion: V3
+    serviceName: direct-route-dest
+  lbPolicy: LEAST_REQUEST
+  name: direct-route-dest
+  outlierDetection: {}
+  perConnectionBufferLimitBytes: 32768
+  trackClusterStats:
+    perEndpointStats: true
+  type: EDS
+- circuitBreakers:
+    thresholds:
+    - maxRetries: 1024
+  commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_ONLY
+  dnsRefreshRate: 30s
+  lbPolicy: LEAST_REQUEST
+  loadAssignment:
+    clusterName: accesslog|otel-collector.default.svc.cluster.local|4317
+    endpoints:
+    - lbEndpoints:
+      - endpoint:
+          address:
+            socketAddress:
+              address: otel-collector.default.svc.cluster.local
+              portValue: 4317
+        loadBalancingWeight: 1
+      loadBalancingWeight: 1
+      locality:
+        region: accesslog|otel-collector.default.svc.cluster.local|4317/backend/0
+  name: accesslog|otel-collector.default.svc.cluster.local|4317
+  outlierDetection: {}
+  perConnectionBufferLimitBytes: 32768
+  respectDnsTtl: true
+  trackClusterStats:
+    perEndpointStats: true
+  type: STRICT_DNS
+  typedExtensionProtocolOptions:
+    envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+      '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+      explicitHttpConfig:
+        http2ProtocolOptions: {}

internal/xds/translator/testdata/out/xds-ir/accesslog-endpoint-stats.endpoints.yaml

@@ -0,0 +1,12 @@
+- clusterName: direct-route-dest
+  endpoints:
+  - lbEndpoints:
+    - endpoint:
+        address:
+          socketAddress:
+            address: 1.2.3.4
+            portValue: 50000
+      loadBalancingWeight: 1
+    loadBalancingWeight: 1
+    locality:
+      region: direct-route-dest/backend/0