metrics(virtualHosts): add support for opt-in virtual host metrics st…

…ats (#1855)

* metrics(virtualHosts): add support for opt-in virtual host metrics stats

Signed-off-by: Pratyush Singhal <psinghal20@gmail.com>

* refactor: use prefix match instead of exact match in virtual cluster configuration

Signed-off-by: Pratyush Singhal <psinghal20@gmail.com>

* fix: update the test output for prefix match

Signed-off-by: Pratyush Singhal <psinghal20@gmail.com>

---------

Signed-off-by: Pratyush Singhal <psinghal20@gmail.com>

api/config/v1alpha1/metric_types.go

@@ -13,6 +13,9 @@ type ProxyMetrics struct {
 	// Matches defines configuration for selecting specific metrics instead of generating all metrics stats
 	// that are enabled by default. This helps reduce CPU and memory overhead in Envoy.
 	Matches []Match `json:"matches,omitempty"`
+
+	// EnableVirtualHostStats enables envoy stat metrics for virtual hosts.
+	EnableVirtualHostStats bool `json:"enableVirtualHostStats,omitempty"`
 }
 
 type MetricSinkType string

charts/gateway-helm/crds/generated/config.gateway.envoyproxy.io_envoyproxies.yaml

@@ -3938,6 +3938,10 @@ spec:
                     description: Metrics defines metrics configuration for managed
                       proxies.
                     properties:
+                      enableVirtualHostStats:
+                        description: EnableVirtualHostStats enables envoy stat metrics
+                          for virtual hosts.
+                        type: boolean
                       matches:
                         description: Matches defines configuration for selecting specific
                           metrics instead of generating all metrics stats that are

docs/latest/api/config_types.md

@@ -837,6 +837,7 @@ _Appears in:_
 | `prometheus` _[PrometheusProvider](#prometheusprovider)_ | Prometheus defines the configuration for Admin endpoint `/stats/prometheus`. |
 | `sinks` _[MetricSink](#metricsink) array_ | Sinks defines the metric sinks where metrics are sent to. |
 | `matches` _[Match](#match) array_ | Matches defines configuration for selecting specific metrics instead of generating all metrics stats that are enabled by default. This helps reduce CPU and memory overhead in Envoy. |
+| `enableVirtualHostStats` _boolean_ | EnableVirtualHostStats enables envoy stat metrics for virtual hosts. |
 
 
 ## ProxyTelemetry

internal/gatewayapi/listener.go

@@ -51,6 +51,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap
 
 		gwXdsIR.AccessLog = processAccessLog(gwInfraIR.Proxy.Config)
 		gwXdsIR.Tracing = processTracing(gateway.Gateway, gwInfraIR.Proxy.Config)
+		gwXdsIR.Metrics = processMetrics(gwInfraIR.Proxy.Config)
 
 		for _, listener := range gateway.listeners {
 			// Process protocol & supported kinds
@@ -234,3 +235,12 @@ func processTracing(gw *v1beta1.Gateway, envoyproxy *configv1a1.EnvoyProxy) *ir.
 		ProxyTracing: *envoyproxy.Spec.Telemetry.Tracing,
 	}
 }
+
+func processMetrics(envoyproxy *configv1a1.EnvoyProxy) *ir.Metrics {
+	if envoyproxy == nil || envoyproxy.Spec.Telemetry.Metrics == nil {
+		return nil
+	}
+	return &ir.Metrics{
+		EnableVirtualHostStats: envoyproxy.Spec.Telemetry.Metrics.EnableVirtualHostStats,
+	}
+}

internal/gatewayapi/listener_test.go

@@ -52,3 +52,37 @@ func TestProcessTracing(t *testing.T) {
 		})
 	}
 }
+
+func TestProcessMetrics(t *testing.T) {
+	cases := []struct {
+		name  string
+		proxy *egcfgv1a1.EnvoyProxy
+
+		expected *ir.Metrics
+	}{
+		{
+			name: "nil proxy config",
+		},
+		{
+			name: "virtual host stats enabled",
+			proxy: &egcfgv1a1.EnvoyProxy{
+				Spec: egcfgv1a1.EnvoyProxySpec{
+					Telemetry: egcfgv1a1.ProxyTelemetry{
+						Metrics: &egcfgv1a1.ProxyMetrics{
+							EnableVirtualHostStats: true,
+						},
+					},
+				},
+			},
+			expected: &ir.Metrics{
+				EnableVirtualHostStats: true,
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := processMetrics(c.proxy)
+			assert.Equal(t, c.expected, got)
+		})
+	}
+}

internal/ir/xds.go

@@ -56,6 +56,8 @@ type Xds struct {
 	AccessLog *AccessLog `json:"accessLog,omitempty" yaml:"accessLog,omitempty"`
 	// Tracing configuration for the gateway.
 	Tracing *Tracing `json:"tracing,omitempty" yaml:"tracing,omitempty"`
+	// Metrics configuration for the gateway.
+	Metrics *Metrics `json:"metrics,omitempty" yaml:"metrics,omitempty"`
 	// HTTP listeners exposed by the gateway.
 	HTTP []*HTTPListener `json:"http,omitempty" yaml:"http,omitempty"`
 	// TCP Listeners exposed by the gateway.
@@ -913,3 +915,9 @@ type Tracing struct {
 
 	egcfgv1a1.ProxyTracing
 }
+
+// Metrics defines the configuration for metrics generated by Envoy
+// +k8s:deepcopy-gen=true
+type Metrics struct {
+	EnableVirtualHostStats bool `json:"enableVirtualHostStats"`
+}

internal/xds/translator/testdata/in/xds-ir/metrics-virtual-host.yaml

@@ -0,0 +1,17 @@
+name: "metrics"
+metrics:
+  enableVirtualHostStats: true
+http:
+- name: "first-listener"
+  address: "0.0.0.0"
+  port: 10080
+  hostnames:
+  - "*"
+  routes:
+  - name: "first-route"
+    hostname: "*"
+    destination:
+      name: "first-route-dest"
+      endpoints:
+      - host: "1.2.3.4"
+        port: 50000

internal/xds/translator/testdata/out/xds-ir/metrics-virtual-host.clusters.yaml

@@ -0,0 +1,13 @@
+- commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_ONLY
+  edsClusterConfig:
+    edsConfig:
+      ads: {}
+      resourceApiVersion: V3
+    serviceName: first-route-dest
+  name: first-route-dest
+  outlierDetection: {}
+  perConnectionBufferLimitBytes: 32768
+  type: EDS

internal/xds/translator/testdata/out/xds-ir/metrics-virtual-host.endpoints.yaml

@@ -0,0 +1,10 @@
+- clusterName: first-route-dest
+  endpoints:
+  - lbEndpoints:
+    - endpoint:
+        address:
+          socketAddress:
+            address: 1.2.3.4
+            portValue: 50000
+    loadBalancingWeight: 1
+    locality: {}

internal/xds/translator/testdata/out/xds-ir/metrics-virtual-host.listeners.yaml

@@ -0,0 +1,33 @@
+- address:
+    socketAddress:
+      address: 0.0.0.0
+      portValue: 10080
+  defaultFilterChain:
+    filters:
+    - name: envoy.filters.network.http_connection_manager
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+        commonHttpProtocolOptions:
+          headersWithUnderscoresAction: REJECT_REQUEST
+        http2ProtocolOptions:
+          initialConnectionWindowSize: 1048576
+          initialStreamWindowSize: 65536
+          maxConcurrentStreams: 100
+        httpFilters:
+        - name: envoy.filters.http.router
+          typedConfig:
+            '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+        mergeSlashes: true
+        normalizePath: true
+        pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT
+        rds:
+          configSource:
+            ads: {}
+            resourceApiVersion: V3
+          routeConfigName: first-listener
+        statPrefix: http
+        upgradeConfigs:
+        - upgradeType: websocket
+        useRemoteAddress: true
+  name: first-listener
+  perConnectionBufferLimitBytes: 32768

internal/xds/translator/testdata/out/xds-ir/metrics-virtual-host.routes.yaml

@@ -0,0 +1,18 @@
+- ignorePortInHostMatching: true
+  name: first-listener
+  virtualHosts:
+  - domains:
+    - '*'
+    name: first-listener/*
+    routes:
+    - match:
+        prefix: /
+      name: first-route
+      route:
+        cluster: first-route-dest
+    virtualClusters:
+    - headers:
+      - name: :authority
+        stringMatch:
+          prefix: '*'
+      name: '*'

internal/xds/translator/translator.go

@@ -16,6 +16,7 @@ import (
 	endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	matcherv3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
 	resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3"
 	"github.com/tetratelabs/multierror"
 
@@ -28,6 +29,8 @@ var (
 	ErrXdsClusterExists = errors.New("xds cluster exists")
 )
 
+const AuthorityHeaderKey = ":authority"
+
 // Translator translates the xDS IR into xDS resources.
 type Translator struct {
 	// GlobalRateLimit holds the global rate limit settings
@@ -61,7 +64,7 @@ func (t *Translator) Translate(ir *ir.Xds) (*types.ResourceVersionTable, error)
 
 	tCtx := new(types.ResourceVersionTable)
 
-	if err := t.processHTTPListenerXdsTranslation(tCtx, ir.HTTP, ir.AccessLog, ir.Tracing); err != nil {
+	if err := t.processHTTPListenerXdsTranslation(tCtx, ir.HTTP, ir.AccessLog, ir.Tracing, ir.Metrics); err != nil {
 		return nil, err
 	}
 
@@ -94,7 +97,7 @@ func (t *Translator) Translate(ir *ir.Xds) (*types.ResourceVersionTable, error)
 }
 
 func (t *Translator) processHTTPListenerXdsTranslation(tCtx *types.ResourceVersionTable, httpListeners []*ir.HTTPListener,
-	accesslog *ir.AccessLog, tracing *ir.Tracing) error {
+	accesslog *ir.AccessLog, tracing *ir.Tracing, metrics *ir.Metrics) error {
 	for _, httpListener := range httpListeners {
 		addFilterChain := true
 		var xdsRouteCfg *routev3.RouteConfiguration
@@ -173,6 +176,25 @@ func (t *Translator) processHTTPListenerXdsTranslation(tCtx *types.ResourceVersi
 					Name:    fmt.Sprintf("%s/%s", httpListener.Name, underscoredHostname),
 					Domains: []string{httpRoute.Hostname},
 				}
+				if metrics != nil && metrics.EnableVirtualHostStats {
+					vHost.VirtualClusters = []*routev3.VirtualCluster{
+						{
+							Name: underscoredHostname,
+							Headers: []*routev3.HeaderMatcher{
+								{
+									Name: AuthorityHeaderKey,
+									HeaderMatchSpecifier: &routev3.HeaderMatcher_StringMatch{
+										StringMatch: &matcherv3.StringMatcher{
+											MatchPattern: &matcherv3.StringMatcher_Prefix{
+												Prefix: httpRoute.Hostname,
+											},
+										},
+									},
+								},
+							},
+						},
+					}
+				}
 				vHosts[httpRoute.Hostname] = vHost
 				vHostsList = append(vHostsList, vHost)
 			}

internal/xds/translator/translator_test.go

@@ -158,6 +158,9 @@ func TestTranslateXds(t *testing.T) {
 		{
 			name: "tracing",
 		},
+		{
+			name: "metrics-virtual-host",
+		},
 		{
 			name:                      "jsonpatch",
 			requireEnvoyPatchPolicies: true,