diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index 00f4c38a7c4..76da4babce1 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -42,7 +42,9 @@ type EnvoyProxySpec struct { Logging ProxyLogging `json:"logging,omitempty"` // Telemetry defines telemetry parameters for managed proxies. - Telemetry ProxyTelemetry `json:"telemetry,omitempty"` + // + // +optional + Telemetry *ProxyTelemetry `json:"telemetry,omitempty"` // Bootstrap defines the Envoy Bootstrap as a YAML string. // Visit https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/bootstrap/v3/bootstrap.proto#envoy-v3-api-msg-config-bootstrap-v3-bootstrap diff --git a/api/v1alpha1/metric_types.go b/api/v1alpha1/metric_types.go index 03efd6fa04f..b3bf977a761 100644 --- a/api/v1alpha1/metric_types.go +++ b/api/v1alpha1/metric_types.go @@ -72,4 +72,6 @@ type OpenTelemetrySink struct { } type PrometheusProvider struct { + // Disable the Prometheus endpoint. + Disable bool `json:"disable,omitempty"` } diff --git a/api/v1alpha1/validation/envoyproxy_validate.go b/api/v1alpha1/validation/envoyproxy_validate.go index 73f8a5e7aaf..1ba367ef0b8 100644 --- a/api/v1alpha1/validation/envoyproxy_validate.go +++ b/api/v1alpha1/validation/envoyproxy_validate.go @@ -182,7 +182,9 @@ func validateBootstrap(boostrapConfig *egv1a1.ProxyBootstrap) error { func validateProxyTelemetry(spec *egv1a1.EnvoyProxySpec) []error { var errs []error - if spec != nil && spec.Telemetry.AccessLog != nil { + if spec != nil && + spec.Telemetry != nil && + spec.Telemetry.AccessLog != nil { accessLogErrs := validateProxyAccessLog(spec.Telemetry.AccessLog) if len(accessLogErrs) > 0 { errs = append(errs, accessLogErrs...) diff --git a/api/v1alpha1/validation/envoyproxy_validate_test.go b/api/v1alpha1/validation/envoyproxy_validate_test.go index a692c0caba0..7f4b4b7a952 100644 --- a/api/v1alpha1/validation/envoyproxy_validate_test.go +++ b/api/v1alpha1/validation/envoyproxy_validate_test.go @@ -361,7 +361,7 @@ func TestValidateEnvoyProxy(t *testing.T) { Name: "test", }, Spec: egv1a1.EnvoyProxySpec{ - Telemetry: egv1a1.ProxyTelemetry{ + Telemetry: &egv1a1.ProxyTelemetry{ AccessLog: &egv1a1.ProxyAccessLog{ Settings: []egv1a1.ProxyAccessLogSetting{ { @@ -384,7 +384,7 @@ func TestValidateEnvoyProxy(t *testing.T) { Name: "test", }, Spec: egv1a1.EnvoyProxySpec{ - Telemetry: egv1a1.ProxyTelemetry{ + Telemetry: &egv1a1.ProxyTelemetry{ AccessLog: &egv1a1.ProxyAccessLog{ Settings: []egv1a1.ProxyAccessLogSetting{ { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 44e453e4fa8..90aea0d0055 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -776,7 +776,11 @@ func (in *EnvoyProxySpec) DeepCopyInto(out *EnvoyProxySpec) { (*in).DeepCopyInto(*out) } in.Logging.DeepCopyInto(&out.Logging) - in.Telemetry.DeepCopyInto(&out.Telemetry) + if in.Telemetry != nil { + in, out := &in.Telemetry, &out.Telemetry + *out = new(ProxyTelemetry) + (*in).DeepCopyInto(*out) + } if in.Bootstrap != nil { in, out := &in.Bootstrap, &out.Bootstrap *out = new(ProxyBootstrap) diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index f3af9f56801..33aed66b3a6 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5433,6 +5433,10 @@ spec: prometheus: description: Prometheus defines the configuration for Admin endpoint `/stats/prometheus`. + properties: + disable: + description: Disable the Prometheus endpoint. + type: boolean type: object sinks: description: Sinks defines the metric sinks where metrics diff --git a/examples/kubernetes/metric/prometheus.yaml b/examples/kubernetes/metric/disable-prometheus.yaml similarity index 91% rename from examples/kubernetes/metric/prometheus.yaml rename to examples/kubernetes/metric/disable-prometheus.yaml index 15412ccc8e0..ea611a5278c 100644 --- a/examples/kubernetes/metric/prometheus.yaml +++ b/examples/kubernetes/metric/disable-prometheus.yaml @@ -18,4 +18,5 @@ metadata: spec: telemetry: metrics: - prometheus: {} + prometheus: + disable: true diff --git a/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.json index 6e78b572351..18146a09411 100644 --- a/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.json +++ b/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.json @@ -60,6 +60,30 @@ }, "staticResources": { "clusters": [ + { + "connectTimeout": "0.250s", + "loadAssignment": { + "clusterName": "prometheus_stats", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 19000 + } + } + } + } + ] + } + ] + }, + "name": "prometheus_stats", + "type": "STATIC" + }, { "connectTimeout": "10s", "http2ProtocolOptions": { @@ -171,7 +195,25 @@ } ], "routeConfig": { - "name": "local_route" + "name": "local_route", + "virtualHosts": [ + { + "domains": [ + "*" + ], + "name": "prometheus_stats", + "routes": [ + { + "match": { + "prefix": "/stats/prometheus" + }, + "route": { + "cluster": "prometheus_stats" + } + } + ] + } + ] }, "statPrefix": "eg-ready-http" } diff --git a/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.yaml index 51df9493f5c..07c0abf429b 100644 --- a/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.yaml @@ -37,6 +37,18 @@ xds: resourceApiVersion: V3 staticResources: clusters: + - connectTimeout: 0.250s + loadAssignment: + clusterName: prometheus_stats + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 19000 + name: prometheus_stats + type: STATIC - connectTimeout: 10s http2ProtocolOptions: connectionKeepalive: @@ -103,6 +115,15 @@ xds: '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router routeConfig: name: local_route + virtualHosts: + - domains: + - '*' + name: prometheus_stats + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats statPrefix: eg-ready-http name: envoy-gateway-proxy-ready-0.0.0.0-19001 - '@type': type.googleapis.com/envoy.admin.v3.EndpointsConfigDump diff --git a/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.bootstrap.yaml b/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.bootstrap.yaml index f2597643d2f..ccf66263b20 100644 --- a/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.bootstrap.yaml +++ b/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.bootstrap.yaml @@ -36,6 +36,18 @@ xds: resourceApiVersion: V3 staticResources: clusters: + - connectTimeout: 0.250s + loadAssignment: + clusterName: prometheus_stats + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 19000 + name: prometheus_stats + type: STATIC - connectTimeout: 10s http2ProtocolOptions: connectionKeepalive: @@ -102,5 +114,14 @@ xds: '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router routeConfig: name: local_route + virtualHosts: + - domains: + - '*' + name: prometheus_stats + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats statPrefix: eg-ready-http name: envoy-gateway-proxy-ready-0.0.0.0-19001 diff --git a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml index 5ad8bc104a4..4d1084bfec4 100644 --- a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml @@ -47,6 +47,15 @@ envoyProxy: stat_prefix: eg-ready-http route_config: name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats http_filters: - name: envoy.filters.http.health_check typed_config: @@ -60,6 +69,19 @@ envoyProxy: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 - connect_timeout: 10s load_assignment: cluster_name: xds_cluster @@ -111,7 +133,6 @@ envoyProxy: resource_api_version: V3 name: runtime-0 logging: {} - telemetry: {} status: {} gatewayClass: metadata: @@ -493,6 +514,18 @@ xds: resourceApiVersion: V3 staticResources: clusters: + - connectTimeout: 0.250s + loadAssignment: + clusterName: prometheus_stats + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 19000 + name: prometheus_stats + type: STATIC - connectTimeout: 10s http2ProtocolOptions: connectionKeepalive: @@ -559,6 +592,15 @@ xds: '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router routeConfig: name: local_route + virtualHosts: + - domains: + - '*' + name: prometheus_stats + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats statPrefix: eg-ready-http name: envoy-gateway-proxy-ready-0.0.0.0-19001 - '@type': type.googleapis.com/envoy.admin.v3.EndpointsConfigDump diff --git a/internal/cmd/egctl/testdata/translate/out/envoy-patch-policy.all.yaml b/internal/cmd/egctl/testdata/translate/out/envoy-patch-policy.all.yaml index 6d8e6b4c6d6..6906115a854 100644 --- a/internal/cmd/egctl/testdata/translate/out/envoy-patch-policy.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/envoy-patch-policy.all.yaml @@ -37,6 +37,18 @@ xds: resourceApiVersion: V3 staticResources: clusters: + - connectTimeout: 0.250s + loadAssignment: + clusterName: prometheus_stats + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 19000 + name: prometheus_stats + type: STATIC - connectTimeout: 10s http2ProtocolOptions: connectionKeepalive: @@ -103,6 +115,15 @@ xds: '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router routeConfig: name: local_route + virtualHosts: + - domains: + - '*' + name: prometheus_stats + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats statPrefix: eg-ready-http name: envoy-gateway-proxy-ready-0.0.0.0-19001 - '@type': type.googleapis.com/envoy.admin.v3.EndpointsConfigDump diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json index 9498c6c7761..90b22d925b6 100644 --- a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json +++ b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json @@ -60,6 +60,30 @@ }, "staticResources": { "clusters": [ + { + "connectTimeout": "0.250s", + "loadAssignment": { + "clusterName": "prometheus_stats", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 19000 + } + } + } + } + ] + } + ] + }, + "name": "prometheus_stats", + "type": "STATIC" + }, { "connectTimeout": "10s", "http2ProtocolOptions": { @@ -171,7 +195,25 @@ } ], "routeConfig": { - "name": "local_route" + "name": "local_route", + "virtualHosts": [ + { + "domains": [ + "*" + ], + "name": "prometheus_stats", + "routes": [ + { + "match": { + "prefix": "/stats/prometheus" + }, + "route": { + "cluster": "prometheus_stats" + } + } + ] + } + ] }, "statPrefix": "eg-ready-http" } diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml index 31991019843..65d48975c58 100644 --- a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml @@ -37,6 +37,18 @@ xds: resourceApiVersion: V3 staticResources: clusters: + - connectTimeout: 0.250s + loadAssignment: + clusterName: prometheus_stats + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 19000 + name: prometheus_stats + type: STATIC - connectTimeout: 10s http2ProtocolOptions: connectionKeepalive: @@ -103,6 +115,15 @@ xds: '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router routeConfig: name: local_route + virtualHosts: + - domains: + - '*' + name: prometheus_stats + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats statPrefix: eg-ready-http name: envoy-gateway-proxy-ready-0.0.0.0-19001 - '@type': type.googleapis.com/envoy.admin.v3.EndpointsConfigDump diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.bootstrap.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.bootstrap.yaml index 645f1321d03..1341b58afb7 100644 --- a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.bootstrap.yaml +++ b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.bootstrap.yaml @@ -36,6 +36,18 @@ xds: resourceApiVersion: V3 staticResources: clusters: + - connectTimeout: 0.250s + loadAssignment: + clusterName: prometheus_stats + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 19000 + name: prometheus_stats + type: STATIC - connectTimeout: 10s http2ProtocolOptions: connectionKeepalive: @@ -102,5 +114,14 @@ xds: '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router routeConfig: name: local_route + virtualHosts: + - domains: + - '*' + name: prometheus_stats + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats statPrefix: eg-ready-http name: envoy-gateway-proxy-ready-0.0.0.0-19001 diff --git a/internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml b/internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml index aa07e2fa2c4..cbb15ee7d7f 100644 --- a/internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml @@ -18,7 +18,6 @@ envoyProxy: address: 127.0.0.1 port_value: 19000 logging: {} - telemetry: {} status: {} gatewayClass: metadata: diff --git a/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.json index aa09a1fb5e1..22af80fcc98 100644 --- a/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.json +++ b/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.json @@ -60,6 +60,30 @@ }, "staticResources": { "clusters": [ + { + "connectTimeout": "0.250s", + "loadAssignment": { + "clusterName": "prometheus_stats", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 19000 + } + } + } + } + ] + } + ] + }, + "name": "prometheus_stats", + "type": "STATIC" + }, { "connectTimeout": "10s", "http2ProtocolOptions": { @@ -171,7 +195,25 @@ } ], "routeConfig": { - "name": "local_route" + "name": "local_route", + "virtualHosts": [ + { + "domains": [ + "*" + ], + "name": "prometheus_stats", + "routes": [ + { + "match": { + "prefix": "/stats/prometheus" + }, + "route": { + "cluster": "prometheus_stats" + } + } + ] + } + ] }, "statPrefix": "eg-ready-http" } diff --git a/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.yaml index 220704ed613..ac36c71ef08 100644 --- a/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.yaml @@ -37,6 +37,18 @@ xds: resourceApiVersion: V3 staticResources: clusters: + - connectTimeout: 0.250s + loadAssignment: + clusterName: prometheus_stats + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 19000 + name: prometheus_stats + type: STATIC - connectTimeout: 10s http2ProtocolOptions: connectionKeepalive: @@ -103,6 +115,15 @@ xds: '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router routeConfig: name: local_route + virtualHosts: + - domains: + - '*' + name: prometheus_stats + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats statPrefix: eg-ready-http name: envoy-gateway-proxy-ready-0.0.0.0-19001 - '@type': type.googleapis.com/envoy.admin.v3.EndpointsConfigDump diff --git a/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.bootstrap.yaml b/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.bootstrap.yaml index f2597643d2f..ccf66263b20 100644 --- a/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.bootstrap.yaml +++ b/internal/cmd/egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.bootstrap.yaml @@ -36,6 +36,18 @@ xds: resourceApiVersion: V3 staticResources: clusters: + - connectTimeout: 0.250s + loadAssignment: + clusterName: prometheus_stats + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 19000 + name: prometheus_stats + type: STATIC - connectTimeout: 10s http2ProtocolOptions: connectionKeepalive: @@ -102,5 +114,14 @@ xds: '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router routeConfig: name: local_route + virtualHosts: + - domains: + - '*' + name: prometheus_stats + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats statPrefix: eg-ready-http name: envoy-gateway-proxy-ready-0.0.0.0-19001 diff --git a/internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml b/internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml index d4f86e54a53..240c41716ee 100644 --- a/internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml @@ -11,7 +11,6 @@ envoyProxy: annotations: custom1: svc-annotation1 type: Kubernetes - telemetry: {} status: {} gatewayClass: metadata: diff --git a/internal/gatewayapi/listener.go b/internal/gatewayapi/listener.go index 8549728f92d..e24d58f7fbd 100644 --- a/internal/gatewayapi/listener.go +++ b/internal/gatewayapi/listener.go @@ -154,6 +154,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap func processAccessLog(envoyproxy *egv1a1.EnvoyProxy) *ir.AccessLog { if envoyproxy == nil || + envoyproxy.Spec.Telemetry == nil || envoyproxy.Spec.Telemetry.AccessLog == nil || (!envoyproxy.Spec.Telemetry.AccessLog.Disable && len(envoyproxy.Spec.Telemetry.AccessLog.Settings) == 0) { // use the default access log @@ -226,7 +227,9 @@ func processAccessLog(envoyproxy *egv1a1.EnvoyProxy) *ir.AccessLog { } func processTracing(gw *gwapiv1.Gateway, envoyproxy *egv1a1.EnvoyProxy) *ir.Tracing { - if envoyproxy == nil || envoyproxy.Spec.Telemetry.Tracing == nil { + if envoyproxy == nil || + envoyproxy.Spec.Telemetry == nil || + envoyproxy.Spec.Telemetry.Tracing == nil { return nil } @@ -237,7 +240,9 @@ func processTracing(gw *gwapiv1.Gateway, envoyproxy *egv1a1.EnvoyProxy) *ir.Trac } func processMetrics(envoyproxy *egv1a1.EnvoyProxy) *ir.Metrics { - if envoyproxy == nil || envoyproxy.Spec.Telemetry.Metrics == nil { + if envoyproxy == nil || + envoyproxy.Spec.Telemetry == nil || + envoyproxy.Spec.Telemetry.Metrics == nil { return nil } return &ir.Metrics{ diff --git a/internal/gatewayapi/listener_test.go b/internal/gatewayapi/listener_test.go index b803aaf15ab..ddecda17ada 100644 --- a/internal/gatewayapi/listener_test.go +++ b/internal/gatewayapi/listener_test.go @@ -33,7 +33,7 @@ func TestProcessTracing(t *testing.T) { }, proxy: &egcfgv1a1.EnvoyProxy{ Spec: egcfgv1a1.EnvoyProxySpec{ - Telemetry: egcfgv1a1.ProxyTelemetry{ + Telemetry: &egcfgv1a1.ProxyTelemetry{ Tracing: &egcfgv1a1.ProxyTracing{}, }, }, @@ -68,7 +68,7 @@ func TestProcessMetrics(t *testing.T) { name: "virtual host stats enabled", proxy: &egcfgv1a1.EnvoyProxy{ Spec: egcfgv1a1.EnvoyProxySpec{ - Telemetry: egcfgv1a1.ProxyTelemetry{ + Telemetry: &egcfgv1a1.ProxyTelemetry{ Metrics: &egcfgv1a1.ProxyMetrics{ EnableVirtualHostStats: true, }, diff --git a/internal/gatewayapi/testdata/envoyproxy-valid.out.yaml b/internal/gatewayapi/testdata/envoyproxy-valid.out.yaml index 825a6e7abea..ea18760f8c5 100644 --- a/internal/gatewayapi/testdata/envoyproxy-valid.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-valid.out.yaml @@ -100,7 +100,6 @@ infraIR: envoyService: type: LoadBalancer type: Kubernetes - telemetry: {} status: {} listeners: - address: "" diff --git a/internal/gatewayapi/testdata/merge-invalid-multiple-gateways.out.yaml b/internal/gatewayapi/testdata/merge-invalid-multiple-gateways.out.yaml index 93775aaf5f7..fa4b57bcb56 100755 --- a/internal/gatewayapi/testdata/merge-invalid-multiple-gateways.out.yaml +++ b/internal/gatewayapi/testdata/merge-invalid-multiple-gateways.out.yaml @@ -112,7 +112,6 @@ infraIR: spec: logging: {} mergeGateways: true - telemetry: {} status: {} listeners: - address: "" diff --git a/internal/gatewayapi/testdata/merge-valid-multiple-gateways-multiple-routes.out.yaml b/internal/gatewayapi/testdata/merge-valid-multiple-gateways-multiple-routes.out.yaml index aebb2aff709..720f7ec14a1 100755 --- a/internal/gatewayapi/testdata/merge-valid-multiple-gateways-multiple-routes.out.yaml +++ b/internal/gatewayapi/testdata/merge-valid-multiple-gateways-multiple-routes.out.yaml @@ -205,7 +205,6 @@ infraIR: spec: logging: {} mergeGateways: true - telemetry: {} status: {} listeners: - address: "" diff --git a/internal/gatewayapi/testdata/merge-valid-multiple-gateways.out.yaml b/internal/gatewayapi/testdata/merge-valid-multiple-gateways.out.yaml index bd4578160a6..76639b23b93 100755 --- a/internal/gatewayapi/testdata/merge-valid-multiple-gateways.out.yaml +++ b/internal/gatewayapi/testdata/merge-valid-multiple-gateways.out.yaml @@ -121,7 +121,6 @@ infraIR: spec: logging: {} mergeGateways: true - telemetry: {} status: {} listeners: - address: "" diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go index 08106c458f3..a0e746dc833 100644 --- a/internal/infrastructure/kubernetes/proxy/resource.go +++ b/internal/infrastructure/kubernetes/proxy/resource.go @@ -84,8 +84,21 @@ func envoyLabels(extraLabels map[string]string) map[string]string { return labels } +func enablePrometheus(infra *ir.ProxyInfra) bool { + if infra.Config != nil && + infra.Config.Spec.Telemetry != nil && + infra.Config.Spec.Telemetry.Metrics != nil && + infra.Config.Spec.Telemetry.Metrics.Prometheus != nil && + infra.Config.Spec.Telemetry.Metrics.Prometheus.Disable { + return false + } + + return true +} + // expectedProxyContainers returns expected proxy containers. -func expectedProxyContainers(infra *ir.ProxyInfra, deploymentConfig *egv1a1.KubernetesDeploymentSpec) ([]corev1.Container, error) { +func expectedProxyContainers(infra *ir.ProxyInfra, + deploymentConfig *egv1a1.KubernetesDeploymentSpec) ([]corev1.Container, error) { // Define slice to hold container ports var ports []corev1.ContainerPort @@ -110,12 +123,7 @@ func expectedProxyContainers(infra *ir.ProxyInfra, deploymentConfig *egv1a1.Kube } } - var proxyMetrics *egv1a1.ProxyMetrics - if infra.Config != nil { - proxyMetrics = infra.Config.Spec.Telemetry.Metrics - } - - if proxyMetrics != nil && proxyMetrics.Prometheus != nil { + if enablePrometheus(infra) { ports = append(ports, corev1.ContainerPort{ Name: "metrics", ContainerPort: bootstrap.EnvoyReadinessPort, // TODO: make this configurable @@ -125,6 +133,11 @@ func expectedProxyContainers(infra *ir.ProxyInfra, deploymentConfig *egv1a1.Kube var bootstrapConfigurations string + var proxyMetrics *egv1a1.ProxyMetrics + if infra.Config != nil && + infra.Config.Spec.Telemetry != nil { + proxyMetrics = infra.Config.Spec.Telemetry.Metrics + } // Get the default Bootstrap bootstrapConfigurations, err := bootstrap.GetRenderedBootstrapConfig(proxyMetrics) if err != nil { diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index efcc1e6a1e1..50f5fda1698 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -156,13 +156,6 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { } deploymentConfig := provider.GetEnvoyProxyKubeProvider().EnvoyDeployment - enablePrometheus := false - if r.infra.Config != nil && - r.infra.Config.Spec.Telemetry.Metrics != nil && - r.infra.Config.Spec.Telemetry.Metrics.Prometheus != nil { - enablePrometheus = true - } - // Get expected bootstrap configurations rendered ProxyContainers containers, err := expectedProxyContainers(r.infra, deploymentConfig) if err != nil { @@ -185,7 +178,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { if deploymentConfig.Pod.Annotations != nil { annotations = deploymentConfig.Pod.Annotations } - if enablePrometheus { + if enablePrometheus(r.infra) { if annotations == nil { annotations = make(map[string]string, 2) } diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index bfbfdd88da4..9e1602f44d4 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -349,7 +349,15 @@ func TestDeployment(t *testing.T) { } if tc.telemetry != nil { - tc.infra.Proxy.Config.Spec.Telemetry = *tc.telemetry + tc.infra.Proxy.Config.Spec.Telemetry = tc.telemetry + } else { + tc.infra.Proxy.Config.Spec.Telemetry = &egv1a1.ProxyTelemetry{ + Metrics: &egv1a1.ProxyMetrics{ + Prometheus: &egv1a1.PrometheusProvider{ + Disable: true, + }, + }, + } } if len(tc.proxyLogging) > 0 { diff --git a/internal/xds/bootstrap/bootstrap.go b/internal/xds/bootstrap/bootstrap.go index dc6fe78e48e..3ed28e05010 100644 --- a/internal/xds/bootstrap/bootstrap.go +++ b/internal/xds/bootstrap/bootstrap.go @@ -122,14 +122,14 @@ func (b *bootstrapConfig) render() error { // GetRenderedBootstrapConfig renders the bootstrap YAML string func GetRenderedBootstrapConfig(proxyMetrics *egv1a1.ProxyMetrics) (string, error) { var ( - enablePrometheus bool + enablePrometheus = true metricSinks []metricSink StatsMatcher StatsMatcherParameters ) if proxyMetrics != nil { if proxyMetrics.Prometheus != nil { - enablePrometheus = true + enablePrometheus = !proxyMetrics.Prometheus.Disable } addresses := sets.NewString() @@ -152,7 +152,6 @@ func GetRenderedBootstrapConfig(proxyMetrics *egv1a1.ProxyMetrics) (string, erro } if proxyMetrics.Matches != nil { - // Add custom envoy proxy stats for _, match := range proxyMetrics.Matches { switch match.Type { @@ -165,7 +164,6 @@ func GetRenderedBootstrapConfig(proxyMetrics *egv1a1.ProxyMetrics) (string, erro } } } - } cfg := &bootstrapConfig{ diff --git a/internal/xds/bootstrap/bootstrap_test.go b/internal/xds/bootstrap/bootstrap_test.go index bbee8f02026..bd246c212ea 100644 --- a/internal/xds/bootstrap/bootstrap_test.go +++ b/internal/xds/bootstrap/bootstrap_test.go @@ -22,7 +22,12 @@ func TestGetRenderedBootstrapConfig(t *testing.T) { proxyMetrics *egv1a1.ProxyMetrics }{ { - name: "default", + name: "disable-prometheus", + proxyMetrics: &egv1a1.ProxyMetrics{ + Prometheus: &egv1a1.PrometheusProvider{ + Disable: true, + }, + }, }, { name: "enable-prometheus", @@ -33,6 +38,9 @@ func TestGetRenderedBootstrapConfig(t *testing.T) { { name: "otel-metrics", proxyMetrics: &egv1a1.ProxyMetrics{ + Prometheus: &egv1a1.PrometheusProvider{ + Disable: true, + }, Sinks: []egv1a1.MetricSink{ { Type: egv1a1.MetricSinkTypeOpenTelemetry, @@ -65,7 +73,6 @@ func TestGetRenderedBootstrapConfig(t *testing.T) { Value: "cluster", }, }, - Prometheus: &egv1a1.PrometheusProvider{}, }, }, } diff --git a/internal/xds/bootstrap/testdata/default.yaml b/internal/xds/bootstrap/testdata/disable-prometheus.yaml similarity index 100% rename from internal/xds/bootstrap/testdata/default.yaml rename to internal/xds/bootstrap/testdata/disable-prometheus.yaml diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 53bd49902c5..41cccc4cfd8 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -995,6 +995,9 @@ _Appears in:_ _Appears in:_ - [ProxyMetrics](#proxymetrics) +| Field | Description | +| --- | --- | +| `disable` _boolean_ | Disable the Prometheus endpoint. | #### ProviderType diff --git a/site/content/en/latest/design/metrics.md b/site/content/en/latest/design/metrics.md index 644e3c7a634..c87b23ad4e5 100644 --- a/site/content/en/latest/design/metrics.md +++ b/site/content/en/latest/design/metrics.md @@ -23,14 +23,15 @@ Envoy Gateway leverages [Gateway API](https://gateway-api.sigs.k8s.io/) for conf ## Use-Cases -- Enable prometheus metric +- Enable prometheus metric by default +- Disable prometheus metric - Push metrics via Open Telemetry Sink - TODO: Customize histogram buckets of target metric - TODO: Support stats matcher ### ProxyMetric API Type -```golang mdox-exec="sed '1,7d' api/config/v1alpha1/metric_types.go" +```golang mdox-exec="sed '1,7d' api/v1alpha1/metric_types.go" type ProxyMetrics struct { // Prometheus defines the configuration for Admin endpoint `/stats/prometheus`. Prometheus *PrometheusProvider `json:"prometheus,omitempty"` @@ -70,14 +71,16 @@ type OpenTelemetrySink struct { } type PrometheusProvider struct { + // Disable the Prometheus endpoint. + Disable bool `json:"disable,omitempty"` } ``` ### Example -1. The following is an example to enable prometheus metric. +1. The following is an example to disable prometheus metric. -```yaml mdox-exec="sed '1,12d' examples/kubernetes/metric/prometheus.yaml" +```yaml mdox-exec="sed '1,12d' examples/kubernetes/metric/disable-prometheus.yaml" apiVersion: gateway.envoyproxy.io/v1alpha1 kind: EnvoyProxy metadata: @@ -86,7 +89,8 @@ metadata: spec: telemetry: metrics: - prometheus: {} + prometheus: + disable: true ``` 1. The following is an example to send metric via Open Telemetry sink. diff --git a/site/content/en/latest/user/proxy-observability.md b/site/content/en/latest/user/proxy-observability.md index 8755332262d..5afd1064b8d 100644 --- a/site/content/en/latest/user/proxy-observability.md +++ b/site/content/en/latest/user/proxy-observability.md @@ -50,14 +50,7 @@ TEMPO_IP=$(kubectl get svc tempo -n monitoring -o jsonpath='{.status.loadBalance ## Metrics -By default, Envoy Gateway doesn't expose metrics of the EnvoyProxy instances. -You can enable metrics by setting the `telemetry.metrics.prometheus` in the `EnvoyProxy` CRD. - -Expose prometheus metrics endpoints: - -```shell -kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/kubernetes/metric/prometheus.yaml -``` +By default, Envoy Gateway expose metrics with prometheus endpoint. Verify metrics: @@ -69,6 +62,12 @@ kubectl port-forward pod/$ENVOY_POD_NAME -n envoy-gateway-system 19001:19001 curl localhost:19001/stats/prometheus | grep "default/backend/rule/0/match/0-www" ``` +You can disable metrics by setting the `telemetry.metrics.prometheus.disable` to `true` in the `EnvoyProxy` CRD. + +```shell +kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/kubernetes/metric/disable-prometheus.yaml +``` + Envoy Gateway can send metrics to OpenTelemetry Sink. Send metrics to OTel-Collector: