From 11ecc75fed2f0e0f215f6727092a4390336c05bc Mon Sep 17 00:00:00 2001 From: zufardhiyaulhaq Date: Mon, 15 Apr 2024 17:38:30 +0700 Subject: [PATCH] feat: support compression on prometheus Signed-off-by: zufardhiyaulhaq --- api/v1alpha1/envoyproxy_metric_types.go | 6 + .../gateway.envoyproxy.io_envoyproxies.yaml | 11 ++ internal/xds/bootstrap/bootstrap.go | 24 ++- internal/xds/bootstrap/bootstrap.yaml.tpl | 23 +++ internal/xds/bootstrap/bootstrap_test.go | 33 +++++ .../enable-prometheus-brotli-compression.yaml | 137 ++++++++++++++++++ .../enable-prometheus-gzip-compression.yaml | 137 ++++++++++++++++++ .../enable-prometheus-zstd-compression.yaml | 137 ++++++++++++++++++ site/content/en/latest/api/extension_types.md | 2 + 9 files changed, 505 insertions(+), 5 deletions(-) create mode 100644 internal/xds/bootstrap/testdata/render/enable-prometheus-brotli-compression.yaml create mode 100644 internal/xds/bootstrap/testdata/render/enable-prometheus-gzip-compression.yaml create mode 100644 internal/xds/bootstrap/testdata/render/enable-prometheus-zstd-compression.yaml diff --git a/api/v1alpha1/envoyproxy_metric_types.go b/api/v1alpha1/envoyproxy_metric_types.go index 59c6a6f2dd9..e01ebee4dac 100644 --- a/api/v1alpha1/envoyproxy_metric_types.go +++ b/api/v1alpha1/envoyproxy_metric_types.go @@ -79,4 +79,10 @@ type ProxyOpenTelemetrySink struct { type ProxyPrometheusProvider struct { // Disable the Prometheus endpoint. Disable bool `json:"disable,omitempty"` + // Enable the compression on Prometheus endpoint. + EnableCompression bool `json:"enableCompression,omitempty"` + // Compression type + // +kubebuilder:validation:Enum=gzip;brotli;zstd + // +kubebuilder:default=gzip + CompressionLibrary string `json:"compressionLibrary,omitempty"` } diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 76d96eccaea..7b78ae6ee67 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -6227,9 +6227,20 @@ spec: description: Prometheus defines the configuration for Admin endpoint `/stats/prometheus`. properties: + compressionLibrary: + default: gzip + description: Compression type + enum: + - gzip + - brotli + - zstd + type: string disable: description: Disable the Prometheus endpoint. type: boolean + enableCompression: + description: Enable the compression on Prometheus endpoint. + type: boolean type: object sinks: description: Sinks defines the metric sinks where metrics diff --git a/internal/xds/bootstrap/bootstrap.go b/internal/xds/bootstrap/bootstrap.go index 115494fd350..8b3b1f0a439 100644 --- a/internal/xds/bootstrap/bootstrap.go +++ b/internal/xds/bootstrap/bootstrap.go @@ -63,6 +63,11 @@ type bootstrapParameters struct { ReadyServer readyServerParameters // EnablePrometheus defines whether to enable metrics endpoint for prometheus. EnablePrometheus bool + // EnablePrometheusCompression defines whether to enable HTTP compression on metrics endpoint for prometheus. + EnablePrometheusCompression bool + // PrometheusCompressionLibrary defines the HTTP compression library for metrics endpoint for prometheus. + PrometheusCompressionLibrary string + // OtelMetricSinks defines the configuration of the OpenTelemetry sinks. OtelMetricSinks []metricSink // EnableStatConfig defines whether to to customize the Envoy proxy stats. @@ -136,9 +141,11 @@ func (b *bootstrapConfig) render() error { // GetRenderedBootstrapConfig renders the bootstrap YAML string func GetRenderedBootstrapConfig(opts *RenderBootsrapConfigOptions) (string, error) { var ( - enablePrometheus = true - metricSinks []metricSink - StatsMatcher StatsMatcherParameters + enablePrometheus = true + enablePrometheusCompression = false + PrometheusCompressionLibrary = "gzip" + metricSinks []metricSink + StatsMatcher StatsMatcherParameters ) if opts != nil && opts.ProxyMetrics != nil { @@ -146,6 +153,11 @@ func GetRenderedBootstrapConfig(opts *RenderBootsrapConfigOptions) (string, erro if proxyMetrics.Prometheus != nil { enablePrometheus = !proxyMetrics.Prometheus.Disable + enablePrometheusCompression = proxyMetrics.Prometheus.EnableCompression + + if proxyMetrics.Prometheus.EnableCompression { + PrometheusCompressionLibrary = proxyMetrics.Prometheus.CompressionLibrary + } } addresses := sets.NewString() @@ -216,8 +228,10 @@ func GetRenderedBootstrapConfig(opts *RenderBootsrapConfigOptions) (string, erro Port: EnvoyReadinessPort, ReadinessPath: EnvoyReadinessPath, }, - EnablePrometheus: enablePrometheus, - OtelMetricSinks: metricSinks, + EnablePrometheus: enablePrometheus, + EnablePrometheusCompression: enablePrometheusCompression, + PrometheusCompressionLibrary: PrometheusCompressionLibrary, + OtelMetricSinks: metricSinks, }, } if opts != nil && opts.ProxyMetrics != nil && opts.ProxyMetrics.Matches != nil { diff --git a/internal/xds/bootstrap/bootstrap.yaml.tpl b/internal/xds/bootstrap/bootstrap.yaml.tpl index 1c2a2f12edf..cc59b913862 100644 --- a/internal/xds/bootstrap/bootstrap.yaml.tpl +++ b/internal/xds/bootstrap/bootstrap.yaml.tpl @@ -86,6 +86,29 @@ static_resources: prefix: /stats/prometheus route: cluster: prometheus_stats + {{- if .EnablePrometheusCompression }} + typed_per_filter_config: + envoy.filters.http.compression: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.CompressorPerRoute + {{- if eq .PrometheusCompressionLibrary "gzip"}} + compressor_library: + name: text_optimized + typed_config: + "@type": type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip + {{- end }} + {{- if eq .PrometheusCompressionLibrary "brotli"}} + compressor_library: + name: text_optimized + typed_config: + "@type": type.googleapis.com/envoy.extensions.compression.brotli.compressor.v3.Brotli + {{- end }} + {{- if eq .PrometheusCompressionLibrary "zstd"}} + compressor_library: + name: text_optimized + typed_config: + "@type": type.googleapis.com/envoy.extensions.compression.zstd.compressor.v3.Zstd + {{- end }} + {{- end }} {{- end }} http_filters: - name: envoy.filters.http.health_check diff --git a/internal/xds/bootstrap/bootstrap_test.go b/internal/xds/bootstrap/bootstrap_test.go index 87c5ecb7958..821284d4cbf 100644 --- a/internal/xds/bootstrap/bootstrap_test.go +++ b/internal/xds/bootstrap/bootstrap_test.go @@ -42,6 +42,39 @@ func TestGetRenderedBootstrapConfig(t *testing.T) { }, }, }, + { + name: "enable-prometheus-gzip-compression", + opts: &RenderBootsrapConfigOptions{ + ProxyMetrics: &egv1a1.ProxyMetrics{ + Prometheus: &egv1a1.ProxyPrometheusProvider{ + EnableCompression: true, + CompressionLibrary: "gzip", + }, + }, + }, + }, + { + name: "enable-prometheus-brotli-compression", + opts: &RenderBootsrapConfigOptions{ + ProxyMetrics: &egv1a1.ProxyMetrics{ + Prometheus: &egv1a1.ProxyPrometheusProvider{ + EnableCompression: true, + CompressionLibrary: "brotli", + }, + }, + }, + }, + { + name: "enable-prometheus-zstd-compression", + opts: &RenderBootsrapConfigOptions{ + ProxyMetrics: &egv1a1.ProxyMetrics{ + Prometheus: &egv1a1.ProxyPrometheusProvider{ + EnableCompression: true, + CompressionLibrary: "zstd", + }, + }, + }, + }, { name: "otel-metrics", opts: &RenderBootsrapConfigOptions{ diff --git a/internal/xds/bootstrap/testdata/render/enable-prometheus-brotli-compression.yaml b/internal/xds/bootstrap/testdata/render/enable-prometheus-brotli-compression.yaml new file mode 100644 index 00000000000..b37944e2c8a --- /dev/null +++ b/internal/xds/bootstrap/testdata/render/enable-prometheus-brotli-compression.yaml @@ -0,0 +1,137 @@ +admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 +layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 +dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 +static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + typed_per_filter_config: + envoy.filters.http.compression: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.CompressorPerRoute + compressor_library: + name: text_optimized + typed_config: + "@type": type.googleapis.com/envoy.extensions.compression.brotli.compressor.v3.Brotli + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 +overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 diff --git a/internal/xds/bootstrap/testdata/render/enable-prometheus-gzip-compression.yaml b/internal/xds/bootstrap/testdata/render/enable-prometheus-gzip-compression.yaml new file mode 100644 index 00000000000..ca82e1996b4 --- /dev/null +++ b/internal/xds/bootstrap/testdata/render/enable-prometheus-gzip-compression.yaml @@ -0,0 +1,137 @@ +admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 +layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 +dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 +static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + typed_per_filter_config: + envoy.filters.http.compression: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.CompressorPerRoute + compressor_library: + name: text_optimized + typed_config: + "@type": type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 +overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 diff --git a/internal/xds/bootstrap/testdata/render/enable-prometheus-zstd-compression.yaml b/internal/xds/bootstrap/testdata/render/enable-prometheus-zstd-compression.yaml new file mode 100644 index 00000000000..922247b8340 --- /dev/null +++ b/internal/xds/bootstrap/testdata/render/enable-prometheus-zstd-compression.yaml @@ -0,0 +1,137 @@ +admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 +layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 +dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 +static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + typed_per_filter_config: + envoy.filters.http.compression: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.CompressorPerRoute + compressor_library: + name: text_optimized + typed_config: + "@type": type.googleapis.com/envoy.extensions.compression.zstd.compressor.v3.Zstd + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 +overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 7c9b355fefb..2f395d635c7 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -2316,6 +2316,8 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `disable` | _boolean_ | true | Disable the Prometheus endpoint. | +| `enableCompression` | _boolean_ | true | Enable the compression on Prometheus endpoint. | +| `compressionLibrary` | _string_ | true | Compression type | #### ProxyProtocol