diff --git a/api/v1alpha1/envoygateway_helpers.go b/api/v1alpha1/envoygateway_helpers.go index d9673a0cd2a..e9e369f4e74 100644 --- a/api/v1alpha1/envoygateway_helpers.go +++ b/api/v1alpha1/envoygateway_helpers.go @@ -19,10 +19,11 @@ func DefaultEnvoyGateway() *EnvoyGateway { APIVersion: GroupVersion.String(), }, EnvoyGatewaySpec{ - Gateway: DefaultGateway(), - Provider: DefaultEnvoyGatewayProvider(), - Logging: DefaultEnvoyGatewayLogging(), - Admin: DefaultEnvoyGatewayAdmin(), + Gateway: DefaultGateway(), + Provider: DefaultEnvoyGatewayProvider(), + Logging: DefaultEnvoyGatewayLogging(), + Admin: DefaultEnvoyGatewayAdmin(), + Telemetry: DefaultEnvoyGatewayTelemetry(), }, } } @@ -47,6 +48,9 @@ func (e *EnvoyGateway) SetEnvoyGatewayDefaults() { if e.Admin == nil { e.Admin = DefaultEnvoyGatewayAdmin() } + if e.Telemetry == nil { + e.Telemetry = DefaultEnvoyGatewayTelemetry() + } } // GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified. @@ -88,6 +92,50 @@ func DefaultEnvoyGatewayLogging() *EnvoyGatewayLogging { } } +// GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified. +func (e *EnvoyGateway) GetEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry { + if e.Telemetry != nil { + if e.Telemetry.Metrics.Prometheus == nil { + e.Telemetry.Metrics.Prometheus = DefaultEnvoyGatewayPrometheus() + } + + if e.Telemetry.Metrics == nil { + e.Telemetry.Metrics = DefaultEnvoyGatewayMetrics() + } + return e.Telemetry + } + e.Telemetry = DefaultEnvoyGatewayTelemetry() + + return e.Telemetry +} + +// IfDisablePrometheus returns if disable prometheus. +func (e *EnvoyGateway) IfDisablePrometheus() bool { + return e.GetEnvoyGatewayTelemetry().Metrics.Prometheus.Disable +} + +// DefaultEnvoyGatewayTelemetry returns a new EnvoyGatewayTelemetry with default configuration parameters. +func DefaultEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry { + return &EnvoyGatewayTelemetry{ + Metrics: DefaultEnvoyGatewayMetrics(), + } +} + +// DefaultEnvoyGatewayMetrics returns a new EnvoyGatewayMetrics with default configuration parameters. +func DefaultEnvoyGatewayMetrics() *EnvoyGatewayMetrics { + return &EnvoyGatewayMetrics{ + Prometheus: DefaultEnvoyGatewayPrometheus(), + } +} + +// DefaultEnvoyGatewayPrometheus returns a new EnvoyGatewayMetrics with default configuration parameters. +func DefaultEnvoyGatewayPrometheus() *EnvoyGatewayPrometheusProvider { + return &EnvoyGatewayPrometheusProvider{ + // Enable prometheus pull by default. + Disable: false, + } +} + // DefaultEnvoyGatewayProvider returns a new EnvoyGatewayProvider with default configuration parameters. func DefaultEnvoyGatewayProvider() *EnvoyGatewayProvider { return &EnvoyGatewayProvider{ diff --git a/api/v1alpha1/envoygateway_metrics_types.go b/api/v1alpha1/envoygateway_metrics_types.go new file mode 100644 index 00000000000..0b827a06afa --- /dev/null +++ b/api/v1alpha1/envoygateway_metrics_types.go @@ -0,0 +1,48 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package v1alpha1 + +// EnvoyGatewayMetrics defines control plane push/pull metrics configurations. +type EnvoyGatewayMetrics struct { + // Sinks defines the metric sinks where metrics are sent to. + Sinks []EnvoyGatewayMetricSink `json:"sinks,omitempty"` + // Prometheus defines the configuration for prometheus endpoint. + Prometheus *EnvoyGatewayPrometheusProvider `json:"prometheus,omitempty"` +} + +// EnvoyGatewayMetricSink defines control plane +// metric sinks where metrics are sent to. +type EnvoyGatewayMetricSink struct { + // Type defines the metric sink type. + // EG control plane currently supports OpenTelemetry. + // +kubebuilder:validation:Enum=OpenTelemetry + // +kubebuilder:default=OpenTelemetry + Type MetricSinkType `json:"type"` + // OpenTelemetry defines the configuration for OpenTelemetry sink. + // It's required if the sink type is OpenTelemetry. + OpenTelemetry *EnvoyGatewayOpenTelemetrySink `json:"openTelemetry,omitempty"` +} + +type EnvoyGatewayOpenTelemetrySink struct { + // Host define the sink service hostname. + Host string `json:"host"` + // Protocol define the sink service protocol. + // +kubebuilder:validation:Enum=grpc;http + Protocol string `json:"protocol"` + // Port defines the port the sink service is exposed on. + // + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=4317 + Port int32 `json:"port,omitempty"` +} + +// EnvoyGatewayPrometheusProvider will expose prometheus endpoint in pull mode. +type EnvoyGatewayPrometheusProvider struct { + // Disable defines if disables the prometheus metrics in pull mode. + // + Disable bool `json:"disable,omitempty"` +} diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index efdf826eae5..bfbf4101e50 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -19,6 +19,10 @@ const ( GatewayAdminPort = 19000 // GatewayAdminHost is the host of envoy gateway admin server. GatewayAdminHost = "127.0.0.1" + // GatewayMetricsPort is the port which envoy gateway metrics server is listening on. + GatewayMetricsPort = 19001 + // GatewayMetricsHost is the host of envoy gateway metrics server. + GatewayMetricsHost = "0.0.0.0" ) // +kubebuilder:object:root=true @@ -59,6 +63,12 @@ type EnvoyGatewaySpec struct { // +optional Admin *EnvoyGatewayAdmin `json:"admin,omitempty"` + // Telemetry defines the desired control plane telemetry related abilities. + // If unspecified, the telemetry is used with default configuration. + // + // +optional + Telemetry *EnvoyGatewayTelemetry `json:"telemetry,omitempty"` + // RateLimit defines the configuration associated with the Rate Limit service // deployed by Envoy Gateway required to implement the Global Rate limiting // functionality. The specific rate limit service used here is the reference @@ -80,6 +90,13 @@ type EnvoyGatewaySpec struct { ExtensionAPIs *ExtensionAPISettings `json:"extensionApis,omitempty"` } +// EnvoyGatewayTelemetry defines telemetry configurations for envoy gateway control plane. +// Control plane will focus on metrics observability telemetry and tracing telemetry later. +type EnvoyGatewayTelemetry struct { + // Metrics defines metrics configuration for envoy gateway. + Metrics *EnvoyGatewayMetrics `json:"metrics,omitempty"` +} + // EnvoyGatewayLogging defines logging for Envoy Gateway. type EnvoyGatewayLogging struct { // Level is the logging level. If unspecified, defaults to "info". diff --git a/api/v1alpha1/metric_types.go b/api/v1alpha1/envoyproxy_metric_types.go similarity index 90% rename from api/v1alpha1/metric_types.go rename to api/v1alpha1/envoyproxy_metric_types.go index b3bf977a761..170a1686b00 100644 --- a/api/v1alpha1/metric_types.go +++ b/api/v1alpha1/envoyproxy_metric_types.go @@ -5,11 +5,17 @@ package v1alpha1 +type MetricSinkType string + +const ( + MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry" +) + type ProxyMetrics struct { // Prometheus defines the configuration for Admin endpoint `/stats/prometheus`. - Prometheus *PrometheusProvider `json:"prometheus,omitempty"` + Prometheus *ProxyPrometheusProvider `json:"prometheus,omitempty"` // Sinks defines the metric sinks where metrics are sent to. - Sinks []MetricSink `json:"sinks,omitempty"` + Sinks []ProxyMetricSink `json:"sinks,omitempty"` // Matches defines configuration for selecting specific metrics instead of generating all metrics stats // that are enabled by default. This helps reduce CPU and memory overhead in Envoy, but eliminating some stats // may after critical functionality. Here are the stats that we strongly recommend not disabling: @@ -23,13 +29,7 @@ type ProxyMetrics struct { EnableVirtualHostStats bool `json:"enableVirtualHostStats,omitempty"` } -type MetricSinkType string - -const ( - MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry" -) - -type MetricSink struct { +type ProxyMetricSink struct { // Type defines the metric sink type. // EG currently only supports OpenTelemetry. // +kubebuilder:validation:Enum=OpenTelemetry @@ -37,27 +37,10 @@ type MetricSink struct { Type MetricSinkType `json:"type"` // OpenTelemetry defines the configuration for OpenTelemetry sink. // It's required if the sink type is OpenTelemetry. - OpenTelemetry *OpenTelemetrySink `json:"openTelemetry,omitempty"` + OpenTelemetry *ProxyOpenTelemetrySink `json:"openTelemetry,omitempty"` } -// Match defines the stats match configuration. -type Match struct { - // MatcherType defines the stats matcher type - // - // +kubebuilder:validation:Enum=RegularExpression;Prefix;Suffix - Type MatcherType `json:"type"` - Value string `json:"value"` -} - -type MatcherType string - -const ( - Prefix MatcherType = "Prefix" - RegularExpression MatcherType = "RegularExpression" - Suffix MatcherType = "Suffix" -) - -type OpenTelemetrySink struct { +type ProxyOpenTelemetrySink struct { // Host define the service hostname. Host string `json:"host"` // Port defines the port the service is exposed on. @@ -71,7 +54,24 @@ type OpenTelemetrySink struct { // TODO: add support for customizing OpenTelemetry sink in https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/stat_sinks/open_telemetry/v3/open_telemetry.proto#envoy-v3-api-msg-extensions-stat-sinks-open-telemetry-v3-sinkconfig } -type PrometheusProvider struct { +type ProxyPrometheusProvider struct { // Disable the Prometheus endpoint. Disable bool `json:"disable,omitempty"` } + +// Match defines the stats match configuration. +type Match struct { + // MatcherType defines the stats matcher type + // + // +kubebuilder:validation:Enum=RegularExpression;Prefix;Suffix + Type MatcherType `json:"type"` + Value string `json:"value"` +} + +type MatcherType string + +const ( + Prefix MatcherType = "Prefix" + RegularExpression MatcherType = "RegularExpression" + Suffix MatcherType = "Suffix" +) diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go index 1f254696ef2..657cb9ee73d 100644 --- a/api/v1alpha1/validation/envoygateway_validate.go +++ b/api/v1alpha1/validation/envoygateway_validate.go @@ -82,6 +82,16 @@ func ValidateEnvoyGateway(eg *v1alpha1.EnvoyGateway) error { return fmt.Errorf("unsupported extension server TLS certificateRef %v", certificateRefKind) } } + case eg.Telemetry != nil: + if eg.Telemetry.Metrics != nil { + for _, sink := range eg.Telemetry.Metrics.Sinks { + if sink.Type == v1alpha1.MetricSinkTypeOpenTelemetry { + if sink.OpenTelemetry == nil { + return fmt.Errorf("OpenTelemetry is required when sink Type is OpenTelemetry") + } + } + } + } } return nil } diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go index a88caf5d517..06a3043be12 100644 --- a/api/v1alpha1/validation/envoygateway_validate_test.go +++ b/api/v1alpha1/validation/envoygateway_validate_test.go @@ -413,6 +413,47 @@ func TestValidateEnvoyGateway(t *testing.T) { }, }, expect: false, + }, { + name: "valid gateway metrics sink", + eg: &v1alpha1.EnvoyGateway{ + EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ + Gateway: v1alpha1.DefaultGateway(), + Provider: v1alpha1.DefaultEnvoyGatewayProvider(), + Telemetry: &v1alpha1.EnvoyGatewayTelemetry{ + Metrics: &v1alpha1.EnvoyGatewayMetrics{ + Sinks: []v1alpha1.EnvoyGatewayMetricSink{ + { + Type: v1alpha1.MetricSinkTypeOpenTelemetry, + OpenTelemetry: &v1alpha1.EnvoyGatewayOpenTelemetrySink{ + Host: "x.x.x.x", + Port: 4317, + Protocol: "grpc", + }, + }, + }, + }, + }, + }, + }, + expect: true, + }, { + name: "invalid gateway metrics sink", + eg: &v1alpha1.EnvoyGateway{ + EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ + Gateway: v1alpha1.DefaultGateway(), + Provider: v1alpha1.DefaultEnvoyGatewayProvider(), + Telemetry: &v1alpha1.EnvoyGatewayTelemetry{ + Metrics: &v1alpha1.EnvoyGatewayMetrics{ + Sinks: []v1alpha1.EnvoyGatewayMetricSink{ + { + Type: v1alpha1.MetricSinkTypeOpenTelemetry, + }, + }, + }, + }, + }, + }, + expect: false, }, } diff --git a/api/v1alpha1/validation/envoyproxy_validate.go b/api/v1alpha1/validation/envoyproxy_validate.go index 1ba367ef0b8..b1ebbbab80d 100644 --- a/api/v1alpha1/validation/envoyproxy_validate.go +++ b/api/v1alpha1/validation/envoyproxy_validate.go @@ -191,6 +191,17 @@ func validateProxyTelemetry(spec *egv1a1.EnvoyProxySpec) []error { } } + if spec != nil && spec.Telemetry != nil && spec.Telemetry.Metrics != nil { + for _, sink := range spec.Telemetry.Metrics.Sinks { + if sink.Type == egv1a1.MetricSinkTypeOpenTelemetry { + if sink.OpenTelemetry == nil { + err := fmt.Errorf("opentelemetry is required if the sink type is OpenTelemetry") + errs = append(errs, err) + } + } + } + } + return errs } diff --git a/api/v1alpha1/validation/envoyproxy_validate_test.go b/api/v1alpha1/validation/envoyproxy_validate_test.go index 7f4b4b7a952..0f60a6fa593 100644 --- a/api/v1alpha1/validation/envoyproxy_validate_test.go +++ b/api/v1alpha1/validation/envoyproxy_validate_test.go @@ -404,6 +404,50 @@ func TestValidateEnvoyProxy(t *testing.T) { }, }, expected: false, + }, { + name: "should invalid when metrics type is OpenTelemetry, but `OpenTelemetry` field being empty", + proxy: &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "test", + }, + Spec: egv1a1.EnvoyProxySpec{ + Telemetry: &egv1a1.ProxyTelemetry{ + Metrics: &egv1a1.ProxyMetrics{ + Sinks: []egv1a1.ProxyMetricSink{ + { + Type: egv1a1.MetricSinkTypeOpenTelemetry, + }, + }, + }, + }, + }, + }, + expected: false, + }, { + name: "should valid when metrics type is OpenTelemetry and `OpenTelemetry` field being not empty", + proxy: &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "test", + }, + Spec: egv1a1.EnvoyProxySpec{ + Telemetry: &egv1a1.ProxyTelemetry{ + Metrics: &egv1a1.ProxyMetrics{ + Sinks: []egv1a1.ProxyMetricSink{ + { + Type: egv1a1.MetricSinkTypeOpenTelemetry, + OpenTelemetry: &egv1a1.ProxyOpenTelemetrySink{ + Host: "0.0.0.0", + Port: 3217, + }, + }, + }, + }, + }, + }, + }, + expected: true, }, } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c9b8600c45f..be820081ebf 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -545,6 +545,83 @@ func (in *EnvoyGatewayLogging) DeepCopy() *EnvoyGatewayLogging { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvoyGatewayMetricSink) DeepCopyInto(out *EnvoyGatewayMetricSink) { + *out = *in + if in.OpenTelemetry != nil { + in, out := &in.OpenTelemetry, &out.OpenTelemetry + *out = new(EnvoyGatewayOpenTelemetrySink) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyGatewayMetricSink. +func (in *EnvoyGatewayMetricSink) DeepCopy() *EnvoyGatewayMetricSink { + if in == nil { + return nil + } + out := new(EnvoyGatewayMetricSink) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvoyGatewayMetrics) DeepCopyInto(out *EnvoyGatewayMetrics) { + *out = *in + if in.Sinks != nil { + in, out := &in.Sinks, &out.Sinks + *out = make([]EnvoyGatewayMetricSink, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Prometheus != nil { + in, out := &in.Prometheus, &out.Prometheus + *out = new(EnvoyGatewayPrometheusProvider) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyGatewayMetrics. +func (in *EnvoyGatewayMetrics) DeepCopy() *EnvoyGatewayMetrics { + if in == nil { + return nil + } + out := new(EnvoyGatewayMetrics) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvoyGatewayOpenTelemetrySink) DeepCopyInto(out *EnvoyGatewayOpenTelemetrySink) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyGatewayOpenTelemetrySink. +func (in *EnvoyGatewayOpenTelemetrySink) DeepCopy() *EnvoyGatewayOpenTelemetrySink { + if in == nil { + return nil + } + out := new(EnvoyGatewayOpenTelemetrySink) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvoyGatewayPrometheusProvider) DeepCopyInto(out *EnvoyGatewayPrometheusProvider) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyGatewayPrometheusProvider. +func (in *EnvoyGatewayPrometheusProvider) DeepCopy() *EnvoyGatewayPrometheusProvider { + if in == nil { + return nil + } + out := new(EnvoyGatewayPrometheusProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EnvoyGatewayProvider) DeepCopyInto(out *EnvoyGatewayProvider) { *out = *in @@ -613,6 +690,11 @@ func (in *EnvoyGatewaySpec) DeepCopyInto(out *EnvoyGatewaySpec) { *out = new(EnvoyGatewayAdmin) (*in).DeepCopyInto(*out) } + if in.Telemetry != nil { + in, out := &in.Telemetry, &out.Telemetry + *out = new(EnvoyGatewayTelemetry) + (*in).DeepCopyInto(*out) + } if in.RateLimit != nil { in, out := &in.RateLimit, &out.RateLimit *out = new(RateLimit) @@ -640,6 +722,26 @@ func (in *EnvoyGatewaySpec) DeepCopy() *EnvoyGatewaySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvoyGatewayTelemetry) DeepCopyInto(out *EnvoyGatewayTelemetry) { + *out = *in + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = new(EnvoyGatewayMetrics) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyGatewayTelemetry. +func (in *EnvoyGatewayTelemetry) DeepCopy() *EnvoyGatewayTelemetry { + if in == nil { + return nil + } + out := new(EnvoyGatewayTelemetry) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EnvoyJSONPatchConfig) DeepCopyInto(out *EnvoyJSONPatchConfig) { *out = *in @@ -1406,26 +1508,6 @@ func (in *Match) DeepCopy() *Match { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MetricSink) DeepCopyInto(out *MetricSink) { - *out = *in - if in.OpenTelemetry != nil { - in, out := &in.OpenTelemetry, &out.OpenTelemetry - *out = new(OpenTelemetrySink) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricSink. -func (in *MetricSink) DeepCopy() *MetricSink { - if in == nil { - return nil - } - out := new(MetricSink) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenTelemetryEnvoyProxyAccessLog) DeepCopyInto(out *OpenTelemetryEnvoyProxyAccessLog) { *out = *in @@ -1448,36 +1530,6 @@ func (in *OpenTelemetryEnvoyProxyAccessLog) DeepCopy() *OpenTelemetryEnvoyProxyA return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OpenTelemetrySink) DeepCopyInto(out *OpenTelemetrySink) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenTelemetrySink. -func (in *OpenTelemetrySink) DeepCopy() *OpenTelemetrySink { - if in == nil { - return nil - } - out := new(OpenTelemetrySink) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PrometheusProvider) DeepCopyInto(out *PrometheusProvider) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusProvider. -func (in *PrometheusProvider) DeepCopy() *PrometheusProvider { - if in == nil { - return nil - } - out := new(PrometheusProvider) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxyAccessLog) DeepCopyInto(out *ProxyAccessLog) { *out = *in @@ -1617,17 +1669,37 @@ func (in *ProxyLogging) DeepCopy() *ProxyLogging { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyMetricSink) DeepCopyInto(out *ProxyMetricSink) { + *out = *in + if in.OpenTelemetry != nil { + in, out := &in.OpenTelemetry, &out.OpenTelemetry + *out = new(ProxyOpenTelemetrySink) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyMetricSink. +func (in *ProxyMetricSink) DeepCopy() *ProxyMetricSink { + if in == nil { + return nil + } + out := new(ProxyMetricSink) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxyMetrics) DeepCopyInto(out *ProxyMetrics) { *out = *in if in.Prometheus != nil { in, out := &in.Prometheus, &out.Prometheus - *out = new(PrometheusProvider) + *out = new(ProxyPrometheusProvider) **out = **in } if in.Sinks != nil { in, out := &in.Sinks, &out.Sinks - *out = make([]MetricSink, len(*in)) + *out = make([]ProxyMetricSink, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1649,6 +1721,36 @@ func (in *ProxyMetrics) DeepCopy() *ProxyMetrics { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyOpenTelemetrySink) DeepCopyInto(out *ProxyOpenTelemetrySink) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyOpenTelemetrySink. +func (in *ProxyOpenTelemetrySink) DeepCopy() *ProxyOpenTelemetrySink { + if in == nil { + return nil + } + out := new(ProxyOpenTelemetrySink) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyPrometheusProvider) DeepCopyInto(out *ProxyPrometheusProvider) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyPrometheusProvider. +func (in *ProxyPrometheusProvider) DeepCopy() *ProxyPrometheusProvider { + if in == nil { + return nil + } + out := new(ProxyPrometheusProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxyTelemetry) DeepCopyInto(out *ProxyTelemetry) { *out = *in diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 9e1602f44d4..1c784927385 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -259,7 +259,7 @@ func TestDeployment(t *testing.T) { infra: newTestInfra(), telemetry: &egv1a1.ProxyTelemetry{ Metrics: &egv1a1.ProxyMetrics{ - Prometheus: &egv1a1.PrometheusProvider{}, + Prometheus: &egv1a1.ProxyPrometheusProvider{}, }, }, }, @@ -353,7 +353,7 @@ func TestDeployment(t *testing.T) { } else { tc.infra.Proxy.Config.Spec.Telemetry = &egv1a1.ProxyTelemetry{ Metrics: &egv1a1.ProxyMetrics{ - Prometheus: &egv1a1.PrometheusProvider{ + Prometheus: &egv1a1.ProxyPrometheusProvider{ Disable: true, }, }, diff --git a/internal/xds/bootstrap/bootstrap_test.go b/internal/xds/bootstrap/bootstrap_test.go index bd246c212ea..b145045a8b3 100644 --- a/internal/xds/bootstrap/bootstrap_test.go +++ b/internal/xds/bootstrap/bootstrap_test.go @@ -24,7 +24,7 @@ func TestGetRenderedBootstrapConfig(t *testing.T) { { name: "disable-prometheus", proxyMetrics: &egv1a1.ProxyMetrics{ - Prometheus: &egv1a1.PrometheusProvider{ + Prometheus: &egv1a1.ProxyPrometheusProvider{ Disable: true, }, }, @@ -32,19 +32,19 @@ func TestGetRenderedBootstrapConfig(t *testing.T) { { name: "enable-prometheus", proxyMetrics: &egv1a1.ProxyMetrics{ - Prometheus: &egv1a1.PrometheusProvider{}, + Prometheus: &egv1a1.ProxyPrometheusProvider{}, }, }, { name: "otel-metrics", proxyMetrics: &egv1a1.ProxyMetrics{ - Prometheus: &egv1a1.PrometheusProvider{ + Prometheus: &egv1a1.ProxyPrometheusProvider{ Disable: true, }, - Sinks: []egv1a1.MetricSink{ + Sinks: []egv1a1.ProxyMetricSink{ { Type: egv1a1.MetricSinkTypeOpenTelemetry, - OpenTelemetry: &egv1a1.OpenTelemetrySink{ + OpenTelemetry: &egv1a1.ProxyOpenTelemetrySink{ Host: "otel-collector.monitoring.svc", Port: 4317, }, diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 1f3545304b6..d718cd49290 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -253,6 +253,7 @@ EnvoyGateway is the schema for the envoygateways API. | `provider` _[EnvoyGatewayProvider](#envoygatewayprovider)_ | Provider defines the desired provider and provider-specific configuration. If unspecified, the Kubernetes provider is used with default configuration parameters. | | `logging` _[EnvoyGatewayLogging](#envoygatewaylogging)_ | Logging defines logging parameters for Envoy Gateway. | | `admin` _[EnvoyGatewayAdmin](#envoygatewayadmin)_ | Admin defines the desired admin related abilities. If unspecified, the Admin is used with default configuration parameters. | +| `telemetry` _[EnvoyGatewayTelemetry](#envoygatewaytelemetry)_ | Telemetry defines the desired control plane telemetry related abilities. If unspecified, the telemetry is used with default configuration. | | `rateLimit` _[RateLimit](#ratelimit)_ | RateLimit defines the configuration associated with the Rate Limit service deployed by Envoy Gateway required to implement the Global Rate limiting functionality. The specific rate limit service used here is the reference implementation in Envoy. For more details visit https://github.com/envoyproxy/ratelimit. This configuration is unneeded for "Local" rate limiting. | | `extensionManager` _[ExtensionManager](#extensionmanager)_ | ExtensionManager defines an extension manager to register for the Envoy Gateway Control Plane. | | `extensionApis` _[ExtensionAPISettings](#extensionapisettings)_ | ExtensionAPIs defines the settings related to specific Gateway API Extensions implemented by Envoy Gateway | @@ -388,6 +389,66 @@ _Appears in:_ | `level` _object (keys:[EnvoyGatewayLogComponent](#envoygatewaylogcomponent), values:[LogLevel](#loglevel))_ | Level is the logging level. If unspecified, defaults to "info". EnvoyGatewayLogComponent options: default/provider/gateway-api/xds-translator/xds-server/infrastructure/global-ratelimit. LogLevel options: debug/info/error/warn. | +#### EnvoyGatewayMetricSink + + + +EnvoyGatewayMetricSink defines control plane metric sinks where metrics are sent to. + +_Appears in:_ +- [EnvoyGatewayMetrics](#envoygatewaymetrics) + +| Field | Description | +| --- | --- | +| `type` _[MetricSinkType](#metricsinktype)_ | Type defines the metric sink type. EG control plane currently supports OpenTelemetry. | +| `openTelemetry` _[EnvoyGatewayOpenTelemetrySink](#envoygatewayopentelemetrysink)_ | OpenTelemetry defines the configuration for OpenTelemetry sink. It's required if the sink type is OpenTelemetry. | + + +#### EnvoyGatewayMetrics + + + +EnvoyGatewayMetrics defines control plane push/pull metrics configurations. + +_Appears in:_ +- [EnvoyGatewayTelemetry](#envoygatewaytelemetry) + +| Field | Description | +| --- | --- | +| `sinks` _[EnvoyGatewayMetricSink](#envoygatewaymetricsink) array_ | Sinks defines the metric sinks where metrics are sent to. | +| `prometheus` _[EnvoyGatewayPrometheusProvider](#envoygatewayprometheusprovider)_ | Prometheus defines the configuration for prometheus endpoint. | + + +#### EnvoyGatewayOpenTelemetrySink + + + + + +_Appears in:_ +- [EnvoyGatewayMetricSink](#envoygatewaymetricsink) + +| Field | Description | +| --- | --- | +| `host` _string_ | Host define the sink service hostname. | +| `protocol` _string_ | Protocol define the sink service protocol. | +| `port` _integer_ | Port defines the port the sink service is exposed on. | + + +#### EnvoyGatewayPrometheusProvider + + + +EnvoyGatewayPrometheusProvider will expose prometheus endpoint in pull mode. + +_Appears in:_ +- [EnvoyGatewayMetrics](#envoygatewaymetrics) + +| Field | Description | +| --- | --- | +| `disable` _boolean_ | Disable defines if disables the prometheus metrics in pull mode. | + + #### EnvoyGatewayProvider @@ -435,11 +496,27 @@ _Appears in:_ | `provider` _[EnvoyGatewayProvider](#envoygatewayprovider)_ | Provider defines the desired provider and provider-specific configuration. If unspecified, the Kubernetes provider is used with default configuration parameters. | | `logging` _[EnvoyGatewayLogging](#envoygatewaylogging)_ | Logging defines logging parameters for Envoy Gateway. | | `admin` _[EnvoyGatewayAdmin](#envoygatewayadmin)_ | Admin defines the desired admin related abilities. If unspecified, the Admin is used with default configuration parameters. | +| `telemetry` _[EnvoyGatewayTelemetry](#envoygatewaytelemetry)_ | Telemetry defines the desired control plane telemetry related abilities. If unspecified, the telemetry is used with default configuration. | | `rateLimit` _[RateLimit](#ratelimit)_ | RateLimit defines the configuration associated with the Rate Limit service deployed by Envoy Gateway required to implement the Global Rate limiting functionality. The specific rate limit service used here is the reference implementation in Envoy. For more details visit https://github.com/envoyproxy/ratelimit. This configuration is unneeded for "Local" rate limiting. | | `extensionManager` _[ExtensionManager](#extensionmanager)_ | ExtensionManager defines an extension manager to register for the Envoy Gateway Control Plane. | | `extensionApis` _[ExtensionAPISettings](#extensionapisettings)_ | ExtensionAPIs defines the settings related to specific Gateway API Extensions implemented by Envoy Gateway | +#### EnvoyGatewayTelemetry + + + +EnvoyGatewayTelemetry defines telemetry configurations for envoy gateway control plane. Control plane will focus on metrics observability telemetry and tracing telemetry later. + +_Appears in:_ +- [EnvoyGateway](#envoygateway) +- [EnvoyGatewaySpec](#envoygatewayspec) + +| Field | Description | +| --- | --- | +| `metrics` _[EnvoyGatewayMetrics](#envoygatewaymetrics)_ | Metrics defines metrics configuration for envoy gateway. | + + #### EnvoyJSONPatchConfig @@ -980,21 +1057,6 @@ _Appears in:_ -#### MetricSink - - - - - -_Appears in:_ -- [ProxyMetrics](#proxymetrics) - -| Field | Description | -| --- | --- | -| `type` _[MetricSinkType](#metricsinktype)_ | Type defines the metric sink type. EG currently only supports OpenTelemetry. | -| `openTelemetry` _[OpenTelemetrySink](#opentelemetrysink)_ | OpenTelemetry defines the configuration for OpenTelemetry sink. It's required if the sink type is OpenTelemetry. | - - #### MetricSinkType _Underlying type:_ `string` @@ -1002,7 +1064,8 @@ _Underlying type:_ `string` _Appears in:_ -- [MetricSink](#metricsink) +- [EnvoyGatewayMetricSink](#envoygatewaymetricsink) +- [ProxyMetricSink](#proxymetricsink) @@ -1022,35 +1085,6 @@ _Appears in:_ | `resources` _object (keys:string, values:string)_ | Resources is a set of labels that describe the source of a log entry, including envoy node info. It's recommended to follow [semantic conventions](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/). | -#### OpenTelemetrySink - - - - - -_Appears in:_ -- [MetricSink](#metricsink) - -| Field | Description | -| --- | --- | -| `host` _string_ | Host define the service hostname. | -| `port` _integer_ | Port defines the port the service is exposed on. | - - -#### PrometheusProvider - - - - - -_Appears in:_ -- [ProxyMetrics](#proxymetrics) - -| Field | Description | -| --- | --- | -| `disable` _boolean_ | Disable the Prometheus endpoint. | - - #### ProviderType _Underlying type:_ `string` @@ -1187,6 +1221,21 @@ _Appears in:_ | `level` _object (keys:[ProxyLogComponent](#proxylogcomponent), values:[LogLevel](#loglevel))_ | Level is a map of logging level per component, where the component is the key and the log level is the value. If unspecified, defaults to "default: warn". | +#### ProxyMetricSink + + + + + +_Appears in:_ +- [ProxyMetrics](#proxymetrics) + +| Field | Description | +| --- | --- | +| `type` _[MetricSinkType](#metricsinktype)_ | Type defines the metric sink type. EG currently only supports OpenTelemetry. | +| `openTelemetry` _[ProxyOpenTelemetrySink](#proxyopentelemetrysink)_ | OpenTelemetry defines the configuration for OpenTelemetry sink. It's required if the sink type is OpenTelemetry. | + + #### ProxyMetrics @@ -1198,12 +1247,41 @@ _Appears in:_ | Field | Description | | --- | --- | -| `prometheus` _[PrometheusProvider](#prometheusprovider)_ | Prometheus defines the configuration for Admin endpoint `/stats/prometheus`. | -| `sinks` _[MetricSink](#metricsink) array_ | Sinks defines the metric sinks where metrics are sent to. | +| `prometheus` _[ProxyPrometheusProvider](#proxyprometheusprovider)_ | Prometheus defines the configuration for Admin endpoint `/stats/prometheus`. | +| `sinks` _[ProxyMetricSink](#proxymetricsink) array_ | Sinks defines the metric sinks where metrics are sent to. | | `matches` _[Match](#match) array_ | Matches defines configuration for selecting specific metrics instead of generating all metrics stats that are enabled by default. This helps reduce CPU and memory overhead in Envoy, but eliminating some stats may after critical functionality. Here are the stats that we strongly recommend not disabling: `cluster_manager.warming_clusters`, `cluster..membership_total`,`cluster..membership_healthy`, `cluster..membership_degraded`,reference https://github.com/envoyproxy/envoy/issues/9856, https://github.com/envoyproxy/envoy/issues/14610 | | `enableVirtualHostStats` _boolean_ | EnableVirtualHostStats enables envoy stat metrics for virtual hosts. | +#### ProxyOpenTelemetrySink + + + + + +_Appears in:_ +- [ProxyMetricSink](#proxymetricsink) + +| Field | Description | +| --- | --- | +| `host` _string_ | Host define the service hostname. | +| `port` _integer_ | Port defines the port the service is exposed on. | + + +#### ProxyPrometheusProvider + + + + + +_Appears in:_ +- [ProxyMetrics](#proxymetrics) + +| Field | Description | +| --- | --- | +| `disable` _boolean_ | Disable the Prometheus endpoint. | + + #### ProxyTelemetry diff --git a/site/content/en/latest/design/eg-metrics.md b/site/content/en/latest/design/eg-metrics.md new file mode 100644 index 00000000000..f43af77be12 --- /dev/null +++ b/site/content/en/latest/design/eg-metrics.md @@ -0,0 +1,225 @@ +--- +date: 2023-10-10 +title: "Control Plane Observability: Metrics" +--- + +This document aims to cover all aspects of envoy gateway control plane metrics observability. + +{{% alert title="Note" color="secondary" %}} +**Data plane** observability (while important) is outside of scope for this document. For dataplane observability, refer to [here](./metrics). +{{% /alert %}} + +## Current State + +At present, the Envoy Gateway control plane provides logs and controller-runtime metrics, without traces. Logs are managed through our proprietary library (`internal/logging`, a shim to `zap`) and are written to `/dev/stdout`. + +## Goals + +Our objectives include: + ++ Supporting **PULL** mode for Prometheus metrics and exposing these metrics on the admin address. ++ Supporting **PUSH** mode for Prometheus metrics, thereby sending metrics to the Open Telemetry Stats sink via gRPC or HTTP. + +## Non-Goals + +Our non-goals include: + ++ Supporting other stats sinks. + +## Use-Cases + +The use-cases include: + ++ Exposing Prometheus metrics in the Envoy Gateway Control Plane. ++ Pushing Envoy Gateway Control Plane metrics via the Open Telemetry Sink. + +## Design + +### Standards + +Our metrics, will be built upon the [OpenTelemetry](https://opentelemetry.io/) standards. All metrics will be configured via the [OpenTelemetry SDK](https://opentelemetry.io/docs/specs/otel/metrics/sdk/), which offers neutral libraries that can be connected to various backends. + +This approach allows the Envoy Gateway code to concentrate on the crucial aspect - generating the metrics - and delegate all other tasks to systems designed for telemetry ingestion. + +### Attributes + +OpenTelemetry defines a set of [Semantic Conventions](https://opentelemetry.io/docs/concepts/semantic-conventions/), including [Kubernetes specific ones](https://opentelemetry.io/docs/specs/otel/resource/semantic_conventions/k8s/). + +These attributes can be expressed in logs (as keys of structured logs), traces (as attributes), and metrics (as labels). + +We aim to use attributes consistently where applicable. Where possible, these should adhere to codified Semantic Conventions; when not possible, they should maintain consistency across the project. + +### Extensibility + +Envoy Gateway supports both **PULL/PUSH** mode metrics, with Metrics exported via Prometheus by default. + +Additionally, Envoy Gateway can export metrics using both the [OTEL gRPC metrics exporter](https://opentelemetry.io/docs/specs/otel/metrics/sdk_exporters/otlp/#general) and [OTEL HTTP metrics exporter](https://opentelemetry.io/docs/specs/otel/metrics/sdk_exporters/otlp/#general), which pushes metrics by grpc/http to a remote OTEL collector. + +Users can extend these in two ways: + +#### Downstream Collection + +Based on the exported data, other tools can collect, process, and export telemetry as needed. Some examples include: + ++ Metrics in **PULL** mode: The OTEL collector can scrape Prometheus and export to X. ++ Metrics in **PUSH** mode: The OTEL collector can receive OTEL gRPC/HTTP exporter metrics and export to X. + +While the examples above involve OTEL collectors, there are numerous other systems available. + +#### Vendor extensions + +The OTEL libraries allow for the registration of Providers/Handlers. While we will offer the default ones (PULL via Prometheus, PUSH via OTEL HTTP metrics exporter) mentioned in Envoy Gateway's extensibility, we can easily allow custom builds of Envoy Gateway to plug in alternatives if the default options don't meet their needs. + +For instance, users may prefer to write metrics over the OTLP gRPC metrics exporter instead of the HTTP metrics exporter. This is perfectly acceptable -- and almost impossible to prevent. The OTEL has ways to register their providers/exporters, and Envoy Gateway can ensure its usage is such that it's not overly difficult to swap out a different provider/exporter. + +### Stability + +Observability is, in essence, a user-facing API. Its primary purpose is to be consumed - by both humans and tooling. Therefore, having well-defined guarantees around their formats is crucial. + +Please note that this refers only to the contents of the telemetry - what we emit, the names of things, semantics, etc. Other settings like Prometheus vs OTLP, JSON vs plaintext, logging levels, etc., are not considered. + +I propose the following: + +#### Metrics + +Metrics offer the greatest potential for providing guarantees. They often directly influence alerts and dashboards, making changes highly impactful. This contrasts with traces and logs, which are often used for ad-hoc analysis, where minor changes to information can be easily understood by a human. + +Moreover, there is precedent for this: [Kubernetes Metrics Lifecycle](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#metric-lifecycle) has well-defined processes, and Envoy Gateway's dataplane (Envoy Proxy) metrics are de facto stable. + +Currently, all Envoy Gateway metrics lack defined stability. I suggest we categorize all existing metrics as either: + ++ ***Deprecated***: a metric that is intended to be phased out. ++ ***Experimental***: a metric that is off by default. ++ ***Alpha***: a metric that is on by default. + +We should aim to promote a core set of metrics to **Stable** within a few releases. + +## Envoy Gateway API Types + +New APIs will be added to Envoy Gateway config, which are used to manage Control Plane Telemetry bootstrap configs. + +### EnvoyGatewayTelemetry + +``` go +// EnvoyGatewayTelemetry defines telemetry configurations for envoy gateway control plane. +// Control plane will focus on metrics observability telemetry and tracing telemetry later. +type EnvoyGatewayTelemetry struct { + // Metrics defines metrics configuration for envoy gateway. + Metrics *EnvoyGatewayMetrics `json:"metrics,omitempty"` +} +``` + +### EnvoyGatewayMetrics + +> Prometheus will be exposed on 0.0.0.0:19001, which is not supported to be configured yet. + +``` go +// EnvoyGatewayMetrics defines control plane push/pull metrics configurations. +type EnvoyGatewayMetrics struct { + // Sinks defines the metric sinks where metrics are sent to. + Sinks []EnvoyGatewayMetricSink `json:"sinks,omitempty"` + // Prometheus defines the configuration for prometheus endpoint. + Prometheus *EnvoyGatewayPrometheusProvider `json:"prometheus,omitempty"` +} + +// EnvoyGatewayMetricSink defines control plane +// metric sinks where metrics are sent to. +type EnvoyGatewayMetricSink struct { + // Type defines the metric sink type. + // EG control plane currently supports OpenTelemetry. + // +kubebuilder:validation:Enum=OpenTelemetry + // +kubebuilder:default=OpenTelemetry + Type MetricSinkType `json:"type"` + // OpenTelemetry defines the configuration for OpenTelemetry sink. + // It's required if the sink type is OpenTelemetry. + OpenTelemetry *EnvoyGatewayOpenTelemetrySink `json:"openTelemetry,omitempty"` +} + +type EnvoyGatewayOpenTelemetrySink struct { + // Host define the sink service hostname. + Host string `json:"host"` + // Protocol define the sink service protocol. + // +kubebuilder:validation:Enum=grpc;http + Protocol string `json:"protocol"` + // Port defines the port the sink service is exposed on. + // + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=4317 + Port int32 `json:"port,omitempty"` +} + +// EnvoyGatewayPrometheusProvider will expose prometheus endpoint in pull mode. +type EnvoyGatewayPrometheusProvider struct { + // Disable defines if disables the prometheus metrics in pull mode. + // + Disable bool `json:"disable,omitempty"` +} + +``` + +#### Example + ++ The following is an example to disable prometheus metric. + +``` yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +logging: + level: null + default: info +provider: + type: Kubernetes +telemetry: + metrics: + prometheus: + disable: true +``` + ++ The following is an example to send metric via Open Telemetry sink to OTEL gRPC Collector. + +``` yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +logging: + level: null + default: info +provider: + type: Kubernetes +telemetry: + metrics: + sinks: + - type: OpenTelemetry + openTelemetry: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + protocol: grpc +``` + ++ The following is an example to disable prometheus metric and send metric via Open Telemetry sink to OTEL HTTP Collector at the same time. + +``` yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +logging: + level: null + default: info +provider: + type: Kubernetes +telemetry: + metrics: + prometheus: + disable: false + sinks: + - type: OpenTelemetry + openTelemetry: + host: otel-collector.monitoring.svc.cluster.local + port: 4318 + protocol: http +``` diff --git a/site/content/en/latest/install/install-helm.md b/site/content/en/latest/install/install-helm.md index e83575ea5b1..3f3c57e1db9 100644 --- a/site/content/en/latest/install/install-helm.md +++ b/site/content/en/latest/install/install-helm.md @@ -125,12 +125,12 @@ These are the ports used by Envoy Gateway and the managed Envoy Proxy. ### Envoy Gateway | Envoy Gateway | Address | Port | Configurable | -|:----------------------:|:---------:|:------:|:--------------:| -| Xds EnvoyProxy Server | 0.0.0.0 | 18000 | Yes | -| Xds RateLimit Server | 0.0.0.0 | 18001 | Yes | +|:----------------------:|:---------:|:------:| :------: | +| Xds EnvoyProxy Server | 0.0.0.0 | 18000 | No | +| Xds RateLimit Server | 0.0.0.0 | 18001 | No | | Admin Server | 127.0.0.1 | 19000 | Yes | -| Metrics Server | 0.0.0.0 | 19001 | Yes | -| Health Check | 127.0.0.1 | 8081 | No | +| Metrics Server | 0.0.0.0 | 19001 | No | +| Health Check | 127.0.0.1 | 8081 | No | ### EnvoyProxy