design(docs/api): control plane metrics monitoring

api/v1alpha1/envoygateway_helpers.go

@@ -19,10 +19,11 @@ func DefaultEnvoyGateway() *EnvoyGateway {
 			APIVersion: GroupVersion.String(),
 		},
 		EnvoyGatewaySpec{
-			Gateway:  DefaultGateway(),
-			Provider: DefaultEnvoyGatewayProvider(),
-			Logging:  DefaultEnvoyGatewayLogging(),
-			Admin:    DefaultEnvoyGatewayAdmin(),
+			Gateway:   DefaultGateway(),
+			Provider:  DefaultEnvoyGatewayProvider(),
+			Logging:   DefaultEnvoyGatewayLogging(),
+			Admin:     DefaultEnvoyGatewayAdmin(),
+			Telemetry: DefaultEnvoyGatewayTelemetry(),
 		},
 	}
 }
@@ -47,6 +48,9 @@ func (e *EnvoyGateway) SetEnvoyGatewayDefaults() {
 	if e.Admin == nil {
 		e.Admin = DefaultEnvoyGatewayAdmin()
 	}
+	if e.Telemetry == nil {
+		e.Telemetry = DefaultEnvoyGatewayTelemetry()
+	}
 }
 
 // GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
@@ -88,6 +92,50 @@ func DefaultEnvoyGatewayLogging() *EnvoyGatewayLogging {
 	}
 }
 
+// GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
+func (e *EnvoyGateway) GetEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
+	if e.Telemetry != nil {
+		if e.Telemetry.Metrics.Prometheus == nil {
+			e.Telemetry.Metrics.Prometheus = DefaultEnvoyGatewayPrometheus()
+		}
+
+		if e.Telemetry.Metrics == nil {
+			e.Telemetry.Metrics = DefaultEnvoyGatewayMetrics()
+		}
+		return e.Telemetry
+	}
+	e.Telemetry = DefaultEnvoyGatewayTelemetry()
+
+	return e.Telemetry
+}
+
+// IfDisablePrometheus returns if disable prometheus.
+func (e *EnvoyGateway) IfDisablePrometheus() bool {
+	return e.GetEnvoyGatewayTelemetry().Metrics.Prometheus.Disable
+}
+
+// DefaultEnvoyGatewayTelemetry returns a new EnvoyGatewayTelemetry with default configuration parameters.
+func DefaultEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
+	return &EnvoyGatewayTelemetry{
+		Metrics: DefaultEnvoyGatewayMetrics(),
+	}
+}
+
+// DefaultEnvoyGatewayMetrics returns a new EnvoyGatewayMetrics with default configuration parameters.
+func DefaultEnvoyGatewayMetrics() *EnvoyGatewayMetrics {
+	return &EnvoyGatewayMetrics{
+		Prometheus: DefaultEnvoyGatewayPrometheus(),
+	}
+}
+
+// DefaultEnvoyGatewayPrometheus returns a new EnvoyGatewayMetrics with default configuration parameters.
+func DefaultEnvoyGatewayPrometheus() *EnvoyGatewayPrometheusProvider {
+	return &EnvoyGatewayPrometheusProvider{
+		// Enable prometheus pull by default.
+		Disable: false,
+	}
+}
+
 // DefaultEnvoyGatewayProvider returns a new EnvoyGatewayProvider with default configuration parameters.
 func DefaultEnvoyGatewayProvider() *EnvoyGatewayProvider {
 	return &EnvoyGatewayProvider{

api/v1alpha1/envoygateway_metrics_types.go

@@ -0,0 +1,48 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package v1alpha1
+
+// EnvoyGatewayMetrics defines control plane push/pull metrics configurations.
+type EnvoyGatewayMetrics struct {
+	// Sinks defines the metric sinks where metrics are sent to.
+	Sinks []EnvoyGatewayMetricSink `json:"sinks,omitempty"`
+	// Prometheus defines the configuration for prometheus endpoint.
+	Prometheus *EnvoyGatewayPrometheusProvider `json:"prometheus,omitempty"`
+}
+
+// EnvoyGatewayMetricSink defines control plane
+// metric sinks where metrics are sent to.
+type EnvoyGatewayMetricSink struct {
+	// Type defines the metric sink type.
+	// EG control plane currently supports OpenTelemetry.
+	// +kubebuilder:validation:Enum=OpenTelemetry
+	// +kubebuilder:default=OpenTelemetry
+	Type MetricSinkType `json:"type"`
+	// OpenTelemetry defines the configuration for OpenTelemetry sink.
+	// It's required if the sink type is OpenTelemetry.
+	OpenTelemetry *EnvoyGatewayOpenTelemetrySink `json:"openTelemetry,omitempty"`
+}
+
+type EnvoyGatewayOpenTelemetrySink struct {
+	// Host define the sink service hostname.
+	Host string `json:"host"`
+	// Protocol define the sink service protocol.
+	// +kubebuilder:validation:Enum=grpc;http
+	Protocol string `json:"protocol"`
+	// Port defines the port the sink service is exposed on.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:default=4317
+	Port int32 `json:"port,omitempty"`
+}
+
+// EnvoyGatewayPrometheusProvider will expose prometheus endpoint in pull mode.
+type EnvoyGatewayPrometheusProvider struct {
+	// Disable defines if disables the prometheus metrics in pull mode.
+	//
+	Disable bool `json:"disable,omitempty"`
+}

api/v1alpha1/envoygateway_types.go

@@ -19,6 +19,10 @@ const (
 	GatewayAdminPort = 19000
 	// GatewayAdminHost is the host of envoy gateway admin server.
 	GatewayAdminHost = "127.0.0.1"
+	// GatewayMetricsPort is the port which envoy gateway metrics server is listening on.
+	GatewayMetricsPort = 19001
+	// GatewayMetricsHost is the host of envoy gateway metrics server.
+	GatewayMetricsHost = "0.0.0.0"
 )
 
 // +kubebuilder:object:root=true
@@ -59,6 +63,12 @@ type EnvoyGatewaySpec struct {
 	// +optional
 	Admin *EnvoyGatewayAdmin `json:"admin,omitempty"`
 
+	// Telemetry defines the desired control plane telemetry related abilities.
+	// If unspecified, the telemetry is used with default configuration.
+	//
+	// +optional
+	Telemetry *EnvoyGatewayTelemetry `json:"telemetry,omitempty"`
+
 	// RateLimit defines the configuration associated with the Rate Limit service
 	// deployed by Envoy Gateway required to implement the Global Rate limiting
 	// functionality. The specific rate limit service used here is the reference
@@ -80,6 +90,13 @@ type EnvoyGatewaySpec struct {
 	ExtensionAPIs *ExtensionAPISettings `json:"extensionApis,omitempty"`
 }
 
+// EnvoyGatewayTelemetry defines telemetry configurations for envoy gateway control plane.
+// Control plane will focus on metrics observability telemetry and tracing telemetry later.
+type EnvoyGatewayTelemetry struct {
+	// Metrics defines metrics configuration for envoy gateway.
+	Metrics *EnvoyGatewayMetrics `json:"metrics,omitempty"`
+}
+
 // EnvoyGatewayLogging defines logging for Envoy Gateway.
 type EnvoyGatewayLogging struct {
 	// Level is the logging level. If unspecified, defaults to "info".

api/v1alpha1/metric_types.go → api/v1alpha1/envoyproxy_metric_types.go

@@ -5,11 +5,17 @@
 
 package v1alpha1
 
+type MetricSinkType string
+
+const (
+	MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
+)
+
 type ProxyMetrics struct {
 	// Prometheus defines the configuration for Admin endpoint `/stats/prometheus`.
-	Prometheus *PrometheusProvider `json:"prometheus,omitempty"`
+	Prometheus *ProxyPrometheusProvider `json:"prometheus,omitempty"`
 	// Sinks defines the metric sinks where metrics are sent to.
-	Sinks []MetricSink `json:"sinks,omitempty"`
+	Sinks []ProxyMetricSink `json:"sinks,omitempty"`
 	// Matches defines configuration for selecting specific metrics instead of generating all metrics stats
 	// that are enabled by default. This helps reduce CPU and memory overhead in Envoy, but eliminating some stats
 	// may after critical functionality. Here are the stats that we strongly recommend not disabling:
@@ -23,41 +29,18 @@ type ProxyMetrics struct {
 	EnableVirtualHostStats bool `json:"enableVirtualHostStats,omitempty"`
 }
 
-type MetricSinkType string
-
-const (
-	MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
-)
-
-type MetricSink struct {
+type ProxyMetricSink struct {
 	// Type defines the metric sink type.
 	// EG currently only supports OpenTelemetry.
 	// +kubebuilder:validation:Enum=OpenTelemetry
 	// +kubebuilder:default=OpenTelemetry
 	Type MetricSinkType `json:"type"`
 	// OpenTelemetry defines the configuration for OpenTelemetry sink.
 	// It's required if the sink type is OpenTelemetry.
-	OpenTelemetry *OpenTelemetrySink `json:"openTelemetry,omitempty"`
+	OpenTelemetry *ProxyOpenTelemetrySink `json:"openTelemetry,omitempty"`
 }
 
-// Match defines the stats match configuration.
-type Match struct {
-	// MatcherType defines the stats matcher type
-	//
-	// +kubebuilder:validation:Enum=RegularExpression;Prefix;Suffix
-	Type  MatcherType `json:"type"`
-	Value string      `json:"value"`
-}
-
-type MatcherType string
-
-const (
-	Prefix            MatcherType = "Prefix"
-	RegularExpression MatcherType = "RegularExpression"
-	Suffix            MatcherType = "Suffix"
-)
-
-type OpenTelemetrySink struct {
+type ProxyOpenTelemetrySink struct {
 	// Host define the service hostname.
 	Host string `json:"host"`
 	// Port defines the port the service is exposed on.
@@ -71,7 +54,24 @@ type OpenTelemetrySink struct {
 	// TODO: add support for customizing OpenTelemetry sink in https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/stat_sinks/open_telemetry/v3/open_telemetry.proto#envoy-v3-api-msg-extensions-stat-sinks-open-telemetry-v3-sinkconfig
 }
 
-type PrometheusProvider struct {
+type ProxyPrometheusProvider struct {
 	// Disable the Prometheus endpoint.
 	Disable bool `json:"disable,omitempty"`
 }
+
+// Match defines the stats match configuration.
+type Match struct {
+	// MatcherType defines the stats matcher type
+	//
+	// +kubebuilder:validation:Enum=RegularExpression;Prefix;Suffix
+	Type  MatcherType `json:"type"`
+	Value string      `json:"value"`
+}
+
+type MatcherType string
+
+const (
+	Prefix            MatcherType = "Prefix"
+	RegularExpression MatcherType = "RegularExpression"
+	Suffix            MatcherType = "Suffix"
+)

api/v1alpha1/validation/envoygateway_validate.go

@@ -82,6 +82,16 @@ func ValidateEnvoyGateway(eg *v1alpha1.EnvoyGateway) error {
 				return fmt.Errorf("unsupported extension server TLS certificateRef %v", certificateRefKind)
 			}
 		}
+	case eg.Telemetry != nil:
+		if eg.Telemetry.Metrics != nil {
+			for _, sink := range eg.Telemetry.Metrics.Sinks {
+				if sink.Type == v1alpha1.MetricSinkTypeOpenTelemetry {
+					if sink.OpenTelemetry == nil {
+						return fmt.Errorf("OpenTelemetry is required when sink Type is OpenTelemetry")
+					}
+				}
+			}
+		}
 	}
 	return nil
 }

api/v1alpha1/validation/envoygateway_validate_test.go

@@ -413,6 +413,47 @@ func TestValidateEnvoyGateway(t *testing.T) {
 				},
 			},
 			expect: false,
+		}, {
+			name: "valid gateway metrics sink",
+			eg: &v1alpha1.EnvoyGateway{
+				EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{
+					Gateway:  v1alpha1.DefaultGateway(),
+					Provider: v1alpha1.DefaultEnvoyGatewayProvider(),
+					Telemetry: &v1alpha1.EnvoyGatewayTelemetry{
+						Metrics: &v1alpha1.EnvoyGatewayMetrics{
+							Sinks: []v1alpha1.EnvoyGatewayMetricSink{
+								{
+									Type: v1alpha1.MetricSinkTypeOpenTelemetry,
+									OpenTelemetry: &v1alpha1.EnvoyGatewayOpenTelemetrySink{
+										Host:     "x.x.x.x",
+										Port:     4317,
+										Protocol: "grpc",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			expect: true,
+		}, {
+			name: "invalid gateway metrics sink",
+			eg: &v1alpha1.EnvoyGateway{
+				EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{
+					Gateway:  v1alpha1.DefaultGateway(),
+					Provider: v1alpha1.DefaultEnvoyGatewayProvider(),
+					Telemetry: &v1alpha1.EnvoyGatewayTelemetry{
+						Metrics: &v1alpha1.EnvoyGatewayMetrics{
+							Sinks: []v1alpha1.EnvoyGatewayMetricSink{
+								{
+									Type: v1alpha1.MetricSinkTypeOpenTelemetry,
+								},
+							},
+						},
+					},
+				},
+			},
+			expect: false,
 		},
 	}
 

api/v1alpha1/validation/envoyproxy_validate.go

@@ -191,6 +191,17 @@ func validateProxyTelemetry(spec *egv1a1.EnvoyProxySpec) []error {
 		}
 	}
 
+	if spec != nil && spec.Telemetry != nil && spec.Telemetry.Metrics != nil {
+		for _, sink := range spec.Telemetry.Metrics.Sinks {
+			if sink.Type == egv1a1.MetricSinkTypeOpenTelemetry {
+				if sink.OpenTelemetry == nil {
+					err := fmt.Errorf("opentelemetry is required if the sink type is OpenTelemetry")
+					errs = append(errs, err)
+				}
+			}
+		}
+	}
+
 	return errs
 }
 

api/v1alpha1/validation/envoyproxy_validate_test.go

@@ -404,6 +404,50 @@ func TestValidateEnvoyProxy(t *testing.T) {
 				},
 			},
 			expected: false,
+		}, {
+			name: "should invalid when metrics type is OpenTelemetry, but `OpenTelemetry` field being empty",
+			proxy: &egv1a1.EnvoyProxy{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test",
+					Name:      "test",
+				},
+				Spec: egv1a1.EnvoyProxySpec{
+					Telemetry: &egv1a1.ProxyTelemetry{
+						Metrics: &egv1a1.ProxyMetrics{
+							Sinks: []egv1a1.ProxyMetricSink{
+								{
+									Type: egv1a1.MetricSinkTypeOpenTelemetry,
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: false,
+		}, {
+			name: "should valid when metrics type is OpenTelemetry and `OpenTelemetry` field being not empty",
+			proxy: &egv1a1.EnvoyProxy{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test",
+					Name:      "test",
+				},
+				Spec: egv1a1.EnvoyProxySpec{
+					Telemetry: &egv1a1.ProxyTelemetry{
+						Metrics: &egv1a1.ProxyMetrics{
+							Sinks: []egv1a1.ProxyMetricSink{
+								{
+									Type: egv1a1.MetricSinkTypeOpenTelemetry,
+									OpenTelemetry: &egv1a1.ProxyOpenTelemetrySink{
+										Host: "0.0.0.0",
+										Port: 3217,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: true,
 		},
 	}