design/impl: control plane metrics monitoring

Signed-off-by: bitliu <bitliu@tencent.com>

api/v1alpha1/envoygateway_helpers.go

@@ -19,10 +19,11 @@ func DefaultEnvoyGateway() *EnvoyGateway {
 			APIVersion: GroupVersion.String(),
 		},
 		EnvoyGatewaySpec{
-			Gateway:  DefaultGateway(),
-			Provider: DefaultEnvoyGatewayProvider(),
-			Logging:  DefaultEnvoyGatewayLogging(),
-			Admin:    DefaultEnvoyGatewayAdmin(),
+			Gateway:   DefaultGateway(),
+			Provider:  DefaultEnvoyGatewayProvider(),
+			Logging:   DefaultEnvoyGatewayLogging(),
+			Admin:     DefaultEnvoyGatewayAdmin(),
+			Telemetry: DefaultEnvoyGatewayTelemetry(),
 		},
 	}
 }
@@ -47,6 +48,9 @@ func (e *EnvoyGateway) SetEnvoyGatewayDefaults() {
 	if e.Admin == nil {
 		e.Admin = DefaultEnvoyGatewayAdmin()
 	}
+	if e.Telemetry == nil {
+		e.Telemetry = DefaultEnvoyGatewayTelemetry()
+	}
 }
 
 // GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
@@ -88,6 +92,70 @@ func DefaultEnvoyGatewayLogging() *EnvoyGatewayLogging {
 	}
 }
 
+// GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
+func (e *EnvoyGateway) GetEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
+	if e.Telemetry != nil {
+		if e.Telemetry.Metrics.Prometheus == nil {
+			e.Telemetry.Metrics.Prometheus = DefaultEnvoyGatewayPrometheus()
+		}
+		if e.Telemetry.Metrics.Address == nil {
+			e.Telemetry.Metrics.Address = DefaultEnvoyGatewayMetricsAddress()
+		}
+		if e.Telemetry.Metrics == nil {
+			e.Telemetry.Metrics = DefaultEnvoyGatewayMetrics()
+		}
+		return e.Telemetry
+	}
+	e.Telemetry = DefaultEnvoyGatewayTelemetry()
+
+	return e.Telemetry
+}
+
+// GetEnvoyGatewayMetricsAddress returns the EnvoyGateway Metrics Address.
+func (e *EnvoyGateway) GetEnvoyGatewayMetricsAddress() string {
+	address := e.GetEnvoyGatewayTelemetry().Metrics.Address
+	if address != nil {
+		return fmt.Sprintf("%s:%d", address.Host, address.Port)
+	}
+
+	return ""
+}
+
+func (e *EnvoyGateway) IfEnablePrometheus() bool {
+	return e.GetEnvoyGatewayTelemetry().Metrics.Prometheus.Enable
+}
+
+// DefaultEnvoyGatewayTelemetry returns a new EnvoyGatewayTelemetry with default configuration parameters.
+func DefaultEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
+	return &EnvoyGatewayTelemetry{
+		Metrics: DefaultEnvoyGatewayMetrics(),
+	}
+}
+
+// DefaultEnvoyGatewayMetrics returns a new EnvoyGatewayMetrics with default configuration parameters.
+func DefaultEnvoyGatewayMetrics() *EnvoyGatewayMetrics {
+	return &EnvoyGatewayMetrics{
+		Address:    DefaultEnvoyGatewayMetricsAddress(),
+		Prometheus: DefaultEnvoyGatewayPrometheus(),
+	}
+}
+
+// DefaultEnvoyGatewayMetricsAddress returns a new EnvoyGatewayMetrics with default configuration parameters.
+func DefaultEnvoyGatewayMetricsAddress() *EnvoyGatewayMetricsAddress {
+	return &EnvoyGatewayMetricsAddress{
+		Host: GatewayMetricsHost,
+		Port: GatewayMetricsPort,
+	}
+}
+
+// DefaultEnvoyGatewayPrometheus returns a new EnvoyGatewayMetrics with default configuration parameters.
+func DefaultEnvoyGatewayPrometheus() *EnvoyGatewayPrometheusProvider {
+	return &EnvoyGatewayPrometheusProvider{
+		// Enable prometheus pull by default.
+		Enable: true,
+	}
+}
+
 // DefaultEnvoyGatewayProvider returns a new EnvoyGatewayProvider with default configuration parameters.
 func DefaultEnvoyGatewayProvider() *EnvoyGatewayProvider {
 	return &EnvoyGatewayProvider{

api/v1alpha1/envoygateway_metric_types.go

@@ -0,0 +1,61 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package v1alpha1
+
+// EnvoyGatewayMetrics defines control plane push/pull metrics configurations.
+type EnvoyGatewayMetrics struct {
+	// Address defines the address of Envoy Gateway Metrics Server.
+	Address *EnvoyGatewayMetricsAddress
+	// Sinks defines the metric sinks where metrics are sent to.
+	Sinks []EnvoyGatewayMetricSink `json:"sinks,omitempty"`
+	// Prometheus defines the configuration for prometheus endpoint.
+	Prometheus *EnvoyGatewayPrometheusProvider `json:"prometheus,omitempty"`
+}
+
+// EnvoyGatewayMetricSink defines control plane
+// metric sinks where metrics are sent to.
+type EnvoyGatewayMetricSink struct {
+	// Type defines the metric sink type.
+	// EG control plane currently supports OpenTelemetry.
+	// +kubebuilder:validation:Enum=OpenTelemetry
+	// +kubebuilder:default=OpenTelemetry
+	Type MetricSinkType `json:"type"`
+	// Host define the sink service hostname.
+	Host string `json:"host"`
+	// Protocol define the sink service protocol.
+	// +kubebuilder:validation:Enum=grpc;http
+	Protocol string `json:"protocol"`
+	// Port defines the port the sink service is exposed on.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:default=4317
+	Port int32 `json:"port,omitempty"`
+}
+
+// EnvoyGatewayPrometheusProvider will expose prometheus endpoint in pull mode.
+type EnvoyGatewayPrometheusProvider struct {
+	// Enable defines if enables the prometheus metrics in pull mode. Default is true.
+	//
+	// +optional
+	// +kubebuilder:default=true
+	Enable bool `json:"enable,omitempty"`
+}
+
+// EnvoyGatewayMetricsAddress defines the Envoy Gateway Metrics Address configuration.
+type EnvoyGatewayMetricsAddress struct {
+	// Port defines the port the metrics server is exposed on.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:default=19010
+	Port int `json:"port,omitempty"`
+	// Host defines the metrics server hostname.
+	//
+	// +optional
+	// +kubebuilder:default="0.0.0.0"
+	Host string `json:"host,omitempty"`
+}

api/v1alpha1/envoygateway_types.go

@@ -19,6 +19,10 @@ const (
 	GatewayAdminPort = 19000
 	// GatewayAdminHost is the host of envoy gateway admin server.
 	GatewayAdminHost = "127.0.0.1"
+	// GatewayMetricsPort is the port which envoy gateway metrics server is listening on.
+	GatewayMetricsPort = 19010
+	// GatewayMetricsHost is the host of envoy gateway metrics server.
+	GatewayMetricsHost = "0.0.0.0"
 )
 
 // +kubebuilder:object:root=true
@@ -59,6 +63,12 @@ type EnvoyGatewaySpec struct {
 	// +optional
 	Admin *EnvoyGatewayAdmin `json:"admin,omitempty"`
 
+	// Telemetry defines the desired control plane telemetry related abilities.
+	// If unspecified, the telemetry is used with default configuration.
+	//
+	// +optional
+	Telemetry *EnvoyGatewayTelemetry `json:"telemetry,omitempty"`
+
 	// RateLimit defines the configuration associated with the Rate Limit service
 	// deployed by Envoy Gateway required to implement the Global Rate limiting
 	// functionality. The specific rate limit service used here is the reference
@@ -80,6 +90,13 @@ type EnvoyGatewaySpec struct {
 	ExtensionAPIs *ExtensionAPISettings `json:"extensionApis,omitempty"`
 }
 
+// EnvoyGatewayTelemetry defines telemetry configurations for envoy gateway control plane.
+// Control plane will focus on metrics observability telemetry and tracing telemetry later.
+type EnvoyGatewayTelemetry struct {
+	// Metrics defines metrics configuration for envoy gateway.
+	Metrics *EnvoyGatewayMetrics `json:"metrics,omitempty"`
+}
+
 // EnvoyGatewayLogging defines logging for Envoy Gateway.
 type EnvoyGatewayLogging struct {
 	// Level is the logging level. If unspecified, defaults to "info".

api/v1alpha1/metric_types.go

@@ -5,9 +5,15 @@
 
 package v1alpha1
 
+type MetricSinkType string
+
+const (
+	MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
+)
+
 type ProxyMetrics struct {
 	// Prometheus defines the configuration for Admin endpoint `/stats/prometheus`.
-	Prometheus *PrometheusProvider `json:"prometheus,omitempty"`
+	Prometheus *ProxyPrometheusProvider `json:"prometheus,omitempty"`
 	// Sinks defines the metric sinks where metrics are sent to.
 	Sinks []MetricSink `json:"sinks,omitempty"`
 	// Matches defines configuration for selecting specific metrics instead of generating all metrics stats
@@ -23,12 +29,6 @@ type ProxyMetrics struct {
 	EnableVirtualHostStats bool `json:"enableVirtualHostStats,omitempty"`
 }
 
-type MetricSinkType string
-
-const (
-	MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
-)
-
 type MetricSink struct {
 	// Type defines the metric sink type.
 	// EG currently only supports OpenTelemetry.
@@ -71,5 +71,5 @@ type OpenTelemetrySink struct {
 	// TODO: add support for customizing OpenTelemetry sink in https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/stat_sinks/open_telemetry/v3/open_telemetry.proto#envoy-v3-api-msg-extensions-stat-sinks-open-telemetry-v3-sinkconfig
 }
 
-type PrometheusProvider struct {
+type ProxyPrometheusProvider struct {
 }

api/v1alpha1/validation/validate.go

@@ -161,8 +161,7 @@ func validateBootstrap(boostrapConfig *egv1a1.ProxyBootstrap) error {
 		}
 	}
 
-	// nolint // Circumvents this error "Error: copylocks: call of reflect.DeepEqual copies lock value:"
-	if userXdsCluster == nil || !reflect.DeepEqual(*userXdsCluster.LoadAssignment, *defaultXdsCluster.LoadAssignment) {
+	if userXdsCluster == nil || !reflect.DeepEqual(userXdsCluster.LoadAssignment, defaultXdsCluster.LoadAssignment) {
 		return fmt.Errorf("xds_cluster's loadAssigntment cannot be modified")
 	}
 

api/v1alpha1/validation/validate_test.go

@@ -567,6 +567,64 @@ func TestEnvoyGatewayAdmin(t *testing.T) {
 	assert.True(t, eg.Admin.EnablePprof == false)
 }
 
+func TestEnvoyGatewayTelemetry(t *testing.T) {
+	// default envoygateway config telemetry should not be nil
+	eg := egv1a1.DefaultEnvoyGateway()
+	assert.True(t, eg.Telemetry != nil)
+
+	// get default telemetry config from envoygateway
+	// values should be set in default
+	egTelemetry := eg.GetEnvoyGatewayTelemetry()
+	assert.True(t, egTelemetry != nil)
+	assert.True(t, egTelemetry.Metrics != nil)
+	assert.True(t, egTelemetry.Metrics.Address != nil)
+	assert.True(t, egTelemetry.Metrics.Address.Host == egv1a1.GatewayMetricsHost)
+	assert.True(t, egTelemetry.Metrics.Address.Port == egv1a1.GatewayMetricsPort)
+	assert.True(t, egTelemetry.Metrics.Prometheus.Enable == true)
+	assert.True(t, egTelemetry.Metrics.Sinks == nil)
+
+	// override the telemetry config
+	// values should be updated
+	eg.Telemetry.Metrics = &egv1a1.EnvoyGatewayMetrics{
+		Address: &egv1a1.EnvoyGatewayMetricsAddress{
+			Host: "127.0.0.1",
+			Port: 19011,
+		},
+		Prometheus: &egv1a1.EnvoyGatewayPrometheusProvider{
+			Enable: false,
+		},
+		Sinks: []egv1a1.EnvoyGatewayMetricSink{
+			{
+				Type:     egv1a1.MetricSinkTypeOpenTelemetry,
+				Host:     "otel-collector.monitoring.svc.cluster.local",
+				Protocol: "grpc",
+				Port:     4317,
+			}, {
+				Type:     egv1a1.MetricSinkTypeOpenTelemetry,
+				Host:     "otel-collector.monitoring.svc.cluster.local",
+				Protocol: "http",
+				Port:     4318,
+			},
+		},
+	}
+
+	assert.True(t, eg.GetEnvoyGatewayTelemetry().Metrics.Prometheus.Enable == false)
+	assert.True(t, eg.GetEnvoyGatewayTelemetry().Metrics.Address.Host == "127.0.0.1")
+	assert.True(t, eg.GetEnvoyGatewayTelemetry().Metrics.Address.Port == 19011)
+	assert.True(t, len(eg.GetEnvoyGatewayTelemetry().Metrics.Sinks) == 2)
+	assert.True(t, eg.GetEnvoyGatewayTelemetry().Metrics.Sinks[0].Type == egv1a1.MetricSinkTypeOpenTelemetry)
+
+	// set eg defaults when telemetry is nil
+	// the telemetry should not be nil
+	eg.Telemetry = nil
+	eg.SetEnvoyGatewayDefaults()
+	assert.True(t, eg.Telemetry != nil)
+	assert.True(t, eg.Telemetry.Metrics != nil)
+	assert.True(t, egTelemetry.Metrics.Address != nil)
+	assert.True(t, eg.Telemetry.Metrics.Prometheus.Enable == true)
+	assert.True(t, eg.Telemetry.Metrics.Sinks == nil)
+}
+
 func TestGetEnvoyProxyDefaultComponentLevel(t *testing.T) {
 	cases := []struct {
 		logging   egv1a1.ProxyLogging