Skip to content

Commit

Permalink
design/impl: control plane metrics monitoring
Browse files Browse the repository at this point in the history
Signed-off-by: bitliu <bitliu@tencent.com>
  • Loading branch information
Xunzhuo committed Oct 17, 2023
1 parent fa55804 commit aec4cb8
Show file tree
Hide file tree
Showing 49 changed files with 2,876 additions and 172 deletions.
76 changes: 72 additions & 4 deletions api/v1alpha1/envoygateway_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@ func DefaultEnvoyGateway() *EnvoyGateway {
APIVersion: GroupVersion.String(),
},
EnvoyGatewaySpec{
Gateway: DefaultGateway(),
Provider: DefaultEnvoyGatewayProvider(),
Logging: DefaultEnvoyGatewayLogging(),
Admin: DefaultEnvoyGatewayAdmin(),
Gateway: DefaultGateway(),
Provider: DefaultEnvoyGatewayProvider(),
Logging: DefaultEnvoyGatewayLogging(),
Admin: DefaultEnvoyGatewayAdmin(),
Telemetry: DefaultEnvoyGatewayTelemetry(),
},
}
}
Expand All @@ -47,6 +48,9 @@ func (e *EnvoyGateway) SetEnvoyGatewayDefaults() {
if e.Admin == nil {
e.Admin = DefaultEnvoyGatewayAdmin()
}
if e.Telemetry == nil {
e.Telemetry = DefaultEnvoyGatewayTelemetry()
}
}

// GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
Expand Down Expand Up @@ -88,6 +92,70 @@ func DefaultEnvoyGatewayLogging() *EnvoyGatewayLogging {
}
}

// GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
func (e *EnvoyGateway) GetEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
if e.Telemetry != nil {
if e.Telemetry.Metrics.Prometheus == nil {
e.Telemetry.Metrics.Prometheus = DefaultEnvoyGatewayPrometheus()
}
if e.Telemetry.Metrics.Address == nil {
e.Telemetry.Metrics.Address = DefaultEnvoyGatewayMetricsAddress()
}
if e.Telemetry.Metrics == nil {
e.Telemetry.Metrics = DefaultEnvoyGatewayMetrics()
}
return e.Telemetry
}
e.Telemetry = DefaultEnvoyGatewayTelemetry()

return e.Telemetry
}

// GetEnvoyGatewayMetricsAddress returns the EnvoyGateway Metrics Address.
func (e *EnvoyGateway) GetEnvoyGatewayMetricsAddress() string {
address := e.GetEnvoyGatewayTelemetry().Metrics.Address
if address != nil {
return fmt.Sprintf("%s:%d", address.Host, address.Port)
}

return ""
}

func (e *EnvoyGateway) IfEnablePrometheus() bool {
return e.GetEnvoyGatewayTelemetry().Metrics.Prometheus.Enable
}

// DefaultEnvoyGatewayTelemetry returns a new EnvoyGatewayTelemetry with default configuration parameters.
func DefaultEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
return &EnvoyGatewayTelemetry{
Metrics: DefaultEnvoyGatewayMetrics(),
}
}

// DefaultEnvoyGatewayMetrics returns a new EnvoyGatewayMetrics with default configuration parameters.
func DefaultEnvoyGatewayMetrics() *EnvoyGatewayMetrics {
return &EnvoyGatewayMetrics{
Address: DefaultEnvoyGatewayMetricsAddress(),
Prometheus: DefaultEnvoyGatewayPrometheus(),
}
}

// DefaultEnvoyGatewayMetricsAddress returns a new EnvoyGatewayMetrics with default configuration parameters.
func DefaultEnvoyGatewayMetricsAddress() *EnvoyGatewayMetricsAddress {
return &EnvoyGatewayMetricsAddress{
Host: GatewayMetricsHost,
Port: GatewayMetricsPort,
}
}

// DefaultEnvoyGatewayPrometheus returns a new EnvoyGatewayMetrics with default configuration parameters.
func DefaultEnvoyGatewayPrometheus() *EnvoyGatewayPrometheusProvider {
return &EnvoyGatewayPrometheusProvider{
// Enable prometheus pull by default.
Enable: true,
}
}

// DefaultEnvoyGatewayProvider returns a new EnvoyGatewayProvider with default configuration parameters.
func DefaultEnvoyGatewayProvider() *EnvoyGatewayProvider {
return &EnvoyGatewayProvider{
Expand Down
61 changes: 61 additions & 0 deletions api/v1alpha1/envoygateway_metric_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// Copyright Envoy Gateway Authors
// SPDX-License-Identifier: Apache-2.0
// The full text of the Apache license is available in the LICENSE file at
// the root of the repo.

package v1alpha1

// EnvoyGatewayMetrics defines control plane push/pull metrics configurations.
type EnvoyGatewayMetrics struct {
// Address defines the address of Envoy Gateway Metrics Server.
Address *EnvoyGatewayMetricsAddress
// Sinks defines the metric sinks where metrics are sent to.
Sinks []EnvoyGatewayMetricSink `json:"sinks,omitempty"`
// Prometheus defines the configuration for prometheus endpoint.
Prometheus *EnvoyGatewayPrometheusProvider `json:"prometheus,omitempty"`
}

// EnvoyGatewayMetricSink defines control plane
// metric sinks where metrics are sent to.
type EnvoyGatewayMetricSink struct {
// Type defines the metric sink type.
// EG control plane currently supports OpenTelemetry.
// +kubebuilder:validation:Enum=OpenTelemetry
// +kubebuilder:default=OpenTelemetry
Type MetricSinkType `json:"type"`
// Host define the sink service hostname.
Host string `json:"host"`
// Protocol define the sink service protocol.
// +kubebuilder:validation:Enum=grpc;http
Protocol string `json:"protocol"`
// Port defines the port the sink service is exposed on.
//
// +optional
// +kubebuilder:validation:Minimum=0
// +kubebuilder:default=4317
Port int32 `json:"port,omitempty"`
}

// EnvoyGatewayPrometheusProvider will expose prometheus endpoint in pull mode.
type EnvoyGatewayPrometheusProvider struct {
// Enable defines if enables the prometheus metrics in pull mode. Default is true.
//
// +optional
// +kubebuilder:default=true
Enable bool `json:"enable,omitempty"`
}

// EnvoyGatewayMetricsAddress defines the Envoy Gateway Metrics Address configuration.
type EnvoyGatewayMetricsAddress struct {
// Port defines the port the metrics server is exposed on.
//
// +optional
// +kubebuilder:validation:Minimum=0
// +kubebuilder:default=19010
Port int `json:"port,omitempty"`
// Host defines the metrics server hostname.
//
// +optional
// +kubebuilder:default="0.0.0.0"
Host string `json:"host,omitempty"`
}
17 changes: 17 additions & 0 deletions api/v1alpha1/envoygateway_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ const (
GatewayAdminPort = 19000
// GatewayAdminHost is the host of envoy gateway admin server.
GatewayAdminHost = "127.0.0.1"
// GatewayMetricsPort is the port which envoy gateway metrics server is listening on.
GatewayMetricsPort = 19010
// GatewayMetricsHost is the host of envoy gateway metrics server.
GatewayMetricsHost = "0.0.0.0"
)

// +kubebuilder:object:root=true
Expand Down Expand Up @@ -59,6 +63,12 @@ type EnvoyGatewaySpec struct {
// +optional
Admin *EnvoyGatewayAdmin `json:"admin,omitempty"`

// Telemetry defines the desired control plane telemetry related abilities.
// If unspecified, the telemetry is used with default configuration.
//
// +optional
Telemetry *EnvoyGatewayTelemetry `json:"telemetry,omitempty"`

// RateLimit defines the configuration associated with the Rate Limit service
// deployed by Envoy Gateway required to implement the Global Rate limiting
// functionality. The specific rate limit service used here is the reference
Expand All @@ -80,6 +90,13 @@ type EnvoyGatewaySpec struct {
ExtensionAPIs *ExtensionAPISettings `json:"extensionApis,omitempty"`
}

// EnvoyGatewayTelemetry defines telemetry configurations for envoy gateway control plane.
// Control plane will focus on metrics observability telemetry and tracing telemetry later.
type EnvoyGatewayTelemetry struct {
// Metrics defines metrics configuration for envoy gateway.
Metrics *EnvoyGatewayMetrics `json:"metrics,omitempty"`
}

// EnvoyGatewayLogging defines logging for Envoy Gateway.
type EnvoyGatewayLogging struct {
// Level is the logging level. If unspecified, defaults to "info".
Expand Down
16 changes: 8 additions & 8 deletions api/v1alpha1/metric_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,15 @@

package v1alpha1

type MetricSinkType string

const (
MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
)

type ProxyMetrics struct {
// Prometheus defines the configuration for Admin endpoint `/stats/prometheus`.
Prometheus *PrometheusProvider `json:"prometheus,omitempty"`
Prometheus *ProxyPrometheusProvider `json:"prometheus,omitempty"`
// Sinks defines the metric sinks where metrics are sent to.
Sinks []MetricSink `json:"sinks,omitempty"`
// Matches defines configuration for selecting specific metrics instead of generating all metrics stats
Expand All @@ -23,12 +29,6 @@ type ProxyMetrics struct {
EnableVirtualHostStats bool `json:"enableVirtualHostStats,omitempty"`
}

type MetricSinkType string

const (
MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
)

type MetricSink struct {
// Type defines the metric sink type.
// EG currently only supports OpenTelemetry.
Expand Down Expand Up @@ -71,5 +71,5 @@ type OpenTelemetrySink struct {
// TODO: add support for customizing OpenTelemetry sink in https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/stat_sinks/open_telemetry/v3/open_telemetry.proto#envoy-v3-api-msg-extensions-stat-sinks-open-telemetry-v3-sinkconfig
}

type PrometheusProvider struct {
type ProxyPrometheusProvider struct {
}
3 changes: 1 addition & 2 deletions api/v1alpha1/validation/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,7 @@ func validateBootstrap(boostrapConfig *egv1a1.ProxyBootstrap) error {
}
}

// nolint // Circumvents this error "Error: copylocks: call of reflect.DeepEqual copies lock value:"
if userXdsCluster == nil || !reflect.DeepEqual(*userXdsCluster.LoadAssignment, *defaultXdsCluster.LoadAssignment) {
if userXdsCluster == nil || !reflect.DeepEqual(userXdsCluster.LoadAssignment, defaultXdsCluster.LoadAssignment) {
return fmt.Errorf("xds_cluster's loadAssigntment cannot be modified")
}

Expand Down
58 changes: 58 additions & 0 deletions api/v1alpha1/validation/validate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,64 @@ func TestEnvoyGatewayAdmin(t *testing.T) {
assert.True(t, eg.Admin.EnablePprof == false)
}

func TestEnvoyGatewayTelemetry(t *testing.T) {
// default envoygateway config telemetry should not be nil
eg := egv1a1.DefaultEnvoyGateway()
assert.True(t, eg.Telemetry != nil)

// get default telemetry config from envoygateway
// values should be set in default
egTelemetry := eg.GetEnvoyGatewayTelemetry()
assert.True(t, egTelemetry != nil)
assert.True(t, egTelemetry.Metrics != nil)
assert.True(t, egTelemetry.Metrics.Address != nil)
assert.True(t, egTelemetry.Metrics.Address.Host == egv1a1.GatewayMetricsHost)
assert.True(t, egTelemetry.Metrics.Address.Port == egv1a1.GatewayMetricsPort)
assert.True(t, egTelemetry.Metrics.Prometheus.Enable == true)
assert.True(t, egTelemetry.Metrics.Sinks == nil)

// override the telemetry config
// values should be updated
eg.Telemetry.Metrics = &egv1a1.EnvoyGatewayMetrics{
Address: &egv1a1.EnvoyGatewayMetricsAddress{
Host: "127.0.0.1",
Port: 19011,
},
Prometheus: &egv1a1.EnvoyGatewayPrometheusProvider{
Enable: false,
},
Sinks: []egv1a1.EnvoyGatewayMetricSink{
{
Type: egv1a1.MetricSinkTypeOpenTelemetry,
Host: "otel-collector.monitoring.svc.cluster.local",
Protocol: "grpc",
Port: 4317,
}, {
Type: egv1a1.MetricSinkTypeOpenTelemetry,
Host: "otel-collector.monitoring.svc.cluster.local",
Protocol: "http",
Port: 4318,
},
},
}

assert.True(t, eg.GetEnvoyGatewayTelemetry().Metrics.Prometheus.Enable == false)
assert.True(t, eg.GetEnvoyGatewayTelemetry().Metrics.Address.Host == "127.0.0.1")
assert.True(t, eg.GetEnvoyGatewayTelemetry().Metrics.Address.Port == 19011)
assert.True(t, len(eg.GetEnvoyGatewayTelemetry().Metrics.Sinks) == 2)
assert.True(t, eg.GetEnvoyGatewayTelemetry().Metrics.Sinks[0].Type == egv1a1.MetricSinkTypeOpenTelemetry)

// set eg defaults when telemetry is nil
// the telemetry should not be nil
eg.Telemetry = nil
eg.SetEnvoyGatewayDefaults()
assert.True(t, eg.Telemetry != nil)
assert.True(t, eg.Telemetry.Metrics != nil)
assert.True(t, egTelemetry.Metrics.Address != nil)
assert.True(t, eg.Telemetry.Metrics.Prometheus.Enable == true)
assert.True(t, eg.Telemetry.Metrics.Sinks == nil)
}

func TestGetEnvoyProxyDefaultComponentLevel(t *testing.T) {
cases := []struct {
logging egv1a1.ProxyLogging
Expand Down
Loading

0 comments on commit aec4cb8

Please sign in to comment.