Skip to content

Commit

Permalink
design: control plane metrics monitoring
Browse files Browse the repository at this point in the history
Signed-off-by: bitliu <bitliu@tencent.com>
  • Loading branch information
Xunzhuo committed Oct 16, 2023
1 parent d4a4c42 commit 4f90b09
Show file tree
Hide file tree
Showing 48 changed files with 2,685 additions and 264 deletions.
87 changes: 82 additions & 5 deletions api/v1alpha1/envoygateway_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
package v1alpha1

import (
"fmt"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

Expand All @@ -17,10 +19,12 @@ func DefaultEnvoyGateway() *EnvoyGateway {
APIVersion: GroupVersion.String(),
},
EnvoyGatewaySpec{
Gateway: DefaultGateway(),
Provider: DefaultEnvoyGatewayProvider(),
Logging: DefaultEnvoyGatewayLogging(),
Admin: DefaultEnvoyGatewayAdmin(),
Gateway: DefaultGateway(),
Provider: DefaultEnvoyGatewayProvider(),
Logging: DefaultEnvoyGatewayLogging(),
Admin: DefaultEnvoyGatewayAdmin(),
Debug: DefaultEnvoyGatewayDebug(),
Telemetry: DefaultEnvoyGatewayTelemetry(),
},
}
}
Expand All @@ -45,6 +49,12 @@ func (e *EnvoyGateway) SetEnvoyGatewayDefaults() {
if e.Admin == nil {
e.Admin = DefaultEnvoyGatewayAdmin()
}
if e.Telemetry == nil {
e.Telemetry = DefaultEnvoyGatewayTelemetry()
}
if e.Debug == nil {
e.Debug = DefaultEnvoyGatewayDebug()
}
}

// GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
Expand All @@ -60,6 +70,22 @@ func (e *EnvoyGateway) GetEnvoyGatewayAdmin() *EnvoyGatewayAdmin {
return e.Admin
}

// GetEnvoyGatewayDebug returns the EnvoyGatewayDebug of EnvoyGateway or a default EnvoyGatewayDebug if unspecified.
func (e *EnvoyGateway) GetEnvoyGatewayDebug() *EnvoyGatewayDebug {
if e.Debug != nil {
return e.Debug
}
e.Debug = DefaultEnvoyGatewayDebug()

return e.Debug
}

// GetEnvoyGatewayAdminAddress returns the EnvoyGateway Admin Address.
func (e *EnvoyGateway) GetEnvoyGatewayAdminAddress() string {
address := e.GetEnvoyGatewayAdmin().Address
return fmt.Sprintf("%s:%d", address.Host, address.Port)
}

// DefaultGateway returns a new Gateway with default configuration parameters.
func DefaultGateway() *Gateway {
return &Gateway{
Expand All @@ -76,6 +102,51 @@ func DefaultEnvoyGatewayLogging() *EnvoyGatewayLogging {
}
}

// GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
func (e *EnvoyGateway) GetEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
if e.Telemetry != nil {
if e.Telemetry.Metrics.Prometheus == nil {
e.Telemetry.Metrics.Prometheus = DefaultEnvoyGatewayPrometheus()
}
if e.Telemetry.Metrics == nil {
e.Telemetry.Metrics = DefaultEnvoyGatewayMetrics()
}
return e.Telemetry
}
e.Telemetry = DefaultEnvoyGatewayTelemetry()

return e.Telemetry
}

func (e *EnvoyGateway) IfEnablePrometheus() bool {
return e.GetEnvoyGatewayTelemetry().Metrics.Prometheus.Enable
}

// DefaultEnvoyGatewayTelemetry returns a new EnvoyGatewayTelemetry with default configuration parameters.
func DefaultEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
return &EnvoyGatewayTelemetry{
Metrics: DefaultEnvoyGatewayMetrics(),
}
}

// DefaultEnvoyGatewayMetrics returns a new EnvoyGatewayMetrics with default configuration parameters.
func DefaultEnvoyGatewayMetrics() *EnvoyGatewayMetrics {
return &EnvoyGatewayMetrics{
// Enable prometheus pull by default.
Prometheus: &EnvoyGatewayPrometheusProvider{
Enable: true,
},
}
}

// DefaultEnvoyGatewayPrometheus returns a new EnvoyGatewayMetrics with default configuration parameters.
func DefaultEnvoyGatewayPrometheus() *EnvoyGatewayPrometheusProvider {
return &EnvoyGatewayPrometheusProvider{
// Enable prometheus pull by default.
Enable: true,
}
}

// DefaultEnvoyGatewayProvider returns a new EnvoyGatewayProvider with default configuration parameters.
func DefaultEnvoyGatewayProvider() *EnvoyGatewayProvider {
return &EnvoyGatewayProvider{
Expand Down Expand Up @@ -103,11 +174,17 @@ func DefaultEnvoyGatewayKubeProvider() *EnvoyGatewayKubernetesProvider {
// DefaultEnvoyGatewayAdmin returns a new EnvoyGatewayAdmin with default configuration parameters.
func DefaultEnvoyGatewayAdmin() *EnvoyGatewayAdmin {
return &EnvoyGatewayAdmin{
Debug: false,
Address: DefaultEnvoyGatewayAdminAddress(),
}
}

// DefaultEnvoyGatewayDebug returns a new EnvoyGatewayDebug with default configuration parameters.
func DefaultEnvoyGatewayDebug() *EnvoyGatewayDebug {
return &EnvoyGatewayDebug{
DumpConfig: false,
}
}

// DefaultEnvoyGatewayAdminAddress returns a new EnvoyGatewayAdminAddress with default configuration parameters.
func DefaultEnvoyGatewayAdminAddress() *EnvoyGatewayAdminAddress {
return &EnvoyGatewayAdminAddress{
Expand Down
43 changes: 43 additions & 0 deletions api/v1alpha1/envoygateway_metric_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Copyright Envoy Gateway Authors
// SPDX-License-Identifier: Apache-2.0
// The full text of the Apache license is available in the LICENSE file at
// the root of the repo.

package v1alpha1

// EnvoyGatewayMetrics defines control plane push/pull metrics configurations.
type EnvoyGatewayMetrics struct {
// Sinks defines the metric sinks where metrics are sent to.
Sinks []EnvoyGatewayMetricSink `json:"sinks,omitempty"`
// Prometheus defines the configuration for prometheus endpoint.
Prometheus *EnvoyGatewayPrometheusProvider `json:"prometheus,omitempty"`
}

// EnvoyGatewayMetricSink defines control plane
// metric sinks where metrics are sent to.
type EnvoyGatewayMetricSink struct {
// Type defines the metric sink type.
// EG control plane currently supports OpenTelemetry.
// +kubebuilder:validation:Enum=OpenTelemetry
// +kubebuilder:default=OpenTelemetry
Type MetricSinkType `json:"type"`
// Host define the sink service hostname.
Host string `json:"host"`
// Protocol define the sink service protocol.
Protocol string `json:"protocol"`
// Port defines the port the sink service is exposed on.
//
// +optional
// +kubebuilder:validation:Minimum=0
// +kubebuilder:default=4317
Port int32 `json:"port,omitempty"`
}

// EnvoyGatewayPrometheusProvider will expose prometheus endpoint in pull mode.
type EnvoyGatewayPrometheusProvider struct {
// Enable defines if enables the prometheus metrics in pull mode. Default is true.
//
// +optional
// +kubebuilder:default=true
Enable bool `json:"enable,omitempty"`
}
48 changes: 46 additions & 2 deletions api/v1alpha1/envoygateway_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,25 @@ type EnvoyGatewaySpec struct {
// +optional
// +kubebuilder:default={default: info}
Logging *EnvoyGatewayLogging `json:"logging,omitempty"`

// Telemetry defines telemetry related configurations for envoy gateway.
//
// +optional
Telemetry *EnvoyGatewayTelemetry `json:"telemetry,omitempty"`

// Admin defines the desired admin related abilities.
// If unspecified, the Admin is used with default configuration
// parameters.
//
// +optional
Admin *EnvoyGatewayAdmin `json:"admin,omitempty"`

// Debug defines the desired debug related abilities.
// If unspecified, the debug will not be running, including pprof, dump config etc.
//
// +optional
Debug *EnvoyGatewayDebug `json:"debug,omitempty"`

// RateLimit defines the configuration associated with the Rate Limit service
// deployed by Envoy Gateway required to implement the Global Rate limiting
// functionality. The specific rate limit service used here is the reference
Expand All @@ -79,6 +91,13 @@ type EnvoyGatewaySpec struct {
ExtensionAPIs *ExtensionAPISettings `json:"extensionApis,omitempty"`
}

// EnvoyGatewayTelemetry defines telemetry configurations for envoy gateway control plane.
// Control plane will focus on metrics observability telemetry and tracing telemetry later.
type EnvoyGatewayTelemetry struct {
// Metrics defines metrics configuration for envoy gateway.
Metrics *EnvoyGatewayMetrics `json:"metrics,omitempty"`
}

// EnvoyGatewayLogging defines logging for Envoy Gateway.
type EnvoyGatewayLogging struct {
// Level is the logging level. If unspecified, defaults to "info".
Expand Down Expand Up @@ -424,11 +443,21 @@ type EnvoyGatewayAdmin struct {
//
// +optional
Address *EnvoyGatewayAdminAddress `json:"address,omitempty"`
}

// EnvoyGatewayDebug defines the Envoy Gateway Debug configuration.
type EnvoyGatewayDebug struct {

// DumpConfig defines if dump the Envoy Gateway config in logs.
//
// +optional
DumpConfig bool `json:"dumpConfig,omitempty"`

// Debug defines if enable the /debug endpoint of Envoy Gateway.
// Address defines the address of Envoy Gateway Debug Server.
// Pprof will use the debug address, if you set it to non-nil.
//
// +optional
Debug bool `json:"debug,omitempty"`
Address *EnvoyGatewayDebugAddress `json:"address,omitempty"`
}

// EnvoyGatewayAdminAddress defines the Envoy Gateway Admin Address configuration.
Expand All @@ -446,6 +475,21 @@ type EnvoyGatewayAdminAddress struct {
Host string `json:"host,omitempty"`
}

// EnvoyGatewayDebugAddress defines the Envoy Gateway Debug Address configuration.
type EnvoyGatewayDebugAddress struct {
// Port defines the port the debug server is exposed on.
//
// +optional
// +kubebuilder:validation:Minimum=0
// +kubebuilder:default=19010
Port int `json:"port,omitempty"`
// Host defines the debug server hostname.
//
// +optional
// +kubebuilder:default="127.0.0.1"
Host string `json:"host,omitempty"`
}

func init() {
SchemeBuilder.Register(&EnvoyGateway{})
}
26 changes: 13 additions & 13 deletions api/v1alpha1/envoyproxy_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,44 +129,44 @@ type ProxyLogging struct {
// and the log level is the value. If unspecified, defaults to "default: warn".
//
// +kubebuilder:default={default: warn}
Level map[LogComponent]LogLevel `json:"level,omitempty"`
Level map[ProxyLogComponent]LogLevel `json:"level,omitempty"`
}

// LogComponent defines a component that supports a configured logging level.
// ProxyLogComponent defines a component that supports a configured logging level.
// +kubebuilder:validation:Enum=system;upstream;http;connection;admin;client;filter;main;router;runtime
type LogComponent string
type ProxyLogComponent string

const (
// LogComponentDefault defines the default logging component.
// See more details: https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-l
LogComponentDefault LogComponent = "default"
LogComponentDefault ProxyLogComponent = "default"

// LogComponentUpstream defines the "upstream" logging component.
LogComponentUpstream LogComponent = "upstream"
LogComponentUpstream ProxyLogComponent = "upstream"

// LogComponentHTTP defines the "http" logging component.
LogComponentHTTP LogComponent = "http"
LogComponentHTTP ProxyLogComponent = "http"

// LogComponentConnection defines the "connection" logging component.
LogComponentConnection LogComponent = "connection"
LogComponentConnection ProxyLogComponent = "connection"

// LogComponentAdmin defines the "admin" logging component.
LogComponentAdmin LogComponent = "admin"
LogComponentAdmin ProxyLogComponent = "admin"

// LogComponentClient defines the "client" logging component.
LogComponentClient LogComponent = "client"
LogComponentClient ProxyLogComponent = "client"

// LogComponentFilter defines the "filter" logging component.
LogComponentFilter LogComponent = "filter"
LogComponentFilter ProxyLogComponent = "filter"

// LogComponentMain defines the "main" logging component.
LogComponentMain LogComponent = "main"
LogComponentMain ProxyLogComponent = "main"

// LogComponentRouter defines the "router" logging component.
LogComponentRouter LogComponent = "router"
LogComponentRouter ProxyLogComponent = "router"

// LogComponentRuntime defines the "runtime" logging component.
LogComponentRuntime LogComponent = "runtime"
LogComponentRuntime ProxyLogComponent = "runtime"
)

// ProxyBootstrap defines Envoy Bootstrap configuration.
Expand Down
16 changes: 8 additions & 8 deletions api/v1alpha1/metric_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,15 @@

package v1alpha1

type MetricSinkType string

const (
MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
)

type ProxyMetrics struct {
// Prometheus defines the configuration for Admin endpoint `/stats/prometheus`.
Prometheus *PrometheusProvider `json:"prometheus,omitempty"`
Prometheus *ProxyPrometheusProvider `json:"prometheus,omitempty"`
// Sinks defines the metric sinks where metrics are sent to.
Sinks []MetricSink `json:"sinks,omitempty"`
// Matches defines configuration for selecting specific metrics instead of generating all metrics stats
Expand All @@ -23,12 +29,6 @@ type ProxyMetrics struct {
EnableVirtualHostStats bool `json:"enableVirtualHostStats,omitempty"`
}

type MetricSinkType string

const (
MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
)

type MetricSink struct {
// Type defines the metric sink type.
// EG currently only supports OpenTelemetry.
Expand Down Expand Up @@ -71,5 +71,5 @@ type OpenTelemetrySink struct {
// TODO: add support for customizing OpenTelemetry sink in https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/stat_sinks/open_telemetry/v3/open_telemetry.proto#envoy-v3-api-msg-extensions-stat-sinks-open-telemetry-v3-sinkconfig
}

type PrometheusProvider struct {
type ProxyPrometheusProvider struct {
}
Loading

0 comments on commit 4f90b09

Please sign in to comment.