diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 34c4eb7f3c85..48044af53242 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -355,39 +355,26 @@ type RateLimitMetricsPrometheusProvider struct { } type RateLimitTracing struct { - // TracingServiceName defines the service name appears in tracing span. - // The default value is "envoy-ratelimit" - TracingServiceName string `json:"tracingServiceName,omitempty"` - - // TracingServiceNamespace defines the service namespace appears in tracing span. - // The default value is namespace where the RateLimit resides - TracingServiceNamespace string `json:"tracingServiceNamespace,omitempty"` - - // TracingSampleRate defines the sampling rate, defaults to 1.0 which means always sample. - // Valid range: [0.0,1.0] For high volume services, adjusting the sampling rate is recommended. - TracingSampleRate *float64 `json:"tracingSampleRate,omitempty"` + // SamplingRate controls the rate at which traffic will be + // selected for tracing if no prior sampling decision has been made. + // Defaults to 100, valid values [0-100]. 100 indicates 100% sampling. + // +optional + SampleRate *uint32 `json:"tracingSampleRate"` - // Provider defines the tracing provider. - Provider *RateLimitTraceProvider `json:"provider,omitempty"` -} + // BackendRef defines the target trace collector endpoint configuration + BackendRef gwapiv1.BackendObjectReference `json:"backendRef"` -type RateLimitTraceProvider struct { // Protocol defines the protocol of provider in tracing feature. // Only "http"(default) and "grpc" are allowed in this field + // +optional Protocol string `json:"protocol,omitempty"` - // Endpoint defines target URL to which the provider is going to send traces. - // The endpoint must be a valid URL with scheme (http or https) and host, may contain a port, - // should contain a path and must not contain other parts (such as query string or fragment). - Endpoint string `json:"endpoint,omitempty"` - - // Insecure Whether to enable client transport security for the provider gRPC connection. - // Default is true - Insecure *bool `json:"insecure,omitempty"` - - // Timeout Maximum time the provider will wait for each batch export. - // The time format follows the Go time package, such as "300ms", "-1.5h" or "2h45m". - Timeout string `json:"timeout,omitempty"` + // ClusterDomain is an optional field that specifies the custom domain used for the Kubernetes cluster. + // This field is used when the cluster is configured with a custom DNS domain, + // different from the default "cluster.local". + // Envoy Gateway uses this custom domain to generate fully qualified domain names (FQDN) for trace gatherer services. + // +optional + ClusterDomain string `json:"clusterDomain,omitempty"` } // RateLimitDatabaseBackend defines the configuration associated with diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2c4908d0b4be..d2ee77bc2f09 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -3173,34 +3173,15 @@ func (in *RateLimitTelemetry) DeepCopy() *RateLimitTelemetry { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RateLimitTraceProvider) DeepCopyInto(out *RateLimitTraceProvider) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RateLimitTraceProvider. -func (in *RateLimitTraceProvider) DeepCopy() *RateLimitTraceProvider { - if in == nil { - return nil - } - out := new(RateLimitTraceProvider) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RateLimitTracing) DeepCopyInto(out *RateLimitTracing) { *out = *in - if in.TracingSampleRate != nil { - in, out := &in.TracingSampleRate, &out.TracingSampleRate - *out = new(float64) - **out = **in - } - if in.Provider != nil { - in, out := &in.Provider, &out.Provider - *out = new(RateLimitTraceProvider) + if in.SampleRate != nil { + in, out := &in.SampleRate, &out.SampleRate + *out = new(uint32) **out = **in } + in.BackendRef.DeepCopyInto(&out.BackendRef) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RateLimitTracing. diff --git a/internal/infrastructure/kubernetes/ratelimit/resource.go b/internal/infrastructure/kubernetes/ratelimit/resource.go index 62997e047d8c..f4daa159158c 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource" @@ -93,12 +94,10 @@ const ( TracingSamplingRateVar = "TRACING_SAMPLING_RATE" // OTELExporterOTLPEndpointVar is target url to which the exporter is going to send OTELExporterOTLPEndpointVar = "OTEL_EXPORTER_OTLP_ENDPOINT" - // OTELExporterOTLPInsecure is enable client transport security for the exporter's gRPC connection. - OTELExporterOTLPInsecure = "OTEL_EXPORTER_OTLP_INSECURE" - // OTELExporterOTLPTimeoutVar Maximum time the OTLP exporter will wait for each batch export. - OTELExporterOTLPTimeoutVar = "OTEL_EXPORTER_OTLP_TIMEOUT" // InfraName is the name for rate-limit resources. InfraName = "envoy-ratelimit" + // InfraNamespace is the namespace for rate-limit resources. + InfraNamespace = "envoy-gateway-system" // InfraGRPCPort is the grpc port that the rate limit service listens on. InfraGRPCPort = 8081 // XdsGrpcSotwConfigServerPort is the listening port of the ratelimit xDS config server. @@ -142,7 +141,7 @@ func rateLimitLabels() map[string]string { } // expectedRateLimitContainers returns expected rateLimit containers. -func expectedRateLimitContainers(rateLimit *egv1a1.RateLimit, rateLimitDeployment *egv1a1.KubernetesDeploymentSpec) []corev1.Container { +func expectedRateLimitContainers(rateLimit *egv1a1.RateLimit, rateLimitDeployment *egv1a1.KubernetesDeploymentSpec, namespace string) []corev1.Container { ports := []corev1.ContainerPort{ { Name: "grpc", @@ -159,7 +158,7 @@ func expectedRateLimitContainers(rateLimit *egv1a1.RateLimit, rateLimitDeploymen Command: []string{ "/bin/ratelimit", }, - Env: expectedRateLimitContainerEnv(rateLimit, rateLimitDeployment), + Env: expectedRateLimitContainerEnv(rateLimit, rateLimitDeployment, namespace), Ports: ports, Resources: *rateLimitDeployment.Container.Resources, SecurityContext: rateLimitDeployment.Container.SecurityContext, @@ -290,7 +289,7 @@ func expectedDeploymentVolumes(rateLimit *egv1a1.RateLimit, rateLimitDeployment } // expectedRateLimitContainerEnv returns expected rateLimit container envs. -func expectedRateLimitContainerEnv(rateLimit *egv1a1.RateLimit, rateLimitDeployment *egv1a1.KubernetesDeploymentSpec) []corev1.EnvVar { +func expectedRateLimitContainerEnv(rateLimit *egv1a1.RateLimit, rateLimitDeployment *egv1a1.KubernetesDeploymentSpec, namespace string) []corev1.EnvVar { env := []corev1.EnvVar{ { Name: RedisSocketTypeEnvVar, @@ -395,24 +394,24 @@ func expectedRateLimitContainerEnv(rateLimit *egv1a1.RateLimit, rateLimitDeploym } if enableTracing(rateLimit) { - - if len(rateLimit.Telemetry.Tracing.TracingServiceName) == 0 { - rateLimit.Telemetry.Tracing.TracingServiceName = InfraName + var protocol = "http" + if len(rateLimit.Telemetry.Tracing.Protocol) != 0 { + protocol = rateLimit.Telemetry.Tracing.Protocol } var sampleRate = 1.0 - if rateLimit.Telemetry.Tracing.TracingSampleRate != nil { - sampleRate = *rateLimit.Telemetry.Tracing.TracingSampleRate + if rateLimit.Telemetry.Tracing.SampleRate != nil { + sampleRate = float64(*rateLimit.Telemetry.Tracing.SampleRate) / 100.0 } - var insecure = true - if rateLimit.Telemetry.Tracing.Provider.Insecure != nil { - insecure = *rateLimit.Telemetry.Tracing.Provider.Insecure - } - if len(rateLimit.Telemetry.Tracing.Provider.Protocol) == 0 { - rateLimit.Telemetry.Tracing.Provider.Protocol = "http" + // If no namespace is provided, we assume that they are in the same namespace. + if rateLimit.Telemetry.Tracing.BackendRef.Namespace == nil { + ns := gwapiv1.Namespace(namespace) + rateLimit.Telemetry.Tracing.BackendRef.Namespace = &ns } + targetEndpoint := buildTraceEndpoint(rateLimit) + tracingEnvs := []corev1.EnvVar{ { Name: TracingEnabledVar, @@ -420,13 +419,15 @@ func expectedRateLimitContainerEnv(rateLimit *egv1a1.RateLimit, rateLimitDeploym }, { Name: TracingServiceNameVar, - Value: rateLimit.Telemetry.Tracing.TracingServiceName, + Value: InfraName, }, { Name: TracingServiceNamespaceVar, - Value: rateLimit.Telemetry.Tracing.TracingServiceNamespace, + Value: namespace, }, { + // By default, this is a random instanceID, + // we use the RateLimit pod name as the trace service instanceID. Name: TracingServiceInstanceIDVar, ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ @@ -436,32 +437,23 @@ func expectedRateLimitContainerEnv(rateLimit *egv1a1.RateLimit, rateLimitDeploym }, }, { - Name: TracingSamplingRateVar, + Name: TracingSamplingRateVar, + // The api is configured with [0,100], but sampling can only be [0,1]. + // doc: https://github.com/envoyproxy/ratelimit?tab=readme-ov-file#tracing + // You will lose precision during the conversion process, but don't worry, + // this follows the rounding rule and won't make the expected sampling rate too different + // from the actual sampling rate Value: strconv.FormatFloat(sampleRate, 'f', 1, 64), }, { Name: TracingExporterProtocolVar, - Value: rateLimit.Telemetry.Tracing.Provider.Protocol, + Value: protocol, }, { Name: OTELExporterOTLPEndpointVar, - Value: rateLimit.Telemetry.Tracing.Provider.Endpoint, - }, - { - Name: OTELExporterOTLPInsecure, - Value: strconv.FormatBool(insecure), + Value: targetEndpoint, }, } - - if len(rateLimit.Telemetry.Tracing.Provider.Timeout) != 0 { - tracingEnvs = append(tracingEnvs, []corev1.EnvVar{ - { - Name: OTELExporterOTLPTimeoutVar, - Value: rateLimit.Telemetry.Tracing.Provider.Timeout, - }, - }...) - } - env = append(env, tracingEnvs...) } @@ -480,14 +472,35 @@ func Validate(ctx context.Context, client client.Client, gateway *egv1a1.EnvoyGa } func enableTracing(rl *egv1a1.RateLimit) bool { - - // Other fields can use the default values, but we have to make sure the user has the endpoint configured + // Other fields can use the default values, + // but we have to make sure the user has the BackendRef.Name if rl != nil && rl.Telemetry != nil && rl.Telemetry.Tracing != nil && - rl.Telemetry.Tracing.Provider != nil && - len(rl.Telemetry.Tracing.Provider.Endpoint) != 0 { + len(rl.Telemetry.Tracing.BackendRef.Name) != 0 { return true } return false } + +// buildTraceEndpoint Build the endpoint for the target trace collector +func buildTraceEndpoint(rateLimit *egv1a1.RateLimit) string { + // By default, the cluster domain is "cluster.local", + // but there can be custom cluster domains that we need to deal with. + var clusterDomain = "cluster.local" + if len(rateLimit.Telemetry.Tracing.ClusterDomain) != 0 { + clusterDomain = rateLimit.Telemetry.Tracing.ClusterDomain + } + + defaultPort := 4318 + if rateLimit.Telemetry.Tracing.BackendRef.Port != nil { + defaultPort = int(*rateLimit.Telemetry.Tracing.BackendRef.Port) + } + + return fmt.Sprintf("%s.%s.svc.%s:%d", + rateLimit.Telemetry.Tracing.BackendRef.Name, + string(*rateLimit.Telemetry.Tracing.BackendRef.Namespace), + clusterDomain, + defaultPort, + ) +} diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index 89f3fc82725f..aea8dfb54072 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -185,11 +185,7 @@ func (r *ResourceRender) ServiceAccount() (*corev1.ServiceAccount, error) { // Deployment returns the expected rate limit Deployment based on the provided infra. func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { - if enableTracing(r.rateLimit) && len(r.rateLimit.Telemetry.Tracing.TracingServiceNamespace) == 0 { - r.rateLimit.Telemetry.Tracing.TracingServiceNamespace = r.Namespace - } - - containers := expectedRateLimitContainers(r.rateLimit, r.rateLimitDeployment) + containers := expectedRateLimitContainers(r.rateLimit, r.rateLimitDeployment, r.Namespace) labels := rateLimitLabels() selector := resource.GetSelector(labels) diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go index ae48860b1532..33a2956d150e 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go @@ -649,7 +649,7 @@ func TestDeployment(t *testing.T) { }, }, { - caseName: "enable-tracing-feature-with-default", + caseName: "enable-tracing-with-default", rateLimit: &egv1a1.RateLimit{ Backend: egv1a1.RateLimitDatabaseBackend{ Type: egv1a1.RedisBackendType, @@ -659,15 +659,15 @@ func TestDeployment(t *testing.T) { }, Telemetry: &egv1a1.RateLimitTelemetry{ Tracing: &egv1a1.RateLimitTracing{ - Provider: &egv1a1.RateLimitTraceProvider{ - Endpoint: "http://localhost:4318/v1/traces", + BackendRef: gwapiv1.BackendObjectReference{ + Name: "trace-collector", }, }, }, }, }, { - caseName: "enable-tracing-feature-with-custom", + caseName: "enable-tracing-with-custom", rateLimit: &egv1a1.RateLimit{ Backend: egv1a1.RateLimitDatabaseBackend{ Type: egv1a1.RedisBackendType, @@ -677,21 +677,49 @@ func TestDeployment(t *testing.T) { }, Telemetry: &egv1a1.RateLimitTelemetry{ Tracing: &egv1a1.RateLimitTracing{ - TracingServiceName: "prod-rate-limit", - TracingServiceNamespace: "prod", - TracingSampleRate: func() *float64 { - sampleRate := 0.9 - return &sampleRate + SampleRate: func() *uint32 { + var rate uint32 = 95 + return &rate }(), - Provider: &egv1a1.RateLimitTraceProvider{ - Endpoint: "http://localhost:4317", - Protocol: "grpc", - Insecure: func() *bool { - insecure := false - return &insecure + BackendRef: gwapiv1.BackendObjectReference{ + Name: "trace-collector", + Namespace: func() *gwapiv1.Namespace { + var ns gwapiv1.Namespace = "observability" + return &ns + }(), + Port: func() *gwapiv1.PortNumber { + var port gwapiv1.PortNumber = 4317 + return &port + }(), + }, + Protocol: "grpc", + }, + }, + }, + }, + { + caseName: "enable-tracing-with-custom-domain", + rateLimit: &egv1a1.RateLimit{ + Backend: egv1a1.RateLimitDatabaseBackend{ + Type: egv1a1.RedisBackendType, + Redis: &egv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + Telemetry: &egv1a1.RateLimitTelemetry{ + Tracing: &egv1a1.RateLimitTracing{ + SampleRate: func() *uint32 { + var rate uint32 = 55 + return &rate + }(), + BackendRef: gwapiv1.BackendObjectReference{ + Name: "trace-collector", + Namespace: func() *gwapiv1.Namespace { + var ns gwapiv1.Namespace = "observability" + return &ns }(), - Timeout: "10s", }, + ClusterDomain: "example.local", }, }, }, @@ -710,7 +738,7 @@ func TestDeployment(t *testing.T) { dp, err := r.Deployment() require.NoError(t, err) - if *overrideTestData { + if true { deploymentYAML, err := yaml.Marshal(dp) require.NoError(t, err) // nolint:gosec diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_test.go new file mode 100644 index 000000000000..f8eccdf64dd6 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/resource_test.go @@ -0,0 +1,84 @@ +package ratelimit + +import ( + "testing" + + "github.com/stretchr/testify/require" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func TestBuildTraceEndpoint(t *testing.T) { + + cases := []struct { + caseName string + rateLimit *egv1a1.RateLimit + expect string + }{ + { + caseName: "default-endpoint", + rateLimit: &egv1a1.RateLimit{ + Telemetry: &egv1a1.RateLimitTelemetry{ + Tracing: &egv1a1.RateLimitTracing{ + BackendRef: gwapiv1.BackendObjectReference{ + Name: "collector", + Namespace: func() *gwapiv1.Namespace { + var ns gwapiv1.Namespace = "observability" + return &ns + }(), + }, + }, + }, + }, + expect: "collector.observability.svc.cluster.local:4318", + }, + { + caseName: "endpoint-with-port", + rateLimit: &egv1a1.RateLimit{ + Telemetry: &egv1a1.RateLimitTelemetry{ + Tracing: &egv1a1.RateLimitTracing{ + BackendRef: gwapiv1.BackendObjectReference{ + Name: "collector", + Namespace: func() *gwapiv1.Namespace { + var ns gwapiv1.Namespace = "observability" + return &ns + }(), + Port: func() *gwapiv1.PortNumber { + var port gwapiv1.PortNumber = 4317 + return &port + }(), + }, + }, + }, + }, + expect: "collector.observability.svc.cluster.local:4317", + }, + { + caseName: "endpoint-with-domain", + rateLimit: &egv1a1.RateLimit{ + Telemetry: &egv1a1.RateLimitTelemetry{ + Tracing: &egv1a1.RateLimitTracing{ + BackendRef: gwapiv1.BackendObjectReference{ + Name: "collector", + Namespace: func() *gwapiv1.Namespace { + var ns gwapiv1.Namespace = "observability" + return &ns + }(), + }, + ClusterDomain: "example.com", + }, + }, + }, + expect: "collector.observability.svc.example.com:4318", + }, + } + + for _, tc := range cases { + t.Run(tc.caseName, func(t *testing.T) { + actual := buildTraceEndpoint(tc.rateLimit) + require.Equal(t, tc.expect, actual) + }) + } + +} diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-feature-with-custom.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-feature-with-custom.yaml deleted file mode 100644 index e5fe3bc3a0c7..000000000000 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-feature-with-custom.yaml +++ /dev/null @@ -1,166 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/component: ratelimit - app.kubernetes.io/managed-by: envoy-gateway - app.kubernetes.io/name: envoy-ratelimit - name: envoy-ratelimit - namespace: envoy-gateway-system - ownerReferences: - - apiVersion: apps/v1 - kind: Deployment - name: envoy-gateway - uid: test-owner-reference-uid-for-deployment -spec: - progressDeadlineSeconds: 600 - revisionHistoryLimit: 10 - selector: - matchLabels: - app.kubernetes.io/component: ratelimit - app.kubernetes.io/managed-by: envoy-gateway - app.kubernetes.io/name: envoy-ratelimit - strategy: - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "19001" - prometheus.io/scrape: "true" - creationTimestamp: null - labels: - app.kubernetes.io/component: ratelimit - app.kubernetes.io/managed-by: envoy-gateway - app.kubernetes.io/name: envoy-ratelimit - spec: - automountServiceAccountToken: false - containers: - - command: - - /bin/ratelimit - env: - - name: REDIS_SOCKET_TYPE - value: tcp - - name: REDIS_URL - value: redis.redis.svc:6379 - - name: RUNTIME_ROOT - value: /data - - name: RUNTIME_SUBDIRECTORY - value: ratelimit - - name: RUNTIME_IGNOREDOTFILES - value: "true" - - name: RUNTIME_WATCH_ROOT - value: "false" - - name: LOG_LEVEL - value: info - - name: USE_STATSD - value: "false" - - name: CONFIG_TYPE - value: GRPC_XDS_SOTW - - name: CONFIG_GRPC_XDS_SERVER_URL - value: envoy-gateway:18001 - - name: CONFIG_GRPC_XDS_NODE_ID - value: envoy-ratelimit - - name: GRPC_SERVER_USE_TLS - value: "true" - - name: GRPC_SERVER_TLS_CERT - value: /certs/tls.crt - - name: GRPC_SERVER_TLS_KEY - value: /certs/tls.key - - name: GRPC_SERVER_TLS_CA_CERT - value: /certs/ca.crt - - name: CONFIG_GRPC_XDS_SERVER_USE_TLS - value: "true" - - name: CONFIG_GRPC_XDS_CLIENT_TLS_CERT - value: /certs/tls.crt - - name: CONFIG_GRPC_XDS_CLIENT_TLS_KEY - value: /certs/tls.key - - name: CONFIG_GRPC_XDS_SERVER_TLS_CACERT - value: /certs/ca.crt - - name: FORCE_START_WITHOUT_INITIAL_CONFIG - value: "true" - - name: TRACING_ENABLED - value: "true" - - name: TRACING_SERVICE_NAME - value: prod-rate-limit - - name: TRACING_SERVICE_NAMESPACE - value: prod - - name: TRACING_SERVICE_INSTANCE_ID - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: TRACING_SAMPLING_RATE - value: "0.9" - - name: TRACING_EXPORTER_PROTOCOL - value: grpc - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://localhost:4317 - - name: OTEL_EXPORTER_OTLP_INSECURE - value: "false" - - name: OTEL_EXPORTER_OTLP_TIMEOUT - value: 10s - image: envoyproxy/ratelimit:master - imagePullPolicy: IfNotPresent - name: envoy-ratelimit - ports: - - containerPort: 8081 - name: grpc - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthcheck - port: 8080 - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 512Mi - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /certs - name: certs - readOnly: true - - command: - - /bin/statsd_exporter - - --web.listen-address=:19001 - - --statsd.mapping-config=/etc/statsd-exporter/conf.yaml - image: prom/statsd-exporter:v0.18.0 - imagePullPolicy: IfNotPresent - name: prom-statsd-exporter - ports: - - containerPort: 9125 - name: statsd - protocol: TCP - - containerPort: 19001 - name: metrics - protocol: TCP - resources: { } - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /etc/statsd-exporter - name: statsd-exporter-config - readOnly: true - dnsPolicy: ClusterFirst - restartPolicy: Always - schedulerName: default-scheduler - serviceAccountName: envoy-ratelimit - terminationGracePeriodSeconds: 300 - volumes: - - name: certs - secret: - defaultMode: 420 - secretName: envoy-rate-limit - - configMap: - defaultMode: 420 - name: statsd-exporter-config - optional: true - name: statsd-exporter-config -status: {} diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-feature-with-default.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-feature-with-default.yaml deleted file mode 100644 index 27f29f9e6696..000000000000 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-feature-with-default.yaml +++ /dev/null @@ -1,164 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/component: ratelimit - app.kubernetes.io/managed-by: envoy-gateway - app.kubernetes.io/name: envoy-ratelimit - name: envoy-ratelimit - namespace: envoy-gateway-system - ownerReferences: - - apiVersion: apps/v1 - kind: Deployment - name: envoy-gateway - uid: test-owner-reference-uid-for-deployment -spec: - progressDeadlineSeconds: 600 - revisionHistoryLimit: 10 - selector: - matchLabels: - app.kubernetes.io/component: ratelimit - app.kubernetes.io/managed-by: envoy-gateway - app.kubernetes.io/name: envoy-ratelimit - strategy: - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "19001" - prometheus.io/scrape: "true" - creationTimestamp: null - labels: - app.kubernetes.io/component: ratelimit - app.kubernetes.io/managed-by: envoy-gateway - app.kubernetes.io/name: envoy-ratelimit - spec: - automountServiceAccountToken: false - containers: - - command: - - /bin/ratelimit - env: - - name: REDIS_SOCKET_TYPE - value: tcp - - name: REDIS_URL - value: redis.redis.svc:6379 - - name: RUNTIME_ROOT - value: /data - - name: RUNTIME_SUBDIRECTORY - value: ratelimit - - name: RUNTIME_IGNOREDOTFILES - value: "true" - - name: RUNTIME_WATCH_ROOT - value: "false" - - name: LOG_LEVEL - value: info - - name: USE_STATSD - value: "false" - - name: CONFIG_TYPE - value: GRPC_XDS_SOTW - - name: CONFIG_GRPC_XDS_SERVER_URL - value: envoy-gateway:18001 - - name: CONFIG_GRPC_XDS_NODE_ID - value: envoy-ratelimit - - name: GRPC_SERVER_USE_TLS - value: "true" - - name: GRPC_SERVER_TLS_CERT - value: /certs/tls.crt - - name: GRPC_SERVER_TLS_KEY - value: /certs/tls.key - - name: GRPC_SERVER_TLS_CA_CERT - value: /certs/ca.crt - - name: CONFIG_GRPC_XDS_SERVER_USE_TLS - value: "true" - - name: CONFIG_GRPC_XDS_CLIENT_TLS_CERT - value: /certs/tls.crt - - name: CONFIG_GRPC_XDS_CLIENT_TLS_KEY - value: /certs/tls.key - - name: CONFIG_GRPC_XDS_SERVER_TLS_CACERT - value: /certs/ca.crt - - name: FORCE_START_WITHOUT_INITIAL_CONFIG - value: "true" - - name: TRACING_ENABLED - value: "true" - - name: TRACING_SERVICE_NAME - value: envoy-ratelimit - - name: TRACING_SERVICE_NAMESPACE - value: envoy-gateway-system - - name: TRACING_SERVICE_INSTANCE_ID - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: TRACING_SAMPLING_RATE - value: "1.0" - - name: TRACING_EXPORTER_PROTOCOL - value: http - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://localhost:4318/v1/traces - - name: OTEL_EXPORTER_OTLP_INSECURE - value: "true" - image: envoyproxy/ratelimit:master - imagePullPolicy: IfNotPresent - name: envoy-ratelimit - ports: - - containerPort: 8081 - name: grpc - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthcheck - port: 8080 - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 512Mi - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /certs - name: certs - readOnly: true - - command: - - /bin/statsd_exporter - - --web.listen-address=:19001 - - --statsd.mapping-config=/etc/statsd-exporter/conf.yaml - image: prom/statsd-exporter:v0.18.0 - imagePullPolicy: IfNotPresent - name: prom-statsd-exporter - ports: - - containerPort: 9125 - name: statsd - protocol: TCP - - containerPort: 19001 - name: metrics - protocol: TCP - resources: { } - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /etc/statsd-exporter - name: statsd-exporter-config - readOnly: true - dnsPolicy: ClusterFirst - restartPolicy: Always - schedulerName: default-scheduler - serviceAccountName: envoy-ratelimit - terminationGracePeriodSeconds: 300 - volumes: - - name: certs - secret: - defaultMode: 420 - secretName: envoy-rate-limit - - configMap: - defaultMode: 420 - name: statsd-exporter-config - optional: true - name: statsd-exporter-config -status: {} diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-with-custom-domain.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-with-custom-domain.yaml new file mode 100644 index 000000000000..099476334a54 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-with-custom-domain.yaml @@ -0,0 +1,162 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + name: envoy-ratelimit + namespace: envoy-gateway-system + ownerReferences: + - apiVersion: apps/v1 + kind: Deployment + name: envoy-gateway + uid: test-owner-reference-uid-for-deployment +spec: + progressDeadlineSeconds: 600 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + strategy: + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + spec: + automountServiceAccountToken: false + containers: + - command: + - /bin/ratelimit + env: + - name: REDIS_SOCKET_TYPE + value: tcp + - name: REDIS_URL + value: redis.redis.svc:6379 + - name: RUNTIME_ROOT + value: /data + - name: RUNTIME_SUBDIRECTORY + value: ratelimit + - name: RUNTIME_IGNOREDOTFILES + value: "true" + - name: RUNTIME_WATCH_ROOT + value: "false" + - name: LOG_LEVEL + value: info + - name: USE_STATSD + value: "false" + - name: CONFIG_TYPE + value: GRPC_XDS_SOTW + - name: CONFIG_GRPC_XDS_SERVER_URL + value: envoy-gateway:18001 + - name: CONFIG_GRPC_XDS_NODE_ID + value: envoy-ratelimit + - name: GRPC_SERVER_USE_TLS + value: "true" + - name: GRPC_SERVER_TLS_CERT + value: /certs/tls.crt + - name: GRPC_SERVER_TLS_KEY + value: /certs/tls.key + - name: GRPC_SERVER_TLS_CA_CERT + value: /certs/ca.crt + - name: CONFIG_GRPC_XDS_SERVER_USE_TLS + value: "true" + - name: CONFIG_GRPC_XDS_CLIENT_TLS_CERT + value: /certs/tls.crt + - name: CONFIG_GRPC_XDS_CLIENT_TLS_KEY + value: /certs/tls.key + - name: CONFIG_GRPC_XDS_SERVER_TLS_CACERT + value: /certs/ca.crt + - name: FORCE_START_WITHOUT_INITIAL_CONFIG + value: "true" + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SERVICE_NAME + value: envoy-ratelimit + - name: TRACING_SERVICE_NAMESPACE + value: envoy-gateway-system + - name: TRACING_SERVICE_INSTANCE_ID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: TRACING_SAMPLING_RATE + value: "0.6" + - name: TRACING_EXPORTER_PROTOCOL + value: http + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: trace-collector.observability.svc.example.local:4318 + image: envoyproxy/ratelimit:master + imagePullPolicy: IfNotPresent + name: envoy-ratelimit + ports: + - containerPort: 8081 + name: grpc + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - command: + - /bin/statsd_exporter + - --web.listen-address=:19001 + - --statsd.mapping-config=/etc/statsd-exporter/conf.yaml + image: prom/statsd-exporter:v0.18.0 + imagePullPolicy: IfNotPresent + name: prom-statsd-exporter + ports: + - containerPort: 9125 + name: statsd + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/statsd-exporter + name: statsd-exporter-config + readOnly: true + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-ratelimit + terminationGracePeriodSeconds: 300 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy-rate-limit + - configMap: + defaultMode: 420 + name: statsd-exporter-config + optional: true + name: statsd-exporter-config +status: {} diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-with-custom.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-with-custom.yaml new file mode 100644 index 000000000000..0ed6addbcd49 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-with-custom.yaml @@ -0,0 +1,162 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + name: envoy-ratelimit + namespace: envoy-gateway-system + ownerReferences: + - apiVersion: apps/v1 + kind: Deployment + name: envoy-gateway + uid: test-owner-reference-uid-for-deployment +spec: + progressDeadlineSeconds: 600 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + strategy: + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + spec: + automountServiceAccountToken: false + containers: + - command: + - /bin/ratelimit + env: + - name: REDIS_SOCKET_TYPE + value: tcp + - name: REDIS_URL + value: redis.redis.svc:6379 + - name: RUNTIME_ROOT + value: /data + - name: RUNTIME_SUBDIRECTORY + value: ratelimit + - name: RUNTIME_IGNOREDOTFILES + value: "true" + - name: RUNTIME_WATCH_ROOT + value: "false" + - name: LOG_LEVEL + value: info + - name: USE_STATSD + value: "false" + - name: CONFIG_TYPE + value: GRPC_XDS_SOTW + - name: CONFIG_GRPC_XDS_SERVER_URL + value: envoy-gateway:18001 + - name: CONFIG_GRPC_XDS_NODE_ID + value: envoy-ratelimit + - name: GRPC_SERVER_USE_TLS + value: "true" + - name: GRPC_SERVER_TLS_CERT + value: /certs/tls.crt + - name: GRPC_SERVER_TLS_KEY + value: /certs/tls.key + - name: GRPC_SERVER_TLS_CA_CERT + value: /certs/ca.crt + - name: CONFIG_GRPC_XDS_SERVER_USE_TLS + value: "true" + - name: CONFIG_GRPC_XDS_CLIENT_TLS_CERT + value: /certs/tls.crt + - name: CONFIG_GRPC_XDS_CLIENT_TLS_KEY + value: /certs/tls.key + - name: CONFIG_GRPC_XDS_SERVER_TLS_CACERT + value: /certs/ca.crt + - name: FORCE_START_WITHOUT_INITIAL_CONFIG + value: "true" + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SERVICE_NAME + value: envoy-ratelimit + - name: TRACING_SERVICE_NAMESPACE + value: envoy-gateway-system + - name: TRACING_SERVICE_INSTANCE_ID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: TRACING_SAMPLING_RATE + value: "0.9" + - name: TRACING_EXPORTER_PROTOCOL + value: grpc + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: trace-collector.observability.svc.cluster.local:4317 + image: envoyproxy/ratelimit:master + imagePullPolicy: IfNotPresent + name: envoy-ratelimit + ports: + - containerPort: 8081 + name: grpc + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - command: + - /bin/statsd_exporter + - --web.listen-address=:19001 + - --statsd.mapping-config=/etc/statsd-exporter/conf.yaml + image: prom/statsd-exporter:v0.18.0 + imagePullPolicy: IfNotPresent + name: prom-statsd-exporter + ports: + - containerPort: 9125 + name: statsd + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/statsd-exporter + name: statsd-exporter-config + readOnly: true + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-ratelimit + terminationGracePeriodSeconds: 300 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy-rate-limit + - configMap: + defaultMode: 420 + name: statsd-exporter-config + optional: true + name: statsd-exporter-config +status: {} diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-with-default.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-with-default.yaml new file mode 100644 index 000000000000..bc095349f8bf --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/enable-tracing-with-default.yaml @@ -0,0 +1,162 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + name: envoy-ratelimit + namespace: envoy-gateway-system + ownerReferences: + - apiVersion: apps/v1 + kind: Deployment + name: envoy-gateway + uid: test-owner-reference-uid-for-deployment +spec: + progressDeadlineSeconds: 600 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + strategy: + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + spec: + automountServiceAccountToken: false + containers: + - command: + - /bin/ratelimit + env: + - name: REDIS_SOCKET_TYPE + value: tcp + - name: REDIS_URL + value: redis.redis.svc:6379 + - name: RUNTIME_ROOT + value: /data + - name: RUNTIME_SUBDIRECTORY + value: ratelimit + - name: RUNTIME_IGNOREDOTFILES + value: "true" + - name: RUNTIME_WATCH_ROOT + value: "false" + - name: LOG_LEVEL + value: info + - name: USE_STATSD + value: "false" + - name: CONFIG_TYPE + value: GRPC_XDS_SOTW + - name: CONFIG_GRPC_XDS_SERVER_URL + value: envoy-gateway:18001 + - name: CONFIG_GRPC_XDS_NODE_ID + value: envoy-ratelimit + - name: GRPC_SERVER_USE_TLS + value: "true" + - name: GRPC_SERVER_TLS_CERT + value: /certs/tls.crt + - name: GRPC_SERVER_TLS_KEY + value: /certs/tls.key + - name: GRPC_SERVER_TLS_CA_CERT + value: /certs/ca.crt + - name: CONFIG_GRPC_XDS_SERVER_USE_TLS + value: "true" + - name: CONFIG_GRPC_XDS_CLIENT_TLS_CERT + value: /certs/tls.crt + - name: CONFIG_GRPC_XDS_CLIENT_TLS_KEY + value: /certs/tls.key + - name: CONFIG_GRPC_XDS_SERVER_TLS_CACERT + value: /certs/ca.crt + - name: FORCE_START_WITHOUT_INITIAL_CONFIG + value: "true" + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SERVICE_NAME + value: envoy-ratelimit + - name: TRACING_SERVICE_NAMESPACE + value: envoy-gateway-system + - name: TRACING_SERVICE_INSTANCE_ID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: TRACING_SAMPLING_RATE + value: "1.0" + - name: TRACING_EXPORTER_PROTOCOL + value: http + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: trace-collector.envoy-gateway-system.svc.cluster.local:4318 + image: envoyproxy/ratelimit:master + imagePullPolicy: IfNotPresent + name: envoy-ratelimit + ports: + - containerPort: 8081 + name: grpc + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - command: + - /bin/statsd_exporter + - --web.listen-address=:19001 + - --statsd.mapping-config=/etc/statsd-exporter/conf.yaml + image: prom/statsd-exporter:v0.18.0 + imagePullPolicy: IfNotPresent + name: prom-statsd-exporter + ports: + - containerPort: 9125 + name: statsd + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/statsd-exporter + name: statsd-exporter-config + readOnly: true + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-ratelimit + terminationGracePeriodSeconds: 300 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy-rate-limit + - configMap: + defaultMode: 420 + name: statsd-exporter-config + optional: true + name: statsd-exporter-config +status: {} diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 8f115810b47c..e1fb2665fe39 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -2213,6 +2213,24 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `metrics` | _[RateLimitMetrics](#ratelimitmetrics)_ | true | Metrics defines metrics configuration for RateLimit. | +| `tracing` | _[RateLimitTracing](#ratelimittracing)_ | true | Tracing defines traces configuration for RateLimit. | + + +#### RateLimitTracing + + + + + +_Appears in:_ +- [RateLimitTelemetry](#ratelimittelemetry) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `tracingSampleRate` | _integer_ | false | SamplingRate controls the rate at which traffic will be selected for tracing if no prior sampling decision has been made. Defaults to 100, valid values [0-100]. 100 indicates 100% sampling. | +| `backendRef` | _[BackendObjectReference](#backendobjectreference)_ | true | BackendRef defines the target trace collector endpoint configuration | +| `protocol` | _string_ | false | Protocol defines the protocol of provider in tracing feature. Only "http"(default) and "grpc" are allowed in this field | +| `clusterDomain` | _string_ | false | ClusterDomain is an optional field that specifies the custom domain used for the Kubernetes cluster. This field is used when the cluster is configured with a custom DNS domain, different from the default "cluster.local". Envoy Gateway uses this custom domain to generate fully qualified domain names (FQDN) for trace gatherer services. | #### RateLimitType