From 5b9ebea5bc513d14f1b48a4abf72a086d957d95b Mon Sep 17 00:00:00 2001 From: David Alger Date: Mon, 1 Apr 2024 09:28:50 -0500 Subject: [PATCH 1/6] api: gRPC Access Log Service (ALS) logging sink Signed-off-by: David Alger --- api/v1alpha1/accesslogging_types.go | 66 ++++++++- api/v1alpha1/zz_generated.deepcopy.go | 56 +++++++ .../gateway.envoyproxy.io_envoyproxies.yaml | 137 ++++++++++++++++++ site/content/en/latest/api/extension_types.md | 55 +++++++ 4 files changed, 313 insertions(+), 1 deletion(-) diff --git a/api/v1alpha1/accesslogging_types.go b/api/v1alpha1/accesslogging_types.go index 483a224fab4..64b6ea391d5 100644 --- a/api/v1alpha1/accesslogging_types.go +++ b/api/v1alpha1/accesslogging_types.go @@ -5,6 +5,8 @@ package v1alpha1 +import gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + type ProxyAccessLog struct { // Disable disables access logging for managed proxies if set to true. Disable bool `json:"disable,omitempty"` @@ -60,6 +62,10 @@ type ProxyAccessLogFormat struct { type ProxyAccessLogSinkType string const ( + // ProxyAccessLogSinkTypeALS defines the gRPC Access Log Service (ALS) sink. + // The service must implement the Envoy gRPC Access Log Service streaming API: + // https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/accesslog/v3/als.proto + ProxyAccessLogSinkTypeALS ProxyAccessLogSinkType = "ALS" // ProxyAccessLogSinkTypeFile defines the file accesslog sink. ProxyAccessLogSinkTypeFile ProxyAccessLogSinkType = "File" // ProxyAccessLogSinkTypeOpenTelemetry defines the OpenTelemetry accesslog sink. @@ -71,13 +77,17 @@ const ( // ProxyAccessLogSink defines the sink of accesslog. // +union // +// +kubebuilder:validation:XValidation:rule="self.type == 'ALS' ? has(self.als) : !has(self.als)",message="If AccessLogSink type is ALS, als field needs to be set." // +kubebuilder:validation:XValidation:rule="self.type == 'File' ? has(self.file) : !has(self.file)",message="If AccessLogSink type is File, file field needs to be set." // +kubebuilder:validation:XValidation:rule="self.type == 'OpenTelemetry' ? has(self.openTelemetry) : !has(self.openTelemetry)",message="If AccessLogSink type is OpenTelemetry, openTelemetry field needs to be set." type ProxyAccessLogSink struct { // Type defines the type of accesslog sink. - // +kubebuilder:validation:Enum=File;OpenTelemetry + // +kubebuilder:validation:Enum=ALS;File;OpenTelemetry // +unionDiscriminator Type ProxyAccessLogSinkType `json:"type,omitempty"` + // ALS defines the gRPC Access Log Service (ALS) sink. + // +optional + ALS *ALSEnvoyProxyAccessLog `json:"als,omitempty"` // File defines the file accesslog sink. // +optional File *FileEnvoyProxyAccessLog `json:"file,omitempty"` @@ -86,6 +96,60 @@ type ProxyAccessLogSink struct { OpenTelemetry *OpenTelemetryEnvoyProxyAccessLog `json:"openTelemetry,omitempty"` } +type ALSEnvoyProxyAccessLogType string + +const ( + // ALSEnvoyProxyAccessLogTypeHTTP defines the HTTP access log type and will populate StreamAccessLogsMessage.http_logs. + ALSEnvoyProxyAccessLogTypeHTTP ALSEnvoyProxyAccessLogType = "HTTP" + // ALSEnvoyProxyAccessLogTypeTCP defines the TCP access log type and will populate StreamAccessLogsMessage.tcp_logs. + ALSEnvoyProxyAccessLogTypeTCP ALSEnvoyProxyAccessLogType = "TCP" +) + +// ALSEnvoyProxyAccessLog defines the gRPC Access Log Service (ALS) sink. +// The service must implement the Envoy gRPC Access Log Service streaming API: +// https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/accesslog/v3/als.proto +// Access log format information is passed in the form of gRPC metadata when the +// stream is established. Specifically, the following metadata is passed: +// +// - `x-accesslog-text` - The access log format string when a Text format is used. +// +// - `x-accesslog-attr` - JSON encoded key/value pairs when a JSON format is used. +// +// +kubebuilder:validation:XValidation:message="BackendRef only supports Service Kind.",rule="!has(self.backendRef.kind) || self.backendRef.kind == 'Service'" +// +kubebuilder:validation:XValidation:rule="self.type == 'HTTP' || !has(self.http)",message="The http field may only be set when type is HTTP." +type ALSEnvoyProxyAccessLog struct { + // BackendRef references a Kubernetes object that represents the gRPC service to which + // the access logs will be sent. Currently only Service is supported. + BackendRef gwapiv1.BackendObjectReference `json:"backendRef,omitempty"` + // LogName defines the friendly name of the access log to be returned in + // StreamAccessLogsMessage.Identifier. This allows the access log server + // to differentiate between different access logs coming from the same Envoy. + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:default="accesslog" + LogName string `json:"logName,omitempty"` + // Type defines the type of accesslog. Supported types are "HTTP" and "TCP". Defaults to "HTTP" when not specified. + // +kubebuilder:validation:Enum=HTTP;TCP + // +unionDiscriminator + // +kubebuilder:default="HTTP" + Type ALSEnvoyProxyAccessLogType `json:"type,omitempty"` + // HTTP defines additional configuration specific to HTTP access logs. + // +optional + HTTP *ALSEnvoyProxyHTTPAccessLogConfig `json:"http,omitempty"` +} + +type ALSEnvoyProxyHTTPAccessLogConfig struct { + // RequestHeaders defines request headers to include in log entries sent to the access log service. + // +optional + RequestHeaders []string `json:"requestHeaders,omitempty" yaml:"requestHeaders,omitempty"` + // ResponseHeaders defines response headers to include in log entries sent to the access log service. + // +optional + ResponseHeaders []string `json:"responseHeaders,omitempty" yaml:"responseHeaders,omitempty"` + // ResponseTrailers defines response trailers to include in log entries sent to the access log service. + // +optional + ResponseTrailers []string `json:"responseTrailers,omitempty" yaml:"responseTrailers,omitempty"` +} + type FileEnvoyProxyAccessLog struct { // Path defines the file path used to expose envoy access log(e.g. /dev/stdout). // +kubebuilder:validation:MinLength=1 diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d170dcf1f43..0d532b2be9f 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -19,6 +19,57 @@ import ( "sigs.k8s.io/gateway-api/apis/v1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ALSEnvoyProxyAccessLog) DeepCopyInto(out *ALSEnvoyProxyAccessLog) { + *out = *in + in.BackendRef.DeepCopyInto(&out.BackendRef) + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(ALSEnvoyProxyHTTPAccessLogConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ALSEnvoyProxyAccessLog. +func (in *ALSEnvoyProxyAccessLog) DeepCopy() *ALSEnvoyProxyAccessLog { + if in == nil { + return nil + } + out := new(ALSEnvoyProxyAccessLog) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ALSEnvoyProxyHTTPAccessLogConfig) DeepCopyInto(out *ALSEnvoyProxyHTTPAccessLogConfig) { + *out = *in + if in.RequestHeaders != nil { + in, out := &in.RequestHeaders, &out.RequestHeaders + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ResponseHeaders != nil { + in, out := &in.ResponseHeaders, &out.ResponseHeaders + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ResponseTrailers != nil { + in, out := &in.ResponseTrailers, &out.ResponseTrailers + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ALSEnvoyProxyHTTPAccessLogConfig. +func (in *ALSEnvoyProxyHTTPAccessLogConfig) DeepCopy() *ALSEnvoyProxyHTTPAccessLogConfig { + if in == nil { + return nil + } + out := new(ALSEnvoyProxyHTTPAccessLogConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ActiveHealthCheck) DeepCopyInto(out *ActiveHealthCheck) { *out = *in @@ -2804,6 +2855,11 @@ func (in *ProxyAccessLogSetting) DeepCopy() *ProxyAccessLogSetting { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxyAccessLogSink) DeepCopyInto(out *ProxyAccessLogSink) { *out = *in + if in.ALS != nil { + in, out := &in.ALS, &out.ALS + *out = new(ALSEnvoyProxyAccessLog) + (*in).DeepCopyInto(*out) + } if in.File != nil { in, out := &in.File, &out.File *out = new(FileEnvoyProxyAccessLog) diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 4e092a8e739..98c2fcf0d68 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5886,6 +5886,139 @@ spec: description: ProxyAccessLogSink defines the sink of accesslog. properties: + als: + description: ALS defines the gRPC Access Log Service + (ALS) sink. + properties: + backendRef: + description: |- + BackendRef references a Kubernetes object that represents the gRPC service to which + the access logs will be sent. Currently only Service is supported. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + + Defaults to "Service" when not specified. + + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations SHOULD NOT + support ExternalName Services. + + + Support: Core (Services with a type other than ExternalName) + + + Support: Implementation-specific (Services with type ExternalName) + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the backend. When unspecified, the local + namespace is inferred. + + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port specifies the destination port number to use for this resource. + Port is required when the referent is a Kubernetes Service. In this + case, the port number is the service port number, not the target port. + For other resources, destination port might be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + x-kubernetes-validations: + - message: Must have port for Service reference + rule: '(size(self.group) == 0 && self.kind + == ''Service'') ? has(self.port) : true' + http: + description: HTTP defines additional configuration + specific to HTTP access logs. + properties: + requestHeaders: + description: RequestHeaders defines request + headers to include in log entries sent + to the access log service. + items: + type: string + type: array + responseHeaders: + description: ResponseHeaders defines response + headers to include in log entries sent + to the access log service. + items: + type: string + type: array + responseTrailers: + description: ResponseTrailers defines + response trailers to include in log + entries sent to the access log service. + items: + type: string + type: array + type: object + logName: + default: accesslog + description: |- + LogName defines the friendly name of the access log to be returned in + StreamAccessLogsMessage.Identifier. This allows the access log server + to differentiate between different access logs coming from the same Envoy. + minLength: 1 + type: string + type: + default: HTTP + description: Type defines the type of accesslog. + Supported types are "HTTP" and "TCP". Defaults + to "HTTP" when not specified. + enum: + - HTTP + - TCP + type: string + type: object + x-kubernetes-validations: + - message: BackendRef only supports Service Kind. + rule: '!has(self.backendRef.kind) || self.backendRef.kind + == ''Service''' + - message: The http field may only be set when + type is HTTP. + rule: self.type == 'HTTP' || !has(self.http) file: description: File defines the file accesslog sink. properties: @@ -5924,11 +6057,15 @@ spec: description: Type defines the type of accesslog sink. enum: + - ALS - File - OpenTelemetry type: string type: object x-kubernetes-validations: + - message: If AccessLogSink type is ALS, als field + needs to be set. + rule: 'self.type == ''ALS'' ? has(self.als) : !has(self.als)' - message: If AccessLogSink type is File, file field needs to be set. rule: 'self.type == ''File'' ? has(self.file) : diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index a289a7530b8..6f69cfc2a89 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -40,6 +40,60 @@ _Appears in:_ +#### ALSEnvoyProxyAccessLog + + + +ALSEnvoyProxyAccessLog defines the gRPC Access Log Service (ALS) sink. +The service must implement the Envoy gRPC Access Log Service streaming API: +https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/accesslog/v3/als.proto +Access log format information is passed in the form of gRPC metadata when the +stream is established. Specifically, the following metadata is passed: + + +- `x-accesslog-text` - The access log format string when a Text format is used. + + +- `x-accesslog-attr` - JSON encoded key/value pairs when a JSON format is used. + +_Appears in:_ +- [ProxyAccessLogSink](#proxyaccesslogsink) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `backendRef` | _[BackendObjectReference](#backendobjectreference)_ | true | BackendRef references a Kubernetes object that represents the gRPC service to which
the access logs will be sent. Currently only Service is supported. | +| `logName` | _string_ | false | LogName defines the friendly name of the access log to be returned in
StreamAccessLogsMessage.Identifier. This allows the access log server
to differentiate between different access logs coming from the same Envoy. | +| `type` | _[ALSEnvoyProxyAccessLogType](#alsenvoyproxyaccesslogtype)_ | true | Type defines the type of accesslog. Supported types are "HTTP" and "TCP". Defaults to "HTTP" when not specified. | +| `http` | _[ALSEnvoyProxyHTTPAccessLogConfig](#alsenvoyproxyhttpaccesslogconfig)_ | false | HTTP defines additional configuration specific to HTTP access logs. | + + +#### ALSEnvoyProxyAccessLogType + +_Underlying type:_ _string_ + + + +_Appears in:_ +- [ALSEnvoyProxyAccessLog](#alsenvoyproxyaccesslog) + + + +#### ALSEnvoyProxyHTTPAccessLogConfig + + + + + +_Appears in:_ +- [ALSEnvoyProxyAccessLog](#alsenvoyproxyaccesslog) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `requestHeaders` | _string array_ | false | RequestHeaders defines request headers to include in log entries sent to the access log service. | +| `responseHeaders` | _string array_ | false | ResponseHeaders defines response headers to include in log entries sent to the access log service. | +| `responseTrailers` | _string array_ | false | ResponseTrailers defines response trailers to include in log entries sent to the access log service. | + + #### ActiveHealthCheck @@ -1974,6 +2028,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `type` | _[ProxyAccessLogSinkType](#proxyaccesslogsinktype)_ | true | Type defines the type of accesslog sink. | +| `als` | _[ALSEnvoyProxyAccessLog](#alsenvoyproxyaccesslog)_ | false | ALS defines the gRPC Access Log Service (ALS) sink. | | `file` | _[FileEnvoyProxyAccessLog](#fileenvoyproxyaccesslog)_ | false | File defines the file accesslog sink. | | `openTelemetry` | _[OpenTelemetryEnvoyProxyAccessLog](#opentelemetryenvoyproxyaccesslog)_ | false | OpenTelemetry defines the OpenTelemetry accesslog sink. | From f5c13a382b44050b2ff05f94569df4a836f4bc60 Mon Sep 17 00:00:00 2001 From: David Alger Date: Tue, 2 Apr 2024 09:25:45 -0500 Subject: [PATCH 2/6] Make type mandatory and remove kubebuilder default for LogName Signed-off-by: David Alger --- api/v1alpha1/accesslogging_types.go | 5 +---- api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ .../crds/generated/gateway.envoyproxy.io_envoyproxies.yaml | 2 -- site/content/en/latest/api/extension_types.md | 2 -- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/accesslogging_types.go b/api/v1alpha1/accesslogging_types.go index 3311c2f5eab..8d6690c02c6 100644 --- a/api/v1alpha1/accesslogging_types.go +++ b/api/v1alpha1/accesslogging_types.go @@ -112,7 +112,6 @@ const ( // stream is established. Specifically, the following metadata is passed: // // - `x-accesslog-text` - The access log format string when a Text format is used. -// // - `x-accesslog-attr` - JSON encoded key/value pairs when a JSON format is used. // // +kubebuilder:validation:XValidation:message="BackendRef only supports Service Kind.",rule="!has(self.backendRef.kind) || self.backendRef.kind == 'Service'" @@ -126,12 +125,10 @@ type ALSEnvoyProxyAccessLog struct { // to differentiate between different access logs coming from the same Envoy. // +optional // +kubebuilder:validation:MinLength=1 - // +kubebuilder:default="accesslog" - LogName string `json:"logName,omitempty"` + LogName *string `json:"logName,omitempty"` // Type defines the type of accesslog. Supported types are "HTTP" and "TCP". Defaults to "HTTP" when not specified. // +kubebuilder:validation:Enum=HTTP;TCP // +unionDiscriminator - // +kubebuilder:default="HTTP" Type ALSEnvoyProxyAccessLogType `json:"type,omitempty"` // HTTP defines additional configuration specific to HTTP access logs. // +optional diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f9a1f32ae57..445222b93fd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,6 +23,11 @@ import ( func (in *ALSEnvoyProxyAccessLog) DeepCopyInto(out *ALSEnvoyProxyAccessLog) { *out = *in in.BackendRef.DeepCopyInto(&out.BackendRef) + if in.LogName != nil { + in, out := &in.LogName, &out.LogName + *out = new(string) + **out = **in + } if in.HTTP != nil { in, out := &in.HTTP, &out.HTTP *out = new(ALSEnvoyProxyHTTPAccessLogConfig) diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 20c439f0102..80ac072f6be 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5995,7 +5995,6 @@ spec: type: array type: object logName: - default: accesslog description: |- LogName defines the friendly name of the access log to be returned in StreamAccessLogsMessage.Identifier. This allows the access log server @@ -6003,7 +6002,6 @@ spec: minLength: 1 type: string type: - default: HTTP description: Type defines the type of accesslog. Supported types are "HTTP" and "TCP". Defaults to "HTTP" when not specified. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 74742c009c0..1a64a043138 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -52,8 +52,6 @@ stream is established. Specifically, the following metadata is passed: - `x-accesslog-text` - The access log format string when a Text format is used. - - - `x-accesslog-attr` - JSON encoded key/value pairs when a JSON format is used. _Appears in:_ From 64890e9ec8c6e3edc8bc53119e4df656c638a8bb Mon Sep 17 00:00:00 2001 From: David Alger Date: Tue, 2 Apr 2024 11:02:21 -0500 Subject: [PATCH 3/6] Add CEL validations Signed-off-by: David Alger --- api/v1alpha1/accesslogging_types.go | 5 +- .../gateway.envoyproxy.io_envoyproxies.yaml | 3 + test/cel-validation/envoyproxy_test.go | 124 ++++++++++++++++++ 3 files changed, 129 insertions(+), 3 deletions(-) diff --git a/api/v1alpha1/accesslogging_types.go b/api/v1alpha1/accesslogging_types.go index 8d6690c02c6..726dbc50d09 100644 --- a/api/v1alpha1/accesslogging_types.go +++ b/api/v1alpha1/accesslogging_types.go @@ -119,7 +119,7 @@ const ( type ALSEnvoyProxyAccessLog struct { // BackendRef references a Kubernetes object that represents the gRPC service to which // the access logs will be sent. Currently only Service is supported. - BackendRef gwapiv1.BackendObjectReference `json:"backendRef,omitempty"` + BackendRef gwapiv1.BackendObjectReference `json:"backendRef"` // LogName defines the friendly name of the access log to be returned in // StreamAccessLogsMessage.Identifier. This allows the access log server // to differentiate between different access logs coming from the same Envoy. @@ -128,8 +128,7 @@ type ALSEnvoyProxyAccessLog struct { LogName *string `json:"logName,omitempty"` // Type defines the type of accesslog. Supported types are "HTTP" and "TCP". Defaults to "HTTP" when not specified. // +kubebuilder:validation:Enum=HTTP;TCP - // +unionDiscriminator - Type ALSEnvoyProxyAccessLogType `json:"type,omitempty"` + Type ALSEnvoyProxyAccessLogType `json:"type"` // HTTP defines additional configuration specific to HTTP access logs. // +optional HTTP *ALSEnvoyProxyHTTPAccessLogConfig `json:"http,omitempty"` diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 80ac072f6be..2ca621f1f64 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -6009,6 +6009,9 @@ spec: - HTTP - TCP type: string + required: + - backendRef + - type type: object x-kubernetes-validations: - message: BackendRef only supports Service Kind. diff --git a/test/cel-validation/envoyproxy_test.go b/test/cel-validation/envoyproxy_test.go index b7850a084ad..416fe2fe6b6 100644 --- a/test/cel-validation/envoyproxy_test.go +++ b/test/cel-validation/envoyproxy_test.go @@ -321,6 +321,33 @@ func TestEnvoyProxyProvider(t *testing.T) { }, wantErrors: []string{"If AccessLogFormat type is JSON, json field needs to be set"}, }, + { + desc: "ProxyAccessLogSink-with-TypeALS-but-no-als", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Telemetry: &egv1a1.ProxyTelemetry{ + AccessLog: &egv1a1.ProxyAccessLog{ + Settings: []egv1a1.ProxyAccessLogSetting{ + { + Format: egv1a1.ProxyAccessLogFormat{ + Type: egv1a1.ProxyAccessLogFormatTypeJSON, + JSON: map[string]string{ + "foo": "bar", + }, + }, + Sinks: []egv1a1.ProxyAccessLogSink{ + { + Type: egv1a1.ProxyAccessLogSinkTypeALS, + }, + }, + }, + }, + }, + }, + } + }, + wantErrors: []string{"If AccessLogSink type is ALS, als field needs to be set"}, + }, { desc: "ProxyAccessLogSink-with-TypeFile-but-no-file", mutate: func(envoy *egv1a1.EnvoyProxy) { @@ -399,6 +426,103 @@ func TestEnvoyProxyProvider(t *testing.T) { }, wantErrors: []string{}, }, + { + desc: "accesslog-ALS", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Telemetry: &egv1a1.ProxyTelemetry{ + AccessLog: &egv1a1.ProxyAccessLog{ + Settings: []egv1a1.ProxyAccessLogSetting{ + { + Format: egv1a1.ProxyAccessLogFormat{ + Type: egv1a1.ProxyAccessLogFormatTypeJSON, + JSON: map[string]string{ + "attr1": "value1", + "attr2": "value2", + }, + }, + Sinks: []egv1a1.ProxyAccessLogSink{ + { + Type: egv1a1.ProxyAccessLogSinkTypeALS, + ALS: &egv1a1.ALSEnvoyProxyAccessLog{ + BackendRef: gwapiv1.BackendObjectReference{ + Name: "fake-service", + Port: ptr.To(gwapiv1.PortNumber(9000)), + }, + }, + }, + }, + }, + }, + }, + }, + } + }, + }, + { + desc: "invalid-accesslog-ALS-type", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Telemetry: &egv1a1.ProxyTelemetry{ + AccessLog: &egv1a1.ProxyAccessLog{ + Settings: []egv1a1.ProxyAccessLogSetting{ + { + Format: egv1a1.ProxyAccessLogFormat{ + Type: "Text", + Text: ptr.To("[%START_TIME%]"), + }, + Sinks: []egv1a1.ProxyAccessLogSink{ + { + Type: egv1a1.ProxyAccessLogSinkTypeALS, + ALS: &egv1a1.ALSEnvoyProxyAccessLog{ + BackendRef: gwapiv1.BackendObjectReference{ + Name: "fake-service", + Port: ptr.To(gwapiv1.PortNumber(9000)), + }, + Type: egv1a1.ALSEnvoyProxyAccessLogTypeTCP, + HTTP: &egv1a1.ALSEnvoyProxyHTTPAccessLogConfig{}, + }, + }, + }, + }, + }, + }, + }, + } + }, + wantErrors: []string{"The http field may only be set when type is HTTP."}, + }, + { + desc: "invalid-accesslog-ALS-backendref", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Telemetry: &egv1a1.ProxyTelemetry{ + AccessLog: &egv1a1.ProxyAccessLog{ + Settings: []egv1a1.ProxyAccessLogSetting{ + { + Format: egv1a1.ProxyAccessLogFormat{ + Type: "Text", + Text: ptr.To("[%START_TIME%]"), + }, + Sinks: []egv1a1.ProxyAccessLogSink{ + { + Type: egv1a1.ProxyAccessLogSinkTypeALS, + ALS: &egv1a1.ALSEnvoyProxyAccessLog{ + BackendRef: gwapiv1.BackendObjectReference{ + Name: "fake-service", + Kind: ptr.To(gwapiv1.Kind("foo")), + }, + }, + }, + }, + }, + }, + }, + }, + } + }, + wantErrors: []string{"BackendRef only supports Service Kind."}, + }, { desc: "accesslog-OpenTelemetry", mutate: func(envoy *egv1a1.EnvoyProxy) { From a32679ee15e6c94c4e7ade4f6a8a27171d4fe9ea Mon Sep 17 00:00:00 2001 From: David Alger Date: Tue, 2 Apr 2024 11:11:46 -0500 Subject: [PATCH 4/6] Cleanup Signed-off-by: David Alger --- api/v1alpha1/accesslogging_types.go | 8 ++++---- .../generated/gateway.envoyproxy.io_envoyproxies.yaml | 3 +-- site/content/en/latest/api/extension_types.md | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/api/v1alpha1/accesslogging_types.go b/api/v1alpha1/accesslogging_types.go index 726dbc50d09..97acad15568 100644 --- a/api/v1alpha1/accesslogging_types.go +++ b/api/v1alpha1/accesslogging_types.go @@ -126,7 +126,7 @@ type ALSEnvoyProxyAccessLog struct { // +optional // +kubebuilder:validation:MinLength=1 LogName *string `json:"logName,omitempty"` - // Type defines the type of accesslog. Supported types are "HTTP" and "TCP". Defaults to "HTTP" when not specified. + // Type defines the type of accesslog. Supported types are "HTTP" and "TCP". // +kubebuilder:validation:Enum=HTTP;TCP Type ALSEnvoyProxyAccessLogType `json:"type"` // HTTP defines additional configuration specific to HTTP access logs. @@ -137,13 +137,13 @@ type ALSEnvoyProxyAccessLog struct { type ALSEnvoyProxyHTTPAccessLogConfig struct { // RequestHeaders defines request headers to include in log entries sent to the access log service. // +optional - RequestHeaders []string `json:"requestHeaders,omitempty" yaml:"requestHeaders,omitempty"` + RequestHeaders []string `json:"requestHeaders,omitempty"` // ResponseHeaders defines response headers to include in log entries sent to the access log service. // +optional - ResponseHeaders []string `json:"responseHeaders,omitempty" yaml:"responseHeaders,omitempty"` + ResponseHeaders []string `json:"responseHeaders,omitempty"` // ResponseTrailers defines response trailers to include in log entries sent to the access log service. // +optional - ResponseTrailers []string `json:"responseTrailers,omitempty" yaml:"responseTrailers,omitempty"` + ResponseTrailers []string `json:"responseTrailers,omitempty"` } type FileEnvoyProxyAccessLog struct { diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 2ca621f1f64..50220f93acd 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -6003,8 +6003,7 @@ spec: type: string type: description: Type defines the type of accesslog. - Supported types are "HTTP" and "TCP". Defaults - to "HTTP" when not specified. + Supported types are "HTTP" and "TCP". enum: - HTTP - TCP diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 1a64a043138..9caa9b9aa85 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -61,7 +61,7 @@ _Appears in:_ | --- | --- | --- | --- | | `backendRef` | _[BackendObjectReference](#backendobjectreference)_ | true | BackendRef references a Kubernetes object that represents the gRPC service to which
the access logs will be sent. Currently only Service is supported. | | `logName` | _string_ | false | LogName defines the friendly name of the access log to be returned in
StreamAccessLogsMessage.Identifier. This allows the access log server
to differentiate between different access logs coming from the same Envoy. | -| `type` | _[ALSEnvoyProxyAccessLogType](#alsenvoyproxyaccesslogtype)_ | true | Type defines the type of accesslog. Supported types are "HTTP" and "TCP". Defaults to "HTTP" when not specified. | +| `type` | _[ALSEnvoyProxyAccessLogType](#alsenvoyproxyaccesslogtype)_ | true | Type defines the type of accesslog. Supported types are "HTTP" and "TCP". | | `http` | _[ALSEnvoyProxyHTTPAccessLogConfig](#alsenvoyproxyhttpaccesslogconfig)_ | false | HTTP defines additional configuration specific to HTTP access logs. | From 21ae2a5290b629297963eb5d1aaa31b6b79a31de Mon Sep 17 00:00:00 2001 From: David Alger Date: Tue, 2 Apr 2024 11:14:11 -0500 Subject: [PATCH 5/6] Fix missing type in validation Signed-off-by: David Alger --- test/cel-validation/envoyproxy_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cel-validation/envoyproxy_test.go b/test/cel-validation/envoyproxy_test.go index 416fe2fe6b6..5c0f82c359f 100644 --- a/test/cel-validation/envoyproxy_test.go +++ b/test/cel-validation/envoyproxy_test.go @@ -449,6 +449,7 @@ func TestEnvoyProxyProvider(t *testing.T) { Name: "fake-service", Port: ptr.To(gwapiv1.PortNumber(9000)), }, + Type: egv1a1.ALSEnvoyProxyAccessLogTypeHTTP, }, }, }, From 31ace50039eed317ea8790e715b4bf83f11be2c6 Mon Sep 17 00:00:00 2001 From: David Alger Date: Thu, 28 Mar 2024 21:43:56 -0500 Subject: [PATCH 6/6] feat: gRPC Access Log Service (ALS) logging sink Signed-off-by: David Alger --- .../kubernetes/accesslog/als-accesslog.yaml | 39 ++++ .../kubernetes/accesslog/multi-sinks.yaml | 7 + internal/gatewayapi/listener.go | 93 ++++++++- .../envoyproxy-accesslog-als-json.in.yaml | 132 ++++++++++++ .../envoyproxy-accesslog-als-json.out.yaml | 195 ++++++++++++++++++ ...nvoyproxy-accesslog-with-bad-sinks.in.yaml | 1 + ...voyproxy-accesslog-with-bad-sinks.out.yaml | 1 + .../testdata/envoyproxy-accesslog.in.yaml | 53 +++++ .../testdata/envoyproxy-accesslog.out.yaml | 56 +++++ internal/ir/xds.go | 20 ++ internal/ir/zz_generated.deepcopy.go | 74 +++++++ internal/provider/kubernetes/controller.go | 86 ++++++-- internal/provider/kubernetes/indexers.go | 35 ++++ internal/provider/kubernetes/predicates.go | 33 ++- internal/xds/translator/accesslog.go | 84 ++++++++ .../testdata/in/xds-ir/accesslog-als-tcp.yaml | 23 +++ .../testdata/in/xds-ir/accesslog.yaml | 23 +++ .../xds-ir/accesslog-als-tcp.clusters.yaml | 22 ++ .../xds-ir/accesslog-als-tcp.endpoints.yaml | 12 ++ .../xds-ir/accesslog-als-tcp.listeners.yaml | 1 + .../out/xds-ir/accesslog-als-tcp.routes.yaml | 1 + .../out/xds-ir/accesslog.clusters.yaml | 22 ++ .../out/xds-ir/accesslog.endpoints.yaml | 12 ++ .../out/xds-ir/accesslog.listeners.yaml | 49 +++++ internal/xds/translator/translator_test.go | 3 + 25 files changed, 1050 insertions(+), 27 deletions(-) create mode 100644 examples/kubernetes/accesslog/als-accesslog.yaml create mode 100644 internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.in.yaml create mode 100755 internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.out.yaml create mode 100644 internal/xds/translator/testdata/in/xds-ir/accesslog-als-tcp.yaml create mode 100755 internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.clusters.yaml create mode 100755 internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.endpoints.yaml create mode 100755 internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.listeners.yaml create mode 100755 internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.routes.yaml diff --git a/examples/kubernetes/accesslog/als-accesslog.yaml b/examples/kubernetes/accesslog/als-accesslog.yaml new file mode 100644 index 00000000000..1b28389d7d3 --- /dev/null +++ b/examples/kubernetes/accesslog/als-accesslog.yaml @@ -0,0 +1,39 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: als-access-logging + namespace: envoy-gateway-system +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: als-access-logging + namespace: envoy-gateway-system +spec: + telemetry: + accessLog: + settings: + - format: + type: JSON + json: + attr1: val1 + attr2: val2 + sinks: + - type: ALS + als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + type: HTTP diff --git a/examples/kubernetes/accesslog/multi-sinks.yaml b/examples/kubernetes/accesslog/multi-sinks.yaml index ae2cf3b55d5..4048cb4983c 100644 --- a/examples/kubernetes/accesslog/multi-sinks.yaml +++ b/examples/kubernetes/accesslog/multi-sinks.yaml @@ -27,6 +27,13 @@ spec: - type: File file: path: /dev/stdout + - type: ALS + als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + type: HTTP - type: OpenTelemetry openTelemetry: host: otel-collector.monitoring.svc.cluster.local diff --git a/internal/gatewayapi/listener.go b/internal/gatewayapi/listener.go index 4702d2a4e26..3a27cba6898 100644 --- a/internal/gatewayapi/listener.go +++ b/internal/gatewayapi/listener.go @@ -8,7 +8,9 @@ package gatewayapi import ( "fmt" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" @@ -43,7 +45,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap infraIR[irKey].Proxy.Config = resources.EnvoyProxy } - xdsIR[irKey].AccessLog = processAccessLog(infraIR[irKey].Proxy.Config) + xdsIR[irKey].AccessLog = t.processAccessLog(infraIR[irKey].Proxy.Config, gateway.Gateway, resources) xdsIR[irKey].Tracing = processTracing(gateway.Gateway, infraIR[irKey].Proxy.Config) xdsIR[irKey].Metrics = processMetrics(infraIR[irKey].Proxy.Config) @@ -160,7 +162,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap } } -func processAccessLog(envoyproxy *egv1a1.EnvoyProxy) *ir.AccessLog { +func (t *Translator) processAccessLog(envoyproxy *egv1a1.EnvoyProxy, gw *gwapiv1.Gateway, resources *Resources) *ir.AccessLog { if envoyproxy == nil || envoyproxy.Spec.Telemetry == nil || envoyproxy.Spec.Telemetry.AccessLog == nil || @@ -208,6 +210,52 @@ func processAccessLog(envoyproxy *egv1a1.EnvoyProxy) *ir.AccessLog { } irAccessLog.JSON = append(irAccessLog.JSON, al) } + case egv1a1.ProxyAccessLogSinkTypeALS: + if sink.ALS == nil { + continue + } + + var logName string + if sink.ALS.LogName != nil { + logName = *sink.ALS.LogName + } else { + logName = fmt.Sprintf("accesslog/%s/%s", gw.Namespace, gw.Name) + } + + clusterName := fmt.Sprintf("accesslog/%s/%s/port/%d", + NamespaceDerefOr(sink.ALS.BackendRef.Namespace, envoyproxy.Namespace), + string(sink.ALS.BackendRef.Name), + *sink.ALS.BackendRef.Port, + ) + + al := &ir.ALSAccessLog{ + LogName: logName, + Destination: ir.RouteDestination{ + Name: clusterName, + Settings: []*ir.DestinationSetting{ + t.processServiceDestination(sink.ALS.BackendRef, ir.GRPC, envoyproxy, resources), + }, + }, + Type: sink.ALS.Type, + } + + if al.Type == egv1a1.ALSEnvoyProxyAccessLogTypeHTTP { + http := &ir.ALSAccessLogHTTP{ + RequestHeaders: sink.ALS.HTTP.RequestHeaders, + ResponseHeaders: sink.ALS.HTTP.ResponseHeaders, + ResponseTrailers: sink.ALS.HTTP.ResponseTrailers, + } + al.HTTP = http + } + + switch accessLog.Format.Type { + case egv1a1.ProxyAccessLogFormatTypeJSON: + al.Attributes = accessLog.Format.JSON + case egv1a1.ProxyAccessLogFormatTypeText: + al.Text = accessLog.Format.Text + } + + irAccessLog.ALS = append(irAccessLog.ALS, al) case egv1a1.ProxyAccessLogSinkTypeOpenTelemetry: if sink.OpenTelemetry == nil { continue @@ -234,6 +282,47 @@ func processAccessLog(envoyproxy *egv1a1.EnvoyProxy) *ir.AccessLog { return irAccessLog } +func (t *Translator) processServiceDestination(backendRef gwapiv1.BackendObjectReference, protocol ir.AppProtocol, envoyproxy *egv1a1.EnvoyProxy, resources *Resources) *ir.DestinationSetting { + var ( + endpoints []*ir.DestinationEndpoint + addrType *ir.DestinationAddressType + servicePort v1.ServicePort + backendTLS *ir.TLSUpstreamConfig + ) + + serviceNamespace := NamespaceDerefOr(backendRef.Namespace, envoyproxy.Namespace) + service := resources.GetService(serviceNamespace, string(backendRef.Name)) + for _, port := range service.Spec.Ports { + if port.Port == int32(*backendRef.Port) { + servicePort = port + break + } + } + + if servicePort.AppProtocol != nil && + *servicePort.AppProtocol == "kubernetes.io/h2c" { + protocol = ir.HTTP2 + } + + // Route to endpoints by default + if !t.EndpointRoutingDisabled { + endpointSlices := resources.GetEndpointSlicesForBackend(serviceNamespace, string(backendRef.Name), KindDerefOr(backendRef.Kind, KindService)) + endpoints, addrType = getIREndpointsFromEndpointSlices(endpointSlices, servicePort.Name, servicePort.Protocol) + } else { + // Fall back to Service ClusterIP routing + ep := ir.NewDestEndpoint(service.Spec.ClusterIP, uint32(*backendRef.Port)) + endpoints = append(endpoints, ep) + } + + return &ir.DestinationSetting{ + Weight: ptr.To(uint32(1)), + Protocol: protocol, + Endpoints: endpoints, + AddressType: addrType, + TLS: backendTLS, + } +} + func processTracing(gw *gwapiv1.Gateway, envoyproxy *egv1a1.EnvoyProxy) *ir.Tracing { if envoyproxy == nil || envoyproxy.Spec.Telemetry == nil || diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.in.yaml new file mode 100644 index 00000000000..cb7c82b4afe --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.in.yaml @@ -0,0 +1,132 @@ +envoyproxy: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + namespace: envoy-gateway-system + name: test + spec: + telemetry: + accessLog: + settings: + - format: + type: JSON + json: + attr1: val1 + attr2: val2 + sinks: + - type: ALS + als: + logName: accesslog + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + type: HTTP + - type: ALS + als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + type: TCP + provider: + type: Kubernetes + kubernetes: + envoyService: + type: LoadBalancer + envoyDeployment: + replicas: 2 + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: "envoyproxy/envoy:distroless-dev" + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + runAsUser: 2000 + allowPrivilegeEscalation: false + pod: + annotations: + key1: val1 + key2: val2 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: "router" + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + fsGroupChangePolicy: "OnRootMismatch" + volumes: + - name: certs + secret: + secretName: envoy-cert +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same +services: +- apiVersion: v1 + kind: Service + metadata: + name: envoy-als + namespace: monitoring + spec: + type: ClusterIP + ports: + - name: grpc + port: 9000 + protocol: TCP + targetPort: 9000 +endpointSlices: +- apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: endpointslice-envoy-als + namespace: monitoring + labels: + kubernetes.io/service-name: envoy-als + addressType: IPv4 + ports: + - name: grpc + protocol: TCP + port: 9090 + endpoints: + - addresses: + - "10.240.0.10" + conditions: + ready: true diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.out.yaml new file mode 100755 index 00000000000..2ab9d534a6f --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.out.yaml @@ -0,0 +1,195 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: Same + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +infraIR: + envoy-gateway/gateway-1: + proxy: + config: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + creationTimestamp: null + name: test + namespace: envoy-gateway-system + spec: + logging: {} + provider: + kubernetes: + envoyDeployment: + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: envoyproxy/envoy:distroless-dev + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + runAsUser: 2000 + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + annotations: + key1: val1 + key2: val2 + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 3000 + runAsUser: 1000 + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: router + volumes: + - name: certs + secret: + secretName: envoy-cert + replicas: 2 + envoyService: + type: LoadBalancer + type: Kubernetes + telemetry: + accessLog: + settings: + - format: + json: + attr1: val1 + attr2: val2 + type: JSON + sinks: + - als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + logName: accesslog + type: HTTP + type: ALS + - als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + type: TCP + type: ALS + status: {} + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + als: + - attributes: + attr1: val1 + attr2: val2 + destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + weight: 1 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + name: accesslog + type: HTTP + - attributes: + attr1: val1 + attr2: val2 + destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + weight: 1 + name: accesslog/envoy-gateway/gateway-1 + type: TCP + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml index cccabc9cd4d..58277d8a3b5 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml @@ -14,6 +14,7 @@ envoyproxy: [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n sinks: - type: File + - type: ALS - type: OpenTelemetry provider: type: Kubernetes diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml index 583ce720c9a..2d9ec30b5c5 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml @@ -108,6 +108,7 @@ infraIR: type: Text sinks: - type: File + - type: ALS - type: OpenTelemetry status: {} listeners: diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml index 0d691420576..95f003afff3 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml @@ -16,6 +16,28 @@ envoyproxy: - type: File file: path: /dev/stdout + - type: ALS + als: + logName: accesslog + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + type: HTTP + - type: ALS + als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + type: TCP - type: OpenTelemetry openTelemetry: host: otel-collector.monitoring.svc.cluster.local @@ -85,3 +107,34 @@ gateways: allowedRoutes: namespaces: from: Same +services: +- apiVersion: v1 + kind: Service + metadata: + name: envoy-als + namespace: monitoring + spec: + type: ClusterIP + ports: + - name: grpc + port: 9000 + protocol: TCP + targetPort: 9000 +endpointSlices: +- apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: endpointslice-envoy-als + namespace: monitoring + labels: + kubernetes.io/service-name: envoy-als + addressType: IPv4 + ports: + - name: grpc + protocol: TCP + port: 9090 + endpoints: + - addresses: + - "10.240.0.10" + conditions: + ready: true diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml index 74a090dc6e8..a383c24095b 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml @@ -110,6 +110,28 @@ infraIR: - file: path: /dev/stdout type: File + - als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + logName: accesslog + type: HTTP + type: ALS + - als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + type: TCP + type: ALS - openTelemetry: host: otel-collector.monitoring.svc.cluster.local port: 4317 @@ -133,6 +155,40 @@ infraIR: xdsIR: envoy-gateway/gateway-1: accessLog: + als: + - destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + weight: 1 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + name: accesslog + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: HTTP + - destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + weight: 1 + name: accesslog/envoy-gateway/gateway-1 + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: TCP openTelemetry: - host: otel-collector.monitoring.svc.cluster.local port: 4317 diff --git a/internal/ir/xds.go b/internal/ir/xds.go index 33d241d2247..930d3cd9434 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -1300,6 +1300,7 @@ type RateLimitValue struct { type AccessLog struct { Text []*TextAccessLog `json:"text,omitempty" yaml:"text,omitempty"` JSON []*JSONAccessLog `json:"json,omitempty" yaml:"json,omitempty"` + ALS []*ALSAccessLog `json:"als,omitempty" yaml:"als,omitempty"` OpenTelemetry []*OpenTelemetryAccessLog `json:"openTelemetry,omitempty" yaml:"openTelemetry,omitempty"` } @@ -1317,6 +1318,25 @@ type JSONAccessLog struct { Path string `json:"path" yaml:"path"` } +// ALSAccessLog holds the configuration for gRPC ALS access logging. +// +k8s:deepcopy-gen=true +type ALSAccessLog struct { + LogName string `json:"name" yaml:"name"` + Destination RouteDestination `json:"destination,omitempty" yaml:"destination,omitempty"` + Type egv1a1.ALSEnvoyProxyAccessLogType `json:"type" yaml:"type"` + Text *string `json:"text,omitempty" yaml:"text,omitempty"` + Attributes map[string]string `json:"attributes,omitempty" yaml:"attributes,omitempty"` + HTTP *ALSAccessLogHTTP `json:"http,omitempty" yaml:"http,omitempty"` +} + +// ALSAccessLogHTTP holds the configuration for HTTP ALS access logging. +// +k8s:deepcopy-gen=true +type ALSAccessLogHTTP struct { + RequestHeaders []string `json:"requestHeaders,omitempty" yaml:"requestHeaders,omitempty"` + ResponseHeaders []string `json:"responseHeaders,omitempty" yaml:"responseHeaders,omitempty"` + ResponseTrailers []string `json:"responseTrailers,omitempty" yaml:"responseTrailers,omitempty"` +} + // OpenTelemetryAccessLog holds the configuration for OpenTelemetry access logging. // +k8s:deepcopy-gen=true type OpenTelemetryAccessLog struct { diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index e957d8b2cc4..3a9ffc24623 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -16,6 +16,69 @@ import ( "sigs.k8s.io/gateway-api/apis/v1alpha2" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ALSAccessLog) DeepCopyInto(out *ALSAccessLog) { + *out = *in + in.Destination.DeepCopyInto(&out.Destination) + if in.Text != nil { + in, out := &in.Text, &out.Text + *out = new(string) + **out = **in + } + if in.Attributes != nil { + in, out := &in.Attributes, &out.Attributes + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(ALSAccessLogHTTP) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ALSAccessLog. +func (in *ALSAccessLog) DeepCopy() *ALSAccessLog { + if in == nil { + return nil + } + out := new(ALSAccessLog) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ALSAccessLogHTTP) DeepCopyInto(out *ALSAccessLogHTTP) { + *out = *in + if in.RequestHeaders != nil { + in, out := &in.RequestHeaders, &out.RequestHeaders + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ResponseHeaders != nil { + in, out := &in.ResponseHeaders, &out.ResponseHeaders + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ResponseTrailers != nil { + in, out := &in.ResponseTrailers, &out.ResponseTrailers + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ALSAccessLogHTTP. +func (in *ALSAccessLogHTTP) DeepCopy() *ALSAccessLogHTTP { + if in == nil { + return nil + } + out := new(ALSAccessLogHTTP) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AccessLog) DeepCopyInto(out *AccessLog) { *out = *in @@ -41,6 +104,17 @@ func (in *AccessLog) DeepCopyInto(out *AccessLog) { } } } + if in.ALS != nil { + in, out := &in.ALS, &out.ALS + *out = make([]*ALSAccessLog, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ALSAccessLog) + (*in).DeepCopyInto(*out) + } + } + } if in.OpenTelemetry != nil { in, out := &in.OpenTelemetry, &out.OpenTelemetry *out = make([]*OpenTelemetryAccessLog, len(*in)) diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index c3ea9daab3d..1c019d7a0b7 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -174,6 +174,11 @@ func (r *gatewayAPIReconciler) Reconcile(ctx context.Context, _ reconcile.Reques return reconcile.Result{}, err } + // Add all EnvoyProxies to the resourceTree + if err = r.processEnvoyProxies(ctx, managedGC, gwcResource, resourceMappings); err != nil { + return reconcile.Result{}, err + } + // Add all EnvoyPatchPolicies to the resourceTree if err = r.processEnvoyPatchPolicies(ctx, gwcResource); err != nil { return reconcile.Result{}, err @@ -219,26 +224,6 @@ func (r *gatewayAPIReconciler) Reconcile(ctx context.Context, _ reconcile.Reques gwcResource.Namespaces = append(gwcResource.Namespaces, namespace) } - // Process the parametersRef of the accepted GatewayClass. - if managedGC.Spec.ParametersRef != nil && managedGC.DeletionTimestamp == nil { - if err := r.processParamsRef(ctx, managedGC, gwcResource); err != nil { - msg := fmt.Sprintf("%s: %v", status.MsgGatewayClassInvalidParams, err) - if err := r.updateStatusForGatewayClass(ctx, managedGC, false, string(gwapiv1.GatewayClassReasonInvalidParameters), msg); err != nil { - r.log.Error(err, "unable to update GatewayClass status") - } - r.log.Error(err, "failed to process parametersRef for gatewayclass", "name", managedGC.Name) - return reconcile.Result{}, err - } - } - - if gwcResource.EnvoyProxy != nil && gwcResource.EnvoyProxy.Spec.MergeGateways != nil { - if *gwcResource.EnvoyProxy.Spec.MergeGateways { - r.mergeGateways.Insert(managedGC.Name) - } else { - r.mergeGateways.Delete(managedGC.Name) - } - } - if err := r.updateStatusForGatewayClass(ctx, managedGC, true, string(gwapiv1.GatewayClassReasonAccepted), status.MsgValidGatewayClass); err != nil { r.log.Error(err, "unable to update GatewayClass status") return reconcile.Result{}, err @@ -774,6 +759,64 @@ func (r *gatewayAPIReconciler) processGateways(ctx context.Context, managedGC *g return nil } +// processEnvoyProxies adds EnvoyProxies to the resourceTree +func (r *gatewayAPIReconciler) processEnvoyProxies(ctx context.Context, managedGC *gwapiv1.GatewayClass, resourceTree *gatewayapi.Resources, resourceMap *resourceMappings) error { + // Process the parametersRef of the accepted GatewayClass. + if managedGC.Spec.ParametersRef != nil && managedGC.DeletionTimestamp == nil { + if err := r.processParamsRef(ctx, managedGC, resourceTree); err != nil { + msg := fmt.Sprintf("%s: %v", status.MsgGatewayClassInvalidParams, err) + if err := r.updateStatusForGatewayClass(ctx, managedGC, false, string(gwapiv1.GatewayClassReasonInvalidParameters), msg); err != nil { + r.log.Error(err, "unable to update GatewayClass status") + } + r.log.Error(err, "failed to process parametersRef for gatewayclass", "name", managedGC.Name) + return err + } + } + + if resourceTree.EnvoyProxy != nil && resourceTree.EnvoyProxy.Spec.MergeGateways != nil { + if *resourceTree.EnvoyProxy.Spec.MergeGateways { + r.mergeGateways.Insert(managedGC.Name) + } else { + r.mergeGateways.Delete(managedGC.Name) + } + } + + // Add the referenced Resources in EnvoyProxies to the resourceTree + r.processEnvoyProxyObjectRefs(resourceTree, resourceMap) + + return nil +} + +// processEnvoyProxyObjectRefs adds the referenced resources in EnvoyProxies +// to the resourceTree +// - BackendRefs for AccessLog +func (r *gatewayAPIReconciler) processEnvoyProxyObjectRefs(resourceTree *gatewayapi.Resources, resourceMap *resourceMappings) { + if resourceTree.EnvoyProxy == nil { + return + } + envoyproxy := resourceTree.EnvoyProxy + + if envoyproxy.Spec.Telemetry != nil && envoyproxy.Spec.Telemetry.AccessLog != nil { + + // Add the referenced BackendRefs in AccessLog to Maps for later processing + accessLog := envoyproxy.Spec.Telemetry.AccessLog + for _, settings := range accessLog.Settings { + for _, sink := range settings.Sinks { + if sink.ALS != nil { + backendRef := sink.ALS.BackendRef + backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, envoyproxy.Namespace) + resourceMap.allAssociatedBackendRefs[gwapiv1.BackendObjectReference{ + Group: backendRef.Group, + Kind: backendRef.Kind, + Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Name: backendRef.Name, + }] = struct{}{} + } + } + } + } +} + // processEnvoyPatchPolicies adds EnvoyPatchPolicies to the resourceTree func (r *gatewayAPIReconciler) processEnvoyPatchPolicies(ctx context.Context, resourceTree *gatewayapi.Resources) error { envoyPatchPolicies := egv1a1.EnvoyPatchPolicyList{} @@ -926,6 +969,9 @@ func (r *gatewayAPIReconciler) watchResources(ctx context.Context, mgr manager.M ); err != nil { return err } + if err := addEnvoyProxyIndexers(ctx, mgr); err != nil { + return err + } // Watch Gateway CRUDs and reconcile affected GatewayClass. gPredicates := []predicate.Predicate{ diff --git a/internal/provider/kubernetes/indexers.go b/internal/provider/kubernetes/indexers.go index 3e638a17aae..40e6a31ee80 100644 --- a/internal/provider/kubernetes/indexers.go +++ b/internal/provider/kubernetes/indexers.go @@ -28,6 +28,7 @@ const ( gatewayUDPRouteIndex = "gatewayUDPRouteIndex" secretGatewayIndex = "secretGatewayIndex" targetRefGrantRouteIndex = "targetRefGrantRouteIndex" + backendEnvoyProxyIndex = "backendEnvoyProxyIndex" backendHTTPRouteIndex = "backendHTTPRouteIndex" backendGRPCRouteIndex = "backendGRPCRouteIndex" backendTLSRouteIndex = "backendTLSRouteIndex" @@ -54,6 +55,40 @@ func addReferenceGrantIndexers(ctx context.Context, mgr manager.Manager) error { return nil } +func addEnvoyProxyIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &v1alpha1.EnvoyProxy{}, backendEnvoyProxyIndex, backendEnvoyProxyIndexFunc); err != nil { + return err + } + return nil +} + +func backendEnvoyProxyIndexFunc(rawObj client.Object) []string { + envoyproxy := rawObj.(*v1alpha1.EnvoyProxy) + var backendRefs []string + + if envoyproxy.Spec.Telemetry != nil && envoyproxy.Spec.Telemetry.AccessLog != nil { + for _, settings := range envoyproxy.Spec.Telemetry.AccessLog.Settings { + for _, sink := range settings.Sinks { + if sink.ALS != nil { + backend := sink.ALS.BackendRef + if backend.Kind == nil || string(*backend.Kind) == gatewayapi.KindService { + // If an explicit Backend namespace is not provided, use the EnvoyProxy namespace to + // lookup the provided Gateway Name. + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gatewayapi.NamespaceDerefOr(backend.Namespace, envoyproxy.Namespace), + Name: string(backend.Name), + }.String(), + ) + } + } + } + } + } + + return backendRefs +} + // addHTTPRouteIndexers adds indexing on HTTPRoute. // - For Service, ServiceImports objects that are referenced in HTTPRoute objects via `.spec.rules.backendRefs`. // This helps in querying for HTTPRoutes that are affected by a particular Service CRUD. diff --git a/internal/provider/kubernetes/predicates.go b/internal/provider/kubernetes/predicates.go index 3585a2913ae..716bce3cae1 100644 --- a/internal/provider/kubernetes/predicates.go +++ b/internal/provider/kubernetes/predicates.go @@ -247,11 +247,29 @@ func (r *gatewayAPIReconciler) validateServiceForReconcile(obj client.Object) bo } nsName := utils.NamespacedName(svc) - if r.isRouteReferencingBackend(&nsName) { + + switch { + case r.isRouteReferencingBackend(&nsName): + return true + case r.isEnvoyProxyReferencingBackend(&nsName): + return true + case r.isSecurityPolicyReferencingBackend(&nsName): return true } - return r.isSecurityPolicyReferencingBackend(&nsName) + return false +} + +func (r *gatewayAPIReconciler) isEnvoyProxyReferencingBackend(nsName *types.NamespacedName) bool { + epList := &egv1a1.EnvoyProxyList{} + if err := r.client.List(context.Background(), epList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(backendEnvoyProxyIndex, nsName.String()), + }); err != nil { + r.log.Error(err, "unable to find associated EnvoyProxy") + return false + } + + return len(epList.Items) > 0 } func (r *gatewayAPIReconciler) isSecurityPolicyReferencingBackend(nsName *types.NamespacedName) bool { @@ -335,7 +353,7 @@ func (r *gatewayAPIReconciler) isRouteReferencingBackend(nsName *types.Namespace } // validateEndpointSliceForReconcile returns true if the endpointSlice references -// a service that is referenced by a xRoute +// a service that is referenced by an xRoute, EnvoyProxy, or SecurityPolicy. func (r *gatewayAPIReconciler) validateEndpointSliceForReconcile(obj client.Object) bool { ep, ok := obj.(*discoveryv1.EndpointSlice) if !ok { @@ -359,11 +377,16 @@ func (r *gatewayAPIReconciler) validateEndpointSliceForReconcile(obj client.Obje nsName.Name = multiClusterSvcName } - if r.isRouteReferencingBackend(&nsName) { + switch { + case r.isRouteReferencingBackend(&nsName): + return true + case r.isEnvoyProxyReferencingBackend(&nsName): + return true + case r.isSecurityPolicyReferencingBackend(&nsName): return true } - return r.isSecurityPolicyReferencingBackend(&nsName) + return false } // validateDeploymentForReconcile tries finding the owning Gateway of the Deployment diff --git a/internal/xds/translator/accesslog.go b/internal/xds/translator/accesslog.go index a74315a255e..8b0c15271cd 100644 --- a/internal/xds/translator/accesslog.go +++ b/internal/xds/translator/accesslog.go @@ -6,8 +6,10 @@ package translator import ( + "encoding/json" "errors" "sort" + "strings" accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" cfgcore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -21,6 +23,7 @@ import ( "google.golang.org/protobuf/types/known/structpb" "k8s.io/utils/ptr" + "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/xds/types" ) @@ -132,6 +135,74 @@ func buildXdsAccessLog(al *ir.AccessLog, forListener bool) []*accesslog.AccessLo }, }) } + // handle ALS access logs + for _, als := range al.ALS { + cc := &grpcaccesslog.CommonGrpcAccessLogConfig{ + LogName: als.LogName, + GrpcService: &cfgcore.GrpcService{ + TargetSpecifier: &cfgcore.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &cfgcore.GrpcService_EnvoyGrpc{ + ClusterName: als.Destination.Name, + }, + }, + }, + TransportApiVersion: cfgcore.ApiVersion_V3, + } + + // include text and json format as metadata when initiating stream + md := make([]*cfgcore.HeaderValue, 0, 2) + + if als.Text != nil && *als.Text != "" { + md = append(md, &cfgcore.HeaderValue{ + Key: "x-accesslog-text", + Value: strings.ReplaceAll(strings.Trim(*als.Text, "\x00\n\r"), "\x00\n\r", " "), + }) + } + + if len(als.Attributes) > 0 { + if attr, err := json.Marshal(als.Attributes); err == nil { + md = append(md, &cfgcore.HeaderValue{ + Key: "x-accesslog-attr", + Value: string(attr), + }) + } + } + + cc.GrpcService.InitialMetadata = md + + switch als.Type { + case v1alpha1.ALSEnvoyProxyAccessLogTypeHTTP: + al := &grpcaccesslog.HttpGrpcAccessLogConfig{ + CommonConfig: cc, + } + + if als.HTTP != nil { + al.AdditionalRequestHeadersToLog = als.HTTP.RequestHeaders + al.AdditionalResponseHeadersToLog = als.HTTP.ResponseHeaders + al.AdditionalResponseTrailersToLog = als.HTTP.ResponseTrailers + } + + accesslogAny, _ := anypb.New(al) + accessLogs = append(accessLogs, &accesslog.AccessLog{ + Name: "envoy.access_loggers.http_grpc", + ConfigType: &accesslog.AccessLog_TypedConfig{ + TypedConfig: accesslogAny, + }, + }) + case v1alpha1.ALSEnvoyProxyAccessLogTypeTCP: + al := &grpcaccesslog.TcpGrpcAccessLogConfig{ + CommonConfig: cc, + } + + accesslogAny, _ := anypb.New(al) + accessLogs = append(accessLogs, &accesslog.AccessLog{ + Name: "envoy.access_loggers.tcp_grpc", + ConfigType: &accesslog.AccessLog_TypedConfig{ + TypedConfig: accesslogAny, + }, + }) + } + } // handle open telemetry access logs for _, otel := range al.OpenTelemetry { al := &otelaccesslog.OpenTelemetryAccessLogConfig{ @@ -238,6 +309,19 @@ func processClusterForAccessLog(tCtx *types.ResourceVersionTable, al *ir.AccessL return nil } + // add clusters for ALS access logs + for _, als := range al.ALS { + if err := addXdsCluster(tCtx, &xdsClusterArgs{ + name: als.Destination.Name, + settings: als.Destination.Settings, + tSocket: nil, + endpointType: EndpointTypeStatic, + }); err != nil && !errors.Is(err, ErrXdsClusterExists) { + return err + } + } + + // add clusters for Open Telemetry access logs for _, otel := range al.OpenTelemetry { clusterName := buildClusterName("accesslog", otel.Host, otel.Port) diff --git a/internal/xds/translator/testdata/in/xds-ir/accesslog-als-tcp.yaml b/internal/xds/translator/testdata/in/xds-ir/accesslog-als-tcp.yaml new file mode 100644 index 00000000000..2d8f0c6aa48 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/accesslog-als-tcp.yaml @@ -0,0 +1,23 @@ +name: "accesslog" +accesslog: + json: + - json: + start_time: "%START_TIME%" + method: "%REQ(:METHOD)%" + path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + response_code: "%RESPONSE_CODE%" + als: + - destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 1.1.1.1 + port: 9000 + protocol: GRPC + weight: 1 + attributes: + attr1: value1 + attr2: value2 + type: TCP diff --git a/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml b/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml index f4ec43ac072..b96e79c8d24 100644 --- a/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml @@ -12,6 +12,29 @@ accesslog: path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" protocol: "%PROTOCOL%" response_code: "%RESPONSE_CODE%" + als: + - destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 1.1.1.1 + port: 9000 + protocol: GRPC + weight: 1 + text: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + attributes: + attr1: value1 + attr2: value2 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + type: HTTP openTelemetry: - text: | [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.clusters.yaml new file mode 100755 index 00000000000..18b309bb74d --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.clusters.yaml @@ -0,0 +1,22 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: accesslog/monitoring/envoy-als/port/9000 + lbPolicy: LEAST_REQUEST + name: accesslog/monitoring/envoy-als/port/9000 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.endpoints.yaml new file mode 100755 index 00000000000..9159d4e31f2 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: accesslog/monitoring/envoy-als/port/9000 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.1.1.1 + portValue: 9000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: accesslog/monitoring/envoy-als/port/9000/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.listeners.yaml new file mode 100755 index 00000000000..fe51488c706 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.listeners.yaml @@ -0,0 +1 @@ +[] diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.routes.yaml new file mode 100755 index 00000000000..fe51488c706 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.routes.yaml @@ -0,0 +1 @@ +[] diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml index 7168156486d..f76b1efc1d5 100644 --- a/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml @@ -15,6 +15,28 @@ outlierDetection: {} perConnectionBufferLimitBytes: 32768 type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: accesslog/monitoring/envoy-als/port/9000 + lbPolicy: LEAST_REQUEST + name: accesslog/monitoring/envoy-als/port/9000 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} - circuitBreakers: thresholds: - maxRetries: 1024 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml index 20c80b3aaaa..2ce2e6da08c 100644 --- a/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml @@ -10,3 +10,15 @@ loadBalancingWeight: 1 locality: region: direct-route-dest/backend/0 +- clusterName: accesslog/monitoring/envoy-als/port/9000 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.1.1.1 + portValue: 9000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: accesslog/monitoring/envoy-als/port/9000/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml index e6d5535eb15..0a7a5431abb 100644 --- a/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml @@ -26,6 +26,32 @@ response_code: '%RESPONSE_CODE%' start_time: '%START_TIME%' path: /dev/stdout + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.http_grpc + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig + additionalRequestHeadersToLog: + - x-client-ip-address + additionalResponseHeadersToLog: + - cache-control + additionalResponseTrailersToLog: + - expires + commonConfig: + grpcService: + envoyGrpc: + clusterName: accesslog/monitoring/envoy-als/port/9000 + initialMetadata: + - key: x-accesslog-text + value: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% + %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" + "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"' + - key: x-accesslog-attr + value: '{"attr1":"value1","attr2":"value2"}' + transportApiVersion: V3 - filter: responseFlagFilter: flags: @@ -88,6 +114,29 @@ response_code: '%RESPONSE_CODE%' start_time: '%START_TIME%' path: /dev/stdout + - name: envoy.access_loggers.http_grpc + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig + additionalRequestHeadersToLog: + - x-client-ip-address + additionalResponseHeadersToLog: + - cache-control + additionalResponseTrailersToLog: + - expires + commonConfig: + grpcService: + envoyGrpc: + clusterName: accesslog/monitoring/envoy-als/port/9000 + initialMetadata: + - key: x-accesslog-text + value: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% + %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% + "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" + "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"' + - key: x-accesslog-attr + value: '{"attr1":"value1","attr2":"value2"}' + transportApiVersion: V3 - name: envoy.access_loggers.open_telemetry typedConfig: '@type': type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index 88030fc15c9..06974c63c45 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -172,6 +172,9 @@ func TestTranslateXds(t *testing.T) { { name: "ratelimit-sourceip", }, + { + name: "accesslog-als-tcp", + }, { name: "accesslog", },