diff --git a/examples/kubernetes/accesslog/als-accesslog.yaml b/examples/kubernetes/accesslog/als-accesslog.yaml new file mode 100644 index 000000000000..1b28389d7d3b --- /dev/null +++ b/examples/kubernetes/accesslog/als-accesslog.yaml @@ -0,0 +1,39 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: als-access-logging + namespace: envoy-gateway-system +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: als-access-logging + namespace: envoy-gateway-system +spec: + telemetry: + accessLog: + settings: + - format: + type: JSON + json: + attr1: val1 + attr2: val2 + sinks: + - type: ALS + als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + type: HTTP diff --git a/examples/kubernetes/accesslog/multi-sinks.yaml b/examples/kubernetes/accesslog/multi-sinks.yaml index ae2cf3b55d5e..4048cb4983cf 100644 --- a/examples/kubernetes/accesslog/multi-sinks.yaml +++ b/examples/kubernetes/accesslog/multi-sinks.yaml @@ -27,6 +27,13 @@ spec: - type: File file: path: /dev/stdout + - type: ALS + als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + type: HTTP - type: OpenTelemetry openTelemetry: host: otel-collector.monitoring.svc.cluster.local diff --git a/internal/gatewayapi/listener.go b/internal/gatewayapi/listener.go index 4702d2a4e26f..3a27cba68981 100644 --- a/internal/gatewayapi/listener.go +++ b/internal/gatewayapi/listener.go @@ -8,7 +8,9 @@ package gatewayapi import ( "fmt" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" @@ -43,7 +45,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap infraIR[irKey].Proxy.Config = resources.EnvoyProxy } - xdsIR[irKey].AccessLog = processAccessLog(infraIR[irKey].Proxy.Config) + xdsIR[irKey].AccessLog = t.processAccessLog(infraIR[irKey].Proxy.Config, gateway.Gateway, resources) xdsIR[irKey].Tracing = processTracing(gateway.Gateway, infraIR[irKey].Proxy.Config) xdsIR[irKey].Metrics = processMetrics(infraIR[irKey].Proxy.Config) @@ -160,7 +162,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap } } -func processAccessLog(envoyproxy *egv1a1.EnvoyProxy) *ir.AccessLog { +func (t *Translator) processAccessLog(envoyproxy *egv1a1.EnvoyProxy, gw *gwapiv1.Gateway, resources *Resources) *ir.AccessLog { if envoyproxy == nil || envoyproxy.Spec.Telemetry == nil || envoyproxy.Spec.Telemetry.AccessLog == nil || @@ -208,6 +210,52 @@ func processAccessLog(envoyproxy *egv1a1.EnvoyProxy) *ir.AccessLog { } irAccessLog.JSON = append(irAccessLog.JSON, al) } + case egv1a1.ProxyAccessLogSinkTypeALS: + if sink.ALS == nil { + continue + } + + var logName string + if sink.ALS.LogName != nil { + logName = *sink.ALS.LogName + } else { + logName = fmt.Sprintf("accesslog/%s/%s", gw.Namespace, gw.Name) + } + + clusterName := fmt.Sprintf("accesslog/%s/%s/port/%d", + NamespaceDerefOr(sink.ALS.BackendRef.Namespace, envoyproxy.Namespace), + string(sink.ALS.BackendRef.Name), + *sink.ALS.BackendRef.Port, + ) + + al := &ir.ALSAccessLog{ + LogName: logName, + Destination: ir.RouteDestination{ + Name: clusterName, + Settings: []*ir.DestinationSetting{ + t.processServiceDestination(sink.ALS.BackendRef, ir.GRPC, envoyproxy, resources), + }, + }, + Type: sink.ALS.Type, + } + + if al.Type == egv1a1.ALSEnvoyProxyAccessLogTypeHTTP { + http := &ir.ALSAccessLogHTTP{ + RequestHeaders: sink.ALS.HTTP.RequestHeaders, + ResponseHeaders: sink.ALS.HTTP.ResponseHeaders, + ResponseTrailers: sink.ALS.HTTP.ResponseTrailers, + } + al.HTTP = http + } + + switch accessLog.Format.Type { + case egv1a1.ProxyAccessLogFormatTypeJSON: + al.Attributes = accessLog.Format.JSON + case egv1a1.ProxyAccessLogFormatTypeText: + al.Text = accessLog.Format.Text + } + + irAccessLog.ALS = append(irAccessLog.ALS, al) case egv1a1.ProxyAccessLogSinkTypeOpenTelemetry: if sink.OpenTelemetry == nil { continue @@ -234,6 +282,47 @@ func processAccessLog(envoyproxy *egv1a1.EnvoyProxy) *ir.AccessLog { return irAccessLog } +func (t *Translator) processServiceDestination(backendRef gwapiv1.BackendObjectReference, protocol ir.AppProtocol, envoyproxy *egv1a1.EnvoyProxy, resources *Resources) *ir.DestinationSetting { + var ( + endpoints []*ir.DestinationEndpoint + addrType *ir.DestinationAddressType + servicePort v1.ServicePort + backendTLS *ir.TLSUpstreamConfig + ) + + serviceNamespace := NamespaceDerefOr(backendRef.Namespace, envoyproxy.Namespace) + service := resources.GetService(serviceNamespace, string(backendRef.Name)) + for _, port := range service.Spec.Ports { + if port.Port == int32(*backendRef.Port) { + servicePort = port + break + } + } + + if servicePort.AppProtocol != nil && + *servicePort.AppProtocol == "kubernetes.io/h2c" { + protocol = ir.HTTP2 + } + + // Route to endpoints by default + if !t.EndpointRoutingDisabled { + endpointSlices := resources.GetEndpointSlicesForBackend(serviceNamespace, string(backendRef.Name), KindDerefOr(backendRef.Kind, KindService)) + endpoints, addrType = getIREndpointsFromEndpointSlices(endpointSlices, servicePort.Name, servicePort.Protocol) + } else { + // Fall back to Service ClusterIP routing + ep := ir.NewDestEndpoint(service.Spec.ClusterIP, uint32(*backendRef.Port)) + endpoints = append(endpoints, ep) + } + + return &ir.DestinationSetting{ + Weight: ptr.To(uint32(1)), + Protocol: protocol, + Endpoints: endpoints, + AddressType: addrType, + TLS: backendTLS, + } +} + func processTracing(gw *gwapiv1.Gateway, envoyproxy *egv1a1.EnvoyProxy) *ir.Tracing { if envoyproxy == nil || envoyproxy.Spec.Telemetry == nil || diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.in.yaml new file mode 100644 index 000000000000..cb7c82b4afe4 --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.in.yaml @@ -0,0 +1,132 @@ +envoyproxy: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + namespace: envoy-gateway-system + name: test + spec: + telemetry: + accessLog: + settings: + - format: + type: JSON + json: + attr1: val1 + attr2: val2 + sinks: + - type: ALS + als: + logName: accesslog + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + type: HTTP + - type: ALS + als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + type: TCP + provider: + type: Kubernetes + kubernetes: + envoyService: + type: LoadBalancer + envoyDeployment: + replicas: 2 + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: "envoyproxy/envoy:distroless-dev" + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + runAsUser: 2000 + allowPrivilegeEscalation: false + pod: + annotations: + key1: val1 + key2: val2 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: "router" + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + fsGroupChangePolicy: "OnRootMismatch" + volumes: + - name: certs + secret: + secretName: envoy-cert +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same +services: +- apiVersion: v1 + kind: Service + metadata: + name: envoy-als + namespace: monitoring + spec: + type: ClusterIP + ports: + - name: grpc + port: 9000 + protocol: TCP + targetPort: 9000 +endpointSlices: +- apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: endpointslice-envoy-als + namespace: monitoring + labels: + kubernetes.io/service-name: envoy-als + addressType: IPv4 + ports: + - name: grpc + protocol: TCP + port: 9090 + endpoints: + - addresses: + - "10.240.0.10" + conditions: + ready: true diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.out.yaml new file mode 100755 index 000000000000..2ab9d534a6f8 --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.out.yaml @@ -0,0 +1,195 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: Same + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +infraIR: + envoy-gateway/gateway-1: + proxy: + config: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + creationTimestamp: null + name: test + namespace: envoy-gateway-system + spec: + logging: {} + provider: + kubernetes: + envoyDeployment: + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: envoyproxy/envoy:distroless-dev + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + runAsUser: 2000 + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + annotations: + key1: val1 + key2: val2 + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 3000 + runAsUser: 1000 + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: router + volumes: + - name: certs + secret: + secretName: envoy-cert + replicas: 2 + envoyService: + type: LoadBalancer + type: Kubernetes + telemetry: + accessLog: + settings: + - format: + json: + attr1: val1 + attr2: val2 + type: JSON + sinks: + - als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + logName: accesslog + type: HTTP + type: ALS + - als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + type: TCP + type: ALS + status: {} + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + als: + - attributes: + attr1: val1 + attr2: val2 + destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + weight: 1 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + name: accesslog + type: HTTP + - attributes: + attr1: val1 + attr2: val2 + destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + weight: 1 + name: accesslog/envoy-gateway/gateway-1 + type: TCP + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml index cccabc9cd4de..58277d8a3b52 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml @@ -14,6 +14,7 @@ envoyproxy: [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n sinks: - type: File + - type: ALS - type: OpenTelemetry provider: type: Kubernetes diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml index 583ce720c9af..2d9ec30b5c52 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml @@ -108,6 +108,7 @@ infraIR: type: Text sinks: - type: File + - type: ALS - type: OpenTelemetry status: {} listeners: diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml index 0d6914205760..95f003afff3e 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml @@ -16,6 +16,28 @@ envoyproxy: - type: File file: path: /dev/stdout + - type: ALS + als: + logName: accesslog + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + type: HTTP + - type: ALS + als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + type: TCP - type: OpenTelemetry openTelemetry: host: otel-collector.monitoring.svc.cluster.local @@ -85,3 +107,34 @@ gateways: allowedRoutes: namespaces: from: Same +services: +- apiVersion: v1 + kind: Service + metadata: + name: envoy-als + namespace: monitoring + spec: + type: ClusterIP + ports: + - name: grpc + port: 9000 + protocol: TCP + targetPort: 9000 +endpointSlices: +- apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: endpointslice-envoy-als + namespace: monitoring + labels: + kubernetes.io/service-name: envoy-als + addressType: IPv4 + ports: + - name: grpc + protocol: TCP + port: 9090 + endpoints: + - addresses: + - "10.240.0.10" + conditions: + ready: true diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml index 74a090dc6e87..a383c24095bc 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml @@ -110,6 +110,28 @@ infraIR: - file: path: /dev/stdout type: File + - als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + logName: accesslog + type: HTTP + type: ALS + - als: + backendRef: + name: envoy-als + namespace: monitoring + port: 9000 + type: TCP + type: ALS - openTelemetry: host: otel-collector.monitoring.svc.cluster.local port: 4317 @@ -133,6 +155,40 @@ infraIR: xdsIR: envoy-gateway/gateway-1: accessLog: + als: + - destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + weight: 1 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + name: accesslog + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: HTTP + - destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + weight: 1 + name: accesslog/envoy-gateway/gateway-1 + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: TCP openTelemetry: - host: otel-collector.monitoring.svc.cluster.local port: 4317 diff --git a/internal/ir/xds.go b/internal/ir/xds.go index 33d241d2247c..930d3cd94346 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -1300,6 +1300,7 @@ type RateLimitValue struct { type AccessLog struct { Text []*TextAccessLog `json:"text,omitempty" yaml:"text,omitempty"` JSON []*JSONAccessLog `json:"json,omitempty" yaml:"json,omitempty"` + ALS []*ALSAccessLog `json:"als,omitempty" yaml:"als,omitempty"` OpenTelemetry []*OpenTelemetryAccessLog `json:"openTelemetry,omitempty" yaml:"openTelemetry,omitempty"` } @@ -1317,6 +1318,25 @@ type JSONAccessLog struct { Path string `json:"path" yaml:"path"` } +// ALSAccessLog holds the configuration for gRPC ALS access logging. +// +k8s:deepcopy-gen=true +type ALSAccessLog struct { + LogName string `json:"name" yaml:"name"` + Destination RouteDestination `json:"destination,omitempty" yaml:"destination,omitempty"` + Type egv1a1.ALSEnvoyProxyAccessLogType `json:"type" yaml:"type"` + Text *string `json:"text,omitempty" yaml:"text,omitempty"` + Attributes map[string]string `json:"attributes,omitempty" yaml:"attributes,omitempty"` + HTTP *ALSAccessLogHTTP `json:"http,omitempty" yaml:"http,omitempty"` +} + +// ALSAccessLogHTTP holds the configuration for HTTP ALS access logging. +// +k8s:deepcopy-gen=true +type ALSAccessLogHTTP struct { + RequestHeaders []string `json:"requestHeaders,omitempty" yaml:"requestHeaders,omitempty"` + ResponseHeaders []string `json:"responseHeaders,omitempty" yaml:"responseHeaders,omitempty"` + ResponseTrailers []string `json:"responseTrailers,omitempty" yaml:"responseTrailers,omitempty"` +} + // OpenTelemetryAccessLog holds the configuration for OpenTelemetry access logging. // +k8s:deepcopy-gen=true type OpenTelemetryAccessLog struct { diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index e957d8b2cc4c..3a9ffc246234 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -16,6 +16,69 @@ import ( "sigs.k8s.io/gateway-api/apis/v1alpha2" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ALSAccessLog) DeepCopyInto(out *ALSAccessLog) { + *out = *in + in.Destination.DeepCopyInto(&out.Destination) + if in.Text != nil { + in, out := &in.Text, &out.Text + *out = new(string) + **out = **in + } + if in.Attributes != nil { + in, out := &in.Attributes, &out.Attributes + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(ALSAccessLogHTTP) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ALSAccessLog. +func (in *ALSAccessLog) DeepCopy() *ALSAccessLog { + if in == nil { + return nil + } + out := new(ALSAccessLog) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ALSAccessLogHTTP) DeepCopyInto(out *ALSAccessLogHTTP) { + *out = *in + if in.RequestHeaders != nil { + in, out := &in.RequestHeaders, &out.RequestHeaders + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ResponseHeaders != nil { + in, out := &in.ResponseHeaders, &out.ResponseHeaders + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ResponseTrailers != nil { + in, out := &in.ResponseTrailers, &out.ResponseTrailers + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ALSAccessLogHTTP. +func (in *ALSAccessLogHTTP) DeepCopy() *ALSAccessLogHTTP { + if in == nil { + return nil + } + out := new(ALSAccessLogHTTP) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AccessLog) DeepCopyInto(out *AccessLog) { *out = *in @@ -41,6 +104,17 @@ func (in *AccessLog) DeepCopyInto(out *AccessLog) { } } } + if in.ALS != nil { + in, out := &in.ALS, &out.ALS + *out = make([]*ALSAccessLog, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ALSAccessLog) + (*in).DeepCopyInto(*out) + } + } + } if in.OpenTelemetry != nil { in, out := &in.OpenTelemetry, &out.OpenTelemetry *out = make([]*OpenTelemetryAccessLog, len(*in)) diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index c3ea9daab3d9..1c019d7a0b7d 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -174,6 +174,11 @@ func (r *gatewayAPIReconciler) Reconcile(ctx context.Context, _ reconcile.Reques return reconcile.Result{}, err } + // Add all EnvoyProxies to the resourceTree + if err = r.processEnvoyProxies(ctx, managedGC, gwcResource, resourceMappings); err != nil { + return reconcile.Result{}, err + } + // Add all EnvoyPatchPolicies to the resourceTree if err = r.processEnvoyPatchPolicies(ctx, gwcResource); err != nil { return reconcile.Result{}, err @@ -219,26 +224,6 @@ func (r *gatewayAPIReconciler) Reconcile(ctx context.Context, _ reconcile.Reques gwcResource.Namespaces = append(gwcResource.Namespaces, namespace) } - // Process the parametersRef of the accepted GatewayClass. - if managedGC.Spec.ParametersRef != nil && managedGC.DeletionTimestamp == nil { - if err := r.processParamsRef(ctx, managedGC, gwcResource); err != nil { - msg := fmt.Sprintf("%s: %v", status.MsgGatewayClassInvalidParams, err) - if err := r.updateStatusForGatewayClass(ctx, managedGC, false, string(gwapiv1.GatewayClassReasonInvalidParameters), msg); err != nil { - r.log.Error(err, "unable to update GatewayClass status") - } - r.log.Error(err, "failed to process parametersRef for gatewayclass", "name", managedGC.Name) - return reconcile.Result{}, err - } - } - - if gwcResource.EnvoyProxy != nil && gwcResource.EnvoyProxy.Spec.MergeGateways != nil { - if *gwcResource.EnvoyProxy.Spec.MergeGateways { - r.mergeGateways.Insert(managedGC.Name) - } else { - r.mergeGateways.Delete(managedGC.Name) - } - } - if err := r.updateStatusForGatewayClass(ctx, managedGC, true, string(gwapiv1.GatewayClassReasonAccepted), status.MsgValidGatewayClass); err != nil { r.log.Error(err, "unable to update GatewayClass status") return reconcile.Result{}, err @@ -774,6 +759,64 @@ func (r *gatewayAPIReconciler) processGateways(ctx context.Context, managedGC *g return nil } +// processEnvoyProxies adds EnvoyProxies to the resourceTree +func (r *gatewayAPIReconciler) processEnvoyProxies(ctx context.Context, managedGC *gwapiv1.GatewayClass, resourceTree *gatewayapi.Resources, resourceMap *resourceMappings) error { + // Process the parametersRef of the accepted GatewayClass. + if managedGC.Spec.ParametersRef != nil && managedGC.DeletionTimestamp == nil { + if err := r.processParamsRef(ctx, managedGC, resourceTree); err != nil { + msg := fmt.Sprintf("%s: %v", status.MsgGatewayClassInvalidParams, err) + if err := r.updateStatusForGatewayClass(ctx, managedGC, false, string(gwapiv1.GatewayClassReasonInvalidParameters), msg); err != nil { + r.log.Error(err, "unable to update GatewayClass status") + } + r.log.Error(err, "failed to process parametersRef for gatewayclass", "name", managedGC.Name) + return err + } + } + + if resourceTree.EnvoyProxy != nil && resourceTree.EnvoyProxy.Spec.MergeGateways != nil { + if *resourceTree.EnvoyProxy.Spec.MergeGateways { + r.mergeGateways.Insert(managedGC.Name) + } else { + r.mergeGateways.Delete(managedGC.Name) + } + } + + // Add the referenced Resources in EnvoyProxies to the resourceTree + r.processEnvoyProxyObjectRefs(resourceTree, resourceMap) + + return nil +} + +// processEnvoyProxyObjectRefs adds the referenced resources in EnvoyProxies +// to the resourceTree +// - BackendRefs for AccessLog +func (r *gatewayAPIReconciler) processEnvoyProxyObjectRefs(resourceTree *gatewayapi.Resources, resourceMap *resourceMappings) { + if resourceTree.EnvoyProxy == nil { + return + } + envoyproxy := resourceTree.EnvoyProxy + + if envoyproxy.Spec.Telemetry != nil && envoyproxy.Spec.Telemetry.AccessLog != nil { + + // Add the referenced BackendRefs in AccessLog to Maps for later processing + accessLog := envoyproxy.Spec.Telemetry.AccessLog + for _, settings := range accessLog.Settings { + for _, sink := range settings.Sinks { + if sink.ALS != nil { + backendRef := sink.ALS.BackendRef + backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, envoyproxy.Namespace) + resourceMap.allAssociatedBackendRefs[gwapiv1.BackendObjectReference{ + Group: backendRef.Group, + Kind: backendRef.Kind, + Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Name: backendRef.Name, + }] = struct{}{} + } + } + } + } +} + // processEnvoyPatchPolicies adds EnvoyPatchPolicies to the resourceTree func (r *gatewayAPIReconciler) processEnvoyPatchPolicies(ctx context.Context, resourceTree *gatewayapi.Resources) error { envoyPatchPolicies := egv1a1.EnvoyPatchPolicyList{} @@ -926,6 +969,9 @@ func (r *gatewayAPIReconciler) watchResources(ctx context.Context, mgr manager.M ); err != nil { return err } + if err := addEnvoyProxyIndexers(ctx, mgr); err != nil { + return err + } // Watch Gateway CRUDs and reconcile affected GatewayClass. gPredicates := []predicate.Predicate{ diff --git a/internal/provider/kubernetes/indexers.go b/internal/provider/kubernetes/indexers.go index 3e638a17aae8..40e6a31ee803 100644 --- a/internal/provider/kubernetes/indexers.go +++ b/internal/provider/kubernetes/indexers.go @@ -28,6 +28,7 @@ const ( gatewayUDPRouteIndex = "gatewayUDPRouteIndex" secretGatewayIndex = "secretGatewayIndex" targetRefGrantRouteIndex = "targetRefGrantRouteIndex" + backendEnvoyProxyIndex = "backendEnvoyProxyIndex" backendHTTPRouteIndex = "backendHTTPRouteIndex" backendGRPCRouteIndex = "backendGRPCRouteIndex" backendTLSRouteIndex = "backendTLSRouteIndex" @@ -54,6 +55,40 @@ func addReferenceGrantIndexers(ctx context.Context, mgr manager.Manager) error { return nil } +func addEnvoyProxyIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &v1alpha1.EnvoyProxy{}, backendEnvoyProxyIndex, backendEnvoyProxyIndexFunc); err != nil { + return err + } + return nil +} + +func backendEnvoyProxyIndexFunc(rawObj client.Object) []string { + envoyproxy := rawObj.(*v1alpha1.EnvoyProxy) + var backendRefs []string + + if envoyproxy.Spec.Telemetry != nil && envoyproxy.Spec.Telemetry.AccessLog != nil { + for _, settings := range envoyproxy.Spec.Telemetry.AccessLog.Settings { + for _, sink := range settings.Sinks { + if sink.ALS != nil { + backend := sink.ALS.BackendRef + if backend.Kind == nil || string(*backend.Kind) == gatewayapi.KindService { + // If an explicit Backend namespace is not provided, use the EnvoyProxy namespace to + // lookup the provided Gateway Name. + backendRefs = append(backendRefs, + types.NamespacedName{ + Namespace: gatewayapi.NamespaceDerefOr(backend.Namespace, envoyproxy.Namespace), + Name: string(backend.Name), + }.String(), + ) + } + } + } + } + } + + return backendRefs +} + // addHTTPRouteIndexers adds indexing on HTTPRoute. // - For Service, ServiceImports objects that are referenced in HTTPRoute objects via `.spec.rules.backendRefs`. // This helps in querying for HTTPRoutes that are affected by a particular Service CRUD. diff --git a/internal/provider/kubernetes/predicates.go b/internal/provider/kubernetes/predicates.go index 3585a2913aeb..716bce3cae10 100644 --- a/internal/provider/kubernetes/predicates.go +++ b/internal/provider/kubernetes/predicates.go @@ -247,11 +247,29 @@ func (r *gatewayAPIReconciler) validateServiceForReconcile(obj client.Object) bo } nsName := utils.NamespacedName(svc) - if r.isRouteReferencingBackend(&nsName) { + + switch { + case r.isRouteReferencingBackend(&nsName): + return true + case r.isEnvoyProxyReferencingBackend(&nsName): + return true + case r.isSecurityPolicyReferencingBackend(&nsName): return true } - return r.isSecurityPolicyReferencingBackend(&nsName) + return false +} + +func (r *gatewayAPIReconciler) isEnvoyProxyReferencingBackend(nsName *types.NamespacedName) bool { + epList := &egv1a1.EnvoyProxyList{} + if err := r.client.List(context.Background(), epList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(backendEnvoyProxyIndex, nsName.String()), + }); err != nil { + r.log.Error(err, "unable to find associated EnvoyProxy") + return false + } + + return len(epList.Items) > 0 } func (r *gatewayAPIReconciler) isSecurityPolicyReferencingBackend(nsName *types.NamespacedName) bool { @@ -335,7 +353,7 @@ func (r *gatewayAPIReconciler) isRouteReferencingBackend(nsName *types.Namespace } // validateEndpointSliceForReconcile returns true if the endpointSlice references -// a service that is referenced by a xRoute +// a service that is referenced by an xRoute, EnvoyProxy, or SecurityPolicy. func (r *gatewayAPIReconciler) validateEndpointSliceForReconcile(obj client.Object) bool { ep, ok := obj.(*discoveryv1.EndpointSlice) if !ok { @@ -359,11 +377,16 @@ func (r *gatewayAPIReconciler) validateEndpointSliceForReconcile(obj client.Obje nsName.Name = multiClusterSvcName } - if r.isRouteReferencingBackend(&nsName) { + switch { + case r.isRouteReferencingBackend(&nsName): + return true + case r.isEnvoyProxyReferencingBackend(&nsName): + return true + case r.isSecurityPolicyReferencingBackend(&nsName): return true } - return r.isSecurityPolicyReferencingBackend(&nsName) + return false } // validateDeploymentForReconcile tries finding the owning Gateway of the Deployment diff --git a/internal/xds/translator/accesslog.go b/internal/xds/translator/accesslog.go index a74315a255e9..8b0c15271cdb 100644 --- a/internal/xds/translator/accesslog.go +++ b/internal/xds/translator/accesslog.go @@ -6,8 +6,10 @@ package translator import ( + "encoding/json" "errors" "sort" + "strings" accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" cfgcore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -21,6 +23,7 @@ import ( "google.golang.org/protobuf/types/known/structpb" "k8s.io/utils/ptr" + "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/xds/types" ) @@ -132,6 +135,74 @@ func buildXdsAccessLog(al *ir.AccessLog, forListener bool) []*accesslog.AccessLo }, }) } + // handle ALS access logs + for _, als := range al.ALS { + cc := &grpcaccesslog.CommonGrpcAccessLogConfig{ + LogName: als.LogName, + GrpcService: &cfgcore.GrpcService{ + TargetSpecifier: &cfgcore.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &cfgcore.GrpcService_EnvoyGrpc{ + ClusterName: als.Destination.Name, + }, + }, + }, + TransportApiVersion: cfgcore.ApiVersion_V3, + } + + // include text and json format as metadata when initiating stream + md := make([]*cfgcore.HeaderValue, 0, 2) + + if als.Text != nil && *als.Text != "" { + md = append(md, &cfgcore.HeaderValue{ + Key: "x-accesslog-text", + Value: strings.ReplaceAll(strings.Trim(*als.Text, "\x00\n\r"), "\x00\n\r", " "), + }) + } + + if len(als.Attributes) > 0 { + if attr, err := json.Marshal(als.Attributes); err == nil { + md = append(md, &cfgcore.HeaderValue{ + Key: "x-accesslog-attr", + Value: string(attr), + }) + } + } + + cc.GrpcService.InitialMetadata = md + + switch als.Type { + case v1alpha1.ALSEnvoyProxyAccessLogTypeHTTP: + al := &grpcaccesslog.HttpGrpcAccessLogConfig{ + CommonConfig: cc, + } + + if als.HTTP != nil { + al.AdditionalRequestHeadersToLog = als.HTTP.RequestHeaders + al.AdditionalResponseHeadersToLog = als.HTTP.ResponseHeaders + al.AdditionalResponseTrailersToLog = als.HTTP.ResponseTrailers + } + + accesslogAny, _ := anypb.New(al) + accessLogs = append(accessLogs, &accesslog.AccessLog{ + Name: "envoy.access_loggers.http_grpc", + ConfigType: &accesslog.AccessLog_TypedConfig{ + TypedConfig: accesslogAny, + }, + }) + case v1alpha1.ALSEnvoyProxyAccessLogTypeTCP: + al := &grpcaccesslog.TcpGrpcAccessLogConfig{ + CommonConfig: cc, + } + + accesslogAny, _ := anypb.New(al) + accessLogs = append(accessLogs, &accesslog.AccessLog{ + Name: "envoy.access_loggers.tcp_grpc", + ConfigType: &accesslog.AccessLog_TypedConfig{ + TypedConfig: accesslogAny, + }, + }) + } + } // handle open telemetry access logs for _, otel := range al.OpenTelemetry { al := &otelaccesslog.OpenTelemetryAccessLogConfig{ @@ -238,6 +309,19 @@ func processClusterForAccessLog(tCtx *types.ResourceVersionTable, al *ir.AccessL return nil } + // add clusters for ALS access logs + for _, als := range al.ALS { + if err := addXdsCluster(tCtx, &xdsClusterArgs{ + name: als.Destination.Name, + settings: als.Destination.Settings, + tSocket: nil, + endpointType: EndpointTypeStatic, + }); err != nil && !errors.Is(err, ErrXdsClusterExists) { + return err + } + } + + // add clusters for Open Telemetry access logs for _, otel := range al.OpenTelemetry { clusterName := buildClusterName("accesslog", otel.Host, otel.Port) diff --git a/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml b/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml index f4ec43ac0721..d560e389de97 100644 --- a/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml @@ -12,6 +12,29 @@ accesslog: path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" protocol: "%PROTOCOL%" response_code: "%RESPONSE_CODE%" + als: + - destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 1.1.1.1 + port: 9000 + protocol: HTTP + weight: 1 + text: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + attributes: + attr1: value1 + attr2: value2 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + type: HTTP openTelemetry: - text: | [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml index 7168156486d7..c5cb65f0a942 100644 --- a/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml @@ -15,6 +15,23 @@ outlierDetection: {} perConnectionBufferLimitBytes: 32768 type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: accesslog/monitoring/envoy-als/port/9000 + lbPolicy: LEAST_REQUEST + name: accesslog/monitoring/envoy-als/port/9000 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS - circuitBreakers: thresholds: - maxRetries: 1024 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml index 20c80b3aaaa8..2ce2e6da08c7 100644 --- a/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml @@ -10,3 +10,15 @@ loadBalancingWeight: 1 locality: region: direct-route-dest/backend/0 +- clusterName: accesslog/monitoring/envoy-als/port/9000 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.1.1.1 + portValue: 9000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: accesslog/monitoring/envoy-als/port/9000/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml index e6d5535eb15b..0a7a5431abb7 100644 --- a/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml @@ -26,6 +26,32 @@ response_code: '%RESPONSE_CODE%' start_time: '%START_TIME%' path: /dev/stdout + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.http_grpc + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig + additionalRequestHeadersToLog: + - x-client-ip-address + additionalResponseHeadersToLog: + - cache-control + additionalResponseTrailersToLog: + - expires + commonConfig: + grpcService: + envoyGrpc: + clusterName: accesslog/monitoring/envoy-als/port/9000 + initialMetadata: + - key: x-accesslog-text + value: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% + %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" + "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"' + - key: x-accesslog-attr + value: '{"attr1":"value1","attr2":"value2"}' + transportApiVersion: V3 - filter: responseFlagFilter: flags: @@ -88,6 +114,29 @@ response_code: '%RESPONSE_CODE%' start_time: '%START_TIME%' path: /dev/stdout + - name: envoy.access_loggers.http_grpc + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig + additionalRequestHeadersToLog: + - x-client-ip-address + additionalResponseHeadersToLog: + - cache-control + additionalResponseTrailersToLog: + - expires + commonConfig: + grpcService: + envoyGrpc: + clusterName: accesslog/monitoring/envoy-als/port/9000 + initialMetadata: + - key: x-accesslog-text + value: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% + %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% + "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" + "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"' + - key: x-accesslog-attr + value: '{"attr1":"value1","attr2":"value2"}' + transportApiVersion: V3 - name: envoy.access_loggers.open_telemetry typedConfig: '@type': type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig