diff --git a/examples/kubernetes/accesslog/als-accesslog.yaml b/examples/kubernetes/accesslog/als-accesslog.yaml new file mode 100644 index 00000000000..686c827f108 --- /dev/null +++ b/examples/kubernetes/accesslog/als-accesslog.yaml @@ -0,0 +1,39 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: als-access-logging + namespace: envoy-gateway-system +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: als-access-logging + namespace: envoy-gateway-system +spec: + telemetry: + accessLog: + settings: + - format: + type: JSON + json: + attr1: val1 + attr2: val2 + sinks: + - type: ALS + als: + backendRefs: + - name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + type: HTTP diff --git a/examples/kubernetes/accesslog/multi-sinks.yaml b/examples/kubernetes/accesslog/multi-sinks.yaml index ae2cf3b55d5..99ac318d914 100644 --- a/examples/kubernetes/accesslog/multi-sinks.yaml +++ b/examples/kubernetes/accesslog/multi-sinks.yaml @@ -27,6 +27,13 @@ spec: - type: File file: path: /dev/stdout + - type: ALS + als: + backendRefs: + - name: envoy-als + namespace: monitoring + port: 9000 + type: HTTP - type: OpenTelemetry openTelemetry: host: otel-collector.monitoring.svc.cluster.local diff --git a/internal/gatewayapi/listener.go b/internal/gatewayapi/listener.go index aa47af35182..25afaacee7d 100644 --- a/internal/gatewayapi/listener.go +++ b/internal/gatewayapi/listener.go @@ -246,8 +246,8 @@ func (t *Translator) processAccessLog(envoyproxy *egv1a1.EnvoyProxy, resources * irAccessLog := &ir.AccessLog{} // translate the access log configuration to the IR - for idx, accessLog := range envoyproxy.Spec.Telemetry.AccessLog.Settings { - for _, sink := range accessLog.Sinks { + for i, accessLog := range envoyproxy.Spec.Telemetry.AccessLog.Settings { + for j, sink := range accessLog.Sinks { switch sink.Type { case egv1a1.ProxyAccessLogSinkTypeFile: if sink.File == nil { @@ -273,6 +273,50 @@ func (t *Translator) processAccessLog(envoyproxy *egv1a1.EnvoyProxy, resources * } irAccessLog.JSON = append(irAccessLog.JSON, al) } + case egv1a1.ProxyAccessLogSinkTypeALS: + if sink.ALS == nil { + continue + } + + var logName string + if sink.ALS.LogName != nil { + logName = *sink.ALS.LogName + } else { + logName = fmt.Sprintf("%s/%s", envoyproxy.Namespace, envoyproxy.Name) + } + + // TODO: how to get authority from the backendRefs? + ds, err := t.processBackendRefs(sink.ALS.BackendRefs, envoyproxy.Namespace, resources, envoyproxy) + if err != nil { + return nil, err + } + + al := &ir.ALSAccessLog{ + LogName: logName, + Destination: ir.RouteDestination{ + Name: fmt.Sprintf("accesslog_als_%d_%d", i, j), // TODO: rename this, so that we can share backend with tracing? + Settings: ds, + }, + Type: sink.ALS.Type, + } + + if al.Type == egv1a1.ALSEnvoyProxyAccessLogTypeHTTP && sink.ALS.HTTP != nil { + http := &ir.ALSAccessLogHTTP{ + RequestHeaders: sink.ALS.HTTP.RequestHeaders, + ResponseHeaders: sink.ALS.HTTP.ResponseHeaders, + ResponseTrailers: sink.ALS.HTTP.ResponseTrailers, + } + al.HTTP = http + } + + switch accessLog.Format.Type { + case egv1a1.ProxyAccessLogFormatTypeJSON: + al.Attributes = accessLog.Format.JSON + case egv1a1.ProxyAccessLogFormatTypeText: + al.Text = accessLog.Format.Text + } + + irAccessLog.ALS = append(irAccessLog.ALS, al) case egv1a1.ProxyAccessLogSinkTypeOpenTelemetry: if sink.OpenTelemetry == nil { continue @@ -289,7 +333,7 @@ func (t *Translator) processAccessLog(envoyproxy *egv1a1.EnvoyProxy, resources * return nil, err } al.Destination = ir.RouteDestination{ - Name: fmt.Sprintf("accesslog-%d", idx), // TODO: rename this, so that we can share backend with tracing? + Name: fmt.Sprintf("accesslog_otel_%d_%d", i, j), // TODO: rename this, so that we can share backend with tracing? Settings: ds, } diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.in.yaml new file mode 100644 index 00000000000..28850133563 --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.in.yaml @@ -0,0 +1,132 @@ +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + namespace: envoy-gateway-system + name: test + spec: + telemetry: + accessLog: + settings: + - format: + type: JSON + json: + attr1: val1 + attr2: val2 + sinks: + - type: ALS + als: + logName: accesslog + backendRefs: + - name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + type: HTTP + - type: ALS + als: + backendRefs: + - name: envoy-als + namespace: monitoring + port: 9000 + type: TCP + provider: + type: Kubernetes + kubernetes: + envoyService: + type: LoadBalancer + envoyDeployment: + replicas: 2 + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: "envoyproxy/envoy:distroless-dev" + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + runAsUser: 2000 + allowPrivilegeEscalation: false + pod: + annotations: + key1: val1 + key2: val2 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: "router" + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + fsGroupChangePolicy: "OnRootMismatch" + volumes: + - name: certs + secret: + secretName: envoy-cert +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same +services: +- apiVersion: v1 + kind: Service + metadata: + name: envoy-als + namespace: monitoring + spec: + type: ClusterIP + ports: + - name: grpc + port: 9000 + protocol: TCP + targetPort: 9000 +endpointSlices: +- apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: endpointslice-envoy-als + namespace: monitoring + labels: + kubernetes.io/service-name: envoy-als + addressType: IPv4 + ports: + - name: grpc + protocol: TCP + port: 9090 + endpoints: + - addresses: + - "10.240.0.10" + conditions: + ready: true diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.out.yaml new file mode 100755 index 00000000000..27b9952fca5 --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-als-json.out.yaml @@ -0,0 +1,193 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: Same + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +infraIR: + envoy-gateway/gateway-1: + proxy: + config: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + creationTimestamp: null + name: test + namespace: envoy-gateway-system + spec: + logging: {} + provider: + kubernetes: + envoyDeployment: + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: envoyproxy/envoy:distroless-dev + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + runAsUser: 2000 + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + annotations: + key1: val1 + key2: val2 + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 3000 + runAsUser: 1000 + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: router + volumes: + - name: certs + secret: + secretName: envoy-cert + replicas: 2 + envoyService: + type: LoadBalancer + type: Kubernetes + telemetry: + accessLog: + settings: + - format: + json: + attr1: val1 + attr2: val2 + type: JSON + sinks: + - als: + backendRefs: + - name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + logName: accesslog + type: HTTP + type: ALS + - als: + backendRefs: + - name: envoy-als + namespace: monitoring + port: 9000 + type: TCP + type: ALS + status: {} + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + als: + - attributes: + attr1: val1 + attr2: val2 + destination: + name: accesslog_als_0_0 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + name: accesslog + type: HTTP + - attributes: + attr1: val1 + attr2: val2 + destination: + name: accesslog_als_0_1 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + name: envoy-gateway-system/test + type: TCP + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-backend.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-backend.out.yaml index c1eab74dd87..94763fd2522 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog-backend.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-backend.out.yaml @@ -137,7 +137,7 @@ xdsIR: accessLog: openTelemetry: - destination: - name: accesslog-0 + name: accesslog_otel_0_1 settings: - addressType: IP endpoints: diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-cel.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-cel.out.yaml index c71eb38b0b5..5802c511e97 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog-cel.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-cel.out.yaml @@ -140,7 +140,7 @@ xdsIR: openTelemetry: - authority: otel-collector.monitoring.svc.cluster.local destination: - name: accesslog-0 + name: accesslog_otel_0_1 settings: - endpoints: - host: otel-collector.monitoring.svc.cluster.local diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml index 488d9a14957..9a1b4a3601d 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.in.yaml @@ -14,6 +14,7 @@ envoyProxyForGatewayClass: [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n sinks: - type: File + - type: ALS - type: OpenTelemetry provider: type: Kubernetes diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml index 7b6845bf734..db14cc4b8e3 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml @@ -108,6 +108,7 @@ infraIR: type: Text sinks: - type: File + - type: ALS - type: OpenTelemetry status: {} listeners: diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml index 3eec4aa41fc..ea6b29424ca 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog.in.yaml @@ -16,6 +16,28 @@ envoyProxyForGatewayClass: - type: File file: path: /dev/stdout + - type: ALS + als: + logName: accesslog + backendRefs: + - name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + type: HTTP + - type: ALS + als: + backendRefs: + - name: envoy-als + namespace: monitoring + port: 9000 + type: TCP - type: OpenTelemetry openTelemetry: host: otel-collector.monitoring.svc.cluster.local @@ -85,3 +107,34 @@ gateways: allowedRoutes: namespaces: from: Same +services: +- apiVersion: v1 + kind: Service + metadata: + name: envoy-als + namespace: monitoring + spec: + type: ClusterIP + ports: + - name: grpc + port: 9000 + protocol: TCP + targetPort: 9000 +endpointSlices: +- apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: endpointslice-envoy-als + namespace: monitoring + labels: + kubernetes.io/service-name: envoy-als + addressType: IPv4 + ports: + - name: grpc + protocol: TCP + port: 9090 + endpoints: + - addresses: + - "10.240.0.10" + conditions: + ready: true diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml index d146b4f1925..9694dd07ad7 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog.out.yaml @@ -110,6 +110,28 @@ infraIR: - file: path: /dev/stdout type: File + - als: + backendRefs: + - name: envoy-als + namespace: monitoring + port: 9000 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + logName: accesslog + type: HTTP + type: ALS + - als: + backendRefs: + - name: envoy-als + namespace: monitoring + port: 9000 + type: TCP + type: ALS - openTelemetry: host: otel-collector.monitoring.svc.cluster.local port: 4317 @@ -133,10 +155,42 @@ infraIR: xdsIR: envoy-gateway/gateway-1: accessLog: + als: + - destination: + name: accesslog_als_0_1 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + name: accesslog + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: HTTP + - destination: + name: accesslog_als_0_2 + settings: + - addressType: IP + endpoints: + - host: 10.240.0.10 + port: 9090 + protocol: GRPC + name: envoy-gateway-system/test + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: TCP openTelemetry: - authority: otel-collector.monitoring.svc.cluster.local destination: - name: accesslog-0 + name: accesslog_otel_0_3 settings: - endpoints: - host: otel-collector.monitoring.svc.cluster.local diff --git a/internal/ir/xds.go b/internal/ir/xds.go index dea151a88c2..42704cffe75 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -1606,6 +1606,7 @@ type AccessLog struct { CELMatches []string `json:"celMatches,omitempty" yaml:"celMatches,omitempty"` Text []*TextAccessLog `json:"text,omitempty" yaml:"text,omitempty"` JSON []*JSONAccessLog `json:"json,omitempty" yaml:"json,omitempty"` + ALS []*ALSAccessLog `json:"als,omitempty" yaml:"als,omitempty"` OpenTelemetry []*OpenTelemetryAccessLog `json:"openTelemetry,omitempty" yaml:"openTelemetry,omitempty"` } @@ -1623,6 +1624,25 @@ type JSONAccessLog struct { Path string `json:"path" yaml:"path"` } +// ALSAccessLog holds the configuration for gRPC ALS access logging. +// +k8s:deepcopy-gen=true +type ALSAccessLog struct { + LogName string `json:"name" yaml:"name"` + Destination RouteDestination `json:"destination,omitempty" yaml:"destination,omitempty"` + Type egv1a1.ALSEnvoyProxyAccessLogType `json:"type" yaml:"type"` + Text *string `json:"text,omitempty" yaml:"text,omitempty"` + Attributes map[string]string `json:"attributes,omitempty" yaml:"attributes,omitempty"` + HTTP *ALSAccessLogHTTP `json:"http,omitempty" yaml:"http,omitempty"` +} + +// ALSAccessLogHTTP holds the configuration for HTTP ALS access logging. +// +k8s:deepcopy-gen=true +type ALSAccessLogHTTP struct { + RequestHeaders []string `json:"requestHeaders,omitempty" yaml:"requestHeaders,omitempty"` + ResponseHeaders []string `json:"responseHeaders,omitempty" yaml:"responseHeaders,omitempty"` + ResponseTrailers []string `json:"responseTrailers,omitempty" yaml:"responseTrailers,omitempty"` +} + // OpenTelemetryAccessLog holds the configuration for OpenTelemetry access logging. // +k8s:deepcopy-gen=true type OpenTelemetryAccessLog struct { diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index 15cba6a4c94..171c2c28d40 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -16,6 +16,69 @@ import ( "sigs.k8s.io/gateway-api/apis/v1alpha2" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ALSAccessLog) DeepCopyInto(out *ALSAccessLog) { + *out = *in + in.Destination.DeepCopyInto(&out.Destination) + if in.Text != nil { + in, out := &in.Text, &out.Text + *out = new(string) + **out = **in + } + if in.Attributes != nil { + in, out := &in.Attributes, &out.Attributes + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(ALSAccessLogHTTP) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ALSAccessLog. +func (in *ALSAccessLog) DeepCopy() *ALSAccessLog { + if in == nil { + return nil + } + out := new(ALSAccessLog) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ALSAccessLogHTTP) DeepCopyInto(out *ALSAccessLogHTTP) { + *out = *in + if in.RequestHeaders != nil { + in, out := &in.RequestHeaders, &out.RequestHeaders + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ResponseHeaders != nil { + in, out := &in.ResponseHeaders, &out.ResponseHeaders + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ResponseTrailers != nil { + in, out := &in.ResponseTrailers, &out.ResponseTrailers + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ALSAccessLogHTTP. +func (in *ALSAccessLogHTTP) DeepCopy() *ALSAccessLogHTTP { + if in == nil { + return nil + } + out := new(ALSAccessLogHTTP) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AccessLog) DeepCopyInto(out *AccessLog) { *out = *in @@ -46,6 +109,17 @@ func (in *AccessLog) DeepCopyInto(out *AccessLog) { } } } + if in.ALS != nil { + in, out := &in.ALS, &out.ALS + *out = make([]*ALSAccessLog, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ALSAccessLog) + (*in).DeepCopyInto(*out) + } + } + } if in.OpenTelemetry != nil { in, out := &in.OpenTelemetry, &out.OpenTelemetry *out = make([]*OpenTelemetryAccessLog, len(*in)) diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index c1135c9c6f4..f7e88da222d 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -1632,22 +1632,17 @@ func (r *gatewayAPIReconciler) processEnvoyProxy(ep *egv1a1.EnvoyProxy, resource } if ep.Spec.Telemetry != nil { + var backendRefs []egv1a1.BackendRef telemetry := ep.Spec.Telemetry if telemetry.AccessLog != nil { for _, setting := range telemetry.AccessLog.Settings { for _, sink := range setting.Sinks { - if sink.OpenTelemetry == nil { - continue + if sink.OpenTelemetry != nil { + backendRefs = append(backendRefs, sink.OpenTelemetry.BackendRefs...) } - for _, backendRef := range sink.OpenTelemetry.BackendRefs { - backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, ep.Namespace) - resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ - Group: backendRef.BackendObjectReference.Group, - Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtr(backendNamespace), - Name: backendRef.Name, - }) + if sink.ALS != nil { + backendRefs = append(backendRefs, sink.ALS.BackendRefs...) } } } @@ -1655,32 +1650,24 @@ func (r *gatewayAPIReconciler) processEnvoyProxy(ep *egv1a1.EnvoyProxy, resource if telemetry.Metrics != nil { for _, sink := range telemetry.Metrics.Sinks { - if sink.OpenTelemetry == nil { - continue - } - for _, backendRef := range sink.OpenTelemetry.BackendRefs { - backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, ep.Namespace) - resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ - Group: backendRef.BackendObjectReference.Group, - Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtr(backendNamespace), - Name: backendRef.Name, - }) + if sink.OpenTelemetry != nil { + backendRefs = append(backendRefs, sink.OpenTelemetry.BackendRefs...) } } } if telemetry.Tracing != nil { - for _, backendRef := range telemetry.Tracing.Provider.BackendRefs { - backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, ep.Namespace) - resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ - Group: backendRef.BackendObjectReference.Group, - Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtr(backendNamespace), - Name: backendRef.Name, - }) + backendRefs = append(backendRefs, telemetry.Tracing.Provider.BackendRefs...) + } - } + for _, backendRef := range backendRefs { + backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, ep.Namespace) + resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ + Group: backendRef.BackendObjectReference.Group, + Kind: backendRef.BackendObjectReference.Kind, + Namespace: gatewayapi.NamespacePtr(backendNamespace), + Name: backendRef.Name, + }) } } diff --git a/internal/provider/kubernetes/indexers.go b/internal/provider/kubernetes/indexers.go index 3b6883746a4..443c667e349 100644 --- a/internal/provider/kubernetes/indexers.go +++ b/internal/provider/kubernetes/indexers.go @@ -156,23 +156,29 @@ func backendEnvoyProxyTelemetryIndexFunc(rawObj client.Object) []string { func accessLogRefs(ep *egv1a1.EnvoyProxy) []string { var refs []string - if ep.Spec.Telemetry == nil || ep.Spec.Telemetry.Metrics == nil { + if ep.Spec.Telemetry == nil || ep.Spec.Telemetry.AccessLog == nil { return refs } - for _, sink := range ep.Spec.Telemetry.Metrics.Sinks { - if sink.OpenTelemetry != nil { - otel := sink.OpenTelemetry - if otel.BackendRefs != nil { - for _, ref := range otel.BackendRefs { - if ref.Kind == nil || string(*ref.Kind) == gatewayapi.KindService { - refs = append(refs, - types.NamespacedName{ - Namespace: gatewayapi.NamespaceDerefOr(ref.Namespace, ep.Namespace), - Name: string(ref.Name), - }.String(), - ) - } + for _, setting := range ep.Spec.Telemetry.AccessLog.Settings { + for _, sink := range setting.Sinks { + var backendRefs []egv1a1.BackendRef + if sink.OpenTelemetry != nil { + backendRefs = append(backendRefs, sink.OpenTelemetry.BackendRefs...) + } + + if sink.ALS != nil { + backendRefs = append(backendRefs, sink.ALS.BackendRefs...) + } + + for _, ref := range backendRefs { + if ref.Kind == nil || string(*ref.Kind) == gatewayapi.KindService { + refs = append(refs, + types.NamespacedName{ + Namespace: gatewayapi.NamespaceDerefOr(ref.Namespace, ep.Namespace), + Name: string(ref.Name), + }.String(), + ) } } } diff --git a/internal/xds/translator/accesslog.go b/internal/xds/translator/accesslog.go index 05e05bee30d..4a22b20500b 100644 --- a/internal/xds/translator/accesslog.go +++ b/internal/xds/translator/accesslog.go @@ -6,6 +6,7 @@ package translator import ( + "encoding/json" "errors" "sort" "strings" @@ -25,6 +26,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/structpb" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/utils/protocov" "github.com/envoyproxy/gateway/internal/xds/types" @@ -56,7 +58,8 @@ const ( metadataCommandOperator = "%METADATA" celCommandOperator = "%CEL" - celFilter = "envoy.access_loggers.extension_filters.cel" + tcpGRPCAccessLog = "envoy.access_loggers.tcp_grpc" + celFilter = "envoy.access_loggers.extension_filters.cel" ) // for the case when a route does not exist to upstream, hcm logs will not be present @@ -173,6 +176,74 @@ func buildXdsAccessLog(al *ir.AccessLog, forListener bool) []*accesslog.AccessLo }, }) } + // handle ALS access logs + for _, als := range al.ALS { + cc := &grpcaccesslog.CommonGrpcAccessLogConfig{ + LogName: als.LogName, + GrpcService: &cfgcore.GrpcService{ + TargetSpecifier: &cfgcore.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &cfgcore.GrpcService_EnvoyGrpc{ + ClusterName: als.Destination.Name, + }, + }, + }, + TransportApiVersion: cfgcore.ApiVersion_V3, + } + + // include text and json format as metadata when initiating stream + md := make([]*cfgcore.HeaderValue, 0, 2) + + if als.Text != nil && *als.Text != "" { + md = append(md, &cfgcore.HeaderValue{ + Key: "x-accesslog-text", + Value: strings.ReplaceAll(strings.Trim(*als.Text, "\x00\n\r"), "\x00\n\r", " "), + }) + } + + if len(als.Attributes) > 0 { + if attr, err := json.Marshal(als.Attributes); err == nil { + md = append(md, &cfgcore.HeaderValue{ + Key: "x-accesslog-attr", + Value: string(attr), + }) + } + } + + cc.GrpcService.InitialMetadata = md + + switch als.Type { + case egv1a1.ALSEnvoyProxyAccessLogTypeHTTP: + alCfg := &grpcaccesslog.HttpGrpcAccessLogConfig{ + CommonConfig: cc, + } + + if als.HTTP != nil { + alCfg.AdditionalRequestHeadersToLog = als.HTTP.RequestHeaders + alCfg.AdditionalResponseHeadersToLog = als.HTTP.ResponseHeaders + alCfg.AdditionalResponseTrailersToLog = als.HTTP.ResponseTrailers + } + + accesslogAny, _ := anypb.New(alCfg) + accessLogs = append(accessLogs, &accesslog.AccessLog{ + Name: wellknown.HTTPGRPCAccessLog, + ConfigType: &accesslog.AccessLog_TypedConfig{ + TypedConfig: accesslogAny, + }, + }) + case egv1a1.ALSEnvoyProxyAccessLogTypeTCP: + alCfg := &grpcaccesslog.TcpGrpcAccessLogConfig{ + CommonConfig: cc, + } + + accesslogAny, _ := anypb.New(alCfg) + accessLogs = append(accessLogs, &accesslog.AccessLog{ + Name: tcpGRPCAccessLog, + ConfigType: &accesslog.AccessLog_TypedConfig{ + TypedConfig: accesslogAny, + }, + }) + } + } // handle open telemetry access logs for _, otel := range al.OpenTelemetry { al := &otelaccesslog.OpenTelemetryAccessLogConfig{ @@ -431,6 +502,19 @@ func processClusterForAccessLog(tCtx *types.ResourceVersionTable, al *ir.AccessL return nil } + // add clusters for ALS access logs + for _, als := range al.ALS { + if err := addXdsCluster(tCtx, &xdsClusterArgs{ + name: als.Destination.Name, + settings: als.Destination.Settings, + tSocket: nil, + endpointType: EndpointTypeStatic, + }); err != nil && !errors.Is(err, ErrXdsClusterExists) { + return err + } + } + + // add clusters for Open Telemetry access logs for _, otel := range al.OpenTelemetry { if err := addXdsCluster(tCtx, &xdsClusterArgs{ name: otel.Destination.Name, diff --git a/internal/xds/translator/testdata/in/xds-ir/accesslog-als-tcp.yaml b/internal/xds/translator/testdata/in/xds-ir/accesslog-als-tcp.yaml new file mode 100644 index 00000000000..2d8f0c6aa48 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/accesslog-als-tcp.yaml @@ -0,0 +1,23 @@ +name: "accesslog" +accesslog: + json: + - json: + start_time: "%START_TIME%" + method: "%REQ(:METHOD)%" + path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + response_code: "%RESPONSE_CODE%" + als: + - destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 1.1.1.1 + port: 9000 + protocol: GRPC + weight: 1 + attributes: + attr1: value1 + attr2: value2 + type: TCP diff --git a/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml b/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml index 07d910197a7..26f0f5663f8 100644 --- a/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/accesslog.yaml @@ -12,6 +12,29 @@ accesslog: path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" protocol: "%PROTOCOL%" response_code: "%RESPONSE_CODE%" + als: + - destination: + name: accesslog/monitoring/envoy-als/port/9000 + settings: + - addressType: IP + endpoints: + - host: 1.1.1.1 + port: 9000 + protocol: GRPC + weight: 1 + text: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + attributes: + attr1: value1 + attr2: value2 + http: + requestHeaders: + - x-client-ip-address + responseHeaders: + - cache-control + responseTrailers: + - expires + type: HTTP openTelemetry: - text: | [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.clusters.yaml new file mode 100755 index 00000000000..18b309bb74d --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.clusters.yaml @@ -0,0 +1,22 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: accesslog/monitoring/envoy-als/port/9000 + lbPolicy: LEAST_REQUEST + name: accesslog/monitoring/envoy-als/port/9000 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.endpoints.yaml new file mode 100755 index 00000000000..9159d4e31f2 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: accesslog/monitoring/envoy-als/port/9000 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.1.1.1 + portValue: 9000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: accesslog/monitoring/envoy-als/port/9000/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.listeners.yaml new file mode 100755 index 00000000000..fe51488c706 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.listeners.yaml @@ -0,0 +1 @@ +[] diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.routes.yaml new file mode 100755 index 00000000000..fe51488c706 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-als-tcp.routes.yaml @@ -0,0 +1 @@ +[] diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml index b8874bf24f9..6ba4705c13f 100644 --- a/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog.clusters.yaml @@ -15,6 +15,28 @@ outlierDetection: {} perConnectionBufferLimitBytes: 32768 type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: accesslog/monitoring/envoy-als/port/9000 + lbPolicy: LEAST_REQUEST + name: accesslog/monitoring/envoy-als/port/9000 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} - circuitBreakers: thresholds: - maxRetries: 1024 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml index 20c80b3aaaa..2ce2e6da08c 100644 --- a/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog.endpoints.yaml @@ -10,3 +10,15 @@ loadBalancingWeight: 1 locality: region: direct-route-dest/backend/0 +- clusterName: accesslog/monitoring/envoy-als/port/9000 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.1.1.1 + portValue: 9000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: accesslog/monitoring/envoy-als/port/9000/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml index 8e582b05b7e..21621ff674e 100644 --- a/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog.listeners.yaml @@ -26,6 +26,32 @@ response_code: '%RESPONSE_CODE%' start_time: '%START_TIME%' path: /dev/stdout + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.http_grpc + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig + additionalRequestHeadersToLog: + - x-client-ip-address + additionalResponseHeadersToLog: + - cache-control + additionalResponseTrailersToLog: + - expires + commonConfig: + grpcService: + envoyGrpc: + clusterName: accesslog/monitoring/envoy-als/port/9000 + initialMetadata: + - key: x-accesslog-text + value: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% + %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" + "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"' + - key: x-accesslog-attr + value: '{"attr1":"value1","attr2":"value2"}' + transportApiVersion: V3 - filter: responseFlagFilter: flags: @@ -88,6 +114,29 @@ response_code: '%RESPONSE_CODE%' start_time: '%START_TIME%' path: /dev/stdout + - name: envoy.access_loggers.http_grpc + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig + additionalRequestHeadersToLog: + - x-client-ip-address + additionalResponseHeadersToLog: + - cache-control + additionalResponseTrailersToLog: + - expires + commonConfig: + grpcService: + envoyGrpc: + clusterName: accesslog/monitoring/envoy-als/port/9000 + initialMetadata: + - key: x-accesslog-text + value: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% + %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% + "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" + "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"' + - key: x-accesslog-attr + value: '{"attr1":"value1","attr2":"value2"}' + transportApiVersion: V3 - name: envoy.access_loggers.open_telemetry typedConfig: '@type': type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig diff --git a/test/config/gatewayclass.yaml b/test/config/gatewayclass.yaml index 6178e6b2d25..455636b15a5 100644 --- a/test/config/gatewayclass.yaml +++ b/test/config/gatewayclass.yaml @@ -58,6 +58,13 @@ spec: port: 4317 resources: k8s.cluster.name: "envoy-gateway" + - type: ALS + als: + backendRefs: + - name: envoy-als + namespace: monitoring + port: 8080 + type: HTTP tracing: provider: backendRefs: diff --git a/test/e2e/base/manifests.yaml b/test/e2e/base/manifests.yaml index c1e08e63c8b..ded6c648a0c 100644 --- a/test/e2e/base/manifests.yaml +++ b/test/e2e/base/manifests.yaml @@ -863,3 +863,238 @@ metadata: name: backend-tls-certificate namespace: gateway-conformance-infra type: kubernetes.io/tls +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: envoy-als + namespace: monitoring +data: + go.mod: | + module envoy-als + go 1.22 + require ( + github.com/envoyproxy/go-control-plane v0.12.0 + github.com/prometheus/client_golang v1.19.1 + google.golang.org/grpc v1.64.0 + ) + + require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/protobuf v1.33.0 // indirect + ) + go.sum: | + github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= + github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= + github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= + github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= + github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc= + github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= + github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= + github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= + github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= + github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= + github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= + github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= + github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= + github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= + github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= + github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= + github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= + github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= + github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= + github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= + github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= + github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= + github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= + github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= + golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= + golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= + golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= + golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= + golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= + golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= + google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= + google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= + google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= + google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= + main.go: | + package main + + import ( + "log" + "net" + "net/http" + + alsv2 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v2" + alsv3 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "google.golang.org/grpc" + ) + + var ( + LogCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "log_count", + Help: "The total number of logs received.", + }, []string{"api_version"}) + ) + + func init() { + // Register the summary and the histogram with Prometheus's default registry. + prometheus.MustRegister(LogCount) + } + + type ALSServer struct { + } + + func (a *ALSServer) StreamAccessLogs(logStream alsv2.AccessLogService_StreamAccessLogsServer) error { + log.Println("Streaming als v2 logs") + for { + data, err := logStream.Recv() + if err != nil { + return err + } + + httpLogs := data.GetHttpLogs() + if httpLogs != nil { + LogCount.WithLabelValues("v2").Add(float64(len(httpLogs.LogEntry))) + } + + log.Printf("Received v2 log data: %s\n", data.String()) + } + } + + type ALSServerV3 struct { + } + + func (a *ALSServerV3) StreamAccessLogs(logStream alsv3.AccessLogService_StreamAccessLogsServer) error { + log.Println("Streaming als v3 logs") + for { + data, err := logStream.Recv() + if err != nil { + return err + } + + httpLogs := data.GetHttpLogs() + if httpLogs != nil { + LogCount.WithLabelValues("v3").Add(float64(len(httpLogs.LogEntry))) + } + + log.Printf("Received v3 log data: %s\n", data.String()) + } + } + + func NewALSServer() *ALSServer { + return &ALSServer{} + } + + func NewALSServerV3() *ALSServerV3 { + return &ALSServerV3{} + } + + func main() { + mux := http.NewServeMux() + if err := addMonitor(mux); err != nil { + log.Printf("could not establish self-monitoring: %v\n", err) + } + + s := &http.Server{ + Addr: ":19001", + Handler: mux, + } + + go func() { + s.ListenAndServe() + }() + + listener, err := net.Listen("tcp", "0.0.0.0:8080") + if err != nil { + log.Fatalf("Failed to start listener on port 8080: %v", err) + } + + var opts []grpc.ServerOption + grpcServer := grpc.NewServer(opts...) + alsv2.RegisterAccessLogServiceServer(grpcServer, NewALSServer()) + alsv3.RegisterAccessLogServiceServer(grpcServer, NewALSServerV3()) + log.Println("Starting ALS Server") + if err := grpcServer.Serve(listener); err != nil { + log.Fatalf("grpc serve err: %v", err) + } + } + + func addMonitor(mux *http.ServeMux) error { + mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{EnableOpenMetrics: true})) + + return nil + } + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-als + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: envoy-als + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "19001" + labels: + app: envoy-als + spec: + containers: + - name: envoy-als + command: + - sh + - "-c" + - "cp -a /app /app-live && cd /app-live && go run . " + image: golang:1.22.3-alpine + ports: + - containerPort: 8080 + - containerPort: 19001 + volumeMounts: + - name: envoy-als + mountPath: /app + volumes: + - name: envoy-als + configMap: + name: envoy-als +--- +apiVersion: v1 +kind: Service +metadata: + name: envoy-als + namespace: monitoring +spec: + selector: + app: envoy-als + type: LoadBalancer + ports: + - name: grpc-als + protocol: TCP + port: 8080 + targetPort: 8080 + - name: http-monitoring + protocol: TCP + port: 19001 + targetPort: 19001 diff --git a/test/e2e/testdata/accesslog-als.yaml b/test/e2e/testdata/accesslog-als.yaml new file mode 100644 index 00000000000..86606c3c053 --- /dev/null +++ b/test/e2e/testdata/accesslog-als.yaml @@ -0,0 +1,16 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: accesslog-als + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + rules: + - matches: + - path: + type: PathPrefix + value: /als + backendRefs: + - name: infra-backend-v1 + port: 8080 diff --git a/test/e2e/tests/accesslog.go b/test/e2e/tests/accesslog.go index d2bfba32731..d7d79f089c0 100644 --- a/test/e2e/tests/accesslog.go +++ b/test/e2e/tests/accesslog.go @@ -29,7 +29,7 @@ import ( ) func init() { - ConformanceTests = append(ConformanceTests, FileAccessLogTest, OpenTelemetryTest) + ConformanceTests = append(ConformanceTests, FileAccessLogTest, OpenTelemetryTest, ALSTest) } var FileAccessLogTest = suite.ConformanceTest{ @@ -191,6 +191,64 @@ var OpenTelemetryTest = suite.ConformanceTest{ }, } +var ALSTest = suite.ConformanceTest{ + ShortName: "ALS", + Description: "Make sure ALS access log is working", + Manifests: []string{"testdata/accesslog-als.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + t.Run("HTTP", func(t *testing.T) { + ns := "gateway-conformance-infra" + routeNN := types.NamespacedName{Name: "accesslog-als", Namespace: ns} + gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + preCount := ALSLogCount(t, suite) + expectedResponse := httputils.ExpectedResponse{ + Request: httputils.Request{ + Path: "/als", + }, + Response: httputils.Response{ + StatusCode: 200, + }, + Namespace: ns, + } + // make sure listener is ready + httputils.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) + + if err := wait.PollUntilContextTimeout(context.TODO(), time.Second, time.Minute, true, + func(ctx context.Context) (bool, error) { + curCount := ALSLogCount(t, suite) + return preCount < curCount, nil + }); err != nil { + t.Errorf("failed to get log count from loki: %v", err) + } + }) + }, +} + +func ALSLogCount(t *testing.T, suite *suite.ConformanceTestSuite) int { + metricPath, err := RetrieveURL(suite.Client, types.NamespacedName{ + Namespace: "monitoring", + Name: "envoy-als", + }, 19001, "/metrics") + if err != nil { + t.Fatalf("failed to get metric url: %v", err) + } + + countMetric, err := RetrieveMetric(metricPath, "log_count", time.Second) + if err != nil { + t.Fatalf("failed to get metric: %v", err) + } + + total := 0 + for _, m := range countMetric.Metric { + if m.Counter != nil && m.Counter.Value != nil { + total += int(*m.Counter.Value) + } + } + + return total +} + // QueryLogCountFromLoki queries log count from loki // TODO: move to utils package if needed func QueryLogCountFromLoki(t *testing.T, c client.Client, nn types.NamespacedName, keyValues map[string]string, match string) (int, error) { diff --git a/test/e2e/tests/metric.go b/test/e2e/tests/metric.go index fe658252198..fff4809edda 100644 --- a/test/e2e/tests/metric.go +++ b/test/e2e/tests/metric.go @@ -10,17 +10,11 @@ package tests import ( "context" - "errors" - "fmt" - "net/http" "testing" "time" - "github.com/prometheus/common/expfmt" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - "sigs.k8s.io/controller-runtime/pkg/client" httputils "sigs.k8s.io/gateway-api/conformance/utils/http" "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" "sigs.k8s.io/gateway-api/conformance/utils/suite" @@ -104,49 +98,3 @@ var MetricTest = suite.ConformanceTest{ }) }, } - -func ScrapeMetrics(t *testing.T, c client.Client, nn types.NamespacedName, port int32, path string) error { - svc := corev1.Service{} - if err := c.Get(context.Background(), nn, &svc); err != nil { - return err - } - host := "" - switch svc.Spec.Type { - case corev1.ServiceTypeLoadBalancer: - for _, ing := range svc.Status.LoadBalancer.Ingress { - if ing.IP != "" { - host = ing.IP - break - } - } - default: - host = fmt.Sprintf("%s.%s.svc", nn.Name, nn.Namespace) - } - - url := fmt.Sprintf("http://%s:%d%s", host, port, path) - t.Logf("try to request: %s", url) - - httpClient := http.Client{ - Timeout: 1 * time.Second, - } - res, err := httpClient.Get(url) - if err != nil { - return fmt.Errorf("failed to scrape metrics: %w", err) - } - if res.StatusCode != http.StatusOK { - return fmt.Errorf("failed to scrape metrics: %s", res.Status) - } - - metrics, err := (&expfmt.TextParser{}).TextToMetricFamilies(res.Body) - if err != nil { - return err - } - - // TODO: support metric matching - // for now, just check metric exists - if len(metrics) > 0 { - return nil - } - - return errors.New("no metrics found") -} diff --git a/test/e2e/tests/utils.go b/test/e2e/tests/utils.go index 8e72ca0e0c5..5dfc9af494e 100644 --- a/test/e2e/tests/utils.go +++ b/test/e2e/tests/utils.go @@ -7,8 +7,10 @@ package tests import ( "context" + "errors" "fmt" "io" + "net/http" "strings" "testing" "time" @@ -17,6 +19,8 @@ import ( "fortio.org/fortio/periodic" flog "fortio.org/log" "github.com/google/go-cmp/cmp" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -268,6 +272,78 @@ func EnvoyExtensionPolicyMustBeAccepted(t *testing.T, client client.Client, poli require.NoErrorf(t, waitErr, "error waiting for EnvoyExtensionPolicy to be accepted") } +func ScrapeMetrics(t *testing.T, c client.Client, nn types.NamespacedName, port int32, path string) error { + url, err := RetrieveURL(c, nn, port, path) + if err != nil { + return err + } + + t.Logf("scraping metrics from %s", url) + + metrics, err := RetrieveMetrics(url, time.Second) + if err != nil { + return err + } + + // TODO: support metric matching + // for now, just check metric exists + if len(metrics) > 0 { + return nil + } + + return errors.New("no metrics found") +} + +func RetrieveURL(c client.Client, nn types.NamespacedName, port int32, path string) (string, error) { + svc := corev1.Service{} + if err := c.Get(context.Background(), nn, &svc); err != nil { + return "", err + } + host := "" + switch svc.Spec.Type { + case corev1.ServiceTypeLoadBalancer: + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ing.IP != "" { + host = ing.IP + break + } + } + default: + host = fmt.Sprintf("%s.%s.svc", nn.Name, nn.Namespace) + } + return fmt.Sprintf("http://%s:%d%s", host, port, path), nil +} + +var metricParser = &expfmt.TextParser{} + +func RetrieveMetrics(url string, timeout time.Duration) (map[string]*dto.MetricFamily, error) { + httpClient := http.Client{ + Timeout: timeout, + } + res, err := httpClient.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to scrape metrics: %w", err) + } + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to scrape metrics: %s", res.Status) + } + + return metricParser.TextToMetricFamilies(res.Body) +} + +func RetrieveMetric(url string, name string, timeout time.Duration) (*dto.MetricFamily, error) { + metrics, err := RetrieveMetrics(url, timeout) + if err != nil { + return nil, err + } + + if mf, ok := metrics[name]; ok { + return mf, nil + } + + return nil, fmt.Errorf("metric %s not found", name) +} + func WaitForLoadBalancerAddress(t *testing.T, client client.Client, timeout time.Duration, nn types.NamespacedName) (string, error) { t.Helper()