feat: support access log formatters (envoyproxy#3303)

* Support access log formatters in EnvoyProxy Signed-off-by: Dingkang Li <dingkang1743@gmail.com> * Configure access log formatters based on command operators automatically Signed-off-by: Dingkang Li <dingkang1743@gmail.com> * Use ToAny function in protocov package Signed-off-by: Dingkang Li <dingkang1743@gmail.com> --------- Signed-off-by: Dingkang Li <dingkang1743@gmail.com> Co-authored-by: zirain <zirain2009@gmail.com>
guydc · May 8, 2024 · 0457ee7 · 0457ee7
1 parent 04fb089
commit 0457ee7
Show file tree

Hide file tree

Showing 6 changed files with 420 additions and 0 deletions.
diff --git a/internal/xds/translator/accesslog.go b/internal/xds/translator/accesslog.go
@@ -8,12 +8,16 @@ package translator
 import (
 	"errors"
 	"sort"
+	"strings"
 
 	accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3"
 	cfgcore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	fileaccesslog "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3"
 	grpcaccesslog "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3"
 	otelaccesslog "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/open_telemetry/v3"
+	celformatter "github.com/envoyproxy/go-control-plane/envoy/extensions/formatter/cel/v3"
+	metadataformatter "github.com/envoyproxy/go-control-plane/envoy/extensions/formatter/metadata/v3"
+	reqwithoutqueryformatter "github.com/envoyproxy/go-control-plane/envoy/extensions/formatter/req_without_query/v3"
 	"github.com/envoyproxy/go-control-plane/pkg/wellknown"
 	otlpcommonv1 "go.opentelemetry.io/proto/otlp/common/v1"
 	"golang.org/x/exp/maps"
@@ -22,6 +26,7 @@ import (
 	"k8s.io/utils/ptr"
 
 	"github.com/envoyproxy/gateway/internal/ir"
+	"github.com/envoyproxy/gateway/internal/utils/protocov"
 	"github.com/envoyproxy/gateway/internal/xds/types"
 )
 
@@ -46,6 +51,10 @@ const (
 
 	otelLogName   = "otel_envoy_accesslog"
 	otelAccessLog = "envoy.access_loggers.open_telemetry"
+
+	reqWithoutQueryCommandOperator = "%REQ_WITHOUT_QUERY"
+	metadataCommandOperator        = "%METADATA"
+	celCommandOperator             = "%CEL"
 )
 
 // for the case when a route does not exist to upstream, hcm logs will not be present
@@ -55,6 +64,28 @@ var listenerAccessLogFilter = &accesslog.AccessLogFilter{
 	},
 }
 
+var (
+	// reqWithoutQueryFormatter configures additional formatters needed for some of the format strings like "REQ_WITHOUT_QUERY"
+	reqWithoutQueryFormatter = &cfgcore.TypedExtensionConfig{
+		Name:        "envoy.formatter.req_without_query",
+		TypedConfig: protocov.ToAny(&reqwithoutqueryformatter.ReqWithoutQuery{}),
+	}
+
+	// metadataFormatter configures additional formatters needed for some of the format strings like "METADATA"
+	// for more information, see https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/formatter/metadata/v3/metadata.proto
+	metadataFormatter = &cfgcore.TypedExtensionConfig{
+		Name:        "envoy.formatter.metadata",
+		TypedConfig: protocov.ToAny(&metadataformatter.Metadata{}),
+	}
+
+	// celFormatter configures additional formatters needed for some of the format strings like "CEL"
+	// for more information, see https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/formatter/cel/v3/cel.proto
+	celFormatter = &cfgcore.TypedExtensionConfig{
+		Name:        "envoy.formatter.cel",
+		TypedConfig: protocov.ToAny(&celformatter.Cel{}),
+	}
+)
+
 func buildXdsAccessLog(al *ir.AccessLog, forListener bool) []*accesslog.AccessLog {
 	if al == nil {
 		return nil
@@ -84,6 +115,11 @@ func buildXdsAccessLog(al *ir.AccessLog, forListener bool) []*accesslog.AccessLo
 			},
 		}
 
+		formatters := accessLogTextFormatters(format)
+		if len(formatters) != 0 {
+			filelog.GetLogFormat().Formatters = formatters
+		}
+
 		// TODO: find a better way to handle this
 		accesslogAny, _ := anypb.New(filelog)
 		accessLogs = append(accessLogs, &accesslog.AccessLog{
@@ -122,6 +158,11 @@ func buildXdsAccessLog(al *ir.AccessLog, forListener bool) []*accesslog.AccessLo
 			},
 		}
 
+		formatters := accessLogJSONFormatters(json.JSON)
+		if len(formatters) != 0 {
+			filelog.GetLogFormat().Formatters = formatters
+		}
+
 		accesslogAny, _ := anypb.New(filelog)
 		accessLogs = append(accessLogs, &accesslog.AccessLog{
 			Name: wellknown.FileAccessLog,
@@ -182,6 +223,62 @@ func buildXdsAccessLog(al *ir.AccessLog, forListener bool) []*accesslog.AccessLo
 	return accessLogs
 }
 
+func accessLogTextFormatters(text string) []*cfgcore.TypedExtensionConfig {
+	formatters := make([]*cfgcore.TypedExtensionConfig, 0, 3)
+
+	if strings.Contains(text, reqWithoutQueryCommandOperator) {
+		formatters = append(formatters, reqWithoutQueryFormatter)
+	}
+
+	if strings.Contains(text, metadataCommandOperator) {
+		formatters = append(formatters, metadataFormatter)
+	}
+
+	if strings.Contains(text, celCommandOperator) {
+		formatters = append(formatters, celFormatter)
+	}
+
+	return formatters
+}
+
+func accessLogJSONFormatters(json map[string]string) []*cfgcore.TypedExtensionConfig {
+	reqWithoutQuery, metadata, cel := false, false, false
+
+	for _, value := range json {
+		if reqWithoutQuery && metadata && cel {
+			break
+		}
+
+		if strings.Contains(value, reqWithoutQueryCommandOperator) {
+			reqWithoutQuery = true
+		}
+
+		if strings.Contains(value, metadataCommandOperator) {
+			metadata = true
+		}
+
+		if strings.Contains(value, celCommandOperator) {
+			cel = true
+		}
+	}
+
+	formatters := make([]*cfgcore.TypedExtensionConfig, 0, 3)
+
+	if reqWithoutQuery {
+		formatters = append(formatters, reqWithoutQueryFormatter)
+	}
+
+	if metadata {
+		formatters = append(formatters, metadataFormatter)
+	}
+
+	if cel {
+		formatters = append(formatters, celFormatter)
+	}
+
+	return formatters
+}
+
 // read more here: https://opentelemetry.io/docs/specs/otel/resource/semantic_conventions/k8s/
 const (
 	k8sNamespaceNameKey = "k8s.namespace.name"

diff --git a/internal/xds/translator/testdata/in/xds-ir/accesslog-formatters.yaml b/internal/xds/translator/testdata/in/xds-ir/accesslog-formatters.yaml
@@ -0,0 +1,52 @@
+name: "accesslog"
+accesslog:
+  text:
+  - path: "/dev/stdout"
+    format: |
+      [%START_TIME%] "%CEL(request.method)% %CEL(request.path)% %CEL(request.protocol)%" %CEL(response.code)% %CEL(response.flags)% %CEL(request.total_size)% %CEL(response.total_size)% %CEL(request.duration)% %CEL(response.headers['X-ENVOY-UPSTREAM-SERVICE-TIME'])% "%CEL(request.headers['X-FORWARDED-FOR'])%" "%CEL(request.useragent)%" "%CEL(request.id)%" "%CEL(request.host)%" "%CEL(upstream.address)%"
+  json:
+  - path: "/dev/stdout"
+    json:
+      start_time: "%START_TIME%"
+      method: "%REQ(:METHOD)%"
+      path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"
+      protocol: "%PROTOCOL%"
+      response_code: "%RESPONSE_CODE%"
+      test_key: "%METADATA(DYNAMIC:com.test.my_filter:test_key)%"
+  - path: "/dev/stdout"
+    json:
+      start_time: "%START_TIME%"
+      method: "%REQ(:METHOD)%"
+      path_without_query: "%REQ_WITHOUT_QUERY(X-ENVOY-ORIGINAL-PATH?:PATH)%"
+      protocol: "%PROTOCOL%"
+      response_code: "%RESPONSE_CODE%"
+  openTelemetry:
+  - text: |
+      [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"
+    attributes:
+      "response_code": "%RESPONSE_CODE%"
+    resources:
+      "cluster_name": "cluster1"
+    host: otel-collector.default.svc.cluster.local
+    port: 4317
+http:
+- name: "first-listener"
+  address: "0.0.0.0"
+  port: 10080
+  hostnames:
+  - "*"
+  path:
+    mergeSlashes: true
+    escapedSlashesAction: UnescapeAndRedirect
+  routes:
+  - name: "direct-route"
+    hostname: "*"
+    destination:
+      name: "direct-route-dest"
+      settings:
+      - endpoints:
+        - host: "1.2.3.4"
+          port: 50000
+    directResponse:
+      body: "Unknown custom filter type: UnsupportedType"
+      statusCode: 500
diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-formatters.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-formatters.clusters.yaml
@@ -0,0 +1,49 @@
+- circuitBreakers:
+    thresholds:
+    - maxRetries: 1024
+  commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_ONLY
+  edsClusterConfig:
+    edsConfig:
+      ads: {}
+      resourceApiVersion: V3
+    serviceName: direct-route-dest
+  lbPolicy: LEAST_REQUEST
+  name: direct-route-dest
+  outlierDetection: {}
+  perConnectionBufferLimitBytes: 32768
+  type: EDS
+- circuitBreakers:
+    thresholds:
+    - maxRetries: 1024
+  commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_ONLY
+  dnsRefreshRate: 30s
+  lbPolicy: LEAST_REQUEST
+  loadAssignment:
+    clusterName: accesslog|otel-collector.default.svc.cluster.local|4317
+    endpoints:
+    - lbEndpoints:
+      - endpoint:
+          address:
+            socketAddress:
+              address: otel-collector.default.svc.cluster.local
+              portValue: 4317
+        loadBalancingWeight: 1
+      loadBalancingWeight: 1
+      locality:
+        region: accesslog|otel-collector.default.svc.cluster.local|4317/backend/0
+  name: accesslog|otel-collector.default.svc.cluster.local|4317
+  outlierDetection: {}
+  perConnectionBufferLimitBytes: 32768
+  respectDnsTtl: true
+  type: STRICT_DNS
+  typedExtensionProtocolOptions:
+    envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+      '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+      explicitHttpConfig:
+        http2ProtocolOptions: {}
diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-formatters.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-formatters.endpoints.yaml
@@ -0,0 +1,12 @@
+- clusterName: direct-route-dest
+  endpoints:
+  - lbEndpoints:
+    - endpoint:
+        address:
+          socketAddress:
+            address: 1.2.3.4
+            portValue: 50000
+      loadBalancingWeight: 1
+    loadBalancingWeight: 1
+    locality:
+      region: direct-route-dest/backend/0