support dnstap policy (#593)

* support dnstap policy
* new tests

README.md

@@ -1,8 +1,8 @@
 <p align="center">
 <img src="https://goreportcard.com/badge/github.com/dmachard/go-dns-collector" alt="Go Report"/>
 <img src="https://img.shields.io/badge/go%20version-min%201.20-green" alt="Go version"/>
-<img src="https://img.shields.io/badge/go%20tests-411-green" alt="Go tests"/>
-<img src="https://img.shields.io/badge/go%20lines-37991-green" alt="Go lines"/>
+<img src="https://img.shields.io/badge/go%20tests-414-green" alt="Go tests"/>
+<img src="https://img.shields.io/badge/go%20lines-38159-green" alt="Go lines"/>
 </p>
 
 <p align="center">

dnsutils/message.go

@@ -142,6 +142,11 @@ type DNSTap struct {
 	LatencySec       string  `json:"latency" msgpack:"latency"`
 	Payload          []byte  `json:"-" msgpack:"-"`
 	Extra            string  `json:"extra" msgpack:"extra"`
+	PolicyRule       string  `json:"policy-rule" msgpack:"policy-rule"`
+	PolicyType       string  `json:"policy-type" msgpack:"policy-type"`
+	PolicyMatch      string  `json:"policy-match" msgpack:"policy-match"`
+	PolicyAction     string  `json:"policy-action" msgpack:"policy-action"`
+	PolicyValue      string  `json:"policy-value" msgpack:"policy-value"`
 }
 
 type PowerDNS struct {
@@ -254,6 +259,11 @@ func (dm *DNSMessage) Init() {
 		TimestampRFC3339: "-",
 		LatencySec:       "-",
 		Extra:            "-",
+		PolicyRule:       "-",
+		PolicyType:       "-",
+		PolicyMatch:      "-",
+		PolicyAction:     "-",
+		PolicyValue:      "-",
 	}
 
 	dm.DNS = DNS{
@@ -349,25 +359,25 @@ func (dm *DNSMessage) handlePdnsDirectives(directives []string, s *strings.Build
 			} else {
 				s.WriteString("-")
 			}
-		case directive == "powerdns-applied-hit":
+		case directive == "powerdns-applied-policy-hit":
 			if len(dm.PowerDNS.AppliedPolicyHit) > 0 {
 				s.WriteString(dm.PowerDNS.AppliedPolicyHit)
 			} else {
 				s.WriteString("-")
 			}
-		case directive == "powerdns-applied-kind":
+		case directive == "powerdns-applied-policy-kind":
 			if len(dm.PowerDNS.AppliedPolicyKind) > 0 {
 				s.WriteString(dm.PowerDNS.AppliedPolicyKind)
 			} else {
 				s.WriteString("-")
 			}
-		case directive == "powerdns-applied-trigger":
+		case directive == "powerdns-applied-policy-trigger":
 			if len(dm.PowerDNS.AppliedPolicyTrigger) > 0 {
 				s.WriteString(dm.PowerDNS.AppliedPolicyTrigger)
 			} else {
 				s.WriteString("-")
 			}
-		case directive == "powerdns-applied-type":
+		case directive == "powerdns-applied-policy-type":
 			if len(dm.PowerDNS.AppliedPolicyType) > 0 {
 				s.WriteString(dm.PowerDNS.AppliedPolicyType)
 			} else {
@@ -597,6 +607,16 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo
 			s.WriteString(dm.DNSTap.Version)
 		case directive == "extra":
 			s.WriteString(dm.DNSTap.Extra)
+		case directive == "policy-rule":
+			s.WriteString(dm.DNSTap.PolicyRule)
+		case directive == "policy-type":
+			s.WriteString(dm.DNSTap.PolicyType)
+		case directive == "policy-action":
+			s.WriteString(dm.DNSTap.PolicyAction)
+		case directive == "policy-match":
+			s.WriteString(dm.DNSTap.PolicyMatch)
+		case directive == "policy-value":
+			s.WriteString(dm.DNSTap.PolicyValue)
 		case directive == "operation":
 			s.WriteString(dm.DNSTap.Operation)
 		case directive == "rcode":

dnsutils/message_test.go

@@ -199,7 +199,12 @@ func TestDnsMessage_Json_Reference(t *testing.T) {
 				  "version": "-",
 				  "timestamp-rfc3339ns": "-",
 				  "latency": "-",
-				  "extra": "-"
+				  "extra": "-",
+				  "policy-type": "-",
+				  "policy-action": "-",
+				  "policy-match": "-",
+				  "policy-value": "-",
+				  "policy-rule": "-"
 				}
 			}
 			`
@@ -250,6 +255,11 @@ func TestDnsMessage_JsonFlatten_Reference(t *testing.T) {
 					"dnstap.timestamp-rfc3339ns": "-",
 					"dnstap.version": "-",
 					"dnstap.extra": "-",
+					"dnstap.policy-rule": "-",
+					"dnstap.policy-type": "-",
+					"dnstap.policy-action": "-",
+					"dnstap.policy-match": "-",
+					"dnstap.policy-value": "-",
 					"edns.dnssec-ok": 0,
 					"edns.options": [],
 					"edns.rcode": 0,
@@ -282,6 +292,65 @@ func TestDnsMessage_JsonFlatten_Reference(t *testing.T) {
 	}
 }
 
+func TestDnsMessage_Json_Collectors_Reference(t *testing.T) {
+	testcases := []struct {
+		collector string
+		dmRef     DNSMessage
+		jsonRef   string
+	}{
+		{
+			collector: "powerdns",
+			dmRef: DNSMessage{PowerDNS: &PowerDNS{
+				OriginalRequestSubnet: "subnet",
+				AppliedPolicy:         "basicrpz",
+				AppliedPolicyHit:      "hit",
+				AppliedPolicyKind:     "kind",
+				AppliedPolicyTrigger:  "trigger",
+				AppliedPolicyType:     "type",
+				Tags:                  []string{"tag1"},
+				Metadata:              map[string]string{"stream_id": "collector"},
+			}},
+
+			jsonRef: `{
+						"powerdns": {
+							"original-request-subnet": "subnet",
+							"applied-policy": "basicrpz",
+							"applied-policy-hit": "hit",
+							"applied-policy-kind": "kind",
+							"applied-policy-trigger": "trigger",
+							"applied-policy-type": "type",
+							"tags": ["tag1"],
+							"metadata": {
+								"stream_id": "collector"
+							}
+						}
+					}`,
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.collector, func(t *testing.T) {
+
+			tc.dmRef.Init()
+
+			var dmMap map[string]interface{}
+			err := json.Unmarshal([]byte(tc.dmRef.ToJSON()), &dmMap)
+			if err != nil {
+				t.Fatalf("could not unmarshal dm json: %s\n", err)
+			}
+
+			var refMap map[string]interface{}
+			err = json.Unmarshal([]byte(tc.jsonRef), &refMap)
+			if err != nil {
+				t.Fatalf("could not unmarshal ref json: %s\n", err)
+			}
+
+			if !reflect.DeepEqual(dmMap[tc.collector], refMap[tc.collector]) {
+				t.Errorf("json format different from reference, Get=%s Want=%s", dmMap[tc.collector], refMap[tc.collector])
+			}
+		})
+	}
+}
+
 func TestDnsMessage_Json_Transforms_Reference(t *testing.T) {
 
 	testcases := []struct {
@@ -514,6 +583,13 @@ func TestDnsMessage_TextFormat_DefaultDirectives(t *testing.T) {
 			dm:       DNSMessage{NetworkInfo: DNSNetInfo{ResponseIP: "1.2.3.4", ResponsePort: "4200"}},
 			expected: "1.2.3.4 4200",
 		},
+		{
+			format: "policy-rule policy-type policy-action policy-match policy-value",
+			dm: DNSMessage{DNSTap: DNSTap{PolicyRule: "rule", PolicyType: "type",
+				PolicyAction: "action", PolicyMatch: "match",
+				PolicyValue: "value"}},
+			expected: "rule type action match value",
+		},
 	}
 
 	for _, tc := range testcases {
@@ -687,10 +763,16 @@ func TestDnsMessage_TextFormat_Directives_Pdns(t *testing.T) {
 			expected: "- - - -",
 		},
 		{
-			name:     "applied_policy",
-			format:   "powerdns-applied-policy",
-			dm:       DNSMessage{PowerDNS: &PowerDNS{AppliedPolicy: "test"}},
-			expected: "test",
+			name:   "applied_policy",
+			format: "powerdns-applied-policy powerdns-applied-policy-hit powerdns-applied-policy-kind powerdns-applied-policy-trigger powerdns-applied-policy-type",
+			dm: DNSMessage{PowerDNS: &PowerDNS{
+				AppliedPolicy:        "policy",
+				AppliedPolicyHit:     "hit",
+				AppliedPolicyKind:    "kind",
+				AppliedPolicyTrigger: "trigger",
+				AppliedPolicyType:    "type",
+			}},
+			expected: "policy hit kind trigger type",
 		},
 		{
 			name:     "original_request_subnet",

docs/collectors/collector_dnstap.md

@@ -7,42 +7,33 @@ The traffic can be a tcp or unix DNStap stream. TLS is also supported.
 
 Options:
 
-- `listen-ip`: (string) listen on ip
-- `listen-port`: (integer) listening on port
-- `sock-path`: (string) unix socket path
-- `tls-support:`: (boolean) to enable, set to true
-- `tls-min-version`: (string) min tls version
-- `cert-file`: (string) certificate server file
-- `key-file`: (string) private key server file
-- `sock-rcvbuf`: (integer) sets the socket receive buffer in bytes SO_RCVBUF, set to zero to use the default system value
+- `listen-ip` (str) local address to bind to. Defaults to `0.0.0.0`.
+  > Set the local address that the server will bind to. If not provided, the server will bind to all available network interfaces (0.0.0.0).
+- `listen-port` (int) local port to bind to. Defaults to `6000`.
+  > Set the local port that the server will listen on. If not provided, use the default port.
+- `sock-path` (str) Unix socket path. Default to `null`.
+  > Specify the path for the Unix socket to be created.
+- `tls-support` (bool) set to true to enable TLS. Defaults to `false`.
+  > Enables or disables TLS (Transport Layer Security) support. If set to true, TLS will be used for secure communication.
+- `tls-min-version` (str) Minimun TLS version to use. Default to `1.2`.
+  > Specifies the minimum TLS version that the server will support.
+- `cert-file` (str) path to a certificate server file to use. Default to `(empty)`.
+  > Specifies the path to the certificate file to be used for TLS. This is a required parameter if TLS support is enabled.
+- `key-file`(str) path to a key server file to use. Default to `(empty)`.
+  > Specifies the path to the key file corresponding to the certificate file. This is a required parameter if TLS support is enabled.
+- `sock-rcvbuf` (int) sets the socket receive buffer in bytes SO_RCVBUF. Default to `0`.
   > This advanced parameter allows fine-tuning of network performance by adjusting the amount of data the socket can receive before signaling to the sender to slow down.
-- `reset-conn`: (bool) Reset TCP connection on exit
-  > Send a TCP reset to the remote sender to ensure a proper termination of the connection.
-- `chan-buffer-size`: (integer) channel buffer size used on incoming packet, number of packet before to drop it.
-- `disable-dnsparser"`: (bool) disable the minimalist DNS parser
+  > Set to zero to use the default system value.
+- `reset-conn` (bool) reset TCP connection on exit. Default to `true`.
+  > Set whether to send a TCP Reset to force the cleanup of the connection on the remote side when the server exits.
+- `chan-buffer-size` (int) incoming channel size, number of packet before to drop it. Default to `65535`.
+  > Specifies the maximum number of packets that can be buffered before dropping additional packets.
+- `disable-dnsparser"` (bool) disable the minimalist DNS parser. Defaults to `false`.
   > Some JSON keys should not be available, such as `dns.id`, `dns.flags`, ...
-- `extended-support`: (boolen) decode the extended extra field sent by DNScollector
+- `extended-support` (bool) decode the extended extra field sent by DNScollector. Defaults to `false`.
   > If this setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./../../dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector.
   > This field will contain additional metadata generated by various transformations such as filtering, ATags, and others.
 
-Default values:
-
-```yaml
-dnstap:
-  listen-ip: 0.0.0.0
-  listen-port: 6000
-  sock-path: null
-  tls-support: false
-  tls-min-version: 1.2
-  cert-file: ""
-  key-file: ""
-  sock-rcvbuf: 0
-  reset-conn: true
-  chan-buffer-size: 65535
-  disable-dnsparser: false
-  extended-support: false
-```
-
 ## DNS tap Proxifier
 
 Collector that receives DNSTAP traffic and relays it without decoding or transformations.
@@ -54,23 +45,17 @@ For config examples, take a look to the following [one](../_examples/use-case-12
 
 Options:
 
-- `listen-ip`: (string) listen on ip
-- `listen-port`: (integer) listening on port
-- `sock-path`: (string) unix socket path
-- `tls-support:`: (boolean) to enable, set to true
-- `tls-min-version`: (string) min tls version
-- `cert-file`: (string) certificate server file
-- `key-file`: (string) private key server file
-
-Default values:
-
-```yaml
-dnstap-relay:
-  listen-ip: 0.0.0.0
-  listen-port: 6000
-  sock-path: null
-  tls-support: false
-  tls-min-version: 1.2
-  cert-file: ""
-  key-file: ""
-```
+- `listen-ip` (str) local address to bind to. Defaults to `0.0.0.0`.
+  > Set the local address that the server will bind to. If not provided, the server will bind to all available network interfaces (0.0.0.0).
+- `listen-port` (int) local port to bind to. Defaults to `6000`.
+  > Set the local port that the server will listen on. If not provided, use the default port.
+- `sock-path` (str) Unix socket path. Default to `null`.
+  > Specify the path for the Unix socket to be created.
+- `tls-support` (bool) set to true to enable TLS. Defaults to `false`.
+  > Enables or disables TLS (Transport Layer Security) support. If set to true, TLS will be used for secure communication.
+- `tls-min-version` (str) Minimun TLS version to use. Default to `1.2`.
+  > Specifies the minimum TLS version that the server will support.
+- `cert-file` (str) path to a certificate server file to use. Default to `(empty)`.
+  > Specifies the path to the certificate file to be used for TLS. This is a required parameter if TLS support is enabled.
+- `key-file`(str) path to a key server file to use. Default to `(empty)`.
+  > Specifies the path to the key file corresponding to the certificate file. This is a required parameter if TLS support is enabled.

docs/collectors/collector_powerdns.md

@@ -7,7 +7,7 @@ Settings:
 - `listen-ip` (str) local address to bind to. Defaults to `0.0.0.0`.
   > Set the local address that the server will bind to. If not provided, the server will bind to all available network interfaces (0.0.0.0).
 - `listen-port` (int) local port to bind to. Defaults to `6001`.
-  > Set the local port that the server will listen on. If not provided, the default port is 6001.
+  > Set the local port that the server will listen on. If not provided, use the default port.
 - `tls-support` (bool) set to true to enable TLS. Defaults to `false`.
   > Enables or disables TLS (Transport Layer Security) support. If set to true, TLS will be used for secure communication.
 - `tls-min-version` (str) Minimun TLS version to use. Default to `1.2`.
@@ -17,6 +17,7 @@ Settings:
 - `key-file`(str) path to a key server file to use. Default to `(empty)`.
   > Specifies the path to the key file corresponding to the certificate file. This is a required parameter if TLS support is enabled.
 - `sock-rcvbuf` (int) sets the socket receive buffer in bytes SO_RCVBUF. Default to `0`.
+  > This advanced parameter allows fine-tuning of network performance by adjusting the amount of data the socket can receive before signaling to the sender to slow down.
   > Set to zero to use the default system value.
 - `reset-conn` (bool) reset TCP connection on exit. Default to `true`.
   > Set whether to send a TCP Reset to force the cleanup of the connection on the remote side when the server exits.

docs/configuration.md

@@ -70,6 +70,11 @@ Default directives:
 - `version`: dnstap version
 - `extra`: dnstap extra as string
 - `operation`: dnstap operation
+- `policy-rule`: dnstap policy rule
+- `policy-type`: dnstap policy type
+- `policy-action`: dnstap policy action
+- `policy-match`: dnstap policy match
+- `policy-value`: dnstap policy value
 - `opcode`: dns opcode (integer)
 - `rcode`: dns return code
 - `queryip`: dns query ip

docs/dnsjson.md

@@ -75,7 +75,12 @@ Example:
     "version": "-",
     "extra": "-",
     "timestamp-rfc3339ns": "2021-12-27T14:33:44.559002118Z",
-    "latency": "0.014617"
+    "latency": "0.014617",
+    "policy-rule": "-",
+    "policy-type": "-",
+    "policy-action": "-",
+    "policy-match": "-",
+    "policy-value": "-",
   }
 }
 ```
@@ -113,6 +118,11 @@ Using flat-json requires more processing on the host running go-dnscollector but
   "dnstap.timestamp-rfc3339ns": "2023-03-31T10:14:46.664534902Z",
   "dnstap.version": "BIND 9.18.13-1+ubuntu20.04.1+isc+1-Ubuntu",
   "dnstap.extra": "-",
+  "dnstap.policy-rule": "-",
+  "dnstap.policy-type": "-",
+  "dnstap.policy-action": "-",
+  "dnstap.policy-match": "-",
+  "dnstap.policy-value": "-",
   "edns.dnssec-ok": 0,
   "edns.options.0.code": 10,
   "edns.options.0.data": "-",

processors/dnstap.go

@@ -276,6 +276,32 @@ RUN_LOOP:
 				dm.DNSTap.TimeNsec = int(dt.GetMessage().GetResponseTimeNsec())
 			}
 
+			// policy
+			policyType := dt.GetMessage().GetPolicy().GetType()
+			if len(policyType) > 0 {
+				dm.DNSTap.PolicyType = policyType
+			}
+
+			policyRule := string(dt.GetMessage().GetPolicy().GetRule())
+			if len(policyRule) > 0 {
+				dm.DNSTap.PolicyRule = policyRule
+			}
+
+			policyAction := dt.GetMessage().GetPolicy().GetAction().String()
+			if len(policyAction) > 0 {
+				dm.DNSTap.PolicyAction = policyAction
+			}
+
+			policyMatch := dt.GetMessage().GetPolicy().GetMatch().String()
+			if len(policyMatch) > 0 {
+				dm.DNSTap.PolicyMatch = policyMatch
+			}
+
+			policyValue := string(dt.GetMessage().GetPolicy().GetValue())
+			if len(policyValue) > 0 {
+				dm.DNSTap.PolicyValue = policyValue
+			}
+
 			// compute timestamp
 			ts := time.Unix(int64(dm.DNSTap.TimeSec), int64(dm.DNSTap.TimeNsec))
 			dm.DNSTap.Timestamp = ts.UnixNano()