dnsmessage collector: add missing docs and tests (#717)

* docs for dns message
* add test for #570

README.md

@@ -1,7 +1,7 @@
 <p align="center">
 <img src="https://goreportcard.com/badge/github.com/dmachard/go-dns-collector" alt="Go Report"/>
 <img src="https://img.shields.io/badge/go%20version-min%201.21-green" alt="Go version"/>
-<img src="https://img.shields.io/badge/go%20tests-425-green" alt="Go tests"/>
+<img src="https://img.shields.io/badge/go%20tests-426-green" alt="Go tests"/>
 <img src="https://img.shields.io/badge/go%20bench-20-green" alt="Go bench"/>
 <img src="https://img.shields.io/badge/go%20lines-29697-green" alt="Go lines"/>
 </p>
@@ -37,7 +37,7 @@
   - *Listen for logging traffic with streaming network protocols*
     - [`DNStap`](docs/collectors/collector_dnstap.md#dns-tap) with `tls`|`tcp`|`unix` transports support and [`proxifier`](docs/collectors/collector_dnstap.md#dns-tap-proxifier)
     - [`PowerDNS`](docs/collectors/collector_powerdns.md) streams with full  support
-    - [`DNSMessage`](docs/collectors/collector_dnsmessage.md) for internal DNS data structure
+    - [`DNSMessage`](docs/collectors/collector_dnsmessage.md) to route DNS messages based on specific dns fields
     - [`TZSP`](docs/collectors/collector_tzsp.md) protocol support
   - *Live capture on a network interface*
     - [`AF_PACKET`](docs/collectors/collector_afpacket.md) socket with BPF filter

docs/collectors/collector_dnsmessage.md

@@ -1,5 +1,75 @@
 # Collector: DNSMessage
 
-> Only available with pipelines!
+Collector to match specific DNS messages.
 
-Collector to handle internal DNS data structure.
+Options:
+
+* `chan-buffer-size` (int)
+  > Specifies the maximum number of packets that can be buffered before dropping additional packets.
+
+* `matching` (map)
+    * `include` (map)
+    > Defines the list of fields (flat-json) which must be present in the DNS message (regex are supported).
+
+    * `exclude` (map)
+    > Defines the list of fields (flat-json) which must not be present in the DNS message (regex are supported).
+
+
+The matching functionality support any type of values. For each fields, the advanced settings can be  used:
+* `greater-than` (int) 
+> Enable to match an integer value greater than the provided value.
+
+* `match-source` (string) 
+>  This specifies a URL or local file containing a list of strings to match string field
+
+* `source-kind` (string) 
+> This indicates that the `match-source`  is a list of strings or a list of regular expressions.
+> expected values: `regexp_list`, `string_list`
+
+
+To match specific answers only with a TTL greater than 300 and RDATA equal to a list of IPs.
+
+```yaml
+include:
+  dns.resource-records.an.*.ttl:
+    greater-than: 300
+  dns.resource-records.an.*.rdata:
+    - "^142\\.250\\.185\\.(196|132)$"
+    - "^143\\.251\\.185\\.(196|132)$"
+```
+Second example to match a tag at position 0
+
+```yaml
+include:
+  atags.tags.0: "TXT:apple"
+```
+
+Finally a complete full example:
+
+```yaml
+  - name: filter
+    dnsmessage:
+      matching:
+        include:
+          dns.flags.qr: false
+          dns.opcode: 0
+          dns.length:
+            greater-than: 50
+          dns.qname:
+            match-source: "file://./testsdata/filtering_keep_domains_regex.txt"
+            source-kind: "regexp_list"
+          dnstap.operation:
+            match-source: "http://127.0.0.1/operation.txt"
+            source-kind: "string_list"
+        exclude:
+          dns.qtype: [ "TXT", "MX" ]
+          dns.qname:
+            - ".*\\.github\\.com$"
+            - "^www\\.google\\.com$"
+    transforms:
+      atags:
+        tags: [ "TXT:apple", "TXT:google" ]
+    routing-policy:
+      dropped: [ outputfile ]
+      default: [ console ]
+```

docs/workers.md

@@ -10,7 +10,7 @@ A worker can act as a collector or a logger.
 | [XDP Sniffer](collectors/collector_xdp.md)            | Live capture on network interface with XDP              |
 | [AF_PACKET Sniffer](collectors/collector_afpacket.md) | Live capture on network interface with AF_PACKET socket |
 | [File Ingestor](collectors/collector_fileingestor.md) | File ingestor like pcap                                 |
-| [DNS Message](collectors/collector_dnsmessage.md)     | Internal DNS data structure                             |
+| [DNS Message](collectors/collector_dnsmessage.md)     | Matching specific DNS message                           |
 | [Console](loggers/logger_stdout.md)                   | Print logs to stdout in text, json or binary formats.   |
 | [File](loggers/logger_file.md)                        | Save logs to file in plain text or binary formats       |
 | [DNStap](loggers/logger_dnstap.md)                    | Send logs as DNStap format to a remote collector        |

workers/dnsmessage_test.go

@@ -11,7 +11,49 @@ import (
 	"github.com/dmachard/go-logger"
 )
 
-func Test_DnsMessage_BufferLoggerIsFull(t *testing.T) {
+func TestDnsMessage_RoutingPolicy(t *testing.T) {
+	// simulate next workers
+	kept := GetWorkerForTest(pkgconfig.DefaultBufferSize)
+	dropped := GetWorkerForTest(pkgconfig.DefaultBufferSize)
+
+	// config for the collector
+	config := pkgconfig.GetDefaultConfig()
+	config.Collectors.DNSMessage.Enable = true
+	config.Collectors.DNSMessage.Matching.Include = map[string]interface{}{
+		"dns.qname": "dns.collector",
+	}
+
+	// init the collector
+	c := NewDNSMessage(nil, config, logger.New(false), "test")
+	c.SetDefaultRoutes([]Worker{kept})
+	c.SetDefaultDropped([]Worker{dropped})
+
+	// start to collect and send DNS messages on it
+	go c.StartCollect()
+
+	// this message should be kept by the collector
+	dm := dnsutils.GetFakeDNSMessage()
+	c.GetInputChannel() <- dm
+
+	// this message should dropped by the collector
+	dm.DNS.Qname = "dropped.collector"
+	c.GetInputChannel() <- dm
+
+	// the 1er message should be in th k worker
+	dmKept := <-kept.GetInputChannel()
+	if dmKept.DNS.Qname != "dns.collector" {
+		t.Errorf("invalid dns message with default routing policy")
+	}
+
+	// the 2nd message should be in the d worker
+	dmDropped := <-dropped.GetInputChannel()
+	if dmDropped.DNS.Qname != "dropped.collector" {
+		t.Errorf("invalid dns message with dropped routing policy")
+	}
+
+}
+
+func TestDnsMessage_BufferLoggerIsFull(t *testing.T) {
 	// redirect stdout output to bytes buffer
 	logsChan := make(chan logger.LogEntry, 50)
 	lg := logger.New(true)