diff --git a/README.md b/README.md index 570c529f..586d4882 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

Go Report Go version -Go tests +Go tests Go bench Go lines

@@ -37,7 +37,7 @@ - *Listen for logging traffic with streaming network protocols* - [`DNStap`](docs/collectors/collector_dnstap.md#dns-tap) with `tls`|`tcp`|`unix` transports support and [`proxifier`](docs/collectors/collector_dnstap.md#dns-tap-proxifier) - [`PowerDNS`](docs/collectors/collector_powerdns.md) streams with full support - - [`DNSMessage`](docs/collectors/collector_dnsmessage.md) for internal DNS data structure + - [`DNSMessage`](docs/collectors/collector_dnsmessage.md) to route DNS messages based on specific dns fields - [`TZSP`](docs/collectors/collector_tzsp.md) protocol support - *Live capture on a network interface* - [`AF_PACKET`](docs/collectors/collector_afpacket.md) socket with BPF filter diff --git a/docs/collectors/collector_dnsmessage.md b/docs/collectors/collector_dnsmessage.md index 7073d087..03bf34bd 100644 --- a/docs/collectors/collector_dnsmessage.md +++ b/docs/collectors/collector_dnsmessage.md @@ -1,5 +1,75 @@ # Collector: DNSMessage -> Only available with pipelines! +Collector to match specific DNS messages. -Collector to handle internal DNS data structure. +Options: + +* `chan-buffer-size` (int) + > Specifies the maximum number of packets that can be buffered before dropping additional packets. + +* `matching` (map) + * `include` (map) + > Defines the list of fields (flat-json) which must be present in the DNS message (regex are supported). + + * `exclude` (map) + > Defines the list of fields (flat-json) which must not be present in the DNS message (regex are supported). + + +The matching functionality support any type of values. For each fields, the advanced settings can be used: +* `greater-than` (int) +> Enable to match an integer value greater than the provided value. + +* `match-source` (string) +> This specifies a URL or local file containing a list of strings to match string field + +* `source-kind` (string) +> This indicates that the `match-source` is a list of strings or a list of regular expressions. +> expected values: `regexp_list`, `string_list` + + +To match specific answers only with a TTL greater than 300 and RDATA equal to a list of IPs. + +```yaml +include: + dns.resource-records.an.*.ttl: + greater-than: 300 + dns.resource-records.an.*.rdata: + - "^142\\.250\\.185\\.(196|132)$" + - "^143\\.251\\.185\\.(196|132)$" +``` +Second example to match a tag at position 0 + +```yaml +include: + atags.tags.0: "TXT:apple" +``` + +Finally a complete full example: + +```yaml + - name: filter + dnsmessage: + matching: + include: + dns.flags.qr: false + dns.opcode: 0 + dns.length: + greater-than: 50 + dns.qname: + match-source: "file://./testsdata/filtering_keep_domains_regex.txt" + source-kind: "regexp_list" + dnstap.operation: + match-source: "http://127.0.0.1/operation.txt" + source-kind: "string_list" + exclude: + dns.qtype: [ "TXT", "MX" ] + dns.qname: + - ".*\\.github\\.com$" + - "^www\\.google\\.com$" + transforms: + atags: + tags: [ "TXT:apple", "TXT:google" ] + routing-policy: + dropped: [ outputfile ] + default: [ console ] +``` \ No newline at end of file diff --git a/docs/workers.md b/docs/workers.md index 075da665..3b25c183 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -10,7 +10,7 @@ A worker can act as a collector or a logger. | [XDP Sniffer](collectors/collector_xdp.md) | Live capture on network interface with XDP | | [AF_PACKET Sniffer](collectors/collector_afpacket.md) | Live capture on network interface with AF_PACKET socket | | [File Ingestor](collectors/collector_fileingestor.md) | File ingestor like pcap | -| [DNS Message](collectors/collector_dnsmessage.md) | Internal DNS data structure | +| [DNS Message](collectors/collector_dnsmessage.md) | Matching specific DNS message | | [Console](loggers/logger_stdout.md) | Print logs to stdout in text, json or binary formats. | | [File](loggers/logger_file.md) | Save logs to file in plain text or binary formats | | [DNStap](loggers/logger_dnstap.md) | Send logs as DNStap format to a remote collector | diff --git a/workers/dnsmessage_test.go b/workers/dnsmessage_test.go index bc8322a5..0ed65fc4 100644 --- a/workers/dnsmessage_test.go +++ b/workers/dnsmessage_test.go @@ -11,7 +11,49 @@ import ( "github.com/dmachard/go-logger" ) -func Test_DnsMessage_BufferLoggerIsFull(t *testing.T) { +func TestDnsMessage_RoutingPolicy(t *testing.T) { + // simulate next workers + kept := GetWorkerForTest(pkgconfig.DefaultBufferSize) + dropped := GetWorkerForTest(pkgconfig.DefaultBufferSize) + + // config for the collector + config := pkgconfig.GetDefaultConfig() + config.Collectors.DNSMessage.Enable = true + config.Collectors.DNSMessage.Matching.Include = map[string]interface{}{ + "dns.qname": "dns.collector", + } + + // init the collector + c := NewDNSMessage(nil, config, logger.New(false), "test") + c.SetDefaultRoutes([]Worker{kept}) + c.SetDefaultDropped([]Worker{dropped}) + + // start to collect and send DNS messages on it + go c.StartCollect() + + // this message should be kept by the collector + dm := dnsutils.GetFakeDNSMessage() + c.GetInputChannel() <- dm + + // this message should dropped by the collector + dm.DNS.Qname = "dropped.collector" + c.GetInputChannel() <- dm + + // the 1er message should be in th k worker + dmKept := <-kept.GetInputChannel() + if dmKept.DNS.Qname != "dns.collector" { + t.Errorf("invalid dns message with default routing policy") + } + + // the 2nd message should be in the d worker + dmDropped := <-dropped.GetInputChannel() + if dmDropped.DNS.Qname != "dropped.collector" { + t.Errorf("invalid dns message with dropped routing policy") + } + +} + +func TestDnsMessage_BufferLoggerIsFull(t *testing.T) { // redirect stdout output to bytes buffer logsChan := make(chan logger.LogEntry, 50) lg := logger.New(true)