feat(transformer): add configurable unique fields to traffic reducer (#…

…905)

* feat(transformer): add configurable unique fields to traffic reducer

README.md

@@ -1,9 +1,9 @@
 <p align="center">
   <img src="https://goreportcard.com/badge/github.com/dmachard/go-dns-collector" alt="Go Report"/>
   <img src="https://img.shields.io/badge/go%20version-min%201.21-green" alt="Go version"/>
-  <img src="https://img.shields.io/badge/go%20tests-516-green" alt="Go tests"/>
+  <img src="https://img.shields.io/badge/go%20tests-517-green" alt="Go tests"/>
   <img src="https://img.shields.io/badge/go%20bench-21-green" alt="Go bench"/>
-  <img src="https://img.shields.io/badge/go%20lines-32515-green" alt="Go lines"/>
+  <img src="https://img.shields.io/badge/go%20lines-32537-green" alt="Go lines"/>
 </p>
 
 <p align="center">

dnsutils/dnsmessage_matching.go

@@ -22,7 +22,7 @@ func (dm *DNSMessage) Matching(matching map[string]interface{}) (error, bool) {
 	var isMatch = true
 
 	for nestedKeys, value := range matching {
-		realValue, found := getFieldByJSONTag(dmValue, nestedKeys)
+		realValue, found := GetFieldByJSONTag(dmValue, nestedKeys)
 		if !found {
 			return nil, false
 		}
@@ -429,7 +429,7 @@ func matchUserPattern(realValue, expectedValue reflect.Value) (bool, error) {
 }
 
 // getFieldByJSONTag retrieves a field value from a struct based on JSON tags.
-func getFieldByJSONTag(value reflect.Value, nestedKeys string) (reflect.Value, bool) {
+func GetFieldByJSONTag(value reflect.Value, nestedKeys string) (reflect.Value, bool) {
 	listKeys := strings.SplitN(nestedKeys, ".", 2)
 	jsonKey := listKeys[0]
 	var remainingKeys string
@@ -464,18 +464,18 @@ func getFieldByJSONTag(value reflect.Value, nestedKeys string) (reflect.Value, b
 			// Recurse into structs or handle slices
 			switch fieldValue.Kind() {
 			case reflect.Struct:
-				return getFieldByJSONTag(fieldValue, remainingKeys)
+				return GetFieldByJSONTag(fieldValue, remainingKeys)
 			case reflect.Slice:
 				if sliceElem, leftKey, found := getSliceElement(fieldValue, remainingKeys); found {
 					// Handle the slice element based on its kind
 					switch sliceElem.Kind() {
 					case reflect.Struct:
-						return getFieldByJSONTag(sliceElem, leftKey)
+						return GetFieldByJSONTag(sliceElem, leftKey)
 					case reflect.Slice, reflect.Array:
 						var result []interface{}
 						for i := 0; i < sliceElem.Len(); i++ {
 							if subElem := sliceElem.Index(i); subElem.Kind() == reflect.Struct {
-								if nestedValue, found := getFieldByJSONTag(subElem, leftKey); found {
+								if nestedValue, found := GetFieldByJSONTag(subElem, leftKey); found {
 									result = append(result, nestedValue.Interface())
 								}
 							} else {

docs/transformers/transform_newdomaintracker.md

@@ -27,6 +27,8 @@ The **New Domain Tracker** transformer identifies domains that are newly observe
 * `white-domains-file` (string)
   > path file to domain white list, domains list can be a partial domain name with regexp expression
 
+* `persistence-file` (string)
+  > enable the persistence feature by specifying a file path
 
 ```yaml
 transforms:

docs/transformers/transform_trafficreducer.md

@@ -2,26 +2,31 @@
 # Transformer: Traffic Reducer
 
 Use this transformer to detect repetitive traffic.
-A query or reply is repeated when the following criterias are the same.
+A query or reply is considered repeated  when the specified criteria match.
 
-The following criterias are used:
+The following criteria can be configured for detecting repetitions (default one):
 
-- server identity
-- operation
-- qname or qname+1
-- query ip
-- qtype
+- Server identity
+- Operation
+- Qname or Qname+1
+- Query IP and Response IP
+- Qtype
 
 Options:
 
 * `repetitive-traffic-detector` (boolean)
-  > detect repetitive traffic
+  > Detect repetitive traffic
 
 * `qname-plus-one` (boolean)
-  > use qname+1 instead of the complete one
+  > Use qname+1 instead of the full Qname for matching.
 
 * `watch-interval` (integer)
-  > watch interval in seconds
+  > Interval in seconds to aggregate and process the traffic.
+
+* `unique-fields` (array of strings)  
+  > Define custom fields for uniqueness matching (limited to string and integer values). 
+  > This allows greater flexibility in detecting repetitive traffic.
+  > Complete list of [fields](../dnsconversions.md#json-encoding) available.
 
 Default values:
 
@@ -30,13 +35,20 @@ transforms:
   reducer:
     repetitive-traffic-detector: true
     qname-plus-one: false
-    watch-interval: 5
+    watch-interval: 2
+    unique-fields:
+    - dnstap.identity
+    - dnstap.operation
+    - network.query-ip
+    - network.response-ip
+    - dns.qname
+    - dns.qtype
 ```
 
-Specific text directive(s) available for the text format:
+Specific directives available for the text output format:
 
 * `reducer-occurrences`: display the number of detected duplication
-* `cumulative-length`: sum of the length of each occurrences
+* `cumulative-length`: sums the lengths of all occurrences.
 
 When the feature is enabled, the following json field are populated in your DNS message:
 

pkgconfig/transformers.go

@@ -38,10 +38,11 @@ type ConfigTransformers struct {
 		QueriesTimeout    int  `yaml:"queries-timeout" default:"2"`
 	} `yaml:"latency"`
 	Reducer struct {
-		Enable                    bool `yaml:"enable" default:"false"`
-		RepetitiveTrafficDetector bool `yaml:"repetitive-traffic-detector" default:"false"`
-		QnamePlusOne              bool `yaml:"qname-plus-one" default:"false"`
-		WatchInterval             int  `yaml:"watch-interval" default:"5"`
+		Enable                    bool     `yaml:"enable" default:"false"`
+		RepetitiveTrafficDetector bool     `yaml:"repetitive-traffic-detector" default:"false"`
+		QnamePlusOne              bool     `yaml:"qname-plus-one" default:"false"`
+		WatchInterval             int      `yaml:"watch-interval" default:"2"`
+		UniqueFields              []string `yaml:"unique-fields" default:"[\"dnstap.identity\", \"dnstap.operation\", \"network.query-ip\", \"network.response-ip\",  \"dns.qname\", \"dns.qtype\"]"`
 	} `yaml:"reducer"`
 	Filtering struct {
 		Enable          bool     `yaml:"enable" default:"false"`

transformers/reducer.go

@@ -2,6 +2,8 @@ package transformers
 
 import (
 	"container/list"
+	"fmt"
+	"reflect"
 	"strings"
 	"sync"
 	"time"
@@ -133,18 +135,30 @@ func (t *ReducerTransform) repetitiveTrafficDetector(dm *dnsutils.DNSMessage) (i
 	}
 
 	t.strBuilder.Reset()
-	t.strBuilder.WriteString(dm.DNSTap.Identity)
-	t.strBuilder.WriteString(dm.DNSTap.Operation)
-	t.strBuilder.WriteString(dm.NetworkInfo.QueryIP)
+
+	// update qname ?
 	if t.config.Reducer.QnamePlusOne {
 		qname := strings.ToLower(dm.DNS.Qname)
 		qname = strings.TrimSuffix(qname, ".")
 		if etld, err := publicsuffixlist.EffectiveTLDPlusOne(qname); err == nil {
 			dm.DNS.Qname = etld
 		}
 	}
-	t.strBuilder.WriteString(dm.DNS.Qname)
-	t.strBuilder.WriteString(dm.DNS.Qtype)
+
+	dmValue := reflect.ValueOf(dm).Elem() // Get the struct value of the DNSMessage
+	for _, field := range t.config.Reducer.UniqueFields {
+		if value, found := dnsutils.GetFieldByJSONTag(dmValue, field); found {
+			// Check if the field's kind is either int or string
+			switch value.Kind() {
+			case reflect.Int, reflect.String:
+				t.strBuilder.WriteString(fmt.Sprintf("%v", value.Interface())) // Append field value
+			default:
+				// Skip unsupported types
+				continue
+			}
+		}
+	}
+
 	dmTag := t.strBuilder.String()
 
 	dmCopy := *dm

transformers/reducer_test.go

@@ -81,7 +81,7 @@ func TestReducer_RepetitiveTrafficDetector(t *testing.T) {
 		dnsMessagesIn  []dnsutils.DNSMessage
 	}{
 		{
-			name: "norepeat",
+			name: "no_reduce",
 			dnsMessagesIn: []dnsutils.DNSMessage{
 				{
 					DNSTap:      dnsutils.DNSTap{Operation: "CLIENT_QUERY"},
@@ -104,27 +104,50 @@ func TestReducer_RepetitiveTrafficDetector(t *testing.T) {
 			},
 		},
 		{
-			name: "reduce",
+			name: "reduce_default_unique_fields",
+			dnsMessagesIn: []dnsutils.DNSMessage{
+				{
+					DNSTap:      dnsutils.DNSTap{Operation: "CLIENT_QUERY", Identity: "test"},
+					DNS:         dnsutils.DNS{Qname: "hello.world", Qtype: "A"},
+					NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1", ResponseIP: "8.8.8.8"},
+				},
+				{
+					DNSTap:      dnsutils.DNSTap{Operation: "CLIENT_QUERY", Identity: "test"},
+					DNS:         dnsutils.DNS{Qname: "hello.world", Qtype: "A"},
+					NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1", ResponseIP: "8.8.8.8"},
+				},
+			},
+			dnsMessagesOut: []dnsutils.DNSMessage{
+				{
+					Reducer: &dnsutils.TransformReducer{Occurrences: 2},
+				},
+			},
+		},
+		{
+			name: "no_reduce_responseip_different",
 			dnsMessagesIn: []dnsutils.DNSMessage{
 				{
 					DNSTap:      dnsutils.DNSTap{Operation: "CLIENT_QUERY"},
 					DNS:         dnsutils.DNS{Qname: "hello.world", Qtype: "A"},
-					NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1"},
+					NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1", ResponseIP: "1.1.1.1"},
 				},
 				{
 					DNSTap:      dnsutils.DNSTap{Operation: "CLIENT_QUERY"},
 					DNS:         dnsutils.DNS{Qname: "hello.world", Qtype: "A"},
-					NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1"},
+					NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1", ResponseIP: "8.8.8.8"},
 				},
 			},
 			dnsMessagesOut: []dnsutils.DNSMessage{
 				{
-					Reducer: &dnsutils.TransformReducer{Occurrences: 2},
+					Reducer: &dnsutils.TransformReducer{Occurrences: 1},
+				},
+				{
+					Reducer: &dnsutils.TransformReducer{Occurrences: 1},
 				},
 			},
 		},
 		{
-			name: "norepeat_qtype",
+			name: "no_reduce_qtype_different",
 			dnsMessagesIn: []dnsutils.DNSMessage{
 				{
 					DNSTap:      dnsutils.DNSTap{Operation: "CLIENT_QUERY"},