- 
+ 
- 
+ 
diff --git a/dnsutils/dnsmessage_matching.go b/dnsutils/dnsmessage_matching.go index c4d3ec2d..374be4c4 100644 --- a/dnsutils/dnsmessage_matching.go +++ b/dnsutils/dnsmessage_matching.go @@ -22,7 +22,7 @@ func (dm *DNSMessage) Matching(matching map[string]interface{}) (error, bool) { var isMatch = true for nestedKeys, value := range matching { - realValue, found := getFieldByJSONTag(dmValue, nestedKeys) + realValue, found := GetFieldByJSONTag(dmValue, nestedKeys) if !found { return nil, false } @@ -429,7 +429,7 @@ func matchUserPattern(realValue, expectedValue reflect.Value) (bool, error) { } // getFieldByJSONTag retrieves a field value from a struct based on JSON tags. -func getFieldByJSONTag(value reflect.Value, nestedKeys string) (reflect.Value, bool) { +func GetFieldByJSONTag(value reflect.Value, nestedKeys string) (reflect.Value, bool) { listKeys := strings.SplitN(nestedKeys, ".", 2) jsonKey := listKeys[0] var remainingKeys string @@ -464,18 +464,18 @@ func getFieldByJSONTag(value reflect.Value, nestedKeys string) (reflect.Value, b // Recurse into structs or handle slices switch fieldValue.Kind() { case reflect.Struct: - return getFieldByJSONTag(fieldValue, remainingKeys) + return GetFieldByJSONTag(fieldValue, remainingKeys) case reflect.Slice: if sliceElem, leftKey, found := getSliceElement(fieldValue, remainingKeys); found { // Handle the slice element based on its kind switch sliceElem.Kind() { case reflect.Struct: - return getFieldByJSONTag(sliceElem, leftKey) + return GetFieldByJSONTag(sliceElem, leftKey) case reflect.Slice, reflect.Array: var result []interface{} for i := 0; i < sliceElem.Len(); i++ { if subElem := sliceElem.Index(i); subElem.Kind() == reflect.Struct { - if nestedValue, found := getFieldByJSONTag(subElem, leftKey); found { + if nestedValue, found := GetFieldByJSONTag(subElem, leftKey); found { result = append(result, nestedValue.Interface()) } } else { diff --git a/docs/transformers/transform_newdomaintracker.md b/docs/transformers/transform_newdomaintracker.md index b31ffdf2..c464f11d 100644 --- a/docs/transformers/transform_newdomaintracker.md +++ b/docs/transformers/transform_newdomaintracker.md @@ -27,6 +27,8 @@ The **New Domain Tracker** transformer identifies domains that are newly observe * `white-domains-file` (string) > path file to domain white list, domains list can be a partial domain name with regexp expression +* `persistence-file` (string) + > enable the persistence feature by specifying a file path ```yaml transforms: diff --git a/docs/transformers/transform_trafficreducer.md b/docs/transformers/transform_trafficreducer.md index 196b3dc1..f0a31580 100644 --- a/docs/transformers/transform_trafficreducer.md +++ b/docs/transformers/transform_trafficreducer.md @@ -2,26 +2,31 @@ # Transformer: Traffic Reducer Use this transformer to detect repetitive traffic. -A query or reply is repeated when the following criterias are the same. +A query or reply is considered repeated when the specified criteria match. -The following criterias are used: +The following criteria can be configured for detecting repetitions (default one): -- server identity -- operation -- qname or qname+1 -- query ip -- qtype +- Server identity +- Operation +- Qname or Qname+1 +- Query IP and Response IP +- Qtype Options: * `repetitive-traffic-detector` (boolean) - > detect repetitive traffic + > Detect repetitive traffic * `qname-plus-one` (boolean) - > use qname+1 instead of the complete one + > Use qname+1 instead of the full Qname for matching. * `watch-interval` (integer) - > watch interval in seconds + > Interval in seconds to aggregate and process the traffic. + +* `unique-fields` (array of strings) + > Define custom fields for uniqueness matching (limited to string and integer values). + > This allows greater flexibility in detecting repetitive traffic. + > Complete list of [fields](../dnsconversions.md#json-encoding) available. Default values: @@ -30,13 +35,20 @@ transforms: reducer: repetitive-traffic-detector: true qname-plus-one: false - watch-interval: 5 + watch-interval: 2 + unique-fields: + - dnstap.identity + - dnstap.operation + - network.query-ip + - network.response-ip + - dns.qname + - dns.qtype ``` -Specific text directive(s) available for the text format: +Specific directives available for the text output format: * `reducer-occurrences`: display the number of detected duplication -* `cumulative-length`: sum of the length of each occurrences +* `cumulative-length`: sums the lengths of all occurrences. When the feature is enabled, the following json field are populated in your DNS message: diff --git a/pkgconfig/transformers.go b/pkgconfig/transformers.go index a6c83dc3..52823d80 100644 --- a/pkgconfig/transformers.go +++ b/pkgconfig/transformers.go @@ -38,10 +38,11 @@ type ConfigTransformers struct { QueriesTimeout int `yaml:"queries-timeout" default:"2"` } `yaml:"latency"` Reducer struct { - Enable bool `yaml:"enable" default:"false"` - RepetitiveTrafficDetector bool `yaml:"repetitive-traffic-detector" default:"false"` - QnamePlusOne bool `yaml:"qname-plus-one" default:"false"` - WatchInterval int `yaml:"watch-interval" default:"5"` + Enable bool `yaml:"enable" default:"false"` + RepetitiveTrafficDetector bool `yaml:"repetitive-traffic-detector" default:"false"` + QnamePlusOne bool `yaml:"qname-plus-one" default:"false"` + WatchInterval int `yaml:"watch-interval" default:"2"` + UniqueFields []string `yaml:"unique-fields" default:"[\"dnstap.identity\", \"dnstap.operation\", \"network.query-ip\", \"network.response-ip\", \"dns.qname\", \"dns.qtype\"]"` } `yaml:"reducer"` Filtering struct { Enable bool `yaml:"enable" default:"false"` diff --git a/transformers/reducer.go b/transformers/reducer.go index c5f75e59..c2f6d97f 100644 --- a/transformers/reducer.go +++ b/transformers/reducer.go @@ -2,6 +2,8 @@ package transformers import ( "container/list" + "fmt" + "reflect" "strings" "sync" "time" @@ -133,9 +135,8 @@ func (t *ReducerTransform) repetitiveTrafficDetector(dm *dnsutils.DNSMessage) (i } t.strBuilder.Reset() - t.strBuilder.WriteString(dm.DNSTap.Identity) - t.strBuilder.WriteString(dm.DNSTap.Operation) - t.strBuilder.WriteString(dm.NetworkInfo.QueryIP) + + // update qname ? if t.config.Reducer.QnamePlusOne { qname := strings.ToLower(dm.DNS.Qname) qname = strings.TrimSuffix(qname, ".") @@ -143,8 +144,21 @@ func (t *ReducerTransform) repetitiveTrafficDetector(dm *dnsutils.DNSMessage) (i dm.DNS.Qname = etld } } - t.strBuilder.WriteString(dm.DNS.Qname) - t.strBuilder.WriteString(dm.DNS.Qtype) + + dmValue := reflect.ValueOf(dm).Elem() // Get the struct value of the DNSMessage + for _, field := range t.config.Reducer.UniqueFields { + if value, found := dnsutils.GetFieldByJSONTag(dmValue, field); found { + // Check if the field's kind is either int or string + switch value.Kind() { + case reflect.Int, reflect.String: + t.strBuilder.WriteString(fmt.Sprintf("%v", value.Interface())) // Append field value + default: + // Skip unsupported types + continue + } + } + } + dmTag := t.strBuilder.String() dmCopy := *dm diff --git a/transformers/reducer_test.go b/transformers/reducer_test.go index fccfe4c9..236de071 100644 --- a/transformers/reducer_test.go +++ b/transformers/reducer_test.go @@ -81,7 +81,7 @@ func TestReducer_RepetitiveTrafficDetector(t *testing.T) { dnsMessagesIn []dnsutils.DNSMessage }{ { - name: "norepeat", + name: "no_reduce", dnsMessagesIn: []dnsutils.DNSMessage{ { DNSTap: dnsutils.DNSTap{Operation: "CLIENT_QUERY"}, @@ -104,27 +104,50 @@ func TestReducer_RepetitiveTrafficDetector(t *testing.T) { }, }, { - name: "reduce", + name: "reduce_default_unique_fields", + dnsMessagesIn: []dnsutils.DNSMessage{ + { + DNSTap: dnsutils.DNSTap{Operation: "CLIENT_QUERY", Identity: "test"}, + DNS: dnsutils.DNS{Qname: "hello.world", Qtype: "A"}, + NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1", ResponseIP: "8.8.8.8"}, + }, + { + DNSTap: dnsutils.DNSTap{Operation: "CLIENT_QUERY", Identity: "test"}, + DNS: dnsutils.DNS{Qname: "hello.world", Qtype: "A"}, + NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1", ResponseIP: "8.8.8.8"}, + }, + }, + dnsMessagesOut: []dnsutils.DNSMessage{ + { + Reducer: &dnsutils.TransformReducer{Occurrences: 2}, + }, + }, + }, + { + name: "no_reduce_responseip_different", dnsMessagesIn: []dnsutils.DNSMessage{ { DNSTap: dnsutils.DNSTap{Operation: "CLIENT_QUERY"}, DNS: dnsutils.DNS{Qname: "hello.world", Qtype: "A"}, - NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1"}, + NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1", ResponseIP: "1.1.1.1"}, }, { DNSTap: dnsutils.DNSTap{Operation: "CLIENT_QUERY"}, DNS: dnsutils.DNS{Qname: "hello.world", Qtype: "A"}, - NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1"}, + NetworkInfo: dnsutils.DNSNetInfo{QueryIP: "127.0.0.1", ResponseIP: "8.8.8.8"}, }, }, dnsMessagesOut: []dnsutils.DNSMessage{ { - Reducer: &dnsutils.TransformReducer{Occurrences: 2}, + Reducer: &dnsutils.TransformReducer{Occurrences: 1}, + }, + { + Reducer: &dnsutils.TransformReducer{Occurrences: 1}, }, }, }, { - name: "norepeat_qtype", + name: "no_reduce_qtype_different", dnsMessagesIn: []dnsutils.DNSMessage{ { DNSTap: dnsutils.DNSTap{Operation: "CLIENT_QUERY"},