normalize transformer: add icann managed flag in dnsmessage & new ben…

…ch tests (#640)

* add bench tests for normalize trasnform
* add icann managed flag in dnsmessage

.github/workflows/bench-go.yml

@@ -56,4 +56,4 @@ jobs:
     - name: Bench transformers
       run: |
         cd transformers/
-        go test -benchmem -run=^$ -bench=^BenchmarkUserPrivacy.*\|BenchmarkTransforms.*$
+        go test -benchmem -run=^$ -bench=^BenchmarkUserPrivacy.*\|BenchmarkTransforms.*\|BenchmarkNormalize.*$

README.md

@@ -2,7 +2,7 @@
 <img src="https://goreportcard.com/badge/github.com/dmachard/go-dns-collector" alt="Go Report"/>
 <img src="https://img.shields.io/badge/go%20version-min%201.20-green" alt="Go version"/>
 <img src="https://img.shields.io/badge/go%20tests-429-green" alt="Go tests"/>
-<img src="https://img.shields.io/badge/go%20bench-14-green" alt="Go bench"/>
+<img src="https://img.shields.io/badge/go%20bench-18-green" alt="Go bench"/>
 <img src="https://img.shields.io/badge/go%20lines-38661-green" alt="Go lines"/>
 </p>
 

config.yml

@@ -826,6 +826,7 @@ multiplexer:
 # # additionnals directive for text format
 # # - publicsuffix-tld: tld
 # # - publicsuffix-etld+1: effective tld plus one
+# # - publicsuffix-managed-icann: flag for managed icann domains
 # normalize:
 #   # Wwww.GooGlE.com will be equal to www.google.com
 #   qname-lowercase: true

dnsutils/message.go

@@ -186,6 +186,7 @@ type TransformSuspicious struct {
 type TransformPublicSuffix struct {
 	QnamePublicSuffix        string `json:"tld"`
 	QnameEffectiveTLDPlusOne string `json:"etld+1"`
+	ManagedByICANN           bool   `json:"managed-icann"`
 }
 
 type TransformExtracted struct {
@@ -456,6 +457,12 @@ func (dm *DNSMessage) handlePublicSuffixDirectives(directive string, s *strings.
 			s.WriteString(dm.PublicSuffix.QnamePublicSuffix)
 		case directive == "publixsuffix-etld+1":
 			s.WriteString(dm.PublicSuffix.QnameEffectiveTLDPlusOne)
+		case directive == "publixsuffix-managed-icann":
+			if dm.PublicSuffix.ManagedByICANN {
+				s.WriteString("managed")
+			} else {
+				s.WriteString("private")
+			}
 		default:
 			return errors.New(ErrorUnexpectedDirective + directive)
 		}
@@ -1169,6 +1176,7 @@ func (dm *DNSMessage) Flatten() (map[string]interface{}, error) {
 	if dm.PublicSuffix != nil {
 		dnsFields["publicsuffix.tld"] = dm.PublicSuffix.QnamePublicSuffix
 		dnsFields["publicsuffix.etld+1"] = dm.PublicSuffix.QnameEffectiveTLDPlusOne
+		dnsFields["publicsuffix.managed-icann"] = dm.PublicSuffix.ManagedByICANN
 	}
 
 	// Add TransformExtracted fields

dnsutils/message_test.go

@@ -363,12 +363,14 @@ func TestDnsMessage_Json_Transforms_Reference(t *testing.T) {
 				PublicSuffix: &TransformPublicSuffix{
 					QnamePublicSuffix:        "com",
 					QnameEffectiveTLDPlusOne: "hello.com",
+					ManagedByICANN:           true,
 				},
 			},
 			jsonRef: `{
 						"publicsuffix": {
 							"tld": "com",
-							"etld+1": "hello.com"
+							"etld+1": "hello.com",
+							"managed-icann": true
 						}
 					}`,
 		},

docs/transformers/transform_normalize.md

@@ -51,10 +51,12 @@ Example:
 "publicsuffix": {
   "etld+1": "eu.org",
   "tld": "org",
+  "managed-icann": true
 }
 ```
 
 Specific directives added for text format:
 
 - `publicsuffix-tld`: [Public Suffix](https://publicsuffix.org/) of the DNS QNAME
 - `publicsuffix-etld+1`: [Public Suffix](https://publicsuffix.org/) plus one label of the DNS QNAME
+- `publicsuffix-managed-icann`: [Public Suffix](https://publicsuffix.org/) flag for managed icann domains

transformers/normalize.go

@@ -138,6 +138,7 @@ func (p *NormalizeProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) {
 		dm.PublicSuffix = &dnsutils.TransformPublicSuffix{
 			QnamePublicSuffix:        "-",
 			QnameEffectiveTLDPlusOne: "-",
+			ManagedByICANN:           false,
 		}
 	}
 }
@@ -174,8 +175,9 @@ func (p *NormalizeProcessor) GetEffectiveTld(dm *dnsutils.DNSMessage) int {
 	etld, icann := publicsuffixlist.PublicSuffix(qname)
 	if icann {
 		dm.PublicSuffix.QnamePublicSuffix = etld
+		dm.PublicSuffix.ManagedByICANN = true
 	} else {
-		p.logError("suffix unmanaged by icann: %s", qname)
+		dm.PublicSuffix.ManagedByICANN = false
 	}
 	return ReturnSuccess
 }

transformers/normalize_test.go

@@ -1,8 +1,6 @@
 package transformers
 
 import (
-	"encoding/json"
-	"reflect"
 	"strings"
 	"testing"
 
@@ -11,52 +9,6 @@ import (
 	"github.com/dmachard/go-logger"
 )
 
-func TestNormalize_Json(t *testing.T) {
-	// enable feature
-	config := pkgconfig.GetFakeConfigTransformers()
-
-	log := logger.New(false)
-	outChans := []chan dnsutils.DNSMessage{}
-
-	// get fake
-	dm := dnsutils.GetFakeDNSMessage()
-	dm.Init()
-
-	// init subproccesor
-	qnameNorm := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, outChans, log.Info, log.Error)
-	qnameNorm.InitDNSMessage(&dm)
-
-	// expected json
-	refJSON := `
-			{
-				"publicsuffix": {
-					"tld":"-",
-					"etld+1":"-"
-				}
-			}
-			`
-
-	var dmMap map[string]interface{}
-	err := json.Unmarshal([]byte(dm.ToJSON()), &dmMap)
-	if err != nil {
-		t.Fatalf("could not unmarshal dm json: %s\n", err)
-	}
-
-	var refMap map[string]interface{}
-	err = json.Unmarshal([]byte(refJSON), &refMap)
-	if err != nil {
-		t.Fatalf("could not unmarshal ref json: %s\n", err)
-	}
-
-	if _, ok := dmMap["publicsuffix"]; !ok {
-		t.Fatalf("transformer key is missing")
-	}
-
-	if !reflect.DeepEqual(dmMap["publicsuffix"], refMap["publicsuffix"]) {
-		t.Errorf("json format different from reference")
-	}
-}
-
 func TestNormalize_LowercaseQname(t *testing.T) {
 	// enable feature
 	config := pkgconfig.GetFakeConfigTransformers()
@@ -199,3 +151,122 @@ func TestNormalize_AddTldPlusOne(t *testing.T) {
 		})
 	}
 }
+
+func TestNormalize_SuffixUnmanaged(t *testing.T) {
+	// enable feature
+	config := pkgconfig.GetFakeConfigTransformers()
+	log := logger.New(true)
+	outChans := []chan dnsutils.DNSMessage{}
+
+	// init the processor
+	psl := NewNormalizeSubprocessor(config, logger.New(true), "test", 0, outChans, log.Info, log.Error)
+
+	dm := dnsutils.GetFakeDNSMessage()
+	// https://publicsuffix.org/list/effective_tld_names.dat
+	// // ===BEGIN ICANN DOMAINS===
+	// ....
+	// // ===END ICANN DOMAINS===
+	// ===BEGIN PRIVATE DOMAINS===
+	// ..
+	dm.DNS.Qname = "play.googleapis.com"
+	// // ===END PRIVATE DOMAINS===
+
+	psl.InitDNSMessage(&dm)
+	psl.GetEffectiveTld(&dm)
+	if dm.PublicSuffix.ManagedByICANN {
+		t.Errorf("Qname %s should be private domains", dm.DNS.Qname)
+	}
+}
+
+func TestNormalize_SuffixICANNManaged(t *testing.T) {
+	// enable feature
+	config := pkgconfig.GetFakeConfigTransformers()
+	log := logger.New(true)
+	outChans := []chan dnsutils.DNSMessage{}
+
+	// init the processor
+	psl := NewNormalizeSubprocessor(config, logger.New(true), "test", 0, outChans, log.Info, log.Error)
+
+	dm := dnsutils.GetFakeDNSMessage()
+	// https://publicsuffix.org/list/effective_tld_names.dat
+	// // ===BEGIN ICANN DOMAINS===
+	dm.DNS.Qname = "fr.wikipedia.org"
+	// // ===END ICANN DOMAINS===
+	// ===BEGIN PRIVATE DOMAINS===
+	// ..
+	// // ===END PRIVATE DOMAINS===
+
+	psl.InitDNSMessage(&dm)
+	psl.GetEffectiveTld(&dm)
+	if !dm.PublicSuffix.ManagedByICANN {
+		t.Errorf("Qname %s should be ICANN managed", dm.DNS.Qname)
+	}
+}
+
+// bench tests
+
+func BenchmarkNormalize_GetEffectiveTld(b *testing.B) {
+	config := pkgconfig.GetFakeConfigTransformers()
+
+	log := logger.New(false)
+	channels := []chan dnsutils.DNSMessage{}
+
+	subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+	dm := dnsutils.GetFakeDNSMessage()
+	dm.DNS.Qname = "en.wikipedia.org"
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		subprocessor.InitDNSMessage(&dm)
+		subprocessor.GetEffectiveTld(&dm)
+	}
+}
+
+func BenchmarkNormalize_GetEffectiveTldPlusOne(b *testing.B) {
+	config := pkgconfig.GetFakeConfigTransformers()
+
+	log := logger.New(false)
+	channels := []chan dnsutils.DNSMessage{}
+
+	subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+	dm := dnsutils.GetFakeDNSMessage()
+	dm.DNS.Qname = "en.wikipedia.org"
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		subprocessor.InitDNSMessage(&dm)
+		subprocessor.GetEffectiveTld(&dm)
+	}
+}
+
+func BenchmarkNormalize_QnameLowercase(b *testing.B) {
+	config := pkgconfig.GetFakeConfigTransformers()
+
+	log := logger.New(false)
+	channels := []chan dnsutils.DNSMessage{}
+
+	subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+	dm := dnsutils.GetFakeDNSMessage()
+	dm.DNS.Qname = "EN.Wikipedia.Org"
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		subprocessor.LowercaseQname(&dm)
+	}
+}
+
+func BenchmarkNormalize_QuietText(b *testing.B) {
+	config := pkgconfig.GetFakeConfigTransformers()
+
+	log := logger.New(false)
+	channels := []chan dnsutils.DNSMessage{}
+
+	subprocessor := NewNormalizeSubprocessor(config, logger.New(false), "test", 0, channels, log.Info, log.Error)
+	dm := dnsutils.GetFakeDNSMessage()
+	dm.DNS.Qname = "EN.Wikipedia.Org"
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		subprocessor.QuietText(&dm)
+	}
+}