Merge pull request #48 from nixys/feat/44

feat(#44): Delete rows during anonymization
nixys · Oct 16, 2024 · 11a1161 · 11a1161
2 parents 3b480fa + 8870a26
commit 11a1161
Show file tree

Hide file tree

Showing 8 changed files with 740 additions and 46 deletions.
diff --git a/misc/template.go b/misc/template.go
@@ -7,7 +7,10 @@ import (
 	"github.com/Masterminds/sprig/v3"
 )
 
-var null = "::NULL::"
+var (
+	null = "::NULL::"
+	drop = "::DROP::"
+)
 
 type TemplateData struct {
 	TableName string
@@ -16,7 +19,7 @@ type TemplateData struct {
 }
 
 // TemplateExec makes message from given template `tpl` and data `d`
-func TemplateExec(tpl string, d any) ([]byte, error) {
+func TemplateExec(tpl string, d any) ([]byte, bool, error) {
 
 	var b bytes.Buffer
 
@@ -36,28 +39,36 @@ func TemplateExec(tpl string, d any) ([]byte, error) {
 			}
 			return false
 		}
+		t["drop"] = func() string {
+			return drop
+		}
 
 		return t
 	}()).Parse(tpl)
 	if err != nil {
-		return []byte{}, err
+		return []byte{}, false, err
 	}
 
 	err = t.Execute(&b, d)
 	if err != nil {
-		return []byte{}, err
+		return []byte{}, false, err
 	}
 
 	// Return empty line if buffer is nil
 	if b.Bytes() == nil {
-		return []byte{}, nil
+		return []byte{}, false, nil
 	}
 
 	// Return nil if buffer is NULL (with special key)
 	if bytes.Compare(b.Bytes(), []byte(null)) == 0 {
-		return nil, nil
+		return nil, false, nil
+	}
+
+	// Return `drop` value if buffer is DROP (with special key)
+	if bytes.Compare(b.Bytes(), []byte(drop)) == 0 {
+		return nil, true, nil
 	}
 
 	// Return buffer content otherwise
-	return b.Bytes(), nil
+	return b.Bytes(), false, nil
 }
diff --git a/modules/anonymizers/mysql/dh.go b/modules/anonymizers/mysql/dh.go
@@ -13,6 +13,7 @@ func dhSecurityInsertInto(usrCtx any, deferred, token []byte) ([]byte, error) {
 	uctx := usrCtx.(*userCtx)
 
 	uctx.security.tmpBuf = token
+	uctx.insertIntoBuf = nil
 
 	return deferred, nil
 }
@@ -27,14 +28,45 @@ func dhSecurityInsertIntoTableNameSearch(usrCtx any, deferred, token []byte) ([]
 	return []byte{}, nil
 }
 
-func dhSecurityNil(usrCtx any, deferred, token []byte) ([]byte, error) {
+func dhSecurityInsertIntoValues(usrCtx any, deferred, token []byte) ([]byte, error) {
 
 	uctx := usrCtx.(*userCtx)
 
 	if uctx.security.isSkip == true {
 		return []byte{}, nil
 	}
 
+	uctx.insertIntoBuf = append(uctx.insertIntoBuf, deferred...)
+	uctx.insertIntoBuf = append(uctx.insertIntoBuf, token...)
+
+	return []byte{}, nil
+}
+
+func dhSecurityInsertIntoValueSearch(usrCtx any, deferred, token []byte) ([]byte, error) {
+
+	uctx := usrCtx.(*userCtx)
+
+	if uctx.security.isSkip == true {
+		return []byte{}, nil
+	}
+
+	uctx.insertIntoBuf = append(uctx.insertIntoBuf, deferred...)
+
+	return []byte{}, nil
+}
+
+func dhSecurityValuesEnd(usrCtx any, deferred, token []byte) ([]byte, error) {
+
+	uctx := usrCtx.(*userCtx)
+
+	if uctx.security.isSkip == true {
+		return []byte{}, nil
+	}
+
+	if uctx.insertIntoBuf != nil {
+		return []byte{}, nil
+	}
+
 	return append(deferred, token...), nil
 }
 
@@ -111,17 +143,17 @@ func dhInsertIntoTableName(usrCtx any, deferred, token []byte) ([]byte, error) {
 		return []byte{}, nil
 	}
 
-	d := append(uctx.security.tmpBuf, append(deferred, token...)...)
+	uctx.insertIntoBuf = append(uctx.security.tmpBuf, append(deferred, token...)...)
 
 	uctx.security.isSkip = false
 	uctx.security.tmpBuf = []byte{}
 
 	// Check insert into table name
 	if tn != uctx.filter.TableNameGet() {
-		return d, fmt.Errorf("`create` and `insert into` table names are mismatch (create table: '%s', insert into table: '%s')", uctx.filter.TableNameGet(), tn)
+		return []byte{}, fmt.Errorf("`create` and `insert into` table names are mismatch (create table: '%s', insert into table: '%s')", uctx.filter.TableNameGet(), tn)
 	}
 
-	return d, nil
+	return []byte{}, nil
 }
 
 func dhCreateTableValues(usrCtx any, deferred, token []byte) ([]byte, error) {
@@ -173,7 +205,19 @@ func dhCreateTableValuesEnd(usrCtx any, deferred, token []byte) ([]byte, error)
 		return []byte{}, err
 	}
 
-	return rowDataGen(uctx), nil
+	b := rowDataGen(uctx)
+	if b == nil {
+		return []byte{}, nil
+	} else {
+		if uctx.insertIntoBuf != nil {
+			b = append(uctx.insertIntoBuf, b...)
+			uctx.insertIntoBuf = nil
+		} else {
+			b = append([]byte{','}, b...)
+		}
+	}
+
+	return b, nil
 }
 
 func dhCreateTableValuesStringEnd(usrCtx any, deferred, token []byte) ([]byte, error) {
@@ -189,14 +233,29 @@ func dhCreateTableValuesStringEnd(usrCtx any, deferred, token []byte) ([]byte, e
 		return []byte{}, err
 	}
 
-	return rowDataGen(uctx), nil
+	b := rowDataGen(uctx)
+	if b == nil {
+		return []byte{}, nil
+	} else {
+		if uctx.insertIntoBuf != nil {
+			b = append(uctx.insertIntoBuf, b...)
+			uctx.insertIntoBuf = nil
+		} else {
+			b = append([]byte{','}, b...)
+		}
+	}
+
+	return b, nil
 }
 
 func rowDataGen(uctx *userCtx) []byte {
 
 	var out string
 
 	row := uctx.filter.ValuePop()
+	if row.Values == nil {
+		return nil
+	}
 
 	for i, v := range row.Values {
 

diff --git a/modules/anonymizers/mysql/mysql.go b/modules/anonymizers/mysql/mysql.go
@@ -38,11 +38,12 @@ type SecurityOpts struct {
 }
 
 type userCtx struct {
-	filter     *relfilter.Filter
-	columnName string
-	security   securityCtx
-	tables     map[string]map[string]columnType
-	optKinds   []optKind
+	filter        *relfilter.Filter
+	columnName    string
+	security      securityCtx
+	tables        map[string]map[string]columnType
+	optKinds      []optKind
+	insertIntoBuf []byte
 }
 
 type securityCtx struct {
@@ -449,7 +450,7 @@ func (m *MySQL) Run(ctx context.Context, w io.Writer) error {
 									R: []byte{' ', '\n'},
 								},
 							},
-							DataHandler: dhSecurityNil,
+							DataHandler: dhSecurityInsertIntoValues,
 						},
 					},
 				},
@@ -460,7 +461,7 @@ func (m *MySQL) Run(ctx context.Context, w io.Writer) error {
 							Switch: fsm.Switch{
 								Trigger: []byte("("),
 							},
-							DataHandler: fsm.DataHandlerGenericSkipToken,
+							DataHandler: dhSecurityInsertIntoValueSearch,
 						},
 					},
 				},
@@ -526,14 +527,14 @@ func (m *MySQL) Run(ctx context.Context, w io.Writer) error {
 							Switch: fsm.Switch{
 								Trigger: []byte(","),
 							},
-							DataHandler: dhSecurityNil,
+							DataHandler: fsm.DataHandlerGenericVoid,
 						},
 						{
 							Name: stateSomeIntermediateState,
 							Switch: fsm.Switch{
 								Trigger: []byte(";"),
 							},
-							DataHandler: dhSecurityNil,
+							DataHandler: dhSecurityValuesEnd,
 						},
 					},
 				},

diff --git a/modules/anonymizers/mysql/mysql_test.go b/modules/anonymizers/mysql/mysql_test.go
@@ -0,0 +1,82 @@
+package mysql_anonymize
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"testing"
+
+	"github.com/nixys/nxs-data-anonymizer/misc"
+	"github.com/nixys/nxs-data-anonymizer/modules/filters/relfilter"
+)
+
+func TestMySQL(t *testing.T) {
+
+	var r, e bytes.Buffer
+
+	fin, err := os.Open("mysql_test.in.sql")
+	if err != nil {
+		t.Fatal("open input SQL:", err)
+	}
+
+	m, err := Init(
+		fin,
+		InitOpts{
+			Rules: RulesOpts{
+				TableRules: map[string]map[string]relfilter.ColumnRuleOpts{
+
+					// Delete only row with id `2`
+					"table1": {
+						"id": relfilter.ColumnRuleOpts{
+							Type:   misc.ValueTypeTemplate,
+							Value:  "{{ if eq .Values.id \"2\" }}{{ drop }}{{ else }}{{ .Values.id }}{{ end }}",
+							Unique: false,
+						},
+					},
+
+					// Delete all rows from table
+					"table2": {
+						"id": relfilter.ColumnRuleOpts{
+							Type:   misc.ValueTypeTemplate,
+							Value:  "{{ drop }}",
+							Unique: false,
+						},
+					},
+
+					// Delete no rows
+					"table3": {
+						"id": relfilter.ColumnRuleOpts{
+							Type:   misc.ValueTypeTemplate,
+							Value:  "{{ if eq .Values.id \"4\" }}{{ drop }}{{ else }}{{ .Values.id }}{{ end }}",
+							Unique: false,
+						},
+					},
+				},
+			},
+		},
+	)
+	if err != nil {
+		t.Fatal("init MySQL:", err)
+	}
+
+	if err := m.Run(context.Background(), &r); err != nil {
+		t.Fatal("run MySQL:", err)
+	}
+
+	fout, err := os.Open("mysql_test.out.sql")
+	if err != nil {
+		t.Fatal("open output SQL:", err)
+	}
+
+	if _, err := e.ReadFrom(fout); err != nil {
+		t.Fatal("read output SQL:", err)
+	}
+
+	// os.WriteFile("mysql_test.out.sql", r.Bytes(), 0644)
+
+	if r.String() != e.String() {
+		t.Fatal("incorrect anonymization result")
+	}
+
+	t.Logf("success")
+}