From 0c1a02d7c1c06fa11f121bc2f837dfaee309090f Mon Sep 17 00:00:00 2001 From: Boris Ershov Date: Wed, 16 Oct 2024 17:58:19 +0700 Subject: [PATCH 1/2] feat(#44): Add `drop` template function in MySQL anonymizer --- misc/template.go | 25 +- modules/anonymizers/mysql/dh.go | 70 ++++- modules/anonymizers/mysql/mysql.go | 19 +- modules/anonymizers/mysql/mysql_test.go | 82 +++++ modules/anonymizers/mysql/mysql_test.in.sql | 106 +++++++ modules/anonymizers/mysql/mysql_test.out.sql | 106 +++++++ modules/filters/relfilter/filter.go | 68 ++-- modules/filters/relfilter/filter_test.go | 309 +++++++++++++++++++ 8 files changed, 739 insertions(+), 46 deletions(-) create mode 100644 modules/anonymizers/mysql/mysql_test.go create mode 100644 modules/anonymizers/mysql/mysql_test.in.sql create mode 100644 modules/anonymizers/mysql/mysql_test.out.sql create mode 100644 modules/filters/relfilter/filter_test.go diff --git a/misc/template.go b/misc/template.go index 8cf215d..7ecc1e7 100644 --- a/misc/template.go +++ b/misc/template.go @@ -7,7 +7,10 @@ import ( "github.com/Masterminds/sprig/v3" ) -var null = "::NULL::" +var ( + null = "::NULL::" + drop = "::DROP::" +) type TemplateData struct { TableName string @@ -16,7 +19,7 @@ type TemplateData struct { } // TemplateExec makes message from given template `tpl` and data `d` -func TemplateExec(tpl string, d any) ([]byte, error) { +func TemplateExec(tpl string, d any) ([]byte, bool, error) { var b bytes.Buffer @@ -36,28 +39,36 @@ func TemplateExec(tpl string, d any) ([]byte, error) { } return false } + t["drop"] = func() string { + return drop + } return t }()).Parse(tpl) if err != nil { - return []byte{}, err + return []byte{}, false, err } err = t.Execute(&b, d) if err != nil { - return []byte{}, err + return []byte{}, false, err } // Return empty line if buffer is nil if b.Bytes() == nil { - return []byte{}, nil + return []byte{}, false, nil } // Return nil if buffer is NULL (with special key) if bytes.Compare(b.Bytes(), []byte(null)) == 0 { - return nil, nil + return nil, false, nil + } + + // Return `drop` value if buffer is DROP (with special key) + if bytes.Compare(b.Bytes(), []byte(drop)) == 0 { + return nil, true, nil } // Return buffer content otherwise - return b.Bytes(), nil + return b.Bytes(), false, nil } diff --git a/modules/anonymizers/mysql/dh.go b/modules/anonymizers/mysql/dh.go index c57fe00..451ac8c 100644 --- a/modules/anonymizers/mysql/dh.go +++ b/modules/anonymizers/mysql/dh.go @@ -27,7 +27,7 @@ func dhSecurityInsertIntoTableNameSearch(usrCtx any, deferred, token []byte) ([] return []byte{}, nil } -func dhSecurityNil(usrCtx any, deferred, token []byte) ([]byte, error) { +func dhSecurityInsertIntoValues(usrCtx any, deferred, token []byte) ([]byte, error) { uctx := usrCtx.(*userCtx) @@ -35,6 +35,37 @@ func dhSecurityNil(usrCtx any, deferred, token []byte) ([]byte, error) { return []byte{}, nil } + uctx.insertIntoBuf = append(uctx.insertIntoBuf, deferred...) + uctx.insertIntoBuf = append(uctx.insertIntoBuf, token...) + + return []byte{}, nil +} + +func dhSecurityInsertIntoValueSearch(usrCtx any, deferred, token []byte) ([]byte, error) { + + uctx := usrCtx.(*userCtx) + + if uctx.security.isSkip == true { + return []byte{}, nil + } + + uctx.insertIntoBuf = append(uctx.insertIntoBuf, deferred...) + + return []byte{}, nil +} + +func dhSecurityValuesEnd(usrCtx any, deferred, token []byte) ([]byte, error) { + + uctx := usrCtx.(*userCtx) + + if uctx.security.isSkip == true { + return []byte{}, nil + } + + if uctx.insertIntoBuf != nil { + return []byte{}, nil + } + return append(deferred, token...), nil } @@ -111,17 +142,17 @@ func dhInsertIntoTableName(usrCtx any, deferred, token []byte) ([]byte, error) { return []byte{}, nil } - d := append(uctx.security.tmpBuf, append(deferred, token...)...) + uctx.insertIntoBuf = append(uctx.security.tmpBuf, append(deferred, token...)...) uctx.security.isSkip = false uctx.security.tmpBuf = []byte{} // Check insert into table name if tn != uctx.filter.TableNameGet() { - return d, fmt.Errorf("`create` and `insert into` table names are mismatch (create table: '%s', insert into table: '%s')", uctx.filter.TableNameGet(), tn) + return []byte{}, fmt.Errorf("`create` and `insert into` table names are mismatch (create table: '%s', insert into table: '%s')", uctx.filter.TableNameGet(), tn) } - return d, nil + return []byte{}, nil } func dhCreateTableValues(usrCtx any, deferred, token []byte) ([]byte, error) { @@ -173,7 +204,19 @@ func dhCreateTableValuesEnd(usrCtx any, deferred, token []byte) ([]byte, error) return []byte{}, err } - return rowDataGen(uctx), nil + b := rowDataGen(uctx) + if b == nil { + return []byte{}, nil + } else { + if uctx.insertIntoBuf != nil { + b = append(uctx.insertIntoBuf, b...) + uctx.insertIntoBuf = nil + } else { + b = append([]byte{','}, b...) + } + } + + return b, nil } func dhCreateTableValuesStringEnd(usrCtx any, deferred, token []byte) ([]byte, error) { @@ -189,7 +232,19 @@ func dhCreateTableValuesStringEnd(usrCtx any, deferred, token []byte) ([]byte, e return []byte{}, err } - return rowDataGen(uctx), nil + b := rowDataGen(uctx) + if b == nil { + return []byte{}, nil + } else { + if uctx.insertIntoBuf != nil { + b = append(uctx.insertIntoBuf, b...) + uctx.insertIntoBuf = nil + } else { + b = append([]byte{','}, b...) + } + } + + return b, nil } func rowDataGen(uctx *userCtx) []byte { @@ -197,6 +252,9 @@ func rowDataGen(uctx *userCtx) []byte { var out string row := uctx.filter.ValuePop() + if row.Values == nil { + return nil + } for i, v := range row.Values { diff --git a/modules/anonymizers/mysql/mysql.go b/modules/anonymizers/mysql/mysql.go index b7e3054..93678f3 100644 --- a/modules/anonymizers/mysql/mysql.go +++ b/modules/anonymizers/mysql/mysql.go @@ -38,11 +38,12 @@ type SecurityOpts struct { } type userCtx struct { - filter *relfilter.Filter - columnName string - security securityCtx - tables map[string]map[string]columnType - optKinds []optKind + filter *relfilter.Filter + columnName string + security securityCtx + tables map[string]map[string]columnType + optKinds []optKind + insertIntoBuf []byte } type securityCtx struct { @@ -449,7 +450,7 @@ func (m *MySQL) Run(ctx context.Context, w io.Writer) error { R: []byte{' ', '\n'}, }, }, - DataHandler: dhSecurityNil, + DataHandler: dhSecurityInsertIntoValues, }, }, }, @@ -460,7 +461,7 @@ func (m *MySQL) Run(ctx context.Context, w io.Writer) error { Switch: fsm.Switch{ Trigger: []byte("("), }, - DataHandler: fsm.DataHandlerGenericSkipToken, + DataHandler: dhSecurityInsertIntoValueSearch, }, }, }, @@ -526,14 +527,14 @@ func (m *MySQL) Run(ctx context.Context, w io.Writer) error { Switch: fsm.Switch{ Trigger: []byte(","), }, - DataHandler: dhSecurityNil, + DataHandler: fsm.DataHandlerGenericVoid, }, { Name: stateSomeIntermediateState, Switch: fsm.Switch{ Trigger: []byte(";"), }, - DataHandler: dhSecurityNil, + DataHandler: dhSecurityValuesEnd, }, }, }, diff --git a/modules/anonymizers/mysql/mysql_test.go b/modules/anonymizers/mysql/mysql_test.go new file mode 100644 index 0000000..90cfd30 --- /dev/null +++ b/modules/anonymizers/mysql/mysql_test.go @@ -0,0 +1,82 @@ +package mysql_anonymize + +import ( + "bytes" + "context" + "os" + "testing" + + "github.com/nixys/nxs-data-anonymizer/misc" + "github.com/nixys/nxs-data-anonymizer/modules/filters/relfilter" +) + +func TestMySQL(t *testing.T) { + + var r, e bytes.Buffer + + fin, err := os.Open("mysql_test.in.sql") + if err != nil { + t.Fatal("open input SQL:", err) + } + + m, err := Init( + fin, + InitOpts{ + Rules: RulesOpts{ + TableRules: map[string]map[string]relfilter.ColumnRuleOpts{ + + // Delete only row with id `2` + "table1": { + "id": relfilter.ColumnRuleOpts{ + Type: misc.ValueTypeTemplate, + Value: "{{ if eq .Values.id \"2\" }}{{ drop }}{{ else }}{{ .Values.id }}{{ end }}", + Unique: false, + }, + }, + + // Delete all rows from table + "table2": { + "id": relfilter.ColumnRuleOpts{ + Type: misc.ValueTypeTemplate, + Value: "{{ drop }}", + Unique: false, + }, + }, + + // Delete no rows + "table3": { + "id": relfilter.ColumnRuleOpts{ + Type: misc.ValueTypeTemplate, + Value: "{{ if eq .Values.id \"4\" }}{{ drop }}{{ else }}{{ .Values.id }}{{ end }}", + Unique: false, + }, + }, + }, + }, + }, + ) + if err != nil { + t.Fatal("init MySQL:", err) + } + + if err := m.Run(context.Background(), &r); err != nil { + t.Fatal("run MySQL:", err) + } + + fout, err := os.Open("mysql_test.out.sql") + if err != nil { + t.Fatal("open output SQL:", err) + } + + if _, err := e.ReadFrom(fout); err != nil { + t.Fatal("read output SQL:", err) + } + + // os.WriteFile("mysql_test.out.sql", r.Bytes(), 0644) + + if r.String() != e.String() { + t.Fatal("incorrect anonymization result") + } + + t.Logf("success") +} diff --git a/modules/anonymizers/mysql/mysql_test.in.sql b/modules/anonymizers/mysql/mysql_test.in.sql new file mode 100644 index 0000000..92e604a --- /dev/null +++ b/modules/anonymizers/mysql/mysql_test.in.sql @@ -0,0 +1,106 @@ +-- MariaDB dump 10.19 Distrib 10.6.12-MariaDB, for debian-linux-gnu (x86_64) +-- +-- Host: 127.0.0.1 Database: db +-- ------------------------------------------------------ +-- Server version 8.0.32 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `table1` +-- + +DROP TABLE IF EXISTS `table1`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `table1` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(50) NOT NULL, + `url` varchar(255) NOT NULL, + `auth_method` varchar(20) NOT NULL DEFAULT 'none', + `auth_data` text NOT NULL, + `username` varchar(20) NOT NULL DEFAULT '', + `password` varchar(20) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `id` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `table1` +-- + +LOCK TABLES `table1` WRITE; +/*!40000 ALTER TABLE `table1` DISABLE KEYS */; +INSERT INTO `table1` VALUES (1,'test4','https://github.com/nixys/nxs-rbac-operator0.git','none','','',''),(2,'test5','https://github.com/nixys/nxs-rbac-operator.git','none','','',''),(3,'test6','https://github.com/nixys/nxs-rbac-operator2.git','none','','',''); +/*!40000 ALTER TABLE `table1` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `table2` +-- + +DROP TABLE IF EXISTS `table2`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `table2` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(50) NOT NULL, + `url` varchar(255) NOT NULL, + `auth_method` varchar(20) NOT NULL DEFAULT 'none', + `auth_data` text NOT NULL, + `username` varchar(20) NOT NULL DEFAULT '', + `password` varchar(20) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `id` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `table2` +-- + +LOCK TABLES `table2` WRITE; +/*!40000 ALTER TABLE `table2` DISABLE KEYS */; +INSERT INTO `table2` VALUES (1,'test4','https://github.com/nixys/nxs-rbac-operator0.git','none','','',''),(2,'test5','https://github.com/nixys/nxs-rbac-operator.git','none','','',''),(3,'test6','https://github.com/nixys/nxs-rbac-operator2.git','none','','',''); +/*!40000 ALTER TABLE `table2` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `table3` +-- + +DROP TABLE IF EXISTS `table3`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `table3` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(50) NOT NULL, + `url` varchar(255) NOT NULL, + `auth_method` varchar(20) NOT NULL DEFAULT 'none', + `auth_data` text NOT NULL, + `username` varchar(20) NOT NULL DEFAULT '', + `password` varchar(20) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `id` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `table3` +-- + +LOCK TABLES `table3` WRITE; +/*!40000 ALTER TABLE `table3` DISABLE KEYS */; +INSERT INTO `table3` VALUES (1,'test4','https://github.com/nixys/nxs-rbac-operator0.git','none','','',''),(2,'test5','https://github.com/nixys/nxs-rbac-operator.git','none','','',''),(3,'test6','https://github.com/nixys/nxs-rbac-operator2.git','none','','',''); +/*!40000 ALTER TABLE `table3` ENABLE KEYS */; +UNLOCK TABLES; \ No newline at end of file diff --git a/modules/anonymizers/mysql/mysql_test.out.sql b/modules/anonymizers/mysql/mysql_test.out.sql new file mode 100644 index 0000000..b49b62e --- /dev/null +++ b/modules/anonymizers/mysql/mysql_test.out.sql @@ -0,0 +1,106 @@ +-- MariaDB dump 10.19 Distrib 10.6.12-MariaDB, for debian-linux-gnu (x86_64) +-- +-- Host: 127.0.0.1 Database: db +-- ------------------------------------------------------ +-- Server version 8.0.32 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `table1` +-- + +DROP TABLE IF EXISTS `table1`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `table1` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(50) NOT NULL, + `url` varchar(255) NOT NULL, + `auth_method` varchar(20) NOT NULL DEFAULT 'none', + `auth_data` text NOT NULL, + `username` varchar(20) NOT NULL DEFAULT '', + `password` varchar(20) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `id` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `table1` +-- + +LOCK TABLES `table1` WRITE; +/*!40000 ALTER TABLE `table1` DISABLE KEYS */; +INSERT INTO `table1` VALUES (1,'test4','https://github.com/nixys/nxs-rbac-operator0.git','none','','',''),(3,'test6','https://github.com/nixys/nxs-rbac-operator2.git','none','','',''); +/*!40000 ALTER TABLE `table1` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `table2` +-- + +DROP TABLE IF EXISTS `table2`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `table2` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(50) NOT NULL, + `url` varchar(255) NOT NULL, + `auth_method` varchar(20) NOT NULL DEFAULT 'none', + `auth_data` text NOT NULL, + `username` varchar(20) NOT NULL DEFAULT '', + `password` varchar(20) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `id` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `table2` +-- + +LOCK TABLES `table2` WRITE; +/*!40000 ALTER TABLE `table2` DISABLE KEYS */; + +/*!40000 ALTER TABLE `table2` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `table3` +-- + +DROP TABLE IF EXISTS `table3`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `table3` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(50) NOT NULL, + `url` varchar(255) NOT NULL, + `auth_method` varchar(20) NOT NULL DEFAULT 'none', + `auth_data` text NOT NULL, + `username` varchar(20) NOT NULL DEFAULT '', + `password` varchar(20) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `id` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `table3` +-- + +LOCK TABLES `table3` WRITE; +/*!40000 ALTER TABLE `table3` DISABLE KEYS */; +INSERT INTO `table3` VALUES (1,'test4','https://github.com/nixys/nxs-rbac-operator0.git','none','','',''),(2,'test5','https://github.com/nixys/nxs-rbac-operator.git','none','','',''),(3,'test6','https://github.com/nixys/nxs-rbac-operator2.git','none','','',''); +/*!40000 ALTER TABLE `table3` ENABLE KEYS */; +UNLOCK TABLES; \ No newline at end of file diff --git a/modules/filters/relfilter/filter.go b/modules/filters/relfilter/filter.go index 14f0c4c..313e557 100644 --- a/modules/filters/relfilter/filter.go +++ b/modules/filters/relfilter/filter.go @@ -175,7 +175,7 @@ func Init(opts InitOpts) (*Filter, error) { vars := make(map[string]string) for n, f := range opts.Variables { - v, err := execFilter( + v, _, err := execFilter( execFilterOpts{ t: f.Type, v: f.Value, @@ -340,7 +340,7 @@ func (filter *Filter) Apply() error { // Check direct rules for column if tr != nil { - if cr, e := tr[c.n]; e == true { + if cr, e := tr[c.n]; e { rls = append( rls, @@ -355,7 +355,7 @@ func (filter *Filter) Apply() error { } // Check default rules for column - if cr, e := filter.rules.defaultRules[c.n]; e == true { + if cr, e := filter.rules.defaultRules[c.n]; e { rls = append( rls, applyRule{ @@ -427,6 +427,7 @@ func (filter *Filter) applyRules(tname string, rls []applyRule) error { var ( v []byte + d bool err error ) @@ -445,10 +446,16 @@ func (filter *Filter) applyRules(tname string, rls []applyRule) error { if e, b := r.v[string(valOld[r.c.n])]; b { v = e } else { - v, err = filter.applyLinkFilter(r.c.n, r.cr, r.u, td, valEnvGlob) + v, d, err = filter.applyLinkFilter(r.c.n, r.cr, r.u, td, valEnvGlob) if err != nil { return fmt.Errorf("rules: %w", err) } + + if d { + filter.tableData.values = nil + return nil + } + r.v[string(valOld[r.c.n])] = v } } else { @@ -480,10 +487,15 @@ func (filter *Filter) applyRules(tname string, rls []applyRule) error { tde = append(tde, valEnvGlob...) tde = append(tde, r.c.t.env...) - v, err = filter.applyColumnFilter(r.c.n, r.cr, td, tde) + v, d, err = filter.applyColumnFilter(r.c.n, r.cr, td, tde) if err != nil { return fmt.Errorf("rules: %w", err) } + + if d { + filter.tableData.values = nil + return nil + } } // Set specified value in accordance with filter @@ -493,11 +505,11 @@ func (filter *Filter) applyRules(tname string, rls []applyRule) error { return nil } -func (filter *Filter) applyColumnFilter(cn string, cr ColumnRuleOpts, td any, tde []string) ([]byte, error) { +func (filter *Filter) applyColumnFilter(cn string, cr ColumnRuleOpts, td any, tde []string) ([]byte, bool, error) { for i := 0; i < uniqueAttempts; i++ { - v, err := execFilter( + v, d, err := execFilter( execFilterOpts{ t: cr.Type, v: cr.Value, @@ -505,11 +517,15 @@ func (filter *Filter) applyColumnFilter(cn string, cr ColumnRuleOpts, td any, td td, tde) if err != nil { - return []byte{}, fmt.Errorf("apply filter: %w", err) + return []byte{}, false, fmt.Errorf("apply filter: %w", err) + } + + if d { + return []byte{}, true, nil } if cr.Unique == false { - return v, nil + return v, false, nil } var uv map[string]any @@ -523,18 +539,18 @@ func (filter *Filter) applyColumnFilter(cn string, cr ColumnRuleOpts, td any, td if _, b := uv[string(v)]; b == false { uv[string(v)] = nil filter.tableData.uniques[cn] = uv - return v, nil + return v, false, nil } } - return []byte{}, fmt.Errorf("filter: unable to generate unique value for column `%s.%s`, check filter value for this column in config", filter.tableData.name, cn) + return []byte{}, false, fmt.Errorf("filter: unable to generate unique value for column `%s.%s`, check filter value for this column in config", filter.tableData.name, cn) } -func (filter *Filter) applyLinkFilter(cn string, cr ColumnRuleOpts, u map[string]any, td any, tde []string) ([]byte, error) { +func (filter *Filter) applyLinkFilter(cn string, cr ColumnRuleOpts, u map[string]any, td any, tde []string) ([]byte, bool, error) { for i := 0; i < uniqueAttempts; i++ { - v, err := execFilter( + v, d, err := execFilter( execFilterOpts{ t: cr.Type, v: cr.Value, @@ -542,20 +558,24 @@ func (filter *Filter) applyLinkFilter(cn string, cr ColumnRuleOpts, u map[string td, tde) if err != nil { - return []byte{}, fmt.Errorf("apply link filter: %w", err) + return []byte{}, false, fmt.Errorf("apply link filter: %w", err) + } + + if d { + return []byte{}, true, nil } if cr.Unique == false { - return v, nil + return v, false, nil } if _, b := u[string(v)]; b == false { u[string(v)] = nil - return v, nil + return v, false, nil } } - return []byte{}, fmt.Errorf("apply link filter: unable to generate unique value for column `%s.%s`, check filter value for this column in config", filter.tableData.name, cn) + return []byte{}, false, fmt.Errorf("apply link filter: unable to generate unique value for column `%s.%s`, check filter value for this column in config", filter.tableData.name, cn) } // rowCleanup cleanups current row values @@ -576,16 +596,16 @@ func bcopy(b []byte) []byte { return d } -func execFilter(f execFilterOpts, td any, tde []string) (v []byte, err error) { +func execFilter(f execFilterOpts, td any, tde []string) (v []byte, d bool, err error) { switch f.t { case misc.ValueTypeTemplate: - v, err = misc.TemplateExec( + v, d, err = misc.TemplateExec( f.v, td, ) if err != nil { - return []byte{}, fmt.Errorf("filter: value compile template: %w", err) + return []byte{}, false, fmt.Errorf("filter: value compile template: %w", err) } case misc.ValueTypeCommand: @@ -602,17 +622,17 @@ func execFilter(f execFilterOpts, td any, tde []string) (v []byte, err error) { e, b := err.(*exec.ExitError) if b == false { - return []byte{}, fmt.Errorf("filter: value exec command: %w", err) + return []byte{}, false, fmt.Errorf("filter: value exec command: %w", err) } - return []byte{}, fmt.Errorf("filter: value exec command: bad exit code %d: %s", e.ExitCode(), stderr.String()) + return []byte{}, false, fmt.Errorf("filter: value exec command: bad exit code %d: %s", e.ExitCode(), stderr.String()) } v = stdout.Bytes() default: - return []byte{}, fmt.Errorf("filter: value compile: unknown type") + return []byte{}, false, fmt.Errorf("filter: value compile: unknown type") } - return bytes.ReplaceAll(v, []byte("\n"), []byte("\\n")), nil + return bytes.ReplaceAll(v, []byte("\n"), []byte("\\n")), d, nil } diff --git a/modules/filters/relfilter/filter_test.go b/modules/filters/relfilter/filter_test.go new file mode 100644 index 0000000..4e71f7c --- /dev/null +++ b/modules/filters/relfilter/filter_test.go @@ -0,0 +1,309 @@ +package relfilter + +import ( + "bytes" + "testing" + + "github.com/nixys/nxs-data-anonymizer/misc" +) + +func TestExecFilter(t *testing.T) { + + // Test `drop` template function + v, d, err := execFilter( + execFilterOpts{ + t: misc.ValueTypeTemplate, + v: "{{- drop -}}", + }, + nil, + nil) + if err != nil { + t.Fatal("`drop` function:", err) + } + if v != nil || d == false { + t.Fatal("`drop` function: incorrect return value") + } + t.Logf("`drop` function: success") + + // Test `null` template function + v, d, err = execFilter( + execFilterOpts{ + t: misc.ValueTypeTemplate, + v: "{{- null -}}", + }, + nil, + nil) + if err != nil { + t.Fatal("`null` function:", err) + } + if v != nil || d == true { + t.Fatal("`null` function: incorrect return value") + } + t.Logf("`null` function: success") +} + +func TestFilterApply(t *testing.T) { + TestFilterApplyDropFunction(t) + TestFilterApplyNullFunction(t) + TestLinkFilterApplyDropFunction(t) +} + +func TestFilterApplyDropFunction(t *testing.T) { + + f, err := Init( + InitOpts{ + TableRules: map[string]map[string]ColumnRuleOpts{ + "testTable": { + "testColumn1": ColumnRuleOpts{ + Type: misc.ValueTypeTemplate, + Value: "{{- drop -}}", + Unique: false, + }, + }, + }, + }, + ) + if err != nil { + t.Fatal("init:", err) + } + + testFilterTableInit(f) + + // Apply filters for row + if err := f.Apply(); err != nil { + t.Fatal("apply:", err) + } + + // Get row values + r := f.ValuePop() + + if r.Values != nil { + t.Fatal("`drop` function: unexpected behaviour") + } + t.Logf("`drop` function: success") +} + +func TestFilterApplyNullFunction(t *testing.T) { + + f, err := Init( + InitOpts{ + TableRules: map[string]map[string]ColumnRuleOpts{ + "testTable": { + "testColumn1": ColumnRuleOpts{ + Type: misc.ValueTypeTemplate, + Value: "{{- null -}}", + Unique: false, + }, + }, + }, + }, + ) + if err != nil { + t.Fatal("init:", err) + } + + testFilterTableInit(f) + + // Apply filters for row + if err := f.Apply(); err != nil { + t.Fatal("apply:", err) + } + + // Get row values + r := f.ValuePop() + + if len(r.Values) < 2 || r.Values[0].V != nil { + t.Fatal("`null` function: unexpected behaviour") + } + t.Logf("`null` function: success") +} + +func TestLinkFilterApply(t *testing.T) { + + f, err := Init( + InitOpts{ + TableRules: map[string]map[string]ColumnRuleOpts{ + "testTable1": { + "testColumn1": ColumnRuleOpts{ + Type: misc.ValueTypeTemplate, + Value: "{{- 11 -}}", + Unique: false, + }, + }, + "testTable2": { + "testColumn1": ColumnRuleOpts{ + Type: misc.ValueTypeTemplate, + Value: "{{- 22 -}}", + Unique: false, + }, + }, + }, + + Link: []LinkOpts{ + { + Rule: ColumnRuleOpts{ + Type: misc.ValueTypeTemplate, + Value: "{{- uuidv4 -}}", + Unique: false, + }, + With: map[string][]string{ + "testTable1": { + "testColumn2", + }, + "testTable2": { + "testColumn2", + }, + }, + }, + }, + }, + ) + if err != nil { + t.Fatal("init:", err) + } + + // Fill table 1 + testLinkFilterTable1Init(f) + + // Apply filters for row + if err := f.Apply(); err != nil { + t.Fatal("apply:", err) + } + + // Get row values + r1 := f.ValuePop() + + if len(r1.Values) < 2 { + t.Fatal("incorrect row len for table 1") + } + + // Fill table 2 + testLinkFilterTable2Init(f) + + // Apply filters for row + if err := f.Apply(); err != nil { + t.Fatal("apply:", err) + } + + // Get row values + r2 := f.ValuePop() + + if len(r2.Values) < 2 { + t.Fatal("incorrect row len for table 2") + } + + if bytes.Compare(r1.Values[1].V, r2.Values[1].V) != 0 { + t.Fatal("incorrect values for tables after filter apply") + } + + t.Logf("success") +} + +func TestLinkFilterApplyDropFunction(t *testing.T) { + + f, err := Init( + InitOpts{ + TableRules: map[string]map[string]ColumnRuleOpts{ + "testTable1": { + "testColumn1": ColumnRuleOpts{ + Type: misc.ValueTypeTemplate, + Value: "{{- 1 -}}", + Unique: false, + }, + }, + }, + + Link: []LinkOpts{ + { + Rule: ColumnRuleOpts{ + Type: misc.ValueTypeTemplate, + Value: "{{- drop -}}", + Unique: false, + }, + With: map[string][]string{ + "testTable1": { + "testColumn2", + }, + "testTable2": { + "testColumn2", + }, + }, + }, + }, + }, + ) + if err != nil { + t.Fatal("init:", err) + } + + // Fill table 1 + testLinkFilterTable1Init(f) + + // Apply filters for row + if err := f.Apply(); err != nil { + t.Fatal("apply:", err) + } + + // Get row values + r1 := f.ValuePop() + + if r1.Values != nil { + t.Fatal("`drop` function: unexpected behaviour for table 1") + } + + // Fill table 2 + testLinkFilterTable2Init(f) + + // Apply filters for row + if err := f.Apply(); err != nil { + t.Fatal("apply:", err) + } + + // Get row values + r2 := f.ValuePop() + + if r2.Values != nil { + t.Fatal("`drop` function: unexpected behaviour for table 2") + } + + t.Logf("`drop` function: success") +} + +func testFilterTableInit(f *Filter) { + // Create table + f.TableCreate("testTable") + + // Add column `testColumn1` with value + f.ColumnAdd("testColumn1", "int") + f.ValueAdd([]byte("0")) + + // Add column `testColumn2` with value + f.ColumnAdd("testColumn2", "varchar(10)") + f.ValueAdd([]byte("a")) +} + +func testLinkFilterTable1Init(f *Filter) { + // Create table + f.TableCreate("testTable1") + + // Add column `testColumn1` with value + f.ColumnAdd("testColumn1", "int") + f.ValueAdd([]byte("1")) + + // Add column `testColumn2` with value + f.ColumnAdd("testColumn2", "varchar(100)") + f.ValueAdd([]byte("a")) +} + +func testLinkFilterTable2Init(f *Filter) { + // Create table + f.TableCreate("testTable2") + + // Add column `testColumn1` with value + f.ColumnAdd("testColumn1", "int") + f.ValueAdd([]byte("2")) + + // Add column `testColumn2` with value + f.ColumnAdd("testColumn2", "varchar(100)") + f.ValueAdd([]byte("a")) +} From 8870a26d697d85914159fa508c86d5bce6659cf9 Mon Sep 17 00:00:00 2001 From: Boris Ershov Date: Wed, 16 Oct 2024 19:06:28 +0700 Subject: [PATCH 2/2] fix(#44): Set `insertIntoBuf` to nil --- modules/anonymizers/mysql/dh.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/anonymizers/mysql/dh.go b/modules/anonymizers/mysql/dh.go index 451ac8c..b1d2cf2 100644 --- a/modules/anonymizers/mysql/dh.go +++ b/modules/anonymizers/mysql/dh.go @@ -13,6 +13,7 @@ func dhSecurityInsertInto(usrCtx any, deferred, token []byte) ([]byte, error) { uctx := usrCtx.(*userCtx) uctx.security.tmpBuf = token + uctx.insertIntoBuf = nil return deferred, nil }