diff --git a/ctx/context.go b/ctx/context.go index 6b6b0b2..10be5ee 100644 --- a/ctx/context.go +++ b/ctx/context.go @@ -51,18 +51,8 @@ type progressCtx struct { } type SecurityCtx struct { - Policy securityPolicyCtx - Exceptions securityExceptionsCtx -} - -type securityPolicyCtx struct { - Tables misc.SecurityPolicyTablesType - Columns misc.SecurityPolicyColumnsType -} - -type securityExceptionsCtx struct { - Tables map[string]any - Columns map[string]any + TablePolicy misc.SecurityPolicyTablesType + TableExceptions map[string]any } // Init initiates application custom context @@ -143,6 +133,10 @@ func AppCtxInit() (any, error) { c.Rules.Tables = make(map[string]relfilter.TableRules) + if misc.SecurityPolicyColumnsTypeFromString(conf.Security.Policy.Columns) == misc.SecurityPolicyColumnsRandomize { + c.Rules.RandomizeTypes = relfilter.RandomizeTypesDefault + } + for t, f := range conf.Filters { c.Rules.Tables[t] = relfilter.TableRules{ @@ -160,6 +154,28 @@ func AppCtxInit() (any, error) { } } + c.Rules.Defaults = relfilter.TableRules{ + Columns: func() map[string]relfilter.ColumnRule { + cc := make(map[string]relfilter.ColumnRule) + for c, cf := range conf.Security.Defaults.Columns { + cc[c] = relfilter.ColumnRule{ + Type: misc.ValueTypeFromString(cf.Type), + Value: cf.Value, + Unique: cf.Unique, + } + } + return cc + }(), + } + + c.Rules.ExceptionColumns = func() map[string]any { + v := make(map[string]any) + for _, e := range conf.Security.Exceptions.Columns { + v[e] = nil + } + return v + }() + // Progress settings c.Progress.Humanize = conf.Progress.Humanize @@ -172,26 +188,14 @@ func AppCtxInit() (any, error) { } c.Security = SecurityCtx{ - Policy: securityPolicyCtx{ - Tables: misc.SecurityPolicyTablesTypeFromString(conf.Security.Policy.Tables), - Columns: misc.SecurityPolicyColumnsTypeFromString(conf.Security.Policy.Columns), - }, - Exceptions: securityExceptionsCtx{ - Tables: func() map[string]any { - v := make(map[string]any) - for _, e := range conf.Security.Exceptions.Tables { - v[e] = nil - } - return v - }(), - Columns: func() map[string]any { - v := make(map[string]any) - for _, e := range conf.Security.Exceptions.Columns { - v[e] = nil - } - return v - }(), - }, + TablePolicy: misc.SecurityPolicyTablesTypeFromString(conf.Security.Policy.Tables), + TableExceptions: func() map[string]any { + v := make(map[string]any) + for _, e := range conf.Security.Exceptions.Tables { + v[e] = nil + } + return v + }(), } return c, nil diff --git a/modules/anonymizers/mysql/dh.go b/modules/anonymizers/mysql/dh.go index 4181aeb..a9d5c9e 100644 --- a/modules/anonymizers/mysql/dh.go +++ b/modules/anonymizers/mysql/dh.go @@ -243,17 +243,17 @@ func rowDataGen(filter *relfilter.Filter) []byte { func securityPolicyCheck(uctx *userCtx, tname string) bool { // Continue if security policy is `skip` - if uctx.security.policy.tables != misc.SecurityPolicyTablesSkip { + if uctx.security.tablePolicy != misc.SecurityPolicyTablesSkip { return true } // Check rules for specified table name - if _, b := uctx.filter.TableNameLookup(tname); b == true { + if tr := uctx.filter.TableRulesLookup(tname); tr != nil { return true } // Check specified table name in exceptions - if _, b := uctx.security.exceptions.tables[tname]; b == true { + if _, b := uctx.security.tableExceptions[tname]; b == true { return true } diff --git a/modules/anonymizers/mysql/mysql.go b/modules/anonymizers/mysql/mysql.go index cb65802..065b6f5 100644 --- a/modules/anonymizers/mysql/mysql.go +++ b/modules/anonymizers/mysql/mysql.go @@ -17,18 +17,8 @@ type InitSettings struct { } type SecuritySettings struct { - Policy SecurityPolicySettings - Exceptions SecurityExceptionsSettings -} - -type SecurityPolicySettings struct { - Tables misc.SecurityPolicyTablesType - Columns misc.SecurityPolicyColumnsType -} - -type SecurityExceptionsSettings struct { - Tables map[string]any - Columns map[string]any + TablePolicy misc.SecurityPolicyTablesType + TableExceptions map[string]any } type userCtx struct { @@ -48,18 +38,8 @@ type securityCtx struct { tmpBuf []byte isSkip bool - policy securityPolicyCtx - exceptions securityExceptionsCtx -} - -type securityPolicyCtx struct { - tables misc.SecurityPolicyTablesType - columns misc.SecurityPolicyColumnsType -} - -type securityExceptionsCtx struct { - tables map[string]any - columns map[string]any + tablePolicy misc.SecurityPolicyTablesType + tableExceptions map[string]any } var typeKeys = map[string]relfilter.ColumnType{ @@ -112,14 +92,8 @@ func userCtxInit(s InitSettings) *userCtx { return &userCtx{ filter: relfilter.Init(s.Rules), security: securityCtx{ - policy: securityPolicyCtx{ - tables: s.Security.Policy.Tables, - columns: s.Security.Policy.Columns, - }, - exceptions: securityExceptionsCtx{ - tables: s.Security.Exceptions.Tables, - columns: s.Security.Exceptions.Columns, - }, + tablePolicy: s.Security.TablePolicy, + tableExceptions: s.Security.TableExceptions, }, } } diff --git a/modules/filters/relfilter/filter.go b/modules/filters/relfilter/filter.go index f0ca96c..4ae5bbb 100644 --- a/modules/filters/relfilter/filter.go +++ b/modules/filters/relfilter/filter.go @@ -9,7 +9,10 @@ import ( ) type Rules struct { - Tables map[string]TableRules + Tables map[string]TableRules + ExceptionColumns map[string]any + Defaults TableRules + RandomizeTypes map[ColumnType]ColumnRule } type TableRules struct { @@ -51,9 +54,33 @@ const uniqueAttempts = 5 const ( envVarTable = "ENVVARTABLE" envVarColumnPrefix = "ENVVARCOLUMN_" - envVarCurTable = "ENVVARCURCOLUMN" + envVarCurColumn = "ENVVARCURCOLUMN" ) +type rule struct { + c *column + i int + cr ColumnRule +} + +var RandomizeTypesDefault = map[ColumnType]ColumnRule{ + ColumnTypeBinary: { + Type: misc.ValueTypeTemplate, + Value: "cmFuZG9taXplZCBiaW5hcnkgZGF0YQo=", + Unique: false, + }, + ColumnTypeNum: { + Type: misc.ValueTypeTemplate, + Value: "0", + Unique: false, + }, + ColumnTypeString: { + Type: misc.ValueTypeTemplate, + Value: "randomized string data", + Unique: false, + }, +} + func Init(rules Rules) *Filter { return &Filter{ rules: rules, @@ -74,10 +101,12 @@ func (filter *Filter) TableNameGet() string { return filter.tableData.name } -// TableNameLookup looks up filters for specified table name -func (filter *Filter) TableNameLookup(name string) (TableRules, bool) { - t, b := filter.rules.Tables[name] - return t, b +// TableRulesLookup looks up filters for specified table name +func (filter *Filter) TableRulesLookup(name string) *TableRules { + if t, b := filter.rules.Tables[name]; b { + return &t + } + return nil } // ColumnAdd adds new column into current data set @@ -111,132 +140,191 @@ func (filter *Filter) ValuePop() Row { } } -// Apply applies filter rules for current data set func (filter *Filter) Apply() error { + var rls []rule + tname := filter.tableData.name - // Check current table exist in rules - t, b := filter.TableNameLookup(tname) - if b == true { + // Check rules exist for current table + tr := filter.TableRulesLookup(tname) - td := misc.TemplateData{ - TableName: tname, - Values: make(map[string][]byte), - } + // Create rules for every column within current table + for i, c := range filter.tableData.columns.cc { - // Init table data env variables with current table name - tdenv := []string{ - fmt.Sprintf("%s=%s", envVarTable, tname), - } + // Check direct rules for column + if tr != nil { + if cr, e := tr.Columns[c.n]; e == true { - if len(filter.tableData.columns.cc) > len(filter.tableData.values) { - return fmt.Errorf("mismatch count of columns and values for table '%s'", tname) + rls = append( + rls, + rule{ + c: c, + i: i, + cr: cr, + }, + ) + continue + } } - for i, d := range filter.tableData.columns.cc { - td.Values[d.n] = filter.tableData.values[i].V - - // Fill env variables with columns and its values - tdenv = append( - tdenv, - fmt.Sprintf("%s%s=%s", envVarColumnPrefix, d.n, string(filter.tableData.values[i].V)), + // Check default rules for column + if cr, e := filter.rules.Defaults.Columns[c.n]; e == true { + rls = append( + rls, + rule{ + c: c, + i: i, + cr: cr, + }, ) + continue } - // Filter all columns with specified rules - for n, d := range filter.tableData.columns.cc { - - // Check rule set for current column - c, e := t.Columns[d.n] - if e == false { + // Check randomize rules for column + if cr, b := filter.rules.RandomizeTypes[c.t]; b { + + // Check that column excepted + if _, b := filter.rules.ExceptionColumns[c.n]; !b { + rls = append( + rls, + rule{ + c: c, + i: i, + cr: cr, + }, + ) continue } + } - // Create tmp env variables with current column name - tde := append( - tdenv, - fmt.Sprintf("%s=%s", envVarCurTable, d.n), - ) + // Other rules if required + } + + // Apply rules + if err := filter.applyRules(tname, rls); err != nil { + return fmt.Errorf("filters apply: %w", err) + } + + return nil +} + +func (filter *Filter) applyRules(tname string, rls []rule) error { + + // If no columns has rules + if len(rls) == 0 { + return nil + } + + // Fill table data and table envs + td := misc.TemplateData{ + TableName: tname, + Values: make(map[string][]byte), + } + + tdenv := []string{ + fmt.Sprintf("%s=%s", envVarTable, tname), + } + + for i, c := range filter.tableData.columns.cc { + td.Values[c.n] = filter.tableData.values[i].V - v, err := func() ([]byte, error) { + tdenv = append( + tdenv, + fmt.Sprintf("%s%s=%s", envVarColumnPrefix, c.n, string(filter.tableData.values[i].V)), + ) + } - for i := 0; i < uniqueAttempts; i++ { + // Apply rule for each specified column + for _, r := range rls { - var ( - v []byte - err error - ) + var tde []string - switch c.Type { - case misc.ValueTypeTemplate: - v, err = misc.TemplateExec( - c.Value, - td, - ) - if err != nil { - return []byte{}, fmt.Errorf("value compile template: %w", err) - } - case misc.ValueTypeCommand: + // Create tmp env variables with current column name + tde = append( + tdenv, + fmt.Sprintf("%s=%s", envVarCurColumn, r.c.n), + ) - var stderr, stdout bytes.Buffer + v, err := filter.applyFilter(r.c.n, r.cr, td, tde) + if err != nil { + return fmt.Errorf("rules: %w", err) + } - cmd := exec.Command(c.Value) + // Set specified value in accordance with filter + filter.tableData.values[r.i].V = v + } - cmd.Stdout = &stdout - cmd.Stderr = &stderr + return nil +} - cmd.Env = tde +func (filter *Filter) applyFilter(cn string, cr ColumnRule, td misc.TemplateData, tde []string) ([]byte, error) { - if err := cmd.Run(); err != nil { + for i := 0; i < uniqueAttempts; i++ { - e, b := err.(*exec.ExitError) - if b == false { - return []byte{}, fmt.Errorf("value exec command: %w", err) - } + var ( + v []byte + err error + ) - return []byte{}, fmt.Errorf("value exec command: bad exit code %d: %s", e.ExitCode(), stderr.String()) - } + switch cr.Type { + case misc.ValueTypeTemplate: + v, err = misc.TemplateExec( + cr.Value, + td, + ) + if err != nil { + return []byte{}, fmt.Errorf("filter: value compile template: %w", err) + } + case misc.ValueTypeCommand: - v = stdout.Bytes() + var stderr, stdout bytes.Buffer - default: - return []byte{}, fmt.Errorf("value compile: unknown type") - } + cmd := exec.Command(cr.Value) - v = bytes.ReplaceAll(v, []byte("\n"), []byte("\\n")) + cmd.Stdout = &stdout + cmd.Stderr = &stderr - if c.Unique == false { - return v, nil - } + cmd.Env = tde - var uv map[string]any - if _, b := filter.tableData.uniques[d.n]; b == false { - // For first values - uv = make(map[string]any) - } else { - uv = filter.tableData.uniques[d.n] - } + if err := cmd.Run(); err != nil { - if _, b := uv[string(v)]; b == false { - uv[string(v)] = nil - filter.tableData.uniques[d.n] = uv - return v, nil - } + e, b := err.(*exec.ExitError) + if b == false { + return []byte{}, fmt.Errorf("filter: value exec command: %w", err) } - return []byte{}, fmt.Errorf("unable to generate unique value for column `%s.%s`, check filter value for this column in config", filter.tableData.name, d.n) - }() - if err != nil { - return err + return []byte{}, fmt.Errorf("filter: value exec command: bad exit code %d: %s", e.ExitCode(), stderr.String()) } - // Set specified value in accordance with filter - filter.tableData.values[n].V = v + v = stdout.Bytes() + + default: + return []byte{}, fmt.Errorf("filter: value compile: unknown type") + } + + v = bytes.ReplaceAll(v, []byte("\n"), []byte("\\n")) + + if cr.Unique == false { + return v, nil + } + + var uv map[string]any + if _, b := filter.tableData.uniques[cn]; b == false { + // For first values + uv = make(map[string]any) + } else { + uv = filter.tableData.uniques[cn] + } + + if _, b := uv[string(v)]; b == false { + uv[string(v)] = nil + filter.tableData.uniques[cn] = uv + return v, nil } } - return nil + return []byte{}, fmt.Errorf("filter: unable to generate unique value for column `%s.%s`, check filter value for this column in config", filter.tableData.name, cn) } // rowCleanup cleanups current row values diff --git a/routines/anonymizer/anonymizer.go b/routines/anonymizer/anonymizer.go index 0bd8c4e..6bbe3a7 100644 --- a/routines/anonymizer/anonymizer.go +++ b/routines/anonymizer/anonymizer.go @@ -138,14 +138,8 @@ func anomymize(st anomymizeSettings) error { st.pr, mysql_anonymize.InitSettings{ Security: mysql_anonymize.SecuritySettings{ - Policy: mysql_anonymize.SecurityPolicySettings{ - Tables: st.s.Policy.Tables, - Columns: st.s.Policy.Columns, - }, - Exceptions: mysql_anonymize.SecurityExceptionsSettings{ - Tables: st.s.Exceptions.Tables, - Columns: st.s.Exceptions.Columns, - }, + TablePolicy: st.s.TablePolicy, + TableExceptions: st.s.TableExceptions, }, Rules: st.rs, },