From f89170e0a02fb8ebb57dc1838b8f60b33670416b Mon Sep 17 00:00:00 2001 From: Boris Ershov Date: Fri, 17 May 2024 21:13:53 +0700 Subject: [PATCH] feat(#3): Add security enforcement for tables --- ctx/conf.go | 34 ++++++++- ctx/context.go | 39 ++++++++++ misc/security.go | 47 ++++++++++++ modules/anonymizers/mysql/dh.go | 109 +++++++++++++++++++++++++++- modules/anonymizers/mysql/mysql.go | 99 +++++++++++++++++++------ modules/filters/relfilter/filter.go | 8 +- routines/anonymizer/anonymizer.go | 20 ++++- 7 files changed, 327 insertions(+), 29 deletions(-) create mode 100644 misc/security.go diff --git a/ctx/conf.go b/ctx/conf.go index d3298c6..1d03e47 100644 --- a/ctx/conf.go +++ b/ctx/conf.go @@ -13,7 +13,9 @@ type confOpts struct { Progress progressConf `conf:"progress"` Filters map[string]filterConf `conf:"filters"` - MySQL *mysqlConf `conf:"mysql"` + Security securityConf `conf:"security"` + + MySQL *mysqlConf `conf:"mysql"` } type progressConf struct { @@ -31,6 +33,22 @@ type columnFilterConf struct { Unique bool `conf:"unique"` } +type securityConf struct { + Policy securityPolicyConf `conf:"policy"` + Exceptions securityExceptionsConf `conf:"exceptions"` + Defaults filterConf `conf:"defaults"` +} + +type securityPolicyConf struct { + Tables string `conf:"tables" conf_extraopts:"default=pass"` + Columns string `conf:"columns" conf_extraopts:"default=pass"` +} + +type securityExceptionsConf struct { + Tables []string `conf:"tables"` + Columns []string `conf:"columns"` +} + type mysqlConf struct { Host string `conf:"host" conf_extraopts:"required"` Port int `conf:"port" conf_extraopts:"required"` @@ -60,5 +78,19 @@ func confRead(confPath string) (confOpts, error) { } } + if misc.SecurityPolicyTablesTypeFromString(c.Security.Policy.Tables) == misc.SecurityPolicyTablesUnknown { + return c, fmt.Errorf("conf read: unknown security policy tables type") + } + + if misc.SecurityPolicyColumnsTypeFromString(c.Security.Policy.Columns) == misc.SecurityPolicyColumnsUnknown { + return c, fmt.Errorf("conf read: unknown security policy columns type") + } + + for _, cf := range c.Security.Defaults.Columns { + if misc.ValueTypeFromString(cf.Type) == misc.ValueTypeUnknown { + return c, fmt.Errorf("conf read: unknown default filter type") + } + } + return c, nil } diff --git a/ctx/context.go b/ctx/context.go index cef4525..6b6b0b2 100644 --- a/ctx/context.go +++ b/ctx/context.go @@ -21,6 +21,7 @@ type Ctx struct { Output io.Writer Rules relfilter.Rules Progress progressCtx + Security SecurityCtx DB DBCtx } @@ -49,6 +50,21 @@ type progressCtx struct { Humanize bool } +type SecurityCtx struct { + Policy securityPolicyCtx + Exceptions securityExceptionsCtx +} + +type securityPolicyCtx struct { + Tables misc.SecurityPolicyTablesType + Columns misc.SecurityPolicyColumnsType +} + +type securityExceptionsCtx struct { + Tables map[string]any + Columns map[string]any +} + // Init initiates application custom context func AppCtxInit() (any, error) { @@ -155,6 +171,29 @@ func AppCtxInit() (any, error) { return nil, err } + c.Security = SecurityCtx{ + Policy: securityPolicyCtx{ + Tables: misc.SecurityPolicyTablesTypeFromString(conf.Security.Policy.Tables), + Columns: misc.SecurityPolicyColumnsTypeFromString(conf.Security.Policy.Columns), + }, + Exceptions: securityExceptionsCtx{ + Tables: func() map[string]any { + v := make(map[string]any) + for _, e := range conf.Security.Exceptions.Tables { + v[e] = nil + } + return v + }(), + Columns: func() map[string]any { + v := make(map[string]any) + for _, e := range conf.Security.Exceptions.Columns { + v[e] = nil + } + return v + }(), + }, + } + return c, nil } diff --git a/misc/security.go b/misc/security.go new file mode 100644 index 0000000..dd15de0 --- /dev/null +++ b/misc/security.go @@ -0,0 +1,47 @@ +package misc + +type SecurityPolicyTablesType string + +const ( + SecurityPolicyTablesUnknown SecurityPolicyTablesType = "unknown" + SecurityPolicyTablesPass SecurityPolicyTablesType = "pass" + SecurityPolicyTablesSkip SecurityPolicyTablesType = "skip" +) + +func (v SecurityPolicyTablesType) String() string { + return string(v) +} + +func SecurityPolicyTablesTypeFromString(v string) SecurityPolicyTablesType { + switch v { + case string(SecurityPolicyTablesPass): + return SecurityPolicyTablesPass + case string(SecurityPolicyTablesSkip): + return SecurityPolicyTablesSkip + default: + return SecurityPolicyTablesUnknown + } +} + +type SecurityPolicyColumnsType string + +const ( + SecurityPolicyColumnsUnknown SecurityPolicyColumnsType = "unknown" + SecurityPolicyColumnsPass SecurityPolicyColumnsType = "pass" + SecurityPolicyColumnsRandomize SecurityPolicyColumnsType = "randomize" +) + +func (v SecurityPolicyColumnsType) String() string { + return string(v) +} + +func SecurityPolicyColumnsTypeFromString(v string) SecurityPolicyColumnsType { + switch v { + case string(SecurityPolicyColumnsPass): + return SecurityPolicyColumnsPass + case string(SecurityPolicyColumnsRandomize): + return SecurityPolicyColumnsRandomize + default: + return SecurityPolicyColumnsUnknown + } +} diff --git a/modules/anonymizers/mysql/dh.go b/modules/anonymizers/mysql/dh.go index b9b4035..4181aeb 100644 --- a/modules/anonymizers/mysql/dh.go +++ b/modules/anonymizers/mysql/dh.go @@ -5,20 +5,74 @@ import ( "fmt" "strings" + "github.com/nixys/nxs-data-anonymizer/misc" "github.com/nixys/nxs-data-anonymizer/modules/filters/relfilter" ) -func dhCreateTableName(usrCtx any, deferred, token []byte) ([]byte, error) { +func dhSecurityCreateTable(usrCtx any, deferred, token []byte) ([]byte, error) { + + uctx := usrCtx.(*userCtx) + + uctx.security.tmpBuf = append(uctx.security.tmpBuf, token...) + + return deferred, nil +} + +func dhSecurityCreateTableName(usrCtx any, deferred, token []byte) ([]byte, error) { + + uctx := usrCtx.(*userCtx) + + uctx.security.tmpBuf = append(uctx.security.tmpBuf, deferred...) + uctx.security.tmpBuf = append(uctx.security.tmpBuf, token...) + + return []byte{}, nil +} + +func dhSecurityNil(usrCtx any, deferred, token []byte) ([]byte, error) { uctx := usrCtx.(*userCtx) - uctx.filter.TableCreate(string(deferred)) + + if uctx.security.isSkip == true { + return []byte{}, nil + } return append(deferred, token...), nil } +func dhCreateTableName(usrCtx any, deferred, token []byte) ([]byte, error) { + + uctx := usrCtx.(*userCtx) + + tn := string(deferred) + + // Check table pass through security rules + if !securityPolicyCheck(uctx, tn) { + + // If not: table will be skipped from result dump + + uctx.security.isSkip = true + uctx.security.tmpBuf = []byte{} + return []byte{}, nil + } + + uctx.filter.TableCreate(tn) + + d := append(uctx.security.tmpBuf, append(deferred, token...)...) + + uctx.security.isSkip = false + uctx.security.tmpBuf = []byte{} + + return d, nil +} + func dhCreateTableFieldName(usrCtx any, deferred, token []byte) ([]byte, error) { uctx := usrCtx.(*userCtx) + + if uctx.security.isSkip == true { + return []byte{}, nil + } + uctx.column.name = string(deferred) return append(deferred, token...), nil @@ -28,6 +82,10 @@ func dhCreateTableColumnTypeAdd(usrCtx any, deferred, token []byte) ([]byte, err uctx := usrCtx.(*userCtx) + if uctx.security.isSkip == true { + return []byte{}, nil + } + for k, v := range typeKeys { if k == "generated" { if k == string(token) || strings.ToUpper(k) == string(token) { @@ -53,6 +111,10 @@ func dhCreateTableColumnAdd(usrCtx any, deferred, token []byte) ([]byte, error) uctx := usrCtx.(*userCtx) + if uctx.security.isSkip == true { + return []byte{}, nil + } + if uctx.column.isSkip == false { uctx.filter.ColumnAdd(uctx.column.name, uctx.column.columnType) } @@ -66,6 +128,10 @@ func dhInsertIntoTableName(usrCtx any, deferred, token []byte) ([]byte, error) { uctx := usrCtx.(*userCtx) + if uctx.security.isSkip == true { + return []byte{}, nil + } + // Check insert into table name if bytes.Compare([]byte(uctx.filter.TableNameGet()), deferred) != 0 { return append(deferred, token...), fmt.Errorf("`create` and `insert into` table names are mismatch (create table: '%s', insert into table: '%s')", uctx.filter.TableNameGet(), string(deferred)) @@ -78,6 +144,10 @@ func dhCreateTableValues(usrCtx any, deferred, token []byte) ([]byte, error) { uctx := usrCtx.(*userCtx) + if uctx.security.isSkip == true { + return []byte{}, nil + } + if bytes.Compare(deferred, []byte("NULL")) == 0 { uctx.filter.ValueAdd(nil) } else { @@ -91,6 +161,10 @@ func dhCreateTableValuesString(usrCtx any, deferred, token []byte) ([]byte, erro uctx := usrCtx.(*userCtx) + if uctx.security.isSkip == true { + return []byte{}, nil + } + uctx.filter.ValueAdd(deferred) return []byte{}, nil @@ -100,6 +174,10 @@ func dhCreateTableValuesEnd(usrCtx any, deferred, token []byte) ([]byte, error) uctx := usrCtx.(*userCtx) + if uctx.security.isSkip == true { + return []byte{}, nil + } + if bytes.Compare(deferred, []byte("NULL")) == 0 { uctx.filter.ValueAdd(nil) } else { @@ -118,6 +196,10 @@ func dhCreateTableValuesStringEnd(usrCtx any, deferred, token []byte) ([]byte, e uctx := usrCtx.(*userCtx) + if uctx.security.isSkip == true { + return []byte{}, nil + } + // Apply filter for row if err := uctx.filter.Apply(); err != nil { return []byte{}, err @@ -154,3 +236,26 @@ func rowDataGen(filter *relfilter.Filter) []byte { return []byte(fmt.Sprintf("(%s)", out)) } + +// SecurityPolicyCheck checks the table passes the security rules +// true: pass +// false: skip +func securityPolicyCheck(uctx *userCtx, tname string) bool { + + // Continue if security policy is `skip` + if uctx.security.policy.tables != misc.SecurityPolicyTablesSkip { + return true + } + + // Check rules for specified table name + if _, b := uctx.filter.TableNameLookup(tname); b == true { + return true + } + + // Check specified table name in exceptions + if _, b := uctx.security.exceptions.tables[tname]; b == true { + return true + } + + return false +} diff --git a/modules/anonymizers/mysql/mysql.go b/modules/anonymizers/mysql/mysql.go index 4059ed2..cb65802 100644 --- a/modules/anonymizers/mysql/mysql.go +++ b/modules/anonymizers/mysql/mysql.go @@ -5,14 +5,37 @@ import ( "io" "strings" + "github.com/nixys/nxs-data-anonymizer/misc" "github.com/nixys/nxs-data-anonymizer/modules/filters/relfilter" fsm "github.com/nixys/nxs-go-fsm" ) +type InitSettings struct { + Security SecuritySettings + Rules relfilter.Rules +} + +type SecuritySettings struct { + Policy SecurityPolicySettings + Exceptions SecurityExceptionsSettings +} + +type SecurityPolicySettings struct { + Tables misc.SecurityPolicyTablesType + Columns misc.SecurityPolicyColumnsType +} + +type SecurityExceptionsSettings struct { + Tables map[string]any + Columns map[string]any +} + type userCtx struct { filter *relfilter.Filter column userColumnCtx + + security securityCtx } type userColumnCtx struct { @@ -21,6 +44,24 @@ type userColumnCtx struct { isSkip bool } +type securityCtx struct { + tmpBuf []byte + isSkip bool + + policy securityPolicyCtx + exceptions securityExceptionsCtx +} + +type securityPolicyCtx struct { + tables misc.SecurityPolicyTablesType + columns misc.SecurityPolicyColumnsType +} + +type securityExceptionsCtx struct { + tables map[string]any + columns map[string]any +} + var typeKeys = map[string]relfilter.ColumnType{ // Special @@ -67,19 +108,29 @@ var typeKeys = map[string]relfilter.ColumnType{ "longblob": relfilter.ColumnTypeBinary, } -func userCtxInit(rules relfilter.Rules) *userCtx { +func userCtxInit(s InitSettings) *userCtx { return &userCtx{ - filter: relfilter.Init(rules), + filter: relfilter.Init(s.Rules), + security: securityCtx{ + policy: securityPolicyCtx{ + tables: s.Security.Policy.Tables, + columns: s.Security.Policy.Columns, + }, + exceptions: securityExceptionsCtx{ + tables: s.Security.Exceptions.Tables, + columns: s.Security.Exceptions.Columns, + }, + }, } } -func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { +func Init(ctx context.Context, r io.Reader, s InitSettings) io.Reader { return fsm.Init( r, fsm.Description{ Ctx: ctx, - UserCtx: userCtxInit(rules), + UserCtx: userCtxInit(s), InitState: stateCreateSearch, States: map[fsm.StateName]fsm.State{ @@ -94,7 +145,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityCreateTable, }, }, }, @@ -109,7 +160,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityCreateTableName, }, }, }, @@ -120,7 +171,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { Switch: fsm.Switch{ Trigger: []byte("`"), }, - DataHandler: nil, + DataHandler: dhSecurityCreateTableName, }, }, }, @@ -142,7 +193,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { Switch: fsm.Switch{ Trigger: []byte("("), }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, }, }, @@ -158,7 +209,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, { // Skip table keys description @@ -170,7 +221,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, { // Skip table keys description @@ -182,7 +233,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, { // Skip table keys description @@ -194,7 +245,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, { // Skip table keys description @@ -206,14 +257,14 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, { Name: stateFieldsDescriptionName, Switch: fsm.Switch{ Trigger: []byte("`"), }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, }, }, @@ -227,7 +278,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{'\n'}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, { Name: statefFieldsDescriptionBlockEnd, @@ -237,7 +288,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { L: []byte{'\n'}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, }, }, @@ -314,7 +365,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{'\n'}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, }, }, @@ -329,7 +380,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityCreateTable, }, { Name: stateInsertInto, @@ -340,7 +391,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, }, }, @@ -356,7 +407,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, }, }, @@ -367,7 +418,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { Switch: fsm.Switch{ Trigger: []byte("`"), }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, }, }, @@ -393,7 +444,7 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { R: []byte{' ', '\n'}, }, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, }, }, @@ -470,14 +521,14 @@ func Init(ctx context.Context, r io.Reader, rules relfilter.Rules) io.Reader { Switch: fsm.Switch{ Trigger: []byte(","), }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, { Name: stateSomeIntermediateState, Switch: fsm.Switch{ Trigger: []byte(";"), }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, }, }, diff --git a/modules/filters/relfilter/filter.go b/modules/filters/relfilter/filter.go index fae0c2f..f0ca96c 100644 --- a/modules/filters/relfilter/filter.go +++ b/modules/filters/relfilter/filter.go @@ -74,6 +74,12 @@ func (filter *Filter) TableNameGet() string { return filter.tableData.name } +// TableNameLookup looks up filters for specified table name +func (filter *Filter) TableNameLookup(name string) (TableRules, bool) { + t, b := filter.rules.Tables[name] + return t, b +} + // ColumnAdd adds new column into current data set func (filter *Filter) ColumnAdd(name string, t ColumnType) { filter.tableData.columns.add(name, t) @@ -111,7 +117,7 @@ func (filter *Filter) Apply() error { tname := filter.tableData.name // Check current table exist in rules - t, b := filter.rules.Tables[tname] + t, b := filter.TableNameLookup(tname) if b == true { td := misc.TemplateData{ diff --git a/routines/anonymizer/anonymizer.go b/routines/anonymizer/anonymizer.go index 800fd1a..0bd8c4e 100644 --- a/routines/anonymizer/anonymizer.go +++ b/routines/anonymizer/anonymizer.go @@ -27,6 +27,7 @@ type anomymizeSettings struct { db ctx.DBCtx rs relfilter.Rules w io.Writer + s ctx.SecurityCtx } func Runtime(app appctx.App) error { @@ -63,6 +64,7 @@ func Runtime(app appctx.App) error { db: cc.DB, rs: cc.Rules, w: cc.Output, + s: cc.Security, }, ); err != nil { return err @@ -131,7 +133,23 @@ func anomymize(st anomymizeSettings) error { } } - ar = mysql_anonymize.Init(st.c, st.pr, st.rs) + ar = mysql_anonymize.Init( + st.c, + st.pr, + mysql_anonymize.InitSettings{ + Security: mysql_anonymize.SecuritySettings{ + Policy: mysql_anonymize.SecurityPolicySettings{ + Tables: st.s.Policy.Tables, + Columns: st.s.Policy.Columns, + }, + Exceptions: mysql_anonymize.SecurityExceptionsSettings{ + Tables: st.s.Exceptions.Tables, + Columns: st.s.Exceptions.Columns, + }, + }, + Rules: st.rs, + }, + ) case ctx.DBTypePgSQL: ar = pgsql_anonymize.Init(st.c, st.pr, st.rs) default: