From 193bc2138fb06c310b9d5f3bd865219afa49ccac Mon Sep 17 00:00:00 2001 From: Boris Ershov Date: Fri, 31 May 2024 16:03:24 +0700 Subject: [PATCH] feat(#3): Add security enforcement for PgSQL --- ctx/context.go | 10 +++- modules/anonymizers/mysql/dh.go | 4 +- modules/anonymizers/mysql/mysql.go | 92 ++++++++++++++++++----------- modules/anonymizers/pgsql/dh.go | 80 +++++++++++++++++++++++-- modules/anonymizers/pgsql/pgsql.go | 86 ++++++++++++++++++++++----- modules/anonymizers/pgsql/states.go | 6 +- modules/filters/relfilter/column.go | 5 +- modules/filters/relfilter/filter.go | 18 ------ routines/anonymizer/anonymizer.go | 6 +- 9 files changed, 226 insertions(+), 81 deletions(-) diff --git a/ctx/context.go b/ctx/context.go index 10be5ee..34537d1 100644 --- a/ctx/context.go +++ b/ctx/context.go @@ -6,6 +6,9 @@ import ( "os" "time" + mysql_anonymize "github.com/nixys/nxs-data-anonymizer/modules/anonymizers/mysql" + pgsql_anonymize "github.com/nixys/nxs-data-anonymizer/modules/anonymizers/pgsql" + "github.com/nixys/nxs-data-anonymizer/ds/mysql" "github.com/nixys/nxs-data-anonymizer/misc" "github.com/nixys/nxs-data-anonymizer/modules/filters/relfilter" @@ -134,7 +137,12 @@ func AppCtxInit() (any, error) { c.Rules.Tables = make(map[string]relfilter.TableRules) if misc.SecurityPolicyColumnsTypeFromString(conf.Security.Policy.Columns) == misc.SecurityPolicyColumnsRandomize { - c.Rules.RandomizeTypes = relfilter.RandomizeTypesDefault + switch args.DBType { + case DBTypeMySQL: + c.Rules.RandomizeTypes = mysql_anonymize.RandomizeTypesDefault + case DBTypePgSQL: + c.Rules.RandomizeTypes = pgsql_anonymize.RandomizeTypesDefault + } } for t, f := range conf.Filters { diff --git a/modules/anonymizers/mysql/dh.go b/modules/anonymizers/mysql/dh.go index a9d5c9e..82215ac 100644 --- a/modules/anonymizers/mysql/dh.go +++ b/modules/anonymizers/mysql/dh.go @@ -224,9 +224,9 @@ func rowDataGen(filter *relfilter.Filter) []byte { out += "NULL" } else { switch filter.ColumnTypeGet(i) { - case relfilter.ColumnTypeString: + case columnTypeString: out += fmt.Sprintf("'%s'", v.V) - case relfilter.ColumnTypeBinary: + case columnTypeBinary: out += fmt.Sprintf("_binary '%s'", v.V) default: out += fmt.Sprintf("%s", v.V) diff --git a/modules/anonymizers/mysql/mysql.go b/modules/anonymizers/mysql/mysql.go index 065b6f5..c18b2f7 100644 --- a/modules/anonymizers/mysql/mysql.go +++ b/modules/anonymizers/mysql/mysql.go @@ -42,50 +42,74 @@ type securityCtx struct { tableExceptions map[string]any } +const ( + columnTypeString relfilter.ColumnType = "string" + columnTypeNum relfilter.ColumnType = "numeric" + columnTypeBinary relfilter.ColumnType = "binary" +) + var typeKeys = map[string]relfilter.ColumnType{ // Special "generated": relfilter.ColumnTypeNone, // Strings - "char": relfilter.ColumnTypeString, - "varchar": relfilter.ColumnTypeString, - "tinytext": relfilter.ColumnTypeString, - "text": relfilter.ColumnTypeString, - "mediumtext": relfilter.ColumnTypeString, - "longtext": relfilter.ColumnTypeString, - "enum": relfilter.ColumnTypeString, - "set": relfilter.ColumnTypeString, - "date": relfilter.ColumnTypeString, - "datetime": relfilter.ColumnTypeString, - "timestamp": relfilter.ColumnTypeString, - "time": relfilter.ColumnTypeString, - "year": relfilter.ColumnTypeString, - "json": relfilter.ColumnTypeString, + "char": columnTypeString, + "varchar": columnTypeString, + "tinytext": columnTypeString, + "text": columnTypeString, + "mediumtext": columnTypeString, + "longtext": columnTypeString, + "enum": columnTypeString, + "set": columnTypeString, + "date": columnTypeString, + "datetime": columnTypeString, + "timestamp": columnTypeString, + "time": columnTypeString, + "year": columnTypeString, + "json": columnTypeString, // Numeric - "bit": relfilter.ColumnTypeNum, - "bool": relfilter.ColumnTypeNum, - "boolean": relfilter.ColumnTypeNum, - "tinyint": relfilter.ColumnTypeNum, - "smallint": relfilter.ColumnTypeNum, - "mediumint": relfilter.ColumnTypeNum, - "int": relfilter.ColumnTypeNum, - "integer": relfilter.ColumnTypeNum, - "bigint": relfilter.ColumnTypeNum, - "float": relfilter.ColumnTypeNum, - "double": relfilter.ColumnTypeNum, - "double precision": relfilter.ColumnTypeNum, - "decimal": relfilter.ColumnTypeNum, - "dec": relfilter.ColumnTypeNum, + "bit": columnTypeNum, + "bool": columnTypeNum, + "boolean": columnTypeNum, + "tinyint": columnTypeNum, + "smallint": columnTypeNum, + "mediumint": columnTypeNum, + "int": columnTypeNum, + "integer": columnTypeNum, + "bigint": columnTypeNum, + "float": columnTypeNum, + "double": columnTypeNum, + "double precision": columnTypeNum, + "decimal": columnTypeNum, + "dec": columnTypeNum, // Binary - "binary": relfilter.ColumnTypeBinary, - "varbinary": relfilter.ColumnTypeBinary, - "tinyblob": relfilter.ColumnTypeBinary, - "blob": relfilter.ColumnTypeBinary, - "mediumblob": relfilter.ColumnTypeBinary, - "longblob": relfilter.ColumnTypeBinary, + "binary": columnTypeBinary, + "varbinary": columnTypeBinary, + "tinyblob": columnTypeBinary, + "blob": columnTypeBinary, + "mediumblob": columnTypeBinary, + "longblob": columnTypeBinary, +} + +var RandomizeTypesDefault = map[relfilter.ColumnType]relfilter.ColumnRule{ + columnTypeBinary: { + Type: misc.ValueTypeTemplate, + Value: "cmFuZG9taXplZCBiaW5hcnkgZGF0YQo=", + Unique: false, + }, + columnTypeNum: { + Type: misc.ValueTypeTemplate, + Value: "0", + Unique: false, + }, + columnTypeString: { + Type: misc.ValueTypeTemplate, + Value: "randomized string data", + Unique: false, + }, } func userCtxInit(s InitSettings) *userCtx { diff --git a/modules/anonymizers/pgsql/dh.go b/modules/anonymizers/pgsql/dh.go index 14cadc9..87640cf 100644 --- a/modules/anonymizers/pgsql/dh.go +++ b/modules/anonymizers/pgsql/dh.go @@ -4,9 +4,30 @@ import ( "bytes" "fmt" + "github.com/nixys/nxs-data-anonymizer/misc" "github.com/nixys/nxs-data-anonymizer/modules/filters/relfilter" ) +func dhSecurityCopy(usrCtx any, deferred, token []byte) ([]byte, error) { + + uctx := usrCtx.(*userCtx) + + uctx.security.tmpBuf = append(uctx.security.tmpBuf, token...) + + return deferred, nil +} + +func dhSecurityNil(usrCtx any, deferred, token []byte) ([]byte, error) { + + uctx := usrCtx.(*userCtx) + + if uctx.security.isSkip == true { + return []byte{}, nil + } + + return append(deferred, token...), nil +} + func dhCreateTableName(usrCtx any, deferred, token []byte) ([]byte, error) { tname := string(bytes.TrimSpace(deferred)) @@ -50,12 +71,28 @@ func dhCreateTableDesc(usrCtx any, deferred, token []byte) ([]byte, error) { func dhTableName(usrCtx any, deferred, token []byte) ([]byte, error) { - tname := bytes.TrimSpace(deferred) - uctx := usrCtx.(*userCtx) - uctx.filter.TableCreate(string(tname)) - return append(deferred, token...), nil + tname := string(bytes.TrimSpace(deferred)) + + if !securityPolicyCheck(uctx, tname) { + + // If not: table will be skipped from result dump + + uctx.security.isSkip = true + uctx.security.tmpBuf = []byte{} + + return []byte{}, nil + } + + uctx.filter.TableCreate(tname) + + d := append(uctx.security.tmpBuf, append(deferred, token...)...) + + uctx.security.isSkip = false + uctx.security.tmpBuf = []byte{} + + return d, nil } func dhFieldName(usrCtx any, deferred, token []byte) ([]byte, error) { @@ -64,6 +101,10 @@ func dhFieldName(usrCtx any, deferred, token []byte) ([]byte, error) { uctx := usrCtx.(*userCtx) + if uctx.security.isSkip == true { + return []byte{}, nil + } + t, b := uctx.tables[uctx.filter.TableNameGet()][string(fname)] if b == false { t = relfilter.ColumnTypeNone @@ -78,6 +119,10 @@ func dhValue(usrCtx any, deferred, token []byte) ([]byte, error) { uctx := usrCtx.(*userCtx) + if uctx.security.isSkip == true { + return []byte{}, nil + } + if bytes.Compare(deferred, []byte("\\N")) == 0 { uctx.filter.ValueAdd(nil) } else { @@ -91,6 +136,10 @@ func dhValueEnd(usrCtx any, deferred, token []byte) ([]byte, error) { uctx := usrCtx.(*userCtx) + if uctx.security.isSkip == true { + return []byte{}, nil + } + if bytes.Compare(deferred, []byte("\\N")) == 0 { uctx.filter.ValueAdd(nil) } else { @@ -126,3 +175,26 @@ func rowDataGen(filter *relfilter.Filter) []byte { return []byte(fmt.Sprintf("%s\n", out)) } + +// SecurityPolicyCheck checks the table passes the security rules +// true: pass +// false: skip +func securityPolicyCheck(uctx *userCtx, tname string) bool { + + // Continue if security policy is `skip` + if uctx.security.tablePolicy != misc.SecurityPolicyTablesSkip { + return true + } + + // Check rules for specified table name + if tr := uctx.filter.TableRulesLookup(tname); tr != nil { + return true + } + + // Check specified table name in exceptions + if _, b := uctx.security.tableExceptions[tname]; b == true { + return true + } + + return false +} diff --git a/modules/anonymizers/pgsql/pgsql.go b/modules/anonymizers/pgsql/pgsql.go index 543ac52..89dd7af 100644 --- a/modules/anonymizers/pgsql/pgsql.go +++ b/modules/anonymizers/pgsql/pgsql.go @@ -4,49 +4,107 @@ import ( "context" "io" + "github.com/nixys/nxs-data-anonymizer/misc" "github.com/nixys/nxs-data-anonymizer/modules/filters/relfilter" fsm "github.com/nixys/nxs-go-fsm" ) -type InitSettings struct { - Rules relfilter.Rules +type InitOpts struct { + Security SecurityOpts + Rules relfilter.Rules +} + +type SecurityOpts struct { + TablePolicy misc.SecurityPolicyTablesType + TableExceptions map[string]any } type userCtx struct { filter *relfilter.Filter + security securityCtx + tn *string tables map[string]map[string]relfilter.ColumnType } +type securityCtx struct { + tmpBuf []byte + isSkip bool + + tablePolicy misc.SecurityPolicyTablesType + tableExceptions map[string]any +} + +const ( + columnTypeString relfilter.ColumnType = "string" + columnTypeInt relfilter.ColumnType = "integer" + columnTypeFloat relfilter.ColumnType = "float" +) + var typeKeys = map[string]relfilter.ColumnType{ + // Integer + "smallint": columnTypeInt, + "integer": columnTypeInt, + "bigint": columnTypeInt, + "smallserial": columnTypeInt, + "serial": columnTypeInt, + "bigserial": columnTypeInt, + + // Float + "decimal": columnTypeFloat, + "numeric": columnTypeFloat, + "real": columnTypeFloat, + "double": columnTypeFloat, + // Strings - "character": relfilter.ColumnTypeString, + "character": columnTypeString, + "bpchar": columnTypeString, + "text": columnTypeString, +} - // Numeric - "integer": relfilter.ColumnTypeNum, +var RandomizeTypesDefault = map[relfilter.ColumnType]relfilter.ColumnRule{ + columnTypeInt: { + Type: misc.ValueTypeTemplate, + Value: "0", + Unique: false, + }, + columnTypeFloat: { + Type: misc.ValueTypeTemplate, + Value: "0.0", + Unique: false, + }, + columnTypeString: { + Type: misc.ValueTypeTemplate, + Value: "randomized string data", + Unique: false, + }, } -func userCtxInit(s InitSettings) *userCtx { +func userCtxInit(s InitOpts) *userCtx { return &userCtx{ filter: relfilter.Init(s.Rules), tables: make(map[string]map[string]relfilter.ColumnType), + security: securityCtx{ + tablePolicy: s.Security.TablePolicy, + tableExceptions: s.Security.TableExceptions, + }, } } -func Init(ctx context.Context, r io.Reader, s InitSettings) io.Reader { +func Init(ctx context.Context, r io.Reader, s InitOpts) io.Reader { return fsm.Init( r, fsm.Description{ Ctx: ctx, UserCtx: userCtxInit(s), - InitState: stateCopySearch, + InitState: stateInit, States: map[fsm.StateName]fsm.State{ - stateCopySearch: { + stateInit: { NextStates: []fsm.NextState{ { Name: stateTableName, @@ -57,7 +115,7 @@ func Init(ctx context.Context, r io.Reader, s InitSettings) io.Reader { R: []byte{' '}, }, }, - DataHandler: nil, + DataHandler: dhSecurityCopy, }, { Name: stateCreateTableName, @@ -91,7 +149,7 @@ func Init(ctx context.Context, r io.Reader, s InitSettings) io.Reader { stateCreateTableTail: { NextStates: []fsm.NextState{ { - Name: stateCopySearch, + Name: stateInit, Switch: fsm.Switch{ Trigger: []byte(");"), Delimiters: fsm.Delimiters{ @@ -139,14 +197,14 @@ func Init(ctx context.Context, r io.Reader, s InitSettings) io.Reader { Switch: fsm.Switch{ Trigger: []byte(";\n"), }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, }, }, stateTableValues: { NextStates: []fsm.NextState{ { - Name: stateCopySearch, + Name: stateInit, Switch: fsm.Switch{ Trigger: []byte("\\."), Delimiters: fsm.Delimiters{ @@ -155,7 +213,7 @@ func Init(ctx context.Context, r io.Reader, s InitSettings) io.Reader { }, Escape: false, }, - DataHandler: nil, + DataHandler: dhSecurityNil, }, { Name: stateTableValues, diff --git a/modules/anonymizers/pgsql/states.go b/modules/anonymizers/pgsql/states.go index 2d11d06..339b657 100644 --- a/modules/anonymizers/pgsql/states.go +++ b/modules/anonymizers/pgsql/states.go @@ -3,9 +3,9 @@ package pgsql_anonymize import fsm "github.com/nixys/nxs-go-fsm" var ( - stateCreateTableName = fsm.StateName("creat table name") - stateCreateTableTail = fsm.StateName("creat table tail") - stateCopySearch = fsm.StateName("copy search") + stateInit = fsm.StateName("init") + stateCreateTableName = fsm.StateName("create table name") + stateCreateTableTail = fsm.StateName("create table tail") stateTableName = fsm.StateName("table name") stateFieldName = fsm.StateName("field name") stateCopyTail = fsm.StateName("copy tail") diff --git a/modules/filters/relfilter/column.go b/modules/filters/relfilter/column.go index 559a97b..887013d 100644 --- a/modules/filters/relfilter/column.go +++ b/modules/filters/relfilter/column.go @@ -13,10 +13,7 @@ type column struct { type ColumnType string const ( - ColumnTypeNone ColumnType = "none" - ColumnTypeString ColumnType = "string" - ColumnTypeBinary ColumnType = "binary" - ColumnTypeNum ColumnType = "numeric" + ColumnTypeNone ColumnType = "none" ) func (c ColumnType) String() string { diff --git a/modules/filters/relfilter/filter.go b/modules/filters/relfilter/filter.go index 4ae5bbb..8278393 100644 --- a/modules/filters/relfilter/filter.go +++ b/modules/filters/relfilter/filter.go @@ -63,24 +63,6 @@ type rule struct { cr ColumnRule } -var RandomizeTypesDefault = map[ColumnType]ColumnRule{ - ColumnTypeBinary: { - Type: misc.ValueTypeTemplate, - Value: "cmFuZG9taXplZCBiaW5hcnkgZGF0YQo=", - Unique: false, - }, - ColumnTypeNum: { - Type: misc.ValueTypeTemplate, - Value: "0", - Unique: false, - }, - ColumnTypeString: { - Type: misc.ValueTypeTemplate, - Value: "randomized string data", - Unique: false, - }, -} - func Init(rules Rules) *Filter { return &Filter{ rules: rules, diff --git a/routines/anonymizer/anonymizer.go b/routines/anonymizer/anonymizer.go index 40d6711..8aa85ef 100644 --- a/routines/anonymizer/anonymizer.go +++ b/routines/anonymizer/anonymizer.go @@ -148,7 +148,11 @@ func anonymize(st anonymizeSettings) error { ar = pgsql_anonymize.Init( st.c, st.pr, - pgsql_anonymize.InitSettings{ + pgsql_anonymize.InitOpts{ + Security: pgsql_anonymize.SecurityOpts{ + TablePolicy: st.s.TablePolicy, + TableExceptions: st.s.TableExceptions, + }, Rules: st.rs, }, )