Skip to content

Commit

Permalink
Merge pull request #56 from nixys/feat/32
Browse files Browse the repository at this point in the history 
feat(#32): Add support for DOS line endings
  • Loading branch information
@borisershov
borisershov authored Nov 13, 2024
2 parents bc37adc + 4be6f6a commit c2b79a5
Show file tree
Hide file tree
Showing 12 changed files with 719 additions and 13 deletions.
106 changes: 106 additions & 0 deletions modules/anonymizers/mysql/.testdata/mysql_test.dos.in.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
-- MariaDB dump 10.19 Distrib 10.6.12-MariaDB, for debian-linux-gnu (x86_64)
--
-- Host: 127.0.0.1 Database: db
-- ------------------------------------------------------
-- Server version 8.0.32

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8mb4 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `table1`
--

DROP TABLE IF EXISTS `table1`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `table1` (
`id` int NOT NULL AUTO_INCREMENT,
`name` varchar(50) NOT NULL,
`url` varchar(255) NOT NULL,
`auth_method` varchar(20) NOT NULL DEFAULT 'none',
`auth_data` text NOT NULL,
`username` varchar(20) NOT NULL DEFAULT '',
`password` varchar(20) NOT NULL DEFAULT '',
PRIMARY KEY (`id`),
UNIQUE KEY `id` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `table1`
--

LOCK TABLES `table1` WRITE;
/*!40000 ALTER TABLE `table1` DISABLE KEYS */;
INSERT INTO `table1` VALUES (1,'test4','https://github.com/nixys/nxs-rbac-operator0.git','none','','',''),(2,'test5','https://github.com/nixys/nxs-rbac-operator.git','none','','',''),(3,'test6','https://github.com/nixys/nxs-rbac-operator2.git','none','','','');
/*!40000 ALTER TABLE `table1` ENABLE KEYS */;
UNLOCK TABLES;

--
-- Table structure for table `table2`
--

DROP TABLE IF EXISTS `table2`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `table2` (
`id` int NOT NULL AUTO_INCREMENT,
`name` varchar(50) NOT NULL,
`url` varchar(255) NOT NULL,
`auth_method` varchar(20) NOT NULL DEFAULT 'none',
`auth_data` text NOT NULL,
`username` varchar(20) NOT NULL DEFAULT '',
`password` varchar(20) NOT NULL DEFAULT '',
PRIMARY KEY (`id`),
UNIQUE KEY `id` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `table2`
--

LOCK TABLES `table2` WRITE;
/*!40000 ALTER TABLE `table2` DISABLE KEYS */;
INSERT INTO `table2` VALUES (1,'test4','https://github.com/nixys/nxs-rbac-operator0.git','none','','',''),(2,'test5','https://github.com/nixys/nxs-rbac-operator.git','none','','',''),(3,'test6','https://github.com/nixys/nxs-rbac-operator2.git','none','','','');
/*!40000 ALTER TABLE `table2` ENABLE KEYS */;
UNLOCK TABLES;

--
-- Table structure for table `table3`
--

DROP TABLE IF EXISTS `table3`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `table3` (
`id` int NOT NULL AUTO_INCREMENT,
`name` varchar(50) NOT NULL,
`url` varchar(255) NOT NULL,
`auth_method` varchar(20) NOT NULL DEFAULT 'none',
`auth_data` text NOT NULL,
`username` varchar(20) NOT NULL DEFAULT '',
`password` varchar(20) NOT NULL DEFAULT '',
PRIMARY KEY (`id`),
UNIQUE KEY `id` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `table3`
--

LOCK TABLES `table3` WRITE;
/*!40000 ALTER TABLE `table3` DISABLE KEYS */;
INSERT INTO `table3` VALUES (1,'test4','https://github.com/nixys/nxs-rbac-operator0.git','none','','',''),(2,'test5','https://github.com/nixys/nxs-rbac-operator.git','none','','',''),(3,'test6','https://github.com/nixys/nxs-rbac-operator2.git','none','','','');
/*!40000 ALTER TABLE `table3` ENABLE KEYS */;
UNLOCK TABLES;
106 changes: 106 additions & 0 deletions modules/anonymizers/mysql/.testdata/mysql_test.dos.out.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
-- MariaDB dump 10.19 Distrib 10.6.12-MariaDB, for debian-linux-gnu (x86_64)
--
-- Host: 127.0.0.1 Database: db
-- ------------------------------------------------------
-- Server version 8.0.32

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8mb4 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `table1`
--

DROP TABLE IF EXISTS `table1`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `table1` (
`id` int NOT NULL AUTO_INCREMENT,
`name` varchar(50) NOT NULL,
`url` varchar(255) NOT NULL,
`auth_method` varchar(20) NOT NULL DEFAULT 'none',
`auth_data` text NOT NULL,
`username` varchar(20) NOT NULL DEFAULT '',
`password` varchar(20) NOT NULL DEFAULT '',
PRIMARY KEY (`id`),
UNIQUE KEY `id` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `table1`
--

LOCK TABLES `table1` WRITE;
/*!40000 ALTER TABLE `table1` DISABLE KEYS */;
INSERT INTO `table1` VALUES (1,'test4','https://github.com/nixys/nxs-rbac-operator0.git','none','','',''),(3,'test6','https://github.com/nixys/nxs-rbac-operator2.git','none','','','');
/*!40000 ALTER TABLE `table1` ENABLE KEYS */;
UNLOCK TABLES;

--
-- Table structure for table `table2`
--

DROP TABLE IF EXISTS `table2`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `table2` (
`id` int NOT NULL AUTO_INCREMENT,
`name` varchar(50) NOT NULL,
`url` varchar(255) NOT NULL,
`auth_method` varchar(20) NOT NULL DEFAULT 'none',
`auth_data` text NOT NULL,
`username` varchar(20) NOT NULL DEFAULT '',
`password` varchar(20) NOT NULL DEFAULT '',
PRIMARY KEY (`id`),
UNIQUE KEY `id` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `table2`
--

LOCK TABLES `table2` WRITE;
/*!40000 ALTER TABLE `table2` DISABLE KEYS */;

/*!40000 ALTER TABLE `table2` ENABLE KEYS */;
UNLOCK TABLES;

--
-- Table structure for table `table3`
--

DROP TABLE IF EXISTS `table3`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `table3` (
`id` int NOT NULL AUTO_INCREMENT,
`name` varchar(50) NOT NULL,
`url` varchar(255) NOT NULL,
`auth_method` varchar(20) NOT NULL DEFAULT 'none',
`auth_data` text NOT NULL,
`username` varchar(20) NOT NULL DEFAULT '',
`password` varchar(20) NOT NULL DEFAULT '',
PRIMARY KEY (`id`),
UNIQUE KEY `id` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb3;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `table3`
--

LOCK TABLES `table3` WRITE;
/*!40000 ALTER TABLE `table3` DISABLE KEYS */;
INSERT INTO `table3` VALUES (1,'test4','https://github.com/nixys/nxs-rbac-operator0.git','none','','',''),(2,'test5','https://github.com/nixys/nxs-rbac-operator.git','none','','',''),(3,'test6','https://github.com/nixys/nxs-rbac-operator2.git','none','','','');
/*!40000 ALTER TABLE `table3` ENABLE KEYS */;
UNLOCK TABLES;
0 modules/anonymizers/mysql/mysql_test.in.sql → ...ymizers/mysql/.testdata/mysql_test.in.sql
View file
File renamed without changes.
0 modules/anonymizers/mysql/mysql_test.out.sql → ...mizers/mysql/.testdata/mysql_test.out.sql
View file
File renamed without changes.
8 changes: 4 additions & 4 deletions modules/anonymizers/mysql/mysql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,7 @@ func (m *MySQL) Run(ctx context.Context, w io.Writer) error {
Switch: fsm.Switch{
Trigger: []byte(","),
Delimiters: fsm.Delimiters{
R: []byte{'\n'},
R: []byte{'\n', '\r'},
},
},
DataHandler: nil,
Expand Down Expand Up @@ -359,7 +359,7 @@ func (m *MySQL) Run(ctx context.Context, w io.Writer) error {
Switch: fsm.Switch{
Trigger: []byte(","),
Delimiters: fsm.Delimiters{
R: []byte{'\n'},
R: []byte{'\n', '\r'},
},
},
DataHandler: dhCreateTableColumnAdd,
Expand All @@ -383,7 +383,7 @@ func (m *MySQL) Run(ctx context.Context, w io.Writer) error {
Switch: fsm.Switch{
Trigger: []byte(";"),
Delimiters: fsm.Delimiters{
R: []byte{'\n'},
R: []byte{'\n', '\r'},
},
},
DataHandler: nil,
Expand Down Expand Up @@ -447,7 +447,7 @@ func (m *MySQL) Run(ctx context.Context, w io.Writer) error {
Trigger: []byte("VALUES"),
Delimiters: fsm.Delimiters{
L: []byte{' ', '\n'},
R: []byte{' ', '\n'},
R: []byte{' ', '\n', '\r'},
},
},
DataHandler: dhSecurityInsertIntoValues,
Expand Down
77 changes: 74 additions & 3 deletions modules/anonymizers/mysql/mysql_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ func TestMySQL(t *testing.T) {

var r, e bytes.Buffer

fin, err := os.Open("mysql_test.in.sql")
fin, err := os.Open(".testdata/mysql_test.in.sql")
if err != nil {
t.Fatal("open input SQL:", err)
}
Expand Down Expand Up @@ -63,7 +63,7 @@ func TestMySQL(t *testing.T) {
t.Fatal("run MySQL:", err)
}

fout, err := os.Open("mysql_test.out.sql")
fout, err := os.Open(".testdata/mysql_test.out.sql")
if err != nil {
t.Fatal("open output SQL:", err)
}
Expand All @@ -72,7 +72,78 @@ func TestMySQL(t *testing.T) {
t.Fatal("read output SQL:", err)
}

// os.WriteFile("mysql_test.out.sql", r.Bytes(), 0644)
// os.WriteFile(".testdata/mysql_test.out.sql", r.Bytes(), 0644)

if r.String() != e.String() {
t.Fatal("incorrect anonymization result")
}

t.Logf("success")
}

func TestMySQLDos(t *testing.T) {

var r, e bytes.Buffer

fin, err := os.Open(".testdata/mysql_test.dos.in.sql")
if err != nil {
t.Fatal("open input SQL:", err)
}

m, err := Init(
fin,
InitOpts{
Rules: RulesOpts{
TableRules: map[string]map[string]relfilter.ColumnRuleOpts{

// Delete only row with id `2`
"table1": {
"id": relfilter.ColumnRuleOpts{
Type: misc.ValueTypeTemplate,
Value: "{{ if eq .Values.id \"2\" }}{{ drop }}{{ else }}{{ .Values.id }}{{ end }}",
Unique: false,
},
},

// Delete all rows from table
"table2": {
"id": relfilter.ColumnRuleOpts{
Type: misc.ValueTypeTemplate,
Value: "{{ drop }}",
Unique: false,
},
},

// Delete no rows
"table3": {
"id": relfilter.ColumnRuleOpts{
Type: misc.ValueTypeTemplate,
Value: "{{ if eq .Values.id \"4\" }}{{ drop }}{{ else }}{{ .Values.id }}{{ end }}",
Unique: false,
},
},
},
},
},
)
if err != nil {
t.Fatal("init MySQL:", err)
}

if err := m.Run(context.Background(), &r); err != nil {
t.Fatal("run MySQL:", err)
}

fout, err := os.Open(".testdata/mysql_test.dos.out.sql")
if err != nil {
t.Fatal("open output SQL:", err)
}

if _, err := e.ReadFrom(fout); err != nil {
t.Fatal("read output SQL:", err)
}

// os.WriteFile(".testdata/mysql_test.dos.out.sql", r.Bytes(), 0644)

if r.String() != e.String() {
t.Fatal("incorrect anonymization result")
Expand Down
Loading

0 comments on commit c2b79a5

Please sign in to comment.