SidecarDB: Don't ignore table charset and collation

go/vt/sidecardb/schema/misc/heartbeat.sql

@@ -20,4 +20,4 @@ CREATE TABLE IF NOT EXISTS heartbeat
     tabletUid     INT UNSIGNED    NOT NULL,
     ts            BIGINT UNSIGNED NOT NULL,
     PRIMARY KEY (`keyspaceShard`)
-) engine = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/misc/reparent_journal.sql

@@ -22,4 +22,4 @@ CREATE TABLE IF NOT EXISTS reparent_journal
     `replication_position` varbinary(64000) DEFAULT NULL,
 
     PRIMARY KEY (`time_created_ns`)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/onlineddl/schema_migrations.sql

@@ -84,4 +84,4 @@ CREATE TABLE IF NOT EXISTS schema_migrations
     KEY `table_complete_idx` (`migration_status`, `keyspace`(64), `mysql_table`(64), `completed_timestamp`),
     KEY `migration_context_idx` (`migration_context`(64)),
     KEY `reverted_uuid_idx` (`reverted_uuid`)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/schemaengine/tables.sql

@@ -21,4 +21,4 @@ CREATE TABLE IF NOT EXISTS tables
     CREATE_STATEMENT longtext,
     CREATE_TIME BIGINT,
     PRIMARY KEY (TABLE_SCHEMA, TABLE_NAME)
-) engine = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/schemaengine/udfs.sql

@@ -20,4 +20,4 @@ CREATE TABLE IF NOT EXISTS udfs
     FUNCTION_RETURN_TYPE varchar(20) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL,
     FUNCTION_TYPE varchar(20) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL,
     PRIMARY KEY (FUNCTION_NAME)
-) engine = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/schemaengine/views.sql

@@ -21,4 +21,4 @@ CREATE TABLE IF NOT EXISTS views
     CREATE_STATEMENT longtext,
     VIEW_DEFINITION longtext NOT NULL,
     PRIMARY KEY (TABLE_SCHEMA, TABLE_NAME)
-) engine = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/twopc/dt_participant.sql

@@ -21,4 +21,4 @@ CREATE TABLE IF NOT EXISTS dt_participant
     keyspace varchar(256) NOT NULL,
     shard varchar(256) NOT NULL,
     primary key(dtid, id)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/twopc/dt_state.sql

@@ -21,4 +21,4 @@ CREATE TABLE IF NOT EXISTS dt_state
   time_created bigint NOT NULL,
   primary key(dtid),
   key (time_created)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/twopc/redo_state.sql

@@ -19,4 +19,4 @@ CREATE TABLE IF NOT EXISTS redo_state(
   state bigint NOT NULL,
   time_created bigint NOT NULL,
   primary key(dtid)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/twopc/redo_statement.sql

@@ -19,4 +19,4 @@ CREATE TABLE IF NOT EXISTS redo_statement(
   id bigint NOT NULL,
   statement mediumblob NOT NULL,
   primary key(dtid, id)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/vdiff/vdiff.sql

@@ -33,4 +33,4 @@ CREATE TABLE IF NOT EXISTS vdiff
     UNIQUE KEY `uuid_idx` (`vdiff_uuid`),
     KEY `state` (`state`),
     KEY `ks_wf_idx` (`keyspace`(64), `workflow`(64))
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/vdiff/vdiff_log.sql

@@ -21,4 +21,4 @@ CREATE TABLE IF NOT EXISTS vdiff_log
     `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
     `message`    text      NOT NULL,
     PRIMARY KEY (`id`)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/vdiff/vdiff_table.sql

@@ -27,4 +27,4 @@ CREATE TABLE IF NOT EXISTS vdiff_table
     `created_at`    timestamp      NOT NULL DEFAULT CURRENT_TIMESTAMP,
     `updated_at`    timestamp      NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     PRIMARY KEY (`vdiff_id`, `table_name`)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/vreplication/copy_state.sql

@@ -22,4 +22,4 @@ CREATE TABLE IF NOT EXISTS copy_state
     `lastpk`     varbinary(2000) DEFAULT NULL,
     PRIMARY KEY (`id`),
     KEY `vrepl_id` (`vrepl_id`,`table_name`)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/vreplication/post_copy_action.sql

@@ -21,4 +21,4 @@ CREATE TABLE IF NOT EXISTS post_copy_action(
   action JSON NOT NULL,
   UNIQUE KEY (vrepl_id, table_name),
   PRIMARY KEY(id)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/vreplication/resharding_journal.sql

@@ -20,4 +20,4 @@ CREATE TABLE IF NOT EXISTS resharding_journal
     `db_name` varbinary(255) DEFAULT NULL,
     `val`     blob,
     PRIMARY KEY (`id`)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/vreplication/schema_version.sql

@@ -22,4 +22,4 @@ CREATE TABLE IF NOT EXISTS schema_version
     ddl          BLOB DEFAULT NULL,
     schemax      LONGBLOB         NOT NULL,
     PRIMARY KEY (id)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/vreplication/vreplication.sql

@@ -47,4 +47,4 @@ CREATE TABLE IF NOT EXISTS vreplication
     `options`               json             NOT NULL,
     PRIMARY KEY (`id`),
     KEY `workflow_idx` (`workflow`(64))
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/schema/vreplication/vreplication_log.sql

@@ -26,4 +26,4 @@ CREATE TABLE IF NOT EXISTS vreplication_log
     `count`      bigint         NOT NULL DEFAULT '1',
     PRIMARY KEY (`id`),
     KEY `vrepl_id_idx` (`vrepl_id`)
-) ENGINE = InnoDB
+) ENGINE = InnoDB CHARSET = utf8mb4

go/vt/sidecardb/sidecardb.go

@@ -396,7 +396,7 @@ func (si *schemaInit) getCurrentSchema(tableName string) (string, error) {
 // or an ALTER if the table exists but has a different schema.
 func (si *schemaInit) findTableSchemaDiff(tableName, current, desired string) (string, error) {
 	hints := &schemadiff.DiffHints{
-		TableCharsetCollateStrategy: schemadiff.TableCharsetCollateIgnoreAlways,
+		TableCharsetCollateStrategy: schemadiff.TableCharsetCollateIgnoreEmpty,
 		AlterTableAlgorithmStrategy: schemadiff.AlterTableAlgorithmStrategyCopy,
 	}
 	env := schemadiff.NewEnv(si.env, si.coll)

go/vt/sidecardb/sidecardb_test.go

@@ -250,3 +250,89 @@ func TestAlterTableAlgorithm(t *testing.T) {
 		})
 	}
 }
+
+// TestTableSchemaDiff ensures that the diff produced by schemaInit.findTableSchemaDiff
+// is resulting in the expected alter table statements in a variety of scenarios.
+func TestTableSchemaDiff(t *testing.T) {
+	si := &schemaInit{
+		env: vtenv.NewTestEnv(),
+	}
+
+	type testCase struct {
+		name          string
+		table         string
+		oldSchema     string
+		newSchema     string
+		expectNoDiff  bool
+		expectedAlter string
+	}
+	testCases := []testCase{
+		{
+			name:          "modify table charset",
+			table:         "t1",
+			oldSchema:     "create table if not exists _vt.t1(i int) charset=utf8mb4",
+			newSchema:     "create table if not exists _vt.t(i int) charset=utf8mb3",
+			expectedAlter: "alter table _vt.t1 charset utf8mb3, algorithm = copy",
+		},
+		{
+			name:         "empty charset",
+			table:        "t1",
+			oldSchema:    "create table if not exists _vt.t1(i int) charset=utf8mb4",
+			newSchema:    "create table if not exists _vt.t(i int)",
+			expectNoDiff: true, // We're not specifying an explicit charset in the new schema, so we shouldn't see a diff.
+		},
+		{
+			name:          "modify table engine",
+			table:         "t1",
+			oldSchema:     "create table if not exists _vt.t1(i int) engine=myisam",
+			newSchema:     "create table if not exists _vt.t(i int) engine=innodb",
+			expectedAlter: "alter table _vt.t1 engine innodb, algorithm = copy",
+		},
+		{
+			name:          "add, modify, transfer PK",
+			table:         "t1",
+			oldSchema:     "create table _vt.t1 (i int primary key, i1 varchar(10)) charset utf8mb4",
+			newSchema:     "create table _vt.t1 (i int, i1 varchar(20) character set utf8mb3 collate utf8mb3_bin, i2 int, primary key (i2)) charset utf8mb4",
+			expectedAlter: "alter table _vt.t1 drop primary key, modify column i1 varchar(20) character set utf8mb3 collate utf8mb3_bin, add column i2 int, add primary key (i2), algorithm = copy",
+		},
+		{
+			name:          "modify visibility and add comment",
+			table:         "t1",
+			oldSchema:     "create table if not exists _vt.t1(c1 int, c2 int, c3 varchar(100)) charset utf8mb4",
+			newSchema:     "create table if not exists _vt.t1(c1 int, c2 int, c3 varchar(100) invisible comment 'hoping to drop') charset utf8mb4",
+			expectedAlter: "alter table _vt.t1 modify column c3 varchar(100) comment 'hoping to drop' invisible, algorithm = copy",
+		},
+		{
+			name:          "add PK and remove index",
+			table:         "t1",
+			oldSchema:     "create table if not exists _vt.t1(c1 int, c2 int, c3 varchar(100), key (c2)) charset utf8mb4",
+			newSchema:     "create table if not exists _vt.t1(c1 int primary key, c2 int, c3 varchar(100)) charset utf8mb4",
+			expectedAlter: "alter table _vt.t1 drop key c2, add primary key (c1), algorithm = copy",
+		},
+		{
+			name:          "add generated col",
+			table:         "t1",
+			oldSchema:     "create table if not exists _vt.t1(c1 int primary key) charset utf8mb4",
+			newSchema:     "create table if not exists _vt.t1(c1 int primary key, c2 varchar(10) generated always as ('hello')) charset utf8mb4",
+			expectedAlter: "alter table _vt.t1 add column c2 varchar(10) as ('hello') virtual, algorithm = copy",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			diff, err := si.findTableSchemaDiff(tc.table, tc.oldSchema, tc.newSchema)
+			require.NoError(t, err)
+			if tc.expectNoDiff {
+				require.Empty(t, diff)
+				return
+			}
+			stmt, err := si.env.Parser().Parse(diff)
+			require.NoError(t, err)
+			alter, ok := stmt.(*sqlparser.AlterTable)
+			require.True(t, ok)
+			require.NotNil(t, alter)
+			t.Logf("alter: %s", sqlparser.String(alter))
+			require.Equal(t, strings.ToLower(tc.expectedAlter), strings.ToLower(sqlparser.String(alter)))
+		})
+	}
+}