Merge remote-tracking branch 'origin/main' into vstream_enum_strings

Signed-off-by: Matt Lord <mattalord@gmail.com>

config/init_db.sql

@@ -1,30 +1,22 @@
 # This file is executed immediately after initializing a fresh data directory.
 
-###############################################################################
-# WARNING: This sql is *NOT* safe for production use,
-# as it contains default well-known users and passwords.
-# Care should be taken to change these users and passwords
-# for production.
-###############################################################################
-
 ###############################################################################
 # Equivalent of mysql_secure_installation
 ###############################################################################
 # We need to ensure that super_read_only is disabled so that we can execute
 # these commands. Note that disabling it does NOT disable read_only.
 # We save the current value so that we only re-enable it at the end if it was
 # enabled before.
+
 SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF');
 SET GLOBAL super_read_only='OFF';
 
 # Changes during the init db should not make it to the binlog.
 # They could potentially create errant transactions on replicas.
 SET sql_log_bin = 0;
-# Remove anonymous users.
-DELETE FROM mysql.user WHERE User = '';
 
-# Disable remote root access (only allow UNIX socket).
-DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
+# Remove anonymous users & disable remote root access (only allow UNIX socket).
+DROP USER IF EXISTS ''@'%', ''@'localhost', 'root'@'%';
 
 # Remove test database.
 DROP DATABASE IF EXISTS test;
@@ -78,8 +70,6 @@ GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD
 GRANT SELECT, UPDATE, DELETE, DROP
  ON performance_schema.* TO 'vt_monitoring'@'localhost';
 
-FLUSH PRIVILEGES;
-
 RESET SLAVE ALL;
 RESET MASTER;
 

examples/compose/config/init_db.sql

@@ -12,10 +12,8 @@ SET GLOBAL super_read_only='OFF';
 # Changes during the init db should not make it to the binlog.
 # They could potentially create errant transactions on replicas.
 SET sql_log_bin = 0;
-# Remove anonymous users.
-DELETE FROM mysql.user WHERE User = '';
-# Disable remote root access (only allow UNIX socket).
-DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
+# Remove anonymous users & disable remote root access (only allow UNIX socket).
+DROP USER IF EXISTS ''@'%', ''@'localhost', 'root'@'%';
 # Remove test database.
 DROP DATABASE IF EXISTS test;
 ###############################################################################
@@ -70,7 +68,7 @@ GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
  LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
  SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
  ON *.* TO 'vt_filtered'@'localhost';
-FLUSH PRIVILEGES;
+
 RESET SLAVE ALL;
 RESET MASTER;
 

examples/compose/external_db/mysql/grant.sh

@@ -3,5 +3,5 @@ echo '**********GRANTING PRIVILEGES START*******************'
 echo ${mysql[@]}
 # PURGE BINARY LOGS BEFORE DATE(NOW());
 mysql --protocol=socket -uroot -hlocalhost --socket=/var/run/mysqld/mysqld.sock -p$MYSQL_ROOT_PASSWORD -e \
-"GRANT ALL PRIVILEGES ON *.* TO '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD'; FLUSH PRIVILEGES;"
-echo '*************GRANTING PRIVILEGES END****************'
+"GRANT ALL PRIVILEGES ON *.* TO '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD'"
+echo '*************GRANTING PRIVILEGES END****************'

examples/compose/vttablet-up.sh

@@ -68,7 +68,7 @@ if [ "$external" = "1" ]; then
  # We need a common user for the unmanaged and managed tablets else tools like orchestrator will not function correctly
  echo "Creating matching user for managed tablets..."
  echo "CREATE USER IF NOT EXISTS '$DB_USER'@'%' IDENTIFIED BY '$DB_PASS';" >> $init_db_sql_file
- echo "GRANT ALL ON *.* TO '$DB_USER'@'%';FLUSH PRIVILEGES;" >> $init_db_sql_file
+ echo "GRANT ALL ON *.* TO '$DB_USER'@'%';" >> $init_db_sql_file
 fi
 echo "##[CUSTOM_SQL_END]##" >> $init_db_sql_file
 

examples/operator/101_initial_cluster.yaml

@@ -145,11 +145,8 @@ stringData:
  # Changes during the init db should not make it to the binlog.
  # They could potentially create errant transactions on replicas.
  SET sql_log_bin = 0;
- # Remove anonymous users.
- DELETE FROM mysql.user WHERE User = '';
-
- # Disable remote root access (only allow UNIX socket).
- DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
+ # Remove anonymous users & disable remote root access (only allow UNIX socket).
+ DROP USER IF EXISTS ''@'%', ''@'localhost', 'root'@'%';
 
  # Remove test database.
  DROP DATABASE IF EXISTS test;
@@ -215,8 +212,6 @@ stringData:
  SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
  ON *.* TO 'vt_filtered'@'localhost';
 
- FLUSH PRIVILEGES;
-
  RESET SLAVE ALL;
  RESET MASTER;
 

go/cmd/vtctldclient/command/vreplication/movetables/movetables.go

@@ -48,7 +48,6 @@ func registerCommands(root *cobra.Command) {
  create.Flags().BoolVar(&createOptions.NoRoutingRules, "no-routing-rules", false, "(Advanced) Do not create routing rules while creating the workflow. See the reference documentation for limitations if you use this flag.")
  create.Flags().BoolVar(&createOptions.AtomicCopy, "atomic-copy", false, "(EXPERIMENTAL) A single copy phase is run for all tables from the source. Use this, for example, if your source keyspace has tables which use foreign key constraints.")
  create.Flags().StringVar(&createOptions.WorkflowOptions.TenantId, "tenant-id", "", "(EXPERIMENTAL) The tenant ID to use for the MoveTables workflow into a multi-tenant keyspace.")
- create.Flags().StringVar(&createOptions.WorkflowOptions.SourceKeyspaceAlias, "source-keyspace-alias", "", "(EXPERIMENTAL) Used currently only for multi-tenant migrations. This value will be used instead of the source keyspace name in the keyspace routing rules.")
  create.Flags().BoolVar(&createOptions.WorkflowOptions.StripShardedAutoIncrement, "remove-sharded-auto-increment", true, "If moving the table(s) to a sharded keyspace, remove any auto_increment clauses when copying the schema to the target as sharded keyspaces should rely on either user/application generated values or Vitess sequences to ensure uniqueness.")
  base.AddCommand(create)
 

go/test/endtoend/cluster/cluster_util.go

@@ -351,7 +351,6 @@ func GetPasswordUpdateSQL(localCluster *LocalProcessCluster) string {
  SET PASSWORD FOR 'vt_repl'@'%' = 'VtReplPass';
  SET PASSWORD FOR 'vt_filtered'@'localhost' = 'VtFilteredPass';
  SET PASSWORD FOR 'vt_appdebug'@'localhost' = 'VtDebugPass';
- FLUSH PRIVILEGES;
  `
  return pwdChangeCmd
 }

go/test/endtoend/throttler/util.go

@@ -34,9 +34,10 @@ import (
  "vitess.io/vitess/go/test/endtoend/cluster"
  "vitess.io/vitess/go/vt/concurrency"
  "vitess.io/vitess/go/vt/log"
- topodatapb "vitess.io/vitess/go/vt/proto/topodata"
  "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base"
  "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/throttlerapp"
+
+ topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 )
 
 type Config struct {

go/test/endtoend/vreplication/config_test.go

@@ -144,6 +144,22 @@ create table nopk (name varchar(128), age int unsigned);
  "sequence": "customer_seq"
  }
  },
+ "customer_name": {
+ "column_vindexes": [
+ {
+ "column": "cid",
+ "name": "xxhash"
+ }
+ ]
+ },
+ "customer_type": {
+ "column_vindexes": [
+ {
+ "column": "cid",
+ "name": "xxhash"
+ }
+ ]
+ },
  "customer2": {
  "column_vindexes": [
  {
@@ -401,6 +417,32 @@ create table nopk (name varchar(128), age int unsigned);
  "create_ddl": "create table cproduct(pid bigint, description varchar(128), date1 datetime not null default '0000-00-00 00:00:00', date2 datetime not null default '2021-00-01 00:00:00', primary key(pid)) CHARSET=utf8mb4"
  }]
 }
+`
+
+ materializeCustomerNameSpec = `
+{
+ "workflow": "customer_name",
+ "source_keyspace": "customer",
+ "target_keyspace": "customer",
+ "table_settings": [{
+ "target_table": "customer_name",
+ "source_expression": "select cid, name from customer",
+ "create_ddl": "create table if not exists customer_name (cid bigint not null, name varchar(128), primary key(cid), key(name))"
+ }]
+}
+`
+
+ materializeCustomerTypeSpec = `
+{
+ "workflow": "customer_type",
+ "source_keyspace": "customer",
+ "target_keyspace": "customer",
+ "table_settings": [{
+ "target_table": "customer_type",
+ "source_expression": "select cid, typ from customer",
+ "create_ddl": "create table if not exists customer_type (cid bigint not null, typ enum('individual','soho','enterprise'), primary key(cid), key(typ))"
+ }]
+}
 `
 
  merchantOrdersVSchema = `

go/test/endtoend/vreplication/multi_tenant_test.go

@@ -52,9 +52,8 @@ const (
  tenantMigrationStatusMigrating
  tenantMigrationStatusMigrated
 
- sourceKeyspaceTemplate = "s%d"
- sourceAliasKeyspaceTemplate = "a%d"
- targetKeyspaceName = "mt"
+ sourceKeyspaceTemplate = "s%d"
+ targetKeyspaceName = "mt"
 
  numTenants = 10
  numInitialRowsPerTenant = 10
@@ -130,7 +129,6 @@ func TestMultiTenantSimple(t *testing.T) {
 
  tenantId := int64(1)
  sourceKeyspace := getSourceKeyspace(tenantId)
- sourceAliasKeyspace := getSourceAliasKeyspace(tenantId)
  _, err = vc.AddKeyspace(t, []*Cell{vc.Cells["zone1"]}, sourceKeyspace, "0", stVSchema, stSchema, 1, 0, getInitialTabletIdForTenant(tenantId), nil)
  require.NoError(t, err)
 
@@ -163,19 +161,16 @@ func TestMultiTenantSimple(t *testing.T) {
  sourceKeyspace: sourceKeyspace,
  createFlags: []string{
  "--tenant-id", strconv.FormatInt(tenantId, 10),
- "--source-keyspace-alias", sourceAliasKeyspace,
  },
  })
 
  preSwitchRules := &vschemapb.KeyspaceRoutingRules{
  Rules: []*vschemapb.KeyspaceRoutingRule{
- {FromKeyspace: "a1", ToKeyspace: "s1"},
  {FromKeyspace: "s1", ToKeyspace: "s1"},
  },
  }
  postSwitchRules := &vschemapb.KeyspaceRoutingRules{
  Rules: []*vschemapb.KeyspaceRoutingRule{
- {FromKeyspace: "a1", ToKeyspace: "mt"},
  {FromKeyspace: "s1", ToKeyspace: "mt"},
  },
  }
@@ -187,25 +182,21 @@ func TestMultiTenantSimple(t *testing.T) {
  mt.Create()
  validateKeyspaceRoutingRules(t, vc, primaries, rulesMap, false)
  // Note: we cannot insert into the target keyspace since that is never routed to the source keyspace.
- for _, ks := range []string{sourceKeyspace, sourceAliasKeyspace} {
- lastIndex = insertRows(lastIndex, ks)
- }
+ lastIndex = insertRows(lastIndex, sourceKeyspace)
  waitForWorkflowState(t, vc, fmt.Sprintf("%s.%s", targetKeyspace, mt.workflowName), binlogdatapb.VReplicationWorkflowState_Running.String())
  mt.SwitchReadsAndWrites()
  validateKeyspaceRoutingRules(t, vc, primaries, rulesMap, true)
- // Note: here we have already switched and we can insert into the target keyspace and it should get reverse
- // replicated to the source keyspace. The source and alias are also routed to the target keyspace at this point.
- for _, ks := range []string{sourceKeyspace, sourceAliasKeyspace, targetKeyspace} {
- lastIndex = insertRows(lastIndex, ks)
- }
+ // Note: here we have already switched, and we can insert into the target keyspace, and it should get reverse
+ // replicated to the source keyspace. The source keyspace is routed to the target keyspace at this point.
+ lastIndex = insertRows(lastIndex, sourceKeyspace)
  mt.Complete()
  require.Zero(t, len(getKeyspaceRoutingRules(t, vc).Rules))
  actualRowsInserted := getRowCount(t, vtgateConn, fmt.Sprintf("%s.%s", targetKeyspace, "t1"))
  log.Infof("Migration completed, total rows in target: %d", actualRowsInserted)
  require.Equal(t, lastIndex, int64(actualRowsInserted))
 }
 
-// If switched queries with source/alias qualifiers should execute on target, else on source. Confirm that
+// If switched, queries with source qualifiers should execute on target, else on source. Confirm that
 // the routing rules are as expected and that the query executes on the expected tablet.
 func validateKeyspaceRoutingRules(t *testing.T, vc *VitessCluster, primaries map[string]*cluster.VttabletProcess, rulesMap map[string]*vschemapb.KeyspaceRoutingRules, switched bool) {
  currentRules := getKeyspaceRoutingRules(t, vc)
@@ -224,25 +215,19 @@ func validateKeyspaceRoutingRules(t *testing.T, vc *VitessCluster, primaries map
  require.ElementsMatch(t, rulesMap["post"].Rules, currentRules.Rules)
  validateQueryRoute("mt", "target")
  validateQueryRoute("s1", "target")
- validateQueryRoute("a1", "target")
  } else {
  require.ElementsMatch(t, rulesMap["pre"].Rules, currentRules.Rules)
  // Note that with multi-tenant migration, we cannot redirect the target keyspace since
  // there are multiple source keyspaces and the target has the aggregate of all the tenants.
  validateQueryRoute("mt", "target")
  validateQueryRoute("s1", "source")
- validateQueryRoute("a1", "source")
  }
 }
 
 func getSourceKeyspace(tenantId int64) string {
  return fmt.Sprintf(sourceKeyspaceTemplate, tenantId)
 }
 
-func getSourceAliasKeyspace(tenantId int64) string {
- return fmt.Sprintf(sourceAliasKeyspaceTemplate, tenantId)
-}
-
 func (mtm *multiTenantMigration) insertSomeData(t *testing.T, tenantId int64, keyspace string, numRows int64) {
  vtgateConn, closeConn := getVTGateConn()
  defer closeConn()
@@ -362,7 +347,7 @@ func (mtm *multiTenantMigration) getLastID(tenantId int64) int64 {
  return mtm.lastIDs[tenantId]
 }
 
-func (mtm *multiTenantMigration) initTenantData(t *testing.T, tenantId int64, sourceAliasKeyspace string) {
+func (mtm *multiTenantMigration) initTenantData(t *testing.T, tenantId int64) {
  mtm.insertSomeData(t, tenantId, getSourceKeyspace(tenantId), numInitialRowsPerTenant)
 }
 
@@ -374,16 +359,14 @@ func (mtm *multiTenantMigration) setup(tenantId int64) {
  log.Infof("Creating MoveTables for tenant %d", tenantId)
  mtm.setLastID(tenantId, 0)
  sourceKeyspace := getSourceKeyspace(tenantId)
- sourceAliasKeyspace := getSourceAliasKeyspace(tenantId)
  _, err := vc.AddKeyspace(mtm.t, []*Cell{vc.Cells["zone1"]}, sourceKeyspace, "0", stVSchema, stSchema,
  1, 0, getInitialTabletIdForTenant(tenantId), nil)
  require.NoError(mtm.t, err)
- mtm.initTenantData(mtm.t, tenantId, sourceAliasKeyspace)
+ mtm.initTenantData(mtm.t, tenantId)
 }
 
 func (mtm *multiTenantMigration) start(tenantId int64) {
  sourceKeyspace := getSourceKeyspace(tenantId)
- sourceAliasKeyspace := getSourceAliasKeyspace(tenantId)
  mtm.setTenantMigrationStatus(tenantId, tenantMigrationStatusMigrating)
  mt := newVtctldMoveTables(&moveTablesWorkflow{
  workflowInfo: &workflowInfo{
@@ -395,7 +378,6 @@ func (mtm *multiTenantMigration) start(tenantId int64) {
  tables: mtm.tables,
  createFlags: []string{
  "--tenant-id", strconv.FormatInt(tenantId, 10),
- "--source-keyspace-alias", sourceAliasKeyspace,
  },
  })
  mtm.setActiveMoveTables(tenantId, mt)
@@ -404,13 +386,11 @@ func (mtm *multiTenantMigration) start(tenantId int64) {
 
 func (mtm *multiTenantMigration) switchTraffic(tenantId int64) {
  t := mtm.t
- sourceAliasKeyspace := getSourceAliasKeyspace(tenantId)
  sourceKeyspaceName := getSourceKeyspace(tenantId)
  mt := mtm.getActiveMoveTables(tenantId)
  ksWorkflow := fmt.Sprintf("%s.%s", mtm.targetKeyspace, mt.workflowName)
  waitForWorkflowState(t, vc, ksWorkflow, binlogdatapb.VReplicationWorkflowState_Running.String())
- // we intentionally insert first into the source alias keyspace and then the source keyspace to test routing rules for both.
- mtm.insertSomeData(t, tenantId, sourceAliasKeyspace, numAdditionalRowsPerTenant)
+ mtm.insertSomeData(t, tenantId, sourceKeyspaceName, numAdditionalRowsPerTenant)
  mt.SwitchReadsAndWrites()
  mtm.insertSomeData(t, tenantId, sourceKeyspaceName, numAdditionalRowsPerTenant)
 }