Feature: Multi Target Update Support (#15402)

Signed-off-by: Harshit Gangal <harshit@planetscale.com>

changelog/20.0/20.0.0/summary.md

@@ -9,7 +9,9 @@
     - [Vindex Hints](#vindex-hints)
     - [Update with Limit Support](#update-limit)
     - [Update with Multi Table Support](#multi-table-update)
+    - [Update with Multi Target Support](#update-multi-target)
     - [Delete with Subquery Support](#delete-subquery)
+    - [Delete with Multi Target Support](#delete-multi-target)
   - **[Flag changes](#flag-changes)**
     - [`pprof-http` default change](#pprof-http-default)
     - [New `healthcheck-dial-concurrency` flag](#healthcheck-dial-concurrency-flag)
@@ -59,12 +61,27 @@ Example: `update t1 join t2 on t1.id = t2.id join t3 on t1.col = t3.col set t1.b
 
 More details about how it works is available in [MySQL Docs](https://dev.mysql.com/doc/refman/8.0/en/update.html)
 
+#### <a id="update-multi-target"/> Update with Multi Target Support
+
+Support is added for sharded multi table target update.
+
+Example: `update t1 join t2 on t1.id = t2.id set t1.foo = 'abc', t2.bar = 23`
+
+More details about how it works is available in [MySQL Docs](https://dev.mysql.com/doc/refman/8.0/en/update.html)
+
 #### <a id="delete-subquery"/> Delete with Subquery Support
 
 Support is added for sharded table delete with subquery
 
 Example: `delete from t1 where id in (select col from t2 where foo = 32 and bar = 43)`
 
+#### <a id="delete-multi-target"/> Delete with Multi Target Support
+
+Support is added for sharded multi table target delete.
+
+Example: `delete t1, t3 from t1 join t2 on t1.id = t2.id join t3 on t1.col = t3.col`
+
+More details about how it works is available in [MySQL Docs](https://dev.mysql.com/doc/refman/8.0/en/delete.html)
 
 ### <a id="flag-changes"/>Flag Changes
 

go/test/endtoend/vtgate/queries/dml/dml_test.go

@@ -54,12 +54,6 @@ func TestMultiTableDelete(t *testing.T) {
 	mcmp.Exec("insert into order_tbl(region_id, oid, cust_no) values (1,1,4), (1,2,2), (2,3,5), (2,4,55)")
 	mcmp.Exec("insert into oevent_tbl(oid, ename) values (1,'a'), (2,'b'), (3,'a'), (4,'c')")
 
-	// check rows
-	mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`,
-		`[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(55)]]`)
-	mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`,
-		`[[INT64(1) VARCHAR("a")] [INT64(2) VARCHAR("b")] [INT64(3) VARCHAR("a")] [INT64(4) VARCHAR("c")]]`)
-
 	// multi table delete
 	qr := mcmp.Exec(`delete o from order_tbl o join oevent_tbl ev where o.oid = ev.oid and ev.ename = 'a'`)
 	assert.EqualValues(t, 2, qr.RowsAffected)
@@ -91,12 +85,6 @@ func TestDeleteWithLimit(t *testing.T) {
 	mcmp.Exec("insert into s_tbl(id, num) values (1,10), (2,10), (3,10), (4,20), (5,5), (6,15), (7,17), (8,80)")
 	mcmp.Exec("insert into order_tbl(region_id, oid, cust_no) values (1,1,4), (1,2,2), (2,3,5), (2,4,55)")
 
-	// check rows
-	mcmp.AssertMatches(`select id, num from s_tbl order by id`,
-		`[[INT64(1) INT64(10)] [INT64(2) INT64(10)] [INT64(3) INT64(10)] [INT64(4) INT64(20)] [INT64(5) INT64(5)] [INT64(6) INT64(15)] [INT64(7) INT64(17)] [INT64(8) INT64(80)]]`)
-	mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`,
-		`[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(55)]]`)
-
 	// delete with limit
 	qr := mcmp.Exec(`delete from s_tbl order by num, id limit 3`)
 	require.EqualValues(t, 3, qr.RowsAffected)
@@ -152,12 +140,6 @@ func TestUpdateWithLimit(t *testing.T) {
 	mcmp.Exec("insert into s_tbl(id, num) values (1,10), (2,10), (3,10), (4,20), (5,5), (6,15), (7,17), (8,80)")
 	mcmp.Exec("insert into order_tbl(region_id, oid, cust_no) values (1,1,4), (1,2,2), (2,3,5), (2,4,55)")
 
-	// check rows
-	mcmp.AssertMatches(`select id, num from s_tbl order by id`,
-		`[[INT64(1) INT64(10)] [INT64(2) INT64(10)] [INT64(3) INT64(10)] [INT64(4) INT64(20)] [INT64(5) INT64(5)] [INT64(6) INT64(15)] [INT64(7) INT64(17)] [INT64(8) INT64(80)]]`)
-	mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`,
-		`[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(55)]]`)
-
 	// update with limit
 	qr := mcmp.Exec(`update s_tbl set num = 12 order by num, id limit 3`)
 	require.EqualValues(t, 3, qr.RowsAffected)
@@ -216,13 +198,7 @@ func TestMultiTableUpdate(t *testing.T) {
 	mcmp.Exec("insert into order_tbl(region_id, oid, cust_no) values (1,1,4), (1,2,2), (2,3,5), (2,4,55)")
 	mcmp.Exec("insert into oevent_tbl(oid, ename) values (1,'a'), (2,'b'), (3,'a'), (4,'c')")
 
-	// check rows
-	mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`,
-		`[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(55)]]`)
-	mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`,
-		`[[INT64(1) VARCHAR("a")] [INT64(2) VARCHAR("b")] [INT64(3) VARCHAR("a")] [INT64(4) VARCHAR("c")]]`)
-
-	// multi table delete
+	// multi table update
 	qr := mcmp.Exec(`update order_tbl o join oevent_tbl ev on o.oid = ev.oid set ev.ename = 'a' where ev.oid > 3`)
 	assert.EqualValues(t, 1, qr.RowsAffected)
 
@@ -253,12 +229,6 @@ func TestDeleteWithSubquery(t *testing.T) {
 	mcmp.Exec("insert into s_tbl(id, num) values (1,10), (2,10), (3,10), (4,20), (5,5), (6,15), (7,17), (8,80)")
 	mcmp.Exec("insert into order_tbl(region_id, oid, cust_no) values (1,1,4), (1,2,2), (2,3,5), (2,4,55)")
 
-	// check rows
-	mcmp.AssertMatches(`select id, num from s_tbl order by id`,
-		`[[INT64(1) INT64(10)] [INT64(2) INT64(10)] [INT64(3) INT64(10)] [INT64(4) INT64(20)] [INT64(5) INT64(5)] [INT64(6) INT64(15)] [INT64(7) INT64(17)] [INT64(8) INT64(80)]]`)
-	mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`,
-		`[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(55)]]`)
-
 	// delete with subquery on s_tbl
 	qr := mcmp.Exec(`delete from s_tbl where id in (select oid from order_tbl)`)
 	require.EqualValues(t, 4, qr.RowsAffected)
@@ -305,12 +275,6 @@ func TestMultiTargetDelete(t *testing.T) {
 	mcmp.Exec("insert into order_tbl(region_id, oid, cust_no) values (1,1,4), (1,2,2), (2,3,5), (2,4,55)")
 	mcmp.Exec("insert into oevent_tbl(oid, ename) values (1,'a'), (2,'b'), (3,'a'), (2,'c')")
 
-	// check rows
-	mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`,
-		`[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(55)]]`)
-	mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`,
-		`[[INT64(1) VARCHAR("a")] [INT64(2) VARCHAR("b")] [INT64(2) VARCHAR("c")] [INT64(3) VARCHAR("a")]]`)
-
 	// multi table delete
 	qr := mcmp.Exec(`delete o, ev from order_tbl o join oevent_tbl ev where o.oid = ev.oid and ev.ename = 'a'`)
 	assert.EqualValues(t, 4, qr.RowsAffected)
@@ -368,3 +332,34 @@ func TestMultiTargetDeleteMore(t *testing.T) {
 	mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`,
 		`[[INT64(1) VARCHAR("a")] [INT64(2) VARCHAR("b")] [INT64(2) VARCHAR("c")] [INT64(3) VARCHAR("a")]]`)
 }
+
+// TestMultiTargetUpdate executed multi-target update queries
+func TestMultiTargetUpdate(t *testing.T) {
+	utils.SkipIfBinaryIsBelowVersion(t, 20, "vtgate")
+
+	mcmp, closer := start(t)
+	defer closer()
+
+	// initial rows
+	mcmp.Exec("insert into order_tbl(region_id, oid, cust_no) values (1,1,4), (1,2,2), (2,3,5), (2,4,55)")
+	mcmp.Exec("insert into oevent_tbl(oid, ename) values (1,'a'), (2,'b'), (3,'a'), (4,'c')")
+
+	// multi target update
+	qr := mcmp.Exec(`update order_tbl o join oevent_tbl ev on o.oid = ev.oid set ev.ename = 'a', o.cust_no = 1 where ev.oid > 3`)
+	assert.EqualValues(t, 2, qr.RowsAffected)
+
+	// check rows
+	mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`,
+		`[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(1)]]`)
+	mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`,
+		`[[INT64(1) VARCHAR("a")] [INT64(2) VARCHAR("b")] [INT64(3) VARCHAR("a")] [INT64(4) VARCHAR("a")]]`)
+
+	qr = mcmp.Exec(`update order_tbl o, oevent_tbl ev set ev.ename = 'xyz', o.oid = 40 where o.cust_no = ev.oid and ev.ename = 'b'`)
+	assert.EqualValues(t, 2, qr.RowsAffected)
+
+	// check rows
+	mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid, region_id`,
+		`[[INT64(1) INT64(1) INT64(4)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(1)] [INT64(1) INT64(40) INT64(2)]]`)
+	mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`,
+		`[[INT64(1) VARCHAR("a")] [INT64(2) VARCHAR("xyz")] [INT64(3) VARCHAR("a")] [INT64(4) VARCHAR("a")]]`)
+}

go/test/endtoend/vtgate/queries/dml/sharded_schema.sql

@@ -25,7 +25,8 @@ create table order_tbl
     oid       bigint,
     region_id bigint,
     cust_no   bigint unique key,
-    primary key (oid, region_id)
+    primary key (oid, region_id),
+    unique key (oid)
 ) Engine = InnoDB;
 
 create table oid_vdx_tbl

go/vt/sqlparser/ast_funcs.go

@@ -2686,3 +2686,11 @@ func (node *Update) SetWherePredicate(expr Expr) {
 		Expr: expr,
 	}
 }
+
+// GetHighestOrderLock returns the higher level lock between the current lock and the new lock
+func (lock Lock) GetHighestOrderLock(newLock Lock) Lock {
+	if newLock > lock {
+		return newLock
+	}
+	return lock
+}

go/vt/sqlparser/constants.go

@@ -522,11 +522,11 @@ const (
 // Constants for Enum Type - Lock
 const (
 	NoLock Lock = iota
-	ForUpdateLock
 	ShareModeLock
 	ForShareLock
 	ForShareLockNoWait
 	ForShareLockSkipLocked
+	ForUpdateLock
 	ForUpdateLockNoWait
 	ForUpdateLockSkipLocked
 )

go/vt/vtgate/planbuilder/operator_transformers.go

@@ -26,6 +26,7 @@ import (
 	"vitess.io/vitess/go/slice"
 	"vitess.io/vitess/go/sqltypes"
 	"vitess.io/vitess/go/vt/sqlparser"
+	"vitess.io/vitess/go/vt/sysvars"
 	"vitess.io/vitess/go/vt/vtenv"
 	"vitess.io/vitess/go/vt/vterrors"
 	"vitess.io/vitess/go/vt/vtgate/engine"
@@ -701,6 +702,9 @@ func buildUpdateLogicalPlan(
 			return nil, vterrors.VT12001("Vindex update should have ORDER BY clause when using LIMIT")
 		}
 	}
+	if upd.VerifyAll {
+		stmt.SetComments(stmt.GetParsedComments().SetMySQLSetVarValue(sysvars.ForeignKeyChecks, "OFF"))
+	}
 
 	edml := createDMLPrimitive(ctx, rb, hints, upd.Target.VTable, generateQuery(stmt), vindexes, vQuery)
 

go/vt/vtgate/planbuilder/operators/delete.go

@@ -17,8 +17,6 @@ limitations under the License.
 package operators
 
 import (
-	"sort"
-
 	"vitess.io/vitess/go/slice"
 	"vitess.io/vitess/go/vt/sqlparser"
 	"vitess.io/vitess/go/vt/vterrors"
@@ -35,13 +33,6 @@ type Delete struct {
 	noPredicates
 }
 
-// delOp stores intermediary value for Delete Operator with the vindexes.Table for ordering.
-type delOp struct {
-	op   Operator
-	vTbl *vindexes.Table
-	cols []*sqlparser.ColName
-}
-
 // Clone implements the Operator interface
 func (d *Delete) Clone(inputs []Operator) Operator {
 	newD := *d
@@ -73,7 +64,7 @@ func (d *Delete) ShortDescription() string {
 }
 
 func createOperatorFromDelete(ctx *plancontext.PlanningContext, deleteStmt *sqlparser.Delete) (op Operator) {
-	childFks := ctx.SemTable.GetChildForeignKeysForTable(deleteStmt.Targets[0])
+	childFks := ctx.SemTable.GetChildForeignKeysForTargets()
 
 	// We check if delete with input plan is required. DML with input planning is generally
 	// slower, because it does a selection and then creates a delete statement wherein we have to
@@ -136,34 +127,17 @@ func createDeleteWithInputOp(ctx *plancontext.PlanningContext, del *sqlparser.De
 		Lock:    sqlparser.ForUpdateLock,
 	}
 
-	var delOps []delOp
-	for _, target := range del.Targets {
-		op := createDeleteOpWithTarget(ctx, target)
+	var delOps []dmlOp
+	for _, target := range ctx.SemTable.Targets.Constituents() {
+		op := createDeleteOpWithTarget(ctx, target, del.Ignore)
 		delOps = append(delOps, op)
 	}
 
-	// sort the operator based on sharding vindex type.
-	// Unsharded < Lookup Vindex < Any
-	// This is needed to ensure all the rows are deleted from unowned sharding tables first.
-	// Otherwise, those table rows will be missed from getting deleted as
-	// the owned table row won't have matching values.
-	sort.Slice(delOps, func(i, j int) bool {
-		a, b := delOps[i], delOps[j]
-		// Get the first Vindex of a and b, if available
-		aVdx, bVdx := getFirstVindex(a.vTbl), getFirstVindex(b.vTbl)
-
-		// Sort nil Vindexes to the start
-		if aVdx == nil || bVdx == nil {
-			return aVdx != nil // true if bVdx is nil and aVdx is not nil
-		}
-
-		// Among non-nil Vindexes, those that need VCursor come first
-		return aVdx.NeedsVCursor() && !bVdx.NeedsVCursor()
-	})
+	delOps = sortDmlOps(delOps)
 
 	// now map the operator and column list.
 	var colsList [][]*sqlparser.ColName
-	dmls := slice.Map(delOps, func(from delOp) Operator {
+	dmls := slice.Map(delOps, func(from dmlOp) Operator {
 		colsList = append(colsList, from.cols)
 		for _, col := range from.cols {
 			selectStmt.SelectExprs = append(selectStmt.SelectExprs, aeWrap(col))
@@ -194,9 +168,8 @@ func getFirstVindex(vTbl *vindexes.Table) vindexes.Vindex {
 	return nil
 }
 
-func createDeleteOpWithTarget(ctx *plancontext.PlanningContext, target sqlparser.TableName) delOp {
-	ts := ctx.SemTable.Targets[target.Name]
-	ti, err := ctx.SemTable.TableInfoFor(ts)
+func createDeleteOpWithTarget(ctx *plancontext.PlanningContext, target semantics.TableSet, ignore sqlparser.Ignore) dmlOp {
+	ti, err := ctx.SemTable.TableInfoFor(target)
 	if err != nil {
 		panic(vterrors.VT13001(err.Error()))
 	}
@@ -205,14 +178,18 @@ func createDeleteOpWithTarget(ctx *plancontext.PlanningContext, target sqlparser
 	if len(vTbl.PrimaryKey) == 0 {
 		panic(vterrors.VT09015())
 	}
+	tblName, err := ti.Name()
+	if err != nil {
+		panic(err)
+	}
 
 	var leftComp sqlparser.ValTuple
 	cols := make([]*sqlparser.ColName, 0, len(vTbl.PrimaryKey))
 	for _, col := range vTbl.PrimaryKey {
-		colName := sqlparser.NewColNameWithQualifier(col.String(), target)
+		colName := sqlparser.NewColNameWithQualifier(col.String(), tblName)
 		cols = append(cols, colName)
 		leftComp = append(leftComp, colName)
-		ctx.SemTable.Recursive[colName] = ts
+		ctx.SemTable.Recursive[colName] = target
 	}
 	// optimize for case when there is only single column on left hand side.
 	var lhs sqlparser.Expr = leftComp
@@ -222,11 +199,12 @@ func createDeleteOpWithTarget(ctx *plancontext.PlanningContext, target sqlparser
 	compExpr := sqlparser.NewComparisonExpr(sqlparser.InOp, lhs, sqlparser.ListArg(engine.DmlVals), nil)
 
 	del := &sqlparser.Delete{
+		Ignore:     ignore,
 		TableExprs: sqlparser.TableExprs{ti.GetAliasedTableExpr()},
-		Targets:    sqlparser.TableNames{target},
+		Targets:    sqlparser.TableNames{tblName},
 		Where:      sqlparser.NewWhere(sqlparser.WhereClause, compExpr),
 	}
-	return delOp{
+	return dmlOp{
 		createOperatorFromDelete(ctx, del),
 		vTbl,
 		cols,
@@ -241,10 +219,9 @@ func createDeleteOperator(ctx *plancontext.PlanningContext, del *sqlparser.Delet
 		op = addWherePredsToSubQueryBuilder(ctx, del.Where.Expr, op, sqc)
 	}
 
-	target := del.Targets[0]
-	tblID, exists := ctx.SemTable.Targets[target.Name]
-	if !exists {
-		panic(vterrors.VT13001("delete target table should be part of semantic analyzer"))
+	tblID, err := ctx.SemTable.GetTargetTableSetForTableName(del.Targets[0])
+	if err != nil {
+		panic(err)
 	}
 	tblInfo, err := ctx.SemTable.TableInfoFor(tblID)
 	if err != nil {