Avoid rewriting unsharded queries and split semantic analysis in two (#…

…15217)

Signed-off-by: Andres Taylor <andres@planetscale.com>
Co-authored-by: Andres Taylor <andres@planetscale.com>

go/vt/sqlparser/impossible_query.go

@@ -27,6 +27,9 @@ package sqlparser
 func FormatImpossibleQuery(buf *TrackedBuffer, node SQLNode) {
 	switch node := node.(type) {
 	case *Select:
+		if node.With != nil {
+			node.With.Format(buf)
+		}
 		buf.Myprintf("select %v from ", node.SelectExprs)
 		var prefix string
 		for _, n := range node.From {

go/vt/vtgate/planbuilder/operators/aggregator.go

@@ -80,10 +80,7 @@ func (a *Aggregator) SetInputs(operators []Operator) {
 }
 
 func (a *Aggregator) AddPredicate(_ *plancontext.PlanningContext, expr sqlparser.Expr) Operator {
-	return &Filter{
-		Source:     a,
-		Predicates: []sqlparser.Expr{expr},
-	}
+	return newFilter(a, expr)
 }
 
 func (a *Aggregator) addColumnWithoutPushing(_ *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, addToGroupBy bool) int {

go/vt/vtgate/planbuilder/operators/apply_join.go

@@ -109,7 +109,7 @@ func (aj *ApplyJoin) Clone(inputs []Operator) Operator {
 }
 
 func (aj *ApplyJoin) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Expr) Operator {
-	return AddPredicate(ctx, aj, expr, false, newFilter)
+	return AddPredicate(ctx, aj, expr, false, newFilterSinglePredicate)
 }
 
 // Inputs implements the Operator interface

go/vt/vtgate/planbuilder/operators/filter.go

@@ -39,9 +39,13 @@ type Filter struct {
 	Truncate int
 }
 
-func newFilter(op Operator, expr sqlparser.Expr) Operator {
+func newFilterSinglePredicate(op Operator, expr sqlparser.Expr) Operator {
+	return newFilter(op, expr)
+}
+
+func newFilter(op Operator, expr ...sqlparser.Expr) Operator {
 	return &Filter{
-		Source: op, Predicates: []sqlparser.Expr{expr},
+		Source: op, Predicates: expr,
 	}
 }
 

go/vt/vtgate/planbuilder/operators/hash_join.go

@@ -106,7 +106,7 @@ func (hj *HashJoin) SetInputs(operators []Operator) {
 }
 
 func (hj *HashJoin) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Expr) Operator {
-	return AddPredicate(ctx, hj, expr, false, newFilter)
+	return AddPredicate(ctx, hj, expr, false, newFilterSinglePredicate)
 }
 
 func (hj *HashJoin) AddColumn(ctx *plancontext.PlanningContext, reuseExisting bool, addToGroupBy bool, expr *sqlparser.AliasedExpr) int {

go/vt/vtgate/planbuilder/operators/horizon.go

@@ -94,17 +94,14 @@ func (h *Horizon) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.
 	tableInfo, err := ctx.SemTable.TableInfoForExpr(expr)
 	if err != nil {
 		if errors.Is(err, semantics.ErrNotSingleTable) {
-			return &Filter{
-				Source:     h,
-				Predicates: []sqlparser.Expr{expr},
-			}
+			return newFilter(h, expr)
 		}
 		panic(err)
 	}
 
 	newExpr := semantics.RewriteDerivedTableExpression(expr, tableInfo)
 	if sqlparser.ContainsAggregation(newExpr) {
-		return &Filter{Source: h, Predicates: []sqlparser.Expr{expr}}
+		return newFilter(h, expr)
 	}
 	h.Source = h.Source.AddPredicate(ctx, newExpr)
 	return h

go/vt/vtgate/planbuilder/operators/join.go

@@ -128,7 +128,7 @@ func createInnerJoin(ctx *plancontext.PlanningContext, tableExpr *sqlparser.Join
 }
 
 func (j *Join) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Expr) Operator {
-	return AddPredicate(ctx, j, expr, false, newFilter)
+	return AddPredicate(ctx, j, expr, false, newFilterSinglePredicate)
 }
 
 var _ JoinOp = (*Join)(nil)

go/vt/vtgate/planbuilder/operators/route_planning.go

@@ -533,7 +533,7 @@ func pushJoinPredicates(ctx *plancontext.PlanningContext, exprs []sqlparser.Expr
 	}
 
 	for _, expr := range exprs {
-		AddPredicate(ctx, op, expr, true, newFilter)
+		AddPredicate(ctx, op, expr, true, newFilterSinglePredicate)
 	}
 
 	return op

go/vt/vtgate/planbuilder/operators/subquery.go

@@ -269,10 +269,7 @@ func (sq *SubQuery) settleFilter(ctx *plancontext.PlanningContext, outer Operato
 		predicates = append(predicates, rhsPred)
 		sq.SubqueryValueName = sq.ArgName
 	}
-	return &Filter{
-		Source:     outer,
-		Predicates: predicates,
-	}
+	return newFilter(outer, predicates...)
 }
 
 func dontEnterSubqueries(node, _ sqlparser.SQLNode) bool {

go/vt/vtgate/planbuilder/operators/subquery_planning.go

@@ -458,7 +458,7 @@ func tryMergeSubqueriesRecursively(
 		finalResult = finalResult.Merge(res)
 	}
 
-	op.Source = &Filter{Source: outer.Source, Predicates: []sqlparser.Expr{subQuery.Original}}
+	op.Source = newFilter(outer.Source, subQuery.Original)
 	return op, finalResult.Merge(Rewrote("merge outer of two subqueries"))
 }
 
@@ -477,7 +477,7 @@ func tryMergeSubqueryWithOuter(ctx *plancontext.PlanningContext, subQuery *SubQu
 		return outer, NoRewrite
 	}
 	if !subQuery.IsProjection {
-		op.Source = &Filter{Source: outer.Source, Predicates: []sqlparser.Expr{subQuery.Original}}
+		op.Source = newFilter(outer.Source, subQuery.Original)
 	}
 	ctx.MergedSubqueries = append(ctx.MergedSubqueries, subQuery.originalSubquery)
 	return op, Rewrote("merged subquery with outer")
@@ -582,10 +582,7 @@ func (s *subqueryRouteMerger) merge(ctx *plancontext.PlanningContext, inner, out
 	if isSharded {
 		src = s.outer.Source
 		if !s.subq.IsProjection {
-			src = &Filter{
-				Source:     s.outer.Source,
-				Predicates: []sqlparser.Expr{s.original},
-			}
+			src = newFilter(s.outer.Source, s.original)
 		}
 	} else {
 		src = s.rewriteASTExpression(ctx, inner)
@@ -655,10 +652,7 @@ func (s *subqueryRouteMerger) rewriteASTExpression(ctx *plancontext.PlanningCont
 				cursor.Replace(subq)
 			}
 		}, ctx.SemTable.CopySemanticInfo).(sqlparser.Expr)
-		src = &Filter{
-			Source:     s.outer.Source,
-			Predicates: []sqlparser.Expr{sQuery},
-		}
+		src = newFilter(s.outer.Source, sQuery)
 	}
 	return src
 }

go/vt/vtgate/planbuilder/operators/union.go

@@ -105,10 +105,7 @@ func (u *Union) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Ex
 
 	needsFilter, exprPerSource := u.predicatePerSource(expr, offsets)
 	if needsFilter {
-		return &Filter{
-			Source:     u,
-			Predicates: []sqlparser.Expr{expr},
-		}
+		return newFilter(u, expr)
 	}
 
 	for i, src := range u.Sources {

go/vt/vtgate/planbuilder/testdata/cte_cases.json

@@ -1369,8 +1369,8 @@
           "Name": "main",
           "Sharded": false
         },
-        "FieldQuery": "select u.* from (select * from unsharded where 1 != 1) as u where 1 != 1",
-        "Query": "select u.* from (select * from unsharded) as u",
+        "FieldQuery": "with u as (select * from unsharded where 1 != 1) select u.* from u where 1 != 1",
+        "Query": "with u as (select * from unsharded) select u.* from u",
         "Table": "unsharded"
       },
       "TablesUsed": [
@@ -1709,8 +1709,8 @@
           "Name": "main",
           "Sharded": false
         },
-        "FieldQuery": "select col from (select col from unsharded join unsharded_b where 1 != 1) as u join unsharded_a as ua where 1 != 1",
-        "Query": "select col from (select col from unsharded join unsharded_b) as u join unsharded_a as ua limit 1",
+        "FieldQuery": "with u as (select col from unsharded join unsharded_b where 1 != 1) select col from u join unsharded_a as ua where 1 != 1",
+        "Query": "with u as (select col from unsharded join unsharded_b) select col from u join unsharded_a as ua limit 1",
         "Table": "unsharded, unsharded_a, unsharded_b"
       },
       "TablesUsed": [
@@ -1840,5 +1840,28 @@
         "user.user"
       ]
     }
+  },
+  {
+    "comment": "recursive WITH against an unsharded database",
+    "query": "WITH RECURSIVE cte (n) AS ( SELECT 1 UNION ALL SELECT n + 1 FROM cte WHERE n < 5 ) SELECT cte.n FROM unsharded join cte on unsharded.id = cte.n ",
+    "plan": {
+      "QueryType": "SELECT",
+      "Original": "WITH RECURSIVE cte (n) AS ( SELECT 1 UNION ALL SELECT n + 1 FROM cte WHERE n < 5 ) SELECT cte.n FROM unsharded join cte on unsharded.id = cte.n ",
+      "Instructions": {
+        "OperatorType": "Route",
+        "Variant": "Unsharded",
+        "Keyspace": {
+          "Name": "main",
+          "Sharded": false
+        },
+        "FieldQuery": "with recursive cte(n) as (select 1 from dual where 1 != 1 union all select n + 1 from cte where 1 != 1) select cte.n from unsharded join cte on unsharded.id = cte.n where 1 != 1",
+        "Query": "with recursive cte(n) as (select 1 from dual union all select n + 1 from cte where n < 5) select cte.n from unsharded join cte on unsharded.id = cte.n",
+        "Table": "dual, unsharded"
+      },
+      "TablesUsed": [
+        "main.dual",
+        "main.unsharded"
+      ]
+    }
   }
 ]

go/vt/vtgate/planbuilder/testdata/ddl_cases.json

@@ -260,7 +260,7 @@
           "Name": "main",
           "Sharded": false
         },
-        "Query": "create view view_a as select col1, col2 from (select col1, col2 from unsharded where id = 1 union select col1, col2 from unsharded where id = 3) as a"
+        "Query": "create view view_a as select * from (select col1, col2 from unsharded where id = 1 union select col1, col2 from unsharded where id = 3) as a"
       },
       "TablesUsed": [
         "main.view_a"

go/vt/vtgate/planbuilder/testdata/filter_cases.json

@@ -4086,7 +4086,7 @@
           "Sharded": false
         },
         "FieldQuery": "select col + 2 as a from unsharded where 1 != 1",
-        "Query": "select col + 2 as a from unsharded having col + 2 = 42",
+        "Query": "select col + 2 as a from unsharded having a = 42",
         "Table": "unsharded"
       },
       "TablesUsed": [

go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json

@@ -816,7 +816,7 @@
               "Sharded": false
             },
             "FieldQuery": "select 1 from u_tbl2 left join u_tbl1 on u_tbl1.col1 = cast(u_tbl2.col1 + 'bar' as CHAR) where 1 != 1",
-            "Query": "select 1 from u_tbl2 left join u_tbl1 on u_tbl1.col1 = cast(u_tbl2.col1 + 'bar' as CHAR) where cast(u_tbl2.col1 + 'bar' as CHAR) is not null and not (u_tbl2.col2) <=> (cast(u_tbl2.col1 + 'bar' as CHAR)) and u_tbl2.id = 1 and u_tbl1.col1 is null limit 1 for share",
+            "Query": "select 1 from u_tbl2 left join u_tbl1 on u_tbl1.col1 = cast(u_tbl2.col1 + 'bar' as CHAR) where u_tbl1.col1 is null and cast(u_tbl2.col1 + 'bar' as CHAR) is not null and not (u_tbl2.col2) <=> (cast(u_tbl2.col1 + 'bar' as CHAR)) and u_tbl2.id = 1 limit 1 for share",
             "Table": "u_tbl1, u_tbl2"
           },
           {
@@ -1523,7 +1523,7 @@
                   "Sharded": false
                 },
                 "FieldQuery": "select 1 from u_tbl8 left join u_tbl9 on u_tbl9.col9 = cast('foo' as CHAR) where 1 != 1",
-                "Query": "select 1 from u_tbl8 left join u_tbl9 on u_tbl9.col9 = cast('foo' as CHAR) where not (u_tbl8.col8) <=> (cast('foo' as CHAR)) and (u_tbl8.col8) in ::fkc_vals and u_tbl9.col9 is null limit 1 for share nowait",
+                "Query": "select 1 from u_tbl8 left join u_tbl9 on u_tbl9.col9 = cast('foo' as CHAR) where u_tbl9.col9 is null and cast('foo' as CHAR) is not null and not (u_tbl8.col8) <=> (cast('foo' as CHAR)) and (u_tbl8.col8) in ::fkc_vals limit 1 for share nowait",
                 "Table": "u_tbl8, u_tbl9"
               },
               {
@@ -1599,7 +1599,7 @@
                   "Sharded": false
                 },
                 "FieldQuery": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = cast('foo' as CHAR) where 1 != 1",
-                "Query": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = cast('foo' as CHAR) where not (u_tbl4.col4) <=> (cast('foo' as CHAR)) and (u_tbl4.col4) in ::fkc_vals and u_tbl3.col3 is null limit 1 for share",
+                "Query": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = cast('foo' as CHAR) where u_tbl3.col3 is null and cast('foo' as CHAR) is not null and not (u_tbl4.col4) <=> (cast('foo' as CHAR)) and (u_tbl4.col4) in ::fkc_vals limit 1 for share",
                 "Table": "u_tbl3, u_tbl4"
               },
               {
@@ -1611,7 +1611,7 @@
                   "Sharded": false
                 },
                 "FieldQuery": "select 1 from u_tbl4, u_tbl9 where 1 != 1",
-                "Query": "select 1 from u_tbl4, u_tbl9 where (u_tbl4.col4) in ::fkc_vals and (u_tbl9.col9) not in ((cast('foo' as CHAR))) and u_tbl4.col4 = u_tbl9.col9 limit 1 for share",
+                "Query": "select 1 from u_tbl4, u_tbl9 where u_tbl4.col4 = u_tbl9.col9 and (u_tbl4.col4) in ::fkc_vals and (cast('foo' as CHAR) is null or (u_tbl9.col9) not in ((cast('foo' as CHAR)))) limit 1 for share",
                 "Table": "u_tbl4, u_tbl9"
               },
               {
@@ -1688,7 +1688,7 @@
                   "Sharded": false
                 },
                 "FieldQuery": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = cast(:v1 as CHAR) where 1 != 1",
-                "Query": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = cast(:v1 as CHAR) where not (u_tbl4.col4) <=> (cast(:v1 as CHAR)) and (u_tbl4.col4) in ::fkc_vals and cast(:v1 as CHAR) is not null and u_tbl3.col3 is null limit 1 for share",
+                "Query": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = cast(:v1 as CHAR) where u_tbl3.col3 is null and cast(:v1 as CHAR) is not null and not (u_tbl4.col4) <=> (cast(:v1 as CHAR)) and (u_tbl4.col4) in ::fkc_vals limit 1 for share",
                 "Table": "u_tbl3, u_tbl4"
               },
               {
@@ -1700,7 +1700,7 @@
                   "Sharded": false
                 },
                 "FieldQuery": "select 1 from u_tbl4, u_tbl9 where 1 != 1",
-                "Query": "select 1 from u_tbl4, u_tbl9 where (u_tbl4.col4) in ::fkc_vals and (cast(:v1 as CHAR) is null or (u_tbl9.col9) not in ((cast(:v1 as CHAR)))) and u_tbl4.col4 = u_tbl9.col9 limit 1 for share",
+                "Query": "select 1 from u_tbl4, u_tbl9 where u_tbl4.col4 = u_tbl9.col9 and (u_tbl4.col4) in ::fkc_vals and (cast(:v1 as CHAR) is null or (u_tbl9.col9) not in ((cast(:v1 as CHAR)))) limit 1 for share",
                 "Table": "u_tbl4, u_tbl9"
               },
               {
@@ -2362,7 +2362,7 @@
                   "Sharded": false
                 },
                 "FieldQuery": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = cast(:fkc_upd as CHAR) where 1 != 1",
-                "Query": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = cast(:fkc_upd as CHAR) where not (u_tbl4.col4) <=> (cast(:fkc_upd as CHAR)) and (u_tbl4.col4) in ::fkc_vals and cast(:fkc_upd as CHAR) is not null and u_tbl3.col3 is null limit 1 for share",
+                "Query": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = cast(:fkc_upd as CHAR) where u_tbl3.col3 is null and cast(:fkc_upd as CHAR) is not null and not (u_tbl4.col4) <=> (cast(:fkc_upd as CHAR)) and (u_tbl4.col4) in ::fkc_vals limit 1 for share",
                 "Table": "u_tbl3, u_tbl4"
               },
               {
@@ -2374,7 +2374,7 @@
                   "Sharded": false
                 },
                 "FieldQuery": "select 1 from u_tbl4, u_tbl9 where 1 != 1",
-                "Query": "select 1 from u_tbl4, u_tbl9 where (u_tbl4.col4) in ::fkc_vals and (cast(:fkc_upd as CHAR) is null or (u_tbl9.col9) not in ((cast(:fkc_upd as CHAR)))) and u_tbl4.col4 = u_tbl9.col9 limit 1 for share",
+                "Query": "select 1 from u_tbl4, u_tbl9 where u_tbl4.col4 = u_tbl9.col9 and (u_tbl4.col4) in ::fkc_vals and (cast(:fkc_upd as CHAR) is null or (u_tbl9.col9) not in ((cast(:fkc_upd as CHAR)))) limit 1 for share",
                 "Table": "u_tbl4, u_tbl9"
               },
               {
@@ -2537,7 +2537,7 @@
               "Sharded": false
             },
             "FieldQuery": "select 1 from u_multicol_tbl2 left join u_multicol_tbl1 on u_multicol_tbl1.cola = 2 and u_multicol_tbl1.colb = u_multicol_tbl2.colc - 2 where 1 != 1",
-            "Query": "select 1 from u_multicol_tbl2 left join u_multicol_tbl1 on u_multicol_tbl1.cola = 2 and u_multicol_tbl1.colb = u_multicol_tbl2.colc - 2 where u_multicol_tbl2.colc - 2 is not null and not (u_multicol_tbl2.cola, u_multicol_tbl2.colb) <=> (2, u_multicol_tbl2.colc - 2) and u_multicol_tbl2.id = 7 and u_multicol_tbl1.cola is null and u_multicol_tbl1.colb is null limit 1 for share",
+            "Query": "select 1 from u_multicol_tbl2 left join u_multicol_tbl1 on u_multicol_tbl1.cola = 2 and u_multicol_tbl1.colb = u_multicol_tbl2.colc - 2 where u_multicol_tbl1.cola is null and 2 is not null and u_multicol_tbl1.colb is null and u_multicol_tbl2.colc - 2 is not null and not (u_multicol_tbl2.cola, u_multicol_tbl2.colb) <=> (2, u_multicol_tbl2.colc - 2) and u_multicol_tbl2.id = 7 limit 1 for share",
             "Table": "u_multicol_tbl1, u_multicol_tbl2"
           },
           {
@@ -3139,7 +3139,7 @@
               "Sharded": false
             },
             "FieldQuery": "select u_tbl6.col6 from u_tbl6 as u, u_tbl5 as m where 1 != 1",
-            "Query": "select u_tbl6.col6 from u_tbl6 as u, u_tbl5 as m where u.col2 = 4 and m.col3 = 6 and u.col = m.col for update",
+            "Query": "select u_tbl6.col6 from u_tbl6 as u, u_tbl5 as m where u.col = m.col and u.col2 = 4 and m.col3 = 6 for update",
             "Table": "u_tbl5, u_tbl6"
           },
           {
@@ -3197,7 +3197,7 @@
               "Sharded": false
             },
             "FieldQuery": "select u_tbl10.col from u_tbl10, u_tbl11 where 1 != 1",
-            "Query": "select u_tbl10.col from u_tbl10, u_tbl11 where u_tbl10.id = 5 and u_tbl10.id = u_tbl11.id for update",
+            "Query": "select u_tbl10.col from u_tbl10, u_tbl11 where u_tbl10.id = u_tbl11.id and u_tbl10.id = 5 for update",
             "Table": "u_tbl10, u_tbl11"
           },
           {
@@ -3479,7 +3479,7 @@
                   "Sharded": false
                 },
                 "FieldQuery": "select 1 from u_tbl4 left join u_tbl1 on u_tbl1.col14 = cast(:__sq1 as SIGNED) where 1 != 1",
-                "Query": "select /*+ SET_VAR(foreign_key_checks=OFF) */ 1 from u_tbl4 left join u_tbl1 on u_tbl1.col14 = cast(:__sq1 as SIGNED) where not (u_tbl4.col41) <=> (cast(:__sq1 as SIGNED)) and u_tbl4.col4 = 3 and cast(:__sq1 as SIGNED) is not null and u_tbl1.col14 is null limit 1 lock in share mode",
+                "Query": "select /*+ SET_VAR(foreign_key_checks=OFF) */ 1 from u_tbl4 left join u_tbl1 on u_tbl1.col14 = cast(:__sq1 as SIGNED) where u_tbl1.col14 is null and cast(:__sq1 as SIGNED) is not null and not (u_tbl4.col41) <=> (cast(:__sq1 as SIGNED)) and u_tbl4.col4 = 3 limit 1 lock in share mode",
                 "Table": "u_tbl1, u_tbl4"
               },
               {