From d325cc18ff29e23343217858b27db0fb283a3c7f Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 24 Apr 2024 14:23:43 +0200 Subject: [PATCH 01/11] Adding e2e test to repro error with reference table routing Signed-off-by: Rohit Nayak --- go/test/endtoend/vreplication/helper_test.go | 34 ++++- .../endtoend/vreplication/reference_test.go | 131 ++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 go/test/endtoend/vreplication/reference_test.go diff --git a/go/test/endtoend/vreplication/helper_test.go b/go/test/endtoend/vreplication/helper_test.go index b109cc0e996..e952670f6ab 100644 --- a/go/test/endtoend/vreplication/helper_test.go +++ b/go/test/endtoend/vreplication/helper_test.go @@ -32,6 +32,8 @@ import ( "testing" "time" + "vitess.io/vitess/go/json2" + "github.com/buger/jsonparser" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -463,7 +465,7 @@ func validateDryRunResults(t *testing.T, output string, want []string) { w = strings.TrimSpace(w[1:]) result := strings.HasPrefix(g, w) match = result - //t.Logf("Partial match |%v|%v|%v\n", w, g, match) + // t.Logf("Partial match |%v|%v|%v\n", w, g, match) } else { match = g == w } @@ -831,3 +833,33 @@ func (lg *loadGenerator) waitForCount(want int64) { } } } + +// VExplainPlan is the struct that represents the json output of a vexplain query. +type VExplainPlan struct { + OperatorType string `json:"OperatorType"` + Variant string `json:"Variant"` + Keyspace VExplainKeyspace `json:"Keyspace"` + FieldQuery string `json:"FieldQuery"` + Query string `json:"Query"` + Table string `json:"TableName"` +} + +type VExplainKeyspace struct { + Name string `json:"Name"` + Sharded bool `json:"Sharded"` +} + +// vexplain runs vexplain on the given query and returns the plan. Useful for validating routing rules. +func vexplain(t *testing.T, database, query string) *VExplainPlan { + vtgateConn := vc.GetVTGateConn(t) + defer vtgateConn.Close() + + qr := execVtgateQuery(t, vtgateConn, database, fmt.Sprintf("vexplain %s", query)) + require.NotNil(t, qr) + require.Equal(t, 1, len(qr.Rows)) + json := qr.Rows[0][0].ToString() + + var plan VExplainPlan + require.NoError(t, json2.Unmarshal([]byte(json), &plan)) + return &plan +} diff --git a/go/test/endtoend/vreplication/reference_test.go b/go/test/endtoend/vreplication/reference_test.go new file mode 100644 index 00000000000..40a8b1e39c3 --- /dev/null +++ b/go/test/endtoend/vreplication/reference_test.go @@ -0,0 +1,131 @@ +package vreplication + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + uksSchema = ` +create table product (id int, mfg_id int, cat_id int, name varchar(128), primary key(id)); +create table cat (id int, name varchar(128), primary key(id)); +create table mfg (id int, name varchar(128), primary key(id)); +` + sksSchema = ` +create table product (id int, mfg_id int, cat_id int, name varchar(128), primary key(id)); +create table cat (id int, name varchar(128), primary key(id)); +create table mfg2 (id int, name varchar(128), primary key(id)); +` + uksVSchema = ` +{ + "sharded": false, + "tables": { + "product": {}, + "cat": {}, + "mfg": {} + } +}` + + sksVSchema = ` +{ + "sharded": true, + "tables": { + "product": { + "column_vindexes": [ + { + "column": "id", + "name": "hash" + } + ] + }, + "cat": { + "type": "reference", + "source": "uks.cat" + }, + "mfg2": { + "type": "reference", + "source": "uks.mfg" + } + }, + "vindexes": { + "hash": { + "type": "hash" + } + } +}` + materializeCatSpec = ` +{ + "workflow": "wfCat", + "source_keyspace": "uks", + "target_keyspace": "sks", + "table_settings": [ {"target_table": "cat", "source_expression": "select id, name from cat" }] +}` + materializeMfgSpec = ` +{ + "workflow": "wfMfg", + "source_keyspace": "uks", + "target_keyspace": "sks", + "table_settings": [ {"target_table": "mfg2", "source_expression": "select id, name from mfg" }] +}` + initializeTables = ` +use uks; +insert into product values (1, 1, 1, 'p1'); +insert into product values (2, 2, 2, 'p2'); +insert into product values (3, 3, 3, 'p3'); +insert into cat values (1, 'c1'); +insert into cat values (2, 'c2'); +insert into cat values (3, 'c3'); +insert into mfg values (1, 'm1'); +insert into mfg values (2, 'm2'); +insert into mfg values (3, 'm3'); +` +) + +func TestReferenceTableMaterializationAndRouting(t *testing.T) { + var err error + defaultCellName := "zone1" + allCells := []string{defaultCellName} + allCellNames = defaultCellName + vc = NewVitessCluster(t, "TestReferenceTableMaterializationAndRouting", allCells, mainClusterConfig) + defer vc.TearDown(t) + defaultReplicas = 0 // because of CI resource constraints we can only run this test with primary tablets + defer func() { defaultReplicas = 1 }() + uks := "uks" + sks := "sks" + + defaultCell := vc.Cells[defaultCellName] + vc.AddKeyspace(t, []*Cell{defaultCell}, uks, "0", uksVSchema, uksSchema, defaultReplicas, defaultRdonly, 100, nil) + vc.AddKeyspace(t, []*Cell{defaultCell}, sks, "-80,80-", sksVSchema, sksSchema, defaultReplicas, defaultRdonly, 200, nil) + vtgateConn := getConnection(t, vc.ClusterConfig.hostname, vc.ClusterConfig.vtgateMySQLPort) + verifyClusterHealth(t, vc) + _, _, err = vtgateConn.ExecuteFetchMulti(initializeTables, 0, false) + require.NoError(t, err) + materialize(t, materializeCatSpec, false) + materialize(t, materializeMfgSpec, false) + + tabDash80 := vc.getPrimaryTablet(t, sks, "-80") + tab80Dash := vc.getPrimaryTablet(t, sks, "80-") + catchup(t, tabDash80, "wfCat", "Materialize Category") + catchup(t, tab80Dash, "wfCat", "Materialize Category") + catchup(t, tabDash80, "wfMfg", "Materialize Manufacturer") + catchup(t, tab80Dash, "wfMfg", "Materialize Manufacturer") + + vtgateConn.Close() + vtgateConn = getConnection(t, vc.ClusterConfig.hostname, vc.ClusterConfig.vtgateMySQLPort) + defer vtgateConn.Close() + waitForRowCount(t, vtgateConn, sks, "cat", 3) + waitForRowCount(t, vtgateConn, sks, "mfg2", 3) + + insertQuery := "insert into mfg values (4, 'm4')" + _, err = vtgateConn.ExecuteFetch(insertQuery, 0, false) + require.Contains(t, err.Error(), "table mfg not found") + + insertQuery = "insert into mfg2 values (4, 'm4')" + _, err = vtgateConn.ExecuteFetch(insertQuery, 0, false) + require.Contains(t, err.Error(), "Table 'vt_uks.mfg2' doesn't exist") + + insertQuery = "insert into uks.mfg values (4, 'm4')" + _, err = vtgateConn.ExecuteFetch(insertQuery, 0, false) + require.NoError(t, err) +} From f564ef378db894ba2ad33d708c706ff29a655438 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 25 Apr 2024 12:15:42 +0200 Subject: [PATCH 02/11] Update e2e test to test dmls addressing the reference table differently Signed-off-by: Rohit Nayak --- .../endtoend/vreplication/reference_test.go | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/go/test/endtoend/vreplication/reference_test.go b/go/test/endtoend/vreplication/reference_test.go index 40a8b1e39c3..2e560b22f0c 100644 --- a/go/test/endtoend/vreplication/reference_test.go +++ b/go/test/endtoend/vreplication/reference_test.go @@ -79,6 +79,7 @@ insert into cat values (3, 'c3'); insert into mfg values (1, 'm1'); insert into mfg values (2, 'm2'); insert into mfg values (3, 'm3'); +insert into mfg values (4, 'm4'); ` ) @@ -98,9 +99,12 @@ func TestReferenceTableMaterializationAndRouting(t *testing.T) { vc.AddKeyspace(t, []*Cell{defaultCell}, uks, "0", uksVSchema, uksSchema, defaultReplicas, defaultRdonly, 100, nil) vc.AddKeyspace(t, []*Cell{defaultCell}, sks, "-80,80-", sksVSchema, sksSchema, defaultReplicas, defaultRdonly, 200, nil) vtgateConn := getConnection(t, vc.ClusterConfig.hostname, vc.ClusterConfig.vtgateMySQLPort) + verifyClusterHealth(t, vc) _, _, err = vtgateConn.ExecuteFetchMulti(initializeTables, 0, false) require.NoError(t, err) + vtgateConn.Close() + materialize(t, materializeCatSpec, false) materialize(t, materializeMfgSpec, false) @@ -111,21 +115,38 @@ func TestReferenceTableMaterializationAndRouting(t *testing.T) { catchup(t, tabDash80, "wfMfg", "Materialize Manufacturer") catchup(t, tab80Dash, "wfMfg", "Materialize Manufacturer") - vtgateConn.Close() vtgateConn = getConnection(t, vc.ClusterConfig.hostname, vc.ClusterConfig.vtgateMySQLPort) defer vtgateConn.Close() waitForRowCount(t, vtgateConn, sks, "cat", 3) waitForRowCount(t, vtgateConn, sks, "mfg2", 3) - insertQuery := "insert into mfg values (4, 'm4')" - _, err = vtgateConn.ExecuteFetch(insertQuery, 0, false) - require.Contains(t, err.Error(), "table mfg not found") + execRefQuery(t, "insert into mfg values (5, 'm5')") + execRefQuery(t, "insert into mfg2 values (6, 'm6')") + execRefQuery(t, "insert into uks.mfg values (7, 'm7')") + execRefQuery(t, "insert into sks.mfg2 values (8, 'm8')") + waitForRowCount(t, vtgateConn, uks, "mfg", 8) + + execRefQuery(t, "update mfg set name = concat(name, '-updated') where id = 1") + execRefQuery(t, "update mfg2 set name = concat(name, '-updated') where id = 2") + execRefQuery(t, "update uks.mfg set name = concat(name, '-updated') where id = 3") + execRefQuery(t, "update sks.mfg set name = concat(name, '-updated') where id = 4") - insertQuery = "insert into mfg2 values (4, 'm4')" - _, err = vtgateConn.ExecuteFetch(insertQuery, 0, false) - require.Contains(t, err.Error(), "Table 'vt_uks.mfg2' doesn't exist") + waitForRowCount(t, vtgateConn, uks, "mfg", 8) + qr := execVtgateQuery(t, vtgateConn, "uks", "select count(*) from uks.mfg where name like '%updated%'") + require.NotNil(t, qr) + require.Equal(t, "4", qr.Rows[0][0].ToString()) - insertQuery = "insert into uks.mfg values (4, 'm4')" - _, err = vtgateConn.ExecuteFetch(insertQuery, 0, false) + execRefQuery(t, "delete from mfg where id = 5") + execRefQuery(t, "delete from mfg2 where id = 6") + execRefQuery(t, "delete from uks.mfg where id = 7") + execRefQuery(t, "delete from sks.mfg where id = 8") + waitForRowCount(t, vtgateConn, uks, "mfg", 4) + +} + +func execRefQuery(t *testing.T, query string) { + vtgateConn := getConnection(t, vc.ClusterConfig.hostname, vc.ClusterConfig.vtgateMySQLPort) + defer vtgateConn.Close() + _, err := vtgateConn.ExecuteFetch(query, 0, false) require.NoError(t, err) } From 363f90ddc950510e0aa33fd951bf7868efd3dfaf Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 25 Apr 2024 12:16:43 +0200 Subject: [PATCH 03/11] Update table rerouting to account for reference tables Signed-off-by: Rohit Nayak --- go/vt/vtgate/planbuilder/dml_planner.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/go/vt/vtgate/planbuilder/dml_planner.go b/go/vt/vtgate/planbuilder/dml_planner.go index 675df1b91cb..35d755434a1 100644 --- a/go/vt/vtgate/planbuilder/dml_planner.go +++ b/go/vt/vtgate/planbuilder/dml_planner.go @@ -20,6 +20,7 @@ import ( "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" + "vitess.io/vitess/go/vt/vtgate/vindexes" ) func rewriteRoutedTables(stmt sqlparser.Statement, vschema plancontext.VSchema) error { @@ -42,7 +43,22 @@ func rewriteRoutedTables(stmt sqlparser.Statement, vschema plancontext.VSchema) return false, vterrors.VT09014() } - if vschemaTable.Name.String() != tableName.Name.String() { + if vschemaTable.Type == vindexes.TypeReference && vschemaTable.Source != nil { + // This is a reference table. We need to rewrite the table name and keyspace to that of the source table. + vschemaTable, vindexTbl, _, _, _, err = vschema.FindTableOrVindex(vschemaTable.Source.TableName) + if err != nil { + return false, err + } + if vindexTbl != nil { + // vindex cannot be present in a dml statement. + return false, vterrors.VT09014() + } + tableName.Qualifier = vschemaTable.GetTableName().Qualifier + tableName.Name = sqlparser.NewIdentifierCS(vschemaTable.Name.String()) + newAliasTbl := sqlparser.NewAliasedTableExpr(tableName, "") + aliasTbl.Expr = newAliasTbl.Expr + aliasTbl.As = newAliasTbl.As + } else if vschemaTable.Name.String() != tableName.Name.String() { name := tableName.Name if aliasTbl.As.IsEmpty() { // if the user hasn't specified an alias, we'll insert one here so the old table name still works From 237cdfc544f8674c238a76fe38ebc90a412f80a3 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 25 Apr 2024 12:19:05 +0200 Subject: [PATCH 04/11] Add test cases for insert/update/delete rerouting for reference tables Signed-off-by: Rohit Nayak --- .../vtgate/planbuilder/testdata/onecase.json | 67 +++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/go/vt/vtgate/planbuilder/testdata/onecase.json b/go/vt/vtgate/planbuilder/testdata/onecase.json index da7543f706a..59e8391678d 100644 --- a/go/vt/vtgate/planbuilder/testdata/onecase.json +++ b/go/vt/vtgate/planbuilder/testdata/onecase.json @@ -1,9 +1,68 @@ [ { - "comment": "Add your test case here for debugging and run go test -run=One.", - "query": "", + "comment": "delete from reference table - query send to source table", + "query": "delete from user.ref_with_source where col = 1", "plan": { - + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table - query send to source table", + "query": "update user.ref_with_source set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table - query send to source table", + "query": "insert into user.ref_with_source(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into user.ref_with_source(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] } } -] \ No newline at end of file +] From 8ad019ed32ffa7d5d81f06cfc0fa453c7b0f5618 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 25 Apr 2024 12:43:45 +0200 Subject: [PATCH 05/11] Revert onecase.json. Add separate test case/file for this PR Signed-off-by: Rohit Nayak --- go/vt/vtgate/planbuilder/plan_test.go | 14 ++++ .../vtgate/planbuilder/testdata/onecase.json | 66 +----------------- .../testdata/reference_table_cases.json | 68 +++++++++++++++++++ 3 files changed, 84 insertions(+), 64 deletions(-) create mode 100644 go/vt/vtgate/planbuilder/testdata/reference_table_cases.json diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index a7450db3a05..a848d225ec4 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -732,3 +732,17 @@ func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCase } } } + +func TestReferenceRerouting(t *testing.T) { + reset := oprewriters.EnableDebugPrinting() + defer reset() + + lv := loadSchema(t, "vschemas/schema.json", true) + setFks(t, lv) + vschema := &vschemawrapper.VSchemaWrapper{ + V: lv, + TestBuilder: TestBuilder, + } + + testFile(t, "reference_table_cases.json", "", vschema, false) +} diff --git a/go/vt/vtgate/planbuilder/testdata/onecase.json b/go/vt/vtgate/planbuilder/testdata/onecase.json index 59e8391678d..a15e0c0894b 100644 --- a/go/vt/vtgate/planbuilder/testdata/onecase.json +++ b/go/vt/vtgate/planbuilder/testdata/onecase.json @@ -1,68 +1,6 @@ [ { - "comment": "delete from reference table - query send to source table", - "query": "delete from user.ref_with_source where col = 1", - "plan": { - "QueryType": "DELETE", - "Original": "delete from user.ref_with_source where col = 1", - "Instructions": { - "OperatorType": "Delete", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "update from reference table - query send to source table", - "query": "update user.ref_with_source set x = 4 where col = 1", - "plan": { - "QueryType": "UPDATE", - "Original": "update user.ref_with_source set x = 4 where col = 1", - "Instructions": { - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "insert from reference table - query send to source table", - "query": "insert into user.ref_with_source(x) values(4)", - "plan": { - "QueryType": "INSERT", - "Original": "insert into user.ref_with_source(x) values(4)", - "Instructions": { - "OperatorType": "Insert", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "insert into source_of_ref(x) values (4)", - "TableName": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } + "comment": "Add your test case here for debugging and run go test -run=One.", + "query": "" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json new file mode 100644 index 00000000000..59e8391678d --- /dev/null +++ b/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json @@ -0,0 +1,68 @@ +[ + { + "comment": "delete from reference table - query send to source table", + "query": "delete from user.ref_with_source where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table - query send to source table", + "query": "update user.ref_with_source set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table - query send to source table", + "query": "insert into user.ref_with_source(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into user.ref_with_source(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + } +] From 416f2d42c217fb15842cea76274be2c874b9474c Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 25 Apr 2024 13:59:45 +0200 Subject: [PATCH 06/11] Add cases for select with reference tables Signed-off-by: Rohit Nayak --- .../testdata/reference_table_cases.json | 256 +++++++++++++++++- 1 file changed, 254 insertions(+), 2 deletions(-) diff --git a/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json index 59e8391678d..b86b41e9ffa 100644 --- a/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json @@ -1,6 +1,6 @@ [ { - "comment": "delete from reference table - query send to source table", + "comment": "delete from reference table with another name - query send to source table", "query": "delete from user.ref_with_source where col = 1", "plan": { "QueryType": "DELETE", @@ -22,7 +22,7 @@ } }, { - "comment": "update from reference table - query send to source table", + "comment": "update from reference table with another name - query send to source table", "query": "update user.ref_with_source set x = 4 where col = 1", "plan": { "QueryType": "UPDATE", @@ -43,8 +43,206 @@ ] } }, + { + "comment": "insert from reference table with another name - query send to source table", + "query": "insert into user.ref_with_source(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into user.ref_with_source(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table - query send to source table", + "query": "delete from source_of_ref where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from source_of_ref where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table - query send to source table", + "query": "update source_of_ref set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update source_of_ref set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, { "comment": "insert from reference table - query send to source table", + "query": "insert into source_of_ref(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into source_of_ref(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table qualified with unsharded - query send to source table", + "query": "delete from main.source_of_ref where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from main.source_of_ref where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table qualified with unsharded - query send to source table", + "query": "update main.source_of_ref set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update main.source_of_ref set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table qualified with unsharded - query send to source table", + "query": "insert into main.source_of_ref(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into main.source_of_ref(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table with another name - query send to source table", + "query": "delete from user.ref_with_source where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table with another name - query send to source table", + "query": "update user.ref_with_source set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table with another name - query send to source table", "query": "insert into user.ref_with_source(x) values(4)", "plan": { "QueryType": "INSERT", @@ -64,5 +262,59 @@ "main.source_of_ref" ] } + }, + { + "comment": "select with join to reference table in sharded keyspace: should route shard-scoped", + "query": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "plan": { + "QueryType": "SELECT", + "Original": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "Instructions": { + "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", + "OperatorType": "Route", + "Variant": "EqualUnique", + "Vindex": "user_index", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", + "Table": "`user`, ref_with_source", + "Values": [ + "INT64(2)" + ] + }, + "TablesUsed": [ + "user.ref_with_source", + "user.user" + ] + } + }, + { + "comment": "select with join to reference table in unsharded keyspace: should route shard-scoped", + "query": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "plan": { + "QueryType": "SELECT", + "Original": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "Instructions": { + "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", + "OperatorType": "Route", + "Variant": "EqualUnique", + "Vindex": "user_index", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", + "Table": "`user`, ref_with_source", + "Values": [ + "INT64(2)" + ] + }, + "TablesUsed": [ + "user.ref_with_source", + "user.user" + ] + } } ] From 679e3fd41c1c3c8a5cb2440c86ad34141c733503 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 26 Apr 2024 15:35:11 +0200 Subject: [PATCH 07/11] Refactor and revert unnecessary code/changes Signed-off-by: Rohit Nayak --- go/test/endtoend/vreplication/helper_test.go | 32 ------------------- .../endtoend/vreplication/reference_test.go | 6 ++-- .../vtgate/planbuilder/testdata/onecase.json | 7 ++-- 3 files changed, 8 insertions(+), 37 deletions(-) diff --git a/go/test/endtoend/vreplication/helper_test.go b/go/test/endtoend/vreplication/helper_test.go index e952670f6ab..a378f70af77 100644 --- a/go/test/endtoend/vreplication/helper_test.go +++ b/go/test/endtoend/vreplication/helper_test.go @@ -32,8 +32,6 @@ import ( "testing" "time" - "vitess.io/vitess/go/json2" - "github.com/buger/jsonparser" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -833,33 +831,3 @@ func (lg *loadGenerator) waitForCount(want int64) { } } } - -// VExplainPlan is the struct that represents the json output of a vexplain query. -type VExplainPlan struct { - OperatorType string `json:"OperatorType"` - Variant string `json:"Variant"` - Keyspace VExplainKeyspace `json:"Keyspace"` - FieldQuery string `json:"FieldQuery"` - Query string `json:"Query"` - Table string `json:"TableName"` -} - -type VExplainKeyspace struct { - Name string `json:"Name"` - Sharded bool `json:"Sharded"` -} - -// vexplain runs vexplain on the given query and returns the plan. Useful for validating routing rules. -func vexplain(t *testing.T, database, query string) *VExplainPlan { - vtgateConn := vc.GetVTGateConn(t) - defer vtgateConn.Close() - - qr := execVtgateQuery(t, vtgateConn, database, fmt.Sprintf("vexplain %s", query)) - require.NotNil(t, qr) - require.Equal(t, 1, len(qr.Rows)) - json := qr.Rows[0][0].ToString() - - var plan VExplainPlan - require.NoError(t, json2.Unmarshal([]byte(json), &plan)) - return &plan -} diff --git a/go/test/endtoend/vreplication/reference_test.go b/go/test/endtoend/vreplication/reference_test.go index 2e560b22f0c..2fbb3cb4265 100644 --- a/go/test/endtoend/vreplication/reference_test.go +++ b/go/test/endtoend/vreplication/reference_test.go @@ -118,7 +118,7 @@ func TestReferenceTableMaterializationAndRouting(t *testing.T) { vtgateConn = getConnection(t, vc.ClusterConfig.hostname, vc.ClusterConfig.vtgateMySQLPort) defer vtgateConn.Close() waitForRowCount(t, vtgateConn, sks, "cat", 3) - waitForRowCount(t, vtgateConn, sks, "mfg2", 3) + waitForRowCount(t, vtgateConn, sks, "mfg2", 4) execRefQuery(t, "insert into mfg values (5, 'm5')") execRefQuery(t, "insert into mfg2 values (6, 'm6')") @@ -129,7 +129,7 @@ func TestReferenceTableMaterializationAndRouting(t *testing.T) { execRefQuery(t, "update mfg set name = concat(name, '-updated') where id = 1") execRefQuery(t, "update mfg2 set name = concat(name, '-updated') where id = 2") execRefQuery(t, "update uks.mfg set name = concat(name, '-updated') where id = 3") - execRefQuery(t, "update sks.mfg set name = concat(name, '-updated') where id = 4") + execRefQuery(t, "update sks.mfg2 set name = concat(name, '-updated') where id = 4") waitForRowCount(t, vtgateConn, uks, "mfg", 8) qr := execVtgateQuery(t, vtgateConn, "uks", "select count(*) from uks.mfg where name like '%updated%'") @@ -139,7 +139,7 @@ func TestReferenceTableMaterializationAndRouting(t *testing.T) { execRefQuery(t, "delete from mfg where id = 5") execRefQuery(t, "delete from mfg2 where id = 6") execRefQuery(t, "delete from uks.mfg where id = 7") - execRefQuery(t, "delete from sks.mfg where id = 8") + execRefQuery(t, "delete from sks.mfg2 where id = 8") waitForRowCount(t, vtgateConn, uks, "mfg", 4) } diff --git a/go/vt/vtgate/planbuilder/testdata/onecase.json b/go/vt/vtgate/planbuilder/testdata/onecase.json index a15e0c0894b..da7543f706a 100644 --- a/go/vt/vtgate/planbuilder/testdata/onecase.json +++ b/go/vt/vtgate/planbuilder/testdata/onecase.json @@ -1,6 +1,9 @@ [ { "comment": "Add your test case here for debugging and run go test -run=One.", - "query": "" + "query": "", + "plan": { + + } } -] +] \ No newline at end of file From cf3cf6c3faa2fb5d9e2ca265cb373871c74fce5f Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 26 Apr 2024 17:17:10 +0200 Subject: [PATCH 08/11] Address review comments. Move tests into reference_cases.json and remove additional test Signed-off-by: Rohit Nayak --- go/test/endtoend/vreplication/helper_test.go | 1 - go/vt/vtgate/planbuilder/plan_test.go | 14 - .../planbuilder/testdata/reference_cases.json | 318 +++++++++++++++++ .../testdata/reference_table_cases.json | 320 ------------------ 4 files changed, 318 insertions(+), 335 deletions(-) delete mode 100644 go/vt/vtgate/planbuilder/testdata/reference_table_cases.json diff --git a/go/test/endtoend/vreplication/helper_test.go b/go/test/endtoend/vreplication/helper_test.go index a378f70af77..445a15f7767 100644 --- a/go/test/endtoend/vreplication/helper_test.go +++ b/go/test/endtoend/vreplication/helper_test.go @@ -463,7 +463,6 @@ func validateDryRunResults(t *testing.T, output string, want []string) { w = strings.TrimSpace(w[1:]) result := strings.HasPrefix(g, w) match = result - // t.Logf("Partial match |%v|%v|%v\n", w, g, match) } else { match = g == w } diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index a848d225ec4..a7450db3a05 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -732,17 +732,3 @@ func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCase } } } - -func TestReferenceRerouting(t *testing.T) { - reset := oprewriters.EnableDebugPrinting() - defer reset() - - lv := loadSchema(t, "vschemas/schema.json", true) - setFks(t, lv) - vschema := &vschemawrapper.VSchemaWrapper{ - V: lv, - TestBuilder: TestBuilder, - } - - testFile(t, "reference_table_cases.json", "", vschema, false) -} diff --git a/go/vt/vtgate/planbuilder/testdata/reference_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_cases.json index 032ebcc6374..e67006d5d88 100644 --- a/go/vt/vtgate/planbuilder/testdata/reference_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/reference_cases.json @@ -428,5 +428,323 @@ "main.global_ref" ] } + }, + { + "comment": "delete from reference table with another name - query send to source table", + "query": "delete from user.ref_with_source where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table with another name - query send to source table", + "query": "update user.ref_with_source set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table with another name - query send to source table", + "query": "insert into user.ref_with_source(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into user.ref_with_source(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table - query send to source table", + "query": "delete from source_of_ref where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from source_of_ref where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table - query send to source table", + "query": "update source_of_ref set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update source_of_ref set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table - query send to source table", + "query": "insert into source_of_ref(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into source_of_ref(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table qualified with unsharded - query send to source table", + "query": "delete from main.source_of_ref where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from main.source_of_ref where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table qualified with unsharded - query send to source table", + "query": "update main.source_of_ref set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update main.source_of_ref set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table qualified with unsharded - query send to source table", + "query": "insert into main.source_of_ref(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into main.source_of_ref(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table with another name - query send to source table", + "query": "delete from user.ref_with_source where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table with another name - query send to source table", + "query": "update user.ref_with_source set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table with another name - query send to source table", + "query": "insert into user.ref_with_source(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into user.ref_with_source(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "select with join to reference table in sharded keyspace: should route shard-scoped", + "query": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "plan": { + "QueryType": "SELECT", + "Original": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "Instructions": { + "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", + "OperatorType": "Route", + "Variant": "EqualUnique", + "Vindex": "user_index", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", + "Table": "`user`, ref_with_source", + "Values": [ + "INT64(2)" + ] + }, + "TablesUsed": [ + "user.ref_with_source", + "user.user" + ] + } + }, + { + "comment": "select with join to reference table in unsharded keyspace: should route shard-scoped", + "query": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "plan": { + "QueryType": "SELECT", + "Original": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "Instructions": { + "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", + "OperatorType": "Route", + "Variant": "EqualUnique", + "Vindex": "user_index", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", + "Table": "`user`, ref_with_source", + "Values": [ + "INT64(2)" + ] + }, + "TablesUsed": [ + "user.ref_with_source", + "user.user" + ] + } } ] diff --git a/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json deleted file mode 100644 index b86b41e9ffa..00000000000 --- a/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json +++ /dev/null @@ -1,320 +0,0 @@ -[ - { - "comment": "delete from reference table with another name - query send to source table", - "query": "delete from user.ref_with_source where col = 1", - "plan": { - "QueryType": "DELETE", - "Original": "delete from user.ref_with_source where col = 1", - "Instructions": { - "OperatorType": "Delete", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "update from reference table with another name - query send to source table", - "query": "update user.ref_with_source set x = 4 where col = 1", - "plan": { - "QueryType": "UPDATE", - "Original": "update user.ref_with_source set x = 4 where col = 1", - "Instructions": { - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "insert from reference table with another name - query send to source table", - "query": "insert into user.ref_with_source(x) values(4)", - "plan": { - "QueryType": "INSERT", - "Original": "insert into user.ref_with_source(x) values(4)", - "Instructions": { - "OperatorType": "Insert", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "insert into source_of_ref(x) values (4)", - "TableName": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "delete from reference table - query send to source table", - "query": "delete from source_of_ref where col = 1", - "plan": { - "QueryType": "DELETE", - "Original": "delete from source_of_ref where col = 1", - "Instructions": { - "OperatorType": "Delete", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "update from reference table - query send to source table", - "query": "update source_of_ref set x = 4 where col = 1", - "plan": { - "QueryType": "UPDATE", - "Original": "update source_of_ref set x = 4 where col = 1", - "Instructions": { - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "insert from reference table - query send to source table", - "query": "insert into source_of_ref(x) values(4)", - "plan": { - "QueryType": "INSERT", - "Original": "insert into source_of_ref(x) values(4)", - "Instructions": { - "OperatorType": "Insert", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "insert into source_of_ref(x) values (4)", - "TableName": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "delete from reference table qualified with unsharded - query send to source table", - "query": "delete from main.source_of_ref where col = 1", - "plan": { - "QueryType": "DELETE", - "Original": "delete from main.source_of_ref where col = 1", - "Instructions": { - "OperatorType": "Delete", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "update from reference table qualified with unsharded - query send to source table", - "query": "update main.source_of_ref set x = 4 where col = 1", - "plan": { - "QueryType": "UPDATE", - "Original": "update main.source_of_ref set x = 4 where col = 1", - "Instructions": { - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "insert from reference table qualified with unsharded - query send to source table", - "query": "insert into main.source_of_ref(x) values(4)", - "plan": { - "QueryType": "INSERT", - "Original": "insert into main.source_of_ref(x) values(4)", - "Instructions": { - "OperatorType": "Insert", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "insert into source_of_ref(x) values (4)", - "TableName": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "delete from reference table with another name - query send to source table", - "query": "delete from user.ref_with_source where col = 1", - "plan": { - "QueryType": "DELETE", - "Original": "delete from user.ref_with_source where col = 1", - "Instructions": { - "OperatorType": "Delete", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "update from reference table with another name - query send to source table", - "query": "update user.ref_with_source set x = 4 where col = 1", - "plan": { - "QueryType": "UPDATE", - "Original": "update user.ref_with_source set x = 4 where col = 1", - "Instructions": { - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "insert from reference table with another name - query send to source table", - "query": "insert into user.ref_with_source(x) values(4)", - "plan": { - "QueryType": "INSERT", - "Original": "insert into user.ref_with_source(x) values(4)", - "Instructions": { - "OperatorType": "Insert", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "insert into source_of_ref(x) values (4)", - "TableName": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "select with join to reference table in sharded keyspace: should route shard-scoped", - "query": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", - "plan": { - "QueryType": "SELECT", - "Original": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", - "Instructions": { - "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", - "OperatorType": "Route", - "Variant": "EqualUnique", - "Vindex": "user_index", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", - "Table": "`user`, ref_with_source", - "Values": [ - "INT64(2)" - ] - }, - "TablesUsed": [ - "user.ref_with_source", - "user.user" - ] - } - }, - { - "comment": "select with join to reference table in unsharded keyspace: should route shard-scoped", - "query": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", - "plan": { - "QueryType": "SELECT", - "Original": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", - "Instructions": { - "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", - "OperatorType": "Route", - "Variant": "EqualUnique", - "Vindex": "user_index", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", - "Table": "`user`, ref_with_source", - "Values": [ - "INT64(2)" - ] - }, - "TablesUsed": [ - "user.ref_with_source", - "user.user" - ] - } - } -] From 89a8ebe12987fe49f0fa0405e45d6664fcb2490f Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 1 May 2024 19:00:56 +0200 Subject: [PATCH 09/11] Simplify logic for handling reference tables with different names. Add test cases for aliases Signed-off-by: Rohit Nayak --- go/vt/vtgate/planbuilder/dml_planner.go | 8 +- .../planbuilder/testdata/reference_cases.json | 184 +++++++++++++++++- 2 files changed, 182 insertions(+), 10 deletions(-) diff --git a/go/vt/vtgate/planbuilder/dml_planner.go b/go/vt/vtgate/planbuilder/dml_planner.go index 35d755434a1..2c9b8d13e64 100644 --- a/go/vt/vtgate/planbuilder/dml_planner.go +++ b/go/vt/vtgate/planbuilder/dml_planner.go @@ -53,12 +53,8 @@ func rewriteRoutedTables(stmt sqlparser.Statement, vschema plancontext.VSchema) // vindex cannot be present in a dml statement. return false, vterrors.VT09014() } - tableName.Qualifier = vschemaTable.GetTableName().Qualifier - tableName.Name = sqlparser.NewIdentifierCS(vschemaTable.Name.String()) - newAliasTbl := sqlparser.NewAliasedTableExpr(tableName, "") - aliasTbl.Expr = newAliasTbl.Expr - aliasTbl.As = newAliasTbl.As - } else if vschemaTable.Name.String() != tableName.Name.String() { + } + if vschemaTable.Name.String() != tableName.Name.String() { name := tableName.Name if aliasTbl.As.IsEmpty() { // if the user hasn't specified an alias, we'll insert one here so the old table name still works diff --git a/go/vt/vtgate/planbuilder/testdata/reference_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_cases.json index e67006d5d88..89cbf2425da 100644 --- a/go/vt/vtgate/planbuilder/testdata/reference_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/reference_cases.json @@ -443,7 +443,7 @@ "Sharded": false }, "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", + "Query": "delete from source_of_ref as ref_with_source where col = 1", "Table": "source_of_ref" }, "TablesUsed": [ @@ -465,7 +465,51 @@ "Sharded": false }, "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", + "Query": "update source_of_ref as ref_with_source set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table with another name - query send to source table - with alias", + "query": "delete from user.ref_with_source alias where alias.col = 1 and x = 2", + "plan": { + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source alias where alias.col = 1 and x = 2", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref as alias where alias.col = 1 and x = 2", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table with another name - query send to source table - with alias", + "query": "update user.ref_with_source as alias set x = 4 where alias.col = 1 and x = 2", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source as alias set x = 4 where alias.col = 1 and x = 2", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref as alias set x = 4 where alias.col = 1 and x = 2", "Table": "source_of_ref" }, "TablesUsed": [ @@ -539,6 +583,50 @@ ] } }, + { + "comment": "delete from reference table - query send to source table - with alias", + "query": "delete from source_of_ref where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from source_of_ref where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table - query send to source table - with alias", + "query": "update source_of_ref set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update source_of_ref set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, { "comment": "insert from reference table - query send to source table", "query": "insert into source_of_ref(x) values(4)", @@ -605,6 +693,50 @@ ] } }, + { + "comment": "delete from reference table qualified with unsharded - query send to source table - with alias", + "query": "delete from main.source_of_ref where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from main.source_of_ref where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table qualified with unsharded - query send to source table - with alias", + "query": "update main.source_of_ref set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update main.source_of_ref set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, { "comment": "insert from reference table qualified with unsharded - query send to source table", "query": "insert into main.source_of_ref(x) values(4)", @@ -641,7 +773,7 @@ "Sharded": false }, "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", + "Query": "delete from source_of_ref as ref_with_source where col = 1", "Table": "source_of_ref" }, "TablesUsed": [ @@ -663,7 +795,51 @@ "Sharded": false }, "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", + "Query": "update source_of_ref as ref_with_source set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table with another name - query send to source table - with alias", + "query": "delete from user.ref_with_source where ref_with_source.col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source where ref_with_source.col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref as ref_with_source where ref_with_source.col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table with another name - query send to source table - with alias", + "query": "update user.ref_with_source set x = 4 where ref_with_source.col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source set x = 4 where ref_with_source.col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref as ref_with_source set x = 4 where ref_with_source.col = 1", "Table": "source_of_ref" }, "TablesUsed": [ From 3ed27d03a3975b579095a13d16b757224f22e173 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 1 May 2024 19:51:36 +0200 Subject: [PATCH 10/11] Improve comments Signed-off-by: Rohit Nayak --- go/vt/vtgate/planbuilder/dml_planner.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/go/vt/vtgate/planbuilder/dml_planner.go b/go/vt/vtgate/planbuilder/dml_planner.go index 2c9b8d13e64..b536cff058f 100644 --- a/go/vt/vtgate/planbuilder/dml_planner.go +++ b/go/vt/vtgate/planbuilder/dml_planner.go @@ -43,8 +43,8 @@ func rewriteRoutedTables(stmt sqlparser.Statement, vschema plancontext.VSchema) return false, vterrors.VT09014() } + // For reference tables, managed by Vitess (Source not nil), we need to use the source table as the target. if vschemaTable.Type == vindexes.TypeReference && vschemaTable.Source != nil { - // This is a reference table. We need to rewrite the table name and keyspace to that of the source table. vschemaTable, vindexTbl, _, _, _, err = vschema.FindTableOrVindex(vschemaTable.Source.TableName) if err != nil { return false, err @@ -54,10 +54,16 @@ func rewriteRoutedTables(stmt sqlparser.Statement, vschema plancontext.VSchema) return false, vterrors.VT09014() } } + + // The vschema table name can be different due to table routing rules pointing the table to a different one + // or if it backed by reference tables whose source table name is named differently. if vschemaTable.Name.String() != tableName.Name.String() { name := tableName.Name if aliasTbl.As.IsEmpty() { - // if the user hasn't specified an alias, we'll insert one here so the old table name still works + // If the user has not specified an alias, we will insert one here so the old table name still works, + // since the original name might be used in a where clause, for example. + // If the user already specified an alias we should not change it, since the alias might be used in + // a where clause, for example. aliasTbl.As = sqlparser.NewIdentifierCS(name.String()) } tableName.Qualifier = sqlparser.IdentifierCS{} From 91d5865122e141a0bd5fe0c04f099f7228a3daab Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Mon, 6 May 2024 16:10:00 +0200 Subject: [PATCH 11/11] Add license header Signed-off-by: Rohit Nayak --- go/test/endtoend/vreplication/reference_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/go/test/endtoend/vreplication/reference_test.go b/go/test/endtoend/vreplication/reference_test.go index 2fbb3cb4265..1d3f8423ff5 100644 --- a/go/test/endtoend/vreplication/reference_test.go +++ b/go/test/endtoend/vreplication/reference_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package vreplication import (