[release-19.0] Improve performance for BaseShowTablesWithSizes quer…

…y. (#15713) (#15795)

Signed-off-by: Arthur Schreiber <arthurschreiber@github.com>
Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com>
Co-authored-by: Deepthi Sigireddi <deepthi@planetscale.com>
Co-authored-by: Arthur Schreiber <arthurschreiber@github.com>

go/mysql/flavor_mysql.go

@@ -329,41 +329,25 @@ GROUP BY t.table_name, t.table_type, t.create_time, t.table_comment`
 
 // TablesWithSize80 is a query to select table along with size for mysql 8.0
 //
-// We join with a subquery that materializes the data from `information_schema.innodb_sys_tablespaces`
-// early for performance reasons. This effectively causes only a single read of `information_schema.innodb_tablespaces`
-// per query.
 // Note the following:
-//   - We use UNION ALL to deal differently with partitioned tables vs. non-partitioned tables.
-//     Originally, the query handled both, but that introduced "WHERE ... OR" conditions that led to poor query
-//     optimization. By separating to UNION ALL we remove all "OR" conditions.
+//   - We use a single query to fetch both partitioned and non-partitioned tables. This is because
+//     accessing `information_schema.innodb_tablespaces` is expensive on servers with many tablespaces,
+//     and every query that loads the table needs to perform full table scans on it. Doing a single
+//     table scan is more efficient than doing more than one.
 //   - We utilize `INFORMATION_SCHEMA`.`TABLES`.`CREATE_OPTIONS` column to do early pruning before the JOIN.
 //   - `TABLES`.`TABLE_NAME` has `utf8mb4_0900_ai_ci` collation.  `INNODB_TABLESPACES`.`NAME` has `utf8mb3_general_ci`.
 //     We normalize the collation to get better query performance (we force the casting at the time of our choosing)
-//   - `create_options` is NULL for views, and therefore we need an additional UNION ALL to include views
 const TablesWithSize80 = `SELECT t.table_name,
-		t.table_type,
-		UNIX_TIMESTAMP(t.create_time),
-		t.table_comment,
-		i.file_size,
-		i.allocated_size
-	FROM information_schema.tables t
-		LEFT JOIN information_schema.innodb_tablespaces i
-	ON i.name = CONCAT(t.table_schema, '/', t.table_name) COLLATE utf8mb3_general_ci
-	WHERE
-		t.table_schema = database() AND not t.create_options <=> 'partitioned'
-UNION ALL
-	SELECT
-		t.table_name,
 		t.table_type,
 		UNIX_TIMESTAMP(t.create_time),
 		t.table_comment,
 		SUM(i.file_size),
 		SUM(i.allocated_size)
 	FROM information_schema.tables t
 		LEFT JOIN information_schema.innodb_tablespaces i
-	ON i.name LIKE (CONCAT(t.table_schema, '/', t.table_name, '#p#%') COLLATE utf8mb3_general_ci )
+	ON i.name LIKE CONCAT(t.table_schema, '/', t.table_name, IF(t.create_options <=> 'partitioned', '#p#%', '')) COLLATE utf8mb3_general_ci
 	WHERE
-		t.table_schema = database() AND t.create_options <=> 'partitioned'
+		t.table_schema = database()
 	GROUP BY
 		t.table_schema, t.table_name, t.table_type, t.create_time, t.table_comment
 `

go/test/endtoend/vtgate/queries/normalize/normalize_test.go

@@ -40,9 +40,13 @@ func TestNormalizeAllFields(t *testing.T) {
 	defer conn.Close()
 
 	insertQuery := `insert into t1 values (1, "chars", "variable chars", x'73757265', 0x676F, 0.33, 9.99, 1, "1976-06-08", "small", "b", "{\"key\":\"value\"}", point(1,5), b'011', 0b0101)`
-	normalizedInsertQuery := `insert into t1 values (:vtg1 /* INT64 */, :vtg2 /* VARCHAR */, :vtg3 /* VARCHAR */, :vtg4 /* HEXVAL */, :vtg5 /* HEXNUM */, :vtg6 /* DECIMAL */, :vtg7 /* DECIMAL */, :vtg8 /* INT64 */, :vtg9 /* VARCHAR */, :vtg10 /* VARCHAR */, :vtg11 /* VARCHAR */, :vtg12 /* VARCHAR */, point(:vtg13 /* INT64 */, :vtg14 /* INT64 */), :vtg15 /* BITNUM */, :vtg16 /* BITNUM */)`
+
+	normalizedInsertQuery := `insert into t1 values (:vtg1 /* INT64 */, :vtg2 /* VARCHAR */, :vtg3 /* VARCHAR */, :vtg4 /* HEXVAL */, :vtg5 /* HEXNUM */, :vtg6 /* DECIMAL(3,2) */, :vtg7 /* DECIMAL(3,2) */, :vtg8 /* INT64 */, :vtg9 /* VARCHAR */, :vtg10 /* VARCHAR */, :vtg11 /* VARCHAR */, :vtg12 /* VARCHAR */, point(:vtg13 /* INT64 */, :vtg14 /* INT64 */), :vtg15 /* BITNUM */, :vtg16 /* BITNUM */)`
 	vtgateVersion, err := cluster.GetMajorVersion("vtgate")
 	require.NoError(t, err)
+	if vtgateVersion < 20 {
+		normalizedInsertQuery = `insert into t1 values (:vtg1 /* INT64 */, :vtg2 /* VARCHAR */, :vtg3 /* VARCHAR */, :vtg4 /* HEXVAL */, :vtg5 /* HEXNUM */, :vtg6 /* DECIMAL */, :vtg7 /* DECIMAL */, :vtg8 /* INT64 */, :vtg9 /* VARCHAR */, :vtg10 /* VARCHAR */, :vtg11 /* VARCHAR */, :vtg12 /* VARCHAR */, point(:vtg13 /* INT64 */, :vtg14 /* INT64 */), :vtg15 /* BITNUM */, :vtg16 /* BITNUM */)`
+	}
 	if vtgateVersion < 19 {
 		normalizedInsertQuery = `insert into t1 values (:vtg1 /* INT64 */, :vtg2 /* VARCHAR */, :vtg3 /* VARCHAR */, :vtg4 /* HEXVAL */, :vtg5 /* HEXNUM */, :vtg6 /* DECIMAL */, :vtg7 /* DECIMAL */, :vtg8 /* INT64 */, :vtg9 /* VARCHAR */, :vtg10 /* VARCHAR */, :vtg11 /* VARCHAR */, :vtg12 /* VARCHAR */, point(:vtg13 /* INT64 */, :vtg14 /* INT64 */), :vtg15 /* HEXNUM */, :vtg16 /* HEXNUM */)`
 	}

go/vt/vttablet/endtoend/misc_test.go

@@ -906,28 +906,91 @@ func TestShowTablesWithSizes(t *testing.T) {
 		_, err := conn.ExecuteFetch(query, 1, false)
 		require.NoError(t, err)
 	}
-	expectTables := map[string]([]string){ // TABLE_TYPE, TABLE_COMMENT
-		"show_tables_with_sizes_t1":        {"BASE TABLE", ""},
-		"show_tables_with_sizes_v1":        {"VIEW", "VIEW"},
-		"show_tables_with_sizes_employees": {"BASE TABLE", ""},
+
+	expectedTables := []string{
+		"show_tables_with_sizes_t1",
+		"show_tables_with_sizes_v1",
+		"show_tables_with_sizes_employees",
 	}
+	actualTables := []string{}
 
 	rs, err := conn.ExecuteFetch(conn.BaseShowTablesWithSizes(), -1, false)
 	require.NoError(t, err)
 	require.NotEmpty(t, rs.Rows)
 
-	assert.GreaterOrEqual(t, len(rs.Rows), len(expectTables))
-	matchedTables := map[string]bool{}
+	assert.GreaterOrEqual(t, len(rs.Rows), len(expectedTables))
+
 	for _, row := range rs.Rows {
+		assert.Equal(t, 6, len(row))
+
 		tableName := row[0].ToString()
-		vals, ok := expectTables[tableName]
-		if ok {
-			assert.Equal(t, vals[0], row[1].ToString()) // TABLE_TYPE
-			assert.Equal(t, vals[1], row[3].ToString()) // TABLE_COMMENT
-			matchedTables[tableName] = true
+		if tableName == "show_tables_with_sizes_t1" {
+			// TABLE_TYPE
+			assert.Equal(t, "BASE TABLE", row[1].ToString())
+
+			assert.True(t, row[2].IsIntegral())
+			createTime, err := row[2].ToCastInt64()
+			assert.NoError(t, err)
+			assert.Greater(t, createTime, int64(0))
+
+			// TABLE_COMMENT
+			assert.Equal(t, "", row[3].ToString())
+
+			assert.True(t, row[4].IsDecimal())
+			fileSize, err := row[4].ToCastInt64()
+			assert.NoError(t, err)
+			assert.Greater(t, fileSize, int64(0))
+
+			assert.True(t, row[4].IsDecimal())
+			allocatedSize, err := row[5].ToCastInt64()
+			assert.NoError(t, err)
+			assert.Greater(t, allocatedSize, int64(0))
+
+			actualTables = append(actualTables, tableName)
+		} else if tableName == "show_tables_with_sizes_v1" {
+			// TABLE_TYPE
+			assert.Equal(t, "VIEW", row[1].ToString())
+
+			assert.True(t, row[2].IsIntegral())
+			createTime, err := row[2].ToCastInt64()
+			assert.NoError(t, err)
+			assert.Greater(t, createTime, int64(0))
+
+			// TABLE_COMMENT
+			assert.Equal(t, "VIEW", row[3].ToString())
+
+			assert.True(t, row[4].IsNull())
+			assert.True(t, row[5].IsNull())
+
+			actualTables = append(actualTables, tableName)
+		} else if tableName == "show_tables_with_sizes_employees" {
+			// TABLE_TYPE
+			assert.Equal(t, "BASE TABLE", row[1].ToString())
+
+			assert.True(t, row[2].IsIntegral())
+			createTime, err := row[2].ToCastInt64()
+			assert.NoError(t, err)
+			assert.Greater(t, createTime, int64(0))
+
+			// TABLE_COMMENT
+			assert.Equal(t, "", row[3].ToString())
+
+			assert.True(t, row[4].IsDecimal())
+			fileSize, err := row[4].ToCastInt64()
+			assert.NoError(t, err)
+			assert.Greater(t, fileSize, int64(0))
+
+			assert.True(t, row[5].IsDecimal())
+			allocatedSize, err := row[5].ToCastInt64()
+			assert.NoError(t, err)
+			assert.Greater(t, allocatedSize, int64(0))
+
+			actualTables = append(actualTables, tableName)
 		}
 	}
-	assert.Equalf(t, len(expectTables), len(matchedTables), "%v", matchedTables)
+
+	assert.Equal(t, len(expectedTables), len(actualTables))
+	assert.ElementsMatch(t, expectedTables, actualTables)
 }
 
 // TestTuple tests that bind variables having tuple values work with vttablet.