From 8fb0297f77c1f229020ce4e39de4ee2b613c1ab0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 25 Jun 2023 10:34:42 +0000 Subject: [PATCH 1/6] BaseShowTablesWithSizes: optimize MySQL 8.0 query Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/mysql/flavor_mysql.go | 47 ++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 407c14e34b0..2f4fa42edda 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -356,17 +356,42 @@ GROUP BY t.table_name, t.table_type, t.create_time, t.table_comment` // We join with a subquery that materializes the data from `information_schema.innodb_sys_tablespaces` // early for performance reasons. This effectively causes only a single read of `information_schema.innodb_tablespaces` // per query. -const TablesWithSize80 = `SELECT t.table_name, - t.table_type, - UNIX_TIMESTAMP(t.create_time), - t.table_comment, - SUM(i.file_size), - SUM(i.allocated_size) -FROM information_schema.tables t -LEFT JOIN information_schema.innodb_tablespaces i - ON i.name LIKE CONCAT(database(), '/%') AND (i.name = CONCAT(t.table_schema, '/', t.table_name) OR i.name LIKE CONCAT(t.table_schema, '/', t.table_name, '#p#%')) -WHERE t.table_schema = database() -GROUP BY t.table_name, t.table_type, t.create_time, t.table_comment` +// Note the following: +// - We use UNION ALL to deal differently with partitioned tables vs. non-partitioned tables. +// Originally, the query handled both, but that introduced "WHERE ... OR" conditions that led to poor query +// optimization. By separating to UNION ALL we remove all "OR" conditions. +// - We utilize `INFORMATION_SCHEMA`.`TABLES`.`CREATE_OPTIONS` column to do early pruning before the JOIN. +// - `TABLES`.`TABLE_NAME` has `utf8mb4_0900_ai_ci` collation. `INNODB_TABLESPACES`.`NAME` has `utf8mb3_general_ci`. +// We normalize the collation to get better query performance (we force the casting at the time of our choosing) +const TablesWithSize80 = ` +SELECT + t1.table_name, + t1.table_type, + UNIX_TIMESTAMP(t1.create_time), + t1.table_comment, + i1.file_size, + i1.allocated_size + FROM information_schema.tables t1 + LEFT JOIN information_schema.innodb_tablespaces i1 + ON i1.name = CONCAT(t1.table_schema, '/', t1.table_name) COLLATE utf8_general_ci + WHERE + t1.table_schema = database() AND t1.create_options != 'partitioned' +UNION ALL +SELECT + t2.table_name, + t2.table_type, + UNIX_TIMESTAMP(t2.create_time), + t2.table_comment, + SUM(i2.file_size), + SUM(i2.allocated_size) + FROM information_schema.tables t2 + LEFT JOIN information_schema.innodb_tablespaces i2 + ON i2.name LIKE (CONCAT(t2.table_schema, '/', t2.table_name, '#p#%') COLLATE utf8_general_ci ) + WHERE + t2.table_schema = database() AND t2.create_options = 'partitioned' + GROUP BY + t2.table_name, t2.table_type, t2.create_time, t2.table_comment +` // baseShowTablesWithSizes is part of the Flavor interface. func (mysqlFlavor56) baseShowTablesWithSizes() string { From 28eeb2acc4073a8b2394f581e4bab8beb063823d Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 26 Jun 2023 07:47:15 +0000 Subject: [PATCH 2/6] adapt regular expression Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/schema/engine_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index c38a63e2481..072e39400ef 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -43,7 +43,7 @@ import ( querypb "vitess.io/vitess/go/vt/proto/query" ) -const baseShowTablesPattern = `SELECT t\.table_name.*` +const baseShowTablesPattern = `[\s]*SELECT[\s]+(t|t1)[.]table_name.*` var mustMatch = utils.MustMatchFn(".Mutex") From 90523dc9b7b06c6377cb05da97c9a0f30b3163f7 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 26 Jun 2023 08:26:10 +0000 Subject: [PATCH 3/6] adapt regular expression, 2 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/query_executor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/query_executor_test.go b/go/vt/vttablet/tabletserver/query_executor_test.go index ca8fc7ebf9d..c9ecae09d8c 100644 --- a/go/vt/vttablet/tabletserver/query_executor_test.go +++ b/go/vt/vttablet/tabletserver/query_executor_test.go @@ -1533,7 +1533,7 @@ func setUpQueryExecutorTest(t *testing.T) *fakesqldb.DB { return db } -const baseShowTablesPattern = `SELECT t\.table_name.*` +const baseShowTablesPattern = `[\s]*SELECT[\s]+(t|t1)[.]table_name.*` func initQueryExecutorTestDB(db *fakesqldb.DB) { addQueryExecutorSupportedQueries(db) From 5f4350aa06747afe72934aac154188a9e985b5a4 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 26 Jun 2023 09:28:55 +0000 Subject: [PATCH 4/6] refactored query again to combat the many unit test query format assumptions Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/mysql/flavor_mysql.go | 46 +++++++++---------- .../tabletserver/query_executor_test.go | 2 +- .../tabletserver/schema/engine_test.go | 2 +- 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 2f4fa42edda..f0c26a1ace0 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -363,34 +363,32 @@ GROUP BY t.table_name, t.table_type, t.create_time, t.table_comment` // - We utilize `INFORMATION_SCHEMA`.`TABLES`.`CREATE_OPTIONS` column to do early pruning before the JOIN. // - `TABLES`.`TABLE_NAME` has `utf8mb4_0900_ai_ci` collation. `INNODB_TABLESPACES`.`NAME` has `utf8mb3_general_ci`. // We normalize the collation to get better query performance (we force the casting at the time of our choosing) -const TablesWithSize80 = ` -SELECT - t1.table_name, - t1.table_type, - UNIX_TIMESTAMP(t1.create_time), - t1.table_comment, - i1.file_size, - i1.allocated_size - FROM information_schema.tables t1 - LEFT JOIN information_schema.innodb_tablespaces i1 - ON i1.name = CONCAT(t1.table_schema, '/', t1.table_name) COLLATE utf8_general_ci +const TablesWithSize80 = `SELECT t.table_name, + t.table_type, + UNIX_TIMESTAMP(t.create_time), + t.table_comment, + i.file_size, + i.allocated_size + FROM information_schema.tables t + LEFT JOIN information_schema.innodb_tablespaces i + ON i.name = CONCAT(t.table_schema, '/', t.table_name) COLLATE utf8_general_ci WHERE - t1.table_schema = database() AND t1.create_options != 'partitioned' + t.table_schema = database() AND t.create_options != 'partitioned' UNION ALL -SELECT - t2.table_name, - t2.table_type, - UNIX_TIMESTAMP(t2.create_time), - t2.table_comment, - SUM(i2.file_size), - SUM(i2.allocated_size) - FROM information_schema.tables t2 - LEFT JOIN information_schema.innodb_tablespaces i2 - ON i2.name LIKE (CONCAT(t2.table_schema, '/', t2.table_name, '#p#%') COLLATE utf8_general_ci ) + SELECT + t.table_name, + t.table_type, + UNIX_TIMESTAMP(t.create_time), + t.table_comment, + SUM(i.file_size), + SUM(i.allocated_size) + FROM information_schema.tables t + LEFT JOIN information_schema.innodb_tablespaces i + ON i.name LIKE (CONCAT(t.table_schema, '/', t.table_name, '#p#%') COLLATE utf8_general_ci ) WHERE - t2.table_schema = database() AND t2.create_options = 'partitioned' + t.table_schema = database() AND t.create_options = 'partitioned' GROUP BY - t2.table_name, t2.table_type, t2.create_time, t2.table_comment + t.table_name, t.table_type, t.create_time, t.table_comment ` // baseShowTablesWithSizes is part of the Flavor interface. diff --git a/go/vt/vttablet/tabletserver/query_executor_test.go b/go/vt/vttablet/tabletserver/query_executor_test.go index c9ecae09d8c..ca8fc7ebf9d 100644 --- a/go/vt/vttablet/tabletserver/query_executor_test.go +++ b/go/vt/vttablet/tabletserver/query_executor_test.go @@ -1533,7 +1533,7 @@ func setUpQueryExecutorTest(t *testing.T) *fakesqldb.DB { return db } -const baseShowTablesPattern = `[\s]*SELECT[\s]+(t|t1)[.]table_name.*` +const baseShowTablesPattern = `SELECT t\.table_name.*` func initQueryExecutorTestDB(db *fakesqldb.DB) { addQueryExecutorSupportedQueries(db) diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index 072e39400ef..c38a63e2481 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -43,7 +43,7 @@ import ( querypb "vitess.io/vitess/go/vt/proto/query" ) -const baseShowTablesPattern = `[\s]*SELECT[\s]+(t|t1)[.]table_name.*` +const baseShowTablesPattern = `SELECT t\.table_name.*` var mustMatch = utils.MustMatchFn(".Mutex") From e398329925cb779f80c30536df7edaa149f4e8b6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 26 Jun 2023 09:30:45 +0000 Subject: [PATCH 5/6] use t.table_schema in GROUP BY. Although not strictly necessary this may be faster Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/mysql/flavor_mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index f0c26a1ace0..84c4564d7a0 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -388,7 +388,7 @@ UNION ALL WHERE t.table_schema = database() AND t.create_options = 'partitioned' GROUP BY - t.table_name, t.table_type, t.create_time, t.table_comment + t.table_schema, t.table_name, t.table_type, t.create_time, t.table_comment ` // baseShowTablesWithSizes is part of the Flavor interface. From de0a0d40ad6822f872bdb8a65cf044584d438fc6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:34:23 +0300 Subject: [PATCH 6/6] Support views in BaseShowTablesWithSizes for MySQL 8.0 (#13394) * Support views in BaseShowTablesWithSizes for MySQL 8.0 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * added test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fixed test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * schema tracker: use null safe comparison Signed-off-by: Andres Taylor --------- Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Signed-off-by: Andres Taylor Co-authored-by: Andres Taylor --- go/mysql/flavor_mysql.go | 5 +-- go/vt/vttablet/endtoend/misc_test.go | 50 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 84c4564d7a0..3650d085960 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -363,6 +363,7 @@ GROUP BY t.table_name, t.table_type, t.create_time, t.table_comment` // - We utilize `INFORMATION_SCHEMA`.`TABLES`.`CREATE_OPTIONS` column to do early pruning before the JOIN. // - `TABLES`.`TABLE_NAME` has `utf8mb4_0900_ai_ci` collation. `INNODB_TABLESPACES`.`NAME` has `utf8mb3_general_ci`. // We normalize the collation to get better query performance (we force the casting at the time of our choosing) +// - `create_options` is NULL for views, and therefore we need an additional UNION ALL to include views const TablesWithSize80 = `SELECT t.table_name, t.table_type, UNIX_TIMESTAMP(t.create_time), @@ -373,7 +374,7 @@ const TablesWithSize80 = `SELECT t.table_name, LEFT JOIN information_schema.innodb_tablespaces i ON i.name = CONCAT(t.table_schema, '/', t.table_name) COLLATE utf8_general_ci WHERE - t.table_schema = database() AND t.create_options != 'partitioned' + t.table_schema = database() AND not t.create_options <=> 'partitioned' UNION ALL SELECT t.table_name, @@ -386,7 +387,7 @@ UNION ALL LEFT JOIN information_schema.innodb_tablespaces i ON i.name LIKE (CONCAT(t.table_schema, '/', t.table_name, '#p#%') COLLATE utf8_general_ci ) WHERE - t.table_schema = database() AND t.create_options = 'partitioned' + t.table_schema = database() AND t.create_options <=> 'partitioned' GROUP BY t.table_schema, t.table_name, t.table_type, t.create_time, t.table_comment ` diff --git a/go/vt/vttablet/endtoend/misc_test.go b/go/vt/vttablet/endtoend/misc_test.go index 913af99fc4f..455376576ab 100644 --- a/go/vt/vttablet/endtoend/misc_test.go +++ b/go/vt/vttablet/endtoend/misc_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "io" + "math" "net/http" "reflect" "strings" @@ -943,3 +944,52 @@ func TestHexAndBitBindVar(t *testing.T) { require.NoError(t, err) assert.Equal(t, `[[INT64(10) UINT64(10) INT64(2480) UINT64(2480)]]`, fmt.Sprintf("%v", qr.Rows)) } + +// Test will validate drop view ddls. +func TestShowTablesWithSizes(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &connParams) + require.NoError(t, err) + defer conn.Close() + + setupQueries := []string{ + `drop view if exists show_tables_with_sizes_v1`, + `drop table if exists show_tables_with_sizes_t1`, + `drop table if exists show_tables_with_sizes_employees`, + `create table show_tables_with_sizes_t1 (id int primary key)`, + `create view show_tables_with_sizes_v1 as select * from show_tables_with_sizes_t1`, + `CREATE TABLE show_tables_with_sizes_employees (id INT NOT NULL, store_id INT) PARTITION BY HASH(store_id) PARTITIONS 4`, + } + + defer func() { + _, _ = conn.ExecuteFetch(`drop view if exists show_tables_with_sizes_v1`, 1, false) + _, _ = conn.ExecuteFetch(`drop table if exists show_tables_with_sizes_t1`, 1, false) + _, _ = conn.ExecuteFetch(`drop table if exists show_tables_with_sizes_employees`, 1, false) + }() + for _, query := range setupQueries { + _, err := conn.ExecuteFetch(query, 1, false) + require.NoError(t, err) + } + expectTables := map[string]([]string){ // TABLE_TYPE, TABLE_COMMENT + "show_tables_with_sizes_t1": {"BASE TABLE", ""}, + "show_tables_with_sizes_v1": {"VIEW", "VIEW"}, + "show_tables_with_sizes_employees": {"BASE TABLE", ""}, + } + + rs, err := conn.ExecuteFetch(conn.BaseShowTablesWithSizes(), math.MaxInt, false) + require.NoError(t, err) + require.NotEmpty(t, rs.Rows) + + assert.GreaterOrEqual(t, len(rs.Rows), len(expectTables)) + matchedTables := map[string]bool{} + for _, row := range rs.Rows { + tableName := row[0].ToString() + vals, ok := expectTables[tableName] + if ok { + assert.Equal(t, vals[0], row[1].ToString()) // TABLE_TYPE + assert.Equal(t, vals[1], row[3].ToString()) // TABLE_COMMENT + matchedTables[tableName] = true + } + } + assert.Equalf(t, len(expectTables), len(matchedTables), "%v", matchedTables) +}