From 1bc77da5fb80b004dac68c7e77a17a54676772ed Mon Sep 17 00:00:00 2001 From: Neha Deodhar Date: Mon, 25 Feb 2019 12:08:51 -0800 Subject: [PATCH] ENG-4682 (#906): Support RelationType (user table, index, system table) in TableInfo Summary: Added RelationType information to TableInfo. This is needed by Yugaware to distinguish between system and user tables (specifically for postgres) and in future, for index tables. Design note: Test Plan: Modified unit tests. `ctest -R master-test` Reviewers: mikhail, mihnea, bogdan, neil Reviewed By: neil Subscribers: yql, bharat Differential Revision: https://phabricator.dev.yugabyte.com/D6239 --- src/yb/master/catalog_manager.cc | 36 ++++++++- src/yb/master/master-test.cc | 125 +++++++++++++++++++++---------- src/yb/master/master.proto | 10 +++ 3 files changed, 128 insertions(+), 43 deletions(-) diff --git a/src/yb/master/catalog_manager.cc b/src/yb/master/catalog_manager.cc index bf8f71b62918..aff66dde9e68 100644 --- a/src/yb/master/catalog_manager.cc +++ b/src/yb/master/catalog_manager.cc @@ -3462,7 +3462,24 @@ Status CatalogManager::ListTables(const ListTablesRequestPB* req, namespace_id = ns->id(); } + bool has_rel_filter = req->relation_type_filter_size() > 0; + bool include_user_table = has_rel_filter ? false : true; + bool include_user_index = has_rel_filter ? false : true; + bool include_system_table = req->exclude_system_tables() ? false + : (has_rel_filter ? false : true); + + for (const auto &relation : req->relation_type_filter()) { + if (relation == SYSTEM_TABLE_RELATION) { + include_system_table = true; + } else if (relation == USER_TABLE_RELATION) { + include_user_table = true; + } else if (relation == INDEX_TABLE_RELATION) { + include_user_index = true; + } + } + boost::shared_lock l(lock_); + RelationType relation_type; for (const auto& entry : table_ids_map_) { auto& table_info = *entry.second; @@ -3481,15 +3498,28 @@ Status CatalogManager::ListTables(const ListTablesRequestPB* req, } } - if (req->exclude_system_tables() && - (IsSystemTable(table_info) || table_info.IsRedisTable())) { - continue; + if (IsUserIndex(table_info)) { + if (!include_user_index) { + continue; + } + relation_type = INDEX_TABLE_RELATION; + } else if (IsUserTable(table_info)) { + if (!include_user_table) { + continue; + } + relation_type = USER_TABLE_RELATION; + } else { + if (!include_system_table) { + continue; + } + relation_type = SYSTEM_TABLE_RELATION; } ListTablesResponsePB::TableInfo *table = resp->add_tables(); table->set_id(entry.second->id()); table->set_name(ltm->data().name()); table->set_table_type(ltm->data().table_type()); + table->set_relation_type(relation_type); scoped_refptr ns = FindPtrOrNull(namespace_ids_map_, ltm->data().namespace_id()); diff --git a/src/yb/master/master-test.cc b/src/yb/master/master-test.cc index d180bf71c479..3da2321b96f7 100644 --- a/src/yb/master/master-test.cc +++ b/src/yb/master/master-test.cc @@ -84,28 +84,32 @@ DECLARE_double(leader_failure_max_missed_heartbeat_periods); EXPECTED_SYSTEM_NAMESPACES \ /**/ -#define TABLE_ENTRY(namespace, table) \ +#define TABLE_ENTRY(namespace, table, relation_type) \ std::make_tuple(k##namespace##table##TableName, \ - k##namespace##NamespaceName, k##namespace##NamespaceId) + k##namespace##NamespaceName, k##namespace##NamespaceId, relation_type) + +#define SYSTEM_TABLE_ENTRY(namespace, table) \ + TABLE_ENTRY(namespace, table, SYSTEM_TABLE_RELATION) #define EXPECTED_SYSTEM_TABLES \ - TABLE_ENTRY(System, Peers), \ - TABLE_ENTRY(System, Local), \ - TABLE_ENTRY(System, Partitions), \ - TABLE_ENTRY(System, SizeEstimates), \ - std::make_tuple(kSysCatalogTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId), \ - TABLE_ENTRY(SystemSchema, Aggregates), \ - TABLE_ENTRY(SystemSchema, Columns), \ - TABLE_ENTRY(SystemSchema, Functions), \ - TABLE_ENTRY(SystemSchema, Indexes), \ - TABLE_ENTRY(SystemSchema, Triggers), \ - TABLE_ENTRY(SystemSchema, Types), \ - TABLE_ENTRY(SystemSchema, Views), \ - TABLE_ENTRY(SystemSchema, Keyspaces), \ - TABLE_ENTRY(SystemSchema, Tables), \ - TABLE_ENTRY(SystemAuth, Roles), \ - TABLE_ENTRY(SystemAuth, RolePermissions), \ - TABLE_ENTRY(SystemAuth, ResourceRolePermissionsIndex) + SYSTEM_TABLE_ENTRY(System, Peers), \ + SYSTEM_TABLE_ENTRY(System, Local), \ + SYSTEM_TABLE_ENTRY(System, Partitions), \ + SYSTEM_TABLE_ENTRY(System, SizeEstimates), \ + std::make_tuple(kSysCatalogTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId, \ + SYSTEM_TABLE_RELATION), \ + SYSTEM_TABLE_ENTRY(SystemSchema, Aggregates), \ + SYSTEM_TABLE_ENTRY(SystemSchema, Columns), \ + SYSTEM_TABLE_ENTRY(SystemSchema, Functions), \ + SYSTEM_TABLE_ENTRY(SystemSchema, Indexes), \ + SYSTEM_TABLE_ENTRY(SystemSchema, Triggers), \ + SYSTEM_TABLE_ENTRY(SystemSchema, Types), \ + SYSTEM_TABLE_ENTRY(SystemSchema, Views), \ + SYSTEM_TABLE_ENTRY(SystemSchema, Keyspaces), \ + SYSTEM_TABLE_ENTRY(SystemSchema, Tables), \ + SYSTEM_TABLE_ENTRY(SystemAuth, Roles), \ + SYSTEM_TABLE_ENTRY(SystemAuth, RolePermissions), \ + SYSTEM_TABLE_ENTRY(SystemAuth, ResourceRolePermissionsIndex) /**/ namespace yb { @@ -209,12 +213,14 @@ class MasterTest : public YBTest { ASSERT_EQ(namespaces.namespaces_size(), namespace_info.size()); } - void CheckTables(const std::set>& table_info, - const ListTablesResponsePB& tables) { + void CheckTables( + const std::set>& table_info, + const ListTablesResponsePB& tables) { for (int i = 0; i < tables.tables_size(); i++) { auto search_key = std::make_tuple(tables.tables(i).name(), tables.tables(i).namespace_().name(), - tables.tables(i).namespace_().id()); + tables.tables(i).namespace_().id(), + tables.tables(i).relation_type()); ASSERT_TRUE(table_info.find(search_key) != table_info.end()) << strings::Substitute("Couldn't find table $0.$1", tables.tables(i).namespace_().name(), tables.tables(i).name()); @@ -520,7 +526,8 @@ TEST_F(MasterTest, TestCatalog) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, default_namespace_name, default_namespace_id), + std::make_tuple(kTableName, default_namespace_name, default_namespace_id, + USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -568,7 +575,8 @@ TEST_F(MasterTest, TestCatalog) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, default_namespace_name, default_namespace_id), + std::make_tuple(kTableName, default_namespace_name, default_namespace_id, + USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -619,6 +627,35 @@ TEST_F(MasterTest, TestCatalog) { ASSERT_EQ(1, tables.tables_size()); ASSERT_EQ(kSystemPeersTableName, tables.tables(0).name()); } + + { + ListTablesRequestPB req; + req.add_relation_type_filter(USER_TABLE_RELATION); + DoListTables(req, &tables); + ASSERT_EQ(2, tables.tables_size()); + } + + { + ListTablesRequestPB req; + req.add_relation_type_filter(INDEX_TABLE_RELATION); + DoListTables(req, &tables); + ASSERT_EQ(0, tables.tables_size()); + } + + { + ListTablesRequestPB req; + req.add_relation_type_filter(SYSTEM_TABLE_RELATION); + DoListTables(req, &tables); + ASSERT_EQ(kNumSystemTables, tables.tables_size()); + } + + { + ListTablesRequestPB req; + req.add_relation_type_filter(SYSTEM_TABLE_RELATION); + req.add_relation_type_filter(USER_TABLE_RELATION); + DoListTables(req, &tables); + ASSERT_EQ(kNumSystemTables + 2, tables.tables_size()); + } } // Regression test for KUDU-253/KUDU-592: crash if the schema passed to CreateTable @@ -998,7 +1035,7 @@ TEST_F(MasterTest, TestDeletingNonEmptyNamespace) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, other_ns_name, other_ns_id), + std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1092,7 +1129,8 @@ TEST_F(MasterTest, TestTablesWithNamespace) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, default_namespace_name, default_namespace_id), + std::make_tuple(kTableName, default_namespace_name, default_namespace_id, + USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1114,7 +1152,8 @@ TEST_F(MasterTest, TestTablesWithNamespace) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, default_namespace_name, default_namespace_id), + std::make_tuple(kTableName, default_namespace_name, default_namespace_id, + USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1171,7 +1210,7 @@ TEST_F(MasterTest, TestTablesWithNamespace) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, other_ns_name, other_ns_id), + std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1193,7 +1232,7 @@ TEST_F(MasterTest, TestTablesWithNamespace) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, other_ns_name, other_ns_id), + std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1215,7 +1254,7 @@ TEST_F(MasterTest, TestTablesWithNamespace) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, other_ns_name, other_ns_id), + std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1234,7 +1273,8 @@ TEST_F(MasterTest, TestTablesWithNamespace) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, default_namespace_name, default_namespace_id), + std::make_tuple(kTableName, default_namespace_name, default_namespace_id, + USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1280,7 +1320,8 @@ TEST_F(MasterTest, TestFullTableName) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, default_namespace_name, default_namespace_id), + std::make_tuple(kTableName, default_namespace_name, default_namespace_id, + USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1311,8 +1352,9 @@ TEST_F(MasterTest, TestFullTableName) { ASSERT_EQ(2 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, default_namespace_name, default_namespace_id), - std::make_tuple(kTableName, other_ns_name, other_ns_id), + std::make_tuple(kTableName, default_namespace_name, default_namespace_id, + USER_TABLE_RELATION), + std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1322,14 +1364,15 @@ TEST_F(MasterTest, TestFullTableName) { ASSERT_EQ(1, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, default_namespace_name, default_namespace_id), + std::make_tuple(kTableName, default_namespace_name, default_namespace_id, + USER_TABLE_RELATION), }, tables); ASSERT_NO_FATALS(DoListAllTables(&tables, other_ns_name)); ASSERT_EQ(1, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, other_ns_name, other_ns_id) + std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION) }, tables); // Try to alter table: change namespace name into the default one. @@ -1354,8 +1397,9 @@ TEST_F(MasterTest, TestFullTableName) { ASSERT_EQ(2 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, other_ns_name, other_ns_id), - std::make_tuple(kTableName, default_namespace_name, default_namespace_id), + std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION), + std::make_tuple(kTableName, default_namespace_name, default_namespace_id, + USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1366,7 +1410,8 @@ TEST_F(MasterTest, TestFullTableName) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, default_namespace_name, default_namespace_id), + std::make_tuple(kTableName, default_namespace_name, default_namespace_id, + USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); @@ -1445,7 +1490,7 @@ TEST_F(MasterTest, TestGetTableSchema) { ASSERT_EQ(1 + kNumSystemTables, tables.tables_size()); CheckTables( { - std::make_tuple(kTableName, other_ns_name, other_ns_id), + std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION), EXPECTED_SYSTEM_TABLES }, tables); diff --git a/src/yb/master/master.proto b/src/yb/master/master.proto index 0525855be279..5ab7ff1872de 100644 --- a/src/yb/master/master.proto +++ b/src/yb/master/master.proto @@ -729,6 +729,12 @@ message IsDeleteTableDoneResponsePB { optional bool done = 2; } +enum RelationType { + SYSTEM_TABLE_RELATION = 1; + USER_TABLE_RELATION = 2; + INDEX_TABLE_RELATION = 3; +} + message ListTablesRequestPB { // When used, only returns tables that satisfy a substring match on name_filter. optional string name_filter = 1; @@ -738,6 +744,9 @@ message ListTablesRequestPB { // Exclude system tables. optional bool exclude_system_tables = 3 [default = false]; + + // Can be used to filter tables based on RelationType + repeated RelationType relation_type_filter = 4; } message ListTablesResponsePB { @@ -749,6 +758,7 @@ message ListTablesResponsePB { required string name = 2; optional TableType table_type = 3; optional NamespaceIdentifierPB namespace = 4; + optional RelationType relation_type = 5 [default = USER_TABLE_RELATION]; } repeated TableInfo tables = 2;