Skip to content

Commit

Permalink
ENG-4682 (#906): Support RelationType (user table, index, system tabl…
Browse files Browse the repository at this point in the history 
…e) in TableInfo

Summary:
Added RelationType information to TableInfo. This is needed by Yugaware to distinguish between system and user tables (specifically for postgres) and in future, for index tables.
Design note: <doc-link>

Test Plan:
Modified unit tests.
`ctest -R master-test`

Reviewers: mikhail, mihnea, bogdan, neil

Reviewed By: neil

Subscribers: yql, bharat

Differential Revision: https://phabricator.dev.yugabyte.com/D6239
  • Loading branch information
@ndeodhar
ndeodhar committed Apr 2, 2019
1 parent 1dedcd1 commit 1bc77da
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 43 deletions.
36 changes: 33 additions & 3 deletions src/yb/master/catalog_manager.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3462,7 +3462,24 @@ Status CatalogManager::ListTables(const ListTablesRequestPB* req,
namespace_id = ns->id();
}

bool has_rel_filter = req->relation_type_filter_size() > 0;
bool include_user_table = has_rel_filter ? false : true;
bool include_user_index = has_rel_filter ? false : true;
bool include_system_table = req->exclude_system_tables() ? false
: (has_rel_filter ? false : true);

for (const auto &relation : req->relation_type_filter()) {
if (relation == SYSTEM_TABLE_RELATION) {
include_system_table = true;
} else if (relation == USER_TABLE_RELATION) {
include_user_table = true;
} else if (relation == INDEX_TABLE_RELATION) {
include_user_index = true;
}
}

boost::shared_lock<LockType> l(lock_);
RelationType relation_type;

for (const auto& entry : table_ids_map_) {
auto& table_info = *entry.second;
Expand All @@ -3481,15 +3498,28 @@ Status CatalogManager::ListTables(const ListTablesRequestPB* req,
}
}

if (req->exclude_system_tables() &&
(IsSystemTable(table_info) || table_info.IsRedisTable())) {
continue;
if (IsUserIndex(table_info)) {
if (!include_user_index) {
continue;
}
relation_type = INDEX_TABLE_RELATION;
} else if (IsUserTable(table_info)) {
if (!include_user_table) {
continue;
}
relation_type = USER_TABLE_RELATION;
} else {
if (!include_system_table) {
continue;
}
relation_type = SYSTEM_TABLE_RELATION;
}

ListTablesResponsePB::TableInfo *table = resp->add_tables();
table->set_id(entry.second->id());
table->set_name(ltm->data().name());
table->set_table_type(ltm->data().table_type());
table->set_relation_type(relation_type);

scoped_refptr<NamespaceInfo> ns = FindPtrOrNull(namespace_ids_map_,
ltm->data().namespace_id());
Expand Down
125 changes: 85 additions & 40 deletions src/yb/master/master-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,28 +84,32 @@ DECLARE_double(leader_failure_max_missed_heartbeat_periods);
EXPECTED_SYSTEM_NAMESPACES \
/**/

#define TABLE_ENTRY(namespace, table) \
#define TABLE_ENTRY(namespace, table, relation_type) \
std::make_tuple(k##namespace##table##TableName, \
k##namespace##NamespaceName, k##namespace##NamespaceId)
k##namespace##NamespaceName, k##namespace##NamespaceId, relation_type)

#define SYSTEM_TABLE_ENTRY(namespace, table) \
TABLE_ENTRY(namespace, table, SYSTEM_TABLE_RELATION)

#define EXPECTED_SYSTEM_TABLES \
TABLE_ENTRY(System, Peers), \
TABLE_ENTRY(System, Local), \
TABLE_ENTRY(System, Partitions), \
TABLE_ENTRY(System, SizeEstimates), \
std::make_tuple(kSysCatalogTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId), \
TABLE_ENTRY(SystemSchema, Aggregates), \
TABLE_ENTRY(SystemSchema, Columns), \
TABLE_ENTRY(SystemSchema, Functions), \
TABLE_ENTRY(SystemSchema, Indexes), \
TABLE_ENTRY(SystemSchema, Triggers), \
TABLE_ENTRY(SystemSchema, Types), \
TABLE_ENTRY(SystemSchema, Views), \
TABLE_ENTRY(SystemSchema, Keyspaces), \
TABLE_ENTRY(SystemSchema, Tables), \
TABLE_ENTRY(SystemAuth, Roles), \
TABLE_ENTRY(SystemAuth, RolePermissions), \
TABLE_ENTRY(SystemAuth, ResourceRolePermissionsIndex)
SYSTEM_TABLE_ENTRY(System, Peers), \
SYSTEM_TABLE_ENTRY(System, Local), \
SYSTEM_TABLE_ENTRY(System, Partitions), \
SYSTEM_TABLE_ENTRY(System, SizeEstimates), \
std::make_tuple(kSysCatalogTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId, \
SYSTEM_TABLE_RELATION), \
SYSTEM_TABLE_ENTRY(SystemSchema, Aggregates), \
SYSTEM_TABLE_ENTRY(SystemSchema, Columns), \
SYSTEM_TABLE_ENTRY(SystemSchema, Functions), \
SYSTEM_TABLE_ENTRY(SystemSchema, Indexes), \
SYSTEM_TABLE_ENTRY(SystemSchema, Triggers), \
SYSTEM_TABLE_ENTRY(SystemSchema, Types), \
SYSTEM_TABLE_ENTRY(SystemSchema, Views), \
SYSTEM_TABLE_ENTRY(SystemSchema, Keyspaces), \
SYSTEM_TABLE_ENTRY(SystemSchema, Tables), \
SYSTEM_TABLE_ENTRY(SystemAuth, Roles), \
SYSTEM_TABLE_ENTRY(SystemAuth, RolePermissions), \
SYSTEM_TABLE_ENTRY(SystemAuth, ResourceRolePermissionsIndex)
/**/

namespace yb {
Expand Down Expand Up @@ -209,12 +213,14 @@ class MasterTest : public YBTest {
ASSERT_EQ(namespaces.namespaces_size(), namespace_info.size());
}

void CheckTables(const std::set<std::tuple<TableName, NamespaceName, NamespaceId>>& table_info,
const ListTablesResponsePB& tables) {
void CheckTables(
const std::set<std::tuple<TableName, NamespaceName, NamespaceId, bool>>& table_info,
const ListTablesResponsePB& tables) {
for (int i = 0; i < tables.tables_size(); i++) {
auto search_key = std::make_tuple(tables.tables(i).name(),
tables.tables(i).namespace_().name(),
tables.tables(i).namespace_().id());
tables.tables(i).namespace_().id(),
tables.tables(i).relation_type());
ASSERT_TRUE(table_info.find(search_key) != table_info.end())
<< strings::Substitute("Couldn't find table $0.$1",
tables.tables(i).namespace_().name(), tables.tables(i).name());
Expand Down Expand Up @@ -520,7 +526,8 @@ TEST_F(MasterTest, TestCatalog) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, default_namespace_name, default_namespace_id),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id,
USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand Down Expand Up @@ -568,7 +575,8 @@ TEST_F(MasterTest, TestCatalog) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, default_namespace_name, default_namespace_id),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id,
USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand Down Expand Up @@ -619,6 +627,35 @@ TEST_F(MasterTest, TestCatalog) {
ASSERT_EQ(1, tables.tables_size());
ASSERT_EQ(kSystemPeersTableName, tables.tables(0).name());
}

{
ListTablesRequestPB req;
req.add_relation_type_filter(USER_TABLE_RELATION);
DoListTables(req, &tables);
ASSERT_EQ(2, tables.tables_size());
}

{
ListTablesRequestPB req;
req.add_relation_type_filter(INDEX_TABLE_RELATION);
DoListTables(req, &tables);
ASSERT_EQ(0, tables.tables_size());
}

{
ListTablesRequestPB req;
req.add_relation_type_filter(SYSTEM_TABLE_RELATION);
DoListTables(req, &tables);
ASSERT_EQ(kNumSystemTables, tables.tables_size());
}

{
ListTablesRequestPB req;
req.add_relation_type_filter(SYSTEM_TABLE_RELATION);
req.add_relation_type_filter(USER_TABLE_RELATION);
DoListTables(req, &tables);
ASSERT_EQ(kNumSystemTables + 2, tables.tables_size());
}
}

// Regression test for KUDU-253/KUDU-592: crash if the schema passed to CreateTable
Expand Down Expand Up @@ -998,7 +1035,7 @@ TEST_F(MasterTest, TestDeletingNonEmptyNamespace) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, other_ns_name, other_ns_id),
std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand Down Expand Up @@ -1092,7 +1129,8 @@ TEST_F(MasterTest, TestTablesWithNamespace) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, default_namespace_name, default_namespace_id),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id,
USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand All @@ -1114,7 +1152,8 @@ TEST_F(MasterTest, TestTablesWithNamespace) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, default_namespace_name, default_namespace_id),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id,
USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand Down Expand Up @@ -1171,7 +1210,7 @@ TEST_F(MasterTest, TestTablesWithNamespace) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, other_ns_name, other_ns_id),
std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand All @@ -1193,7 +1232,7 @@ TEST_F(MasterTest, TestTablesWithNamespace) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, other_ns_name, other_ns_id),
std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand All @@ -1215,7 +1254,7 @@ TEST_F(MasterTest, TestTablesWithNamespace) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, other_ns_name, other_ns_id),
std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand All @@ -1234,7 +1273,8 @@ TEST_F(MasterTest, TestTablesWithNamespace) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, default_namespace_name, default_namespace_id),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id,
USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand Down Expand Up @@ -1280,7 +1320,8 @@ TEST_F(MasterTest, TestFullTableName) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, default_namespace_name, default_namespace_id),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id,
USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand Down Expand Up @@ -1311,8 +1352,9 @@ TEST_F(MasterTest, TestFullTableName) {
ASSERT_EQ(2 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, default_namespace_name, default_namespace_id),
std::make_tuple(kTableName, other_ns_name, other_ns_id),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id,
USER_TABLE_RELATION),
std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand All @@ -1322,14 +1364,15 @@ TEST_F(MasterTest, TestFullTableName) {
ASSERT_EQ(1, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, default_namespace_name, default_namespace_id),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id,
USER_TABLE_RELATION),
}, tables);

ASSERT_NO_FATALS(DoListAllTables(&tables, other_ns_name));
ASSERT_EQ(1, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, other_ns_name, other_ns_id)
std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION)
}, tables);

// Try to alter table: change namespace name into the default one.
Expand All @@ -1354,8 +1397,9 @@ TEST_F(MasterTest, TestFullTableName) {
ASSERT_EQ(2 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, other_ns_name, other_ns_id),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id),
std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id,
USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand All @@ -1366,7 +1410,8 @@ TEST_F(MasterTest, TestFullTableName) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, default_namespace_name, default_namespace_id),
std::make_tuple(kTableName, default_namespace_name, default_namespace_id,
USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand Down Expand Up @@ -1445,7 +1490,7 @@ TEST_F(MasterTest, TestGetTableSchema) {
ASSERT_EQ(1 + kNumSystemTables, tables.tables_size());
CheckTables(
{
std::make_tuple(kTableName, other_ns_name, other_ns_id),
std::make_tuple(kTableName, other_ns_name, other_ns_id, USER_TABLE_RELATION),
EXPECTED_SYSTEM_TABLES
}, tables);

Expand Down
10 changes: 10 additions & 0 deletions src/yb/master/master.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -729,6 +729,12 @@ message IsDeleteTableDoneResponsePB {
optional bool done = 2;
}

enum RelationType {
SYSTEM_TABLE_RELATION = 1;
USER_TABLE_RELATION = 2;
INDEX_TABLE_RELATION = 3;
}

message ListTablesRequestPB {
// When used, only returns tables that satisfy a substring match on name_filter.
optional string name_filter = 1;
Expand All @@ -738,6 +744,9 @@ message ListTablesRequestPB {

// Exclude system tables.
optional bool exclude_system_tables = 3 [default = false];

// Can be used to filter tables based on RelationType
repeated RelationType relation_type_filter = 4;
}

message ListTablesResponsePB {
Expand All @@ -749,6 +758,7 @@ message ListTablesResponsePB {
required string name = 2;
optional TableType table_type = 3;
optional NamespaceIdentifierPB namespace = 4;
optional RelationType relation_type = 5 [default = USER_TABLE_RELATION];
}

repeated TableInfo tables = 2;
Expand Down

0 comments on commit 1bc77da

Please sign in to comment.