Skip to content

Commit

Permalink
Add metadata for views
Browse files Browse the repository at this point in the history
  • Loading branch information
@electrum
electrum committed Jun 11, 2014
1 parent 390a1c9 commit 5a727b8
Show file tree
Hide file tree
Showing 6 changed files with 210 additions and 14 deletions.
40 changes: 38 additions & 2 deletions .../com/facebook/presto/connector/informationSchema/InformationSchemaDataStreamProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.facebook.presto.metadata.QualifiedTableName;
import com.facebook.presto.metadata.QualifiedTablePrefix;
import com.facebook.presto.metadata.TableHandle;
import com.facebook.presto.metadata.ViewDefinition;
import com.facebook.presto.operator.AlignmentOperator;
import com.facebook.presto.operator.Operator;
import com.facebook.presto.operator.OperatorContext;
Expand All @@ -44,25 +45,29 @@
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import io.airlift.slice.Slice;

import javax.inject.Inject;

import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_COLUMNS;
import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_INTERNAL_FUNCTIONS;
import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_INTERNAL_PARTITIONS;
import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_SCHEMATA;
import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_TABLES;
import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_VIEWS;
import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.informationSchemaTableColumns;
import static com.facebook.presto.util.Types.checkType;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Strings.nullToEmpty;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.collect.Sets.union;
import static java.lang.String.format;

public class InformationSchemaDataStreamProvider
Expand Down Expand Up @@ -113,6 +118,9 @@ public InternalTable getInformationSchemaTable(ConnectorSession session, String
if (table.equals(TABLE_TABLES)) {
return buildTables(session, catalog, filters);
}
if (table.equals(TABLE_VIEWS)) {
return buildViews(session, catalog, filters);
}
if (table.equals(TABLE_SCHEMATA)) {
return buildSchemata(session, catalog);
}
Expand Down Expand Up @@ -157,13 +165,18 @@ private Map<QualifiedTableName, List<ColumnMetadata>> getColumnsList(ConnectorSe

private InternalTable buildTables(ConnectorSession session, String catalogName, Map<String, Object> filters)
{
Set<QualifiedTableName> tables = ImmutableSet.copyOf(getTablesList(session, catalogName, filters));
Set<QualifiedTableName> views = ImmutableSet.copyOf(getViewsList(session, catalogName, filters));

InternalTable.Builder table = InternalTable.builder(informationSchemaTableColumns(TABLE_TABLES));
for (QualifiedTableName name : getTablesList(session, catalogName, filters)) {
for (QualifiedTableName name : union(tables, views)) {
// if table and view names overlap, the view wins
String type = views.contains(name) ? "VIEW" : "BASE TABLE";
table.add(
name.getCatalogName(),
name.getSchemaName(),
name.getTableName(),
"BASE TABLE");
type);
}
return table.build();
}
Expand All @@ -173,6 +186,29 @@ private List<QualifiedTableName> getTablesList(ConnectorSession session, String
return metadata.listTables(session, extractQualifiedTablePrefix(catalogName, filters));
}

private List<QualifiedTableName> getViewsList(ConnectorSession session, String catalogName, Map<String, Object> filters)
{
return metadata.listViews(session, extractQualifiedTablePrefix(catalogName, filters));
}

private InternalTable buildViews(ConnectorSession session, String catalogName, Map<String, Object> filters)
{
InternalTable.Builder table = InternalTable.builder(informationSchemaTableColumns(TABLE_VIEWS));
for (Entry<QualifiedTableName, ViewDefinition> entry : getViews(session, catalogName, filters).entrySet()) {
table.add(
entry.getKey().getCatalogName(),
entry.getKey().getSchemaName(),
entry.getKey().getTableName(),
entry.getValue().getOriginalSql());
}
return table.build();
}

private Map<QualifiedTableName, ViewDefinition> getViews(ConnectorSession session, String catalogName, Map<String, Object> filters)
{
return metadata.getViews(session, extractQualifiedTablePrefix(catalogName, filters));
}

private InternalTable buildFunctions()
{
InternalTable.Builder table = InternalTable.builder(informationSchemaTableColumns(TABLE_INTERNAL_FUNCTIONS));
Expand Down
7 changes: 7 additions & 0 deletions .../main/java/com/facebook/presto/connector/informationSchema/InformationSchemaMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ public class InformationSchemaMetadata

public static final SchemaTableName TABLE_COLUMNS = new SchemaTableName(INFORMATION_SCHEMA, "columns");
public static final SchemaTableName TABLE_TABLES = new SchemaTableName(INFORMATION_SCHEMA, "tables");
public static final SchemaTableName TABLE_VIEWS = new SchemaTableName(INFORMATION_SCHEMA, "views");
public static final SchemaTableName TABLE_SCHEMATA = new SchemaTableName(INFORMATION_SCHEMA, "schemata");
public static final SchemaTableName TABLE_INTERNAL_FUNCTIONS = new SchemaTableName(INFORMATION_SCHEMA, "__internal_functions__");
public static final SchemaTableName TABLE_INTERNAL_PARTITIONS = new SchemaTableName(INFORMATION_SCHEMA, "__internal_partitions__");
Expand All @@ -70,6 +71,12 @@ public class InformationSchemaMetadata
.column("table_name", VARCHAR)
.column("table_type", VARCHAR)
.build())
.table(tableMetadataBuilder(TABLE_VIEWS)
.column("table_catalog", VARCHAR)
.column("table_schema", VARCHAR)
.column("table_name", VARCHAR)
.column("view_definition", VARCHAR)
.build())
.table(tableMetadataBuilder(TABLE_SCHEMATA)
.column("catalog_name", VARCHAR)
.column("schema_name", VARCHAR)
Expand Down
12 changes: 12 additions & 0 deletions presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,18 @@ FunctionInfo getExactOperator(OperatorType operatorType, Type returnType, List<?
@NotNull
Map<String, String> getCatalogNames();

/**
* Get the names that match the specified table prefix (never null).
*/
@NotNull
List<QualifiedTableName> listViews(ConnectorSession session, QualifiedTablePrefix prefix);

/**
* Get the view definitions that match the specified table prefix (never null).
*/
@NotNull
Map<QualifiedTableName, ViewDefinition> getViews(ConnectorSession session, QualifiedTablePrefix prefix);

/**
* Returns the view definition for the specified view name.
*/
Expand Down
85 changes: 74 additions & 11 deletions presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import javax.inject.Singleton;

import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
Expand All @@ -60,6 +61,7 @@
import static com.facebook.presto.metadata.MetadataUtil.checkCatalogName;
import static com.facebook.presto.metadata.MetadataUtil.checkColumnName;
import static com.facebook.presto.metadata.QualifiedTableName.convertFromSchemaTableName;
import static com.facebook.presto.metadata.ViewDefinition.ViewColumn;
import static com.facebook.presto.spi.StandardErrorCode.INVALID_VIEW;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
Expand Down Expand Up @@ -297,15 +299,37 @@ public boolean canCreateSampledTables(ConnectorSession session, String catalogNa
public Map<QualifiedTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, QualifiedTablePrefix prefix)
{
checkNotNull(prefix, "prefix is null");
SchemaTablePrefix tablePrefix = prefix.asSchemaTablePrefix();

Map<QualifiedTableName, List<ColumnMetadata>> tableColumns = new LinkedHashMap<>();
Map<QualifiedTableName, List<ColumnMetadata>> tableColumns = new HashMap<>();
for (ConnectorMetadataEntry connectorMetadata : allConnectorsFor(prefix.getCatalogName())) {
for (Entry<SchemaTableName, List<ColumnMetadata>> entry : connectorMetadata.getMetadata().listTableColumns(session, prefix.asSchemaTablePrefix()).entrySet()) {
QualifiedTableName tableName = new QualifiedTableName(prefix.getCatalogName(), entry.getKey().getSchemaName(), entry.getKey().getTableName());
if (!tableColumns.containsKey(tableName)) {
tableColumns.put(tableName, entry.getValue());
ConnectorMetadata metadata = connectorMetadata.getMetadata();

for (Entry<SchemaTableName, List<ColumnMetadata>> entry : metadata.listTableColumns(session, tablePrefix).entrySet()) {
QualifiedTableName tableName = new QualifiedTableName(
prefix.getCatalogName(),
entry.getKey().getSchemaName(),
entry.getKey().getTableName());
tableColumns.put(tableName, entry.getValue());
}

// if table and view names overlap, the view wins
for (Entry<SchemaTableName, String> entry : metadata.getViews(session, tablePrefix).entrySet()) {
QualifiedTableName tableName = new QualifiedTableName(
prefix.getCatalogName(),
entry.getKey().getSchemaName(),
entry.getKey().getTableName());

int ordinalPosition = 0;
ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder();
for (ViewColumn column : deserializeView(entry.getValue()).getColumns()) {
columns.add(new ColumnMetadata(column.getName(), column.getType(), ordinalPosition, false));
ordinalPosition++;
}

tableColumns.put(tableName, columns.build());
}

}
return ImmutableMap.copyOf(tableColumns);
}
Expand Down Expand Up @@ -351,6 +375,40 @@ public Map<String, String> getCatalogNames()
return catalogsMap.build();
}

@Override
public List<QualifiedTableName> listViews(ConnectorSession session, QualifiedTablePrefix prefix)
{
checkNotNull(prefix, "prefix is null");

String schemaNameOrNull = prefix.getSchemaName().orNull();
Set<QualifiedTableName> tables = new LinkedHashSet<>();
for (ConnectorMetadataEntry entry : allConnectorsFor(prefix.getCatalogName())) {
for (QualifiedTableName tableName : transform(entry.getMetadata().listViews(session, schemaNameOrNull), convertFromSchemaTableName(prefix.getCatalogName()))) {
tables.add(tableName);
}
}
return ImmutableList.copyOf(tables);
}

@Override
public Map<QualifiedTableName, ViewDefinition> getViews(ConnectorSession session, QualifiedTablePrefix prefix)
{
checkNotNull(prefix, "prefix is null");
SchemaTablePrefix tablePrefix = prefix.asSchemaTablePrefix();

Map<QualifiedTableName, ViewDefinition> views = new LinkedHashMap<>();
for (ConnectorMetadataEntry metadata : allConnectorsFor(prefix.getCatalogName())) {
for (Entry<SchemaTableName, String> entry : metadata.getMetadata().getViews(session, tablePrefix).entrySet()) {
QualifiedTableName viewName = new QualifiedTableName(
prefix.getCatalogName(),
entry.getKey().getSchemaName(),
entry.getKey().getTableName());
views.put(viewName, deserializeView(entry.getValue()));
}
}
return ImmutableMap.copyOf(views);
}

@Override
public Optional<ViewDefinition> getView(ConnectorSession session, QualifiedTableName viewName)
{
Expand All @@ -359,12 +417,7 @@ public Optional<ViewDefinition> getView(ConnectorSession session, QualifiedTable
Map<SchemaTableName, String> views = entry.getMetadata().getViews(session, prefix);
String view = views.get(viewName.asSchemaTableName());
if (view != null) {
try {
return Optional.of(viewCodec.fromJson(view));
}
catch (IllegalArgumentException e) {
throw new PrestoException(INVALID_VIEW.toErrorCode(), "Invalid view JSON: " + view, e);
}
return Optional.of(deserializeView(view));
}
}
return Optional.absent();
Expand All @@ -386,6 +439,16 @@ public void dropView(ConnectorSession session, QualifiedTableName viewName)
connectorMetadata.getMetadata().dropView(session, viewName.asSchemaTableName());
}

private ViewDefinition deserializeView(String data)
{
try {
return viewCodec.fromJson(data);
}
catch (IllegalArgumentException e) {
throw new PrestoException(INVALID_VIEW.toErrorCode(), "Invalid view JSON: " + data, e);
}
}

private List<ConnectorMetadataEntry> allConnectorsFor(String catalogName)
{
ImmutableList.Builder<ConnectorMetadataEntry> builder = ImmutableList.builder();
Expand Down
3 changes: 2 additions & 1 deletion presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,8 @@ protected TupleDescriptor visitShowColumns(ShowColumns showColumns, AnalysisCont
{
QualifiedTableName tableName = MetadataUtil.createQualifiedTableName(session, showColumns.getTable());

if (!metadata.getTableHandle(session, tableName).isPresent()) {
if (!metadata.getView(session, tableName).isPresent() &&
!metadata.getTableHandle(session, tableName).isPresent()) {
throw new SemanticException(MISSING_TABLE, showColumns, "Table '%s' does not exist", tableName);
}

Expand Down
77 changes: 77 additions & 0 deletions presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,26 @@

import com.facebook.presto.spi.ConnectorSession;
import com.facebook.presto.testing.MaterializedResult;
import com.facebook.presto.testing.MaterializedRow;
import com.facebook.presto.testing.QueryRunner;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
import org.intellij.lang.annotations.Language;
import org.testng.annotations.Test;

import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.INFORMATION_SCHEMA;
import static com.facebook.presto.spi.type.BooleanType.BOOLEAN;
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
import static com.facebook.presto.sql.SqlFormatter.formatSql;
import static com.facebook.presto.sql.parser.SqlParser.createStatement;
import static com.facebook.presto.testing.MaterializedResult.resultBuilder;
import static com.google.common.collect.Iterables.transform;
import static java.lang.String.format;
import static java.util.Collections.nCopies;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;

public abstract class AbstractTestDistributedQueries
extends AbstractTestApproximateQueries
Expand Down Expand Up @@ -127,6 +134,67 @@ public void testView()
assertQueryTrue("DROP VIEW test_view");
}

@Test
public void testViewMetadata()
throws Exception
{
@Language("SQL") String query = "SELECT 123 x, 'foo' y";
assertQueryTrue("CREATE VIEW meta_test_view AS " + query);

// test INFORMATION_SCHEMA.TABLES
MaterializedResult actual = computeActual(format(
"SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = '%s'",
getSession().getSchema()));

MaterializedResult expected = resultBuilder(getSession(), actual.getTypes())
.row("customer", "BASE TABLE")
.row("lineitem", "BASE TABLE")
.row("meta_test_view", "VIEW")
.row("nation", "BASE TABLE")
.row("orders", "BASE TABLE")
.row("part", "BASE TABLE")
.row("partsupp", "BASE TABLE")
.row("region", "BASE TABLE")
.row("supplier", "BASE TABLE")
.build();

assertContains(actual, expected);

// test SHOW TABLES
actual = computeActual("SHOW TABLES");

MaterializedResult.Builder builder = resultBuilder(getSession(), actual.getTypes());
for (MaterializedRow row : expected.getMaterializedRows()) {
builder.row(row.getField(0));
}
expected = builder.build();

assertContains(actual, expected);

// test INFORMATION_SCHEMA.VIEWS
actual = computeActual(format(
"SELECT table_name, view_definition FROM information_schema.views WHERE table_schema = '%s'",
getSession().getSchema()));

expected = resultBuilder(getSession(), actual.getTypes())
.row("meta_test_view", formatSql(createStatement(query)))
.build();

assertContains(actual, expected);

// test SHOW COLUMNS
actual = computeActual("SHOW COLUMNS FROM meta_test_view");

expected = resultBuilder(getSession(), VARCHAR, VARCHAR, BOOLEAN, BOOLEAN)
.row("x", "bigint", true, false)
.row("y", "varchar", true, false)
.build();

assertEquals(actual, expected);

assertQueryTrue("DROP VIEW meta_test_view");
}

@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*statement is too large.*")
public void testLargeQueryFailure()
throws Exception
Expand Down Expand Up @@ -178,4 +246,13 @@ public void testTableSampleSystemBoundaryValues()
assertTrue(all.getMaterializedRows().containsAll(fullSample.getMaterializedRows()));
assertEquals(emptySample.getMaterializedRows().size(), 0);
}

private static void assertContains(MaterializedResult actual, MaterializedResult expected)
{
for (MaterializedRow row : expected.getMaterializedRows()) {
if (!actual.getMaterializedRows().contains(row)) {
fail(format("expected row missing: %s%nActual:%n %s%nExpected:%n %s", row, actual, expected));
}
}
}
}

0 comments on commit 5a727b8

Please sign in to comment.