Skip to content

Commit

Permalink
Disallow querying Delta Lake system tables
Browse files Browse the repository at this point in the history 
Hive connector cannot read from Delta Lake tables
reason why querying system tables for such tables
shouldn't be permitted within the hive connector.

In case of trying to query on the hive connector
the special hive tables:

- $properties
- $partitions
on a Delta Lake table, the user will receive a table
not found exception.
  • Loading branch information
@findinpath @findepi
findinpath authored and findepi committed Jan 19, 2022
1 parent 835438a commit e239035
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 23 deletions.
28 changes: 24 additions & 4 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/PartitionsSystemTableProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package io.trino.plugin.hive;

import io.trino.plugin.hive.authentication.HiveIdentity;
import io.trino.plugin.hive.metastore.Table;
import io.trino.spi.connector.ColumnMetadata;
import io.trino.spi.connector.ConnectorSession;
import io.trino.spi.connector.ConnectorTableMetadata;
Expand All @@ -22,6 +23,7 @@
import io.trino.spi.connector.SchemaTableName;
import io.trino.spi.connector.SystemTable;
import io.trino.spi.type.Type;
import io.trino.spi.type.TypeManager;

import javax.inject.Inject;

Expand All @@ -33,7 +35,14 @@
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.collect.Streams.stream;
import static io.trino.plugin.hive.HiveSessionProperties.getTimestampPrecision;
import static io.trino.plugin.hive.SystemTableHandler.PARTITIONS;
import static io.trino.plugin.hive.metastore.MetastoreUtil.getProtectMode;
import static io.trino.plugin.hive.metastore.MetastoreUtil.verifyOnline;
import static io.trino.plugin.hive.util.HiveBucketing.getHiveBucketHandle;
import static io.trino.plugin.hive.util.HiveUtil.getPartitionKeyColumnHandles;
import static io.trino.plugin.hive.util.HiveUtil.getRegularColumnHandles;
import static io.trino.plugin.hive.util.HiveUtil.isDeltaLakeTable;
import static io.trino.plugin.hive.util.SystemTables.createSystemTable;
import static java.util.Objects.requireNonNull;
import static java.util.function.Function.identity;
Expand All @@ -43,11 +52,13 @@ public class PartitionsSystemTableProvider
implements SystemTableProvider
{
private final HivePartitionManager partitionManager;
private final TypeManager typeManager;

@Inject
public PartitionsSystemTableProvider(HivePartitionManager partitionManager)
public PartitionsSystemTableProvider(HivePartitionManager partitionManager, TypeManager typeManager)
{
this.partitionManager = requireNonNull(partitionManager, "partitionManager is null");
this.typeManager = requireNonNull(typeManager, "typeManager is null");
}

@Override
Expand All @@ -68,11 +79,20 @@ public Optional<SystemTable> getSystemTable(HiveMetadata metadata, ConnectorSess
}

SchemaTableName sourceTableName = PARTITIONS.getSourceTableName(tableName);
HiveTableHandle sourceTableHandle = metadata.getTableHandle(session, sourceTableName);

if (sourceTableHandle == null) {
Table sourceTable = metadata.getMetastore()
.getTable(new HiveIdentity(session), sourceTableName.getSchemaName(), sourceTableName.getTableName())
.orElse(null);
if (sourceTable == null || isDeltaLakeTable(sourceTable)) {
return Optional.empty();
}
verifyOnline(sourceTableName, Optional.empty(), getProtectMode(sourceTable), sourceTable.getParameters());
HiveTableHandle sourceTableHandle = new HiveTableHandle(
sourceTableName.getSchemaName(),
sourceTableName.getTableName(),
sourceTable.getParameters(),
getPartitionKeyColumnHandles(sourceTable, typeManager),
getRegularColumnHandles(sourceTable, typeManager, getTimestampPrecision(session)),
getHiveBucketHandle(session, sourceTable, typeManager));

List<HiveColumnHandle> partitionColumns = sourceTableHandle.getPartitionColumns();
if (partitionColumns.isEmpty()) {
Expand Down
5 changes: 1 addition & 4 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/PropertiesSystemTableProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import com.google.common.collect.ImmutableSortedMap;
import io.trino.plugin.hive.authentication.HiveIdentity;
import io.trino.plugin.hive.metastore.Table;
import io.trino.spi.TrinoException;
import io.trino.spi.connector.ColumnMetadata;
import io.trino.spi.connector.ConnectorSession;
import io.trino.spi.connector.ConnectorTableMetadata;
Expand All @@ -33,11 +32,9 @@
import java.util.Optional;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.trino.plugin.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT;
import static io.trino.plugin.hive.SystemTableHandler.PROPERTIES;
import static io.trino.plugin.hive.util.HiveUtil.isDeltaLakeTable;
import static io.trino.plugin.hive.util.SystemTables.createSystemTable;
import static java.lang.String.format;

public class PropertiesSystemTableProvider
implements SystemTableProvider
Expand Down Expand Up @@ -65,7 +62,7 @@ public Optional<SystemTable> getSystemTable(HiveMetadata metadata, ConnectorSess
.orElseThrow(() -> new TableNotFoundException(tableName));

if (isDeltaLakeTable(table)) {
throw new TrinoException(HIVE_UNSUPPORTED_FORMAT, format("Cannot query Delta Lake table '%s'", sourceTableName));
return Optional.empty();
}
Map<String, String> sortedTableParameters = ImmutableSortedMap.copyOf(table.getParameters());
List<ColumnMetadata> columns = sortedTableParameters.keySet().stream()
Expand Down
20 changes: 6 additions & 14 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -839,7 +839,7 @@ protected final void setup(String databaseName, HiveConfig hiveConfig, HiveMetas
TupleDomain.all()));
},
ImmutableSet.of(
new PartitionsSystemTableProvider(partitionManager),
new PartitionsSystemTableProvider(partitionManager, TESTING_TYPE_MANAGER),
new PropertiesSystemTableProvider()),
(metastore) -> new NoneHiveMaterializedViewMetadata()
{
Expand Down Expand Up @@ -2899,22 +2899,14 @@ public void testHideDeltaLakeTables()
.hasMessage(format("Cannot query Delta Lake table '%s'", tableName));
}

// Verify the hidden `$properties` Delta Lake table handle can't be obtained within the hive connector
SchemaTableName propertiesTableName = new SchemaTableName(tableName.getSchemaName(), format("%s$properties", tableName.getTableName()));
// Verify the hidden `$properties` and `$partitions` Delta Lake table handle can't be obtained within the hive connector
try (Transaction transaction = newTransaction()) {
ConnectorMetadata metadata = transaction.getMetadata();
metadata.beginQuery(session);
assertThatThrownBy(() -> metadata.getSystemTable(newSession(), propertiesTableName))
.hasMessage(format("Cannot query Delta Lake table '%s'", tableName));
}

// Verify the hidden `$partitions` Delta Lake table handle can't be obtained within the hive connector
SchemaTableName partitionsTableName = new SchemaTableName(tableName.getSchemaName(), format("%s$partitions", tableName.getTableName()));
try (Transaction transaction = newTransaction()) {
ConnectorMetadata metadata = transaction.getMetadata();
metadata.beginQuery(session);
assertThatThrownBy(() -> metadata.getSystemTable(newSession(), partitionsTableName))
.hasMessage(format("Cannot query Delta Lake table '%s'", tableName));
SchemaTableName propertiesTableName = new SchemaTableName(tableName.getSchemaName(), format("%s$properties", tableName.getTableName()));
assertThat(metadata.getSystemTable(newSession(), propertiesTableName)).isEmpty();
SchemaTableName partitionsTableName = new SchemaTableName(tableName.getSchemaName(), format("%s$partitions", tableName.getTableName()));
assertThat(metadata.getSystemTable(newSession(), partitionsTableName)).isEmpty();
}

// Assert that table is hidden
Expand Down
2 changes: 1 addition & 1 deletion plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ protected void setup(String host, int port, String databaseName, boolean s3Selec
new NodeVersion("test_version"),
new NoneHiveRedirectionsProvider(),
ImmutableSet.of(
new PartitionsSystemTableProvider(hivePartitionManager),
new PartitionsSystemTableProvider(hivePartitionManager, TESTING_TYPE_MANAGER),
new PropertiesSystemTableProvider()),
new DefaultHiveMaterializedViewMetadataFactory(),
SqlStandardAccessControlMetadata::new,
Expand Down

0 comments on commit e239035

Please sign in to comment.