Skip to content

Commit

Permalink
Increase max writer version to 4 for delta lake
Browse files Browse the repository at this point in the history
  • Loading branch information
@homar
homar committed Oct 11, 2022
1 parent 2496fac commit 863ed73
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 12 deletions.
2 changes: 1 addition & 1 deletion plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ public class DeltaLakeMetadata
// The required writer version used by tables created by Trino
private static final int WRITER_VERSION = 2;
// The highest writer version Trino supports writing to
private static final int MAX_WRITER_VERSION = 3;
private static final int MAX_WRITER_VERSION = 4;
// Matches the dummy column Databricks stores in the metastore
private static final List<Column> DUMMY_DATA_COLUMNS = ImmutableList.of(
new Column("col", HiveType.toHiveType(new ArrayType(VarcharType.createUnboundedVarcharType())), Optional.empty()));
Expand Down
12 changes: 6 additions & 6 deletions .../src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,14 @@ public void testAddColumnUnsupportedWriterVersion()
onDelta().executeQuery(format("" +
"CREATE TABLE default.%s (col int) " +
"USING DELTA LOCATION 's3://%s/%s'" +
"TBLPROPERTIES ('delta.minWriterVersion'='4')",
"TBLPROPERTIES ('delta.minWriterVersion'='5')",
tableName,
bucketName,
tableDirectory));

try {
assertQueryFailure(() -> onTrino().executeQuery("ALTER TABLE delta.default." + tableName + " ADD COLUMN new_col int"))
.hasMessageMatching(".* Table .* requires Delta Lake writer version 4 which is not supported");
.hasMessageMatching(".* Table .* requires Delta Lake writer version 5 which is not supported");
}
finally {
onDelta().executeQuery("DROP TABLE default." + tableName);
Expand Down Expand Up @@ -180,14 +180,14 @@ public void testCommentOnTableUnsupportedWriterVersion()
onDelta().executeQuery(format("" +
"CREATE TABLE default.%s (col int) " +
"USING DELTA LOCATION 's3://%s/%s'" +
"TBLPROPERTIES ('delta.minWriterVersion'='4')",
"TBLPROPERTIES ('delta.minWriterVersion'='5')",
tableName,
bucketName,
tableDirectory));

try {
assertQueryFailure(() -> onTrino().executeQuery("COMMENT ON TABLE delta.default." + tableName + " IS 'test comment'"))
.hasMessageMatching(".* Table .* requires Delta Lake writer version 4 which is not supported");
.hasMessageMatching(".* Table .* requires Delta Lake writer version 5 which is not supported");
}
finally {
onTrino().executeQuery("DROP TABLE delta.default." + tableName);
Expand Down Expand Up @@ -224,14 +224,14 @@ public void testCommentOnColumnUnsupportedWriterVersion()
onDelta().executeQuery(format("" +
"CREATE TABLE default.%s (col int) " +
"USING DELTA LOCATION 's3://%s/%s'" +
"TBLPROPERTIES ('delta.minWriterVersion'='4')",
"TBLPROPERTIES ('delta.minWriterVersion'='5')",
tableName,
bucketName,
tableDirectory));

try {
assertQueryFailure(() -> onTrino().executeQuery("COMMENT ON COLUMN delta.default." + tableName + ".col IS 'test column comment'"))
.hasMessageMatching(".* Table .* requires Delta Lake writer version 4 which is not supported");
.hasMessageMatching(".* Table .* requires Delta Lake writer version 5 which is not supported");
}
finally {
onTrino().executeQuery("DROP TABLE delta.default." + tableName);
Expand Down
11 changes: 6 additions & 5 deletions ...ain/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksInsertCompatibility.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,11 +368,11 @@ public void testGeneratedColumnsCompatibility()
.containsOnly(row(1, 2), row(2, 4), row(3, 6));

assertQueryFailure(() -> onTrino().executeQuery("INSERT INTO delta.default." + tableName + " VALUES (1, 2, 4)"))
.hasMessageMatching(".*Table default." + tableName + " requires Delta Lake writer version 4 which is not supported");
.hasMessageMatching(".*Writing to tables with generated columns is not supported");
assertQueryFailure(() -> onTrino().executeQuery("DELETE FROM delta.default." + tableName + " WHERE a_number = 1"))
.hasMessageMatching(".*Table default." + tableName + " requires Delta Lake writer version 4 which is not supported");
.hasMessageMatching(".*Writing to tables with generated columns is not supported");
assertQueryFailure(() -> onTrino().executeQuery("UPDATE delta.default." + tableName + " SET a_number_times_two = 5 WHERE a_number = 2"))
.hasMessageMatching(".*Table default." + tableName + " requires Delta Lake writer version 4 which is not supported");
.hasMessageMatching(".*Writing to tables with generated columns is not supported");
}
finally {
onDelta().executeQuery("DROP TABLE default." + tableName);
Expand Down Expand Up @@ -552,15 +552,16 @@ public void testWritesToTableWithCDFFails()
"LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'" +
"TBLPROPERTIES (delta.enableChangeDataFeed = true)");

assertQueryFailure(() -> onTrino().executeQuery("INSERT INTO delta.default." + tableName + " VALUES (1, 2)"))
.hasMessageMatching(".* Table .* requires Delta Lake writer version 4 which is not supported");
onTrino().executeQuery("INSERT INTO delta.default." + tableName + " VALUES (1, 2)");
assertQueryFailure(() -> onTrino().executeQuery("UPDATE delta.default." + tableName + " SET a = 3 WHERE b = 3"))
.hasMessageContaining("Writing to tables with Change Data Feed enabled is not supported");
assertQueryFailure(() -> onTrino().executeQuery("DELETE FROM delta.default." + tableName + " WHERE a = 3"))
.hasMessageContaining("Writing to tables with Change Data Feed enabled is not supported");
assertQueryFailure(() -> onTrino().executeQuery("MERGE INTO delta.default." + tableName + " t USING delta.default." + tableName + " s " +
"ON (t.a = s.a) WHEN MATCHED THEN UPDATE SET b = 42"))
.hasMessageContaining("Writing to tables with Change Data Feed enabled is not supported");
assertThat(onTrino().executeQuery("SELECT * FROM delta.default." + tableName))
.containsOnly(row(1, 2));
}
finally {
onDelta().executeQuery("DROP TABLE IF EXISTS default." + tableName);
Expand Down

0 comments on commit 863ed73

Please sign in to comment.