From 632eb585dccf539f9da6e90154762edb95a3a717 Mon Sep 17 00:00:00 2001 From: Ge Gao Date: Sun, 9 Jun 2024 19:54:40 -0700 Subject: [PATCH] [Verifier] Wrap checksum queries with partition predicate for reused output table For Insert and CTAS queries, when output table reuse is on, build the partition predicate and apply it when assembling the checksum queries. --- presto-verifier/pom.xml | 21 +++ .../verifier/checksum/ChecksumValidator.java | 13 +- .../verifier/framework/DataVerification.java | 4 +- .../framework/DeterminismAnalyzer.java | 2 +- .../framework/ExtendedVerification.java | 4 +- .../framework/QueryConfiguration.java | 25 +++- .../verifier/framework/QueryObjectBundle.java | 9 ++ .../verifier/rewrite/QueryRewriter.java | 124 +++++++++++++++--- .../VerificationQueryRewriterFactory.java | 5 + .../PrestoQuerySourceQuerySupplier.java | 13 +- .../presto/verifier/source/VerifierDao.java | 8 +- .../presto/verifier/VerifierTestUtil.java | 1 + .../checksum/TestChecksumValidator.java | 3 +- .../framework/AbstractVerificationTest.java | 17 ++- .../framework/TestDataVerification.java | 11 +- .../framework/TestDeterminismAnalyzer.java | 6 +- .../framework/TestQueryConfiguration.java | 26 ++-- .../framework/TestVerificationManager.java | 7 +- .../prestoaction/TestJdbcPrestoAction.java | 2 +- .../TestIgnoredFunctionsMismatchResolver.java | 1 + ...tTooManyOpenPartitionsFailureResolver.java | 1 + .../verifier/rewrite/TestQueryRewriter.java | 17 +-- .../TestPrestoQuerySourceQuerySupplier.java | 6 +- 23 files changed, 262 insertions(+), 64 deletions(-) diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index f2c6174b97eb..fd5bdd3b205a 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -29,11 +29,27 @@ + + com.facebook.presto + presto-hive + + + com.facebook.presto + presto-cache + + + + com.facebook.presto presto-hive-common + + com.facebook.presto + presto-hive-metastore + + com.facebook.presto presto-jdbc @@ -198,6 +214,11 @@ provided + + joda-time + joda-time + + com.facebook.presto diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java index 1e3c76d970e3..2caa6c5f86d6 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.verifier.checksum; +import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.GroupBy; import com.facebook.presto.sql.tree.GroupingElement; @@ -52,17 +53,17 @@ public ChecksumValidator(Map> columnValidato this.columnValidators = columnValidators; } - public Query generateChecksumQuery(QualifiedName tableName, List columns) + public Query generateChecksumQuery(QualifiedName tableName, List columns, Optional partitionPredicate) { ImmutableList.Builder selectItems = ImmutableList.builder(); selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of()))); for (Column column : columns) { selectItems.addAll(columnValidators.get(column.getCategory()).get().generateChecksumColumns(column)); } - return simpleQuery(new Select(false, selectItems.build()), new Table(tableName)); + return simpleQuery(new Select(false, selectItems.build()), new Table(tableName), partitionPredicate, Optional.empty()); } - public Query generatePartitionChecksumQuery(QualifiedName tableName, List dataColumns, List partitionColumns) + public Query generatePartitionChecksumQuery(QualifiedName tableName, List dataColumns, List partitionColumns, Optional partitionPredicate) { ImmutableList.Builder selectItems = ImmutableList.builder(); selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of()))); @@ -79,7 +80,7 @@ public Query generatePartitionChecksumQuery(QualifiedName tableName, List partitionColumns, List dataColumns) + public Query generateBucketChecksumQuery(QualifiedName tableName, List partitionColumns, List dataColumns, Optional partitionPredicate) { ImmutableList.Builder selectItems = ImmutableList.builder(); selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of()))); @@ -106,7 +107,7 @@ public Query generateBucketChecksumQuery(QualifiedName tableName, List p return simpleQuery( new Select(false, selectItems.build()), new Table(tableName), - Optional.empty(), + partitionPredicate, Optional.of(new GroupBy(false, groupByList.build())), Optional.empty(), Optional.of(new OrderBy(orderByList.build())), diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java index 1e9cf1b4e12d..73aba566d818 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java @@ -108,7 +108,7 @@ public DataMatchResult verify( ChecksumQueryContext testChecksumQueryContext) { List testColumns = getColumns(getHelperAction(), typeManager, test.getObjectName()); - Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getObjectName(), testColumns); + Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getObjectName(), testColumns, test.getPartitionsPredicate()); testChecksumQueryContext.setChecksumQuery(formatSql(testChecksumQuery)); List controlColumns = null; @@ -116,7 +116,7 @@ public DataMatchResult verify( if (isControlEnabled()) { controlColumns = getColumns(getHelperAction(), typeManager, control.getObjectName()); - Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getObjectName(), controlColumns); + Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getObjectName(), controlColumns, control.getPartitionsPredicate()); controlChecksumQueryContext.setChecksumQuery(formatSql(controlChecksumQuery)); QueryResult controlChecksum = callAndConsume( diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java index 3283aaa367be..dabf0121b3f1 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java @@ -144,7 +144,7 @@ private DeterminismAnalysis analyze(QueryObjectBundle control, ChecksumResult co stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setQueryId)); // Run checksum query - Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns); + Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns, Optional.empty()); ChecksumResult testChecksum = getOnlyElement(callAndConsume( () -> prestoAction.execute(checksumQuery, DETERMINISM_ANALYSIS_CHECKSUM, ChecksumResult::fromResultSet), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setChecksumQueryId)).getResults()); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java index c36dd5f52afb..fee119d50360 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java @@ -273,7 +273,7 @@ private List runPartitionChecksum( ChecksumQueryContext checksumQueryContext, QueryStage queryStage) { - Query partitionChecksumQuery = checksumValidator.generatePartitionChecksumQuery(bundle.getObjectName(), dataColumns, partitionColumns); + Query partitionChecksumQuery = checksumValidator.generatePartitionChecksumQuery(bundle.getObjectName(), dataColumns, partitionColumns, bundle.getPartitionsPredicate()); checksumQueryContext.setPartitionChecksumQuery(formatSql(partitionChecksumQuery)); return callAndConsume( () -> getHelperAction().execute(partitionChecksumQuery, queryStage, ChecksumResult::fromResultSet), @@ -287,7 +287,7 @@ private List runBucketChecksum( ChecksumQueryContext checksumQueryContext, QueryStage queryStage) { - Query bucketChecksumQuery = checksumValidator.generateBucketChecksumQuery(bundle.getObjectName(), partitionColumns, dataColumns); + Query bucketChecksumQuery = checksumValidator.generateBucketChecksumQuery(bundle.getObjectName(), partitionColumns, dataColumns, bundle.getPartitionsPredicate()); List checksumResults = callAndConsume( () -> getHelperAction().execute(bucketChecksumQuery, queryStage, ChecksumResult::fromResultSet), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(checksumQueryContext::setBucketChecksumQueryId)).getResults(); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java index 3ae5f0e2d771..eecee85523c8 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.verifier.framework; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.jdbi.v3.core.mapper.reflect.ColumnName; import org.jdbi.v3.core.mapper.reflect.JdbiConstructor; @@ -37,6 +38,7 @@ public class QueryConfiguration private final Optional password; private final Map sessionProperties; private final boolean isReusableTable; + private final List partitions; @JdbiConstructor public QueryConfiguration( @@ -45,9 +47,10 @@ public QueryConfiguration( @ColumnName("username") Optional username, @ColumnName("password") Optional password, @ColumnName("session_properties") Optional> sessionProperties, - @ColumnName("client_tags") Optional> clientTags) + @ColumnName("client_tags") Optional> clientTags, + @ColumnName("partitions") Optional> partitions) { - this(catalog, schema, username, password, sessionProperties, clientTags.filter(tags -> tags.contains(CLIENT_TAG_OUTPUT_RETAINED)).isPresent()); + this(catalog, schema, username, password, sessionProperties, clientTags.filter(tags -> tags.contains(CLIENT_TAG_OUTPUT_RETAINED)).isPresent(), partitions); } public QueryConfiguration( @@ -56,7 +59,8 @@ public QueryConfiguration( Optional username, Optional password, Optional> sessionProperties, - boolean isReusableTable) + boolean isReusableTable, + Optional> partitions) { this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); @@ -64,6 +68,7 @@ public QueryConfiguration( this.password = requireNonNull(password, "password is null"); this.sessionProperties = ImmutableMap.copyOf(sessionProperties.orElse(ImmutableMap.of())); this.isReusableTable = isReusableTable; + this.partitions = ImmutableList.copyOf(partitions.orElse(ImmutableList.of())); } public QueryConfiguration applyOverrides(QueryConfigurationOverrides overrides) @@ -85,7 +90,8 @@ public QueryConfiguration applyOverrides(QueryConfigurationOverrides overrides) Optional.ofNullable(overrides.getUsernameOverride().orElse(username.orElse(null))), Optional.ofNullable(overrides.getPasswordOverride().orElse(password.orElse(null))), Optional.of(sessionProperties), - isReusableTable); + isReusableTable, + Optional.of(partitions)); } public String getCatalog() @@ -118,6 +124,11 @@ public boolean isReusableTable() return isReusableTable; } + public List getPartitions() + { + return partitions; + } + @Override public boolean equals(Object obj) { @@ -133,13 +144,14 @@ public boolean equals(Object obj) Objects.equals(username, o.username) && Objects.equals(password, o.password) && Objects.equals(sessionProperties, o.sessionProperties) && - isReusableTable == o.isReusableTable; + isReusableTable == o.isReusableTable && + Objects.equals(partitions, partitions); } @Override public int hashCode() { - return Objects.hash(catalog, schema, username, password, sessionProperties, isReusableTable); + return Objects.hash(catalog, schema, username, password, sessionProperties, isReusableTable, partitions); } @Override @@ -152,6 +164,7 @@ public String toString() .add("password", password) .add("sessionProperties", sessionProperties) .add("isReusableTable", isReusableTable) + .add("partitions", partitions) .toString(); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java index 0465157956e2..96ebe68c277d 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Statement; @@ -25,6 +26,7 @@ public class QueryObjectBundle extends QueryBundle { private final QualifiedName objectName; + private final Optional partitionsPredicate; private final boolean reuseTable; private final Optional rewrittenFunctionCalls; @@ -35,11 +37,13 @@ public QueryObjectBundle( List teardownQueries, ClusterType cluster, Optional rewrittenFunctionCalls, + Optional partitionsPredicate, boolean reuseTable) { super(setupQueries, query, teardownQueries, cluster); this.objectName = requireNonNull(objectName, "objectName is null"); this.rewrittenFunctionCalls = requireNonNull(rewrittenFunctionCalls, "rewrittenFunctionCalls is null"); + this.partitionsPredicate = partitionsPredicate; this.reuseTable = reuseTable; } @@ -57,4 +61,9 @@ public boolean isReuseTable() { return reuseTable; } + + public Optional getPartitionsPredicate() + { + return partitionsPredicate; + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java index dbd704853583..c3b9e6e694c2 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.verifier.rewrite; +import com.facebook.presto.common.block.BlockEncodingSerde; +import com.facebook.presto.common.predicate.NullableValue; import com.facebook.presto.common.type.ArrayType; import com.facebook.presto.common.type.DecimalType; import com.facebook.presto.common.type.MapType; @@ -22,8 +24,11 @@ import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.common.type.TypeSignatureParameter; import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.planner.LiteralEncoder; import com.facebook.presto.sql.tree.AllColumns; import com.facebook.presto.sql.tree.Cast; +import com.facebook.presto.sql.tree.ComparisonExpression; +import com.facebook.presto.sql.tree.ComparisonExpression.Operator; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateTableAsSelect; import com.facebook.presto.sql.tree.CreateView; @@ -32,7 +37,9 @@ import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Identifier; import com.facebook.presto.sql.tree.Insert; +import com.facebook.presto.sql.tree.IsNullPredicate; import com.facebook.presto.sql.tree.LikeClause; +import com.facebook.presto.sql.tree.LogicalBinaryExpression; import com.facebook.presto.sql.tree.Property; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Query; @@ -43,6 +50,7 @@ import com.facebook.presto.sql.tree.SingleColumn; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.verifier.framework.ClusterType; +import com.facebook.presto.verifier.framework.Column; import com.facebook.presto.verifier.framework.QueryConfiguration; import com.facebook.presto.verifier.framework.QueryException; import com.facebook.presto.verifier.framework.QueryObjectBundle; @@ -53,6 +61,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; import org.intellij.lang.annotations.Language; +import org.joda.time.DateTimeZone; import java.sql.ResultSetMetaData; import java.util.ArrayList; @@ -61,6 +70,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TimeZone; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -74,9 +84,12 @@ import static com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.common.type.UnknownType.UNKNOWN; import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.hive.HiveUtil.parsePartitionValue; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionNamesAndValues; import static com.facebook.presto.sql.tree.LikeClause.PropertiesOption.INCLUDING; import static com.facebook.presto.sql.tree.ShowCreate.Type.VIEW; import static com.facebook.presto.verifier.framework.CreateViewVerification.SHOW_CREATE_VIEW_CONVERTER; +import static com.facebook.presto.verifier.framework.DataVerificationUtil.getColumns; import static com.facebook.presto.verifier.framework.QueryStage.REWRITE; import static com.facebook.presto.verifier.framework.VerifierUtil.PARSING_OPTIONS; import static com.facebook.presto.verifier.framework.VerifierUtil.getColumnNames; @@ -96,6 +109,7 @@ public class QueryRewriter { private final SqlParser sqlParser; private final TypeManager typeManager; + private final BlockEncodingSerde blockEncodingSerde; private final PrestoAction prestoAction; private final Map prefixes; private final Map> tableProperties; @@ -105,17 +119,19 @@ public class QueryRewriter public QueryRewriter( SqlParser sqlParser, TypeManager typeManager, + BlockEncodingSerde blockEncodingSerde, PrestoAction prestoAction, Map tablePrefixes, Map> tableProperties, Map reuseTables) { - this(sqlParser, typeManager, prestoAction, tablePrefixes, tableProperties, reuseTables, ImmutableMultimap.of()); + this(sqlParser, typeManager, blockEncodingSerde, prestoAction, tablePrefixes, tableProperties, reuseTables, ImmutableMultimap.of()); } public QueryRewriter( SqlParser sqlParser, TypeManager typeManager, + BlockEncodingSerde blockEncodingSerde, PrestoAction prestoAction, Map tablePrefixes, Map> tableProperties, @@ -124,6 +140,7 @@ public QueryRewriter( { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerge"); this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); this.prefixes = ImmutableMap.copyOf(tablePrefixes); this.tableProperties = ImmutableMap.copyOf(tableProperties); @@ -156,14 +173,18 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig functionSubstitutions = rewriterResult.getSubstitutions(); } if (shouldReuseTable && !functionSubstitutions.isPresent()) { - return new QueryObjectBundle( - createTableAsSelect.getName(), - ImmutableList.of(), - createTableAsSelect, - ImmutableList.of(), - clusterType, - Optional.empty(), - true); + Optional partitionsPredicate = getPartitionsPredicate(createTableAsSelect.getName(), queryConfiguration.getPartitions()); + if (partitionsPredicate.isPresent()) { + return new QueryObjectBundle( + createTableAsSelect.getName(), + ImmutableList.of(), + createTableAsSelect, + ImmutableList.of(), + clusterType, + Optional.empty(), + partitionsPredicate, + true); + } } QualifiedName temporaryTableName = generateTemporaryName(Optional.of(createTableAsSelect.getName()), prefix); @@ -181,6 +202,7 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, functionSubstitutions, + Optional.empty(), false); } if (statement instanceof Insert) { @@ -195,14 +217,18 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig functionSubstitutions = rewriterResult.getSubstitutions(); } if (shouldReuseTable && !functionSubstitutions.isPresent()) { - return new QueryObjectBundle( - originalTableName, - ImmutableList.of(), - insert, - ImmutableList.of(), - clusterType, - Optional.empty(), - true); + Optional partitionsPredicate = getPartitionsPredicate(originalTableName, queryConfiguration.getPartitions()); + if (partitionsPredicate.isPresent()) { + return new QueryObjectBundle( + originalTableName, + ImmutableList.of(), + insert, + ImmutableList.of(), + clusterType, + Optional.empty(), + partitionsPredicate, + true); + } } QualifiedName temporaryTableName = generateTemporaryName(Optional.of(originalTableName), prefix); @@ -222,6 +248,7 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, functionSubstitutions, + Optional.empty(), false); } if (statement instanceof Query) { @@ -252,6 +279,7 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, functionSubstitutions, + Optional.empty(), false); } if (statement instanceof CreateView) { @@ -288,6 +316,7 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig ImmutableList.of(new DropView(temporaryViewName, true)), clusterType, Optional.empty(), + Optional.empty(), false); } if (statement instanceof CreateTable) { @@ -305,6 +334,7 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, Optional.empty(), + Optional.empty(), false); } @@ -480,4 +510,64 @@ private static List applyPropertyOverride(List properties, L .map(entry -> new Property(new Identifier(entry.getKey()), entry.getValue())) .collect(toImmutableList()); } + + private Optional getPartitionsPredicate(QualifiedName tableName, List partitions) + { + if (partitions.isEmpty()) { + return Optional.empty(); + } + + List columns = getColumns(prestoAction, typeManager, tableName); + + Expression disjunct = null; + for (String partition : partitions) { + Optional conjunct = Optional.empty(); + try { + conjunct = getPartitionConjunct(partition, columns); + } + catch (Exception e) { + } + if (!conjunct.isPresent()) { + return Optional.empty(); + } + disjunct = disjunct == null ? conjunct.get() : new LogicalBinaryExpression(LogicalBinaryExpression.Operator.OR, disjunct, conjunct.get()); + } + + return Optional.ofNullable(disjunct); + } + + private Optional getPartitionConjunct(String partition, List columns) + { + Expression conjunct = null; + Map partitionRawKeyValues = toPartitionNamesAndValues(partition); + // TryCatch + for (String partitionKey : partitionRawKeyValues.keySet()) { + Type type = null; + for (Column column : columns) { + if (column.getName().equals(partitionKey)) { + type = column.getType(); + break; + } + } + if (type == null) { + // LOG + return Optional.empty(); + } + + NullableValue partitionValue = parsePartitionValue(partitionKey, partitionRawKeyValues.get(partitionKey), type, DateTimeZone.forTimeZone(TimeZone.getDefault())); + + Expression equalPredicate = null; + if (partitionValue.isNull()) { + equalPredicate = new IsNullPredicate(new Identifier(partitionKey)); + } + else { + LiteralEncoder literalEncoder = new LiteralEncoder(blockEncodingSerde); + equalPredicate = new ComparisonExpression(Operator.EQUAL, new Identifier(partitionKey), + literalEncoder.toExpression(partitionValue.getValue(), partitionValue.getType(), false)); + } + conjunct = conjunct == null ? equalPredicate : new LogicalBinaryExpression(LogicalBinaryExpression.Operator.AND, conjunct, equalPredicate); + } + + return Optional.ofNullable(conjunct); + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java index 8b8d0ff8f40e..19a2ca0d353e 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.verifier.rewrite; +import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.tree.BooleanLiteral; @@ -48,6 +49,7 @@ public class VerificationQueryRewriterFactory { private final SqlParser sqlParser; private final TypeManager typeManager; + private final BlockEncodingSerde blockEncodingSerde; private final QualifiedName controlTablePrefix; private final QualifiedName testTablePrefix; private final List controlTableProperties; @@ -61,12 +63,14 @@ public class VerificationQueryRewriterFactory public VerificationQueryRewriterFactory( SqlParser sqlParser, TypeManager typeManager, + BlockEncodingSerde blockEncodingSerde, @ForControl QueryRewriteConfig controlConfig, @ForTest QueryRewriteConfig testConfig, VerifierConfig verifierConfig) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerge is null"); this.controlTablePrefix = requireNonNull(controlConfig.getTablePrefix(), "controlTablePrefix is null"); this.testTablePrefix = requireNonNull(testConfig.getTablePrefix(), "testTablePrefix is null"); this.controlTableProperties = constructProperties(controlConfig.getTableProperties()); @@ -82,6 +86,7 @@ public QueryRewriter create(PrestoAction prestoAction) return new QueryRewriter( sqlParser, typeManager, + blockEncodingSerde, prestoAction, ImmutableMap.of(CONTROL, controlTablePrefix, TEST, testTablePrefix), ImmutableMap.of(CONTROL, controlTableProperties, TEST, testTableProperties), diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java index 1791c29f4ed5..dbccf93b0499 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java @@ -51,6 +51,8 @@ public class PrestoQuerySourceQuerySupplier Optional.ofNullable(resultSet.getString("control_session_properties")) .map(StringToStringMapColumnMapper.CODEC::fromJson), Optional.ofNullable(resultSet.getString("control_client_tags")) + .map(StringListColumnMapper.CODEC::fromJson), + Optional.ofNullable(resultSet.getString("control_partitions")) .map(StringListColumnMapper.CODEC::fromJson)), new QueryConfiguration( resultSet.getString("test_catalog"), @@ -60,6 +62,8 @@ public class PrestoQuerySourceQuerySupplier Optional.ofNullable(resultSet.getString("test_session_properties")) .map(StringToStringMapColumnMapper.CODEC::fromJson), Optional.ofNullable(resultSet.getString("test_client_tags")) + .map(StringListColumnMapper.CODEC::fromJson), + Optional.ofNullable(resultSet.getString("test_partitions")) .map(StringListColumnMapper.CODEC::fromJson)))); private final PrestoAction helperAction; @@ -73,7 +77,14 @@ public PrestoQuerySourceQuerySupplier( PrestoQuerySourceQueryConfig config) { this.helperAction = helperActionFactory.create( - new QueryConfiguration(config.getCatalog(), config.getSchema(), config.getUsername(), config.getPassword(), Optional.empty(), Optional.empty()), + new QueryConfiguration( + config.getCatalog(), + config.getSchema(), + config.getUsername(), + config.getPassword(), + Optional.empty(), + Optional.empty(), + Optional.empty()), VerificationContext.create("", "")); this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.query = requireNonNull(config.getQuery(), "query is null"); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java index ef088b2ce5e5..5908e1f61c0c 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java @@ -40,6 +40,7 @@ public interface VerifierDao " control_password varchar(256) DEFAULT NULL,\n" + " control_session_properties text DEFAULT NULL,\n" + " control_client_tags text DEFAULT NULL,\n" + + " control_partitions text DEFAULT NULL,\n" + " test_catalog varchar(256) NOT NULL,\n" + " test_schema varchar(256) NOT NULL,\n" + " test_query text NOT NULL,\n" + @@ -47,7 +48,8 @@ public interface VerifierDao " test_username varchar(256) DEFAULT NULL,\n" + " test_password varchar(256) DEFAULT NULL,\n" + " test_session_properties text DEFAULT NULL,\n" + - " test_client_tags text DEFAULT NULL)") + " test_client_tags text DEFAULT NULL,\n" + + " test_partitions text DEFAULT NULL)") void createVerifierQueriesTable(@Define("table_name") String tableName); @SqlQuery("SELECT\n" + @@ -61,6 +63,7 @@ public interface VerifierDao " control_password,\n" + " control_session_properties,\n" + " control_client_tags,\n" + + " control_partitions,\n" + " test_query,\n" + " test_query_id,\n" + " test_catalog,\n" + @@ -68,7 +71,8 @@ public interface VerifierDao " test_username,\n" + " test_password,\n" + " test_session_properties,\n" + - " test_client_tags\n" + + " test_client_tags,\n" + + " test_partitions\n" + "FROM\n" + " \n" + "WHERE\n" + diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java index 4da570f0e7fb..9663a24c52bc 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java @@ -72,6 +72,7 @@ public class VerifierTestUtil ImmutableList.of(), CONTROL, Optional.empty(), + Optional.empty(), false); private static final MySqlOptions MY_SQL_OPTIONS = MySqlOptions.builder() diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java index b4aee00bf709..0aaab41f6d0a 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java @@ -123,7 +123,8 @@ public void testChecksumQuery() MAP_COLUMN, MAP_FLOAT_NON_FLOAT_COLUMN, MAP_NON_ORDERABLE_COLUMN, - ROW_COLUMN)); + ROW_COLUMN), + Optional.empty()); Statement expectedChecksumQuery = sqlParser.createStatement( "SELECT\n" + " \"count\"(*)\n" + diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java index d6b90e425ef5..017f176ca8cc 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java @@ -15,6 +15,8 @@ import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.presto.common.block.BlockEncodingManager; +import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; @@ -62,7 +64,8 @@ public abstract class AbstractVerificationTest protected static final String SUITE = "test-suite"; protected static final String NAME = "test-query"; protected static final String TEST_ID = "test-id"; - protected static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), true); + protected static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), + Optional.empty(), true, Optional.empty()); protected static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build(); protected static final String CONTROL_TABLE_PREFIX = "tmp_verifier_c"; protected static final String TEST_TABLE_PREFIX = "tmp_verifier_t"; @@ -78,6 +81,7 @@ public abstract class AbstractVerificationTest private final Injector injector; private final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); + private final BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(); private final PrestoExceptionClassifier exceptionClassifier = PrestoExceptionClassifier.defaultBuilder().build(); private final DeterminismAnalyzerConfig determinismAnalyzerConfig = new DeterminismAnalyzerConfig().setMaxAnalysisRuns(3).setRunTeardown(true); private final FailureResolverManagerFactory failureResolverManagerFactory; @@ -134,6 +138,11 @@ protected SourceQuery getSourceQuery(String controlQuery, String testQuery, Stri return new SourceQuery(SUITE, NAME, controlQuery, testQuery, Optional.of(controlQueryId), Optional.of(testQueryId), QUERY_CONFIGURATION, QUERY_CONFIGURATION); } + protected SourceQuery getSourceQuery(String controlQuery, String testQuery, String controlQueryId, String testQueryId, QueryConfiguration controlQueryConfiguration, QueryConfiguration testQueryConfiguration) + { + return new SourceQuery(SUITE, NAME, controlQuery, testQuery, Optional.of(controlQueryId), Optional.of(testQueryId), controlQueryConfiguration, testQueryConfiguration); + } + protected Optional runExplain(String controlQuery, String testQuery) { return verify(getSourceQuery(controlQuery, testQuery), true, Optional.empty(), Optional.empty()); @@ -159,6 +168,11 @@ protected Optional runVerification(String controlQuery, Stri return verify(getSourceQuery(controlQuery, testQuery, controlQueryId, testQueryId), false, Optional.empty(), Optional.of(settings)); } + protected Optional runVerification(String controlQuery, String testQuery, String controlQueryId, String testQueryId, QueryConfiguration controlQueryConfiguration, QueryConfiguration testQueryConfiguration, VerificationSettings settings) + { + return verify(getSourceQuery(controlQuery, testQuery, controlQueryId, testQueryId, controlQueryConfiguration, testQueryConfiguration), false, Optional.empty(), Optional.of(settings)); + } + protected Optional verify(SourceQuery sourceQuery, boolean explain) { return verify(sourceQuery, explain, Optional.empty(), Optional.empty()); @@ -216,6 +230,7 @@ private Optional verify( QueryRewriterFactory queryRewriterFactory = new VerificationQueryRewriterFactory( sqlParser, typeManager, + blockEncodingSerde, controlRewriteConfig, testRewriteConfig, verifierConfig); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java index 90858af58f69..29467e3de3fb 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java @@ -18,6 +18,7 @@ import com.facebook.presto.verifier.event.VerifierQueryEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -226,9 +227,13 @@ public void testReuseTable() getQueryRunner().execute(controlQuery); String controlQueryId = "control_query_id"; - Optional event = runVerification(testQuery, controlQuery, controlQueryId, testQueryId, reuseTableSettings); + Optional event = runVerification(testQuery, controlQuery, controlQueryId, testQueryId, + new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), + Optional.empty(), true, Optional.of(ImmutableList.of("test_column=1"))), + new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), + Optional.empty(), true, Optional.empty()), reuseTableSettings); assertTrue(event.get().getControlQueryInfo().getIsReuseTable()); - assertTrue(event.get().getTestQueryInfo().getIsReuseTable()); + assertFalse(event.get().getTestQueryInfo().getIsReuseTable()); assertEvent(event.get(), SUCCEEDED, Optional.empty(), Optional.empty(), Optional.empty()); } @@ -377,7 +382,7 @@ public void testChecksumQueryCompilerError() public void testExecutionTimeSessionProperty() { QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of(QUERY_MAX_EXECUTION_TIME, - "20m")), Optional.empty()); + "20m")), Optional.empty(), Optional.empty()); SourceQuery sourceQuery = new SourceQuery(SUITE, NAME, "SELECT 1.0", "SELECT 1.00001", Optional.empty(), Optional.empty(), configuration, configuration); Optional event = verify(sourceQuery, false); assertTrue(event.isPresent()); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java index 3fd179d4c783..71c6784911ea 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.presto.common.block.BlockEncodingManager; +import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; @@ -50,6 +52,7 @@ public class TestDeterminismAnalyzer private static final String SUITE = "test-suite"; private static final String NAME = "test-query"; private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); + private static final BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(); @Test public void testMutableCatalog() @@ -66,7 +69,7 @@ private static boolean isMutableCatalogReferenced(DeterminismAnalyzer determinis private static DeterminismAnalyzer createDeterminismAnalyzer(String mutableCatalogPattern) { - QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty()); + QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); VerificationContext verificationContext = VerificationContext.create(SUITE, NAME); VerifierConfig verifierConfig = new VerifierConfig().setTestId("test-id"); RetryConfig retryConfig = new RetryConfig(); @@ -86,6 +89,7 @@ private static DeterminismAnalyzer createDeterminismAnalyzer(String mutableCatal QueryRewriter queryRewriter = new QueryRewriter( sqlParser, typeManager, + blockEncodingSerde, prestoAction, ImmutableMap.of(CONTROL, QualifiedName.of("tmp_verifier_c"), TEST, QualifiedName.of("tmp_verifier_t")), ImmutableMap.of(), diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java index f03b305fde5f..1cba3d981c04 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java @@ -50,15 +50,17 @@ public class TestQueryConfiguration private static final List CLIENT_TAGS = ImmutableList.of(QueryConfiguration.CLIENT_TAG_OUTPUT_RETAINED); private static final QueryConfiguration CONFIGURATION_1 = new QueryConfiguration(CATALOG, SCHEMA, Optional.of(USERNAME), Optional.of(PASSWORD), - Optional.of(SESSION_PROPERTIES), Optional.of(CLIENT_TAGS)); - private static final QueryConfiguration CONFIGURATION_2 = new QueryConfiguration(CATALOG, SCHEMA, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + Optional.of(SESSION_PROPERTIES), Optional.of(CLIENT_TAGS), Optional.empty()); + private static final QueryConfiguration CONFIGURATION_2 = new QueryConfiguration(CATALOG, SCHEMA, Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty(), Optional.empty()); private static final QueryConfiguration CONFIGURATION_FULL_OVERRIDE = new QueryConfiguration( CATALOG_OVERRIDE, SCHEMA_OVERRIDE, Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(SESSION_PROPERTIES_OVERRIDE), - Optional.of(CLIENT_TAGS)); + Optional.of(CLIENT_TAGS), + Optional.empty()); private QueryConfigurationOverridesConfig overrides; @@ -91,7 +93,8 @@ public void testOverrides() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(SESSION_PROPERTIES), - Optional.of(CLIENT_TAGS))); + Optional.of(CLIENT_TAGS), + Optional.empty())); assertEquals(CONFIGURATION_2.applyOverrides(overrides), new QueryConfiguration( CATALOG_OVERRIDE, @@ -99,6 +102,7 @@ public void testOverrides() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.empty(), + Optional.empty(), Optional.empty())); } @@ -113,6 +117,7 @@ public void testSessionPropertyOverride() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(SESSION_PROPERTIES_OVERRIDE), + false, Optional.empty()); assertEquals(CONFIGURATION_2.applyOverrides(overrides), overridden); } @@ -127,7 +132,8 @@ public void testSessionPropertySubstitute() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(ImmutableMap.of("property_1", "value_x", "property_2", "value_2", "property_3", "value_3")), - Optional.of(CLIENT_TAGS)); + Optional.of(CLIENT_TAGS), + Optional.empty()); assertEquals(CONFIGURATION_1.applyOverrides(overrides), substituted1); @@ -137,6 +143,7 @@ public void testSessionPropertySubstitute() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(SESSION_PROPERTIES_OVERRIDE), + false, Optional.empty()); assertEquals(CONFIGURATION_2.applyOverrides(overrides), substituted2); } @@ -153,7 +160,8 @@ public void testSessionPropertyRemovalWithOverrides() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(ImmutableMap.of("property_3", "value_3")), - Optional.of(CLIENT_TAGS)); + Optional.of(CLIENT_TAGS), + Optional.empty()); assertEquals(CONFIGURATION_1.applyOverrides(overrides), removed); } @@ -169,7 +177,8 @@ public void testSessionPropertySubstituteAndRemove() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(SESSION_PROPERTIES_OVERRIDE), - Optional.of(CLIENT_TAGS)); + Optional.of(CLIENT_TAGS), + Optional.empty()); assertEquals(CONFIGURATION_1.applyOverrides(overrides), removed); } @@ -185,7 +194,8 @@ public void testSessionPropertyRemoval() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(ImmutableMap.of("property_1", "value_1")), - Optional.of(CLIENT_TAGS)); + Optional.of(CLIENT_TAGS), + Optional.empty()); assertEquals(CONFIGURATION_1.applyOverrides(overrides), removed); } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java index 4361fe751879..6c67276c870c 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java @@ -14,6 +14,8 @@ package com.facebook.presto.verifier.framework; import com.facebook.airlift.event.client.AbstractEventClient; +import com.facebook.presto.common.block.BlockEncodingManager; +import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; @@ -115,7 +117,8 @@ public List getEvents() private static final String NAME = "test-query"; private static final QualifiedName TABLE_PREFIX = QualifiedName.of("tmp_verifier"); private static final SqlParser SQL_PARSER = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(AT_SIGN, COLON)); - private static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration("test", "di", Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty()); + private static final BlockEncodingSerde BLOCK_ENCODING_SERDE = new BlockEncodingManager(); + private static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration("test", "di", Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); private static final SourceQuery SOURCE_QUERY = new SourceQuery( SUITE, NAME, @@ -218,7 +221,7 @@ private VerificationManager getVerificationManager(List sourceQueri new VerificationFactory( SQL_PARSER, (sourceQuery, verificationContext) -> new QueryActions(prestoAction, prestoAction, prestoAction), - presto -> new QueryRewriter(SQL_PARSER, createTypeManager(), presto, ImmutableMap.of(CONTROL, TABLE_PREFIX, TEST, TABLE_PREFIX), + presto -> new QueryRewriter(SQL_PARSER, createTypeManager(), BLOCK_ENCODING_SERDE, presto, ImmutableMap.of(CONTROL, TABLE_PREFIX, TEST, TABLE_PREFIX), ImmutableMap.of(), ImmutableMap.of(CONTROL, false, TEST, false)), new FailureResolverManagerFactory(ImmutableSet.of(), ImmutableSet.of()), diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java index 7869b152a91d..a965b2c919b9 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java @@ -56,7 +56,7 @@ public class TestJdbcPrestoAction private static final String SUITE = "test-suite"; private static final String NAME = "test-query"; private static final QueryStage QUERY_STAGE = CONTROL_MAIN; - private static final QueryConfiguration CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty()); + private static final QueryConfiguration CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); private static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DECIMAL).build(); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java index e6b749a1dc23..3321c011777f 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java @@ -69,6 +69,7 @@ private static QueryObjectBundle createBundle(String query) ImmutableList.of(), CONTROL, Optional.empty(), + Optional.empty(), false); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java index 827c8a892970..858f75152436 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java @@ -89,6 +89,7 @@ public QueryResult execute(Statement statement, QueryStage queryStage, Re ImmutableList.of(), TEST, Optional.empty(), + Optional.empty(), false); private static final QueryException HIVE_TOO_MANY_OPEN_PARTITIONS_EXCEPTION = new PrestoQueryException( new RuntimeException(), diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java index 95b060e73b81..7bef00a47c23 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.verifier.rewrite; +import com.facebook.presto.common.block.BlockEncodingManager; +import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; @@ -76,6 +78,7 @@ public class TestQueryRewriter Optional.of("user"), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); private static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build(); private static final QueryRewriteConfig QUERY_REWRITE_CONFIG = new QueryRewriteConfig() @@ -83,6 +86,7 @@ public class TestQueryRewriter .setTableProperties("{\"p_int\": 30, \"p_long\": 4294967297, \"p_double\": 1.5, \"p_varchar\": \"test\", \"p_bool\": true}"); private static final VerifierConfig VERIFIER_CONFIG = new VerifierConfig(); private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); + private static final BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(); private static StandaloneQueryRunner queryRunner; private static PrestoAction prestoAction; @@ -583,14 +587,11 @@ public void testRewriteFunctionCalls() @Test public void testReuseTableRewrite() { - String query = "INSERT INTO dest_table SELECT * FROM test_table"; + String query = "CREATE TABLE dest_table AS SELECT a, b FROM test_table WHERE a = 1 AND b = 'xyz'"; + queryRunner.execute(query); + List partitions = ImmutableList.of("a=1", "b=xyz"); QueryConfiguration configuration = new QueryConfiguration( - CATALOG, - SCHEMA, - Optional.of("user"), - Optional.empty(), - Optional.empty(), - true); + CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), true, Optional.of(partitions)); assertShadowed( getQueryRewriter(new QueryRewriteConfig().setReuseTable(true), VERIFIER_CONFIG), query, @@ -674,6 +675,6 @@ private QueryRewriter getQueryRewriter() private QueryRewriter getQueryRewriter(QueryRewriteConfig rewriteConfig, VerifierConfig verifierConfig) { - return new VerificationQueryRewriterFactory(sqlParser, createTypeManager(), rewriteConfig, rewriteConfig, verifierConfig).create(prestoAction); + return new VerificationQueryRewriterFactory(sqlParser, createTypeManager(), blockEncodingSerde, rewriteConfig, rewriteConfig, verifierConfig).create(prestoAction); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java index 3127831975ef..96860c7ec6cb 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java @@ -60,6 +60,7 @@ public class TestPrestoQuerySourceQuerySupplier " 'user' control_username,\n" + " '{\"a\": \"b\"}' control_session_properties,\n" + " '[\"x\"]' control_client_tags,\n" + + " NULL control_partitions,\n" + " query_id control_query_id,\n" + " NULL control_password,\n" + " query test_query,\n" + @@ -69,6 +70,7 @@ public class TestPrestoQuerySourceQuerySupplier " NULL test_password,\n" + " '{\"c\": \"d\"}' test_session_properties,\n" + " '[\"y\"]' test_client_tags,\n" + + " NULL test_partitions,\n" + " query_id test_query_id\n" + "FROM (\n" + " VALUES\n" + @@ -76,9 +78,9 @@ public class TestPrestoQuerySourceQuerySupplier " ('Q2', 'INSERT INTO test_table SELECT 1', 'T2')\n" + ") queries(name, query, query_id)"; private static final QueryConfiguration CONTROL_CONFIGURATION = new QueryConfiguration( - "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("a", "b")), Optional.of(ImmutableList.of("x"))); + "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("a", "b")), Optional.of(ImmutableList.of("x")), Optional.empty()); private static final QueryConfiguration TEST_CONFIGURATION = new QueryConfiguration( - "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("c", "d")), Optional.of(ImmutableList.of("y"))); + "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("c", "d")), Optional.of(ImmutableList.of("y")), Optional.empty()); private static final List SOURCE_QUERIES = ImmutableList.of( new SourceQuery("test", "Q1", "SELECT 1", "SELECT 1", Optional.of("T1"), Optional.of("T1"), CONTROL_CONFIGURATION, TEST_CONFIGURATION), new SourceQuery("test", "Q2", "INSERT INTO test_table SELECT 1", "INSERT INTO test_table SELECT 1", Optional.of("T2"), Optional.of("T2"), CONTROL_CONFIGURATION,