diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index f2c6174b97eb..fd5bdd3b205a 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -29,11 +29,27 @@ + + com.facebook.presto + presto-hive + + + com.facebook.presto + presto-cache + + + + com.facebook.presto presto-hive-common + + com.facebook.presto + presto-hive-metastore + + com.facebook.presto presto-jdbc @@ -198,6 +214,11 @@ provided + + joda-time + joda-time + + com.facebook.presto diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java index 1e3c76d970e3..2caa6c5f86d6 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.verifier.checksum; +import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.GroupBy; import com.facebook.presto.sql.tree.GroupingElement; @@ -52,17 +53,17 @@ public ChecksumValidator(Map> columnValidato this.columnValidators = columnValidators; } - public Query generateChecksumQuery(QualifiedName tableName, List columns) + public Query generateChecksumQuery(QualifiedName tableName, List columns, Optional partitionPredicate) { ImmutableList.Builder selectItems = ImmutableList.builder(); selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of()))); for (Column column : columns) { selectItems.addAll(columnValidators.get(column.getCategory()).get().generateChecksumColumns(column)); } - return simpleQuery(new Select(false, selectItems.build()), new Table(tableName)); + return simpleQuery(new Select(false, selectItems.build()), new Table(tableName), partitionPredicate, Optional.empty()); } - public Query generatePartitionChecksumQuery(QualifiedName tableName, List dataColumns, List partitionColumns) + public Query generatePartitionChecksumQuery(QualifiedName tableName, List dataColumns, List partitionColumns, Optional partitionPredicate) { ImmutableList.Builder selectItems = ImmutableList.builder(); selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of()))); @@ -79,7 +80,7 @@ public Query generatePartitionChecksumQuery(QualifiedName tableName, List partitionColumns, List dataColumns) + public Query generateBucketChecksumQuery(QualifiedName tableName, List partitionColumns, List dataColumns, Optional partitionPredicate) { ImmutableList.Builder selectItems = ImmutableList.builder(); selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of()))); @@ -106,7 +107,7 @@ public Query generateBucketChecksumQuery(QualifiedName tableName, List p return simpleQuery( new Select(false, selectItems.build()), new Table(tableName), - Optional.empty(), + partitionPredicate, Optional.of(new GroupBy(false, groupByList.build())), Optional.empty(), Optional.of(new OrderBy(orderByList.build())), diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java index 1e9cf1b4e12d..73aba566d818 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java @@ -108,7 +108,7 @@ public DataMatchResult verify( ChecksumQueryContext testChecksumQueryContext) { List testColumns = getColumns(getHelperAction(), typeManager, test.getObjectName()); - Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getObjectName(), testColumns); + Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getObjectName(), testColumns, test.getPartitionsPredicate()); testChecksumQueryContext.setChecksumQuery(formatSql(testChecksumQuery)); List controlColumns = null; @@ -116,7 +116,7 @@ public DataMatchResult verify( if (isControlEnabled()) { controlColumns = getColumns(getHelperAction(), typeManager, control.getObjectName()); - Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getObjectName(), controlColumns); + Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getObjectName(), controlColumns, control.getPartitionsPredicate()); controlChecksumQueryContext.setChecksumQuery(formatSql(controlChecksumQuery)); QueryResult controlChecksum = callAndConsume( diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java index 3283aaa367be..dabf0121b3f1 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java @@ -144,7 +144,7 @@ private DeterminismAnalysis analyze(QueryObjectBundle control, ChecksumResult co stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setQueryId)); // Run checksum query - Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns); + Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns, Optional.empty()); ChecksumResult testChecksum = getOnlyElement(callAndConsume( () -> prestoAction.execute(checksumQuery, DETERMINISM_ANALYSIS_CHECKSUM, ChecksumResult::fromResultSet), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setChecksumQueryId)).getResults()); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java index c36dd5f52afb..fee119d50360 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java @@ -273,7 +273,7 @@ private List runPartitionChecksum( ChecksumQueryContext checksumQueryContext, QueryStage queryStage) { - Query partitionChecksumQuery = checksumValidator.generatePartitionChecksumQuery(bundle.getObjectName(), dataColumns, partitionColumns); + Query partitionChecksumQuery = checksumValidator.generatePartitionChecksumQuery(bundle.getObjectName(), dataColumns, partitionColumns, bundle.getPartitionsPredicate()); checksumQueryContext.setPartitionChecksumQuery(formatSql(partitionChecksumQuery)); return callAndConsume( () -> getHelperAction().execute(partitionChecksumQuery, queryStage, ChecksumResult::fromResultSet), @@ -287,7 +287,7 @@ private List runBucketChecksum( ChecksumQueryContext checksumQueryContext, QueryStage queryStage) { - Query bucketChecksumQuery = checksumValidator.generateBucketChecksumQuery(bundle.getObjectName(), partitionColumns, dataColumns); + Query bucketChecksumQuery = checksumValidator.generateBucketChecksumQuery(bundle.getObjectName(), partitionColumns, dataColumns, bundle.getPartitionsPredicate()); List checksumResults = callAndConsume( () -> getHelperAction().execute(bucketChecksumQuery, queryStage, ChecksumResult::fromResultSet), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(checksumQueryContext::setBucketChecksumQueryId)).getResults(); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java index 3ae5f0e2d771..eecee85523c8 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.verifier.framework; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.jdbi.v3.core.mapper.reflect.ColumnName; import org.jdbi.v3.core.mapper.reflect.JdbiConstructor; @@ -37,6 +38,7 @@ public class QueryConfiguration private final Optional password; private final Map sessionProperties; private final boolean isReusableTable; + private final List partitions; @JdbiConstructor public QueryConfiguration( @@ -45,9 +47,10 @@ public QueryConfiguration( @ColumnName("username") Optional username, @ColumnName("password") Optional password, @ColumnName("session_properties") Optional> sessionProperties, - @ColumnName("client_tags") Optional> clientTags) + @ColumnName("client_tags") Optional> clientTags, + @ColumnName("partitions") Optional> partitions) { - this(catalog, schema, username, password, sessionProperties, clientTags.filter(tags -> tags.contains(CLIENT_TAG_OUTPUT_RETAINED)).isPresent()); + this(catalog, schema, username, password, sessionProperties, clientTags.filter(tags -> tags.contains(CLIENT_TAG_OUTPUT_RETAINED)).isPresent(), partitions); } public QueryConfiguration( @@ -56,7 +59,8 @@ public QueryConfiguration( Optional username, Optional password, Optional> sessionProperties, - boolean isReusableTable) + boolean isReusableTable, + Optional> partitions) { this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); @@ -64,6 +68,7 @@ public QueryConfiguration( this.password = requireNonNull(password, "password is null"); this.sessionProperties = ImmutableMap.copyOf(sessionProperties.orElse(ImmutableMap.of())); this.isReusableTable = isReusableTable; + this.partitions = ImmutableList.copyOf(partitions.orElse(ImmutableList.of())); } public QueryConfiguration applyOverrides(QueryConfigurationOverrides overrides) @@ -85,7 +90,8 @@ public QueryConfiguration applyOverrides(QueryConfigurationOverrides overrides) Optional.ofNullable(overrides.getUsernameOverride().orElse(username.orElse(null))), Optional.ofNullable(overrides.getPasswordOverride().orElse(password.orElse(null))), Optional.of(sessionProperties), - isReusableTable); + isReusableTable, + Optional.of(partitions)); } public String getCatalog() @@ -118,6 +124,11 @@ public boolean isReusableTable() return isReusableTable; } + public List getPartitions() + { + return partitions; + } + @Override public boolean equals(Object obj) { @@ -133,13 +144,14 @@ public boolean equals(Object obj) Objects.equals(username, o.username) && Objects.equals(password, o.password) && Objects.equals(sessionProperties, o.sessionProperties) && - isReusableTable == o.isReusableTable; + isReusableTable == o.isReusableTable && + Objects.equals(partitions, partitions); } @Override public int hashCode() { - return Objects.hash(catalog, schema, username, password, sessionProperties, isReusableTable); + return Objects.hash(catalog, schema, username, password, sessionProperties, isReusableTable, partitions); } @Override @@ -152,6 +164,7 @@ public String toString() .add("password", password) .add("sessionProperties", sessionProperties) .add("isReusableTable", isReusableTable) + .add("partitions", partitions) .toString(); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java index 0465157956e2..96ebe68c277d 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryObjectBundle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Statement; @@ -25,6 +26,7 @@ public class QueryObjectBundle extends QueryBundle { private final QualifiedName objectName; + private final Optional partitionsPredicate; private final boolean reuseTable; private final Optional rewrittenFunctionCalls; @@ -35,11 +37,13 @@ public QueryObjectBundle( List teardownQueries, ClusterType cluster, Optional rewrittenFunctionCalls, + Optional partitionsPredicate, boolean reuseTable) { super(setupQueries, query, teardownQueries, cluster); this.objectName = requireNonNull(objectName, "objectName is null"); this.rewrittenFunctionCalls = requireNonNull(rewrittenFunctionCalls, "rewrittenFunctionCalls is null"); + this.partitionsPredicate = partitionsPredicate; this.reuseTable = reuseTable; } @@ -57,4 +61,9 @@ public boolean isReuseTable() { return reuseTable; } + + public Optional getPartitionsPredicate() + { + return partitionsPredicate; + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java index dbd704853583..c3b9e6e694c2 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.verifier.rewrite; +import com.facebook.presto.common.block.BlockEncodingSerde; +import com.facebook.presto.common.predicate.NullableValue; import com.facebook.presto.common.type.ArrayType; import com.facebook.presto.common.type.DecimalType; import com.facebook.presto.common.type.MapType; @@ -22,8 +24,11 @@ import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.common.type.TypeSignatureParameter; import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.planner.LiteralEncoder; import com.facebook.presto.sql.tree.AllColumns; import com.facebook.presto.sql.tree.Cast; +import com.facebook.presto.sql.tree.ComparisonExpression; +import com.facebook.presto.sql.tree.ComparisonExpression.Operator; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateTableAsSelect; import com.facebook.presto.sql.tree.CreateView; @@ -32,7 +37,9 @@ import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Identifier; import com.facebook.presto.sql.tree.Insert; +import com.facebook.presto.sql.tree.IsNullPredicate; import com.facebook.presto.sql.tree.LikeClause; +import com.facebook.presto.sql.tree.LogicalBinaryExpression; import com.facebook.presto.sql.tree.Property; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Query; @@ -43,6 +50,7 @@ import com.facebook.presto.sql.tree.SingleColumn; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.verifier.framework.ClusterType; +import com.facebook.presto.verifier.framework.Column; import com.facebook.presto.verifier.framework.QueryConfiguration; import com.facebook.presto.verifier.framework.QueryException; import com.facebook.presto.verifier.framework.QueryObjectBundle; @@ -53,6 +61,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; import org.intellij.lang.annotations.Language; +import org.joda.time.DateTimeZone; import java.sql.ResultSetMetaData; import java.util.ArrayList; @@ -61,6 +70,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TimeZone; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -74,9 +84,12 @@ import static com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.common.type.UnknownType.UNKNOWN; import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.hive.HiveUtil.parsePartitionValue; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionNamesAndValues; import static com.facebook.presto.sql.tree.LikeClause.PropertiesOption.INCLUDING; import static com.facebook.presto.sql.tree.ShowCreate.Type.VIEW; import static com.facebook.presto.verifier.framework.CreateViewVerification.SHOW_CREATE_VIEW_CONVERTER; +import static com.facebook.presto.verifier.framework.DataVerificationUtil.getColumns; import static com.facebook.presto.verifier.framework.QueryStage.REWRITE; import static com.facebook.presto.verifier.framework.VerifierUtil.PARSING_OPTIONS; import static com.facebook.presto.verifier.framework.VerifierUtil.getColumnNames; @@ -96,6 +109,7 @@ public class QueryRewriter { private final SqlParser sqlParser; private final TypeManager typeManager; + private final BlockEncodingSerde blockEncodingSerde; private final PrestoAction prestoAction; private final Map prefixes; private final Map> tableProperties; @@ -105,17 +119,19 @@ public class QueryRewriter public QueryRewriter( SqlParser sqlParser, TypeManager typeManager, + BlockEncodingSerde blockEncodingSerde, PrestoAction prestoAction, Map tablePrefixes, Map> tableProperties, Map reuseTables) { - this(sqlParser, typeManager, prestoAction, tablePrefixes, tableProperties, reuseTables, ImmutableMultimap.of()); + this(sqlParser, typeManager, blockEncodingSerde, prestoAction, tablePrefixes, tableProperties, reuseTables, ImmutableMultimap.of()); } public QueryRewriter( SqlParser sqlParser, TypeManager typeManager, + BlockEncodingSerde blockEncodingSerde, PrestoAction prestoAction, Map tablePrefixes, Map> tableProperties, @@ -124,6 +140,7 @@ public QueryRewriter( { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerge"); this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); this.prefixes = ImmutableMap.copyOf(tablePrefixes); this.tableProperties = ImmutableMap.copyOf(tableProperties); @@ -156,14 +173,18 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig functionSubstitutions = rewriterResult.getSubstitutions(); } if (shouldReuseTable && !functionSubstitutions.isPresent()) { - return new QueryObjectBundle( - createTableAsSelect.getName(), - ImmutableList.of(), - createTableAsSelect, - ImmutableList.of(), - clusterType, - Optional.empty(), - true); + Optional partitionsPredicate = getPartitionsPredicate(createTableAsSelect.getName(), queryConfiguration.getPartitions()); + if (partitionsPredicate.isPresent()) { + return new QueryObjectBundle( + createTableAsSelect.getName(), + ImmutableList.of(), + createTableAsSelect, + ImmutableList.of(), + clusterType, + Optional.empty(), + partitionsPredicate, + true); + } } QualifiedName temporaryTableName = generateTemporaryName(Optional.of(createTableAsSelect.getName()), prefix); @@ -181,6 +202,7 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, functionSubstitutions, + Optional.empty(), false); } if (statement instanceof Insert) { @@ -195,14 +217,18 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig functionSubstitutions = rewriterResult.getSubstitutions(); } if (shouldReuseTable && !functionSubstitutions.isPresent()) { - return new QueryObjectBundle( - originalTableName, - ImmutableList.of(), - insert, - ImmutableList.of(), - clusterType, - Optional.empty(), - true); + Optional partitionsPredicate = getPartitionsPredicate(originalTableName, queryConfiguration.getPartitions()); + if (partitionsPredicate.isPresent()) { + return new QueryObjectBundle( + originalTableName, + ImmutableList.of(), + insert, + ImmutableList.of(), + clusterType, + Optional.empty(), + partitionsPredicate, + true); + } } QualifiedName temporaryTableName = generateTemporaryName(Optional.of(originalTableName), prefix); @@ -222,6 +248,7 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, functionSubstitutions, + Optional.empty(), false); } if (statement instanceof Query) { @@ -252,6 +279,7 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, functionSubstitutions, + Optional.empty(), false); } if (statement instanceof CreateView) { @@ -288,6 +316,7 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig ImmutableList.of(new DropView(temporaryViewName, true)), clusterType, Optional.empty(), + Optional.empty(), false); } if (statement instanceof CreateTable) { @@ -305,6 +334,7 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, QueryConfig ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType, Optional.empty(), + Optional.empty(), false); } @@ -480,4 +510,64 @@ private static List applyPropertyOverride(List properties, L .map(entry -> new Property(new Identifier(entry.getKey()), entry.getValue())) .collect(toImmutableList()); } + + private Optional getPartitionsPredicate(QualifiedName tableName, List partitions) + { + if (partitions.isEmpty()) { + return Optional.empty(); + } + + List columns = getColumns(prestoAction, typeManager, tableName); + + Expression disjunct = null; + for (String partition : partitions) { + Optional conjunct = Optional.empty(); + try { + conjunct = getPartitionConjunct(partition, columns); + } + catch (Exception e) { + } + if (!conjunct.isPresent()) { + return Optional.empty(); + } + disjunct = disjunct == null ? conjunct.get() : new LogicalBinaryExpression(LogicalBinaryExpression.Operator.OR, disjunct, conjunct.get()); + } + + return Optional.ofNullable(disjunct); + } + + private Optional getPartitionConjunct(String partition, List columns) + { + Expression conjunct = null; + Map partitionRawKeyValues = toPartitionNamesAndValues(partition); + // TryCatch + for (String partitionKey : partitionRawKeyValues.keySet()) { + Type type = null; + for (Column column : columns) { + if (column.getName().equals(partitionKey)) { + type = column.getType(); + break; + } + } + if (type == null) { + // LOG + return Optional.empty(); + } + + NullableValue partitionValue = parsePartitionValue(partitionKey, partitionRawKeyValues.get(partitionKey), type, DateTimeZone.forTimeZone(TimeZone.getDefault())); + + Expression equalPredicate = null; + if (partitionValue.isNull()) { + equalPredicate = new IsNullPredicate(new Identifier(partitionKey)); + } + else { + LiteralEncoder literalEncoder = new LiteralEncoder(blockEncodingSerde); + equalPredicate = new ComparisonExpression(Operator.EQUAL, new Identifier(partitionKey), + literalEncoder.toExpression(partitionValue.getValue(), partitionValue.getType(), false)); + } + conjunct = conjunct == null ? equalPredicate : new LogicalBinaryExpression(LogicalBinaryExpression.Operator.AND, conjunct, equalPredicate); + } + + return Optional.ofNullable(conjunct); + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java index 8b8d0ff8f40e..19a2ca0d353e 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.verifier.rewrite; +import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.tree.BooleanLiteral; @@ -48,6 +49,7 @@ public class VerificationQueryRewriterFactory { private final SqlParser sqlParser; private final TypeManager typeManager; + private final BlockEncodingSerde blockEncodingSerde; private final QualifiedName controlTablePrefix; private final QualifiedName testTablePrefix; private final List controlTableProperties; @@ -61,12 +63,14 @@ public class VerificationQueryRewriterFactory public VerificationQueryRewriterFactory( SqlParser sqlParser, TypeManager typeManager, + BlockEncodingSerde blockEncodingSerde, @ForControl QueryRewriteConfig controlConfig, @ForTest QueryRewriteConfig testConfig, VerifierConfig verifierConfig) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerge is null"); this.controlTablePrefix = requireNonNull(controlConfig.getTablePrefix(), "controlTablePrefix is null"); this.testTablePrefix = requireNonNull(testConfig.getTablePrefix(), "testTablePrefix is null"); this.controlTableProperties = constructProperties(controlConfig.getTableProperties()); @@ -82,6 +86,7 @@ public QueryRewriter create(PrestoAction prestoAction) return new QueryRewriter( sqlParser, typeManager, + blockEncodingSerde, prestoAction, ImmutableMap.of(CONTROL, controlTablePrefix, TEST, testTablePrefix), ImmutableMap.of(CONTROL, controlTableProperties, TEST, testTableProperties), diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java index 1791c29f4ed5..dbccf93b0499 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/PrestoQuerySourceQuerySupplier.java @@ -51,6 +51,8 @@ public class PrestoQuerySourceQuerySupplier Optional.ofNullable(resultSet.getString("control_session_properties")) .map(StringToStringMapColumnMapper.CODEC::fromJson), Optional.ofNullable(resultSet.getString("control_client_tags")) + .map(StringListColumnMapper.CODEC::fromJson), + Optional.ofNullable(resultSet.getString("control_partitions")) .map(StringListColumnMapper.CODEC::fromJson)), new QueryConfiguration( resultSet.getString("test_catalog"), @@ -60,6 +62,8 @@ public class PrestoQuerySourceQuerySupplier Optional.ofNullable(resultSet.getString("test_session_properties")) .map(StringToStringMapColumnMapper.CODEC::fromJson), Optional.ofNullable(resultSet.getString("test_client_tags")) + .map(StringListColumnMapper.CODEC::fromJson), + Optional.ofNullable(resultSet.getString("test_partitions")) .map(StringListColumnMapper.CODEC::fromJson)))); private final PrestoAction helperAction; @@ -73,7 +77,14 @@ public PrestoQuerySourceQuerySupplier( PrestoQuerySourceQueryConfig config) { this.helperAction = helperActionFactory.create( - new QueryConfiguration(config.getCatalog(), config.getSchema(), config.getUsername(), config.getPassword(), Optional.empty(), Optional.empty()), + new QueryConfiguration( + config.getCatalog(), + config.getSchema(), + config.getUsername(), + config.getPassword(), + Optional.empty(), + Optional.empty(), + Optional.empty()), VerificationContext.create("", "")); this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.query = requireNonNull(config.getQuery(), "query is null"); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java index ef088b2ce5e5..5908e1f61c0c 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java @@ -40,6 +40,7 @@ public interface VerifierDao " control_password varchar(256) DEFAULT NULL,\n" + " control_session_properties text DEFAULT NULL,\n" + " control_client_tags text DEFAULT NULL,\n" + + " control_partitions text DEFAULT NULL,\n" + " test_catalog varchar(256) NOT NULL,\n" + " test_schema varchar(256) NOT NULL,\n" + " test_query text NOT NULL,\n" + @@ -47,7 +48,8 @@ public interface VerifierDao " test_username varchar(256) DEFAULT NULL,\n" + " test_password varchar(256) DEFAULT NULL,\n" + " test_session_properties text DEFAULT NULL,\n" + - " test_client_tags text DEFAULT NULL)") + " test_client_tags text DEFAULT NULL,\n" + + " test_partitions text DEFAULT NULL)") void createVerifierQueriesTable(@Define("table_name") String tableName); @SqlQuery("SELECT\n" + @@ -61,6 +63,7 @@ public interface VerifierDao " control_password,\n" + " control_session_properties,\n" + " control_client_tags,\n" + + " control_partitions,\n" + " test_query,\n" + " test_query_id,\n" + " test_catalog,\n" + @@ -68,7 +71,8 @@ public interface VerifierDao " test_username,\n" + " test_password,\n" + " test_session_properties,\n" + - " test_client_tags\n" + + " test_client_tags,\n" + + " test_partitions\n" + "FROM\n" + " \n" + "WHERE\n" + diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java index 4da570f0e7fb..9663a24c52bc 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java @@ -72,6 +72,7 @@ public class VerifierTestUtil ImmutableList.of(), CONTROL, Optional.empty(), + Optional.empty(), false); private static final MySqlOptions MY_SQL_OPTIONS = MySqlOptions.builder() diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java index b4aee00bf709..0aaab41f6d0a 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/checksum/TestChecksumValidator.java @@ -123,7 +123,8 @@ public void testChecksumQuery() MAP_COLUMN, MAP_FLOAT_NON_FLOAT_COLUMN, MAP_NON_ORDERABLE_COLUMN, - ROW_COLUMN)); + ROW_COLUMN), + Optional.empty()); Statement expectedChecksumQuery = sqlParser.createStatement( "SELECT\n" + " \"count\"(*)\n" + diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java index d6b90e425ef5..017f176ca8cc 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/AbstractVerificationTest.java @@ -15,6 +15,8 @@ import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.presto.common.block.BlockEncodingManager; +import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; @@ -62,7 +64,8 @@ public abstract class AbstractVerificationTest protected static final String SUITE = "test-suite"; protected static final String NAME = "test-query"; protected static final String TEST_ID = "test-id"; - protected static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), true); + protected static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), + Optional.empty(), true, Optional.empty()); protected static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build(); protected static final String CONTROL_TABLE_PREFIX = "tmp_verifier_c"; protected static final String TEST_TABLE_PREFIX = "tmp_verifier_t"; @@ -78,6 +81,7 @@ public abstract class AbstractVerificationTest private final Injector injector; private final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); + private final BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(); private final PrestoExceptionClassifier exceptionClassifier = PrestoExceptionClassifier.defaultBuilder().build(); private final DeterminismAnalyzerConfig determinismAnalyzerConfig = new DeterminismAnalyzerConfig().setMaxAnalysisRuns(3).setRunTeardown(true); private final FailureResolverManagerFactory failureResolverManagerFactory; @@ -134,6 +138,11 @@ protected SourceQuery getSourceQuery(String controlQuery, String testQuery, Stri return new SourceQuery(SUITE, NAME, controlQuery, testQuery, Optional.of(controlQueryId), Optional.of(testQueryId), QUERY_CONFIGURATION, QUERY_CONFIGURATION); } + protected SourceQuery getSourceQuery(String controlQuery, String testQuery, String controlQueryId, String testQueryId, QueryConfiguration controlQueryConfiguration, QueryConfiguration testQueryConfiguration) + { + return new SourceQuery(SUITE, NAME, controlQuery, testQuery, Optional.of(controlQueryId), Optional.of(testQueryId), controlQueryConfiguration, testQueryConfiguration); + } + protected Optional runExplain(String controlQuery, String testQuery) { return verify(getSourceQuery(controlQuery, testQuery), true, Optional.empty(), Optional.empty()); @@ -159,6 +168,11 @@ protected Optional runVerification(String controlQuery, Stri return verify(getSourceQuery(controlQuery, testQuery, controlQueryId, testQueryId), false, Optional.empty(), Optional.of(settings)); } + protected Optional runVerification(String controlQuery, String testQuery, String controlQueryId, String testQueryId, QueryConfiguration controlQueryConfiguration, QueryConfiguration testQueryConfiguration, VerificationSettings settings) + { + return verify(getSourceQuery(controlQuery, testQuery, controlQueryId, testQueryId, controlQueryConfiguration, testQueryConfiguration), false, Optional.empty(), Optional.of(settings)); + } + protected Optional verify(SourceQuery sourceQuery, boolean explain) { return verify(sourceQuery, explain, Optional.empty(), Optional.empty()); @@ -216,6 +230,7 @@ private Optional verify( QueryRewriterFactory queryRewriterFactory = new VerificationQueryRewriterFactory( sqlParser, typeManager, + blockEncodingSerde, controlRewriteConfig, testRewriteConfig, verifierConfig); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java index 90858af58f69..29467e3de3fb 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java @@ -18,6 +18,7 @@ import com.facebook.presto.verifier.event.VerifierQueryEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -226,9 +227,13 @@ public void testReuseTable() getQueryRunner().execute(controlQuery); String controlQueryId = "control_query_id"; - Optional event = runVerification(testQuery, controlQuery, controlQueryId, testQueryId, reuseTableSettings); + Optional event = runVerification(testQuery, controlQuery, controlQueryId, testQueryId, + new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), + Optional.empty(), true, Optional.of(ImmutableList.of("test_column=1"))), + new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), + Optional.empty(), true, Optional.empty()), reuseTableSettings); assertTrue(event.get().getControlQueryInfo().getIsReuseTable()); - assertTrue(event.get().getTestQueryInfo().getIsReuseTable()); + assertFalse(event.get().getTestQueryInfo().getIsReuseTable()); assertEvent(event.get(), SUCCEEDED, Optional.empty(), Optional.empty(), Optional.empty()); } @@ -377,7 +382,7 @@ public void testChecksumQueryCompilerError() public void testExecutionTimeSessionProperty() { QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of(QUERY_MAX_EXECUTION_TIME, - "20m")), Optional.empty()); + "20m")), Optional.empty(), Optional.empty()); SourceQuery sourceQuery = new SourceQuery(SUITE, NAME, "SELECT 1.0", "SELECT 1.00001", Optional.empty(), Optional.empty(), configuration, configuration); Optional event = verify(sourceQuery, false); assertTrue(event.isPresent()); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java index 3fd179d4c783..71c6784911ea 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDeterminismAnalyzer.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.presto.common.block.BlockEncodingManager; +import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; @@ -50,6 +52,7 @@ public class TestDeterminismAnalyzer private static final String SUITE = "test-suite"; private static final String NAME = "test-query"; private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); + private static final BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(); @Test public void testMutableCatalog() @@ -66,7 +69,7 @@ private static boolean isMutableCatalogReferenced(DeterminismAnalyzer determinis private static DeterminismAnalyzer createDeterminismAnalyzer(String mutableCatalogPattern) { - QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty()); + QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); VerificationContext verificationContext = VerificationContext.create(SUITE, NAME); VerifierConfig verifierConfig = new VerifierConfig().setTestId("test-id"); RetryConfig retryConfig = new RetryConfig(); @@ -86,6 +89,7 @@ private static DeterminismAnalyzer createDeterminismAnalyzer(String mutableCatal QueryRewriter queryRewriter = new QueryRewriter( sqlParser, typeManager, + blockEncodingSerde, prestoAction, ImmutableMap.of(CONTROL, QualifiedName.of("tmp_verifier_c"), TEST, QualifiedName.of("tmp_verifier_t")), ImmutableMap.of(), diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java index f03b305fde5f..1cba3d981c04 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java @@ -50,15 +50,17 @@ public class TestQueryConfiguration private static final List CLIENT_TAGS = ImmutableList.of(QueryConfiguration.CLIENT_TAG_OUTPUT_RETAINED); private static final QueryConfiguration CONFIGURATION_1 = new QueryConfiguration(CATALOG, SCHEMA, Optional.of(USERNAME), Optional.of(PASSWORD), - Optional.of(SESSION_PROPERTIES), Optional.of(CLIENT_TAGS)); - private static final QueryConfiguration CONFIGURATION_2 = new QueryConfiguration(CATALOG, SCHEMA, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + Optional.of(SESSION_PROPERTIES), Optional.of(CLIENT_TAGS), Optional.empty()); + private static final QueryConfiguration CONFIGURATION_2 = new QueryConfiguration(CATALOG, SCHEMA, Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty(), Optional.empty()); private static final QueryConfiguration CONFIGURATION_FULL_OVERRIDE = new QueryConfiguration( CATALOG_OVERRIDE, SCHEMA_OVERRIDE, Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(SESSION_PROPERTIES_OVERRIDE), - Optional.of(CLIENT_TAGS)); + Optional.of(CLIENT_TAGS), + Optional.empty()); private QueryConfigurationOverridesConfig overrides; @@ -91,7 +93,8 @@ public void testOverrides() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(SESSION_PROPERTIES), - Optional.of(CLIENT_TAGS))); + Optional.of(CLIENT_TAGS), + Optional.empty())); assertEquals(CONFIGURATION_2.applyOverrides(overrides), new QueryConfiguration( CATALOG_OVERRIDE, @@ -99,6 +102,7 @@ public void testOverrides() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.empty(), + Optional.empty(), Optional.empty())); } @@ -113,6 +117,7 @@ public void testSessionPropertyOverride() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(SESSION_PROPERTIES_OVERRIDE), + false, Optional.empty()); assertEquals(CONFIGURATION_2.applyOverrides(overrides), overridden); } @@ -127,7 +132,8 @@ public void testSessionPropertySubstitute() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(ImmutableMap.of("property_1", "value_x", "property_2", "value_2", "property_3", "value_3")), - Optional.of(CLIENT_TAGS)); + Optional.of(CLIENT_TAGS), + Optional.empty()); assertEquals(CONFIGURATION_1.applyOverrides(overrides), substituted1); @@ -137,6 +143,7 @@ public void testSessionPropertySubstitute() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(SESSION_PROPERTIES_OVERRIDE), + false, Optional.empty()); assertEquals(CONFIGURATION_2.applyOverrides(overrides), substituted2); } @@ -153,7 +160,8 @@ public void testSessionPropertyRemovalWithOverrides() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(ImmutableMap.of("property_3", "value_3")), - Optional.of(CLIENT_TAGS)); + Optional.of(CLIENT_TAGS), + Optional.empty()); assertEquals(CONFIGURATION_1.applyOverrides(overrides), removed); } @@ -169,7 +177,8 @@ public void testSessionPropertySubstituteAndRemove() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(SESSION_PROPERTIES_OVERRIDE), - Optional.of(CLIENT_TAGS)); + Optional.of(CLIENT_TAGS), + Optional.empty()); assertEquals(CONFIGURATION_1.applyOverrides(overrides), removed); } @@ -185,7 +194,8 @@ public void testSessionPropertyRemoval() Optional.of(USERNAME_OVERRIDE), Optional.of(PASSWORD_OVERRIDE), Optional.of(ImmutableMap.of("property_1", "value_1")), - Optional.of(CLIENT_TAGS)); + Optional.of(CLIENT_TAGS), + Optional.empty()); assertEquals(CONFIGURATION_1.applyOverrides(overrides), removed); } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java index 4361fe751879..6c67276c870c 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java @@ -14,6 +14,8 @@ package com.facebook.presto.verifier.framework; import com.facebook.airlift.event.client.AbstractEventClient; +import com.facebook.presto.common.block.BlockEncodingManager; +import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; @@ -115,7 +117,8 @@ public List getEvents() private static final String NAME = "test-query"; private static final QualifiedName TABLE_PREFIX = QualifiedName.of("tmp_verifier"); private static final SqlParser SQL_PARSER = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(AT_SIGN, COLON)); - private static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration("test", "di", Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty()); + private static final BlockEncodingSerde BLOCK_ENCODING_SERDE = new BlockEncodingManager(); + private static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration("test", "di", Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); private static final SourceQuery SOURCE_QUERY = new SourceQuery( SUITE, NAME, @@ -218,7 +221,7 @@ private VerificationManager getVerificationManager(List sourceQueri new VerificationFactory( SQL_PARSER, (sourceQuery, verificationContext) -> new QueryActions(prestoAction, prestoAction, prestoAction), - presto -> new QueryRewriter(SQL_PARSER, createTypeManager(), presto, ImmutableMap.of(CONTROL, TABLE_PREFIX, TEST, TABLE_PREFIX), + presto -> new QueryRewriter(SQL_PARSER, createTypeManager(), BLOCK_ENCODING_SERDE, presto, ImmutableMap.of(CONTROL, TABLE_PREFIX, TEST, TABLE_PREFIX), ImmutableMap.of(), ImmutableMap.of(CONTROL, false, TEST, false)), new FailureResolverManagerFactory(ImmutableSet.of(), ImmutableSet.of()), diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java index 7869b152a91d..a965b2c919b9 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java @@ -56,7 +56,7 @@ public class TestJdbcPrestoAction private static final String SUITE = "test-suite"; private static final String NAME = "test-query"; private static final QueryStage QUERY_STAGE = CONTROL_MAIN; - private static final QueryConfiguration CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty()); + private static final QueryConfiguration CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); private static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DECIMAL).build(); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java index e6b749a1dc23..3321c011777f 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestIgnoredFunctionsMismatchResolver.java @@ -69,6 +69,7 @@ private static QueryObjectBundle createBundle(String query) ImmutableList.of(), CONTROL, Optional.empty(), + Optional.empty(), false); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java index 827c8a892970..858f75152436 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java @@ -89,6 +89,7 @@ public QueryResult execute(Statement statement, QueryStage queryStage, Re ImmutableList.of(), TEST, Optional.empty(), + Optional.empty(), false); private static final QueryException HIVE_TOO_MANY_OPEN_PARTITIONS_EXCEPTION = new PrestoQueryException( new RuntimeException(), diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java index 95b060e73b81..7bef00a47c23 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.verifier.rewrite; +import com.facebook.presto.common.block.BlockEncodingManager; +import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; @@ -76,6 +78,7 @@ public class TestQueryRewriter Optional.of("user"), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); private static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build(); private static final QueryRewriteConfig QUERY_REWRITE_CONFIG = new QueryRewriteConfig() @@ -83,6 +86,7 @@ public class TestQueryRewriter .setTableProperties("{\"p_int\": 30, \"p_long\": 4294967297, \"p_double\": 1.5, \"p_varchar\": \"test\", \"p_bool\": true}"); private static final VerifierConfig VERIFIER_CONFIG = new VerifierConfig(); private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); + private static final BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(); private static StandaloneQueryRunner queryRunner; private static PrestoAction prestoAction; @@ -583,14 +587,11 @@ public void testRewriteFunctionCalls() @Test public void testReuseTableRewrite() { - String query = "INSERT INTO dest_table SELECT * FROM test_table"; + String query = "CREATE TABLE dest_table AS SELECT a, b FROM test_table WHERE a = 1 AND b = 'xyz'"; + queryRunner.execute(query); + List partitions = ImmutableList.of("a=1", "b=xyz"); QueryConfiguration configuration = new QueryConfiguration( - CATALOG, - SCHEMA, - Optional.of("user"), - Optional.empty(), - Optional.empty(), - true); + CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty(), true, Optional.of(partitions)); assertShadowed( getQueryRewriter(new QueryRewriteConfig().setReuseTable(true), VERIFIER_CONFIG), query, @@ -674,6 +675,6 @@ private QueryRewriter getQueryRewriter() private QueryRewriter getQueryRewriter(QueryRewriteConfig rewriteConfig, VerifierConfig verifierConfig) { - return new VerificationQueryRewriterFactory(sqlParser, createTypeManager(), rewriteConfig, rewriteConfig, verifierConfig).create(prestoAction); + return new VerificationQueryRewriterFactory(sqlParser, createTypeManager(), blockEncodingSerde, rewriteConfig, rewriteConfig, verifierConfig).create(prestoAction); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java index 3127831975ef..96860c7ec6cb 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestPrestoQuerySourceQuerySupplier.java @@ -60,6 +60,7 @@ public class TestPrestoQuerySourceQuerySupplier " 'user' control_username,\n" + " '{\"a\": \"b\"}' control_session_properties,\n" + " '[\"x\"]' control_client_tags,\n" + + " NULL control_partitions,\n" + " query_id control_query_id,\n" + " NULL control_password,\n" + " query test_query,\n" + @@ -69,6 +70,7 @@ public class TestPrestoQuerySourceQuerySupplier " NULL test_password,\n" + " '{\"c\": \"d\"}' test_session_properties,\n" + " '[\"y\"]' test_client_tags,\n" + + " NULL test_partitions,\n" + " query_id test_query_id\n" + "FROM (\n" + " VALUES\n" + @@ -76,9 +78,9 @@ public class TestPrestoQuerySourceQuerySupplier " ('Q2', 'INSERT INTO test_table SELECT 1', 'T2')\n" + ") queries(name, query, query_id)"; private static final QueryConfiguration CONTROL_CONFIGURATION = new QueryConfiguration( - "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("a", "b")), Optional.of(ImmutableList.of("x"))); + "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("a", "b")), Optional.of(ImmutableList.of("x")), Optional.empty()); private static final QueryConfiguration TEST_CONFIGURATION = new QueryConfiguration( - "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("c", "d")), Optional.of(ImmutableList.of("y"))); + "catalog", "schema", Optional.of("user"), Optional.empty(), Optional.of(ImmutableMap.of("c", "d")), Optional.of(ImmutableList.of("y")), Optional.empty()); private static final List SOURCE_QUERIES = ImmutableList.of( new SourceQuery("test", "Q1", "SELECT 1", "SELECT 1", Optional.of("T1"), Optional.of("T1"), CONTROL_CONFIGURATION, TEST_CONFIGURATION), new SourceQuery("test", "Q2", "INSERT INTO test_table SELECT 1", "INSERT INTO test_table SELECT 1", Optional.of("T2"), Optional.of("T2"), CONTROL_CONFIGURATION,