diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml
index f2c6174b97eb..fd5bdd3b205a 100644
--- a/presto-verifier/pom.xml
+++ b/presto-verifier/pom.xml
@@ -29,11 +29,27 @@
+
+ com.facebook.presto
+ presto-hive
+
+
+ com.facebook.presto
+ presto-cache
+
+
+
+
com.facebook.presto
presto-hive-common
+
+ com.facebook.presto
+ presto-hive-metastore
+
+
com.facebook.presto
presto-jdbc
@@ -198,6 +214,11 @@
provided
+
+ joda-time
+ joda-time
+
+
com.facebook.presto
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java
index 1e3c76d970e3..2caa6c5f86d6 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumValidator.java
@@ -13,6 +13,7 @@
*/
package com.facebook.presto.verifier.checksum;
+import com.facebook.presto.sql.tree.Expression;
import com.facebook.presto.sql.tree.FunctionCall;
import com.facebook.presto.sql.tree.GroupBy;
import com.facebook.presto.sql.tree.GroupingElement;
@@ -52,17 +53,17 @@ public ChecksumValidator(Map> columnValidato
this.columnValidators = columnValidators;
}
- public Query generateChecksumQuery(QualifiedName tableName, List columns)
+ public Query generateChecksumQuery(QualifiedName tableName, List columns, Optional partitionPredicate)
{
ImmutableList.Builder selectItems = ImmutableList.builder();
selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of())));
for (Column column : columns) {
selectItems.addAll(columnValidators.get(column.getCategory()).get().generateChecksumColumns(column));
}
- return simpleQuery(new Select(false, selectItems.build()), new Table(tableName));
+ return simpleQuery(new Select(false, selectItems.build()), new Table(tableName), partitionPredicate, Optional.empty());
}
- public Query generatePartitionChecksumQuery(QualifiedName tableName, List dataColumns, List partitionColumns)
+ public Query generatePartitionChecksumQuery(QualifiedName tableName, List dataColumns, List partitionColumns, Optional partitionPredicate)
{
ImmutableList.Builder selectItems = ImmutableList.builder();
selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of())));
@@ -79,7 +80,7 @@ public Query generatePartitionChecksumQuery(QualifiedName tableName, List partitionColumns, List dataColumns)
+ public Query generateBucketChecksumQuery(QualifiedName tableName, List partitionColumns, List dataColumns, Optional partitionPredicate)
{
ImmutableList.Builder selectItems = ImmutableList.builder();
selectItems.add(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of())));
@@ -106,7 +107,7 @@ public Query generateBucketChecksumQuery(QualifiedName tableName, List p
return simpleQuery(
new Select(false, selectItems.build()),
new Table(tableName),
- Optional.empty(),
+ partitionPredicate,
Optional.of(new GroupBy(false, groupByList.build())),
Optional.empty(),
Optional.of(new OrderBy(orderByList.build())),
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java
index 1e9cf1b4e12d..73aba566d818 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java
@@ -108,7 +108,7 @@ public DataMatchResult verify(
ChecksumQueryContext testChecksumQueryContext)
{
List testColumns = getColumns(getHelperAction(), typeManager, test.getObjectName());
- Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getObjectName(), testColumns);
+ Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getObjectName(), testColumns, test.getPartitionsPredicate());
testChecksumQueryContext.setChecksumQuery(formatSql(testChecksumQuery));
List controlColumns = null;
@@ -116,7 +116,7 @@ public DataMatchResult verify(
if (isControlEnabled()) {
controlColumns = getColumns(getHelperAction(), typeManager, control.getObjectName());
- Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getObjectName(), controlColumns);
+ Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getObjectName(), controlColumns, control.getPartitionsPredicate());
controlChecksumQueryContext.setChecksumQuery(formatSql(controlChecksumQuery));
QueryResult controlChecksum = callAndConsume(
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java
index 3283aaa367be..dabf0121b3f1 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalyzer.java
@@ -144,7 +144,7 @@ private DeterminismAnalysis analyze(QueryObjectBundle control, ChecksumResult co
stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setQueryId));
// Run checksum query
- Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns);
+ Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns, Optional.empty());
ChecksumResult testChecksum = getOnlyElement(callAndConsume(
() -> prestoAction.execute(checksumQuery, DETERMINISM_ANALYSIS_CHECKSUM, ChecksumResult::fromResultSet),
stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setChecksumQueryId)).getResults());
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java
index c36dd5f52afb..fee119d50360 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExtendedVerification.java
@@ -273,7 +273,7 @@ private List runPartitionChecksum(
ChecksumQueryContext checksumQueryContext,
QueryStage queryStage)
{
- Query partitionChecksumQuery = checksumValidator.generatePartitionChecksumQuery(bundle.getObjectName(), dataColumns, partitionColumns);
+ Query partitionChecksumQuery = checksumValidator.generatePartitionChecksumQuery(bundle.getObjectName(), dataColumns, partitionColumns, bundle.getPartitionsPredicate());
checksumQueryContext.setPartitionChecksumQuery(formatSql(partitionChecksumQuery));
return callAndConsume(
() -> getHelperAction().execute(partitionChecksumQuery, queryStage, ChecksumResult::fromResultSet),
@@ -287,7 +287,7 @@ private List runBucketChecksum(
ChecksumQueryContext checksumQueryContext,
QueryStage queryStage)
{
- Query bucketChecksumQuery = checksumValidator.generateBucketChecksumQuery(bundle.getObjectName(), partitionColumns, dataColumns);
+ Query bucketChecksumQuery = checksumValidator.generateBucketChecksumQuery(bundle.getObjectName(), partitionColumns, dataColumns, bundle.getPartitionsPredicate());
List checksumResults = callAndConsume(
() -> getHelperAction().execute(bucketChecksumQuery, queryStage, ChecksumResult::fromResultSet),
stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(checksumQueryContext::setBucketChecksumQueryId)).getResults();
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java
index 3ae5f0e2d771..eecee85523c8 100644
--- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java
+++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java
@@ -13,6 +13,7 @@
*/
package com.facebook.presto.verifier.framework;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.jdbi.v3.core.mapper.reflect.ColumnName;
import org.jdbi.v3.core.mapper.reflect.JdbiConstructor;
@@ -37,6 +38,7 @@ public class QueryConfiguration
private final Optional password;
private final Map sessionProperties;
private final boolean isReusableTable;
+ private final List partitions;
@JdbiConstructor
public QueryConfiguration(
@@ -45,9 +47,10 @@ public QueryConfiguration(
@ColumnName("username") Optional username,
@ColumnName("password") Optional password,
@ColumnName("session_properties") Optional