Skip to content

Commit

Permalink
Merge pull request #3253 from shollyman/morestuff
Browse files Browse the repository at this point in the history 
BigQuery: add missing query statistics.
  • Loading branch information
@shollyman
shollyman committed May 22, 2018
2 parents d3fe98c + ddb1ce4 commit f1a0d3b
Show file tree
Hide file tree
Showing 2 changed files with 253 additions and 1 deletion.
212 changes: 212 additions & 0 deletions ...-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,13 @@

package com.google.cloud.bigquery;

import com.google.api.core.ApiFunction;
import com.google.api.services.bigquery.model.JobConfiguration;
import com.google.api.services.bigquery.model.JobStatistics2;
import com.google.api.services.bigquery.model.JobStatistics3;
import com.google.api.services.bigquery.model.JobStatistics4;
import com.google.cloud.StringEnumType;
import com.google.cloud.StringEnumValue;
import com.google.common.base.MoreObjects;
import com.google.common.base.MoreObjects.ToStringHelper;
import com.google.common.collect.Lists;
Expand Down Expand Up @@ -311,6 +314,7 @@ static LoadStatistics fromPb(com.google.api.services.bigquery.model.JobStatistic
}
}


/**
* A Google BigQuery Query Job statistics.
*/
Expand All @@ -320,18 +324,92 @@ public static class QueryStatistics extends JobStatistics {

private final Integer billingTier;
private final Boolean cacheHit;
private final String ddlOperationPerformed;
private final TableId ddlTargetTable;
private final Long estimatedBytesProcessed;
private final Long numDmlAffectedRows;
private final List<TableId> referencedTables;
private final StatementType statementType;
private final Long totalBytesBilled;
private final Long totalBytesProcessed;
private final Long totalPartitionsProcessed;
private final Long totalSlotMs;
private final List<QueryStage> queryPlan;
private final List<TimelineSample> timeline;
private final Schema schema;


/**
* StatementType represents possible types of SQL statements reported as part of the
* QueryStatistics of a BigQuery job.
*/
public static final class StatementType extends StringEnumValue {
private static final long serialVersionUID = 818920627219751204L;

private static final ApiFunction<String, StatementType> CONSTRUCTOR =
new ApiFunction<String, StatementType>() {
@Override
public StatementType apply(String constant) {
return new StatementType(constant);
}
};

private static final StringEnumType<StatementType> type = new StringEnumType(
StatementType.class,
CONSTRUCTOR);

public static final StatementType SELECT = type.createAndRegister("SELECT");
public static final StatementType UPDATE = type.createAndRegister("UPDATE");
public static final StatementType INSERT = type.createAndRegister("INSERT");
public static final StatementType DELETE = type.createAndRegister("DELETE");
public static final StatementType CREATE_TABLE = type.createAndRegister("CREATE_TABLE");
public static final StatementType CREATE_TABLE_AS_SELECT = type.createAndRegister("CREATE_TABLE_AS_SELECT");
public static final StatementType CREATE_VIEW = type.createAndRegister("CREATE_VIEW");
public static final StatementType DROP_TABLE = type.createAndRegister("DROP_TABLE");
public static final StatementType DROP_VIEW = type.createAndRegister("DROP_VIEW");
public static final StatementType MERGE = type.createAndRegister("MERGE");

private StatementType(String constant) {
super(constant);
}

/**
* Get the StatementType for the given String constant, and throw an exception if the constant is
* not recognized.
*/
public static StatementType valueOfStrict(String constant) {
return type.valueOfStrict(constant);
}

/**
* Get the State for the given String constant, and allow unrecognized values.
*/
public static StatementType valueOf(String constant) {
return type.valueOf(constant);
}

/**
* Return the known values for State.
*/
public static StatementType[] values() {
return type.values();
}
}

static final class Builder extends JobStatistics.Builder<QueryStatistics, Builder> {

private Integer billingTier;
private Boolean cacheHit;
private String ddlOperationPerformed;
private TableId ddlTargetTable;
private Long estimatedBytesProcessed;
private Long numDmlAffectedRows;
private List<TableId> referencedTables;
private StatementType statementType;
private Long totalBytesBilled;
private Long totalBytesProcessed;
private Long totalPartitionsProcessed;
private Long totalSlotMs;
private List<QueryStage> queryPlan;
private List<TimelineSample> timeline;
private Schema schema;
Expand All @@ -343,8 +421,25 @@ private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsP
if (statisticsPb.getQuery() != null) {
this.billingTier = statisticsPb.getQuery().getBillingTier();
this.cacheHit = statisticsPb.getQuery().getCacheHit();
this.ddlOperationPerformed = statisticsPb.getQuery().getDdlOperationPerformed();
if (statisticsPb.getQuery().getDdlTargetTable() != null) {
this.ddlTargetTable = TableId.fromPb(statisticsPb.getQuery().getDdlTargetTable());
}
this.estimatedBytesProcessed = statisticsPb.getQuery().getEstimatedBytesProcessed();
this.numDmlAffectedRows = statisticsPb.getQuery().getNumDmlAffectedRows();
this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled();
this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed();
this.totalPartitionsProcessed = statisticsPb.getQuery().getTotalPartitionsProcessed();
this.totalSlotMs = statisticsPb.getQuery().getTotalSlotMs();
if (statisticsPb.getQuery().getStatementType() != null) {
this.statementType = StatementType.valueOf(statisticsPb.getQuery().getStatementType());
}

if (statisticsPb.getQuery().getReferencedTables() != null) {
this.referencedTables =
Lists.transform(
statisticsPb.getQuery().getReferencedTables(), TableId.FROM_PB_FUNCTION);
}
if (statisticsPb.getQuery().getQueryPlan() != null) {
this.queryPlan =
Lists.transform(
Expand All @@ -371,6 +466,41 @@ Builder setCacheHit(Boolean cacheHit) {
return self();
}

Builder setDDLOperationPerformed(String ddlOperationPerformed) {
this.ddlOperationPerformed = ddlOperationPerformed;
return self();
}

Builder setDDLTargetTable(TableId ddlTargetTable) {
this.ddlTargetTable = ddlTargetTable;
return self();
}

Builder setEstimatedBytesProcessed(Long estimatedBytesProcessed) {
this.estimatedBytesProcessed = estimatedBytesProcessed;
return self();
}

Builder setNumDmlAffectedRows(Long numDmlAffectedRows) {
this.numDmlAffectedRows = numDmlAffectedRows;
return self();
}

Builder setReferenceTables(List<TableId> referencedTables) {
this.referencedTables = referencedTables;
return self();
}

Builder setStatementType(StatementType statementType) {
this.statementType = statementType;
return self();
}

Builder setStatementType(String strStatementType) {
this.statementType = StatementType.valueOf(strStatementType);
return self();
}

Builder setTotalBytesBilled(Long totalBytesBilled) {
this.totalBytesBilled = totalBytesBilled;
return self();
Expand All @@ -381,6 +511,16 @@ Builder setTotalBytesProcessed(Long totalBytesProcessed) {
return self();
}

Builder setTotalPartitionsProcessed(Long totalPartitionsProcessed) {
this.totalPartitionsProcessed = totalPartitionsProcessed;
return self();
}

Builder setTotalSlotMs(Long totalSlotMs) {
this.totalSlotMs = totalSlotMs;
return self();
}

Builder setQueryPlan(List<QueryStage> queryPlan) {
this.queryPlan = queryPlan;
return self();
Expand All @@ -406,8 +546,16 @@ private QueryStatistics(Builder builder) {
super(builder);
this.billingTier = builder.billingTier;
this.cacheHit = builder.cacheHit;
this.ddlOperationPerformed = builder.ddlOperationPerformed;
this.ddlTargetTable = builder.ddlTargetTable;
this.estimatedBytesProcessed = builder.estimatedBytesProcessed;
this.numDmlAffectedRows = builder.numDmlAffectedRows;
this.referencedTables = builder.referencedTables;
this.statementType = builder.statementType;
this.totalBytesBilled = builder.totalBytesBilled;
this.totalBytesProcessed = builder.totalBytesProcessed;
this.totalPartitionsProcessed = builder.totalPartitionsProcessed;
this.totalSlotMs = builder.totalSlotMs;
this.queryPlan = builder.queryPlan;
this.timeline = builder.timeline;
this.schema = builder.schema;
Expand All @@ -432,6 +580,47 @@ public Boolean getCacheHit() {
return cacheHit;
}

/**
* [BETA] For DDL queries, returns the operation applied to the DDL target table.
*/
public String getDdlOperationPerformed() { return ddlOperationPerformed; }

/**
* [BETA] For DDL queries, returns the TableID of the targeted table.
*/
public TableId getDdlTargetTable() { return ddlTargetTable; }

/**
* The original estimate of bytes processed for the job.
*/
public Long getEstimatedBytesProcessed() { return estimatedBytesProcessed; }

/**
* The number of rows affected by a DML statement.
* Present only for DML statements INSERT, UPDATE or DELETE.
*/
public Long getNumDmlAffectedRows() { return numDmlAffectedRows; }

/**
* Referenced tables for the job.
* Queries that reference more than 50 tables will not have a complete list.
*/
public List<TableId> getReferencedTables() { return referencedTables; }

/**
* [BETA] The type of query statement, if valid.
* Possible values include:
* SELECT
* INSERT
* UPDATE
* DELETE
* CREATE_TABLE
* CREATE_TABLE_AS_SELECT
* DROP_TABLE
* CREATE_VIEW
* DROP_VIEW
*/
public StatementType getStatementType() { return statementType; }

/**
* Returns the total number of bytes billed for the job.
Expand All @@ -448,6 +637,15 @@ public Long getTotalBytesProcessed() {
return totalBytesProcessed;
}

/**
* Total number of partitions processed from all partitioned tables referenced in the job.
*/
public Long getTotalPartitionsProcessed() { return totalPartitionsProcessed; }

/**
* Returns the slot-milliseconds consumed by the query.
*/
public Long getTotalSlotMs() { return totalSlotMs; }

/**
* Returns the query plan as a list of stages or {@code null} if a query plan is not available.
Expand Down Expand Up @@ -509,8 +707,22 @@ com.google.api.services.bigquery.model.JobStatistics toPb() {
JobStatistics2 queryStatisticsPb = new JobStatistics2();
queryStatisticsPb.setBillingTier(billingTier);
queryStatisticsPb.setCacheHit(cacheHit);
queryStatisticsPb.setDdlOperationPerformed(ddlOperationPerformed);
queryStatisticsPb.setEstimatedBytesProcessed(estimatedBytesProcessed);
queryStatisticsPb.setTotalBytesBilled(totalBytesBilled);
queryStatisticsPb.setTotalBytesProcessed(totalBytesProcessed);
queryStatisticsPb.setTotalPartitionsProcessed(totalPartitionsProcessed);
queryStatisticsPb.setTotalSlotMs(totalSlotMs);
if (ddlTargetTable != null) {
queryStatisticsPb.setDdlTargetTable(ddlTargetTable.toPb());
}

if (referencedTables != null) {
queryStatisticsPb.setReferencedTables(Lists.transform(referencedTables, TableId.TO_PB_FUNCTION));
}
if (statementType != null) {
queryStatisticsPb.setStatementType(statementType.toString());
}
if (queryPlan != null) {
queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION));
}
Expand Down
42 changes: 41 additions & 1 deletion ...ents/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,22 @@ public class JobStatisticsTest {

private static final Integer BILLING_TIER = 42;
private static final Boolean CACHE_HIT = true;
private static final String DDL_OPERATION_PERFORMED = "SKIP";
private static final TableId DDL_TARGET_TABLE = TableId.of("foo", "bar", "baz");
private static final Long ESTIMATE_BYTES_PROCESSED = 101L;
private static final Long NUM_DML_AFFECTED_ROWS = 88L;
private static final QueryStatistics.StatementType STATEMENT_TYPE = QueryStatistics.StatementType.SELECT;
private static final Long TOTAL_BYTES_BILLED = 24L;
private static final Long TOTAL_BYTES_PROCESSED = 42L;
private static final Long TOTAL_PARTITION_PROCESSED = 63L;
private static final Long TOTAL_SLOT_MS = 10202L;
private static final Long INPUT_BYTES = 1L;
private static final Long INPUT_FILES = 2L;
private static final Long OUTPUT_BYTES = 3L;
private static final Long OUTPUT_ROWS = 4L;
private static final List<TableId> REFERENCED_TABLES = ImmutableList.of(
TableId.of("foo", "bar", "table1"),
TableId.of("foo","bar","table2"));
private static final List<Long> FILE_COUNT = ImmutableList.of(1L, 2L, 3L);
private static final Long CREATION_TIME = 10L;
private static final Long END_TIME = 20L;
Expand Down Expand Up @@ -112,8 +122,16 @@ public class JobStatisticsTest {
.setStartTime(START_TIME)
.setBillingTier(BILLING_TIER)
.setCacheHit(CACHE_HIT)
.setDDLOperationPerformed(DDL_OPERATION_PERFORMED)
.setDDLTargetTable(DDL_TARGET_TABLE)
.setEstimatedBytesProcessed(ESTIMATE_BYTES_PROCESSED)
.setNumDmlAffectedRows(NUM_DML_AFFECTED_ROWS)
.setReferenceTables(REFERENCED_TABLES)
.setStatementType(STATEMENT_TYPE)
.setTotalBytesBilled(TOTAL_BYTES_BILLED)
.setTotalBytesProcessed(TOTAL_BYTES_PROCESSED)
.setTotalPartitionsProcessed(TOTAL_PARTITION_PROCESSED)
.setTotalSlotMs(TOTAL_SLOT_MS)
.setQueryPlan(QUERY_PLAN)
.setTimeline(TIMELINE)
.setSchema(SCHEMA)
Expand Down Expand Up @@ -146,9 +164,16 @@ public void testBuilder() {
assertEquals(END_TIME, QUERY_STATISTICS.getEndTime());
assertEquals(BILLING_TIER, QUERY_STATISTICS.getBillingTier());
assertEquals(CACHE_HIT, QUERY_STATISTICS.getCacheHit());
assertEquals(DDL_OPERATION_PERFORMED, QUERY_STATISTICS.getDdlOperationPerformed());
assertEquals(DDL_TARGET_TABLE, QUERY_STATISTICS.getDdlTargetTable());
assertEquals(ESTIMATE_BYTES_PROCESSED, QUERY_STATISTICS.getEstimatedBytesProcessed());
assertEquals(NUM_DML_AFFECTED_ROWS, QUERY_STATISTICS.getNumDmlAffectedRows());
assertEquals(REFERENCED_TABLES, QUERY_STATISTICS.getReferencedTables());
assertEquals(STATEMENT_TYPE, QUERY_STATISTICS.getStatementType());
assertEquals(TOTAL_BYTES_BILLED, QUERY_STATISTICS.getTotalBytesBilled());
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
assertEquals(TOTAL_PARTITION_PROCESSED, QUERY_STATISTICS.getTotalPartitionsProcessed());
assertEquals(TOTAL_SLOT_MS, QUERY_STATISTICS.getTotalSlotMs());
assertEquals(QUERY_PLAN, QUERY_STATISTICS.getQueryPlan());
assertEquals(TIMELINE, QUERY_STATISTICS.getTimeline());

Expand All @@ -165,8 +190,15 @@ public void testBuilder() {
assertEquals(END_TIME, QUERY_STATISTICS_INCOMPLETE.getEndTime());
assertEquals(BILLING_TIER, QUERY_STATISTICS_INCOMPLETE.getBillingTier());
assertEquals(CACHE_HIT, QUERY_STATISTICS_INCOMPLETE.getCacheHit());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getDdlOperationPerformed());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getDdlTargetTable());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getEstimatedBytesProcessed());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getNumDmlAffectedRows());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalBytesBilled());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalBytesProcessed());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalPartitionsProcessed());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalSlotMs());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getReferencedTables());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getQueryPlan());
}

Expand Down Expand Up @@ -235,10 +267,18 @@ private void compareQueryStatistics(QueryStatistics expected, QueryStatistics va
compareStatistics(expected, value);
assertEquals(expected.getBillingTier(), value.getBillingTier());
assertEquals(expected.getCacheHit(), value.getCacheHit());
assertEquals(expected.getDdlOperationPerformed(), value.getDdlOperationPerformed());
assertEquals(expected.getDdlTargetTable(), value.getDdlTargetTable());
assertEquals(expected.getEstimatedBytesProcessed(), value.getEstimatedBytesProcessed());
assertEquals(expected.getTotalBytesBilled(), value.getTotalBytesBilled());
assertEquals(expected.getTotalBytesProcessed(), value.getTotalBytesProcessed());
assertEquals(expected.getTotalPartitionsProcessed(), value.getTotalPartitionsProcessed());
assertEquals(expected.getTotalSlotMs(), value.getTotalSlotMs());
assertEquals(expected.getQueryPlan(), value.getQueryPlan());
assertEquals(expected.getReferencedTables(), value.getReferencedTables());
assertEquals(expected.getSchema(), value.getSchema());
assertEquals(expected.getStatementType(), value.getStatementType());
assertEquals(expected.getTimeline(), value.getTimeline());
}

private void compareStatistics(JobStatistics expected, JobStatistics value) {
Expand Down

0 comments on commit f1a0d3b

Please sign in to comment.