From 92077437d759705151f7778207616ecf024371ba Mon Sep 17 00:00:00 2001 From: Farhan Ahmed Date: Mon, 19 Jun 2023 10:56:18 -0400 Subject: [PATCH] feat: Partial Projection of Table Metadata (#2756) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/java-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes # ☕️ If you write sample code, please follow the [samples format]( https://togithub.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md). --- .../com/google/cloud/bigquery/BigQuery.java | 19 +++ .../java/com/google/cloud/bigquery/Table.java | 4 +- .../cloud/bigquery/spi/v2/BigQueryRpc.java | 5 +- .../bigquery/spi/v2/HttpBigQueryRpc.java | 8 ++ .../cloud/bigquery/it/ITBigQueryTest.java | 128 ++++++++++++++++++ 5 files changed, 160 insertions(+), 4 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 8f11d03fa..271278357 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -121,6 +121,20 @@ public String getSelector() { } } + /** + * Metadata of a BigQuery Table. + * + * @see Table + * Resource + */ + enum TableMetadataView { + BASIC, + FULL, + STORAGE_STATS, + TABLE_METADATA_VIEW_UNSPECIFIED; + } + /** * Fields of a BigQuery Model resource. * @@ -384,6 +398,11 @@ public static TableOption fields(TableField... fields) { public static TableOption autodetectSchema(boolean autodetect) { return new TableOption(BigQueryRpc.Option.AUTODETECT_SCHEMA, autodetect); } + + /** Returns an option to specify the metadata of the table. */ + public static TableOption tableMetadataView(TableMetadataView tableMetadataView) { + return new TableOption(BigQueryRpc.Option.TABLE_METADATA_VIEW, tableMetadataView); + } } /* Class for specifying IAM options. */ diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java index cf1dfb4f7..3a58005ac 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java @@ -49,10 +49,10 @@ public static class Builder extends TableInfo.Builder { private final BigQuery bigquery; private final TableInfo.BuilderImpl infoBuilder; - Builder(BigQuery bigquery, TableId tableId, TableDefinition defintion) { + Builder(BigQuery bigquery, TableId tableId, TableDefinition definition) { this.bigquery = bigquery; this.infoBuilder = new TableInfo.BuilderImpl(); - this.infoBuilder.setTableId(tableId).setDefinition(defintion); + this.infoBuilder.setTableId(tableId).setDefinition(definition); } Builder(Table table) { diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java index 33480a91b..341640919 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java @@ -40,7 +40,7 @@ @InternalExtensionOnly public interface BigQueryRpc extends ServiceRpc { - // These options are part of the Google Cloud BigQuery query parameters + // These options are part of the Google Cloud BigQuery query parameters. enum Option { FIELDS("fields"), DELETE_CONTENTS("deleteContents"), @@ -56,7 +56,8 @@ enum Option { START_INDEX("startIndex"), STATE_FILTER("stateFilter"), TIMEOUT("timeoutMs"), - REQUESTED_POLICY_VERSION("requestedPolicyVersion"); + REQUESTED_POLICY_VERSION("requestedPolicyVersion"), + TABLE_METADATA_VIEW("view"); private final String value; diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index a63e3cc2b..96a10deb3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -294,6 +294,7 @@ public Table getTable( .get(projectId, datasetId, tableId) .setPrettyPrint(false) .setFields(Option.FIELDS.getString(options)) + .setView(getTableMetadataOption(options)) .execute(); } catch (IOException ex) { BigQueryException serviceException = translate(ex); @@ -304,6 +305,13 @@ public Table getTable( } } + private String getTableMetadataOption(Map options) { + if (options.containsKey(Option.TABLE_METADATA_VIEW)) { + return options.get(Option.TABLE_METADATA_VIEW).toString(); + } + return "STORAGE_STATS"; + } + @Override public Tuple> listTables( String projectId, String datasetId, Map options) { diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 1bb395e27..9d68e8710 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -49,6 +49,7 @@ import com.google.cloud.bigquery.BigQuery.JobListOption; import com.google.cloud.bigquery.BigQuery.JobOption; import com.google.cloud.bigquery.BigQuery.TableField; +import com.google.cloud.bigquery.BigQuery.TableMetadataView; import com.google.cloud.bigquery.BigQuery.TableOption; import com.google.cloud.bigquery.BigQueryDryRunResult; import com.google.cloud.bigquery.BigQueryError; @@ -1549,6 +1550,133 @@ public void testCreateAndGetTable() { assertTrue(remoteTable.delete()); } + @Test + public void testCreateAndGetTableWithBasicTableMetadataView() { + String tableName = "test_create_and_get_table_with_basic_metadata_view"; + TableId tableId = TableId.of(DATASET, tableName); + TimePartitioning partitioning = TimePartitioning.of(Type.DAY); + Clustering clustering = + Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build(); + StandardTableDefinition tableDefinition = + StandardTableDefinition.newBuilder() + .setSchema(TABLE_SCHEMA) + .setTimePartitioning(partitioning) + .setClustering(clustering) + .build(); + Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition)); + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + TableOption tableOption = BigQuery.TableOption.tableMetadataView(TableMetadataView.BASIC); + Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption); + assertNotNull(remoteTable); + assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType()); + assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); + // Next four values are considered transient fields that should not be calculated + assertNull(remoteTable.getLastModifiedTime()); + assertNull(remoteTable.getDefinition().getNumBytes()); + assertNull(remoteTable.getDefinition().getNumLongTermBytes()); + assertNull(remoteTable.getDefinition().getNumRows()); + assertTrue(remoteTable.delete()); + } + + @Test + public void testCreateAndGetTableWithFullTableMetadataView() { + String tableName = "test_create_and_get_table_with_full_metadata_view"; + TableId tableId = TableId.of(DATASET, tableName); + TimePartitioning partitioning = TimePartitioning.of(Type.DAY); + Clustering clustering = + Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build(); + StandardTableDefinition tableDefinition = + StandardTableDefinition.newBuilder() + .setSchema(TABLE_SCHEMA) + .setTimePartitioning(partitioning) + .setClustering(clustering) + .build(); + Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition)); + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + TableOption tableOption = BigQuery.TableOption.tableMetadataView(TableMetadataView.FULL); + Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption); + assertNotNull(remoteTable); + assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType()); + assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); + assertNotNull(remoteTable.getLastModifiedTime()); + assertNotNull(remoteTable.getDefinition().getNumBytes()); + assertNotNull(remoteTable.getDefinition().getNumLongTermBytes()); + assertNotNull(remoteTable.getDefinition().getNumRows()); + assertTrue(remoteTable.delete()); + } + + @Test + public void testCreateAndGetTableWithStorageStatsTableMetadataView() { + String tableName = "test_create_and_get_table_with_storage_stats_metadata_view"; + TableId tableId = TableId.of(DATASET, tableName); + TimePartitioning partitioning = TimePartitioning.of(Type.DAY); + Clustering clustering = + Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build(); + StandardTableDefinition tableDefinition = + StandardTableDefinition.newBuilder() + .setSchema(TABLE_SCHEMA) + .setTimePartitioning(partitioning) + .setClustering(clustering) + .build(); + Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition)); + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + TableOption tableOption = + BigQuery.TableOption.tableMetadataView(TableMetadataView.STORAGE_STATS); + Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption); + assertNotNull(remoteTable); + assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType()); + assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); + assertNotNull(remoteTable.getLastModifiedTime()); + assertNotNull(remoteTable.getDefinition().getNumBytes()); + assertNotNull(remoteTable.getDefinition().getNumLongTermBytes()); + assertNotNull(remoteTable.getDefinition().getNumRows()); + assertTrue(remoteTable.delete()); + } + + @Test + public void testCreateAndGetTableWithUnspecifiedTableMetadataView() { + String tableName = "test_create_and_get_table_with_unspecified_metadata_view"; + TableId tableId = TableId.of(DATASET, tableName); + TimePartitioning partitioning = TimePartitioning.of(Type.DAY); + Clustering clustering = + Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build(); + StandardTableDefinition tableDefinition = + StandardTableDefinition.newBuilder() + .setSchema(TABLE_SCHEMA) + .setTimePartitioning(partitioning) + .setClustering(clustering) + .build(); + Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition)); + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + TableOption tableOption = + BigQuery.TableOption.tableMetadataView(TableMetadataView.TABLE_METADATA_VIEW_UNSPECIFIED); + Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption); + assertNotNull(remoteTable); + assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType()); + assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); + assertNotNull(remoteTable.getLastModifiedTime()); + assertNotNull(remoteTable.getDefinition().getNumBytes()); + assertNotNull(remoteTable.getDefinition().getNumLongTermBytes()); + assertNotNull(remoteTable.getDefinition().getNumRows()); + assertTrue(remoteTable.delete()); + } + @Test public void testCreateAndGetTableWithSelectedField() { String tableName = "test_create_and_get_selected_fields_table";