Allow configuring view expiration time in BigQuery

Additionally, set the config property as 30 minutes in BigQuery query runner.
trinodb · Mar 4, 2022 · d146e11 · d146e11
1 parent 132452e
commit d146e11
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 3 deletions.
diff --git a/docs/src/main/sphinx/connector/bigquery.rst b/docs/src/main/sphinx/connector/bigquery.rst
@@ -122,6 +122,7 @@ Property                                              Description
 ``bigquery.views-enabled``                            Enables the connector to read from views and not only tables.  ``false``
                                                       Please read `this section <#reading-from-views>`_ before
                                                       enabling this feature.
+``bigquery.view-expire-duration``                     Expire duration for the materialized view.                     ``24h``
 ``bigquery.view-materialization-project``             The project where the materialized view is going to be created The view's project
 ``bigquery.view-materialization-dataset``             The dataset where the materialized view is going to be created The view's dataset
 ``bigquery.views-cache-ttl``                          Duration for which the materialization of a view will be       ``15m``

diff --git a/plugin/trino-bigquery/pom.xml b/plugin/trino-bigquery/pom.xml
@@ -205,6 +205,11 @@
             <artifactId>grpc-api</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>javax.inject</groupId>
             <artifactId>javax.inject</artifactId>

diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConfig.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConfig.java
@@ -20,11 +20,13 @@
 import io.airlift.units.Duration;
 import io.airlift.units.MinDuration;
 
+import javax.annotation.PostConstruct;
 import javax.validation.constraints.Min;
 import javax.validation.constraints.NotNull;
 
 import java.util.Optional;
 
+import static com.google.common.base.Preconditions.checkState;
 import static java.util.concurrent.TimeUnit.HOURS;
 import static java.util.concurrent.TimeUnit.MINUTES;
 
@@ -38,6 +40,7 @@ public class BigQueryConfig
     private Optional<String> parentProjectId = Optional.empty();
     private Optional<Integer> parallelism = Optional.empty();
     private boolean viewsEnabled;
+    private Duration viewExpireDuration = new Duration(24, HOURS);
     private Optional<String> viewMaterializationProject = Optional.empty();
     private Optional<String> viewMaterializationDataset = Optional.empty();
     private int maxReadRowsRetries = DEFAULT_MAX_READ_ROWS_RETRIES;
@@ -98,9 +101,17 @@ public BigQueryConfig setViewsEnabled(boolean viewsEnabled)
         return this;
     }
 
-    public Duration getViewExpiration()
+    @NotNull
+    public Duration getViewExpireDuration()
+    {
+        return viewExpireDuration;
+    }
+
+    @Config("bigquery.view-expire-duration")
+    public BigQueryConfig setViewExpireDuration(Duration viewExpireDuration)
     {
-        return new Duration(24, HOURS);
+        this.viewExpireDuration = viewExpireDuration;
+        return this;
     }
 
     public Optional<String> getViewMaterializationProject()
@@ -186,4 +197,10 @@ public BigQueryConfig setServiceCacheTtl(Duration serviceCacheTtl)
         this.serviceCacheTtl = serviceCacheTtl;
         return this;
     }
+
+    @PostConstruct
+    public void validate()
+    {
+        checkState(viewExpireDuration.toMillis() > viewsCacheTtl.toMillis(), "View expiration duration must be longer than view cache TTL");
+    }
 }
diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitManager.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitManager.java
@@ -74,7 +74,7 @@ public BigQuerySplitManager(
         this.bigQueryReadClientFactory = requireNonNull(bigQueryReadClientFactory, "bigQueryReadClientFactory cannot be null");
         this.parallelism = config.getParallelism();
         this.viewEnabled = config.isViewsEnabled();
-        this.viewExpiration = config.getViewExpiration();
+        this.viewExpiration = config.getViewExpireDuration();
         this.nodeManager = requireNonNull(nodeManager, "nodeManager cannot be null");
     }
 

diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BigQueryQueryRunner.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BigQueryQueryRunner.java
@@ -71,6 +71,7 @@ public static DistributedQueryRunner createQueryRunner(Map<String, String> extra
 
             connectorProperties = new HashMap<>(ImmutableMap.copyOf(connectorProperties));
             connectorProperties.putIfAbsent("bigquery.views-enabled", "true");
+            connectorProperties.putIfAbsent("bigquery.view-expire-duration", "30m");
 
             queryRunner.installPlugin(new BigQueryPlugin());
             queryRunner.createCatalog(

diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConfig.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConfig.java
@@ -23,7 +23,9 @@
 import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
 import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;
 import static java.util.concurrent.TimeUnit.DAYS;
+import static java.util.concurrent.TimeUnit.HOURS;
 import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 public class TestBigQueryConfig
 {
@@ -34,6 +36,7 @@ public void testDefaults()
                 .setProjectId(null)
                 .setParentProjectId(null)
                 .setParallelism(null)
+                .setViewExpireDuration(new Duration(24, HOURS))
                 .setViewMaterializationProject(null)
                 .setViewMaterializationDataset(null)
                 .setMaxReadRowsRetries(3)
@@ -51,6 +54,7 @@ public void testExplicitPropertyMappingsWithCredentialsKey()
                 .put("bigquery.parent-project-id", "ppid")
                 .put("bigquery.parallelism", "20")
                 .put("bigquery.views-enabled", "true")
+                .put("bigquery.view-expire-duration", "30m")
                 .put("bigquery.view-materialization-project", "vmproject")
                 .put("bigquery.view-materialization-dataset", "vmdataset")
                 .put("bigquery.max-read-rows-retries", "10")
@@ -64,6 +68,7 @@ public void testExplicitPropertyMappingsWithCredentialsKey()
                 .setParentProjectId("ppid")
                 .setParallelism(20)
                 .setViewsEnabled(true)
+                .setViewExpireDuration(new Duration(30, MINUTES))
                 .setViewMaterializationProject("vmproject")
                 .setViewMaterializationDataset("vmdataset")
                 .setMaxReadRowsRetries(10)
@@ -73,4 +78,15 @@ public void testExplicitPropertyMappingsWithCredentialsKey()
 
         assertFullMapping(properties, expected);
     }
+
+    @Test
+    public void testInvalidViewSetting()
+    {
+        assertThatThrownBy(() -> new BigQueryConfig()
+                .setViewExpireDuration(new Duration(5, MINUTES))
+                .setViewsCacheTtl(new Duration(10, MINUTES))
+                .validate())
+                .isInstanceOf(IllegalStateException.class)
+                .hasMessage("View expiration duration must be longer than view cache TTL");
+    }
 }