GoogleCloudPlatform · danawillow · May 18, 2020 · May 15, 2020
diff --git a/products/bigquery/terraform.yaml b/products/bigquery/terraform.yaml
@@ -116,13 +116,32 @@ overrides: !ruby/object:Overrides::ResourceOverrides
           account_name: "bqowner"
         ignore_read_extra:
           - "etag"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "bigquery_job_query_table_reference"
+        primary_resource_id: "job"
+        vars:
+          job_id: "job_query"
+          account_name: "bqowner"
+        ignore_read_extra:
+          - "etag"
+          - "query.0.default_dataset.0.dataset_id"
+          - "query.0.destination_table.0.table_id"
       - !ruby/object:Provider::Terraform::Examples
         name: "bigquery_job_load"
         primary_resource_id: "job"
         vars:
           job_id: "job_load"
         ignore_read_extra:
           - "etag"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "bigquery_job_load_table_reference"
+        primary_resource_id: "job"
+        vars:
+          job_id: "job_load"
+        ignore_read_extra:
+          - "etag"
+          - "load.0.destination_table.0.table_id"
+        skip_docs: true  # there are a lot of examples for this resource, so omitting some that are similar to others
       - !ruby/object:Provider::Terraform::Examples
         name: "bigquery_job_copy"
         primary_resource_id: "job"
@@ -135,6 +154,22 @@ overrides: !ruby/object:Overrides::ResourceOverrides
           project: :PROJECT_NAME
         ignore_read_extra:
           - "etag"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "bigquery_job_copy_table_reference"
+        primary_resource_id: "job"
+        vars:
+          job_id: "job_copy"
+          account_name: "bqowner"
+          key_name: "example-key"
+          keyring_name: "example-keyring"
+        test_env_vars:
+          project: :PROJECT_NAME
+        ignore_read_extra:
+          - "etag"
+          - "copy.0.destination_table.0.table_id"
+          - "copy.0.source_tables.0.table_id"
+          - "copy.0.source_tables.1.table_id"
+        skip_docs: true  # there are a lot of examples for this resource, so omitting some that are similar to others
       - !ruby/object:Provider::Terraform::Examples
         name: "bigquery_job_extract"
         primary_resource_id: "job"
@@ -143,11 +178,60 @@ overrides: !ruby/object:Overrides::ResourceOverrides
           account_name: "bqowner"
         ignore_read_extra:
           - "etag"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "bigquery_job_extract_table_reference"
+        primary_resource_id: "job"
+        vars:
+          job_id: "job_extract"
+          account_name: "bqowner"
+        ignore_read_extra:
+          - "etag"
+          - "extract.0.source_table.0.table_id"
+        skip_docs: true  # there are a lot of examples for this resource, so omitting some that are similar to others
     properties:
       id: !ruby/object:Overrides::Terraform::PropertyOverride
         exclude: true
       configuration: !ruby/object:Overrides::Terraform::PropertyOverride
         flatten_object: true
+      configuration.copy.destinationTable: !ruby/object:Overrides::Terraform::PropertyOverride
+        custom_expand: 'templates/terraform/custom_expand/bigquery_table_ref.go.erb'
+        custom_flatten: 'templates/terraform/custom_flatten/bigquery_table_ref_copy_destinationtable.go.erb'
+      configuration.copy.destinationTable.projectId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.copy.destinationTable.datasetId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.copy.destinationTable.tableId: !ruby/object:Overrides::Terraform::PropertyOverride
+        description: |
+          The table. Can be specified `{{table_id}}` if `project_id` and `dataset_id` are also set,
+          or of the form `projects/{{project}}/datasets/{{dataset_id}}/tables/{{table_id}}` if not.
+      configuration.copy.sourceTables: !ruby/object:Overrides::Terraform::PropertyOverride
+        custom_expand: 'templates/terraform/custom_expand/bigquery_table_ref_array.go.erb'
+        custom_flatten: 'templates/terraform/custom_flatten/bigquery_table_ref_copy_sourcetables.go.erb'
+      configuration.copy.sourceTables.projectId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.copy.sourceTables.datasetId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.copy.sourceTables.tableId: !ruby/object:Overrides::Terraform::PropertyOverride
+        description: |
+          The table. Can be specified `{{table_id}}` if `project_id` and `dataset_id` are also set,
+          or of the form `projects/{{project}}/datasets/{{dataset_id}}/tables/{{table_id}}` if not.
+      configuration.load.destinationTable: !ruby/object:Overrides::Terraform::PropertyOverride
+        custom_expand: 'templates/terraform/custom_expand/bigquery_table_ref.go.erb'
+        custom_flatten: 'templates/terraform/custom_flatten/bigquery_table_ref_load_destinationtable.go.erb'
+      configuration.load.destinationTable.projectId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.load.destinationTable.datasetId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.load.destinationTable.tableId: !ruby/object:Overrides::Terraform::PropertyOverride
+        description: |
+          The table. Can be specified `{{table_id}}` if `project_id` and `dataset_id` are also set,
+          or of the form `projects/{{project}}/datasets/{{dataset_id}}/tables/{{table_id}}` if not.
       configuration.load.skipLeadingRows: !ruby/object:Overrides::Terraform::PropertyOverride
         validation: !ruby/object:Provider::Terraform::Validation
           function: 'validation.IntAtLeast(0)'
@@ -159,11 +243,48 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         default_from_api: true
       configuration.extract.destinationFormat: !ruby/object:Overrides::Terraform::PropertyOverride
         default_from_api: true
+      configuration.extract.sourceTable: !ruby/object:Overrides::Terraform::PropertyOverride
+        custom_expand: 'templates/terraform/custom_expand/bigquery_table_ref.go.erb'
+        custom_flatten: 'templates/terraform/custom_flatten/bigquery_table_ref_extract_sourcetable.go.erb'
+      configuration.extract.sourceTable.projectId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.extract.sourceTable.datasetId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.extract.sourceTable.tableId: !ruby/object:Overrides::Terraform::PropertyOverride
+        description: |
+          The table. Can be specified `{{table_id}}` if `project_id` and `dataset_id` are also set,
+          or of the form `projects/{{project}}/datasets/{{dataset_id}}/tables/{{table_id}}` if not.
+      configuration.query.destinationTable: !ruby/object:Overrides::Terraform::PropertyOverride
+        custom_expand: 'templates/terraform/custom_expand/bigquery_table_ref.go.erb'
+        custom_flatten: 'templates/terraform/custom_flatten/bigquery_table_ref_query_destinationtable.go.erb'
+      configuration.query.destinationTable.projectId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.query.destinationTable.datasetId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.query.destinationTable.tableId: !ruby/object:Overrides::Terraform::PropertyOverride
+        description: |
+          The table. Can be specified `{{table_id}}` if `project_id` and `dataset_id` are also set,
+          or of the form `projects/{{project}}/datasets/{{dataset_id}}/tables/{{table_id}}` if not.
+      configuration.query.defaultDataset: !ruby/object:Overrides::Terraform::PropertyOverride
+        custom_expand: 'templates/terraform/custom_expand/bigquery_dataset_ref.go.erb'
+        custom_flatten: 'templates/terraform/custom_flatten/bigquery_dataset_ref.go.erb'
+      configuration.query.defaultDataset.projectId: !ruby/object:Overrides::Terraform::PropertyOverride
+        required: false
+        default_from_api: true
+      configuration.query.defaultDataset.datasetId: !ruby/object:Overrides::Terraform::PropertyOverride
+        description: |
+          The dataset. Can be specified `{{dataset_id}}` if `project_id` is also set,
+          or of the form `projects/{{project}}/datasets/{{dataset_id}}` if not.
       jobReference: !ruby/object:Overrides::Terraform::PropertyOverride
         flatten_object: true
       jobReference.projectId: !ruby/object:Overrides::Terraform::PropertyOverride
         exclude: true
     custom_code: !ruby/object:Provider::Terraform::CustomCode
+      constants: templates/terraform/constants/bigquery_job.go
       encoder: templates/terraform/encoders/bigquery_job.go.erb
   Table: !ruby/object:Overrides::Terraform::ResourceOverride
     exclude: true

diff --git a/provider/terraform/examples.rb b/provider/terraform/examples.rb
@@ -111,6 +111,9 @@ class Examples < Api::Object
       # Whether to skip generating tests for this resource
       attr_reader :skip_test
 
+      # Whether to skip generating docs for this example
+      attr_reader :skip_docs
+
       # The name of the primary resource for use in IAM tests. IAM tests need
       # a reference to the primary resource to create IAM policies for
       attr_reader :primary_resource_name
@@ -267,6 +270,7 @@ def validate
         check :ignore_read_extra, type: Array, item_type: String, default: []
         check :primary_resource_name, type: String
         check :skip_test, type: TrueClass
+        check :skip_docs, type: TrueClass
         check :config_path, type: String, default: "templates/terraform/examples/#{name}.tf.erb"
         check :skip_vcr, type: TrueClass
       end

diff --git a/templates/terraform/constants/bigquery_job.go b/templates/terraform/constants/bigquery_job.go
@@ -0,0 +1,18 @@
+<%# The license inside this block applies to this file.
+	# Copyright 2020 Google Inc.
+	# Licensed under the Apache License, Version 2.0 (the "License");
+	# you may not use this file except in compliance with the License.
+	# You may obtain a copy of the License at
+	#
+	#     http://www.apache.org/licenses/LICENSE-2.0
+	#
+	# Unless required by applicable law or agreed to in writing, software
+	# distributed under the License is distributed on an "AS IS" BASIS,
+	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	# See the License for the specific language governing permissions and
+	# limitations under the License.
+-%>
+var (
+	bigqueryDatasetRegexp = regexp.MustCompile("projects/(.+)/datasets/(.+)")
+	bigqueryTableRegexp = regexp.MustCompile("projects/(.+)/datasets/(.+)/tables/(.+)")
+)
diff --git a/templates/terraform/custom_expand/bigquery_dataset_ref.go.erb b/templates/terraform/custom_expand/bigquery_dataset_ref.go.erb
@@ -0,0 +1,40 @@
+<%# # the license inside this if block pertains to this file
+		# Copyright 2020 Google Inc.
+		# Licensed under the Apache License, Version 2.0 (the "License");
+		# you may not use this file except in compliance with the License.
+		# You may obtain a copy of the License at
+		#
+		#     http://www.apache.org/licenses/LICENSE-2.0
+		#
+		# Unless required by applicable law or agreed to in writing, software
+		# distributed under the License is distributed on an "AS IS" BASIS,
+		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+		# See the License for the specific language governing permissions and
+		# limitations under the License.
+#%>
+func expand<%= prefix -%><%= titlelize_property(property) -%>(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	l := v.([]interface{})
+	if len(l) == 0 || l[0] == nil {
+			return nil, nil
+	}
+	raw := l[0]
+	original := raw.(map[string]interface{})
+	transformed := make(map[string]interface{})
+
+	transformedProjectId := original["project_id"]
+	if val := reflect.ValueOf(transformedProjectId); val.IsValid() && !isEmptyValue(val) {
+		transformed["projectId"] = transformedProjectId
+	}
+
+	transformedDatasetId := original["dataset_id"]
+	if val := reflect.ValueOf(transformedDatasetId); val.IsValid() && !isEmptyValue(val) {
+		transformed["datasetId"] = transformedDatasetId
+	}
+
+	if parts := bigqueryDatasetRegexp.FindStringSubmatch(transformedDatasetId.(string)); parts != nil {
+		transformed["projectId"] = parts[1]
+		transformed["datasetId"] = parts[2]
+	}
+
+	return transformed, nil
+}
diff --git a/templates/terraform/custom_expand/bigquery_table_ref.go.erb b/templates/terraform/custom_expand/bigquery_table_ref.go.erb
@@ -0,0 +1,46 @@
+<%# # the license inside this if block pertains to this file
+		# Copyright 2020 Google Inc.
+		# Licensed under the Apache License, Version 2.0 (the "License");
+		# you may not use this file except in compliance with the License.
+		# You may obtain a copy of the License at
+		#
+		#     http://www.apache.org/licenses/LICENSE-2.0
+		#
+		# Unless required by applicable law or agreed to in writing, software
+		# distributed under the License is distributed on an "AS IS" BASIS,
+		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+		# See the License for the specific language governing permissions and
+		# limitations under the License.
+#%>
+func expand<%= prefix -%><%= titlelize_property(property) -%>(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	l := v.([]interface{})
+	if len(l) == 0 || l[0] == nil {
+			return nil, nil
+	}
+	raw := l[0]
+	original := raw.(map[string]interface{})
+	transformed := make(map[string]interface{})
+
+	transformedProjectId := original["project_id"]
+	if val := reflect.ValueOf(transformedProjectId); val.IsValid() && !isEmptyValue(val) {
+		transformed["projectId"] = transformedProjectId
+	}
+
+	transformedDatasetId := original["dataset_id"]
+	if val := reflect.ValueOf(transformedDatasetId); val.IsValid() && !isEmptyValue(val) {
+		transformed["datasetId"] = transformedDatasetId
+	}
+
+	transformedTableId := original["table_id"]
+	if val := reflect.ValueOf(transformedTableId); val.IsValid() && !isEmptyValue(val) {
+		transformed["tableId"] = transformedTableId
+	}
+
+	if parts := bigqueryTableRegexp.FindStringSubmatch(transformedTableId.(string)); parts != nil {
+		transformed["projectId"] = parts[1]
+		transformed["datasetId"] = parts[2]
+		transformed["tableId"] =  parts[3]
+	}
+
+	return transformed, nil
+}
diff --git a/templates/terraform/custom_expand/bigquery_table_ref_array.go.erb b/templates/terraform/custom_expand/bigquery_table_ref_array.go.erb
@@ -0,0 +1,50 @@
+<%# # the license inside this if block pertains to this file
+		# Copyright 2020 Google Inc.
+		# Licensed under the Apache License, Version 2.0 (the "License");
+		# you may not use this file except in compliance with the License.
+		# You may obtain a copy of the License at
+		#
+		#     http://www.apache.org/licenses/LICENSE-2.0
+		#
+		# Unless required by applicable law or agreed to in writing, software
+		# distributed under the License is distributed on an "AS IS" BASIS,
+		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+		# See the License for the specific language governing permissions and
+		# limitations under the License.
+#%>
+func expand<%= prefix -%><%= titlelize_property(property) -%>(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	l := v.([]interface{})
+	req := make([]interface{}, 0, len(l))
+	for _, raw := range l {
+		if raw == nil {
+			continue
+		}
+		original := raw.(map[string]interface{})
+		transformed := make(map[string]interface{})
+
+		transformedProjectId := original["project_id"]
+		if val := reflect.ValueOf(transformedProjectId); val.IsValid() && !isEmptyValue(val) {
+			transformed["projectId"] = transformedProjectId
+		}
+
+		transformedDatasetId := original["dataset_id"]
+		if val := reflect.ValueOf(transformedDatasetId); val.IsValid() && !isEmptyValue(val) {
+			transformed["datasetId"] = transformedDatasetId
+		}
+
+		transformedTableId := original["table_id"]
+		if val := reflect.ValueOf(transformedTableId); val.IsValid() && !isEmptyValue(val) {
+			transformed["tableId"] = transformedTableId
+		}
+
+		tableRef := regexp.MustCompile("projects/(.+)/datasets/(.+)/tables/(.+)")
+		if parts := tableRef.FindStringSubmatch(transformedTableId.(string)); parts != nil {
+			transformed["projectId"] = parts[1]
+			transformed["datasetId"] = parts[2]
+			transformed["tableId"] =  parts[3]
+		}
+
+		req = append(req, transformed)
+	}
+	return req, nil
+}
diff --git a/templates/terraform/custom_flatten/bigquery_dataset_ref.go.erb b/templates/terraform/custom_flatten/bigquery_dataset_ref.go.erb
@@ -0,0 +1,32 @@
+<%# The license inside this block applies to this file.
+	# Copyright 2020 Google Inc.
+	# Licensed under the Apache License, Version 2.0 (the "License");
+	# you may not use this file except in compliance with the License.
+	# You may obtain a copy of the License at
+	#
+	#     http://www.apache.org/licenses/LICENSE-2.0
+	#
+	# Unless required by applicable law or agreed to in writing, software
+	# distributed under the License is distributed on an "AS IS" BASIS,
+	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	# See the License for the specific language governing permissions and
+	# limitations under the License.
+-%>
+func flatten<%= prefix -%><%= titlelize_property(property) -%>(v interface{}, d *schema.ResourceData, config *Config) interface{} {
+	if v == nil {
+		return nil
+	}
+	original := v.(map[string]interface{})
+	if len(original) == 0 {
+		return nil
+	}
+	transformed := make(map[string]interface{})
+	transformed["project_id"] = original["projectId"]
+	transformed["dataset_id"] = original["datasetId"]
+
+	if bigqueryDatasetRegexp.MatchString(d.Get("query.0.default_dataset.0.dataset_id").(string)) {
+		// The user specified the dataset_id as a URL, so store it in state that way
+		transformed["dataset_id"] = fmt.Sprintf("projects/%s/datasets/%s", transformed["project_id"], transformed["dataset_id"])
+	}
+	return []interface{}{transformed}
+}