Merge pull request #1142 from GoogleCloudPlatform/jccb/bq-factory-fix

Fix bq factory docs

blueprints/factories/bigquery-factory/README.md

@@ -1,36 +1,18 @@
 # Google Cloud BQ Factory
 
-This module allows creation and management of BigQuery datasets and views as well as tables by defining them in well formatted `yaml` files.
+This module allows creation and management of BigQuery datasets tables and views by defining them in well-formatted YAML files. YAML abstraction for BQ can simplify users onboarding and also makes creation of tables easier compared to HCL.
 
-Yaml abstraction for BQ can simplify users onboarding and also makes creation of tables easier compared to HCL.
+This factory is based on the [BQ dataset module](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/modules/bigquery-dataset) which currently only supports tables and views. As soon as external table and materialized view support is added, this factory will be enhanced accordingly.
 
-Subfolders distinguish between views and tables and ensures easier navigation for users.
-
-This factory is based on the [BQ dataset module](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/modules/bigquery-dataset) which currently only supports tables and views. As soon as external table and materialized view support is added, factory will be enhanced accordingly.
-
-You can create as many files as you like, the code will loop through it and create the required variables in order to execute everything accordingly.
+You can create as many files as you like, the code will loop through it and create everything accordingly.
 
 ## Example
 
 ### Terraform code
 
-```hcl
-module "bq" {
-  source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric/modules/bigquery-dataset"
-
-  for_each   = local.output
-  project_id = var.project_id
-  id         = each.key
-  views      = try(each.value.views, null)
-  tables     = try(each.value.tables, null)
-}
-# tftest skip
-```
-
-### Configuration Structure
-
+In this section we show how to create tables and views from a file structure simlar to the one shown below.
 ```bash
-base_folder
+bigquery
 │
 ├── tables
 │   ├── table_a.yaml
@@ -40,32 +22,43 @@ base_folder
 │   ├── view_b.yaml
 ```
 
-## YAML structure and definition formatting
+First we create the table definition in `bigquery/tables/countries.yaml`.
 
-### Tables
+```yaml
+# tftest-file id=table path=bigquery/tables/countries.yaml
+dataset: my_dataset
+table: countries
+deletion_protection: true
+labels:
+  env: prod
+schema:
+  - name: country
+    type: STRING
+  - name: population
+    type: INT64
+```
 
-Table definition to be placed in a set of yaml files in the corresponding subfolder. Structure should look as following:
+And a view in  `bigquery/views/population.yaml`.
 
 ```yaml
-
-dataset: # required name of the dataset the table is to be placed in
-table: # required descriptive name of the table
-schema: # required schema in JSON FORMAT Example: [{name: "test", type: "STRING"},{name: "test2", type: "INT64"}]
-labels: # not required, defaults to {}, Example: {"a":"thisislabela","b":"thisislabelb"}
-use_legacy_sql: boolean # not required, defaults to false
-deletion_protection: boolean # not required, defaults to false
+# tftest-file id=view path=bigquery/views/population.yaml
+dataset: my_dataset
+view: department
+query: SELECT SUM(population) from  my_dataset.countries
+labels:
+  env: prod
 ```
 
-### Views
-View definition to be placed in a set of yaml files in the corresponding subfolder. Structure should look as following:
+With this file structure, we can use the factory as follows:
 
-```yaml
-dataset: # required, name of the dataset the view is to be placed in
-view: # required, descriptive name of the view
-query: # required, SQL Query for the view in quotes
-labels: # not required, defaults to {}, Example: {"a":"thisislabela","b":"thisislabelb"}
-use_legacy_sql: bool # not required, defaults to false
-deletion_protection: bool # not required, defaults to false
+```hcl
+module "bq" {
+  source      = "./fabric/blueprints/factories/bigquery-factory"
+  project_id  = var.project_id
+  tables_path = "bigquery/tables"
+  views_path  = "bigquery/views"
+}
+# tftest modules=2 resources=3 files=table,view inventory=simple.yaml
 ```
 <!-- BEGIN TFDOC -->
 
@@ -74,8 +67,8 @@ deletion_protection: bool # not required, defaults to false
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
 | [project_id](variables.tf#L17) | Project ID. | <code>string</code> | ✓ |  |
-| [tables_dir](variables.tf#L22) | Relative path for the folder storing table data. | <code>string</code> | ✓ |  |
-| [views_dir](variables.tf#L27) | Relative path for the folder storing view data. | <code>string</code> | ✓ |  |
+| [tables_path](variables.tf#L22) | Relative path for the folder storing table data. | <code>string</code> | ✓ |  |
+| [views_path](variables.tf#L27) | Relative path for the folder storing view data. | <code>string</code> | ✓ |  |
 
 <!-- END TFDOC -->
 ## TODO

blueprints/factories/bigquery-factory/main.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,17 +16,22 @@
 
 locals {
   views = {
-    for f in fileset("${var.views_dir}", "**/*.yaml") :
-    trimsuffix(f, ".yaml") => yamldecode(file("${var.views_dir}/${f}"))
+    for f in fileset(var.views_path, "**/*.yaml") :
+    trimsuffix(f, ".yaml") => yamldecode(file("${var.views_path}/${f}"))
   }
 
   tables = {
-    for f in fileset("${var.tables_dir}", "**/*.yaml") :
-    trimsuffix(f, ".yaml") => yamldecode(file("${var.tables_dir}/${f}"))
+    for f in fileset(var.tables_path, "**/*.yaml") :
+    trimsuffix(f, ".yaml") => yamldecode(file("${var.tables_path}/${f}"))
   }
 
-  output = {
-    for dataset in distinct([for v in values(merge(local.views, local.tables)) : v.dataset]) :
+  all_datasets = distinct(concat(
+    [for x in values(local.tables) : x.dataset],
+    [for x in values(local.views) : x.dataset]
+  ))
+
+  datasets = {
+    for dataset in local.all_datasets :
     dataset => {
       "views" = {
         for k, v in local.views :
@@ -57,9 +62,8 @@ locals {
 }
 
 module "bq" {
-  source = "../../../modules/bigquery-dataset"
-
-  for_each   = local.output
+  source     = "../../../modules/bigquery-dataset"
+  for_each   = local.datasets
   project_id = var.project_id
   id         = each.key
   views      = try(each.value.views, null)

blueprints/factories/bigquery-factory/variables.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,12 +19,12 @@ variable "project_id" {
   type        = string
 }
 
-variable "tables_dir" {
+variable "tables_path" {
   description = "Relative path for the folder storing table data."
   type        = string
 }
 
-variable "views_dir" {
+variable "views_path" {
   description = "Relative path for the folder storing view data."
   type        = string
 }

tests/blueprints/factories/bigquery_factory/examples/simple.yaml

@@ -0,0 +1,40 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+  module.bq.module.bq["my_dataset"].google_bigquery_dataset.default:
+    dataset_id: my_dataset
+    project: project-id
+  module.bq.module.bq["my_dataset"].google_bigquery_table.default["countries"]:
+    dataset_id: my_dataset
+    friendly_name: countries
+    labels:
+      env: prod
+    project: project-id
+    schema: '[{"name":"country","type":"STRING"},{"name":"population","type":"INT64"}]'
+    table_id: countries
+  module.bq.module.bq["my_dataset"].google_bigquery_table.views["department"]:
+    dataset_id: my_dataset
+    friendly_name: department
+    labels:
+      env: prod
+    project: project-id
+    table_id: department
+    view:
+    - query: SELECT SUM(population) from  my_dataset.countries
+      use_legacy_sql: false
+
+counts:
+  google_bigquery_dataset: 1
+  google_bigquery_table: 2