From a98e86fc0b12663cb43da70193f1c191ca3c3d61 Mon Sep 17 00:00:00 2001 From: Sylvio Date: Tue, 15 Oct 2024 20:09:59 +0000 Subject: [PATCH 1/8] fix cloudbuild service account email --- cloud-composer-etl/prereq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud-composer-etl/prereq.sh b/cloud-composer-etl/prereq.sh index 8c5fe2f..f4e902e 100644 --- a/cloud-composer-etl/prereq.sh +++ b/cloud-composer-etl/prereq.sh @@ -54,7 +54,7 @@ gcloud services enable cloudbuild.googleapis.com \ echo "Granting Cloud Build's Service Account IAM roles to deploy the resources..." PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format='value(projectNumber)') -MEMBER=serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com +MEMBER=serviceAccount:$PROJECT_NUMBER-compute@developer.gserviceaccount.com add_iam_member $MEMBER roles/editor add_iam_member $MEMBER roles/iam.securityAdmin add_iam_member $MEMBER roles/compute.networkAdmin From 8731a1d8e535c9eecc4b380e922cbdf41a8fceab Mon Sep 17 00:00:00 2001 From: Sylvio Date: Tue, 15 Oct 2024 20:52:20 +0000 Subject: [PATCH 2/8] bump composer module to 5.1 --- cloud-composer-etl/infra/composer.tf | 7 +------ cloud-composer-etl/infra/provider.tf | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/cloud-composer-etl/infra/composer.tf b/cloud-composer-etl/infra/composer.tf index eb12a64..bfa6e22 100644 --- a/cloud-composer-etl/infra/composer.tf +++ b/cloud-composer-etl/infra/composer.tf @@ -12,19 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -data "google_composer_image_versions" "all" { - region = var.region -} - module "composer" { source = "terraform-google-modules/composer/google//modules/create_environment_v2" - version = "~> 3.4" + version = "~> 5.1" project_id = var.project_id region = var.region composer_env_name = var.composer_env_name composer_service_account = google_service_account.service_account.email - image_version = data.google_composer_image_versions.all.image_versions[0].image_version_id environment_size = "ENVIRONMENT_SIZE_SMALL" labels = local.resource_labels diff --git a/cloud-composer-etl/infra/provider.tf b/cloud-composer-etl/infra/provider.tf index 7faa364..41aff5c 100644 --- a/cloud-composer-etl/infra/provider.tf +++ b/cloud-composer-etl/infra/provider.tf @@ -18,11 +18,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "~> 4" + version = "~> 5.44" } google-beta = { source = "hashicorp/google-beta" - version = "~> 4" + version = "~> 5.44" } } provider_meta "google" { From 758d2d2d67728e7e9fd163a30f646723e597d8d7 Mon Sep 17 00:00:00 2001 From: Sylvio Date: Wed, 16 Oct 2024 12:15:10 +0000 Subject: [PATCH 3/8] bump cloudsql version --- cloud-composer-etl/infra/sql.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud-composer-etl/infra/sql.tf b/cloud-composer-etl/infra/sql.tf index ee108cf..9c2c82d 100644 --- a/cloud-composer-etl/infra/sql.tf +++ b/cloud-composer-etl/infra/sql.tf @@ -24,11 +24,11 @@ resource "random_id" "db_name_suffix" { resource "google_sql_database_instance" "instance" { name = local.db_instance_name region = var.region - database_version = "POSTGRES_14" + database_version = "POSTGRES_16" deletion_protection = false # not recommended for PROD settings { - tier = "db-custom-1-3840" + tier = "db-f1-micro" user_labels = local.resource_labels ip_configuration { From 65afa9b6a6e7915f78a8a9947a2750f5c842b592 Mon Sep 17 00:00:00 2001 From: Sylvio Date: Wed, 16 Oct 2024 12:15:27 +0000 Subject: [PATCH 4/8] remove google-beta provider --- cloud-composer-etl/infra/composer.tf | 1 - cloud-composer-etl/infra/network.tf | 1 - cloud-composer-etl/infra/provider.tf | 9 --------- 3 files changed, 11 deletions(-) diff --git a/cloud-composer-etl/infra/composer.tf b/cloud-composer-etl/infra/composer.tf index bfa6e22..9cb9d66 100644 --- a/cloud-composer-etl/infra/composer.tf +++ b/cloud-composer-etl/infra/composer.tf @@ -46,7 +46,6 @@ module "composer" { } depends_on = [ - module.vpc, google_project_iam_member.composer_v2_extension ] } diff --git a/cloud-composer-etl/infra/network.tf b/cloud-composer-etl/infra/network.tf index ccb5e99..0897a20 100644 --- a/cloud-composer-etl/infra/network.tf +++ b/cloud-composer-etl/infra/network.tf @@ -52,7 +52,6 @@ resource "google_compute_global_address" "service_range" { } resource "google_service_networking_connection" "private_service_connection" { - provider = google-beta network = module.vpc.network_id service = "servicenetworking.googleapis.com" reserved_peering_ranges = [google_compute_global_address.service_range.name] diff --git a/cloud-composer-etl/infra/provider.tf b/cloud-composer-etl/infra/provider.tf index 41aff5c..079d116 100644 --- a/cloud-composer-etl/infra/provider.tf +++ b/cloud-composer-etl/infra/provider.tf @@ -20,10 +20,6 @@ terraform { source = "hashicorp/google" version = "~> 5.44" } - google-beta = { - source = "hashicorp/google-beta" - version = "~> 5.44" - } } provider_meta "google" { module_name = "cloud-solutions/cloud-composer-etl-v0.1" @@ -34,8 +30,3 @@ provider "google" { project = var.project_id region = var.region } - -provider "google-beta" { - project = var.project_id - region = var.region -} \ No newline at end of file From 56094fa4596cd9ae2f5a19c0801f91c6e05eda52 Mon Sep 17 00:00:00 2001 From: Sylvio Date: Wed, 16 Oct 2024 12:23:56 +0000 Subject: [PATCH 5/8] bump vpc and add known issues --- cloud-composer-etl/README.md | 9 +++++++++ cloud-composer-etl/infra/network.tf | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cloud-composer-etl/README.md b/cloud-composer-etl/README.md index 77656ee..92c3698 100644 --- a/cloud-composer-etl/README.md +++ b/cloud-composer-etl/README.md @@ -111,3 +111,12 @@ Next step: do this same analysis with other dags. ``` gcloud builds submit . --config build/cloudbuild_destroy.yaml ``` + +## Known issues +If you face problems to delete the Service Networking peering: +``` + Error: Unable to remove Service Networking Connection, err: Error waiting for Delete Service Networking Connection: Error code 9, message: Failed to delete connection; Producer services (e.g. CloudSQL, Cloud Memstore, etc.) are still using this connection + ``` +Go to the [Console](https://console.cloud.google.com/networking/peering/list) and manually delete the peering. + +Then, run the Cloud Build Destroy job again. diff --git a/cloud-composer-etl/infra/network.tf b/cloud-composer-etl/infra/network.tf index 0897a20..d0b93fc 100644 --- a/cloud-composer-etl/infra/network.tf +++ b/cloud-composer-etl/infra/network.tf @@ -14,7 +14,7 @@ module "vpc" { source = "terraform-google-modules/network/google" - version = "~> 9.0" + version = "~> 9.3" project_id = var.project_id network_name = var.network_name routing_mode = "GLOBAL" From 68165e8ecf51ada277f5dfd37ee65249a3e207f7 Mon Sep 17 00:00:00 2001 From: Sylvio Date: Wed, 16 Oct 2024 12:46:25 +0000 Subject: [PATCH 6/8] replace module by native resource for better version control --- cloud-composer-etl/infra/composer.tf | 4 +-- cloud-composer-etl/infra/network.tf | 54 +++++++++++++--------------- cloud-composer-etl/infra/sql.tf | 2 +- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/cloud-composer-etl/infra/composer.tf b/cloud-composer-etl/infra/composer.tf index 9cb9d66..8595f3a 100644 --- a/cloud-composer-etl/infra/composer.tf +++ b/cloud-composer-etl/infra/composer.tf @@ -23,8 +23,8 @@ module "composer" { environment_size = "ENVIRONMENT_SIZE_SMALL" labels = local.resource_labels - network = module.vpc.network_name - subnetwork = var.composer_env_name + network = google_compute_network.vpc_network.id + subnetwork = google_compute_subnetwork.composer_subnetwork.id master_ipv4_cidr = var.composer_ip_ranges.master service_ip_allocation_range_name = "services" pod_ip_allocation_range_name = "pods" diff --git a/cloud-composer-etl/infra/network.tf b/cloud-composer-etl/infra/network.tf index d0b93fc..18f5f0a 100644 --- a/cloud-composer-etl/infra/network.tf +++ b/cloud-composer-etl/infra/network.tf @@ -12,54 +12,50 @@ # See the License for the specific language governing permissions and # limitations under the License. -module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 9.3" - project_id = var.project_id - network_name = var.network_name + +resource "google_compute_network" "vpc_network" { + name = var.network_name + description = "VPC for Data Platform" routing_mode = "GLOBAL" + auto_create_subnetworks = false +} - subnets = [ - { - subnet_name = var.composer_env_name - subnet_ip = var.composer_ip_ranges.nodes - subnet_region = var.region - subnet_private_access = true - }, - ] +resource "google_compute_subnetwork" "composer_subnetwork" { + name = var.composer_env_name + ip_cidr_range = var.composer_ip_ranges.nodes + region = var.region + network = google_compute_network.vpc_network.id + private_ip_google_access = true - secondary_ranges = { - "${var.composer_env_name}" = [ - { - range_name = "pods" - ip_cidr_range = var.composer_ip_ranges.pods - }, - { - range_name = "services" - ip_cidr_range = var.composer_ip_ranges.services - }, - ] + secondary_ip_range { + range_name = "pods" + ip_cidr_range = var.composer_ip_ranges.pods + } + secondary_ip_range { + range_name = "services" + ip_cidr_range = var.composer_ip_ranges.services } } + resource "google_compute_global_address" "service_range" { name = "service-networking-address" purpose = "VPC_PEERING" address_type = "INTERNAL" address = "10.200.0.0" prefix_length = 16 - network = module.vpc.network_name + network = google_compute_network.vpc_network.id } resource "google_service_networking_connection" "private_service_connection" { - network = module.vpc.network_id + network = google_compute_network.vpc_network.id service = "servicenetworking.googleapis.com" reserved_peering_ranges = [google_compute_global_address.service_range.name] } resource "google_compute_router" "nat_router" { - name = "${module.vpc.network_name}-nat-router" - network = module.vpc.network_self_link + name = "${google_compute_network.vpc_network.id}-nat-router" + network = google_compute_network.vpc_network.id region = var.region bgp { @@ -68,7 +64,7 @@ resource "google_compute_router" "nat_router" { } resource "google_compute_router_nat" "nat_gateway" { - name = "${module.vpc.network_name}-nat-gw" + name = "${google_compute_network.vpc_network.id}-nat-gw" router = google_compute_router.nat_router.name region = google_compute_router.nat_router.region nat_ip_allocate_option = "AUTO_ONLY" diff --git a/cloud-composer-etl/infra/sql.tf b/cloud-composer-etl/infra/sql.tf index 9c2c82d..d76ae4d 100644 --- a/cloud-composer-etl/infra/sql.tf +++ b/cloud-composer-etl/infra/sql.tf @@ -33,7 +33,7 @@ resource "google_sql_database_instance" "instance" { ip_configuration { ipv4_enabled = true - private_network = module.vpc.network_self_link + private_network = google_compute_network.vpc_network.id } } From eb10775776eb61278c8a4955ec50e42685a21b61 Mon Sep 17 00:00:00 2001 From: Sylvio Date: Wed, 16 Oct 2024 13:28:56 +0000 Subject: [PATCH 7/8] fix network references --- cloud-composer-etl/infra/network.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud-composer-etl/infra/network.tf b/cloud-composer-etl/infra/network.tf index 18f5f0a..bf01007 100644 --- a/cloud-composer-etl/infra/network.tf +++ b/cloud-composer-etl/infra/network.tf @@ -54,7 +54,7 @@ resource "google_service_networking_connection" "private_service_connection" { } resource "google_compute_router" "nat_router" { - name = "${google_compute_network.vpc_network.id}-nat-router" + name = "${google_compute_network.vpc_network.name}-nat-router" network = google_compute_network.vpc_network.id region = var.region @@ -64,7 +64,7 @@ resource "google_compute_router" "nat_router" { } resource "google_compute_router_nat" "nat_gateway" { - name = "${google_compute_network.vpc_network.id}-nat-gw" + name = "${google_compute_network.vpc_network.name}-nat-gw" router = google_compute_router.nat_router.name region = google_compute_router.nat_router.region nat_ip_allocate_option = "AUTO_ONLY" From 9d92b7e864c1ce96ee7b1d22bde645c5822a1973 Mon Sep 17 00:00:00 2001 From: Sylvio Date: Wed, 16 Oct 2024 13:32:14 +0000 Subject: [PATCH 8/8] fix network reference --- cloud-composer-etl/infra/composer.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud-composer-etl/infra/composer.tf b/cloud-composer-etl/infra/composer.tf index 8595f3a..3783e0b 100644 --- a/cloud-composer-etl/infra/composer.tf +++ b/cloud-composer-etl/infra/composer.tf @@ -23,8 +23,8 @@ module "composer" { environment_size = "ENVIRONMENT_SIZE_SMALL" labels = local.resource_labels - network = google_compute_network.vpc_network.id - subnetwork = google_compute_subnetwork.composer_subnetwork.id + network = google_compute_network.vpc_network.name + subnetwork = google_compute_subnetwork.composer_subnetwork.name master_ipv4_cidr = var.composer_ip_ranges.master service_ip_allocation_range_name = "services" pod_ip_allocation_range_name = "pods"