Merge pull request #207 from GoogleCloudPlatform/composer-fixes

Cloud Composer fixes
GoogleCloudPlatform · Oct 16, 2024 · 93a4823 · 93a4823
2 parents b67e43c + 9d92b7e
commit 93a4823
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 53 deletions.
diff --git a/cloud-composer-etl/README.md b/cloud-composer-etl/README.md
@@ -111,3 +111,12 @@ Next step: do this same analysis with other dags.
 ```
 gcloud builds submit . --config build/cloudbuild_destroy.yaml
 ```
+
+## Known issues
+If you face problems to delete the Service Networking peering:
+```
+ Error: Unable to remove Service Networking Connection, err: Error waiting for Delete Service Networking Connection: Error code 9, message: Failed to delete connection; Producer services (e.g. CloudSQL, Cloud Memstore, etc.) are still using this connection
+ ```
+Go to the [Console](https://console.cloud.google.com/networking/peering/list) and manually delete the peering.
+
+Then, run the Cloud Build Destroy job again.
diff --git a/cloud-composer-etl/infra/composer.tf b/cloud-composer-etl/infra/composer.tf
@@ -12,24 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-data "google_composer_image_versions" "all" {
-  region = var.region
-}
-
 module "composer" {
   source  = "terraform-google-modules/composer/google//modules/create_environment_v2"
-  version = "~> 3.4"
+  version = "~> 5.1"
 
   project_id               = var.project_id
   region                   = var.region
   composer_env_name        = var.composer_env_name
   composer_service_account = google_service_account.service_account.email
-  image_version            = data.google_composer_image_versions.all.image_versions[0].image_version_id
   environment_size         = "ENVIRONMENT_SIZE_SMALL"
   labels                   = local.resource_labels
 
-  network                          = module.vpc.network_name
-  subnetwork                       = var.composer_env_name
+  network                          = google_compute_network.vpc_network.name
+  subnetwork                       = google_compute_subnetwork.composer_subnetwork.name
   master_ipv4_cidr                 = var.composer_ip_ranges.master
   service_ip_allocation_range_name = "services"
   pod_ip_allocation_range_name     = "pods"
@@ -51,7 +46,6 @@ module "composer" {
   }
 
   depends_on = [
-    module.vpc,
     google_project_iam_member.composer_v2_extension
   ]
 }
diff --git a/cloud-composer-etl/infra/network.tf b/cloud-composer-etl/infra/network.tf
@@ -12,55 +12,50 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-module "vpc" {
-  source       = "terraform-google-modules/network/google"
-  version      = "~> 9.0"
-  project_id   = var.project_id
-  network_name = var.network_name
+
+resource "google_compute_network" "vpc_network" {
+  name = var.network_name
+  description = "VPC for Data Platform"
   routing_mode = "GLOBAL"
+  auto_create_subnetworks = false
+}
 
-  subnets = [
-    {
-      subnet_name           = var.composer_env_name
-      subnet_ip             = var.composer_ip_ranges.nodes
-      subnet_region         = var.region
-      subnet_private_access = true
-    },
-  ]
+resource "google_compute_subnetwork" "composer_subnetwork" {
+  name          = var.composer_env_name
+  ip_cidr_range = var.composer_ip_ranges.nodes
+  region        = var.region
+  network       = google_compute_network.vpc_network.id
+  private_ip_google_access = true
 
-  secondary_ranges = {
-    "${var.composer_env_name}" = [
-      {
-        range_name    = "pods"
-        ip_cidr_range = var.composer_ip_ranges.pods
-      },
-      {
-        range_name    = "services"
-        ip_cidr_range = var.composer_ip_ranges.services
-      },
-    ]
+  secondary_ip_range {
+    range_name    = "pods"
+    ip_cidr_range = var.composer_ip_ranges.pods
+  }
+  secondary_ip_range {
+    range_name    = "services"
+    ip_cidr_range = var.composer_ip_ranges.services
   }
 }
 
+
 resource "google_compute_global_address" "service_range" {
   name          = "service-networking-address"
   purpose       = "VPC_PEERING"
   address_type  = "INTERNAL"
   address       = "10.200.0.0"
   prefix_length = 16
-  network       = module.vpc.network_name
+  network       = google_compute_network.vpc_network.id
 }
 
 resource "google_service_networking_connection" "private_service_connection" {
-  provider                = google-beta
-  network                 = module.vpc.network_id
+  network                 = google_compute_network.vpc_network.id
   service                 = "servicenetworking.googleapis.com"
   reserved_peering_ranges = [google_compute_global_address.service_range.name]
 }
 
 resource "google_compute_router" "nat_router" {
-  name    = "${module.vpc.network_name}-nat-router"
-  network = module.vpc.network_self_link
+  name    = "${google_compute_network.vpc_network.name}-nat-router"
+  network = google_compute_network.vpc_network.id
   region  = var.region
 
   bgp {
@@ -69,7 +64,7 @@ resource "google_compute_router" "nat_router" {
 }
 
 resource "google_compute_router_nat" "nat_gateway" {
-  name                               = "${module.vpc.network_name}-nat-gw"
+  name                               = "${google_compute_network.vpc_network.name}-nat-gw"
   router                             = google_compute_router.nat_router.name
   region                             = google_compute_router.nat_router.region
   nat_ip_allocate_option             = "AUTO_ONLY"

diff --git a/cloud-composer-etl/infra/provider.tf b/cloud-composer-etl/infra/provider.tf
@@ -18,11 +18,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4"
-    }
-    google-beta = {
-      source  = "hashicorp/google-beta"
-      version = "~> 4"
+      version = "~> 5.44"
     }
   }
   provider_meta "google" {
@@ -34,8 +30,3 @@ provider "google" {
   project = var.project_id
   region  = var.region
 }
-
-provider "google-beta" {
-  project = var.project_id
-  region  = var.region
-}
diff --git a/cloud-composer-etl/infra/sql.tf b/cloud-composer-etl/infra/sql.tf
@@ -24,16 +24,16 @@ resource "random_id" "db_name_suffix" {
 resource "google_sql_database_instance" "instance" {
   name                = local.db_instance_name
   region              = var.region
-  database_version    = "POSTGRES_14"
+  database_version    = "POSTGRES_16"
   deletion_protection = false # not recommended for PROD
 
   settings {
-    tier        = "db-custom-1-3840"
+    tier        = "db-f1-micro"
     user_labels = local.resource_labels
 
     ip_configuration {
       ipv4_enabled    = true
-      private_network = module.vpc.network_self_link
+      private_network = google_compute_network.vpc_network.id
     }
   }
 

diff --git a/cloud-composer-etl/prereq.sh b/cloud-composer-etl/prereq.sh
@@ -54,7 +54,7 @@ gcloud services enable cloudbuild.googleapis.com \
 
 echo "Granting Cloud Build's Service Account IAM roles to deploy the resources..."
 PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format='value(projectNumber)')
-MEMBER=serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com
+MEMBER=serviceAccount:$PROJECT_NUMBER-compute@developer.gserviceaccount.com
 add_iam_member $MEMBER roles/editor
 add_iam_member $MEMBER roles/iam.securityAdmin
 add_iam_member $MEMBER roles/compute.networkAdmin