network.tf file to create networking resources for Splunk export

main.tf

@@ -30,13 +30,13 @@ locals {
   dataflow_temporary_gcs_bucket_name = "${var.project}-${var.dataflow_job_name}-${random_id.bucket_suffix.hex}"
   dataflow_temporary_gcs_bucket_path = "tmp/"
 
-  project_log_sink_name = "${var.dataflow_job_name}-project-log-sink"
+  project_log_sink_name      = "${var.dataflow_job_name}-project-log-sink"
   organization_log_sink_name = "${var.dataflow_job_name}-organization-log-sink"
 
-  dataflow_input_topic_name = "${var.dataflow_job_name}-input-topic"
-  dataflow_input_subscription_name = "${var.dataflow_job_name}-input-subscription"
+  dataflow_input_topic_name             = "${var.dataflow_job_name}-input-topic"
+  dataflow_input_subscription_name      = "${var.dataflow_job_name}-input-subscription"
   dataflow_output_deadletter_topic_name = "${var.dataflow_job_name}-deadletter-topic"
-  dataflow_output_deadletter_sub_name = "${var.dataflow_job_name}-deadletter-subscription"
+  dataflow_output_deadletter_sub_name   = "${var.dataflow_job_name}-deadletter-subscription"
 
   dataflow_replay_job_name = "${var.dataflow_job_name}-replay"
 
@@ -54,7 +54,7 @@ resource "google_pubsub_subscription" "dataflow_input_pubsub_subscription" {
 
   # messages retained for 7 days (max)
   message_retention_duration = "604800s"
-  ack_deadline_seconds = 30
+  ack_deadline_seconds       = 30
 
   # subscription never expires
   expiration_policy {
@@ -63,9 +63,9 @@ resource "google_pubsub_subscription" "dataflow_input_pubsub_subscription" {
 }
 
 resource "google_logging_project_sink" "project_log_sink" {
-  name = local.project_log_sink_name
+  name        = local.project_log_sink_name
   destination = "pubsub.googleapis.com/projects/${var.project}/topics/${google_pubsub_topic.dataflow_input_pubsub_topic.name}"
-  filter = var.log_filter
+  filter      = var.log_filter
 
   unique_writer_identity = true
 }
@@ -81,25 +81,25 @@ resource "google_logging_project_sink" "project_log_sink" {
 
 resource "google_pubsub_topic_iam_binding" "pubsub_iam_binding" {
   project = google_pubsub_topic.dataflow_input_pubsub_topic.project
-  topic = google_pubsub_topic.dataflow_input_pubsub_topic.name
-  role = "roles/pubsub.publisher"
+  topic   = google_pubsub_topic.dataflow_input_pubsub_topic.name
+  role    = "roles/pubsub.publisher"
   members = [
     google_logging_project_sink.project_log_sink.writer_identity,
   ]
 }
 
 output "dataflow_job_id" {
-    value = google_dataflow_job.dataflow_job.job_id
+  value = google_dataflow_job.dataflow_job.job_id
 }
 
 output "dataflow_input_topic" {
-    value = google_pubsub_topic.dataflow_input_pubsub_topic.name
+  value = google_pubsub_topic.dataflow_input_pubsub_topic.name
 }
 
 output "dataflow_output_deadletter_subscription" {
-    value = google_pubsub_subscription.dataflow_deadletter_pubsub_sub.name
+  value = google_pubsub_subscription.dataflow_deadletter_pubsub_sub.name
 }
 
 output "dataflow_log_export_dashboard" {
-    value = var.workspace != "" ? google_monitoring_dashboard.splunk-export-pipeline-dashboard[0].id : ""
+  value = var.workspace != "" ? google_monitoring_dashboard.splunk-export-pipeline-dashboard[0].id : ""
 }

monitoring.tf

@@ -13,18 +13,18 @@
 # limitations under the License.
 
 resource "google_monitoring_group" "splunk-export-pipeline-group" {
-  count  = var.workspace != "" ? 1 : 0
+  count = var.workspace != "" ? 1 : 0
 
   display_name = "Splunk Log Export Group"
-  project = var.workspace
+  project      = var.workspace
 
   filter = "resource.metadata.name=starts_with(\"${var.dataflow_job_name}\")"
 }
 
 resource "google_monitoring_dashboard" "splunk-export-pipeline-dashboard" {
-  count  = var.workspace != "" ? 1 : 0
+  count = var.workspace != "" ? 1 : 0
 
-  project = var.workspace
+  project        = var.workspace
   dashboard_json = <<EOF
 {
   "displayName": "Splunk Log Export Ops",

network.tf

@@ -0,0 +1,77 @@
+resource "google_compute_network" "splunk_export" {
+  count = var.create_network == true ? 1 : 0
+
+  name                    = var.network
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "splunk_subnet" {
+  count = var.create_network == true ? 1 : 0
+
+  name                     = local.subnet_name
+  ip_cidr_range            = var.primary_subnet_cidr
+  region                   = var.region
+  network                  = google_compute_network.splunk_export.id
+  private_ip_google_access = true
+
+# Optional configuration to log network traffic at the subnet level 
+#   log_config {
+#     aggregation_interval = "INTERVAL_15_MIN"
+#     flow_sampling        = 0.1
+#     metadata             = "INCLUDE_ALL_METADATA"
+#   }
+
+}
+
+resource "google_compute_router" "dataflow_to_splunk_router" {
+  count = var.create_network == true ? 1 : 0
+
+  name    = "${var.network}-${var.region}-router"
+  region  = google_compute_subnetwork.splunk_subnet.region
+  network = google_compute_network.splunk_export.id
+}
+
+resource "google_compute_address" "dataflow_nat_ip_address" {
+  count = var.create_network == true ? 1 : 0
+
+  name   = "dataflow-splunk-nat-ip-address"
+  region = google_compute_subnetwork.splunk_subnet.region
+}
+
+resource "google_compute_router_nat" "dataflow_nat" {
+  count = var.create_network == true ? 1 : 0
+
+  name                               = "${var.network}-${var.region}-router-nat"
+  router                             = google_compute_router.dataflow_to_splunk_router.name
+  region                             = google_compute_router.dataflow_to_splunk_router.region
+  nat_ip_allocate_option             = "MANUAL_ONLY"
+  source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
+  nat_ips                            = google_compute_address.dataflow_nat_ip_address.*.self_link
+  min_ports_per_vm                   = 1024
+  subnetwork {
+    name                    = google_compute_subnetwork.splunk_subnet.id
+    source_ip_ranges_to_nat = ["PRIMARY_IP_RANGE"]
+  }
+
+  log_config {
+    enable = true
+    filter = "ERRORS_ONLY"
+  }
+}
+
+# Creating firewall rule so that dataflow jobs with > 1 worker can communicate over internal IPs.
+# Source: https://cloud.google.com/dataflow/docs/guides/routes-firewall#firewall_rules_required_by
+resource "google_compute_firewall" "connect_dataflow_workers" {
+  count = var.create_network == true ? 1 : 0
+
+  name    = "dataflow-internal-ip-fwr"
+  network = google_compute_network.splunk_export.id
+
+  allow {
+    protocol = "tcp"
+    ports    = ["12345-12346"]
+  }
+
+  source_tags = ["dataflow"]
+  target_tags = ["dataflow"]
+}

pipeline.tf

@@ -31,34 +31,34 @@ resource "google_pubsub_subscription" "dataflow_deadletter_pubsub_sub" {
 }
 
 resource "google_storage_bucket" "dataflow_job_temp_bucket" {
-  name = local.dataflow_temporary_gcs_bucket_name
-  location = var.region
+  name          = local.dataflow_temporary_gcs_bucket_name
+  location      = var.region
   storage_class = "REGIONAL"
 }
 
 resource "google_storage_bucket_object" "dataflow_job_temp_object" {
-  name = local.dataflow_temporary_gcs_bucket_path
+  name    = local.dataflow_temporary_gcs_bucket_path
   content = "Placeholder for Dataflow to write temporary files"
-  bucket = google_storage_bucket.dataflow_job_temp_bucket.name
+  bucket  = google_storage_bucket.dataflow_job_temp_bucket.name
 }
 
 resource "google_dataflow_job" "dataflow_job" {
-  name = var.dataflow_job_name
+  name              = var.dataflow_job_name
   template_gcs_path = var.dataflow_template_path
   temp_gcs_location = "gs://${local.dataflow_temporary_gcs_bucket_name}/${local.dataflow_temporary_gcs_bucket_path}"
-  machine_type = var.dataflow_job_machine_type
-  max_workers = var.dataflow_job_machine_count
+  machine_type      = var.dataflow_job_machine_type
+  max_workers       = var.dataflow_job_machine_count
   parameters = {
-    inputSubscription	= google_pubsub_subscription.dataflow_input_pubsub_subscription.id
-    outputDeadletterTopic = google_pubsub_topic.dataflow_deadletter_pubsub_topic.id
-    url	= var.splunk_hec_url
-    token	= var.splunk_hec_token
-    parallelism = var.dataflow_job_parallelism
-    batchCount = var.dataflow_job_batch_count
-    includePubsubMessage = local.dataflow_job_include_pubsub_message
+    inputSubscription            = google_pubsub_subscription.dataflow_input_pubsub_subscription.id
+    outputDeadletterTopic        = google_pubsub_topic.dataflow_deadletter_pubsub_topic.id
+    url                          = var.splunk_hec_url
+    token                        = var.splunk_hec_token
+    parallelism                  = var.dataflow_job_parallelism
+    batchCount                   = var.dataflow_job_batch_count
+    includePubsubMessage         = local.dataflow_job_include_pubsub_message
     disableCertificateValidation = var.dataflow_job_disable_certificate_validation
   }
-  region = var.region
-  network = var.network
+  region           = var.region
+  network          = var.create_netowork == true ? google_compute_network.splunk_export.id : var.network
   ip_configuration = "WORKER_IP_PRIVATE"
 }

variables.tf

@@ -22,18 +22,24 @@ variable "region" {
 
 variable "zone" {
   description = "Zone to deploy into"
-  default = ""
+  default     = ""
 }
 
 variable "network" {
   description = "Network to deploy into"
 }
 
+variable "create_network" {
+  description = "Boolean value if a new network needs to be created."
+  default     = false
+  type        = bool
+}
+
 # Dashboard parameters
 
 variable "workspace" {
   description = "Cloud Monitoring Workspace to create dashboard under. This assumes Workspace is already created and project provided is already added to it. If parameter is empty, no dashboard will be created"
-  default = ""
+  default     = ""
 }
 
 # Log sink details
@@ -46,26 +52,26 @@ variable "log_filter" {
 
 variable "splunk_hec_url" {
   description = "Splunk HEC URL to write data to. Example: https://[MY_SPLUNK_IP_OR_FQDN]:8088"
-  
+
   validation {
-    condition = can(regex("https?://.*(:[0-9]+)?", var.splunk_hec_url))
+    condition     = can(regex("https?://.*(:[0-9]+)?", var.splunk_hec_url))
     error_message = "Splunk HEC url must of the form <protocol>://<host>:<port> ."
   }
 }
 
 variable "splunk_hec_token" {
   description = "Splunk HEC token"
-  sensitive = true
+  sensitive   = true
 }
 
 # Dataflow job parameters
 
 variable "dataflow_template_path" {
   description = "Dataflow template path. Defaults to latest version of Google-hosted Pub/Sub to Splunk template"
-  default = "gs://dataflow-templates/latest/Cloud_PubSub_to_Splunk"
+  default     = "gs://dataflow-templates/latest/Cloud_PubSub_to_Splunk"
 
   validation {
-    condition = can(regex("gs://.+", var.dataflow_template_path))
+    condition     = can(regex("gs://.+", var.dataflow_template_path))
     error_message = "Splunk Dataflow template path must be a GCS object path gs://<bucket_name>/<path> ."
   }
 }
@@ -76,29 +82,29 @@ variable "dataflow_job_name" {
 
 variable "dataflow_job_machine_type" {
   description = "Dataflow job worker machine type"
-  default = "n1-standard-4"
+  default     = "n1-standard-4"
 }
 
 variable "dataflow_job_machine_count" {
   description = "Dataflow job max worker count. Defaults to 2."
-  type = number
-  default = 2
+  type        = number
+  default     = 2
 }
 
 variable "dataflow_job_parallelism" {
   description = "Maximum parallel requests to Splunk. Defaults to 8."
-  type = number
-  default = 8
+  type        = number
+  default     = 8
 }
 
 variable "dataflow_job_batch_count" {
   description = "Batch count of messages in single request to Splunk. Defaults to 50."
-  type = number
-  default = 50
+  type        = number
+  default     = 50
 }
 
 variable "dataflow_job_disable_certificate_validation" {
   description = "Disable SSL certificate validation (default: false)"
-  type = bool
-  default = false
+  type        = bool
+  default     = false
 }