Template infra deploy #7187990327

.template-version

@@ -1 +1 @@
-8bfab4387277bb7789d1aebc062bb3f6ba9a1e1d
+d7fa7698f3aecdefb48d11756ba411e1210659f0

docs/infra/set-up-network.md

@@ -11,7 +11,11 @@ The network setup process will configure and deploy network resources needed by
 Before setting up the database you'll need to have:
 
 1. [Set up the AWS account](./set-up-aws-account.md)
-2. Optionally configure the networks you want to have on your project in the [project-config module](/infra/project-config/main.tf). By default there is configuration for three networks, one for each application environment.
+2. Optionally adjust the configuration for the networks you want to have on your project in the [project-config module](/infra/project-config/main.tf). By default there is configuration for three networks, one for each application environment. By default, there is one network for each application environment. If you have multiple apps and want your applications in separate networks, you may want to give the networks differentiating names (e.g. "foo-dev", "foo-prod", "bar-dev", "bar-prod", instead of just "dev", "prod").
+3. [Configure the app](/infra/app/app-config/main.tf).
+   1. Update `has_database` to `true` or `false` depending on whether or not your application has a database to integrate with. This setting determines whether or not to create VPC endpoints needed by the database layer.
+   2. Update `has_external_non_aws_service` to `true` or `false` depending on whether or not your application makes calls to an external non-AWS service. This setting determines whether or not to create NAT gateways, which allows the service in the private subnet to make requests to the internet.
+   3. Update `network_name` for your application environments. This mapping ensures that each network is configured appropriately based on the application(s) in that network (see `local.apps_in_network` in [/infra/networks/main.tf](/infra/networks/main.tf)) Failure to set the network name properly means that the network layer may not receive the correct application configurations for `has_database` and `has_external_non_aws_service`.
 
 ## 1. Configure backend
 
@@ -28,3 +32,7 @@ Now run the following commands to create the resources. Review the terraform bef
 ```bash
 make infra-update-network NETWORK_NAME=<NETWORK_NAME>
 ```
+
+## Updating the network
+
+If you make changes to your application's configuration that impacts the network (such as `has_database` and `has_external_non_aws_service`), make sure to update the network before you update or deploy subsequent infrastructure layers.

infra/app/app-config/main.tf

@@ -1,9 +1,24 @@
 locals {
-  app_name                        = "app"
-  environments                    = ["dev", "staging", "prod"]
-  project_name                    = module.project_config.project_name
-  image_repository_name           = "${local.project_name}-${local.app_name}"
-  has_database                    = false
+  app_name              = "app"
+  environments          = ["dev", "staging", "prod"]
+  project_name          = module.project_config.project_name
+  image_repository_name = "${local.project_name}-${local.app_name}"
+
+  # Whether or not the application has a database
+  # If enabled:
+  # 1. The networks associated with this application's environments will have
+  #    VPC endpoints needed by the database layer
+  # 2. Each environment's config will have a database_config property that is used to
+  #    pass db_vars into the infra/modules/service module, which provides the necessary
+  #    configuration for the service to access the database
+  has_database = false
+
+  # Whether or not the application depends on external non-AWS services.
+  # If enabled, the networks associated with this application's environments
+  # will have NAT gateways, which allows the service in the private subnet to
+  # make calls to the internet.
+  has_external_non_aws_service = false
+
   has_incident_management_service = false
 
   feature_flags = ["foo", "bar"]

infra/app/app-config/outputs.tf

@@ -18,6 +18,10 @@ output "has_database" {
   value = local.has_database
 }
 
+output "has_external_non_aws_service" {
+  value = local.has_external_non_aws_service
+}
+
 output "has_incident_management_service" {
   value = local.has_incident_management_service
 }

infra/modules/network/main.tf

@@ -22,9 +22,10 @@ module "aws_vpc" {
   database_subnet_tags       = { subnet_type = "database" }
   database_subnet_group_name = var.database_subnet_group_name
 
-  enable_nat_gateway     = var.nat_gateway_config != "none" ? true : false
-  single_nat_gateway     = var.nat_gateway_config == "shared" ? true : false
-  one_nat_gateway_per_az = var.nat_gateway_config == "per_az" ? true : false
+  # If application needs external services, then create one NAT gateway per availability zone
+  enable_nat_gateway     = var.has_external_non_aws_service
+  single_nat_gateway     = false
+  one_nat_gateway_per_az = var.has_external_non_aws_service
 
   enable_dns_hostnames = true
   enable_dns_support   = true

infra/modules/network/variables.tf

@@ -3,20 +3,24 @@ variable "name" {
   description = "Name to give the VPC. Will be added to the VPC under the 'network_name' tag."
 }
 
+variable "aws_services_security_group_name_prefix" {
+  type        = string
+  description = "Prefix for the name of the security group attached to VPC endpoints"
+}
+
 variable "database_subnet_group_name" {
   type        = string
   description = "Name of the database subnet group"
 }
 
-variable "nat_gateway_config" {
-  # nat gateways are pricey, unless outgoing traffic
-  # is part of your service, one should be enough
-  # (i.e. to allow your app to fetch some resources on startup)
-  type        = string
-  default     = "none"
-  description = "How many NAT gateways (which can be pricey) to create. None, a single one that is shared across all subnets, one per availability zone, or one per private subnet"
-  validation {
-    condition     = contains(["none", "shared", "per_az", "per_subnet"], var.nat_gateway_config)
-    error_message = "Allowed values for nat_gateway_config are 'none', 'shared', 'per_az', 'per_subnet'"
-  }
+variable "has_database" {
+  type        = bool
+  description = "Whether the application(s) in this network have a database. Determines whether to create VPC endpoints needed by the database layer."
+  default     = false
+}
+
+variable "has_external_non_aws_service" {
+  type        = bool
+  description = "Whether the application(s) in this network need to call external non-AWS services. Determines whether or not to create NAT gateways."
+  default     = false
 }

infra/modules/network/vpc-endpoints.tf

@@ -0,0 +1,135 @@
+locals {
+  # List of AWS services used by this VPC
+  # This list is used to create VPC endpoints so that the AWS services can
+  # be accessed without network traffic ever leaving the VPC's private network
+  # For a list of AWS services that integrate with AWS PrivateLink
+  # see https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html
+  #
+  # The database module requires VPC access from private networks to SSM, KMS, and RDS
+  aws_service_integrations = setunion(
+    # AWS services used by ECS Fargate: ECR to fetch images, S3 for image layers, and CloudWatch for logs
+    ["ecr.api", "ecr.dkr", "s3", "logs"],
+
+    # Workaround: Feature flags use AWS Evidently, but we are going to create that VPC endpoint separately
+    # rather than as part of this list in order to get around the limitation that AWS Evidently
+    # is not available in some availability zones (at the time of writing)
+
+    # AWS services used by the database's role manager
+    var.has_database ? ["ssm", "kms", "secretsmanager"] : [],
+  )
+
+  # S3 and DynamoDB use Gateway VPC endpoints. All other services use Interface VPC endpoints
+  interface_vpc_endpoints = toset([for aws_service in local.aws_service_integrations : aws_service if !contains(["s3", "dynamodb"], aws_service)])
+  gateway_vpc_endpoints   = toset([for aws_service in local.aws_service_integrations : aws_service if contains(["s3", "dynamodb"], aws_service)])
+}
+
+data "aws_region" "current" {}
+
+# VPC Endpoints for accessing AWS Services
+# ----------------------------------------
+#
+# Since the role manager Lambda function is in the VPC (which is needed to be
+# able to access the database) we need to allow the Lambda function to access
+# AWS Systems Manager Parameter Store (to fetch the database password) and
+# KMS (to decrypt SecureString parameters from Parameter Store). We can do
+# this by either allowing internet access to the Lambda, or by using a VPC
+# endpoint. The latter is more secure.
+# See https://repost.aws/knowledge-center/lambda-vpc-parameter-store
+# See https://docs.aws.amazon.com/vpc/latest/privatelink/create-interface-endpoint.html#create-interface-endpoint
+
+resource "aws_security_group" "aws_services" {
+  name_prefix = var.aws_services_security_group_name_prefix
+  description = "VPC endpoints to access AWS services from the VPCs private subnets"
+  vpc_id      = module.aws_vpc.vpc_id
+}
+
+resource "aws_vpc_endpoint" "interface" {
+  for_each = local.interface_vpc_endpoints
+
+  vpc_id              = module.aws_vpc.vpc_id
+  service_name        = "com.amazonaws.${data.aws_region.current.name}.${each.key}"
+  vpc_endpoint_type   = "Interface"
+  security_group_ids  = [aws_security_group.aws_services.id]
+  subnet_ids          = module.aws_vpc.private_subnets
+  private_dns_enabled = true
+}
+
+resource "aws_vpc_endpoint" "gateway" {
+  for_each = local.gateway_vpc_endpoints
+
+  vpc_id            = module.aws_vpc.vpc_id
+  service_name      = "com.amazonaws.${data.aws_region.current.name}.${each.key}"
+  vpc_endpoint_type = "Gateway"
+  route_table_ids   = module.aws_vpc.private_route_table_ids
+}
+
+# Interface VPC Endpoint for AWS CloudWatch Evidently (Workaround)
+# ----------------------------------------------------------------
+#
+# Add Interface VPC Endpoint for AWS CloudWatch Evidently separately from other VPC Endpoints,
+# because at the time of writing, Evidently isn't supported in certain availability zones.
+# So we filter down the list of private subnets by the ones in the availability zones that are
+# supported by Evidently before creating the VPC endpoint.
+
+data "aws_subnet" "private" {
+  count = length(module.aws_vpc.private_subnets)
+  id    = module.aws_vpc.private_subnets[count.index]
+}
+
+locals {
+  # At the time of writing, these are the only availability zones supported by AWS CloudWatch Evidently
+  # This list was obtained by using the AWS Console, going through each US region, attempting to add
+  # a VPC endpoint for Evidently in the default VPC, and seeing which availability zones show up as
+  # options.
+  evidently_az_ids = [
+    "use1-az2",
+    "use1-az4",
+    "use1-az6",
+    "use2-az1",
+    "use2-az2",
+    "use2-az3",
+    "usw2-az1",
+    "usw2-az2",
+    "usw2-az3",
+  ]
+
+  evidently_dataplane_az_ids = [
+    "use1-az1",
+    "use1-az4",
+    "use1-az6",
+    "use2-az1",
+    "use2-az2",
+    "use2-az3",
+    "usw2-az1",
+    "usw2-az2",
+    "usw2-az3",
+  ]
+
+  aws_evidently_subnet_ids = [
+    for subnet in data.aws_subnet.private[*] : subnet.id
+    if contains(local.evidently_az_ids, subnet.availability_zone_id)
+  ]
+
+  aws_evidently_dataplane_subnet_ids = [
+    for subnet in data.aws_subnet.private[*] : subnet.id
+    if contains(local.evidently_dataplane_az_ids, subnet.availability_zone_id)
+  ]
+}
+
+resource "aws_vpc_endpoint" "evidently" {
+  vpc_id              = module.aws_vpc.vpc_id
+  service_name        = "com.amazonaws.${data.aws_region.current.name}.evidently"
+  vpc_endpoint_type   = "Interface"
+  security_group_ids  = [aws_security_group.aws_services.id]
+  subnet_ids          = local.aws_evidently_subnet_ids
+  private_dns_enabled = true
+}
+
+resource "aws_vpc_endpoint" "evidently_dataplane" {
+  vpc_id              = module.aws_vpc.vpc_id
+  service_name        = "com.amazonaws.${data.aws_region.current.name}.evidently-dataplane"
+  vpc_endpoint_type   = "Interface"
+  security_group_ids  = [aws_security_group.aws_services.id]
+  subnet_ids          = local.aws_evidently_dataplane_subnet_ids
+  private_dns_enabled = true
+}

infra/networks/main.tf

@@ -2,31 +2,35 @@
 # that currently just adds resources to the default VPC
 # The full network implementation is part of https://github.com/navapbc/template-infra/issues/152
 
-data "aws_region" "current" {}
-
 locals {
   tags = merge(module.project_config.default_tags, {
     network_name = var.network_name
     description  = "VPC resources"
   })
   region = module.project_config.default_region
 
-  # List of AWS services used by this VPC
-  # This list is used to create VPC endpoints so that the AWS services can
-  # be accessed without network traffic ever leaving the VPC's private network
-  # For a list of AWS services that integrate with AWS PrivateLink
-  # see https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html
-  #
-  # The database module requires VPC access from private networks to SSM, KMS, and RDS
-  aws_service_integrations = setunion(
-    # AWS services used by ECS Fargate: ECR to fetch images, S3 for image layers, and CloudWatch for logs
-    ["ecr.api", "ecr.dkr", "s3", "logs"],
-
-    # AWS services used by the database's role manager
-    module.app_config.has_database ? ["ssm", "kms", "secretsmanager"] : [],
-  )
-
   network_config = module.project_config.network_configs[var.network_name]
+
+  # List of configuration for all applications, even ones that are not in the current network
+  # If project has multiple applications, add other app configs to this list
+  app_configs = [module.app_config]
+
+  # List of configuration for applications that are in the current network
+  # An application is in the current network if at least one of its environments
+  # is mapped to the network
+  apps_in_network = [
+    for app in local.app_configs :
+    app
+    if anytrue([
+      for environment_config in app.environment_configs : true if environment_config.network_name == var.network_name
+    ])
+  ]
+
+  # Whether any of the applications in the network have a database
+  has_database = anytrue([for app in local.apps_in_network : app.has_database])
+
+  # Whether any of the applications in the network have dependencies on an external non-AWS service
+  has_external_non_aws_service = anytrue([for app in local.apps_in_network : app.has_external_non_aws_service])
 }
 
 terraform {
@@ -60,45 +64,10 @@ module "app_config" {
 }
 
 module "network" {
-  source                     = "../modules/network"
-  name                       = var.network_name
-  database_subnet_group_name = local.network_config.database_subnet_group_name
-  nat_gateway_config         = "none"
-}
-
-data "aws_route_table" "private" {
-  count     = length(module.network.private_subnet_ids)
-  subnet_id = module.network.private_subnet_ids[count.index]
-}
-
-# VPC Endpoints for accessing AWS Services
-# ----------------------------------------
-#
-# Since the role manager Lambda function is in the VPC (which is needed to be
-# able to access the database) we need to allow the Lambda function to access
-# AWS Systems Manager Parameter Store (to fetch the database password) and
-# KMS (to decrypt SecureString parameters from Parameter Store). We can do
-# this by either allowing internet access to the Lambda, or by using a VPC
-# endpoint. The latter is more secure.
-# See https://repost.aws/knowledge-center/lambda-vpc-parameter-store
-# See https://docs.aws.amazon.com/vpc/latest/privatelink/create-interface-endpoint.html#create-interface-endpoint
-
-resource "aws_security_group" "aws_services" {
-  count = length(local.aws_service_integrations) > 0 ? 1 : 0
-
-  name_prefix = module.project_config.aws_services_security_group_name_prefix
-  description = "VPC endpoints to access AWS services from the VPCs private subnets"
-  vpc_id      = module.network.vpc_id
-}
-
-resource "aws_vpc_endpoint" "aws_service" {
-  for_each = local.aws_service_integrations
-
-  vpc_id              = module.network.vpc_id
-  service_name        = "com.amazonaws.${data.aws_region.current.name}.${each.key}"
-  vpc_endpoint_type   = each.key == "s3" ? "Gateway" : "Interface"
-  security_group_ids  = each.key == "s3" ? null : [aws_security_group.aws_services[0].id]
-  subnet_ids          = each.key == "s3" ? null : module.network.private_subnet_ids
-  private_dns_enabled = each.key == "s3" ? null : true
-  route_table_ids     = each.key == "s3" ? data.aws_route_table.private[*].id : null
+  source                                  = "../modules/network"
+  name                                    = var.network_name
+  aws_services_security_group_name_prefix = module.project_config.aws_services_security_group_name_prefix
+  database_subnet_group_name              = local.network_config.database_subnet_group_name
+  has_database                            = local.has_database
+  has_external_non_aws_service            = local.has_external_non_aws_service
 }