diff --git a/README.md b/README.md index 14d182b8..be7af1a8 100644 --- a/README.md +++ b/README.md @@ -238,14 +238,22 @@ No modules. | [azurerm_log_analytics_solution.main](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_solution) | resource | | [azurerm_log_analytics_workspace.main](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource | | [azurerm_role_assignment.acr](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.application_gateway_existing_vnet_network_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.application_gateway_new_vnet_network_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.application_gateway_resource_group_reader](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.existing_application_gateway_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.network_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.network_contributor_on_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [null_resource.kubernetes_version_keeper](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.pool_name_keeper](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [tls_private_key.ssh](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | +| [azurerm_client_config.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | | [azurerm_log_analytics_workspace.main](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/log_analytics_workspace) | data source | +| [azurerm_resource_group.aks_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | +| [azurerm_resource_group.ingress_gw](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | | [azurerm_resource_group.main](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | | [azurerm_user_assigned_identity.cluster_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/user_assigned_identity) | data source | +| [azurerm_virtual_network.application_gateway_vnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/virtual_network) | data source | ## Inputs @@ -292,28 +300,26 @@ No modules. | [auto\_scaler\_profile\_skip\_nodes\_with\_system\_pods](#input\_auto\_scaler\_profile\_skip\_nodes\_with\_system\_pods) | If `true` cluster autoscaler will never delete nodes with pods from kube-system (except for DaemonSet or mirror pods). Defaults to `true`. | `bool` | `true` | no | | [automatic\_channel\_upgrade](#input\_automatic\_channel\_upgrade) | (Optional) The upgrade channel for this Kubernetes Cluster. Possible values are `patch`, `rapid`, `node-image` and `stable`. By default automatic-upgrades are turned off. Note that you cannot specify the patch version using `kubernetes_version` or `orchestrator_version` when using the `patch` upgrade channel. See [the documentation](https://learn.microsoft.com/en-us/azure/aks/auto-upgrade-cluster) for more information | `string` | `null` | no | | [azure\_policy\_enabled](#input\_azure\_policy\_enabled) | Enable Azure Policy Addon. | `bool` | `false` | no | +| [brown\_field\_application\_gateway\_for\_ingress](#input\_brown\_field\_application\_gateway\_for\_ingress) | [Definition of `brown_field`](https://learn.microsoft.com/en-us/azure/application-gateway/tutorial-ingress-controller-add-on-existing)
* `id` - (Required) The ID of the Application Gateway that be used as cluster ingress.
* `subnet_id` - (Required) The ID of the Subnet which the Application Gateway is connected to. Must be set when `create_role_assignments` is `true`. | 
object({
    id        = string
    subnet_id = string
  })
| `null` | no | | [client\_id](#input\_client\_id) | (Optional) The Client ID (appId) for the Service Principal used for the AKS deployment | `string` | `""` | no | | [client\_secret](#input\_client\_secret) | (Optional) The Client Secret (password) for the Service Principal used for the AKS deployment | `string` | `""` | no | | [cluster\_log\_analytics\_workspace\_name](#input\_cluster\_log\_analytics\_workspace\_name) | (Optional) The name of the Analytics workspace | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | (Optional) The name for the AKS resources created in the specified Azure Resource Group. This variable overwrites the 'prefix' var (The 'prefix' var will still be applied to the dns\_prefix if it is set) | `string` | `null` | no | | [confidential\_computing](#input\_confidential\_computing) | (Optional) Enable Confidential Computing. | 
object({
    sgx_quote_helper_enabled = bool
  })
| `null` | no | | [create\_role\_assignment\_network\_contributor](#input\_create\_role\_assignment\_network\_contributor) | (Deprecated) Create a role assignment for the AKS Service Principal to be a Network Contributor on the subnets used for the AKS Cluster | `bool` | `false` | no | +| [create\_role\_assignments\_for\_application\_gateway](#input\_create\_role\_assignments\_for\_application\_gateway) | (Optional) Whether to create the corresponding role assignments for application gateway or not. Defaults to `true`. | `bool` | `true` | no | | [default\_node\_pool\_fips\_enabled](#input\_default\_node\_pool\_fips\_enabled) | (Optional) Should the nodes in this Node Pool have Federal Information Processing Standard enabled? Changing this forces a new resource to be created. | `bool` | `null` | no | | [disk\_encryption\_set\_id](#input\_disk\_encryption\_set\_id) | (Optional) The ID of the Disk Encryption Set which should be used for the Nodes and Volumes. More information [can be found in the documentation](https://docs.microsoft.com/azure/aks/azure-disk-customer-managed-keys). Changing this forces a new resource to be created. | `string` | `null` | no | | [ebpf\_data\_plane](#input\_ebpf\_data\_plane) | (Optional) Specifies the eBPF data plane used for building the Kubernetes network. Possible value is `cilium`. Changing this forces a new resource to be created. | `string` | `null` | no | | [enable\_auto\_scaling](#input\_enable\_auto\_scaling) | Enable node pool autoscaling | `bool` | `false` | no | | [enable\_host\_encryption](#input\_enable\_host\_encryption) | Enable Host Encryption for default node pool. Encryption at host feature must be enabled on the subscription: https://docs.microsoft.com/azure/virtual-machines/linux/disks-enable-host-based-encryption-cli | `bool` | `false` | no | | [enable\_node\_public\_ip](#input\_enable\_node\_public\_ip) | (Optional) Should nodes in this Node Pool have a Public IP Address? Defaults to false. | `bool` | `false` | no | +| [green\_field\_application\_gateway\_for\_ingress](#input\_green\_field\_application\_gateway\_for\_ingress) | [Definition of `green_field`](https://learn.microsoft.com/en-us/azure/application-gateway/tutorial-ingress-controller-add-on-new)
* `name` - (Optional) The name of the Application Gateway to be used or created in the Nodepool Resource Group, which in turn will be integrated with the ingress controller of this Kubernetes Cluster.
* `subnet_cidr` - (Optional) The subnet CIDR to be used to create an Application Gateway, which in turn will be integrated with the ingress controller of this Kubernetes Cluster.
* `subnet_id` - (Optional) The ID of the subnet on which to create an Application Gateway, which in turn will be integrated with the ingress controller of this Kubernetes Cluster. | 
object({
    name        = optional(string)
    subnet_cidr = optional(string)
    subnet_id   = optional(string)
  })
| `null` | no | | [http\_application\_routing\_enabled](#input\_http\_application\_routing\_enabled) | Enable HTTP Application Routing Addon (forces recreation). | `bool` | `false` | no | | [identity\_ids](#input\_identity\_ids) | (Optional) Specifies a list of User Assigned Managed Identity IDs to be assigned to this Kubernetes Cluster. | `list(string)` | `null` | no | | [identity\_type](#input\_identity\_type) | (Optional) The type of identity used for the managed cluster. Conflicts with `client_id` and `client_secret`. Possible values are `SystemAssigned` and `UserAssigned`. If `UserAssigned` is set, an `identity_ids` must be set as well. | `string` | `"SystemAssigned"` | no | | [image\_cleaner\_enabled](#input\_image\_cleaner\_enabled) | (Optional) Specifies whether Image Cleaner is enabled. | `bool` | `false` | no | | [image\_cleaner\_interval\_hours](#input\_image\_cleaner\_interval\_hours) | (Optional) Specifies the interval in hours when images should be cleaned up. Defaults to `48`. | `number` | `48` | no | -| [ingress\_application\_gateway\_enabled](#input\_ingress\_application\_gateway\_enabled) | Whether to deploy the Application Gateway ingress controller to this Kubernetes Cluster? | `bool` | `false` | no | -| [ingress\_application\_gateway\_id](#input\_ingress\_application\_gateway\_id) | The ID of the Application Gateway to integrate with the ingress controller of this Kubernetes Cluster. | `string` | `null` | no | -| [ingress\_application\_gateway\_name](#input\_ingress\_application\_gateway\_name) | The name of the Application Gateway to be used or created in the Nodepool Resource Group, which in turn will be integrated with the ingress controller of this Kubernetes Cluster. | `string` | `null` | no | -| [ingress\_application\_gateway\_subnet\_cidr](#input\_ingress\_application\_gateway\_subnet\_cidr) | The subnet CIDR to be used to create an Application Gateway, which in turn will be integrated with the ingress controller of this Kubernetes Cluster. | `string` | `null` | no | -| [ingress\_application\_gateway\_subnet\_id](#input\_ingress\_application\_gateway\_subnet\_id) | The ID of the subnet on which to create an Application Gateway, which in turn will be integrated with the ingress controller of this Kubernetes Cluster. | `string` | `null` | no | | [key\_vault\_secrets\_provider\_enabled](#input\_key\_vault\_secrets\_provider\_enabled) | (Optional) Whether to use the Azure Key Vault Provider for Secrets Store CSI Driver in an AKS cluster. For more details: https://docs.microsoft.com/en-us/azure/aks/csi-secrets-store-driver | `bool` | `false` | no | | [kms\_enabled](#input\_kms\_enabled) | (Optional) Enable Azure KeyVault Key Management Service. | `bool` | `false` | no | | [kms\_key\_vault\_key\_id](#input\_kms\_key\_vault\_key\_id) | (Optional) Identifier of Azure Key Vault key. When Azure Key Vault key management service is enabled, this field is required and must be a valid key identifier. | `string` | `null` | no | @@ -336,7 +342,7 @@ No modules. | [log\_analytics\_workspace\_resource\_group\_name](#input\_log\_analytics\_workspace\_resource\_group\_name) | (Optional) Resource group name to create azurerm\_log\_analytics\_solution. | `string` | `null` | no | | [log\_analytics\_workspace\_sku](#input\_log\_analytics\_workspace\_sku) | The SKU (pricing level) of the Log Analytics workspace. For new subscriptions the SKU should be set to PerGB2018 | `string` | `"PerGB2018"` | no | | [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period for the logs in days | `number` | `30` | no | -| [maintenance\_window](#input\_maintenance\_window) | (Optional) Maintenance configuration of the managed cluster. | 
object({
    allowed = optional(list(object({
      day   = string
      hours = set(number)
    })), []),
    not_allowed = optional(list(object({
      end   = string
      start = string
    })), []),
  })
| `null` | no | +| [maintenance\_window](#input\_maintenance\_window) | (Optional) Maintenance configuration of the managed cluster. | 
object({
    allowed = optional(list(object({
      day   = string
      hours = set(number)
      })), [
    ]),
    not_allowed = optional(list(object({
      end   = string
      start = string
    })), []),
  })
| `null` | no | | [maintenance\_window\_node\_os](#input\_maintenance\_window\_node\_os) | - `day_of_month` -
- `day_of_week` - (Optional) The day of the week for the maintenance run. Options are `Monday`, `Tuesday`, `Wednesday`, `Thurday`, `Friday`, `Saturday` and `Sunday`. Required in combination with weekly frequency.
- `duration` - (Required) The duration of the window for maintenance to run in hours.
- `frequency` - (Required) Frequency of maintenance. Possible options are `Daily`, `Weekly`, `AbsoluteMonthly` and `RelativeMonthly`.
- `interval` - (Required) The interval for maintenance runs. Depending on the frequency this interval is week or month based.
- `start_date` - (Optional) The date on which the maintenance window begins to take effect.
- `start_time` - (Optional) The time for maintenance to begin, based on the timezone determined by `utc_offset`. Format is `HH:mm`.
- `utc_offset` - (Optional) Used to determine the timezone for cluster maintenance.
- `week_index` - (Optional) The week in the month used for the maintenance run. Options are `First`, `Second`, `Third`, `Fourth`, and `Last`.

---
`not_allowed` block supports the following:
- `end` - (Required) The end of a time span, formatted as an RFC3339 string.
- `start` - (Required) The start of a time span, formatted as an RFC3339 string. | 
object({
    day_of_month = optional(number)
    day_of_week  = optional(string)
    duration     = number
    frequency    = string
    interval     = number
    start_date   = optional(string)
    start_time   = optional(string)
    utc_offset   = optional(string)
    week_index   = optional(string)
    not_allowed = optional(set(object({
      end   = string
      start = string
    })))
  })
| `null` | no | | [microsoft\_defender\_enabled](#input\_microsoft\_defender\_enabled) | (Optional) Is Microsoft Defender on the cluster enabled? Requires `var.log_analytics_workspace_enabled` to be `true` to set this variable to `true`. | `bool` | `false` | no | | [monitor\_metrics](#input\_monitor\_metrics) | (Optional) Specifies a Prometheus add-on profile for the Kubernetes Cluster
object({
annotations\_allowed = "(Optional) Specifies a comma-separated list of Kubernetes annotation keys that will be used in the resource's labels metric."
labels\_allowed = "(Optional) Specifies a Comma-separated list of additional Kubernetes label keys that will be used in the resource's labels metric."
}) | 
object({
    annotations_allowed = optional(string)
    labels_allowed      = optional(string)
  })
| `null` | no | diff --git a/examples/application_gateway_ingress/k8s_workload.tf b/examples/application_gateway_ingress/k8s_workload.tf new file mode 100644 index 00000000..c4d6d0fa --- /dev/null +++ b/examples/application_gateway_ingress/k8s_workload.tf @@ -0,0 +1,111 @@ +resource "kubernetes_namespace_v1" "example" { + metadata { + name = "example" + } + + depends_on = [module.aks] +} + +resource "kubernetes_pod" "aspnet_app" { + #checkov:skip=CKV_K8S_8:We don't need readiness probe for this simple example. + #checkov:skip=CKV_K8S_9:We don't need readiness probe for this simple example. + #checkov:skip=CKV_K8S_22:readOnlyRootFilesystem would block our pod from working + #checkov:skip=CKV_K8S_28:capabilities would block our pod from working + metadata { + labels = { + app = "aspnetapp" + } + name = "aspnetapp" + namespace = kubernetes_namespace_v1.example.metadata[0].name + } + spec { + container { + name = "aspnetapp-image" + image = "mcr.microsoft.com/dotnet/samples@sha256:7070894cc10d2b1e68e72057cca22040c5984cfae2ec3e079e34cf0a4da7fcea" + image_pull_policy = "Always" + + port { + container_port = 80 + protocol = "TCP" + } + resources { + limits = { + cpu = "250m" + memory = "256Mi" + } + requests = { + cpu = "250m" + memory = "256Mi" + } + } + security_context {} + } + } +} + +resource "kubernetes_service" "svc" { + metadata { + name = "aspnetapp" + namespace = kubernetes_namespace_v1.example.metadata[0].name + } + spec { + selector = { + app = "aspnetapp" + } + + port { + port = 80 + protocol = "TCP" + target_port = 80 + } + } +} + +resource "kubernetes_ingress_v1" "ing" { + metadata { + annotations = { + "kubernetes.io/ingress.class" : "azure/application-gateway" + } + name = "aspnetapp" + namespace = kubernetes_namespace_v1.example.metadata[0].name + } + spec { + rule { + http { + path { + path = "/" + path_type = "Exact" + + backend { + service { + name = "aspnetapp" + + port { + number = 80 + } + } + } + } + } + } + } + + depends_on = [ + module.aks, + ] +} + +resource "time_sleep" "wait_for_ingress" { + create_duration = "15m" + + depends_on = [kubernetes_ingress_v1.ing] +} + +data "kubernetes_ingress_v1" "ing" { + metadata { + name = "aspnetapp" + namespace = kubernetes_namespace_v1.example.metadata[0].name + } + + depends_on = [time_sleep.wait_for_ingress] +} \ No newline at end of file diff --git a/examples/application_gateway_ingress/main.tf b/examples/application_gateway_ingress/main.tf new file mode 100644 index 00000000..7dba26ab --- /dev/null +++ b/examples/application_gateway_ingress/main.tf @@ -0,0 +1,194 @@ +resource "random_id" "prefix" { + byte_length = 8 +} + +resource "random_id" "name" { + byte_length = 8 +} + +resource "azurerm_resource_group" "main" { + count = var.create_resource_group ? 1 : 0 + + location = var.location + name = coalesce(var.resource_group_name, "${random_id.prefix.hex}-rg") +} + +locals { + resource_group = { + name = var.create_resource_group ? azurerm_resource_group.main[0].name : var.resource_group_name + location = var.location + } +} + +resource "azurerm_virtual_network" "test" { + count = var.bring_your_own_vnet ? 1 : 0 + + address_space = ["10.52.0.0/16"] + location = local.resource_group.location + name = "${random_id.prefix.hex}-vn" + resource_group_name = local.resource_group.name +} + +resource "azurerm_subnet" "test" { + count = var.bring_your_own_vnet ? 1 : 0 + + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test[0].name +} + +locals { + appgw_cidr = !var.use_brown_field_application_gateway && !var.bring_your_own_vnet ? "10.225.0.0/16" : "10.52.1.0/24" +} + +resource "azurerm_subnet" "appgw" { + count = var.use_brown_field_application_gateway && var.bring_your_own_vnet ? 1 : 0 + + address_prefixes = [local.appgw_cidr] + name = "${random_id.prefix.hex}-gw" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test[0].name +} + +# Locals block for hardcoded names +locals { + backend_address_pool_name = try("${azurerm_virtual_network.test[0].name}-beap", "") + frontend_ip_configuration_name = try("${azurerm_virtual_network.test[0].name}-feip", "") + frontend_port_name = try("${azurerm_virtual_network.test[0].name}-feport", "") + http_setting_name = try("${azurerm_virtual_network.test[0].name}-be-htst", "") + listener_name = try("${azurerm_virtual_network.test[0].name}-httplstn", "") + request_routing_rule_name = try("${azurerm_virtual_network.test[0].name}-rqrt", "") +} + +resource "azurerm_public_ip" "pip" { + count = var.use_brown_field_application_gateway && var.bring_your_own_vnet ? 1 : 0 + + allocation_method = "Static" + location = local.resource_group.location + name = "appgw-pip" + resource_group_name = local.resource_group.name + sku = "Standard" +} + +resource "azurerm_application_gateway" "appgw" { + count = var.use_brown_field_application_gateway && var.bring_your_own_vnet ? 1 : 0 + + location = local.resource_group.location + #checkov:skip=CKV_AZURE_120:We don't need the WAF for this simple example + name = "ingress" + resource_group_name = local.resource_group.name + + backend_address_pool { + name = local.backend_address_pool_name + } + backend_http_settings { + cookie_based_affinity = "Disabled" + name = local.http_setting_name + port = 80 + protocol = "Http" + request_timeout = 1 + } + frontend_ip_configuration { + name = local.frontend_ip_configuration_name + public_ip_address_id = azurerm_public_ip.pip[0].id + } + frontend_port { + name = local.frontend_port_name + port = 80 + } + gateway_ip_configuration { + name = "appGatewayIpConfig" + subnet_id = azurerm_subnet.appgw[0].id + } + http_listener { + frontend_ip_configuration_name = local.frontend_ip_configuration_name + frontend_port_name = local.frontend_port_name + name = local.listener_name + protocol = "Http" + } + request_routing_rule { + http_listener_name = local.listener_name + name = local.request_routing_rule_name + rule_type = "Basic" + backend_address_pool_name = local.backend_address_pool_name + backend_http_settings_name = local.http_setting_name + priority = 1 + } + sku { + name = "Standard_v2" + tier = "Standard_v2" + capacity = 1 + } + + lifecycle { + ignore_changes = [ + tags, + backend_address_pool, + backend_http_settings, + http_listener, + probe, + request_routing_rule, + url_path_map, + ] + } +} + +module "aks" { + #checkov:skip=CKV_AZURE_141:We enable admin account here so we can provision K8s resources directly in this simple example + source = "../.." + + prefix = random_id.name.hex + resource_group_name = local.resource_group.name + kubernetes_version = "1.26" # don't specify the patch version! + automatic_channel_upgrade = "patch" + agents_availability_zones = ["1", "2"] + agents_count = null + agents_max_count = 2 + agents_max_pods = 100 + agents_min_count = 1 + agents_pool_name = "testnodepool" + agents_pool_linux_os_configs = [ + { + transparent_huge_page_enabled = "always" + sysctl_configs = [ + { + fs_aio_max_nr = 65536 + fs_file_max = 100000 + fs_inotify_max_user_watches = 1000000 + } + ] + } + ] + agents_type = "VirtualMachineScaleSets" + azure_policy_enabled = true + enable_auto_scaling = true + enable_host_encryption = true + http_application_routing_enabled = true + green_field_application_gateway_for_ingress = var.use_brown_field_application_gateway ? null : { + name = "ingress" + subnet_cidr = local.appgw_cidr + } + brown_field_application_gateway_for_ingress = var.use_brown_field_application_gateway ? { + id = azurerm_application_gateway.appgw[0].id + subnet_id = azurerm_subnet.appgw[0].id + } : null + create_role_assignments_for_application_gateway = var.create_role_assignments_for_application_gateway + local_account_disabled = false + log_analytics_workspace_enabled = false + net_profile_dns_service_ip = "10.0.0.10" + net_profile_service_cidr = "10.0.0.0/16" + network_plugin = "azure" + network_policy = "azure" + os_disk_size_gb = 60 + private_cluster_enabled = false + public_network_access_enabled = true + rbac_aad = true + rbac_aad_managed = true + role_based_access_control_enabled = true + sku_tier = "Standard" + vnet_subnet_id = var.bring_your_own_vnet ? azurerm_subnet.test[0].id : null + depends_on = [ + azurerm_subnet.test, + ] +} \ No newline at end of file diff --git a/examples/application_gateway_ingress/outputs.tf b/examples/application_gateway_ingress/outputs.tf new file mode 100644 index 00000000..bd10335d --- /dev/null +++ b/examples/application_gateway_ingress/outputs.tf @@ -0,0 +1,4 @@ +output "ingress_endpoint" { + depends_on = [time_sleep.wait_for_ingress] + value = "http://${data.kubernetes_ingress_v1.ing.status[0].load_balancer[0].ingress[0].ip}" +} diff --git a/examples/application_gateway_ingress/providers.tf b/examples/application_gateway_ingress/providers.tf new file mode 100644 index 00000000..ef95b157 --- /dev/null +++ b/examples/application_gateway_ingress/providers.tf @@ -0,0 +1,47 @@ +terraform { + required_version = ">=1.3" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.51, < 4.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.22.0" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + time = { + source = "hashicorp/time" + version = "0.9.1" + } + } +} + +provider "azurerm" { + features { + resource_group { + prevent_deletion_if_contains_resources = false + } + } +} + +resource "local_sensitive_file" "k8s_config" { + filename = "${path.module}/k8sconfig" + content = module.aks.kube_config_raw + + depends_on = [module.aks] +} + +# DO NOT DO THIS IN PRODUCTION ENVIRONMENT +provider "kubernetes" { + config_path = local_sensitive_file.k8s_config.filename + # host = module.aks.admin_host + # client_certificate = base64decode(module.aks.admin_client_certificate) + # client_key = base64decode(module.aks.admin_client_key) + # cluster_ca_certificate = base64decode(module.aks.admin_cluster_ca_certificate) +} + +provider "random" {} \ No newline at end of file diff --git a/examples/application_gateway_ingress/variables.tf b/examples/application_gateway_ingress/variables.tf new file mode 100644 index 00000000..b461f41f --- /dev/null +++ b/examples/application_gateway_ingress/variables.tf @@ -0,0 +1,29 @@ +variable "create_resource_group" { + type = bool + default = true + nullable = false +} + +variable "create_role_assignments_for_application_gateway" { + type = bool + default = true +} + +variable "location" { + default = "eastus" +} + +variable "resource_group_name" { + type = string + default = null +} + +variable "use_brown_field_application_gateway" { + type = bool + default = false +} + +variable "bring_your_own_vnet" { + type = bool + default = true +} diff --git a/examples/startup/main.tf b/examples/startup/main.tf index e72cb59c..30457b18 100644 --- a/examples/startup/main.tf +++ b/examples/startup/main.tf @@ -67,16 +67,19 @@ module "aks" { confidential_computing = { sgx_quote_helper_enabled = true } - disk_encryption_set_id = azurerm_disk_encryption_set.des.id - enable_auto_scaling = true - enable_host_encryption = true - http_application_routing_enabled = true - ingress_application_gateway_enabled = true - ingress_application_gateway_name = "${random_id.prefix.hex}-agw" - ingress_application_gateway_subnet_cidr = "10.52.1.0/24" - local_account_disabled = true - log_analytics_workspace_enabled = true - cluster_log_analytics_workspace_name = random_id.name.hex + disk_encryption_set_id = azurerm_disk_encryption_set.des.id + enable_auto_scaling = true + enable_host_encryption = true + http_application_routing_enabled = true + application_gateway_for_ingress = { + new_gw = { + name = "${random_id.prefix.hex}-agw" + subnet_cidr = "10.52.1.0/24" + } + } + local_account_disabled = true + log_analytics_workspace_enabled = true + cluster_log_analytics_workspace_name = random_id.name.hex maintenance_window = { allowed = [ { diff --git a/locals.tf b/locals.tf index b5c09745..e34cc7b1 100644 --- a/locals.tf +++ b/locals.tf @@ -10,8 +10,20 @@ locals { (contains(["rapid", "stable", "node-image"], var.automatic_channel_upgrade) && var.kubernetes_version == null && var.orchestrator_version == null) ) # Abstract the decision whether to create an Analytics Workspace or not. - create_analytics_solution = var.log_analytics_workspace_enabled && var.log_analytics_solution == null - create_analytics_workspace = var.log_analytics_workspace_enabled && var.log_analytics_workspace == null + create_analytics_solution = var.log_analytics_workspace_enabled && var.log_analytics_solution == null + create_analytics_workspace = var.log_analytics_workspace_enabled && var.log_analytics_workspace == null + default_nodepool_subnet_segments = try(split("/", var.vnet_subnet_id), []) + # Application Gateway ID: /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mygroup1/providers/Microsoft.Network/applicationGateways/myGateway1 + existing_application_gateway_for_ingress_id = try(var.brown_field_application_gateway_for_ingress.id, null) + existing_application_gateway_resource_group_for_ingress = var.brown_field_application_gateway_for_ingress == null ? null : local.existing_application_gateway_segments_for_ingress[4] + existing_application_gateway_segments_for_ingress = var.brown_field_application_gateway_for_ingress == null ? null : split("/", local.existing_application_gateway_for_ingress_id) + existing_application_gateway_subnet_resource_group_name = try(local.existing_application_gateway_subnet_segments[4], null) + # Subnet ID: /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mygroup1/providers/Microsoft.Network/virtualNetworks/myvnet1/subnets/mysubnet1 + existing_application_gateway_subnet_segments = try(split("/", var.brown_field_application_gateway_for_ingress.subnet_id), []) + existing_application_gateway_subnet_subscription_id_for_ingress = try(local.existing_application_gateway_subnet_segments[2], null) + existing_application_gateway_subnet_vnet_name = try(local.existing_application_gateway_subnet_segments[8], null) + existing_application_gateway_subscription_id_for_ingress = try(local.existing_application_gateway_segments_for_ingress[2], null) + ingress_application_gateway_enabled = var.brown_field_application_gateway_for_ingress != null || var.green_field_application_gateway_for_ingress != null # Abstract the decision whether to use an Analytics Workspace supplied via vars, provision one ourselves or leave it null. # This guarantees that local.log_analytics_workspace will contain a valid `id` and `name` IFF log_analytics_workspace_enabled # is set to `true`. @@ -41,5 +53,6 @@ locals { ], [var.vnet_subnet_id])) query_datasource_for_log_analytics_workspace_location = var.log_analytics_workspace_enabled && (var.log_analytics_workspace != null ? var.log_analytics_workspace.location == null : false) subnet_ids = toset([for id in local.potential_subnet_ids : id if id != null]) + use_brown_field_gw_for_ingress = var.brown_field_application_gateway_for_ingress != null } diff --git a/main.tf b/main.tf index d0a0507e..8bf66eb7 100644 --- a/main.tf +++ b/main.tf @@ -330,13 +330,13 @@ resource "azurerm_kubernetes_cluster" "main" { } } dynamic "ingress_application_gateway" { - for_each = var.ingress_application_gateway_enabled ? ["ingress_application_gateway"] : [] + for_each = local.ingress_application_gateway_enabled ? ["ingress_application_gateway"] : [] content { - gateway_id = var.ingress_application_gateway_id - gateway_name = var.ingress_application_gateway_name - subnet_cidr = var.ingress_application_gateway_subnet_cidr - subnet_id = var.ingress_application_gateway_subnet_id + gateway_id = try(var.brown_field_application_gateway_for_ingress.id, null) + gateway_name = try(var.green_field_application_gateway_for_ingress.name, null) + subnet_cidr = try(var.green_field_application_gateway_for_ingress.subnet_cidr, null) + subnet_id = try(var.green_field_application_gateway_for_ingress.subnet_id, null) } } dynamic "key_management_service" { @@ -571,6 +571,10 @@ resource "azurerm_kubernetes_cluster" "main" { (var.client_id == "" || var.client_secret == "") && var.identity_type == "UserAssigned" && try(length(var.identity_ids), 0) > 0) error_message = "When `kubelet_identity` is enabled - The `type` field in the `identity` block must be set to `UserAssigned` and `identity_ids` must be set." } + precondition { + condition = var.brown_field_application_gateway_for_ingress == null || var.green_field_application_gateway_for_ingress == null + error_message = "Either one of `var.existing_application_gateway_for_ingress` or `var.new_application_gateway_for_ingress` must be `null`." + } } } @@ -884,3 +888,77 @@ resource "azurerm_role_assignment" "network_contributor_on_subnet" { } } } + +data "azurerm_client_config" "this" {} + +data "azurerm_virtual_network" "application_gateway_vnet" { + count = var.create_role_assignments_for_application_gateway && local.use_brown_field_gw_for_ingress ? 1 : 0 + + name = local.existing_application_gateway_subnet_vnet_name + resource_group_name = local.existing_application_gateway_subnet_resource_group_name +} + +resource "azurerm_role_assignment" "application_gateway_existing_vnet_network_contributor" { + count = var.create_role_assignments_for_application_gateway && local.use_brown_field_gw_for_ingress ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.main.ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id + scope = data.azurerm_virtual_network.application_gateway_vnet[0].id + role_definition_name = "Network Contributor" + + lifecycle { + precondition { + condition = data.azurerm_client_config.this.subscription_id == local.existing_application_gateway_subnet_subscription_id_for_ingress + error_message = "Application Gateway's subnet must be in the same subscription, or `var.application_gateway_for_ingress.create_role_assignments` must be set to `false`." + } + } +} + +resource "azurerm_role_assignment" "application_gateway_new_vnet_network_contributor" { + count = var.create_role_assignments_for_application_gateway && !local.use_brown_field_gw_for_ingress ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.main.ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id + scope = join("/", slice(local.default_nodepool_subnet_segments, 0, length(local.default_nodepool_subnet_segments) - 2)) + role_definition_name = "Network Contributor" + + lifecycle { + precondition { + condition = var.green_field_application_gateway_for_ingress == null || !(var.create_role_assignments_for_application_gateway && var.vnet_subnet_id == null) + error_message = "When `var.vnet_subnet_id` is `null`, you must set `var.create_role_assignments_for_application_gateway` to `false`, set `var.new_application_gateway_for_ingress` to `null`." + } + } +} + +resource "azurerm_role_assignment" "existing_application_gateway_contributor" { + count = var.create_role_assignments_for_application_gateway && local.use_brown_field_gw_for_ingress ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.main.ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id + scope = var.brown_field_application_gateway_for_ingress.id + role_definition_name = "Contributor" + + lifecycle { + precondition { + condition = var.brown_field_application_gateway_for_ingress == null ? true : data.azurerm_client_config.this.subscription_id == local.existing_application_gateway_subscription_id_for_ingress + error_message = "Application Gateway must be in the same subscription, or `var.create_role_assignments_for_application_gateway` must be set to `false`." + } + } +} + +data "azurerm_resource_group" "ingress_gw" { + count = var.create_role_assignments_for_application_gateway && local.use_brown_field_gw_for_ingress ? 1 : 0 + + name = local.existing_application_gateway_resource_group_for_ingress +} + +data "azurerm_resource_group" "aks_rg" { + count = var.create_role_assignments_for_application_gateway ? 1 : 0 + + name = var.resource_group_name +} + +resource "azurerm_role_assignment" "application_gateway_resource_group_reader" { + count = var.create_role_assignments_for_application_gateway ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.main.ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id + scope = local.use_brown_field_gw_for_ingress ? data.azurerm_resource_group.ingress_gw[0].id : data.azurerm_resource_group.aks_rg[0].id + role_definition_name = "Reader" +} diff --git a/test/e2e/terraform_aks_test.go b/test/e2e/terraform_aks_test.go index c5a7044e..2affbf05 100644 --- a/test/e2e/terraform_aks_test.go +++ b/test/e2e/terraform_aks_test.go @@ -1,10 +1,17 @@ package e2e import ( + "fmt" + "io" "os" "regexp" + "strings" "testing" + "github.com/hashicorp/go-retryablehttp" + + "github.com/stretchr/testify/require" + test_helper "github.com/Azure/terraform-module-test-helper" "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" @@ -118,3 +125,66 @@ func TestExamples_differentLocationForLogAnalyticsSolution(t *testing.T) { Vars: vars, }, nil) } + +func TestExamples_applicationGatewayIngress(t *testing.T) { + useExistingAppGw := []struct { + useBrownFieldAppGw bool + bringYourOwnVnet bool + createRoleBindingForAppGw bool + }{ + { + bringYourOwnVnet: true, + useBrownFieldAppGw: true, + createRoleBindingForAppGw: true, + }, + { + bringYourOwnVnet: true, + useBrownFieldAppGw: false, + createRoleBindingForAppGw: true, + }, + { + bringYourOwnVnet: false, + useBrownFieldAppGw: false, + createRoleBindingForAppGw: false, + }, + } + for _, u := range useExistingAppGw { + t.Run(fmt.Sprintf("useExistingAppGw %t %t %t", u.bringYourOwnVnet, u.useBrownFieldAppGw, u.createRoleBindingForAppGw), func(t *testing.T) { + test_helper.RunE2ETest(t, "../../", "examples/application_gateway_ingress", terraform.Options{ + Upgrade: true, + Vars: map[string]interface{}{ + "bring_your_own_vnet": u.bringYourOwnVnet, + "use_brown_field_application_gateway": u.useBrownFieldAppGw, + "create_role_assignments_for_application_gateway": u.createRoleBindingForAppGw, + }, + }, func(t *testing.T, output test_helper.TerraformOutput) { + url, ok := output["ingress_endpoint"].(string) + require.True(t, ok) + html, err := getHTML(url) + require.NoError(t, err) + if strings.Contains(html, "Welcome to .NET") { + return + } + }) + }) + } +} + +func getHTML(url string) (string, error) { + client := retryablehttp.NewClient() + client.RetryMax = 10 + resp, err := client.Get(url) + if err != nil { + return "", err + } + defer func() { + _ = resp.Body.Close() + }() + + bytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return string(bytes), nil +} diff --git a/test/go.mod b/test/go.mod index 5e4125f3..e4e29b92 100644 --- a/test/go.mod +++ b/test/go.mod @@ -7,6 +7,7 @@ toolchain go1.21.2 require ( github.com/Azure/terraform-module-test-helper v0.17.0 github.com/gruntwork-io/terratest v0.46.1 + github.com/hashicorp/go-retryablehttp v0.7.5 github.com/stretchr/testify v1.8.4 ) diff --git a/variables.tf b/variables.tf index 46d7d3c8..f8e5c637 100644 --- a/variables.tf +++ b/variables.tf @@ -362,6 +362,19 @@ variable "azure_policy_enabled" { description = "Enable Azure Policy Addon." } +variable "brown_field_application_gateway_for_ingress" { + type = object({ + id = string + subnet_id = string + }) + default = null + description = <<-EOT + [Definition of `brown_field`](https://learn.microsoft.com/en-us/azure/application-gateway/tutorial-ingress-controller-add-on-existing) + * `id` - (Required) The ID of the Application Gateway that be used as cluster ingress. + * `subnet_id` - (Required) The ID of the Subnet which the Application Gateway is connected to. Must be set when `create_role_assignments` is `true`. + EOT +} + variable "client_id" { type = string default = "" @@ -403,6 +416,13 @@ variable "create_role_assignment_network_contributor" { nullable = false } +variable "create_role_assignments_for_application_gateway" { + type = bool + default = true + description = "(Optional) Whether to create the corresponding role assignments for application gateway or not. Defaults to `true`." + nullable = false +} + variable "default_node_pool_fips_enabled" { type = bool default = null @@ -439,6 +459,21 @@ variable "enable_node_public_ip" { description = "(Optional) Should nodes in this Node Pool have a Public IP Address? Defaults to false." } +variable "green_field_application_gateway_for_ingress" { + type = object({ + name = optional(string) + subnet_cidr = optional(string) + subnet_id = optional(string) + }) + default = null + description = <<-EOT + [Definition of `green_field`](https://learn.microsoft.com/en-us/azure/application-gateway/tutorial-ingress-controller-add-on-new) + * `name` - (Optional) The name of the Application Gateway to be used or created in the Nodepool Resource Group, which in turn will be integrated with the ingress controller of this Kubernetes Cluster. + * `subnet_cidr` - (Optional) The subnet CIDR to be used to create an Application Gateway, which in turn will be integrated with the ingress controller of this Kubernetes Cluster. + * `subnet_id` - (Optional) The ID of the subnet on which to create an Application Gateway, which in turn will be integrated with the ingress controller of this Kubernetes Cluster. +EOT +} + variable "http_application_routing_enabled" { type = bool default = false @@ -474,37 +509,6 @@ variable "image_cleaner_interval_hours" { description = "(Optional) Specifies the interval in hours when images should be cleaned up. Defaults to `48`." } -variable "ingress_application_gateway_enabled" { - type = bool - default = false - description = "Whether to deploy the Application Gateway ingress controller to this Kubernetes Cluster?" - nullable = false -} - -variable "ingress_application_gateway_id" { - type = string - default = null - description = "The ID of the Application Gateway to integrate with the ingress controller of this Kubernetes Cluster." -} - -variable "ingress_application_gateway_name" { - type = string - default = null - description = "The name of the Application Gateway to be used or created in the Nodepool Resource Group, which in turn will be integrated with the ingress controller of this Kubernetes Cluster." -} - -variable "ingress_application_gateway_subnet_cidr" { - type = string - default = null - description = "The subnet CIDR to be used to create an Application Gateway, which in turn will be integrated with the ingress controller of this Kubernetes Cluster." -} - -variable "ingress_application_gateway_subnet_id" { - type = string - default = null - description = "The ID of the subnet on which to create an Application Gateway, which in turn will be integrated with the ingress controller of this Kubernetes Cluster." -} - variable "key_vault_secrets_provider_enabled" { type = bool default = false @@ -676,7 +680,8 @@ variable "maintenance_window" { allowed = optional(list(object({ day = string hours = set(number) - })), []), + })), [ + ]), not_allowed = optional(list(object({ end = string start = string