Add support for extra node pools

examples/multiple_node_pools/main.tf

@@ -0,0 +1,54 @@
+resource "random_id" "prefix" {
+  byte_length = 8
+}
+
+resource "azurerm_resource_group" "main" {
+  count = var.create_resource_group ? 1 : 0
+
+  location = var.location
+  name     = coalesce(var.resource_group_name, "${random_id.prefix.hex}-rg")
+}
+
+locals {
+  resource_group = {
+    name     = var.create_resource_group ? azurerm_resource_group.main[0].name : var.resource_group_name
+    location = var.location
+  }
+}
+
+resource "azurerm_virtual_network" "test" {
+  address_space       = ["10.52.0.0/16"]
+  location            = local.resource_group.location
+  name                = "${random_id.prefix.hex}-vn"
+  resource_group_name = local.resource_group.name
+}
+
+resource "azurerm_subnet" "test" {
+  address_prefixes                               = ["10.52.0.0/24"]
+  name                                           = "${random_id.prefix.hex}-sn"
+  resource_group_name                            = local.resource_group.name
+  virtual_network_name                           = azurerm_virtual_network.test.name
+  enforce_private_link_endpoint_network_policies = true
+}
+
+locals {
+  nodes = {
+    for i in range(3) : "worker${i}" => {
+      name       = substr("worker${i}${random_id.prefix.hex}", 0, 12)
+      vm_size    = "Standard_D2s_v3"
+      node_count = 1
+    }
+  }
+}
+
+module "aks" {
+  source = "../.."
+
+  prefix              = "prefix-${random_id.prefix.hex}"
+  resource_group_name = local.resource_group.name
+  os_disk_size_gb     = 60
+  sku_tier            = "Paid"
+  rbac_aad            = false
+  vnet_subnet_id      = azurerm_subnet.test.id
+  node_pools          = local.nodes
+}

examples/multiple_node_pools/providers.tf

@@ -0,0 +1,23 @@
+terraform {
+  required_version = ">=1.2"
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = ">= 3.40, < 4.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.3.2"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {
+    resource_group {
+      prevent_deletion_if_contains_resources = false
+    }
+  }
+}
+
+provider "random" {}

examples/multiple_node_pools/variables.tf

@@ -0,0 +1,14 @@
+variable "create_resource_group" {
+  type     = bool
+  default  = true
+  nullable = false
+}
+
+variable "location" {
+  default = "centralus"
+}
+
+variable "resource_group_name" {
+  type    = string
+  default = null
+}

main.tf

@@ -387,6 +387,137 @@ resource "azurerm_kubernetes_cluster" "main" {
   }
 }
 
+resource "azurerm_kubernetes_cluster_node_pool" "node_pool" {
+  for_each = var.node_pools
+
+  kubernetes_cluster_id         = azurerm_kubernetes_cluster.main.id
+  name                          = each.value.name
+  vm_size                       = each.value.vm_size
+  capacity_reservation_group_id = each.value.capacity_reservation_group_id
+  custom_ca_trust_enabled       = each.value.custom_ca_trust_enabled
+  enable_auto_scaling           = each.value.enable_auto_scaling
+  enable_host_encryption        = each.value.enable_host_encryption
+  enable_node_public_ip         = each.value.enable_node_public_ip
+  eviction_policy               = each.value.eviction_policy
+  fips_enabled                  = each.value.fips_enabled
+  host_group_id                 = each.value.host_group_id
+  kubelet_disk_type             = each.value.kubelet_disk_type
+  max_count                     = each.value.max_count
+  max_pods                      = each.value.max_pods
+  message_of_the_day            = each.value.message_of_the_day
+  min_count                     = each.value.min_count
+  mode                          = each.value.mode
+  node_count                    = each.value.node_count
+  node_labels                   = each.value.node_labels
+  node_public_ip_prefix_id      = each.value.node_public_ip_prefix_id
+  node_taints                   = each.value.node_taints
+  orchestrator_version          = each.value.orchestrator_version
+  os_disk_size_gb               = each.value.os_disk_size_gb
+  os_disk_type                  = each.value.os_disk_type
+  os_sku                        = each.value.os_sku
+  os_type                       = each.value.os_type
+  pod_subnet_id                 = each.value.pod_subnet_id
+  priority                      = each.value.priority
+  proximity_placement_group_id  = each.value.proximity_placement_group_id
+  scale_down_mode               = each.value.scale_down_mode
+  spot_max_price                = each.value.spot_max_price
+  tags                          = each.value.tags
+  ultra_ssd_enabled             = each.value.ultra_ssd_enabled
+  vnet_subnet_id                = each.value.vnet_subnet_id
+  workload_runtime              = each.value.workload_runtime
+  zones                         = each.value.zones
+
+  dynamic "kubelet_config" {
+    for_each = each.value.kubelet_config == null ? [] : ["kubelet_config"]
+
+    content {
+      allowed_unsafe_sysctls    = each.value.kubelet_config.allowed_unsafe_sysctls
+      container_log_max_line    = each.value.kubelet_config.container_log_max_files
+      container_log_max_size_mb = each.value.kubelet_config.container_log_max_size_mb
+      cpu_cfs_quota_enabled     = each.value.kubelet_config.cpu_cfs_quota_enabled
+      cpu_cfs_quota_period      = each.value.kubelet_config.cpu_cfs_quota_period
+      cpu_manager_policy        = each.value.kubelet_config.cpu_manager_policy
+      image_gc_high_threshold   = each.value.kubelet_config.image_gc_high_threshold
+      image_gc_low_threshold    = each.value.kubelet_config.image_gc_low_threshold
+      pod_max_pid               = each.value.kubelet_config.pod_max_pid
+      topology_manager_policy   = each.value.kubelet_config.topology_manager_policy
+    }
+  }
+  dynamic "linux_os_config" {
+    for_each = each.value.linux_os_config == null ? [] : ["linux_os_config"]
+
+    content {
+      swap_file_size_mb             = each.value.linux_os_config.swap_file_size_mb
+      transparent_huge_page_defrag  = each.value.linux_os_config.transparent_huge_page_defrag
+      transparent_huge_page_enabled = each.value.linux_os_config.transparent_huge_page_enabled
+
+      dynamic "sysctl_config" {
+        for_each = each.value.linux_os_config.sysctl_config == null ? [] : ["sysctl_config"]
+
+        content {
+          fs_aio_max_nr                      = each.value.linux_os_config.sysctl_config.fs_aio_max_nr
+          fs_file_max                        = each.value.linux_os_config.sysctl_config.fs_file_max
+          fs_inotify_max_user_watches        = each.value.linux_os_config.sysctl_config.fs_inotify_max_user_watches
+          fs_nr_open                         = each.value.linux_os_config.sysctl_config.fs_nr_open
+          kernel_threads_max                 = each.value.linux_os_config.sysctl_config.kernel_threads_max
+          net_core_netdev_max_backlog        = each.value.linux_os_config.sysctl_config.net_core_netdev_max_backlog
+          net_core_optmem_max                = each.value.linux_os_config.sysctl_config.net_core_optmem_max
+          net_core_rmem_default              = each.value.linux_os_config.sysctl_config.net_core_rmem_default
+          net_core_rmem_max                  = each.value.linux_os_config.sysctl_config.net_core_rmem_max
+          net_core_somaxconn                 = each.value.linux_os_config.sysctl_config.net_core_somaxconn
+          net_core_wmem_default              = each.value.linux_os_config.sysctl_config.net_core_wmem_default
+          net_core_wmem_max                  = each.value.linux_os_config.sysctl_config.net_core_wmem_max
+          net_ipv4_ip_local_port_range_max   = each.value.linux_os_config.sysctl_config.net_ipv4_ip_local_port_range_max
+          net_ipv4_ip_local_port_range_min   = each.value.linux_os_config.sysctl_config.net_ipv4_ip_local_port_range_min
+          net_ipv4_neigh_default_gc_thresh1  = each.value.linux_os_config.sysctl_config.net_ipv4_neigh_default_gc_thresh1
+          net_ipv4_neigh_default_gc_thresh2  = each.value.linux_os_config.sysctl_config.net_ipv4_neigh_default_gc_thresh2
+          net_ipv4_neigh_default_gc_thresh3  = each.value.linux_os_config.sysctl_config.net_ipv4_neigh_default_gc_thresh3
+          net_ipv4_tcp_fin_timeout           = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_fin_timeout
+          net_ipv4_tcp_keepalive_intvl       = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_keepalive_intvl
+          net_ipv4_tcp_keepalive_probes      = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_keepalive_probes
+          net_ipv4_tcp_keepalive_time        = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_keepalive_time
+          net_ipv4_tcp_max_syn_backlog       = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_max_syn_backlog
+          net_ipv4_tcp_max_tw_buckets        = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_max_tw_buckets
+          net_ipv4_tcp_tw_reuse              = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_tw_reuse
+          net_netfilter_nf_conntrack_buckets = each.value.linux_os_config.sysctl_config.net_netfilter_nf_conntrack_buckets
+          net_netfilter_nf_conntrack_max     = each.value.linux_os_config.sysctl_config.net_netfilter_nf_conntrack_max
+          vm_max_map_count                   = each.value.linux_os_config.sysctl_config.vm_max_map_count
+          vm_swappiness                      = each.value.linux_os_config.sysctl_config.vm_swappiness
+          vm_vfs_cache_pressure              = each.value.linux_os_config.sysctl_config.vm_vfs_cache_pressure
+        }
+      }
+    }
+  }
+  dynamic "node_network_profile" {
+    for_each = each.value.node_network_profile == null ? [] : ["node_network_profile"]
+
+    content {
+      node_public_ip_tags = each.value.node_network_profile.node_public_ip_tags
+    }
+  }
+  dynamic "upgrade_settings" {
+    for_each = each.value.upgrade_settings == null ? [] : ["upgrade_settings"]
+
+    content {
+      max_surge = each.value.upgrade_settings.max_surge
+    }
+  }
+  dynamic "windows_profile" {
+    for_each = each.value.windows_profile == null ? [] : ["windows_profile"]
+
+    content {
+      outbound_nat_enabled = each.value.windows_profile.outbound_nat_enabled
+    }
+  }
+
+  lifecycle {
+    precondition {
+      condition     = var.agents_type == "VirtualMachineScaleSets"
+      error_message = "Multiple Node Pools are only supported when the Kubernetes Cluster is using Virtual Machine Scale Sets."
+    }
+  }
+}
+
 resource "azurerm_log_analytics_workspace" "main" {
   count = local.create_analytics_workspace ? 1 : 0
 

test/e2e/terraform_aks_test.go

@@ -89,8 +89,17 @@ func TestExamplesNamedCluster(t *testing.T) {
 	})
 }
 
-func TestExamplesWithACR(t *testing.T) {
-	test_helper.RunE2ETest(t, "../../", "examples/with_acr", terraform.Options{
-		Upgrade: true,
-	}, nil)
+func TestExamplesWithoutAssertion(t *testing.T) {
+	examples := []string{
+		"examples/with_acr",
+		"examples/multiple_node_pools",
+	}
+	for _, e := range examples {
+		example := e
+		t.Run(example, func(t *testing.T) {
+			test_helper.RunE2ETest(t, "../../", example, terraform.Options{
+				Upgrade: true,
+			}, nil)
+		})
+	}
 }

test/upgrade/upgrade_test.go

@@ -75,16 +75,25 @@ func TestExampleUpgrade_named_cluster(t *testing.T) {
 	}, currentMajorVersion)
 }
 
-func TestExampleUpgrade_withACR(t *testing.T) {
-	currentRoot, err := test_helper.GetCurrentModuleRootPath()
-	if err != nil {
-		t.FailNow()
+func TestExampleUpgrade(t *testing.T) {
+	examples := []string{
+		"examples/with_acr",
+		"examples/multiple_node_pools",
 	}
-	currentMajorVersion, err := test_helper.GetCurrentMajorVersionFromEnv()
-	if err != nil {
-		t.FailNow()
+	for _, e := range examples {
+		example := e
+		t.Run(example, func(t *testing.T) {
+			currentRoot, err := test_helper.GetCurrentModuleRootPath()
+			if err != nil {
+				t.FailNow()
+			}
+			currentMajorVersion, err := test_helper.GetCurrentMajorVersionFromEnv()
+			if err != nil {
+				t.FailNow()
+			}
+			test_helper.ModuleUpgradeTest(t, "Azure", "terraform-azurerm-aks", example, currentRoot, terraform.Options{
+				Upgrade: true,
+			}, currentMajorVersion)
+		})
 	}
-	test_helper.ModuleUpgradeTest(t, "Azure", "terraform-azurerm-aks", "examples/with_acr", currentRoot, terraform.Options{
-		Upgrade: true,
-	}, currentMajorVersion)
 }