[openstack] for-each master nodes

contrib/terraform/openstack/kubespray.tf

@@ -24,6 +24,7 @@ module "ips" {
   network_name                  = var.network_name
   router_id                     = module.network.router_id
   k8s_nodes                     = var.k8s_nodes
+  k8s_masters                   = var.k8s_masters
   k8s_master_fips               = var.k8s_master_fips
   bastion_fips                  = var.bastion_fips
   router_internal_port_id       = module.network.router_internal_port_id
@@ -44,6 +45,7 @@ module "compute" {
   number_of_bastions                           = var.number_of_bastions
   number_of_k8s_nodes_no_floating_ip           = var.number_of_k8s_nodes_no_floating_ip
   number_of_gfs_nodes_no_floating_ip           = var.number_of_gfs_nodes_no_floating_ip
+  k8s_masters                                  = var.k8s_masters
   k8s_nodes                                    = var.k8s_nodes
   bastion_root_volume_size_in_gb               = var.bastion_root_volume_size_in_gb
   etcd_root_volume_size_in_gb                  = var.etcd_root_volume_size_in_gb
@@ -70,6 +72,7 @@ module "compute" {
   flavor_bastion                               = var.flavor_bastion
   k8s_master_fips                              = module.ips.k8s_master_fips
   k8s_master_no_etcd_fips                      = module.ips.k8s_master_no_etcd_fips
+  k8s_masters_fips                             = module.ips.k8s_masters_fips
   k8s_node_fips                                = module.ips.k8s_node_fips
   k8s_nodes_fips                               = module.ips.k8s_nodes_fips
   bastion_fips                                 = module.ips.bastion_fips

contrib/terraform/openstack/modules/compute/main.tf

@@ -297,6 +297,64 @@ resource "openstack_compute_instance_v2" "k8s_master" {
   }
 }
 
+resource "openstack_networking_port_v2" "k8s_masters_port" {
+  for_each              = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {}
+  name                  = "${var.cluster_name}-k8s-${each.key}"
+  network_id            = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id
+  admin_state_up        = "true"
+  port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
+  security_group_ids    = var.port_security_enabled ? local.master_sec_groups : null
+  no_security_groups    = var.port_security_enabled ? null : false
+
+  depends_on = [
+    var.network_router_id
+  ]
+}
+
+resource "openstack_compute_instance_v2" "k8s_masters" {
+  for_each          = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {}
+  name              = "${var.cluster_name}-k8s-${each.key}"
+  availability_zone = each.value.az
+  image_id          = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null
+  flavor_id         = each.value.flavor
+  key_pair          = openstack_compute_keypair_v2.k8s.name
+
+  dynamic "block_device" {
+    for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : []
+    content {
+      uuid                  = local.image_to_use_master
+      source_type           = "image"
+      volume_size           = var.master_root_volume_size_in_gb
+      volume_type           = var.master_volume_type
+      boot_index            = 0
+      destination_type      = "volume"
+      delete_on_termination = true
+    }
+  }
+
+  network {
+    port = openstack_networking_port_v2.k8s_masters_port[each.key].id
+  }
+
+  dynamic "scheduler_hints" {
+    for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []
+    content {
+      group = openstack_compute_servergroup_v2.k8s_master[0].id
+    }
+  }
+
+  metadata = {
+    ssh_user         = var.ssh_user
+    kubespray_groups = "%{if each.value.etcd == true}etcd,%{endif}kube_control_plane,${var.supplementary_master_groups},k8s_cluster%{if each.value.floating_ip == false},no_floating%{endif}"
+    depends_on       = var.network_router_id
+    use_access_ip    = var.use_access_ip
+  }
+
+  provisioner "local-exec" {
+    command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_masters_fips : value.address]), 0)}/ > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}"
+  }
+}
+
 resource "openstack_networking_port_v2" "k8s_master_no_etcd_port" {
   count                 = var.number_of_k8s_masters_no_etcd
   name                  = "${var.cluster_name}-k8s-master-ne-${count.index + 1}"
@@ -760,6 +818,12 @@ resource "openstack_networking_floatingip_associate_v2" "k8s_master" {
   port_id               = element(openstack_networking_port_v2.k8s_master_port.*.id, count.index)
 }
 
+resource "openstack_networking_floatingip_associate_v2" "k8s_masters" {
+  for_each              = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip } : {}
+  floating_ip           = var.k8s_masters_fips[each.key].address
+  port_id               = openstack_networking_port_v2.k8s_masters_port[each.key].id
+}
+
 resource "openstack_networking_floatingip_associate_v2" "k8s_master_no_etcd" {
   count                 = var.master_root_volume_size_in_gb == 0 ? var.number_of_k8s_masters_no_etcd : 0
   floating_ip           = var.k8s_master_no_etcd_fips[count.index]

contrib/terraform/openstack/modules/compute/variables.tf

@@ -88,6 +88,10 @@ variable "k8s_node_fips" {
   type = list
 }
 
+variable "k8s_masters_fips" {
+  type = map
+}
+
 variable "k8s_nodes_fips" {
   type = map
 }
@@ -112,6 +116,8 @@ variable "k8s_allowed_egress_ips" {
   type = list
 }
 
+variable "k8s_masters" {}
+
 variable "k8s_nodes" {}
 
 variable "supplementary_master_groups" {

contrib/terraform/openstack/modules/ips/main.tf

@@ -14,6 +14,12 @@ resource "openstack_networking_floatingip_v2" "k8s_master" {
   depends_on = [null_resource.dummy_dependency]
 }
 
+resource "openstack_networking_floatingip_v2" "k8s_masters" {
+  for_each   = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip } : {}
+  pool       = var.floatingip_pool
+  depends_on = [null_resource.dummy_dependency]
+}
+
 # If user specifies pre-existing IPs to use in k8s_master_fips, do not create new ones.
 resource "openstack_networking_floatingip_v2" "k8s_master_no_etcd" {
   count      = length(var.k8s_master_fips) > 0 ? 0 : var.number_of_k8s_masters_no_etcd

contrib/terraform/openstack/modules/ips/outputs.tf

@@ -3,6 +3,10 @@ output "k8s_master_fips" {
   value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master[*].address
 }
 
+output "k8s_masters_fips" {
+  value = openstack_networking_floatingip_v2.k8s_masters
+}
+
 # If k8s_master_fips is already defined as input, keep the same value since new FIPs have not been created.
 output "k8s_master_no_etcd_fips" {
   value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master_no_etcd[*].address

contrib/terraform/openstack/modules/ips/variables.tf

@@ -16,6 +16,8 @@ variable "router_id" {
   default = ""
 }
 
+variable "k8s_masters" {}
+
 variable "k8s_nodes" {}
 
 variable "k8s_master_fips" {}

contrib/terraform/openstack/sample-inventory/cluster.tfvars

@@ -32,6 +32,28 @@ number_of_k8s_masters_no_floating_ip_no_etcd = 0
 
 flavor_k8s_master = "<UUID>"
 
+k8s_masters = {
+  # "master-1" = {
+  #   "az"          = "nova"
+  #   "flavor"      = "<UUID>"
+  #   "floating_ip" = true
+  #   "etcd" = true
+  # },
+  # "master-2" = {
+  #   "az"          = "nova"
+  #   "flavor"      = "<UUID>"
+  #   "floating_ip" = false
+  #   "etcd" = true
+  # },
+  # "master-3" = {
+  #   "az"          = "nova"
+  #   "flavor"      = "<UUID>"
+  #   "floating_ip" = true
+  #   "etcd" = true
+  # },
+}
+
+
 # nodes
 number_of_k8s_nodes = 2
 

contrib/terraform/openstack/variables.tf

@@ -286,6 +286,10 @@ variable "router_internal_port_id" {
   default     = null
 }
 
+variable "k8s_masters" {
+  default = {}
+}
+
 variable "k8s_nodes" {
   default = {}
 }