From f8e03c0f13a62499cf516f22e7c6bda3b524bdd4 Mon Sep 17 00:00:00 2001 From: Robin Wallace Date: Wed, 13 Apr 2022 07:24:54 +0200 Subject: [PATCH] [openstack] for-each master nodes --- contrib/terraform/openstack/kubespray.tf | 3 + .../openstack/modules/compute/main.tf | 64 +++++++++++++++++++ .../openstack/modules/compute/variables.tf | 6 ++ .../terraform/openstack/modules/ips/main.tf | 6 ++ .../openstack/modules/ips/outputs.tf | 4 ++ .../openstack/modules/ips/variables.tf | 2 + .../openstack/sample-inventory/cluster.tfvars | 22 +++++++ contrib/terraform/openstack/variables.tf | 4 ++ 8 files changed, 111 insertions(+) diff --git a/contrib/terraform/openstack/kubespray.tf b/contrib/terraform/openstack/kubespray.tf index 5093b881970..8e09c0d959a 100644 --- a/contrib/terraform/openstack/kubespray.tf +++ b/contrib/terraform/openstack/kubespray.tf @@ -24,6 +24,7 @@ module "ips" { network_name = var.network_name router_id = module.network.router_id k8s_nodes = var.k8s_nodes + k8s_masters = var.k8s_masters k8s_master_fips = var.k8s_master_fips bastion_fips = var.bastion_fips router_internal_port_id = module.network.router_internal_port_id @@ -44,6 +45,7 @@ module "compute" { number_of_bastions = var.number_of_bastions number_of_k8s_nodes_no_floating_ip = var.number_of_k8s_nodes_no_floating_ip number_of_gfs_nodes_no_floating_ip = var.number_of_gfs_nodes_no_floating_ip + k8s_masters = var.k8s_masters k8s_nodes = var.k8s_nodes bastion_root_volume_size_in_gb = var.bastion_root_volume_size_in_gb etcd_root_volume_size_in_gb = var.etcd_root_volume_size_in_gb @@ -70,6 +72,7 @@ module "compute" { flavor_bastion = var.flavor_bastion k8s_master_fips = module.ips.k8s_master_fips k8s_master_no_etcd_fips = module.ips.k8s_master_no_etcd_fips + k8s_masters_fips = module.ips.k8s_masters_fips k8s_node_fips = module.ips.k8s_node_fips k8s_nodes_fips = module.ips.k8s_nodes_fips bastion_fips = module.ips.bastion_fips diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index a6a15c015e9..16898353a08 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -297,6 +297,64 @@ resource "openstack_compute_instance_v2" "k8s_master" { } } +resource "openstack_networking_port_v2" "k8s_masters_port" { + for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {} + name = "${var.cluster_name}-k8s-${each.key}" + network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id + admin_state_up = "true" + port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled + security_group_ids = var.port_security_enabled ? local.master_sec_groups : null + no_security_groups = var.port_security_enabled ? null : false + + depends_on = [ + var.network_router_id + ] +} + +resource "openstack_compute_instance_v2" "k8s_masters" { + for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {} + name = "${var.cluster_name}-k8s-${each.key}" + availability_zone = each.value.az + image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null + flavor_id = each.value.flavor + key_pair = openstack_compute_keypair_v2.k8s.name + + dynamic "block_device" { + for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : [] + content { + uuid = local.image_to_use_master + source_type = "image" + volume_size = var.master_root_volume_size_in_gb + volume_type = var.master_volume_type + boot_index = 0 + destination_type = "volume" + delete_on_termination = true + } + } + + network { + port = openstack_networking_port_v2.k8s_masters_port[each.key].id + } + + dynamic "scheduler_hints" { + for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : [] + content { + group = openstack_compute_servergroup_v2.k8s_master[0].id + } + } + + metadata = { + ssh_user = var.ssh_user + kubespray_groups = "%{if each.value.etcd == true}etcd,%{endif}kube_control_plane,${var.supplementary_master_groups},k8s_cluster%{if each.value.floating_ip == false},no_floating%{endif}" + depends_on = var.network_router_id + use_access_ip = var.use_access_ip + } + + provisioner "local-exec" { + command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_masters_fips : value.address]), 0)}/ > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}" + } +} + resource "openstack_networking_port_v2" "k8s_master_no_etcd_port" { count = var.number_of_k8s_masters_no_etcd name = "${var.cluster_name}-k8s-master-ne-${count.index + 1}" @@ -760,6 +818,12 @@ resource "openstack_networking_floatingip_associate_v2" "k8s_master" { port_id = element(openstack_networking_port_v2.k8s_master_port.*.id, count.index) } +resource "openstack_networking_floatingip_associate_v2" "k8s_masters" { + for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip } : {} + floating_ip = var.k8s_masters_fips[each.key].address + port_id = openstack_networking_port_v2.k8s_masters_port[each.key].id +} + resource "openstack_networking_floatingip_associate_v2" "k8s_master_no_etcd" { count = var.master_root_volume_size_in_gb == 0 ? var.number_of_k8s_masters_no_etcd : 0 floating_ip = var.k8s_master_no_etcd_fips[count.index] diff --git a/contrib/terraform/openstack/modules/compute/variables.tf b/contrib/terraform/openstack/modules/compute/variables.tf index 09447684441..ca8034bb5a7 100644 --- a/contrib/terraform/openstack/modules/compute/variables.tf +++ b/contrib/terraform/openstack/modules/compute/variables.tf @@ -88,6 +88,10 @@ variable "k8s_node_fips" { type = list } +variable "k8s_masters_fips" { + type = map +} + variable "k8s_nodes_fips" { type = map } @@ -112,6 +116,8 @@ variable "k8s_allowed_egress_ips" { type = list } +variable "k8s_masters" {} + variable "k8s_nodes" {} variable "supplementary_master_groups" { diff --git a/contrib/terraform/openstack/modules/ips/main.tf b/contrib/terraform/openstack/modules/ips/main.tf index 243572162f2..3f962fdfc97 100644 --- a/contrib/terraform/openstack/modules/ips/main.tf +++ b/contrib/terraform/openstack/modules/ips/main.tf @@ -14,6 +14,12 @@ resource "openstack_networking_floatingip_v2" "k8s_master" { depends_on = [null_resource.dummy_dependency] } +resource "openstack_networking_floatingip_v2" "k8s_masters" { + for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip } : {} + pool = var.floatingip_pool + depends_on = [null_resource.dummy_dependency] +} + # If user specifies pre-existing IPs to use in k8s_master_fips, do not create new ones. resource "openstack_networking_floatingip_v2" "k8s_master_no_etcd" { count = length(var.k8s_master_fips) > 0 ? 0 : var.number_of_k8s_masters_no_etcd diff --git a/contrib/terraform/openstack/modules/ips/outputs.tf b/contrib/terraform/openstack/modules/ips/outputs.tf index 591cac2502f..3ff4622abf6 100644 --- a/contrib/terraform/openstack/modules/ips/outputs.tf +++ b/contrib/terraform/openstack/modules/ips/outputs.tf @@ -3,6 +3,10 @@ output "k8s_master_fips" { value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master[*].address } +output "k8s_masters_fips" { + value = openstack_networking_floatingip_v2.k8s_masters +} + # If k8s_master_fips is already defined as input, keep the same value since new FIPs have not been created. output "k8s_master_no_etcd_fips" { value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master_no_etcd[*].address diff --git a/contrib/terraform/openstack/modules/ips/variables.tf b/contrib/terraform/openstack/modules/ips/variables.tf index a30fffde025..b52888b847f 100644 --- a/contrib/terraform/openstack/modules/ips/variables.tf +++ b/contrib/terraform/openstack/modules/ips/variables.tf @@ -16,6 +16,8 @@ variable "router_id" { default = "" } +variable "k8s_masters" {} + variable "k8s_nodes" {} variable "k8s_master_fips" {} diff --git a/contrib/terraform/openstack/sample-inventory/cluster.tfvars b/contrib/terraform/openstack/sample-inventory/cluster.tfvars index a9464029824..3c2576775fd 100644 --- a/contrib/terraform/openstack/sample-inventory/cluster.tfvars +++ b/contrib/terraform/openstack/sample-inventory/cluster.tfvars @@ -32,6 +32,28 @@ number_of_k8s_masters_no_floating_ip_no_etcd = 0 flavor_k8s_master = "" +k8s_masters = { + # "master-1" = { + # "az" = "nova" + # "flavor" = "" + # "floating_ip" = true + # "etcd" = true + # }, + # "master-2" = { + # "az" = "nova" + # "flavor" = "" + # "floating_ip" = false + # "etcd" = true + # }, + # "master-3" = { + # "az" = "nova" + # "flavor" = "" + # "floating_ip" = true + # "etcd" = true + # }, +} + + # nodes number_of_k8s_nodes = 2 diff --git a/contrib/terraform/openstack/variables.tf b/contrib/terraform/openstack/variables.tf index 0ed836cf477..8f58a84fc7b 100644 --- a/contrib/terraform/openstack/variables.tf +++ b/contrib/terraform/openstack/variables.tf @@ -286,6 +286,10 @@ variable "router_internal_port_id" { default = null } +variable "k8s_masters" { + default = {} +} + variable "k8s_nodes" { default = {} }