From 86dcc7fa870c31915d7985835839451b34c5a1a7 Mon Sep 17 00:00:00 2001 From: Young-ook Kim Date: Tue, 7 Dec 2021 21:46:03 +0900 Subject: [PATCH 1/7] fix(bottlerocket): bottlerocket ami_type name update to follow up aws official release --- examples/bottlerocket/README.md | 11 +++++++++++ examples/bottlerocket/default.auto.tfvars | 11 ++++++++++- examples/bottlerocket/tc1.tfvars | 11 ++++++++--- main.tf | 4 ++-- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/examples/bottlerocket/README.md b/examples/bottlerocket/README.md index dfdb86c5..320bab66 100644 --- a/examples/bottlerocket/README.md +++ b/examples/bottlerocket/README.md @@ -21,6 +21,17 @@ terraform plan -var-file tc1.tfvars terraform apply -var-file tc1.tfvars ``` +## Launch Bottlerocket managed node group +You can configure an AMI type for your (aws managed or self managed) node groups. For GPU instance types, you can set the `ami_type` parameter in the node group definition. GPU instance types should use the AL2_x86_64_GPU for its ami type or Non-GPU instances should use the AL2_x86_64. And ARM architecture based instance should use AL2_ARM_64. + +Possible values: +- AL2_x86_64 +- AL2_x86_64_GPU +- AL2_ARM_64 +- CUSTOM +- BOTTLEROCKET_ARM_64 +- BOTTLEROCKET_x86_64 + ## Clean up To remove all infrastrcuture, run terraform: ```sh diff --git a/examples/bottlerocket/default.auto.tfvars b/examples/bottlerocket/default.auto.tfvars index e221f922..49bb3845 100644 --- a/examples/bottlerocket/default.auto.tfvars +++ b/examples/bottlerocket/default.auto.tfvars @@ -9,6 +9,15 @@ node_groups = [ { name = "bottlerocket" instance_type = "t3.small" - ami_type = "BR_x86_64" + ami_type = "BOTTLEROCKET_x86_64" }, ] + + +# allowed values for 'ami_type' +# - AL2_x86_64 +# - AL2_x86_64_GPU +# - AL2_ARM_64 +# - CUSTOM +# - BOTTLEROCKET_ARM_64 +# - BOTTLEROCKET_x86_64 diff --git a/examples/bottlerocket/tc1.tfvars b/examples/bottlerocket/tc1.tfvars index adfef13d..364525f1 100644 --- a/examples/bottlerocket/tc1.tfvars +++ b/examples/bottlerocket/tc1.tfvars @@ -14,7 +14,12 @@ managed_node_groups = [ desired_size = 1 instance_type = "t3.small" ami_type = "AL2_x86_64" - } + }, + { + name = "bottlerocket" + instance_type = "t3.small" + ami_type = "BOTTLEROCKET_x86_64" + }, ] node_groups = [ { @@ -29,7 +34,7 @@ node_groups = [ { name = "bottlerocket" instance_type = "t3.small" - ami_type = "BR_x86_64" + ami_type = "BOTTLEROCKET_x86_64" }, { name = "al2-gpu" @@ -40,5 +45,5 @@ node_groups = [ name = "al2-arm" instance_type = "m6g.medium" ami_type = "AL2_ARM_64" - } + }, ] diff --git a/main.tf b/main.tf index 246c1927..7d39db5e 100644 --- a/main.tf +++ b/main.tf @@ -130,7 +130,7 @@ data "aws_ami" "br" { } filter { name = "architecture" - values = [length(regexall("ARM", lookup(each.value, "ami_type", "BR_x86_64"))) > 0 ? "arm64" : "x86_64"] + values = [length(regexall("ARM", lookup(each.value, "ami_type", "BOTTLEROCKET_x86_64"))) > 0 ? "arm64" : "x86_64"] } } @@ -337,7 +337,7 @@ resource "aws_eks_node_group" "ng" { node_group_name = join("-", [aws_eks_cluster.cp.name, each.key]) node_role_arn = aws_iam_role.ng.0.arn subnet_ids = local.subnet_ids - ami_type = lookup(each.value, "ami_type", "AL2_x86_64") # available values ["AL2_x86_64", "AL2_x86_64_GPU", "AL2_ARM_64"] + ami_type = lookup(each.value, "ami_type", "AL2_x86_64") instance_types = [lookup(each.value, "instance_type", "m5.xlarge")] version = aws_eks_cluster.cp.version tags = merge(local.default-tags, var.tags) From 632a2f26afe2c284186b5462debd9a18767419b1 Mon Sep 17 00:00:00 2001 From: Young-ook Kim Date: Wed, 8 Dec 2021 16:03:27 +0900 Subject: [PATCH 2/7] fix(bottlerocket): added the minimal required version definition for aws provider --- examples/bottlerocket/main.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/bottlerocket/main.tf b/examples/bottlerocket/main.tf index acb6f7ed..43171c98 100644 --- a/examples/bottlerocket/main.tf +++ b/examples/bottlerocket/main.tf @@ -2,6 +2,12 @@ terraform { required_version = "~> 1.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.64" + } + } } provider "aws" { From 65c23a342f54cc50a071e177fcd0b60dad068f22 Mon Sep 17 00:00:00 2001 From: Young-ook Kim Date: Wed, 8 Dec 2021 17:11:56 +0900 Subject: [PATCH 3/7] feat(bottlerocket): added multi-arch bottle rocket node groups test case --- .../{tc1.tfvars => fixture.tc1.tfvars} | 5 --- examples/bottlerocket/fixture.tc2.tfvars | 33 +++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) rename examples/bottlerocket/{tc1.tfvars => fixture.tc1.tfvars} (88%) create mode 100644 examples/bottlerocket/fixture.tc2.tfvars diff --git a/examples/bottlerocket/tc1.tfvars b/examples/bottlerocket/fixture.tc1.tfvars similarity index 88% rename from examples/bottlerocket/tc1.tfvars rename to examples/bottlerocket/fixture.tc1.tfvars index 364525f1..ae7fc3f2 100644 --- a/examples/bottlerocket/tc1.tfvars +++ b/examples/bottlerocket/fixture.tc1.tfvars @@ -15,11 +15,6 @@ managed_node_groups = [ instance_type = "t3.small" ami_type = "AL2_x86_64" }, - { - name = "bottlerocket" - instance_type = "t3.small" - ami_type = "BOTTLEROCKET_x86_64" - }, ] node_groups = [ { diff --git a/examples/bottlerocket/fixture.tc2.tfvars b/examples/bottlerocket/fixture.tc2.tfvars new file mode 100644 index 00000000..8f68cfa7 --- /dev/null +++ b/examples/bottlerocket/fixture.tc2.tfvars @@ -0,0 +1,33 @@ +aws_region = "ap-northeast-2" +name = "eks-bottlerocket-tc2" +tags = { + env = "dev" + test = "tc2" +} +kubernetes_version = "1.21" +enable_ssm = true +managed_node_groups = [ + { + name = "bottlerocket-x86" + instance_type = "t3.small" + ami_type = "BOTTLEROCKET_x86_64" + }, + { + name = "bottlerocket-arm" + instance_type = "m6g.medium" + ami_type = "BOTTLEROCKET_ARM_64" + }, + +] +node_groups = [ + { + name = "bottlerocket-x86" + instance_type = "t3.small" + ami_type = "BOTTLEROCKET_x86_64" + }, + { + name = "bottlerocket-arm" + instance_type = "m6g.medium" + ami_type = "BOTTLEROCKET_ARM_64" + }, +] From 0857bb09dc3e1fa54b193233bb584da4000f4005 Mon Sep 17 00:00:00 2001 From: Young-ook Kim Date: Wed, 8 Dec 2021 23:05:07 +0900 Subject: [PATCH 4/7] feat(bottlerocket): aded new userdata for bottlerocket managed node groups --- main.tf | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index 7d39db5e..e689033a 100644 --- a/main.tf +++ b/main.tf @@ -157,7 +157,7 @@ data "template_cloudinit_config" "ng" { } } -data "template_file" "br" { +data "template_file" "ng_br" { for_each = { for ng in var.node_groups : ng.name => ng } template = file("${path.module}/templates/bottlerocket.tpl") vars = { @@ -175,7 +175,7 @@ resource "aws_launch_template" "ng" { name = format("eks-%s", uuid()) tags = merge(local.default-tags, local.eks-tag, var.tags) image_id = length(regexall("^AL2", lookup(each.value, "ami_type", "AL2_x86_64"))) > 0 ? data.aws_ami.eks[each.key].id : data.aws_ami.br[each.key].id - user_data = base64encode(length(regexall("^AL2", lookup(each.value, "ami_type", "AL2_x86_64"))) > 0 ? data.template_cloudinit_config.ng[each.key].rendered : data.template_file.br[each.key].rendered) + user_data = base64encode(length(regexall("^AL2", lookup(each.value, "ami_type", "AL2_x86_64"))) > 0 ? data.template_cloudinit_config.ng[each.key].rendered : data.template_file.ng_br[each.key].rendered) instance_type = lookup(each.value, "instance_type", "t3.medium") iam_instance_profile { @@ -305,11 +305,24 @@ data "template_cloudinit_config" "mng" { } } +data "template_file" "mng_br" { + for_each = { for ng in var.managed_node_groups : ng.name => ng } + template = file("${path.module}/templates/bottlerocket.tpl") + vars = { + cluster_name = aws_eks_cluster.cp.name + cluster_endpoint = aws_eks_cluster.cp.endpoint + cluster_ca_data = aws_eks_cluster.cp.certificate_authority.0.data + admin_container_enabled = false + admin_container_superpowered = false + admin_container_source = "" + } +} + resource "aws_launch_template" "mng" { for_each = { for ng in var.managed_node_groups : ng.name => ng } name = format("eks-%s", uuid()) tags = merge(local.default-tags, local.eks-tag, var.tags) - user_data = data.template_cloudinit_config.mng[each.key].rendered + user_data = (length(regexall("^AL2", lookup(each.value, "ami_type", "AL2_x86_64"))) > 0) ? data.template_cloudinit_config.mng[each.key].rendered : base64encode(data.template_file.mng_br[each.key].rendered) block_device_mappings { device_name = "/dev/xvda" From 7d7b540df6628b8423f9da44dcbe53e67a660c30 Mon Sep 17 00:00:00 2001 From: Young-ook Kim Date: Fri, 10 Dec 2021 19:36:21 +0900 Subject: [PATCH 5/7] fix(bottlerocket): updated user-data config for both al2 and bottlerocket os --- examples/bottlerocket/fixture.tc2.tfvars | 1 - main.tf | 76 ++++++++++-------------- variables.tf | 5 ++ 3 files changed, 37 insertions(+), 45 deletions(-) diff --git a/examples/bottlerocket/fixture.tc2.tfvars b/examples/bottlerocket/fixture.tc2.tfvars index 8f68cfa7..fa0592c5 100644 --- a/examples/bottlerocket/fixture.tc2.tfvars +++ b/examples/bottlerocket/fixture.tc2.tfvars @@ -17,7 +17,6 @@ managed_node_groups = [ instance_type = "m6g.medium" ami_type = "BOTTLEROCKET_ARM_64" }, - ] node_groups = [ { diff --git a/main.tf b/main.tf index e689033a..ad4397ba 100644 --- a/main.tf +++ b/main.tf @@ -110,8 +110,13 @@ data "aws_ami" "eks" { most_recent = true filter { - name = "name" - values = [format(length(regexall("ARM|GPU$", lookup(each.value, "ami_type", "AL2_x86_64"))) > 0 ? "amazon-eks-*-node-%s-*" : "amazon-eks-node-%s-*", var.kubernetes_version)] + name = "name" + values = [ + format(length(regexall("^AL2", lookup(each.value, "ami_type", local.default_eks_config.ami_type))) > 0 ? + (length(regexall("ARM|GPU$", lookup(each.value, "ami_type", local.default_eks_config.ami_type))) > 0 ? "amazon-eks-*-node-%s-*" : "amazon-eks-node-%s-*") : + (length(regexall("^BOTTLEROCKET", lookup(each.value, "ami_type", local.default_eks_config.ami_type))) > 0 ? "bottlerocket-aws-k8s-%s-*" : "amazon-eks-node-%s-*") + , var.kubernetes_version) + ] } filter { name = "architecture" @@ -119,21 +124,6 @@ data "aws_ami" "eks" { } } -data "aws_ami" "br" { - for_each = { for ng in var.node_groups : ng.name => ng } - owners = ["amazon"] - most_recent = true - - filter { - name = "name" - values = [format("bottlerocket-aws-k8s-%s-*", var.kubernetes_version)] - } - filter { - name = "architecture" - values = [length(regexall("ARM", lookup(each.value, "ami_type", "BOTTLEROCKET_x86_64"))) > 0 ? "arm64" : "x86_64"] - } -} - data "template_cloudinit_config" "ng" { for_each = { for ng in var.node_groups : ng.name => ng } base64_encode = true @@ -157,16 +147,15 @@ data "template_cloudinit_config" "ng" { } } -data "template_file" "ng_br" { - for_each = { for ng in var.node_groups : ng.name => ng } +data "template_file" "br" { template = file("${path.module}/templates/bottlerocket.tpl") vars = { cluster_name = aws_eks_cluster.cp.name cluster_endpoint = aws_eks_cluster.cp.endpoint cluster_ca_data = aws_eks_cluster.cp.certificate_authority.0.data - admin_container_enabled = false - admin_container_superpowered = false - admin_container_source = "" + admin_container_enabled = lookup(var.bottlerocket_config, "admin_container_enabled", local.default_bottlerocket_config.admin_container_enabled) + admin_container_superpowered = lookup(var.bottlerocket_config, "admin_container_superpowered", local.default_bottlerocket_config.admin_container_superpowered) + admin_container_source = lookup(var.bottlerocket_config, "admin_container_source", local.default_bottlerocket_config.admin_container_source) } } @@ -174,9 +163,15 @@ resource "aws_launch_template" "ng" { for_each = { for ng in var.node_groups : ng.name => ng } name = format("eks-%s", uuid()) tags = merge(local.default-tags, local.eks-tag, var.tags) - image_id = length(regexall("^AL2", lookup(each.value, "ami_type", "AL2_x86_64"))) > 0 ? data.aws_ami.eks[each.key].id : data.aws_ami.br[each.key].id - user_data = base64encode(length(regexall("^AL2", lookup(each.value, "ami_type", "AL2_x86_64"))) > 0 ? data.template_cloudinit_config.ng[each.key].rendered : data.template_file.ng_br[each.key].rendered) - instance_type = lookup(each.value, "instance_type", "t3.medium") + image_id = data.aws_ami.eks[each.key].id + instance_type = lookup(each.value, "instance_type", local.default_eks_config.instance_type) + user_data = ( + length(regexall("^AL2", lookup(each.value, "ami_type", local.default_eks_config.ami_type))) > 0 ? + data.template_cloudinit_config.ng[each.key].rendered : + length(regexall("^BOTTLEROCKET", lookup(each.value, "ami_type", local.default_eks_config.ami_type))) > 0 ? + base64encode(data.template_file.br.rendered) : + data.template_cloudinit_config.ng[each.key].rendered + ) iam_instance_profile { arn = aws_iam_instance_profile.ng.0.arn @@ -305,24 +300,17 @@ data "template_cloudinit_config" "mng" { } } -data "template_file" "mng_br" { - for_each = { for ng in var.managed_node_groups : ng.name => ng } - template = file("${path.module}/templates/bottlerocket.tpl") - vars = { - cluster_name = aws_eks_cluster.cp.name - cluster_endpoint = aws_eks_cluster.cp.endpoint - cluster_ca_data = aws_eks_cluster.cp.certificate_authority.0.data - admin_container_enabled = false - admin_container_superpowered = false - admin_container_source = "" - } -} - resource "aws_launch_template" "mng" { - for_each = { for ng in var.managed_node_groups : ng.name => ng } - name = format("eks-%s", uuid()) - tags = merge(local.default-tags, local.eks-tag, var.tags) - user_data = (length(regexall("^AL2", lookup(each.value, "ami_type", "AL2_x86_64"))) > 0) ? data.template_cloudinit_config.mng[each.key].rendered : base64encode(data.template_file.mng_br[each.key].rendered) + for_each = { for ng in var.managed_node_groups : ng.name => ng } + name = format("eks-%s", uuid()) + tags = merge(local.default-tags, local.eks-tag, var.tags) + user_data = ( + length(regexall("^AL2", lookup(each.value, "ami_type", local.default_eks_config.ami_type))) > 0 ? + data.template_cloudinit_config.mng[each.key].rendered : + length(regexall("^BOTTLEROCKET", lookup(each.value, "ami_type", local.default_eks_config.ami_type))) > 0 ? + base64encode(data.template_file.br.rendered) : + data.template_cloudinit_config.mng[each.key].rendered + ) block_device_mappings { device_name = "/dev/xvda" @@ -350,8 +338,8 @@ resource "aws_eks_node_group" "ng" { node_group_name = join("-", [aws_eks_cluster.cp.name, each.key]) node_role_arn = aws_iam_role.ng.0.arn subnet_ids = local.subnet_ids - ami_type = lookup(each.value, "ami_type", "AL2_x86_64") - instance_types = [lookup(each.value, "instance_type", "m5.xlarge")] + ami_type = lookup(each.value, "ami_type", local.default_eks_config.ami_type) + instance_types = [lookup(each.value, "instance_type", local.default_eks_config.instance_type)] version = aws_eks_cluster.cp.version tags = merge(local.default-tags, var.tags) diff --git a/variables.tf b/variables.tf index 8a36b0a0..e5f19cc1 100644 --- a/variables.tf +++ b/variables.tf @@ -45,6 +45,11 @@ variable "enable_ssm" { default = false } +variable "bottlerocket_config" { + description = "Bottlerocket OS configuration" + default = {} +} + ### security variable "policy_arns" { description = "A list of policy ARNs to attach the node groups role" From 2329494865ac18e79b50549ecd946c9bc34a0917 Mon Sep 17 00:00:00 2001 From: Young-ook Kim Date: Fri, 10 Dec 2021 20:48:53 +0900 Subject: [PATCH 6/7] feat(bottlerocket): option to disable control container --- main.tf | 1 + templates/bottlerocket.tpl | 3 +++ 2 files changed, 4 insertions(+) diff --git a/main.tf b/main.tf index ad4397ba..a1d7fa48 100644 --- a/main.tf +++ b/main.tf @@ -156,6 +156,7 @@ data "template_file" "br" { admin_container_enabled = lookup(var.bottlerocket_config, "admin_container_enabled", local.default_bottlerocket_config.admin_container_enabled) admin_container_superpowered = lookup(var.bottlerocket_config, "admin_container_superpowered", local.default_bottlerocket_config.admin_container_superpowered) admin_container_source = lookup(var.bottlerocket_config, "admin_container_source", local.default_bottlerocket_config.admin_container_source) + control_container_enabled = var.enable_ssm } } diff --git a/templates/bottlerocket.tpl b/templates/bottlerocket.tpl index fca9a51f..f53d0941 100644 --- a/templates/bottlerocket.tpl +++ b/templates/bottlerocket.tpl @@ -9,3 +9,6 @@ superpowered = ${admin_container_superpowered} %{ if admin_container_source != "" } source = "${admin_container_source}" %{ endif } + +[settings.host-containers.control] +enabled = ${control_container_enabled} From f862b24b512ee4369fc07f35463b55605ee7d363 Mon Sep 17 00:00:00 2001 From: Young-ook Kim Date: Fri, 10 Dec 2021 23:50:19 +0900 Subject: [PATCH 7/7] feat(bottlerocket): default eks config values --- defaults.tf | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 defaults.tf diff --git a/defaults.tf b/defaults.tf new file mode 100644 index 00000000..6c634ed7 --- /dev/null +++ b/defaults.tf @@ -0,0 +1,13 @@ +### default values + +locals { + default_eks_config = { + ami_type = "AL2_x86_64" + instance_type = "t3.medium" + } + default_bottlerocket_config = { + admin_container_enabled = false + admin_container_superpowered = false + admin_container_source = "" + } +}