From 64bef4c948112ea7399041638d2164e926ca48a0 Mon Sep 17 00:00:00 2001 From: Jon Morehouse Date: Tue, 5 Mar 2024 12:56:13 -0600 Subject: [PATCH] feat: introduce `azure-aks` sandbox (#45) feat: initial azure-aks sandbox --- .terraform.lock.hcl | 101 ++++++++++++++++++++++++++++++++++++++++++++ README.md | 34 ++++++++++++++- aks.tf | 56 ++++++++++++++++++++++++ network.tf | 42 ++++++++++++++++++ outputs.tf | 34 +++++++++++++++ providers.tf | 5 +++ registry.tf | 35 +++++++++++++++ resource_group.tf | 4 ++ ssh.tf | 20 +++++++++ terraform.tfvars | 7 +++ variables.tf | 62 +++++++++++++++++++++++++++ versions.tf | 16 +++++++ 12 files changed, 414 insertions(+), 2 deletions(-) create mode 100644 .terraform.lock.hcl create mode 100644 aks.tf create mode 100644 network.tf create mode 100644 outputs.tf create mode 100644 providers.tf create mode 100644 registry.tf create mode 100644 resource_group.tf create mode 100644 ssh.tf create mode 100644 terraform.tfvars create mode 100644 variables.tf create mode 100644 versions.tf diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl new file mode 100644 index 0000000..fa20695 --- /dev/null +++ b/.terraform.lock.hcl @@ -0,0 +1,101 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/azure/azapi" { + version = "1.12.1" + constraints = "~> 1.12.1" + hashes = [ + "h1:EaQL7pQCRm5iL2zy/dG7rOe2OZ0ZypuyVnpQAiAwJmM=", + "zh:1cf52e685ceb04e73e13fbf3f3036bff23a3274a4ceda8693c0612076a588166", + "zh:321b59c2a67c6cb4e5cf0dbe2cc978f5389d781e8b391f9b75bf4d830abd2ffe", + "zh:49046bd8020c3b44c6b5dc67041f181e4fff45e3bc1a9ff0646dd20c21c8ce47", + "zh:5784d0c326ec4825571577bc39b253019bd3b1030c19d67ca3436df2d7ba01c8", + "zh:5ad7e18d26f170c01888d8e65dab7aa475089aac7bf0106526fd57cdd56533bc", + "zh:6695854f4f655673bea85e37444bf0c070b440dba4bc269aa144d0f6b7c1cc5f", + "zh:7f372c897da6b9ad90869a8eb85b37dad4dff2d5d311b3eca1a2e6373e2271ed", + "zh:8afa1a2be1dada4e8be4ab72d9d56f36af1e486c9353d04aabf6e79db7310125", + "zh:90809364619238c45185bff25c7d9c4fde34253561d8183ebbe797456c44bc9c", + "zh:9338d44650c9e68e10a6bc2d69f7beacd5059e6ac681d2e388e80a1652d9c183", + "zh:c94ee6fb1df2c1d35f338107b5e73cdba86c4ecf9dcde95e2ca0132cbbd4bd7c", + "zh:de231d363b1a664c6b5d3af8d3b9cf542d04d4506fb9458ba6c8ebf94e0e32ae", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.94.0" + constraints = "~> 3.94.0" + hashes = [ + "h1:t3fM/PO8PLAA5mK3esAypp01V6Vh75kjPnNqxQeVrV0=", + "zh:20d102bc63096ade82f8da81c91afaffa858aa56fe9a7ad02f24f5ae5618bc53", + "zh:3ddb9d6173a4fdb9b2352a76324ee321976915544ae66cbb863c7a60f0593f05", + "zh:4bc6c62142f67192d2def11f4fd419c54dddd89a5448af036bfc60b15eb0509a", + "zh:4c5120c2101a51524af32c4220c5e376f97a227730dd92ec0b06ac677e4b39f2", + "zh:585fa7ab876d09899cd2d842f12bc28c34556b4d47919eceadefab6fa47f909f", + "zh:59de7ea462470dee7088fc4deeff48e1ffd286eaca1185c219be68dadde745b8", + "zh:8421a46dd3bc4bc2eb56f7eb9b91cc84a66070b72195a805862c6022adee2da0", + "zh:a2fcb5a091d5944dc50f1e51f53fa4d370810a507fbf4122920d756083d8df19", + "zh:beb6b93a2a16942625bb6ac1e52bf26878e35f5562f3173279423ca66553b6d7", + "zh:c6846892ea68f49c838d90b75793d1f3a866871dd701ccb575b1eecccd4e7051", + "zh:ddd59492b6d5ce4c83f06a5b16c520048f3e9bb898bab4f3910042f5c01ffeda", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.2" + constraints = ">= 3.0.0" + hashes = [ + "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.0" + hashes = [ + "h1:I8MBeauYA8J8yheLJ8oSMWqB0kovn16dF/wKZ1QTdkk=", + "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", + "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", + "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", + "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", + "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", + "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", + "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", + "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", + "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", + "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.5" + constraints = ">= 3.1.0" + hashes = [ + "h1:zeG5RmggBZW/8JWIVrdaeSJa0OG62uFX5HY1eE8SjzY=", + "zh:01cfb11cb74654c003f6d4e32bbef8f5969ee2856394a96d127da4949c65153e", + "zh:0472ea1574026aa1e8ca82bb6df2c40cd0478e9336b7a8a64e652119a2fa4f32", + "zh:1a8ddba2b1550c5d02003ea5d6cdda2eef6870ece86c5619f33edd699c9dc14b", + "zh:1e3bb505c000adb12cdf60af5b08f0ed68bc3955b0d4d4a126db5ca4d429eb4a", + "zh:6636401b2463c25e03e68a6b786acf91a311c78444b1dc4f97c539f9f78de22a", + "zh:76858f9d8b460e7b2a338c477671d07286b0d287fd2d2e3214030ae8f61dd56e", + "zh:a13b69fb43cb8746793b3069c4d897bb18f454290b496f19d03c3387d1c9a2dc", + "zh:a90ca81bb9bb509063b736842250ecff0f886a91baae8de65c8430168001dad9", + "zh:c4de401395936e41234f1956ebadbd2ed9f414e6908f27d578614aaa529870d4", + "zh:c657e121af8fde19964482997f0de2d5173217274f6997e16389e7707ed8ece8", + "zh:d68b07a67fbd604c38ec9733069fbf23441436fecf554de6c75c032f82e1ef19", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/README.md b/README.md index d2971e4..c450428 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,32 @@ -# terraform-azure-aks-sandbox -Azure AKS sandbox for Nuon apps. +# Azure AKS + +Standard Azure sandbox that provisions the following: + +* VPN +* AKS Cluster + +## Usage + +To use this in your BYOC app, please use the `azure-aks` runner type: + +```toml +version = "v1" + +[runner] +runner_type = "azure-aks" + +[sandbox] +terraform_version = "1.5.4" +[sandbox.public_repo] +directory = "azure-aks" +repo = "nuonco/sandboxes" +branch = "main" +``` + +## Testing + +This sandbox can be tested outside of `nuon` by following these steps: + +1. Ensure you have an Azure account setup and `az` installed +1. [Create Service Principal Credentials](https://learn.microsoft.com/en-us/azure/aks/learn/quick-kubernetes-deploy-terraform?tabs=bash#create-a-service-principal) +1. Create a `terraform.tfvars` with the correct variable inputs diff --git a/aks.tf b/aks.tf new file mode 100644 index 0000000..1d49164 --- /dev/null +++ b/aks.tf @@ -0,0 +1,56 @@ +module "aks" { + source = "Azure/aks/azurerm" + + prefix = var.nuon_id + resource_group_name = azurerm_resource_group.rg.name + kubernetes_version = var.cluster_version + automatic_channel_upgrade = "patch" + agents_availability_zones = ["1", "2"] + agents_count = null + agents_max_count = 2 + agents_max_pods = 100 + agents_min_count = 1 + agents_pool_name = "agents" + agents_pool_linux_os_configs = [ + { + transparent_huge_page_enabled = "always" + sysctl_configs = [ + { + fs_aio_max_nr = 65536 + fs_file_max = 100000 + fs_inotify_max_user_watches = 1000000 + } + ] + } + ] + agents_type = "VirtualMachineScaleSets" + azure_policy_enabled = true + enable_auto_scaling = true + enable_host_encryption = false + + green_field_application_gateway_for_ingress = { + name = "ingress" + subnet_cidr = local.appgw_cidr + } + create_role_assignments_for_application_gateway = true + local_account_disabled = false + log_analytics_workspace_enabled = false + net_profile_dns_service_ip = local.dns_service_ip + net_profile_service_cidr = local.service_cidr + network_plugin = "azure" + network_policy = "azure" + os_disk_size_gb = 60 + private_cluster_enabled = false + rbac_aad = true + rbac_aad_managed = true + role_based_access_control_enabled = true + sku_tier = "Standard" + vnet_subnet_id = module.network.vnet_subnets[0] + attached_acr_id_map = { + "${azurerm_container_registry.acr.name}" = azurerm_container_registry.acr.id + } + + depends_on = [ + module.network, + ] +} diff --git a/network.tf b/network.tf new file mode 100644 index 0000000..2187830 --- /dev/null +++ b/network.tf @@ -0,0 +1,42 @@ +locals { + // we create a network with two address spaces - one for node pool subnets and one for services, gateways etc. + address_spaces = ["10.0.0.0/16", "10.2.0.0/16"] + // node pool subnets + subnet_cidrs = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + subnet_names = ["a", "b", "c"] + + // app and services + appgw_cidr = "10.2.0.0/24" + service_cidr = "10.2.1.0/24" + dns_service_ip = "10.2.1.10" +} + +module "network" { + source = "Azure/network/azurerm" + resource_group_name = azurerm_resource_group.rg.name + address_spaces = local.address_spaces + + // we create three subnets - one for the nodes, one for ingresses and one for pods + subnet_prefixes = local.subnet_cidrs + subnet_names = local.subnet_names + + subnet_service_endpoints = { + "subnet1" : ["Microsoft.Sql"], + "subnet2" : ["Microsoft.Sql"], + "subnet3" : ["Microsoft.Sql"] + } + use_for_each = true + tags = { + environment = "dev" + costcenter = "it" + } + + depends_on = [azurerm_resource_group.rg] +} + +#resource "azurerm_subnet" "appgw" { + #address_prefixes = [local.appgw_cidr] + #name = "${var.nuon_id}-gw" + #resource_group_name = azurerm_resource_group.rg.name + #virtual_network_name = module.network.vnet_name +#} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..f6b64e7 --- /dev/null +++ b/outputs.tf @@ -0,0 +1,34 @@ +output "runner" { + value = {} +} + +output "vpn" { + value = { + name = module.network.vnet_name + subnet_ids = module.network.vnet_subnets + } +} + +output "acr" { + value = { + id = azurerm_container_registry.acr.id + login_server = azurerm_container_registry.acr.login_server + token_id = azurerm_container_registry_token.runner.id + password = nonsensitive(azurerm_container_registry_token_password.runner.password1[0].value) + } +} + +output "cluster" { + value = { + "id" = module.aks.aks_id + "name" = module.aks.aks_name + "client_certificate" = nonsensitive(module.aks.client_certificate) + "client_key" = nonsensitive(module.aks.client_key) + "cluster_ca_certificate" = nonsensitive(module.aks.cluster_ca_certificate) + "cluster_fqdn" = module.aks.cluster_fqdn + "oidc_issuer_url" = module.aks.oidc_issuer_url + "location" = module.aks.location + "kube_config_raw" = nonsensitive(module.aks.kube_config_raw) + "kube_admin_config_raw" = nonsensitive(module.aks.kube_admin_config_raw) + } +} diff --git a/providers.tf b/providers.tf new file mode 100644 index 0000000..8477b37 --- /dev/null +++ b/providers.tf @@ -0,0 +1,5 @@ +provider "azurerm" { + features {} +} + +provider "azapi" {} diff --git a/registry.tf b/registry.tf new file mode 100644 index 0000000..7f0b812 --- /dev/null +++ b/registry.tf @@ -0,0 +1,35 @@ +resource "azurerm_container_registry" "acr" { + name = var.nuon_id + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + sku = "Premium" + admin_enabled = false +} + +resource "azurerm_container_registry_scope_map" "acr" { + name = var.nuon_id + container_registry_name = azurerm_container_registry.acr.name + resource_group_name = azurerm_resource_group.rg.name + actions = [ + "repositories/${var.nuon_id}/content/read", + "repositories/${var.nuon_id}/content/write" + ] +} + +resource "random_pet" "token_name" { + prefix = "runner" + separator = "" +} + +resource "azurerm_container_registry_token" "runner" { + name = random_pet.token_name.id + container_registry_name = azurerm_container_registry.acr.name + resource_group_name = azurerm_resource_group.rg.name + scope_map_id = azurerm_container_registry_scope_map.acr.id +} + +resource "azurerm_container_registry_token_password" "runner" { + container_registry_token_id = azurerm_container_registry_token.runner.id + + password1 {} +} diff --git a/resource_group.tf b/resource_group.tf new file mode 100644 index 0000000..12cd31e --- /dev/null +++ b/resource_group.tf @@ -0,0 +1,4 @@ +resource "azurerm_resource_group" "rg" { + location = var.location + name = var.nuon_id +} diff --git a/ssh.tf b/ssh.tf new file mode 100644 index 0000000..17d71a1 --- /dev/null +++ b/ssh.tf @@ -0,0 +1,20 @@ +resource "random_pet" "ssh_key_name" { + prefix = "ssh" + separator = "" +} + +resource "azapi_resource_action" "ssh_public_key_gen" { + type = "Microsoft.Compute/sshPublicKeys@2022-11-01" + resource_id = azapi_resource.ssh_public_key.id + action = "generateKeyPair" + method = "POST" + + response_export_values = ["publicKey", "privateKey"] +} + +resource "azapi_resource" "ssh_public_key" { + type = "Microsoft.Compute/sshPublicKeys@2022-11-01" + name = random_pet.ssh_key_name.id + location = azurerm_resource_group.rg.location + parent_id = azurerm_resource_group.rg.id +} diff --git a/terraform.tfvars b/terraform.tfvars new file mode 100644 index 0000000..c9c7d99 --- /dev/null +++ b/terraform.tfvars @@ -0,0 +1,7 @@ +internal_root_domain = "foo.nuon.run" +public_root_domain = "public.nuon.run" +nuon_id = "plzxletmtwkqzizk" +location = "eastus" +tags = { + "managed_by" = "nuon" +} diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..344cf65 --- /dev/null +++ b/variables.tf @@ -0,0 +1,62 @@ +variable "nuon_id" { + type = string + description = "The nuon id for this install. Used for naming purposes." +} + +variable "tags" { + type = map(string) + description = "List of custom tags to add to the install resources. Used for taxonomic purposes." +} + +variable "location" { + type = string + description = "The location to launch the cluster in" +} + +// NOTE: if you would like to create an internal load balancer, with TLS, you will have to use the public domain. +variable "internal_root_domain" { + type = string + description = "internal root domain." +} + +variable "public_root_domain" { + type = string + description = "public root domain." +} + +// cluster configuration +variable "cluster_name" { + type = string + description = "The name of the EKS cluster. Will use the install ID by default." + default = "" +} + +variable "cluster_version" { + type = string + description = "The Kubernetes version to use for the EKS cluster." + default = "1.28" +} + +variable "vm_size" { + type = string + default = "Standard_D2_v2" + description = "The image size" +} + +variable "node_count" { + type = number + default = 2 + description = "The minimum number of nodes in the managed node pool." +} + +variable "instance_types" { + type = list(string) + default = ["t3a.medium"] + description = "The EC2 instance types to use for the EKS cluster." +} + +variable "username" { + type = string + description = "The admin username for the new cluster." + default = "azureadmin" +} diff --git a/versions.tf b/versions.tf new file mode 100644 index 0000000..8d269a5 --- /dev/null +++ b/versions.tf @@ -0,0 +1,16 @@ +terraform { + // TODO: uncomment when done testing + backend "s3" {} + + required_providers { + azapi = { + source = "Azure/azapi" + version = "~> 1.12.1" + } + + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.94.0" + } + } +}