From 0abb55e3f48d8b082b6c307561d2af850e5d407d Mon Sep 17 00:00:00 2001 From: Lawrence McDaniel Date: Sat, 25 Feb 2023 07:07:25 -0600 Subject: [PATCH] EKS release 1.25 w dual managed node groups and pod affinity (#46) * actively manage the EBS drive volume to give us more control over its lifecycle * add node affinity * lint * lint * rename pod affinity label key * add a hosting node group * refactor * lint * lint * add a managed EBS volume * add post-deployment script * bump kubernetes releast to 1.25 * deprecate cert_manager_image_version * lint * lint * bump versions: vpc-cni=v1.16.0-eksbuild.1, aws-ebs-csi-driver=v1.16.0-eksbuild.1 * testing * add bastion IAM arn to aws-auth.mapUsers * add bastion IAM user to aws-auth.mapUsers via Terragrunt so that it can be customized * lint * add change notes --- CHANGELOG.md | 9 + cookiecutter.json | 21 +- doc/README.rst | 22 +- .../doc/QUICKSTART.rst | 6 +- {{cookiecutter.github_repo_name}}/make.sh | 16 +- .../environments/modules/mysql/kubernetes.tf | 3 +- .../wordpress/config/resource-quota.yaml.tpl | 9 - .../config/wordpress-values.yaml.tpl | 9 +- .../modules/wordpress/ebs_volume.tf | 140 +++ .../wordpress/kubernetes-resource-quota.tf | 29 - .../environments/modules/wordpress/main.tf | 32 +- .../environments/modules/wordpress/outputs.tf | 51 + .../modules/wordpress/variables.tf | 4 + .../client.hcl | 28 +- .../terragrunt.hcl | 3 + .../stacks/modules/kubernetes/main.tf | 53 +- .../stacks/modules/kubernetes/outputs.tf | 17 +- .../stacks/modules/kubernetes/variables.tf | 20 +- .../kubernetes_cert_manager/certificate.tf | 21 + .../kubernetes_cert_manager/cluster-issuer.tf | 22 + .../modules/kubernetes_cert_manager/main.tf | 108 +- .../manifests/cert-manager-values.yaml.tpl | 24 +- .../kubernetes_cert_manager/variables.tf | 4 - .../kubernetes_dashboard/yml/values.yaml | 358 +------ .../modules/kubernetes_ingress_clb/main.tf | 17 + .../yml/nginx-values.yaml | 10 + .../modules/kubernetes_karpenter/main.tf | 60 +- .../yml/karpenter-values.yaml | 15 + .../modules/kubernetes_kubeapps/main.tf | 4 + .../yml/kubeapps-values.yaml | 10 + .../config/kubecost-values.yaml | 951 +----------------- .../kubernetes_kubecost/config/values.yaml | 41 - .../modules/kubernetes_kubecost/main.tf | 8 +- .../config/metrics-server-values.yaml | 10 + .../modules/kubernetes_metricsserver/main.tf | 10 +- .../modules/kubernetes_prometheus/main.tf | 30 +- .../yml/prometheus-values.yaml | 10 + .../kubernetes_prometheus/yml/values.yaml | 20 +- .../stacks/modules/kubernetes_vpa/main.tf | 11 +- .../yml/vertical-pod-autoscaler-values.yaml | 10 + .../kubernetes/terragrunt.hcl | 67 +- .../kubernetes_cert_manager/terragrunt.hcl | 2 - .../stack.hcl | 28 +- 43 files changed, 689 insertions(+), 1634 deletions(-) delete mode 100644 {{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/config/resource-quota.yaml.tpl create mode 100644 {{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/ebs_volume.tf delete mode 100644 {{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/kubernetes-resource-quota.tf create mode 100644 {{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/certificate.tf create mode 100644 {{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/cluster-issuer.tf create mode 100644 {{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_ingress_clb/yml/nginx-values.yaml create mode 100644 {{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_karpenter/yml/karpenter-values.yaml create mode 100644 {{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubeapps/yml/kubeapps-values.yaml delete mode 100644 {{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/config/values.yaml create mode 100644 {{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_metricsserver/config/metrics-server-values.yaml create mode 100644 {{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/yml/prometheus-values.yaml create mode 100644 {{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_vpa/yml/vertical-pod-autoscaler-values.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index a0ad34ce..cca15fee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.0.25] (2023-2-25) + +- bump AWS EKS to release 1.25 +- bump AWS EKS Add-on versions +- parameterize aws-auth.mapUsers +- refactor AWS EKS Managed Node groups into two groups, service and hosting. Default the service group to 3 nodes and the hosting group to 0. +- remove AWS EKS service node taints. replace these with node affinity for service pods to encourage isolation of these from nodes running mostly application software. +- create AWS EBS volumes for Wordpress deployments so that we can control lifecycle, naming and drive volume attributes. + ## [1.0.24] (2023-2-17) - refactor MySQL and MongoDB remote backup solutions diff --git a/cookiecutter.json b/cookiecutter.json index c9b9cd28..c895050f 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -26,10 +26,9 @@ "stack_add_remote_mysql": ["Y", "N"], "stack_add_remote_mongodb": ["Y", "N"], "stack_add_remote_redis": ["Y", "N"], - "cert_manager_image_version": "v1.11.0", "ci_build_tutor_version": "15.2.0", - "ci_build_kubectl_version": "1.24/stable", - "kubernetes_cluster_version": "1.24", + "ci_build_kubectl_version": "1.25/stable", + "kubernetes_cluster_version": "1.25", "ci_build_theme_repository": "edx-theme-example", "ci_build_theme_repository_organization": "lpm0073", "ci_build_theme_ref": "main", @@ -84,15 +83,15 @@ "ci_openedx_actions_tutor_plugin_enable_notes_version": "v1.0.2", "ci_openedx_actions_tutor_plugin_enable_s3_version": "v1.0.2", "ci_openedx_actions_tutor_plugin_enable_xqueue_version": "v1.0.0", - "eks_worker_group_instance_type": "t3.xlarge", + "eks_hosting_group_instance_type": "t3.large", "eks_create_kms_key": ["Y", "N"], - "eks_worker_group_min_size": 0, - "eks_worker_group_max_size": 1, - "eks_worker_group_desired_size": 0, - "eks_karpenter_group_instance_type": "t3.large", - "eks_karpenter_group_min_size": 3, - "eks_karpenter_group_max_size": 10, - "eks_karpenter_group_desired_size": 3, + "eks_hosting_group_min_size": 0, + "eks_hosting_group_max_size": 1, + "eks_hosting_group_desired_size": 0, + "eks_service_group_instance_type": "t3.large", + "eks_service_group_min_size": 3, + "eks_service_group_max_size": 10, + "eks_service_group_desired_size": 3, "mongodb_instance_type": "t3.medium", "mongodb_allocated_storage": 10, "bastion_instance_type": "t3.micro", diff --git a/doc/README.rst b/doc/README.rst index fd556171..99491d8c 100644 --- a/doc/README.rst +++ b/doc/README.rst @@ -265,7 +265,7 @@ AWS Elastics Kubernetes Service Configuration Options **BE AWARE:** there are far-reaching and often times irreversible consequences to changing this value. DO NOT change this value unless you're certain that you understand whaat you're doing. - *default value: 1.24* + *default value: 1.25* - **eks_create_kms_key:** @@ -285,7 +285,7 @@ their rightful 'owner' and can be called back into service by the 'owner' at any for you, immediately replacing any node that is called back by its owner. This happens infrequently, with the exception of the eu-west-2 (London) AWS data center. -- **eks_karpenter_group_instance_type:** +- **eks_service_group_instance_type:** The *preferred* instance type that Karpenter will acquire on your behalf from the spot-price marketplace. Note that the Terraform scripts include several fallback options in the event that your preferred instance type is not @@ -293,7 +293,7 @@ AWS data center. *default value: t3.large* -- **eks_karpenter_group_min_size:** +- **eks_service_group_min_size:** The minimum number of EC2 instance compute nodes to maintain inside the compute plane of your cluster. This value needs to be at least 1 in order for Karpenter to gather real-time load and performance metrics that it uses @@ -303,7 +303,7 @@ AWS data center. *default value: 3* -- **eks_karpenter_group_max_size:** +- **eks_service_group_max_size:** The maximum number of EC2 instances that Karpenter is permitted to add to the Kubernetes compute plane regardless of real-time load metrics. @@ -314,7 +314,7 @@ AWS data center. *default value: 10* -- **eks_karpenter_group_desired_size:** +- **eks_service_group_desired_size:** The initial setting that Karpenter will use when the EKS cluster is created and initialized. This value will poentially change (higher or lower) as soon as metrics-server and promethus @@ -322,27 +322,27 @@ AWS data center. *default value: 3* -eks_worker_group is an optional, supplemental EC2 node worker group that is included in the +eks_hosting_group is an optional, supplemental EC2 node worker group that is included in the AWS EKS build. If you chose to install Karpenter then you can ignore these options. Nodes created in this group will use on-demand pricing, which will cost around 3x as compared to the Karpenter nodes, which use spot-pricing. However, availability of on-demand nodes is guaranteed by AWS. -- **eks_worker_group_min_size:** +- **eks_hosting_group_min_size:** The minimum allowed quanity of nodes for this group. *default value: 0* -- **eks_worker_group_max_size:** +- **eks_hosting_group_max_size:** The maximum allowed quanity of nodes for this group. *default value: 0* -- **eks_worker_group_desired_size:** +- **eks_hosting_group_desired_size:** The current run-time requested quanity of nodes for this group. *default value: 0* -- **eks_worker_group_instance_type:** +- **eks_hosting_group_instance_type:** The AWS EC2 instance type that will be created for all nodes in this group. *default value: t3.xlarge* @@ -571,7 +571,7 @@ The scaffolding that is generated by Cookiecutter provides the code samples that singular tool at your disposal for programatically administering your Kubernetes cluster. Your choice of `kubectl `_ version (and its installation method) have an equally significan impact to the reliability of your deploy workflows. - *default value: 1.24/stable* + *default value: 1.25/stable* - **ci_build_theme_repository:** diff --git a/{{cookiecutter.github_repo_name}}/doc/QUICKSTART.rst b/{{cookiecutter.github_repo_name}}/doc/QUICKSTART.rst index 5b51e7bc..093d30a7 100644 --- a/{{cookiecutter.github_repo_name}}/doc/QUICKSTART.rst +++ b/{{cookiecutter.github_repo_name}}/doc/QUICKSTART.rst @@ -93,10 +93,10 @@ Review your production environment parameters. redis_node_type = "cache.t2.small" # 2 vCPU 8gb - eks_worker_group_instance_type = "t3.large" + eks_hosting_group_instance_type = "t3.large" # 2 vCPU 8gb - eks_karpenter_group_instance_type = "t3.large" + eks_service_group_instance_type = "t3.large" } @@ -225,7 +225,7 @@ Following is an example aws-auth configMap with additional IAM user accounts add resourceVersion: "499488" uid: 52d6e7fd-01b7-4c80-b831-b971507e5228 -Note that by default, Kubernetes version 1.24 and newer encrypts all secrets data using `AWS Key Management Service (KMS) `_. +Note that by default, Kubernetes version 1.25 and newer encrypts all secrets data using `AWS Key Management Service (KMS) `_. The Cookiecutter automatically adds the IAM user for the bastion server. For any other IAM users you'll need to modify the following in terraform/stacks/modules/kubernetes/main.tf: diff --git a/{{cookiecutter.github_repo_name}}/make.sh b/{{cookiecutter.github_repo_name}}/make.sh index dd0532d9..43e4ab7d 100755 --- a/{{cookiecutter.github_repo_name}}/make.sh +++ b/{{cookiecutter.github_repo_name}}/make.sh @@ -35,14 +35,14 @@ cookiecutter --checkout $GITHUB_BRANCH \ environment_name={{ cookiecutter.environment_name }} \ environment_subdomain={{ cookiecutter.environment_subdomain }} \ eks_create_kms_key={{ cookiecutter.eks_create_kms_key }} \ - eks_worker_group_instance_type={{ cookiecutter.eks_worker_group_instance_type }} \ - eks_worker_group_min_size={{ cookiecutter.eks_worker_group_min_size }} \ - eks_worker_group_max_size={{ cookiecutter.eks_worker_group_max_size }} \ - eks_worker_group_desired_size={{ cookiecutter.eks_worker_group_desired_size }} \ - eks_karpenter_group_instance_type={{ cookiecutter.eks_karpenter_group_instance_type }} \ - eks_karpenter_group_min_size={{ cookiecutter.eks_karpenter_group_min_size }} \ - eks_karpenter_group_max_size={{ cookiecutter.eks_karpenter_group_max_size }} \ - eks_karpenter_group_desired_size={{ cookiecutter.eks_karpenter_group_desired_size }} \ + eks_hosting_group_instance_type={{ cookiecutter.eks_hosting_group_instance_type }} \ + eks_hosting_group_min_size={{ cookiecutter.eks_hosting_group_min_size }} \ + eks_hosting_group_max_size={{ cookiecutter.eks_hosting_group_max_size }} \ + eks_hosting_group_desired_size={{ cookiecutter.eks_hosting_group_desired_size }} \ + eks_service_group_instance_type={{ cookiecutter.eks_service_group_instance_type }} \ + eks_service_group_min_size={{ cookiecutter.eks_service_group_min_size }} \ + eks_service_group_max_size={{ cookiecutter.eks_service_group_max_size }} \ + eks_service_group_desired_size={{ cookiecutter.eks_service_group_desired_size }} \ mysql_instance_class={{ cookiecutter.mysql_instance_class }} \ mysql_allocated_storage={{ cookiecutter.mysql_allocated_storage }} \ redis_node_type={{ cookiecutter.redis_node_type }} \ diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/mysql/kubernetes.tf b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/mysql/kubernetes.tf index f403321d..2f369d3e 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/mysql/kubernetes.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/mysql/kubernetes.tf @@ -194,8 +194,7 @@ resource "kubernetes_secret" "credentials" { MYSQL_PORT = data.kubernetes_secret.mysql_root.data.MYSQL_PORT } } -{% endif %} -{% if cookiecutter.ci_deploy_install_license_manager_service|upper == "Y" -%} +{% endif %}{% if cookiecutter.ci_deploy_install_license_manager_service|upper == "Y" -%} resource "random_password" "mysql_license_manager" { length = 16 special = true diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/config/resource-quota.yaml.tpl b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/config/resource-quota.yaml.tpl deleted file mode 100644 index 55426a18..00000000 --- a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/config/resource-quota.yaml.tpl +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: ResourceQuota -metadata: - name: ${namespace} - namespace: ${namespace} -spec: - hard: - limits.cpu: "${resource_quota_cpu}" - limits.memory: "${resource_quota_memory}" diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/config/wordpress-values.yaml.tpl b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/config/wordpress-values.yaml.tpl index 4e20c4b0..f475d2a4 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/config/wordpress-values.yaml.tpl +++ b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/config/wordpress-values.yaml.tpl @@ -71,7 +71,7 @@ affinity: - weight: 1 preference: matchExpressions: - - key: application-group + - key: node-group operator: In values: - wordpress @@ -82,6 +82,11 @@ nodeSelector: {} ## @param tolerations Tolerations for pod assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## +tolerations: + - effect: NoSchedule + key: role + operator: Equal + value: pvc-pods ## WordPress containers' resource requests and limits ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ ## @param resources.limits The resources limits for the WordPress containers @@ -101,7 +106,7 @@ containerPorts: ingress: enabled: false persistence: - size: ${persistenceSize} + existingClaim: "${wordpressDomain}" serviceAccount: create: ${serviceAccountCreate} name: ${serviceAccountName} diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/ebs_volume.tf b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/ebs_volume.tf new file mode 100644 index 00000000..9e4bef3f --- /dev/null +++ b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/ebs_volume.tf @@ -0,0 +1,140 @@ +#------------------------------------------------------------------------------ +# written by: Lawrence McDaniel +# https://lawrencemcdaniel.com/ +# +# date: Feb-2023 +# +# usage: create a detachable EBS volume to be used as the PVC for the Wordpress pod. +# +# Problems we're trying to solve: the Bitnami Wordpress chart provides +# dynamic PVC and volume management by default, but there are shortcomings: +# 1. the EBS drive volume gets destroyed whenever we run Terraform destroy on +# a Wordpress site, which is usually **not** what we want. +# +# 2. the EBS volumes are generically named and tagged. we'd prefer to see +# identifying information that helps us understand which EBS volume belongs +# to which Wordpress site. +# +# 3. The Bitnami charts lacks granular control over the design attributes of the +# EBS volume, the PV and the PVC. We want to maintain the potential to fine +# tune these in the future. +#------------------------------------------------------------------------------ + +#------------------------------------------------------------------------------ +# KUBERNETES RESOURCES +#------------------------------------------------------------------------------ +resource "kubernetes_persistent_volume_claim" "wordpress" { + metadata { + name = local.wordpressDomain + namespace = local.wordpressNamespace + labels = { + "ebs_volume_id" = "${aws_ebs_volume.wordpress.id}" + "name" = "${local.wordpressDomain}" + "namespace" = "${local.wordpressNamespace}" + } + } + + spec { + access_modes = ["ReadWriteOnce"] + storage_class_name = "gp2" + resources { + requests = { + storage = "${local.persistenceSize / 2}Gi" + } + } + volume_name = kubernetes_persistent_volume.wordpress.metadata.0.name + } + + depends_on = [ + kubernetes_persistent_volume.wordpress + ] +} + +resource "kubernetes_persistent_volume" "wordpress" { + metadata { + name = local.wordpressDomain + labels = { + "topology.kubernetes.io/region" = "${var.aws_region}" + "topology.kubernetes.io/zone" = "${aws_ebs_volume.wordpress.availability_zone}" + "ebs_volume_id" = "${aws_ebs_volume.wordpress.id}" + "name" = "${local.wordpressDomain}" + "namespace" = "${local.wordpressNamespace}" + } + annotations = { + } + } + + spec { + capacity = { + storage = "${local.persistenceSize}Gi" + } + access_modes = ["ReadWriteOnce"] + storage_class_name = "gp2" + persistent_volume_source { + aws_elastic_block_store { + volume_id = aws_ebs_volume.wordpress.id + fs_type = "ext4" + } + } + node_affinity { + required { + node_selector_term { + match_expressions { + key = "topology.kubernetes.io/zone" + operator = "In" + values = ["${aws_ebs_volume.wordpress.availability_zone}"] + } + match_expressions { + key = "topology.kubernetes.io/region" + operator = "In" + values = ["${var.aws_region}"] + } + } + } + } + } + + depends_on = [ + aws_ebs_volume.wordpress + ] +} + +# create a detachable EBS volume for the wordpress databases +#------------------------------------------------------------------------------ +# AWS ELASTIC BLOCK STORE RESOURCES +#------------------------------------------------------------------------------ +resource "aws_ebs_volume" "wordpress" { + availability_zone = data.aws_subnet.private_subnet.availability_zone + size = local.persistenceSize + tags = var.tags + + # local.ebsVolumePreventDestroy defaults to 'Y' + # for anything other than an upper case 'N' we'll assume that + # we should not destroy this resource. + lifecycle { + prevent_destroy = false + } + + depends_on = [ + data.aws_subnet.private_subnet + ] +} + + +#------------------------------------------------------------------------------ +# SUPPORTING RESOURCES +#------------------------------------------------------------------------------ + +data "aws_subnet" "private_subnet" { + id = var.subnet_ids[random_integer.subnet_id.result] +} + +# randomize the choice of subnet. Each of the +# possible subnets corresponds to the AWS availability +# zones in the data center. Most data center have three +# availability zones, but some like us-east-1 have more than +# three. +resource "random_integer" "subnet_id" { + min = 0 + max = length(var.subnet_ids) - 1 +} diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/kubernetes-resource-quota.tf b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/kubernetes-resource-quota.tf deleted file mode 100644 index d1fd8f04..00000000 --- a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/kubernetes-resource-quota.tf +++ /dev/null @@ -1,29 +0,0 @@ -#------------------------------------------------------------------------------ -# written by: Lawrence McDaniel -# https://lawrencemcdaniel.com -# -# date: Feb-2023 -# -# usage: Wordpress Kubernetes resource quota -#------------------------------------------------------------------------------ - -data "template_file" "resource_quota" { - template = file("${path.module}/config/resource-quota.yaml.tpl") - vars = { - namespace = local.wordpressNamespace - resource_quota_cpu = var.resource_quota_cpu - resource_quota_memory = var.resource_quota_memory - } -} - -resource "kubectl_manifest" "resource_quota" { - count = var.resource_quota == "Y" ? 1 : 0 - yaml_body = data.template_file.resource_quota.rendered - - depends_on = [ - kubernetes_secret.wordpress_config, - ssh_sensitive_resource.mysql, - kubernetes_namespace.wordpress, - helm_release.wordpress - ] -} diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/main.tf b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/main.tf index ec2d6907..f7e916e3 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/main.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/main.tf @@ -17,6 +17,11 @@ # helm show all bitnami/wordpress # helm show values bitnami/wordpress # +# Trouble shooting an installation: +# helm ls --namespace my-wordpress-site +# helm history wordpress --namespace my-wordpress-site +# helm rollback wordpress 4 --namespace my-wordpress-site +# # see: https://jmrobles.medium.com/launch-a-wordpress-site-on-kubernetes-in-just-1-minute-193914cb4902 #----------------------------------------------------------- locals { @@ -34,6 +39,8 @@ locals { externalDatabaseUser = var.wordpressConfig["DatabaseUser"] externalDatabaseDatabase = var.wordpressConfig["Database"] persistenceSize = var.wordpressConfig["DiskVolumeSize"] + ebsVolumePreventDestroy = var.wordpressConfig["DiskVolumePreventDestroy"] + aws_ebs_volume_id = var.wordpressConfig["AWSEBSVolumeId"] serviceAccountName = local.wordpressDomain HorizontalAutoscalingMinReplicas = 1 HorizontalAutoscalingMaxReplicas = 1 @@ -77,7 +84,7 @@ data "template_file" "wordpress-values" { allowEmptyPassword = false extraVolumes = data.template_file.extraVolumes.rendered extraVolumeMounts = data.template_file.extraVolumeMounts.rendered - persistenceSize = local.persistenceSize + pvcEbs_volume_id = local.aws_ebs_volume_id != "" ? local.aws_ebs_volume_id : aws_ebs_volume.wordpress.id serviceAccountCreate = true serviceAccountName = local.serviceAccountName serviceAccountAnnotations = data.template_file.serviceAccountAnnotations.rendered @@ -119,3 +126,26 @@ resource "helm_release" "wordpress" { ssh_sensitive_resource.mysql ] } + +resource "null_resource" "wordpress_post_deployment" { + + provisioner "local-exec" { + command = <<-EOT + # 1. Switch to namespace for this Wordpress deployment. + # Find the name of the Wordpress pod using one of the Helm-generated labels + # --------------------------------------- + kubectl config set-context --current --namespace=lawrencemcdaniel-api + POD=$(kubectl get pod -l app.kubernetes.io/name=wordpress -o jsonpath="{.items[0].metadata.name}") + + # 2. shell into the wordpress container of the deployed pod + # and execute the post deployment ops + # --------------------------------------- + echo "running post deployments scripts on ${POD}" + kubectl exec -it $POD --container=wordpress -- /bin/bash -c "touch /opt/bitnami/wordpress/wordfence-waf.php" + EOT + } + + depends_on = [ + helm_release.wordfence + ] +} diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/outputs.tf b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/outputs.tf index 7caf35d5..10831d0c 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/outputs.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/outputs.tf @@ -111,3 +111,54 @@ output "kubernetes_resource_quota_memory" { description = "The namespace resource limit for memory" value = var.resource_quota == "Y" ? var.resource_quota_memory : "" } + +# EBS Volume outputs +output "wordpress_volume_id" { + description = "The id of the EC2 EBS volume" + value = aws_ebs_volume.wordpress.id +} + +output "wordpress_volume_arn" { + description = "The arn of the EC2 EBS volume" + value = aws_ebs_volume.wordpress.arn +} + +output "wordpress_volume_availability_zone" { + description = "The availability zone in which the EC2 EBS volume was created" + value = aws_ebs_volume.wordpress.availability_zone +} + +output "wordpress_volume_subnet_id" { + description = "The availability zone index in which the EC2 EBS volume was created" + value = data.aws_subnet.private_subnet.id +} + +output "wordpress_volume_size" { + description = "The allocated size of the EC2 EBS volume" + value = aws_ebs_volume.wordpress.size +} + +output "wordpress_volume_tags" { + description = "The resource tags of the EC2 EBS volume" + value = aws_ebs_volume.wordpress.tags +} + +output "wordpress_volume_multi_attach_enabled" { + description = "Boolean. whether wordpress_volume_multi_attach_enabled is enabled for the EC2 EBS volume" + value = aws_ebs_volume.wordpress.multi_attach_enabled +} + +output "wordpress_volume_iops" { + description = "The iops setting of the EC2 EBS volume" + value = aws_ebs_volume.wordpress.iops +} + +output "wordpress_volume_throughput" { + description = "The throughput setting of the EC2 EBS volume" + value = aws_ebs_volume.wordpress.throughput +} + +output "wordpress_volume_type" { + description = "The type of the EC2 EBS volume" + value = aws_ebs_volume.wordpress.type +} diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/variables.tf b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/variables.tf index 8f08dd77..f9d58ed6 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/variables.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/wordpress/variables.tf @@ -45,3 +45,7 @@ variable "resource_quota_memory" { type = string default = "1Gi" } + +variable "subnet_ids" { + type = list(string) +} diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/wordpress/{{ cookiecutter.wordpress_namespace }}/client.hcl b/{{cookiecutter.github_repo_name}}/terraform/environments/wordpress/{{ cookiecutter.wordpress_namespace }}/client.hcl index 6b31717c..c214fe3e 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/environments/wordpress/{{ cookiecutter.wordpress_namespace }}/client.hcl +++ b/{{cookiecutter.github_repo_name}}/terraform/environments/wordpress/{{ cookiecutter.wordpress_namespace }}/client.hcl @@ -7,17 +7,19 @@ # usage: create client parameters for a single Wordpress site. #------------------------------------------------------------------------------ locals { - wp_hosted_zone_id = "{{ cookiecutter.wordpress_hosted_zone_id }}" - wp_domain = "{{ cookiecutter.wordpress_domain }}" - wp_subdomain = "{{ cookiecutter.wordpress_subdomain }}" - wp_namespace = "{{ cookiecutter.wordpress_namespace }}" - wp_username = "{{ cookiecutter.wordpress_username }}" - wp_email = "{{ cookiecutter.wordpress_email }}" - wp_user_firstname = "{{ cookiecutter.wordpress_user_firstname }}" - wp_user_lastname = "{{ cookiecutter.wordpress_user_lastname }}" - wp_blog_name = "{{ cookiecutter.wordpress_blog_name }}" - wp_database_user = "{{ cookiecutter.wordpress_database_user }}" - wp_database = "{{ cookiecutter.wordpress_database }}" - wp_disk_volume_size = "{{ cookiecutter.wordpress_disk_volume_size }}" - phpmyadmin = "Y" + wp_hosted_zone_id = "{{ cookiecutter.wordpress_hosted_zone_id }}" + wp_domain = "{{ cookiecutter.wordpress_domain }}" + wp_subdomain = "{{ cookiecutter.wordpress_subdomain }}" + wp_namespace = "{{ cookiecutter.wordpress_namespace }}" + wp_username = "{{ cookiecutter.wordpress_username }}" + wp_email = "{{ cookiecutter.wordpress_email }}" + wp_user_firstname = "{{ cookiecutter.wordpress_user_firstname }}" + wp_user_lastname = "{{ cookiecutter.wordpress_user_lastname }}" + wp_blog_name = "{{ cookiecutter.wordpress_blog_name }}" + wp_database_user = "{{ cookiecutter.wordpress_database_user }}" + wp_database = "{{ cookiecutter.wordpress_database }}" + wp_disk_volume_size = "{{ cookiecutter.wordpress_disk_volume_size }}" + phpmyadmin = "N" + aws_ebs_volume_id = "" + aws_ebs_volume_prevent_destroy = "Y" } diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/wordpress/{{ cookiecutter.wordpress_namespace }}/terragrunt.hcl b/{{cookiecutter.github_repo_name}}/terraform/environments/wordpress/{{ cookiecutter.wordpress_namespace }}/terragrunt.hcl index 2392772f..9fa254b1 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/environments/wordpress/{{ cookiecutter.wordpress_namespace }}/terragrunt.hcl +++ b/{{cookiecutter.github_repo_name}}/terraform/environments/wordpress/{{ cookiecutter.wordpress_namespace }}/terragrunt.hcl @@ -30,6 +30,8 @@ locals { DatabaseUser = local.client_vars.locals.wp_database_user, Database = local.client_vars.locals.wp_database, DiskVolumeSize = local.client_vars.locals.wp_disk_volume_size + DiskVolumePreventDestroy = local.client_vars.locals.aws_ebs_volume_prevent_destroy + AWSEBSVolumeId = local.client_vars.locals.aws_ebs_volume_id } tags = merge( @@ -121,5 +123,6 @@ inputs = { aws_region = local.aws_region wordpressConfig = local.wordpressConfig phpmyadmin = local.phpmyadmin + subnet_ids = dependency.vpc.outputs.private_subnets tags = local.tags } diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/main.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/main.tf index e3e26bac..fb0cccbe 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/main.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/main.tf @@ -19,7 +19,7 @@ locals { } resource "aws_security_group" "worker_group_mgmt" { - name_prefix = "${var.namespace}-eks_worker_group_mgmt" + name_prefix = "${var.namespace}-eks_hosting_group_mgmt" description = "openedx_devops: Ingress CLB worker group management" vpc_id = var.vpc_id @@ -88,7 +88,7 @@ module "eks" { # add more IAM users to the KMS key owners list kms_key_owners = [ - "arn:aws:iam::${var.account_id}:user/system/bastion-user/${var.namespace}-bastion" + "${var.bastion_iam_arn}" ] tags = merge( @@ -101,13 +101,13 @@ module "eks" { cluster_addons = { vpc-cni = { - addon_version = "v1.12.2-eksbuild.1" + addon_version = "v1.12.5-eksbuild.1" } coredns = { addon_version = "v1.9.3-eksbuild.2" } kube-proxy = { - addon_version = "v1.24.9-eksbuild.1" + addon_version = "v1.25.6-eksbuild.1" } aws-ebs-csi-driver = { service_account_role_arn = aws_iam_role.AmazonEKS_EBS_CSI_DriverRole.arn @@ -156,15 +156,15 @@ module "eks" { # (a few hours or less) as these are usually only instantiated during # bursts of user activity such as at the start of a scheduled lecture or # exam on a large mooc. - karpenter = { + {{ cookiecutter.global_platform_shared_resource_identifier }} = { capacity_type = "SPOT" enable_monitoring = false - desired_size = var.eks_karpenter_group_desired_size - min_size = var.eks_karpenter_group_min_size - max_size = var.eks_karpenter_group_max_size + desired_size = var.eks_service_group_desired_size + min_size = var.eks_service_group_min_size + max_size = var.eks_service_group_max_size labels = { - application-group = "service" + node-group = "{{ cookiecutter.global_platform_shared_resource_identifier }}" } iam_role_additional_policies = { @@ -175,15 +175,46 @@ module "eks" { AmazonEBSCSIDriverPolicy = data.aws_iam_policy.AmazonEBSCSIDriverPolicy.arn } - instance_types = ["${var.eks_karpenter_group_instance_type}"] + instance_types = ["${var.eks_service_group_instance_type}"] tags = merge( var.tags, - { Name = "eks-${var.shared_resource_identifier}-karpenter" } + { Name = "eks-${var.shared_resource_identifier}" } + ) + } + + hosting = { + capacity_type = "SPOT" + enable_monitoring = false + desired_size = var.eks_hosting_group_desired_size + min_size = var.eks_hosting_group_min_size + max_size = var.eks_hosting_group_max_size + + labels = { + node-group = "{{ cookiecutter.global_platform_shared_resource_identifier }}" + } + + iam_role_additional_policies = { + # Required by Karpenter + AmazonSSMManagedInstanceCore = "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + + # Required by EBS CSI Add-on + AmazonEBSCSIDriverPolicy = data.aws_iam_policy.AmazonEBSCSIDriverPolicy.arn + } + + instance_types = ["${var.eks_hosting_group_instance_type}"] + tags = merge( + var.tags, + { Name = "eks-${var.shared_resource_identifier}-hosting" } ) } } + # add the bastion IAM user to aws-auth.mapUsers so that + # kubectl and k9s work from inside the bastion server by default. + manage_aws_auth_configmap = true + aws_auth_users = var.map_users + } resource "kubernetes_namespace" "namespace-shared" { diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/outputs.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/outputs.tf index bb6a21aa..729ad68d 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/outputs.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/outputs.tf @@ -154,14 +154,21 @@ output "fargate_profiles" { } ################################################################################ -# EKS Managed Node Group +# EKS Managed Node Groups ################################################################################ -output "karpenter_node_group_iam_role_name" { - value = module.eks.eks_managed_node_groups["karpenter"].iam_role_name +output "service_node_group_iam_role_name" { + value = module.eks.eks_managed_node_groups["service"].iam_role_name } -output "karpenter_node_group_iam_role_arn" { - value = module.eks.eks_managed_node_groups["karpenter"].iam_role_arn +output "service_node_group_iam_role_arn" { + value = module.eks.eks_managed_node_groups["service"].iam_role_arn +} +output "hosting_node_group_iam_role_name" { + value = module.eks.eks_managed_node_groups["hosting"].iam_role_name +} + +output "hosting_node_group_iam_role_arn" { + value = module.eks.eks_managed_node_groups["hosting"].iam_role_arn } output "eks_managed_node_groups" { description = "Map of attribute maps for all EKS managed node groups created" diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/variables.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/variables.tf index 509786cc..d39051d2 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/variables.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes/variables.tf @@ -53,35 +53,35 @@ variable "eks_create_kms_key" { type = bool } -variable "eks_worker_group_min_size" { +variable "eks_hosting_group_min_size" { type = number } -variable "eks_worker_group_max_size" { +variable "eks_hosting_group_max_size" { type = number } -variable "eks_karpenter_group_instance_type" { +variable "eks_service_group_instance_type" { type = string } -variable "eks_karpenter_group_min_size" { +variable "eks_service_group_min_size" { type = number } -variable "eks_karpenter_group_max_size" { +variable "eks_service_group_max_size" { type = number } -variable "eks_karpenter_group_desired_size" { +variable "eks_service_group_desired_size" { type = number } -variable "eks_worker_group_desired_size" { +variable "eks_hosting_group_desired_size" { type = number } -variable "eks_worker_group_instance_type" { +variable "eks_hosting_group_instance_type" { type = string } @@ -104,3 +104,7 @@ variable "map_roles" { })) default = [] } + +variable "bastion_iam_arn" { + type = string +} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/certificate.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/certificate.tf new file mode 100644 index 00000000..21f98176 --- /dev/null +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/certificate.tf @@ -0,0 +1,21 @@ +data "aws_route53_zone" "services_subdomain" { + name = var.services_subdomain +} + +data "template_file" "certificate" { + template = file("${path.module}/manifests/certificate.yml.tpl") + vars = { + services_subdomain = var.services_subdomain + namespace = var.namespace + } +} + +resource "kubectl_manifest" "certificate" { + yaml_body = data.template_file.certificate.rendered + + depends_on = [ + module.cert_manager_irsa, + helm_release.cert-manager, + aws_iam_policy.cert_manager_policy, + ] +} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/cluster-issuer.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/cluster-issuer.tf new file mode 100644 index 00000000..0c829988 --- /dev/null +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/cluster-issuer.tf @@ -0,0 +1,22 @@ +data "template_file" "cluster-issuer" { + template = file("${path.module}/manifests/cluster-issuer.yml.tpl") + vars = { + root_domain = var.root_domain + services_subdomain = var.services_subdomain + namespace = var.namespace + aws_region = var.aws_region + hosted_zone_id = data.aws_route53_zone.services_subdomain.id + } +} + + + +resource "kubectl_manifest" "cluster-issuer" { + yaml_body = data.template_file.cluster-issuer.rendered + + depends_on = [ + module.cert_manager_irsa, + helm_release.cert-manager, + aws_iam_policy.cert_manager_policy, + ] +} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/main.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/main.tf index acfc0e7c..6c756223 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/main.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/main.tf @@ -7,14 +7,39 @@ # usage: Add tls certs for EKS cluster load balancer # see https://cert-manager.io/docs/ # -# prequisites: -# helm repo add jetstack https://charts.jetstack.io -# helm repo update +# helm reference: +# brew install helm +# +# helm repo add jetstack https://charts.jetstack.io +# helm repo update +# helm show all jetstack/cert-manager +# helm show values jetstack/cert-manager #------------------------------------------------------------------------------ -data "aws_route53_zone" "services_subdomain" { - name = var.services_subdomain + +data "template_file" "cert-manager-values" { + template = file("${path.module}/manifests/cert-manager-values.yaml.tpl") + vars = { + role_arn = module.cert_manager_irsa.iam_role_arn + namespace = var.cert_manager_namespace + } } +resource "helm_release" "cert-manager" { + name = "cert-manager" + namespace = var.cert_manager_namespace + create_namespace = true + + chart = "cert-manager" + repository = "jetstack" + version = "{{ cookiecutter.terraform_helm_cert_manager }}" + values = [ + data.template_file.cert-manager-values.rendered + ] +} + +#------------------------------------------------------------------------------ +# SUPPORTING RESOURCES +#------------------------------------------------------------------------------ resource "aws_iam_policy" "cert_manager_policy" { name = "${var.namespace}-cert-manager-policy" path = "/" @@ -55,76 +80,3 @@ module "cert_manager_irsa" { role_policy_arns = [aws_iam_policy.cert_manager_policy.arn] oidc_fully_qualified_subjects = ["system:serviceaccount:${var.cert_manager_namespace}:cert-manager"] } - -data "template_file" "cert-manager-values" { - template = file("${path.module}/manifests/cert-manager-values.yaml.tpl") - vars = { - role_arn = module.cert_manager_irsa.iam_role_arn - namespace = var.cert_manager_namespace - cert_manager_image_version = var.cert_manager_image_version - } -} - -#----------------------------------------------------------- -# NOTE: you must initialize a local helm repo in order to run -# this script. -# -# brew install helm -# -# kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.1/cert-manager.crds.yaml -# helm repo add jetstack https://charts.jetstack.io -# helm repo update -#----------------------------------------------------------- -resource "helm_release" "cert-manager" { - name = "cert-manager" - namespace = var.cert_manager_namespace - create_namespace = true - - chart = "cert-manager" - repository = "jetstack" - version = "{{ cookiecutter.terraform_helm_cert_manager }}" - values = [ - data.template_file.cert-manager-values.rendered - ] -} - -data "template_file" "certificate" { - template = file("${path.module}/manifests/certificate.yml.tpl") - vars = { - services_subdomain = var.services_subdomain - namespace = var.namespace - } -} - -resource "kubectl_manifest" "certificate" { - yaml_body = data.template_file.certificate.rendered - - depends_on = [ - module.cert_manager_irsa, - helm_release.cert-manager, - aws_iam_policy.cert_manager_policy, - ] -} - -data "template_file" "cluster-issuer" { - template = file("${path.module}/manifests/cluster-issuer.yml.tpl") - vars = { - root_domain = var.root_domain - services_subdomain = var.services_subdomain - namespace = var.namespace - aws_region = var.aws_region - hosted_zone_id = data.aws_route53_zone.services_subdomain.id - } -} - - - -resource "kubectl_manifest" "cluster-issuer" { - yaml_body = data.template_file.cluster-issuer.rendered - - depends_on = [ - module.cert_manager_irsa, - helm_release.cert-manager, - aws_iam_policy.cert_manager_policy, - ] -} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/manifests/cert-manager-values.yaml.tpl b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/manifests/cert-manager-values.yaml.tpl index 66b6d2e0..2bd6372a 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/manifests/cert-manager-values.yaml.tpl +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/manifests/cert-manager-values.yaml.tpl @@ -1,17 +1,13 @@ -global: - podSecurityPolicy: - enabled: true - useAppArmor: false -image: - tag: ${cert_manager_image_version} -webhook: - image: - tag: ${cert_manager_image_version} -prometheus: - enabled: false -installCRDs: true -extraArgs: - - --issuer-ambient-credentials +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-group + operator: In + values: + - {{ cookiecutter.global_platform_shared_resource_identifier }} serviceAccount: annotations: eks.amazonaws.com/role-arn: ${role_arn} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/variables.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/variables.tf index d296eb05..d9536126 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/variables.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_cert_manager/variables.tf @@ -25,7 +25,3 @@ variable "aws_region" { variable "services_subdomain" { type = string } - -variable "cert_manager_image_version" { - type = string -} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_dashboard/yml/values.yaml b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_dashboard/yml/values.yaml index 05c47df3..94776632 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_dashboard/yml/values.yaml +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_dashboard/yml/values.yaml @@ -1,130 +1,15 @@ -# Copyright 2020 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for kubernetes-dashboard -# This is a YAML-formatted file. -# Declare name/value pairs to be passed into your templates. -# name: value - -image: - ## Repository for container - repository: kubernetesui/dashboard - tag: v2.7.0 - pullPolicy: IfNotPresent - pullSecrets: [] - -## Number of replicas -replicaCount: 1 - -## @param commonLabels Labels to add to all deployed objects -## -commonLabels: {} -## @param commonAnnotations Annotations to add to all deployed objects -## -commonAnnotations: {} - -## Here annotations can be added to the kubernetes dashboard deployment -annotations: {} -## Here labels can be added to the kubernetes dashboard deployment -labels: {} - -## Additional container arguments -## -# extraArgs: -# - --enable-skip-login -# - --enable-insecure-login -# - --system-banner="Welcome to Kubernetes" - -## Additional container environment variables -## -extraEnv: [] -# - name: SOME_VAR -# value: 'some value' - -## Additional volumes to be added to kubernetes dashboard pods -## -extraVolumes: [] -# - name: dashboard-kubeconfig -# secret: -# defaultMode: 420 -# secretName: dashboard-kubeconfig - -## Additional volumeMounts to be added to kubernetes dashboard container -## -extraVolumeMounts: [] -# - mountPath: /kubeconfig -# name: dashboard-kubeconfig -# readOnly: true - -## Array of extra K8s manifests to deploy -## -extraManifests: [] -# - apiVersion: v1 -# kind: ConfigMap -# metadata: -# name: additional-configmap -# data: -# mykey: myvalue - -## Annotations to be added to kubernetes dashboard pods -# podAnnotations: - -## SecurityContext to be added to kubernetes dashboard pods -## To disable set the following configuration to null: -# securityContext: null -securityContext: - seccompProfile: - type: RuntimeDefault - -## SecurityContext defaults for the kubernetes dashboard container and metrics scraper container -## To disable set the following configuration to null: -# containerSecurityContext: null -containerSecurityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsUser: 1001 - runAsGroup: 2001 - -## @param podLabels Extra labels for OAuth2 Proxy pods -## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ -## -podLabels: {} -## @param podAnnotations Annotations for OAuth2 Proxy pods -## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ -## -podAnnotations: {} - -## Node labels for pod assignment -## Ref: https://kubernetes.io/docs/user-guide/node-selection/ -## -nodeSelector: {} - -## List of node taints to tolerate (requires Kubernetes >= 1.6) -tolerations: [] -# - key: "key" -# operator: "Equal|Exists" -# value: "value" -# effect: "NoSchedule|PreferNoSchedule|NoExecute" - ## Affinity for pod assignment ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity -affinity: {} - -## Name of Priority Class of pods -# priorityClassName: "" - -## Pod resource requests & limits +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-group + operator: In + values: + - {{ cookiecutter.global_platform_shared_resource_identifier }} resources: requests: cpu: 100m @@ -132,226 +17,3 @@ resources: limits: cpu: 2 memory: 200Mi - -## Serve application over HTTP without TLS -## -## Note: If set to true, you may want to add --enable-insecure-login to extraArgs -protocolHttp: false - -service: - type: ClusterIP - # Dashboard service port - externalPort: 443 - - ## LoadBalancerSourcesRange is a list of allowed CIDR values, which are combined with ServicePort to - ## set allowed inbound rules on the security group assigned to the master load balancer - # loadBalancerSourceRanges: [] - - # clusterIP: "" - - ## A user-specified IP address for load balancer to use as External IP (if supported) - # loadBalancerIP: - - ## Additional Kubernetes Dashboard Service annotations - annotations: {} - - ## Here labels can be added to the Kubernetes Dashboard service - labels: {} - - ## Enable or disable the kubernetes.io/cluster-service label. Should be disabled for GKE clusters >=1.15. - ## Otherwise, the addon manager will presume ownership of the service and try to delete it. - clusterServiceLabel: - enabled: true - key: "kubernetes.io/cluster-service" - -ingress: - ## If true, Kubernetes Dashboard Ingress will be created. - ## - enabled: false - - ## Kubernetes Dashboard Ingress labels - # labels: - # key: value - - ## Kubernetes Dashboard Ingress annotations - # annotations: - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: 'true' - - ## If you plan to use TLS backend with enableInsecureLogin set to false - ## (default), you need to uncomment the below. - ## If you use ingress-nginx < 0.21.0 - # nginx.ingress.kubernetes.io/secure-backends: "true" - ## if you use ingress-nginx >= 0.21.0 - # nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - - ## Kubernetes Dashboard Ingress Class - # className: "example-lb" - - ## Kubernetes Dashboard Ingress paths - ## Both `/` and `/*` are required to work on gce ingress. - paths: - - / - # - /* - - ## Custom Kubernetes Dashboard Ingress paths. Will override default paths. - ## - customPaths: [] - # - pathType: ImplementationSpecific - # backend: - # service: - # name: ssl-redirect - # port: - # name: use-annotation - # - pathType: ImplementationSpecific - # backend: - # service: - # name: >- - # {% raw %}{{ include "kubernetes-dashboard.fullname" . }}{% endraw %} - # port: - # # Don't use string here, use only integer value! - # number: 443 - ## Kubernetes Dashboard Ingress hostnames - ## Must be provided if Ingress is enabled - ## - # hosts: - # - kubernetes-dashboard.domain.com - ## Kubernetes Dashboard Ingress TLS configuration - ## Secrets must be manually created in the namespace - ## - # tls: - # - secretName: kubernetes-dashboard-tls - # hosts: - # - kubernetes-dashboard.domain.com - -# Global dashboard settings -settings: - {} - ## Cluster name that appears in the browser window title if it is set - # clusterName: "" - ## Max number of items that can be displayed on each list page - # itemsPerPage: 10 - ## Number of seconds between every auto-refresh of logs - # logsAutoRefreshTimeInterval: 5 - ## Number of seconds between every auto-refresh of every resource. Set 0 to disable - # resourceAutoRefreshTimeInterval: 5 - ## Hide all access denied warnings in the notification panel - # disableAccessDeniedNotifications: false - -## Pinned CRDs that will be displayed in dashboard's menu -pinnedCRDs: - [] - # - kind: customresourcedefinition - ## Fully qualified name of a CRD - # name: prometheuses.monitoring.coreos.com - ## Display name - # displayName: Prometheus - ## Is this CRD namespaced? - # namespaced: true - -## Metrics Scraper -## Container to scrape, store, and retrieve a window of time from the Metrics Server. -## refs: https://github.com/kubernetes-sigs/dashboard-metrics-scraper -metricsScraper: - ## Wether to enable dashboard-metrics-scraper - enabled: false - image: - repository: kubernetesui/metrics-scraper - tag: v1.0.8 - resources: {} - ## SecurityContext especially for the kubernetes dashboard metrics scraper container - ## If not set, the global containterSecurityContext values will define these values - # containerSecurityContext: - # allowPrivilegeEscalation: false - # readOnlyRootFilesystem: true - # runAsUser: 1001 - # runAsGroup: 2001 -# args: -# - --log-level=info -# - --logtostderr=true - -## Optional Metrics Server sub-chart -## Enable this if you don't already have metrics-server enabled on your cluster and -## want to use it with dashboard metrics-scraper -## refs: -## - https://github.com/kubernetes-sigs/metrics-server -## - https://github.com/kubernetes-sigs/metrics-server/tree/master/charts/metrics-server -metrics-server: - enabled: false - ## Example for additional args - # args: - # - --kubelet-preferred-address-types=InternalIP - # - --kubelet-insecure-tls - -rbac: - # Specifies whether namespaced RBAC resources (Role, Rolebinding) should be created - create: true - - # Specifies whether cluster-wide RBAC resources (ClusterRole, ClusterRolebinding) to access metrics should be created - # Independent from rbac.create parameter. - clusterRoleMetrics: true - - # Start in ReadOnly mode. - # Specifies whether cluster-wide RBAC resources (ClusterRole, ClusterRolebinding) with read only permissions to all resources listed inside the cluster should be created - # Only dashboard-related Secrets and ConfigMaps will still be available for writing. - # - # The basic idea of the clusterReadOnlyRole - # is not to hide all the secrets and sensitive data but more - # to avoid accidental changes in the cluster outside the standard CI/CD. - # - # It is NOT RECOMMENDED to use this version in production. - # Instead you should review the role and remove all potentially sensitive parts such as - # access to persistentvolumes, pods/log etc. - # - # Independent from rbac.create parameter. - clusterReadOnlyRole: false - # It is possible to add additional rules if read only role is enabled. - # This can be useful, for example, to show CRD resources. - # clusterReadOnlyRoleAdditionalRules: [] - - # If the default role permissions are not enough, it is possible to add additional permissions. - # roleAdditionalRules: [] - -serviceAccount: - # Specifies whether a service account should be created - create: true - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: - -livenessProbe: - # Number of seconds to wait before sending first probe - initialDelaySeconds: 30 - # Number of seconds to wait for probe response - timeoutSeconds: 30 - -## podDisruptionBudget -## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ -podDisruptionBudget: - enabled: false - ## Minimum available instances; ignored if there is no PodDisruptionBudget - minAvailable: - ## Maximum unavailable instances; ignored if there is no PodDisruptionBudget - maxUnavailable: - -## PodSecurityContext for pod level securityContext -# securityContext: -# runAsUser: 1001 -# runAsGroup: 2001 - -networkPolicy: - # Whether to create a network policy that allows/restricts access to the service - enabled: false - -## podSecurityPolicy for fine-grained authorization of pod creation and updates -podSecurityPolicy: - # Specifies whether a pod security policy should be created - enabled: false - -serviceMonitor: - # Whether or not to create a Prometheus Operator service monitor. - enabled: false - ## Here labels can be added to the serviceMonitor - labels: {} - ## Here annotations can be added to the serviceMonitor - annotations: {} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_ingress_clb/main.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_ingress_clb/main.tf index 5c05587d..59d94d57 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_ingress_clb/main.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_ingress_clb/main.tf @@ -10,8 +10,21 @@ # # see: https://github.com/kubernetes/ingress-nginx/issues/5593 # https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class +# +# helm reference: +# brew install helm +# +# helm repo add ingress-nginx https://github.com/kubernetes/ingress-nginx +# helm repo update +# helm show all ingress-nginx/ingress-nginx +# helm show values ingress-nginx/ingress-nginx + #------------------------------------------------------------------------------ +data "template_file" "nginx-values" { + template = file("${path.module}/yml/nginx-values.yaml") +} + resource "helm_release" "ingress_nginx_controller" { name = "common" namespace = var.namespace @@ -21,6 +34,10 @@ resource "helm_release" "ingress_nginx_controller" { repository = "https://kubernetes.github.io/ingress-nginx" version = "{{ cookiecutter.terraform_helm_ingress_nginx_controller }}" + values = [ + data.template_file.nginx-values.rendered + ] + set { name = "service.type" value = "ClusterIP" diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_ingress_clb/yml/nginx-values.yaml b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_ingress_clb/yml/nginx-values.yaml new file mode 100644 index 00000000..727f21be --- /dev/null +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_ingress_clb/yml/nginx-values.yaml @@ -0,0 +1,10 @@ +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-group + operator: In + values: + - {{ cookiecutter.global_platform_shared_resource_identifier }} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_karpenter/main.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_karpenter/main.tf index a0ffb859..4a084fc7 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_karpenter/main.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_karpenter/main.tf @@ -26,30 +26,8 @@ # # see: https://registry.terraform.io/modules/terraform-aws-modules/iam/aws/latest/submodules/iam-role-for-service-accounts-eks -module "karpenter_controller_irsa_role" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - # mcdaniel aug-2022: specifying an explicit version causes this module to create - # an incomplete IAM policy. - #version = "~> 5.3" - - role_name = "karpenter-controller-${var.stack_namespace}" - create_role = true - attach_karpenter_controller_policy = true - - karpenter_controller_cluster_id = data.aws_eks_cluster.eks.name - karpenter_controller_node_iam_role_arns = [ - var.karpenter_node_group_iam_role_arn - ] - - oidc_providers = { - ex = { - provider_arn = var.oidc_provider_arn - namespace_service_accounts = ["karpenter:karpenter"] - } - } - - tags = var.tags - +data "template_file" "karpenter-values" { + template = file("${path.module}/yml/karpenter-values.yaml") } @@ -63,6 +41,10 @@ resource "helm_release" "karpenter" { version = "{{ cookiecutter.terraform_helm_karpenter }}" + values = [ + data.template_file.karpenter-values.rendered + ] + set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" value = module.karpenter_controller_irsa_role.iam_role_arn @@ -85,6 +67,36 @@ resource "helm_release" "karpenter" { } +#------------------------------------------------------------------------------ +# SUPPORTING RESOURCES +#------------------------------------------------------------------------------ +module "karpenter_controller_irsa_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + # mcdaniel aug-2022: specifying an explicit version causes this module to create + # an incomplete IAM policy. + #version = "~> 5.3" + + role_name = "karpenter-controller-${var.stack_namespace}" + create_role = true + attach_karpenter_controller_policy = true + + karpenter_controller_cluster_id = data.aws_eks_cluster.eks.name + karpenter_controller_node_iam_role_arns = [ + var.karpenter_node_group_iam_role_arn + ] + + oidc_providers = { + ex = { + provider_arn = var.oidc_provider_arn + namespace_service_accounts = ["karpenter:karpenter"] + } + } + + tags = var.tags + +} + + resource "random_pet" "this" { length = 2 } diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_karpenter/yml/karpenter-values.yaml b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_karpenter/yml/karpenter-values.yaml new file mode 100644 index 00000000..690d85c8 --- /dev/null +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_karpenter/yml/karpenter-values.yaml @@ -0,0 +1,15 @@ +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: karpenter.sh/provisioner-name + operator: DoesNotExist + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-group + operator: In + values: + - {{ cookiecutter.global_platform_shared_resource_identifier }} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubeapps/main.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubeapps/main.tf index d2ff50ed..e4a02b66 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubeapps/main.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubeapps/main.tf @@ -29,6 +29,10 @@ locals { } +data "template_file" "kubeapps-values" { + template = file("${path.module}/yml/kubeapps-values.yaml") +} + resource "helm_release" "kubeapps" { namespace = local.kubeapps_namespace create_namespace = false diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubeapps/yml/kubeapps-values.yaml b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubeapps/yml/kubeapps-values.yaml new file mode 100644 index 00000000..727f21be --- /dev/null +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubeapps/yml/kubeapps-values.yaml @@ -0,0 +1,10 @@ +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-group + operator: In + values: + - {{ cookiecutter.global_platform_shared_resource_identifier }} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/config/kubecost-values.yaml b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/config/kubecost-values.yaml index df7b7204..1251938a 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/config/kubecost-values.yaml +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/config/kubecost-values.yaml @@ -1,960 +1,31 @@ -global: - # zone: cluster.local (use only if your DNS server doesn't live in the same zone as kubecost) - prometheus: - enabled: true # If false, Prometheus will not be installed -- please read this before disabling: https://github.com/kubecost/docs/blob/main/custom-prom.md - fqdn: http://cost-analyzer-prometheus-server.default.svc #example address of a prometheus to connect to. Include protocol (http:// or https://) Ignored if enabled: true - # insecureSkipVerify : false # If true, kubecost will not check the TLS cert of prometheus - # queryServiceBasicAuthSecretName: dbsecret # kubectl create secret generic dbsecret -n kubecost --from-file=USERNAME --from-file=PASSWORD - # queryServiceBearerTokenSecretName: mcdbsecret # kubectl create secret generic mcdbsecret -n kubecost --from-file=TOKEN - - # Durable storage option, product key required - thanos: - enabled: false - # queryService: http://kubecost-thanos-query-frontend-http.kubecost:{% raw %}{{ .Values.thanos.queryFrontend.http.port }}{% endraw %} # an address of the thanos query-frontend endpoint, if different from installed thanos - # queryServiceBasicAuthSecretName: mcdbsecret # kubectl create secret generic mcdbsecret -n kubecost --from-file=USERNAME --from-file=PASSWORD <---enter basic auth credentials like that - # queryServiceBearerTokenSecretName mcdbsecret # kubectl create secret generic mcdbsecret -n kubecost --from-file=TOKEN - # queryOffset: 3h # The offset to apply to all thanos queries in order to achieve syncronization on all cluster block stores - - grafana: - enabled: true # If false, Grafana will not be installed - domainName: cost-analyzer-grafana.default.svc #example grafana domain Ignored if enabled: true - scheme: "http" # http or https, for the domain name above. - proxy: true # If true, the kubecost frontend will route to your grafana through its service endpoint -# fqdn: cost-analyzer-grafana.default.svc - - # Amazon Managed Service for Prometheus - amp: - enabled: false # If true, kubecost will be configured to remote_write and query from Amazon Managed Service for Prometheus. - prometheusServerEndpoint: https://localhost:8085// # The prometheus service endpoint used by kubecost. The calls are forwarded through the SigV4Proxy side car to the AMP workspace. - remoteWriteService: https://aps-workspaces.us-west-2.amazonaws.com/workspaces//api/v1/remote_write # The remote_write endpoint for the AMP workspace. - sigv4: - region: us-west-2 - # access_key: ACCESS_KEY # AWS Access key - # secret_key: SECRET_KEY # AWS Secret key - # role_arn: ROLE_ARN # AWS role arn - # profile: PROFILE # AWS profile - - notifications: - # Kubecost alerting configuration - # Ref: http://docs.kubecost.com/alerts - # alertConfigs: - # frontendUrl: http://localhost:9090 # optional, used for linkbacks - # globalSlackWebhookUrl: https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX # optional, used for Slack alerts - # globalMsTeamsWebhookUrl: https://xxxxx.webhook.office.com/webhookb2/XXXXXXXXXXXXXXXXXXXXXXXX/IncomingWebhook/XXXXXXXXXXXXXXXXXXXXXXXX # optional, used for Microsoft Teams alerts - # globalAlertEmails: - # - recipient@example.com - # - additionalRecipient@example.com - # Alerts generated by kubecost, about cluster data - # alerts: - # Daily namespace budget alert on namespace `kubecost` - # - type: budget # supported: budget, recurringUpdate - # threshold: 50 # optional, required for budget alerts - # window: daily # or 1d - # aggregation: namespace - # filter: kubecost - # ownerContact: # optional, overrides globalAlertEmails default - # - owner@example.com - # - owner2@example.com - # # optional, used for alert-specific Slack and Microsoft Teams alerts - # slackWebhookUrl: https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX - # msTeamsWebhookUrl: https://xxxxx.webhook.office.com/webhookb2/XXXXXXXXXXXXXXXXXXXXXXXX/IncomingWebhook/XXXXXXXXXXXXXXXXXXXXXXXX - - # Daily cluster budget alert on cluster `cluster-one` - # - type: budget - # threshold: 200.8 # optional, required for budget alerts - # window: daily # or 1d - # aggregation: cluster - # filter: cluster-one # does not accept csv - - # Recurring weekly update (weeklyUpdate alert) - # - type: recurringUpdate - # window: weekly # or 7d - # aggregation: namespace - # filter: '*' - - # Recurring weekly namespace update on kubecost namespace - # - type: recurringUpdate - # window: weekly # or 7d - # aggregation: namespace - # filter: kubecost - - # Spend Change Alert - # - type: spendChange # change relative to moving avg - # relativeThreshold: 0.20 # Proportional change relative to baseline. Must be greater than -1 (can be negative) - # window: 1d # accepts ‘d’, ‘h’ - # baselineWindow: 30d # previous window, offset by window - # aggregation: namespace - # filter: kubecost, default # accepts csv - - # Health Score Alert - # - type: health # Alerts when health score changes by a threshold - # window: 10m - # threshold: 5 # Send Alert if health scores changes by 5 or more - - # Kubecost Health Diagnostic - # - type: diagnostic # Alerts when kubecost is unable to compute costs - ie: Prometheus unreachable - # window: 10m - - alertmanager: # Supply an alertmanager FQDN to receive notifications from the app. - enabled: false # If true, allow kubecost to write to your alertmanager - fqdn: http://cost-analyzer-prometheus-server.default.svc #example fqdn. Ignored if prometheus.enabled: true - - # Set saved Cost Allocation report(s) accessible from /reports - # Ref: http://docs.kubecost.com/saved-reports - savedReports: - enabled: false # If true, overwrites report parameters set through UI - reports: - - title: "Example Saved Report 0" - window: "today" - aggregateBy: "namespace" - idle: "separate" - accumulate: false # daily resolution - filters: - - property: "cluster" - value: "cluster-one,cluster*" # supports wildcard filtering and multiple comma separated values - - property: "namespace" - value: "kubecost" - - title: "Example Saved Report 1" - window: "month" - aggregateBy: "controllerKind" - idle: "share" - accumulate: false - filters: - - property: "label" - value: "app:cost*,environment:kube*" - - property: "namespace" - value: "kubecost" - - title: "Example Saved Report 2" - window: "2020-11-11T00:00:00Z,2020-12-09T23:59:59Z" - aggregateBy: "service" - idle: "hide" - accumulate: true # entire window resolution - filters: [] # if no filters, specify empty array - - # Set saved Asset report(s) accessible from /reports - # Ref: http://docs.kubecost.com/saved-reports - assetReports: - enabled: false # If true, overwrites report parameters set through UI - reports: - - title: "Example Asset Report 0" - window: "today" - aggregateBy: "type" - accumulate: false # daily resolution - filters: - - property: "cluster" - value: "cluster-one" - - # Set saved Advanced report(s) accessible from /reports - # Ref: http://docs.kubecost.com/saved-reports - advancedReports: - enabled: false # If true, overwrites report parameters set through UI - reports: - - title: "Example Advanced Report 0" - window: "7d" - aggregateBy: "namespace" - filters: - - property: "cluster" - value: "cluster-one" - cloudBreakdown: "service" - cloudJoin: "label:kubernetes_namespace" - - podAnnotations: {} - # iam.amazonaws.com/role: role-arn - additionalLabels: {} - # generated at http://kubecost.com/install, used for alerts tracking and free trials -kubecostToken: # "" - -# Advanced pipeline for custom prices, enterprise key required -pricingCsv: - enabled: false - location: - provider: "AWS" - region: "us-east-1" - URI: s3://kc-csv-test/pricing_schema.csv # a valid file URI - csvAccessCredentials: pricing-schema-access-secret - -# SAML integration for user management and RBAC, enterprise key required -# Ref: https://github.com/kubecost/docs/blob/master/user-management.md -saml: - enabled: false - secretName: "kubecost-authzero" - #metadataSecretName: "kubecost-authzero-metadata" # One of metadataSecretName or idpMetadataURL must be set. defaults to metadataURL if set - idpMetadataURL: "https://dev-elu2z98r.auth0.com/samlp/metadata/c6nY4M37rBP0qSO1IYIqBPPyIPxLS8v2" - appRootURL: "http://localhost:9090" # sample URL - authTimeout: 1440 # number of minutes the JWT will be valid - redirectURL: "https://dev-elu2z98r.auth0.com/v2/logout" # callback URL redirected to after logout - # audienceURI: "http://localhost:9090" # by convention, the same as the appRootURL, but any string uniquely identifying kubecost to your samp IDP. Optional if you follow the convention - # nameIDFormat: "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified" If your SAML provider requires a specific nameid format - # isGLUUProvider: false # An additional URL parameter must be appended for GLUU providers - rbac: +kubecostToken: ${kubecostToken} +prometheus: + nodeExporter: enabled: false - groups: - - name: admin - enabled: false # if admin is disabled, all SAML users will be able to make configuration changes to the kubecost frontend - assertionName: "http://schemas.auth0.com/userType" # a SAML Assertion, one of whose elements has a value that matches on of the values in assertionValues - assertionValues: - - "admin" - - "superusers" - - name: readonly - enabled: false # if readonly is disabled, all users authorized on SAML will default to readonly - assertionName: "http://schemas.auth0.com/userType" - assertionvalues: - - "readonly" - - name: editor - enabled: true # if editor is enabled, editors will be allowed to edit reports/alerts scoped to them, and act as readers otherwise. Users will never default to editor. - assertionName: "http://schemas.auth0.com/userType" - assertionValues: - - "editor" - -oidc: - enabled: false - clientID: "" # application/client client_id paramter obtained from provider, used to make requests to server - clientSecret: "" # application/client client_secret paramter obtained from provider, used to make requests to server - secretName: "kubecost-oidc-secret" # k8s secret where clientsecret will be stored - authURL: "https://my.auth.server/authorize" # endpoint for login to auth server - loginRedirectURL: "http://my.kubecost.url/model/oidc/authorize" # Kubecost url configured in provider for redirect after authentication - discoveryURL: "https://my.auth.server/.well-known/openid-configuration" # url for OIDC endpoint discovery -# hostedDomain: "example.com" # optional, blocks access to the auth domain specified in the hd claim of the provider ID token - -# Adds an httpProxy as an environment variable. systemProxy.enabled must be `true`to have any effect. -# Ref: https://www.oreilly.com/library/view/security-with-go/9781788627917/5ea6a02b-3d96-44b1-ad3c-6ab60fcbbe4f.xhtml -systemProxy: - enabled: false - httpProxyUrl: "" - httpsProxyUrl: "" - noProxy: "" - -# imagePullSecrets: -# - name: "image-pull-secret" - -kubecostFrontend: - image: "gcr.io/kubecost1/frontend" - imagePullPolicy: Always - # extraEnv: - # - name: NGINX_ENTRYPOINT_WORKER_PROCESSES_AUTOTUNE - # value: "1" - # securityContext: - # readOnlyRootFilesystem: true - resources: - requests: - cpu: "10m" - memory: "55Mi" - #limits: - # cpu: "100m" - # memory: "256Mi" - ipv6: - enabled: true # disable if the cluster does not support ipv6 -# api: -# fqdn: kubecost-api.kubecost.svc.cluster.local:9001 -# model: -# fqdn: kubecost-model.kubecost.svc.cluster.local:9003 - -# Kubecost Metrics deploys a separate pod which will emit kubernetes specific metrics required -# by the cost-model. This pod is designed to remain active and decoupled from the cost-model itself. -# However, disabling this service/pod deployment will flag the cost-model to emit the metrics instead. -kubecostMetrics: - # emitPodAnnotations: false - # emitNamespaceAnnotations: false - # emitKsmV1Metrics: true # emit all KSM metrics in KSM v1. - # emitKsmV1MetricsOnly: false # emit only the KSM metrics missing from KSM v2. Advanced users only. - - # Optional - # The metrics exporter is a separate deployment and service (for prometheus scrape auto-discovery) - # which emits metrics cost-model relies on. Enabling this deployment also removes the KSM dependency - # from the cost-model. If the deployment is not enabled, the metrics will continue to be emitted from - # the cost-model. - exporter: + # kubecost emits pre-2.0 KSM metrics, KSM is enabled by default here for backwards compatibity, but can be disabled to save resources without concern to kubecost metrics + kubeStateMetrics: enabled: false - port: 9005 - # Adds the default Prometheus scrape annotations to the metrics exporter service. - # Set to false and use service.annotations (below) to set custom scrape annotations. - prometheusScrape: true - resources: {} - # requests: - # cpu: "200m" - # memory: "55Mi" - ## Node tolerations for server scheduling to nodes with taints - ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - tolerations: [] - - # - key: "key" - # operator: "Equal|Exists" - # value: "value" - # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: application-group - operator: In - values: - - service - service: - annotations: {} - - # Service Monitor for Kubecost Metrics - serviceMonitor: # the kubecost included prometheus uses scrapeConfigs and does not support service monitors. The following options assume an existing prometheus that supports serviceMonitors. - enabled: false - additionalLabels: {} - networkCosts: - enabled: false - scrapeTimeout: 10s - additionalLabels: {} - ## PriorityClassName - ## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass - priorityClassName: [] - additionalLabels: {} - nodeSelector: {} - extraArgs: [] - -sigV4Proxy: - image: public.ecr.aws/aws-observability/aws-sigv4-proxy:latest - imagePullPolicy: Always - name: aps - port: 8005 - region: us-west-2 # The AWS region - host: aps-workspaces.us-west-2.amazonaws.com # The hostname for AMP service. - # role_arn: arn:aws:iam:::role/role-name # The AWS IAM role to assume. - extraEnv: # Pass extra env variables to sigV4Proxy - # - name: AWS_ACCESS_KEY_ID - # value: - # - name: AWS_SECRET_ACCESS_KEY - # value: - -kubecostModel: - image: "gcr.io/kubecost1/cost-model" - imagePullPolicy: Always - # extraEnv: - # - name: SOME_VARIABLE - # value: "some_value" - # securityContext: - # readOnlyRootFilesystem: true - # Enables the emission of the kubecost_cloud_credit_total and - # kubecost_cloud_expense_total metrics - outOfClusterPromMetricsEnabled: false - # Build local cost allocation cache - warmCache: false - # Build local savings cache - warmSavingsCache: true - # Run allocation ETL pipelines - etl: true - # Enable the ETL filestore backing storage - etlFileStoreEnabled: true - # The total number of days the ETL pipelines will build - # Set to 0 to disable daily ETL (not recommended) - etlDailyStoreDurationDays: 91 - # The total number of hours the ETL pipelines will build - # Set to 0 to disable hourly ETL (not recommended) - etlHourlyStoreDurationHours: 49 - # For deploying kubecost in a cluster that does not self-monitor - etlReadOnlyMode: false - - allocation: - # Enables or disables adding node labels to allocation data (i.e. workloads). - # Defaults to "true" and starts with a sensible includeList for basics like - # topology (e.g. zone, region) and instance type labels. - # nodeLabels: - # enabled: true - # includeList: "node.kubernetes.io/instance-type,topology.kubernetes.io/region,topology.kubernetes.io/zone" - - # Enables or disables the ContainerStats pipeline, used for quantile-based - # queries like for request sizing recommendations. - # ContainerStats provides support for quantile-based request right-sizing - # recommendations. - # - # It is disabled by default to avoid problems in extremely high-scale Thanos - # environments. If you would like to try quantile-based request-sizing - # recommendations, enable this! If you are in a high-scale environment, - # please monitor Kubecost logs, Thanos query logs, and Thanos load closely. - # We hope to make major improvements at scale here soon! - # - # containerStatsEnabled: false - - # max number of concurrent Prometheus queries - maxQueryConcurrency: 5 - resources: - requests: - cpu: "200m" - memory: "55Mi" - #limits: - # cpu: "800m" - # memory: "256Mi" - extraArgs: [] - -# Basic Kubecost ingress, more examples available at https://github.com/kubecost/docs/blob/master/ingress-examples.md -ingress: - enabled: false - # className: nginx - annotations: - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - paths: ["/"] # There's no need to route specifically to the pods-- we have an nginx deployed that handles routing - pathType: ImplementationSpecific - hosts: - - cost-analyzer.local - tls: [] - # - secretName: cost-analyzer-tls - # hosts: - # - cost-analyzer.local - -nodeSelector: {} - -tolerations: [] -# - key: "key" -# operator: "Equal|Exists" -# value: "value" -# effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" - + kube-state-metrics: + disabled: false affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - - key: application-group + - key: node-group operator: In values: - - service -# If true, creates a PriorityClass to be used by the cost-analyzer pod -priority: - enabled: false - name: "" # Provide name of existing priority class only. If left blank, upstream chart will create one from default template. - # value: 1000000 - -# If true, enable creation of NetworkPolicy resources. -networkPolicy: - enabled: false - denyEgress: true # create a network policy that denies egress from kubecost - sameNamespace: true # Set to true if cost analyser and prometheus are on the same namespace -# namespace: kubecost # Namespace where prometheus is installed - - # Cost-analyzer specific vars using the new template - costAnalyzer: - enabled: false # If true, create a newtork policy for cost-analzyer - annotations: {} # annotations to be added to the network policy - additionalLabels: {} # additional labels to be added to the network policy - # Examples rules: - # ingressRules: - # - selectors: # allow ingress from self on all ports - # - podSelector: - # matchLabels: - # app.kubernetes.io/name: cost-analyzer - # - selectors: # allow egress access to prometheus - # - namespaceSelector: - # matchLabels: - # name: prometheus - # podSelector: - # matchLabels: - # app: prometheus - # ports: - # - protocol: TCP - # port: 9090 - # egressRules: - # - selectors: # restrict egress to inside cluster - # - namespaceSelector: {} - -podSecurityPolicy: - enabled: true - -## @param extraVolumes A list of volumes to be added to the pod -## -extraVolumes: [] -## @param extraVolumeMounts A list of volume mounts to be added to the pod -## -extraVolumeMounts: [] - -# Define persistence volume for cost-analyzer, more information at https://github.com/kubecost/docs/blob/master/storage.md -persistentVolume: - size: 32Gi - dbSize: 32.0Gi - enabled: true # Note that setting this to false means configurations will be wiped out on pod restart. - # storageClass: "-" # - # existingClaim: kubecost-cost-analyzer # a claim in the same namespace as kubecost - -service: - type: ClusterIP - port: 9090 - targetPort: 9090 - # nodePort: - labels: {} - annotations: {} - -# Enabling long-term durable storage with Postgres requires an enterprise license -remoteWrite: - postgres: - enabled: false - initImage: "gcr.io/kubecost1/sql-init" - initImagePullPolicy: Always - installLocal: true - remotePostgresAddress: "" # ignored if installing locally - persistentVolume: - size: 200Gi - auth: - password: admin # change me - -prometheus: - extraScrapeConfigs: | - - job_name: kubecost - honor_labels: true - scrape_interval: 1m - scrape_timeout: 60s - metrics_path: /metrics - scheme: http - dns_sd_configs: - - names: - - {% raw %}{{ template "cost-analyzer.serviceName" . }}{% endraw %} - type: 'A' - port: 9003 - - job_name: kubecost-networking - kubernetes_sd_configs: - - role: pod - relabel_configs: - # Scrape only the the targets matching the following metadata - - source_labels: [__meta_kubernetes_pod_label_app] - action: keep - regex: {% raw %}{{ template "cost-analyzer.networkCostsName" . }}{% endraw %} - server: - # If clusterIDConfigmap is defined, instead use user-generated configmap with key CLUSTER_ID - # to use as unique cluster ID in kubecost cost-analyzer deployment. - # This overrides the cluster_id set in prometheus.server.global.external_labels. - # NOTE: This does not affect the external_labels set in prometheus config. - # clusterIDConfigmap: cluster-id-configmap - - resources: {} - # limits: - # cpu: 500m - # memory: 512Mi - # requests: - # cpu: 500m - # memory: 512Mi - global: - scrape_interval: 1m - scrape_timeout: 60s - evaluation_interval: 1m - external_labels: - cluster_id: cluster-one # Each cluster should have a unique ID - persistentVolume: - size: 32Gi - enabled: true - extraArgs: - query.max-concurrency: 1 - query.max-samples: 100000000 - tolerations: [] - # - key: "key" - # operator: "Equal|Exists" - # value: "value" - # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" - alertmanager: - enabled: false - persistentVolume: - enabled: true - # node-export must be disabled if there is an existing daemonset: https://guide.kubecost.com/hc/en-us/articles/4407601830679-Troubleshoot-Install#a-name-node-exporter-a-issue-failedscheduling-kubecost-prometheus-node-exporter - nodeExporter: - enabled: true - # kubecost emits pre-2.0 KSM metrics, KSM is enabled by default here for backwards compatibity, but can be disabled to save resources without concern to kubecost metrics - kubeStateMetrics: - enabled: true - kube-state-metrics: - disabled: false - pushgateway: - enabled: false - persistentVolume: - enabled: true - serverFiles: - # prometheus.yml: # Sample block -- enable if using an in cluster durable store. - # remote_write: - # - url: "http://pgprometheus-adapter:9201/write" - # write_relabel_configs: - # - source_labels: [__name__] - # regex: 'container_.*_allocation|container_.*_allocation_bytes|.*_hourly_cost|kube_pod_container_resource_requests{resource="memory", unit="byte"}|container_memory_working_set_bytes|kube_pod_container_resource_requests{resource="cpu", unit="core"}|kube_pod_container_resource_requests|pod_pvc_allocation|kube_namespace_labels|kube_pod_labels' - # action: keep - # queue_config: - # max_samples_per_send: 1000 - #remote_read: - # - url: "http://pgprometheus-adapter:9201/read" - rules: - groups: - - name: CPU - rules: - - expr: sum(rate(container_cpu_usage_seconds_total{container_name!=""}[5m])) - record: cluster:cpu_usage:rate5m - - expr: rate(container_cpu_usage_seconds_total{container_name!=""}[5m]) - record: cluster:cpu_usage_nosum:rate5m - - expr: avg(irate(container_cpu_usage_seconds_total{container_name!="POD", container_name!=""}[5m])) by (container_name,pod_name,namespace) - record: kubecost_container_cpu_usage_irate - - expr: sum(container_memory_working_set_bytes{container_name!="POD",container_name!=""}) by (container_name,pod_name,namespace) - record: kubecost_container_memory_working_set_bytes - - expr: sum(container_memory_working_set_bytes{container_name!="POD",container_name!=""}) - record: kubecost_cluster_memory_working_set_bytes - - name: Savings - rules: - - expr: sum(avg(kube_pod_owner{owner_kind!="DaemonSet"}) by (pod) * sum(container_cpu_allocation) by (pod)) - record: kubecost_savings_cpu_allocation - labels: - daemonset: "false" - - expr: sum(avg(kube_pod_owner{owner_kind="DaemonSet"}) by (pod) * sum(container_cpu_allocation) by (pod)) / sum(kube_node_info) - record: kubecost_savings_cpu_allocation - labels: - daemonset: "true" - - expr: sum(avg(kube_pod_owner{owner_kind!="DaemonSet"}) by (pod) * sum(container_memory_allocation_bytes) by (pod)) - record: kubecost_savings_memory_allocation_bytes - labels: - daemonset: "false" - - expr: sum(avg(kube_pod_owner{owner_kind="DaemonSet"}) by (pod) * sum(container_memory_allocation_bytes) by (pod)) / sum(kube_node_info) - record: kubecost_savings_memory_allocation_bytes - labels: - daemonset: "true" - -## Module for measuring network costs -## Ref: https://github.com/kubecost/docs/blob/master/network-allocation.md + - {{ cookiecutter.global_platform_shared_resource_identifier }} networkCosts: - enabled: false - podSecurityPolicy: - enabled: false - image: gcr.io/kubecost1/kubecost-network-costs:v16.3 - imagePullPolicy: Always - updateStrategy: - type: RollingUpdate - # For existing Prometheus Installs, annotates the Service which generates Endpoints for each of the network-costs pods. - # The Service is annotated with prometheus.io/scrape: "true" to automatically get picked up by the prometheus config. - # NOTE: Setting this option to true and leaving the above extraScrapeConfig "job_name: kubecost-networking" configured will cause the - # NOTE: pods to be scraped twice. - prometheusScrape: false - # Traffic Logging will enable logging the top 5 destinations for each source - # every 30 minutes. - trafficLogging: true - - # Port will set both the containerPort and hostPort to this value. - # These must be identical due to network-costs being run on hostNetwork - port: 3001 - # this daemonset can use significant resources on large clusters: https://guide.kubecost.com/hc/en-us/articles/4407595973527-Network-Traffic-Cost-Allocation - resources: - limits: # remove the limits by setting limits: {} - cpu: 500m # can be less, will depend on cluster size - # memory: it is not recommended to set a memory limit - requests: - cpu: 50m - memory: 20Mi - extraArgs: [] - config: - # Configuration for traffic destinations, including specific classification - # for IPs and CIDR blocks. This configuration will act as an override to the - # automatic classification provided by network-costs. - destinations: - # In Zone contains a list of address/range that will be - # classified as in zone. - in-zone: - # Loopback Addresses in "IANA IPv4 Special-Purpose Address Registry" - - "127.0.0.0/8" - # IPv4 Link Local Address Space - - "169.254.0.0/16" - # Private Address Ranges in RFC-1918 - - "10.0.0.0/8" # Remove this entry if using Multi-AZ Kubernetes - - "172.16.0.0/12" - - "192.168.0.0/16" - - # In Region contains a list of address/range that will be - # classified as in region. This is synonymous with cross - # zone traffic, where the regions between source and destinations - # are the same, but the zone is different. - in-region: [] - - # Cross Region contains a list of address/range that will be - # classified as non-internet egress from one region to another. - cross-region: [] - - # Direct Classification specifically maps an ip address or range - # to a region (required) and/or zone (optional). This classification - # takes priority over in-zone, in-region, and cross-region configurations. - direct-classification: [] - # - region: "us-east1" - # zone: "us-east1-c" - # ips: - # - "10.0.0.0/24" - services: - # google-cloud-services: when set to true, enables labeling traffic metrics with google cloud - # service endpoints - google-cloud-services: false - # amazon-web-services: when set to true, enables labeling traffic metrics with amazon web service - # endpoints. - amazon-web-services: false - # azure-cloud-services: when set to true, enables labeling traffic metrics with azure cloud service - # endpoints - azure-cloud-services: false - # user defined services provide a way to define custom service endpoints which will label traffic metrics - # falling within the defined address range. - #services: - # - service: "test-service-1" - # ips: - # - "19.1.1.2" - # - service: "test-service-2" - # ips: - # - "15.128.15.2" - # - "20.0.0.0/8" - - ## Node tolerations for server scheduling to nodes with taints - ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - ## - tolerations: [] - # - key: "key" - # operator: "Equal|Exists" - # value: "value" - # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" - affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - - key: application-group + - key: node-group operator: In values: - - service - service: - annotations: {} - labels: {} - - ## PriorityClassName - ## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass - priorityClassName: [] - ## PodMonitor - ## Allows scraping of network metrics from a dedicated prometheus operator setup - podMonitor: - enabled: false - additionalLabels: {} - additionalLabels: {} - nodeSelector: {} - annotations: {} - -# Kubecost Deployment Configuration -# Used for HA mode in Business & Enterprise tier -kubecostDeployment: - replicas: 1 - leaderFollower: - enabled: false - # deploymentStrategy: - # rollingUpdate: - # maxSurge: 1 - # maxUnavailable: 1 - # type: RollingUpdate - -# Kubecost Cluster Controller for Right Sizing and Cluster Turndown -clusterController: - enabled: false - image: gcr.io/kubecost1/cluster-controller:v0.6.1 - imagePullPolicy: Always - kubescaler: - # If true, will cause all (supported) workloads to be have their requests - # automatically right-sized on a regular basis. - defaultResizeAll: false -# fqdn: kubecost-cluster-controller.kubecost.svc.cluster.local:9731 - -reporting: - # Kubecost bug report feature: Logs access/collection limited to .Release.Namespace - # Ref: http://docs.kubecost.com/bug-report - logCollection: true - # Basic frontend analytics - productAnalytics: true - - # Report Javascript errors - errorReporting: true - valuesReporting: true - # googleAnalyticsTag allows you to embed your Google Global Site Tag to track usage of Kubecost. - # googleAnalyticsTag is only included in our Enterprise offering. - # googleAnalyticsTag: G-XXXXXXXXX - -serviceMonitor: # the kubecost included prometheus uses scrapeConfigs and does not support service monitors. The following options assume an existing prometheus that supports serviceMonitors. - enabled: false - additionalLabels: {} - networkCosts: - enabled: false - scrapeTimeout: 10s - additionalLabels: {} - -prometheusRule: - enabled: false - additionalLabels: {} - -supportNFS: false -# initChownDataImage ensures all Kubecost filepath permissions on PV or local storage are set up correctly. -initChownDataImage: "busybox" # Supports a fully qualified Docker image, e.g. registry.hub.docker.com/library/busybox:latest -initChownData: - resources: {} - #requests: - # cpu: "50m" - # memory: "20Mi" - - -grafana: - # namespace_datasources: kubecost # override the default namespace here - # namespace_dashboards: kubecost # override the default namespace here - rbac: - # Manage the Grafana Pod Security Policy - pspEnabled: true - datasources: - datasources.yaml: - apiVersion: 1 - datasources: - - name: prometheus-kubecost - type: prometheus - url: http://kubecost-prometheus-server.kubecost.svc.cluster.local - access: proxy - isDefault: false - sidecar: - dashboards: - enabled: true - # label that the configmaps with dashboards are marked with - label: grafana_dashboard - # set sidecar ERROR_THROTTLE_SLEEP env var from default 5s to 0s -> fixes https://github.com/kubecost/cost-analyzer-helm-chart/issues/877 - annotations: {} - error_throttle_sleep: 0 - datasources: - # dataSourceFilename: foo.yml # If you need to change the name of the datasource file - enabled: false - error_throttle_sleep: 0 -# For grafana to be accessible, add the path to root_url. For example, if you run kubecost at www.foo.com:9090/kubecost -# set root_url to "%(protocol)s://%(domain)s:%(http_port)s/kubecost/grafana". No change is necessary here if kubecost runs at a root URL - grafana.ini: - server: - serve_from_sub_path: true - root_url: "%(protocol)s://%(domain)s:%(http_port)s/grafana" -serviceAccount: - create: true # Set this to false if you're bringing your own service account. - annotations: {} - # name: kc-test -awsstore: - useAwsStore: false - createServiceAccount: false - -federatedETL: - # federatedCluster indicates whether this cluster should push data to the Federated store - federatedCluster: false - # primaryCluster indicates whether this cluster should load data from the combined section of the Federated store - primaryCluster: false - # redirectS3Backup changes the dir of S3 backup to the Federated combined store, for using Thanos-federated data in the Federated ETL - # Note S3 backup should be enabled separately for this. - redirectS3Backup: false - # useExistingS3Config will attempt to use existing object-store.yaml configs for S3 backup/Thanos as config for the Federated store - useExistingS3Config: false - federator: - # federationCutoffDate is an RFC 3339-formatted string. All ETL files with windows that fall before this time are not processed by the Federator. - # If this is not set, the Federator will process all files regardless of date. - # federationCutoffDate: "2022-10-18T00:00:00.000Z" - # federator.enabled enables the federator to run inside the costmodel container, federating the data in the Federated store - enabled: false - # federator.clusters is an optional whitelist of clusters by cluster id. - # If not set, the federator will attempt to federated all clusters pushing to the federated storage. - clusters: [] - -kubecostAdmissionController: - enabled: false - -# readonly: false # disable updates to kubecost from the frontend UI and via POST request - -# These configs can also be set from the Settings page in the Kubecost product UI -# Values in this block override config changes in the Settings UI on pod restart -# -# kubecostProductConfigs: -# An optional list of cluster definitions that can be added for frontend access. The local -# cluster is *always* included by default, so this list is for non-local clusters. -# Ref: https://github.com/kubecost/docs/blob/master/multi-cluster.md -# clusters: -# - name: "Cluster A" -# address: http://cluster-a.kubecost.com:9090 -# # Optional authentication credentials - only basic auth is currently supported. -# auth: -# type: basic -# # Secret name should be a secret formatted based on: https://github.com/kubecost/docs/blob/master/ingress-examples.md -# secretName: cluster-a-auth -# # Or pass auth directly as base64 encoded user:pass -# data: YWRtaW46YWRtaW4= -# # Or user and pass directly -# user: admin -# pass: admin -# - name: "Cluster B" -# address: http://cluster-b.kubecost.com:9090 -# defaultModelPricing: # default monthly resource prices, used predominately for on-prem clusters -# CPU: 28.0 -# spotCPU: 4.86 -# RAM: 3.09 -# spotRAM: 0.65 -# GPU: 693.50 -# spotGPU: 225.0 -# storage: 0.04 -# zoneNetworkEgress: 0.01 -# regionNetworkEgress: 0.01 -# internetNetworkEgress: 0.12 -# enabled: true -# # The cluster profile represents a predefined set of parameters to use when calculating savings. -# # Possible values are: [ development, production, high-availability ] -# clusterProfile: production -# customPricesEnabled: false # This makes the default view custom prices-- generally used for on-premises clusters -# spotLabel: lifecycle -# spotLabelValue: Ec2Spot -# gpuLabel: gpu -# gpuLabelValue: true -# awsServiceKeyName: ACCESSKEYID -# awsServiceKeyPassword: fakepassword # Only use if your values.yaml are stored encrypted. Otherwise provide an existing secret via serviceKeySecretName -# awsSpotDataRegion: us-east-1 -# awsSpotDataBucket: spot-data-feed-s3-bucket -# awsSpotDataPrefix: dev -# athenaProjectID: "530337586277" # The AWS AccountID where the Athena CUR is. Generally your masterpayer account -# athenaBucketName: "s3://aws-athena-query-results-530337586277-us-east-1" -# athenaRegion: us-east-1 -# athenaDatabase: athenacurcfn_athena_test1 -# athenaTable: "athena_test1" -# athenaWorkgroup: "primary" # The default workgroup in AWS is 'primary' -# masterPayerARN: "" -# projectID: "123456789" # Also known as AccountID on AWS -- the current account/project that this instance of Kubecost is deployed on. -# gcpSecretName: gcp-secret # Name of a secret representing the gcp service key -# bigQueryBillingDataDataset: billing_data.gcp_billing_export_v1_01AC9F_74CF1D_5565A2 -# labelMappingConfigs: # names of k8s labels or annotations used to designate different allocation concepts -# enabled: true -# owner_label: "owner" -# team_label: "team" -# department_label: "dept" -# product_label: "product" -# environment_label: "env" -# namespace_external_label: "kubernetes_namespace" # external labels/tags are used to map external cloud costs to kubernetes concepts -# cluster_external_label: "kubernetes_cluster" -# controller_external_label: "kubernetes_controller" -# product_external_label: "kubernetes_label_app" -# service_external_label: "kubernetes_service" -# deployment_external_label: "kubernetes_deployment" -# owner_external_label: "kubernetes_label_owner" -# team_external_label: "kubernetes_label_team" -# environment_external_label: "kubernetes_label_env" -# department_external_label: "kubernetes_label_department" -# statefulset_external_label: "kubernetes_statefulset" -# daemonset_external_label: "kubernetes_daemonset" -# pod_external_label: "kubernetes_pod" -# grafanaURL: "" -# clusterName: "" # clusterName is the default context name in settings. -# currencyCode: "USD" # official support for USD, AUD, BRL, CAD, CHF, CNY, DKK, EUR, GBP, INR, JPY, NOK, PLN, SEK -# azureBillingRegion: US # Represents 2-letter region code, e.g. West Europe = NL, Canada = CA. ref: https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes -# azureSubscriptionID: 0bd50fdf-c923-4e1e-850c-196dd3dcc5d3 -# azureClientID: f2ef6f7d-71fb-47c8-b766-8d63a19db017 -# azureTenantID: 72faf3ff-7a3f-4597-b0d9-7b0b201bb23a -# azureClientPassword: fake key # Only use if your values.yaml are stored encrypted. Otherwise provide an existing secret via serviceKeySecretName -# azureOfferDurableID: "MS-AZR-0003p" -# discount: "" # percentage discount applied to compute -# negotiatedDiscount: "" # custom negotiated cloud provider discount -# defaultIdle: false -# serviceKeySecretName: "" # Use an existing AWS or Azure secret with format as in aws-service-key-secret.yaml or azure-service-key-secret.yaml. Leave blank if using createServiceKeySecret -# createServiceKeySecret: true # Creates a secret representing your cloud service key based on data in values.yaml. If you are storing unencrypted values, add a secret manually -# sharedNamespaces: "" # namespaces with shared workloads, example value: "kube-system\,ingress-nginx\,kubecost\,monitoring" -# sharedOverhead: "" # value representing a fixed external cost per month to be distributed among aggregations. -# shareTenancyCosts: true # enable or disable sharing costs such as cluster management fees (defaults to "true" on Settings page) -# metricsConfigs: # configuration for metrics emitted by Kubecost -# disabledMetrics: [] # list of metrics that Kubecost will not emit. Note that disabling metrics can lead to unexpected behavior in the cost-model. -# productKey: # apply business or enterprise product license -# key: "" -# enabled: false -# secretname: productkeysecret # create a secret out of a file named productkey.json of format { "key": "kc-b1325234" } -# mountPath: "/some/custom/path/productkey.json" # (use instead of secretname) declare the path at which the product key file is mounted (eg. by a secrets provisioner). The file must be of format { "key": "kc-b1325234" } -# cloudIntegrationSecret: "cloud-integration" -# ingestPodUID: false # Enables using UIDs to uniquely ID pods. This requires either Kubecost's replicated KSM metrics, or KSM v2.1.0+. This may impact performance, and changes the default cost-model allocation behavior. + - {{ cookiecutter.global_platform_shared_resource_identifier }} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/config/values.yaml b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/config/values.yaml deleted file mode 100644 index 70cea00e..00000000 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/config/values.yaml +++ /dev/null @@ -1,41 +0,0 @@ -global: - # zone: cluster.local (use only if your DNS server doesn't live in the same zone as kubecost) - prometheus: - enabled: true # If false, Prometheus will not be installed -- please read this before disabling: https://github.com/kubecost/docs/blob/main/custom-prom.md - fqdn: http://cost-analyzer-prometheus-server.default.svc #example address of a prometheus to connect to. Include protocol (http:// or https://) Ignored if enabled: true - # insecureSkipVerify : false # If true, kubecost will not check the TLS cert of prometheus - # queryServiceBasicAuthSecretName: dbsecret # kubectl create secret generic dbsecret -n kubecost --from-file=USERNAME --from-file=PASSWORD - # queryServiceBearerTokenSecretName: mcdbsecret # kubectl create secret generic mcdbsecret -n kubecost --from-file=TOKEN - -# generated at http://kubecost.com/install, used for alerts tracking and free trials -kubecostToken: ${kubecostToken} - -persistentVolume: - size: 32Gi - dbSize: 32.0Gi - enabled: true # Note that setting this to false means configurations will be wiped out on pod restart. - # storageClass: "-" # - # existingClaim: kubecost-cost-analyzer # a claim in the same namespace as kubecost -prometheus: - server: - # If clusterIDConfigmap is defined, instead use user-generated configmap with key CLUSTER_ID - # to use as unique cluster ID in kubecost cost-analyzer deployment. - # This overrides the cluster_id set in prometheus.server.global.external_labels. - # NOTE: This does not affect the external_labels set in prometheus config. - # clusterIDConfigmap: cluster-id-configmap - persistentVolume: - size: 32Gi - enabled: true - tolerations: - - effect: NoSchedule - key: role - operator: Equal - value: pvc-pods - # node-export must be disabled if there is an existing daemonset: https://guide.kubecost.com/hc/en-us/articles/4407601830679-Troubleshoot-Install#a-name-node-exporter-a-issue-failedscheduling-kubecost-prometheus-node-exporter - nodeExporter: - enabled: false - # kubecost emits pre-2.0 KSM metrics, KSM is enabled by default here for backwards compatibity, but can be disabled to save resources without concern to kubecost metrics - kubeStateMetrics: - enabled: false - kube-state-metrics: - disabled: true diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/main.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/main.tf index ec61b71a..aed58b8a 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/main.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_kubecost/main.tf @@ -14,13 +14,19 @@ # helm repo update # helm search repo cost-analyzer # helm show values cost-analyzer/cost-analyzer +# +# Trouble shooting an installation: +# helm ls --namespace kubecost +# helm history cost-analyzer --namespace kubecost +# helm rollback cost-analyzer 4 --namespace kubecost + #----------------------------------------------------------- locals { cost_analyzer = "cost-analyzer" } data "template_file" "kubecost-values" { - template = file("${path.module}/config/values.yaml") + template = file("${path.module}/config/kubecost-values.yaml") vars = { # get a free Kubecost token here: # https://www.kubecost.com/install#show-instructions diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_metricsserver/config/metrics-server-values.yaml b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_metricsserver/config/metrics-server-values.yaml new file mode 100644 index 00000000..727f21be --- /dev/null +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_metricsserver/config/metrics-server-values.yaml @@ -0,0 +1,10 @@ +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-group + operator: In + values: + - {{ cookiecutter.global_platform_shared_resource_identifier }} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_metricsserver/main.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_metricsserver/main.tf index 4e3b0fe6..d2546722 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_metricsserver/main.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_metricsserver/main.tf @@ -19,9 +19,13 @@ # helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/ # helm repo update # helm search repo metrics-server -# +# helm show values metrics-server/metrics-server #----------------------------------------------------------- +data "template_file" "metrics-server-values" { + template = file("${path.module}/config/metrics-server-values.yaml") + vars = {} +} resource "helm_release" "metrics_server" { namespace = "metrics-server" @@ -32,4 +36,8 @@ resource "helm_release" "metrics_server" { chart = "metrics-server" version = "{{ cookiecutter.terraform_helm_metrics_server }}" + values = [ + data.template_file.metrics-server-values.rendered + ] + } diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/main.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/main.tf index 4ec08ac3..251e770f 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/main.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/main.tf @@ -35,12 +35,9 @@ # kubectl delete crd thanosrulers.monitoring.coreos.com #----------------------------------------------------------- -resource "random_password" "grafana" { - length = 16 - special = false - keepers = { - version = "1" - } +data "template_file" "prometheus-values" { + template = file("${path.module}/yml/prometheus-values.yaml") + vars = {} } resource "helm_release" "prometheus" { @@ -52,6 +49,10 @@ resource "helm_release" "prometheus" { chart = "kube-prometheus-stack" version = "{{ cookiecutter.terraform_helm_prometheus }}" + values = [ + data.template_file.prometheus-values.rendered + ] + # changes the password (stored in k8s secret prometheus-grafana) from 'prom-operator' # to our own randomly generated 16-character strong password. set { @@ -59,12 +60,15 @@ resource "helm_release" "prometheus" { value = random_password.grafana.result } - # un-comment to include a yaml manifest with configuration overrides. - # To generate a yaml file with all possible configuration options: - # helm show values prometheus-community/kube-prometheus-stack > ./yml/values.yaml - # - #values = [ - # "${file("${path.module}/yml/values.yaml")}" - #] +} +#------------------------------------------------------------------------------ +# SUPPORTING RESOURCES +#------------------------------------------------------------------------------ +resource "random_password" "grafana" { + length = 16 + special = false + keepers = { + version = "1" + } } diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/yml/prometheus-values.yaml b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/yml/prometheus-values.yaml new file mode 100644 index 00000000..727f21be --- /dev/null +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/yml/prometheus-values.yaml @@ -0,0 +1,10 @@ +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-group + operator: In + values: + - {{ cookiecutter.global_platform_shared_resource_identifier }} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/yml/values.yaml b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/yml/values.yaml index 8d95f908..585ab82a 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/yml/values.yaml +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_prometheus/yml/values.yaml @@ -631,10 +631,10 @@ alertmanager: - weight: 1 preference: matchExpressions: - - key: application-group + - key: node-group operator: In values: - - service + - {{ cookiecutter.global_platform_shared_resource_identifier }} # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # nodeSelectorTerms: @@ -1885,10 +1885,10 @@ prometheusOperator: - weight: 1 preference: matchExpressions: - - key: application-group + - key: node-group operator: In values: - - service + - {{ cookiecutter.global_platform_shared_resource_identifier }} tolerations: [] ## SecurityContext holds pod-level security attributes and common container settings. @@ -2120,10 +2120,10 @@ prometheusOperator: - weight: 1 preference: matchExpressions: - - key: application-group + - key: node-group operator: In values: - - service + - {{ cookiecutter.global_platform_shared_resource_identifier }} # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # nodeSelectorTerms: @@ -2935,10 +2935,10 @@ prometheus: - weight: 1 preference: matchExpressions: - - key: application-group + - key: node-group operator: In values: - - service + - {{ cookiecutter.global_platform_shared_resource_identifier }} # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # nodeSelectorTerms: @@ -3705,10 +3705,10 @@ thanosRuler: - weight: 1 preference: matchExpressions: - - key: application-group + - key: node-group operator: In values: - - service + - {{ cookiecutter.global_platform_shared_resource_identifier }} # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # nodeSelectorTerms: diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_vpa/main.tf b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_vpa/main.tf index 36a7f8ee..37959202 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_vpa/main.tf +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_vpa/main.tf @@ -17,12 +17,17 @@ # helm repo add cowboysysop https://cowboysysop.github.io/charts/ # helm repo update # helm search repo cowboysysop -# +# helm show values cowboysysop/vertical-pod-autoscaler # # NOTE: run `helm repo update` prior to running this # Terraform module. #----------------------------------------------------------- +data "template_file" "vertical-pod-autoscaler-values" { + template = file("${path.module}/yml/vertical-pod-autoscaler-values.yaml") + vars = {} +} + resource "helm_release" "vpa" { namespace = "vpa" create_namespace = true @@ -32,4 +37,8 @@ resource "helm_release" "vpa" { chart = "vertical-pod-autoscaler" version = "{{ cookiecutter.terraform_helm_vertical_pod_autoscaler }}" + values = [ + data.template_file.vertical-pod-autoscaler-values.rendered + ] + } diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_vpa/yml/vertical-pod-autoscaler-values.yaml b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_vpa/yml/vertical-pod-autoscaler-values.yaml new file mode 100644 index 00000000..727f21be --- /dev/null +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/modules/kubernetes_vpa/yml/vertical-pod-autoscaler-values.yaml @@ -0,0 +1,10 @@ +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-group + operator: In + values: + - {{ cookiecutter.global_platform_shared_resource_identifier }} diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/kubernetes/terragrunt.hcl b/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/kubernetes/terragrunt.hcl index 2f56e1ca..642264c3 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/kubernetes/terragrunt.hcl +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/kubernetes/terragrunt.hcl @@ -8,8 +8,8 @@ #------------------------------------------------------------------------------ locals { # Automatically load stack-level variables - stack_vars = read_terragrunt_config(find_in_parent_folders("stack.hcl")) - global_vars = read_terragrunt_config(find_in_parent_folders("global.hcl")) + stack_vars = read_terragrunt_config(find_in_parent_folders("stack.hcl")) + global_vars = read_terragrunt_config(find_in_parent_folders("global.hcl")) # Extract out common variables for reuse env = local.stack_vars.locals.stack @@ -22,14 +22,16 @@ locals { shared_resource_identifier = local.global_vars.locals.shared_resource_identifier kubernetes_version = local.stack_vars.locals.kubernetes_version eks_create_kms_key = local.stack_vars.locals.eks_create_kms_key - eks_worker_group_instance_type = local.stack_vars.locals.eks_worker_group_instance_type - eks_worker_group_min_size = local.stack_vars.locals.eks_worker_group_min_size - eks_worker_group_max_size = local.stack_vars.locals.eks_worker_group_max_size - eks_worker_group_desired_size = local.stack_vars.locals.eks_worker_group_desired_size - eks_karpenter_group_instance_type = local.stack_vars.locals.eks_karpenter_group_instance_type - eks_karpenter_group_min_size = local.stack_vars.locals.eks_karpenter_group_min_size - eks_karpenter_group_max_size = local.stack_vars.locals.eks_karpenter_group_max_size - eks_karpenter_group_desired_size = local.stack_vars.locals.eks_karpenter_group_desired_size + eks_hosting_group_instance_type = local.stack_vars.locals.eks_hosting_group_instance_type + eks_hosting_group_min_size = local.stack_vars.locals.eks_hosting_group_min_size + eks_hosting_group_max_size = local.stack_vars.locals.eks_hosting_group_max_size + eks_hosting_group_desired_size = local.stack_vars.locals.eks_hosting_group_desired_size + eks_service_group_instance_type = local.stack_vars.locals.eks_service_group_instance_type + eks_service_group_min_size = local.stack_vars.locals.eks_service_group_min_size + eks_service_group_max_size = local.stack_vars.locals.eks_service_group_max_size + eks_service_group_desired_size = local.stack_vars.locals.eks_service_group_desired_size + bastion_iam_arn = "arn:aws:iam::${local.account_id}:user/system/bastion-user/${local.namespace}-bastion" + bastion_iam_username = "${local.namespace}-bastion" tags = merge( local.stack_vars.locals.tags, @@ -65,33 +67,36 @@ include { # These are the variables we have to pass in to use the module specified in the terragrunt configuration above inputs = { - account_id = local.account_id - shared_resource_identifier = local.shared_resource_identifier - aws_region = local.aws_region - root_domain = local.root_domain - namespace = local.namespace - private_subnet_ids = dependency.vpc.outputs.private_subnets - public_subnet_ids = dependency.vpc.outputs.public_subnets - vpc_id = dependency.vpc.outputs.vpc_id - kubernetes_cluster_version = local.kubernetes_version - eks_create_kms_key = local.eks_create_kms_key - eks_worker_group_instance_type = local.eks_worker_group_instance_type - eks_worker_group_min_size = local.eks_worker_group_min_size - eks_worker_group_max_size = local.eks_worker_group_max_size - eks_worker_group_desired_size = local.eks_worker_group_desired_size - eks_karpenter_group_instance_type = local.eks_karpenter_group_instance_type - eks_karpenter_group_min_size = local.eks_karpenter_group_min_size - eks_karpenter_group_max_size = local.eks_karpenter_group_max_size - eks_karpenter_group_desired_size = local.eks_karpenter_group_desired_size + account_id = local.account_id + shared_resource_identifier = local.shared_resource_identifier + aws_region = local.aws_region + root_domain = local.root_domain + namespace = local.namespace + private_subnet_ids = dependency.vpc.outputs.private_subnets + public_subnet_ids = dependency.vpc.outputs.public_subnets + vpc_id = dependency.vpc.outputs.vpc_id + kubernetes_cluster_version = local.kubernetes_version + eks_create_kms_key = local.eks_create_kms_key + eks_hosting_group_instance_type = local.eks_hosting_group_instance_type + eks_hosting_group_min_size = local.eks_hosting_group_min_size + eks_hosting_group_max_size = local.eks_hosting_group_max_size + eks_hosting_group_desired_size = local.eks_hosting_group_desired_size + eks_service_group_instance_type = local.eks_service_group_instance_type + eks_service_group_min_size = local.eks_service_group_min_size + eks_service_group_max_size = local.eks_service_group_max_size + eks_service_group_desired_size = local.eks_service_group_desired_size + bastion_iam_arn = local.bastion_iam_arn + tags = local.tags - tags = local.tags map_roles = [] map_users = [ { - userarn = "arn:aws:iam::621672204142:user/ci" - username = "ci" + userarn = local.bastion_iam_arn + username = local.bastion_iam_username groups = ["system:masters"] } + # ADD MORE USER IAMS HERE FOR ANYONE WHO NEEDS ACCESS TO + # kubectl AND k9s. ] } diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/kubernetes_cert_manager/terragrunt.hcl b/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/kubernetes_cert_manager/terragrunt.hcl index 259be4a0..2f7d0909 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/kubernetes_cert_manager/terragrunt.hcl +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/kubernetes_cert_manager/terragrunt.hcl @@ -15,7 +15,6 @@ locals { shared_resource_namespace = local.global_vars.locals.shared_resource_namespace aws_region = local.global_vars.locals.aws_region cert_manager_namespace = "cert-manager" - cert_manager_image_version = "{{ cookiecutter.cert_manager_image_version }}" services_subdomain = local.global_vars.locals.services_subdomain } @@ -79,7 +78,6 @@ inputs = { root_domain = local.root_domain aws_region = local.aws_region cert_manager_namespace = local.cert_manager_namespace - cert_manager_image_version = local.cert_manager_image_version namespace = local.shared_resource_namespace services_subdomain = local.services_subdomain } diff --git a/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/stack.hcl b/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/stack.hcl index 56572f7f..36f24f8b 100644 --- a/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/stack.hcl +++ b/{{cookiecutter.github_repo_name}}/terraform/stacks/{{cookiecutter.global_platform_shared_resource_identifier}}/stack.hcl @@ -15,7 +15,7 @@ locals { # AWS instance sizing mysql_instance_class = "{{ cookiecutter.mysql_instance_class }}" - mysql_allocated_storage ={{ cookiecutter.mysql_allocated_storage }} + mysql_allocated_storage = {{ cookiecutter.mysql_allocated_storage }} redis_node_type = "{{ cookiecutter.redis_node_type }}" @@ -37,21 +37,23 @@ locals { # # see: https://aws.amazon.com/ec2/instance-types/ #---------------------------------------------------------------------------- - kubernetes_version = "{{ cookiecutter.kubernetes_cluster_version }}" + kubernetes_version = "{{ cookiecutter.kubernetes_cluster_version }}" {% if cookiecutter.eks_create_kms_key|upper == "Y" -%} - eks_create_kms_key = true + eks_create_kms_key = true {% else -%} - eks_create_kms_key = false + eks_create_kms_key = false {% endif -%} - eks_worker_group_instance_type = "{{ cookiecutter.eks_worker_group_instance_type }}" - eks_worker_group_min_size = {{ cookiecutter.eks_worker_group_min_size }} - eks_worker_group_max_size = {{ cookiecutter.eks_worker_group_max_size }} - eks_worker_group_desired_size = {{ cookiecutter.eks_worker_group_desired_size }} - - eks_karpenter_group_instance_type = "{{ cookiecutter.eks_karpenter_group_instance_type }}" - eks_karpenter_group_min_size = {{ cookiecutter.eks_karpenter_group_min_size }} - eks_karpenter_group_max_size = {{ cookiecutter.eks_karpenter_group_max_size }} - eks_karpenter_group_desired_size = {{ cookiecutter.eks_karpenter_group_desired_size }} + + eks_service_group_instance_type = "{{ cookiecutter.eks_service_group_instance_type }}" + eks_service_group_min_size = {{ cookiecutter.eks_service_group_min_size }} + eks_service_group_max_size = {{ cookiecutter.eks_service_group_max_size }} + eks_service_group_desired_size = {{ cookiecutter.eks_service_group_desired_size }} + + eks_hosting_group_instance_type = "{{ cookiecutter.eks_hosting_group_instance_type }}" + eks_hosting_group_min_size = {{ cookiecutter.eks_hosting_group_min_size }} + eks_hosting_group_max_size = {{ cookiecutter.eks_hosting_group_max_size }} + eks_hosting_group_desired_size = {{ cookiecutter.eks_hosting_group_desired_size }} + tags = merge( local.global_vars.locals.tags,