Add helm as K8s resources IaC (kubeflow#921)

aws/IaC/CDK/test-infra/Makefile

@@ -0,0 +1,16 @@
+
+SHELL := /bin/bash
+
+# Lint python codes
+lint:
+	black -r ./
+
+# Entry-point for cdk logging with a fresh new virtual environment
+cdk-verify:
+	( \
+		python3 -m venv .venv; \
+		source .venv/bin/activate; \
+		pip3 install -r requirements.txt; \
+		echo "To validate if dev environment works:"; \
+		cdk synth; \
+	)

aws/IaC/CDK/test-infra/README.md

@@ -1,8 +1,7 @@
-
-# Infrastructure as Code
+# [AWS Resources] Infrastructure as Code
 
 We utilize [CDK](https://github.com/aws/aws-cdk) which
-serves as the tool of Infrastructure as Code (IAC).
+serves as the AWS resources tool of Infrastructure as Code (IAC).
 
 ## Project Architecture
 We define AWS resources in [static_config](https://github.com/kubeflow/testing/tree/master/aws/IaC/CDK/test-infra/config/static_config),

aws/IaC/CDK/test-infra/app.py

@@ -11,13 +11,11 @@
     CDKProwClusterStack,
     CDKTektonClusterStack,
     CDKWorkerClusterStack,
+    env,
 )
 
 app = core.App()
 
-# Add CDK Environment variable
-env = core.Environment(account="809251082950", region="us-west-2")
-
 # Reference CDK Stacks
 CDKS3Stack(app, "cdk-s3")
 CDKECRStack(app, "cdk-ecr")

aws/IaC/CDK/test-infra/config/static_config/ACM_Resources.py

@@ -8,8 +8,13 @@
     # "cdk-id": ("domain-name", "validation-method")
     # Example:
 
-    # "cdk-poc": (
-    #     "cdk-poc.kubeflow-testing.com",
-    #     aws_certificatemanager.ValidationMethod.DNS,
-    # )
+    "cdk-poc-grafana-dashboard": (
+        "dashboard.cdk-poc.kubeflow-testing.com",
+        aws_certificatemanager.ValidationMethod.DNS,
+    ),
+
+    "cdk-poc-prow-dashboard": (
+        "prow.cdk-poc.kubeflow-testing.com",
+        aws_certificatemanager.ValidationMethod.DNS,
+    )
 }

aws/IaC/CDK/test-infra/config/static_config/EKS_Resources.py

@@ -9,10 +9,10 @@
     # "cdk-id": (("cluster-name", "kubernetes-version"), (instance_type, min_size, max_size, desired_size, disk_size))
     # Example:
 
-    # "prow_cluster": (
-    #     ("cdk-poc-prow", aws_eks.KubernetesVersion.V1_18),
-    #     (aws_ec2.InstanceType("m5.large"), 2, 4, 2, 20),
-    # ),
+    "prow_cluster": (
+        ("cdk-poc-prow", aws_eks.KubernetesVersion.V1_18),
+        (aws_ec2.InstanceType("m5.large"), 2, 4, 2, 20),
+    ),
     # "argo_cluster": (
     #     ("cdk-poc-argo", aws_eks.KubernetesVersion.V1_18),
     #     (aws_ec2.InstanceType("m5.large"), 2, 4, 2, 20),

aws/IaC/CDK/test-infra/config/static_config/S3_Resources.py

@@ -11,4 +11,7 @@
     # "status_reconciler_backup_bucket": "cdk-poc-status-reconciler-backup",
     # "daily_worker_prow_logs_bucket": "cdk-poc-daily-worker-prow-logs",
     # "daily_worker_status_reconciler_backup_bucket": "cdk-poc-daily-worker-status-reconciler-backup",
+    "status_reconciler_backup_bucket": "cdk-poc-status-reconciler-backup",
+    "prow_logs_bucket": "cdk-poc-prow-logs",
+    "tide_bucket": "cdk-poc-tide",
 }

aws/IaC/CDK/test-infra/requirements.txt

@@ -1,39 +1,44 @@
 astroid==2.4.2
 attrs==20.3.0
-aws-cdk.assets==1.89.0
-aws-cdk.aws-applicationautoscaling==1.89.0
+aws-cdk.assets==1.91.0
+aws-cdk.aws-apigateway==1.91.0
+aws-cdk.aws-apigatewayv2==1.91.0
+aws-cdk.aws-applicationautoscaling==1.91.0
 aws-cdk.aws-autoscaling==1.89.0
-aws-cdk.aws-autoscaling-common==1.89.0
-aws-cdk.aws-certificatemanager==1.89.0
-aws-cdk.aws-cloudformation==1.89.0
-aws-cdk.aws-cloudwatch==1.89.0
-aws-cdk.aws-codeguruprofiler==1.89.0
-aws-cdk.aws-ec2==1.89.0
-aws-cdk.aws-ecr==1.89.0
-aws-cdk.aws-ecr-assets==1.89.0
-aws-cdk.aws-efs==1.89.0
+aws-cdk.aws-autoscaling-common==1.91.0
+aws-cdk.aws-certificatemanager==1.91.0
+aws-cdk.aws-cloudformation==1.91.0
+aws-cdk.aws-cloudfront==1.91.0
+aws-cdk.aws-cloudwatch==1.91.0
+aws-cdk.aws-codeguruprofiler==1.91.0
+aws-cdk.aws-cognito==1.91.0
+aws-cdk.aws-ec2==1.91.0
+aws-cdk.aws-ecr==1.91.0
+aws-cdk.aws-ecr-assets==1.91.0
+aws-cdk.aws-efs==1.91.0
 aws-cdk.aws-eks==1.89.0
-aws-cdk.aws-elasticloadbalancing==1.89.0
-aws-cdk.aws-elasticloadbalancingv2==1.89.0
-aws-cdk.aws-events==1.89.0
-aws-cdk.aws-iam==1.89.0
-aws-cdk.aws-kms==1.89.0
-aws-cdk.aws-lambda==1.89.0
-aws-cdk.aws-logs==1.89.0
-aws-cdk.aws-route53==1.89.0
-aws-cdk.aws-s3==1.89.0
-aws-cdk.aws-s3-assets==1.89.0
-aws-cdk.aws-sns==1.89.0
-aws-cdk.aws-sqs==1.89.0
-aws-cdk.aws-ssm==1.89.0
+aws-cdk.aws-elasticloadbalancing==1.91.0
+aws-cdk.aws-elasticloadbalancingv2==1.91.0
+aws-cdk.aws-events==1.91.0
+aws-cdk.aws-iam==1.91.0
+aws-cdk.aws-kms==1.91.0
+aws-cdk.aws-lambda==1.91.0
+aws-cdk.aws-logs==1.91.0
+aws-cdk.aws-route53==1.91.0
+aws-cdk.aws-route53-targets==1.91.0
+aws-cdk.aws-s3==1.91.0
+aws-cdk.aws-s3-assets==1.91.0
+aws-cdk.aws-sns==1.91.0
+aws-cdk.aws-sqs==1.91.0
+aws-cdk.aws-ssm==1.91.0
 aws-cdk.cdk==0.36.1
-aws-cdk.cloud-assembly-schema==1.89.0
-aws-cdk.core==1.89.0
-aws-cdk.custom-resources==1.89.0
-aws-cdk.cx-api==1.89.0
+aws-cdk.cloud-assembly-schema==1.91.0
+aws-cdk.core==1.91.0
+aws-cdk.custom-resources==1.91.0
+aws-cdk.cx-api==1.91.0
 aws-cdk.lambda-layer-awscli==1.89.0
 aws-cdk.lambda-layer-kubectl==1.89.0
-aws-cdk.region-info==1.89.0
+aws-cdk.region-info==1.91.0
 cattrs==1.1.2
 constructs==3.3.21
 isort==5.7.0

aws/IaC/CDK/test-infra/test_infra/test_infra_stack.py

@@ -14,6 +14,9 @@
 from config.static_config.IAM_Resources import IAM_Role_List
 from config.static_config.ACM_Resources import ACM_Certs_List
 
+# Add CDK Environment variable
+env = core.Environment(account="809251082950", region="us-west-2")
+
 
 class CDKS3Stack(core.Stack):
     """CDK Class for S3_Resources.py buckets"""

aws/IaC/README.md

@@ -0,0 +1,55 @@
+# Infrastructure as Code
+
+Test infrastructure is set up as two layers of resources:
+
+1. AWS resources set up via [CDK](CDK/test-infra/README.md)
+2. Kubernetes resources set up via [Helm](helm/test-infra/README.md)
+
+## Set up Prow Cluster as Test Infrastructure
+
+### Set up AWS resources
+
+1. Set up ACM certificates for domain names
+
+Add ACM certificates in [ACM_Resources.py](./CDK/test-infra/config/static_config/ACM_Resources.py), take a note of 
+ <Prow Dashboard Domain Name> and <Prow Dashboard Domain Name ACM Certificates>
+
+2. Set up S3 buckets
+
+Add S3 buckets in [S3_Resources.py](./CDK/test-infra/config/static_config/S3_Resources.py), take a note of
+<Status Reconciler S3 Bucket>, <Prow Logs S3 Bucket>, and <Tide S3 Bucket>.
+
+3. Set up EKS clusters
+
+Add EKS clusters in [EKS_Resources.py](./CDK/test-infra/config/static_config/EKS_Resources.py), take a note of
+<EKS Cluster Name>
+
+Deploy AWS resources via `cdk deploy`, more details [here](./CDK/test-infra/README.md)
+
+### Set up Tokens
+
+1. Create Github Token
+
+Create a [personal access token](https://github.com/settings/tokens) for the GitHub bot account, adding the following scopes
+* Must have the `public_repo` and `repo:status` scopes
+* Add the `repo` scope if you plan on handing private repos
+* Add the `admin:org_hook` scope if you plan on handling a github org
+
+Take a note of <Github Token>
+
+2. Create Github Secret (Hmac Token)
+
+You will need two secrets to talk to GitHub. The `hmac-token` is the token that you give
+ to GitHub for validating webhooks. Generate it using any reasonable randomness-generator, 
+```shell script
+openssl rand -hex 20
+```
+
+Take a note of <Hmac Token>
+
+### Set up Kubernetes resources
+
+In general, follow the [doc](./helm/test-infra/README.md) and fill in parameters with above notes
+
+Note: go to [AWS Route53 console](https://console.aws.amazon.com/route53/v2/hostedzones#) to
+ configure DNS record when you use any of domain names.

aws/IaC/helm/test-infra/README.md

@@ -0,0 +1,55 @@
+# [Kubernetes Resources] Infrastructure as Code
+
+We utilize [helm](https://github.com/helm/helm) which
+serves as the tool of Kubernetes resources Infrastructure as Code (IAC).
+
+## Deploy Helm Charts
+This section describes how to deploy helm charts on existing Kubernetes cluster
+
+### Deploy ALB Ingress Controller
+
+```shell script
+helm install alb-ingress-controller ./alb-ingress-controller \
+		--set clusterName=<EKS Cluster Name> \
+		--set secret.keyValue=<AWS Public Key> \
+		--set secret.secretValue=<AWS Secret key>
+```
+
+### Deploy Grafana and Prometheus
+
+```shell script
+kubectl create namespace prometheus
+kubectl create namespace grafana
+
+helm install prometheus ./prometheus \
+    --namespace prometheus \
+    --set alertmanager.persistentVolume.storageClass="gp2" \
+    --set server.persistentVolume.storageClass="gp2"
+
+helm install grafana ./grafana \
+    --namespace grafana \
+    --set persistence.storageClassName="gp2" \
+    --set persistence.enabled=true \
+    --set adminPassword='KFCI!Awesome' \
+    --values ./grafana/environment/grafana.yaml \
+    --set service.type=LoadBalancer
+```
+
+### Deploy Prow
+
+```shell script
+kubectl create namespace prow 
+kubectl create namespace test-pods
+
+helm install prow ./prow \
+		--set s3Buckets.statusReconciler=<Status Reconciler S3 Bucket> \
+		--set s3Buckets.prowLogs=<Prow Logs S3 Bucket> \
+		--set s3Buckets.tide=<Tide S3 Bucket> \
+		--set secret.keyValue=<AWS Public Key> \
+		--set secret.secretValue=<AWS Secret key> \
+		--set managedGithubOrg=<Prow Managed Github Organization> \
+		--set prowDashboardDomainName=<Prow Dashboard Domain Name> \
+		--set prowDashboardDomainNameACMCerts=<Prow Dashboard Domain Name ACM Certificates> \
+		--set tokens.githubToken=<Github Token> \
+		--set tokens.hmacToken=<Hmac Token>
+```

aws/IaC/helm/test-infra/alb-ingress-controller/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

aws/IaC/helm/test-infra/alb-ingress-controller/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: alb-ingress-controller
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

aws/IaC/helm/test-infra/alb-ingress-controller/templates/NOTES.txt

@@ -0,0 +1 @@
+Kubeflow Test Infra ALB Ingress Controller installed!

aws/IaC/helm/test-infra/alb-ingress-controller/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "alb-ingress-controller.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "alb-ingress-controller.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "alb-ingress-controller.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "alb-ingress-controller.labels" -}}
+helm.sh/chart: {{ include "alb-ingress-controller.chart" . }}
+{{ include "alb-ingress-controller.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "alb-ingress-controller.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "alb-ingress-controller.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "alb-ingress-controller.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "alb-ingress-controller.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}