This repo contains the PoC for a Kafka Cluster running on aks cluster.
- Create an Azure Service Principal
## The keyvault is already configured in ZONDEV azure subscription
SERVICE_PRINCIPAL=kafka-poc-sp
SP_PASSWORD=$(az ad sp create-for-rbac --name http://$SERVICE_PRINCIPAL --query password --output tsv)
SP_ID=$(az ad sp show --id http://$SERVICE_PRINCIPAL --query appId --output tsv)
az group create -n <key_vault_resource_group_name> -l northeurope
az group update -n <key_vault_resource_group_name> --set tags.env=dev --set tags.cluster=kafka-poc
az keyvault create -n <key_vault_name> -g <key_vault_resource_group_name> -l northeurope
ssh-keygen -f ~/.ssh/id_rsa_kafka_poc
az keyvault secret set --vault-name <key_vault_name> --name <private_key_name> -f ~/.ssh/id_rsa_kafka_poc.pub > /dev/null
az keyvault secret set --vault-name <key_vault_name> --name <sp_id_name> --value $SP_ID > /dev/null
az keyvault secret set --vault-name <key_vault_name> --name <sp_password_name> --value $SP_PASSWORD > /dev/null
- Create an AKS Cluster
## Clone this repo to your local envrionment
cd system/
terraform init
terraform validate
terraform plan --out kafkapoc.plan
terraform apply kafkapoc.plan
az aks get-credentials --resource-group <resource_group_name_for_aks_cluster> --name kafka-poc
## If you want k8s dashboard access
az aks get-credentials --resource-group <resource_group_name_for_aks_cluster> --name kafka-poc --admin
- Install Zookeeper
kubectl create namespace zookeeper
helm repo add banzaicloud-stable https://kubernetes-charts.banzaicloud.com/
helm repo update
helm install zookeeper --namespace=zookeeper banzaicloud-stable/zookeeper-operator
cat <<EOF | kubectl apply -f -
apiVersion: zookeeper.pravega.io/v1beta1
kind: ZookeeperCluster
metadata:
name: zookeeper
namespace: zookeeper
spec:
replicas: 3
EOF
- Install Kafka Operator & Cluster
kubectl create namespace kafka
helm install kafka-operator --namespace=kafka banzaicloud-stable/kafka-operator
kubectl create -n kafka -f kafka/00-kafkacluster.yaml
- Install NGINX Ingress Controller
kubectl create namespace ingress-nginx
helm install nginx --namespace ingress-nginx stable/nginx-ingress --set controller.kind=DaemonSet
## Check if the nginx service has an external IP address
kubectl get service nginx-ingress-controller --namespace ingress-nginx -o wide
- Install Prometheus Operator
kubectl create namespace monitoring
## Generate the prometheus crd's
./genpromcrds.sh
helm install prometheus --namespace monitoring stable/prometheus-operator -f prometheus/values.yaml --set prometheusOperator.createCustomResource=false
- Install Kafka & Cruise Control ServiceMonitors
kubectl apply -f kafka/10-kafka-servicemonitor.yaml
kubectl apply -f kafka/20-curisecontrol-servicemonitor.yaml
- Install Kafka AlertRules
kubectl apply -f kafka/30-kafka-alertrules.yaml
- Install istio
You must have the latest
istioctl
binary on your path.
istioctl manifest apply -f istio/istioctl-values.yaml
# check that it's working
kubectl -n isito-system get pods
- Start loadtesting with sangrenel
Just apply what's on the
sangrenel.yaml
file.
terraform destroy