Skip to content

Commit

Permalink
chore: Observability stack components used in examples recovered (#2045)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dadrus
dadrus authored Jan 7, 2025
1 parent 6c22355 commit f626c16
Show file tree
Hide file tree
Showing 59 changed files with 863 additions and 39,887 deletions.
2 changes: 1 addition & 1 deletion examples/docker-compose/quickstarts/docker-compose-traefik.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
services:
proxy:
image: traefik:3.0.0
image: traefik:3.2.3
ports:
- "9090:9090"
command: >
Expand Down
163 changes: 102 additions & 61 deletions examples/kubernetes/Justfile
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,23 @@
grafana_version := '2.7.15'
prometheus_version := '44.2.1'
loki_version := '2.8.9'
tempo_version := '0.16.8'
phlare_version := '0.1.2'
nginx_version := '4.10.1'
grafana_version := '4.9.0'
prometheus_version := '10.2.3'
loki_version := '6.24.0'
tempo_version := '1.16.0'
pyroscope_version := '1.10.0'
nginx_version := '4.11.3'
contour_version := '17.0.12'
emissary_version := '8.9.1'
haproxy_version := '0.14.6'
envoy_gw_version := 'v1.0.1'
traefik_version := '28.0.0'
metallb_version := '0.14.5'
certmanager_version := '1.15.3'
trustmanager_version := '0.12.0'
traefik_version := '33.2.1'
metallb_version := '0.14.9'
certmanager_version := '1.16.2'
trustmanager_version := '0.14.0'
certmanager_istio_csr_version := '0.12.0'
istio_version := '1.23.2'
minio_operator_version := '6.0.4'
alloy_version := '0.10.1'

cluster_name := 'demo-cluster'
cluster_name := 'heimdall-demo'
default_router := "contour"

setup-charts:
Expand All @@ -30,71 +32,87 @@ setup-charts:
helm repo add haproxy https://haproxy-ingress.github.io/charts
helm repo add traefik https://traefik.github.io/charts
helm repo add istio https://istio-release.storage.googleapis.com/charts
helm repo add minio-operator https://operator.min.io
helm repo update

## Installs Alloy
install-alloy:
kubectl apply -f observability/namespace.yaml

kubectl create configmap --namespace monitoring alloy-config "--from-file=config.alloy=./observability/alloy/config.alloy"

helm upgrade --install alloy grafana/alloy \
-n monitoring --create-namespace \
--version {{alloy_version}} \
-f observability/alloy/values.yaml \
--wait

## Installs Grafana
install-grafana: setup-charts
install-grafana:
#!/usr/bin/env bash
lb_address=$(kubectl get svc -A -o json | jq -r '.items[] | select(.spec.type == "LoadBalancer").status.loadBalancer.ingress[0].ip')
kubectl apply -f observability/namespace.yaml

helm upgrade --install grafana bitnami/grafana-operator \
-n monitoring --create-namespace \
--set grafana.ingress.hostname="grafana.${lb_address}.nip.io" \
--set grafana.config.server.root_url="https://grafana.${lb_address}.nip.io" \
--values grafana/helm-values.yaml \
--values observability/grafana/helm-values.yaml \
--version {{grafana_version}} \
--wait
while : ; do
kubectl rollout -n monitoring status deployment grafana-deployment && break
sleep 2
done

kubectl apply -f observability/grafana/data-sources/prometheus.yaml
kubectl apply -f observability/grafana/data-sources/loki.yaml
kubectl apply -f observability/grafana/data-sources/tempo.yaml
kubectl apply -f observability/grafana/data-sources/pyroscope.yaml

# Installs Prometheus
install-prometheus: setup-charts
helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \
install-prometheus:
kubectl apply -f observability/namespace.yaml

helm upgrade --install prometheus bitnami/kube-prometheus \
-n monitoring --create-namespace \
--set grafana.enabled=false \
--set prometheus.enableRemoteWriteReceiver=true \
--version {{prometheus_version}} \
--wait
kubectl apply -f grafana/data-sources/prometheus.yaml

install-monitoring-storage:
# install minio tenant and wait until the first pod of the statefulset is running
kubectl apply -f observability/namespace.yaml
kubectl apply -f observability/minio/tenant.yaml

sleep 10
kubectl wait --timeout=90s --for=create pod -l statefulset.kubernetes.io/pod-name=minio-pool-0-0 -n monitoring
kubectl wait --timeout=90s --for=condition=ready pod -l statefulset.kubernetes.io/pod-name=minio-pool-0-0 -n monitoring

# Installs Loki
install-loki: setup-charts
helm upgrade --install loki grafana/loki-stack \
install-loki:
kubectl apply -f observability/namespace.yaml

helm upgrade --install loki grafana/loki \
-n monitoring --create-namespace \
--version {{loki_version}} \
--set grafana.enabled=false \
--wait
kubectl apply -f grafana/data-sources/loki.yaml
-f observability/loki/values.yaml

# Installs Tempo
install-tempo: setup-charts
install-tempo:
kubectl apply -f observability/namespace.yaml

helm upgrade --install tempo grafana/tempo \
-n monitoring --create-namespace \
--set tempo.searchEnabled=true \
--version {{tempo_version}} \
--wait
kubectl apply -f grafana/data-sources/tempo.yaml
-f observability/tempo/values.yaml

# Installs Phlare
install-phlare: setup-charts
helm upgrade --install phlare grafana/phlare \
-n monitoring --create-namespace \
--version {{phlare_version}} \
--wait
kubectl apply -f grafana/data-sources/phlare.yaml
# Installs Pyroscope
install-pyroscope:
kubectl apply -f observability/namespace.yaml

install-dashboards:
#!/usr/bin/env bash
for f in grafana/dashboards/*.yaml ; do
kubectl apply -f $f
done
install-heimdall-pod-monitor:
kubectl apply -f quickstarts/heimdall/pod_monitor.yaml
helm upgrade --install pyroscope grafana/pyroscope \
-n monitoring --create-namespace \
--set serviceMonitor.enabled=true \
--version {{pyroscope_version}}

install-observability-stack: install-grafana install-prometheus install-loki install-tempo install-phlare install-dashboards install-heimdall-pod-monitor
install-observability-stack: install-monitoring-storage install-prometheus install-loki install-tempo install-pyroscope install-grafana install-alloy

install-nginx-ingress-controller global_ext_auth="true":
#!/usr/bin/env bash
Expand Down Expand Up @@ -197,8 +215,10 @@ install-istio with_ingress_gw="true":

install-lb:
#!/usr/bin/env bash
kubectl apply -f metallb/namespace.yaml
helm upgrade --install metallb metallb/metallb \
-n metallb-system --create-namespace \
-n metallb-system \
--version {{metallb_version}} \
--wait

Expand All @@ -209,12 +229,15 @@ install-cert-manager:
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v{{certmanager_version}}/cert-manager.crds.yaml

kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.0/standard-install.yaml

helm upgrade --install cert-manager jetstack/cert-manager \
-n cert-manager --create-namespace \
--version {{certmanager_version}} \
--set installCRDs=false \
--set featureGates='AdditionalCertificateOutputFormats=true,ExperimentalGatewayAPISupport=true' \
--set featureGates='AdditionalCertificateOutputFormats=true' \
--set webhook.featureGates='AdditionalCertificateOutputFormats=true' \
--set "extraArgs={--enable-gateway-api}" \
--wait

kubectl apply -n cert-manager -f cert-manager/ca.yaml
Expand All @@ -229,6 +252,21 @@ install-cert-manager:

kubectl apply -n cert-manager -f cert-manager/cacerts_bundle.yaml

install-minio-operator:
#!/usr/bin/env bash
kubectl apply -f minio-operator/namespace.yaml
kubectl apply -f minio-operator/tls-certificate.yaml

helm upgrade --install operator minio-operator/operator \
-n minio-operator \
--version {{minio_operator_version}} \
-f minio-operator/values.yaml \
--wait

kubectl -n minio-operator get secrets sts-tls -o=jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/ca.crt
kubectl -n minio-operator create secret generic operator-ca-tls-tenants --from-file=/tmp/ca.crt

install-echo-service router=default_router:
kustomize build quickstarts/demo-app/overlays/{{router}} | kubectl apply -f -

Expand Down Expand Up @@ -270,32 +308,35 @@ install-heimdall router=default_router:
kubectl apply -f quickstarts/heimdall/istio-authorization-policy.yaml
fi

kubectl apply -f quickstarts/heimdall/pod_monitor.yaml

create-cluster:
kind create cluster --config kind/kind.yaml --name {{cluster_name}}

setup-cluster: create-cluster setup-charts install-lb install-cert-manager
setup-cluster: create-cluster setup-charts install-lb install-cert-manager install-minio-operator install-observability-stack

install-ngnix-decision-demo: setup-cluster (install-nginx-ingress-controller "false") (install-heimdall "nginx") (install-echo-service "nginx-route-based")
install-ngnix-demo: setup-cluster (install-nginx-ingress-controller "false") (install-heimdall "nginx") (install-echo-service "nginx-route-based")

install-ngnix-global-decision-demo: setup-cluster install-nginx-ingress-controller (install-heimdall "nginx") (install-echo-service "nginx-global")
install-ngnix-global-demo: setup-cluster install-nginx-ingress-controller (install-heimdall "nginx") (install-echo-service "nginx-global")

install-contour-decision-demo: setup-cluster install-contour-ingress-controller (install-heimdall "contour") (install-echo-service "contour")
install-contour-demo: setup-cluster install-contour-ingress-controller (install-heimdall "contour") (install-echo-service "contour")

install-haproxy-decision-demo: setup-cluster install-haproxy-ingress-controller (install-heimdall "haproxy") (install-echo-service "haproxy")
install-haproxy-demo: setup-cluster install-haproxy-ingress-controller (install-heimdall "haproxy") (install-echo-service "haproxy")

install-emissary-decision-demo: setup-cluster install-emissary-ingress-controller (install-heimdall "emissary") (install-echo-service "emissary")
install-emissary-demo: setup-cluster install-emissary-ingress-controller (install-heimdall "emissary") (install-echo-service "emissary")

install-envoygw-decision-demo: setup-cluster install-envoy-gateway (install-heimdall "envoygw") (install-echo-service "envoygw")
install-envoygw-demo: setup-cluster install-envoy-gateway (install-heimdall "envoygw") (install-echo-service "envoygw")

install-traefik-ingress-decision-demo: setup-cluster install-traefik (install-heimdall "traefik") (install-echo-service "traefik-ingress")
install-traefik-ingress-demo: setup-cluster install-traefik (install-heimdall "traefik") (install-echo-service "traefik-ingress")

install-traefik-ingress-route-decision-demo: setup-cluster (install-traefik "false") (install-heimdall "traefik") (install-echo-service "traefik-ingress-route")
install-traefik-ingress-route-demo: setup-cluster (install-traefik "false") (install-heimdall "traefik") (install-echo-service "traefik-ingress-route")

install-traefik-gw-decision-demo: setup-cluster install-traefik (install-heimdall "traefik") (install-echo-service "traefik-gw")
install-traefik-gw-demo: setup-cluster install-traefik (install-heimdall "traefik") (install-echo-service "traefik-gw")

install-istio-ingress-gw-decision-demo: setup-cluster install-istio (install-heimdall "istio-ingress-gw") (install-echo-service "istio-ingress-gw")
install-istio-ingress-gw-demo: setup-cluster install-istio (install-heimdall "istio-ingress-gw") (install-echo-service "istio-ingress-gw")

install-istio-gw-decision-demo: setup-cluster (install-istio "false") (install-heimdall "istio-gw") (install-echo-service "istio-gw")
install-istio-gw-demo: setup-cluster (install-istio "false") (install-heimdall "istio-gw") (install-echo-service "istio-gw")

delete-cluster:
kind delete clusters {{cluster_name}}

38 changes: 33 additions & 5 deletions examples/kubernetes/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Kubernetes Quickstarts

This directory contains working examples described in the getting started, as well as in the integration guides of the documentation. The demonstration of the decision operation mode is done via integration with the corresponding ingress controllers. As of now, these are [Contour](https://projectcontour.io), the [NGINX Ingress Controller](https://docs.nginx.com/nginx-ingress-controller/) and [HAProxy Ingress Controller](https://haproxy-ingress.github.io/).
This directory contains working examples described in the getting started, as well as in the integration guides of the documentation. The demonstration of the decision operation mode is done via integration with the different Ingress Controller, respectively Gateway API implementations.

**Note:** The main branch may have breaking changes (see pending release PRs for details under https://github.com/dadrus/heimdall/pulls) which would make the usage of the referenced heimdall images impossible (even though the configuration files and rules reflect the latest changes). In such situations you'll have to use the `dev` image, build a heimdall image by yourself and update the setups to use it, or switch to a tagged (released) version.

Expand All @@ -18,19 +18,35 @@ To be able to install and play with quickstarts, you need

# Install the demo

Depending on the Ingress Controller you want to install the demo for, execute
Depending on the Ingress Controller/Gateway API you want to install the demo for, execute

```bash
just install-<ingress controller>-decision-demo
just install-<setup-type>-demo
```

with `<ingress controller>` being either `contour`, `nginx`, `haproxy`, or `emissary`. That command line will install and set up a kind based k8s cluster locally including all required services and configuring the used ingress controller to forward all incoming requests to heimdall as external authorization middleware. Depending on the response from heimdall the ingress controller will either forward the request to the upstream service (in that case a simple echo service), or directly respond with an error from heimdall to the client.
with `<setup-type>` being one of the following options:

* `ngnix` - integration with heimdall happens using annotations on Ingress Resource level.
* `ngnix-global` - heimdall is integrated globally.
* `contour` - heimdall is integrated globally.
* `haproxy` - integration with heimdall happens using annotations on Ingress Resource level.
* `emissary` - heimdall is integrated globally.
* `envoygw` - heimdall is integrated on a gateway level.
* `traefik-ingress` - heimdall is integrated globally. Standard k8s Ingress Resource is used four routing.
* `traefik-ingress-route` - heimdall is integrated as Middleware resource, which is then referenced in a Traefik's IngressRoute resource of the upstream service.
* `traefik-gw` - heimdall is integrated globally.
* `istio-ingress-gw` - heimdall is integrated globally. Routing happens using Istio's VirtualService resource.
* `istio-gw` - heimdall is integrated on a gateway level.

That command line will install and set up a kind based k8s cluster locally including all required services and configuring the used ingress controller, gateway api, respectively a vendor specific router implementation to forward incoming requests to heimdall as external authentication/authorization middleware. Depending on the response from heimdall the router implementation will either forward the request to the upstream service (in that case a simple echo service), or directly respond with an error from heimdall to the client. The above setup does also include an observability stack based on grafana components (Alloy, Loki, Tempo, Prometheus, Pyroscope, Grafana)

Depending on your internet connection, it may take some minutes. So, maybe it's time to grab some coffee :)

**Note:** It might happen that the installation of the MetalLB fails due to a bad IP range configured. In such case, just delete the cluster, change `KIND_SUBNET=$(docker network inspect kind -f "{{(index .IPAM.Config 0).Subnet}}")` to `KIND_SUBNET=$(docker network inspect kind -f "{{(index .IPAM.Config 1).Subnet}}")` or similar (to get an IPv4 subnet) in the `metallb/configure.sh` file and restart the setup of the demo.

# Play with the demo

Check which IP is used for the ingress-controller and set a variable to that value. You can easily achieve this by querying the LB IP address of the used ingress controller with e.g.
Check which IP is used for the router implementation and set a variable to that value. You can easily achieve this by querying the LB IP address of the used router with e.g.

```bash
export SERVICE_IP=$(kubectl get svc -n nginx-ingress-controller nginx-ingress-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
Expand All @@ -53,6 +69,18 @@ and check the responses.

Please note: Since nginx does not support 302 response codes from an external auth service, the call to `https://echo-app.local/redir/foo` will result in a 500 error code returned by nginx.

# Observe Telemetry Data

Establish port forwarding for the grafana service with

```bash
kubectl -n monitoring port-forward service/grafana-grafana-operator-grafana-service 3000:3000
```

Open http://grafana.127.0.0.1.nip.io:3000 in your browser to access it. Username is set to `admin` and password is `monitoring`.

**Note:** If a traefik based setup is used, you'll also be able to see the entire request traces. For all other setups, tracing is limited to heimdall only (configuration of other routers for OTEL is a TODO).

# Delete the demo

```bash
Expand Down
Loading

0 comments on commit f626c16

Please sign in to comment.