From d6ed0bf1baac19e1e3099846a65b21d2692cc7c6 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Fri, 2 Sep 2022 21:03:13 +0300 Subject: [PATCH] Add Flux guide Continuous deployment with Kustomizer and Flux Signed-off-by: Stefan Prodan --- docs/github-actions.md | 21 ++-- docs/guides/fluxcd.md | 251 +++++++++++++++++++++++++++++++++++++++++ docs/index.md | 28 +++-- mkdocs.yml | 1 + 4 files changed, 279 insertions(+), 22 deletions(-) create mode 100644 docs/guides/fluxcd.md diff --git a/docs/github-actions.md b/docs/github-actions.md index 4707f981..6a185737 100644 --- a/docs/github-actions.md +++ b/docs/github-actions.md @@ -31,16 +31,16 @@ on: - 'v*' env: - ARTIFACT: oci://ghcr.io//${{github.repository_owner}}/${{github.event.repository.name}} + ARTIFACT: oci://ghcr.io/${{github.repository_owner}}/${{github.event.repository.name}} jobs: kustomizer: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} @@ -75,20 +75,20 @@ permissions: packages: write # needed for GHCR access env: - ARTIFACT: oci://ghcr.io//${{github.repository_owner}}/${{github.event.repository.name}} + ARTIFACT: oci://ghcr.io/${{github.repository_owner}}/${{github.event.repository.name}} jobs: kustomizer: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup cosign uses: sigstore/cosign-installer@main - name: Setup kustomizer uses: stefanprodan/kustomizer/action@main - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} @@ -118,20 +118,20 @@ on: default: 'latest' env: - ARTIFACT: oci://ghcr.io//${{github.repository_owner}}/${{github.event.repository.name}} + ARTIFACT: oci://ghcr.io/${{github.repository_owner}}/${{github.event.repository.name}} jobs: kustomizer: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup kubeconfig uses: azure/k8s-set-context@v1 with: kubeconfig: ${{ secrets.KUBE_CONFIG }} - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} @@ -166,7 +166,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup kubeconfig uses: azure/k8s-set-context@v1 with: @@ -184,3 +184,4 @@ jobs: --revision=${{ github.sha }} \ -f ./deploy --prune --wait ``` + diff --git a/docs/guides/fluxcd.md b/docs/guides/fluxcd.md new file mode 100644 index 00000000..b78661ae --- /dev/null +++ b/docs/guides/fluxcd.md @@ -0,0 +1,251 @@ +# Continuous deployment with Kustomizer and Flux + +This guide shows you how to continuously deploy applications to Kubernetes +clusters with Flux using OCI artifacts produced by Kustomizer. + +This guide offers a better alternative to deploying applications +with GitHub CI (as showcased in the [Deploy from Git](deploy-from-git.md) guide). +Instead of connecting to each Kubernetes cluster from GitHub Actions, +we'll use CI for pushing OCI artifacts to a container registry, and from +there, the Kubernetes clusters (running Flux) will drive the app deployment +themselves. One major advantage to this approach, is that you no longer have +to deal with securing the access from CI to your production systems. + +## Before you begin + +- Install Kustomizer and the Flux CLI. +- Have a Kubernetes cluster version 1.20 or newer. +- Have a GitHub account. + +!!! info "Install with Homebrew" + + ``` + brew install stefanprodan/tap/kustomizer fluxcd/tap/flux + ``` + +## Login to GitHub Container Registry + +Export you GitHub username: + +```shell +export GITHUB_USER="YOUR-GITHUB-USERNAME" +``` + +Generate a [personal access token (PAT)](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry) +with read and write access to GitHub Container Registry. + +```shell +export GITHUB_TOKEN="YOUR-GITHUB-PAT" +``` + +Use the token to sign in to the container registry service at ghcr.io: + +```console +$ echo $GITHUB_TOKEN | docker login ghcr.io -u ${GITHUB_USER} --password-stdin +> Login Succeeded +``` + +!!! info "Other container registries" + + Besides GHCR, both Kustomizer and Flux are compatible with Docker Hub, ACR, ECR, GCR, + self-hosted Docker Registry v2 and any other registry that conforms + to the [Open Container Initiative](https://opencontainers.org/). + +## Clone the demo app repository + +Clone the Kustomizer Git repository locally: + +```bash +git clone https://github.com/stefanprodan/kustomizer +cd kustomizer +``` + +You'll be using a sample web application composed of two [podinfo](https://github.com/stefanprodan/podinfo) +instances called `frontend` and `backend`, and a redis instance called `cache`. +The web application's Kubernetes configuration is located at `./examples/demo-app`. + +## Publish the app manifests + +Export the repository URL and app version: + +```shell +export APP_REPO="ghcr.io/${GITHUB_USER}/kustomizer-demo-app" +export APP_VERSION="1.0.0" +``` + +Build and push the app manifests to GitHub container registry: + +```console +$ kustomizer push artifact oci://${APP_REPO}:${APP_VERSION} \ + -k ./examples/demo-app \ + --source="$(git config --get remote.origin.url)" \ + --revision="$(git branch --show-current)/$(git rev-parse HEAD)" +building manifests... +Namespace/kustomizer-demo-app +ConfigMap/kustomizer-demo-app/redis-config-bd2fcfgt6k +Service/kustomizer-demo-app/backend +Service/kustomizer-demo-app/cache +Service/kustomizer-demo-app/frontend +Deployment/kustomizer-demo-app/backend +Deployment/kustomizer-demo-app/cache +Deployment/kustomizer-demo-app/frontend +HorizontalPodAutoscaler/kustomizer-demo-app/backend +HorizontalPodAutoscaler/kustomizer-demo-app/frontend +pushing image ghcr.io/stefanprodan/kustomizer-demo-app:1.0.0 +published digest ghcr.io/stefanprodan/kustomizer-demo-app@sha256:91d2bd8e0f1620e17e9d4c308ab87903644a952969d8ff52b601be0bffdca096 +``` + +Tag the config image as latest: + +```shell +kustomizer tag artifact oci://${APP_REPO}:${APP_VERSION} latest +``` + +## Configure Flux to deploy the app + +First install Flux on your cluster with: + +```shell +flux install +``` + +!!! info "GitOps" + + For Flux to manage your cluster in a GitOps manner, + you could use the [flux bootstrap](https://fluxcd.io/flux/installation/#bootstrap) instead of `flux install`. + +Create an image pull secret for ghcr.io with: + +```shell +flux create secret oci ghcr-auth \ + --url=ghcr.io \ + --username=flux \ + --password=${GITHUB_TOKEN} +``` + +Create a Flux `OCIRepository` for pulling the latest artifact from GitHub container registry: + +```shell +flux create source oci demo-app \ + --secret-ref=ghcr-auth \ + --url=oci://${APP_REPO} \ + --tag=latest \ + --interval=1m +``` + +!!! info "Automated updates" + + At every minute, Flux verifies if the latest OCI artifact digest differs from the digest + of the last downloaded artifact. When a new artifact is tagged as latest, Flux will + detect the new version and will pull it inside the cluster. + +Create a Flux `Kustomization` for applying the manifests from the artifact on the cluster: + +```shell +flux create kustomization demo-app \ +--source=OCIRepository/demo-app \ +--prune=true \ +--wait=true \ +--health-check-timeout=3m \ +--interval=10m +``` + +!!! info "Automated reconciliation" + + Every time a new version of the OCI artifact is downloaded, Flux reconciles the + changes in the Kubernetes manifests from the artifact with the cluster state. + + During a reconciliation, Flux performs these tasks: + + * Validates the manifests against the Kubernetes API ([server-side apply](https://kubernetes.io/docs/reference/using-api/server-side-apply/) dry run) + * Applies the Kubernetes objects that changed in order (namespaces and other global objects first) + * Deletes the objects that were removed from the latest artifact version + * Waits for the changes to be successfully rollout (Helm upgrades, deployments, jobs, etc) + * Reports the apply diff or any error as Kubernetes events + +You can see the apply diff and any other Flux events with: + +```sh +kubectl alpha events --for kustomization/demo-app -n flux-system +``` + +!!! info "Drift detection and correction" + + Even if nothing changed in the OCI source, Flux verifies if the cluster + state has drifted from the desired state. If a drift is detected, + Flux re-applies the Kubernetes objects that changed and waits for the drift + to be corrected. Then it emits a Kubernetes events with the list of objects + that were corrected. + +## Promote changes to production + +Assuming you're deploying the `latest` version to staging, you could introduce +a dedicated tag for production e.g. `stable`. + +On the production cluster, you'll configure Flux to reconcile the artifacts tagged as `stable`: + +```shell +flux create source oci demo-app \ + --secret-ref=ghcr-auth \ + --url=oci://${APP_REPO} \ + --tag=stable \ + --interval=1m +``` + +To promote a version tested on staging, you would tag it as stable with: + +```shell +kustomizer tag artifact oci://${APP_REPO}:${APP_VERSION} stable +``` + +## Automate the artifact publishing + +You can automate the publishing process by running Kustomizer in CI. + +Here is an example of a GitHub Actions workflow that pushes an OCI artifact +to GHCR every time there is a change to the Kubernetes configuration: + +```yaml +name: publish +on: + push: + branches: + - 'main' + paths: + - 'examples/demo-app/**' + +permissions: + contents: read + id-token: write + packages: write + +env: + ARTIFACT: oci://ghcr.io/${{github.repository_owner}}/${{github.event.repository.name}} + +jobs: + kustomizer: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Setup kustomizer + uses: stefanprodan/kustomizer/action@main + - name: Push + run: | + kustomizer push artifact ${ARTIFACT}:${GITHUB_REF_NAME} \ + -k=examples/demo-app \ + --source=${{ github.repositoryUrl }} \ + --revision="${{ github.ref_name }}/${{ github.sha }}" + - name: Tag latest + run: | + kustomizer tag artifact ${ARTIFACT}:${GITHUB_REF_NAME} latest +``` + +For more details on how to use Kustomizer within GitHub workflows, +please see the [GitHub Actions documentation](../github-actions.md). diff --git a/docs/index.md b/docs/index.md index 9f03557f..0676f006 100644 --- a/docs/index.md +++ b/docs/index.md @@ -81,6 +81,22 @@ Users can access the artifacts by decrypting them with their age private keys: ## Comparison with other tools +### vs flux + +Kustomizer produces OCI artifacts which are compatible with [Flux](https://fluxcd.io). +For more details on how to continuously deploy applications to Kubernetes clusters +with Flux and Kustomizer please see this [guide](guides/fluxcd.md). + +Kustomizer is akin to Flux's [kustomize-controller](https://github.com/fluxcd/kustomize-controller), and it shares +the same reconcile engine that leverages Kubernetes server-side apply. + +Kustomizer can be used as intermediary step when migrating from CI driven deployments +to [Flux](https://fluxcd.io/) and GitOps. If you're running `kubectl apply` in your CI pipelines, +replacing kubectl with kustomizer, would smooth the transition to a continuous delivery system powered by Flux. + +At times, Kustomizer servers as a testing bench for experimental features that are proposed to the Flux community. +For example, Kustomizer is the project where features like staged-apply, garbage collection and diffing where first introduced. + ### vs kubectl Kustomizer makes use of [k8s.io/cli-runtime](https://pkg.go.dev/k8s.io/cli-runtime) @@ -113,18 +129,6 @@ Compared to `kustomize build`, `kustomizer build -k` does things a little differ - Disallows the usage of Kustomize exec and container-based plugins. - Extra patches can be specified with `kustomizer build -k ./overlay --patch ./patch1.yaml --patch ./patch2.yaml`. -### vs flux - -Kustomizer is akin to Flux's [kustomize-controller](https://github.com/fluxcd/kustomize-controller), and it shares -the same reconcile engine that leverages Kubernetes server-side apply. - -Kustomizer can be used as intermediary step when migrating from CI driven deployments -to [Flux](https://fluxcd.io/) and GitOps. If you're running `kubectl apply` in your CI pipelines, -replacing kubectl with kustomizer, would smooth the transition to a continuous delivery system powered by Flux. - -At times, Kustomizer servers as a testing bench for experimental features that are proposed to the Flux community. -For example, Kustomizer is the project where features like staged-apply, garbage collection and diffing where first introduced. - ## License Kustomizer is [Apache 2.0 licensed](https://github.com/stefanprodan/kustomizer/blob/main/LICENSE) diff --git a/mkdocs.yml b/mkdocs.yml index 653e8dae..4a479a7a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -46,6 +46,7 @@ nav: - Secure your Kubernetes supply chain with Kustomizer and Cosign: guides/secure-supply-chain.md - Encryption at rest with Kustomizer and Age: guides/artifacts-encryption.md - Deploy applications from Git with Kustomizer: guides/deploy-from-git.md + - Continuous deployment with Kustomizer and Flux: guides/fluxcd.md - Command Reference: - Artifact: - Push: cmd/kustomizer_push_artifact.md