feat: deploy apps with ArgoCD

This PR adds the feature to bootstrap a Talos cluster with ArgoCD using
the app-of-apps pattern. ArgoCD, in turn, deploys all the dependencies
required by the application and the application itself.

.github/workflows/semantic-pull-request.yaml

@@ -19,4 +19,4 @@ jobs:
     steps:
       - uses: amannn/action-semantic-pull-request@v5
         env:
-          GITHUB_TOKEN: ${{ secrets.CI_GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

hooks/post_gen_project.py

@@ -27,7 +27,6 @@
     shutil.rmtree("frontend")
     file_names = [
         os.path.join("k8s", "base", "frontend.yaml"),
-        os.path.join("k8s", "prod", "patch-react.yaml"),
     ]
     for file_name in file_names:
         os.remove(file_name)

test-scaf.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Default values
 BRANCH_NAME=""

{{cookiecutter.project_slug}}/.github/workflows/main.yaml

@@ -2,6 +2,10 @@ name: Main Workflow
 
 on: [ push, pull_request ]
 
+env:
+  AWS_REGION: ${{ '{{ vars.AWS_REGION }}' }}
+  AWS_ACCOUNT_ID: ${{ '{{ vars.AWS_ACCOUNT_ID }}' }}
+
 jobs:
   check-lint-and-formatting:
     runs-on: ubuntu-latest
@@ -73,3 +77,64 @@ jobs:
           POSTGRES_HOST: postgres
           POSTGRES_PORT: 5432
 {% if cookiecutter.use_celery == 'y' %}          CELERY_BROKER_URL: redis://127.0.0.1:6379/0{% endif %}
+
+  build-and-push-images:
+    if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs: [check-lint-and-formatting, backend-test]
+
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ '{{secrets.GITHUB_TOKEN}}' }}
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Configure AWS credentials from OIDC
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ '{{ env.AWS_ACCOUNT_ID }}' }}:role/{{ cookiecutter.project_slug }}-github-oidc-role
+          aws-region: ${{ '{{ env.AWS_REGION }}' }}
+
+      - name: Log in to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Set environment variables
+        run: |
+          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          echo "BRANCH_NAME=$(echo ${{ '{{ github.ref }}' }} | sed s,refs/heads/,, | sed s,/,-,g)" >> $GITHUB_ENV
+          export TARGET_ENV=$(if [ "$BRANCH_NAME" = "main" ]; then echo "prod"; else echo "sandbox"; fi)
+          echo "TARGET_ENV=$TARGET_ENV" >> $GITHUB_ENV
+          echo "ECR_REPO_BACKEND=${{ '{{ env.AWS_ACCOUNT_ID }}' }}.dkr.ecr.${{ '{{ env.AWS_REGION }}' }}.amazonaws.com/{{ cookiecutter.project_dash }}-$TARGET_ENV-backend" >> $GITHUB_ENV
+
+      - name: Build and push backend image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./backend
+          push: true
+          tags: ${{ '{{ env.ECR_REPO_BACKEND }}' }}:${{ '{{ env.SHORT_SHA }}' }},${{ '{{ env.ECR_REPO_BACKEND }}' }}:${{ '{{ env.BRANCH_NAME }}' }}
+
+      - name: Tag Images
+        run: |
+          cd k8s/$TARGET_ENV
+          kustomize edit set image backend=$ECR_REPO_BACKEND:$SHORT_SHA
+
+      - name: Commit & Push to GitHub
+        run: |
+          git config user.email "$GITHUB_EMAIL"
+          git config user.name "GitHub User"
+          git add k8s/$TARGET_ENV/kustomization.yaml
+          git commit -m "Update $TARGET_ENV image to $SHORT_SHA [skip ci]"
+          git push

{{cookiecutter.project_slug}}/.gitignore

@@ -28,6 +28,7 @@ dist/
 
 # Unit test reports
 TEST*.xml
+.coverage
 
 # Generated by MacOS
 .DS_Store

{{cookiecutter.project_slug}}/.pre-commit-config.yaml

@@ -5,6 +5,7 @@ repos:
     -   id: check-case-conflict
     -   id: check-merge-conflict
     -   id: debug-statements
+        exclude: ^backend/{{ cookiecutter.project_slug }}/utils/debugger.py
     -   id: detect-aws-credentials
         args: ["--allow-missing-credentials"]
     -   id: detect-private-key

{{cookiecutter.project_slug}}/.yamlfmt

@@ -0,0 +1,5 @@
+formatter:
+  type: basic
+  include_document_start: false
+  drop_merge_tag: true
+  max_line_length: 0

{{cookiecutter.project_slug}}/.yamllint

@@ -0,0 +1,29 @@
+---
+
+extends: default
+
+rules:
+  braces:
+    level: warning
+    max-spaces-inside: 1
+  brackets:
+    level: warning
+    max-spaces-inside: 1
+  colons:
+    level: warning
+  commas:
+    level: warning
+  comments: disable
+  comments-indentation: disable
+  document-start: disable
+  empty-lines:
+    level: warning
+  hyphens:
+    level: warning
+  indentation:
+    level: warning
+    indent-sequences: consistent
+  line-length:
+    level: warning
+    allow-non-breakable-inline-mappings: true
+  truthy: disable

{{cookiecutter.project_slug}}/Makefile

@@ -17,10 +17,14 @@ PROJECT_SLUG:="{{ cookiecutter.project_slug }}"
 .PHONY: ci
 ci: test secure build  ## Execute the same checks used in CI/CD
 
-check-lint-and-formatting:  ## Execute check of lint and formatting using existing pre-commit hooks
+.git/hooks/pre-commit: ## Install pre-commit hooks
 	pip install pre-commit; \
 	cd backend/; \
-	pre-commit install; \
+	pre-commit install
+
+.PHONY: check-lint-and-formatting
+check-lint-and-formatting: .git/hooks/pre-commit ## Execute check of lint and formatting using existing pre-commit hooks
+	cd backend; \
 	pre-commit run -a
 
 {% if cookiecutter.create_nextjs_frontend == 'y' %}
@@ -34,11 +38,15 @@ check-lint-and-test-frontend:  ## Frontend Lint & Typecheck & Test
 
 backend-test: ## Execute backend tests
 ifeq ($(CI),true)
+	pip install uv; \
 	cd backend; \
+	uv venv ./venv; \
+	. ./venv/bin/activate; \
 	uv pip install \
 		-r requirements/production.txt \
 		-r requirements/tests.txt; \
-	cd {{ cookiecutter.project_slug }}; \
+	cd scafman; \
+	export DJANGO_SECRET_KEY="test"; \
 	pytest --cov=./ --cov-report html --ds=config.settings.test
 else
 	$(KUBECTL_EXEC_BACKEND) -c "cd {{ cookiecutter.project_slug }} && pytest --cov=./ --cov-report html --ds=config.settings.test"
@@ -127,8 +135,12 @@ secrets:
 	@$(call check_var,ENVIRONMENT)
 	@echo "Creating sealed secrets for $$(kubectl config current-context) cluster"
 	kubectl config get-contexts --no-headers | \
-		grep {{ cookiecutter.project_slug }}-$(ENVIRONMENT) && \
-		kubectl config use-context admin@{{ cookiecutter.project_slug }}-$(ENVIRONMENT)
+		grep {{ cookiecutter.project_dash }}-$(ENVIRONMENT) && kubectl config use-context admin@{{ cookiecutter.project_dash }}-$(ENVIRONMENT)
+	AWS_S3_ACCESS_KEY_ID=$$(tofu -chdir=./terraform/$(ENVIRONMENT) output -raw cnpg_user_access_key) \
+	AWS_S3_SECRET_ACCESS_KEY=$$(tofu -chdir=./terraform/$(ENVIRONMENT) output -raw cnpg_user_secret_key) \
+{% if cookiecutter.mail_service == "Amazon SES" %}	AWS_SES_ACCESS_KEY_ID=$$(tofu -chdir=./terraform/$(ENVIRONMENT) output -raw amazon_ses_user_key) \
+	AWS_SES_SECRET_ACCESS_KEY=$$(tofu -chdir=./terraform/$(ENVIRONMENT) output -raw amazon_ses_user_secret_key) \{% endif %}
+	DJANGO_SECRET_KEY=$$(LC_CTYPE=C tr -dc A-Za-z0-9 </dev/urandom | head -c 32; echo) \
 	ENVIRONMENT=$(ENVIRONMENT) \
 	envsubst < k8s/templates/secrets.yaml.template | op inject | if [ "$(DEBUG)" = "true" ]; then cat; else kubeseal -n {{ cookiecutter.project_slug }}-$(ENVIRONMENT) --format yaml > k8s/$(ENVIRONMENT)/secrets.yaml; fi
 
@@ -150,49 +162,6 @@ prod-secrets: ## Create sealed secrets for prod
 debug-prod-secrets: ## Create sealed secrets for prod
 	$(MAKE) secrets ENVIRONMENT=prod DEBUG=true
 
-
-argocd-app:
-	envsubst < k8s/templates/argocd.application.yaml.template | kubeseal --format yaml > k8s/argocd/application.yaml
-	envsubst < k8s/templates/repocreds.yaml.template | kubeseal --format yaml > k8s/argocd/repocreds.yaml
-	sed -i '/spec:/ {N; s/spec:\n/  annotations:\n    sealedsecrets.bitnami.com\/managed: "true"\nspec:\n/}' k8s/argocd/repocreds.yaml
-
-## Monitoring
-check_var = $(strip $(if $(filter undefined,$(origin $1)),$(error $1 is not set)))
-
-kube-prometheus-up:  ## Install kube-prometheus
-	@$(call check_var,GRAFANA_ADMIN_PASSWORD)
-	@helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
-	@helm repo update
-	@helm install kube-prometheus prometheus-community/kube-prometheus-stack --set grafana.sidecar.datasources.isDefaultDatasource=false,grafana.persistence.enabled=true,grafana.adminPassword=$(GRAFANA_ADMIN_PASSWORD) --namespace monitoring --create-namespace
-
-loki-up:  ## Install loki
-	@helm repo add grafana https://grafana.github.io/helm-charts
-	@helm repo update
-	@helm install loki grafana/loki-stack --values k8s/_monitoring/loki-stack-values.yaml --namespace monitoring --create-namespace
-
-monitoring-up: kube-prometheus-up loki-up  ## Install the monitoring stack
-
-loki-down:  ## Uninstall loki
-	helm uninstall loki --namespace monitoring
-
-kube-prometheus-down:  ## Uninstall kube-prometheus
-	helm uninstall kube-prometheus --namespace monitoring
-
-monitoring-down: loki-down kube-prometheus-down  ## Uninstall the monitoring stack
-
-monitoring-login:  ## Get the login credentials for the monitoring stack
-	@kubectl get secret --namespace monitoring kube-prometheus-grafana -o jsonpath='{.data.admin-user}' | base64 -d; echo
-	@kubectl get secret --namespace monitoring kube-prometheus-grafana -o jsonpath='{.data.admin-password}' | base64 -d; echo
-
-monitoring-port-forward:  ## Forward the port for the monitoring stack
-	kubectl --namespace monitoring port-forward svc/kube-prometheus-grafana 8080:80
-
-monitoring-dashboard:  ## Create the dashboard for the monitoring stack
-	kubectl apply -f k8s/_monitoring/django-logs-table.yaml -n monitoring
-
-monitoring:  ## Show the status of the monitoring stack
-	kubectl get all -n monitoring
-
 ## Help
 
 help: ## Show the list of all the commands and their help text

{{cookiecutter.project_slug}}/README.md

@@ -74,14 +74,15 @@ This project has a NextJS frontend configured. You can access it at [http://loca
 ## Infrastructure provisioning
 
 Terraform can be used to provision AWS resources for your project deployment.
-terraform/ec2-cluster will create an EC2 instance running a kubernetes cluster for your project.
-Check `terraform/ec2-cluster/README.md` for more information and steps for provisioning resources.
+Read `terraform/README.md` for more information and steps for provisioning
+resources.
 
-## Project deployment
+## Application deployment
 
-ArgoCD and kubernetes can be used to automate the deployment of your project to your infrastructure.
-ArgoCD will watch for changes in your repository and apply the kubernetes manifests.
-Check `k8s/argocd/README.md` for more information on creating and setting up the ArgoCD application.
+Use ArgoCD and Kubernetes to automate the deployment of your application to
+your infrastructure. ArgoCD monitors changes within your repository, promptly
+applying the relevant Kubernetes manifests. Read `bootstrap-cluster/README.md`
+for more details.
 
 ## How to manage passwords and sensitive values
 

{{cookiecutter.project_slug}}/argocd/base/argocd/app.yaml

@@ -0,0 +1,33 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: default
+  source:
+    chart: argo-cd
+    repoURL: https://argoproj.github.io/argo-helm
+    targetRevision: 6.9.2
+    helm:
+      releaseName: argocd
+      # https://argo-cd.readthedocs.io/en/stable/operator-manual/ingress/#traefik-v22
+      valuesObject:
+        configs:
+          params:
+            server.insecure: true
+  destination:
+    server: "https://kubernetes.default.svc"
+    namespace: argocd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m

{{cookiecutter.project_slug}}/argocd/base/argocd/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+namespace: argocd
+resources:
+- app.yaml
+kind: Kustomization

{{cookiecutter.project_slug}}/argocd/base/cert-manager/app.yaml

@@ -0,0 +1,33 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cert-manager
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: default
+  source:
+    chart: cert-manager
+    repoURL: https://charts.jetstack.io
+    targetRevision: v1.14.5
+    helm:
+      releaseName: cert-manager
+      parameters:
+      - name: "installCRDs"
+        value: "true"
+  destination:
+    server: "https://kubernetes.default.svc"
+    namespace: cert-manager
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
+    syncOptions:
+      - CreateNamespace=true

{{cookiecutter.project_slug}}/argocd/base/cert-manager/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+namespace: argocd
+resources:
+- app.yaml
+kind: Kustomization

{{cookiecutter.project_slug}}/argocd/base/cloudnative-pg/app.yaml

@@ -0,0 +1,33 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cloudnative-pg
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: default
+  source:
+    chart: cloudnative-pg
+    repoURL: https://cloudnative-pg.github.io/charts
+    targetRevision: v0.19.1
+    helm:
+      releaseName: cloudnative-pg
+      parameters:
+      - name: "fullnameOverride"
+        value: "cnpg-controller-manager"
+  destination:
+    server: "https://kubernetes.default.svc"
+    namespace: cnpg-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
+    syncOptions:
+      - CreateNamespace=true

{{cookiecutter.project_slug}}/argocd/base/cloudnative-pg/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+namespace: argocd
+resources:
+- app.yaml
+kind: Kustomization