Helm chart for AGIMUS to run in kubernetes (#126)

* Maybe a branch will work * Add the new github actions * Remove windows newlines before running action * Remove windows newlines before running action * And again * merge main into helmify * Update read-me and update .env-example * this isn't ready yet. whoops * Unfuck things * More unfuckery * More unfuckery * Add quickstart instructions for running AGIMUS in KinD
jp00p · Jul 7, 2022 · 9ef42fc · 9ef42fc
1 parent b60ac11
commit 9ef42fc
Show file tree

Hide file tree

Showing 23 changed files with 693 additions and 72 deletions.
diff --git a/.env-example b/.env-example
@@ -2,28 +2,28 @@
 # export LOG_LEVEL=DEBUG
 export TZ=America/Los_Angeles
 export LOG_LEVEL=INFO
-# Tokens
+
+# 3rd-party API Tokens
 export DISCORD_TOKEN=
 export TMDB_KEY=
 export NASA_TOKEN=
 export GIT_TOKEN=
+export GOOGLE_API_KEY=
+export GOOGLE_CX=
+export WOLFRAM_ALPHA_ID=
 
 # Mysql DB Creds
 export DB_SEED_FILEPATH=./data/seed-db.sql
-export DB_HOST=db
+# Uncomment next line to run mysql in docker
+export DB_HOST=host.docker.internal
+# Uncomment next line to run mysql in kubernetes
+# export DB_HOST=mysql-service.agimus.svc.cluster.local
 export DB_NAME=FoD
 export DB_PASS=password
 export DB_USER=root
 export DB_DUMP_FILENAME=bot-dump.sql
 export DB_CONTAINER_NAME=fodmysql
 
 # Path to configuration json
-export BOT_CONTAINER_NAME=fodbot
+export BOT_CONTAINER_NAME=agimus
 export BOT_CONFIGURATION_FILEPATH=./configuration.json
-
-# Google Search Credentials for generate_episode_python.json
-export GOOGLE_API_KEY=
-export GOOGLE_CX=
-
-# Wolfram Alpha Credentials for "Computer:" prompts
-export WOLFRAM_ALPHA_ID=
diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml
@@ -0,0 +1,63 @@
+# Name:        build-and-release.yaml
+# Author:      Mathew Fleisch <mathew.fleisch@gmail.com>
+# Description: This action will build and push a docker container, when triggered by
+#              pushing to the main branch
+name: Release Container and Helm Chart
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+
+  release:
+    name: Release Containers and Helm Chart
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      -
+        name: Set tag environment variable
+        run: echo "RELEASE_VERSION=$(make version)" >> $GITHUB_ENV
+      -
+        name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Docker Login
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and Push Containers
+        id: docker_build
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: .
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            ghcr.io/${{ secrets.REPO_OWNER }}/agimus:latest
+            ghcr.io/${{ secrets.REPO_OWNER }}/agimus:${{ env.RELEASE_VERSION }}
+          cache-from: type=registry,ref=ghcr.io/${{ secrets.REPO_OWNER }}/agimus:latest
+          cache-to: type=inline
+      - 
+        name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - 
+        name: Run chart-releaser
+        uses: helm/chart-releaser-action@v1.1.0
+        env:
+          CR_TOKEN: "${{ secrets.GIT_TOKEN }}"
diff --git a/.github/workflows/pr-test-and-build.yaml b/.github/workflows/pr-test-and-build.yaml
@@ -0,0 +1,132 @@
+# Name:        pr-test-and-build.yaml
+# Author:      Mathew Fleisch <mathew.fleisch@gmail.com>
+# Description: This action will run go lint/unit tests as well as
+#              build a docker container and test it against a KinD cluster.
+#              See Makefile for more details.
+name: PR CI
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  unit_test:
+    name: Lint and Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - 
+        name: Install stuff with asdf
+        uses: asdf-vm/actions/install@v1
+        with:
+          tool_versions: |
+            action-validator 0.1.2
+            dockle 0.4.5
+            helm 3.8.1
+            yq 4.22.1
+      -
+        name: Lint Actions
+        run: make --no-print-directory lint-actions
+      # -
+      #   name: Lint Container Using Dockle
+      #   env:
+      #     DOCKER_CONTENT_TRUST: 1
+      #   run: |
+      #     make --no-print-directory docker-build
+      #     make --no-print-directory docker-lint
+
+  integration_test:
+    name: KinD Integration Tests
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - 
+        name: Install stuff with asdf
+        uses: asdf-vm/actions/install@v1
+        with:
+          tool_versions: |
+            helm 3.8.1
+            yq 4.22.1
+      -
+        name: KinD Tests
+        shell: bash
+        env:
+          CONFIGJSON: ${{ secrets.CONFIGJSON }}
+          ENVFILE: ${{ secrets.ENVFILE }}
+        run: |
+          if [ -z "${CONFIGJSON}" ]; then
+            echo "Loading fallback configuration" \
+              && cp configuration.json local.json;
+          else
+            echo "Loading override configuration" \
+              && echo "${{ secrets.CONFIGJSON }}" | base64 -d | sed -e 's/\r//' > local.json;
+          fi
+          if [ -z "${ENVFILE}" ]; then
+            echo "Problem loading github secret ENVFILE" \
+              && echo "${{ secrets.ENVFILE }}" | base64 -d | sed -e 's/\r//' > .env \
+              && cat .env
+          else
+            echo "Loading secrets" \
+              && echo "${{ secrets.ENVFILE }}" | base64 -d | sed -e 's/\r//' > .env \
+              && source .env
+          fi
+          ls -alF
+          echo "DB_SEED_FILEPATH=$DB_SEED_FILEPATH"
+          echo "DB_HOST=$DB_HOST"
+          echo "DB_NAME=$DB_NAME"
+          echo "DB_PASS=$DB_PASS"
+          echo "DB_USER=$DB_USER"
+          echo "DB_DUMP_FILENAME=$DB_DUMP_FILENAME"
+          echo "DB_CONTAINER_NAME=$DB_CONTAINER_NAME"
+          echo "REPO_OWNER=$REPO_OWNER"
+          echo "BOT_CONTAINER_NAME=$BOT_CONTAINER_NAME"
+          echo "BOT_CONFIGURATION_FILEPATH=$BOT_CONFIGURATION_FILEPATH"
+          make help
+          echo "Building KinD cluster" \
+            && make --no-print-directory kind-create
+          echo "Building docker container and load into KinD cluster" \
+            && make --no-print-directory kind-load
+          echo "Running tests..." \
+            && make --no-print-directory kind-test
+          sleep 5
+          kubectl --namespace agimus delete pod $(kubectl --namespace agimus get pods | grep agimus | awk '{print $1}') || true
+          sleep 10
+          kubectl --namespace agimus get deployments -o wide
+          sleep 10
+          kubectl --namespace agimus get pods -o wide
+          sleep 10
+          kubectl --namespace agimus describe pod $(kubectl --namespace agimus get pods | grep agimus | awk '{print $1}') || true
+          sleep 10
+          kubectl --namespace agimus get deployments -o wide
+          kubectl --namespace agimus get pods -o wide
+          kubectl --namespace agimus get secrets
+          kubectl --namespace agimus get configmaps
+          kubectl --namespace agimus get configmaps agimus-config -o yaml
+          kubectl --namespace agimus logs $(kubectl --namespace agimus get pods | grep agimus | awk '{print $1}') || true
+      -
+        name: Debug failure
+        if: ${{ failure() }}
+        shell: bash
+        env:
+          CONFIGJSON: ${{ secrets.CONFIGJSON }}
+        run: |
+          make help
+          ls -alF
+          kubectl --namespace agimus get deployments -o wide
+          kubectl --namespace agimus get pods -o wide
+          kubectl --namespace agimus get secrets
+          kubectl --namespace agimus get configmaps
+          kubectl --namespace agimus get configmaps agimus-config -o yaml
+          kubectl --namespace agimus describe pod $(kubectl --namespace agimus get pods | grep agimus | awk '{print $1}') || true
+          kubectl --namespace agimus logs $(kubectl --namespace agimus get pods | grep agimus | awk '{print $1}') || true
+          kubectl --namespace agimus describe pod $(kubectl --namespace agimus get pods | grep mysql | awk '{print $1}') || true
+          kubectl --namespace agimus logs $(kubectl --namespace agimus get pods | grep mysql | awk '{print $1}') || true
+
diff --git a/.github/workflows/update-tgg.yaml b/.github/workflows/update-tgg.yaml
@@ -12,7 +12,7 @@ on:
     - cron: '15 21 * * 4'
   repository_dispatch:
     types:
-      - trigger-tgg-update
+      - update-shows
 jobs:
   build:
     name: Update The Greatest Generation Metadata

diff --git a/Dockerfile b/Dockerfile
@@ -6,21 +6,24 @@ RUN rm /bin/sh && ln -s /bin/bash /bin/sh \
     && apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         curl wget apt-utils python3 python3-pip make build-essential fonts-noto-color-emoji locales openssl git jq tzdata sudo \
+    && touch /etc/sudoers.d/bot-user \
+    && echo "bot ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/bot-user \
     && useradd -ms /bin/bash bot \
     && usermod -aG sudo bot \
     && python3 -m pip install --upgrade --force pip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
     && ln -s /usr/bin/python3 /usr/local/bin/python
 
-# configure timezone
+# configure timezone and set UTF8 charset
 ENV TZ="America/Los_Angeles"
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-RUN dpkg-reconfigure --frontend noninteractive tzdata
-
-# set UTF8 charset
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
-    locale-gen
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \
+    && echo $TZ > /etc/timezone \
+    && dpkg-reconfigure --frontend noninteractive tzdata \
+    && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \
+    && locale-gen
 ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8 
+ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 
 # Use 'bot' user to avoid pip warning messages