Merge pull request #2 from kloeckner-i/METAL-3701/add_ci

METAL-3701 add ci

.github/workflows/chart-release.yaml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   chart-release:
-    # if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -28,6 +28,6 @@ jobs:
           version: v3.9.4
 
       - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.2.1
+        uses: helm/chart-releaser-action@v1.4.0
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/test.yaml

@@ -0,0 +1,81 @@
+name: Test
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  get-chart: 
+    runs-on: ubuntu-latest
+    outputs:
+      changed_charts: ${{ steps.get-chart.outputs.changed_charts }}
+    steps: 
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - id: get-chart
+        name: 'Get modified charts'
+        run: |
+          cd charts
+          files_changed="$(git show --pretty="" --name-only)"
+          # Adding || true to avoid "Process exited with code 1" errors
+          charts_dirs_changed="$(echo "$files_changed" | xargs dirname | grep -o "charts/[^/]*" | sed "s|charts/||g" | uniq | tr '\n' ' ' || true)"
+          echo "changed:${charts_dirs_changed}"
+          echo "::set-output name=changed_charts::${charts_dirs_changed}"
+  lint: 
+    runs-on: ubuntu-latest
+    steps: 
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Install Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: v3.9.4
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.3.0
+
+      - name: Run chart-testing (lint)
+        run: ct lint --validate-maintainers=false --target-branch main
+
+  db-operator-test:
+    runs-on: ubuntu-latest
+    needs: get-chart
+    if: ${{ contains(needs.get-chart.outputs.changed_charts, 'db-operator') || contains(needs.get-chart.outputs.changed_charts, 'db-instances') }}
+    strategy:
+      matrix:
+        k8s_version: ['v1.21.6', 'v1.22.3', 'v1.23.1', 'v1.24.3']
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      # The existing apparmor profile for mysql needs to be removed.
+      # https://github.com/actions/virtual-environments/issues/181
+      # https://github.com/moby/moby/issues/7512#issuecomment-51845976
+      - name: Remove MySQL App Armour Configuration
+        run: |
+          set -x
+          sudo apt-get remove mysql-server --purge
+          sudo apt-get install apparmor-profiles
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+      
+      - name: Create k3d cluster
+        env:
+          K8S_VERSION: ${{ matrix.k8s_version }}
+        run: make k3d
+
+      - name: Install Helm chart
+        run: make db-operator
+
+      - name: Integration test
+        run: ./tests/db-operator/integration.sh

Makefile

@@ -0,0 +1,19 @@
+
+.PHONY: all deploy build helm
+.ONESHELL: test
+
+ifeq ($(K8S_VERSION),)
+K8S_VERSION := v1.22.13
+endif
+
+k3d:
+	@curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+	@k3d cluster create myk3s -i rancher/k3s:$(K8S_VERSION)-k3s1
+	@kubectl get pod
+
+lint: ## lint helm manifests
+	@helm lint -f db-operator/values.yaml -f db-operator/ci/ci-1-values.yaml --strict ./db-operator
+	@helm lint -f db-instances/values.yaml --strict ./db-instances
+
+db-operator: ## install db-operator chart if not exist and install local chart using helm upgrade --install command
+	@helm upgrade --install --create-namespace --namespace operator my-dboperator charts/db-operator -f charts/db-operator/values.yaml -f charts/db-operator/values-local.yaml

db-operator/values-local.yaml → charts/db-operator/values-local.yaml

@@ -1,9 +1,3 @@
-image:
-  repository: my-db-operator
-  tag: "1.0.0-dev"
-  pullPolicy: IfNotPresent
-
-
 reconcileInterval: "10"
 
 secrets:

tests/db-operator/integration.sh

@@ -0,0 +1,146 @@
+#!/bin/sh -e
+# requirements: jq
+
+OPERATOR_NAMESPACE="operator"
+TEST_NAMESPACE="test"
+TEST_CHARTS_DIR="tests/db-operator"
+
+retry=30
+interval=15
+
+case $TEST_K8S in
+    "microk8s")
+        export HELM_CMD="sudo microk8s.helm3"
+        export KUBECTL_CMD="sudo microk8s.kubectl"
+        ;;
+    *)
+        export HELM_CMD="helm"
+        export KUBECTL_CMD="kubectl"
+esac
+
+check_requirements() {
+    jq --version > /dev/null 2>&1
+    if [ $? -ne 0 ]; then
+        echo "jq not installed"
+        exit 1;
+    fi
+}
+
+check_dboperator_log() {
+    $KUBECTL_CMD logs -l app=db-operator -n ${OPERATOR_NAMESPACE}
+}
+
+check_instance_status() {
+    echo "[DbInstance] checking"
+    for i in $(seq 1 $retry)
+    do
+        count=$($KUBECTL_CMD get dbin -o json | jq '.items | length')
+        if [ "$count" -eq 0 ]; then
+            echo "DbInstance resource doesn't exists"
+            continue;
+        fi
+
+        ready_count=$($KUBECTL_CMD get dbin -o json | jq '[.items[] | select(.status.status == true)] | length')
+
+        if [ "$ready_count" -eq "$count" ]; then
+            echo "[DbInstance] Status OK!"
+            return 0 # finish check
+        fi
+
+        echo "[DbInstance] Status false"
+        $KUBECTL_CMD get dbin
+        check_dboperator_log
+        echo "Retrying after $interval seconds..."
+        sleep $interval; # retry with interval
+    done # end retry
+    echo "DbInstance not healthy"
+    exit 1 # return false
+}
+
+create_googleapi_mock_server() {
+    $HELM_CMD upgrade --install --namespace ${OPERATOR_NAMESPACE} --create-namespace mock-googleapi ${TEST_CHARTS_DIR}/mock-googleapi --wait
+}
+
+create_test_resources() {
+    echo "[Test] creating"
+    $KUBECTL_CMD create ns ${TEST_NAMESPACE} --dry-run=client -o yaml | $KUBECTL_CMD apply -f - \
+    && $HELM_CMD upgrade --install --namespace ${TEST_NAMESPACE} test-mysql-generic ${TEST_CHARTS_DIR}/mysql-generic --wait \
+    && $HELM_CMD upgrade --install --namespace ${TEST_NAMESPACE} test-pg-generic ${TEST_CHARTS_DIR}/postgres-generic --wait \
+    && $HELM_CMD upgrade --install --namespace ${TEST_NAMESPACE} test-pg-gsql ${TEST_CHARTS_DIR}/postgres-gsql --wait
+    if [ $? -ne 0 ]; then
+        echo "[Test] failed to create"
+        exit 1;
+    fi
+    echo "[Test] created"
+}
+
+check_databases_status() {
+    echo "[Database] checking"
+    for i in $(seq 1 $retry)
+    do
+        count=$($KUBECTL_CMD get db -n ${TEST_NAMESPACE} -o json | jq '.items | length')
+        if [ $count -eq 0 ]; then
+            echo "Database resource doesn't exists"
+            continue;
+        fi
+
+        ready_count=$($KUBECTL_CMD get db -n ${TEST_NAMESPACE} -o json | jq '[.items[] | select(.status.status == true)] | length')
+
+        if [ "$ready_count" -eq "$count" ]; then
+            echo "[Database] Status OK!"
+            return 0 # finish check
+        fi
+
+        echo "[Database] Status false"
+        $KUBECTL_CMD get db -n ${TEST_NAMESPACE}
+        check_dboperator_log
+        echo "Retrying after $interval seconds..."
+        sleep $interval; # retry with interval
+    done # end retry
+    echo "Database not healthy"
+    exit 1 # return false
+}
+
+run_test() {
+    echo "[Test] testing read write to database"
+    $HELM_CMD test test-mysql-generic -n ${TEST_NAMESPACE} \
+    && $HELM_CMD test test-pg-generic -n ${TEST_NAMESPACE}
+    if [ $? -ne 0 ]; then
+        echo "[Test] failed"
+        exit 1;
+    fi
+    echo "[Test] OK!"
+}
+
+delete_databases() {
+    echo "[Database] deleting"
+    $KUBECTL_CMD delete db -n ${TEST_NAMESPACE} --all \
+    && echo "[Database] deleted!"
+}
+
+check_databases_deleted() {
+    echo "[Database] checking deleted"
+    for _ in $(seq 1 $retry)
+    do
+        count=$($KUBECTL_CMD get db -n ${TEST_NAMESPACE} -o json | jq '.items | length')
+        if [ "$count" -ne 0 ]; then
+            echo "[Database] $(echo $item | jq -r '.metadata.name') not deleted"
+            check_dboperator_log
+            continue;
+        fi
+        echo "[Database] All deleted!"
+        return 0 # all good
+    done
+    check_dboperator_log
+    echo "[Database] not deleted"
+    exit 1 # return false
+}
+
+check_requirements
+create_googleapi_mock_server
+create_test_resources
+check_instance_status
+check_databases_status
+run_test
+delete_databases
+check_databases_deleted

tests/db-operator/mock-googleapi/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: Mock of google api server to test gsql instance
+name: mock-googleapi
+version: 0.1.0

tests/db-operator/mock-googleapi/templates/api.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mock-google-api
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: googleAPI
+      role: test
+  template:
+    metadata:
+      labels:
+        app: googleAPI
+        role: test
+    spec:
+      containers:
+        - name: api
+          image: ghcr.io/kloeckner-i/cloudish-sql:v1.0.0
+          ports:
+          - containerPort: 8080
+            name: http
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: LOG_LEVEL
+              value: "DEBUG"
+          command:
+            - /usr/local/bin/cloudish-sql
+            - --db-address=pg-test-db:5432
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mock-google-api
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: http
+  selector:
+    app: googleAPI
+    role: test
+  type: ClusterIP

tests/db-operator/mysql-generic/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: Integration test for db operator mysql
+name: test-mysql-generic-db-operator
+version: 0.1.0

tests/db-operator/mysql-generic/scripts/test_read.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+retry=3
+interval=5
+
+for i in `seq 1 $retry`
+do
+    sleep $interval
+    if [ ! -f "${MYSQL_PASSWORD_FILE}" ]; then
+        echo "Password file does not exists"
+        exit 1;
+    else
+        MYSQL_PASSWORD=$(cat ${MYSQL_PASSWORD_FILE})
+    fi
+
+    TESTDATA="$(cat /tmp/checkdata)"
+
+    echo "reading data from mysql..."
+    FOUNDDATA=$(mysql \
+            -h ${MYSQL_HOST} \
+            -u ${MYSQL_USERNAME} \
+            -p${MYSQL_PASSWORD} ${MYSQL_DB} \
+            -e "SELECT data FROM test WHERE data = '${TESTDATA}';")
+
+    echo "$FOUNDDATA" | grep "$TESTDATA" && break;
+done

tests/db-operator/mysql-generic/scripts/test_write.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+TESTDATA="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)"
+echo ${TESTDATA} > /tmp/checkdata
+
+retry=3
+interval=5
+
+for i in `seq 1 $retry`
+do
+    sleep $interval
+    if [ ! -f "${MYSQL_PASSWORD_FILE}" ]; then
+        echo "Password file does not exists"
+        exit 1;
+    else
+        MYSQL_PASSWORD=$(cat ${MYSQL_PASSWORD_FILE})
+    fi
+
+    echo "writing data into mysql database..."
+    mysql -h ${MYSQL_HOST} -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} ${MYSQL_DB} \
+    -e "CREATE TABLE IF NOT EXISTS test (no INT NOT NULL AUTO_INCREMENT PRIMARY KEY, data VARCHAR(100)); INSERT INTO test (data) VALUES('${TESTDATA}');"\
+    && break
+done