Merge branch 'main' into vrt5

Signed-off-by: André Bauer <monotek@users.noreply.github.com>
kokuwaio · Dec 4, 2023 · fb30a13 · fb30a13
2 parents 05a6739 + eeae159
commit fb30a13
Show file tree

Hide file tree

Showing 13 changed files with 147 additions and 81 deletions.
diff --git a/.github/kubeconform.sh b/.github/kubeconform.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# use kubeconform to validate helm generated kubernetes manifest
+#
+
+set -o errexit
+set -o pipefail
+
+CHART_DIRS="$(git diff --find-renames --name-only "$(git rev-parse --abbrev-ref HEAD)" remotes/origin/main | grep '[cC]hart.yaml' | sed -e 's#/[Cc]hart.yaml##g')"
+
+# install kubeconform
+curl --silent --show-error --fail --location --output /tmp/kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/"${KUBECONFORM_VERSION}"/kubeconform-linux-amd64.tar.gz
+sudo tar -C /usr/local/bin -xf /tmp/kubeconform.tar.gz kubeconform
+
+# validate charts
+for CHART_DIR in ${CHART_DIRS};do
+  echo "helm dependency build..."
+  helm dependency build "${CHART_DIR}"
+
+  echo "kubeconform(ing) ${CHART_DIR##charts/} chart..."
+  helm template "${CHART_DIR}" | kubeconform --strict --verbose --kubernetes-version "${KUBERNETES_VERSION#v}"
+done
diff --git a/.github/kubeval.sh b/.github/kubeval.sh
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -6,34 +6,34 @@ on:
       - 'charts/**/**'
 
 env:
-  helm-version: "v3.10.2"
-  kubeval-version: "v0.16.1"
+  helm-version: "v3.12.2"
+  kubeconform-version: "v0.6.3"
 
 jobs:
   lint-chart:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Helm
         uses: azure/setup-helm@v3.5
         with:
           version: "${{ env.helm-version }}"
-      - uses: actions/setup-python@v4.6.0
+      - uses: actions/setup-python@v4.7.1
         with:
           python-version: 3.7
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.4.0
+        uses: helm/chart-testing-action@v2.6.1
       - name: Run chart-testing (lint)
         run: ct lint --config .github/ct.yaml
 
   super-linter:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Lint Code Base
@@ -43,68 +43,65 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_ALL_CODEBASE: false
           VALIDATE_JSCPD: false
-          VALIDATE_KUBERNETES_KUBEVAL: false
+          VALIDATE_KUBERNETES_KUBECONFORM: false
           VALIDATE_YAML: false
 
-  kubeval-chart:
+  kubeconform-chart:
     runs-on: ubuntu-22.04
     needs:
       - lint-chart
     strategy:
       matrix:
         k8s:
-          - v1.23.13
-          - v1.24.7
-          - v1.25.3
+          - v1.25.11
+          - v1.26.6
+          - v1.27.3
+
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Helm
         uses: azure/setup-helm@v3.5
         with:
           version: "${{ env.helm-version }}"
-      - name: Run kubeval
+      - name: Run kubeconform
         env:
           KUBERNETES_VERSION: ${{ matrix.k8s }}
-          KUBEVAL_VERSION: "${{ env.kubeval-version }}"
-        run: .github/kubeval.sh
+          KUBECONFORM_VERSION: "${{ env.kubeconform-version }}"
+        run: .github/kubeconform.sh
 
   install-chart:
     name: install-chart
     runs-on: ubuntu-20.04
     needs: 
-      - kubeval-chart
+      - kubeconform-chart
     strategy:
       matrix:
         k8s:
-          - v1.23.13
-          - v1.24.7
-          - v1.25.3
+          - v1.25.11
+          - v1.26.6
+          - v1.27.3
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Helm
         uses: azure/setup-helm@v3.5
         with:
           version: "${{ env.helm-version }}"
-      - uses: actions/setup-python@v4.6.0
+      - uses: actions/setup-python@v4.7.1
         with:
           python-version: 3.7
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.4.0
+        uses: helm/chart-testing-action@v2.6.1
       - name: Run chart-testing (list-changed)
         id: list-changed
-        run: |
-          changed=$(ct list-changed --config .github/ct.yaml)
-          if [[ -n "$changed" ]]; then
-            echo "::set-output name=changed::true"
-          fi
+        run: if [[ -n "$(ct list-changed --config .github/ct.yaml)" ]]; then echo 'changed=true' >> "$GITHUB_OUTPUT"; fi
       - name: Create kind cluster
-        uses: helm/kind-action@v1.5.0
+        uses: helm/kind-action@v1.8.0
         if: steps.list-changed.outputs.changed == 'true'
         with:
           config: .github/kind-config.yaml

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -28,6 +28,6 @@ jobs:
           version: "${{ env.helm-version }}"
 
       - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.5.0
+        uses: helm/chart-releaser-action@v1.6.0
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
diff --git a/.github/workflows/sync-readme.yaml b/.github/workflows/sync-readme.yaml
@@ -11,12 +11,12 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - run: |
           cp -f README.md ${{ runner.temp }}/README.md
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: gh-pages
 

diff --git a/charts/fluentd-elasticsearch/Chart.yaml b/charts/fluentd-elasticsearch/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: fluentd-elasticsearch
-version: 13.9.0
-appVersion: v4.2.3
+version: 13.11.0
+appVersion: v4.4.4
 type: application
 home: https://www.fluentd.org/
 description: A Fluentd Helm chart for Kubernetes with Elasticsearch output

diff --git a/charts/mysqldump/Chart.yaml b/charts/mysqldump/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
-version: 6.0.0
-appVersion: v3.0.16
+version: 6.3.0
+appVersion: v3.1.1
 description: A Helm chart to help backup MySQL databases using mysqldump
 name: mysqldump
 keywords:
@@ -18,6 +18,6 @@ maintainers:
 type: application
 dependencies:
   - name: mariadb
-    version: 11.0.14
+    version: 12.2.9
     repository: "https://charts.bitnami.com/bitnami"
     condition: mariadb.enabled
diff --git a/charts/mysqldump/README.md b/charts/mysqldump/README.md
@@ -50,7 +50,7 @@ The following tables lists the configurable parameters of the mysqldump chart an
 | image.repository                              | Name of image to use                                                            | monotek/gcloud-mysql         |
 | image.tag                                     | Version of image to use (uses appVersion form Chart.yaml as default if not set) | ""                           |
 | image.pullPolicy                              | Pull Policy to use for image                                                    | IfNotPresent                 |
-| mysql.db                                      | single mysql db to backup (optional)                                            | mysql                        |
+| mysql.db                                      | mysql db(s) to backup (optional)                                                | mysql                        |
 | mysql.host                                    | mysql host to backup                                                            | mysql                        |
 | mysql.username                                | mysql username                                                                  | root                         |
 | mysql.password                                | mysql password                                                                  | ""                           |
@@ -59,6 +59,7 @@ The following tables lists the configurable parameters of the mysqldump chart an
 | mysql.port                                    | mysql port                                                                      | 3306                         |
 | schedule                                      | crontab schedule to run on. set as `now` to run as a one time job               | "0 3 \* \* \*"               |
 | options                                       | options to pass onto MySQL                                                      | "--opt --single-transaction" |
+| rsync.options                                 | options to pass onto rsync                                                      | "-av"                        |
 | debug                                         | print some extra debug logs during backup                                       | false                        |
 | dumpAllToStdout                               | dump all database contents to stdout when not uploading                         | false                        |
 | additionalSteps                               | run these extra shell steps after all backup jobs completed                     | []                           |
@@ -72,7 +73,7 @@ The following tables lists the configurable parameters of the mysqldump chart an
 | persistence.storageClass                      | storage class to use for PVC                                                    |                              |
 | persistence.subPath                           | subPath for PVC                                                                 |                              |
 | allDatabases.enabled                          | backup all databases                                                            | true                         |
-| allDatabases.SingleSqlFile                    | backup all databases to single file                                             | false                        |
+| allDatabases.SingleSqlFile                    | backup all databases to single file (works with mysql.db too)                   | false                        |
 | housekeeping.enabled                          | delete olf backups in pvc                                                       | true                         |
 | housekeeping.keepDays                         | keep last x days of backups in PVC                                              | 10                           |
 | saveToDirectory                               | saves the sql backup to a directory named like the database or alldatabases     | false                        |
@@ -99,6 +100,14 @@ The following tables lists the configurable parameters of the mysqldump chart an
 | upload.openstack.existingSecret               | optional, specify a secret name to use for password                             |                              |
 | upload.openstack.existingSecretKey            | optional, specify a secret key to use for password                              | openstack-backup-password    |
 | upload.openstack.ttlDays                      | days to set time-to-live on uploaded objects (0 to disable)                     | 30                           |
+| upload.s3.enabled                             | upload backups to s3 storage                                                    | false                        |
+| upload.s3.bucketname                          | s3 bucket name                                                                  | mysql-backup                 |
+| upload.s3.endpoint                            | URL endpoint of the S3 service                                                  | <https://mydomain.com>       |
+| upload.s3.region                              | AWS region to use                                                               | us-east-1                    |
+| upload.s3.accesskey                           | s3 access key                                                                   | ""                           |
+| upload.s3.secretkey                           | s3 secret key                                                                   | ""                           |
+| upload.s3.existingSecret                      | optional, existing secret name, used to get s3 Secret key (if set)              | ""                           |
+| upload.s3.existingSecretKey                   | optional, specify a secret key to use for s3 Secret key                         | S3_SECRET_KEY                |
 | resources                                     | resource definitions                                                            | {}                           |
 | nodeSelector                                  | k8s-node selector                                                               | {}                           |
 | tolerations                                   | tolerations                                                                     | \[]                          |

diff --git a/charts/mysqldump/files/job.tpl b/charts/mysqldump/files/job.tpl
@@ -18,7 +18,7 @@ spec:
     image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
     imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
     command: ["/bin/bash", "/scripts/backup.sh"]
-{{- if or .Values.mysql.existingSecret .Values.upload.openstack.existingSecret }}
+{{- if or .Values.mysql.existingSecret .Values.upload.openstack.existingSecret .Values.upload.s3.existingSecret }}
     env:
 {{- end }}
 {{- if .Values.mysql.existingSecret }}
@@ -42,13 +42,28 @@ spec:
             {{- else }}
             key: "openstack-backup-password"
             {{- end }}
+{{- end }}
+{{- if .Values.upload.s3.existingSecret }}
+      - name: S3_SECRET_KEY
+        valueFrom:
+          secretKeyRef:
+            name: {{ .Values.upload.s3.existingSecret | quote }}
+            {{- if .Values.upload.s3.existingSecretKey }}
+            key: {{ .Values.upload.s3.existingSecretKey | quote }}
+            {{- else }}
+            key: "secret-key"
+            {{- end }}
 {{- end }}
     envFrom:
     - configMapRef:
         name: "{{ template "mysqldump.fullname" . }}"
 {{- if not .Values.mysql.existingSecret }}
     - secretRef:
         name: "{{ template "mysqldump.fullname" . }}"
+{{- end }}
+{{- if and (.Values.upload.s3.enabled) (not .Values.upload.s3.existingSecret) }}
+    - secretRef:
+        name: "{{ template "mysqldump.fullname" . }}-s3-secretkey"
 {{- end }}
     volumeMounts:
     - name: backups

diff --git a/charts/mysqldump/templates/configmap-script.yaml b/charts/mysqldump/templates/configmap-script.yaml
@@ -4,7 +4,7 @@ kind: ConfigMap
 metadata:
   name: {{ template "mysqldump.fullname" . }}-script
   labels:
-{{ include "mysqldump.labels" . | indent 4 }}
+    {{- include "mysqldump.labels" . | nindent 4 }}
 data:
 {{- if .Values.upload.openstack.enabled }}
   openstack-upload.py: |-
@@ -35,36 +35,41 @@ data:
     {{ if or (.Values.persistence.enabled) (.Values.persistentVolumeClaim) }}
     {{ if .Values.housekeeping.enabled }}
     echo "delete old backups"
-    find ${BACKUP_DIR} -maxdepth 2 -mtime +${KEEP_DAYS} -regex "^${BACKUP_DIR}/.*[0-9]*_.*\.sql\.gz$" -type f -exec rm {} \;
+    find ${BACKUP_DIR} -type d -name 'lost+found' -prune -o -maxdepth 2 -mtime +${KEEP_DAYS} -regex "^${BACKUP_DIR}/.*[0-9]*_.*\.sql\.gz$" -type f -exec rm {} \;
     {{ end -}}
     {{ end -}}
 
-    {{ if and (.Values.mysql.db) (eq .Values.allDatabases.enabled false) }}
-    MYSQL_DB="{{ .Values.mysql.db }}"
-    echo "Backing up single db ${MYSQL_DB}"
-    {{ if .Values.saveToDirectory }}mkdir -p "${BACKUP_DIR}"/"${MYSQL_DB}"{{ end }}
-    mysqldump ${MYSQL_OPTS} -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} --databases ${MYSQL_DB} | gzip > ${BACKUP_DIR}/{{ if .Values.saveToDirectory }}${MYSQL_DB}/{{ end }}${TIMESTAMP}_${MYSQL_DB}.sql.gz
-    rc=$?
-    {{ else if and (.Values.allDatabases.enabled) (eq .Values.allDatabases.singleBackupFile false)}}
-    for MYSQL_DB in $(mysql -h "${MYSQL_HOST}" -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} -B -N -e "SHOW DATABASES;"|egrep -v '^(information|performance)_schema$'); do
+    {{ if (eq .Values.allDatabases.singleBackupFile false) }}
+      {{ if and (.Values.mysql.db) (eq .Values.allDatabases.enabled false) }}
+    MYSQL_DBS="{{ .Values.mysql.db }}"
+      {{ else if (.Values.allDatabases.enabled) }}
+    MYSQL_DBS=$(mysql -h "${MYSQL_HOST}" -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} -B -N -e "SHOW DATABASES;"|egrep -v '^(information|performance)_schema$')
+      {{ end }}
+    for MYSQL_DB in $MYSQL_DBS; do
       echo "Backing up db ${MYSQL_DB}"
       {{ if .Values.saveToDirectory }}mkdir -p "${BACKUP_DIR}"/"${MYSQL_DB}"{{ end }}
       mysqldump ${MYSQL_OPTS} -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} --databases ${MYSQL_DB} | gzip > ${BACKUP_DIR}/{{ if .Values.saveToDirectory }}${MYSQL_DB}/{{ end }}${TIMESTAMP}_${MYSQL_DB}.sql.gz
       rc=$?
     done
-
-    {{ else if and (.Values.allDatabases.enabled) (.Values.allDatabases.singleBackupFile) }}
-    echo "Backing up all databases"
+    {{ else if (.Values.allDatabases.singleBackupFile) }}
     MYSQL_DB="alldatabases"
     {{ if .Values.saveToDirectory }}mkdir -p "${BACKUP_DIR}"/"${MYSQL_DB}"{{ end }}
+      {{ if and (.Values.mysql.db) (eq .Values.allDatabases.enabled false) }}
+    echo "Backing up all this databases {{ .Values.mysql.db }}"
+    mysqldump ${MYSQL_OPTS} -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} --databases {{ .Values.mysql.db }} | gzip > ${BACKUP_DIR}/{{ if .Values.saveToDirectory }}${MYSQL_DB}/{{ end }}${TIMESTAMP}_${MYSQL_DB}.sql.gz
+    rc=$?
+      {{ else if (.Values.allDatabases.enabled) }}
+    echo "Backing up all databases"
     mysqldump ${MYSQL_OPTS} -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} --all-databases | gzip > ${BACKUP_DIR}/{{ if .Values.saveToDirectory }}${MYSQL_DB}/{{ end }}${TIMESTAMP}_${MYSQL_DB}.sql.gz
     rc=$?
-    {{- end -}}
+      {{ end }}
+    {{ end }}
+
 
-    {{- if or (.Values.upload.googlestoragebucket.enabled) (.Values.upload.ssh.enabled) (.Values.upload.openstack.enabled) -}}
+    {{- if or (.Values.upload.googlestoragebucket.enabled) (.Values.upload.ssh.enabled) (.Values.upload.openstack.enabled) (.Values.upload.s3.enabled) -}}
     {{ if .Values.upload.ssh.enabled -}}
     echo "upload files via ssh to {{ .Values.upload.ssh.user }}@{{ .Values.upload.ssh.host }}:{{ .Values.upload.ssh.dir }}"
-    rsync -av --delete --exclude=*.state -e 'ssh -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null' ${BACKUP_DIR}/ {{ .Values.upload.ssh.user }}@{{ .Values.upload.ssh.host }}:{{ .Values.upload.ssh.dir }}
+    rsync {{ .Values.rsync.options }} --delete --exclude=*.state -e 'ssh -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null' ${BACKUP_DIR}/ {{ .Values.upload.ssh.user }}@{{ .Values.upload.ssh.host }}:{{ .Values.upload.ssh.dir }}
     rcu=$?
     {{ end -}}
 
@@ -85,6 +90,13 @@ data:
     rcu=$?
     {{ end }}
 
+    {{ if .Values.upload.s3.enabled -}}
+    echo "upload files to s3 storage bucket {{ .Values.upload.s3.bucketname }}"
+    aws configure set aws_access_key_id "$S3_ACCESSKEY" && aws configure set aws_secret_access_key "$S3_SECRET_KEY" && aws configure set region "$S3_REGION"
+    find ${BACKUP_DIR} -type d -name 'lost+found' -prune -o -maxdepth 2 -name "${TIMESTAMP}_*.sql.gz" -type f -exec aws --endpoint-url ${S3_ENDPOINT} s3 cp {} s3://${S3_BUCKETNAME} \;
+    rcu=$?
+    {{ end }}
+
     if [ "$rcu" != "0" ]; then
       echo "upload failed"
       exit 1