Update K8s example

helm/charts/aleph/templates/aleph-upgrade-job.yaml

@@ -82,6 +82,7 @@ spec:
                 secretKeyRef:
                   name: aleph-secrets
                   key: SENTRY_DSN
+                  optional: true
             {{ if .Values.global.google }}
             - name: GOOGLE_APPLICATION_CREDENTIALS
               value: /var/secrets/google/service-account.json

helm/charts/aleph/templates/api.yaml

@@ -91,6 +91,7 @@ spec:
                 secretKeyRef:
                   name: aleph-secrets
                   key: SENTRY_DSN
+                  optional: true
             {{ if .Values.global.google }}
             - name: GOOGLE_APPLICATION_CREDENTIALS
               value: /var/secrets/google/service-account.json

helm/charts/aleph/templates/ingest-file.yaml

@@ -62,6 +62,7 @@ spec:
                 secretKeyRef:
                   name: aleph-secrets
                   key: SENTRY_DSN
+                  optional: true
             {{ if .Values.global.google }}
             - name: GOOGLE_APPLICATION_CREDENTIALS
               value: /var/secrets/google/service-account.json

helm/charts/aleph/templates/worker.yaml

@@ -76,6 +76,7 @@ spec:
                 secretKeyRef:
                   name: aleph-secrets
                   key: SENTRY_DSN
+                  optional: true
             {{ if .Values.global.google }}
             - name: GOOGLE_APPLICATION_CREDENTIALS
               value: /var/secrets/google/service-account.json

helm/charts/aleph/values.yaml

@@ -117,7 +117,7 @@ ingestfile:
 
   image:
     repository: ghcr.io/alephdata/ingest-file
-    tag: "3.18.4"
+    tag: "3.19.2"
     pullPolicy: Always
 
   containerSecurityContext:

helm/examples/dev/.gitignore

@@ -0,0 +1 @@
+charts/

helm/examples/dev/Chart.lock

@@ -0,0 +1,21 @@
+dependencies:
+- name: aleph
+  repository: file://../../charts/aleph
+  version: 3.15.4
+- name: ingress-nginx
+  repository: https://kubernetes.github.io/ingress-nginx
+  version: 4.8.3
+- name: postgresql
+  repository: https://charts.bitnami.com/bitnami
+  version: 12.10.1
+- name: elasticsearch
+  repository: https://helm.elastic.co
+  version: 7.17.3
+- name: minio
+  repository: https://helm.min.io/
+  version: 8.0.10
+- name: redis
+  repository: https://charts.bitnami.com/bitnami
+  version: 18.0.2
+digest: sha256:cfaa9778a9da148217fe65d2bc9eebed248436f932e35b7ac9414321eb7e8f4d
+generated: "2023-11-10T12:47:21.424991+01:00"

helm/examples/dev/Chart.yaml

@@ -0,0 +1,30 @@
+apiVersion: v2
+type: application
+name: aleph-dev
+version: 0.1.0
+dependencies: 
+  - name: aleph
+    repository: file://../../charts/aleph
+    version: 3.15.4
+    condition: aleph.enabled
+
+  - name: ingress-nginx
+    repository: https://kubernetes.github.io/ingress-nginx
+    version: 4.8.3
+
+  - name: postgresql
+    repository: https://charts.bitnami.com/bitnami
+    version: v12.10.1
+
+  - name: elasticsearch
+    alias: elasticsearch
+    repository: https://helm.elastic.co
+    version: v7.17.3
+
+  - name: minio
+    repository: https://helm.min.io/
+    version: v8.0.10
+
+  - name: redis
+    repository: https://charts.bitnami.com/bitnami
+    version: v18.0.2

helm/examples/dev/README.md

@@ -0,0 +1,95 @@
+# Kubernetes Development Environment
+
+The example in this directory shows how to install Aleph using Kubernetes on your development machine.
+
+## Prerequisites
+
+* You need to run a Kuberentes cluster on your development machine. A simple way to run a single-node Kubernetes cluster on Linux, macOS, and Windows is using Docker Desktop. Follow the steps outlined in the [Docker Desktop documentation](https://docs.docker.com/desktop/kubernetes/) to enable Kubernetes.
+
+* If you’re already using the `kubectl` CLI with other clusters, make sure to switch to the `docker-dekstop` context:
+
+  ```
+  kubectl config use-context docker-desktop
+  ```
+
+* Running this example may require up to 8GB of memory, so make sure to [update the resource settings](https://docs.docker.com/desktop/settings/mac/#resources) in Docker Desktop if necessary.
+
+* You also need Helm, a tool for packaging Kubernetes resources. Please refer to the [Helm documentation](https://helm.sh/docs/intro/install/) for instructions on how to install Helm on your machine.
+
+## Installation
+
+Once you have installed Helm and have a Kubernetes cluster running on your machine, follow the following steps to install Aleph and dependent services in your cluster.
+
+### Create secrets
+
+Some sensitive configuration options should be stored using Kubernetes secrets. These configuration options are stored in files in the `secrets` directory.
+
+> [!IMPORTANT]  
+> In a production setting, you have to encrypt the secret files, e.g. using [git-crypt](https://github.com/AGWA/git-crypt). Alternatively, you can configure an external service to retrieve secrets from. For more information, refer to [Secrets Best Practices](https://kubernetes.io/docs/concepts/security/secrets-good-practices/).
+
+Use the `kubectl` CLI to create `Secret` objects based on the files in the `secrets` directory:
+
+```
+kubectl create secret generic aleph-secrets --from-file=./secrets/aleph
+kubectl create secret generic postgresql-secrets --from-file=./secrets/postgresql
+kubectl create secret generic minio-secrets --from-file=./secrets/minio
+```
+
+### Install services
+
+Aleph depends on a number of services, including PostgreSQL and Elasticsearch. Installing Aleph before these services are available will cause errors. Run the following command to install all services required by Aleph, but not Aleph itself:
+
+```
+helm install --dependency-update --set "aleph.enabled=false" aleph .
+```
+
+This command downloads Helm charts for dependencies listed in the `Chart.yaml` file in this directory and installs everything in your local Kubernetes cluster, except for Aleph itself.
+
+Once you’ve run the command above, you can run `watch kubectl get pods` to observe how Kubernetes pods for the multiple services are started. Wait until everything is running and ready.
+
+### Install Aleph
+
+Next, run the following command to install Aleph:
+
+```
+helm upgrade --set "aleph.enabled=true" aleph .
+```
+
+This will create all Kubernetes resources for Aleph itself and runs SQL and Elasticsearch migrations. This may take a few minutes.
+
+### Open the Aleph UI
+
+Wait until all pods are running and ready, then open `http://kubernetes.docker.internal` in your web browser. You should see the Aleph homepage.
+
+> [!NOTE]
+> Usually Docker Desktop should automatically add an entry to `/etc/hosts` to resolve `kubernetes.docker.internal`. If `kubernetes.docker.internal` can’t be resolved, you may need to manually add an entry to `/etc/hosts`:
+>
+> ```
+> 127.0.0.1 kubernetes.docker.internal
+> ```
+
+## Upgrading
+
+In order to upgrade your installation after you’ve made changes to the Aleph Helm chart run:
+
+```
+helm upgrade --set "aleph.enabled=true" --dependency-update aleph .
+```
+
+## Creating a user
+
+Run the following command to open a shell inside of the Aleph API container:
+
+```
+kubectl exec -it svc/aleph-api -- bash
+```
+
+In order to create a new (admin) user run:
+
+```
+aleph createuser --name "Test User" --password "12345678" --admin mail@example.org
+```
+
+## Viewing and downloading files
+
+Because the MinIO endpoint is only accessible from within the cluster network, you won’t be able to preview or download files from the Aleph UI.

helm/examples/dev/secrets/aleph/ALEPH_DATABASE_URI

@@ -0,0 +1 @@
+postgresql://aleph:aleph@aleph-postgresql.default.svc/aleph

helm/examples/dev/secrets/aleph/FTM_STORE_URI

@@ -0,0 +1 @@
+postgresql://aleph:aleph@aleph-postgresql.default.svc/aleph

helm/examples/dev/secrets/postgresql/adminPassword

@@ -0,0 +1 @@
+admin

helm/examples/dev/secrets/postgresql/userPassword

@@ -0,0 +1 @@
+aleph

helm/examples/kind/k8s/ingress.dev.yaml → helm/examples/dev/templates/ingress.yaml

@@ -1,11 +1,12 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
-  name: aleph-ingress-dev
-  namespace: dev
+  name: aleph-ingress
+  namespace: default
 spec:
+  ingressClassName: "nginx"
   rules:
-    - host: aleph.devel
+    - host: kubernetes.docker.internal
       http:
         paths:
           - path: /

helm/examples/dev/values.yaml

@@ -0,0 +1,120 @@
+aleph:
+  global:
+    commonEnv:
+      REDIS_URL: redis://aleph-redis-master.default.svc.cluster.local:6379/0
+      ARCHIVE_TYPE: s3
+      ARCHIVE_BUCKET: aleph-archive
+      ARCHIVE_ENDPOINT_URL: http://aleph-minio.default.svc.cluster.local:9000
+    env:
+      ALEPH_UI_URL: http://kubernetes.docker.internal/
+      ALEPH_ELASTICSEARCH_URI: http://aleph-index-master.default.svc.cluster.local:9200
+
+  ui:
+    replicas: 1
+
+  api:
+    hpa:
+      minReplicas: 1
+      maxReplicas: 1
+
+    containerResources:
+      requests:
+        memory: 1000Mi
+      limits:
+        memory: 1000Mi
+
+  ingestfile:
+    hpa:
+      minReplicas: 1
+      maxreplicas: 1
+
+    containerResources:
+      requests:
+        memory: 1000Mi
+      limits:
+        memory: 1000Mi
+
+  worker:
+    replicas: 1
+
+    containerResources:
+      requests:
+        cpu: 30m
+        memory: 500Mi
+      limits:
+        memory: 500Mi
+
+postgresql:
+  auth:
+    database: aleph
+    username: aleph
+    existingSecret: postgresql-secrets
+    secretKeys:
+      adminPasswordKey: adminPassword
+      userPasswordKey: userPassword
+
+  resources:
+    requests:
+      memory: 500Mi
+    limits:
+      memory: 500Mi
+
+elasticsearch:
+  clusterName: "aleph-index"
+
+  replicas: 1
+  minimumMasterNodes: 1
+
+  image: "ghcr.io/alephdata/aleph-elasticsearch"
+  imageTag: "3bb5dbed97cfdb9955324d11e5c623a5c5bbc410"
+  imagePullPolicy: "IfNotPresent"
+
+  esJavaOpts: "-Xmx2g -Xms2g"
+  esConfig:
+    elasticsearch.yml: |
+      xpack:
+        graph:
+          enabled: "false"
+        ml:
+          enabled: "false"
+        security:
+          enabled: "false"
+
+  resources:
+    requests:
+      cpu: 1000m
+      memory: 3Gi
+    limits:
+      memory: 3Gi
+
+  volumeClaimTemplate:
+    accessModes:
+      - "ReadWriteOnce"
+    resources:
+      requests:
+        storage: 5Gi
+
+  clusterHealthCheckParams: "wait_for_status=red&timeout=1s"
+
+minio:
+  mode: standalone
+  replicas: 1
+  existingSecret: minio-secrets
+
+  resources:
+    requests:
+      memory: 500Mi
+    limits:
+      memory: 500Mi
+
+redis:
+  architecture: standalone
+
+  auth:
+    enabled: false
+
+  resources:
+    requests:
+      memory: 500Mi
+    limits:
+      memory: 500Mi