feat: 🎸 add the admin service (to run admin scripts) (#275)

* feat: 🎸 add the admin service (to run admin scripts)

* feat: 🎸 add admin service to kube

* docs: ✏️ fix doc

* docs: ✏️ run the scripts directly using docker or kubectl exec

.github/workflows/docker.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        service: [api, worker]
+        service: [admin, api, worker]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository

.github/workflows/quality.yml

@@ -10,7 +10,7 @@ jobs:
         poetry-version: [1.1.7]
         os: [ubuntu-latest]
         working-directory:
-          [e2e, services/api, libs/libcache, libs/libqueue, libs/libutils]
+          [e2e, services/admin, services/api, libs/libcache, libs/libqueue, libs/libutils]
     defaults:
       run:
         shell: bash

.github/workflows/unit-tests.yml

@@ -11,7 +11,7 @@ jobs:
         poetry-version: [1.1.7]
         os: [ubuntu-latest]
         working-directory:
-          [services/api, libs/libcache, libs/libqueue, libs/libutils]
+          [services/admin, services/api, libs/libcache, libs/libqueue, libs/libutils]
     defaults:
       run:
         shell: bash

.vscode/monorepo.code-workspace

@@ -16,6 +16,10 @@
       "name": "libs/libutils",
       "path": "../libs/libutils"
     },
+    {
+      "name": "services/admin",
+      "path": "../services/admin"
+    },
     {
       "name": "services/api",
       "path": "../services/api"
@@ -37,7 +41,7 @@
     "python.formatting.provider": "black",
     "python.linting.enabled": true,
     "python.linting.mypyEnabled": true,
-    "python.linting.flake8Enabled": true,
+    "python.linting.flake8Enabled": true
   },
   "extensions": {
     "recommendations": [

Makefile

@@ -3,6 +3,7 @@
 install:
 	$(MAKE) -C services/worker/ install
 	$(MAKE) -C services/api/ install
+	$(MAKE) -C services/admin/ install
 
 .PHONY: lock
 lock:
@@ -11,6 +12,7 @@ lock:
 	$(MAKE) -C libs/libcache/ lock
 	$(MAKE) -C services/worker/ lock
 	$(MAKE) -C services/api/ lock
+	$(MAKE) -C services/admin/ lock
 
 .PHONY: api
 api:
@@ -22,6 +24,7 @@ worker:
 
 .PHONY: test
 test:
+	$(MAKE) -C services/admin/ test
 	$(MAKE) -C services/worker/ test
 	$(MAKE) -C services/api/ test
 	$(MAKE) -C libs/libcache/ test
@@ -30,6 +33,7 @@ test:
 
 .PHONY: coverage
 coverage:
+	$(MAKE) -C services/admin/ coverage
 	$(MAKE) -C services/worker/ coverage
 	$(MAKE) -C services/api/ coverage
 	$(MAKE) -C libs/libcache/ coverage
@@ -43,6 +47,7 @@ quality:
 	$(MAKE) -C infra/charts/datasets-server/ quality
 	$(MAKE) -C services/worker/ quality
 	$(MAKE) -C services/api/ quality
+	$(MAKE) -C services/admin/ quality
 	$(MAKE) -C libs/libcache/ quality
 	$(MAKE) -C libs/libqueue/ quality
 	$(MAKE) -C libs/libutils/ quality
@@ -53,6 +58,7 @@ style:
 	$(MAKE) -C e2e/ style
 	$(MAKE) -C services/worker/ style
 	$(MAKE) -C services/api/ style
+	$(MAKE) -C services/admin/ style
 	$(MAKE) -C libs/libcache/ style
 	$(MAKE) -C libs/libqueue/ style
 	$(MAKE) -C libs/libutils/ style

docker-compose.yml

@@ -66,6 +66,16 @@ services:
       mongodb:
         condition: service_started
     restart: always
+  admin:
+    build:
+      context: .
+      dockerfile: ./services/admin/Dockerfile
+    environment:
+      MONGO_URL: "mongodb://mongodb"
+    depends_on:
+      mongodb:
+        condition: service_started
+    restart: always
   mongodb:
     image: mongo
     volumes:

infra/charts/datasets-server/env/dev.yaml

@@ -61,3 +61,12 @@ splitsWorker:
       cpu: 0.01
     limits:
       cpu: 1
+
+admin:
+  replicas: 1
+
+  resources:
+    requests:
+      cpu: 0.01
+    limits:
+      cpu: 1

infra/charts/datasets-server/env/prod.yaml

@@ -113,3 +113,18 @@ splitsWorker:
 
   # Log level
   logLevel: "DEBUG"
+
+admin:
+  replicas: 1
+
+  nodeSelector:
+    role-datasets-server: 'true'
+
+  resources:
+    requests:
+      cpu: 0.01
+    limits:
+      cpu: 1
+
+  # Log level
+  logLevel: "DEBUG"

infra/charts/datasets-server/templates/_helpers.tpl

@@ -62,6 +62,11 @@ app: "{{ include "release" . }}-datasets-worker"
 app: "{{ include "release" . }}-splits-worker"
 {{- end -}}
 
+{{- define "labels.admin" -}}
+{{ include "labels" . }}
+app: "{{ include "release" . }}-admin"
+{{- end -}}
+
 {{/*
 The assets/ subpath in the NFS
 - in a subdirectory named as the chart (datasets-server/), and below it,

infra/charts/datasets-server/templates/admin/_container.tpl

@@ -0,0 +1,41 @@
+{{- define "containerAdmin" -}}
+- name: "{{ include "name" . }}-admin"
+  env:
+  - name: ASSETS_DIRECTORY
+    value: {{ .Values.splitsWorker.assetsDirectory | quote }}
+  - name: LOG_LEVEL
+    value: {{ .Values.splitsWorker.logLevel | quote }}
+  - name: MONGO_CACHE_DATABASE
+    value: {{ .Values.mongodb.cacheDatabase | quote }}
+  - name: MONGO_QUEUE_DATABASE
+    value: {{ .Values.mongodb.queueDatabase | quote }}
+  - name: MONGO_URL
+  {{- if .Values.mongodb.enabled }}
+    value: mongodb://{{.Release.Name}}-mongodb
+  {{- else }}
+    valueFrom:
+      secretKeyRef:
+        name: {{ .Values.secrets.mongoUrl | quote }}
+        key: MONGO_URL
+        optional: false
+  {{- end }}
+  image: "{{ .Values.admin.image.repository }}/{{ .Values.admin.image.name }}:{{ .Values.docker.tag }}"
+  imagePullPolicy: {{ .Values.admin.image.pullPolicy }}
+  volumeMounts:
+  - mountPath: {{ .Values.admin.assetsDirectory | quote }}
+    mountPropagation: None
+    name: nfs
+    subPath: "{{ include "assets.subpath" . }}"
+    readOnly: false
+  securityContext:
+    allowPrivilegeEscalation: false
+  # TODO: provide readiness and liveness probes
+  # readinessProbe:
+  #   tcpSocket:
+  #     port: {{ .Values.admin.readinessPort }}
+  # livenessProbe:
+  #   tcpSocket:
+  #     port: {{ .Values.admin.readinessPort }}
+  resources:
+    {{ toYaml .Values.admin.resources | nindent 4 }}
+{{- end -}}

infra/charts/datasets-server/templates/admin/deployment.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    {{ include "labels.admin" . | nindent 4 }}
+  name: "{{ include "release" . }}-admin"
+  namespace: {{ .Release.Namespace }}
+spec:
+  progressDeadlineSeconds: 600
+  replicas: {{ .Values.admin.replicas }}
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      {{ include "labels.admin" . | nindent 6 }}
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        {{ include "labels.admin" . | nindent 8 }}
+    spec:
+      initContainers:
+        {{ include "initContainerAssets" . | nindent 8 }}
+      containers:
+        {{ include "containerAdmin" . | nindent 8 }}
+      nodeSelector:
+        {{ toYaml .Values.admin.nodeSelector | nindent 8 }}
+      tolerations:
+        {{ toYaml .Values.admin.tolerations | nindent 8 }}
+      volumes:
+        - name: nfs
+          nfs:
+            server: {{ .Values.storage.nfs.server }}
+            path: {{ .Values.storage.nfs.path }}
+      securityContext:
+        runAsUser: {{ .Values.uid }}
+        runAsGroup: {{ .Values.gid }}
+        runAsNonRoot: true

infra/charts/datasets-server/values.yaml

@@ -28,7 +28,7 @@ gid: 3000
 datasetsBlocklist: ""
 
 docker:
-  tag: sha-8a7c036
+  tag: sha-b1ed584
 
 reverseProxy:
   image:
@@ -192,3 +192,25 @@ splitsWorker:
   rowsMinNumber: 10
   # Number of seconds a worker will sleep before trying to process a new job
   workerSleepSeconds: 15
+
+
+admin:
+  image:
+    repository: 707930574880.dkr.ecr.us-east-1.amazonaws.com
+    name: hub-datasets-server-admin
+    pullPolicy: IfNotPresent
+
+  replicas: 1
+
+  resources:
+    requests:
+      cpu: 1
+    limits:
+      cpu: 1
+  nodeSelector: {}
+  tolerations: []
+
+  # Directory of assets (audio files and images that will be served for the web)
+  assetsDirectory: "/assets"
+  # Log level
+  logLevel: "INFO"