Be more explicit about the current docker images

.github/workflows/_docker.yml

@@ -11,12 +11,11 @@ on:
       aws-secret-access-key:
         required: true
 env:
-  os: "ubuntu-latest"
   region: us-east-1
   repository-prefix: 707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-
 jobs:
   build-and-push-image:
-    runs-on: ${{ env.os }}
+    runs-on: "ubuntu-latest"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2

.github/workflows/_e2e_tests.yml

@@ -11,20 +11,36 @@ on:
       aws-secret-access-key:
         required: true
 env:
-  os: "ubuntu-latest"
   python-version: 3.9.6
   poetry-version: 1.1.13
   # required to get access to use a cached poetry venv in "/home/runner/.cache/pypoetry/virtualenvs"
   POETRY_VIRTUALENVS_IN_PROJECT: false
   region: us-east-1
   working-directory: e2e
 jobs:
+  get-config:
+    runs-on: ubuntu-latest
+    outputs:
+      dockerConfig: ${{ steps.set-var.outputs.dockerConfig }}
+    steps:
+    - name: Get docker images
+      id: set-var
+      run: |
+        # from https://stackoverflow.com/a/61919791/7351594
+        content=`cat ${{ inputs.config-file }}`
+        # the following lines are only required for multi line json
+        content="${content//'%'/'%25'}"
+        content="${content//$'\n'/'%0A'}"
+        content="${content//$'\r'/'%0D'}"
+        # end of optional handling for multi line json
+        echo "::set-output name=dockerConfig::$content"
   e2e-tests:
+    needs: get-config
     defaults:
       run:
         shell: bash
-        working-directory: ${{ env.working-directory }}
-    runs-on: ${{ env.os }}
+        working-directory: e2e
+    runs-on: "ubuntu-latest"
     steps:
       - uses: actions/checkout@v3
       - name: Configure AWS credentials
@@ -45,24 +61,13 @@ jobs:
           cache: 'poetry'
           cache-dependency-path: |
             ${{ env.working-directory }}/poetry.lock
-      - name: Get docker images
-        id: set_var
-        run: |
-          # from https://stackoverflow.com/a/61919791/7351594
-          content=`cat ${{ inputs.config-file }}`
-          # the following lines are only required for multi line json
-          content="${content//'%'/'%25'}"
-          content="${content//$'\n'/'%0A'}"
-          content="${content//$'\r'/'%0D'}"
-          # end of optional handling for multi line json
-          echo "::set-output name=dockerConfig::$content"
       - name: Launch the services
         env:
-          SERVICE_ADMIN_DOCKER_IMAGE: "${{fromJson(steps.set_var.outputs.dockerConfig).dockerImage.admin}}"
-          SERVICE_API_DOCKER_IMAGE: "${{fromJson(steps.set_var.outputs.dockerConfig).dockerImage.api}}"
-          SERVICE_DATASETS_WORKER_DOCKER_IMAGE: "${{fromJson(steps.set_var.outputs.dockerConfig).dockerImage.datasetsWorker}}"
-          SERVICE_REVERSE_PROXY_DOCKER_IMAGE: "${{fromJson(steps.set_var.outputs.dockerConfig).dockerImage.reverseProxy}}"
-          SERVICE_SPLITS_WORKER_DOCKER_IMAGE: "${{fromJson(steps.set_var.outputs.dockerConfig).dockerImage.splitsWorker}}"
+          SERVICE_ADMIN_DOCKER_IMAGE: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.admin}}"
+          SERVICE_API_DOCKER_IMAGE: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.api}}"
+          SERVICE_DATASETS_WORKER_DOCKER_IMAGE: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.datasetsWorker}}"
+          SERVICE_REVERSE_PROXY_DOCKER_IMAGE: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.reverseProxy}}"
+          SERVICE_SPLITS_WORKER_DOCKER_IMAGE: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.splitsWorker}}"
         run: docker-compose -f tests/docker-compose.yml up -d
       - name: Install dependencies
         run: |

.github/workflows/_quality-python.yml

@@ -16,14 +16,13 @@ env:
   POETRY_VIRTUALENVS_IN_PROJECT: false
   python-version: "3.9.6"
   poetry-version: "1.1.13"
-  os: "ubuntu-latest"
 jobs:
   code-quality:
     defaults:
       run:
         shell: bash
         working-directory: ${{ inputs.working-directory }}
-    runs-on: ${{ env.os }}
+    runs-on: "ubuntu-latest"
     steps:
       - uses: actions/checkout@v3
       - name: Install poetry

.github/workflows/_unit-tests-python.yml

@@ -14,14 +14,13 @@ env:
   mongo-port: "27017"
   python-version: "3.9.6"
   poetry-version: "1.1.13"
-  os: "ubuntu-latest"
 jobs:
   unit-tests:
     defaults:
       run:
         shell: bash
         working-directory: ${{ inputs.working-directory }}
-    runs-on: ${{ env.os }}
+    runs-on: "ubuntu-latest"
     steps:
       - uses: actions/checkout@v3
       - name: Install poetry

.github/workflows/e2e.yml

@@ -1,12 +1,11 @@
 name: e2e
 on:
+  workflow_dispatch:
   push:
     paths:
       - 'e2e/**'
-env:
-  working-directory: e2e
 jobs:
   quality:
-    uses: ./.github/workflows/quality-python.yml
+    uses: ./.github/workflows/_quality-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: e2e

.github/workflows/infra.yml

@@ -1,5 +1,6 @@
 name: infra
 on:
+  workflow_dispatch:
   push:
     paths:
       - 'infra/charts/datasets-server/**'

.github/workflows/l-libcache.yml

@@ -1,16 +1,15 @@
 name: libs/libcache
 on:
+  workflow_dispatch:
   push:
     paths:
       - 'libs/libcache/**'
-env:  
-  working-directory: libs/libcache
 jobs:
   quality:
     uses: ./.github/workflows/_quality-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: libs/libcache
   unit-tests:
     uses: ./.github/workflows/_unit-tests-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: libs/libcache

.github/workflows/l-libqueue.yml

@@ -1,16 +1,15 @@
 name: libs/libqueue
 on:
+  workflow_dispatch:
   push:
     paths:
       - 'libs/libqueue/**'
-env:
-  working-directory: libs/libqueue
 jobs:
   quality:
     uses: ./.github/workflows/_quality-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: libs/libqueue
   unit-tests:
     uses: ./.github/workflows/_unit-tests-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: libs/libqueue

.github/workflows/l-libutils.yml

@@ -1,16 +1,15 @@
 name: libs/libutils
 on:
+  workflow_dispatch:
   push:
     paths:
       - 'libs/libutils/**'
-env:
-  working-directory: libs/libutils
 jobs:
   quality:
     uses: ./.github/workflows/_quality-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: libs/libutils
   unit-tests:
     uses: ./.github/workflows/_unit-tests-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: libs/libutils

.github/workflows/s-admin.yaml

@@ -1,24 +1,22 @@
 name: services/admin
 on:
+  workflow_dispatch:
   push:
     paths:
       - 'services/admin/**'
-env:
-  service: admin
-  working-directory: services/admin
 jobs:
   quality:
     uses: ./.github/workflows/_quality-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: services/admin
   unit-tests:
     uses: ./.github/workflows/_unit-tests-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: services/admin
   docker:
     uses: ./.github/workflows/_docker.yml
     with:
-      service: ${{ env.service }}
+      service: admin
     secrets:
       aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
       aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/s-api.yaml

@@ -1,24 +1,22 @@
 name: services/api
 on:
+  workflow_dispatch:
   push:
     paths:
       - 'services/api/**'
-env:
-  service: api
-  working-directory: services/api
 jobs:
   quality:
     uses: ./.github/workflows/_quality-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: services/api
   unit-tests:
     uses: ./.github/workflows/_unit-tests-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: services/api
   docker:
     uses: ./.github/workflows/_docker.yml
     with:
-      service: ${{ env.service }}
+      service: api
     secrets:
       aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
       aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/s-worker.yml

@@ -1,28 +1,26 @@
 name: services/worker
 on:
+  workflow_dispatch:
   push:
     paths:
       - 'services/worker/**'
-env:
-  service: worker
-  working-directory: services/worker
 jobs:
   quality:
     uses: ./.github/workflows/_quality-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: services/worker
       # pillow <9.0.0
       safety-exceptions: "-i 44525 -i 44524 -i 44486 -i 44485 -i 45356 -i 44487"
       is_worker: true
   unit-tests:
     uses: ./.github/workflows/_unit-tests-python.yml
     with:
-      working-directory: ${{ env.working-directory }}
+      working-directory: services/worker
       is_worker: true
   docker:
     uses: ./.github/workflows/_docker.yml
     with:
-      service: ${{ env.service }}
+      service: worker
     secrets:
       aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
       aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

infra/charts/datasets-server/values.yaml

@@ -10,6 +10,14 @@ mongodb:
   # Name of the mongo db database used to store the jobs queue
   queueDatabase: "datasets_server_queue"
 
+# overriden by docker-images.yaml (which must be in JSON format!)
+dockerImage:
+  admin: ''
+  api: ''
+  datasetsWorker: ''
+  reverseProxy: ''
+  splitsWorker: ''
+
 storage:
   nfs: {}