ci: 🎡 build the images before running the e2e tests

.github/workflows/_e2e_tests.yml

@@ -4,58 +4,26 @@
 name: Launch end-to-end tests
 on:
   workflow_call:
-    inputs:
-      config-file:
-        required: true
-        type: string
 env:
   python-version: 3.9.15
   poetry-version: 1.3.2
   # required to get access to use a cached poetry venv in "/home/runner/.cache/pypoetry/virtualenvs"
   POETRY_VIRTUALENVS_IN_PROJECT: false
   working-directory: e2e
 jobs:
-  get-config:
-    runs-on: ubuntu-latest
-    outputs:
-      dockerConfig: ${{ steps.set-var.outputs.dockerConfig }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          path: './'
-      - name: Get docker images
-        id: set-var
-        run: |
-          # from https://trstringer.com/github-actions-multiline-strings/
-          dockerConfig=`cat ${{ inputs.config-file }}`
-          # end of optional handling for multi line json
-          echo "dockerConfig<<EOF" >> $GITHUB_OUTPUT
-          echo "$dockerConfig" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
   e2e-tests:
-    needs: get-config
     defaults:
       run:
         shell: bash
         working-directory: e2e
     runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v3
-      - name: Install poetry
-        run: pipx install poetry==${{ env.poetry-version }}
-      - name: Use Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.python-version }}
-          cache: 'poetry'
-          cache-dependency-path: |
-            ${{ env.working-directory }}/poetry.lock
-      - name: Launch the services
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Build and launch the services (no cache)
         env:
-          IMAGE_REVERSE_PROXY: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.reverseProxy}}"
-          IMAGE_SERVICE_ADMIN: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.services.admin}}"
-          IMAGE_SERVICE_API: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.services.api}}"
-          IMAGE_WORKER_DATASETS_BASED: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.workers.datasets_based}}"
           WORKER_SLEEP_TIME: "1"
           # hard coded, see e2e/tests/fixtures/hub.py
           COMMON_HF_ENDPOINT: "https://hub-ci.huggingface.co"
@@ -69,13 +37,26 @@ jobs:
           ADMIN_UVICORN_NUM_WORKERS: "2"
           ADMIN_UVICORN_PORT: "8081"
           COMMON_LOG_LEVEL: "DEBUG"
-        working-directory: tools
-        run: docker compose -f ./docker-compose-datasets-server.yml up -d
-      - name: Install dependencies
+        run: docker compose -f docker-compose-datasets-server.yml up -d
+        working-directory: ./tools
+      -
+        name: Install poetry
+        run: pipx install poetry==${{ env.poetry-version }}
+      -
+        name: Use Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.python-version }}
+          cache: 'poetry'
+          cache-dependency-path: |
+            ${{ env.working-directory }}/poetry.lock
+      -
+        name: Install dependencies
         run: |
           poetry env use "${{ env.python-version }}"
           poetry install
-      - name: End-to-end tests
+      -
+        name: End-to-end tests
         env:
           QUEUE_SLEEP_TIME: "1"
           # hard coded, see e2e/tests/fixtures/hub.py

.github/workflows/e2e.yml

@@ -21,5 +21,3 @@ jobs:
       working-directory: e2e
   e2e-tests:
     uses: ./.github/workflows/_e2e_tests.yml
-    with:
-      config-file: chart/docker-images.yaml

Makefile

@@ -7,9 +7,9 @@ export COMPOSE_PROJECT_NAME := datasets-server
 
 # makefile variables
 DOCKER_COMPOSE := ./tools/docker-compose-datasets-server.yml
-DOCKER_IMAGES := ./chart/docker-images.yaml
+# DOCKER_IMAGES := ./chart/docker-images.yaml
 
-include tools/DockerRemoteImages.mk
+# include tools/DockerRemoteImages.mk
 include tools/Docker.mk
 
 .PHONY: start

e2e/Makefile

@@ -16,11 +16,11 @@ export COMMON_LOG_LEVEL := DEBUG
 
 # makefile variables
 DOCKER_COMPOSE := ../tools/docker-compose-datasets-server.yml
-DOCKER_IMAGES := ../chart/docker-images.yaml
+# DOCKER_IMAGES := ../chart/docker-images.yaml
 
 include ../tools/Python.mk
 include ../tools/PythonTest.mk
-include ../tools/DockerRemoteImages.mk
+# include ../tools/DockerRemoteImages.mk
 include ../tools/Docker.mk
 
 .PHONY: e2e

tools/docker-compose-datasets-server.yml

@@ -1,7 +1,8 @@
 version: "3.9"
 services:
   reverse-proxy:
-    image: ${IMAGE_REVERSE_PROXY?IMAGE_REVERSE_PROXY env var must be provided}
+    image: docker.io/nginx:1.20
+    # image: ${IMAGE_REVERSE_PROXY?IMAGE_REVERSE_PROXY env var must be provided}
     volumes:
       - ../chart/nginx-templates/:/etc/nginx/templates:ro
       - assets:${ASSETS_STORAGE_DIRECTORY-/assets}:ro
@@ -18,10 +19,10 @@ services:
       - api
       - admin
   admin:
-    # build:
-    #   context: ..
-    #   dockerfile: services/admin/Dockerfile
-    image: ${IMAGE_SERVICE_ADMIN?IMAGE_SERVICE_ADMIN env var must be provided}
+    build:
+      context: ..
+      dockerfile: services/admin/Dockerfile
+    # image: ${IMAGE_SERVICE_ADMIN?IMAGE_SERVICE_ADMIN env var must be provided}
     extends:
       file: docker-compose-base.yml
       service: common
@@ -45,10 +46,10 @@ services:
       # for debug
       - ${ADMIN_UVICORN_PORT-8081}:${ADMIN_UVICORN_PORT-8081}
   api:
-    # build:
-    #   context: ..
-    #   dockerfile: services/api/Dockerfile
-    image: ${IMAGE_SERVICE_API?IMAGE_SERVICE_API env var must be provided}
+    build:
+      context: ..
+      dockerfile: services/api/Dockerfile
+    # image: ${IMAGE_SERVICE_API?IMAGE_SERVICE_API env var must be provided}
     extends:
       file: docker-compose-base.yml
       service: common
@@ -70,10 +71,10 @@ services:
       - mongodb
     restart: unless-stopped
   worker-config-names:
-    # build:
-    #   context: ..
-    #   dockerfile: workers/datasets_based/Dockerfile
-    image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
+    build:
+      context: ..
+      dockerfile: workers/datasets_based/Dockerfile
+    # image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
     volumes:
       - splits-datasets-cache:${HF_DATASETS_CACHE-/datasets-cache}:rw
       - splits-modules-cache:${HF_DATASETS_CACHE-/modules-cache}:rw
@@ -87,10 +88,10 @@ services:
       - mongodb
     restart: always
   worker-split-names:
-    # build:
-    #   context: ..
-    #   dockerfile: workers/datasets_based/Dockerfile
-    image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
+    build:
+      context: ..
+      dockerfile: workers/datasets_based/Dockerfile
+    # image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
     volumes:
       - splits-datasets-cache:${HF_DATASETS_CACHE-/datasets-cache}:rw
       - splits-modules-cache:${HF_DATASETS_CACHE-/modules-cache}:rw
@@ -104,10 +105,10 @@ services:
       - mongodb
     restart: always
   worker-splits:
-    # build:
-    #   context: ..
-    #   dockerfile: workers/datasets_based/Dockerfile
-    image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
+    build:
+      context: ..
+      dockerfile: workers/datasets_based/Dockerfile
+    # image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
     volumes:
       - splits-datasets-cache:${HF_DATASETS_CACHE-/datasets-cache}:rw
       - splits-modules-cache:${HF_DATASETS_CACHE-/modules-cache}:rw
@@ -121,10 +122,10 @@ services:
       - mongodb
     restart: always
   worker-first-rows:
-    # build:
-    #   context: ..
-    #   dockerfile: workers/datasets_based/Dockerfile
-    image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
+    build:
+      context: ..
+      dockerfile: workers/datasets_based/Dockerfile
+    # image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
     volumes:
       - assets:${ASSETS_STORAGE_DIRECTORY-/assets}:rw
       - first-rows-datasets-cache:${HF_DATASETS_CACHE-/datasets-cache}:rw
@@ -148,10 +149,10 @@ services:
       - mongodb
     restart: always
   worker-parquet-and-dataset-info:
-    # build:
-    #   context: ..
-    #   dockerfile: workers/dataset_based/Dockerfile
-    image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
+    build:
+      context: ..
+      dockerfile: workers/datasets_based/Dockerfile
+    # image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
     volumes:
       - parquet-datasets-cache:${HF_DATASETS_CACHE-/datasets-cache}:rw
       - parquet-modules-cache:${HF_DATASETS_CACHE-/modules-cache}:rw
@@ -173,10 +174,10 @@ services:
       - mongodb
     restart: always
   worker-parquet:
-    # build:
-    #   context: ..
-    #   dockerfile: workers/dataset_based/Dockerfile
-    image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
+    build:
+      context: ..
+      dockerfile: workers/datasets_based/Dockerfile
+    # image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
     extends:
       file: docker-compose-base.yml
       service: datasets-worker
@@ -186,10 +187,10 @@ services:
       - mongodb
     restart: always
   worker-dataset-info:
-    # build:
-    #   context: ..
-    #   dockerfile: workers/dataset_based/Dockerfile
-    image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
+    build:
+      context: ..
+      dockerfile: workers/datasets_based/Dockerfile
+    # image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
     extends:
       file: docker-compose-base.yml
       service: datasets-worker
@@ -199,10 +200,10 @@ services:
       - mongodb
     restart: always
   worker-sizes:
-    # build:
-    #   context: ..
-    #   dockerfile: workers/dataset_based/Dockerfile
-    image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
+    build:
+      context: ..
+      dockerfile: workers/datasets_based/Dockerfile
+    # image: ${IMAGE_WORKER_DATASETS_BASED?IMAGE_WORKER_DATASETS_BASED env var must be provided}
     extends:
       file: docker-compose-base.yml
       service: datasets-worker
@@ -212,7 +213,7 @@ services:
       - mongodb
     restart: always
   mongodb:
-    image: mongo
+    image: docker.io/mongo
     volumes:
       - mongo:/data/db:rw
     ports: