From b2d5b4b50fa12e717ecbff9264674cbca27b0146 Mon Sep 17 00:00:00 2001 From: Test User Date: Fri, 3 Jun 2022 12:38:00 +0000 Subject: [PATCH 1/6] =?UTF-8?q?ci:=20=F0=9F=8E=A1=20fix=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 41a9b14914..16796f3bb9 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -7,6 +7,6 @@ env: working-directory: e2e jobs: quality: - uses: ./.github/workflows/quality-python.yml + uses: ./.github/workflows/_quality-python.yml with: working-directory: ${{ env.working-directory }} From b94182858dda23c659af977a92433228a629789f Mon Sep 17 00:00:00 2001 From: Test User Date: Fri, 3 Jun 2022 12:46:38 +0000 Subject: [PATCH 2/6] =?UTF-8?q?ci:=20=F0=9F=8E=A1=20try=20to=20fix=20the?= =?UTF-8?q?=20issue=20with=20env?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/huggingface/datasets-server/actions/runs/2433883992 --- .github/workflows/_e2e_tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/_e2e_tests.yml b/.github/workflows/_e2e_tests.yml index 37792249e5..fa2b81b620 100644 --- a/.github/workflows/_e2e_tests.yml +++ b/.github/workflows/_e2e_tests.yml @@ -10,16 +10,16 @@ on: required: true aws-secret-access-key: required: true -env: - os: "ubuntu-latest" - python-version: 3.9.6 - poetry-version: 1.1.13 - # required to get access to use a cached poetry venv in "/home/runner/.cache/pypoetry/virtualenvs" - POETRY_VIRTUALENVS_IN_PROJECT: false - region: us-east-1 - working-directory: e2e jobs: e2e-tests: + env: + os: "ubuntu-latest" + python-version: 3.9.6 + poetry-version: 1.1.13 + # required to get access to use a cached poetry venv in "/home/runner/.cache/pypoetry/virtualenvs" + POETRY_VIRTUALENVS_IN_PROJECT: false + region: us-east-1 + working-directory: e2e defaults: run: shell: bash From 1bd65857ee1e7d3a39fde6d4e46b28649dab3a49 Mon Sep 17 00:00:00 2001 From: Test User Date: Fri, 3 Jun 2022 12:47:42 +0000 Subject: [PATCH 3/6] =?UTF-8?q?ci:=20=F0=9F=8E=A1=20allow=20to=20manually?= =?UTF-8?q?=20launch=20the=20workflows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/e2e.yml | 1 + .github/workflows/infra.yml | 1 + .github/workflows/l-libcache.yml | 1 + .github/workflows/l-libqueue.yml | 1 + .github/workflows/l-libutils.yml | 1 + .github/workflows/s-admin.yaml | 1 + .github/workflows/s-api.yaml | 1 + .github/workflows/s-worker.yml | 1 + 8 files changed, 8 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 16796f3bb9..ee2063b207 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,5 +1,6 @@ name: e2e on: + workflow_dispatch: push: paths: - 'e2e/**' diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml index 506cf17ee3..d8340f415a 100644 --- a/.github/workflows/infra.yml +++ b/.github/workflows/infra.yml @@ -1,5 +1,6 @@ name: infra on: + workflow_dispatch: push: paths: - 'infra/charts/datasets-server/**' diff --git a/.github/workflows/l-libcache.yml b/.github/workflows/l-libcache.yml index e5e40c0ca4..61342fd26f 100644 --- a/.github/workflows/l-libcache.yml +++ b/.github/workflows/l-libcache.yml @@ -1,5 +1,6 @@ name: libs/libcache on: + workflow_dispatch: push: paths: - 'libs/libcache/**' diff --git a/.github/workflows/l-libqueue.yml b/.github/workflows/l-libqueue.yml index fdab80bc6f..39bdcb3b2b 100644 --- a/.github/workflows/l-libqueue.yml +++ b/.github/workflows/l-libqueue.yml @@ -1,5 +1,6 @@ name: libs/libqueue on: + workflow_dispatch: push: paths: - 'libs/libqueue/**' diff --git a/.github/workflows/l-libutils.yml b/.github/workflows/l-libutils.yml index 92a4aa61b2..d135c6b733 100644 --- a/.github/workflows/l-libutils.yml +++ b/.github/workflows/l-libutils.yml @@ -1,5 +1,6 @@ name: libs/libutils on: + workflow_dispatch: push: paths: - 'libs/libutils/**' diff --git a/.github/workflows/s-admin.yaml b/.github/workflows/s-admin.yaml index c8210fa3e7..1289fab956 100644 --- a/.github/workflows/s-admin.yaml +++ b/.github/workflows/s-admin.yaml @@ -1,5 +1,6 @@ name: services/admin on: + workflow_dispatch: push: paths: - 'services/admin/**' diff --git a/.github/workflows/s-api.yaml b/.github/workflows/s-api.yaml index 20161f9d1c..1766691834 100644 --- a/.github/workflows/s-api.yaml +++ b/.github/workflows/s-api.yaml @@ -1,5 +1,6 @@ name: services/api on: + workflow_dispatch: push: paths: - 'services/api/**' diff --git a/.github/workflows/s-worker.yml b/.github/workflows/s-worker.yml index 265ea75a9b..eaa1e77349 100644 --- a/.github/workflows/s-worker.yml +++ b/.github/workflows/s-worker.yml @@ -1,5 +1,6 @@ name: services/worker on: + workflow_dispatch: push: paths: - 'services/worker/**' From b2620453896b5c4e5dffb3062a3112331a4b50df Mon Sep 17 00:00:00 2001 From: Test User Date: Fri, 3 Jun 2022 12:59:22 +0000 Subject: [PATCH 4/6] =?UTF-8?q?ci:=20=F0=9F=8E=A1=20don't=20use=20env=20at?= =?UTF-8?q?=20"job"=20level?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the env can only be used at "step" level: see https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/6 --- .github/workflows/_docker.yml | 3 +-- .github/workflows/_e2e_tests.yml | 19 +++++++++---------- .github/workflows/_quality-python.yml | 3 +-- .github/workflows/_unit-tests-python.yml | 3 +-- .github/workflows/e2e.yml | 4 +--- .github/workflows/l-libcache.yml | 6 ++---- .github/workflows/l-libqueue.yml | 6 ++---- .github/workflows/l-libutils.yml | 6 ++---- .github/workflows/s-admin.yaml | 9 +++------ .github/workflows/s-api.yaml | 9 +++------ .github/workflows/s-worker.yml | 9 +++------ 11 files changed, 28 insertions(+), 49 deletions(-) diff --git a/.github/workflows/_docker.yml b/.github/workflows/_docker.yml index 20863ba045..3dd84a21fa 100644 --- a/.github/workflows/_docker.yml +++ b/.github/workflows/_docker.yml @@ -11,12 +11,11 @@ on: aws-secret-access-key: required: true env: - os: "ubuntu-latest" region: us-east-1 repository-prefix: 707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server- jobs: build-and-push-image: - runs-on: ${{ env.os }} + runs-on: "ubuntu-latest" steps: - name: Checkout repository uses: actions/checkout@v2 diff --git a/.github/workflows/_e2e_tests.yml b/.github/workflows/_e2e_tests.yml index fa2b81b620..7a3aa71af6 100644 --- a/.github/workflows/_e2e_tests.yml +++ b/.github/workflows/_e2e_tests.yml @@ -10,21 +10,20 @@ on: required: true aws-secret-access-key: required: true +env: + python-version: 3.9.6 + poetry-version: 1.1.13 + # required to get access to use a cached poetry venv in "/home/runner/.cache/pypoetry/virtualenvs" + POETRY_VIRTUALENVS_IN_PROJECT: false + region: us-east-1 + working-directory: e2e jobs: e2e-tests: - env: - os: "ubuntu-latest" - python-version: 3.9.6 - poetry-version: 1.1.13 - # required to get access to use a cached poetry venv in "/home/runner/.cache/pypoetry/virtualenvs" - POETRY_VIRTUALENVS_IN_PROJECT: false - region: us-east-1 - working-directory: e2e defaults: run: shell: bash - working-directory: ${{ env.working-directory }} - runs-on: ${{ env.os }} + working-directory: e2e + runs-on: "ubuntu-latest" steps: - uses: actions/checkout@v3 - name: Configure AWS credentials diff --git a/.github/workflows/_quality-python.yml b/.github/workflows/_quality-python.yml index a87efa7814..f9a90c1002 100644 --- a/.github/workflows/_quality-python.yml +++ b/.github/workflows/_quality-python.yml @@ -16,14 +16,13 @@ env: POETRY_VIRTUALENVS_IN_PROJECT: false python-version: "3.9.6" poetry-version: "1.1.13" - os: "ubuntu-latest" jobs: code-quality: defaults: run: shell: bash working-directory: ${{ inputs.working-directory }} - runs-on: ${{ env.os }} + runs-on: "ubuntu-latest" steps: - uses: actions/checkout@v3 - name: Install poetry diff --git a/.github/workflows/_unit-tests-python.yml b/.github/workflows/_unit-tests-python.yml index fd04b1d285..2dc7f6a210 100644 --- a/.github/workflows/_unit-tests-python.yml +++ b/.github/workflows/_unit-tests-python.yml @@ -14,14 +14,13 @@ env: mongo-port: "27017" python-version: "3.9.6" poetry-version: "1.1.13" - os: "ubuntu-latest" jobs: unit-tests: defaults: run: shell: bash working-directory: ${{ inputs.working-directory }} - runs-on: ${{ env.os }} + runs-on: "ubuntu-latest" steps: - uses: actions/checkout@v3 - name: Install poetry diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ee2063b207..60b9d5c1d2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -4,10 +4,8 @@ on: push: paths: - 'e2e/**' -env: - working-directory: e2e jobs: quality: uses: ./.github/workflows/_quality-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: e2e diff --git a/.github/workflows/l-libcache.yml b/.github/workflows/l-libcache.yml index 61342fd26f..c2ccf945d1 100644 --- a/.github/workflows/l-libcache.yml +++ b/.github/workflows/l-libcache.yml @@ -4,14 +4,12 @@ on: push: paths: - 'libs/libcache/**' -env: - working-directory: libs/libcache jobs: quality: uses: ./.github/workflows/_quality-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: libs/libcache unit-tests: uses: ./.github/workflows/_unit-tests-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: libs/libcache diff --git a/.github/workflows/l-libqueue.yml b/.github/workflows/l-libqueue.yml index 39bdcb3b2b..912ac4fb56 100644 --- a/.github/workflows/l-libqueue.yml +++ b/.github/workflows/l-libqueue.yml @@ -4,14 +4,12 @@ on: push: paths: - 'libs/libqueue/**' -env: - working-directory: libs/libqueue jobs: quality: uses: ./.github/workflows/_quality-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: libs/libqueue unit-tests: uses: ./.github/workflows/_unit-tests-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: libs/libqueue diff --git a/.github/workflows/l-libutils.yml b/.github/workflows/l-libutils.yml index d135c6b733..5309efd10f 100644 --- a/.github/workflows/l-libutils.yml +++ b/.github/workflows/l-libutils.yml @@ -4,14 +4,12 @@ on: push: paths: - 'libs/libutils/**' -env: - working-directory: libs/libutils jobs: quality: uses: ./.github/workflows/_quality-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: libs/libutils unit-tests: uses: ./.github/workflows/_unit-tests-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: libs/libutils diff --git a/.github/workflows/s-admin.yaml b/.github/workflows/s-admin.yaml index 1289fab956..673b41cb84 100644 --- a/.github/workflows/s-admin.yaml +++ b/.github/workflows/s-admin.yaml @@ -4,22 +4,19 @@ on: push: paths: - 'services/admin/**' -env: - service: admin - working-directory: services/admin jobs: quality: uses: ./.github/workflows/_quality-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: services/admin unit-tests: uses: ./.github/workflows/_unit-tests-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: services/admin docker: uses: ./.github/workflows/_docker.yml with: - service: ${{ env.service }} + service: admin secrets: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/s-api.yaml b/.github/workflows/s-api.yaml index 1766691834..37056958fc 100644 --- a/.github/workflows/s-api.yaml +++ b/.github/workflows/s-api.yaml @@ -4,22 +4,19 @@ on: push: paths: - 'services/api/**' -env: - service: api - working-directory: services/api jobs: quality: uses: ./.github/workflows/_quality-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: services/api unit-tests: uses: ./.github/workflows/_unit-tests-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: services/api docker: uses: ./.github/workflows/_docker.yml with: - service: ${{ env.service }} + service: api secrets: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/s-worker.yml b/.github/workflows/s-worker.yml index eaa1e77349..d38e0395dd 100644 --- a/.github/workflows/s-worker.yml +++ b/.github/workflows/s-worker.yml @@ -4,26 +4,23 @@ on: push: paths: - 'services/worker/**' -env: - service: worker - working-directory: services/worker jobs: quality: uses: ./.github/workflows/_quality-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: services/worker # pillow <9.0.0 safety-exceptions: "-i 44525 -i 44524 -i 44486 -i 44485 -i 45356 -i 44487" is_worker: true unit-tests: uses: ./.github/workflows/_unit-tests-python.yml with: - working-directory: ${{ env.working-directory }} + working-directory: services/worker is_worker: true docker: uses: ./.github/workflows/_docker.yml with: - service: ${{ env.service }} + service: worker secrets: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 7705991400cf8fdab8b29e0e3be8b71ab3135e82 Mon Sep 17 00:00:00 2001 From: Test User Date: Fri, 3 Jun 2022 13:04:59 +0000 Subject: [PATCH 5/6] =?UTF-8?q?fix:=20=F0=9F=90=9B=20add=20a=20default=20v?= =?UTF-8?q?alue,=20overriden=20by=20docker-images.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infra/charts/datasets-server/values.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/infra/charts/datasets-server/values.yaml b/infra/charts/datasets-server/values.yaml index 4c70fdc8c7..7f4fa81e69 100644 --- a/infra/charts/datasets-server/values.yaml +++ b/infra/charts/datasets-server/values.yaml @@ -10,6 +10,14 @@ mongodb: # Name of the mongo db database used to store the jobs queue queueDatabase: "datasets_server_queue" +# overriden by docker-images.yaml (which must be in JSON format!) +dockerImage: + admin: '' + api: '' + datasetsWorker: '' + reverseProxy: '' + splitsWorker: '' + storage: nfs: {} From 598f69477517f5193b8da70bc54680c6078e438f Mon Sep 17 00:00:00 2001 From: Test User Date: Fri, 3 Jun 2022 13:13:40 +0000 Subject: [PATCH 6/6] =?UTF-8?q?ci:=20=F0=9F=8E=A1=20try=20to=20fix=20the?= =?UTF-8?q?=20config=20loading=20from=20a=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I have the intuition it comes from the "working dir" in the second job, which is why I created a previous job at the root --- .github/workflows/_e2e_tests.yml | 38 ++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/_e2e_tests.yml b/.github/workflows/_e2e_tests.yml index 7a3aa71af6..a901da2740 100644 --- a/.github/workflows/_e2e_tests.yml +++ b/.github/workflows/_e2e_tests.yml @@ -18,7 +18,24 @@ env: region: us-east-1 working-directory: e2e jobs: + get-config: + runs-on: ubuntu-latest + outputs: + dockerConfig: ${{ steps.set-var.outputs.dockerConfig }} + steps: + - name: Get docker images + id: set-var + run: | + # from https://stackoverflow.com/a/61919791/7351594 + content=`cat ${{ inputs.config-file }}` + # the following lines are only required for multi line json + content="${content//'%'/'%25'}" + content="${content//$'\n'/'%0A'}" + content="${content//$'\r'/'%0D'}" + # end of optional handling for multi line json + echo "::set-output name=dockerConfig::$content" e2e-tests: + needs: get-config defaults: run: shell: bash @@ -44,24 +61,13 @@ jobs: cache: 'poetry' cache-dependency-path: | ${{ env.working-directory }}/poetry.lock - - name: Get docker images - id: set_var - run: | - # from https://stackoverflow.com/a/61919791/7351594 - content=`cat ${{ inputs.config-file }}` - # the following lines are only required for multi line json - content="${content//'%'/'%25'}" - content="${content//$'\n'/'%0A'}" - content="${content//$'\r'/'%0D'}" - # end of optional handling for multi line json - echo "::set-output name=dockerConfig::$content" - name: Launch the services env: - SERVICE_ADMIN_DOCKER_IMAGE: "${{fromJson(steps.set_var.outputs.dockerConfig).dockerImage.admin}}" - SERVICE_API_DOCKER_IMAGE: "${{fromJson(steps.set_var.outputs.dockerConfig).dockerImage.api}}" - SERVICE_DATASETS_WORKER_DOCKER_IMAGE: "${{fromJson(steps.set_var.outputs.dockerConfig).dockerImage.datasetsWorker}}" - SERVICE_REVERSE_PROXY_DOCKER_IMAGE: "${{fromJson(steps.set_var.outputs.dockerConfig).dockerImage.reverseProxy}}" - SERVICE_SPLITS_WORKER_DOCKER_IMAGE: "${{fromJson(steps.set_var.outputs.dockerConfig).dockerImage.splitsWorker}}" + SERVICE_ADMIN_DOCKER_IMAGE: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.admin}}" + SERVICE_API_DOCKER_IMAGE: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.api}}" + SERVICE_DATASETS_WORKER_DOCKER_IMAGE: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.datasetsWorker}}" + SERVICE_REVERSE_PROXY_DOCKER_IMAGE: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.reverseProxy}}" + SERVICE_SPLITS_WORKER_DOCKER_IMAGE: "${{fromJson(needs.get-config.outputs.dockerConfig).dockerImage.splitsWorker}}" run: docker-compose -f tests/docker-compose.yml up -d - name: Install dependencies run: |