(PC-34427)[PRO] feat: parallelized e2e dispatch manualy to not record…

… on cloud
pass-culture · Feb 7, 2025 · a4ad7e7 · a4ad7e7
1 parent fe519f8
commit a4ad7e7
Show file tree

Hide file tree

Showing 3 changed files with 105 additions and 50 deletions.
diff --git a/.github/workflows/dev_on_pull_request_workflow.yml b/.github/workflows/dev_on_pull_request_workflow.yml
@@ -229,7 +229,7 @@ jobs:
       GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }}
 
   test-pro-e2e:
-    name: "Tests pro E2E"
+    name: "Pro E2E Tests"
     needs: [pcapi-init-job, build-pcapi]
     uses: ./.github/workflows/dev_on_workflow_tests_pro_e2e.yml
     if: always() &&

diff --git a/.github/workflows/dev_on_push_workflow_main.yml b/.github/workflows/dev_on_push_workflow_main.yml
@@ -226,7 +226,7 @@ jobs:
       GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }}
 
   test-pro-e2e:
-    name: "Tests pro E2E"
+    name: "Pro E2E Tests"
     needs: [pcapi-init-job, build-pcapi]
     uses: ./.github/workflows/dev_on_workflow_tests_pro_e2e.yml
     if: |

diff --git a/.github/workflows/dev_on_workflow_tests_pro_e2e.yml b/.github/workflows/dev_on_workflow_tests_pro_e2e.yml
@@ -27,17 +27,79 @@ defaults:
     working-directory: pro
 
 jobs:
-  tests-pro-e2e-tests:
-    name: "E2E tests and notifications"
+  install:
+    name: "Install E2E 🔧"
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4.2.2
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - uses: technote-space/workflow-conclusion-action@v3
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: "pro/.nvmrc"
+
+      - name: "Cache the node_modules"
+        id: "yarn-modules-cache"
+        uses: pass-culture-github-actions/cache@v1.0.0
+        with:
+          compression-method: "gzip"
+          bucket: ${{ inputs.CACHE_BUCKET_NAME }}
+          path: |
+            ~/.cache/Cypress
+            **/node_modules
+          key: v1-yarn-pro-cypress-dependency-cache-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            v1-yarn-pro-cypress-dependency-cache-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+
+      - name: install
+        run: yarn install --immutable
+
+      - name: "Build vite application"
+        run: yarn build:development
+
+      - name: Save build folder
+        uses: actions/upload-artifact@v4
+        with:
+          name: build
+          if-no-files-found: error
+          path: pro/build
+
+  setup:
+    name: Setup E2E 🔧
+    runs-on: ubuntu-24.04
+    outputs:
+      # will contain a json string with an array of n elements, each being a string of spec files delimited by ,
+      test-chunks: ${{ steps['set-test-chunks'].outputs['test-chunks'] }}
+    steps:
+      - uses: actions/checkout@v3
+      - id: set-test-chunks
+        name: Set Chunks
+        # get all spec files from the e2e directory, group them to be at most 3 at a time and transform them to json
+        # this will output something like ["cypress/integration/test1.spec.ts, cypress/integration/test2.spec.ts, cypress/integration/test3.spec.ts", "cypress/integration/test4.spec.ts,etc.."]
+        run: echo "test-chunks=$(find cypress/e2e -type f -name "*.cy.ts" | xargs -n4 | tr ' ' ',' | jq -R . | jq -s -cM .)" >> $GITHUB_OUTPUT
+
+  cypress-run:
+    name: "Run E2E"
+    runs-on: ubuntu-24.04
+    needs: [setup, install]
+    strategy:
+      # don't fail the entire matrix on failure
+      fail-fast: false
+      matrix:
+        chunk: ${{ fromJson(needs.setup.outputs['test-chunks']) }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - name: "Authentification to Google"
         uses: "google-github-actions/auth@v2"
         with:
           workload_identity_provider: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }}
+
       - name: "Get Secret"
         id: secrets
         uses: "google-github-actions/get-secretmanager-secrets@v2"
@@ -48,6 +110,24 @@ jobs:
             ARTIFACT_REGISTRY_SERVICE_ACCOUNT:passculture-metier-ehp/passculture-main-artifact-registry-service-account
             CYPRESS_CLOUD_RECORD_KEY:passculture-metier-ehp/e2e-tests-pro-cypress-cloud-record-key
             CYPRESS_CLOUD_PROJECT_ID:passculture-metier-ehp/e2e-tests-pro-cypress-cloud-project-id
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: "pro/.nvmrc"
+
+      - uses: KengoTODA/actions-setup-docker-compose@v1
+        with:
+          version: "2.23.3"
+
+      - name: "Fix local permissions"
+        run: sudo chown -R $PCAPI_UID:$PCAPI_GID .
+        working-directory: api
+        env:
+          PCAPI_UID: 1000
+          PCAPI_GID: 1000
+
+      - run: yarn install --immutable
+
       - name: "OpenID Connect Authentication"
         id: openid-auth
         uses: "google-github-actions/auth@v2"
@@ -56,53 +136,27 @@ jobs:
           token_format: "access_token"
           workload_identity_provider: ${{ steps.secrets.outputs.ARTIFACT_REGISTRY_WORKLOAD_IDENTITY_PROVIDER  }}
           service_account: ${{ steps.secrets.outputs.ARTIFACT_REGISTRY_SERVICE_ACCOUNT }}
+
       - name: "Docker login"
         id: docker-login
         uses: "docker/login-action@v3"
         with:
           registry: "europe-west1-docker.pkg.dev"
           username: "oauth2accesstoken"
           password: "${{ steps.openid-auth.outputs.access_token }}"
+
       - name: "Compute docker image name:tag"
         id: compute-image-name
         run: |
           echo "image_name=${{ env.registry }}/${{ inputs.image }}:${{ inputs.tag }}" | tee -a ${GITHUB_OUTPUT}
           echo "::notice:: Running e2e-tests with ${{ env.registry }}/${{ inputs.image }}:${{ inputs.tag }}"
-      - name: "Download artifact"
-        if: ${{ inputs.tag != 'latest' }}
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ inputs.image }}-${{ inputs.tag }}.tar
-          path: ${{ runner.temp }}
-      - uses: actions/setup-node@v4
-        with:
-          node-version-file: "pro/.nvmrc"
-      - uses: KengoTODA/actions-setup-docker-compose@v1
-        with:
-          version: "2.23.3"
-      - name: "Fix local permissions"
-        run: sudo chown -R $PCAPI_UID:$PCAPI_GID .
-        working-directory: api
-        env:
-          PCAPI_UID: 1000
-          PCAPI_GID: 1000
-      - name: "Cache the node_modules"
-        id: "yarn-modules-cache"
-        uses: pass-culture-github-actions/cache@v1.0.0
-        with:
-          compression-method: "gzip"
-          bucket: ${{ inputs.CACHE_BUCKET_NAME }}
-          path: |
-            ~/.cache/Cypress
-            **/node_modules
-          key: v1-yarn-pro-cypress-dependency-cache-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            v1-yarn-pro-cypress-dependency-cache-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
-      - run: yarn install --immutable
+
       - name: "Run postgres and redis server"
         run: docker-compose -f ../docker-compose-backend.yml up postgres redis -d
+
       - name: "Set up Cloud SDK"
         uses: "google-github-actions/setup-gcloud@v2"
+
       - name: "Run API server"
         run: |
           if [ "${{ inputs.tag }}" != "latest" ]; then
@@ -122,20 +176,27 @@ jobs:
             --entrypoint bash \
             ${{ steps.compute-image-name.outputs.image_name }} \
             -c "set -e ; flask install_postgres_extensions ; alembic upgrade pre@head ; alembic upgrade post@head ; flask install_data ; python src/pcapi/app.py"
+
       - name: "Wait for migrations to be run"
         uses: iFaxity/wait-on-action@v1
         with:
           resource: http://localhost:5001/health/api
           timeout: 120000
-      - name: "Build vite application"
-        run: yarn build:development
+
+      - name: Download the build folder
+        uses: actions/download-artifact@v4
+        with:
+          name: build
+          path: pro/build
+
       - name: "Serve vite preview"
         run: yarn serve &
+
       # Using wait-on does not work well for this service, we use curl
       - name: "Wait for front-end to listen"
         run: |
           timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:3001)" != "200" ]]; do sleep 5; done' || false
-      # Doc : https://github.com/cypress-io/github-action
+
       - name: "Cypress run"
         uses: cypress-io/github-action@v6
         with:
@@ -145,36 +206,30 @@ jobs:
           browser: chrome
           config-file: cypress/cypress.config.ts
           env: TAGS="@P0"
-          record: ${{ github.ref == 'refs/heads/master' }} # for Cypress Cloud
-          spec: "cypress/e2e/*"
+          spec: ${{ matrix.chunk }}
+          record: ${{ github.ref == 'refs/heads/master' }} # for Cypress
         env:
           CYPRESS_RECORD_KEY: ${{ steps.secrets.outputs.CYPRESS_CLOUD_RECORD_KEY }}
           CYPRESS_PROJECT_ID: ${{ steps.secrets.outputs.CYPRESS_CLOUD_PROJECT_ID }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      # - name: "Remove Cypress videos from Allure results"
-      #   run: rm -rf ../allure-results/*.mp4
-
-      # - name: "Upload Allure results for pro E2E tests"
-      #   uses: actions/upload-artifact@v4
-      #   with:
-      #     name: allure-results-pro-e2e
-      #     path: allure-results
-
       - name: "Move cypress videos"
         if: always() && failure() && github.ref != 'refs/heads/master' # useless on master bc Cypress Cloud
         run: |
           mkdir -p cypress/videos/${{ github.ref }}/${{ github.sha }} && \
           mv cypress/videos/*.mp4 cypress/videos/${{ github.ref }}/${{ github.sha }}/
+
       - name: "Archive E2E results"
         if: always() && failure() && github.ref != 'refs/heads/master' # useless on master bc Cypress Cloud
         uses: google-github-actions/upload-cloud-storage@v2
         with:
           path: "pro/cypress/videos"
           destination: "${{ inputs.CACHE_BUCKET_NAME }}/pro/cypress/videos/e2e-artifacts"
+
       - name: "Show pcapi log when it fails"
         if: failure()
         run: docker logs pc-api
+
       - name: "Post to a Slack channel"
         if: always() && failure() && github.ref == 'refs/heads/master'
         uses: slackapi/slack-github-action@v1.27.0