From a4ad7e76dc73da43824576a50c140e01ad847fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Myl=C3=A8ne?= <187286904+mleroy-pass@users.noreply.github.com> Date: Wed, 5 Feb 2025 17:21:38 +0100 Subject: [PATCH] (PC-34427)[PRO] feat: parallelized e2e dispatch manualy to not record on cloud --- .../dev_on_pull_request_workflow.yml | 2 +- .../workflows/dev_on_push_workflow_main.yml | 2 +- .../dev_on_workflow_tests_pro_e2e.yml | 151 ++++++++++++------ 3 files changed, 105 insertions(+), 50 deletions(-) diff --git a/.github/workflows/dev_on_pull_request_workflow.yml b/.github/workflows/dev_on_pull_request_workflow.yml index dcb74843db4..60338de502d 100644 --- a/.github/workflows/dev_on_pull_request_workflow.yml +++ b/.github/workflows/dev_on_pull_request_workflow.yml @@ -229,7 +229,7 @@ jobs: GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} test-pro-e2e: - name: "Tests pro E2E" + name: "Pro E2E Tests" needs: [pcapi-init-job, build-pcapi] uses: ./.github/workflows/dev_on_workflow_tests_pro_e2e.yml if: always() && diff --git a/.github/workflows/dev_on_push_workflow_main.yml b/.github/workflows/dev_on_push_workflow_main.yml index 086096ef97d..83982e644df 100644 --- a/.github/workflows/dev_on_push_workflow_main.yml +++ b/.github/workflows/dev_on_push_workflow_main.yml @@ -226,7 +226,7 @@ jobs: GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} test-pro-e2e: - name: "Tests pro E2E" + name: "Pro E2E Tests" needs: [pcapi-init-job, build-pcapi] uses: ./.github/workflows/dev_on_workflow_tests_pro_e2e.yml if: | diff --git a/.github/workflows/dev_on_workflow_tests_pro_e2e.yml b/.github/workflows/dev_on_workflow_tests_pro_e2e.yml index 080f713b1df..c2fecd436e2 100644 --- a/.github/workflows/dev_on_workflow_tests_pro_e2e.yml +++ b/.github/workflows/dev_on_workflow_tests_pro_e2e.yml @@ -27,17 +27,79 @@ defaults: working-directory: pro jobs: - tests-pro-e2e-tests: - name: "E2E tests and notifications" + install: + name: "Install E2E 🔧" runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4.2.2 + - name: Checkout + uses: actions/checkout@v4 + - uses: technote-space/workflow-conclusion-action@v3 + + - uses: actions/setup-node@v4 + with: + node-version-file: "pro/.nvmrc" + + - name: "Cache the node_modules" + id: "yarn-modules-cache" + uses: pass-culture-github-actions/cache@v1.0.0 + with: + compression-method: "gzip" + bucket: ${{ inputs.CACHE_BUCKET_NAME }} + path: | + ~/.cache/Cypress + **/node_modules + key: v1-yarn-pro-cypress-dependency-cache-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + v1-yarn-pro-cypress-dependency-cache-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} + + - name: install + run: yarn install --immutable + + - name: "Build vite application" + run: yarn build:development + + - name: Save build folder + uses: actions/upload-artifact@v4 + with: + name: build + if-no-files-found: error + path: pro/build + + setup: + name: Setup E2E 🔧 + runs-on: ubuntu-24.04 + outputs: + # will contain a json string with an array of n elements, each being a string of spec files delimited by , + test-chunks: ${{ steps['set-test-chunks'].outputs['test-chunks'] }} + steps: + - uses: actions/checkout@v3 + - id: set-test-chunks + name: Set Chunks + # get all spec files from the e2e directory, group them to be at most 3 at a time and transform them to json + # this will output something like ["cypress/integration/test1.spec.ts, cypress/integration/test2.spec.ts, cypress/integration/test3.spec.ts", "cypress/integration/test4.spec.ts,etc.."] + run: echo "test-chunks=$(find cypress/e2e -type f -name "*.cy.ts" | xargs -n4 | tr ' ' ',' | jq -R . | jq -s -cM .)" >> $GITHUB_OUTPUT + + cypress-run: + name: "Run E2E" + runs-on: ubuntu-24.04 + needs: [setup, install] + strategy: + # don't fail the entire matrix on failure + fail-fast: false + matrix: + chunk: ${{ fromJson(needs.setup.outputs['test-chunks']) }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: "Authentification to Google" uses: "google-github-actions/auth@v2" with: workload_identity_provider: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} + - name: "Get Secret" id: secrets uses: "google-github-actions/get-secretmanager-secrets@v2" @@ -48,6 +110,24 @@ jobs: ARTIFACT_REGISTRY_SERVICE_ACCOUNT:passculture-metier-ehp/passculture-main-artifact-registry-service-account CYPRESS_CLOUD_RECORD_KEY:passculture-metier-ehp/e2e-tests-pro-cypress-cloud-record-key CYPRESS_CLOUD_PROJECT_ID:passculture-metier-ehp/e2e-tests-pro-cypress-cloud-project-id + + - uses: actions/setup-node@v4 + with: + node-version-file: "pro/.nvmrc" + + - uses: KengoTODA/actions-setup-docker-compose@v1 + with: + version: "2.23.3" + + - name: "Fix local permissions" + run: sudo chown -R $PCAPI_UID:$PCAPI_GID . + working-directory: api + env: + PCAPI_UID: 1000 + PCAPI_GID: 1000 + + - run: yarn install --immutable + - name: "OpenID Connect Authentication" id: openid-auth uses: "google-github-actions/auth@v2" @@ -56,6 +136,7 @@ jobs: token_format: "access_token" workload_identity_provider: ${{ steps.secrets.outputs.ARTIFACT_REGISTRY_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ steps.secrets.outputs.ARTIFACT_REGISTRY_SERVICE_ACCOUNT }} + - name: "Docker login" id: docker-login uses: "docker/login-action@v3" @@ -63,46 +144,19 @@ jobs: registry: "europe-west1-docker.pkg.dev" username: "oauth2accesstoken" password: "${{ steps.openid-auth.outputs.access_token }}" + - name: "Compute docker image name:tag" id: compute-image-name run: | echo "image_name=${{ env.registry }}/${{ inputs.image }}:${{ inputs.tag }}" | tee -a ${GITHUB_OUTPUT} echo "::notice:: Running e2e-tests with ${{ env.registry }}/${{ inputs.image }}:${{ inputs.tag }}" - - name: "Download artifact" - if: ${{ inputs.tag != 'latest' }} - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.image }}-${{ inputs.tag }}.tar - path: ${{ runner.temp }} - - uses: actions/setup-node@v4 - with: - node-version-file: "pro/.nvmrc" - - uses: KengoTODA/actions-setup-docker-compose@v1 - with: - version: "2.23.3" - - name: "Fix local permissions" - run: sudo chown -R $PCAPI_UID:$PCAPI_GID . - working-directory: api - env: - PCAPI_UID: 1000 - PCAPI_GID: 1000 - - name: "Cache the node_modules" - id: "yarn-modules-cache" - uses: pass-culture-github-actions/cache@v1.0.0 - with: - compression-method: "gzip" - bucket: ${{ inputs.CACHE_BUCKET_NAME }} - path: | - ~/.cache/Cypress - **/node_modules - key: v1-yarn-pro-cypress-dependency-cache-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - v1-yarn-pro-cypress-dependency-cache-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - - run: yarn install --immutable + - name: "Run postgres and redis server" run: docker-compose -f ../docker-compose-backend.yml up postgres redis -d + - name: "Set up Cloud SDK" uses: "google-github-actions/setup-gcloud@v2" + - name: "Run API server" run: | if [ "${{ inputs.tag }}" != "latest" ]; then @@ -122,20 +176,27 @@ jobs: --entrypoint bash \ ${{ steps.compute-image-name.outputs.image_name }} \ -c "set -e ; flask install_postgres_extensions ; alembic upgrade pre@head ; alembic upgrade post@head ; flask install_data ; python src/pcapi/app.py" + - name: "Wait for migrations to be run" uses: iFaxity/wait-on-action@v1 with: resource: http://localhost:5001/health/api timeout: 120000 - - name: "Build vite application" - run: yarn build:development + + - name: Download the build folder + uses: actions/download-artifact@v4 + with: + name: build + path: pro/build + - name: "Serve vite preview" run: yarn serve & + # Using wait-on does not work well for this service, we use curl - name: "Wait for front-end to listen" run: | timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:3001)" != "200" ]]; do sleep 5; done' || false - # Doc : https://github.com/cypress-io/github-action + - name: "Cypress run" uses: cypress-io/github-action@v6 with: @@ -145,36 +206,30 @@ jobs: browser: chrome config-file: cypress/cypress.config.ts env: TAGS="@P0" - record: ${{ github.ref == 'refs/heads/master' }} # for Cypress Cloud - spec: "cypress/e2e/*" + spec: ${{ matrix.chunk }} + record: ${{ github.ref == 'refs/heads/master' }} # for Cypress env: CYPRESS_RECORD_KEY: ${{ steps.secrets.outputs.CYPRESS_CLOUD_RECORD_KEY }} CYPRESS_PROJECT_ID: ${{ steps.secrets.outputs.CYPRESS_CLOUD_PROJECT_ID }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # - name: "Remove Cypress videos from Allure results" - # run: rm -rf ../allure-results/*.mp4 - - # - name: "Upload Allure results for pro E2E tests" - # uses: actions/upload-artifact@v4 - # with: - # name: allure-results-pro-e2e - # path: allure-results - - name: "Move cypress videos" if: always() && failure() && github.ref != 'refs/heads/master' # useless on master bc Cypress Cloud run: | mkdir -p cypress/videos/${{ github.ref }}/${{ github.sha }} && \ mv cypress/videos/*.mp4 cypress/videos/${{ github.ref }}/${{ github.sha }}/ + - name: "Archive E2E results" if: always() && failure() && github.ref != 'refs/heads/master' # useless on master bc Cypress Cloud uses: google-github-actions/upload-cloud-storage@v2 with: path: "pro/cypress/videos" destination: "${{ inputs.CACHE_BUCKET_NAME }}/pro/cypress/videos/e2e-artifacts" + - name: "Show pcapi log when it fails" if: failure() run: docker logs pc-api + - name: "Post to a Slack channel" if: always() && failure() && github.ref == 'refs/heads/master' uses: slackapi/slack-github-action@v1.27.0