[e2e][pipeline] Add support for e2e tests on pipeline (#1067)

Adding playwright e2e test runs to the pipeline

## Changes

- Added new test job to run the playwright end-to-end tests from the
pipeline
- Updated playwright reporting to produce more types of output
- Added steps to save the artifacts of the test runs including the
docker logs and playwright results
- Found and fixed test failures that had been introduced since starting
this work

## How to test this PR

1. Read the notes
2. Verify that the playwright job ran and produced artifacts

## Notes

- The TSC linting failure is unrelated to this work

.env.example.public

@@ -10,7 +10,7 @@ SBL_OIDC_REDIRECT_URI="http://localhost:${SBL_DEV_PORT}/filing"
 SBL_REGTECH_BASE_URL="http://localhost:8881"
 SBL_FILING_BASE_URL="http://localhost:8882"
 SBL_CLEANUP_BASE_URL="http://localhost:8883"
-SBL_MAIL_BASE_URL="http://localhost:8765"
+SBL_MAIL_BASE_URL="http://localhost:8765/public/case"
 SBL_LOGOUT_REDIRECT_URL=""
 SBL_VALIDATION_TIMEOUT_SECONDS="1200"
 SBL_LONGPOLLING_DELAY_SECONDS="backoff"

.github/variables/.env

@@ -3,39 +3,39 @@
 # WITH AND VISIBLE TO THE CLIENT BELOW
 ###############################################################
 
-SBL_DEV_PORT="8899"
-SBL_OIDC_AUTHORITY="http://localhost:8880/realms/regtech"
-SBL_OIDC_CLIENT_ID="regtech-client"
-SBL_OIDC_REDIRECT_URI="http://localhost:${SBL_DEV_PORT}/filing"
-SBL_REGTECH_BASE_URL="http://localhost:8881"
-SBL_FILING_BASE_URL="http://localhost:8882"
-SBL_CLEANUP_BASE_URL="http://localhost:8883"
-SBL_MAIL_BASE_URL="http://localhost:8765"
-SBL_LOGOUT_REDIRECT_URL=""
-SBL_VALIDATION_TIMEOUT_SECONDS="1200"
-SBL_LONGPOLLING_DELAY_SECONDS="backoff"
-SBL_UPLOAD_FILE_SIZE_LIMIT_BYTES="50000000"
+SBL_DEV_PORT=8898
+SBL_OIDC_AUTHORITY=http://localhost:8880/realms/regtech
+SBL_OIDC_CLIENT_ID=regtech-client
+SBL_OIDC_REDIRECT_URI=http://localhost:8898/filing
+SBL_REGTECH_BASE_URL=http://localhost:8881
+SBL_FILING_BASE_URL=http://localhost:8882
+SBL_CLEANUP_BASE_URL=http://localhost:8883
+SBL_MAIL_BASE_URL=http://localhost:8765/public/case
+SBL_LOGOUT_REDIRECT_URL=
+SBL_VALIDATION_TIMEOUT_SECONDS=1200
+SBL_LONGPOLLING_DELAY_SECONDS=backoff
+SBL_UPLOAD_FILE_SIZE_LIMIT_BYTES=50000000
 
 SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS=false
 
 ###############################################################
 # ONLY ADD ENVIRONMENT VARIABLES THAT ARE NOT MEANT TO BE 
 # PACKAGED WITH AND VISIBLE TO THE CLIENT BELOW
 ###############################################################
-NODE_EXTRA_CA_CERTS="./e2e/certs/entrust_chain.crt.pem"
+NODE_EXTRA_CA_CERTS=./e2e/certs/entrust_chain.crt.pem
 
-SBL_PLAYWRIGHT_TEST_TARGET="http://localhost:${SBL_DEV_PORT}"
-SBL_PLAYWRIGHT_TEST_REGTECH_TARGET="${SBL_REGTECH_BASE_URL}"
-SBL_PLAYWRIGHT_TEST_FILING_TARGET="${SBL_FILING_BASE_URL}"
-SBL_PLAYWRIGHT_TEST_CLEANUP_TARGET="${SBL_CLEANUP_BASE_URL}"
-SBL_PLAYWRIGHT_TEST_MAIL_TARGET="${SBL_MAIL_BASE_URL}"
-SBL_PLAYWRIGHT_TEST_KC_TARGET="http://localhost:8880/"
-SBL_PLAYWRIGHT_TEST_KC_REALM="regtech"
-SBL_PLAYWRIGHT_TEST_KC_CLI_USERNAME="admin"
-SBL_PLAYWRIGHT_TEST_KC_CLI_CLIENT_ID="admin-cli"
-SBL_PLAYWRIGHT_TEST_KC_CLI_CLIENT_SECRET="local_test_only"
-SBL_PLAYWRIGHT_TEST_KC_CLI_GRANT_TYPE="client_credentials"
-SBL_PLAYWRIGHT_TEST_KC_ADMIN_USERNAME="admin1"
-SBL_PLAYWRIGHT_TEST_KC_ADMIN_PASSWORD="admin"
-SBL_PLAYWRIGHT_TEST_KC_ADMIN_CLIENT_ID="regtech-client"
-SBL_PLAYWRIGHT_TEST_KC_ADMIN_GRANT_TYPE="password"
+SBL_PLAYWRIGHT_TEST_TARGET=http://localhost:8898
+SBL_PLAYWRIGHT_TEST_REGTECH_TARGET=http://localhost:8881
+SBL_PLAYWRIGHT_TEST_FILING_TARGET=http://localhost:8882
+SBL_PLAYWRIGHT_TEST_CLEANUP_TARGET=http://localhost:8883
+SBL_PLAYWRIGHT_TEST_MAIL_TARGET=http://localhost:8765
+SBL_PLAYWRIGHT_TEST_KC_TARGET=http://localhost:8880/
+SBL_PLAYWRIGHT_TEST_KC_REALM=regtech
+SBL_PLAYWRIGHT_TEST_KC_CLI_USERNAME=admin
+SBL_PLAYWRIGHT_TEST_KC_CLI_CLIENT_ID=admin-cli
+SBL_PLAYWRIGHT_TEST_KC_CLI_CLIENT_SECRET=local_test_only
+SBL_PLAYWRIGHT_TEST_KC_CLI_GRANT_TYPE=client_credentials
+SBL_PLAYWRIGHT_TEST_KC_ADMIN_USERNAME=admin1
+SBL_PLAYWRIGHT_TEST_KC_ADMIN_PASSWORD=admin
+SBL_PLAYWRIGHT_TEST_KC_ADMIN_CLIENT_ID=regtech-client
+SBL_PLAYWRIGHT_TEST_KC_ADMIN_GRANT_TYPE=password

.github/workflows/.pre.yml

@@ -35,7 +35,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
           cache: 'yarn'
           cache-dependency-path: ./yarn.lock
       - name: Clean Yarn cache

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
           cache: 'yarn'
           cache-dependency-path: ./yarn.lock
 
@@ -38,7 +38,7 @@ jobs:
 
       - uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
           cache: 'yarn'
           cache-dependency-path: ./yarn.lock
 
@@ -58,7 +58,7 @@ jobs:
 
       - uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
           cache: 'yarn'
           cache-dependency-path: ./yarn.lock
 
@@ -78,7 +78,7 @@ jobs:
 
       - uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
           cache: 'yarn'
           cache-dependency-path: ./yarn.lock
 

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: ./.github/actions/setvars
       - uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
           cache: 'yarn'
           cache-dependency-path: ./yarn.lock
 
@@ -30,7 +30,7 @@ jobs:
       - uses: ./.github/actions/setvars
       - uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
           cache: 'yarn'
           cache-dependency-path: ./yarn.lock
 
@@ -39,3 +39,150 @@ jobs:
 
       - name: Run React tests
         run: yarn test:e2e:ci
+
+  playwright:
+    name: Playwright
+    runs-on: ubuntu-latest
+
+    steps:
+      # Log some evironment stuff and fix some permissions
+      - name: Echo and permissions
+        run: |
+          cat /etc/os-release
+          bash --version
+
+          mkdir -p /tmp/filing_uploads
+          chmod 777 /tmp/filing_uploads
+
+      # Checkout resources
+      - name: Checkout Frontend
+        uses: actions/checkout@v4
+        with:
+          path: 'sbl-frontend'
+      - name: Checkout Cleanup API
+        uses: actions/checkout@v4
+        with:
+          repository: 'cfpb/regtech-cleanup-api'
+          path: 'regtech-cleanup-api'
+      - name: Checkout Mail API
+        uses: actions/checkout@v4
+        with:
+          repository: 'cfpb/regtech-mail-api'
+          path: 'regtech-mail-api'
+      - name: Checkout User Fi
+        uses: actions/checkout@v4
+        with:
+          repository: 'cfpb/regtech-user-fi-management'
+          path: 'regtech-user-fi-management'
+      - name: Checkout Filing API
+        uses: actions/checkout@v4
+        with:
+          repository: 'cfpb/sbl-filing-api'
+          path: 'sbl-filing-api'
+      - name: Checkout SBL Project
+        uses: actions/checkout@v4
+        with:
+          repository: 'cfpb/sbl-project'
+          path: 'sbl-project'
+
+      # Set the environment
+      - name: Set Environment
+        uses: ./sbl-frontend/.github/actions/setvars
+        with:
+          varFilePath: ./sbl-frontend/.github/variables/.env
+
+      # Build images      
+      - name: Build Frontend
+        run: |
+          cd sbl-frontend
+          docker build -t sbl-project-sbl-frontend:latest .
+      - name: Build Cleanup API
+        run: |
+          cd regtech-cleanup-api
+          docker build -t sbl-project-cleanup:latest .
+      - name: Build Mail API
+        run: |
+          cd regtech-mail-api
+          docker build -t sbl-project-mail-api:latest .
+      - name: Build User Fi API
+        run: |
+          cd regtech-user-fi-management
+          docker build -t sbl-project-user-fi:latest .
+      - name: Build Filing API
+        run: |
+          cd sbl-filing-api
+          docker build -t sbl-project-filing:latest .
+
+      # Setup node stuff
+      - name: Setup Node 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'yarn'
+          cache-dependency-path: ./sbl-frontend/yarn.lock
+
+      # Standup stack
+      - name: Standup Stack
+        run: |
+          cd sbl-project
+          docker compose --profile="backend" --profile="frontend" up -d --remove-orphans --build
+      
+      - name: Check running containers
+        run: |
+          ls -alh /tmp/filing_uploads
+          docker ps
+
+      # Setup for test
+      - name: Install Local Yarn Dependencies
+        run: | 
+          cd sbl-frontend
+          yarn
+          yarn playwright install --with-deps
+
+      - name: Seed Database    
+        run: |
+          cd sbl-project/dev_setup/mock_data/
+          bash create_institutions.sh
+          bash insert_filing_period.sh
+
+      # Run tests
+      - name: Run Playwright Tests
+        id: run-tests
+        run: |
+          cd sbl-frontend
+          sed "s/8899/8898/" ./.env.example.public > ./.env.example.public
+          sed "s/8899/8898/" ./.env.example.private > ./.env.example.private
+          cat ./.env.example.public > ./.env
+          cat ./.env.example.private >> ./.env
+          yarn playwright test --workers 4
+
+      # Store artifact test results 
+      - name: Export docker logs
+        if: '!cancelled()'
+        run: |
+          mkdir -p docker-logs
+          docker logs sbl-project-sbl-frontend-1 &> ./docker-logs/frontend.log
+          docker logs sbl-project-cleanup-1 &> ./docker-logs/cleanup-api.log
+          docker logs sbl-project-user-fi-1 &> ./docker-logs/user-api.log
+          docker logs sbl-project-mail-api-1 &> ./docker-logs/mail-api.log
+          docker logs sbl-project-filing-1 &> ./docker-logs/filing-api.log
+          docker logs sbl-project-keycloak-1 &> ./docker-logs/keycloak.log
+          docker logs sbl-project-mailpit-1 &> ./docker-logs/mailpit.log
+          docker logs sbl-project-pg-1 &> ./docker-logs/postgres.log
+
+      - name: Archive Test Results
+        uses: actions/upload-artifact@v4
+        if: '!cancelled()'
+        with:
+          name: playwright-reports
+          path: |
+            sbl-frontend/playwright-reports
+
+      - name: Archive Docker Logs
+        uses: actions/upload-artifact@v4
+        if: '!cancelled()'
+        with:
+          name: docker-logs
+          path: |
+            docker-logs
+      

.gitignore

@@ -24,7 +24,9 @@ e2e/utils/downloads/*
 !.yarn/versions
 /test-results/
 /playwright-report/
+/playwright-reports/
 /blob-report/
+/results.json
 /playwright/.cache/
 
 playwright/.auth

ENV-GUIDE.md

@@ -13,7 +13,7 @@ SBL_OIDC_REDIRECT_URI="http://localhost:${SBL_DEV_PORT}/filing"
 SBL_REGTECH_BASE_URL="http://localhost:8881"
 SBL_FILING_BASE_URL="http://localhost:8882"
 SBL_CLEANUP_BASE_URL="http://localhost:8883"
-SBL_MAIL_BASE_URL="http://localhost:8765"
+SBL_MAIL_BASE_URL="http://localhost:8765/public/case"
 SBL_LOGOUT_REDIRECT_URL=""
 SBL_VALIDATION_TIMEOUT_SECONDS="1200"
 SBL_LONGPOLLING_DELAY_SECONDS="backoff"
@@ -25,11 +25,11 @@ SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS=false
 
 ```env
 NODE_EXTRA_CA_CERTS="./e2e/certs/entrust_chain.crt.pem"
-SBL_PLAYWRIGHT_TEST_TARGET="http://localhost:${SBL_DEV_PORT}"
-SBL_PLAYWRIGHT_TEST_REGTECH_TARGET="${SBL_REGTECH_BASE_URL}"
-SBL_PLAYWRIGHT_TEST_FILING_TARGET="${SBL_FILING_BASE_URL}"
-SBL_PLAYWRIGHT_TEST_CLEANUP_TARGET="${SBL_CLEANUP_BASE_URL}"
-SBL_PLAYWRIGHT_TEST_MAIL_TARGET="${SBL_MAIL_BASE_URL}"
+SBL_PLAYWRIGHT_TEST_TARGET="http://localhost:8899"
+SBL_PLAYWRIGHT_TEST_REGTECH_TARGET="http://localhost:8881"
+SBL_PLAYWRIGHT_TEST_FILING_TARGET="http://localhost:8882"
+SBL_PLAYWRIGHT_TEST_CLEANUP_TARGET="http://localhost:8883"
+SBL_PLAYWRIGHT_TEST_MAIL_TARGET="http://localhost:8765"
 SBL_PLAYWRIGHT_TEST_KC_TARGET="http://localhost:8880/"
 SBL_PLAYWRIGHT_TEST_KC_REALM="regtech"
 SBL_PLAYWRIGHT_TEST_KC_CLI_USERNAME="admin"

e2e/fixtures/testFixture.ts

@@ -99,11 +99,13 @@ export const test = baseTest.extend<{
         await createDomainAssociation({ adminToken, testEmailDomain, testLei });
       }
 
-      // console.log the ephemeral user data for debugging
-      // eslint-disable-next-line no-console
-      console.log('testUsername :>>', testUsername);
-      // eslint-disable-next-line no-console
-      console.log('testUserPassword :>>', testUserPassword);
+      if (!process.env.CI) {
+        // console.log the ephemeral user data for debugging
+        // eslint-disable-next-line no-console
+        console.log('testUsername :>>', testUsername);
+        // eslint-disable-next-line no-console
+        console.log('testUserPassword :>>', testUserPassword);
+      }
 
       await test.step('Unauthenticated homepage: navigate to keycloak', async () => {
         await page.goto('/');
@@ -155,11 +157,15 @@ export const test = baseTest.extend<{
 
       await use();
 
-      const healthy = await cleanupHealthcheck({ adminToken });
-      if (healthy) {
-        await cleanup({ adminToken, testLei });
+      if (!process.env.CI) {
+        const cadminToken = await getAdminKeycloakToken();
+
+        const healthy = await cleanupHealthcheck({ adminToken: cadminToken });
+        if (healthy) {
+          await cleanup({ adminToken: cadminToken, testLei });
+        }
+        await deleteKeycloakUser({ id: testUserId });
       }
-      await deleteKeycloakUser({ id: testUserId });
     },
     { auto: true },
   ],