From e110488775b437aa4e80a8347eb335e670ebea28 Mon Sep 17 00:00:00 2001 From: eps1lon Date: Thu, 15 Feb 2024 22:52:14 +0100 Subject: [PATCH 01/11] Add smoke test to flight fixture --- .circleci/config.yml | 45 +++++++++++++++++++ fixtures/flight/.gitignore | 2 + .../flight/__tests__/__e2e__/smoke.test.js | 19 ++++++++ fixtures/flight/package.json | 5 ++- fixtures/flight/playwright.config.js | 31 +++++++++++++ fixtures/flight/yarn.lock | 23 +++++++++- 6 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 fixtures/flight/__tests__/__e2e__/smoke.test.js create mode 100644 fixtures/flight/playwright.config.js diff --git a/.circleci/config.yml b/.circleci/config.yml index a0197b4134cd0..ec8bd0f9a079f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -236,6 +236,48 @@ jobs: RELEASE_CHANNEL: experimental command: ./scripts/circleci/run_devtools_e2e_tests.js + run_fixtures_flight_tests: + docker: *docker + environment: *environment + steps: + - checkout + - attach_workspace: + at: . + - restore_cache: + name: Restore yarn cache + keys: + - v2-yarn_cache_fixtures_flight-{{ arch }}-{{ checksum "yarn.lock" }} + - run: + name: Install dependencies + working_directory: fixtures/flight + command: | + yarn install --frozen-lockfile --cache-folder ~/.cache/yarn + if [ $? -ne 0 ]; then + yarn install --frozen-lockfile --cache-folder ~/.cache/yarn + fi + - save_cache: + name: Save yarn cache + key: v2-yarn_cache_fixtures_flight-{{ arch }}-{{ checksum "yarn.lock" }} + paths: + - ~/.cache/yarn + - run: + working_directory: fixtures/flight + name: Playwright install deps + command: | + npx playwright install + sudo npx playwright install-deps + - run: + working_directory: fixtures/flight + command: cp -r ../../build/oss-experimental/* ./node_modules/ + - run: + name: Run tests + working_directory: fixtures/flight + command: yarn test + - store_artifacts: + path: fixtures/flight/playwright-report + - store_artifacts: + path: fixtures/flight/test-results + run_devtools_tests_for_versions: docker: *docker environment: *environment @@ -516,6 +558,9 @@ workflows: - run_devtools_e2e_tests: requires: - build_devtools_and_process_artifacts + - run_fixtures_flight_tests: + requires: + - yarn_build devtools_regression_tests: unless: << pipeline.parameters.prerelease_commit_sha >> diff --git a/fixtures/flight/.gitignore b/fixtures/flight/.gitignore index 4add43216aee8..79042a7f00900 100644 --- a/fixtures/flight/.gitignore +++ b/fixtures/flight/.gitignore @@ -7,6 +7,8 @@ # testing /coverage +/playwright-report +/test-results # production /build diff --git a/fixtures/flight/__tests__/__e2e__/smoke.test.js b/fixtures/flight/__tests__/__e2e__/smoke.test.js new file mode 100644 index 0000000000000..b97d1143638b9 --- /dev/null +++ b/fixtures/flight/__tests__/__e2e__/smoke.test.js @@ -0,0 +1,19 @@ +import {test, expect} from '@playwright/test'; + +test('smoke test', async ({page}) => { + const consoleErrors = []; + page.on('console', msg => { + const type = msg.type(); + if (type === 'warn' || type === 'error') { + consoleErrors.push({type: type, text: msg.text()}); + } + }); + const pageErrors = []; + page.on('pageerror', error => { + pageErrors.push(error.stack); + }); + await page.goto('/'); + + await expect(consoleErrors).toEqual([]); + await expect(pageErrors).toEqual([]); +}); diff --git a/fixtures/flight/package.json b/fixtures/flight/package.json index 3d45747782c0d..c4a02db11f58a 100644 --- a/fixtures/flight/package.json +++ b/fixtures/flight/package.json @@ -60,6 +60,9 @@ "webpack-hot-middleware": "^2.25.3", "webpack-manifest-plugin": "^4.0.2" }, + "devDependencies": { + "@playwright/test": "^1.41.2" + }, "scripts": { "predev": "cp -r ../../build/oss-experimental/* ./node_modules/", "prebuild": "cp -r ../../build/oss-experimental/* ./node_modules/", @@ -70,7 +73,7 @@ "start:global": "NODE_ENV=production node --experimental-loader ./loader/global.js server/global", "start:region": "NODE_ENV=production node --experimental-loader ./loader/region.js --conditions=react-server server/region", "build": "node scripts/build.js", - "test": "node scripts/test.js --env=jsdom" + "test": "playwright test" }, "browserslist": { "production": [ diff --git a/fixtures/flight/playwright.config.js b/fixtures/flight/playwright.config.js new file mode 100644 index 0000000000000..1bd515da39598 --- /dev/null +++ b/fixtures/flight/playwright.config.js @@ -0,0 +1,31 @@ +import {defineConfig, devices} from '@playwright/test'; + +const isCI = process.env.CI === '1'; + +export default defineConfig({ + // relative to this configuration file. + testDir: '__tests__/__e2e__', + fullyParallel: true, + // Fail the build on CI if you accidentally left test.only in the source code. + forbidOnly: !isCI, + retries: isCI ? 2 : 0, + // Opt out of parallel tests on CI. + workers: isCI ? 1 : undefined, + reporter: 'html', + use: { + baseURL: 'http://localhost:3000', + + trace: 'on-first-retry', + }, + projects: [ + { + name: 'chromium', + use: {...devices['Desktop Chrome']}, + }, + ], + webServer: { + command: 'yarn dev', + url: 'http://localhost:3000', + reuseExistingServer: !isCI, + }, +}); diff --git a/fixtures/flight/yarn.lock b/fixtures/flight/yarn.lock index f9c7dbb98a7ce..4954b0e89a04c 100644 --- a/fixtures/flight/yarn.lock +++ b/fixtures/flight/yarn.lock @@ -2689,6 +2689,13 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" +"@playwright/test@^1.41.2": + version "1.41.2" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.41.2.tgz#bd9db40177f8fd442e16e14e0389d23751cdfc54" + integrity sha512-qQB9h7KbibJzrDpkXkYvsmiDJK14FULCCZgEcoe2AvFAS64oCirWTwzTlAYEbKaRxWs5TFesE1Na6izMv3HfGg== + dependencies: + playwright "1.41.2" + "@pmmmwh/react-refresh-webpack-plugin@0.5.7": version "0.5.7" resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.7.tgz#58f8217ba70069cc6a73f5d7e05e85b458c150e2" @@ -4885,7 +4892,7 @@ fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" -fsevents@^2.3.2, fsevents@~2.3.2: +fsevents@2.3.2, fsevents@^2.3.2, fsevents@~2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== @@ -6644,6 +6651,20 @@ pkg-up@^3.1.0: dependencies: find-up "^3.0.0" +playwright-core@1.41.2: + version "1.41.2" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.41.2.tgz#db22372c708926c697acc261f0ef8406606802d9" + integrity sha512-VaTvwCA4Y8kxEe+kfm2+uUUw5Lubf38RxF7FpBxLPmGe5sdNkSg5e3ChEigaGrX7qdqT3pt2m/98LiyvU2x6CA== + +playwright@1.41.2: + version "1.41.2" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.41.2.tgz#4e760b1c79f33d9129a8c65cc27953be6dd35042" + integrity sha512-v0bOa6H2GJChDL8pAeLa/LZC4feoAMbSQm1/jF/ySsWWoaNItvrMP7GEkvEEFyCTUYKMxjQKaTSg5up7nR6/8A== + dependencies: + playwright-core "1.41.2" + optionalDependencies: + fsevents "2.3.2" + postcss-attribute-case-insensitive@^5.0.2: version "5.0.2" resolved "https://registry.yarnpkg.com/postcss-attribute-case-insensitive/-/postcss-attribute-case-insensitive-5.0.2.tgz#03d761b24afc04c09e757e92ff53716ae8ea2741" From 561495b01ecc7974b7d1693e7cd3f1ffbdcd8011 Mon Sep 17 00:00:00 2001 From: eps1lon Date: Thu, 15 Feb 2024 23:16:22 +0100 Subject: [PATCH 02/11] Debug webserver in CI --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ec8bd0f9a079f..22fa8b6e8236c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -272,7 +272,7 @@ jobs: - run: name: Run tests working_directory: fixtures/flight - command: yarn test + command: yarn dev - store_artifacts: path: fixtures/flight/playwright-report - store_artifacts: From be08cc592f5b44b1f1022e094676aa17a41bbf9d Mon Sep 17 00:00:00 2001 From: eps1lon Date: Thu, 15 Feb 2024 23:18:19 +0100 Subject: [PATCH 03/11] Fix CI value --- fixtures/flight/playwright.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fixtures/flight/playwright.config.js b/fixtures/flight/playwright.config.js index 1bd515da39598..40d86c37c9464 100644 --- a/fixtures/flight/playwright.config.js +++ b/fixtures/flight/playwright.config.js @@ -1,6 +1,6 @@ import {defineConfig, devices} from '@playwright/test'; -const isCI = process.env.CI === '1'; +const isCI = Boolean(process.env.CI); export default defineConfig({ // relative to this configuration file. From 9aa8c4efa39b1151179f5eabc24ce9c68467851d Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Fri, 16 Feb 2024 11:27:28 +0100 Subject: [PATCH 04/11] Better webserver debugging --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 22fa8b6e8236c..f8631cfc74fb7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -272,7 +272,10 @@ jobs: - run: name: Run tests working_directory: fixtures/flight - command: yarn dev + command: yarn test + environment: + # Otherwise the webserver is a blackbox + DEBUG: pw:webserver - store_artifacts: path: fixtures/flight/playwright-report - store_artifacts: From 3a94495d7c1d28138da88f0d8ce956c770f3564c Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Fri, 16 Feb 2024 11:37:29 +0100 Subject: [PATCH 05/11] Ensure dependency constraints are fulfilled --- .circleci/config.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f8631cfc74fb7..20ef05a5cb26e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -243,12 +243,16 @@ jobs: - checkout - attach_workspace: at: . + # Fixture copies some built packages from the workroot after install. + # That means dependencies of the built packages are not installed. + # We need to install dependencies of the workroot to fulfill all dependency constraints + - setup_node_modules - restore_cache: - name: Restore yarn cache + name: Restore yarn cache of fixture keys: - v2-yarn_cache_fixtures_flight-{{ arch }}-{{ checksum "yarn.lock" }} - run: - name: Install dependencies + name: Install fixture dependencies working_directory: fixtures/flight command: | yarn install --frozen-lockfile --cache-folder ~/.cache/yarn @@ -256,7 +260,7 @@ jobs: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn fi - save_cache: - name: Save yarn cache + name: Save yarn cache of fixture key: v2-yarn_cache_fixtures_flight-{{ arch }}-{{ checksum "yarn.lock" }} paths: - ~/.cache/yarn From 99466f8f674c52adcd9771c2cb693f5f48425767 Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Fri, 16 Feb 2024 12:58:26 +0100 Subject: [PATCH 06/11] yarn build is required --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 20ef05a5cb26e..354a3c05586a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -272,7 +272,7 @@ jobs: sudo npx playwright install-deps - run: working_directory: fixtures/flight - command: cp -r ../../build/oss-experimental/* ./node_modules/ + command: yarn build - run: name: Run tests working_directory: fixtures/flight From bdbbce882178987b8eb921a90e5ee76c8dedadf9 Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Fri, 16 Feb 2024 14:53:04 +0100 Subject: [PATCH 07/11] Add hint for require Node.js version --- fixtures/flight/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fixtures/flight/package.json b/fixtures/flight/package.json index c4a02db11f58a..968f404fcf8fd 100644 --- a/fixtures/flight/package.json +++ b/fixtures/flight/package.json @@ -3,6 +3,9 @@ "type": "module", "version": "0.1.0", "private": true, + "devEngines": { + "node": "20.x || 21.x" + }, "dependencies": { "@babel/core": "^7.16.0", "@babel/plugin-proposal-private-property-in-object": "^7.18.6", From 0320dd3582c4d6b54973f05bd7dd48353c3900ac Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Fri, 16 Feb 2024 14:54:58 +0100 Subject: [PATCH 08/11] Ensure correct Node.js version is used --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 354a3c05586a7..1c15e3479dfb3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -237,7 +237,8 @@ jobs: command: ./scripts/circleci/run_devtools_e2e_tests.js run_fixtures_flight_tests: - docker: *docker + docker: + - image: cimg/openjdk:20.0-node environment: *environment steps: - checkout From 76fc01c68ba88cbe9b94926e30b233357969da8d Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Fri, 16 Feb 2024 15:43:30 +0100 Subject: [PATCH 09/11] Ensure build completed --- fixtures/flight/__tests__/__e2e__/smoke.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/fixtures/flight/__tests__/__e2e__/smoke.test.js b/fixtures/flight/__tests__/__e2e__/smoke.test.js index b97d1143638b9..af889d5de16ad 100644 --- a/fixtures/flight/__tests__/__e2e__/smoke.test.js +++ b/fixtures/flight/__tests__/__e2e__/smoke.test.js @@ -13,6 +13,7 @@ test('smoke test', async ({page}) => { pageErrors.push(error.stack); }); await page.goto('/'); + await expect(page.locator('h1')).toHaveText('Hello World'); await expect(consoleErrors).toEqual([]); await expect(pageErrors).toEqual([]); From 0bf35c34b94849feaaf283bacbebd4df43a0c761 Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Fri, 16 Feb 2024 15:51:10 +0100 Subject: [PATCH 10/11] Fix build --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c15e3479dfb3..114c224676dd9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -273,7 +273,11 @@ jobs: sudo npx playwright install-deps - run: working_directory: fixtures/flight - command: yarn build + name: yarn build + command: | + yarn build + # Need to build twice for unknown reasons + yarn build - run: name: Run tests working_directory: fixtures/flight From 0a7acc5072cea13741a19a03ef0b9b1df6113c3d Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Sun, 18 Feb 2024 15:58:36 +0100 Subject: [PATCH 11/11] Fix bundle error --- .circleci/config.yml | 7 ------- fixtures/flight/playwright.config.js | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 114c224676dd9..d89631162852b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -271,13 +271,6 @@ jobs: command: | npx playwright install sudo npx playwright install-deps - - run: - working_directory: fixtures/flight - name: yarn build - command: | - yarn build - # Need to build twice for unknown reasons - yarn build - run: name: Run tests working_directory: fixtures/flight diff --git a/fixtures/flight/playwright.config.js b/fixtures/flight/playwright.config.js index 40d86c37c9464..faecd8219acd4 100644 --- a/fixtures/flight/playwright.config.js +++ b/fixtures/flight/playwright.config.js @@ -24,7 +24,7 @@ export default defineConfig({ }, ], webServer: { - command: 'yarn dev', + command: 'FAST_REFRESH=false yarn dev', url: 'http://localhost:3000', reuseExistingServer: !isCI, },