fix: allow yarn 4 berry to work with cypress webpack preprocessor (#2…

…9901)

* fix: make sure string_decoder is shipped with wbip

* fix: remove pnp-webpack-plugin as it isnt needed with webpack 5

* chore: update enhanced-resolve to 5.17.0 in the yarn-lock and remove the need for you 5.15 patch as the issue we were patching was fixed in 5.16

* build binary [run ci]

* resolve to process lib installed by cypress and not the users home directory

* add changelog entry [run ci]

* build binaries [run ci]

* add binary system test

skip system test install for yarn 4 as it is an exception case [run ci]

adapt dep installer to handle yarn 4.1.1 [run ci]

rebase this out [run ci]

try this [run ci]

temp [run ci]

* update circle cache [run ci]

* fix conditional for yarn install [run ci]

* Update system-tests/projects/yarn-v4.1.1-pnp-dep-resolution/README.md

Co-authored-by: Ryan Manuel <ryanm@cypress.io>

* update from yarn 4.1.1 to 4.3.1

* update docker image name

---------

Co-authored-by: Ryan Manuel <ryanm@cypress.io>

.circleci/cache-version.txt

@@ -1,3 +1,3 @@
 # Bump this version to force CI to re-create the cache from scratch.
 
-07-16-24
+07-24-24

.circleci/workflows.yml

@@ -32,7 +32,7 @@ mainBuildFilters: &mainBuildFilters
         - 'update-v8-snapshot-cache-on-develop'
         - 'remove-json-lint'
         - 'publish-binary'
-        - 'chore/increase_electron_timeout'
+        - 'fix/webpack-batteries-included-for-yarn-pnp'
 
 # usually we don't build Mac app - it takes a long time
 # but sometimes we want to really confirm we are doing the right thing
@@ -43,7 +43,7 @@ macWorkflowFilters: &darwin-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'chore/increase_electron_timeout', << pipeline.git.branch >> ]
+    - equal: [ 'fix/webpack-batteries-included-for-yarn-pnp', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -54,7 +54,7 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'remove-json-lint', << pipeline.git.branch >> ]
+    - equal: [ 'fix/webpack-batteries-included-for-yarn-pnp', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -77,7 +77,7 @@ windowsWorkflowFilters: &windows-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'fix/element_highlighting_origin_test_replay', << pipeline.git.branch >> ]
+    - equal: [ 'fix/webpack-batteries-included-for-yarn-pnp', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -153,7 +153,7 @@ commands:
           name: Set environment variable to determine whether or not to persist artifacts
           command: |
             echo "Setting SHOULD_PERSIST_ARTIFACTS variable"
-            echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "chore/increase_electron_timeout" ]]; then
+            echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "fix/webpack-batteries-included-for-yarn-pnp" ]]; then
                 export SHOULD_PERSIST_ARTIFACTS=true
             fi' >> "$BASH_ENV"
   # You must run `setup_should_persist_artifacts` command and be using bash before running this command
@@ -1725,6 +1725,39 @@ jobs:
     steps:
       - maybe_skip_binary_jobs
       - run-binary-system-tests
+  yarn-pnp-preprocessor-system-test:
+    parallelism: 1
+    working_directory: ~/cypress
+    docker:
+      # we need an image with yarn 4 berry installed on it to run this test
+      - image: cypress/base-internal:18.17.1-yarn-berry
+    environment:
+      # needed to inform the bootstrap-docker-container.sh script to link the binary in the system-test project directory
+      REPO_DIR: /root/cypress
+      TEST_PROJECT_DIR: ./system-tests/projects/yarn-v4.3.1-pnp-dep-resolution
+      USE_YARN_TO_INSTALL_CYPRESS_BINARY: true
+    steps:
+      - maybe_skip_binary_jobs
+      - attach_workspace:
+          at: ~/
+      # required node is set in the docker container already with yarn 4
+      - run:
+          name: Install monorepo dependencies with yarn v1.22.22
+          command: |
+            # we need to install the monorepo dependencies here in a fresh manner.
+            # set to the expected version of yarn and install the dependencies
+            yarn set version 1.22.22
+            yarn install --ignore-scripts
+      - run:
+          name: install dependencies in yarn-v4.3.1-pnp-dep-resolution with yarn berry 4.3.1
+          command: |
+            yarn set version 4.3.1
+            cd ./system-tests/projects/yarn-v4.3.1-pnp-dep-resolution && yarn
+      - run:
+          name: Bootstrap the Cypress binary and run binary system test
+          command: |
+            # we need to bootstrap the binary into our project directory and run the tests
+            source ./system-tests/scripts/bootstrap-docker-container.sh 'yarn cypress run'
 
   system-tests-chrome:
     <<: *defaults
@@ -2862,6 +2895,7 @@ linux-x64-workflow: &linux-x64-workflow
           - test-binary-against-kitchensink
           - test-npm-module-on-minimum-node-version
           - binary-system-tests
+          - yarn-pnp-preprocessor-system-test
           - test-kitchensink
           - unit-tests
           - verify-release-readiness
@@ -2976,6 +3010,11 @@ linux-x64-workflow: &linux-x64-workflow
         requires:
           - get-published-artifacts
           - system-tests-node-modules-install
+    - yarn-pnp-preprocessor-system-test:
+        context: publish-binary
+        requires:
+          - get-published-artifacts
+          - system-tests-node-modules-install
 
 linux-x64-contributor-workflow: &linux-x64-contributor-workflow
   jobs:

cli/CHANGELOG.md

@@ -3,6 +3,10 @@
 
 _Released 7/30/2024 (PENDING)_
 
+**Bugfixes:**
+
+- Fixed an issue where yarn PnP was not working correctly with Cypress and `@cypress/webpack-batteries-included-preprocessor`. Fixes [#27947](https://github.com/cypress-io/cypress/issues/27947).
+
 **Dependency Updates:**
 
 - Updated `@cypress/request` from `3.0.0` to `3.0.1`. Addresses [#29863](https://github.com/cypress-io/cypress/issues/29863).

npm/webpack-batteries-included-preprocessor/index.js

@@ -97,28 +97,6 @@ const addTypeScriptConfig = (file, options) => {
   return options
 }
 
-/**
- * Config yarn pnp plugin for webpack 4
- * @param {*} file file to be processed
- * @param {*} options
- */
-const addYarnPnpConfig = (file, options) => {
-  const { makeResolver } = require('pnp-webpack-plugin/resolver')
-  const findPnpApi = require('module').findPnpApi
-
-  if (findPnpApi && file.filePath) {
-    const pnpapi = findPnpApi(file.filePath)
-
-    if (pnpapi) {
-      const PnpPlugin = {
-        apply: makeResolver({ pnpapi }),
-      }
-
-      options.webpackOptions.resolve.plugins.push(PnpPlugin)
-    }
-  }
-}
-
 const getDefaultWebpackOptions = () => {
   return {
     mode: 'development',
@@ -178,7 +156,13 @@ const getDefaultWebpackOptions = () => {
         // resolve to include the full file extension if a file resolution is provided.
         // @see https://github.com/cypress-io/cypress/issues/27599
         // @see https://webpack.js.org/configuration/module/#resolvefullyspecified
-        process: 'process/browser.js',
+
+        // Due to Pnp compatibility issues, we want to make sure that we resolve to the 'process' library installed with the binary,
+        // which should resolve on leaf app/packages/server/node_modules/@cypress/webpack-batteries-included-preprocessor and up the tree.
+        // In other words, we want to resolve 'process' that is installed with cypress (or the package itself, i.e. @cypress/webpack-batteries-included-preprocessor)
+        // and not in the user's node_modules directory as it may not exist.
+        // @see https://github.com/cypress-io/cypress/issues/27947.
+        process: require.resolve('process/browser.js'),
       }),
     ],
     resolve: {
@@ -240,11 +224,6 @@ const preprocessor = (options = {}) => {
       options = addTypeScriptConfig(file, options)
     }
 
-    if (process.versions.pnp) {
-      // pnp path
-      addYarnPnpConfig(file, options)
-    }
-
     return webpackPreprocessor(options)(file)
   }
 }

npm/webpack-batteries-included-preprocessor/package.json

@@ -32,12 +32,12 @@
     "json5": "2.2.3",
     "os-browserify": "^0.3.0",
     "path-browserify": "^1.0.1",
-    "pnp-webpack-plugin": "^1.7.0",
     "process": "^0.11.10",
     "punycode": "^2.3.0",
     "querystring-es3": "^0.2.1",
     "stream-browserify": "^3.0.0",
     "stream-http": "^3.2.0",
+    "string_decoder": "1.3.0",
     "timers-browserify": "^2.0.12",
     "ts-loader": "9.4.4",
     "tsconfig-aliased-for-wbip": "npm:tsconfig@^7.0.0",

system-tests/projects/yarn-v4.3.1-pnp-dep-resolution/.gitignore

@@ -0,0 +1,3 @@
+.yarn/*
+.pnp.cjs
+.pnp.loader.mjs

system-tests/projects/yarn-v4.3.1-pnp-dep-resolution/README.md

@@ -0,0 +1,6 @@
+# What are we testing?
+
+Project has a support file that uses `cypress-axe`, which uses `axe-core`, both of which are installed by PnP and don't live in `node_modules`, but are preprocessed by `cypress` via `webpack` in `./packages/server/node_modules/@cypress/webpack-batteries-included-preprocessor` (`./npm/webpack-batteries-included-preprocessor` packaged with Cypress server) relative to the monorepo root. That `webpack` binary needs to resolve its build dependencies from within the cypress binary, including built-ins to `process`, but needs to resolve the dependencies bundled in the specs/support file from the `pnp` cache.
+
+Since dependencies cannot be installed normally as we do with regular system tests via yarn v1, we need to run this as a binary system test.
+See `yarn-pnp-preprocessor-system-test` inside `./circleci/workflows.yml`.

system-tests/projects/yarn-v4.3.1-pnp-dep-resolution/cypress.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  e2e: {
+    setupNodeEvents (on, config) {
+      // implement node event listeners here
+    },
+    fixturesFolder: false,
+  },
+}

system-tests/projects/yarn-v4.3.1-pnp-dep-resolution/cypress/e2e/spec.cy.js

@@ -0,0 +1,5 @@
+describe('template spec', () => {
+  it('passes', () => {
+    expect(true).to.equal(true)
+  })
+})

system-tests/projects/yarn-v4.3.1-pnp-dep-resolution/cypress/support/e2e.js

@@ -0,0 +1 @@
+import 'cypress-axe'

system-tests/projects/yarn-v4.3.1-pnp-dep-resolution/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "cypress-pnp-webpack-preprocessor-test",
+  "devDependencies": {
+    "axe-core": "4.9.1",
+    "cypress-axe": "1.5.0"
+  },
+  "packageManager": "yarn@4.3.1"
+}

system-tests/projects/yarn-v4.3.1-pnp-dep-resolution/yarn.lock

@@ -0,0 +1,32 @@
+# This file is generated by running "yarn install" inside your project.
+# Manual changes might be lost - proceed with caution!
+
+__metadata:
+  version: 8
+  cacheKey: 10c0
+
+"axe-core@npm:4.9.1":
+  version: 4.9.1
+  resolution: "axe-core@npm:4.9.1"
+  checksum: 10c0/ac9e5a0c6fa115a43ebffc32a1d2189e1ca6431b5a78e88cdcf94a72a25c5964185682edd94fe6bdb1cb4266c0d06301b022866e0e50dcdf6e3cefe556470110
+  languageName: node
+  linkType: hard
+
+"cypress-axe@npm:1.5.0":
+  version: 1.5.0
+  resolution: "cypress-axe@npm:1.5.0"
+  peerDependencies:
+    axe-core: ^3 || ^4
+    cypress: ^10 || ^11 || ^12 || ^13
+  checksum: 10c0/bc9aab239fbed99bc64801b1f83c4835047264a4f34f801649e41579014cc1bddbd1379efaf3c4a9e6ff42f359bb7fc028df039d6fba637852370d77be0af5ec
+  languageName: node
+  linkType: hard
+
+"cypress-pnp-webpack-preprocessor-test@workspace:.":
+  version: 0.0.0-use.local
+  resolution: "cypress-pnp-webpack-preprocessor-test@workspace:."
+  dependencies:
+    axe-core: "npm:4.9.1"
+    cypress-axe: "npm:1.5.0"
+  languageName: unknown
+  linkType: soft

system-tests/scripts/bootstrap-docker-container.sh

@@ -15,6 +15,7 @@ if [ ! -d "$REPO_DIR" ]; then
 fi
 
 ZIP_PATH=$REPO_DIR/cypress.zip
+TARBALL_PATH=$REPO_DIR/cypress.tgz
 CLI_PATH=$REPO_DIR/cli/build
 
 if [ ! -f "$ZIP_PATH" ]; then
@@ -33,17 +34,23 @@ cd $TEST_PROJECT_DIR
 
 export CYPRESS_INSTALL_BINARY=$ZIP_PATH
 export CYPRESS_CACHE_FOLDER=/tmp/CYPRESS_CACHE_FOLDER/
-export npm_config_cache=/tmp/npm_config_cache/
-export npm_config_package_lock=false
 
-mkdir $npm_config_cache
-chown -R 1000:1000 $npm_config_cache
+if [ "$USE_YARN_TO_INSTALL_CYPRESS_BINARY" = true ]; then
+  # if using yarn to install the built cypress binary, we can just install the tarball that is in the monorepo root directory
+  yarn add cypress@file:$TARBALL_PATH
+else
+  export npm_config_cache=/tmp/npm_config_cache/
+  export npm_config_package_lock=false
 
-npx npm@8 install --unsafe-perm --allow-root --force file:$CLI_PATH
+  mkdir $npm_config_cache
+  chown -R 1000:1000 $npm_config_cache
 
-PATH=$PATH:./node_modules/.bin
+  npx npm@8 install --unsafe-perm --allow-root --force file:$CLI_PATH
 
-cypress install
+  PATH=$PATH:./node_modules/.bin
+
+  cypress install
+fi
 
 # run command passed in argv and store exit code
 set +e

system-tests/scripts/projects-yarn-install.js

@@ -24,6 +24,13 @@ const log = (...args) => console.log(logTag, ...args)
 
   for (const packageJsonPath of packageJsons) {
     const project = path.dirname(packageJsonPath)
+
+    if (project.includes('yarn-v4.3.1-pnp-dep-resolution')) {
+      log('found project yarn-v4.3.1-pnp-dep-resolution, skipping dependency install as this requires corepack for yarn 4')
+      log('this project is an exception and tested inside a docker container with corepack and yarn 4 installed against the built cypress binary')
+      continue
+    }
+
     const timeTag = `${logTag} ${project} node_modules install`
 
     console.time(timeTag)

yarn.lock

@@ -14725,9 +14725,9 @@ enhanced-resolve@^4.5.0:
     tapable "^1.0.0"
 
 enhanced-resolve@^5.0.0, enhanced-resolve@^5.15.0, enhanced-resolve@^5.7.0, enhanced-resolve@^5.9.2:
-  version "5.15.0"
-  resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz#1af946c7d93603eb88e9896cee4904dc012e9c35"
-  integrity sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==
+  version "5.17.0"
+  resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.17.0.tgz#d037603789dd9555b89aaec7eb78845c49089bc5"
+  integrity sha512-dwDPwZL0dmye8Txp2gzFmA6sxALaSvdRDjPH0viLcKrtlOL3tw62nWWweVD1SdILDTJrbrL6tdWVN58Wo6U3eA==
   dependencies:
     graceful-fs "^4.2.4"
     tapable "^2.2.0"
@@ -25100,13 +25100,6 @@ pngjs@^6.0.0:
   resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-6.0.0.tgz#ca9e5d2aa48db0228a52c419c3308e87720da821"
   integrity sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==
 
-pnp-webpack-plugin@^1.7.0:
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/pnp-webpack-plugin/-/pnp-webpack-plugin-1.7.0.tgz#65741384f6d8056f36e2255a8d67ffc20866f5c9"
-  integrity sha512-2Rb3vm+EXble/sMXNSu6eoBx8e79gKqhNq9F5ZWW6ERNCTE/Q0wQNne5541tE5vKjfM8hpNCYL+LGc1YTfI0dg==
-  dependencies:
-    ts-pnp "^1.1.6"
-
 pop-iterate@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/pop-iterate/-/pop-iterate-1.0.1.tgz#ceacfdab4abf353d7a0f2aaa2c1fc7b3f9413ba3"
@@ -28960,7 +28953,7 @@ string_decoder@0.10, string_decoder@~0.10.x:
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
   integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=
 
-string_decoder@^1.0.0, string_decoder@^1.1.1:
+string_decoder@1.3.0, string_decoder@^1.0.0, string_decoder@^1.1.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
   integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
@@ -30055,11 +30048,6 @@ ts-node@^9:
     source-map-support "^0.5.17"
     yn "3.1.1"
 
-ts-pnp@^1.1.6:
-  version "1.1.6"
-  resolved "https://registry.yarnpkg.com/ts-pnp/-/ts-pnp-1.1.6.tgz#389a24396d425a0d3162e96d2b4638900fdc289a"
-  integrity sha512-CrG5GqAAzMT7144Cl+UIFP7mz/iIhiy+xQ6GGcnjTezhALT02uPMRw7tgDSESgB5MsfKt55+GPWw4ir1kVtMIQ==
-
 "tsconfig-aliased-for-wbip@npm:tsconfig@^7.0.0":
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/tsconfig/-/tsconfig-7.0.0.tgz#84538875a4dc216e5c4a5432b3a4dec3d54e91b7"