From 4e17ddd2961513352d317e65afa78ec604907caf Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Wed, 15 May 2024 15:52:48 -0700 Subject: [PATCH] refactor: normalize pnpm lockfile data in parser --- e2e/pnpm_lockfiles/README.md | 2 + e2e/pnpm_lockfiles/base/package.json | 1 + e2e/pnpm_lockfiles/v54/pnpm-lock.yaml | 9 + e2e/pnpm_lockfiles/v54/snapshots/defs.bzl | 39 ++ e2e/pnpm_lockfiles/v60/pnpm-lock.yaml | 10 + e2e/pnpm_lockfiles/v60/snapshots/defs.bzl | 39 ++ e2e/pnpm_lockfiles/v61/pnpm-lock.yaml | 10 + e2e/pnpm_lockfiles/v61/snapshots/defs.bzl | 39 ++ e2e/pnpm_lockfiles/v90/pnpm-lock.yaml | 10 + e2e/pnpm_lockfiles/v90/snapshots/defs.bzl | 39 ++ npm/private/npm_translate_lock_generate.bzl | 2 +- npm/private/test/parse_pnpm_lock_tests.bzl | 148 +++++--- .../test/snapshots/wksp/repositories.bzl | 8 +- npm/private/test/transitive_closure_tests.bzl | 14 +- npm/private/test/utils_tests.bzl | 11 +- npm/private/transitive_closure.bzl | 105 +----- npm/private/utils.bzl | 354 +++++++++++++----- 17 files changed, 594 insertions(+), 246 deletions(-) diff --git a/e2e/pnpm_lockfiles/README.md b/e2e/pnpm_lockfiles/README.md index efccb07a2e..1848438e1b 100644 --- a/e2e/pnpm_lockfiles/README.md +++ b/e2e/pnpm_lockfiles/README.md @@ -13,3 +13,5 @@ TODO: - npm: references: `@aspect-test/a2": "npm:@aspect-test/a"` No :node_modules/\* targets are generated for aliases to npm packages. + + Note: _sometimes_ fails to install with pnpm9 diff --git a/e2e/pnpm_lockfiles/base/package.json b/e2e/pnpm_lockfiles/base/package.json index 1220f9bb77..66bfe003d7 100644 --- a/e2e/pnpm_lockfiles/base/package.json +++ b/e2e/pnpm_lockfiles/base/package.json @@ -8,6 +8,7 @@ "uvu": "0.5.6", "@scoped/a": "workspace:*", "@scoped/b": "link:../projects/b", + "@scoped/c": "file:../projects/c", "@scoped/d": "../projects/d" }, "devDependencies": { diff --git a/e2e/pnpm_lockfiles/v54/pnpm-lock.yaml b/e2e/pnpm_lockfiles/v54/pnpm-lock.yaml index 7eb817d5ae..035ee6f259 100644 --- a/e2e/pnpm_lockfiles/v54/pnpm-lock.yaml +++ b/e2e/pnpm_lockfiles/v54/pnpm-lock.yaml @@ -14,6 +14,7 @@ importers: '@aspect-test/c': 2.0.0 '@scoped/a': workspace:* '@scoped/b': link:../projects/b + '@scoped/c': file:../projects/c '@scoped/d': ../projects/d '@types/node': ~16.18.11 meaning-of-life: 1.0.0 @@ -23,6 +24,7 @@ importers: '@aspect-test/a': 5.0.2 '@scoped/a': link:../projects/a '@scoped/b': link:../projects/b + '@scoped/c': file:../projects/c '@scoped/d': link:../projects/d meaning-of-life: 1.0.0_o3deharooos255qt5xdujc3cuq rollup: 3.2.5 @@ -151,6 +153,13 @@ packages: sade: 1.8.1 dev: false + file:../projects/c: + resolution: {directory: ../projects/c, type: directory} + name: '@scoped/c' + dependencies: + '@scoped/a': link:../a + dev: false + registry.npmjs.org/@types/node/16.18.97: resolution: {integrity: sha512-4muilE1Lbfn57unR+/nT9AFjWk0MtWi5muwCEJqnOvfRQDbSfLCUdN7vCIg8TYuaANfhLOV85ve+FNpiUsbSRg==, registry: https://registry.yarnpkg.com/, tarball: https://registry.npmjs.org/@types/node/-/node-16.18.97.tgz} name: '@types/node' diff --git a/e2e/pnpm_lockfiles/v54/snapshots/defs.bzl b/e2e/pnpm_lockfiles/v54/snapshots/defs.bzl index 898ee73532..6e8a036f7b 100644 --- a/e2e/pnpm_lockfiles/v54/snapshots/defs.bzl +++ b/e2e/pnpm_lockfiles/v54/snapshots/defs.bzl @@ -77,6 +77,41 @@ def npm_link_all_packages(name = "node_modules", imported_links = []): link_targets.append("//{}:{}/@types/node".format(bazel_package, name)) scope_targets["@types"] = scope_targets["@types"] + [link_targets[-1]] if "@types" in scope_targets else [link_targets[-1]] + if is_root: + _npm_package_store( + name = ".aspect_rules_js/{}/@scoped+c@0.0.0".format(name), + src = "//projects/c:pkg", + package = "@scoped/c", + version = "0.0.0", + deps = { + "//:.aspect_rules_js/{}/@scoped+a@0.0.0".format(name): "@scoped/a", + }, + visibility = ["//visibility:public"], + tags = ["manual"], + ) + + for link_package in [""]: + if link_package == native.package_name(): + # terminal target for direct dependencies + _npm_link_package_store( + name = "{}/@scoped/c".format(name), + src = "//:.aspect_rules_js/{}/@scoped+c@0.0.0".format(name), + visibility = ["//visibility:public"], + tags = ["manual"], + ) + + # filegroup target that provides a single file which is + # package directory for use in $(execpath) and $(rootpath) + native.filegroup( + name = "{}/@scoped/c/dir".format(name), + srcs = [":{}/@scoped/c".format(name)], + output_group = "package_directory", + visibility = ["//visibility:public"], + tags = ["manual"], + ) + link_targets.append(":{}/@scoped/c".format(name)) + scope_targets["@scoped"] = scope_targets["@scoped"] + [link_targets[-1]] if "@scoped" in scope_targets else [link_targets[-1]] + if is_root: _npm_package_store( name = ".aspect_rules_js/{}/@scoped+a@0.0.0".format(name), @@ -212,6 +247,10 @@ def npm_link_targets(name = "node_modules", package = None): link_targets.append("//{}:{}/uvu".format(bazel_package, name)) link_targets.append("//{}:{}/@types/node".format(bazel_package, name)) + for link_package in [""]: + if link_package == bazel_package: + link_targets.append("//{}:{}/@scoped/c".format(bazel_package, name)) + for link_package in ["", "projects/b", "projects/c", "projects/d"]: if link_package == bazel_package: link_targets.append("//{}:{}/@scoped/a".format(bazel_package, name)) diff --git a/e2e/pnpm_lockfiles/v60/pnpm-lock.yaml b/e2e/pnpm_lockfiles/v60/pnpm-lock.yaml index 5897f5cdcb..9c8fe130fd 100644 --- a/e2e/pnpm_lockfiles/v60/pnpm-lock.yaml +++ b/e2e/pnpm_lockfiles/v60/pnpm-lock.yaml @@ -21,6 +21,9 @@ importers: '@scoped/b': specifier: link:../projects/b version: link:../projects/b + '@scoped/c': + specifier: file:../projects/c + version: file:../projects/c '@scoped/d': specifier: ../projects/d version: link:../projects/d @@ -167,6 +170,13 @@ packages: sade: 1.8.1 dev: false + file:../projects/c: + resolution: {directory: ../projects/c, type: directory} + name: '@scoped/c' + dependencies: + '@scoped/a': link:../a + dev: false + registry.npmjs.org/@types/node@16.18.11: resolution: {integrity: sha512-3oJbGBUWuS6ahSnEq1eN2XrCyf4YsWI8OyCvo7c64zQJNplk3mO84t53o8lfTk+2ji59g5ycfc6qQ3fdHliHuA==, registry: https://registry.yarnpkg.com/, tarball: https://registry.npmjs.org/@types/node/-/node-16.18.11.tgz} name: '@types/node' diff --git a/e2e/pnpm_lockfiles/v60/snapshots/defs.bzl b/e2e/pnpm_lockfiles/v60/snapshots/defs.bzl index e7a108e965..a9f91ac57c 100644 --- a/e2e/pnpm_lockfiles/v60/snapshots/defs.bzl +++ b/e2e/pnpm_lockfiles/v60/snapshots/defs.bzl @@ -82,6 +82,41 @@ def npm_link_all_packages(name = "node_modules", imported_links = []): link_targets.append("//{}:{}/@types/node".format(bazel_package, name)) scope_targets["@types"] = scope_targets["@types"] + [link_targets[-1]] if "@types" in scope_targets else [link_targets[-1]] + if is_root: + _npm_package_store( + name = ".aspect_rules_js/{}/@scoped+c@0.0.0".format(name), + src = "//projects/c:pkg", + package = "@scoped/c", + version = "0.0.0", + deps = { + "//:.aspect_rules_js/{}/@scoped+a@0.0.0".format(name): "@scoped/a", + }, + visibility = ["//visibility:public"], + tags = ["manual"], + ) + + for link_package in [""]: + if link_package == native.package_name(): + # terminal target for direct dependencies + _npm_link_package_store( + name = "{}/@scoped/c".format(name), + src = "//:.aspect_rules_js/{}/@scoped+c@0.0.0".format(name), + visibility = ["//visibility:public"], + tags = ["manual"], + ) + + # filegroup target that provides a single file which is + # package directory for use in $(execpath) and $(rootpath) + native.filegroup( + name = "{}/@scoped/c/dir".format(name), + srcs = [":{}/@scoped/c".format(name)], + output_group = "package_directory", + visibility = ["//visibility:public"], + tags = ["manual"], + ) + link_targets.append(":{}/@scoped/c".format(name)) + scope_targets["@scoped"] = scope_targets["@scoped"] + [link_targets[-1]] if "@scoped" in scope_targets else [link_targets[-1]] + if is_root: _npm_package_store( name = ".aspect_rules_js/{}/@scoped+a@0.0.0".format(name), @@ -218,6 +253,10 @@ def npm_link_targets(name = "node_modules", package = None): link_targets.append("//{}:{}/uvu".format(bazel_package, name)) link_targets.append("//{}:{}/@types/node".format(bazel_package, name)) + for link_package in [""]: + if link_package == bazel_package: + link_targets.append("//{}:{}/@scoped/c".format(bazel_package, name)) + for link_package in ["", "projects/b", "projects/c", "projects/d"]: if link_package == bazel_package: link_targets.append("//{}:{}/@scoped/a".format(bazel_package, name)) diff --git a/e2e/pnpm_lockfiles/v61/pnpm-lock.yaml b/e2e/pnpm_lockfiles/v61/pnpm-lock.yaml index d688014913..739edad92c 100644 --- a/e2e/pnpm_lockfiles/v61/pnpm-lock.yaml +++ b/e2e/pnpm_lockfiles/v61/pnpm-lock.yaml @@ -25,6 +25,9 @@ importers: '@scoped/b': specifier: link:../projects/b version: link:../projects/b + '@scoped/c': + specifier: file:../projects/c + version: file:../projects/c '@scoped/d': specifier: ../projects/d version: link:../projects/d @@ -171,6 +174,13 @@ packages: sade: 1.8.1 dev: false + file:../projects/c: + resolution: {directory: ../projects/c, type: directory} + name: '@scoped/c' + dependencies: + '@scoped/a': link:../a + dev: false + registry.npmjs.org/@types/node@16.18.11: resolution: {integrity: sha512-3oJbGBUWuS6ahSnEq1eN2XrCyf4YsWI8OyCvo7c64zQJNplk3mO84t53o8lfTk+2ji59g5ycfc6qQ3fdHliHuA==, registry: https://registry.yarnpkg.com/, tarball: https://registry.npmjs.org/@types/node/-/node-16.18.11.tgz} name: '@types/node' diff --git a/e2e/pnpm_lockfiles/v61/snapshots/defs.bzl b/e2e/pnpm_lockfiles/v61/snapshots/defs.bzl index e7a108e965..a9f91ac57c 100644 --- a/e2e/pnpm_lockfiles/v61/snapshots/defs.bzl +++ b/e2e/pnpm_lockfiles/v61/snapshots/defs.bzl @@ -82,6 +82,41 @@ def npm_link_all_packages(name = "node_modules", imported_links = []): link_targets.append("//{}:{}/@types/node".format(bazel_package, name)) scope_targets["@types"] = scope_targets["@types"] + [link_targets[-1]] if "@types" in scope_targets else [link_targets[-1]] + if is_root: + _npm_package_store( + name = ".aspect_rules_js/{}/@scoped+c@0.0.0".format(name), + src = "//projects/c:pkg", + package = "@scoped/c", + version = "0.0.0", + deps = { + "//:.aspect_rules_js/{}/@scoped+a@0.0.0".format(name): "@scoped/a", + }, + visibility = ["//visibility:public"], + tags = ["manual"], + ) + + for link_package in [""]: + if link_package == native.package_name(): + # terminal target for direct dependencies + _npm_link_package_store( + name = "{}/@scoped/c".format(name), + src = "//:.aspect_rules_js/{}/@scoped+c@0.0.0".format(name), + visibility = ["//visibility:public"], + tags = ["manual"], + ) + + # filegroup target that provides a single file which is + # package directory for use in $(execpath) and $(rootpath) + native.filegroup( + name = "{}/@scoped/c/dir".format(name), + srcs = [":{}/@scoped/c".format(name)], + output_group = "package_directory", + visibility = ["//visibility:public"], + tags = ["manual"], + ) + link_targets.append(":{}/@scoped/c".format(name)) + scope_targets["@scoped"] = scope_targets["@scoped"] + [link_targets[-1]] if "@scoped" in scope_targets else [link_targets[-1]] + if is_root: _npm_package_store( name = ".aspect_rules_js/{}/@scoped+a@0.0.0".format(name), @@ -218,6 +253,10 @@ def npm_link_targets(name = "node_modules", package = None): link_targets.append("//{}:{}/uvu".format(bazel_package, name)) link_targets.append("//{}:{}/@types/node".format(bazel_package, name)) + for link_package in [""]: + if link_package == bazel_package: + link_targets.append("//{}:{}/@scoped/c".format(bazel_package, name)) + for link_package in ["", "projects/b", "projects/c", "projects/d"]: if link_package == bazel_package: link_targets.append("//{}:{}/@scoped/a".format(bazel_package, name)) diff --git a/e2e/pnpm_lockfiles/v90/pnpm-lock.yaml b/e2e/pnpm_lockfiles/v90/pnpm-lock.yaml index 9df07ae57c..33a52c2364 100644 --- a/e2e/pnpm_lockfiles/v90/pnpm-lock.yaml +++ b/e2e/pnpm_lockfiles/v90/pnpm-lock.yaml @@ -25,6 +25,9 @@ importers: '@scoped/b': specifier: link:../projects/b version: link:../projects/b + '@scoped/c': + specifier: file:../projects/c + version: file:../projects/c '@scoped/d': specifier: ../projects/d version: link:../projects/d @@ -97,6 +100,9 @@ packages: resolution: {integrity: sha512-GyAxHYKN650db+xnimHnL2LPz65ilmQsVhCasWA7drDNQn/rfmPiEVMzjRiS7m46scXIERaBmiJMzYDf0bIUbA==} hasBin: true + '@scoped/c@file:../projects/c': + resolution: {directory: ../projects/c, type: directory} + '@types/node@16.18.97': resolution: {integrity: sha512-4muilE1Lbfn57unR+/nT9AFjWk0MtWi5muwCEJqnOvfRQDbSfLCUdN7vCIg8TYuaANfhLOV85ve+FNpiUsbSRg==, tarball: https://registry.npmjs.org/@types/node/-/node-16.18.97.tgz} @@ -163,6 +169,10 @@ snapshots: '@aspect-test/e@1.0.0': {} + '@scoped/c@file:../projects/c': + dependencies: + '@scoped/a': link:../a + '@types/node@16.18.97': {} dequal@2.0.3: {} diff --git a/e2e/pnpm_lockfiles/v90/snapshots/defs.bzl b/e2e/pnpm_lockfiles/v90/snapshots/defs.bzl index cfdefc9eff..ee2ad025c9 100644 --- a/e2e/pnpm_lockfiles/v90/snapshots/defs.bzl +++ b/e2e/pnpm_lockfiles/v90/snapshots/defs.bzl @@ -82,6 +82,41 @@ def npm_link_all_packages(name = "node_modules", imported_links = []): link_15(name = "{}/uvu".format(name)) link_targets.append("//{}:{}/uvu".format(bazel_package, name)) + if is_root: + _npm_package_store( + name = ".aspect_rules_js/{}/@scoped+c@0.0.0".format(name), + src = "//projects/c:pkg", + package = "@scoped/c", + version = "0.0.0", + deps = { + "//:.aspect_rules_js/{}/@scoped+a@0.0.0".format(name): "@scoped/a", + }, + visibility = ["//visibility:public"], + tags = ["manual"], + ) + + for link_package in [""]: + if link_package == native.package_name(): + # terminal target for direct dependencies + _npm_link_package_store( + name = "{}/@scoped/c".format(name), + src = "//:.aspect_rules_js/{}/@scoped+c@0.0.0".format(name), + visibility = ["//visibility:public"], + tags = ["manual"], + ) + + # filegroup target that provides a single file which is + # package directory for use in $(execpath) and $(rootpath) + native.filegroup( + name = "{}/@scoped/c/dir".format(name), + srcs = [":{}/@scoped/c".format(name)], + output_group = "package_directory", + visibility = ["//visibility:public"], + tags = ["manual"], + ) + link_targets.append(":{}/@scoped/c".format(name)) + scope_targets["@scoped"] = scope_targets["@scoped"] + [link_targets[-1]] if "@scoped" in scope_targets else [link_targets[-1]] + if is_root: _npm_package_store( name = ".aspect_rules_js/{}/@scoped+a@0.0.0".format(name), @@ -218,6 +253,10 @@ def npm_link_targets(name = "node_modules", package = None): link_targets.append("//{}:{}/rollup".format(bazel_package, name)) link_targets.append("//{}:{}/uvu".format(bazel_package, name)) + for link_package in [""]: + if link_package == bazel_package: + link_targets.append("//{}:{}/@scoped/c".format(bazel_package, name)) + for link_package in ["", "projects/b", "projects/c", "projects/d"]: if link_package == bazel_package: link_targets.append("//{}:{}/@scoped/a".format(bazel_package, name)) diff --git a/npm/private/npm_translate_lock_generate.bzl b/npm/private/npm_translate_lock_generate.bzl index cdc60386d5..f63b93a012 100644 --- a/npm/private/npm_translate_lock_generate.bzl +++ b/npm/private/npm_translate_lock_generate.bzl @@ -161,7 +161,7 @@ sh_binary( dep_path = helpers.link_package(root_package, dep_version[len("file:"):]) dep_key = "{}+{}".format(dep_package, dep_version) if not dep_key in fp_links.keys(): - msg = "Expected to file: referenced package {} in first-party links".format(dep_key) + msg = "Expected to file: referenced package {} in first-party links {}".format(dep_key, fp_links.keys()) fail(msg) fp_links[dep_key]["link_packages"][link_package] = [] elif dep_version.startswith("link:"): diff --git a/npm/private/test/parse_pnpm_lock_tests.bzl b/npm/private/test/parse_pnpm_lock_tests.bzl index e35a89c623..8e80f71104 100644 --- a/npm/private/test/parse_pnpm_lock_tests.bzl +++ b/npm/private/test/parse_pnpm_lock_tests.bzl @@ -15,6 +15,38 @@ def _parse_empty_lock_test_impl(ctx): return unittest.end(env) +expected_importers = { + ".": { + "dependencies": { + "@aspect-test/a": "5.0.0", + }, + "dev_dependencies": {}, + "optional_dependencies": {}, + }, +} +expected_packages = { + "@aspect-test/a@5.0.0": { + "id": None, + "name": "@aspect-test/a", + "dependencies": { + "@aspect-test/b": "5.0.0", + "@aspect-test/c": "1.0.0", + "@aspect-test/d": "2.0.0_at_aspect-test_c_1.0.0", + }, + "optional_dependencies": {}, + "peer_dependencies": {}, + "dev": False, + "has_bin": True, + "optional": False, + "requires_build": False, + "version": "5.0.0", + "friendly_version": "5.0.0", + "resolution": { + "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==", + }, + }, +} + def _parse_lockfile_v5_test_impl(ctx): env = unittest.begin(ctx) @@ -44,30 +76,15 @@ def _parse_lockfile_v5_test_impl(ctx): } """) + # v5 formatted peer deps different in many ways including how peer packages were escaped + v5_expected_packages = dict(expected_packages) + v5_expected_packages["@aspect-test/a@5.0.0"] = dict(v5_expected_packages["@aspect-test/a@5.0.0"]) + v5_expected_packages["@aspect-test/a@5.0.0"]["dependencies"] = dict(v5_expected_packages["@aspect-test/a@5.0.0"]["dependencies"]) + v5_expected_packages["@aspect-test/a@5.0.0"]["dependencies"]["@aspect-test/d"] = "2.0.0_@aspect-test+c@1.0.0" + expected = ( - { - ".": { - "dependencies": { - "@aspect-test/a": "5.0.0", - }, - "optionalDependencies": {}, - "devDependencies": {}, - }, - }, - { - "/@aspect-test/a/5.0.0": { - "resolution": { - "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==", - }, - "hasBin": True, - "dependencies": { - "@aspect-test/b": "5.0.0", - "@aspect-test/c": "1.0.0", - "@aspect-test/d": "2.0.0_@aspect-test+c@1.0.0", - }, - "dev": False, - }, - }, + expected_importers, + v5_expected_packages, {}, 5.4, None, @@ -107,29 +124,8 @@ def _parse_lockfile_v6_test_impl(ctx): """) expected = ( - { - ".": { - "dependencies": { - "@aspect-test/a": "5.0.0", - }, - "optionalDependencies": {}, - "devDependencies": {}, - }, - }, - { - "/@aspect-test/a/5.0.0": { - "resolution": { - "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==", - }, - "hasBin": True, - "dependencies": { - "@aspect-test/b": "5.0.0", - "@aspect-test/c": "1.0.0", - "@aspect-test/d": "2.0.0_at_aspect-test_c_1.0.0", - }, - "dev": False, - }, - }, + expected_importers, + expected_packages, {}, 6.0, None, @@ -139,14 +135,74 @@ def _parse_lockfile_v6_test_impl(ctx): return unittest.end(env) +def _parse_lockfile_v9_test_impl(ctx): + env = unittest.begin(ctx) + + parsed_json = utils.parse_pnpm_lock_json("""\ +{ + "lockfileVersion": "9.0", + "settings": { + "autoInstallPeers": true, + "excludeLinksFromLockfile": false + }, + "importers": { + ".": { + "dependencies": { + "@aspect-test/a": { + "specifier": "5.0.0", + "version": "5.0.0" + } + } + } + }, + "packages": { + "@aspect-test/a@5.0.0": { + "resolution": { + "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==" + }, + "hasBin": true + } + }, + "snapshots": { + "@aspect-test/a@5.0.0": { + "dependencies": { + "@aspect-test/b": "5.0.0", + "@aspect-test/c": "1.0.0", + "@aspect-test/d": "2.0.0(@aspect-test/c@1.0.0)" + } + } + } +} +""") + + # NOTE: unknown properties in >=v9 + v9_expected_packages = dict(expected_packages) + v9_expected_packages["@aspect-test/a@5.0.0"] = dict(v9_expected_packages["@aspect-test/a@5.0.0"]) + v9_expected_packages["@aspect-test/a@5.0.0"]["dev"] = None + v9_expected_packages["@aspect-test/a@5.0.0"]["requires_build"] = None + + expected = ( + expected_importers, + v9_expected_packages, + {}, + 9.0, + None, + ) + + asserts.equals(env, expected, parsed_json) + + return unittest.end(env) + a_test = unittest.make(_parse_empty_lock_test_impl, attrs = {}) b_test = unittest.make(_parse_lockfile_v5_test_impl, attrs = {}) c_test = unittest.make(_parse_lockfile_v6_test_impl, attrs = {}) +d_test = unittest.make(_parse_lockfile_v9_test_impl, attrs = {}) TESTS = [ a_test, b_test, c_test, + d_test, ] def parse_pnpm_lock_tests(name): diff --git a/npm/private/test/snapshots/wksp/repositories.bzl b/npm/private/test/snapshots/wksp/repositories.bzl index b3b5733b08..90f58a2468 100644 --- a/npm/private/test/snapshots/wksp/repositories.bzl +++ b/npm/private/test/snapshots/wksp/repositories.bzl @@ -1723,11 +1723,11 @@ def npm_repositories(): integrity = "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", deps = { "string-width": "5.1.2", - "string-width-cjs": "/string-width/4.2.3", + "string-width-cjs": "/string-width@4.2.3", "strip-ansi": "7.1.0", - "strip-ansi-cjs": "/strip-ansi/6.0.1", + "strip-ansi-cjs": "/strip-ansi@6.0.1", "wrap-ansi": "8.1.0", - "wrap-ansi-cjs": "/wrap-ansi/7.0.0", + "wrap-ansi-cjs": "/wrap-ansi@7.0.0", }, transitive_closure = { "@isaacs/cliui": ["8.0.2"], @@ -17855,7 +17855,7 @@ def npm_repositories(): "point-in-polygon": "1.1.0", "polybooljs": "1.2.0", "probe-image-size": "7.2.3", - "regl": "/@plotly/regl/2.1.2", + "regl": "/@plotly/regl@2.1.2", "regl-error2d": "2.0.12", "regl-line2d": "3.1.2", "regl-scatter2d": "3.3.1", diff --git a/npm/private/test/transitive_closure_tests.bzl b/npm/private/test/transitive_closure_tests.bzl index b10e89f4eb..a164ff6f1b 100644 --- a/npm/private/test/transitive_closure_tests.bzl +++ b/npm/private/test/transitive_closure_tests.bzl @@ -6,7 +6,7 @@ load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") load("//npm/private:transitive_closure.bzl", "gather_transitive_closure") TEST_PACKAGES = { - "@aspect-test/a/5.0.0": { + "@aspect-test/a@5.0.0": { "name": "@aspect-test/a", "version": "5.0.0", "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==", @@ -17,21 +17,21 @@ TEST_PACKAGES = { }, "optional_dependencies": {}, }, - "@aspect-test/b/5.0.0": { + "@aspect-test/b@5.0.0": { "dependencies": {}, "optional_dependencies": { "@aspect-test/c": "2.0.0", }, }, - "@aspect-test/c/1.0.0": { + "@aspect-test/c@1.0.0": { "dependencies": {}, "optional_dependencies": {}, }, - "@aspect-test/c/2.0.0": { + "@aspect-test/c@2.0.0": { "dependencies": {}, "optional_dependencies": {}, }, - "@aspect-test/d/2.0.0_@aspect-test+c@1.0.0": { + "@aspect-test/d@2.0.0_@aspect-test+c@1.0.0": { "dependencies": {}, "optional_dependencies": {}, }, @@ -44,12 +44,12 @@ def test_walk_deps(ctx): not_no_optional = False # Walk the example tree above - closure = gather_transitive_closure(TEST_PACKAGES, "@aspect-test/a/5.0.0", not_no_optional) + closure = gather_transitive_closure(TEST_PACKAGES, "@aspect-test/a@5.0.0", not_no_optional) expected = {"@aspect-test/a": ["5.0.0"], "@aspect-test/b": ["5.0.0"], "@aspect-test/c": ["1.0.0", "2.0.0"], "@aspect-test/d": ["2.0.0_@aspect-test+c@1.0.0"]} asserts.equals(env, expected, closure) # Run again with no_optional set, this means we shouldn't walk the dep from @aspect-test/b/5.0.0 -> @aspect-test/c/2.0.0 - closure = gather_transitive_closure(TEST_PACKAGES, "@aspect-test/a/5.0.0", no_optional) + closure = gather_transitive_closure(TEST_PACKAGES, "@aspect-test/a@5.0.0", no_optional) expected = {"@aspect-test/a": ["5.0.0"], "@aspect-test/b": ["5.0.0"], "@aspect-test/c": ["1.0.0"], "@aspect-test/d": ["2.0.0_@aspect-test+c@1.0.0"]} asserts.equals(env, expected, closure) diff --git a/npm/private/test/utils_tests.bzl b/npm/private/test/utils_tests.bzl index 039d9a7075..9874999641 100644 --- a/npm/private/test/utils_tests.bzl +++ b/npm/private/test/utils_tests.bzl @@ -11,10 +11,10 @@ def test_strip_peer_dep_or_patched_version(ctx): asserts.equals( env, "21.1.0", - utils.strip_peer_dep_or_patched_version("21.1.0_rollup@2.70.2_x@1.1.1"), + utils.strip_v5_peer_dep_or_patched_version("21.1.0_rollup@2.70.2_x@1.1.1"), ) - asserts.equals(env, "1.0.0", utils.strip_peer_dep_or_patched_version("1.0.0_o3deharooos255qt5xdujc3cuq")) - asserts.equals(env, "21.1.0", utils.strip_peer_dep_or_patched_version("21.1.0")) + asserts.equals(env, "1.0.0", utils.strip_v5_peer_dep_or_patched_version("1.0.0_o3deharooos255qt5xdujc3cuq")) + asserts.equals(env, "21.1.0", utils.strip_v5_peer_dep_or_patched_version("21.1.0")) return unittest.end(env) def test_bazel_name(ctx): @@ -34,9 +34,8 @@ def test_bazel_name(ctx): # buildifier: disable=function-docstring def test_pnpm_name(ctx): env = unittest.begin(ctx) - asserts.equals(env, "@scope/y/1.1.1", utils.pnpm_name("@scope/y", "1.1.1")) - asserts.equals(env, ("@scope/y", "1.1.1"), utils.parse_pnpm_package_key("@scope/y", "/@scope/y/1.1.1")) - asserts.equals(env, ("@scope/y", "registry/@scope/y/1.1.1"), utils.parse_pnpm_package_key("@scope/y", "registry/@scope/y/1.1.1")) + asserts.equals(env, "@scope/y@1.1.1", utils.pnpm_name("@scope/y", "1.1.1")) + asserts.equals(env, ("@scope/y", "registry/@scope/y@1.1.1"), utils.parse_pnpm_package_key("@scope/y", "registry/@scope/y@1.1.1")) asserts.equals(env, ("@scope/y", "1.1.1"), utils.parse_pnpm_package_key("@scope/y", "1.1.1")) return unittest.end(env) diff --git a/npm/private/transitive_closure.bzl b/npm/private/transitive_closure.bzl index 1524cbda13..4b65f3089b 100644 --- a/npm/private/transitive_closure.bzl +++ b/npm/private/transitive_closure.bzl @@ -75,71 +75,12 @@ def gather_transitive_closure(packages, package, no_optional, cache = {}): def _get_package_info_deps(package_info, no_optional): return package_info["dependencies"] if no_optional else dicts.add(package_info["dependencies"], package_info["optional_dependencies"]) -def _gather_package_info(package_path, package_snapshot): - if package_path.startswith("/"): - # an aliased dependency - package = package_path[1:] - name, version = utils.parse_pnpm_package_key("", package_path) - friendly_version = utils.strip_peer_dep_or_patched_version(version) - package_key = package - elif package_path.startswith("file:") and utils.is_vendored_tarfile(package_snapshot): - if "name" not in package_snapshot: - fail("expected package %s to have a name field" % package_path) - name = package_snapshot["name"] - package = package_snapshot["name"] - version = package_path - if "version" in package_snapshot: - version = package_snapshot["version"] - package_key = "{}/{}".format(package, version) - friendly_version = version - elif package_path.startswith("file:"): - package = package_path - if "name" not in package_snapshot: - msg = "expected package {} to have a name field".format(package_path) - fail(msg) - name = package_snapshot["name"] - version = package_path - friendly_version = package_snapshot["version"] if "version" in package_snapshot else version - package_key = package - else: - package = package_path - if "name" not in package_snapshot: - msg = "expected package {} to have a name field".format(package_path) - fail(msg) - if "version" not in package_snapshot: - msg = "expected package {} to have a version field".format(package_path) - fail(msg) - name = package_snapshot["name"] - version = package_path - friendly_version = package_snapshot["version"] - package_key = package - - if "resolution" not in package_snapshot: - msg = "package {} has no resolution field".format(package_path) - fail(msg) - id = package_snapshot["id"] if "id" in package_snapshot else None - resolution = package_snapshot["resolution"] - - return package_key, { - "name": name, - "id": id, - "version": version, - "friendly_version": friendly_version, - "resolution": resolution, - "dependencies": package_snapshot.get("dependencies", {}), - "optional_dependencies": package_snapshot.get("optionalDependencies", {}), - "dev": package_snapshot.get("dev", False), - "optional": package_snapshot.get("optional", False), - "has_bin": package_snapshot.get("hasBin", False), - "requires_build": package_snapshot.get("requiresBuild", False), - } - -def translate_to_transitive_closure(lock_importers, lock_packages, prod = False, dev = False, no_optional = False): +def translate_to_transitive_closure(importers, packages, prod = False, dev = False, no_optional = False): """Implementation detail of translate_package_lock, converts pnpm-lock to a different dictionary with more data. Args: - lock_importers: lockfile importers dict - lock_packages: lockfile packages dict + importers: workspace projects (pnpm "importers") + packages: all package info by name prod: If true, only install dependencies dev: If true, only install devDependencies no_optional: If true, optionalDependencies are not installed @@ -148,34 +89,21 @@ def translate_to_transitive_closure(lock_importers, lock_packages, prod = False, Nested dictionary suitable for further processing in our repository rule """ - # All package info mapped by package name - packages = {} - # Packages resolved to a different version package_version_map = {} - for package_path, package_snapshot in lock_packages.items(): - package_key, package_info = _gather_package_info(package_path, package_snapshot) - - if package_key in packages: - msg = "ERROR: duplicate package info: {}\n\t{}\n\t{}".format(package_key, packages[package_key], package_info) - - # buildifier: disable=print - print(msg) - - packages[package_key] = package_info - - # tarbal versions + # tarbal versions + for package_key, package_info in packages.items(): if package_info["resolution"].get("tarball", None) and package_info["resolution"]["tarball"].startswith("file:"): package_version_map[package_key] = package_info # Collect deps of each importer (workspace projects) - importers = {} - for importPath in lock_importers.keys(): - lock_importer = lock_importers[importPath] - prod_deps = {} if dev else lock_importer.get("dependencies", {}) - dev_deps = {} if prod else lock_importer.get("devDependencies", {}) - opt_deps = {} if no_optional else lock_importer.get("optionalDependencies", {}) + importers_deps = {} + for importPath in importers.keys(): + lock_importer = importers[importPath] + prod_deps = {} if dev else lock_importer.get("dependencies") + dev_deps = {} if prod else lock_importer.get("dev_dependencies") + opt_deps = {} if no_optional else lock_importer.get("optional_dependencies") deps = dicts.add(prod_deps, opt_deps) all_deps = dicts.add(prod_deps, dev_deps, opt_deps) @@ -187,7 +115,7 @@ def translate_to_transitive_closure(lock_importers, lock_packages, prod = False, if info["name"] in all_deps: all_deps[info["name"]] = info["version"] - importers[importPath] = { + importers_deps[importPath] = { # deps this importer should pass on if it is linked as a first-party package; this does # not include devDependencies "deps": deps, @@ -199,15 +127,14 @@ def translate_to_transitive_closure(lock_importers, lock_packages, prod = False, # Collect transitive dependencies for each package cache = {} for package in packages.keys(): - package_info = packages[package] - - package_info["transitive_closure"] = gather_transitive_closure( + transitive_closure = gather_transitive_closure( packages, package, no_optional, cache, ) - cache[package] = package_info["transitive_closure"] + packages[package]["transitive_closure"] = transitive_closure + cache[package] = transitive_closure - return (importers, packages) + return (importers_deps, packages) diff --git a/npm/private/utils.bzl b/npm/private/utils.bzl index 292ef41ab6..5d2f2d2e7a 100644 --- a/npm/private/utils.bzl +++ b/npm/private/utils.bzl @@ -34,7 +34,7 @@ def _bazel_name(name, version = None): escaped_version = "%s__%s" % (escaped_version, _sanitize_string(peer_version)) return "%s__%s" % (escaped_name, escaped_version) -def _strip_peer_dep_or_patched_version(version): +def _strip_v5_peer_dep_or_patched_version(version): "Remove peer dependency or patched syntax from version string" # 21.1.0_rollup@2.70.2 becomes 21.1.0 @@ -44,9 +44,19 @@ def _strip_peer_dep_or_patched_version(version): return version[:index] return version +def _strip_v6_peer_dep_or_patched_version(version): + "Remove peer dependency or patched syntax from pnpm6+ version string" + + # 21.1.0(rollup@2.70.2) becomes 21.1.0 + # 21.1.0(patch_hash=...) becomes 21.1.0 + index = version.find("(") + if index != -1: + return version[:index] + return version + def _pnpm_name(name, version): "Make a name/version pnpm-style name for a package name and version" - return "%s/%s" % (name, version) + return "%s@%s" % (name, version) def _parse_pnpm_package_key(pnpm_name, pnpm_version): if pnpm_version.startswith("link:") or pnpm_version.startswith("file:"): @@ -64,15 +74,134 @@ def _parse_pnpm_package_key(pnpm_name, pnpm_version): # registry.com/name/version # # return a (name, version) tuple. This format is found in pnpm lock file v5. + # TODO: what if the registry has '/'s? _, pnpm_version = pnpm_version.split("/", 1) - segments = pnpm_version.rsplit("/", 1) + segments = pnpm_version.rsplit("@", 1) if len(segments) != 2: msg = "unexpected pnpm versioned name {}".format(pnpm_version) fail(msg) return (segments[0], segments[1]) -def _convert_pnpm_v6_version_peer_dep(version): +# Metadata about a pnpm "project" (importer). +# +# Metadata may come from different locations depending on the lockfile, this struct should +# have data normalized across lockfiles. +def _new_import_info(dependencies, dev_dependencies, optional_dependencies): + return { + "dependencies": dependencies, + "dev_dependencies": dev_dependencies, + "optional_dependencies": optional_dependencies, + } + +# Metadata about a package. +# +# Metadata may come from different locations depending on the lockfile, this struct should +# have data normalized across lockfiles. +def _new_package_info(id, name, dependencies, optional_dependencies, peer_dependencies, dev, has_bin, optional, requires_build, version, friendly_version, resolution): + return { + "id": id, + "name": name, + "dependencies": dependencies, + "optional_dependencies": optional_dependencies, + "peer_dependencies": peer_dependencies, + "dev": dev, + "has_bin": has_bin, + "optional": optional, + "requires_build": requires_build, + "version": version, + "friendly_version": friendly_version, + "resolution": resolution, + } + +def _convert_v5_importers(importers): + result = {} + for import_path, importer in importers.items(): + result[import_path] = _new_import_info( + # TODO: normalize edge cases such as: + # - deps with protocols + # - ? + dependencies = importer.get("dependencies", {}), + dev_dependencies = importer.get("devDependencies", {}), + optional_dependencies = importer.get("optionalDependencies", {}), + ) + return result + +def _convert_v5_v6_file_package(package_path, package_snapshot): + if "name" not in package_snapshot: + msg = "expected package {} to have a name field".format(package_path) + fail(msg) + + name = package_snapshot["name"] + version = package_path + if _is_vendored_tarfile(package_snapshot): + if "version" in package_snapshot: + version = package_snapshot["version"] + package_key = "{}@{}".format(name, version) + friendly_version = version + else: + package_key = package_path + friendly_version = package_snapshot["version"] if "version" in package_snapshot else version + + return package_key, name, version, friendly_version + +def _convert_v5_v6_default_package(package_path, package_snapshot): + if "name" not in package_snapshot: + msg = "expected package {} to have a name field".format(package_path) + fail(msg) + if "version" not in package_snapshot: + msg = "expected package {} to have a version field".format(package_path) + fail(msg) + name = package_snapshot["name"] + package_key = package_path + version = package_path + friendly_version = package_snapshot["version"] + + return package_key, name, version, friendly_version + +def _convert_v5_packages(packages): + result = {} + for package_path, package_snapshot in packages.items(): + if "resolution" not in package_snapshot: + msg = "package {} has no resolution field".format(package_path) + fail(msg) + + if package_path.startswith("/"): + # an aliased dependency + name, version = package_path[1:].rsplit("/", 1) + friendly_version = _strip_v5_peer_dep_or_patched_version(version) + package_key = "{}@{}".format(name, version) + elif package_path.startswith("file:"): + # direct reference to file + package_key, name, version, friendly_version = _convert_v5_v6_file_package(package_path, package_snapshot) + else: + package_key, name, version, friendly_version = _convert_v5_v6_default_package(package_path, package_snapshot) + + package_info = _new_package_info( + id = package_snapshot.get("id", None), + name = name, + version = version, + friendly_version = friendly_version, + dependencies = package_snapshot.get("dependencies", {}), + peer_dependencies = package_snapshot.get("peerDependencies", {}), + optional_dependencies = package_snapshot.get("optionalDependencies", {}), + dev = package_snapshot.get("dev", False), + has_bin = package_snapshot.get("hasBin", False), + optional = package_snapshot.get("optional", False), + requires_build = package_snapshot.get("requiresBuild", False), + resolution = package_snapshot.get("resolution"), + ) + + if package_key in result: + msg = "WARNING: duplicate package: {}\n\t{}\n\t{}".format(package_key, result[package_key], package_info) + + # buildifier: disable=print + print(msg) + + result[package_key] = package_info + return result + +def _convert_pnpm_v6_v9_version_peer_dep(version): # Covert a pnpm lock file v6 version string of the format # version(@scope/peer@version)(@scope/peer@version) # to a pnpm lock file v5 version_peer_version that is compatible with rules_js. @@ -92,19 +221,17 @@ def _convert_pnpm_v6_version_peer_dep(version): version = version.rstrip("_") return version -def _convert_pnpm_v6_package_name(package_name): - # Covert a pnpm lock file v6 /name/version string of the format - # @scope/name@version(@scope/name@version)(@scope/name@version) - # to a pnpm lock file v5 @scope/name/version_peer_version format that is compatible with rules_js. - if package_name.startswith("/"): - package_name = _convert_pnpm_v6_version_peer_dep(package_name) - segments = package_name.rsplit("@", 1) - if len(segments) != 2: - msg = "unexpected pnpm versioned name {}".format(package_name) - fail(msg) - return "%s/%s" % (segments[0], segments[1]) - else: - return _convert_pnpm_v6_version_peer_dep(package_name) +def _convert_pnpm_v6_importer_dependency_map(deps): + result = {} + for name, attributes in deps.items(): + result[name] = _convert_pnpm_v6_v9_version_peer_dep(attributes.get("version")) + return result + +def _convert_pnpm_v6_v9_package_dependency_map(deps): + result = {} + for name, version in deps.items(): + result[name] = _convert_pnpm_v6_v9_version_peer_dep(version) + return result def _convert_v6_importers(importers): # Convert pnpm lockfile v6 importers to a rules_js compatible ~v5 format. @@ -132,13 +259,14 @@ def _convert_v6_importers(importers): result = {} for import_path, importer in importers.items(): - result[import_path] = {} - for key in ["dependencies", "optionalDependencies", "devDependencies"]: - deps = importer.get(key, None) - if deps != None: - result[import_path][key] = {} - for name, attributes in deps.items(): - result[import_path][key][name] = _convert_pnpm_v6_package_name(attributes.get("version")) + result[import_path] = _new_import_info( + # TODO: normalize edge cases such as: + # - deps with protocols + # - ? + dependencies = _convert_pnpm_v6_importer_dependency_map(importer.get("dependencies", {})), + dev_dependencies = _convert_pnpm_v6_importer_dependency_map(importer.get("devDependencies", {})), + optional_dependencies = _convert_pnpm_v6_importer_dependency_map(importer.get("optionalDependencies", {})), + ) return result def _convert_v6_packages(packages): @@ -151,36 +279,63 @@ def _convert_v6_packages(packages): # v6: 2.0.0(@aspect-test/c@2.0.2) result = {} - for package, package_info in packages.items(): - # convert v6 package dependencies + optionalDependencies - for key in ["dependencies", "optionalDependencies"]: - deps = package_info.get(key, None) - if deps != None: - dependencies = {} - for dep_name, dep_version in deps.items(): - dependencies[dep_name] = _convert_pnpm_v6_package_name(dep_version) - package_info[key] = dependencies - - package_key = _convert_pnpm_v6_package_name(package) + for package_path, package_snapshot in packages.items(): + if "resolution" not in package_snapshot: + msg = "package {} has no resolution field".format(package_path) + fail(msg) + + package_path = _convert_pnpm_v6_v9_version_peer_dep(package_path) + + if package_path.startswith("/"): + # an aliased dependency + name, version = package_path[1:].rsplit("@", 1) + + # TODO: dont strip twice, but 'friendly_version' may have already been converted from v6 + friendly_version = _strip_v5_peer_dep_or_patched_version(_strip_v6_peer_dep_or_patched_version(version)) + package_key = "{}@{}".format(name, version) + elif package_path.startswith("file:"): + # direct reference to file + package_key, name, version, friendly_version = _convert_v5_v6_file_package(package_path, package_snapshot) + else: + package_key, name, version, friendly_version = _convert_v5_v6_default_package(package_path, package_snapshot) + + package_info = _new_package_info( + id = package_snapshot.get("id", None), + name = name, + version = version, + friendly_version = friendly_version, + dependencies = _convert_pnpm_v6_v9_package_dependency_map(package_snapshot.get("dependencies", {})), + optional_dependencies = _convert_pnpm_v6_v9_package_dependency_map(package_snapshot.get("optionalDependencies", {})), + peer_dependencies = _convert_pnpm_v6_v9_package_dependency_map(package_snapshot.get("peerDependencies", {})), + dev = package_snapshot.get("dev", False), + has_bin = package_snapshot.get("hasBin", False), + optional = package_snapshot.get("optional", False), + requires_build = package_snapshot.get("requiresBuild", False), + resolution = package_snapshot.get("resolution"), + ) if package_key in result: msg = "ERROR: duplicate package: {}\n\t{}\n\t{}".format(package_key, result[package_key], package_info) - fail(msg) + + # buildifier: disable=print + print(msg) result[package_key] = package_info return result -def _convert_pnpm_v9_package_name(package_name): - # Covert a pnpm lock file v9 name/version@version string of the format - # @scope/name@version(@scope/name@version)(@scope/name@version)@version - # to a pnpm lock file v5 @scope/name/version_peer_version format that is compatible with rules_js. - package_name = _convert_pnpm_v6_version_peer_dep(package_name) - segments = package_name.rsplit("@", 1) - if len(segments) != 2: - msg = "unexpected pnpm versioned name {}".format(package_name) - fail(msg) - return "/%s/%s" % (segments[0], segments[1]) +def _convert_pnpm_v9_package_dependency_map(deps): + result = {} + for name, version in deps.items(): + # Raw version or version(peers) or file reference + if version[0].isdigit() or version.startswith("file:") or version.startswith("link:"): + result[name] = _convert_pnpm_v6_v9_version_peer_dep(version) + else: + # Otherwise assume a reference to another package@version(...maybe peers...) + i = version.find("@", 1) + result[version[:i]] = _convert_pnpm_v6_v9_version_peer_dep(version[i + 1:]) + + return result # v9 importers are the same as v6 importers _convert_v9_importers = _convert_v6_importers @@ -190,10 +345,19 @@ def _convert_v9_packages(packages, snapshots): # v9 split package metadata (v6 "packages" field) into 2: # + # The 'snapshots' keys contain the resolved dependencies such as each unique combo of deps/peers/versions + # while 'packages' contain the static information about each and every package@version such as hasBin, + # resolution and static dep data. + # + # Note all static registry info such as URLs has moved from the 'importers[x/pkg@version].version' and 'packages[x/pkg@version]' to + # only being present in the actual packages[pkg@version].resolution.* + # + # Example: + # # packages: # '@scoped/name@5.0.2' # hasBin - # resolution (integrity etc) + # resolution (registry-url, integrity etc) # peerDependencies which *might* be resolved # # snapshots: @@ -202,45 +366,55 @@ def _convert_v9_packages(packages, snapshots): # a-dep@1.2.3 # peer@2.0.2 # b-dep@3.2.1(peer-b@4.5.6) - # - # Where the 'snapshots' keys contain the peer information while 'packages' contain the static information - # such as hasBin, resolution and peerDependencies that require resolution. result = {} # Snapshots contains the packages with the keys (which include peers) to return - for package, snapshot_info in snapshots.items(): - # convert v6 package dependencies + optionalDependencies - for key in ["dependencies", "optionalDependencies"]: - deps = snapshot_info.get(key, None) - if deps != None: - dependencies = {} - for dep_name, dep_version in deps.items(): - dependencies[dep_name] = _convert_pnpm_v6_version_peer_dep(dep_version) - snapshot_info[key] = dependencies - - # Strip peer-dep info off to get the raw package - package_version = package - if package_version[-1] == ")": - package_version = package_version[:package_version.find("(")] - - # Metadata for this snapshot persisted in the 'packages' - package_info = packages[package_version] - if package_info == None: - msg = "Failed to find pnpm-lock snapshot %s (%s) in packages" % (package, package_version) + for package_key, package_snapshot in snapshots.items(): + peer_meta_index = package_key.find("(") + + static_key = package_key[:peer_meta_index] if peer_meta_index > 0 else package_key + if not static_key in packages: + msg = "package {} not found in pnpm 'packages'".format(static_key) fail(msg) - # Also include the static data from the 'packages' - for info_name, info_value in package_info.items(): - snapshot_info[info_name] = info_value + package_data = packages[static_key] + + # the raw name + version are the key, not including peerDeps+patch + name, friendly_version = static_key.rsplit("@", 1) + package_key = _convert_pnpm_v6_v9_version_peer_dep(package_key) + + # Extract the version including peerDeps+patch from the key + version = package_key.rsplit("@", 1)[1] - package_key = _convert_pnpm_v9_package_name(package) + # package_data can have the resolved "version" for things like https:// deps + if "version" in package_data: + friendly_version = package_data["version"] + + if "resolution" not in package_data: + msg = "package {} has no resolution field".format(static_key) + fail(msg) + + package_info = _new_package_info( + id = package_data.get("id", None), + name = name, + version = version, + friendly_version = friendly_version, + dependencies = _convert_pnpm_v9_package_dependency_map(package_snapshot.get("dependencies", {})), + optional_dependencies = _convert_pnpm_v9_package_dependency_map(package_snapshot.get("optionalDependencies", {})), + peer_dependencies = _convert_pnpm_v9_package_dependency_map(package_data.get("peerDependencies", {})), + dev = None, # TODO(pnpm9): must inspect importers.*.devDependencies? + has_bin = package_data.get("hasBin", False), + optional = package_snapshot.get("optional", False), + requires_build = None, # Unknown from lockfile in v9 + resolution = package_data.get("resolution"), + ) if package_key in result: - msg = "ERROR: duplicate package: {}\n\t{}\n\t{}".format(package_key, result[package_key], snapshot_info) + msg = "ERROR: duplicate package: {}\n\t{}\n\t{}".format(package_key, result[package_key], package_info) fail(msg) - result[package_key] = snapshot_info + result[package_key] = package_info return result @@ -282,26 +456,21 @@ def _parse_pnpm_lock_common(parsed, err): lockfile_version = float(lockfile_version) _assert_lockfile_version(lockfile_version) - importers = parsed.get("importers", { - ".": { - "dependencies": parsed.get("dependencies", {}), - "optionalDependencies": parsed.get("optionalDependencies", {}), - "devDependencies": parsed.get("devDependencies", {}), - }, - }) - + # Fallback to {".": parsed} for non-workspace lockfiles where the deps are at the root. + importers = parsed.get("importers", {".": parsed}) packages = parsed.get("packages", {}) + patched_dependencies = parsed.get("patchedDependencies", {}) - if lockfile_version >= 9.0: + if lockfile_version < 6.0: + importers = _convert_v5_importers(importers) + packages = _convert_v5_packages(packages) + elif lockfile_version < 9.0: + importers = _convert_v6_importers(importers) + packages = _convert_v6_packages(packages) + else: # >= 9 snapshots = parsed.get("snapshots", {}) importers = _convert_v9_importers(importers) packages = _convert_v9_packages(packages, snapshots) - elif lockfile_version >= 6.0: - # special handling for lockfile v6 which had breaking changes - importers = _convert_v6_importers(importers) - packages = _convert_v6_packages(packages) - - patched_dependencies = parsed.get("patchedDependencies", {}) return importers, packages, patched_dependencies, lockfile_version, None @@ -380,7 +549,7 @@ def _npm_registry_download_url(package, version, registries, default_registry): registry.removesuffix("/"), package, package_name_no_scope, - _strip_peer_dep_or_patched_version(version), + _strip_v5_peer_dep_or_patched_version(version), ) def _is_git_repository_url(url): @@ -535,7 +704,7 @@ utils = struct( parse_pnpm_lock_json = _parse_pnpm_lock_json, friendly_name = _friendly_name, package_store_name = _package_store_name, - strip_peer_dep_or_patched_version = _strip_peer_dep_or_patched_version, + strip_v5_peer_dep_or_patched_version = _strip_v5_peer_dep_or_patched_version, make_symlink = _make_symlink, # Symlinked node_modules structure package store path under node_modules package_store_root = ".aspect_rules_js", @@ -555,6 +724,5 @@ utils = struct( reverse_force_copy = _reverse_force_copy, exists = _exists, replace_npmrc_token_envvar = _replace_npmrc_token_envvar, - is_vendored_tarfile = _is_vendored_tarfile, is_tarball_extension = _is_tarball_extension, )