Skip to content

Commit

Permalink
refactor: normalize pnpm lockfile data in parser
Browse files Browse the repository at this point in the history
  • Loading branch information
@jbedard
jbedard committed May 15, 2024
1 parent 2ae73ac commit a8f655f
Show file tree
Hide file tree
Showing 6 changed files with 287 additions and 151 deletions.
46 changes: 32 additions & 14 deletions npm/private/test/parse_pnpm_lock_tests.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,22 +87,31 @@ packages:
"dependencies": {
"@aspect-test/a": "5.0.0",
},
"optionalDependencies": {},
"devDependencies": {},
"dev_dependencies": {},
"optional_dependencies": {},
},
},
{
"/@aspect-test/a/5.0.0": {
"resolution": {
"integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==",
},
"hasBin": True,
"@aspect-test/a@5.0.0": {
"id": None,
"name": "@aspect-test/a",
"key": "@aspect-test/a@5.0.0",
"dependencies": {
"@aspect-test/b": "5.0.0",
"@aspect-test/c": "1.0.0",
"@aspect-test/d": "2.0.0_@aspect-test+c@1.0.0",
},
"optional_dependencies": {},
"peer_dependencies": {},
"dev": False,
"has_bin": True,
"optional": False,
"requires_build": False,
"version": "5.0.0",
"friendly_version": "5.0.0",
"resolution": {
"integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==",
},
},
},
{},
Expand Down Expand Up @@ -169,22 +178,31 @@ packages:
"dependencies": {
"@aspect-test/a": "5.0.0",
},
"optionalDependencies": {},
"devDependencies": {},
"dev_dependencies": {},
"optional_dependencies": {},
},
},
{
"/@aspect-test/a/5.0.0": {
"resolution": {
"integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==",
},
"hasBin": True,
"@aspect-test/a@5.0.0": {
"id": None,
"name": "@aspect-test/a",
"key": "@aspect-test/a@5.0.0",
"dependencies": {
"@aspect-test/b": "5.0.0",
"@aspect-test/c": "1.0.0",
"@aspect-test/d": "2.0.0_at_aspect-test_c_1.0.0",
},
"optional_dependencies": {},
"peer_dependencies": {},
"dev": False,
"has_bin": True,
"optional": False,
"requires_build": False,
"version": "5.0.0",
"friendly_version": "5.0.0",
"resolution": {
"integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==",
},
},
},
{},
Expand Down
8 changes: 4 additions & 4 deletions npm/private/test/snapshots/bzlmod/repositories.bzl
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 7 additions & 7 deletions npm/private/test/transitive_closure_tests.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest")
load("//npm/private:transitive_closure.bzl", "gather_transitive_closure")

TEST_PACKAGES = {
"@aspect-test/a/5.0.0": {
"@aspect-test/a@5.0.0": {
"name": "@aspect-test/a",
"version": "5.0.0",
"integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==",
Expand All @@ -17,21 +17,21 @@ TEST_PACKAGES = {
},
"optional_dependencies": {},
},
"@aspect-test/b/5.0.0": {
"@aspect-test/b@5.0.0": {
"dependencies": {},
"optional_dependencies": {
"@aspect-test/c": "2.0.0",
},
},
"@aspect-test/c/1.0.0": {
"@aspect-test/c@1.0.0": {
"dependencies": {},
"optional_dependencies": {},
},
"@aspect-test/c/2.0.0": {
"@aspect-test/c@2.0.0": {
"dependencies": {},
"optional_dependencies": {},
},
"@aspect-test/d/2.0.0_@aspect-test+c@1.0.0": {
"@aspect-test/d@2.0.0_@aspect-test+c@1.0.0": {
"dependencies": {},
"optional_dependencies": {},
},
Expand All @@ -44,12 +44,12 @@ def test_walk_deps(ctx):
not_no_optional = False

# Walk the example tree above
closure = gather_transitive_closure(TEST_PACKAGES, "@aspect-test/a/5.0.0", not_no_optional)
closure = gather_transitive_closure(TEST_PACKAGES, "@aspect-test/a@5.0.0", not_no_optional)
expected = {"@aspect-test/a": ["5.0.0"], "@aspect-test/b": ["5.0.0"], "@aspect-test/c": ["1.0.0", "2.0.0"], "@aspect-test/d": ["2.0.0_@aspect-test+c@1.0.0"]}
asserts.equals(env, expected, closure)

# Run again with no_optional set, this means we shouldn't walk the dep from @aspect-test/b/5.0.0 -> @aspect-test/c/2.0.0
closure = gather_transitive_closure(TEST_PACKAGES, "@aspect-test/a/5.0.0", no_optional)
closure = gather_transitive_closure(TEST_PACKAGES, "@aspect-test/a@5.0.0", no_optional)
expected = {"@aspect-test/a": ["5.0.0"], "@aspect-test/b": ["5.0.0"], "@aspect-test/c": ["1.0.0"], "@aspect-test/d": ["2.0.0_@aspect-test+c@1.0.0"]}
asserts.equals(env, expected, closure)

Expand Down
6 changes: 3 additions & 3 deletions npm/private/test/utils_tests.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ def test_bazel_name(ctx):
# buildifier: disable=function-docstring
def test_pnpm_name(ctx):
env = unittest.begin(ctx)
asserts.equals(env, "@scope/y/1.1.1", utils.pnpm_name("@scope/y", "1.1.1"))
asserts.equals(env, ("@scope/y", "1.1.1"), utils.parse_pnpm_package_key("@scope/y", "/@scope/y/1.1.1"))
asserts.equals(env, ("@scope/y", "registry/@scope/y/1.1.1"), utils.parse_pnpm_package_key("@scope/y", "registry/@scope/y/1.1.1"))
asserts.equals(env, "@scope/y@1.1.1", utils.pnpm_name("@scope/y", "1.1.1"))
asserts.equals(env, ("@scope/y", "1.1.1"), utils.parse_pnpm_package_key("@scope/y", "/@scope/y/1.1.1", "/"))
asserts.equals(env, ("@scope/y", "registry/@scope/y@1.1.1"), utils.parse_pnpm_package_key("@scope/y", "registry/@scope/y@1.1.1"))
asserts.equals(env, ("@scope/y", "1.1.1"), utils.parse_pnpm_package_key("@scope/y", "1.1.1"))
return unittest.end(env)

Expand Down
88 changes: 11 additions & 77 deletions npm/private/transitive_closure.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,72 +75,12 @@ def gather_transitive_closure(packages, package, no_optional, cache = {}):
def _get_package_info_deps(package_info, no_optional):
return package_info["dependencies"] if no_optional else dicts.add(package_info["dependencies"], package_info["optional_dependencies"])

def _gather_package_info(package_path, package_snapshot):
if package_path.startswith("/"):
# an aliased dependency
package = package_path[1:]
name, version = utils.parse_pnpm_package_key("", package_path)
friendly_version = utils.strip_peer_dep_or_patched_version(version)
package_key = package
elif package_path.startswith("file:") and utils.is_vendored_tarfile(package_snapshot):
if "name" not in package_snapshot:
fail("expected package %s to have a name field" % package_path)
name = package_snapshot["name"]
package = package_snapshot["name"]
version = package_path
if "version" in package_snapshot:
version = package_snapshot["version"]
package_key = "{}/{}".format(package, version)
friendly_version = version
elif package_path.startswith("file:"):
package = package_path
if "name" not in package_snapshot:
msg = "expected package {} to have a name field".format(package_path)
fail(msg)
name = package_snapshot["name"]
version = package_path
friendly_version = package_snapshot["version"] if "version" in package_snapshot else version
package_key = package
else:
package = package_path
if "name" not in package_snapshot:
msg = "expected package {} to have a name field".format(package_path)
fail(msg)
if "version" not in package_snapshot:
msg = "expected package {} to have a version field".format(package_path)
fail(msg)
name = package_snapshot["name"]
version = package_path
friendly_version = package_snapshot["version"]
package_key = package

if "resolution" not in package_snapshot:
msg = "package {} has no resolution field".format(package_path)
fail(msg)
id = package_snapshot["id"] if "id" in package_snapshot else None
resolution = package_snapshot["resolution"]

return package_key, {
"name": name,
"id": id,
"version": version,
"friendly_version": friendly_version,
"resolution": resolution,
"dependencies": package_snapshot.get("dependencies", {}),
"optional_dependencies": package_snapshot.get("optionalDependencies", {}),
"dev": package_snapshot.get("dev", False),
"optional": package_snapshot.get("optional", False),
"patched": package_snapshot.get("patched", False),
"has_bin": package_snapshot.get("hasBin", False),
"requires_build": package_snapshot.get("requiresBuild", False),
}

def translate_to_transitive_closure(lock_importers, lock_packages, prod = False, dev = False, no_optional = False):
def translate_to_transitive_closure(importers, packages, prod = False, dev = False, no_optional = False):
"""Implementation detail of translate_package_lock, converts pnpm-lock to a different dictionary with more data.
Args:
lock_importers: lockfile importers dict
lock_packages: lockfile packages dict
importers: workspace projects (pnpm "importers")
packages: all package info by name
prod: If true, only install dependencies
dev: If true, only install devDependencies
no_optional: If true, optionalDependencies are not installed
Expand All @@ -149,27 +89,21 @@ def translate_to_transitive_closure(lock_importers, lock_packages, prod = False,
Nested dictionary suitable for further processing in our repository rule
"""

# All package info mapped by package name
packages = {}

# Packages resolved to a different version
package_version_map = {}

for package_path, package_snapshot in lock_packages.items():
package, package_info = _gather_package_info(package_path, package_snapshot)
packages[package] = package_info

# tarbal versions
# tarbal versions
for package_key, package_info in packages.items():
if package_info["resolution"].get("tarball", None) and package_info["resolution"]["tarball"].startswith("file:"):
package_version_map[package] = package_info
package_version_map[package_key] = package_info

# Collect deps of each importer (workspace projects)
importers = {}
for importPath in lock_importers.keys():
lock_importer = lock_importers[importPath]
prod_deps = {} if dev else lock_importer.get("dependencies", {})
dev_deps = {} if prod else lock_importer.get("devDependencies", {})
opt_deps = {} if no_optional else lock_importer.get("optionalDependencies", {})
for importPath in importers.keys():
lock_importer = importers[importPath]
prod_deps = {} if dev else lock_importer.get("dependencies")
dev_deps = {} if prod else lock_importer.get("dev_dependencies")
opt_deps = {} if no_optional else lock_importer.get("optional_dependencies")

deps = dicts.add(prod_deps, opt_deps)
all_deps = dicts.add(prod_deps, dev_deps, opt_deps)
Expand Down
Loading

0 comments on commit a8f655f

Please sign in to comment.