feat(builtin): first experimental rules for npm tarballs (#2544)

This is not fully designed yet, so it's not included in public API. May be deleted at any time.
The newly added README explains what's going on.
Based on design: https://hackmd.io/gu2Nj0TKS068LKAf8KanuA

WORKSPACE

@@ -79,6 +79,19 @@ npm_install(
     package_lock_json = "//packages/node-patches:package-lock.json",
 )
 
+load("@build_bazel_rules_nodejs//internal/npm_tarballs:translate_package_lock.bzl", "translate_package_lock")
+
+# Translate our package.lock file from JSON to Starlark
+translate_package_lock(
+    name = "npm_node_patches_lock",
+    package_lock = "//packages/node-patches:package-lock.json",
+)
+
+load("@npm_node_patches_lock//:index.bzl", _npm_patches_repositories = "npm_repositories")
+
+# # Declare an external repository for each npm package fetchable by the lock file
+_npm_patches_repositories()
+
 npm_install(
     name = "angular_deps",
     package_json = "//packages/angular:package.json",

internal/common/download.bzl

@@ -0,0 +1,43 @@
+"Repository rule wrapper around Bazel's downloader"
+
+def _bazel_download(repository_ctx):
+    repository_ctx.file("BUILD.bazel", repository_ctx.attr.build_file_content)
+    repository_ctx.download(
+        output = repository_ctx.attr.output,
+        url = repository_ctx.attr.url,
+        integrity = repository_ctx.attr.integrity,
+    )
+
+bazel_download = repository_rule(
+    doc = """Utility to call Bazel downloader.
+
+    This is a simple pass-thru wrapper for Bazel's
+    [repository_ctx#download](https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#download)
+    function.
+    """,
+    implementation = _bazel_download,
+    attrs = {
+        "build_file_content": attr.string(
+            doc = "Content for the generated BUILD file.",
+            mandatory = True,
+        ),
+        "integrity": attr.string(
+            doc = """
+            Expected checksum of the file downloaded, in Subresource Integrity format.
+            This must match the checksum of the file downloaded.
+            It is a security risk to omit the checksum as remote files can change.
+            At best omitting this field will make your build non-hermetic.
+            It is optional to make development easier but should be set before shipping.
+            """,
+            mandatory = True,
+        ),
+        "output": attr.string(
+            doc = "path to the output file, relative to the repository directory",
+            mandatory = True,
+        ),
+        "url": attr.string_list(
+            doc = "List of mirror URLs referencing the same file.",
+            mandatory = True,
+        ),
+    },
+)

internal/npm_tarballs/BUILD.bazel

@@ -0,0 +1 @@
+# No bazel targets in this package

internal/npm_tarballs/README.md

@@ -0,0 +1,68 @@
+# npm_tarballs
+
+This is an expermental feature inspired by external package fetching in rules_go and others.
+
+See the design doc: https://hackmd.io/gu2Nj0TKS068LKAf8KanuA
+
+## Rules
+
+`translate_package_lock.bzl` takes a package-lock.json file and produces a Starlark representation of downloader rules for each package listed.
+
+Currently this is implemented only for npm v7 produced lockfiles (version 2 of the spec) but it could be ported to any other lockfile format.
+
+For example, for https://github.com/bazelbuild/rules_nodejs/blob/stable/packages/node-patches/package-lock.json we produce an `index.bzl` file like:
+
+```
+"Generated by package_lock.bzl from //packages/node-patches:package-lock.json"
+
+load("@build_bazel_rules_nodejs//internal/common:download.bzl", "bazel_download")
+
+def npm_repositories():
+    """Define external repositories to fetch each tarball individually from npm on-demand.
+    """
+
+# [...]
+
+    bazel_download(
+        name = "npm_typescript-3.5.3",
+        output = "typescript-3.5.3.tgz",
+        integrity = "sha512-ACzBtm/PhXBDId6a6sDJfroT2pOWt/oOnk4/dElG5G33ZL776N3Y6/6bKZJBFpd+b05F3Ct9qDjMeJmRWtE2/g==",
+        url = ["https://registry.npmjs.org/typescript/-/typescript-3.5.3.tgz"],
+        build_file_content = """"Generated by package_lock.bzl"
+
+load("@build_bazel_rules_nodejs//internal/npm_tarballs:npm_tarball.bzl", "npm_tarball")
+
+npm_tarball(
+    name = "npm_typescript-3.5.3",
+    src = "typescript-3.5.3.tgz",
+    package_name = "typescript",
+    deps = [],
+    visibility = ["//visibility:public"],
+)
+
+"""
+    )
+
+# [...]
+```
+
+This generated index.bzl can then be loaded in the WORKSPACE and the `npm_repositories` macro called.
+This then declares `bazel_download` rules that are themselves able to fetch packages on-demand.
+We also supply a BUILD file content for each of these packages, using a minimal `npm_tarball` rule that
+represents the location and dependencies of the downloaded .tgz file.
+
+In addition, we give some syntax sugar.
+In the repo produced by `translate_package_lock` we provide "catch-all" targets
+`//:dependencies` and `//:devDependencies` that depend on all tarballs so listed in the package-lock.json.
+For direct dependencies, we also produce a `//somepackage` target that aliases the version of `somepackage` depended on.
+In the above example, that means the user can dep on `@npm_repositories//typescript` rather than
+`@npm_typescript-3.5.3` because we know the package depends on version 3.5.3.
+
+## Future work
+
+So far the resulting tarballs aren't used by anything in rules_nodejs (nothing consumes `NpmTarballInfo`).
+In later work we'll explore what other rules might want to use the tarballs,
+such as a pnpm_install rule that uses pnpm semantics to just symlink things into a tree.
+Or maybe an npm_install rule, one for each package, that unpacks the tarballs and runs the postinstall logic on each.
+We believe some experimentation will be required to find a good path forward that uses the download-as-needed semantics here,
+while keeping most existing semantics of rules_nodejs rules working.

internal/npm_tarballs/npm_tarball.bzl

@@ -0,0 +1,64 @@
+"Info about npm tarball files"
+
+NpmTarballInfo = provider(
+    doc = "Describe tarballs downloaded from npm registry",
+    fields = {
+        "tarballs": "depset of needed tarballs to be able to npm install",
+    },
+)
+
+_DOC = """This rule is a simple reference to a file downloaded from npm.
+
+It is not meant to be used on its own, rather it is generated into BUILD files in external repos
+and its provider can then be referenced in actions by tools like pnpm that need to find the .tgz files.
+"""
+
+_ATTRS = {
+    "deps": attr.label_list(
+        doc = "Other npm_tarball rules for packages this one depends on",
+        providers = [NpmTarballInfo],
+    ),
+    "package_name": attr.string(
+        doc = "the name field from the package.json of the package this tarball contains",
+    ),
+    "src": attr.label(
+        doc = "The downloaded tarball",
+        allow_single_file = [".tgz"],
+    ),
+}
+
+def _npm_tarball(ctx):
+    # Allow aggregate rules like "all_dependencies" to have only deps but no tarball
+    if ctx.attr.src and not ctx.attr.package_name:
+        fail("when given a src, must also tell the package_name for it")
+    direct = []
+    direct_files = []
+    if ctx.attr.src:
+        direct = [struct(
+            package_name = ctx.attr.package_name,
+            tarball = ctx.file.src,
+        )]
+        direct_files = [ctx.file.src]
+
+    transitive = [d[NpmTarballInfo].tarballs for d in ctx.attr.deps]
+    transitive_files = []
+    for dset in transitive:
+        for info in dset.to_list():
+            transitive_files.append(info.tarball)
+    return [
+        NpmTarballInfo(tarballs = depset(
+            direct,
+            transitive = transitive,
+        )),
+        # For testing
+        OutputGroupInfo(
+            direct = direct_files,
+            transitive = transitive_files,
+        ),
+    ]
+
+npm_tarball = rule(
+    implementation = _npm_tarball,
+    attrs = _ATTRS,
+    doc = _DOC,
+)

internal/npm_tarballs/test/BUILD.bazel

@@ -0,0 +1,58 @@
+load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_test")
+
+# In normal usage, some other rule would extract the NpmTarballInfo
+# For testing we simply want to grab the file
+filegroup(
+    name = "get_typescript",
+    # the internal/node-patches package-lock.json depends on this version of typescript
+    srcs = ["@npm_typescript-3.5.3"],
+    output_group = "direct",
+)
+
+nodejs_test(
+    name = "test_some_package_fetched",
+    data = [":get_typescript"],
+    entry_point = "test_some_package_fetched.js",
+)
+
+filegroup(
+    name = "get_typescript_alias",
+    # Since typescript is a direct dependency, we can point to the version used by this package
+    # without having to specify (it's an alias)
+    srcs = ["@npm_node_patches_lock//typescript"],
+    output_group = "direct",
+)
+
+# Run the same test again but point to this filegroup to be sure the same typescript was there
+nodejs_test(
+    name = "test_alias",
+    data = [":get_typescript_alias"],
+    entry_point = "test_some_package_fetched.js",
+)
+
+filegroup(
+    name = "get_all_devdeps",
+    # Check that there's also a syntax-sugar for "all the devDependencies listed"
+    srcs = ["@npm_node_patches_lock//:devDependencies"],
+    output_group = "transitive",
+)
+
+# Run that same test again, typescript should be in here
+nodejs_test(
+    name = "test_alldevdeps",
+    data = [":get_all_devdeps"],
+    entry_point = "test_some_package_fetched.js",
+)
+
+filegroup(
+    name = "get_ansi-align",
+    # According to package-lock.json, it depends on string-width@3.1.0
+    srcs = ["@npm_ansi-align-3.0.0"],
+    output_group = "transitive",
+)
+
+nodejs_test(
+    name = "test_dependencies_available",
+    data = [":get_ansi-align"],
+    entry_point = "test_dependencies_available.js",
+)

internal/npm_tarballs/test/test_dependencies_available.js

@@ -0,0 +1,6 @@
+const assert = require('assert');
+const {existsSync} = require('fs');
+const runfiles = require(process.env['BAZEL_NODE_RUNFILES_HELPER']);
+const tarPath = runfiles.resolve('npm_string-width-3.1.0/string-width-3.1.0.tgz');
+
+assert.ok(existsSync(tarPath));

internal/npm_tarballs/test/test_some_package_fetched.js

@@ -0,0 +1,12 @@
+const assert = require('assert');
+const {existsSync, statSync} = require('fs');
+const runfiles = require(process.env['BAZEL_NODE_RUNFILES_HELPER']);
+const tarPath = runfiles.resolve('npm_typescript-3.5.3/typescript-3.5.3.tgz');
+
+assert.ok(existsSync(tarPath));
+
+// The size of https://www.npmjs.com/package/typescript/v/3.5.3
+expectedSize = 7960741;
+assert.strictEqual(
+    statSync(tarPath).size, expectedSize,
+    `Expected to download the typescript 3.5.3 release which is ${expectedSize} bytes`);