Merge branch 'main' into bzlmod

.bazelci/presubmit.yml

@@ -533,6 +533,22 @@ tasks:
     test_targets:
       - "//..."
     build_flags: *aspects_flags
+  crate_universe_local_path_external:
+    name: Crate Universe Local Path External
+    platform: ubuntu2004
+    working_directory: examples/crate_universe_local_path
+    run_targets:
+      - "//:vendor_lazy_static_out_of_tree"
+    test_targets:
+      - "//..."
+  crate_universe_local_path_in_tree:
+    name: Crate Universe Local Path In Tree
+    platform: ubuntu2004
+    working_directory: examples/crate_universe_local_path
+    run_targets:
+      - "//:vendor_lazy_static_in_tree"
+    test_targets:
+      - "//..."
 # See https://github.com/bazelbuild/rules_rust/issues/2186 about re-enabling these.
 #  crate_universe_examples_windows:
 #    name: Crate Universe Examples

crate_universe/extensions.bzl

@@ -6,6 +6,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "generate_config_file", "generate_splicing_manifest")
 load("//crate_universe/private:generate_utils.bzl", "CARGO_BAZEL_GENERATOR_SHA256", "CARGO_BAZEL_GENERATOR_URL", "GENERATOR_ENV_VARS", "render_config")
+load("//crate_universe/private:local_crate_mirror.bzl", "local_crate_mirror")
 load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS")
 load("//rust/platform:triple.bzl", "get_host_triple")
 load("//rust/platform:triple_mappings.bzl", "system_to_binary_ext")
@@ -133,6 +134,9 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo
     lockfile_path = tag_path.get_child("lockfile.json")
     module_ctx.file(lockfile_path, "")
 
+    paths_to_track_file = module_ctx.path("paths-to-track")
+    warnings_output_file = module_ctx.path("warnings-output-file")
+
     cargo_bazel([
         "generate",
         "--cargo-lockfile",
@@ -148,8 +152,25 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo
         "--repin",
         "--lockfile",
         lockfile_path,
+        "--nonhermetic-root-bazel-workspace-dir",
+        module_ctx.path(Label("@@//:MODULE.bazel")).dirname,
+        "--paths-to-track",
+        paths_to_track_file,
+        "--warnings-output-path",
+        warnings_output_file,
     ])
 
+    paths_to_track = json.decode(module_ctx.read(paths_to_track_file))
+    for path in paths_to_track:
+        # This read triggers watching the file at this path and invalidates the repository_rule which will get re-run.
+        # Ideally we'd use module_ctx.watch, but it doesn't support files outside of the workspace, and we need to support that.
+        module_ctx.read(path)
+
+    warnings_output_file = json.decode(module_ctx.read(warnings_output_file))
+    for warning in warnings_output_file:
+        # buildifier: disable=print
+        print("WARN: {}".format(warning))
+
     crates_dir = tag_path.get_child(cfg.name)
     _generate_repo(
         name = cfg.name,
@@ -212,6 +233,22 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo
                 strip_prefix = repo.get("strip_prefix", None),
                 **kwargs
             )
+        elif "Path" in repo:
+            options = {
+                "config": rendering_config,
+                "crate_context": crate,
+                "platform_conditions": contents["conditions"],
+                "supported_platform_triples": cfg.supported_platform_triples,
+            }
+            kwargs = {}
+            if len(CARGO_BAZEL_URLS) == 0:
+                kwargs["generator"] = "@cargo_bazel_bootstrap//:cargo-bazel"
+            local_crate_mirror(
+                name = crate_repo_name,
+                options_json = json.encode(options),
+                path = repo["Path"]["path"],
+                **kwargs
+            )
         else:
             fail("Invalid repo: expected Http or Git to exist for crate %s-%s, got %s" % (name, version, repo))
 

crate_universe/private/crates_repository.bzl

@@ -71,6 +71,9 @@ def _crates_repository_impl(repository_ctx):
             "metadata": metadata_path,
         })
 
+    paths_to_track_file = repository_ctx.path("paths-to-track")
+    warnings_output_file = repository_ctx.path("warnings-output-file")
+
     # Run the generator
     execute_generator(
         repository_ctx = repository_ctx,
@@ -82,10 +85,23 @@ def _crates_repository_impl(repository_ctx):
         repository_dir = repository_ctx.path("."),
         cargo = cargo_path,
         rustc = rustc_path,
+        paths_to_track_file = paths_to_track_file,
+        warnings_output_file = warnings_output_file,
         # sysroot = tools.sysroot,
         **kwargs
     )
 
+    paths_to_track = json.decode(repository_ctx.read(paths_to_track_file))
+    for path in paths_to_track:
+        # This read triggers watching the file at this path and invalidates the repository_rule which will get re-run.
+        # Ideally we'd use repository_ctx.watch, but it doesn't support files outside of the workspace, and we need to support that.
+        repository_ctx.read(path)
+
+    warnings_output_file = json.decode(repository_ctx.read(warnings_output_file))
+    for warning in warnings_output_file:
+        # buildifier: disable=print
+        print("WARN: {}".format(warning))
+
     # Determine the set of reproducible values
     attrs = {attr: getattr(repository_ctx.attr, attr) for attr in dir(repository_ctx.attr)}
     exclude = ["to_json", "to_proto"]

crate_universe/private/crates_vendor.bzl

@@ -34,6 +34,7 @@ ${{_ENVIRON[@]}} \\
 {env} \\
     "{bin}" \\
     {args} \\
+    --nonhermetic-root-bazel-workspace-dir="${{BUILD_WORKSPACE_DIRECTORY}}" \\
     "$@"
 """
 
@@ -42,7 +43,7 @@ _WINDOWS_WRAPPER = """\
 set RUNTIME_PWD=%CD%
 {env}
 
-{bin} {args} %*
+{bin} {args} --nonhermetic-root-bazel-workspace-dir=%BUILD_WORKSPACE_DIRECTORY% %*
 exit %ERRORLEVEL%
 """
 

crate_universe/private/generate_utils.bzl

@@ -433,6 +433,8 @@ def execute_generator(
         repository_dir,
         cargo,
         rustc,
+        paths_to_track_file,
+        warnings_output_file,
         metadata = None):
     """Execute the `cargo-bazel` binary to produce `BUILD` and `.bzl` files.
 
@@ -446,6 +448,8 @@ def execute_generator(
         repository_dir (path): The output path for the Bazel module and BUILD files.
         cargo (path): The path of a Cargo binary.
         rustc (path): The path of a Rustc binary.
+        paths_to_track_file (path): Path to file where generator should write which files should trigger re-generating as a JSON list.
+        warnings_output_file (path): Path to file where generator should write warnings to print.
         metadata (path, optional): The path to a Cargo metadata json file. If this is set, it indicates to
             the generator that repinning is required. This file must be adjacent to a `Cargo.toml` and
             `Cargo.lock` file.
@@ -470,8 +474,20 @@ def execute_generator(
         cargo,
         "--rustc",
         rustc,
+        "--nonhermetic-root-bazel-workspace-dir",
+        repository_ctx.workspace_root,
+        "--paths-to-track",
+        paths_to_track_file,
+        "--warnings-output-path",
+        warnings_output_file,
     ]
 
+    if repository_ctx.attr.generator:
+        args.extend([
+            "--generator",
+            repository_ctx.attr.generator,
+        ])
+
     if lockfile_path:
         args.extend([
             "--lockfile",

crate_universe/private/local_crate_mirror.bzl

@@ -0,0 +1,64 @@
+"""`local_crate_mirror` rule implementation."""
+
+load("//crate_universe/private:common_utils.bzl", "execute")
+load("//crate_universe/private:generate_utils.bzl", "get_generator")
+load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS")
+load("//rust/platform:triple.bzl", "get_host_triple")
+
+def _local_crate_mirror_impl(repository_ctx):
+    path = repository_ctx.path(repository_ctx.attr.path)
+
+    host_triple = get_host_triple(repository_ctx)
+
+    generator, _generator_sha256 = get_generator(repository_ctx, host_triple.str)
+
+    # TODO: Work out why we can't just symlink here and actually copy.
+    # illicitonion thinks it may be that symlinks didn't get invalidated properly?
+    for child in repository_ctx.path(path).readdir():
+        repository_ctx.execute(["cp", "-r", child, repository_ctx.path(child.basename)])
+
+    paths_to_track = execute(repository_ctx, ["find", path, "-type", "f"]).stdout.strip().split("\n")
+    for path_to_track in paths_to_track:
+        if path_to_track:
+            repository_ctx.read(path_to_track)
+
+    execute(repository_ctx, [generator, "render", "--options-json", repository_ctx.attr.options_json, "--output-path", repository_ctx.path("BUILD.bazel")])
+
+    repository_ctx.file("WORKSPACE.bazel", "")
+
+local_crate_mirror = repository_rule(
+    doc = """This is a private implementation detail of crate_universe, and should not be relied on in manually written code.
+
+This is effectively a `local_repository` rule impementation, but where the BUILD.bazel file is generated using the `cargo-bazel render` command.""",
+    implementation = _local_crate_mirror_impl,
+    attrs = {
+        "generator": attr.string(
+            doc = (
+                "The absolute label of a generator. Eg. `@cargo_bazel_bootstrap//:cargo-bazel`. " +
+                "This is typically used when bootstrapping"
+            ),
+        ),
+        "generator_sha256s": attr.string_dict(
+            doc = "Dictionary of `host_triple` -> `sha256` for a `cargo-bazel` binary.",
+            default = CARGO_BAZEL_SHA256S,
+        ),
+        "generator_urls": attr.string_dict(
+            doc = (
+                "URL template from which to download the `cargo-bazel` binary. `{host_triple}` and will be " +
+                "filled in according to the host platform."
+            ),
+            default = CARGO_BAZEL_URLS,
+        ),
+        "options_json": attr.string(
+            doc = "JSON serialized instance of a crate_universe::context::SingleBuildFileRenderContext",
+        ),
+        "path": attr.string(
+            # TODO: Verify what happens if this is not an absolute path.
+            doc = "Absolute path to the BUILD.bazel file to generate.",
+        ),
+        "quiet": attr.bool(
+            doc = "If stdout and stderr should not be printed to the terminal.",
+            default = True,
+        ),
+    },
+)

crate_universe/private/srcs.bzl

@@ -11,6 +11,7 @@ CARGO_BAZEL_SRCS = [
     Label("//crate_universe:src/cli.rs"),
     Label("//crate_universe:src/cli/generate.rs"),
     Label("//crate_universe:src/cli/query.rs"),
+    Label("//crate_universe:src/cli/render.rs"),
     Label("//crate_universe:src/cli/splice.rs"),
     Label("//crate_universe:src/cli/vendor.rs"),
     Label("//crate_universe:src/config.rs"),

crate_universe/src/cli.rs

@@ -2,6 +2,7 @@
 
 mod generate;
 mod query;
+mod render;
 mod splice;
 mod vendor;
 
@@ -15,12 +16,14 @@ use tracing_subscriber::FmtSubscriber;
 
 pub use self::generate::GenerateOptions;
 pub use self::query::QueryOptions;
+pub use self::render::RenderOptions;
 pub use self::splice::SpliceOptions;
 pub use self::vendor::VendorOptions;
 
 // Entrypoints
 pub use generate::generate;
 pub use query::query;
+pub use render::render;
 pub use splice::splice;
 pub use vendor::vendor;
 
@@ -42,6 +45,9 @@ pub enum Options {
 
     /// Vendor BUILD files to the workspace with either repository definitions or `cargo vendor` generated sources.
     Vendor(VendorOptions),
+
+    /// Render a BUILD file for a single crate.
+    Render(RenderOptions),
 }
 
 // Convenience wrappers to avoid dependencies in the binary
@@ -51,7 +57,7 @@ pub fn parse_args() -> Options {
     Options::parse()
 }
 
-const EXPECTED_LOGGER_NAMES: [&str; 4] = ["Generate", "Splice", "Query", "Vendor"];
+const EXPECTED_LOGGER_NAMES: [&str; 5] = ["Generate", "Splice", "Query", "Vendor", "Render"];
 
 /// A wrapper for the tracing-subscriber default [FormatEvent]
 /// that prepends the name of the active CLI option.