diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index aecde57d75..6d4bfb4177 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -533,6 +533,22 @@ tasks: test_targets: - "//..." build_flags: *aspects_flags + crate_universe_local_path_external: + name: Crate Universe Local Path External + platform: ubuntu2004 + working_directory: examples/crate_universe_local_path + run_targets: + - "//:vendor_lazy_static_out_of_tree" + test_targets: + - "//..." + crate_universe_local_path_in_tree: + name: Crate Universe Local Path In Tree + platform: ubuntu2004 + working_directory: examples/crate_universe_local_path + run_targets: + - "//:vendor_lazy_static_in_tree" + test_targets: + - "//..." # See https://github.com/bazelbuild/rules_rust/issues/2186 about re-enabling these. # crate_universe_examples_windows: # name: Crate Universe Examples diff --git a/crate_universe/extensions.bzl b/crate_universe/extensions.bzl index 943fb73976..279b9d703e 100644 --- a/crate_universe/extensions.bzl +++ b/crate_universe/extensions.bzl @@ -6,6 +6,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "generate_config_file", "generate_splicing_manifest") load("//crate_universe/private:generate_utils.bzl", "CARGO_BAZEL_GENERATOR_SHA256", "CARGO_BAZEL_GENERATOR_URL", "GENERATOR_ENV_VARS", "render_config") +load("//crate_universe/private:local_crate_mirror.bzl", "local_crate_mirror") load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS") load("//rust/platform:triple.bzl", "get_host_triple") load("//rust/platform:triple_mappings.bzl", "system_to_binary_ext") @@ -133,6 +134,9 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo lockfile_path = tag_path.get_child("lockfile.json") module_ctx.file(lockfile_path, "") + paths_to_track_file = module_ctx.path("paths-to-track") + warnings_output_file = module_ctx.path("warnings-output-file") + cargo_bazel([ "generate", "--cargo-lockfile", @@ -148,8 +152,25 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo "--repin", "--lockfile", lockfile_path, + "--nonhermetic-root-bazel-workspace-dir", + module_ctx.path(Label("@@//:MODULE.bazel")).dirname, + "--paths-to-track", + paths_to_track_file, + "--warnings-output-path", + warnings_output_file, ]) + paths_to_track = json.decode(module_ctx.read(paths_to_track_file)) + for path in paths_to_track: + # This read triggers watching the file at this path and invalidates the repository_rule which will get re-run. + # Ideally we'd use module_ctx.watch, but it doesn't support files outside of the workspace, and we need to support that. + module_ctx.read(path) + + warnings_output_file = json.decode(module_ctx.read(warnings_output_file)) + for warning in warnings_output_file: + # buildifier: disable=print + print("WARN: {}".format(warning)) + crates_dir = tag_path.get_child(cfg.name) _generate_repo( name = cfg.name, @@ -212,6 +233,22 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo strip_prefix = repo.get("strip_prefix", None), **kwargs ) + elif "Path" in repo: + options = { + "config": rendering_config, + "crate_context": crate, + "platform_conditions": contents["conditions"], + "supported_platform_triples": cfg.supported_platform_triples, + } + kwargs = {} + if len(CARGO_BAZEL_URLS) == 0: + kwargs["generator"] = "@cargo_bazel_bootstrap//:cargo-bazel" + local_crate_mirror( + name = crate_repo_name, + options_json = json.encode(options), + path = repo["Path"]["path"], + **kwargs + ) else: fail("Invalid repo: expected Http or Git to exist for crate %s-%s, got %s" % (name, version, repo)) diff --git a/crate_universe/private/crates_repository.bzl b/crate_universe/private/crates_repository.bzl index 4a1952c634..70ce40bfb4 100644 --- a/crate_universe/private/crates_repository.bzl +++ b/crate_universe/private/crates_repository.bzl @@ -71,6 +71,9 @@ def _crates_repository_impl(repository_ctx): "metadata": metadata_path, }) + paths_to_track_file = repository_ctx.path("paths-to-track") + warnings_output_file = repository_ctx.path("warnings-output-file") + # Run the generator execute_generator( repository_ctx = repository_ctx, @@ -82,10 +85,23 @@ def _crates_repository_impl(repository_ctx): repository_dir = repository_ctx.path("."), cargo = cargo_path, rustc = rustc_path, + paths_to_track_file = paths_to_track_file, + warnings_output_file = warnings_output_file, # sysroot = tools.sysroot, **kwargs ) + paths_to_track = json.decode(repository_ctx.read(paths_to_track_file)) + for path in paths_to_track: + # This read triggers watching the file at this path and invalidates the repository_rule which will get re-run. + # Ideally we'd use repository_ctx.watch, but it doesn't support files outside of the workspace, and we need to support that. + repository_ctx.read(path) + + warnings_output_file = json.decode(repository_ctx.read(warnings_output_file)) + for warning in warnings_output_file: + # buildifier: disable=print + print("WARN: {}".format(warning)) + # Determine the set of reproducible values attrs = {attr: getattr(repository_ctx.attr, attr) for attr in dir(repository_ctx.attr)} exclude = ["to_json", "to_proto"] diff --git a/crate_universe/private/crates_vendor.bzl b/crate_universe/private/crates_vendor.bzl index b8d95627bf..a195f8c20f 100644 --- a/crate_universe/private/crates_vendor.bzl +++ b/crate_universe/private/crates_vendor.bzl @@ -34,6 +34,7 @@ ${{_ENVIRON[@]}} \\ {env} \\ "{bin}" \\ {args} \\ + --nonhermetic-root-bazel-workspace-dir="${{BUILD_WORKSPACE_DIRECTORY}}" \\ "$@" """ @@ -42,7 +43,7 @@ _WINDOWS_WRAPPER = """\ set RUNTIME_PWD=%CD% {env} -{bin} {args} %* +{bin} {args} --nonhermetic-root-bazel-workspace-dir=%BUILD_WORKSPACE_DIRECTORY% %* exit %ERRORLEVEL% """ diff --git a/crate_universe/private/generate_utils.bzl b/crate_universe/private/generate_utils.bzl index 94dcdd2360..a345b7538f 100644 --- a/crate_universe/private/generate_utils.bzl +++ b/crate_universe/private/generate_utils.bzl @@ -433,6 +433,8 @@ def execute_generator( repository_dir, cargo, rustc, + paths_to_track_file, + warnings_output_file, metadata = None): """Execute the `cargo-bazel` binary to produce `BUILD` and `.bzl` files. @@ -446,6 +448,8 @@ def execute_generator( repository_dir (path): The output path for the Bazel module and BUILD files. cargo (path): The path of a Cargo binary. rustc (path): The path of a Rustc binary. + paths_to_track_file (path): Path to file where generator should write which files should trigger re-generating as a JSON list. + warnings_output_file (path): Path to file where generator should write warnings to print. metadata (path, optional): The path to a Cargo metadata json file. If this is set, it indicates to the generator that repinning is required. This file must be adjacent to a `Cargo.toml` and `Cargo.lock` file. @@ -470,8 +474,20 @@ def execute_generator( cargo, "--rustc", rustc, + "--nonhermetic-root-bazel-workspace-dir", + repository_ctx.workspace_root, + "--paths-to-track", + paths_to_track_file, + "--warnings-output-path", + warnings_output_file, ] + if repository_ctx.attr.generator: + args.extend([ + "--generator", + repository_ctx.attr.generator, + ]) + if lockfile_path: args.extend([ "--lockfile", diff --git a/crate_universe/private/local_crate_mirror.bzl b/crate_universe/private/local_crate_mirror.bzl new file mode 100644 index 0000000000..4298e5d8b7 --- /dev/null +++ b/crate_universe/private/local_crate_mirror.bzl @@ -0,0 +1,64 @@ +"""`local_crate_mirror` rule implementation.""" + +load("//crate_universe/private:common_utils.bzl", "execute") +load("//crate_universe/private:generate_utils.bzl", "get_generator") +load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS") +load("//rust/platform:triple.bzl", "get_host_triple") + +def _local_crate_mirror_impl(repository_ctx): + path = repository_ctx.path(repository_ctx.attr.path) + + host_triple = get_host_triple(repository_ctx) + + generator, _generator_sha256 = get_generator(repository_ctx, host_triple.str) + + # TODO: Work out why we can't just symlink here and actually copy. + # illicitonion thinks it may be that symlinks didn't get invalidated properly? + for child in repository_ctx.path(path).readdir(): + repository_ctx.execute(["cp", "-r", child, repository_ctx.path(child.basename)]) + + paths_to_track = execute(repository_ctx, ["find", path, "-type", "f"]).stdout.strip().split("\n") + for path_to_track in paths_to_track: + if path_to_track: + repository_ctx.read(path_to_track) + + execute(repository_ctx, [generator, "render", "--options-json", repository_ctx.attr.options_json, "--output-path", repository_ctx.path("BUILD.bazel")]) + + repository_ctx.file("WORKSPACE.bazel", "") + +local_crate_mirror = repository_rule( + doc = """This is a private implementation detail of crate_universe, and should not be relied on in manually written code. + +This is effectively a `local_repository` rule impementation, but where the BUILD.bazel file is generated using the `cargo-bazel render` command.""", + implementation = _local_crate_mirror_impl, + attrs = { + "generator": attr.string( + doc = ( + "The absolute label of a generator. Eg. `@cargo_bazel_bootstrap//:cargo-bazel`. " + + "This is typically used when bootstrapping" + ), + ), + "generator_sha256s": attr.string_dict( + doc = "Dictionary of `host_triple` -> `sha256` for a `cargo-bazel` binary.", + default = CARGO_BAZEL_SHA256S, + ), + "generator_urls": attr.string_dict( + doc = ( + "URL template from which to download the `cargo-bazel` binary. `{host_triple}` and will be " + + "filled in according to the host platform." + ), + default = CARGO_BAZEL_URLS, + ), + "options_json": attr.string( + doc = "JSON serialized instance of a crate_universe::context::SingleBuildFileRenderContext", + ), + "path": attr.string( + # TODO: Verify what happens if this is not an absolute path. + doc = "Absolute path to the BUILD.bazel file to generate.", + ), + "quiet": attr.bool( + doc = "If stdout and stderr should not be printed to the terminal.", + default = True, + ), + }, +) diff --git a/crate_universe/private/srcs.bzl b/crate_universe/private/srcs.bzl index a96e424466..bea5cc28f9 100644 --- a/crate_universe/private/srcs.bzl +++ b/crate_universe/private/srcs.bzl @@ -11,6 +11,7 @@ CARGO_BAZEL_SRCS = [ Label("//crate_universe:src/cli.rs"), Label("//crate_universe:src/cli/generate.rs"), Label("//crate_universe:src/cli/query.rs"), + Label("//crate_universe:src/cli/render.rs"), Label("//crate_universe:src/cli/splice.rs"), Label("//crate_universe:src/cli/vendor.rs"), Label("//crate_universe:src/config.rs"), diff --git a/crate_universe/src/cli.rs b/crate_universe/src/cli.rs index 9172c97f66..19a1fea833 100644 --- a/crate_universe/src/cli.rs +++ b/crate_universe/src/cli.rs @@ -2,6 +2,7 @@ mod generate; mod query; +mod render; mod splice; mod vendor; @@ -15,12 +16,14 @@ use tracing_subscriber::FmtSubscriber; pub use self::generate::GenerateOptions; pub use self::query::QueryOptions; +pub use self::render::RenderOptions; pub use self::splice::SpliceOptions; pub use self::vendor::VendorOptions; // Entrypoints pub use generate::generate; pub use query::query; +pub use render::render; pub use splice::splice; pub use vendor::vendor; @@ -42,6 +45,9 @@ pub enum Options { /// Vendor BUILD files to the workspace with either repository definitions or `cargo vendor` generated sources. Vendor(VendorOptions), + + /// Render a BUILD file for a single crate. + Render(RenderOptions), } // Convenience wrappers to avoid dependencies in the binary @@ -51,7 +57,7 @@ pub fn parse_args() -> Options { Options::parse() } -const EXPECTED_LOGGER_NAMES: [&str; 4] = ["Generate", "Splice", "Query", "Vendor"]; +const EXPECTED_LOGGER_NAMES: [&str; 5] = ["Generate", "Splice", "Query", "Vendor", "Render"]; /// A wrapper for the tracing-subscriber default [FormatEvent] /// that prepends the name of the active CLI option. diff --git a/crate_universe/src/cli/generate.rs b/crate_universe/src/cli/generate.rs index 2178f276cb..4ec2638bb4 100644 --- a/crate_universe/src/cli/generate.rs +++ b/crate_universe/src/cli/generate.rs @@ -2,18 +2,21 @@ use std::fs; use std::path::{Path, PathBuf}; +use std::sync::Arc; use anyhow::{bail, Context as AnyhowContext, Result}; +use camino::Utf8PathBuf; use cargo_lock::Lockfile; use clap::Parser; use crate::config::Config; use crate::context::Context; use crate::lockfile::{lock_context, write_lockfile}; -use crate::metadata::{load_metadata, Annotations, Cargo}; +use crate::metadata::{load_metadata, Annotations, Cargo, SourceAnnotation}; use crate::rendering::{write_outputs, Renderer}; use crate::splicing::SplicingManifest; use crate::utils::normalize_cargo_file_paths; +use crate::utils::starlark::Label; /// Command line options for the `generate` subcommand #[derive(Parser, Debug)] @@ -63,6 +66,35 @@ pub struct GenerateOptions { /// If true, outputs will be printed instead of written to disk. #[clap(long)] pub dry_run: bool, + + /// The path to the Bazel root workspace (i.e. the directory containing the WORKSPACE.bazel file or similar). + /// BE CAREFUL with this value. We never want to include it in a lockfile hash (to keep lockfiles portable), + /// which means you also should not use it anywhere that _should_ be guarded by a lockfile hash. + /// You basically never want to use this value. + #[clap(long)] + pub nonhermetic_root_bazel_workspace_dir: Utf8PathBuf, + + /// Path to write a list of files which the repository_rule should watch. + /// If any of these paths change, the repository rule should be rerun. + /// These files may be outside of the Bazel-managed workspace. + /// A (possibly empty) JSON sorted array of strings will be unconditionally written to this file. + #[clap(long)] + pub paths_to_track: PathBuf, + + /// The label of this binary, if it was built in bootstrap mode. + /// BE CAREFUL with this value. We never want to include it in a lockfile hash (to keep lockfiles portable), + /// which means you also should not use it anywhere that _should_ be guarded by a lockfile hash. + /// You basically never want to use this value. + #[clap(long)] + pub(crate) generator: Option