diff --git a/.cargo/config.toml b/.cargo/config.toml index 886b8e0a50194..37b7a62c3e144 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,12 +1,6 @@ [build] -# `cargo_internal_build` is set for cargo builds inside Meta. -# The flag enables features not available in opensource version, -# in particular, dependencies on libraries not available in open source. rustflags = [ - # @oss-disable: "--cfg", - # @oss-disable: "cargo_internal_build", - "--cfg", - "tokio_unstable", + "--cfg=tokio_unstable", ] # @oss-disable: [source.crates-io] diff --git a/Cargo.toml b/Cargo.toml index ef70090ad0460..520bc4823afde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,8 +64,6 @@ members = [ "app/buck2_data", "app/buck2_worker_proto", "app/buck2_wrapper_common", - # @oss-disable: "buck2_tpx", - # @oss-disable: "buck2_tpx_cli", "app/buck2_build_api", "app/buck2_build_api_derive", "app/buck2_build_api_tests", @@ -78,7 +76,7 @@ members = [ "dice/dice_examples", "dice/dice_tests", # @oss-disable: "dice/fuzzy_dice", - # @oss-disable: "dice_replay", + "dice/read_dump", # @oss-disable: "host_sharing", "gazebo/display_container", "gazebo/cmp_any", @@ -372,8 +370,6 @@ buck2_test = { path = "app/buck2_test" } buck2_test_api = { path = "app/buck2_test_api" } buck2_test_proto = { path = "app/buck2_test_proto" } buck2_test_runner = { path = "app/buck2_test_runner" } -buck2_tpx = { path = "buck2_tpx" } -buck2_tpx_cli = { path = "buck2_tpx_cli" } buck2_transition = { path = "app/buck2_transition" } buck2_util = { path = "app/buck2_util" } buck2_worker_proto = { path = "app/buck2_worker_proto" } diff --git a/app/buck2/BUCK b/app/buck2/BUCK index 10eef9a900627..ad33d7ea3408d 100644 --- a/app/buck2/BUCK +++ b/app/buck2/BUCK @@ -11,13 +11,17 @@ rust_library( ( "linux", [ + "fbsource//third-party/rust:dirs", "fbsource//third-party/rust:nix", + "//buck2/app/buck2_forkserver:buck2_forkserver", ], ), ( "macos", [ + "fbsource//third-party/rust:dirs", "fbsource//third-party/rust:nix", + "//buck2/app/buck2_forkserver:buck2_forkserver", ], ), ( @@ -32,7 +36,6 @@ rust_library( "fbsource//third-party/rust:async-trait", "fbsource//third-party/rust:backtrace", "fbsource//third-party/rust:clap-3", - "fbsource//third-party/rust:dirs", "fbsource//third-party/rust:futures", "fbsource//third-party/rust:libc", "fbsource//third-party/rust:rand", @@ -54,7 +57,6 @@ rust_library( "//buck2/app/buck2_error:buck2_error", "//buck2/app/buck2_event_observer:buck2_event_observer", "//buck2/app/buck2_events:buck2_events", - "//buck2/app/buck2_forkserver:buck2_forkserver", "//buck2/app/buck2_query:buck2_query", "//buck2/app/buck2_server:buck2_server", "//buck2/app/buck2_server_ctx:buck2_server_ctx", @@ -64,7 +66,7 @@ rust_library( "//buck2/app/buck2_wrapper_common:buck2_wrapper_common", "//buck2/dice/dice:dice", "//buck2/gazebo/dupe:dupe", - "//common/rust/gflags:gflags", + # @oss-disable: "//common/rust/gflags:gflags", "//common/rust/shed/fbinit:fbinit", ], ) @@ -109,8 +111,8 @@ buck_rust_binary( "//buck2/app/buck2_transition:buck2_transition", "//buck2/app/buck2_wrapper_common:buck2_wrapper_common", "//buck2/gazebo/dupe:dupe", - "//common/rust/folly/logging:logging", - "//common/rust/gflags:gflags", + # @oss-disable: "//common/rust/folly/logging:logging", + # @oss-disable: "//common/rust/gflags:gflags", "//common/rust/shed/fbinit:fbinit", ], ) diff --git a/app/buck2/Cargo.toml b/app/buck2/Cargo.toml index 96a1af6412fd6..2c58829dc221b 100644 --- a/app/buck2/Cargo.toml +++ b/app/buck2/Cargo.toml @@ -10,10 +10,8 @@ async-trait = { workspace = true } backtrace = { workspace = true } blake3 = { workspace = true } clap = { workspace = true } -dirs = { workspace = true } futures = { workspace = true } hex = { workspace = true } -# @oss-disable: hostcaps = { path = "../../../common/rust/shed/hostcaps" } libc = { workspace = true } rand = { workspace = true } serde = { workspace = true } @@ -22,7 +20,6 @@ termimad = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } tracing = { workspace = true } -which = "4.2.4" allocative = { workspace = true } dice = { workspace = true } @@ -30,7 +27,6 @@ dupe = { workspace = true } fbinit = { workspace = true } gazebo = { workspace = true } host_sharing = { workspace = true } -# @oss-disable: user = { path = "../../../common/rust/user", optional = true } buck2_action_impl = { workspace = true } buck2_analysis = { workspace = true } @@ -53,7 +49,6 @@ buck2_data = { workspace = true } buck2_error = { workspace = true } buck2_event_observer = { workspace = true } buck2_events = { workspace = true } -buck2_forkserver = { workspace = true } buck2_interpreter_for_build = { workspace = true } buck2_query = { workspace = true } buck2_query_impls = { workspace = true } @@ -71,8 +66,11 @@ buck2_wrapper_common = { workspace = true } jemallocator = { workspace = true } [target.'cfg(unix)'.dependencies] +dirs = { workspace = true } nix = { workspace = true } +buck2_forkserver = { workspace = true } + [target.'cfg(windows)'.dependencies] libc = { workspace = true } winapi = { workspace = true } diff --git a/app/buck2_action_impl_tests/Cargo.toml b/app/buck2_action_impl_tests/Cargo.toml index a2744b441984e..f39dc9531e312 100644 --- a/app/buck2_action_impl_tests/Cargo.toml +++ b/app/buck2_action_impl_tests/Cargo.toml @@ -4,7 +4,7 @@ edition = "2021" name = "buck2_action_impl_tests" version = "0.1.0" -[dependencies] +[dev-dependencies] anyhow = { workspace = true } ctor = { workspace = true } indoc = { workspace = true } diff --git a/app/buck2_action_impl_tests/src/lib.rs b/app/buck2_action_impl_tests/src/lib.rs index 29d179e5daefb..706a036596a02 100644 --- a/app/buck2_action_impl_tests/src/lib.rs +++ b/app/buck2_action_impl_tests/src/lib.rs @@ -7,8 +7,8 @@ * of this source tree. */ -#![feature(error_generic_member_access)] #![cfg(test)] +#![feature(error_generic_member_access)] mod context; diff --git a/app/buck2_action_metadata_proto/Cargo.toml b/app/buck2_action_metadata_proto/Cargo.toml index d7c8c84fde0b0..a008437d8d49f 100644 --- a/app/buck2_action_metadata_proto/Cargo.toml +++ b/app/buck2_action_metadata_proto/Cargo.toml @@ -6,7 +6,6 @@ version = "0.1.0" [dependencies] prost = { workspace = true } -prost-types = { workspace = true } tonic = { workspace = true } [build-dependencies] diff --git a/app/buck2_build_api/Cargo.toml b/app/buck2_build_api/Cargo.toml index 7b2ebbcae9c58..e3b7247705935 100644 --- a/app/buck2_build_api/Cargo.toml +++ b/app/buck2_build_api/Cargo.toml @@ -9,21 +9,18 @@ anyhow = { workspace = true } async-recursion = { workspace = true } async-trait = { workspace = true } blake3 = { workspace = true } -crossbeam-epoch = { workspace = true } dashmap = { workspace = true } derivative = { workspace = true } derive_more = { workspace = true } either = { workspace = true } fnv = { workspace = true } futures = { workspace = true } -glob = { workspace = true } higher-order-closure = { workspace = true } indexmap = { workspace = true } internment = { workspace = true } inventory = { workspace = true } itertools = { workspace = true } once_cell = { workspace = true } -pin-project = { workspace = true } ref-cast = { workspace = true } regex = { workspace = true } serde = { workspace = true } @@ -31,16 +28,13 @@ serde_json = { workspace = true } shlex = { workspace = true } smallvec = { workspace = true } static_assertions = { workspace = true } -structopt = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } allocative = { workspace = true } dice = { workspace = true } display_container = { workspace = true } dupe = { workspace = true } -fbinit = { workspace = true } gazebo = { workspace = true } more_futures = { workspace = true } provider = { workspace = true } diff --git a/app/buck2_build_api/src/actions/execute/action_execution_target.rs b/app/buck2_build_api/src/actions/execute/action_execution_target.rs index 85c32921c5220..15c125837d4ef 100644 --- a/app/buck2_build_api/src/actions/execute/action_execution_target.rs +++ b/app/buck2_build_api/src/actions/execute/action_execution_target.rs @@ -48,6 +48,7 @@ impl<'a> ActionExecutionTarget<'a> { self.action.owner().dupe(), self.action.category(), self.action.identifier(), + self.action.action_key(), ) .unwrap() } diff --git a/app/buck2_build_api/src/actions/mod.rs b/app/buck2_build_api/src/actions/mod.rs index 2f44f206588f1..4211b28edd3d5 100644 --- a/app/buck2_build_api/src/actions/mod.rs +++ b/app/buck2_build_api/src/actions/mod.rs @@ -338,6 +338,11 @@ impl RegisteredAction { self.key.deferred_key().owner() } + /// Gets the action key, uniquely identifying this action in a target. + pub fn action_key(&self) -> String { + self.key.deferred_key().action_key() + } + pub fn key(&self) -> &ActionKey { &self.key } diff --git a/app/buck2_build_info/Cargo.toml b/app/buck2_build_info/Cargo.toml index c5f6c4970e359..5257331938e7d 100644 --- a/app/buck2_build_info/Cargo.toml +++ b/app/buck2_build_info/Cargo.toml @@ -4,6 +4,4 @@ name = "buck2_build_info" version = "0.1.0" [dependencies] -buck2_error = { workspace = true } buck2_util = { workspace = true } -# @oss-disable: build_info = { path = "../../../common/rust/build_info" } diff --git a/app/buck2_build_info/src/lib.rs b/app/buck2_build_info/src/lib.rs index 978420d736d8d..c39699240fa57 100644 --- a/app/buck2_build_info/src/lib.rs +++ b/app/buck2_build_info/src/lib.rs @@ -40,12 +40,12 @@ pub fn win_internal_version() -> Option<&'static str> { /// Get the time at which this binary was built, if available. pub fn time_iso8601() -> Option<&'static str> { - #[cfg(any(fbcode_build, cargo_internal_build))] + #[cfg(fbcode_build)] { Some(build_info::BuildInfo::get_time_iso8601()) } - #[cfg(not(any(fbcode_build, cargo_internal_build)))] + #[cfg(not(fbcode_build))] { None } diff --git a/app/buck2_bxl/Cargo.toml b/app/buck2_bxl/Cargo.toml index 46c62962e71c9..f7db9d8e32936 100644 --- a/app/buck2_bxl/Cargo.toml +++ b/app/buck2_bxl/Cargo.toml @@ -19,14 +19,10 @@ higher-order-closure = { workspace = true } indexmap = { workspace = true } itertools = { workspace = true } num-bigint = { workspace = true } -once_cell = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -structopt = { workspace = true } tokio = { workspace = true } -fbinit = { workspace = true } - allocative = { workspace = true } buck2_events = { workspace = true } cmp_any = { workspace = true } diff --git a/app/buck2_cfg_constructor/Cargo.toml b/app/buck2_cfg_constructor/Cargo.toml index fb0740008586c..1d2ed431c1329 100644 --- a/app/buck2_cfg_constructor/Cargo.toml +++ b/app/buck2_cfg_constructor/Cargo.toml @@ -16,7 +16,6 @@ starlark = { workspace = true } buck2_build_api = { workspace = true } buck2_common = { workspace = true } -buck2_configured = { workspace = true } buck2_core = { workspace = true } buck2_error = { workspace = true } buck2_events = { workspace = true } diff --git a/app/buck2_client/BUCK b/app/buck2_client/BUCK index 6aca599dd3a84..5967442cd2b17 100644 --- a/app/buck2_client/BUCK +++ b/app/buck2_client/BUCK @@ -54,7 +54,7 @@ rust_library( "//buck2/app/buck2_error:buck2_error", "//buck2/app/buck2_event_observer:buck2_event_observer", "//buck2/app/buck2_events:buck2_events", - "//buck2/app/buck2_execute:buck2_execute", + # @oss-disable: "//buck2/app/buck2_execute:buck2_execute", "//buck2/app/buck2_offline_archive:buck2_offline_archive", "//buck2/app/buck2_query_parser:buck2_query_parser", "//buck2/app/buck2_subscription_proto:buck2_subscription_proto", diff --git a/app/buck2_client/Cargo.toml b/app/buck2_client/Cargo.toml index bbe477475a05f..acbe40fdad02f 100644 --- a/app/buck2_client/Cargo.toml +++ b/app/buck2_client/Cargo.toml @@ -15,7 +15,6 @@ chrono = { workspace = true } clap = { workspace = true } csv = { workspace = true } derive_more = { workspace = true } -dice = { workspace = true } dupe = { workspace = true } futures = { workspace = true } gazebo = { workspace = true } @@ -54,7 +53,6 @@ buck2_data = { workspace = true } buck2_error = { workspace = true } buck2_event_observer = { workspace = true } buck2_events = { workspace = true } -buck2_execute = { workspace = true } buck2_offline_archive = { workspace = true } buck2_query_parser = { workspace = true } buck2_subscription_proto = { workspace = true } diff --git a/app/buck2_client/src/commands/clean.rs b/app/buck2_client/src/commands/clean.rs index cf63dd9b962f2..4aa9b68128792 100644 --- a/app/buck2_client/src/commands/clean.rs +++ b/app/buck2_client/src/commands/clean.rs @@ -220,7 +220,7 @@ fn clean_buck_out(path: &AbsNormPathBuf) -> anyhow::Result<()> { Ok(()) } -#[cfg(any(fbcode_build, cargo_internal_build))] +#[cfg(fbcode_build)] async fn try_clean_eden_buck_out( buck_out: &AbsNormPathBuf, dryrun: bool, @@ -270,7 +270,7 @@ async fn try_clean_eden_buck_out( Ok(Some(vec![buck_out.display().to_string()])) } -#[cfg(not(any(fbcode_build, cargo_internal_build)))] +#[cfg(not(fbcode_build))] async fn try_clean_eden_buck_out( _buck_out: &AbsNormPathBuf, _dryrun: bool, diff --git a/app/buck2_client_ctx/Cargo.toml b/app/buck2_client_ctx/Cargo.toml index 8494cf92da8ed..fbd961feeaee4 100644 --- a/app/buck2_client_ctx/Cargo.toml +++ b/app/buck2_client_ctx/Cargo.toml @@ -14,7 +14,6 @@ bytes = { workspace = true } chrono = { workspace = true } clap = { workspace = true } crossterm = { workspace = true } -csv = { workspace = true } derivative = { workspace = true } derive_more = { workspace = true } dupe = { workspace = true } @@ -27,7 +26,6 @@ httparse = { workspace = true } hyper = { workspace = true } itertools = { workspace = true } libc = { workspace = true } -linked-hash-map = { workspace = true } memmap2 = { workspace = true } object = { workspace = true } once_cell = { workspace = true } @@ -48,8 +46,6 @@ tokio-util = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } -# @oss-disable: detect_eden = { path = "../../../common/rust/shed/detect_eden" } -# @oss-disable: hostcaps = { path = "../../../common/rust/shed/hostcaps" } superconsole = { version = "0.2.0", path = "../../superconsole" } # Please do not add dependency on `buck2_build_api`. @@ -62,7 +58,6 @@ buck2_error = { workspace = true } buck2_event_observer = { workspace = true } buck2_events = { workspace = true } buck2_http = { workspace = true } -buck2_test_api = { workspace = true } buck2_util = { workspace = true } buck2_wrapper_common = { workspace = true } diff --git a/app/buck2_client_ctx/src/manifold.rs b/app/buck2_client_ctx/src/manifold.rs index ae95084be4f72..35b0cb6579f9c 100644 --- a/app/buck2_client_ctx/src/manifold.rs +++ b/app/buck2_client_ctx/src/manifold.rs @@ -129,7 +129,7 @@ impl Bucket { /// Return the place to upload logs, or None to not upload logs at all fn log_upload_url(use_vpnless: bool) -> Option<&'static str> { - #[cfg(any(fbcode_build, cargo_internal_build))] + #[cfg(fbcode_build)] if hostcaps::is_prod() { Some("https://manifold.facebook.net") } else if use_vpnless { @@ -137,7 +137,7 @@ fn log_upload_url(use_vpnless: bool) -> Option<&'static str> { } else { Some("https://manifold.c2p.facebook.net") } - #[cfg(not(any(fbcode_build, cargo_internal_build)))] + #[cfg(not(fbcode_build))] { #[cfg(fbcode_build)] compile_error!("this code is not meant to be compiled in fbcode"); diff --git a/app/buck2_client_ctx/src/subscribers/recorder.rs b/app/buck2_client_ctx/src/subscribers/recorder.rs index d1f428407a8b2..7d09d0b5dd910 100644 --- a/app/buck2_client_ctx/src/subscribers/recorder.rs +++ b/app/buck2_client_ctx/src/subscribers/recorder.rs @@ -1123,7 +1123,7 @@ pub fn try_get_invocation_recorder<'a>( .map(|path| path.resolve(&ctx.working_dir)); let filesystem; - #[cfg(any(fbcode_build, cargo_internal_build))] + #[cfg(fbcode_build)] { let root = std::path::Path::to_owned(ctx.paths()?.project_root().root().to_buf().as_ref()); if detect_eden::is_eden(root).unwrap_or(false) { @@ -1132,7 +1132,7 @@ pub fn try_get_invocation_recorder<'a>( filesystem = "default".to_owned(); } } - #[cfg(not(any(fbcode_build, cargo_internal_build)))] + #[cfg(not(fbcode_build))] { filesystem = "default".to_owned(); } diff --git a/app/buck2_common/BUCK b/app/buck2_common/BUCK index de2865f4970d3..69d3f74efffe2 100644 --- a/app/buck2_common/BUCK +++ b/app/buck2_common/BUCK @@ -77,6 +77,6 @@ rust_library( "//buck2/gazebo/gazebo:gazebo", "//buck2/shed/more_futures:more_futures", "//buck2/starlark-rust/starlark_map:starlark_map", - "//common/rust/folly/memory:memory", + # @oss-disable: "//common/rust/folly/memory:memory", ], ) diff --git a/app/buck2_common/Cargo.toml b/app/buck2_common/Cargo.toml index 6e584f28d9d8f..1d143fa141d39 100644 --- a/app/buck2_common/Cargo.toml +++ b/app/buck2_common/Cargo.toml @@ -33,7 +33,6 @@ sha1 = { workspace = true } sha2 = { workspace = true } smallvec = { workspace = true } tokio = { workspace = true } -tokio-util = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } diff --git a/app/buck2_core/Cargo.toml b/app/buck2_core/Cargo.toml index 39482290a6705..5c1aece1f9b6a 100644 --- a/app/buck2_core/Cargo.toml +++ b/app/buck2_core/Cargo.toml @@ -6,8 +6,6 @@ version = "0.1.0" [dependencies] anyhow = { workspace = true } arc-swap = { workspace = true } -assert_matches = { workspace = true } -async-trait = { workspace = true } blake3 = { workspace = true } compact_str = { workspace = true } dashmap = { workspace = true } @@ -21,7 +19,6 @@ futures = { workspace = true } hostname = { workspace = true } indent_write = { workspace = true } itertools = { workspace = true } -libc = { workspace = true } memchr = { workspace = true } once_cell = { workspace = true } os_str_bytes = { workspace = true } @@ -59,6 +56,7 @@ nix = { workspace = true } common-path = { workspace = true } [dev-dependencies] +assert_matches = { workspace = true } maplit = { workspace = true } serde_json = { workspace = true } test-case = { workspace = true } diff --git a/app/buck2_core/src/fs/buck_out_path.rs b/app/buck2_core/src/fs/buck_out_path.rs index 4e8f8f3b002c4..51c0906d760a1 100644 --- a/app/buck2_core/src/fs/buck_out_path.rs +++ b/app/buck2_core/src/fs/buck_out_path.rs @@ -93,6 +93,7 @@ impl BuckOutScratchPath { owner: BaseDeferredKey, category: &Category, identifier: Option<&str>, + action_key: String, ) -> anyhow::Result { const MAKE_SENSIBLE_PREFIX: &str = "_buck_"; // Windows has MAX_PATH limit (260 chars). @@ -132,6 +133,7 @@ impl BuckOutScratchPath { // FIXME: Should this be a crypto hasher? let mut hasher = DefaultHasher::new(); v.hash(&mut hasher); + action_key.hash(&mut hasher); let output_hash = format!("{}{:x}", MAKE_SENSIBLE_PREFIX, hasher.finish()); path.join_normalized(ForwardRelativePath::new(&output_hash)?)? } @@ -399,6 +401,7 @@ mod tests { BaseDeferredKey::TargetLabel(cfg_target.dupe()), &category, None, + "1_2".to_owned(), ) .unwrap(); @@ -407,6 +410,7 @@ mod tests { BaseDeferredKey::TargetLabel(cfg_target.dupe()), &category, Some(s), + "3_4".to_owned(), ) .unwrap() .path diff --git a/app/buck2_data/Cargo.toml b/app/buck2_data/Cargo.toml index d95ae719044ce..e8208bde000b9 100644 --- a/app/buck2_data/Cargo.toml +++ b/app/buck2_data/Cargo.toml @@ -17,4 +17,3 @@ gazebo = { workspace = true } [build-dependencies] buck2_protoc_dev = { workspace = true } -tonic-build = { workspace = true } diff --git a/app/buck2_downward_api_proto/Cargo.toml b/app/buck2_downward_api_proto/Cargo.toml index 8b9694b9ad149..170d708e8e477 100644 --- a/app/buck2_downward_api_proto/Cargo.toml +++ b/app/buck2_downward_api_proto/Cargo.toml @@ -7,7 +7,6 @@ version = "0.1.0" [dependencies] anyhow = { workspace = true } prost = { workspace = true } -prost-types = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } diff --git a/app/buck2_eden/BUCK b/app/buck2_eden/BUCK index a17c79b906892..742094f763735 100644 --- a/app/buck2_eden/BUCK +++ b/app/buck2_eden/BUCK @@ -7,29 +7,29 @@ rust_library( name = "buck2_eden", srcs = glob(["src/**/*.rs"]), named_deps = { - "edenfs": "//eden/fs/service:thrift-rust", - "fb303_core": "//fb303/thrift:fb303_core-rust", + # @oss-disable: "edenfs": "//eden/fs/service:thrift-rust", + # @oss-disable: "fb303_core": "//fb303/thrift:fb303_core-rust", }, test_deps = [ ], deps = [ - "fbsource//third-party/rust:anyhow", - "fbsource//third-party/rust:async-trait", - "fbsource//third-party/rust:compact_str", - "fbsource//third-party/rust:futures", - "fbsource//third-party/rust:libc", - "fbsource//third-party/rust:parking_lot", - "fbsource//third-party/rust:serde", - "fbsource//third-party/rust:tokio", - "fbsource//third-party/rust:toml", - "fbsource//third-party/rust:tracing", - "//buck2/allocative/allocative:allocative", - "//buck2/app/buck2_common:buck2_common", - "//buck2/app/buck2_core:buck2_core", - "//buck2/app/buck2_error:buck2_error", - "//buck2/gazebo/dupe:dupe", - "//common/rust/shed/fbinit:fbinit", - "//common/rust/shed/sorted_vector_map:sorted_vector_map", - "//common/rust/thrift/bareclient:thriftclient", + # @oss-disable: "fbsource//third-party/rust:anyhow", + # @oss-disable: "fbsource//third-party/rust:async-trait", + # @oss-disable: "fbsource//third-party/rust:compact_str", + # @oss-disable: "fbsource//third-party/rust:futures", + # @oss-disable: "fbsource//third-party/rust:libc", + # @oss-disable: "fbsource//third-party/rust:parking_lot", + # @oss-disable: "fbsource//third-party/rust:serde", + # @oss-disable: "fbsource//third-party/rust:tokio", + # @oss-disable: "fbsource//third-party/rust:toml", + # @oss-disable: "fbsource//third-party/rust:tracing", + # @oss-disable: "//buck2/allocative/allocative:allocative", + # @oss-disable: "//buck2/app/buck2_common:buck2_common", + # @oss-disable: "//buck2/app/buck2_core:buck2_core", + # @oss-disable: "//buck2/app/buck2_error:buck2_error", + # @oss-disable: "//buck2/gazebo/dupe:dupe", + # @oss-disable: "//common/rust/shed/fbinit:fbinit", + # @oss-disable: "//common/rust/shed/sorted_vector_map:sorted_vector_map", + # @oss-disable: "//common/rust/thrift/bareclient:thriftclient", ], ) diff --git a/app/buck2_eden/Cargo.toml b/app/buck2_eden/Cargo.toml index a5ec843d5e811..1a7a0fd3e5d64 100644 --- a/app/buck2_eden/Cargo.toml +++ b/app/buck2_eden/Cargo.toml @@ -5,26 +5,3 @@ edition = "2021" license = "MIT OR Apache-2.0" name = "buck2_eden" version = "0.1.0" - -[dependencies] -allocative = { workspace = true } -anyhow = { workspace = true } -async-trait = { workspace = true } -buck2_common = { workspace = true } -buck2_core = { workspace = true } -buck2_error = { workspace = true } -compact_str = { workspace = true } -dupe = { workspace = true } -# @oss-disable: edenfs = { package = "thrift", path = "../../../eden/fs/service" } -# @oss-disable: fb303_core = { package = "fb303_core", path = "../../../fb303/thrift" } -fbinit = { workspace = true } -futures = { workspace = true } -libc = { workspace = true } -parking_lot = { workspace = true } -serde = { workspace = true } -sorted_vector_map = { workspace = true } -tokio = { workspace = true } -toml = { workspace = true } -tracing = { workspace = true } - -[dev-dependencies] diff --git a/app/buck2_eden/src/lib.rs b/app/buck2_eden/src/lib.rs index bd53d882760d3..a1c70871a7508 100644 --- a/app/buck2_eden/src/lib.rs +++ b/app/buck2_eden/src/lib.rs @@ -7,9 +7,9 @@ * of this source tree. */ -#![feature(error_generic_member_access)] // This can't be built in our OSS implementation. -#![cfg(any(fbcode_build, cargo_internal_build))] +#![cfg(fbcode_build)] +#![feature(error_generic_member_access)] pub mod connection; pub mod io_provider; diff --git a/app/buck2_event_observer/Cargo.toml b/app/buck2_event_observer/Cargo.toml index 7ae3363252193..c6da41777e3be 100644 --- a/app/buck2_event_observer/Cargo.toml +++ b/app/buck2_event_observer/Cargo.toml @@ -16,12 +16,10 @@ buck2_test_api = { workspace = true } buck2_util = { workspace = true } buck2_wrapper_common = { workspace = true } clap = { workspace = true } -csv = { workspace = true } derivative = { workspace = true } derive_more = { workspace = true } dupe = { workspace = true } gazebo = { workspace = true } -itertools = { workspace = true } linked-hash-map = { workspace = true } shlex = { workspace = true } superconsole = { version = "0.2.0", path = "../../superconsole" } diff --git a/app/buck2_events/BUCK b/app/buck2_events/BUCK index 6d2a7f747bb05..f2b20c6f31e49 100644 --- a/app/buck2_events/BUCK +++ b/app/buck2_events/BUCK @@ -18,9 +18,9 @@ rust_library( "fbsource//third-party/rust:is_proc_translated", "fbsource//third-party/rust:once_cell", "fbsource//third-party/rust:pin-project", - "fbsource//third-party/rust:prost", + # @oss-disable: "fbsource//third-party/rust:prost", "fbsource//third-party/rust:serde", - "fbsource//third-party/rust:serde_json", + # @oss-disable: "fbsource//third-party/rust:serde_json", "fbsource//third-party/rust:smallvec", "fbsource//third-party/rust:sys-info", "fbsource//third-party/rust:tokio", @@ -31,7 +31,7 @@ rust_library( "//buck2/app/buck2_core:buck2_core", "//buck2/app/buck2_data:buck2_data", "//buck2/app/buck2_error:buck2_error", - "//buck2/app/buck2_util:buck2_util", + # @oss-disable: "//buck2/app/buck2_util:buck2_util", "//buck2/app/buck2_wrapper_common:buck2_wrapper_common", "//buck2/facebook/scribe_client:scribe_client", "//buck2/gazebo/dupe:dupe", diff --git a/app/buck2_events/Cargo.toml b/app/buck2_events/Cargo.toml index 45d15bcbc49d9..2bfecfa9d3f77 100644 --- a/app/buck2_events/Cargo.toml +++ b/app/buck2_events/Cargo.toml @@ -5,36 +5,27 @@ version = "0.1.0" [dependencies] anyhow = { workspace = true } -async-trait = { workspace = true } -base64 = { workspace = true } -byteorder = { workspace = true } crossbeam-channel = { workspace = true } -crossbeam-epoch = { workspace = true } derive_more = { workspace = true } futures = { workspace = true } hostname = { workspace = true } is_proc_translated = { workspace = true } once_cell = { workspace = true } pin-project = { workspace = true } -prost = { workspace = true } serde = { workspace = true } -serde_json = { workspace = true } smallvec = { workspace = true } sys-info = { workspace = true } tokio = { workspace = true } -tracing = { workspace = true } uuid = { workspace = true } allocative = { workspace = true } dupe = { workspace = true } fbinit = { workspace = true } gazebo = { workspace = true } -# @oss-disable: user = { path = "../../../common/rust/user" } buck2_build_info = { workspace = true } buck2_cli_proto = { workspace = true } buck2_core = { workspace = true } buck2_data = { workspace = true } buck2_error = { workspace = true } -buck2_util = { workspace = true } buck2_wrapper_common = { workspace = true } diff --git a/app/buck2_events/src/metadata.rs b/app/buck2_events/src/metadata.rs index cc039b5c16c12..2573f9f347a24 100644 --- a/app/buck2_events/src/metadata.rs +++ b/app/buck2_events/src/metadata.rs @@ -79,11 +79,11 @@ pub struct SystemInfo { pub fn system_info() -> SystemInfo { let hostname = hostname(); let username; - #[cfg(any(fbcode_build, cargo_internal_build))] + #[cfg(fbcode_build)] { username = user::current_username().ok(); } - #[cfg(not(any(fbcode_build, cargo_internal_build)))] + #[cfg(not(fbcode_build))] { username = None; } diff --git a/app/buck2_execute/BUCK b/app/buck2_execute/BUCK index 0e4a78941c643..e9bd251c4f07d 100644 --- a/app/buck2_execute/BUCK +++ b/app/buck2_execute/BUCK @@ -9,7 +9,7 @@ rust_library( ["src/**/*.rs"], ), named_deps = { - "edenfs": "//eden/fs/service:thrift-rust", + # @oss-disable: "edenfs": "//eden/fs/service:thrift-rust", }, test_deps = [ "fbsource//third-party/rust:assert_matches", diff --git a/app/buck2_execute/Cargo.toml b/app/buck2_execute/Cargo.toml index a051265ee49f2..3b67f3708c83c 100644 --- a/app/buck2_execute/Cargo.toml +++ b/app/buck2_execute/Cargo.toml @@ -24,19 +24,15 @@ num_cpus = { workspace = true } once_cell = { workspace = true } prost = { workspace = true } ref-cast = { workspace = true } -serde = { workspace = true } sha1 = { workspace = true } sha2 = { workspace = true } -slog = { workspace = true } smallvec = { workspace = true } tokio = { workspace = true } -toml = { workspace = true } tracing = { workspace = true } allocative = { workspace = true } dice = { workspace = true } dupe = { workspace = true } -# @oss-disable: edenfs = { package = "thrift", path = "../../../eden/fs/service" } fbinit = { workspace = true } gazebo = { workspace = true } host_sharing = { workspace = true } @@ -50,13 +46,11 @@ buck2_cli_proto = { workspace = true } buck2_common = { workspace = true } buck2_core = { workspace = true } buck2_data = { workspace = true } -buck2_eden = { workspace = true } buck2_error = { workspace = true } buck2_events = { workspace = true } buck2_http = { workspace = true } buck2_miniperf_proto = { workspace = true } buck2_re_configuration = { workspace = true } -buck2_util = { workspace = true } buck2_wrapper_common = { workspace = true } [dev-dependencies] diff --git a/app/buck2_execute/src/materialize/materializer.rs b/app/buck2_execute/src/materialize/materializer.rs index 9b9d8406d3608..d5358920c6546 100644 --- a/app/buck2_execute/src/materialize/materializer.rs +++ b/app/buck2_execute/src/materialize/materializer.rs @@ -34,17 +34,17 @@ use crate::directory::ActionDirectoryMember; use crate::directory::ActionImmutableDirectory; use crate::directory::ActionSharedDirectory; use crate::execute::action_digest::TrackedActionDigest; -#[cfg(any(fbcode_build, cargo_internal_build))] +#[cfg(fbcode_build)] use crate::materialize::eden_api::EdenBuckOut; use crate::materialize::http::Checksum; // Add a stub EdenBuckOut for when we don't have Eden output enabled -#[cfg(not(any(fbcode_build, cargo_internal_build)))] +#[cfg(not(fbcode_build))] pub struct EdenBuckOut { not_implemented: !, } -#[cfg(not(any(fbcode_build, cargo_internal_build)))] +#[cfg(not(fbcode_build))] impl EdenBuckOut { pub async fn remove_paths_recursive( &self, diff --git a/app/buck2_execute/src/materialize/mod.rs b/app/buck2_execute/src/materialize/mod.rs index c98726964df82..feeb6c706d64a 100644 --- a/app/buck2_execute/src/materialize/mod.rs +++ b/app/buck2_execute/src/materialize/mod.rs @@ -7,7 +7,7 @@ * of this source tree. */ -#[cfg(any(fbcode_build, cargo_internal_build))] +#[cfg(fbcode_build)] pub mod eden_api; pub mod http; diff --git a/app/buck2_execute/src/re/client.rs b/app/buck2_execute/src/re/client.rs index c26acf2a8068f..e9451e566d92e 100644 --- a/app/buck2_execute/src/re/client.rs +++ b/app/buck2_execute/src/re/client.rs @@ -645,6 +645,7 @@ impl RemoteExecutionClientImpl { .unwrap_or("remote_execution/features/client_buck2") .to_owned(); + re_client_config.disable_fallocate = static_metadata.disable_fallocate; // TODO(ndmitchell): For now, we just drop RE log messages, but ideally we'd put them in our log stream. let logger = slog::Logger::root(slog::Discard, slog::o!()); REClientBuilder::new(fb) diff --git a/app/buck2_execute_impl/BUCK b/app/buck2_execute_impl/BUCK index 8b90781a0a1fd..3519e609f5388 100644 --- a/app/buck2_execute_impl/BUCK +++ b/app/buck2_execute_impl/BUCK @@ -8,6 +8,20 @@ rust_library( srcs = glob( ["src/**/*.rs"], ), + os_deps = [ + ( + "linux", + [ + "//buck2/app/buck2_forkserver_proto:buck2_forkserver_proto", + ], + ), + ( + "macos", + [ + "//buck2/app/buck2_forkserver_proto:buck2_forkserver_proto", + ], + ), + ], test_deps = [ "fbsource//third-party/rust:assert_matches", ], @@ -42,7 +56,6 @@ rust_library( "//buck2/app/buck2_events:buck2_events", "//buck2/app/buck2_execute:buck2_execute", "//buck2/app/buck2_forkserver:buck2_forkserver", - "//buck2/app/buck2_forkserver_proto:buck2_forkserver_proto", "//buck2/app/buck2_http:buck2_http", "//buck2/app/buck2_util:buck2_util", "//buck2/app/buck2_worker_proto:buck2_worker_proto", diff --git a/app/buck2_execute_impl/Cargo.toml b/app/buck2_execute_impl/Cargo.toml index cb1460381746c..92114d3fb7468 100644 --- a/app/buck2_execute_impl/Cargo.toml +++ b/app/buck2_execute_impl/Cargo.toml @@ -12,11 +12,9 @@ chrono = { workspace = true } derivative = { workspace = true } derive_more = { workspace = true } dupe = { workspace = true } -faccess = { workspace = true } futures = { workspace = true } gazebo = { workspace = true } host_sharing = { workspace = true } -hostname = { workspace = true } indexmap = { workspace = true } itertools = { workspace = true } more_futures = { workspace = true } @@ -43,11 +41,13 @@ buck2_error = { workspace = true } buck2_events = { workspace = true } buck2_execute = { workspace = true } buck2_forkserver = { workspace = true } -buck2_forkserver_proto = { workspace = true } buck2_http = { workspace = true } buck2_util = { workspace = true } buck2_worker_proto = { workspace = true } buck2_wrapper_common = { workspace = true } +[target.'cfg(unix)'.dependencies] +buck2_forkserver_proto = { workspace = true } + [dev-dependencies] assert_matches = { workspace = true } diff --git a/app/buck2_execute_impl/src/materializers/mod.rs b/app/buck2_execute_impl/src/materializers/mod.rs index 956760b0a0375..6c902f4e5fed8 100644 --- a/app/buck2_execute_impl/src/materializers/mod.rs +++ b/app/buck2_execute_impl/src/materializers/mod.rs @@ -7,7 +7,7 @@ * of this source tree. */ -#[cfg(any(fbcode_build, cargo_internal_build))] +#[cfg(fbcode_build)] pub mod eden; pub mod deferred; diff --git a/app/buck2_file_watcher/BUCK b/app/buck2_file_watcher/BUCK index 731164abc1751..6cef3ba208d10 100644 --- a/app/buck2_file_watcher/BUCK +++ b/app/buck2_file_watcher/BUCK @@ -6,11 +6,25 @@ oncall("build_infra") rust_library( name = "buck2_file_watcher", srcs = glob(["src/**/*.rs"]), - test_deps = [ - "fbsource//third-party/rust:assert_matches", - "fbsource//third-party/rust:tempfile", - "//buck2/app/buck2_util:buck2_util", - "//buck2/gazebo/gazebo:gazebo", + test_os_deps = [ + ( + "linux", + [ + "fbsource//third-party/rust:assert_matches", + "fbsource//third-party/rust:tempfile", + "//buck2/app/buck2_util:buck2_util", + "//buck2/gazebo/gazebo:gazebo", + ], + ), + ( + "macos", + [ + "fbsource//third-party/rust:assert_matches", + "fbsource//third-party/rust:tempfile", + "//buck2/app/buck2_util:buck2_util", + "//buck2/gazebo/gazebo:gazebo", + ], + ), ], deps = [ "fbsource//third-party/rust:anyhow", diff --git a/app/buck2_file_watcher/Cargo.toml b/app/buck2_file_watcher/Cargo.toml index 7aec92a6af7f6..700786af27741 100644 --- a/app/buck2_file_watcher/Cargo.toml +++ b/app/buck2_file_watcher/Cargo.toml @@ -27,7 +27,7 @@ buck2_error = { workspace = true } buck2_events = { workspace = true } buck2_util = { workspace = true } -[dev-dependencies] +[target.'cfg(unix)'.dev-dependencies] assert_matches = { workspace = true } buck2_util = { workspace = true } gazebo = { workspace = true } diff --git a/app/buck2_forkserver/BUCK b/app/buck2_forkserver/BUCK index 34b002571ba25..a8979ed1d936d 100644 --- a/app/buck2_forkserver/BUCK +++ b/app/buck2_forkserver/BUCK @@ -19,12 +19,18 @@ rust_library( "linux", [ "fbsource//third-party/rust:nix", + "fbsource//third-party/rust:rand", + "//buck2/app/buck2_grpc:buck2_grpc", + "//buck2/app/buck2_util:buck2_util", ], ), ( "macos", [ "fbsource//third-party/rust:nix", + "fbsource//third-party/rust:rand", + "//buck2/app/buck2_grpc:buck2_grpc", + "//buck2/app/buck2_util:buck2_util", ], ), ( @@ -41,6 +47,14 @@ rust_library( "fbsource//third-party/rust:tempfile", "fbsource//third-party/rust:tokio-stream", ], + test_os_deps = [ + ( + "windows", + [ + "//buck2/app/buck2_util:buck2_util", + ], + ), + ], deps = [ "fbsource//third-party/rust:anyhow", "fbsource//third-party/rust:arc-swap", @@ -49,9 +63,7 @@ rust_library( "fbsource//third-party/rust:bytes", "fbsource//third-party/rust:futures", "fbsource//third-party/rust:libc", - "fbsource//third-party/rust:nix", "fbsource//third-party/rust:pin-project", - "fbsource//third-party/rust:rand", "fbsource//third-party/rust:take_mut", "fbsource//third-party/rust:tokio", "fbsource//third-party/rust:tokio-util", @@ -63,9 +75,7 @@ rust_library( "//buck2/app/buck2_data:buck2_data", "//buck2/app/buck2_error:buck2_error", "//buck2/app/buck2_forkserver_proto:buck2_forkserver_proto", - "//buck2/app/buck2_grpc:buck2_grpc", "//buck2/app/buck2_miniperf_proto:buck2_miniperf_proto", - "//buck2/app/buck2_util:buck2_util", "//buck2/gazebo/dupe:dupe", ], ) diff --git a/app/buck2_forkserver/Cargo.toml b/app/buck2_forkserver/Cargo.toml index 2401e0dfcb9c3..1ffe645c37bf8 100644 --- a/app/buck2_forkserver/Cargo.toml +++ b/app/buck2_forkserver/Cargo.toml @@ -12,7 +12,6 @@ bytes = { workspace = true } futures = { workspace = true } libc = { workspace = true } pin-project = { workspace = true } -rand = { workspace = true } take_mut = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } @@ -21,27 +20,32 @@ tracing = { workspace = true } allocative = { workspace = true } dupe = { workspace = true } -gazebo = { workspace = true } buck2_common = { workspace = true } buck2_core = { workspace = true } buck2_data = { workspace = true } buck2_error = { workspace = true } buck2_forkserver_proto = { workspace = true } -buck2_grpc = { workspace = true } buck2_miniperf_proto = { workspace = true } -buck2_util = { workspace = true } -buck2_wrapper_common = { workspace = true } [target.'cfg(unix)'.dependencies] nix = { workspace = true } +rand = { workspace = true } + +buck2_grpc = { workspace = true } +buck2_util = { workspace = true } [target.'cfg(windows)'.dependencies] winapi = { workspace = true } +buck2_wrapper_common = { workspace = true } + [dev-dependencies] assert_matches = { workspace = true } bytes = { workspace = true } sysinfo = { workspace = true } tempfile = { workspace = true } tokio-stream = { workspace = true } + +[target.'cfg(windows)'.dev-dependencies] +buck2_util = { workspace = true } diff --git a/app/buck2_http/BUCK b/app/buck2_http/BUCK index 34bf2996d7b5d..b3b76e63d9532 100644 --- a/app/buck2_http/BUCK +++ b/app/buck2_http/BUCK @@ -22,7 +22,20 @@ rust_library( ], test_deps = [ "fbsource//third-party/rust:httptest", - "fbsource//third-party/rust:tempfile", + ], + test_os_deps = [ + ( + "linux", + [ + "fbsource//third-party/rust:tempfile", + ], + ), + ( + "macos", + [ + "fbsource//third-party/rust:tempfile", + ], + ), ], deps = [ "fbsource//third-party/rust:anyhow", diff --git a/app/buck2_http/Cargo.toml b/app/buck2_http/Cargo.toml index cff868e07eaa5..a6ef535945790 100644 --- a/app/buck2_http/Cargo.toml +++ b/app/buck2_http/Cargo.toml @@ -34,4 +34,6 @@ hyper-unix-connector = { workspace = true } [dev-dependencies] httptest = { workspace = true } + +[target.'cfg(unix)'.dev-dependencies] tempfile = { workspace = true } diff --git a/app/buck2_http/src/client/mod.rs b/app/buck2_http/src/client/mod.rs index 29af434557d5a..db88db8b07269 100644 --- a/app/buck2_http/src/client/mod.rs +++ b/app/buck2_http/src/client/mod.rs @@ -757,7 +757,7 @@ mod tests { } // TODO(skarlage, T160529958): Debug why these tests fail on CircleCI -#[cfg(all(test, any(fbcode_build, cargo_internal_build)))] +#[cfg(all(test, fbcode_build))] mod proxy_tests { use std::convert::Infallible; use std::net::TcpListener; diff --git a/app/buck2_install_proto/Cargo.toml b/app/buck2_install_proto/Cargo.toml index 674e527513886..6b68cc9c7afb5 100644 --- a/app/buck2_install_proto/Cargo.toml +++ b/app/buck2_install_proto/Cargo.toml @@ -6,7 +6,6 @@ version = "0.1.0" [dependencies] prost = { workspace = true } -prost-types = { workspace = true } tonic = { workspace = true } [build-dependencies] diff --git a/app/buck2_offline_archive/BUCK b/app/buck2_offline_archive/BUCK index d6f2adc3ca091..5ac86ce7673aa 100644 --- a/app/buck2_offline_archive/BUCK +++ b/app/buck2_offline_archive/BUCK @@ -5,8 +5,19 @@ oncall("build_infra") rust_library( name = "buck2_offline_archive", srcs = glob(["src/**/*.rs"]), - test_deps = [ - "fbsource//third-party/rust:tempfile", + test_os_deps = [ + ( + "linux", + [ + "fbsource//third-party/rust:tempfile", + ], + ), + ( + "macos", + [ + "fbsource//third-party/rust:tempfile", + ], + ), ], deps = [ "fbsource//third-party/rust:anyhow", diff --git a/app/buck2_offline_archive/Cargo.toml b/app/buck2_offline_archive/Cargo.toml index 29c89078c98a4..198112bab99ac 100644 --- a/app/buck2_offline_archive/Cargo.toml +++ b/app/buck2_offline_archive/Cargo.toml @@ -10,5 +10,5 @@ serde = { workspace = true } buck2_core = { workspace = true } -[dev-dependencies] +[target.'cfg(unix)'.dev-dependencies] tempfile = { workspace = true } diff --git a/app/buck2_query_parser/Cargo.toml b/app/buck2_query_parser/Cargo.toml index c171dd83c5188..ba1e4780c27e5 100644 --- a/app/buck2_query_parser/Cargo.toml +++ b/app/buck2_query_parser/Cargo.toml @@ -9,7 +9,6 @@ anyhow = { workspace = true } derive_more = { workspace = true } enum-map = { workspace = true } nom = { workspace = true } -structopt = { workspace = true } dupe = { workspace = true } gazebo = { workspace = true } diff --git a/app/buck2_re_configuration/src/lib.rs b/app/buck2_re_configuration/src/lib.rs index 31d90147c70d8..0c8600a8fa5ed 100644 --- a/app/buck2_re_configuration/src/lib.rs +++ b/app/buck2_re_configuration/src/lib.rs @@ -64,6 +64,7 @@ mod fbcode { // ttl management pub minimal_blob_ttl_seconds: Option, + pub disable_fallocate: bool, } impl RemoteExecutionStaticMetadataImpl for RemoteExecutionStaticMetadata { @@ -137,6 +138,9 @@ mod fbcode { )?, minimal_blob_ttl_seconds: legacy_config .parse(BUCK2_RE_CLIENT_CFG_SECTION, "minimal_blob_ttl_seconds")?, + disable_fallocate: legacy_config + .parse(BUCK2_RE_CLIENT_CFG_SECTION, "disable_fallocate")? + .unwrap_or(false), }) } diff --git a/app/buck2_server/Cargo.toml b/app/buck2_server/Cargo.toml index f319e8b188d93..ee5ebcd84b5c0 100644 --- a/app/buck2_server/Cargo.toml +++ b/app/buck2_server/Cargo.toml @@ -55,7 +55,6 @@ buck2_common = { workspace = true } buck2_configured = { workspace = true } buck2_core = { workspace = true } buck2_data = { workspace = true } -buck2_eden = { workspace = true } buck2_error = { workspace = true } buck2_event_observer = { workspace = true } buck2_events = { workspace = true } @@ -78,10 +77,6 @@ host_sharing = { workspace = true } [target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] psutil = { workspace = true } -[target.'cfg(any(fbcode_build, cargo_internal_build))'.dependencies] -# @oss-disable: detect_eden = { path = "../../../common/rust/shed/detect_eden" } -# @oss-disable: fsinfo = { path = "../../../eden/scm/lib/fsinfo" } - [dev-dependencies] assert_matches = { workspace = true } buck2_util = { workspace = true } diff --git a/app/buck2_server/src/daemon/io_provider.rs b/app/buck2_server/src/daemon/io_provider.rs index 170c2219617c7..c2ace871d92ff 100644 --- a/app/buck2_server/src/daemon/io_provider.rs +++ b/app/buck2_server/src/daemon/io_provider.rs @@ -23,7 +23,7 @@ pub async fn create_io_provider( cas_digest_config: CasDigestConfig, trace_io: bool, ) -> anyhow::Result> { - #[cfg(any(fbcode_build, cargo_internal_build))] + #[cfg(fbcode_build)] { use buck2_core::rollout_percentage::RolloutPercentage; diff --git a/app/buck2_server/src/daemon/server.rs b/app/buck2_server/src/daemon/server.rs index 23ebf66253e64..176dc3b6c442a 100644 --- a/app/buck2_server/src/daemon/server.rs +++ b/app/buck2_server/src/daemon/server.rs @@ -670,7 +670,7 @@ where ); let (output_send, output_recv) = tokio::sync::mpsc::unbounded_channel(); - // We run the event consumer on a totally separate tokio runtime to avoid the consumer task from getting stuck behind + // We run the event consumer on new non-tokio thread to avoid the consumer task from getting stuck behind // another tokio task in its lifo task slot. See T96012305 and https://github.com/tokio-rs/tokio/issues/4323 for more // information. let merge_task = thread::Builder::new() diff --git a/app/buck2_server/src/daemon/state.rs b/app/buck2_server/src/daemon/state.rs index b9c73e95c02b1..493348632c9c7 100644 --- a/app/buck2_server/src/daemon/state.rs +++ b/app/buck2_server/src/daemon/state.rs @@ -544,7 +544,7 @@ impl DaemonState { )?)) } MaterializationMethod::Eden => { - #[cfg(any(fbcode_build, cargo_internal_build))] + #[cfg(fbcode_build)] { use buck2_execute::materialize::eden_api::EdenBuckOut; use buck2_execute_impl::materializers::eden::EdenMaterializer; @@ -575,7 +575,7 @@ impl DaemonState { )) } } - #[cfg(not(any(fbcode_build, cargo_internal_build)))] + #[cfg(not(fbcode_build))] { let _unused = buck_out_path; let _unused = fs; @@ -724,7 +724,7 @@ impl DaemonState { } fn validate_buck_out_mount(&self) -> anyhow::Result<()> { - #[cfg(any(fbcode_build, cargo_internal_build))] + #[cfg(fbcode_build)] { use buck2_core::fs::fs_util; use buck2_core::soft_error; diff --git a/app/buck2_server_commands/Cargo.toml b/app/buck2_server_commands/Cargo.toml index 8fc50e0f45e18..ff710e8e2333e 100644 --- a/app/buck2_server_commands/Cargo.toml +++ b/app/buck2_server_commands/Cargo.toml @@ -25,9 +25,7 @@ tokio = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } -allocative = { workspace = true } dice = { workspace = true } dupe = { workspace = true } gazebo = { workspace = true } diff --git a/app/buck2_subscription_proto/Cargo.toml b/app/buck2_subscription_proto/Cargo.toml index 0d73f13bb3fba..044b56ef203c4 100644 --- a/app/buck2_subscription_proto/Cargo.toml +++ b/app/buck2_subscription_proto/Cargo.toml @@ -12,4 +12,3 @@ tonic = { workspace = true } [build-dependencies] buck2_protoc_dev = { workspace = true } -tonic-build = { workspace = true } diff --git a/app/buck2_test/BUCK b/app/buck2_test/BUCK index 581edcf4f65e5..91bcf3a2032fc 100644 --- a/app/buck2_test/BUCK +++ b/app/buck2_test/BUCK @@ -6,6 +6,20 @@ oncall("build_infra") rust_library( name = "buck2_test", srcs = glob(["src/**/*.rs"]), + os_deps = [ + ( + "linux", + [ + "fbsource//third-party/rust:libc", + ], + ), + ( + "macos", + [ + "fbsource//third-party/rust:libc", + ], + ), + ], test_deps = [ "fbsource//third-party/rust:maplit", ], @@ -18,7 +32,6 @@ rust_library( "fbsource//third-party/rust:futures", "fbsource//third-party/rust:indexmap", "fbsource//third-party/rust:itertools", - "fbsource//third-party/rust:libc", "fbsource//third-party/rust:serde", "fbsource//third-party/rust:serde_json", "fbsource//third-party/rust:tokio", diff --git a/app/buck2_test/Cargo.toml b/app/buck2_test/Cargo.toml index b6a19ad0665f6..ff7f994e06b4b 100644 --- a/app/buck2_test/Cargo.toml +++ b/app/buck2_test/Cargo.toml @@ -13,7 +13,6 @@ derive_more = { workspace = true } futures = { workspace = true } indexmap = { workspace = true } itertools = { workspace = true } -libc = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } @@ -44,5 +43,8 @@ buck2_server_ctx = { workspace = true } buck2_test_api = { workspace = true } buck2_util = { workspace = true } +[target.'cfg(unix)'.dependencies] +libc = { workspace = true } + [dev-dependencies] maplit = { workspace = true } diff --git a/app/buck2_util/BUCK b/app/buck2_util/BUCK index becdee87f30a3..d985a40f323f6 100644 --- a/app/buck2_util/BUCK +++ b/app/buck2_util/BUCK @@ -9,6 +9,18 @@ rust_library( ["src/**/*.rs"], ), os_deps = [ + ( + "linux", + [ + "fbsource//third-party/rust:libc", + ], + ), + ( + "macos", + [ + "fbsource//third-party/rust:libc", + ], + ), ( "windows", [ @@ -22,7 +34,6 @@ rust_library( deps = [ "fbsource//third-party/rust:anyhow", "fbsource//third-party/rust:futures", - "fbsource//third-party/rust:libc", "fbsource//third-party/rust:serde", "fbsource//third-party/rust:static_assertions", "fbsource//third-party/rust:tokio", diff --git a/app/buck2_util/Cargo.toml b/app/buck2_util/Cargo.toml index 4f9c691a6c420..034ad7a816ff6 100644 --- a/app/buck2_util/Cargo.toml +++ b/app/buck2_util/Cargo.toml @@ -12,16 +12,17 @@ allocative = { workspace = true } anyhow = { workspace = true } dupe = { workspace = true } futures = { workspace = true } -once_cell = { workspace = true } starlark_map = { workspace = true } tracing = { workspace = true } triomphe = { workspace = true } -libc = { workspace = true } serde = { workspace = true } static_assertions = { workspace = true } tokio = { workspace = true } +[target.'cfg(unix)'.dependencies] +libc = { workspace = true } + [target.'cfg(windows)'.dependencies] winapi = { workspace = true } diff --git a/app/buck2_worker_proto/Cargo.toml b/app/buck2_worker_proto/Cargo.toml index 7f73470fe6fb6..61b8e18429f95 100644 --- a/app/buck2_worker_proto/Cargo.toml +++ b/app/buck2_worker_proto/Cargo.toml @@ -6,7 +6,6 @@ version = "0.1.0" [dependencies] prost = { workspace = true } -prost-types = { workspace = true } tonic = { workspace = true } [build-dependencies] diff --git a/app/buck2_wrapper_common/Cargo.toml b/app/buck2_wrapper_common/Cargo.toml index 6ee89a1e7a8b9..5396220dc70a5 100644 --- a/app/buck2_wrapper_common/Cargo.toml +++ b/app/buck2_wrapper_common/Cargo.toml @@ -13,7 +13,6 @@ version = "0.1.0" allocative = { workspace = true } anyhow = { workspace = true } dupe = { workspace = true } -once_cell = { workspace = true } serde = { workspace = true } shlex = { workspace = true } sysinfo = { workspace = true } diff --git a/dice/dice/BUCK b/dice/dice/BUCK index 2fce47669701a..af44a7c4f3f77 100644 --- a/dice/dice/BUCK +++ b/dice/dice/BUCK @@ -1,4 +1,3 @@ -load("@fbcode_macros//build_defs:rust_binary.bzl", "rust_binary") load("@fbcode_macros//build_defs:rust_library.bzl", "rust_library") load("@fbsource//tools/build_defs:glob_defs.bzl", "glob") @@ -47,15 +46,3 @@ rust_library( "//common/rust/shed/sorted_vector_map:sorted_vector_map", ], ) - -rust_binary( - name = "read_dump", - srcs = ["bin/read_dump.rs"], - deps = [ - "fbsource//third-party/rust:anyhow", - "fbsource//third-party/rust:bincode", - "fbsource//third-party/rust:clap-3", - "fbsource//third-party/rust:serde_json", - ":dice", - ], -) diff --git a/dice/dice/Cargo.toml b/dice/dice/Cargo.toml index a6cf747c273d2..f6694dd44575f 100644 --- a/dice/dice/Cargo.toml +++ b/dice/dice/Cargo.toml @@ -8,8 +8,6 @@ allocative = { workspace = true } anyhow = "1.0.65" anymap = "0.12.1" async-trait = "0.1.24" -bincode = "1.3.3" -clap = { version = "3.2.24", features = ["derive"] } cmp_any = { workspace = true } dashmap = "4.0.2" derivative = { workspace = true } @@ -23,11 +21,9 @@ itertools = "0.10.0" lock_free_hashtable = { workspace = true } lock_free_vec = { workspace = true } more_futures = { path = "../../shed/more_futures" } -once_cell = "1.3" parking_lot = { version = "0.11.2", features = ["send_guard"] } scopeguard = { workspace = true } serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0.48" slab = "0.4.7" # @oss-disable: sorted_vector_map.path = "../../../common/rust/shed/sorted_vector_map" sorted_vector_map.version = "0.1" @@ -41,12 +37,7 @@ triomphe = { workspace = true } [dev-dependencies] anyhow = "1.0.65" assert_matches = "1.5" -cast = "0.2.3" +bincode = { workspace = true } derivative = "2.1.1" tempfile = "3.1" tokio = { version = "1.5", features = ["full"] } - - -[[bin]] -name = "read_dump" -path = "bin/read_dump.rs" diff --git a/dice/fuzzy_dice/Cargo.toml b/dice/fuzzy_dice/Cargo.toml index fae42bce9de11..caf60fb310fbb 100644 --- a/dice/fuzzy_dice/Cargo.toml +++ b/dice/fuzzy_dice/Cargo.toml @@ -17,8 +17,6 @@ higher-order-closure = { workspace = true } itertools = "0.10.3" parking_lot = { version = "0.11.2", features = ["send_guard"] } quickcheck = "1.0" -quickcheck_async = "0.1.1" -ref-cast = "1.0.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.48" thiserror = "1.0.36" diff --git a/dice/read_dump/BUCK b/dice/read_dump/BUCK new file mode 100644 index 0000000000000..924eb02d4ce19 --- /dev/null +++ b/dice/read_dump/BUCK @@ -0,0 +1,15 @@ +load("@fbcode_macros//build_defs:rust_binary.bzl", "rust_binary") + +oncall("build_infra") + +rust_binary( + name = "read_dump", + srcs = ["src/main.rs"], + deps = [ + "fbsource//third-party/rust:anyhow", + "fbsource//third-party/rust:bincode", + "fbsource//third-party/rust:clap-3", + "fbsource//third-party/rust:serde_json", + "//buck2/dice/dice:dice", + ], +) diff --git a/dice/read_dump/Cargo.toml b/dice/read_dump/Cargo.toml new file mode 100644 index 0000000000000..37ddc04136371 --- /dev/null +++ b/dice/read_dump/Cargo.toml @@ -0,0 +1,12 @@ +[package] +edition = "2021" +name = "read_dump" +version = "0.1.0" + +[dependencies] +anyhow = { workspace = true } +bincode = { workspace = true } +clap = { workspace = true } +serde_json = { workspace = true } + +dice = { path = "../dice" } diff --git a/dice/dice/bin/read_dump.rs b/dice/read_dump/src/main.rs similarity index 100% rename from dice/dice/bin/read_dump.rs rename to dice/read_dump/src/main.rs diff --git a/docs/developers/bxl_faq.md b/docs/developers/bxl_faq.md index ef601226f14c2..15f1d40bec2ac 100644 --- a/docs/developers/bxl_faq.md +++ b/docs/developers/bxl_faq.md @@ -60,3 +60,19 @@ execution. Anon targets are meant for sharing work betwen multiple BXLs. The context type is a normal rule analysis `context`. Anon targets are `await`-ed inline with your BXL function. + +## Can I mutate types returned by BXL APIs? + +The data types produced by BXL API calls are always immutable. + +## What is run synchronously vs asynchronously? + +Starlark itself is run synchronously. However, certain BXL APIs are evaluated +asynchronously. + +If you pass in multiple inputs to builds, queries, or analyses, the execution of +these API calls will be blocking, but the inputs themselves will be evaluated in +parallel within the execution. + +Ensuring artifacts, dynamic outputs, anon targets, and resolving promises will +happen _after_ the Starlark script is executed. diff --git a/docs/rule_authors/anon_targets.md b/docs/rule_authors/anon_targets.md index 97e5d686421ce..e443246a25a87 100644 --- a/docs/rule_authors/anon_targets.md +++ b/docs/rule_authors/anon_targets.md @@ -37,6 +37,208 @@ analysis (which is passed in, or after looking at an anon target). In many ways, anon target is the version of `dynamic_output` at analysis time, rather than action time. +The execution platform for an anon target is that of the inherited from the +calling target, which is part of the hash. If that is too restrictive, you could +use execution groups, where an anon target gets told which execution group to +use. + +# Creating anon targets + +## Anon rule + +An anonymous rule is defined using `rule` or `anon_rule`. + +Example: + +```python +my_anon_rule = rule( + impl = _anon_impl, + attrs = {}, +) + +# Or: + +my_anon_rule = anon_rule( + impl = _anon_impl, + attrs = {}, + artifact_promise_mappings = {} # only available for anon_rule +) +``` + +For `rule`, these are normal rules, with the difference that they are not in a +configuration, so `ctx.actions.label` won't show configuration information, but +just `unspecified`. + +For `anon_rule`, the configuration restrictions also apply, and there is an +`artifact_promise_mappings` field which you can specify a dict of artifact +promise names to the map function, which would be applied to the anon target's +promise during rule resolution. + +## Anon target + +An anonymous rule is used via `ctx.actions.anon_target` or +`ctx.actions.anon_targets`, passing in the rule and the attributes for the rule. + +The return values of those functions are a `AnonTarget` and `AnonTargets` type, +respectively. + +Example: + +```python +my_anon_rule1 = anon_rule( + impl = _anon_impl, + attrs = {}, + artifact_promise_mappings = {} +) + +my_anon_rule2 = anon_rule( + impl = _anon_impl, + attrs = {}, + artifact_promise_mappings = {} +) + +# +anon_target = ctx.actions.anon_target(my_anon_rule1, {}) + +anon_targets = ctx.actions.anon_targets([(my_anon_rule1, {}), (my_anon_rule2, {})]) +``` + +### `AnonTarget` and `AnonTargets` + +`AnonTarget` has a `promise` attribute, and `artifact()` and `artifacts()` +functions. `AnonTargets` has a `promise` attribute and `anon_targets` attribute. + +The `promise` attribute for both types returns the anon target's promise (type +is `promise`), which when evaluated returns the providers of the anonymous +target. The `promise` type has a few special behaviors. + +- It has a `map` function, which takes a function and applies it to the future, + returning a new future +- All promises will eventually resolve to a list of providers + +For `AnonTarget`, the `artifact()` and `artifacts()` functions only return +something if using `anon_rule`. `artifact()` takes in an artifact name, which +should be found in the `artifact_promise_mappings` dict, and returns the +artifact promise. `artifacts()` returns the dict of all promise artifact names +to the artifact promise itself, as defined in `artifact_promise_mappings`. See +[Convert promise to artifact](#convert-promise-to-artifact) below for more +information about artifact promises. + +Example: + +```python +HelloInfo = provider(fields = ["output"]) + +my_anon_rule = anon_rule( + impl = _anon_impl, + attrs = {}, + artifact_promise_mappings = { + "hello": lambda x: x[HelloInfo].output, + } +) + +# +anon_target = ctx.actions.anon_target(my_anon_rule, {}) +artifact = anon_target.artifact("hello") +artifact_from_dict = anon_target.artifacts()["hello"] +``` + +For `AnonTargets`, the `anon_targets` attribute returns a list of the underlying +`AnonTarget`s. + +Example: + +```python +HelloInfo = provider(fields = ["output"]) +GoodbyeInfo = provider(fields = ["output"]) + +my_anon_rule1 = anon_rule( + impl = _anon_impl, + attrs = {}, + artifact_promise_mappings = { + "hello": lambda x: x[HelloInfo].output, + } +) + +my_anon_rule2 = anon_rule( + impl = _anon_impl, + attrs = {}, + artifact_promise_mappings = { + "goodbye": lambda x: x[GoodbyeInfo].output, + } +) + +# +all_targets = ctx.actions.anon_targets([(my_anon_rule1, {}), (my_anon_rule2, {})]) +hello = all_targets.anon_targets[0].artifact("hello") +goodbye = all_targets.anon_targets[1].artifact("goodbye") +``` + +# Attributes + +Anon targets only support a subset of attributes that normal rules support. + +Supported attributes: + +- `bool` +- `int` +- `str` +- `enum` +- `dep` + - `deps` attributes do not take strings, but dependencies, already in a + configuration + - `exec_deps` are available if the passed in `dep`'s execution platform + matches + - Default `attr.deps` (as used for toolchains) are not permitted, as the + default can't express a dependency. They must be passed forward from the + caller. that of the anon target's caller +- `source` + - Accepts bound artifacts or promise artifacts +- `arg` + - Can only be used if `anon_target_compatible` is `True` when declaring + `attrs.arg` (ex: `attrs.arg(anon_target_compatible = True)`) +- `label` +- `list` +- `tuple` +- `dict` +- `one_of` +- `option` + +You can use these attributes like you would in normal rules: + +```python +my_anon_rule = anon_rule( + impl = _my_anon_impl, + attrs = { + "my_int": attrs.int(), + "my_string_with_default": attrs.string(default = "foo"), + "my_optional_source": attrs.option(attrs.source()), + "my_list_of_labels": attrs.list(attrs.label()), + }, + artifact_promise_mappings = {} +) + +def _my_anon_impl(ctx: AnalysisContext) -> list[Provider]: + my_int = ctx.attrs.my_int + my_string_with_default = ctx.attrs.my_string_with_default + my_optional_source = ctx.attrs.my_optional_source + my_list_of_labels = ctx.attrs.my_list_of_labels + + # do something with the attributes... + + return [DefaultInfo()] +``` + +## Attribute Resolution + +Attribute resolution is handled differently from normal code: + +- The `name` attribute is optional, but, if present, must be a syntactically + valid target, but can refer to a cell/package that does not exist. +- Transitions and more complex forms of attributes are banned. + +# Examples + ## Simple Example ```python @@ -65,52 +267,6 @@ def impl(ctx): .map(k) ``` -Notes: - -- An anonymous rule is defined using `rule` or `anon_rule`. For `rule`, these - are normal rules, with the difference that they are not in a configuration, so - `ctx.actions.label` won't show configuration information, but just - `unspecified`. For `anon_rule`, the configuration restrictions also apply, and - there is an `artifact_promise_mappings` field which you can specify a dict of - artifact promise names to the map function, which would be applied to the anon - target's promise during rule resolution. -- An anonymous rule is used via `ctx.actions.anon_target`, passing in the rule - and the attributes for the rule. -- The return value is a `AnonTarget` type, which has a `promise` attribute, and - `artifact()` and `artifacts()` functions. -- The `promise` attribute returns the anon target's promise (type is `promise`), - which when evaluated returns the providers of the anonymous target. The - `promise` type has a few special behaviors. - - It has a `map` function, which takes a function and applies it to the - future, returning a new future. - - If analysis returns a `promise` type, the outer Rust layer invokes the - future to get at the analysis result. If that future then returns another - future, Rust keeps going until it has a final result. It must eventually get - to a list of providers. -- Attribute resolution is handled differently from normal code: - - String/Int/Bool happen as normal. - - The name attribute is optional, but, if present, must be a syntactically - valid target, but can refer to a cell/package that does not exist. - - Deps attributes do not take strings, but dependencies, already in a - configuration. - - Exec_deps are available if the passed in dep's execution platform matches - that of the anon target's caller. - - Transitions and more complex forms of attributes are banned. - - Default `attr.deps` (as used for toolchains) are not permitted, as the - default can't express a dependency. They must be passed forward from the - caller. -- The `artifact()` and `artifacts()` functions only return something if using - `anon_rule`. `artifact()` takes in an artifact name, which should be found in - the `artifact_promise_mappings` dict, and returns the artifact promise. - `artifacts()` returns the dict of all promise artifact names to the artifact - promise itself, as defined in `artifact_promise_mappings`. See - [Convert promise to artifact](#convert-promise-to-artifact) below for more - information about artifact promises. -- The execution platform for an anon target is that of the inherited from the - calling target, which is part of the hash. If that is too restrictive, you - could use execution groups, where an anon target gets told which execution - group to use. - ## Longer example The following code represents a scenario for a compile-and-link language where, @@ -187,7 +343,7 @@ silly_binary = rule( It can be challenging to pass around the promises from anon_target and structure functions to support that. If you only need an artifact (or multiple artifacts) -from an anon_target, you can use `artifacts()` function on the anon target to +from an anon_target, you can use `artifact()` function on the anon target to convert a promise to an artifact. This artifact can be passed to most things that expect artifacts, but until it is resolved (at the end of the current analysis) it can't be inspected with artifact functions like `.extension`, etc. @@ -231,5 +387,4 @@ def _use_impl(ctx: AnalysisContext) -> ["provider"]: use_promise_artifact = rule(impl = _use_impl, attrs = { "some_tool": attr.exec_dep(), }) - ``` diff --git a/docs/users/build_observability/interactive_console.md b/docs/users/build_observability/interactive_console.md index 59f44c9066bc3..79d9c2127c328 100644 --- a/docs/users/build_observability/interactive_console.md +++ b/docs/users/build_observability/interactive_console.md @@ -1,17 +1,86 @@ --- id: interactive_console -title: Buck2 Interactive Console +title: Buck2 Consoles --- -This will work as long as stdin is a TTY, which will be true most of the time if -you're not piping anything into Buck2. +Buck2 offers several console types for build-like commands (e.g. `build`, +`install`, `test`, etc.). The console is always written to stderr. -To see what's available you can press `?`. +The console can be specified via the `--console` flag, or the `BUCK_CONSOLE` env +variable. The default console type is `auto`. Supported `--console` types: -To disable to allow alternate use of stdin, or for follow up pasted commands to -not get swallowed: +- `auto` - Default console type. Auto defaults to the superconsole if the stderr + is a TTY. Otherwise will uses simple console +- `simple` - Build a simpleconsole with TTY, if TTY is supported by the OS. See + [Simpleconsole](#simpleconsole) +- `simplenotty` - Build a simpleconsole without TTY. See + [Simpleconsole](#simpleconsole) +- `simpletty` - Build a simpleconsole with TTY. See + [Simpleconsole](#simpleconsole) +- `super` - Build a superconsole regardless of whether stderr is a TTY. See + [Superconsole](#superconsole) +- `none` - See [No console](#no-console) -Environment Variable: `BUCK_NO_INTERACTIVE_CONSOLE` or flag: -`--no-interactive-console` +If `simplenotty` or `none` are specified, or if TTY is not supported by the OS, +then we strip out any color within the error messages. + +All console options will output the build result, whether succeeded or not, to +stdout. Note that action execution stderr is hidden if the build succeeded. + +The simple and superconsole will also print metadata about the build itself, +such as the Buck2 UUID, the percentage of cache hits, and the number of action +commands ran. In addition, they will print the event spans detected within the +build. + +## Simpleconsole + +The simpleconsole prints the stdout/stderr messages and event spans, line by +line. There is no resource usage telemetry emitted. + +### Demo + +![Simpleconsole running a build](simpleconsole.gif) + +## Superconsole + +The superconsole uses the +[superconsole](https://github.com/facebookincubator/superconsole) library to +provide an interactive console which shows the event spans going on within +Buck2. + +### Demo + +![Superconsole running a build](superconsole.gif) + +### Toggles + +The superconsole also provides several toggles to inspect ongoing Buck2 +telemetry. + +To see what's available you can press `?` or `h`. This will work as long as +stdin is a TTY, which will be true most of the time if you're not piping +anything into Buck2. To disable to allow alternate use of stdin, or for follow +up pasted commands to not get swallowed, you can set the +`BUCK_NO_INTERACTIVE_CONSOLE` environment variable, or use the flag +`--no-interactive-console`. + +We support the following toggles: + +- `c` - toggle commands (shown in superconsole by default) +- `d` - toggle DICE key states +- `e` - toggle debugging events, such as spans and instant event counts +- `2` - toggle two lines mode when showing events +- `r` - toggle detailed remote execution info, such as uploads, downloads, and + action cache calls +- `i` - toggle I/O counters +- `p` - display target configurations +- `+` - show more lines +- `-` - show fewer lines +- `h` - show help Note: Not available yet for Windows + +## No console + +When specifying the `none` console type, Buck2 will only print if the build +succeeded, or the error if the build failed. diff --git a/docs/users/build_observability/logging.md b/docs/users/build_observability/logging.md index 24232c9f56845..1d0f1e7c618ea 100644 --- a/docs/users/build_observability/logging.md +++ b/docs/users/build_observability/logging.md @@ -3,7 +3,156 @@ id: logging title: Logging --- -Buck2 produces detailed event logs for each invocation. They follow a schema -outlined in `data.proto`. +Buck2 produces detailed event logs for each invocation, which follow a schema +outlined in `app/buck_data/data.proto` in the buck2 parent directory. The event +logs that Buck2 produces automatically are always in protobuf zstd-compressed +format (see [Viewing the event log](#viewing-the-event-log) for more details). -Those logs can be accessed using commands under `buck2 log`. +## Event log format + +Warning: the schemas are all subject to change, so we do not recommend relying +on the format. For the source of truth, take a look at `data.proto`. + +### Invocation header + +The first line of the event log is always the `Invocation` header: + +```python +Invocation { + # CLI args split into a list of strings + command_line_args: List[str], + # Expanded CLI args, which expand any argsfiles + expanded_command_line_args: List[str], + # Absolute path of the current working directory of the Buck2 command + working_dir: str, + # UUID of the Buck2 command + trace_id: str, +} +``` + +### Command result footer + +The last line is always the `CommandResult`: + +```python +Result { + # One of the result types of CommandResult protobuf type in data.proto + result: BuildResponse | CqueryResponse | BxlResponse | ..., +} +``` + +### Buck events + +The rest of the event log contain `BuckEvent`s, which are either +`SpanStartEvent`s, `SpanEndEvent`s, or `InstantEvent`s. + +The `BuckEvent` format is roughly as follows: + +```python +Event { + # When the event was fired. This is always a 2-item list, where the first + # value is millis, second value is micros + timestamp: List[u64], + # UUID of the Buck2 command, same one as the invocation header + trace_id: str, + # A trace-unique 64-bit integer identifying this event's span ID, + # if this event begins a new span or belongs to one. + span_id: u64, + # A trace-unique 64-bit identifying the span that this event is logically + # parented to. + parent_id: u64, + # See sections below for more details + data: SpanStart | SpanEnd | Instant, +} +``` + +#### Span starts + +The `SpanStartEvent` indicates that a span of work starting: + +```python +SpanStart { + # One of the data types of SpanStartEvent protobuf type in data.proto + data: AnalysisStart | ActionExecutionStart | ..., +} +``` + +#### Span ends + +The `SpanEndEvent` indicates that a span of work has finished: + +```python +SpanEnd { + # Duration of the span + duration_us: u64, + # CPU poll times for this span + stats: SpanStats, + # One of the data types of SpanEndEvent protobuf type in data.proto + data: AnalysisEnd | ActionExecutionEnd | ..., +} + +# CPU poll times for this span +SpanStats { + max_poll_time_us: u64, + total_poll_time_us: u64, +} +``` + +#### Instant events + +The `InstantEvent` represents a single point in time: + +```python +InstantEvent { + # One of the data types of InstantEvent protobuf type in data.proto + data: ConsoleMessage | ActionError | ..., +} +``` + +One specific instant event type that may be of interest is the `SnapShot` event, +which includes some interesting details like RSS, CPU, I/O, remote execution, +and DICE metrics. + +## Viewing the event log + +Event logs can be accessed using commands under `buck2 log show`, which outputs +the event logs in JSONL format. You can run `buck2 log show --help` to see all +available options. Some useful commands: + +- Show the logs for the most recent Buck2 command: + +```sh +buck2 log show +``` + +- Show the logs for a specific Buck2 command, given the command's UUID: + +```sh +buck2 log show --trace-id +``` + +- Show the logs for a recent Buck2 command: + +```sh +buck2 log show --recent +``` + + + +You can also download the logs locally from Buck2 UI. The logs will be +downloaded from Manifold in protobuf zstd-compressed format, and you can view +them in JSONL format by passing the path into `buck2 log show`. + + +The JSON schema is derived from the protobuf types, and the log itself could be +quite large. [jq](https://jqlang.github.io/jq/) can be useful to find specific +things. For example, this jq script shows the max event delay between a snapshot +event creation on the daemon side, and when the client receives it. + +```sh +buck2 log show | jq -s ' + map( + .Event.data.Instant.data.Snapshot.this_event_client_delay_ms + | select(. != null) + ) | max' +``` diff --git a/docs/users/build_observability/simpleconsole.gif b/docs/users/build_observability/simpleconsole.gif new file mode 100644 index 0000000000000..079ae7038d5a5 Binary files /dev/null and b/docs/users/build_observability/simpleconsole.gif differ diff --git a/docs/users/build_observability/superconsole.gif b/docs/users/build_observability/superconsole.gif new file mode 100644 index 0000000000000..f04a970e67d0e Binary files /dev/null and b/docs/users/build_observability/superconsole.gif differ diff --git a/docs/users/faq/common_issues.md b/docs/users/faq/common_issues.md index ce45ef1f9fcdd..55a594322d05b 100644 --- a/docs/users/faq/common_issues.md +++ b/docs/users/faq/common_issues.md @@ -80,3 +80,18 @@ memory) when your machine ran out of memory. You can also use the `--after ` option to see all open spans at a certain point in time of the build. + +## Why does my target not have any outputs? + +If you see that your build succeeded, but the console message stated that your +target did not have any outputs, this means that the underlying rule did not +declare any outputs artifacts, defined as outputs declared in: + +- `default_outputs` and/or `other_outputs` in `DefaultInfo` +- `cmd_args` in `RunInfo` +- `cmd_args` inside the `command` in `ExternalRunnerTestInfo` + +For example, building a target which is uses a `python_library` rule merely +groups source files together, and does not generate any output artifacts such as +a python executable. You would need to build a `python_binary` which uses that +library in order to get an output. diff --git a/examples/hello_world/BUCK b/examples/hello_world/BUCK index ebd90692f7e65..b0e97423fd55c 100644 --- a/examples/hello_world/BUCK +++ b/examples/hello_world/BUCK @@ -7,7 +7,7 @@ cxx_binary( cxx_library( name = "print", - srcs = glob(["**/*.cpp"]), + srcs = ["library.cpp"], exported_headers = glob(["**/*.hpp"]), visibility = ["PUBLIC"], ) diff --git a/prelude/erlang/common_test/cth_hooks/src/cth_tpx.erl b/prelude/erlang/common_test/cth_hooks/src/cth_tpx.erl index 72ee0a2b40e77..fe7be1216c5b4 100644 --- a/prelude/erlang/common_test/cth_hooks/src/cth_tpx.erl +++ b/prelude/erlang/common_test/cth_hooks/src/cth_tpx.erl @@ -105,7 +105,7 @@ fmt_stack(Suite, CasePat, CaseArgs, {_Class, {Reason, ST}}, Label) -> fmt_stack(Suite, CasePat, CaseArgs, {Reason, ST}, Label); fmt_stack(_Suite, _CasePat, _CaseArgs, Reason, _Label) -> Output = ct_error_printer:format_error(Reason, true), - unicode:characters_to_list(io_lib:format("~s", [Output])). + unicode:characters_to_list(io_lib:format("~ts", [Output])). %% ----------------------------------------------------------------------------- %% CT hooks functions diff --git a/prelude/erlang/common_test/test_binary/src/list_test.erl b/prelude/erlang/common_test/test_binary/src/list_test.erl index 05de4291d1259..04abc56599785 100644 --- a/prelude/erlang/common_test/test_binary/src/list_test.erl +++ b/prelude/erlang/common_test/test_binary/src/list_test.erl @@ -72,16 +72,21 @@ list_tests(Suite, Hooks) -> %% @doc Test that all the tests in the list are exported. -spec test_exported_test(suite(), test_name()) -> error | ok. test_exported_test(Suite, Test) -> - case erlang:function_exported(Suite, Test, _Arity = 1) of + case erlang:function_exported(Suite, Test, 1) of false -> - error( - {invalid_test, - io_lib:format( - "The test ~s has been discovered while recursively exploring all/0, " ++ - "groups/0 but is not an exported method of arity 1", - [Test] - )} - ); + case erlang:function_exported(Suite, '$handle_undefined_function', 2) of + true -> + ok; + false -> + error( + {invalid_test, + io_lib:format( + "The test ~s has been discovered while recursively exploring all/0, " ++ + "groups/0 but is not an exported method of arity 1", + [Test] + )} + ) + end; true -> ok end. diff --git a/prelude/rust/cargo_buildscript.bzl b/prelude/rust/cargo_buildscript.bzl index c3d2970d356a2..a369ae87b55f0 100644 --- a/prelude/rust/cargo_buildscript.bzl +++ b/prelude/rust/cargo_buildscript.bzl @@ -47,7 +47,7 @@ def _cargo_buildscript_impl(ctx: AnalysisContext) -> list[Provider]: env["CARGO_PKG_NAME"] = ctx.attrs.package_name env["CARGO_PKG_VERSION"] = ctx.attrs.version env["OUT_DIR"] = out_dir.as_output() - env["RUSTC"] = cmd_args(toolchain_info.compiler_standalone).relative_to(cwd) + env["RUSTC"] = cmd_args(toolchain_info.compiler).relative_to(cwd) env["RUSTC_LINKER"] = "/bin/false" env["RUST_BACKTRACE"] = "1" env["TARGET"] = toolchain_info.rustc_target_triple diff --git a/prelude/rust/rust_toolchain.bzl b/prelude/rust/rust_toolchain.bzl index 3b10d2a35f322..1156e57a03c4b 100644 --- a/prelude/rust/rust_toolchain.bzl +++ b/prelude/rust/rust_toolchain.bzl @@ -65,9 +65,6 @@ rust_toolchain_attrs = { "failure_filter": provider_field(bool, default = False), # The Rust compiler (rustc) "compiler": provider_field(RunInfo | None, default = None), - # A Rust compiler that can be used "standalone", without special settings - # from Rust rules (e.g. this keeps the default sysroot). - "compiler_standalone": provider_field(RunInfo | None, default = None), # Rust documentation extractor (rustdoc) "rustdoc": provider_field(RunInfo | None, default = None), # Clippy (linter) version of the compiler diff --git a/prelude/toolchains/rust.bzl b/prelude/toolchains/rust.bzl index 21ee3d32fd469..85e08b1c85b9a 100644 --- a/prelude/toolchains/rust.bzl +++ b/prelude/toolchains/rust.bzl @@ -42,7 +42,6 @@ def _system_rust_toolchain_impl(ctx): clippy_driver = RunInfo(args = ["clippy-driver"]), clippy_toml = ctx.attrs.clippy_toml[DefaultInfo].default_outputs[0] if ctx.attrs.clippy_toml else None, compiler = RunInfo(args = ["rustc"]), - compiler_standalone = RunInfo(args = ["rustc"]), concat_tool = ctx.attrs.concat_tool[RunInfo], default_edition = ctx.attrs.default_edition, deny_lints = ctx.attrs.deny_lints, diff --git a/shed/more_futures/Cargo.toml b/shed/more_futures/Cargo.toml index 97ca4b85b9154..b467f05f0e0ea 100644 --- a/shed/more_futures/Cargo.toml +++ b/shed/more_futures/Cargo.toml @@ -11,7 +11,6 @@ buck2_events = { workspace = true } buck2_wrapper_common = { workspace = true } dupe = { workspace = true } futures = "0.3" -gazebo = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true } pin-project = "0.4" diff --git a/shim/shims.bzl b/shim/shims.bzl index f18390c605adc..0124fe940775e 100644 --- a/shim/shims.bzl +++ b/shim/shims.bzl @@ -19,10 +19,11 @@ def rust_library( os_deps = None, test_deps = None, test_env = None, + test_os_deps = None, mapped_srcs = {}, visibility = ["PUBLIC"], **kwargs): - _unused = (test_deps, test_env, named_deps, visibility) # @unused + _unused = (test_deps, test_env, test_os_deps, named_deps, visibility) # @unused deps = _maybe_select_map(deps, _fix_deps) mapped_srcs = _maybe_select_map(mapped_srcs, _fix_mapped_srcs) if os_deps: @@ -78,7 +79,6 @@ def rust_protobuf_library( deps = [], test_deps = None, doctests = True): - deps = _maybe_select_map(deps, _fix_deps) if build_env: build_env = { k: _fix_dep_in_string(v) @@ -183,18 +183,21 @@ def _fix_dep(x: str) -> [ None, str, ]: - if x == "//common/rust/folly/logging:logging": - return None + if x == "//common/rust/shed/fbinit:fbinit": + return "fbsource//third-party/rust:fbinit" + elif x == "//common/rust/shed/sorted_vector_map:sorted_vector_map": + return "fbsource//third-party/rust:sorted_vector_map" elif x == "//watchman/rust/watchman_client:watchman_client": return "fbsource//third-party/rust:watchman_client" - elif x.startswith("//common/rust/shed/"): - return "fbsource//third-party/rust:" + x.removeprefix("//common/rust/shed/").split(":")[0] - elif x.startswith("//common/rust/") or x.startswith("//buck2/facebook/") or x.startswith("//eden/") or x.startswith("//remote_execution/"): + elif x.startswith("fbsource//third-party/rust:") or x.startswith(":"): + return x + elif x.startswith("//buck2/facebook/"): return None elif x.startswith("//buck2/"): return "root//" + x.removeprefix("//buck2/") else: - return x + fail("Dependency is unaccounted for `{}`.\n".format(x) + + "Did you forget 'oss-disable'?") def _fix_dep_in_string(x: str) -> str: """Replace internal labels in string values such as env-vars.""" diff --git a/starlark-rust/starlark_lsp/BUCK b/starlark-rust/starlark_lsp/BUCK index 90c2547f471c3..ef378fae15f82 100644 --- a/starlark-rust/starlark_lsp/BUCK +++ b/starlark-rust/starlark_lsp/BUCK @@ -9,10 +9,23 @@ rust_library( "--cfg=rust_nightly", ], test_deps = [ - "fbsource//third-party/rust:maplit", "fbsource//third-party/rust:regex", "fbsource//third-party/rust:textwrap", ], + test_os_deps = [ + ( + "linux", + [ + "fbsource//third-party/rust:maplit", + ], + ), + ( + "macos", + [ + "fbsource//third-party/rust:maplit", + ], + ), + ], deps = [ "fbsource//third-party/rust:anyhow", "fbsource//third-party/rust:derivative", diff --git a/starlark-rust/starlark_lsp/Cargo.toml b/starlark-rust/starlark_lsp/Cargo.toml index 0c9601a204e86..b656afb44b251 100644 --- a/starlark-rust/starlark_lsp/Cargo.toml +++ b/starlark-rust/starlark_lsp/Cargo.toml @@ -29,6 +29,8 @@ starlark = { version = "0.10.0", path = "../starlark" } starlark_syntax = { version = "0.10.0", path = "../starlark_syntax" } [dev-dependencies] -maplit = "1.0.2" regex = "1.5.4" textwrap = "0.11" + +[target.'cfg(not(windows))'.dev-dependencies] +maplit = "1.0.2" diff --git a/superconsole/Cargo.toml b/superconsole/Cargo.toml index 5f517cf5438e0..e153e95a6b0d0 100644 --- a/superconsole/Cargo.toml +++ b/superconsole/Cargo.toml @@ -13,7 +13,6 @@ version = "0.2.0" [dependencies] anyhow = "1.0.65" crossbeam-channel = "0.5" -crossbeam-epoch = "0.9.7" crossterm = "0.27" itertools = "0.10" termwiz = "0.18"