diff --git a/.github/workflows/test-cron.yaml b/.github/workflows/test-cron.yaml index 48c0af7037a..ea1305af2d9 100644 --- a/.github/workflows/test-cron.yaml +++ b/.github/workflows/test-cron.yaml @@ -54,7 +54,7 @@ jobs: uses: actions/cache@v3 with: key: Linux-ARM64-engine-${{ steps.get-engine-hash.outputs.hash }}-v1 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so @@ -90,7 +90,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.Linux-ARM64 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so @@ -155,7 +155,7 @@ jobs: uses: actions/cache@v3 with: key: Linux-x86_64-engine-${{ steps.get-engine-hash.outputs.hash }}-v1 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so @@ -191,7 +191,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so @@ -268,7 +268,7 @@ jobs: uses: actions/cache@v3 with: key: macOS11-x86_64-engine-${{ steps.get-engine-hash.outputs.hash }}-v1 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so @@ -304,7 +304,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.macOS11-x86_64 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f34422cffea..d0e6fc16933 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -58,7 +58,7 @@ jobs: uses: actions/cache@v3 with: key: Linux-ARM64-engine-${{ steps.get-engine-hash.outputs.hash }}-v1 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so @@ -93,7 +93,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.Linux-ARM64 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so @@ -158,7 +158,7 @@ jobs: uses: actions/cache@v3 with: key: Linux-x86_64-engine-${{ steps.get-engine-hash.outputs.hash }}-v1 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so @@ -193,7 +193,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so @@ -270,7 +270,7 @@ jobs: uses: actions/cache@v3 with: key: macOS11-x86_64-engine-${{ steps.get-engine-hash.outputs.hash }}-v1 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so @@ -305,7 +305,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.macOS11-x86_64 - path: '.pants + path: 'src/python/pants/bin/native_client src/python/pants/engine/internals/native_engine.so diff --git a/build-support/bin/generate_github_workflows.py b/build-support/bin/generate_github_workflows.py index e4f847c30e7..3a9bac56615 100644 --- a/build-support/bin/generate_github_workflows.py +++ b/build-support/bin/generate_github_workflows.py @@ -68,7 +68,7 @@ def hashFiles(path: str) -> str: NATIVE_FILES = [ - ".pants", + "src/python/pants/bin/native_client", "src/python/pants/engine/internals/native_engine.so", "src/python/pants/engine/internals/native_engine.so.metadata", ] diff --git a/build-support/bin/rust/bootstrap_code.sh b/build-support/bin/rust/bootstrap_code.sh index b94450f3bf7..0fcdfb69557 100644 --- a/build-support/bin/rust/bootstrap_code.sh +++ b/build-support/bin/rust/bootstrap_code.sh @@ -31,9 +31,9 @@ case "${KERNEL}" in esac readonly NATIVE_ENGINE_BINARY="native_engine.so" +export NATIVE_CLIENT_BINARY="${REPO_ROOT}/src/python/pants/bin/native_client" readonly NATIVE_ENGINE_RESOURCE="${REPO_ROOT}/src/python/pants/engine/internals/${NATIVE_ENGINE_BINARY}" readonly NATIVE_ENGINE_RESOURCE_METADATA="${NATIVE_ENGINE_RESOURCE}.metadata" -readonly NATIVE_CLIENT_PATH="${REPO_ROOT}/.pants" readonly NATIVE_CLIENT_TARGET="${NATIVE_ROOT}/target/${MODE}/pants" function _build_native_code() { @@ -55,7 +55,7 @@ function bootstrap_native_code() { engine_version_in_metadata="$(sed -n 's/^engine_version: //p' "${NATIVE_ENGINE_RESOURCE_METADATA}")" fi - if [[ -f "${NATIVE_ENGINE_RESOURCE}" && -f "${NATIVE_CLIENT_PATH}" && + if [[ -f "${NATIVE_ENGINE_RESOURCE}" && -f "${NATIVE_CLIENT_BINARY}" && "${engine_version_calculated}" == "${engine_version_in_metadata}" ]]; then return 0 fi @@ -79,9 +79,9 @@ function bootstrap_native_code() { # Create the native engine resource. # NB: On Mac Silicon, for some reason, first removing the old native_engine.so is necessary to avoid the Pants # process from being killed when recompiling. - rm -f "${NATIVE_ENGINE_RESOURCE}" "${NATIVE_CLIENT_PATH}" + rm -f "${NATIVE_ENGINE_RESOURCE}" "${NATIVE_CLIENT_BINARY}" cp "${native_binary}" "${NATIVE_ENGINE_RESOURCE}" - cp "${NATIVE_CLIENT_TARGET}" "${NATIVE_CLIENT_PATH}" + cp "${NATIVE_CLIENT_TARGET}" "${NATIVE_CLIENT_BINARY}" # Create the accompanying metadata file. local -r metadata_file=$(mktemp -t pants.native_engine.metadata.XXXXXX) diff --git a/pants b/pants index 471fc51e40a..9795daaf697 100755 --- a/pants +++ b/pants @@ -41,7 +41,6 @@ source "${HERE}/build-support/pants_venv" source "${HERE}/build-support/bin/rust/bootstrap_code.sh" function exec_pants_bare() { - PANTS_NATIVE_EXE="${HERE}/.pants" PANTS_PY_EXE="${HERE}/src/python/pants/bin/pants_loader.py" PANTS_SRCPATH="${HERE}/src/python" @@ -49,9 +48,18 @@ function exec_pants_bare() { activate_pants_venv 1>&2 bootstrap_native_code 1>&2 - if [ -n "${USE_NATIVE_PANTS}" ]; then + if [ -n "${PANTS_DEBUG}" ]; then + if [[ "$*" != *"--no-pantsd"* ]]; then + echo "Error! Must pass '--no-pantsd' when using PANTS_DEBUG" + exit 1 + fi + DEBUG_ARGS="-m debugpy --listen 127.0.0.1:5678 --wait-for-client" + echo "Will launch debugpy server at '127.0.0.1:5678' waiting for client connection." + fi + + if [ -z "${PANTS_NO_NATIVE_CLIENT}" ]; then set +e - "${PANTS_NATIVE_EXE}" "$@" + "${NATIVE_CLIENT_BINARY}" "$@" result=$? # N.B.: The native pants client currently relies on pantsd being up. If it's not, it will fail # with exit code 75 (EX_TEMPFAIL in /usr/include/sysexits.h) and we should fall through to the @@ -63,15 +71,6 @@ function exec_pants_bare() { set -e fi - if [ -n "${PANTS_DEBUG}" ]; then - if [[ "$*" != *"--no-pantsd"* ]]; then - echo "Error! Must pass '--no-pantsd' when using PANTS_DEBUG" - exit 1 - fi - DEBUG_ARGS="-m debugpy --listen 127.0.0.1:5678 --wait-for-client" - echo "Will launch debugpy server at '127.0.0.1:5678' waiting for client connection." - fi - # shellcheck disable=SC2086 PYTHONPATH="${PANTS_SRCPATH}:${PYTHONPATH}" RUNNING_PANTS_FROM_SOURCES=1 NO_SCIE_WARNING=1 \ exec ${PANTS_PREPEND_ARGS:-} "$(venv_dir)/bin/python" ${DEBUG_ARGS} "${PANTS_PY_EXE}" "$@" diff --git a/src/python/pants/BUILD b/src/python/pants/BUILD index e62fe5b62ca..d6ac89fedbb 100644 --- a/src/python/pants/BUILD +++ b/src/python/pants/BUILD @@ -18,6 +18,8 @@ python_distribution( dependencies=[ "./__main__.py", ":resources", + # Include the native client binary in the distribution. + "src/python/pants/bin:native_client", ], # Because we have native code, this will cause the wheel to use whatever the ABI is for the # interpreter used to run setup.py, e.g. `cp36m-macosx_10_15_x86_64`. diff --git a/src/python/pants/bin/.gitignore b/src/python/pants/bin/.gitignore new file mode 100644 index 00000000000..e3f6b86b8dc --- /dev/null +++ b/src/python/pants/bin/.gitignore @@ -0,0 +1 @@ +/native_client diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index d5dae54a32f..3d70e7ad2f1 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -99,5 +99,9 @@ pex_binary( strip_pex_env=False, ) +resources( + name="native_client", + sources=["native_client"], +) python_tests(name="tests") diff --git a/src/rust/engine/client/src/main.rs b/src/rust/engine/client/src/main.rs index 6fdaf9be8b3..e2f5ccc1997 100644 --- a/src/rust/engine/client/src/main.rs +++ b/src/rust/engine/client/src/main.rs @@ -25,13 +25,16 @@ // Arc can be more clear than needing to grok Orderings: #![allow(clippy::mutex_atomic)] -use std::convert::AsRef; +use std::convert::{AsRef, Infallible}; use std::env; +use std::ffi::{CString, OsString}; +use std::os::unix::ffi::OsStringExt; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::time::SystemTime; use log::debug; +use nix::unistd::execv; use strum::VariantNames; use strum_macros::{AsRefStr, EnumString, EnumVariantNames}; @@ -135,6 +138,24 @@ fn find_pantsd( Ok(pantsd_settings) } +fn execv_fallback_client(pants_server: OsString) -> Result { + let exe = PathBuf::from(pants_server.clone()); + let c_exe = CString::new(exe.into_os_string().into_vec()) + .expect("Failed to convert executable to a C string."); + + let mut c_args = vec![c_exe.clone()]; + c_args.extend( + std::env::args_os() + .skip(1) + .map(|arg| CString::new(arg.into_vec()).expect("Failed to convert argument to a C string.")), + ); + + execv(&c_exe, &c_args).map_err(|errno| { + eprintln!("Failed to exec pants at {pants_server:?}: {}", errno.desc()); + 1 + }) +} + // The value is taken from this C precedent: // ``` // $ grep 75 /usr/include/sysexits.h @@ -142,16 +163,32 @@ fn find_pantsd( // ``` const EX_TEMPFAIL: i32 = 75; +// An environment variable which if set, points to a non-native entrypoint to fall back to if +// `pantsd` is not already running with the appropriate fingerprint. +// +// This environment variable constitutes a public API used by `scie-pants` and the `pants` script. +// But in future, the native client may become the only client for `pantsd` (by directly handling +// forking the `pantsd` process and then connecting to it). +const PANTS_SERVER_EXE: &str = "_PANTS_SERVER_EXE"; + #[tokio::main] async fn main() { let start = SystemTime::now(); - match execute(start).await { - Err(err) => { + let pants_server = env::var_os(PANTS_SERVER_EXE); + match (execute(start).await, pants_server) { + (Err(_), Some(pants_server)) => { + // We failed to connect to `pantsd`, but a server variable was provided. Fall back + // to `execv`'ing the legacy Python client, which will handle spawning `pantsd`. + if let Err(exit_code) = execv_fallback_client(pants_server) { + std::process::exit(exit_code); + } + } + (Err(err), None) => { eprintln!("{err}"); // We use this exit code to indicate an error running pants via the nailgun protocol to // differentiate from a successful nailgun protocol session. std::process::exit(EX_TEMPFAIL); } - Ok(exit_code) => std::process::exit(exit_code), + (Ok(exit_code), _) => std::process::exit(exit_code), } } diff --git a/src/rust/engine/options/src/build_root.rs b/src/rust/engine/options/src/build_root.rs index 5cf2fc2ef8b..513228bff6e 100644 --- a/src/rust/engine/options/src/build_root.rs +++ b/src/rust/engine/options/src/build_root.rs @@ -11,7 +11,7 @@ use log::debug; pub struct BuildRoot(PathBuf); impl BuildRoot { - const SENTINEL_FILES: &'static [&'static str] = &["pants", "BUILDROOT", "BUILD_ROOT"]; + const SENTINEL_FILES: &'static [&'static str] = &["pants.toml", "BUILDROOT", "BUILD_ROOT"]; pub fn find() -> Result { let cwd = env::current_dir().map_err(|e| format!("Failed to determine $CWD: {e}"))?; diff --git a/src/rust/engine/options/src/build_root_tests.rs b/src/rust/engine/options/src/build_root_tests.rs index af60d57b390..75d9a05ca7f 100644 --- a/src/rust/engine/options/src/build_root_tests.rs +++ b/src/rust/engine/options/src/build_root_tests.rs @@ -32,7 +32,7 @@ fn test_find_cwd() { assert_sentinel("BUILDROOT"); assert_sentinel("BUILD_ROOT"); - assert_sentinel("pants"); + assert_sentinel("pants.toml"); } #[test] @@ -44,7 +44,7 @@ fn test_find_subdir() { assert!(BuildRoot::find_from(&buildroot_path).is_err()); assert!(BuildRoot::find_from(&subdir).is_err()); - let sentinel = &buildroot.path().join("pants"); + let sentinel = &buildroot.path().join("pants.toml"); fs::write(sentinel, []).unwrap(); assert_eq!( &buildroot_path,