Introduce a native pants client. (#11922)

* Introduce a native pants client. Currently the client only handles talking to a pantsd brought up by other means. The Pants repo ./pants script is updated to optionally use the native client and prop up pantsd using the python client as needed. Work towards #11831 # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Debug macOS test failure. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Tom's feedback. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Fix parser precedence. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Build engine and client together. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Fixup flag parsing precedence / string list gathering. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Use nailgun crate to manage the pantsd ng connection. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Tighten up test CWD lock impl. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Break tests out into seperate module files. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Restructure BuildRoot to be testable. Previously the reliance on CWD forced races across test modules which would require a Mutex shared across those modules. That's awkward and this arrangement provides for the important test coverage still. [ci skip-build-wheels] * Fix clippy lints. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Add best effort tty settings save & restore. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Use nix for termios. [ci skip-build-wheels] * Hack in an attempt at cross-platform process cmdline handling. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Switch from remoteprocess to sysinfo. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Use sysinfo exclusively to check pants process. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Make rust Process name extraction match Python. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Introduce ParseError to delay expensive error formatting. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Fix shell fmt and lint. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Add a client test. Refactor launch_pantsd into a test utility to support this. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Add parse_tests. This found a few bugs, particularly handling escape sequences for implicit adds. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Add Args tests. [ci skip-build-wheels] * Fix test string formatting. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] * Add Env tests. [ci skip-build-wheels] * Clean up Args tests. # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels]
pantsbuild · Jul 6, 2021 · 751fe01 · 751fe01
1 parent 2a496aa
commit 751fe01
Show file tree

Hide file tree

Showing 25 changed files with 2,433 additions and 9 deletions.
diff --git a/.gitignore b/.gitignore
@@ -54,3 +54,4 @@ GRTAGS
 GSYMS
 GTAGS
 .mypy_cache/
+/.pants
diff --git a/build-support/bin/rust/bootstrap_code.sh b/build-support/bin/rust/bootstrap_code.sh
@@ -35,16 +35,19 @@ readonly NATIVE_ENGINE_BINARY_PYO3="native_engine_pyo3.so"
 readonly NATIVE_ENGINE_RESOURCE="${REPO_ROOT}/src/python/pants/engine/internals/${NATIVE_ENGINE_BINARY}"
 readonly NATIVE_ENGINE_RESOURCE_PYO3="${REPO_ROOT}/src/python/pants/engine/internals/${NATIVE_ENGINE_BINARY_PYO3}"
 readonly NATIVE_ENGINE_RESOURCE_METADATA="${NATIVE_ENGINE_RESOURCE}.metadata"
+readonly NATIVE_CLIENT_PATH="${REPO_ROOT}/.pants"
+readonly NATIVE_CLIENT_TARGET="${NATIVE_ROOT}/target/${MODE}/pants"
 
 function _build_native_code() {
-  banner "Building native engine..."
+  banner "Building native code..."
   # NB: See Cargo.toml with regard to the `extension-module` features.
   "${REPO_ROOT}/cargo" build \
     --features=extension-module \
     --features=engine_pyo3/extension-module \
     ${MODE_FLAG} \
     -p engine \
-    -p engine_pyo3 || die
+    -p engine_pyo3 \
+    -p client || die
 }
 
 function bootstrap_native_code() {
@@ -66,7 +69,11 @@ function bootstrap_native_code() {
   if [[ -f "${NATIVE_ENGINE_RESOURCE_METADATA}" ]]; then
     engine_version_in_metadata="$(sed -n 's/^engine_version: //p' "${NATIVE_ENGINE_RESOURCE_METADATA}")"
   fi
-  if [[ ! -f "${NATIVE_ENGINE_RESOURCE}" || ! -f "${NATIVE_ENGINE_RESOURCE_PYO3}" || "${engine_version_calculated}" != "${engine_version_in_metadata}" ]]; then
+  if [[ ! -f "${NATIVE_ENGINE_RESOURCE}" || ! -f \
+    "${NATIVE_ENGINE_RESOURCE_PYO3}" || ! -f \
+    "${NATIVE_CLIENT_PATH}" || \
+    "${engine_version_calculated}" != "${engine_version_in_metadata}" ]]; then
+
     _build_native_code || die
 
     # If bootstrapping the native engine fails, don't attempt to run pants
@@ -80,15 +87,21 @@ function bootstrap_native_code() {
       die "Failed to build native engine, file missing at ${native_binary_pyo3}."
     fi
 
+    # If bootstrapping the native client fails, don't attempt to run pants afterwards.
+    if [[ ! -f "${NATIVE_CLIENT_TARGET}" ]]; then
+      die "Failed to build native client."
+    fi
+
     # Pick up Cargo.lock changes if any caused by the `cargo build`.
     engine_version_calculated="$(calculate_current_hash)"
 
     # Create the native engine resource.
     # NB: On Mac Silicon, for some reason, first removing the old native_engine.so is necessary to avoid the Pants
     #  process from being killed when recompiling.
-    rm -f "${NATIVE_ENGINE_RESOURCE}" "${NATIVE_ENGINE_RESOURCE_PYO3}"
+    rm -f "${NATIVE_ENGINE_RESOURCE}" "${NATIVE_ENGINE_RESOURCE_PYO3}" "${NATIVE_CLIENT_PATH}"
     cp "${native_binary}" "${NATIVE_ENGINE_RESOURCE}"
     cp "${native_binary_pyo3}" "${NATIVE_ENGINE_RESOURCE_PYO3}"
+    cp "${NATIVE_CLIENT_TARGET}" "${NATIVE_CLIENT_PATH}"
 
     # Create the accompanying metadata file.
     local -r metadata_file=$(mktemp -t pants.native_engine.metadata.XXXXXX)

diff --git a/pants b/pants
@@ -38,17 +38,31 @@ source "${HERE}/build-support/pants_venv"
 source "${HERE}/build-support/bin/rust/bootstrap_code.sh"
 
 function exec_pants_bare() {
-
-  PANTS_EXE="${HERE}/src/python/pants/bin/pants_loader.py"
+  PANTS_NATIVE_EXE="${HERE}/.pants"
+  PANTS_PY_EXE="${HERE}/src/python/pants/bin/pants_loader.py"
   PANTS_SRCPATH="${HERE}/src/python"
 
   # Redirect activation and native bootstrap to ensure that they don't interfere with stdout.
   activate_pants_venv 1>&2
   bootstrap_native_code 1>&2
 
+  if [ -n "${USE_NATIVE_PANTS}" ]; then
+    set +e
+    "${PANTS_NATIVE_EXE}" "$@"
+    result=$?
+    # N.B.: The native pants client currently relies on pantsd being up. If it's not, it will fail
+    # with exit code 75 (EX_TEMPFAIL in /usr/include/sysexits.h) and we should fall through to the
+    # python pants client which knows how to start up pantsd. This failure takes O(1ms); so has no
+    # appreciable impact on --no-pantsd runs.
+    if ((result != 75)); then
+      exit ${result}
+    fi
+    set -e
+  fi
+
   # shellcheck disable=SC2086
   PYTHONPATH="${PANTS_SRCPATH}:${PYTHONPATH}" RUNNING_PANTS_FROM_SOURCES=1 \
-    exec ${PANTS_PREPEND_ARGS:-} "$(venv_dir)/bin/python" "${PANTS_EXE}" "$@"
+    exec ${PANTS_PREPEND_ARGS:-} "$(venv_dir)/bin/python" "${PANTS_PY_EXE}" "$@"
 }
 
 exec_pants_bare "$@"
diff --git a/src/rust/engine/Cargo.lock b/src/rust/engine/Cargo.lock
diff --git a/src/rust/engine/Cargo.toml b/src/rust/engine/Cargo.toml
@@ -22,6 +22,7 @@ members = [
   "async_latch",
   "async_semaphore",
   "async_value",
+  "client",
   "concrete_time",
   "engine_pyo3",
   "fs",
@@ -60,6 +61,7 @@ default-members = [
   ".",
   "async_latch",
   "async_semaphore",
+  "client",
   "async_value",
   "concrete_time",
   "engine_pyo3",

diff --git a/src/rust/engine/client/Cargo.toml b/src/rust/engine/client/Cargo.toml
@@ -0,0 +1,30 @@
+[package]
+version = "0.0.1"
+edition = "2018"
+name = "client"
+authors = [ "Pants Build <pantsbuild@gmail.com>" ]
+publish = false
+
+[[bin]]
+name = "pants"
+path = "src/main.rs"
+
+[dependencies]
+env_logger = "0.5.4"
+futures = "0.3"
+libc = "0.2"
+log = "0.4"
+nailgun = { path = "../nailgun" }
+nix = "0.20"
+peg = "0.7"
+sha2 = "0.9"
+shellexpand = "2.1"
+strum = "0.20"
+strum_macros = "0.20"
+sysinfo = "0.17.1"
+tokio = { version = "1.4", features = ["rt-multi-thread", "macros", "net", "io-std", "io-util"] }
+toml = "0.5"
+uname = "0.1"
+
+[dev-dependencies]
+tempdir = "0.3"
diff --git a/src/rust/engine/client/src/build_root.rs b/src/rust/engine/client/src/build_root.rs
@@ -0,0 +1,64 @@
+// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+use std::env;
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+
+use log::debug;
+
+#[derive(Debug)]
+pub struct BuildRoot(PathBuf);
+
+impl BuildRoot {
+  const SENTINEL_FILES: &'static [&'static str] = &["pants", "BUILDROOT", "BUILD_ROOT"];
+
+  pub fn find() -> Result<BuildRoot, String> {
+    let cwd = env::current_dir().map_err(|e| format!("Failed to determine $CWD: {}", e))?;
+    Self::find_from(&cwd)
+  }
+
+  pub(crate) fn find_from(start: &Path) -> Result<BuildRoot, String> {
+    let mut build_root = start;
+    loop {
+      for sentinel in Self::SENTINEL_FILES {
+        let sentinel_path = build_root.join(sentinel);
+        if !sentinel_path.exists() {
+          continue;
+        }
+        let sentinel_path_metadata = sentinel_path.metadata().map_err(|e| {
+          format!(
+            "\
+            Failed to read metadata for {path} to determine if is a build root sentinel file: {err}\
+            ",
+            path = sentinel_path.display(),
+            err = e
+          )
+        })?;
+        if sentinel_path_metadata.is_file() {
+          let root = BuildRoot(build_root.to_path_buf());
+          debug!("Found {:?} starting search from {}.", root, start.display());
+          return Ok(root);
+        }
+      }
+
+      build_root = build_root.parent().ok_or(format!(
+        "\
+          No build root detected for the current directory of {cwd}. Pants detects the build root \
+          by looking for at least one file from {sentinel_files} in the cwd and its ancestors. If \
+          you have none of these files, you can create an empty file in your build root.\
+          ",
+        cwd = start.display(),
+        sentinel_files = Self::SENTINEL_FILES.join(", ")
+      ))?;
+    }
+  }
+}
+
+impl Deref for BuildRoot {
+  type Target = PathBuf;
+
+  fn deref(&self) -> &PathBuf {
+    &self.0
+  }
+}