Enable dependency graph traversal for determining the set of tests to…

… run on Ninja builds

Summary:
Enable dependency graph traversal for determining the set of tests to run on Ninja builds.
This is done by running "ninja -t deps" and "ninja -t commands" to extract the equivalent of
what depend.make and link.txt files provide in a Unix Makefiles CMake build.

Test Plan: Jenkins

Reviewers: timur, bogdan, sergei

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D5227

CMakeLists.txt

@@ -471,6 +471,10 @@ if (NOT "$ENV{YB_DISABLE_LATEST_SYMLINK}" STREQUAL "1")
     COMMENT "Recreating the 'latest' symlink at '${LATEST_BUILD_SYMLINK_PATH}'")
 endif()
 
+add_custom_target(dummy_target ALL
+  cat /dev/null
+  COMMENT "Dummy target for dependency resolution testing")
+
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 include(CMakeParseArguments)
 

bin/remote_build.py

@@ -215,7 +215,7 @@ def main():
     for arg in ybd_args:
         remote_command += " {0}".format(shlex.quote(arg))
     print("Remote command: {0}".format(remote_command))
-    ssh_args = ['ssh', args.host, remote_command]
+    ssh_args = ['ssh', args.host, '-o', 'ControlMaster=no', remote_command]
     proc = subprocess.Popen(ssh_args, shell=False)
     proc.communicate()
     if proc.returncode != 0:

build-support/common-build-env.sh

@@ -690,7 +690,8 @@ set_cmake_build_type_and_compiler_type() {
     make_program=ninja
     if ! which ninja &>/dev/null; then
       if using_linuxbrew; then
-        make_program=$YB_LINUXBREW_DIR/bin/ninja
+        export YB_NINJA_PATH=$YB_LINUXBREW_DIR/bin/ninja
+        make_program=$YB_NINJA_PATH
       elif is_mac; then
         log "Did not find the 'ninja' executable, auto-installing ninja using Homebrew"
         brew install ninja
@@ -1055,8 +1056,9 @@ detect_linuxbrew() {
         candidates=( "${candidates[@]}" "$preferred_linuxbrew_dir" )
       fi
     elif is_jenkins; then
-      log "Warning: Linuxbrew directory referenced by '$version_for_jenkins_file' does not" \
-          "exist: '$preferred_linuxbrew_dir', will attempt to use other location."
+      fail "Warning: Linuxbrew directory referenced by '$version_for_jenkins_file' does not" \
+           "exist: '$preferred_linuxbrew_dir', refusing to proceed to prevent non-deterministic " \
+           "builds."
     fi
   elif is_jenkins; then
     log "Warning: '$version_for_jenkins_file' does not exist"

build-support/jenkins/build-and-test.sh

@@ -100,16 +100,26 @@ build_cpp_code() {
   #
   # We're explicitly disabling third-party rebuilding here as we've already built third-party
   # dependencies (or downloaded them, or picked an existing third-party directory) above.
+
+  local yb_build_args=(
+    --no-rebuild-thirdparty
+    --skip-java
+    "$BUILD_TYPE"
+  )
+
+  if using_ninja; then
+    # TODO: remove this code when it becomes clear why CMake sometimes gets re-run.
+    log "Building a dummy target to check if Ninja re-runs CMake (it should not)."
+    # The "-d explain" option will make Ninja explain why it is building a particular target.
+    time run_centralized_build_cmd "$YB_SRC_ROOT/yb_build.sh" $remote_opt \
+      --make-ninja-extra-args "-d explain" \
+      --target dummy_target \
+      "${yb_build_args[@]}"
+  fi
+
   time run_centralized_build_cmd "$YB_SRC_ROOT/yb_build.sh" $remote_opt \
-    --no-rebuild-thirdparty \
-    --skip-java \
-    "$BUILD_TYPE" 2>&1 | \
+    "${yb_build_args[@]}" 2>&1 | \
     filter_boring_cpp_build_output
-  if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
-    log "C++ build failed!"
-    # TODO: perhaps we shouldn't even try to run C++ tests in this case?
-    EXIT_STATUS=1
-  fi
 
   log "Finished building C++ code (see timing information above)"
 
@@ -157,11 +167,12 @@ readonly build_type
 BUILD_TYPE=$build_type
 readonly BUILD_TYPE
 
-set_cmake_build_type_and_compiler_type
-
 export YB_USE_NINJA=1
 log "YB_USE_NINJA=$YB_USE_NINJA"
 
+set_cmake_build_type_and_compiler_type
+log "YB_NINJA_PATH=${YB_NINJA_PATH:-undefined}"
+
 set_build_root --no-readonly
 
 set_common_test_paths
@@ -192,6 +203,11 @@ log "Finished running Python tests (see timing information above)"
 YB_BUILD_JAVA=${YB_BUILD_JAVA:-1}
 YB_BUILD_CPP=${YB_BUILD_CPP:-1}
 
+if [[ -z ${YB_RUN_AFFECTED_TESTS_ONLY:-} ]] && is_jenkins_phabricator_build; then
+  log "YB_RUN_AFFECTED_TESTS_ONLY is not set, and this is a Jenkins phabricator test." \
+      "Setting YB_RUN_AFFECTED_TESTS_ONLY=1 automatically."
+  export YB_RUN_AFFECTED_TESTS_ONLY=1
+fi
 export YB_RUN_AFFECTED_TESTS_ONLY=${YB_RUN_AFFECTED_TESTS_ONLY:-0}
 log "YB_RUN_AFFECTED_TESTS_ONLY=$YB_RUN_AFFECTED_TESTS_ONLY"
 
@@ -319,35 +335,15 @@ fi
 # We have a retry loop around CMake because it sometimes fails due to NFS unavailability.
 declare -i -r MAX_CMAKE_RETRIES=3
 declare -i cmake_attempt_index=1
-cmake_succeeded=false
 while true; do
-  if [[ $cmake_attempt_index -eq $MAX_CMAKE_RETRIES ]]; then
-    log "This is the last attempt (attempt $MAX_CMAKE_RETRIES), trying to run CMake on the " \
-        "build master host."
-    cmake_cmd_prefix="run_centralized_build_cmd"
-  else
-    cmake_cmd_prefix=""
-  fi
-  # No quotes around $cmake_cmd_prefix below on purpose, because we just want to run the yb_build.sh
-  # script locally if it is empty.
-  #
-  # The --no-remote parameter tells the yb_build.sh command to always run any compiler commands
-  # (in this case, those that CMake runs internally to test compiler capabilities) locally, without
-  # ssh-ing to remote compilation worker nodes.
-  #
-  # The optional $cmake_cmd_prefix (a call to our run_centralized_build_cmd bash function) is
-  # something different: if present, it will ensure that CMake itself runs on the our "central
-  # build master" machine, which is also in most cases the NFS server. This is only done as the
-  # last attempt. We will probably get away from the concept of a "central build machine" at some
-  # point.
-  if ( set -x
-       $cmake_cmd_prefix "$YB_SRC_ROOT/yb_build.sh" "$BUILD_TYPE" --cmake-only --no-remote ); then
+  # We run CMake on the "central build master" in case of a distributed build.
+  if run_centralized_build_cmd "$YB_SRC_ROOT/yb_build.sh" "$BUILD_TYPE" --cmake-only --no-remote
+  then
     log "CMake succeeded after attempt $cmake_attempt_index"
     break
   fi
   if [[ $cmake_attempt_index -eq $MAX_CMAKE_RETRIES ]]; then
-    log "CMake failed after $MAX_CMAKE_RETRIES attempts, giving up."
-    exit 1
+    fatal "CMake failed after $MAX_CMAKE_RETRIES attempts, giving up."
   fi
   heading "CMake failed at attempt $cmake_attempt_index, re-trying"
   let cmake_attempt_index+=1
@@ -445,6 +441,8 @@ if [[ ${YB_TRACK_REGRESSIONS:-} == "1" ]]; then
 
   cd "$YB_SRC_ROOT"
 fi
+# End of special logic for the regression tracking mode.
+
 build_cpp_code "$YB_SRC_ROOT"
 
 if [[ ${YB_TRACK_REGRESSIONS:-} == "1" ]]; then

python/yb/common_util.py

@@ -97,6 +97,10 @@ def convert_to_non_ninja_build_root(build_root):
     return os.path.join(directory, NINJA_BUILD_ROOT_PART_RE.sub('', basename))
 
 
+def is_ninja_build_root(build_root):
+    return build_root != convert_to_non_ninja_build_root(build_root)
+
+
 def get_bool_env_var(env_var_name):
     value = os.environ.get(env_var_name, None)
     if value is None: