SPARK-1455: Better isolation for unit tests.

This is a simple first step towards avoiding running the Hive tests
whenever possible.

Author: Patrick Wendell <pwendell@gmail.com>

Closes #420 from pwendell/test-isolation and squashes the following commits:

350c8af [Patrick Wendell] SPARK-1455: Better isolation for unit tests.
(cherry picked from commit 5aaf983)

Signed-off-by: Patrick Wendell <pwendell@gmail.com>

dev/run-tests

@@ -24,17 +24,27 @@ cd $FWDIR
 # Remove work directory
 rm -rf ./work
 
-# Fail fast
-set -e
-set -o pipefail
 if test -x "$JAVA_HOME/bin/java"; then
     declare java_cmd="$JAVA_HOME/bin/java"
 else 
     declare java_cmd=java
 fi
 JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
 [ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8."
-export SPARK_HIVE=true
+
+# Partial solution for SPARK-1455. Only run Hive tests if there are sql changes.
+if [ -n "$AMPLAB_JENKINS" ]; then
+  git fetch origin master:master
+  diffs=`git diff --dirstat master | awk '{ print $2; }' | grep "^sql/"`
+  if [ -n "$diffs" ]; then
+    echo "Detected changes in SQL. Will run Hive test suite."
+    run_sql_tests=true
+  fi
+fi
+
+# Fail fast
+set -e
+set -o pipefail
 
 echo "========================================================================="
 echo "Running Apache RAT checks"
@@ -49,9 +59,17 @@ dev/scalastyle
 echo "========================================================================="
 echo "Running Spark unit tests"
 echo "========================================================================="
-# echo "q" is needed because sbt on encountering a build file with failure (either resolution or compilation)
-# prompts the user for input either q, r, etc to quit or retry. This echo is there to make it not block.
-echo -e "q\n" | sbt/sbt assembly test |  grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
+# echo "q" is needed because sbt on encountering a build file with failure 
+# (either resolution or compilation) prompts the user for input either q, r, 
+# etc to quit or retry. This echo is there to make it not block.
+echo -e "q\n" | SPARK_HIVE=true sbt/sbt assembly | \
+  grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
+
+if [ -n "$run_sql_tests" ]; then
+  echo -e "q\n" | SPARK_HIVE=true sbt/sbt test | grep -v -e "info.*Resolving" 
+else
+  echo -e "q\n" | sbt/sbt test | grep -v -e "info.*Resolving" 
+fi
 
 echo "========================================================================="
 echo "Running PySpark tests"