diff --git a/dev/run-tests b/dev/run-tests index 0725b681f1a1b..68059933f2795 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -24,9 +24,6 @@ cd $FWDIR # Remove work directory rm -rf ./work -# Fail fast -set -e -set -o pipefail if test -x "$JAVA_HOME/bin/java"; then declare java_cmd="$JAVA_HOME/bin/java" else @@ -34,7 +31,20 @@ else fi JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q') [ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8." -export SPARK_HIVE=true + +# Partial solution for SPARK-1455. Only run Hive tests if there are sql changes. +if [ -n "$AMPLAB_JENKINS" ]; then + git fetch origin master:master + diffs=`git diff --dirstat master | awk '{ print $2; }' | grep "^sql/"` + if [ -n "$diffs" ]; then + echo "Detected changes in SQL. Will run Hive test suite." + run_sql_tests=true + fi +fi + +# Fail fast +set -e +set -o pipefail echo "=========================================================================" echo "Running Apache RAT checks" @@ -49,9 +59,17 @@ dev/scalastyle echo "=========================================================================" echo "Running Spark unit tests" echo "=========================================================================" -# echo "q" is needed because sbt on encountering a build file with failure (either resolution or compilation) -# prompts the user for input either q, r, etc to quit or retry. This echo is there to make it not block. -echo -e "q\n" | sbt/sbt assembly test | grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including" +# echo "q" is needed because sbt on encountering a build file with failure +# (either resolution or compilation) prompts the user for input either q, r, +# etc to quit or retry. This echo is there to make it not block. +echo -e "q\n" | SPARK_HIVE=true sbt/sbt assembly | \ + grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including" + +if [ -n "$run_sql_tests" ]; then + echo -e "q\n" | SPARK_HIVE=true sbt/sbt test | grep -v -e "info.*Resolving" +else + echo -e "q\n" | sbt/sbt test | grep -v -e "info.*Resolving" +fi echo "=========================================================================" echo "Running PySpark tests"