Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SPARK-1455: Better isolation for unit tests. #420

Closed
wants to merge 1 commit into from
Closed

SPARK-1455: Better isolation for unit tests. #420

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 25 additions & 7 deletions dev/run-tests
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,27 @@ cd $FWDIR
# Remove work directory
rm -rf ./work

# Fail fast
set -e
set -o pipefail
if test -x "$JAVA_HOME/bin/java"; then
declare java_cmd="$JAVA_HOME/bin/java"
else
declare java_cmd=java
fi
JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
[ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8."
export SPARK_HIVE=true

# Partial solution for SPARK-1455. Only run Hive tests if there are sql changes.
if [ -n "$AMPLAB_JENKINS" ]; then
git fetch origin master:master
diffs=`git diff --dirstat master | awk '{ print $2; }' | grep "^sql/"`
if [ -n "$diffs" ]; then
echo "Detected changes in SQL. Will run Hive test suite."
run_sql_tests=true
fi
fi

# Fail fast
set -e
set -o pipefail

echo "========================================================================="
echo "Running Apache RAT checks"
Expand All @@ -49,9 +59,17 @@ dev/scalastyle
echo "========================================================================="
echo "Running Spark unit tests"
echo "========================================================================="
# echo "q" is needed because sbt on encountering a build file with failure (either resolution or compilation)
# prompts the user for input either q, r, etc to quit or retry. This echo is there to make it not block.
echo -e "q\n" | sbt/sbt assembly test | grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
# echo "q" is needed because sbt on encountering a build file with failure
# (either resolution or compilation) prompts the user for input either q, r,
# etc to quit or retry. This echo is there to make it not block.
echo -e "q\n" | SPARK_HIVE=true sbt/sbt assembly | \
grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"

if [ -n "$run_sql_tests" ]; then
echo -e "q\n" | SPARK_HIVE=true sbt/sbt test | grep -v -e "info.*Resolving"
else
echo -e "q\n" | sbt/sbt test | grep -v -e "info.*Resolving"
fi

echo "========================================================================="
echo "Running PySpark tests"
Expand Down