From 0ad63f1dd0bc1eec873b3c03d1eb5cc92222b634 Mon Sep 17 00:00:00 2001 From: Frank Austin Nothaft Date: Sat, 17 Dec 2016 22:55:48 -0800 Subject: [PATCH] [ADAM-1317] Add interlocks for move_to_xyz scripts. Resolves #1317. --- scripts/jenkins-test | 84 +++++++++++++++++++++++++++++++---- scripts/move_to_scala_2.10.sh | 16 ++++++- scripts/move_to_scala_2.11.sh | 15 ++++++- scripts/move_to_spark_1.sh | 9 ++++ scripts/move_to_spark_2.sh | 9 ++++ 5 files changed, 120 insertions(+), 13 deletions(-) diff --git a/scripts/jenkins-test b/scripts/jenkins-test index 8cff05ea42..028c37d5a7 100755 --- a/scripts/jenkins-test +++ b/scripts/jenkins-test @@ -10,7 +10,7 @@ find . -name pom.xml \ -exec sed -i.bak \ -e "s:sun.io.serialization.extendedDebugInfo=true:sun.io.serialization.extendedDebugInfo=true -Djava.io.tmpdir=${ADAM_MVN_TMP_DIR}:g" \ {} \; -find . -name "*.bak" -exec rm {} \; +find . -name "*.bak" -exec rm -f {} \; # variable declarations export PATH=${JAVA_HOME}/bin/:${PATH} @@ -37,18 +37,59 @@ then exit 1 fi +# this next line is supposed to fail +set +e + +echo "Rewriting POM.xml files to Scala 2.10 and Spark 1 should error..." +./scripts/move_to_spark_1.sh +if [[ $? == 0 ]]; +then + echo "Running move_to_spark_1.sh when POMs are set up for Spark 1 should fail, but error code was 0 (success)." + exit 1 +fi + +./scripts/move_to_scala_2.10.sh +if [[ $? == 0 ]]; +then + echo "Running move_to_scala_2.10.sh when POMs are set up for Scala 2.10 should fail, but error code was 0 (success)." + exit 1 +fi + +set -e + # are we testing for spark 2.0.0? if so, we need to rewrite our poms first if [ ${SPARK_VERSION} == 2.0.0 ]; then - echo "Rewriting POM.xml files for Scala 2.10." - ./scripts/move_to_spark_2.sh + + echo "Rewriting POM.xml files for Spark 2." + ./scripts/move_to_spark_2.sh + + # shouldn't be able to move to spark 2 twice + set +e + ./scripts/move_to_spark_2.sh + if [[ $? == 0 ]]; + then + echo "We have already moved to Spark 2, so running move_to_spark_2.sh a second time should fail, but error code was 0 (success)." + exit 1 + fi + set -e fi # are we testing for scala 2.11? if so, we need to rewrite our poms to 2.11 first if [ ${SCALAVER} == 2.11 ]; then - echo "Rewriting POM.xml files for Scala 2.10." - ./scripts/move_to_scala_2.11.sh + echo "Rewriting POM.xml files for Scala 2.11." + ./scripts/move_to_scala_2.11.sh + + # shouldn't be able to move to scala 2.11 twice + set +e + ./scripts/move_to_scala_2.11.sh + if [[ $? == 0 ]]; + then + echo "We have already moved to Scala 2.11, so running move_to_scala_2.11.sh a second time should fail, but error code was 0 (success)." + exit 1 + fi + set -e fi # print versions @@ -77,6 +118,33 @@ mvn -U \ find ${ADAM_MVN_TMP_DIR} rm -rf ${ADAM_MVN_TMP_DIR} +# and move our poms back to their original values +# this will allow us to pass our porcelain test at the end +if [ ${SPARK_VERSION} == 2.0.0 ]; +then + + echo "Rewriting POM.xml files back to Spark 1." + ./scripts/move_to_spark_1.sh +fi +if [ ${SCALAVER} == 2.11 ]; +then + echo "Rewriting POM.xml files back to Scala 2.10." + ./scripts/move_to_scala_2.10.sh +fi + +find . -name pom.xml \ + -exec sed -i.bak \ + -e "s:sun.io.serialization.extendedDebugInfo=true -Djava.io.tmpdir=${ADAM_MVN_TMP_DIR}:sun.io.serialization.extendedDebugInfo=true:g" \ + {} \; +find . -name "*.bak" -exec rm -f {} \; + +if test -n "$(git status --porcelain)" +then + echo "Applying move_to_xyz script marred a pom.xml file." + echo "Exiting..." + exit 1 +fi + # run integration tests on scala 2.10; prebuilt spark distributions are not available for 2.11 if [ ${SCALAVER} == 2.10 ]; then @@ -155,17 +223,15 @@ then popd # test that the source is formatted correctly - # we had modified the poms to add a temp dir, so check those out first + # we had modified the poms to add a temp dir, so back out that modification first pushd ${PROJECT_ROOT} - find . -name pom.xml -exec git checkout {} \; ./scripts/format-source if test -n "$(git status --porcelain)" then echo "Please run './scripts/format-source'" exit 1 fi - popd - + popd fi echo diff --git a/scripts/move_to_scala_2.10.sh b/scripts/move_to_scala_2.10.sh index 7aa7a65783..72bcb643c4 100755 --- a/scripts/move_to_scala_2.10.sh +++ b/scripts/move_to_scala_2.10.sh @@ -2,5 +2,17 @@ set +x -find . -name "pom.xml" -exec sed -e "s/2.11.8/2.10.6/g" -e "s/2.11/2.10/g" -i.2.10.bak '{}' \; -find . -name "*.2.10.bak" -exec rm {} \; +grep "" pom.xml | grep -q 2.10 +if [[ $? == 0 ]]; +then + echo "Scala version is already set to 2.10 (Scala artifacts have _2.10 version suffix in artifact name)." + echo "Cowardly refusing to move to Scala 2.10 a second time..." + + exit 1 +fi + +find . -name "pom.xml" -exec sed -e "s/2.11.8/2.10.6/g" \ + -e "s/2.11/2.10/g" -i.2.10.bak \ + -e "/no Scala/ s/Scala 2.10/Scala 2.11/g" -i.2.10.bak \ + '{}' \; +find . -name "*.2.10.bak" -exec rm -f {} \; diff --git a/scripts/move_to_scala_2.11.sh b/scripts/move_to_scala_2.11.sh index b4daadff82..d443329e6b 100755 --- a/scripts/move_to_scala_2.11.sh +++ b/scripts/move_to_scala_2.11.sh @@ -2,9 +2,20 @@ set +x -find . -name "pom.xml" -exec sed -e "s/2.10.6/2.11.8/g" -e "/bdg-utils.version/!s/2.10/2.11/g" -i.2.11.bak '{}' \; +grep "" pom.xml | grep -q 2.11 +if [[ $? == 0 ]]; +then + echo "Scala version is already set to 2.11 (Scala artifacts have _2.11 version suffix in artifact name)." + echo "Cowardly refusing to move to Scala 2.11 a second time..." + + exit 1 +fi + +find . -name "pom.xml" -exec sed -e "s/2.10.6/2.11.8/g" \ + -e "/bdg-utils.version/! s/2.10/2.11/g" \ + -i.2.11.bak '{}' \; # keep parquet-scala at parquet-scala_2.10 find . -name "pom.xml" -exec sed -e "s/parquet-scala_2.11/parquet-scala_2.10/g" -i.2.11.2.bak '{}' \; # keep maven-javadoc-plugin at version 2.10.3 find . -name "pom.xml" -exec sed -e "s/2.11.3/2.10.3/g" -i.2.11.3.bak '{}' \; -find . -name "*.2.11.*bak" -exec rm {} \; +find . -name "*.2.11.*bak" -exec rm -f {} \; diff --git a/scripts/move_to_spark_1.sh b/scripts/move_to_spark_1.sh index 8c3ff9f548..4427317092 100755 --- a/scripts/move_to_spark_1.sh +++ b/scripts/move_to_spark_1.sh @@ -2,6 +2,15 @@ set +x +grep -q "spark2" pom.xml +if [[ $? == 1 ]]; +then + echo "POM is already set up for Spark 1 (Spark 1/2 artifacts are missing -spark2 suffix in artifact names)." + echo "Cowardly refusing to move to Spark 1 a second time..." + + exit 1 +fi + find . -name "pom.xml" -exec sed \ -e "s/-spark2//g" \ -e "/spark.version/ s/2.0.0/1.6.3/g" \ diff --git a/scripts/move_to_spark_2.sh b/scripts/move_to_spark_2.sh index fd8d79d1b6..863ab1365d 100755 --- a/scripts/move_to_spark_2.sh +++ b/scripts/move_to_spark_2.sh @@ -2,6 +2,15 @@ set +x +grep -q "spark2" pom.xml +if [[ $? == 0 ]]; +then + echo "POM is already set up for Spark 2 (Spark 1/2 artifacts have -spark2 suffix in artifact names)." + echo "Cowardly refusing to move to Spark 2 a second time..." + + exit 1 +fi + svp="\${scala.version.prefix}" substitution_cmd="s/_$svp/-spark2_$svp/g"