From 97b801d29cce88f36e1cb6e68b1c1a1aaa557e02 Mon Sep 17 00:00:00 2001
From: Frank Austin Nothaft <fnothaft@alumni.stanford.edu>
Date: Wed, 24 Aug 2016 08:32:19 -0700
Subject: [PATCH] [ADAM-1093] Move to support Spark 2.0.0.

Relies on https://github.com/bigdatagenomics/utils/pull/78. Resolves #1093:

* Clean up logging in VariantContextRDD
* Add move_to_spark_2.sh script and CI hooks
---
 .../rdd/variation/VariantContextRDD.scala     |  4 +-
 pom.xml                                       |  2 +-
 scripts/jenkins-test                          |  7 +++
 scripts/move_to_spark_2.sh                    |  8 ++++
 scripts/release/release.sh                    | 46 +++++++++++++++++--
 5 files changed, 61 insertions(+), 6 deletions(-)
 create mode 100755 scripts/move_to_spark_2.sh

diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/rdd/variation/VariantContextRDD.scala b/adam-core/src/main/scala/org/bdgenomics/adam/rdd/variation/VariantContextRDD.scala
index 4679ee18f5..5fdcc956ed 100644
--- a/adam-core/src/main/scala/org/bdgenomics/adam/rdd/variation/VariantContextRDD.scala
+++ b/adam-core/src/main/scala/org/bdgenomics/adam/rdd/variation/VariantContextRDD.scala
@@ -24,8 +24,7 @@ import htsjdk.variant.variantcontext.writer.{
 import htsjdk.variant.vcf.{ VCFHeader, VCFHeaderLine }
 import java.io.OutputStream
 import org.apache.hadoop.io.LongWritable
-import org.apache.hadoop.fs.{ FileSystem, Path }
-import org.apache.spark.Logging
+import org.apache.hadoop.fs.Path
 import org.apache.spark.rdd.RDD
 import org.bdgenomics.adam.converters.{
   SupportedHeaderLines,
@@ -38,6 +37,7 @@ import org.bdgenomics.adam.models.{
 }
 import org.bdgenomics.adam.rdd.{ FileMerger, MultisampleGenomicRDD }
 import org.bdgenomics.formats.avro.Sample
+import org.bdgenomics.utils.misc.Logging
 import org.bdgenomics.utils.cli.SaveArgs
 import org.seqdoop.hadoop_bam._
 import scala.collection.JavaConversions._
diff --git a/pom.xml b/pom.xml
index 3c397489a3..c084132bdd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
     <hadoop-bam.version>7.6.0</hadoop-bam.version>
     <slf4j.version>1.7.21</slf4j.version>
     <bdg-formats.version>0.9.0</bdg-formats.version>
-    <bdg-utils.version>0.2.7</bdg-utils.version>
+    <bdg-utils.version>0.2.8</bdg-utils.version>
     <htsjdk.version>2.5.0</htsjdk.version>
     <scoverage.plugin.version>1.1.1</scoverage.plugin.version>
   </properties>
diff --git a/scripts/jenkins-test b/scripts/jenkins-test
index 03f487a876..ec2bbf4ef5 100755
--- a/scripts/jenkins-test
+++ b/scripts/jenkins-test
@@ -38,6 +38,13 @@ then
     exit 1
 fi
 
+# are we testing for spark 2.0.0? if so, we need to rewrite our poms first
+if [ ${SPARK_VERSION} == 2.0.0 ];
+then
+        echo "Rewriting POM.xml files for Scala 2.10."
+        ./scripts/move_to_spark_2.sh
+fi
+
 # are we testing for scala 2.11? if so, we need to rewrite our poms to 2.11 first
 if [ ${SCALAVER} == 2.11 ];
 then
diff --git a/scripts/move_to_spark_2.sh b/scripts/move_to_spark_2.sh
new file mode 100755
index 0000000000..5b829e49cc
--- /dev/null
+++ b/scripts/move_to_spark_2.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set +x
+
+find . -name "pom.xml" -exec sed -e "/utils-/ s/_2.10/-spark2_2.10/g" \
+    -e "/adam-/ s/_2.10/-spark2_2.10/g" \
+    -e "/spark.version/ s/1.6.1/2.0.0/g" \
+    -i .spark2.bak '{}' \;
diff --git a/scripts/release/release.sh b/scripts/release/release.sh
index 720d7b2c9d..631fd22953 100755
--- a/scripts/release/release.sh
+++ b/scripts/release/release.sh
@@ -38,7 +38,7 @@ mvn --batch-mode \
   release:perform
 
 if [ $? != 0 ]; then
-  echo "Releasing Scala 2.10 version failed."
+  echo "Releasing Spark 1, Scala 2.10 version failed."
   exit 1
 fi
 
@@ -50,7 +50,7 @@ zip -r adam-${release}.zip adam-${release}.jar adam-${release}.pom
 # do scala 2.11 release
 git checkout -b maint_2.11-${release} ${branch}
 ./scripts/move_to_scala_2.11.sh
-git commit -a -m "Modifying pom.xml files for 2.11 release."
+git commit -a -m "Modifying pom.xml files for Spark 1, Scala 2.11 release."
 mvn --batch-mode \
   -P distribution \
   -Dresume=false \
@@ -63,7 +63,47 @@ mvn --batch-mode \
   release:perform
 
 if [ $? != 0 ]; then
-  echo "Releasing Scala 2.11 version failed."
+  echo "Releasing Spark 1, Scala 2.11 version failed."
+  exit 1
+fi
+
+# do spark 2, scala 2.11 release
+git checkout -b maint_spark2_2.11-${release} ${branch}
+./scripts/move_to_spark2.sh
+git commit -a -m "Modifying pom.xml files for Spark 2, Scala 2.11 release."
+mvn --batch-mode \
+  -P distribution \
+  -Dresume=false \
+  -Dtag=adam-parent-spark2_2.11-${release} \
+  -DreleaseVersion=${release} \
+  -DdevelopmentVersion=${devel} \
+  -DbranchName=adam-spark2_2.11-${release} \
+  release:clean \
+  release:prepare \
+  release:perform
+
+if [ $? != 0 ]; then
+  echo "Releasing Spark 2, Scala 2.11 version failed."
+  exit 1
+fi
+
+# do spark 2, scala 2.10 release
+git checkout -b maint_spark2_2.10-${release} ${branch}
+./scripts/move_to_scala_2.10.sh
+git commit -a -m "Modifying pom.xml files for Spark 2, Scala 2.10 release."
+mvn --batch-mode \
+  -P distribution \
+  -Dresume=false \
+  -Dtag=adam-parent-spark2_2.10-${release} \
+  -DreleaseVersion=${release} \
+  -DdevelopmentVersion=${devel} \
+  -DbranchName=adam-spark2_2.10-${release} \
+  release:clean \
+  release:prepare \
+  release:perform
+
+if [ $? != 0 ]; then
+  echo "Releasing Spark 2, Scala 2.10 version failed."
   exit 1
 fi