diff --git a/common/network-common/src/test/java/org/apache/spark/network/TestUtils.java b/common/network-common/src/test/java/org/apache/spark/network/TestUtils.java index 56a2b805f154c..c2c5ffa43e0ed 100644 --- a/common/network-common/src/test/java/org/apache/spark/network/TestUtils.java +++ b/common/network-common/src/test/java/org/apache/spark/network/TestUtils.java @@ -22,7 +22,9 @@ public class TestUtils { public static String getLocalHost() { try { - return InetAddress.getLocalHost().getHostAddress(); + return (System.getenv().containsKey("SPARK_LOCAL_IP"))? + System.getenv("SPARK_LOCAL_IP"): + InetAddress.getLocalHost().getHostAddress(); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala index f8ef0d08d829c..64ae21bc6ee05 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala @@ -28,7 +28,8 @@ import scala.concurrent.duration._ import com.google.common.io.Files import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.server.MiniYARNCluster -import org.scalatest.{BeforeAndAfterAll, Matchers} +import org.scalactic.source.Position +import org.scalatest.{BeforeAndAfterAll, Matchers, Tag} import org.scalatest.concurrent.Eventually._ import org.apache.spark._ @@ -40,6 +41,7 @@ import org.apache.spark.util.Utils abstract class BaseYarnClusterSuite extends SparkFunSuite with BeforeAndAfterAll with Matchers with Logging { + private var isBindSuccessful = true // log4j configuration for the YARN containers, so that their output is collected // by YARN instead of trying to overwrite unit-tests.log. @@ -63,6 +65,14 @@ abstract class BaseYarnClusterSuite def newYarnConfig(): YarnConfiguration + override protected def test(testName: String, testTags: Tag*)(testFun: => Any) + (implicit pos: Position): Unit = { + super.test(testName, testTags: _*) { + assume(isBindSuccessful, "Mini Yarn cluster should be able to bind.") + testFun + } + } + override def beforeAll(): Unit = { super.beforeAll() @@ -79,9 +89,16 @@ abstract class BaseYarnClusterSuite yarnConf.set("yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage", "100.0") - yarnCluster = new MiniYARNCluster(getClass().getName(), 1, 1, 1) - yarnCluster.init(yarnConf) - yarnCluster.start() + try { + yarnCluster = new MiniYARNCluster(getClass().getName(), 1, 1, 1) + yarnCluster.init(yarnConf) + yarnCluster.start() + } catch { + case e: Throwable if org.apache.commons.lang3.exception.ExceptionUtils.indexOfThrowable( + e, classOf[java.net.BindException]) != -1 => + isBindSuccessful = false + return + } // There's a race in MiniYARNCluster in which start() may return before the RM has updated // its address in the configuration. You can see this in the logs by noticing that when @@ -117,7 +134,7 @@ abstract class BaseYarnClusterSuite override def afterAll(): Unit = { try { - yarnCluster.stop() + if (yarnCluster != null) yarnCluster.stop() } finally { super.afterAll() }