Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SPARK-28704][SQL][TEST] Add back Skiped HiveExternalCatalogVersionsSuite in HiveSparkSubmitSuite at JDK9+ #30451

Closed
wants to merge 9 commits into from
Closed

[SPARK-28704][SQL][TEST] Add back Skiped HiveExternalCatalogVersionsSuite in HiveSparkSubmitSuite at JDK9+ #30451

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d553765
Update HiveExternalCatalogVersionsSuite.scala
AngersZhuuuu Nov 19, 2020
b9dd1e4
Update HiveExternalCatalogVersionsSuite.scala
AngersZhuuuu Nov 20, 2020
9400d9a
Merge branch 'master' into SPARK-28704
AngersZhuuuu Nov 21, 2020
67222ca
Update package.scala
AngersZhuuuu Nov 21, 2020
053974a
with extra Deps
AngersZhuuuu Nov 21, 2020
668db8b
Update package.scala
AngersZhuuuu Nov 21, 2020
b853f5e
test
AngersZhuuuu Nov 21, 2020
c4543ef
hive version change with jdk version
AngersZhuuuu Nov 22, 2020
568150a
Update IsolatedClientLoader.scala
AngersZhuuuu Nov 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 13 additions & 14 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,18 @@ import org.apache.spark.util.Utils
@ExtendedHiveTest
class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
import HiveExternalCatalogVersionsSuite._
private val isTestAtLeastJava9 = SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)
private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse")
private val tmpDataDir = Utils.createTempDir(namePrefix = "test-data")
// For local test, you can set `spark.test.cache-dir` to a static value like `/tmp/test-spark`, to
// avoid downloading Spark of different versions in each run.
private val sparkTestingDir = Option(System.getProperty(SPARK_TEST_CACHE_DIR_SYSTEM_PROPERTY))
.map(new File(_)).getOrElse(Utils.createTempDir(namePrefix = "test-spark"))
private val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
val hiveVersion = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) {
"2.3.7"
} else {
"1.2.1"
}

override def afterAll(): Unit = {
try {
Expand Down Expand Up @@ -149,7 +153,9 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
new String(Files.readAllBytes(contentPath), StandardCharsets.UTF_8)
}

private def prepare(): Unit = {
override def beforeAll(): Unit = {
super.beforeAll()

val tempPyFile = File.createTempFile("test", ".py")
// scalastyle:off line.size.limit
Files.write(tempPyFile.toPath,
Expand Down Expand Up @@ -199,7 +205,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
"--master", "local[2]",
"--conf", s"${UI_ENABLED.key}=false",
"--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false",
"--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=1.2.1",
"--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveVersion",
"--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven",
"--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}",
"--conf", s"spark.sql.test.version.index=$index",
Expand All @@ -211,23 +217,14 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
tempPyFile.delete()
}

override def beforeAll(): Unit = {
super.beforeAll()
if (!isTestAtLeastJava9) {
prepare()
}
}

test("backward compatibility") {
// TODO SPARK-28704 Test backward compatibility on JDK9+ once we have a version supports JDK9+
assume(!isTestAtLeastJava9)
val args = Seq(
"--class", PROCESS_TABLES.getClass.getName.stripSuffix("$"),
"--name", "HiveExternalCatalog backward compatibility test",
"--master", "local[2]",
"--conf", s"${UI_ENABLED.key}=false",
"--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false",
"--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=1.2.1",
"--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveVersion",
"--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven",
"--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}",
"--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath}",
Expand All @@ -252,7 +249,9 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils {
// do not throw exception during object initialization.
case NonFatal(_) => Seq("3.0.1", "2.4.7") // A temporary fallback to use a specific version
}
versions.filter(v => v.startsWith("3") || !TestUtils.isPythonVersionAtLeast38())
versions
.filter(v => v.startsWith("3") || !TestUtils.isPythonVersionAtLeast38())
.filter(v => v.startsWith("3") || !SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9))
}

protected var spark: SparkSession = _
Expand Down