[WIP][test-java11] Test Hadoop 2.7 with JDK 11

dev/deps/spark-deps-hadoop-3.2-hive-2.3

@@ -3,14 +3,17 @@ JLargeArrays-1.5.jar
 JTransforms-3.1.jar
 RoaringBitmap-0.7.45.jar
 ST4-4.0.4.jar
-accessors-smart-1.2.jar
 activation-1.1.1.jar
 aircompressor-0.10.jar
 algebra_2.12-2.0.0-M2.jar
 antlr-runtime-3.5.2.jar
 antlr4-runtime-4.7.1.jar
 aopalliance-1.0.jar
 aopalliance-repackaged-2.5.0.jar
+apacheds-i18n-2.0.0-M15.jar
+apacheds-kerberos-codec-2.0.0-M15.jar
+api-asn1-api-1.0.0-M20.jar
+api-util-1.0.0-M20.jar
 arpack_combined_all-0.1.jar
 arrow-format-0.15.1.jar
 arrow-memory-0.15.1.jar
@@ -32,10 +35,10 @@ commons-codec-1.10.jar
 commons-collections-3.2.2.jar
 commons-compiler-3.0.15.jar
 commons-compress-1.8.1.jar
-commons-configuration2-2.1.1.jar
+commons-configuration-1.6.jar
 commons-crypto-1.0.0.jar
-commons-daemon-1.0.13.jar
 commons-dbcp-1.4.jar
+commons-digester-1.8.jar
 commons-httpclient-3.1.jar
 commons-io-2.4.jar
 commons-lang-2.6.jar
@@ -47,37 +50,35 @@ commons-pool-1.5.4.jar
 commons-text-1.6.jar
 compress-lzf-1.0.3.jar
 core-1.1.2.jar
-curator-client-2.13.0.jar
-curator-framework-2.13.0.jar
-curator-recipes-2.13.0.jar
+curator-client-2.7.1.jar
+curator-framework-2.7.1.jar
+curator-recipes-2.7.1.jar
 datanucleus-api-jdo-4.2.4.jar
 datanucleus-core-4.1.17.jar
 datanucleus-rdbms-4.1.19.jar
 derby-10.12.1.1.jar
-dnsjava-2.1.7.jar
 dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
-ehcache-3.3.1.jar
 flatbuffers-java-1.9.0.jar
 generex-1.0.2.jar
-geronimo-jcache_1.0_spec-1.0-alpha-1.jar
 gson-2.2.4.jar
 guava-14.0.1.jar
-guice-4.0.jar
-guice-servlet-4.0.jar
-hadoop-annotations-3.2.0.jar
-hadoop-auth-3.2.0.jar
-hadoop-client-3.2.0.jar
-hadoop-common-3.2.0.jar
-hadoop-hdfs-client-3.2.0.jar
-hadoop-mapreduce-client-common-3.2.0.jar
-hadoop-mapreduce-client-core-3.2.0.jar
-hadoop-mapreduce-client-jobclient-3.2.0.jar
-hadoop-yarn-api-3.2.0.jar
-hadoop-yarn-client-3.2.0.jar
-hadoop-yarn-common-3.2.0.jar
-hadoop-yarn-registry-3.2.0.jar
-hadoop-yarn-server-common-3.2.0.jar
-hadoop-yarn-server-web-proxy-3.2.0.jar
+guice-3.0.jar
+guice-servlet-3.0.jar
+hadoop-annotations-2.7.4.jar
+hadoop-auth-2.7.4.jar
+hadoop-client-2.7.4.jar
+hadoop-common-2.7.4.jar
+hadoop-hdfs-2.7.4.jar
+hadoop-mapreduce-client-app-2.7.4.jar
+hadoop-mapreduce-client-common-2.7.4.jar
+hadoop-mapreduce-client-core-2.7.4.jar
+hadoop-mapreduce-client-jobclient-2.7.4.jar
+hadoop-mapreduce-client-shuffle-2.7.4.jar
+hadoop-yarn-api-2.7.4.jar
+hadoop-yarn-client-2.7.4.jar
+hadoop-yarn-common-2.7.4.jar
+hadoop-yarn-server-common-2.7.4.jar
+hadoop-yarn-server-web-proxy-2.7.4.jar
 hive-beeline-2.3.6.jar
 hive-cli-2.3.6.jar
 hive-common-2.3.6.jar
@@ -96,7 +97,7 @@ hive-vector-code-gen-2.3.6.jar
 hk2-api-2.5.0.jar
 hk2-locator-2.5.0.jar
 hk2-utils-2.5.0.jar
-htrace-core4-4.1.0-incubating.jar
+htrace-core-3.1.0-incubating.jar
 httpclient-4.5.6.jar
 httpcore-4.4.10.jar
 istack-commons-runtime-3.0.8.jar
@@ -106,12 +107,12 @@ jackson-core-2.10.0.jar
 jackson-core-asl-1.9.13.jar
 jackson-databind-2.10.0.jar
 jackson-dataformat-yaml-2.10.0.jar
-jackson-jaxrs-base-2.9.5.jar
-jackson-jaxrs-json-provider-2.9.5.jar
+jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
 jackson-module-jaxb-annotations-2.10.0.jar
 jackson-module-paranamer-2.10.0.jar
 jackson-module-scala_2.12-2.10.0.jar
+jackson-xc-1.9.13.jar
 jakarta.activation-api-1.2.1.jar
 jakarta.annotation-api-1.3.4.jar
 jakarta.inject-2.5.0.jar
@@ -123,9 +124,8 @@ javax.inject-1.jar
 javax.jdo-3.2.0-m3.jar
 javax.servlet-api-3.1.0.jar
 javolution-5.5.1.jar
-jaxb-api-2.2.11.jar
+jaxb-api-2.2.2.jar
 jaxb-runtime-2.3.2.jar
-jcip-annotations-1.0-1.jar
 jcl-over-slf4j-1.7.16.jar
 jdo-api-3.0.1.jar
 jersey-client-2.29.jar
@@ -135,14 +135,14 @@ jersey-container-servlet-core-2.29.jar
 jersey-hk2-2.29.jar
 jersey-media-jaxb-2.29.jar
 jersey-server-2.29.jar
-jetty-webapp-9.4.18.v20190429.jar
-jetty-xml-9.4.18.v20190429.jar
+jetty-6.1.26.jar
+jetty-sslengine-6.1.26.jar
+jetty-util-6.1.26.jar
 jline-2.14.6.jar
 joda-time-2.10.5.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
 json-1.8.jar
-json-smart-2.3.jar
 json4s-ast_2.12-3.6.6.jar
 json4s-core_2.12-3.6.6.jar
 json4s-jackson_2.12-3.6.6.jar
@@ -151,20 +151,6 @@ jsp-api-2.1.jar
 jsr305-3.0.0.jar
 jta-1.1.jar
 jul-to-slf4j-1.7.16.jar
-kerb-admin-1.0.1.jar
-kerb-client-1.0.1.jar
-kerb-common-1.0.1.jar
-kerb-core-1.0.1.jar
-kerb-crypto-1.0.1.jar
-kerb-identity-1.0.1.jar
-kerb-server-1.0.1.jar
-kerb-simplekdc-1.0.1.jar
-kerb-util-1.0.1.jar
-kerby-asn1-1.0.1.jar
-kerby-config-1.0.1.jar
-kerby-pkix-1.0.1.jar
-kerby-util-1.0.1.jar
-kerby-xdr-1.0.1.jar
 kryo-shaded-4.0.2.jar
 kubernetes-client-4.6.1.jar
 kubernetes-model-4.6.1.jar
@@ -184,12 +170,9 @@ metrics-jmx-4.1.1.jar
 metrics-json-4.1.1.jar
 metrics-jvm-4.1.1.jar
 minlog-1.3.0.jar
-mssql-jdbc-6.2.1.jre7.jar
 netty-all-4.1.42.Final.jar
-nimbus-jose-jwt-4.41.1.jar
 objenesis-2.5.1.jar
 okapi-shade-0.4.2.jar
-okhttp-2.7.5.jar
 okhttp-3.12.0.jar
 okio-1.15.0.jar
 opencsv-2.3.jar
@@ -208,7 +191,6 @@ parquet-jackson-1.10.1.jar
 protobuf-java-2.5.0.jar
 py4j-0.10.8.1.jar
 pyrolite-4.30.jar
-re2j-1.1.jar
 scala-collection-compat_2.12-2.1.1.jar
 scala-compiler-2.12.10.jar
 scala-library-2.12.10.jar
@@ -225,17 +207,17 @@ spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12-0.17.0-M1.jar
 spire_2.12-0.17.0-M1.jar
+stax-api-1.0-2.jar
 stax-api-1.0.1.jar
-stax2-api-3.1.4.jar
 stream-2.9.6.jar
 super-csv-2.2.0.jar
-token-provider-1.0.1.jar
 transaction-api-1.1.jar
 univocity-parsers-2.8.3.jar
 validation-api-2.0.1.Final.jar
 velocity-1.5.jar
-woodstox-core-5.0.3.jar
 xbean-asm7-shaded-4.15.jar
+xercesImpl-2.9.1.jar
+xmlenc-0.52.jar
 xz-1.5.jar
 zjsonpatch-0.3.0.jar
 zookeeper-3.4.14.jar

external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala

@@ -17,14 +17,15 @@
 
 package org.apache.spark.sql.kafka010
 
-import java.io.{File, IOException}
+import java.io.{File, IOException, PrintWriter}
 import java.net.{InetAddress, InetSocketAddress}
 import java.nio.charset.StandardCharsets
 import java.util.{Collections, Properties, UUID}
 import java.util.concurrent.TimeUnit
 import javax.security.auth.login.Configuration
 
 import scala.collection.JavaConverters._
+import scala.io.Source
 import scala.util.Random
 
 import com.google.common.io.Files
@@ -136,9 +137,33 @@ class KafkaTestUtils(
     kdcConf.setProperty(MiniKdc.DEBUG, "true")
     kdc = new MiniKdc(kdcConf, kdcDir)
     kdc.start()
+    rewriteKrb5Conf()
     kdcReady = true
   }
 
+  private def rewriteKrb5Conf(): Unit = {
+    val krb5Conf = Source.fromFile(kdc.getKrb5conf, "UTF-8").getLines()
+    val rewriteKrb5Conf = krb5Conf.map(s =>
+      if (s.contains("libdefaults")) {
+        val addedConfig =
+          addedKrb5Config("default_tkt_enctypes", "aes128-cts-hmac-sha1-96") +
+            addedKrb5Config("default_tgs_enctypes", "aes128-cts-hmac-sha1-96")
+        s + addedConfig
+      } else {
+        s
+      })
+    kdc.getKrb5conf.delete()
+    val writer = new PrintWriter(kdc.getKrb5conf)
+    // scalastyle:off
+    rewriteKrb5Conf.foreach(writer.println)
+    // scalastyle:on
+    writer.close()
+  }
+
+  private def addedKrb5Config(key: String, value: String): String = {
+    System.lineSeparator() + s"    $key=$value"
+  }
+
   private def createKeytabsAndJaasConfigFile(): String = {
     assert(kdcReady, "KDC should be set up beforehand")
     val baseDir = Utils.createTempDir()
@@ -171,6 +196,7 @@ class KafkaTestUtils(
       |  useKeyTab=true
       |  storeKey=true
       |  useTicketCache=false
+      |  refreshKrb5Config=true
       |  keyTab="${zkServerKeytabFile.getAbsolutePath()}"
       |  principal="$zkServerUser@$realm";
       |};
@@ -180,6 +206,7 @@ class KafkaTestUtils(
       |  useKeyTab=true
       |  storeKey=true
       |  useTicketCache=false
+      |  refreshKrb5Config=true
       |  keyTab="${zkClientKeytabFile.getAbsolutePath()}"
       |  principal="$zkClientUser@$realm";
       |};

hadoop-cloud/pom.xml

@@ -202,7 +202,7 @@
      enables store-specific committers.
     -->
     <profile>
-      <id>hadoop-3.2</id>
+      <id>skip-hadoop-3.2-profile</id>
       <properties>
         <extra.source.dir>src/hadoop-3/main/scala</extra.source.dir>
         <extra.testsource.dir>src/hadoop-3/test/scala</extra.testsource.dir>

pom.xml

@@ -2918,8 +2918,8 @@
     <profile>
       <id>hadoop-3.2</id>
       <properties>
-        <hadoop.version>3.2.0</hadoop.version>
-        <curator.version>2.13.0</curator.version>
+        <hadoop.version>2.7.4</hadoop.version>
+        <curator.version>2.7.1</curator.version>
       </properties>
     </profile>