From 85df50d33dd820a3034cab9815e5875ab8fe555d Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 17 Jan 2022 13:05:08 -0500 Subject: [PATCH] Modernize and consolidate JDKs usage across all stages of the build. Use JDK-17 as bundled JDK distribution to run tests Signed-off-by: Andriy Redko --- .../main/java/org/opensearch/gradle/Jdk.java | 2 +- .../opensearch/gradle/JdkDownloadPlugin.java | 24 ++++++++++++++++--- .../gradle/test/DistroTestPlugin.java | 4 ++-- .../gradle/JdkDownloadPluginTests.java | 2 +- buildSrc/version.properties | 6 ++--- libs/nio/build.gradle | 10 ++++++++ libs/ssl-config/build.gradle | 10 ++++++++ .../ssl/DefaultJdkTrustConfigTests.java | 4 +--- plugins/repository-hdfs/build.gradle | 4 ++++ qa/evil-tests/build.gradle | 8 +++++++ server/build.gradle | 6 +++++ server/licenses/joda-time-2.10.12.jar.sha1 | 1 + server/licenses/joda-time-2.10.4.jar.sha1 | 1 - .../common/time/DateUtilsTests.java | 8 +------ .../resources/provision/krb5.conf.template | 2 ++ 15 files changed, 71 insertions(+), 21 deletions(-) create mode 100644 server/licenses/joda-time-2.10.12.jar.sha1 delete mode 100644 server/licenses/joda-time-2.10.4.jar.sha1 diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java index b289a882455d7..792debe3f350a 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java @@ -49,7 +49,7 @@ public class Jdk implements Buildable, Iterable { private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList(Arrays.asList("aarch64", "x64")); - private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptopenjdk", "openjdk")); + private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk")); private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList( Arrays.asList("darwin", "freebsd", "linux", "mac", "windows") ); diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java index b8d4bcb007e8d..b9f3036f7aad9 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java @@ -46,7 +46,7 @@ import org.gradle.api.internal.artifacts.ArtifactAttributes; public class JdkDownloadPlugin implements Plugin { - + public static final String VENDOR_ADOPTIUM = "adoptium"; public static final String VENDOR_ADOPTOPENJDK = "adoptopenjdk"; public static final String VENDOR_OPENJDK = "openjdk"; @@ -108,7 +108,25 @@ private void setupRepository(Project project, Jdk jdk) { String repoUrl; String artifactPattern; - if (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK)) { + if (jdk.getVendor().equals(VENDOR_ADOPTIUM)) { + repoUrl = "https://github.com/adoptium/temurin" + jdk.getMajor() + "-binaries/releases/download/"; + // JDK updates are suffixed with 'U' (fe OpenJDK17U), whereas GA releases are not (fe OpenJDK17). + // To distinguish between those, the GA releases have only major version component (fe 17+32), + // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of + // version separator '.' should be enough. + artifactPattern = "jdk-" + + jdk.getBaseVersion() + + "+" + + jdk.getBuild() + + "/OpenJDK" + + jdk.getMajor() + + (jdk.getBaseVersion().contains(".") ? "U" : "") + + "-jdk_[classifier]_[module]_hotspot_" + + jdk.getBaseVersion() + + "_" + + jdk.getBuild() + + ".[ext]"; + } else if (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK)) { repoUrl = "https://api.adoptopenjdk.net/v3/binary/version/"; if (jdk.getMajor().equals("8")) { // legacy pattern for JDK 8 @@ -167,7 +185,7 @@ public static NamedDomainObjectContainer getContainer(Project project) { private static String dependencyNotation(Jdk jdk) { String platformDep = jdk.getPlatform().equals("darwin") || jdk.getPlatform().equals("mac") - ? (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK) ? "mac" : "osx") + ? (jdk.getVendor().equals(VENDOR_OPENJDK) ? "osx" : "mac") : jdk.getPlatform(); String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz"; diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index 8f66e263dcdd8..81b51f7f1f9ff 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -76,8 +76,8 @@ public class DistroTestPlugin implements Plugin { private static final String SYSTEM_JDK_VERSION = "8u242+b08"; private static final String SYSTEM_JDK_VENDOR = "adoptopenjdk"; - private static final String GRADLE_JDK_VERSION = "14+36@076bab302c7b4508975440c56f6cc26a"; - private static final String GRADLE_JDK_VENDOR = "openjdk"; + private static final String GRADLE_JDK_VERSION = "17.0.1+12"; + private static final String GRADLE_JDK_VENDOR = "adoptium"; // all distributions used by distro tests. this is temporary until tests are per distribution private static final String EXAMPLE_PLUGIN_CONFIGURATION = "examplePlugin"; diff --git a/buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java b/buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java index 1c96d5986c5fd..4dcc65cca4c62 100644 --- a/buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java +++ b/buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java @@ -60,7 +60,7 @@ public void testUnknownVendor() { "11.0.2+33", "linux", "x64", - "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]" + "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptium, adoptopenjdk, openjdk]" ); } diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 3f366f8e2fb3e..f8f297bfe06b0 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,8 +1,8 @@ opensearch = 1.3.0 lucene = 8.10.1 -bundled_jdk_vendor = adoptopenjdk -bundled_jdk = 15.0.1+9 +bundled_jdk_vendor = adoptium +bundled_jdk = 17.0.1+12 @@ -20,7 +20,7 @@ slf4j = 1.6.2 jna = 5.5.0 netty = 4.1.72.Final -joda = 2.10.4 +joda = 2.10.12 # when updating this version, you need to ensure compatibility with: # - plugins/ingest-attachment (transitive dependency, check the upstream POM) diff --git a/libs/nio/build.gradle b/libs/nio/build.gradle index 794a544607c20..cae9f7e6feb26 100644 --- a/libs/nio/build.gradle +++ b/libs/nio/build.gradle @@ -27,6 +27,9 @@ * specific language governing permissions and limitations * under the License. */ + +import org.opensearch.gradle.info.BuildParams + apply plugin: 'opensearch.publish' dependencies { @@ -47,3 +50,10 @@ tasks.named('forbiddenApisMain').configure { // es-all is not checked as we connect and accept sockets replaceSignatureFiles 'jdk-signatures' } + +tasks.test { + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { + jvmArgs += ["--add-opens", "java.base/java.nio.channels=ALL-UNNAMED"] + jvmArgs += ["--add-opens", "java.base/java.net=ALL-UNNAMED"] + } +} diff --git a/libs/ssl-config/build.gradle b/libs/ssl-config/build.gradle index 0186f30696a6e..740d5e309350c 100644 --- a/libs/ssl-config/build.gradle +++ b/libs/ssl-config/build.gradle @@ -27,6 +27,9 @@ * specific language governing permissions and limitations * under the License. */ + +import org.opensearch.gradle.info.BuildParams + apply plugin: "opensearch.publish" dependencies { @@ -52,3 +55,10 @@ forbiddenPatterns { exclude '**/*.p12' exclude '**/*.jks' } + +tasks.test { + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { + jvmArgs += ["--add-opens", "java.base/java.security.cert=ALL-UNNAMED"] + } +} + diff --git a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java index bc83f00575481..e767787d67d6d 100644 --- a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java +++ b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java @@ -68,10 +68,8 @@ private void assertStandardIssuers(X509ExtendedTrustManager trustManager) { assertThat(trustManager.getAcceptedIssuers(), not(emptyArray())); // This is a sample of the CAs that we expect on every JRE. // We can safely change this list if the JRE's issuer list changes, but we want to assert something useful. - assertHasTrustedIssuer(trustManager, "VeriSign"); - assertHasTrustedIssuer(trustManager, "GeoTrust"); + // - https://bugs.openjdk.java.net/browse/JDK-8215012: VeriSign, GeoTrust" and "thawte" are gone assertHasTrustedIssuer(trustManager, "DigiCert"); - assertHasTrustedIssuer(trustManager, "thawte"); assertHasTrustedIssuer(trustManager, "COMODO"); } diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index f5f619cc075fb..b4f0e8405b840 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -228,6 +228,10 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec ) } } + + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { + jvmArgs += ["--add-opens", "java.security.jgss/sun.security.krb5=ALL-UNNAMED"] + } } testClusters."${integTestTaskName}" { diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle index dde39e56df9df..691115864de16 100644 --- a/qa/evil-tests/build.gradle +++ b/qa/evil-tests/build.gradle @@ -33,6 +33,8 @@ * integration, change default filesystem impl, mess with arbitrary * threads, etc. */ + +import org.opensearch.gradle.info.BuildParams apply plugin: 'opensearch.testclusters' apply plugin: 'opensearch.standalone-test' @@ -61,3 +63,9 @@ thirdPartyAudit { 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1' ) } + +tasks.test { + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { + jvmArgs += ["--add-opens", "java.base/java.lang=ALL-UNNAMED"] + } +} diff --git a/server/build.gradle b/server/build.gradle index 2efb9ae9d9f7b..44b88754312ac 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -353,3 +353,9 @@ tasks.named("licenseHeaders").configure { excludes << 'org/apache/lucene/search/RegexpQuery87*' excludes << 'org/opensearch/client/documentation/placeholder.txt' } + +tasks.test { + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { + jvmArgs += ["--add-opens", "java.base/java.nio.file=ALL-UNNAMED"] + } +} diff --git a/server/licenses/joda-time-2.10.12.jar.sha1 b/server/licenses/joda-time-2.10.12.jar.sha1 new file mode 100644 index 0000000000000..538f23152f69d --- /dev/null +++ b/server/licenses/joda-time-2.10.12.jar.sha1 @@ -0,0 +1 @@ +95b3f193ad0493d94dcd7daa9ea575c30e6be5f5 \ No newline at end of file diff --git a/server/licenses/joda-time-2.10.4.jar.sha1 b/server/licenses/joda-time-2.10.4.jar.sha1 deleted file mode 100644 index 878998e54db2b..0000000000000 --- a/server/licenses/joda-time-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8c10bb8815109067ce3c91a8e547b5a52e8a1c1a \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java b/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java index 7897be8f29e31..95786a130106d 100644 --- a/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java +++ b/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java @@ -58,13 +58,7 @@ import static org.hamcrest.Matchers.is; public class DateUtilsTests extends OpenSearchTestCase { - private static final Set IGNORE = new HashSet<>( - Arrays.asList( - "Eire", - "Europe/Dublin", // dublin timezone in joda does not account for DST - "Asia/Qostanay" // this has been added in joda 2.10.2 but is not part of the JDK 12.0.1 tzdata yet - ) - ); + private static final Set IGNORE = new HashSet<>(Arrays.asList("Pacific/Enderbury", "Pacific/Kanton", "Pacific/Niue")); public void testTimezoneIds() { assertNull(DateUtils.dateTimeZoneToZoneId(null)); diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template index 57fd8ac849708..ba0832b2b7d99 100644 --- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template +++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template @@ -43,6 +43,8 @@ # udp_preference_limit = 1 kdc_timeout = 3000 canonicalize = true + # See please https://seanjmullan.org/blog/2021/09/14/jdk17 (deprecate 3DES and RC4 in Kerberos) + allow_weak_crypto = true [realms] ${REALM_NAME} = {