From 27c8bcbbd15339631892c11bd15bea9cb2d6d73b Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 22 Mar 2020 11:58:11 -0400 Subject: [PATCH] Introduce aarch64 packaging (#53914) (#53926) This commit introduces aarch64 packaging, including bundling an aarch64 JDK distribution. We had to make some interesting choices here: - ML binaries are not compiled for aarch64, so for now we disable ML on aarch64 - depending on underlying page sizes, we have to disable class data sharing --- buildSrc/build.gradle | 2 + .../elasticsearch/gradle/Architecture.java | 40 +++++++++++++++ .../java/org/elasticsearch/gradle/Jdk.java | 20 ++++++++ .../gradle/JdkDownloadPlugin.java | 35 +++++++++---- .../gradle/test/DistroTestPlugin.java | 6 ++- .../testclusters/TestClustersPlugin.java | 2 + .../gradle/JdkDownloadPluginTests.java | 51 ++++++++++++++++--- .../testKit/jdk-download/reuse/build.gradle | 1 + .../testKit/jdk-download/subproj/build.gradle | 3 ++ distribution/archives/build.gradle | 42 +++++++++------ .../archives/linux-aarch64-tar/build.gradle | 2 + .../oss-linux-aarch64-tar/build.gradle | 2 + distribution/build.gradle | 10 ++-- .../packages/aarch64-deb/build.gradle | 2 + .../packages/aarch64-oss-deb/build.gradle | 2 + .../packages/aarch64-oss-rpm/build.gradle | 2 + .../packages/aarch64-rpm/build.gradle | 2 + distribution/packages/build.gradle | 50 ++++++++++++------ distribution/src/bin/elasticsearch | 6 ++- distribution/src/bin/elasticsearch-cli | 1 + distribution/src/bin/elasticsearch-env | 8 ++- .../tools/launchers/JvmErgonomics.java | 11 ++-- gradle/runtime-jdk-provision.gradle | 2 + settings.gradle | 6 +++ .../xpack/core/XPackSettings.java | 11 +++- .../sql/src/main/bin/elasticsearch-sql-cli | 1 + 26 files changed, 258 insertions(+), 62 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java create mode 100644 distribution/archives/linux-aarch64-tar/build.gradle create mode 100644 distribution/archives/oss-linux-aarch64-tar/build.gradle create mode 100644 distribution/packages/aarch64-deb/build.gradle create mode 100644 distribution/packages/aarch64-oss-deb/build.gradle create mode 100644 distribution/packages/aarch64-oss-rpm/build.gradle create mode 100644 distribution/packages/aarch64-rpm/build.gradle diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 575031e4bf71e..8448fa8cb9b3f 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -190,8 +190,10 @@ if (project != rootProject) { distribution project(':distribution:archives:oss-windows-zip') distribution project(':distribution:archives:darwin-tar') distribution project(':distribution:archives:oss-darwin-tar') + distribution project(':distribution:archives:linux-aarch64-tar') distribution project(':distribution:archives:linux-tar') distribution project(':distribution:archives:oss-linux-tar') + distribution project(':distribution:archives:oss-linux-aarch64-tar') } // for external projects we want to remove the marker file indicating we are running the Elasticsearch project diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java new file mode 100644 index 0000000000000..f230d9af86e11 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle; + +public enum Architecture { + + X64, + AARCH64; + + public static Architecture current() { + final String architecture = System.getProperty("os.arch", ""); + switch (architecture) { + case "amd64": + case "x86_64": + return X64; + case "aarch64": + return AARCH64; + default: + throw new IllegalArgumentException("can not determine architecture from [" + architecture + "]"); + } + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java index 16fc59c4b4d3d..2672405bbc5c9 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java @@ -35,6 +35,7 @@ public class Jdk implements Buildable, Iterable { + private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList(Arrays.asList("aarch64", "x64")); private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptopenjdk", "openjdk")); private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("darwin", "linux", "windows", "mac")); private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"); @@ -46,6 +47,7 @@ public class Jdk implements Buildable, Iterable { private final Property vendor; private final Property version; private final Property platform; + private final Property architecture; private String baseVersion; private String major; private String build; @@ -57,6 +59,7 @@ public class Jdk implements Buildable, Iterable { this.vendor = objectFactory.property(String.class); this.version = objectFactory.property(String.class); this.platform = objectFactory.property(String.class); + this.architecture = objectFactory.property(String.class); } public String getName() { @@ -99,6 +102,19 @@ public void setPlatform(String platform) { this.platform.set(platform); } + public String getArchitecture() { + return architecture.get(); + } + + public void setArchitecture(final String architecture) { + if (ALLOWED_ARCHITECTURES.contains(architecture) == false) { + throw new IllegalArgumentException( + "unknown architecture [" + architecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES + ); + } + this.architecture.set(architecture); + } + public String getBaseVersion() { return baseVersion; } @@ -167,9 +183,13 @@ void finalizeValues() { if (vendor.isPresent() == false) { throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]"); } + if (architecture.isPresent() == false) { + throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]"); + } version.finalizeValue(); platform.finalizeValue(); vendor.finalizeValue(); + architecture.finalizeValue(); } @Override diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java index 11167d3d3e213..795b99c3dfcc6 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java @@ -47,7 +47,6 @@ import java.util.concurrent.Callable; import java.util.stream.StreamSupport; -import static org.elasticsearch.gradle.Util.capitalize; import static org.elasticsearch.gradle.tool.Boilerplate.findByName; import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate; @@ -72,7 +71,10 @@ public void apply(Project project) { DependencyHandler dependencies = project.getDependencies(); Map depConfig = new HashMap<>(); depConfig.put("path", ":"); // root project - depConfig.put("configuration", configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform())); + depConfig.put( + "configuration", + configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture()) + ); project.getDependencies().add(jdk.getConfigurationName(), dependencies.project(depConfig)); // ensure a root level jdk download task exists @@ -87,7 +89,14 @@ public static NamedDomainObjectContainer getContainer(Project project) { } private static void setupRootJdkDownload(Project rootProject, Jdk jdk) { - String extractTaskName = "extract" + capitalize(jdk.getPlatform()) + "Jdk-" + jdk.getVendor() + "-" + jdk.getVersion(); + String extractTaskName = String.format( + Locale.ROOT, + "extract-%s-%s-jdk-%s-%s", + jdk.getPlatform(), + jdk.getArchitecture(), + jdk.getVendor(), + jdk.getVersion() + ); // Skip setup if we've already configured a JDK for this platform, vendor and version if (findByName(rootProject.getTasks(), extractTaskName) == null) { @@ -107,7 +116,7 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) { repoUrl = "https://artifactory.elstc.co/artifactory/oss-jdk-local/"; artifactPattern = String.format( Locale.ROOT, - "adoptopenjdk/OpenJDK%sU-jdk_x64_[module]_hotspot_[revision]_%s.[ext]", + "adoptopenjdk/OpenJDK%sU-jdk_[classifier]_[module]_hotspot_[revision]_%s.[ext]", jdk.getMajor(), jdk.getBuild() ); @@ -121,14 +130,14 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) { + jdk.getHash() + "/" + jdk.getBuild() - + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"; + + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]"; } else { // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back artifactPattern = "java/GA/jdk" + jdk.getMajor() + "/" + jdk.getBuild() - + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"; + + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]"; } } else { throw new GradleException("Unknown JDK vendor [" + jdk.getVendor() + "]"); @@ -150,14 +159,14 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) { // Declare a configuration and dependency from which to download the remote JDK final ConfigurationContainer configurations = rootProject.getConfigurations(); - String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform()); + String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture()); Configuration downloadConfiguration = maybeCreate(configurations, downloadConfigName); rootProject.getDependencies().add(downloadConfigName, dependencyNotation(jdk)); // Create JDK extract task final Provider extractPath = rootProject.getLayout() .getBuildDirectory() - .dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform()); + .dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform() + "_" + jdk.getArchitecture()); TaskProvider extractTask = createExtractTask( extractTaskName, @@ -168,7 +177,13 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) { ); // Declare a configuration for the extracted JDK archive - String artifactConfigName = configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform()); + String artifactConfigName = configName( + "extracted_jdk", + jdk.getVendor(), + jdk.getVersion(), + jdk.getPlatform(), + jdk.getArchitecture() + ); maybeCreate(configurations, artifactConfigName); rootProject.getArtifacts().add(artifactConfigName, extractPath, artifact -> artifact.builtBy(extractTask)); } @@ -254,7 +269,7 @@ private static String dependencyNotation(Jdk jdk) { : jdk.getPlatform(); String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz"; - return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension; + return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + ":" + jdk.getArchitecture() + "@" + extension; } private static String groupName(Jdk jdk) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java index a9561adda015c..7e425632ede2b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java @@ -173,12 +173,14 @@ private static Jdk createJdk( String name, String vendor, String version, - String platform + String platform, + String architecture ) { Jdk jdk = jdksContainer.create(name); jdk.setVendor(vendor); jdk.setVersion(version); jdk.setPlatform(platform); + jdk.setArchitecture(architecture); return jdk; } @@ -210,7 +212,7 @@ private static List configureVM(Project project) { NamedDomainObjectContainer jdksContainer = JdkDownloadPlugin.getContainer(project); String platform = box.contains("windows") ? "windows" : "linux"; - Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform); + Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform, "x64"); // setup VM used by these tests VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index e74058b04bb52..fc41054049f5b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.gradle.testclusters; +import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.DistributionDownloadPlugin; import org.elasticsearch.gradle.Jdk; import org.elasticsearch.gradle.JdkDownloadPlugin; @@ -65,6 +66,7 @@ public void apply(Project project) { jdk.setVendor(LEGACY_JAVA_VENDOR); jdk.setVersion(LEGACY_JAVA_VERSION); jdk.setPlatform(OS.current().name().toLowerCase()); + jdk.setArchitecture(Architecture.current().name().toLowerCase()); }); // enable the DSL to describe clusters diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java index ea291858913f8..758aacbf90c8c 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java @@ -36,7 +36,7 @@ public static void setupRoot() { } public void testMissingVendor() { - assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "vendor not specified for jdk [testjdk]"); + assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "x64", "vendor not specified for jdk [testjdk]"); } public void testUnknownVendor() { @@ -46,20 +46,29 @@ public void testUnknownVendor() { "unknown", "11.0.2+33", "linux", + "x64", "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]" ); } public void testMissingVersion() { - assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "version not specified for jdk [testjdk]"); + assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "x64", "version not specified for jdk [testjdk]"); } public void testBadVersionFormat() { - assertJdkError(createProject(), "testjdk", "openjdk", "badversion", "linux", "malformed version [badversion] for jdk [testjdk]"); + assertJdkError( + createProject(), + "testjdk", + "openjdk", + "badversion", + "linux", + "x64", + "malformed version [badversion] for jdk [testjdk]" + ); } public void testMissingPlatform() { - assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "platform not specified for jdk [testjdk]"); + assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "x64", "platform not specified for jdk [testjdk]"); } public void testUnknownPlatform() { @@ -69,19 +78,44 @@ public void testUnknownPlatform() { "openjdk", "11.0.2+33", "unknown", + "x64", "unknown platform [unknown] for jdk [testjdk], must be one of [darwin, linux, windows, mac]" ); } - private void assertJdkError(Project project, String name, String vendor, String version, String platform, String message) { + public void testMissingArchitecture() { + assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", "linux", null, "architecture not specified for jdk [testjdk]"); + } + + public void testUnknownArchitecture() { + assertJdkError( + createProject(), + "testjdk", + "openjdk", + "11.0.2+33", + "linux", + "unknown", + "unknown architecture [unknown] for jdk [testjdk], must be one of [aarch64, x64]" + ); + } + + private void assertJdkError( + final Project project, + final String name, + final String vendor, + final String version, + final String platform, + final String architecture, + final String message + ) { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> createJdk(project, name, vendor, version, platform) + () -> createJdk(project, name, vendor, version, platform, architecture) ); assertThat(e.getMessage(), equalTo(message)); } - private void createJdk(Project project, String name, String vendor, String version, String platform) { + private void createJdk(Project project, String name, String vendor, String version, String platform, String architecture) { @SuppressWarnings("unchecked") NamedDomainObjectContainer jdks = (NamedDomainObjectContainer) project.getExtensions().getByName("jdks"); jdks.create(name, jdk -> { @@ -94,6 +128,9 @@ private void createJdk(Project project, String name, String vendor, String versi if (platform != null) { jdk.setPlatform(platform); } + if (architecture != null) { + jdk.setArchitecture(architecture); + } }).finalizeValues(); } diff --git a/buildSrc/src/testKit/jdk-download/reuse/build.gradle b/buildSrc/src/testKit/jdk-download/reuse/build.gradle index 795098c4b5229..c8b08171091b1 100644 --- a/buildSrc/src/testKit/jdk-download/reuse/build.gradle +++ b/buildSrc/src/testKit/jdk-download/reuse/build.gradle @@ -7,5 +7,6 @@ jdks { vendor = fakeJdkVendor version = fakeJdkVersion platform = "linux" + architeecture = "x64" } } diff --git a/buildSrc/src/testKit/jdk-download/subproj/build.gradle b/buildSrc/src/testKit/jdk-download/subproj/build.gradle index a0713bef204d9..a19a6bead4c7b 100644 --- a/buildSrc/src/testKit/jdk-download/subproj/build.gradle +++ b/buildSrc/src/testKit/jdk-download/subproj/build.gradle @@ -10,16 +10,19 @@ jdks { vendor = fakeJdkVendor version = fakeJdkVersion platform = "linux" + architecture = "x64" } darwin { vendor = fakeJdkVendor version = fakeJdkVersion platform = "darwin" + architecture = "x64" } windows { vendor = fakeJdkVendor version = fakeJdkVersion platform = "windows" + architecture = "x64" } } diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 2cf861fc2b820..36a28034815e0 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -52,7 +52,7 @@ task createJvmOptionsDir(type: EmptyDirTask) { dirMode = 0750 } -CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) { +CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, boolean oss, boolean jdk) { return copySpec { into("elasticsearch-${version}") { into('lib') { @@ -72,7 +72,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla } if (jdk) { into("darwin".equals(platform) ? 'jdk.app' : 'jdk') { - with jdkFiles(project, platform) + with jdkFiles(project, platform, architecture) } } into('') { @@ -118,31 +118,31 @@ Closure commonZipConfig = { task buildIntegTestZip(type: Zip) { configure(commonZipConfig) - with archiveFiles(transportModulesFiles, 'zip', null, true, false) + with archiveFiles(transportModulesFiles, 'zip', null, 'x64', true, false) } task buildWindowsZip(type: Zip) { configure(commonZipConfig) archiveClassifier = 'windows-x86_64' - with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', false, true) + with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', 'x64', false, true) } task buildOssWindowsZip(type: Zip) { configure(commonZipConfig) archiveClassifier = 'windows-x86_64' - with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', true, true) + with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', 'x64', true, true) } task buildNoJdkWindowsZip(type: Zip) { configure(commonZipConfig) archiveClassifier = 'no-jdk-windows-x86_64' - with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', false, false) + with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', 'x64', false, false) } task buildOssNoJdkWindowsZip(type: Zip) { configure(commonZipConfig) archiveClassifier = 'no-jdk-windows-x86_64' - with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', true, false) + with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', 'x64', true, false) } Closure commonTarConfig = { @@ -155,49 +155,61 @@ Closure commonTarConfig = { task buildDarwinTar(type: SymbolicLinkPreservingTar) { configure(commonTarConfig) archiveClassifier = 'darwin-x86_64' - with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', false, true) + with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', 'x64', false, true) } task buildOssDarwinTar(type: SymbolicLinkPreservingTar) { configure(commonTarConfig) archiveClassifier = 'darwin-x86_64' - with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', true, true) + with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', 'x64', true, true) } task buildNoJdkDarwinTar(type: SymbolicLinkPreservingTar) { configure(commonTarConfig) archiveClassifier = 'no-jdk-darwin-x86_64' - with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', false, false) + with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', 'x64', false, false) } task buildOssNoJdkDarwinTar(type: SymbolicLinkPreservingTar) { configure(commonTarConfig) archiveClassifier = 'no-jdk-darwin-x86_64' - with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', true, false) + with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', 'x64', true, false) +} + +task buildLinuxAarch64Tar(type: SymbolicLinkPreservingTar) { + configure(commonTarConfig) + archiveClassifier = 'linux-aarch64' + with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'aarch64', false, true) } task buildLinuxTar(type: SymbolicLinkPreservingTar) { configure(commonTarConfig) archiveClassifier = 'linux-x86_64' - with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', false, true) + with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'x64', false, true) +} + +task buildOssLinuxAarch64Tar(type: SymbolicLinkPreservingTar) { + configure(commonTarConfig) + archiveClassifier = 'linux-x86_64' + with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, true) } task buildOssLinuxTar(type: SymbolicLinkPreservingTar) { configure(commonTarConfig) archiveClassifier = 'linux-x86_64' - with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', true, true) + with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, true) } task buildNoJdkLinuxTar(type: SymbolicLinkPreservingTar) { configure(commonTarConfig) archiveClassifier = 'no-jdk-linux-x86_64' - with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', false, false) + with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'x64', false, false) } task buildOssNoJdkLinuxTar(type: SymbolicLinkPreservingTar) { configure(commonTarConfig) archiveClassifier = 'no-jdk-linux-x86_64' - with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', true, false) + with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, false) } Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() } diff --git a/distribution/archives/linux-aarch64-tar/build.gradle b/distribution/archives/linux-aarch64-tar/build.gradle new file mode 100644 index 0000000000000..4a6dde5fc0c92 --- /dev/null +++ b/distribution/archives/linux-aarch64-tar/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// distribution is done in the parent project. diff --git a/distribution/archives/oss-linux-aarch64-tar/build.gradle b/distribution/archives/oss-linux-aarch64-tar/build.gradle new file mode 100644 index 0000000000000..4a6dde5fc0c92 --- /dev/null +++ b/distribution/archives/oss-linux-aarch64-tar/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// distribution is done in the parent project. diff --git a/distribution/build.gradle b/distribution/build.gradle index 221f639769e0b..846eb5f825de6 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -388,16 +388,17 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } - jdkFiles = { Project project, String platform -> + jdkFiles = { Project project, String platform, String architecture -> project.jdks { - "bundled_${platform}" { + "bundled_${platform}_${architecture}" { it.platform = platform it.version = VersionProperties.getBundledJdk(platform) it.vendor = VersionProperties.bundledJdkVendor + it.architecture = architecture } } return copySpec { - from project.jdks."bundled_${platform}" + from project.jdks."bundled_${platform}_${architecture}" exclude "demo/**" eachFile { FileCopyDetails details -> if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') { @@ -605,10 +606,13 @@ subprojects { ['archives:windows-zip', 'archives:oss-windows-zip', 'archives:darwin-tar', 'archives:oss-darwin-tar', + 'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar', 'archives:linux-tar', 'archives:oss-linux-tar', 'archives:integ-test-zip', 'packages:rpm', 'packages:deb', + 'packages:aarch64-rpm', 'packages:aarch64-deb', 'packages:oss-rpm', 'packages:oss-deb', + 'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb' ].forEach { subName -> Project subproject = project("${project.path}:${subName}") Configuration configuration = configurations.create(subproject.name) diff --git a/distribution/packages/aarch64-deb/build.gradle b/distribution/packages/aarch64-deb/build.gradle new file mode 100644 index 0000000000000..4a6dde5fc0c92 --- /dev/null +++ b/distribution/packages/aarch64-deb/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// distribution is done in the parent project. diff --git a/distribution/packages/aarch64-oss-deb/build.gradle b/distribution/packages/aarch64-oss-deb/build.gradle new file mode 100644 index 0000000000000..4a6dde5fc0c92 --- /dev/null +++ b/distribution/packages/aarch64-oss-deb/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// distribution is done in the parent project. diff --git a/distribution/packages/aarch64-oss-rpm/build.gradle b/distribution/packages/aarch64-oss-rpm/build.gradle new file mode 100644 index 0000000000000..4a6dde5fc0c92 --- /dev/null +++ b/distribution/packages/aarch64-oss-rpm/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// distribution is done in the parent project. diff --git a/distribution/packages/aarch64-rpm/build.gradle b/distribution/packages/aarch64-rpm/build.gradle new file mode 100644 index 0000000000000..4a6dde5fc0c92 --- /dev/null +++ b/distribution/packages/aarch64-rpm/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// distribution is done in the parent project. diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 9b1efa3e43d7c..105f244e6bc5e 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -98,17 +98,17 @@ addProcessFilesTask('rpm', false, false) // Common configuration that is package dependent. This can't go in ospackage // since we have different templated files that need to be consumed, but the structure // is the same -Closure commonPackageConfig(String type, boolean oss, boolean jdk) { +Closure commonPackageConfig(String type, boolean oss, boolean jdk, String architecture) { return { onlyIf { OS.current().equals(OS.WINDOWS) == false } dependsOn "process${oss ? 'Oss' : ''}${jdk ? '' : 'NoJdk'}${type.capitalize()}Files" packageName "elasticsearch${oss ? '-oss' : ''}" - arch(type == 'deb' ? 'amd64' : 'X86_64') + arch(architecture == 'aarch64' ? 'aarch64' : type == 'deb' ? 'amd64' : 'X86_64') // Follow elasticsearch's file naming convention String jdkString = jdk ? "" : "no-jdk-" - String prefix = "${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}" + String prefix = "${architecture == 'aarch64' ? 'aarch64-' : ''}${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}" destinationDir = file("${prefix}/build/distributions") // SystemPackagingTask overrides default archive task convention mappings, but doesn't provide a setter so we have to override the convention mapping itself @@ -143,7 +143,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) { } if (jdk) { into('jdk') { - with jdkFiles(project, 'linux') + with jdkFiles(project, 'linux', architecture) } } // we need to specify every intermediate directory in these paths so the package managers know they are explicitly @@ -306,9 +306,9 @@ ospackage { into '/usr/share/elasticsearch' } -Closure commonDebConfig(boolean oss, boolean jdk) { +Closure commonDebConfig(boolean oss, boolean jdk, String architecture) { return { - configure(commonPackageConfig('deb', oss, jdk)) + configure(commonPackageConfig('deb', oss, jdk, architecture)) // jdeb does not provide a way to set the License control attribute, and ospackage // silently ignores setting it. Instead, we set the license as "custom field" @@ -336,25 +336,33 @@ Closure commonDebConfig(boolean oss, boolean jdk) { } } +task buildAarch64Deb(type: Deb) { + configure(commonDebConfig(false, true, 'aarch64')) +} + task buildDeb(type: Deb) { - configure(commonDebConfig(false, true)) + configure(commonDebConfig(false, true, 'x64')) +} + +task buildAarch64OssDeb(type: Deb) { + configure(commonDebConfig(true, true, 'aarch64')) } task buildOssDeb(type: Deb) { - configure(commonDebConfig(true, true)) + configure(commonDebConfig(true, true, 'x64')) } task buildNoJdkDeb(type: Deb) { - configure(commonDebConfig(false, false)) + configure(commonDebConfig(false, false, 'x64')) } task buildOssNoJdkDeb(type: Deb) { - configure(commonDebConfig(true, false)) + configure(commonDebConfig(true, false, 'x64')) } -Closure commonRpmConfig(boolean oss, boolean jdk) { +Closure commonRpmConfig(boolean oss, boolean jdk, String architecture) { return { - configure(commonPackageConfig('rpm', oss, jdk)) + configure(commonPackageConfig('rpm', oss, jdk, architecture)) if (oss) { license 'ASL 2.0' @@ -381,20 +389,28 @@ Closure commonRpmConfig(boolean oss, boolean jdk) { } } +task buildAarch64Rpm(type: Rpm) { + configure(commonRpmConfig(false, true, 'aarch64')) +} + task buildRpm(type: Rpm) { - configure(commonRpmConfig(false, true)) + configure(commonRpmConfig(false, true, 'x64')) +} + +task buildAarch64OssRpm(type: Rpm) { + configure(commonRpmConfig(true, true, 'aarch64')) } task buildOssRpm(type: Rpm) { - configure(commonRpmConfig(true, true)) + configure(commonRpmConfig(true, true, 'x64')) } task buildNoJdkRpm(type: Rpm) { - configure(commonRpmConfig(false, false)) + configure(commonRpmConfig(false, false, 'x64')) } task buildOssNoJdkRpm(type: Rpm) { - configure(commonRpmConfig(true, false)) + configure(commonRpmConfig(true, false, 'x64')) } Closure dpkgExists = { it -> new File('/bin/dpkg-deb').exists() || new File('/usr/bin/dpkg-deb').exists() || new File('/usr/local/bin/dpkg-deb').exists() } @@ -446,6 +462,8 @@ subprojects { final File rpmDatabase = new File(extractionDir, 'rpm-database') commandLine 'rpm', '--badreloc', + '--ignorearch', + '--ignoreos', '--nodeps', '--noscripts', '--notriggers', diff --git a/distribution/src/bin/elasticsearch b/distribution/src/bin/elasticsearch index 136aed6755c5e..e2c2288cb664c 100755 --- a/distribution/src/bin/elasticsearch +++ b/distribution/src/bin/elasticsearch @@ -29,7 +29,7 @@ for option in "$@"; do done if [ -z "$ES_TMPDIR" ]; then - ES_TMPDIR=`"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory` + ES_TMPDIR=`"$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory` fi # get keystore password before setting java options to avoid @@ -52,12 +52,13 @@ fi # - second, JVM options are read from jvm.options and jvm.options.d/*.options # - third, JVM options from ES_JAVA_OPTS are applied # - fourth, ergonomic JVM options are applied -ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"` +ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"` # manual parsing to find out, if process should be detached if [[ $DAEMONIZE = false ]]; then exec \ "$JAVA" \ + "$XSHARE" \ $ES_JAVA_OPTS \ -Des.path.home="$ES_HOME" \ -Des.path.conf="$ES_PATH_CONF" \ @@ -70,6 +71,7 @@ if [[ $DAEMONIZE = false ]]; then else exec \ "$JAVA" \ + "$XSHARE" \ $ES_JAVA_OPTS \ -Des.path.home="$ES_HOME" \ -Des.path.conf="$ES_PATH_CONF" \ diff --git a/distribution/src/bin/elasticsearch-cli b/distribution/src/bin/elasticsearch-cli index 4af827b67caf9..6f03456eb0122 100644 --- a/distribution/src/bin/elasticsearch-cli +++ b/distribution/src/bin/elasticsearch-cli @@ -22,6 +22,7 @@ ES_JAVA_OPTS="-Xms4m -Xmx64m -XX:+UseSerialGC ${ES_JAVA_OPTS}" exec \ "$JAVA" \ + "$XSHARE" \ $ES_JAVA_OPTS \ -Des.path.home="$ES_HOME" \ -Des.path.conf="$ES_PATH_CONF" \ diff --git a/distribution/src/bin/elasticsearch-env b/distribution/src/bin/elasticsearch-env index cbdfbf8facb5c..5a54ad58e0abd 100644 --- a/distribution/src/bin/elasticsearch-env +++ b/distribution/src/bin/elasticsearch-env @@ -67,8 +67,14 @@ if [ ! -z "$JAVA_OPTS" ]; then echo "pass JVM parameters via ES_JAVA_OPTS" fi +if [[ "$("$JAVA" -version 2>/dev/null)" =~ "Unable to map CDS archive" ]]; then + XSHARE="-Xshare:off" +else + XSHARE="-Xshare:auto" +fi + # check the Java version -"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.java_version_checker.JavaVersionChecker +"$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.java_version_checker.JavaVersionChecker export HOSTNAME=$HOSTNAME diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java index 1040181b66af1..67de0ac8854bf 100644 --- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java @@ -103,10 +103,13 @@ private static List flagsFinal(final List userDefinedJvmOptions) */ final String java = Paths.get(System.getProperty("java.home"), "bin", "java").toString(); final List command = Collections.unmodifiableList( - Stream.of(Stream.of(java), userDefinedJvmOptions.stream(), Stream.of("-XX:+PrintFlagsFinal"), Stream.of("-version")) - .reduce(Stream::concat) - .get() - .collect(Collectors.toList()) + Stream.of( + Stream.of(java), + userDefinedJvmOptions.stream(), + Stream.of("-Xshare:off"), + Stream.of("-XX:+PrintFlagsFinal"), + Stream.of("-version") + ).reduce(Stream::concat).get().collect(Collectors.toList()) ); final Process process = new ProcessBuilder().command(command).start(); final List output = readLinesFromInputStream(process.getInputStream()); diff --git a/gradle/runtime-jdk-provision.gradle b/gradle/runtime-jdk-provision.gradle index 7ace3277c0a1e..4c346c450dcf4 100644 --- a/gradle/runtime-jdk-provision.gradle +++ b/gradle/runtime-jdk-provision.gradle @@ -1,3 +1,4 @@ +import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.OS import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.info.BuildParams @@ -9,6 +10,7 @@ jdks { vendor = VersionProperties.bundledJdkVendor version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase()) platform = OS.current().name().toLowerCase() + architecture = Architecture.current().name().toLowerCase() } } diff --git a/settings.gradle b/settings.gradle index 0381049d6b235..155cdfa771870 100644 --- a/settings.gradle +++ b/settings.gradle @@ -27,7 +27,9 @@ List projects = [ 'distribution:archives:darwin-tar', 'distribution:archives:oss-no-jdk-darwin-tar', 'distribution:archives:no-jdk-darwin-tar', + 'distribution:archives:oss-linux-aarch64-tar', 'distribution:archives:oss-linux-tar', + 'distribution:archives:linux-aarch64-tar', 'distribution:archives:linux-tar', 'distribution:archives:oss-no-jdk-linux-tar', 'distribution:archives:no-jdk-linux-tar', @@ -36,11 +38,15 @@ List projects = [ 'distribution:docker:docker-export', 'distribution:docker:oss-docker-build-context', 'distribution:docker:oss-docker-export', + 'distribution:packages:aarch64-oss-deb', 'distribution:packages:oss-deb', + 'distribution:packages:aarch64-deb', 'distribution:packages:deb', 'distribution:packages:oss-no-jdk-deb', 'distribution:packages:no-jdk-deb', + 'distribution:packages:aarch64-oss-rpm', 'distribution:packages:oss-rpm', + 'distribution:packages:aarch64-rpm', 'distribution:packages:rpm', 'distribution:packages:oss-no-jdk-rpm', 'distribution:packages:no-jdk-rpm', diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index 6988559093803..4dc554960b22b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -71,8 +71,15 @@ private XPackSettings() { public static final Setting GRAPH_ENABLED = Setting.boolSetting("xpack.graph.enabled", true, Setting.Property.NodeScope); /** Setting for enabling or disabling machine learning. Defaults to true. */ - public static final Setting MACHINE_LEARNING_ENABLED = Setting.boolSetting("xpack.ml.enabled", true, - Setting.Property.NodeScope); + public static final Setting MACHINE_LEARNING_ENABLED = Setting.boolSetting( + "xpack.ml.enabled", + "aarch64".equals(System.getProperty("os.arch")) ? false : true, + value -> { + if (value && "aarch64".equals(System.getProperty("os.arch"))) { + throw new IllegalArgumentException("[xpack.ml.enabled] can not be set to [true] on [aarch64]"); + } + }, + Setting.Property.NodeScope); /** Setting for enabling or disabling rollup. Defaults to true. */ public static final Setting ROLLUP_ENABLED = Setting.boolSetting("xpack.rollup.enabled", true, diff --git a/x-pack/plugin/sql/src/main/bin/elasticsearch-sql-cli b/x-pack/plugin/sql/src/main/bin/elasticsearch-sql-cli index 47916a7975ee3..4bafba9bda3a8 100755 --- a/x-pack/plugin/sql/src/main/bin/elasticsearch-sql-cli +++ b/x-pack/plugin/sql/src/main/bin/elasticsearch-sql-cli @@ -12,5 +12,6 @@ CLI_JAR=$(ls "$ES_HOME"/bin/elasticsearch-sql-cli-*.jar) exec \ "$JAVA" \ + "$XSHARE" \ -jar "$CLI_JAR" \ "$@"