Merge branch 'main' into assume-old-cluster-features

.ci/init.gradle

@@ -1,95 +1,3 @@
-import com.bettercloud.vault.VaultConfig
-import com.bettercloud.vault.Vault
-
-initscript {
-  repositories {
-    mavenCentral()
-  }
-  dependencies {
-    classpath 'com.bettercloud:vault-java-driver:4.1.0'
-  }
-}
-
-boolean USE_ARTIFACTORY = false
-
-if (System.getenv('VAULT_ADDR') == null) {
-  // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
-  if (System.getenv('CI') == null) {
-    return
-  }
-
-  throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
-}
-
-if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) {
-    // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
-  if (System.getenv('CI') == null) {
-    return
-  }
-
-  throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
-    "or the VAULT_TOKEN environment variable to use this init script.")
-}
-
-final String vaultPathPrefix = System.getenv('VAULT_ADDR') ==~ /.+vault-ci.+\.dev.*/ ? "secret/ci/elastic-elasticsearch/migrated" : "secret/elasticsearch-ci"
-
-final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault(
-  new VaultConfig()
-    .address(System.env.VAULT_ADDR)
-    .engineVersion(1)
-    .build()
-)
-  .withRetries(5, 1000)
-  .auth()
-  .loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
-  .getAuthClientToken()
-
-final Vault vault = new Vault(
-  new VaultConfig()
-    .address(System.env.VAULT_ADDR)
-    .engineVersion(1)
-    .token(vaultToken)
-    .build()
-)
-  .withRetries(5, 1000)
-
-
-if (USE_ARTIFACTORY) {
-  final Map<String, String> artifactoryCredentials = vault.logical()
-    .read("${vaultPathPrefix}/artifactory.elstc.co")
-    .getData()
-  logger.info("Using elastic artifactory repos")
-  Closure configCache = {
-    return {
-      name "artifactory-gradle-release"
-      url "https://artifactory.elstc.co/artifactory/gradle-release"
-      credentials {
-        username artifactoryCredentials.get("username")
-        password artifactoryCredentials.get("token")
-      }
-    }
-  }
-  settingsEvaluated { settings ->
-    settings.pluginManagement {
-      repositories {
-        maven configCache()
-      }
-    }
-  }
-  projectsLoaded {
-    allprojects {
-      buildscript {
-        repositories {
-          maven configCache()
-        }
-      }
-      repositories {
-        maven configCache()
-      }
-    }
-  }
-}
-
 gradle.settingsEvaluated { settings ->
   settings.pluginManager.withPlugin("com.gradle.develocity") {
     settings.develocity {
@@ -98,14 +6,10 @@ gradle.settingsEvaluated { settings ->
   }
 }
 
-
 final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
 final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
 
 if (buildCacheUrl) {
-  final Map<String, String> buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical()
-    .read("${vaultPathPrefix}/gradle-build-cache")
-    .getData()
   gradle.settingsEvaluated { settings ->
     settings.buildCache {
       local {
@@ -116,11 +20,10 @@ if (buildCacheUrl) {
         url = buildCacheUrl
         push = buildCachePush
         credentials {
-          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username")
-          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password")
+          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME")
+          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD")
         }
       }
     }
   }
 }
-

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java

@@ -83,6 +83,8 @@ public class VectorScorerBenchmark {
 
     RandomVectorScorer luceneDotScorerQuery;
     RandomVectorScorer nativeDotScorerQuery;
+    RandomVectorScorer luceneSqrScorerQuery;
+    RandomVectorScorer nativeSqrScorerQuery;
 
     @Setup
     public void setup() throws IOException {
@@ -130,6 +132,8 @@ public void setup() throws IOException {
         }
         luceneDotScorerQuery = luceneScorer(values, VectorSimilarityFunction.DOT_PRODUCT, queryVec);
         nativeDotScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.DOT_PRODUCT, values, queryVec).get();
+        luceneSqrScorerQuery = luceneScorer(values, VectorSimilarityFunction.EUCLIDEAN, queryVec);
+        nativeSqrScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.EUCLIDEAN, values, queryVec).get();
 
         // sanity
         var f1 = dotProductLucene();
@@ -157,6 +161,12 @@ public void setup() throws IOException {
         if (q1 != q2) {
             throw new AssertionError("query: lucene[" + q1 + "] != " + "native[" + q2 + "]");
         }
+
+        var sqr1 = squareDistanceLuceneQuery();
+        var sqr2 = squareDistanceNativeQuery();
+        if (sqr1 != sqr2) {
+            throw new AssertionError("query: lucene[" + q1 + "] != " + "native[" + q2 + "]");
+        }
     }
 
     @TearDown
@@ -217,6 +227,16 @@ public float squareDistanceScalar() {
         return 1 / (1f + adjustedDistance);
     }
 
+    @Benchmark
+    public float squareDistanceLuceneQuery() throws IOException {
+        return luceneSqrScorerQuery.score(1);
+    }
+
+    @Benchmark
+    public float squareDistanceNativeQuery() throws IOException {
+        return nativeSqrScorerQuery.score(1);
+    }
+
     QuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
         var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
         var slice = in.slice("values", 0, in.length());

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java

@@ -10,47 +10,45 @@
 package org.elasticsearch.gradle.internal.conventions;
 
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
+import org.elasticsearch.gradle.internal.conventions.info.GitInfoValueSource;
 import org.elasticsearch.gradle.internal.conventions.util.Util;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
-import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 
-import javax.inject.Inject;
 import java.io.File;
 
-class GitInfoPlugin implements Plugin<Project> {
+import javax.inject.Inject;
 
-    private ProviderFactory factory;
-    private ObjectFactory objectFactory;
+public abstract class GitInfoPlugin implements Plugin<Project> {
 
+    private ProviderFactory factory;
     private Provider<String> revision;
-    private Property<GitInfo> gitInfo;
 
     @Inject
-    GitInfoPlugin(ProviderFactory factory, ObjectFactory objectFactory) {
+    public GitInfoPlugin(ProviderFactory factory) {
         this.factory = factory;
-        this.objectFactory = objectFactory;
     }
 
     @Override
     public void apply(Project project) {
-        File rootDir = Util.locateElasticsearchWorkspace(project.getGradle());
-        gitInfo = objectFactory.property(GitInfo.class).value(factory.provider(() ->
-            GitInfo.gitInfo(rootDir)
-        ));
-        gitInfo.disallowChanges();
-        gitInfo.finalizeValueOnRead();
-
-        revision = gitInfo.map(info -> info.getRevision() == null ? info.getRevision() : "main");
+        File rootDir = getGitRootDir(project);
+        getGitInfo().convention(factory.of(GitInfoValueSource.class, spec -> { spec.getParameters().getPath().set(rootDir); }));
+        revision = getGitInfo().map(info -> info.getRevision() == null ? info.getRevision() : "main");
     }
 
-    public Property<GitInfo> getGitInfo() {
-        return gitInfo;
+    private static File getGitRootDir(Project project) {
+        File rootDir = project.getRootDir();
+        if (new File(rootDir, ".git").exists()) {
+            return rootDir;
+        }
+        return Util.locateElasticsearchWorkspace(project.getGradle());
     }
 
+    public abstract Property<GitInfo> getGitInfo();
+
     public Provider<String> getRevision() {
         return revision;
     }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java

@@ -15,9 +15,10 @@
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 
-import javax.inject.Inject;
 import java.util.Map;
 
+import javax.inject.Inject;
+
 public class LicensingPlugin implements Plugin<Project> {
     static final String ELASTIC_LICENSE_URL_PREFIX = "https://raw.githubusercontent.com/elastic/elasticsearch/";
     static final String ELASTIC_LICENSE_URL_POSTFIX = "/licenses/ELASTIC-LICENSE-2.0.txt";
@@ -33,24 +34,33 @@ public LicensingPlugin(ProviderFactory providerFactory) {
     @Override
     public void apply(Project project) {
         Provider<String> revision = project.getRootProject().getPlugins().apply(GitInfoPlugin.class).getRevision();
-        Provider<String> licenseCommitProvider = providerFactory.provider(() ->
-             isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion()
+        Provider<String> licenseCommitProvider = providerFactory.provider(
+            () -> isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion()
         );
 
-        Provider<String> elasticLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
-                licenseCommit + ELASTIC_LICENSE_URL_POSTFIX);
-        Provider<String> agplLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
-            licenseCommit + AGPL_ELASTIC_LICENSE_URL_POSTFIX);
+        Provider<String> elasticLicenseURL = licenseCommitProvider.map(
+            licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + licenseCommit + ELASTIC_LICENSE_URL_POSTFIX
+        );
+        Provider<String> agplLicenseURL = licenseCommitProvider.map(
+            licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + licenseCommit + AGPL_ELASTIC_LICENSE_URL_POSTFIX
+        );
         // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
         project.getExtensions().getExtraProperties().set("elasticLicenseUrl", elasticLicenseURL);
 
-        MapProperty<String, String> licensesProperty = project.getObjects().mapProperty(String.class, String.class).convention(
-                providerFactory.provider(() -> Map.of(
-                        "Server Side Public License, v 1", "https://www.mongodb.com/licensing/server-side-public-license",
-                        "Elastic License 2.0", elasticLicenseURL.get(),
-                        "GNU Affero General Public License Version 3", agplLicenseURL.get())
+        MapProperty<String, Provider<String>> licensesProperty = project.getObjects()
+            .mapProperty(String.class, (Class<Provider<String>>) (Class<?>) Provider.class)
+            .convention(
+                providerFactory.provider(
+                    () -> Map.of(
+                        "Server Side Public License, v 1",
+                        providerFactory.provider(() -> "https://www.mongodb.com/licensing/server-side-public-license"),
+                        "Elastic License 2.0",
+                        elasticLicenseURL,
+                        "GNU Affero General Public License Version 3",
+                        agplLicenseURL
+                    )
                 )
-        );
+            );
 
         // Default to the SSPL+Elastic dual license
         project.getExtensions().getExtraProperties().set("projectLicenses", licensesProperty);

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java

@@ -28,6 +28,7 @@
 import org.gradle.api.plugins.JavaLibraryPlugin;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.provider.MapProperty;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.publish.PublishingExtension;
 import org.gradle.api.publish.maven.MavenPublication;
@@ -42,6 +43,7 @@
 import java.io.File;
 import java.util.Map;
 import java.util.concurrent.Callable;
+
 import javax.inject.Inject;
 
 public class PublishPlugin implements Plugin<Project> {
@@ -81,15 +83,15 @@ private void configurePublications(Project project) {
             }
         });
         @SuppressWarnings("unchecked")
-        var projectLicenses = (MapProperty<String, String>) project.getExtensions().getExtraProperties().get("projectLicenses");
+        var projectLicenses = (MapProperty<String, Provider<String>>) project.getExtensions().getExtraProperties().get("projectLicenses");
         publication.getPom().withXml(xml -> {
             var node = xml.asNode();
             node.appendNode("inceptionYear", "2009");
             var licensesNode = node.appendNode("licenses");
             projectLicenses.get().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> {
                 Node license = licensesNode.appendNode("license");
                 license.appendNode("name", entry.getKey());
-                license.appendNode("url", entry.getValue());
+                license.appendNode("url", entry.getValue().get());
                 license.appendNode("distribution", "repo");
             });
             var developer = node.appendNode("developers").appendNode("developer");
@@ -194,7 +196,6 @@ static void configureSourcesJar(Project project) {
         });
     }
 
-
     /**
      * Format the generated pom files to be in a sort of reproducible order.
      */

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java

@@ -22,6 +22,7 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -190,4 +191,15 @@ public String urlFromOrigin() {
         }
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (o == null || getClass() != o.getClass()) return false;
+        GitInfo gitInfo = (GitInfo) o;
+        return Objects.equals(revision, gitInfo.revision) && Objects.equals(origin, gitInfo.origin);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(revision, origin);
+    }
 }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfoValueSource.java

@@ -0,0 +1,22 @@
+package org.elasticsearch.gradle.internal.conventions.info;
+
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.ValueSource;
+import org.gradle.api.provider.ValueSourceParameters;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+
+public abstract class GitInfoValueSource implements ValueSource<GitInfo, GitInfoValueSource.Parameters> {
+
+    @Nullable
+    @Override
+    public GitInfo obtain() {
+        File path = getParameters().getPath().get();
+        return GitInfo.gitInfo(path);
+    }
+
+    public interface Parameters extends ValueSourceParameters {
+        Property<File> getPath();
+    }
+}