Merge branch 'main' into esql_heap_wait

muted-tests.yml

@@ -381,6 +381,12 @@ tests:
 - class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
   method: testMatchAllQuery
   issue: https://github.com/elastic/elasticsearch/issues/114607
+- class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests
+  method: testPushSpatialIntersectsEvalToSource {default}
+  issue: https://github.com/elastic/elasticsearch/issues/114627
+- class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests
+  method: testPushWhereEvalToSource {default}
+  issue: https://github.com/elastic/elasticsearch/issues/114628
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml

@@ -21,13 +21,13 @@ setup:
                 element_type: byte
                 index_options:
                   type: hnsw
+                  m: 16
+                  ef_construction: 100
               vector2:
                 type: dense_vector
                 dims: 1024
                 index: true
                 similarity: dot_product
-                index_options:
-                  type: int8_hnsw
               vector3:
                 type: dense_vector
                 dims: 100

x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec

@@ -369,7 +369,7 @@ date1:date               | dd_ms:integer
 
 evalDateDiffMonthAsWhole0Months
 
-ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME
+ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-01-01T00:00:00")
 | EVAL msecs=DATE_DIFF("milliseconds", from, to), months=DATE_DIFF("month", from, to)
 ;
 
@@ -380,7 +380,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIM
 
 evalDateDiffMonthAsWhole1Month
 
-ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-02-01T00:00:00"::DATETIME
+ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-02-01T00:00:00")
 | EVAL secs=DATE_DIFF("seconds", from, to), months=DATE_DIFF("month", from, to)
 ;
 
@@ -392,7 +392,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-02-01T00:00:00"::DATETIM
 evalDateDiffYearAsWhole0Years
 required_capability: date_diff_year_calendarial
 
-ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME
+ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-01-01T00:00:00")
 | EVAL msecs=DATE_DIFF("milliseconds", from, to), years=DATE_DIFF("year", from, to)
 ;
 
@@ -403,7 +403,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIM
 evalDateDiffYearAsWhole1Year
 required_capability: date_diff_year_calendarial
 
-ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME
+ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2025-01-01T00:00:00")
 | EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to)
 ;
 
@@ -414,7 +414,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIM
 evalDateDiffYearAsWhole1Year
 required_capability: date_diff_year_calendarial
 
-ROW from="2024-01-01T00:00:00Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME
+ROW from=TO_DATETIME("2024-01-01T00:00:00Z"), to=TO_DATETIME("2025-01-01T00:00:00")
 | EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to)
 ;
 
@@ -426,9 +426,9 @@ evalDateDiffYearForDocs
 required_capability: date_diff_year_calendarial
 
 // tag::evalDateDiffYearForDocs[]
-ROW end_23="2023-12-31T23:59:59.999Z"::DATETIME,
-  start_24="2024-01-01T00:00:00.000Z"::DATETIME,
-    end_24="2024-12-31T23:59:59.999"::DATETIME
+ROW end_23=TO_DATETIME("2023-12-31T23:59:59.999Z"),
+  start_24=TO_DATETIME("2024-01-01T00:00:00.000Z"),
+    end_24=TO_DATETIME("2024-12-31T23:59:59.999")
 | EVAL end23_to_start24=DATE_DIFF("year", end_23, start_24)
 | EVAL end23_to_end24=DATE_DIFF("year", end_23, end_24)
 | EVAL start_to_end_24=DATE_DIFF("year", start_24, end_24)

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java

@@ -25,7 +25,7 @@
 public class EsDataSource implements DataSource, Wrapper {
 
     static {
-        // invoke Version to perform classpath/jar sanity checks
+        // invoke version to perform classpath/jar sanity checks
         ClientVersion.CURRENT.toString();
     }
 

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java

@@ -23,7 +23,7 @@ public class EsDriver implements Driver {
     private static final EsDriver INSTANCE = new EsDriver();
 
     static {
-        // invoke Version to perform classpath/jar sanity checks
+        // invoke version to perform classpath/jar sanity checks
         ClientVersion.CURRENT.toString();
 
         try {

x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/DriverManagerRegistrationTests.java

@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.test.ESTestCase;
 
 import java.security.AccessController;
@@ -27,8 +26,8 @@ public void testVersioning() throws Exception {
             /* This test will only work properly in gradle because in gradle we run the tests
              * using the jar. */
 
-            assertNotEquals(String.valueOf(Version.CURRENT.major), d.getMajorVersion());
-            assertNotEquals(String.valueOf(Version.CURRENT.minor), d.getMinorVersion());
+            assertNotEquals(String.valueOf(VersionTests.current().major), d.getMajorVersion());
+            assertNotEquals(String.valueOf(VersionTests.current().minor), d.getMinorVersion());
         });
     }
 

x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java

@@ -7,17 +7,21 @@
 
 package org.elasticsearch.xpack.sql.jdbc;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.rest.root.MainResponse;
-import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.sql.client.ClientVersion;
 import org.elasticsearch.xpack.sql.proto.SqlVersion;
+import org.elasticsearch.xpack.sql.proto.SqlVersions;
 
 import java.io.IOException;
 import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_VERSION_COMPATIBILITY;
 
 /**
  * Test class for JDBC-ES server versions checks.
@@ -29,15 +33,11 @@ public class VersionParityTests extends WebServerTestCase {
 
     public void testExceptionThrownOnIncompatibleVersions() throws IOException, SQLException {
         String url = JdbcConfiguration.URL_PREFIX + webServerAddress();
-        Version firstVersion = VersionUtils.getFirstVersion();
-        Version version = Version.V_7_7_0;
-        do {
-            version = VersionUtils.getPreviousVersion(version);
+        for (var version = SqlVersions.getFirstVersion(); version.onOrAfter(INTRODUCING_VERSION_COMPATIBILITY) == false; version =
+            SqlVersions.getNextVersion(version)) {
             logger.info("Checking exception is thrown for version {}", version);
 
             prepareResponse(version);
-            // Client's version is wired up to patch level, excluding the qualifier => generate the test version as the server does it.
-            String versionString = SqlVersion.fromString(version.toString()).toString();
 
             SQLException ex = expectThrows(
                 SQLException.class,
@@ -48,27 +48,30 @@ public void testExceptionThrownOnIncompatibleVersions() throws IOException, SQLE
                     + ClientVersion.CURRENT.majorMinorToString()
                     + " or newer; attempting to connect to a server "
                     + "version "
-                    + versionString,
+                    + version,
                 ex.getMessage()
             );
-        } while (version.compareTo(firstVersion) > 0);
+        }
     }
 
     public void testNoExceptionThrownForCompatibleVersions() throws IOException {
         String url = JdbcConfiguration.URL_PREFIX + webServerAddress();
-        Version version = Version.CURRENT;
-        try {
-            do {
+        List<SqlVersion> afterVersionCompatibility = SqlVersions.getAllVersions()
+            .stream()
+            .filter(v -> v.onOrAfter(INTRODUCING_VERSION_COMPATIBILITY))
+            .collect(Collectors.toCollection(ArrayList::new));
+        afterVersionCompatibility.add(VersionTests.current());
+        for (var version : afterVersionCompatibility) {
+            try {
                 prepareResponse(version);
                 new JdbcHttpClient(new JdbcConnection(JdbcConfiguration.create(url, null, 0), false));
-                version = VersionUtils.getPreviousVersion(version);
-            } while (version.compareTo(Version.V_7_7_0) >= 0);
-        } catch (SQLException sqle) {
-            fail("JDBC driver version and Elasticsearch server version should be compatible. Error: " + sqle);
+            } catch (SQLException sqle) {
+                fail("JDBC driver version and Elasticsearch server version should be compatible. Error: " + sqle);
+            }
         }
     }
 
-    void prepareResponse(Version version) throws IOException {
+    void prepareResponse(SqlVersion version) throws IOException {
         MainResponse response = version == null ? createCurrentVersionMainResponse() : createMainResponse(version);
         webServer().enqueue(
             new MockResponse().setResponseCode(200)

x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java

@@ -6,15 +6,20 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.client.ClientVersion;
+import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 public class VersionTests extends ESTestCase {
     public void testVersionIsCurrent() {
         /* This test will only work properly in gradle because in gradle we run the tests
          * using the jar. */
-        assertEquals(org.elasticsearch.Version.CURRENT.major, ClientVersion.CURRENT.major);
-        assertEquals(org.elasticsearch.Version.CURRENT.minor, ClientVersion.CURRENT.minor);
-        assertEquals(org.elasticsearch.Version.CURRENT.revision, ClientVersion.CURRENT.revision);
+        assertEquals(current(), ClientVersion.CURRENT);
+    }
+
+    /** Returns the current stack version. Can be unreleased. */
+    public static SqlVersion current() {
+        return SqlVersion.fromString(Build.current().version());
     }
 }

x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/WebServerTestCase.java

@@ -8,13 +8,13 @@
 package org.elasticsearch.xpack.sql.jdbc;
 
 import org.elasticsearch.Build;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.rest.root.MainResponse;
 import org.elasticsearch.test.BuildUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.xpack.sql.proto.SqlVersion;
 import org.junit.After;
 import org.junit.Before;
 
@@ -42,10 +42,10 @@ public MockWebServer webServer() {
     }
 
     MainResponse createCurrentVersionMainResponse() {
-        return createMainResponse(Version.CURRENT);
+        return createMainResponse(VersionTests.current());
     }
 
-    MainResponse createMainResponse(Version version) {
+    MainResponse createMainResponse(SqlVersion version) {
         // the SQL client only cares about node version,
         // so ignore index & transport versions here (just set them to current)
         String clusterUuid = randomAlphaOfLength(10);

x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ConnectionTestCase.java

@@ -6,8 +6,6 @@
  */
 package org.elasticsearch.xpack.sql.qa.jdbc;
 
-import org.elasticsearch.Version;
-
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.SQLException;
@@ -22,8 +20,8 @@ public void testConnectionProperties() throws SQLException {
             assertFalse(c.isClosed());
             assertTrue(c.isReadOnly());
             DatabaseMetaData md = c.getMetaData();
-            assertEquals(Version.CURRENT.major, md.getDatabaseMajorVersion());
-            assertEquals(Version.CURRENT.minor, md.getDatabaseMinorVersion());
+            assertEquals(JdbcTestUtils.CURRENT.major, md.getDatabaseMajorVersion());
+            assertEquals(JdbcTestUtils.CURRENT.minor, md.getDatabaseMinorVersion());
         }
     }
 

x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java

@@ -6,8 +6,9 @@
  */
 package org.elasticsearch.xpack.sql.qa.jdbc;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.Build;
 import org.elasticsearch.xpack.sql.jdbc.EsType;
+import org.elasticsearch.xpack.sql.proto.SqlVersion;
 import org.elasticsearch.xpack.sql.proto.StringUtils;
 
 import java.math.BigInteger;
@@ -26,10 +27,11 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 
-import static org.elasticsearch.Version.V_8_2_0;
-import static org.elasticsearch.Version.V_8_4_0;
 import static org.elasticsearch.common.time.DateUtils.toMilliSeconds;
 import static org.elasticsearch.test.ESTestCase.randomLongBetween;
+import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.supportsDateNanos;
+import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.supportsUnsignedLong;
+import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.supportsVersionType;
 
 final class JdbcTestUtils {
 
@@ -43,7 +45,9 @@ private JdbcTestUtils() {}
     static final LocalDate EPOCH = LocalDate.of(1970, 1, 1);
 
     static final String UNSIGNED_LONG_TYPE_NAME = "UNSIGNED_LONG";
-    static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE);
+
+    // Build's version is always a SemVer in JDBC tests
+    public static final SqlVersion CURRENT = SqlVersion.fromString(Build.current().version());
 
     /*
      * The version of the driver that the QA (bwc-)tests run against.
@@ -59,12 +63,11 @@ private JdbcTestUtils() {}
      *     }
      * </code>
      */
-    static final Version JDBC_DRIVER_VERSION;
+    static final SqlVersion JDBC_DRIVER_VERSION;
 
     static {
-        // master's version is x.0.0-SNAPSHOT, tho Version#fromString() won't accept that back for recent versions
-        String jdbcDriverVersion = System.getProperty(DRIVER_VERSION_PROPERTY_NAME, "").replace("-SNAPSHOT", "");
-        JDBC_DRIVER_VERSION = Version.fromString(jdbcDriverVersion); // takes empty and null strings, resolves them to CURRENT
+        String jdbcDriverVersion = System.getProperty(DRIVER_VERSION_PROPERTY_NAME, "");
+        JDBC_DRIVER_VERSION = jdbcDriverVersion.isEmpty() ? CURRENT : SqlVersion.fromString(jdbcDriverVersion);
 
         // Note: keep in sync with org.elasticsearch.xpack.sql.jdbc.TypeUtils#CLASS_TO_TYPE
         Map<Class<?>, EsType> aMap = new LinkedHashMap<>();
@@ -150,15 +153,14 @@ static int extractNanosOnly(long nanos) {
     }
 
     static boolean versionSupportsDateNanos() {
-        return JDBC_DRIVER_VERSION.onOrAfter(Version.V_7_12_0);
+        return supportsDateNanos(JDBC_DRIVER_VERSION);
     }
 
     public static boolean isUnsignedLongSupported() {
-        return JDBC_DRIVER_VERSION.onOrAfter(V_8_2_0);
+        return supportsUnsignedLong(JDBC_DRIVER_VERSION);
     }
 
     public static boolean isVersionFieldTypeSupported() {
-        return JDBC_DRIVER_VERSION.onOrAfter(V_8_4_0);
+        return supportsVersionType(JDBC_DRIVER_VERSION);
     }
-
 }

x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcWarningsTestCase.java

@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.sql.qa.jdbc;
 
-import org.elasticsearch.Version;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -20,14 +19,13 @@
 import java.util.List;
 import java.util.Properties;
 
+import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_WARNING_HANDLING;
 import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.JDBC_DRIVER_VERSION;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 
 public abstract class JdbcWarningsTestCase extends JdbcIntegrationTestCase {
 
-    private static final Version WARNING_HANDLING_ADDED_VERSION = Version.V_8_2_0;
-
     @Before
     public void setupData() throws IOException {
         index("test_data", b -> b.field("foo", 1));
@@ -89,7 +87,7 @@ public void testClearWarnings() throws SQLException {
     }
 
     private void assumeWarningHandlingDriverVersion() {
-        assumeTrue("Driver does not yet handle deprecation warnings", JDBC_DRIVER_VERSION.onOrAfter(WARNING_HANDLING_ADDED_VERSION));
+        assumeTrue("Driver does not yet handle deprecation warnings", JDBC_DRIVER_VERSION.onOrAfter(INTRODUCING_WARNING_HANDLING));
     }
 
 }