[ES|QL] Pushdown string format for BinaryComparison on datetime fields (

elastic#110669) Use string format in datetime field BinaryComparison.
iverase · Jul 12, 2024 · 9f20d47 · 9f20d47
1 parent cecdc9b
commit 9f20d47
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 5 deletions.
diff --git a/docs/changelog/110669.yaml b/docs/changelog/110669.yaml
@@ -0,0 +1,6 @@
+pr: 110669
+summary: "[ES|QL] Use `RangeQuery` and String in `BinaryComparison` on datetime fields"
+area: ES|QL
+type: bug
+issues:
+ - 107900
diff --git a/...l-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java b/...l-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
@@ -66,6 +66,7 @@
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;
 import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
@@ -284,6 +285,8 @@ static Query translate(BinaryComparison bc, TranslatorHandler handler) {
             ZoneId zoneId = null;
             if (DataType.isDateTime(attribute.dataType())) {
                 zoneId = bc.zoneId();
+                value = DEFAULT_DATE_TIME_FORMATTER.formatMillis((Long) value);
+                format = DEFAULT_DATE_TIME_FORMATTER.pattern();
             }
             if (bc instanceof GreaterThan) {
                 return new RangeQuery(source, name, value, false, null, false, format, zoneId);

diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
@@ -1171,3 +1171,27 @@ from employees
 a:datetime
 null
 ;
+
+ImplicitCastingEqual
+required_capability: rangequery_for_datetime
+from employees
+| where birth_date == "1957-05-23T00:00:00Z"
+| keep emp_no, birth_date
+;
+
+emp_no:integer | birth_date:datetime
+10007          | 1957-05-23T00:00:00Z
+;
+
+ImplicitCastingIn
+required_capability: rangequery_for_datetime
+from employees
+| where birth_date IN ("1957-05-23T00:00:00Z", "1958-02-19T00:00:00Z")
+| keep emp_no, birth_date
+| sort emp_no
+;
+
+emp_no:integer | birth_date:datetime
+10007          | 1957-05-23T00:00:00Z
+10008          | 1958-02-19T00:00:00Z
+;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -133,7 +133,12 @@ public enum Cap {
          * Fix a parsing issue where numbers below Long.MIN_VALUE threw an exception instead of parsing as doubles.
          * see <a href="https://github.com/elastic/elasticsearch/issues/104323"> Parsing large numbers is inconsistent #104323 </a>
          */
-        FIX_PARSING_LARGE_NEGATIVE_NUMBERS;
+        FIX_PARSING_LARGE_NEGATIVE_NUMBERS,
+
+        /**
+         * Use RangeQuery for BinaryComparison on DateTime fields.
+         * */
+        RANGEQUERY_FOR_DATETIME;
 
         private final boolean snapshotOnly;
 

diff --git a/...in/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/...in/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
@@ -59,6 +59,7 @@
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.HOUR_MINUTE_SECOND;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString;
 
@@ -204,6 +205,8 @@ static Query translate(BinaryComparison bc, TranslatorHandler handler) {
             ZoneId zoneId = null;
             if (DataType.isDateTime(attribute.dataType())) {
                 zoneId = bc.zoneId();
+                value = dateTimeToString((Long) value);
+                format = DEFAULT_DATE_TIME_FORMATTER.pattern();
             }
             if (bc instanceof GreaterThan) {
                 return new RangeQuery(source, name, value, false, null, false, format, zoneId);

diff --git a/.../plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/.../plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java
@@ -74,7 +74,7 @@ public void testBinaryComparisons() {
 
         assertQueryTranslation("""
             FROM test | WHERE "2007-12-03T10:15:30+01:00" > date""", containsString("""
-            "esql_single_value":{"field":"date","next":{"range":{"date":{"lt":1196673330000,"time_zone":"Z","""));
+            "esql_single_value":{"field":"date","next":{"range":{"date":{"lt":"2007-12-03T09:15:30.000Z","time_zone":"Z","""));
 
         assertQueryTranslation("""
             FROM test | WHERE 2147483648::unsigned_long > unsigned_long""", containsString("""
@@ -94,7 +94,7 @@ public void testBinaryComparisons() {
 
         assertQueryTranslation("""
             FROM test | WHERE "2007-12-03T10:15:30+01:00" == date""", containsString("""
-            "esql_single_value":{"field":"date","next":{"term":{"date":{"value":1196673330000}"""));
+            "esql_single_value":{"field":"date","next":{"term":{"date":{"value":"2007-12-03T09:15:30.000Z"""));
 
         assertQueryTranslation("""
             FROM test | WHERE ip != "127.0.0.1\"""", containsString("""
@@ -135,12 +135,12 @@ public void testRanges() {
         assertQueryTranslation("""
             FROM test | WHERE "2007-12-03T10:15:30+01:00" < date AND date < "2024-01-01T10:15:30+01:00\"""", matchesRegex("""
             .*must.*esql_single_value":\\{"field":"date\"""" + """
-            .*"range":\\{"date":\\{"gt":1196673330000,.*"range":\\{"date":\\{"lt":1704100530000.*"""));
+            .*"range":\\{"date":\\{"gt":\"2007-12-03T09:15:30.000Z\",.*"range":\\{"date":\\{"lt":\"2024-01-01T09:15:30.000Z\".*"""));
 
         assertQueryTranslation("""
             FROM test | WHERE "2007-12-03T10:15:30+01:00" <= date AND date <= "2024-01-01T10:15:30+01:00\"""", matchesRegex("""
             .*must.*esql_single_value":\\{"field":"date\"""" + """
-            .*"range":\\{"date":\\{"gte":1196673330000,.*"range":\\{"date":\\{"lte":1704100530000.*"""));
+            .*"range":\\{"date":\\{"gte":\"2007-12-03T09:15:30.000Z\",.*"range":\\{"date":\\{"lte":\"2024-01-01T09:15:30.000Z\".*"""));
 
         assertQueryTranslation("""
             FROM test | WHERE 2147483648::unsigned_long < unsigned_long AND unsigned_long < 2147483650::unsigned_long""", matchesRegex("""