diff --git a/docs/reference/esql/functions/description/qstr.asciidoc b/docs/reference/esql/functions/description/qstr.asciidoc deleted file mode 100644 index 5ce9316405ad2..0000000000000 --- a/docs/reference/esql/functions/description/qstr.asciidoc +++ /dev/null @@ -1,5 +0,0 @@ -// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. - -*Description* - -Performs a query string query. Returns true if the provided query string matches the row. diff --git a/docs/reference/esql/functions/description/to_date_nanos.asciidoc b/docs/reference/esql/functions/description/to_date_nanos.asciidoc new file mode 100644 index 0000000000000..3fac7295f1bed --- /dev/null +++ b/docs/reference/esql/functions/description/to_date_nanos.asciidoc @@ -0,0 +1,7 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Converts an input to a nanosecond-resolution date value (aka date_nanos). + +NOTE: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. diff --git a/docs/reference/esql/functions/examples/qstr.asciidoc b/docs/reference/esql/functions/examples/qstr.asciidoc deleted file mode 100644 index 003373c84c029..0000000000000 --- a/docs/reference/esql/functions/examples/qstr.asciidoc +++ /dev/null @@ -1,13 +0,0 @@ -// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. - -*Example* - -[source.merge.styled,esql] ----- -include::{esql-specs}/qstr-function.csv-spec[tag=qstr-with-field] ----- -[%header.monospaced.styled,format=dsv,separator=|] -|=== -include::{esql-specs}/qstr-function.csv-spec[tag=qstr-with-field-result] -|=== - diff --git a/docs/reference/esql/functions/kibana/definition/date_diff.json b/docs/reference/esql/functions/kibana/definition/date_diff.json index f4c4de53f72a3..f91b6ef027782 100644 --- a/docs/reference/esql/functions/kibana/definition/date_diff.json +++ b/docs/reference/esql/functions/kibana/definition/date_diff.json @@ -56,6 +56,5 @@ "examples" : [ "ROW date1 = TO_DATETIME(\"2023-12-02T11:00:00.000Z\"), date2 = TO_DATETIME(\"2023-12-02T11:00:00.001Z\")\n| EVAL dd_ms = DATE_DIFF(\"microseconds\", date1, date2)", "ROW end_23=\"2023-12-31T23:59:59.999Z\"::DATETIME,\n start_24=\"2024-01-01T00:00:00.000Z\"::DATETIME,\n end_24=\"2024-12-31T23:59:59.999\"::DATETIME\n| EVAL end23_to_start24=DATE_DIFF(\"year\", end_23, start_24)\n| EVAL end23_to_end24=DATE_DIFF(\"year\", end_23, end_24)\n| EVAL start_to_end_24=DATE_DIFF(\"year\", start_24, end_24)" - ], - "preview" : false + ] } diff --git a/docs/reference/esql/functions/kibana/definition/date_format.json b/docs/reference/esql/functions/kibana/definition/date_format.json index 7bd01d7f4ef31..6dc81ed379718 100644 --- a/docs/reference/esql/functions/kibana/definition/date_format.json +++ b/docs/reference/esql/functions/kibana/definition/date_format.json @@ -42,6 +42,6 @@ } ], "examples" : [ - "FROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL hired = DATE_FORMAT(\"YYYY-MM-dd\", hire_date)" + "FROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL hired = DATE_FORMAT(\"yyyy-MM-dd\", hire_date)" ] } diff --git a/docs/reference/esql/functions/kibana/definition/qstr.json b/docs/reference/esql/functions/kibana/definition/qstr.json deleted file mode 100644 index dfa3dfd3818ad..0000000000000 --- a/docs/reference/esql/functions/kibana/definition/qstr.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", - "type" : "eval", - "name" : "qstr", - "description" : "Performs a query string query. Returns true if the provided query string matches the row.", - "signatures" : [ - { - "params" : [ - { - "name" : "query", - "type" : "keyword", - "optional" : false, - "description" : "Query string in Lucene query string format." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "query", - "type" : "text", - "optional" : false, - "description" : "Query string in Lucene query string format." - } - ], - "variadic" : false, - "returnType" : "boolean" - } - ], - "examples" : [ - "from books \n| where qstr(\"author: Faulkner\")\n| keep book_no, author \n| sort book_no \n| limit 5;" - ], - "preview" : true -} diff --git a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json new file mode 100644 index 0000000000000..84cc23842f537 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json @@ -0,0 +1,8 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "to_date_nanos", + "description" : "Converts an input to a nanosecond-resolution date value (aka date_nanos).", + "note" : "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.", + "signatures" : [ ] +} diff --git a/docs/reference/esql/functions/kibana/docs/date_format.md b/docs/reference/esql/functions/kibana/docs/date_format.md index 19aeb45e0d403..7c75f8eb566da 100644 --- a/docs/reference/esql/functions/kibana/docs/date_format.md +++ b/docs/reference/esql/functions/kibana/docs/date_format.md @@ -8,5 +8,5 @@ Returns a string representation of a date, in the provided format. ``` FROM employees | KEEP first_name, last_name, hire_date -| EVAL hired = DATE_FORMAT("YYYY-MM-dd", hire_date) +| EVAL hired = DATE_FORMAT("yyyy-MM-dd", hire_date) ``` diff --git a/docs/reference/esql/functions/kibana/docs/mv_avg.md b/docs/reference/esql/functions/kibana/docs/mv_avg.md index c5163f36129bf..c3d7e5423f724 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_avg.md +++ b/docs/reference/esql/functions/kibana/docs/mv_avg.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_AVG -Converts a multivalued field into a single valued field containing the average of all the values. +Converts a multivalued field into a single valued field containing the average of all of the values. ``` ROW a=[3, 5, 1, 6] diff --git a/docs/reference/esql/functions/kibana/docs/mv_sum.md b/docs/reference/esql/functions/kibana/docs/mv_sum.md index 987017b34b743..16285d3c7229b 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_sum.md +++ b/docs/reference/esql/functions/kibana/docs/mv_sum.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_SUM -Converts a multivalued field into a single valued field containing the sum of all the values. +Converts a multivalued field into a single valued field containing the sum of all of the values. ``` ROW a=[3, 5, 6] diff --git a/docs/reference/esql/functions/kibana/docs/qstr.md b/docs/reference/esql/functions/kibana/docs/qstr.md deleted file mode 100644 index 37b5777623185..0000000000000 --- a/docs/reference/esql/functions/kibana/docs/qstr.md +++ /dev/null @@ -1,14 +0,0 @@ - - -### QSTR -Performs a query string query. Returns true if the provided query string matches the row. - -``` -from books -| where qstr("author: Faulkner") -| keep book_no, author -| sort book_no -| limit 5; -``` diff --git a/docs/reference/esql/functions/kibana/docs/to_date_nanos.md b/docs/reference/esql/functions/kibana/docs/to_date_nanos.md new file mode 100644 index 0000000000000..0294802485ccb --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/to_date_nanos.md @@ -0,0 +1,8 @@ + + +### TO_DATE_NANOS +Converts an input to a nanosecond-resolution date value (aka date_nanos). + +Note: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. diff --git a/docs/reference/esql/functions/kibana/inline_cast.json b/docs/reference/esql/functions/kibana/inline_cast.json index f1aa283c52e95..81a1966773238 100644 --- a/docs/reference/esql/functions/kibana/inline_cast.json +++ b/docs/reference/esql/functions/kibana/inline_cast.json @@ -3,6 +3,7 @@ "boolean" : "to_boolean", "cartesian_point" : "to_cartesianpoint", "cartesian_shape" : "to_cartesianshape", + "date_nanos" : "to_date_nanos", "date_period" : "to_dateperiod", "datetime" : "to_datetime", "double" : "to_double", diff --git a/docs/reference/esql/functions/layout/qstr.asciidoc b/docs/reference/esql/functions/layout/qstr.asciidoc deleted file mode 100644 index 715a11089f0d4..0000000000000 --- a/docs/reference/esql/functions/layout/qstr.asciidoc +++ /dev/null @@ -1,17 +0,0 @@ -// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. - -[discrete] -[[esql-qstr]] -=== `QSTR` - -preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] - -*Syntax* - -[.text-center] -image::esql/functions/signature/qstr.svg[Embedded,opts=inline] - -include::../parameters/qstr.asciidoc[] -include::../description/qstr.asciidoc[] -include::../types/qstr.asciidoc[] -include::../examples/qstr.asciidoc[] diff --git a/docs/reference/esql/functions/layout/to_date_nanos.asciidoc b/docs/reference/esql/functions/layout/to_date_nanos.asciidoc new file mode 100644 index 0000000000000..5b8b9390e6ce2 --- /dev/null +++ b/docs/reference/esql/functions/layout/to_date_nanos.asciidoc @@ -0,0 +1,16 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-to_date_nanos]] +=== `TO_DATE_NANOS` + +preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/to_date_nanos.svg[Embedded,opts=inline] + +include::../parameters/to_date_nanos.asciidoc[] +include::../description/to_date_nanos.asciidoc[] +include::../types/to_date_nanos.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/qstr.asciidoc b/docs/reference/esql/functions/parameters/to_date_nanos.asciidoc similarity index 58% rename from docs/reference/esql/functions/parameters/qstr.asciidoc rename to docs/reference/esql/functions/parameters/to_date_nanos.asciidoc index e51096084f2f3..224f474fa64e3 100644 --- a/docs/reference/esql/functions/parameters/qstr.asciidoc +++ b/docs/reference/esql/functions/parameters/to_date_nanos.asciidoc @@ -2,5 +2,5 @@ *Parameters* -`query`:: -Query string in Lucene query string format. +`field`:: +Input value. The input can be a single- or multi-valued column or an expression. diff --git a/docs/reference/esql/functions/signature/qstr.svg b/docs/reference/esql/functions/signature/qstr.svg deleted file mode 100644 index 0d3841b071cef..0000000000000 --- a/docs/reference/esql/functions/signature/qstr.svg +++ /dev/null @@ -1 +0,0 @@ -QSTR(query) diff --git a/docs/reference/esql/functions/signature/to_date_nanos.svg b/docs/reference/esql/functions/signature/to_date_nanos.svg new file mode 100644 index 0000000000000..0b24b56429588 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_date_nanos.svg @@ -0,0 +1 @@ +TO_DATE_NANOS(field) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/qstr.asciidoc b/docs/reference/esql/functions/types/to_date_nanos.asciidoc similarity index 80% rename from docs/reference/esql/functions/types/qstr.asciidoc rename to docs/reference/esql/functions/types/to_date_nanos.asciidoc index 866a39e925665..1f50b65f25a77 100644 --- a/docs/reference/esql/functions/types/qstr.asciidoc +++ b/docs/reference/esql/functions/types/to_date_nanos.asciidoc @@ -4,7 +4,6 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== -query | result -keyword | boolean -text | boolean +field | result +date_nanos |=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv index 7e857f5243f58..83a2f3cb1c281 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv @@ -1,9 +1,9 @@ -millis:date,nanos:date_nanos -2023-10-23T13:55:01.543Z,2023-10-23T13:55:01.543123456Z -2023-10-23T13:53:55.832Z,2023-10-23T13:53:55.832987654Z -2023-10-23T13:52:55.015Z,2023-10-23T13:52:55.015787878Z -2023-10-23T13:51:54.732Z,2023-10-23T13:51:54.732102837Z -2023-10-23T13:33:34.937Z,2023-10-23T13:33:34.937193000Z -2023-10-23T12:27:28.948Z,2023-10-23T12:27:28.948000000Z -2023-10-23T12:15:03.360Z,2023-10-23T12:15:03.360103847Z -1999-10-23T12:15:03.360Z,[2023-03-23T12:15:03.360103847Z, 2023-02-23T13:33:34.937193000Z, 2023-01-23T13:55:01.543123456Z] +millis:date,nanos:date_nanos,num:long +2023-10-23T13:55:01.543Z,2023-10-23T13:55:01.543123456Z,1698069301543123456 +2023-10-23T13:53:55.832Z,2023-10-23T13:53:55.832987654Z,1698069235832987654 +2023-10-23T13:52:55.015Z,2023-10-23T13:52:55.015787878Z,1698069175015787878 +2023-10-23T13:51:54.732Z,2023-10-23T13:51:54.732102837Z,1698069114732102837 +2023-10-23T13:33:34.937Z,2023-10-23T13:33:34.937193000Z,1698068014937193000 +2023-10-23T12:27:28.948Z,2023-10-23T12:27:28.948000000Z,1698064048948000000 +2023-10-23T12:15:03.360Z,2023-10-23T12:15:03.360103847Z,1698063303360103847 +1999-10-23T12:15:03.360Z,[2023-03-23T12:15:03.360103847Z, 2023-02-23T13:33:34.937193000Z, 2023-01-23T13:55:01.543123456Z], 0 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index b77689e1b5768..883010eb484db 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -70,3 +70,196 @@ FROM date_nanos | SORT millis asc | EVAL nanos = MV_LAST(nanos) | KEEP nanos | L nanos:date_nanos 2023-03-23T12:15:03.360103847Z ; + +string to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("2023-03-23T12:15:03.360103847"); + +d:date_nanos +2023-03-23T12:15:03.360103847Z +; + +string to date nanos, :: notation +required_capability: to_date_nanos + +ROW d = "2023-03-23T12:15:03.360103847"::date_nanos; + +d:date_nanos +2023-03-23T12:15:03.360103847Z +; + +string to date nanos, milliseconds only +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("2023-03-23T12:15:03.360"); + +d:date_nanos +2023-03-23T12:15:03.360Z +; + +string to date nanos, out of range +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("2262-04-12T00:00:00.000"); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(\"2262-04-12T00:00:00.000\")] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: date[2262-04-12T00:00:00Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution + +d:date_nanos +null +; + +string to date nanos, pre 1970 +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("1969-04-12T00:00:00.000"); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(\"1969-04-12T00:00:00.000\")] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: date[1969-04-12T00:00:00Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution + +d:date_nanos +null +; + +long to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(1724160894123456789); + +d:date_nanos +2024-08-20T13:34:54.123456789Z +; + +long to date nanos, :: notation +required_capability: to_date_nanos + +ROW d = 1724160894123456789::date_nanos; + +d:date_nanos +2024-08-20T13:34:54.123456789Z +; + + +long to date nanos, before 1970 +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_LONG(-1)); + +warning:Line 1:9: evaluation of [TO_DATE_NANOS(TO_LONG(-1))] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported. +d:date_nanos +null +; + +unsigned long to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_UNSIGNED_LONG(1724160894123456789)); + +d:date_nanos +2024-08-20T13:34:54.123456789Z +; + +double to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(1724160894123456789.0); + +d:date_nanos +# Note we've lost some precision here +2024-08-20T13:34:54.123456768Z +; + +datetime to date nanos, in range +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_DATETIME("2024-08-20T13:34:54.123Z")); + +d:date_nanos +2024-08-20T13:34:54.123000000Z +; + +datetime to date nanos, with overflow +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_DATETIME("2262-04-12T00:00:00.000")); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(TO_DATETIME(\"2262-04-12T00:00:00.000\"))] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: milliSeconds [9223372800000] are after 2262-04-11T23:47:16.854775807 and cannot be converted to nanoseconds + +d:date_nanos +null +; + +datetime to date nanos, pre 1970 +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_DATETIME("1969-04-12T00:00:00.000")); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(TO_DATETIME(\"1969-04-12T00:00:00.000\"))] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: milliSeconds [-22809600000] are before the epoch in 1970 and cannot be converted to nanoseconds + +d:date_nanos +null +; + + +date nanos to long, index version +required_capability: to_date_nanos + +FROM date_nanos | WHERE millis > "2020-02-02" | EVAL l = TO_LONG(nanos) | KEEP l; + +l:long +1698069301543123456 +1698069235832987654 +1698069175015787878 +1698069114732102837 +1698068014937193000 +1698064048948000000 +1698063303360103847 +; + +long to date nanos, index version +required_capability: to_date_nanos + +FROM date_nanos | WHERE millis > "2020-02-02" | EVAL d = TO_DATE_NANOS(num) | KEEP d; + +d:date_nanos +2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +2023-10-23T12:15:03.360103847Z +; + +date_nanos to date nanos, index version +required_capability: to_date_nanos + +FROM date_nanos | WHERE millis > "2020-02-02" | EVAL d = TO_DATE_NANOS(nanos) | KEEP d; + +d:date_nanos +2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +2023-10-23T12:15:03.360103847Z +; + +attempt to cast the result of a fold to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(CONCAT("2023-01-01","T12:12:12")); + +d:date_nanos +2023-01-01T12:12:12.000000000Z +; + +attempt to cast nulls to date nanos +required_capability: to_date_nanos + +ROW a = TO_DATE_NANOS(null), b = TO_DATE_NANOS(null + 1::long), c = TO_DATE_NANOS(CONCAT("2024", null)); + +a:date_nanos | b:date_nanos | c:date_nanos +null | null | null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json index 506290d90b4b0..a07f9eeeca7b8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json @@ -5,6 +5,9 @@ }, "nanos": { "type": "date_nanos" + }, + "num": { + "type": "long" } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 2b3fa9dec797d..13c3857a5c497 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -97,6 +97,8 @@ double tau() "boolean to_boolean(field:boolean|keyword|text|double|long|unsigned_long|integer)" "cartesian_point to_cartesianpoint(field:cartesian_point|keyword|text)" "cartesian_shape to_cartesianshape(field:cartesian_point|cartesian_shape|keyword|text)" +"date_nanos to_date_nanos(field:date|date_nanos|keyword|text|double|long|unsigned_long)" +"date_nanos to_datenanos(field:date|date_nanos|keyword|text|double|long|unsigned_long)" "date_period to_dateperiod(field:date_period|keyword|text)" "date to_datetime(field:date|date_nanos|keyword|text|double|long|unsigned_long|integer)" "double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)" @@ -227,6 +229,8 @@ to_bool |field |"boolean|keyword|text|double to_boolean |field |"boolean|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. to_cartesianpo|field |"cartesian_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. to_cartesiansh|field |"cartesian_point|cartesian_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. +to_date_nanos |field |"date|date_nanos|keyword|text|double|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. +to_datenanos |field |"date|date_nanos|keyword|text|double|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. to_dateperiod |field |"date_period|keyword|text" |Input value. The input is a valid constant date period expression. to_datetime |field |"date|date_nanos|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. to_dbl |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. @@ -357,6 +361,8 @@ to_bool |Converts an input value to a boolean value. A string value of *tr to_boolean |Converts an input value to a boolean value. A string value of *true* will be case-insensitive converted to the Boolean *true*. For anything else, including the empty string, the function will return *false*. The numerical value of *0* will be converted to *false*, anything else will be converted to *true*. to_cartesianpo|Converts an input value to a `cartesian_point` value. A string will only be successfully converted if it respects the {wikipedia}/Well-known_text_representation_of_geometry[WKT Point] format. to_cartesiansh|Converts an input value to a `cartesian_shape` value. A string will only be successfully converted if it respects the {wikipedia}/Well-known_text_representation_of_geometry[WKT] format. +to_date_nanos |Converts an input to a nanosecond-resolution date value (aka date_nanos). +to_datenanos |Converts an input to a nanosecond-resolution date value (aka date_nanos). to_dateperiod |Converts an input value into a `date_period` value. to_datetime |Converts an input value to a date value. A string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use <>. to_dbl |Converts an input value to a double value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to double. Boolean *true* will be converted to double *1.0*, *false* to *0.0*. @@ -489,6 +495,8 @@ to_bool |boolean to_boolean |boolean |false |false |false to_cartesianpo|cartesian_point |false |false |false to_cartesiansh|cartesian_shape |false |false |false +to_date_nanos |date_nanos |false |false |false +to_datenanos |date_nanos |false |false |false to_dateperiod |date_period |false |false |false to_datetime |date |false |false |false to_dbl |double |false |false |false @@ -536,5 +544,5 @@ required_capability: meta meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -119 | 119 | 119 +121 | 121 | 121 ; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java new file mode 100644 index 0000000000000..e00e7e044ae12 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java @@ -0,0 +1,122 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromDatetimeEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromDatetimeEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromDatetime"; + } + + @Override + public Block evalVector(Vector v) { + LongVector vector = (LongVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongVector container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromDatetime(value); + } + + @Override + public Block evalBlock(Block b) { + LongBlock block = (LongBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongBlock container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromDatetime(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromDatetimeEvaluator get(DriverContext context) { + return new ToDateNanosFromDatetimeEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromDatetimeEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java new file mode 100644 index 0000000000000..23b30e669241b --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java @@ -0,0 +1,124 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.InvalidArgumentException; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromDoubleEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromDoubleEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromDouble"; + } + + @Override + public Block evalVector(Vector v) { + DoubleVector vector = (DoubleVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (IllegalArgumentException | InvalidArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p)); + } catch (IllegalArgumentException | InvalidArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(DoubleVector container, int index) { + double value = container.getDouble(index); + return ToDateNanos.fromDouble(value); + } + + @Override + public Block evalBlock(Block b) { + DoubleBlock block = (DoubleBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException | InvalidArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(DoubleBlock container, int index) { + double value = container.getDouble(index); + return ToDateNanos.fromDouble(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromDoubleEvaluator get(DriverContext context) { + return new ToDateNanosFromDoubleEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromDoubleEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java new file mode 100644 index 0000000000000..cc52208ce5a25 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java @@ -0,0 +1,122 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromLongEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromLongEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromLong"; + } + + @Override + public Block evalVector(Vector v) { + LongVector vector = (LongVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongVector container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromLong(value); + } + + @Override + public Block evalBlock(Block b) { + LongBlock block = (LongBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongBlock container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromLong(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromLongEvaluator get(DriverContext context) { + return new ToDateNanosFromLongEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromLongEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java new file mode 100644 index 0000000000000..c5a20ac298da7 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromStringEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromStringEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromString"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return ToDateNanos.fromKeyword(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return ToDateNanos.fromKeyword(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromStringEvaluator get(DriverContext context) { + return new ToDateNanosFromStringEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromStringEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f714d4d1808c1..f0fa89dedd9ab 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -273,6 +273,11 @@ public enum Cap { */ DATE_NANOS_TYPE(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + /** + * Support for to_date_nanos function + */ + TO_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + /** * Support CIDRMatch in CombineDisjunctions rule. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 5a6430e0fdfad..e5409e32d5d34 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatePeriod; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDegrees; @@ -349,6 +350,7 @@ private FunctionDefinition[][] functions() { def(ToCartesianShape.class, ToCartesianShape::new, "to_cartesianshape"), def(ToDatePeriod.class, ToDatePeriod::new, "to_dateperiod"), def(ToDatetime.class, ToDatetime::new, "to_datetime", "to_dt"), + def(ToDateNanos.class, ToDateNanos::new, "to_date_nanos", "to_datenanos"), def(ToDegrees.class, ToDegrees::new, "to_degrees"), def(ToDouble.class, ToDouble::new, "to_double", "to_dbl"), def(ToGeoPoint.class, ToGeoPoint::new, "to_geopoint"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java index bdbc9b649c101..4d34033286f52 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDegrees; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; @@ -107,6 +108,7 @@ public static List getNamedWriteables() { entries.add(ToBoolean.ENTRY); entries.add(ToCartesianPoint.ENTRY); entries.add(ToDatetime.ENTRY); + entries.add(ToDateNanos.ENTRY); entries.add(ToDegrees.ENTRY); entries.add(ToDouble.ENTRY); entries.add(ToGeoShape.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java new file mode 100644 index 0000000000000..9a6a91b7ccedd --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.core.InvalidArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.DataTypeConverter; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER; + +public class ToDateNanos extends AbstractConvertFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "ToDateNanos", + ToDateNanos::new + ); + + private static final Map EVALUATORS = Map.ofEntries( + Map.entry(DATETIME, ToDateNanosFromDatetimeEvaluator.Factory::new), + Map.entry(DATE_NANOS, (field, source) -> field), + Map.entry(LONG, ToDateNanosFromLongEvaluator.Factory::new), + Map.entry(KEYWORD, ToDateNanosFromStringEvaluator.Factory::new), + Map.entry(TEXT, ToDateNanosFromStringEvaluator.Factory::new), + Map.entry(DOUBLE, ToDateNanosFromDoubleEvaluator.Factory::new), + Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new) + /* + NB: not including an integer conversion, because max int in nanoseconds is like 2 seconds after epoch, and it seems more likely + a user who tries to convert an int to a nanosecond date has made a mistake that we should catch that at parse time. + TO_DATE_NANOS(TO_LONG(intVal)) is still possible if someone really needs to do this. + */ + ); + + @FunctionInfo( + returnType = "date_nanos", + description = "Converts an input to a nanosecond-resolution date value (aka date_nanos).", + note = "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers " + + "cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.", + preview = true + ) + public ToDateNanos( + Source source, + @Param( + name = "field", + type = { "date", "date_nanos", "keyword", "text", "double", "long", "unsigned_long" }, + description = "Input value. The input can be a single- or multi-valued column or an expression." + ) Expression field + ) { + super(source, field); + } + + protected ToDateNanos(StreamInput in) throws IOException { + super(in); + } + + @Override + public DataType dataType() { + return DATE_NANOS; + } + + @Override + protected Map factories() { + return EVALUATORS; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new ToDateNanos(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, ToDateNanos::new, field()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @ConvertEvaluator(extraName = "FromLong", warnExceptions = { IllegalArgumentException.class }) + static long fromLong(long in) { + if (in < 0L) { + throw new IllegalArgumentException("Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported."); + } + return in; + } + + @ConvertEvaluator(extraName = "FromDouble", warnExceptions = { IllegalArgumentException.class, InvalidArgumentException.class }) + static long fromDouble(double in) { + if (in < 0d) { + throw new IllegalArgumentException("Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported."); + } + return DataTypeConverter.safeDoubleToLong(in); + } + + @ConvertEvaluator(extraName = "FromString", warnExceptions = { IllegalArgumentException.class }) + static long fromKeyword(BytesRef in) { + Instant parsed = DateFormatters.from(DEFAULT_DATE_NANOS_FORMATTER.parse(in.utf8ToString())).toInstant(); + return DateUtils.toLong(parsed); + } + + @ConvertEvaluator(extraName = "FromDatetime", warnExceptions = { IllegalArgumentException.class }) + static long fromDatetime(long in) { + return DateUtils.toNanoSeconds(in); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 0c530bd0eb273..edc3081a33681 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatePeriod; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; @@ -63,6 +64,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; @@ -96,6 +98,7 @@ public class EsqlDataTypeConverter { public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time"); + public static final DateFormatter DEFAULT_DATE_NANOS_FORMATTER = DateFormatter.forPattern("strict_date_optional_time_nanos"); public static final DateFormatter HOUR_MINUTE_SECOND = DateFormatter.forPattern("strict_hour_minute_second_fraction"); @@ -104,6 +107,7 @@ public class EsqlDataTypeConverter { entry(CARTESIAN_POINT, ToCartesianPoint::new), entry(CARTESIAN_SHAPE, ToCartesianShape::new), entry(DATETIME, ToDatetime::new), + entry(DATE_NANOS, ToDateNanos::new), // ToDegrees, typeless entry(DOUBLE, ToDouble::new), entry(GEO_POINT, ToGeoPoint::new), @@ -499,7 +503,7 @@ public static String dateTimeToString(long dateTime) { } public static String nanoTimeToString(long dateTime) { - return DateFormatter.forPattern("strict_date_optional_time_nanos").formatNanos(dateTime); + return DEFAULT_DATE_NANOS_FORMATTER.formatNanos(dateTime); } public static String dateTimeToString(long dateTime, DateFormatter formatter) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index e44ea907518b4..b3942a71edadb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -623,6 +623,7 @@ public static void forUnaryBoolean( /** * Generate positive test cases for a unary function operating on an {@link DataType#DATETIME}. + * This variant defaults to maximum range of possible values */ public static void forUnaryDatetime( List suppliers, @@ -641,6 +642,29 @@ public static void forUnaryDatetime( ); } + /** + * Generate positive test cases for a unary function operating on an {@link DataType#DATETIME}. + * This variant accepts a range of values + */ + public static void forUnaryDatetime( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + long min, + long max, + Function expectedValue, + List warnings + ) { + unaryNumeric( + suppliers, + expectedEvaluatorToString, + dateCases(min, max), + expectedType, + n -> expectedValue.apply(Instant.ofEpochMilli(n.longValue())), + warnings + ); + } + /** * Generate positive test cases for a unary function operating on an {@link DataType#DATE_NANOS}. */ @@ -1044,26 +1068,45 @@ public static List booleanCases() { *

*/ public static List dateCases() { - return List.of( - new TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataType.DATETIME), - new TypedDataSupplier( - "", - () -> ESTestCase.randomLongBetween(0, 10 * (long) 10e11), // 1970-01-01T00:00:00Z - 2286-11-20T17:46:40Z - DataType.DATETIME - ), - new TypedDataSupplier( - "", - // 2286-11-20T17:46:40Z - +292278994-08-17T07:12:55.807Z - () -> ESTestCase.randomLongBetween(10 * (long) 10e11, Long.MAX_VALUE), - DataType.DATETIME - ), - new TypedDataSupplier( - "", - // very close to +292278994-08-17T07:12:55.807Z, the maximum supported millis since epoch - () -> ESTestCase.randomLongBetween(Long.MAX_VALUE / 100 * 99, Long.MAX_VALUE), - DataType.DATETIME - ) - ); + return dateCases(Long.MIN_VALUE, Long.MAX_VALUE); + } + + /** + * Generate cases for {@link DataType#DATETIME}. + *

+ * For multi-row parameters, see {@link MultiRowTestCaseSupplier#dateCases}. + *

+ */ + public static List dateCases(long min, long max) { + List cases = new ArrayList<>(); + if (min <= 0 && max >= 0) { + cases.add(new TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataType.DATETIME)); + } + + // 1970-01-01T00:00:00Z - 2286-11-20T17:46:40Z + long lower1 = Math.max(min, 0); + long upper1 = Math.min(max, 10 * (long) 10e11); + if (lower1 < upper1) { + cases.add(new TypedDataSupplier("", () -> ESTestCase.randomLongBetween(lower1, upper1), DataType.DATETIME)); + } + + // 2286-11-20T17:46:40Z - +292278994-08-17T07:12:55.807Z + long lower2 = Math.max(min, 10 * (long) 10e11); + long upper2 = Math.min(max, Long.MAX_VALUE); + if (lower2 < upper2) { + cases.add(new TypedDataSupplier("", () -> ESTestCase.randomLongBetween(lower2, upper2), DataType.DATETIME)); + } + + // very close to +292278994-08-17T07:12:55.807Z, the maximum supported millis since epoch + long lower3 = Math.max(min, Long.MAX_VALUE / 100 * 99); + long upper3 = Math.min(max, Long.MAX_VALUE); + if (lower3 < upper3) { + cases.add( + new TypedDataSupplier("", () -> ESTestCase.randomLongBetween(lower3, upper3), DataType.DATETIME) + ); + } + + return cases; } /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java new file mode 100644 index 0000000000000..e91a5cc1ebca4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class ToDateNanosTests extends AbstractScalarFunctionTestCase { + public ToDateNanosTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + final String read = "Attribute[channel=0]"; + final List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryDateNanos(suppliers, read, DataType.DATE_NANOS, DateUtils::toLong, List.of()); + TestCaseSupplier.forUnaryDatetime( + suppliers, + "ToDateNanosFromDatetimeEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + 0, + DateUtils.MAX_NANOSECOND_INSTANT.toEpochMilli(), + i -> DateUtils.toNanoSeconds(i.toEpochMilli()), + List.of() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + "ToDateNanosFromLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + l -> l, + 0, + Long.MAX_VALUE, + List.of() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + "ToDateNanosFromLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + l -> null, + Long.MIN_VALUE, + -1L, + List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported." + ) + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "ToLongFromUnsignedLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + BigInteger::longValueExact, + BigInteger.ZERO, + BigInteger.valueOf(Long.MAX_VALUE), + List.of() + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "ToLongFromUnsignedLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + bi -> null, + BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TWO), + UNSIGNED_LONG_MAX, + bi -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + bi + "] out of [long] range" + ) + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToDateNanosFromDoubleEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + d -> null, + Double.NEGATIVE_INFINITY, + -Double.MIN_VALUE, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported." + ) + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToDateNanosFromDoubleEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + d -> null, + 9.223372036854777E18, // a "convenient" value larger than `(double) Long.MAX_VALUE` (== ...776E18) + Double.POSITIVE_INFINITY, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range" + ) + ); + TestCaseSupplier.forUnaryStrings( + suppliers, + "ToDateNanosFromStringEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: " + + (bytesRef.utf8ToString().isEmpty() + ? "cannot parse empty datetime" + : ("failed to parse date field [" + bytesRef.utf8ToString() + "] with format [strict_date_optional_time_nanos]")) + ) + ); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "date_nanos or datetime or double or long or string or unsigned_long" + ); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDateNanos(source, args.get(0)); + } +}