diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 439f31e0ba..ac1be213b6 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -362,6 +362,15 @@ public static FunctionExpression day_of_week( return compile(functionProperties, BuiltinFunctionName.DAY_OF_WEEK, expressions); } + public static FunctionExpression extract(FunctionProperties functionProperties, + Expression... expressions) { + return compile(functionProperties, BuiltinFunctionName.EXTRACT, expressions); + } + + public static FunctionExpression extract(Expression... expressions) { + return extract(FunctionProperties.None, expressions); + } + public static FunctionExpression from_days(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.FROM_DAYS, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index e28ea6322f..2309c99539 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -35,6 +35,7 @@ import static org.opensearch.sql.utils.DateTimeUtils.extractDate; import static org.opensearch.sql.utils.DateTimeUtils.extractDateTime; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableTable; import com.google.common.collect.Table; import java.math.BigDecimal; @@ -56,6 +57,7 @@ import java.time.temporal.TemporalAmount; import java.util.Arrays; import java.util.Locale; +import java.util.Map; import java.util.TimeZone; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -105,6 +107,32 @@ public class DateTimeFunction { // Mode used for week/week_of_year function by default when no argument is provided private static final ExprIntegerValue DEFAULT_WEEK_OF_YEAR_MODE = new ExprIntegerValue(0); + + // Map used to determine format output for the extract function + private static final Map extract_formats = + ImmutableMap.builder() + .put("MICROSECOND", "SSSSSS") + .put("SECOND", "ss") + .put("MINUTE", "mm") + .put("HOUR", "HH") + .put("DAY", "dd") + .put("WEEK", "w") + .put("MONTH", "MM") + .put("YEAR", "yyyy") + .put("SECOND_MICROSECOND", "ssSSSSSS") + .put("MINUTE_MICROSECOND", "mmssSSSSSS") + .put("MINUTE_SECOND", "mmss") + .put("HOUR_MICROSECOND", "HHmmssSSSSSS") + .put("HOUR_SECOND", "HHmmss") + .put("HOUR_MINUTE", "HHmm") + .put("DAY_MICROSECOND", "ddHHmmssSSSSSS") + .put("DAY_SECOND", "ddHHmmss") + .put("DAY_MINUTE", "ddHHmm") + .put("DAY_HOUR", "ddHH") + .put("YEAR_MONTH", "yyyyMM") + .put("QUARTER", "Q") + .build(); + // Map used to determine format output for the get_format function private static final Table formats = ImmutableTable.builder() @@ -157,6 +185,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(dayOfWeek(BuiltinFunctionName.DAY_OF_WEEK.getName())); repository.register(dayOfYear(BuiltinFunctionName.DAYOFYEAR)); repository.register(dayOfYear(BuiltinFunctionName.DAY_OF_YEAR)); + repository.register(extract()); repository.register(from_days()); repository.register(from_unixtime()); repository.register(get_format()); @@ -538,6 +567,17 @@ private DefaultFunctionResolver dayOfYear(BuiltinFunctionName dayOfYear) { ); } + private DefaultFunctionResolver extract() { + return define(BuiltinFunctionName.EXTRACT.getName(), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprExtractForTime), + LONG, STRING, TIME), + impl(nullMissingHandling(DateTimeFunction::exprExtract), LONG, STRING, DATE), + impl(nullMissingHandling(DateTimeFunction::exprExtract), LONG, STRING, DATETIME), + impl(nullMissingHandling(DateTimeFunction::exprExtract), LONG, STRING, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunction::exprExtract), LONG, STRING, STRING) + ); + } + /** * FROM_DAYS(LONG). return the date value given the day number N. */ @@ -1260,6 +1300,48 @@ private ExprValue exprDayOfYear(ExprValue date) { return new ExprIntegerValue(date.dateValue().getDayOfYear()); } + /** + * Obtains a formatted long value for a specified part and datetime for the 'extract' function. + * + * @param part is an ExprValue which comes from a defined list of accepted values. + * @param datetime the date to be formatted as an ExprValue. + * @return is a LONG formatted according to the input arguments. + */ + public ExprLongValue formatExtractFunction(ExprValue part, ExprValue datetime) { + String partName = part.stringValue().toUpperCase(); + LocalDateTime arg = datetime.datetimeValue(); + String text = arg.format(DateTimeFormatter.ofPattern( + extract_formats.get(partName), Locale.ENGLISH)); + + return new ExprLongValue(Long.parseLong(text)); + } + + /** + * Implements extract function. Returns a LONG formatted according to the 'part' argument. + * + * @param part Literal that determines the format of the outputted LONG. + * @param datetime The date/datetime to be formatted. + * @return A LONG + */ + private ExprValue exprExtract(ExprValue part, ExprValue datetime) { + return formatExtractFunction(part, datetime); + } + + /** + * Implements extract function. Returns a LONG formatted according to the 'part' argument. + * + * @param part Literal that determines the format of the outputted LONG. + * @param time The time to be formatted. + * @return A LONG + */ + private ExprValue exprExtractForTime(FunctionProperties functionProperties, + ExprValue part, + ExprValue time) { + return formatExtractFunction( + part, + new ExprDatetimeValue(extractDateTime(time, functionProperties))); + } + /** * From_days implementation for ExprValue. * diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index ca0de55cef..09aaf9b64a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -80,6 +80,7 @@ public enum BuiltinFunctionName { DAYOFYEAR(FunctionName.of("dayofyear")), DAY_OF_WEEK(FunctionName.of("day_of_week")), DAY_OF_YEAR(FunctionName.of("day_of_year")), + EXTRACT(FunctionName.of("extract")), FROM_DAYS(FunctionName.of("from_days")), FROM_UNIXTIME(FunctionName.of("from_unixtime")), GET_FORMAT(FunctionName.of("get_format")), diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java new file mode 100644 index 0000000000..338933333a --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -0,0 +1,156 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression.datetime; + +import static java.time.temporal.ChronoField.ALIGNED_WEEK_OF_YEAR; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; + +import java.time.LocalDate; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.data.model.ExprDateValue; +import org.opensearch.sql.data.model.ExprDatetimeValue; +import org.opensearch.sql.data.model.ExprTimeValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionTestBase; +import org.opensearch.sql.expression.FunctionExpression; + +class ExtractTest extends ExpressionTestBase { + + private final String datetimeInput = "2023-02-11 10:11:12.123"; + + private final String timeInput = "10:11:12.123"; + + private final String dateInput = "2023-02-11"; + + private static Stream getDatetimeResultsForExtractFunction() { + return Stream.of( + Arguments.of("DAY_MICROSECOND", 11101112123000L), + Arguments.of("DAY_SECOND", 11101112), + Arguments.of("DAY_MINUTE", 111011), + Arguments.of("DAY_HOUR", 1110) + ); + } + + private static Stream getTimeResultsForExtractFunction() { + return Stream.of( + Arguments.of("MICROSECOND", 123000), + Arguments.of("SECOND", 12), + Arguments.of("MINUTE", 11), + Arguments.of("HOUR", 10), + Arguments.of("SECOND_MICROSECOND", 12123000), + Arguments.of("MINUTE_MICROSECOND", 1112123000), + Arguments.of("MINUTE_SECOND", 1112), + Arguments.of("HOUR_MICROSECOND", 101112123000L), + Arguments.of("HOUR_SECOND", 101112), + Arguments.of("HOUR_MINUTE", 1011) + ); + } + + private static Stream getDateResultsForExtractFunction() { + return Stream.of( + Arguments.of("DAY", 11), + Arguments.of("WEEK", 6), + Arguments.of("MONTH", 2), + Arguments.of("QUARTER", 1), + Arguments.of("YEAR", 2023), + Arguments.of("YEAR_MONTH", 202302) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource({ + "getDatetimeResultsForExtractFunction", + "getTimeResultsForExtractFunction", + "getDateResultsForExtractFunction"}) + public void testExtractWithDatetime(String part, long expected) { + FunctionExpression datetimeExpression = DSL.extract( + DSL.literal(part), + DSL.literal(new ExprDatetimeValue(datetimeInput))); + + assertEquals(LONG, datetimeExpression.type()); + assertEquals(expected, eval(datetimeExpression).longValue()); + assertEquals( + String.format("extract(\"%s\", DATETIME '2023-02-11 10:11:12.123')", part), + datetimeExpression.toString()); + } + + private void datePartWithTimeArgQuery(String part, String time, long expected) { + ExprTimeValue timeValue = new ExprTimeValue(time); + FunctionExpression datetimeExpression = DSL.extract( + functionProperties, + DSL.literal(part), + DSL.literal(timeValue)); + + assertEquals(LONG, datetimeExpression.type()); + assertEquals(expected, + eval(datetimeExpression).longValue()); + } + + + @Test + public void testExtractDatePartWithTimeType() { + datePartWithTimeArgQuery( + "DAY", + timeInput, + LocalDate.now(functionProperties.getQueryStartClock()).getDayOfMonth()); + + datePartWithTimeArgQuery( + "WEEK", + timeInput, + LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR)); + + datePartWithTimeArgQuery( + "MONTH", + timeInput, + LocalDate.now(functionProperties.getQueryStartClock()).getMonthValue()); + + datePartWithTimeArgQuery( + "YEAR", + timeInput, + LocalDate.now(functionProperties.getQueryStartClock()).getYear()); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("getDateResultsForExtractFunction") + public void testExtractWithDate(String part, long expected) { + FunctionExpression datetimeExpression = DSL.extract( + DSL.literal(part), + DSL.literal(new ExprDateValue(dateInput))); + + assertEquals(LONG, datetimeExpression.type()); + assertEquals(expected, eval(datetimeExpression).longValue()); + assertEquals( + String.format("extract(\"%s\", DATE '2023-02-11')", part), + datetimeExpression.toString()); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("getTimeResultsForExtractFunction") + public void testExtractWithTime(String part, long expected) { + FunctionExpression datetimeExpression = DSL.extract( + functionProperties, + DSL.literal(part), + DSL.literal(new ExprTimeValue(timeInput))); + + assertEquals(LONG, datetimeExpression.type()); + assertEquals(expected, eval(datetimeExpression).longValue()); + assertEquals( + String.format("extract(\"%s\", TIME '10:11:12.123')", part), + datetimeExpression.toString()); + } + + private ExprValue eval(Expression expression) { + return expression.valueOf(); + } +} diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index ea594e99dd..ee40925b4d 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1752,6 +1752,75 @@ Example:: +-------------------------------------------------+ +EXTRACT +_______ + +Description +>>>>>>>>>>> + +Usage: extract(part FROM date) returns a LONG with digits in order according to the given 'part' arguments. +The specific format of the returned long is determined by the table below. + +Argument type: PART +PART must be one of the following tokens in the table below. + +The format specifiers found in this table are the same as those found in the `DATE_FORMAT`_ function. +.. list-table:: The following table describes the mapping of a 'part' to a particular format. + :widths: 20 80 + :header-rows: 1 + + * - Part + - Format + * - MICROSECOND + - %f + * - SECOND + - %s + * - MINUTE + - %i + * - HOUR + - %H + * - DAY + - %d + * - WEEK + - %X + * - MONTH + - %m + * - YEAR + - %V + * - SECOND_MICROSECOND + - %s%f + * - MINUTE_MICROSECOND + - %i%s%f + * - MINUTE_SECOND + - %i%s + * - HOUR_MICROSECOND + - %H%i%s%f + * - HOUR_SECOND + - %H%i%s + * - HOUR_MINUTE + - %H%i + * - DAY_MICROSECOND + - %d%H%i%s%f + * - DAY_SECOND + - %d%H%i%s + * - DAY_MINUTE + - %d%H%i + * - DAY_HOUR + - %d%H% + * - YEAR_MONTH + - %V%m + +Return type: LONG + +Example:: + + os> SELECT extract(YEAR_MONTH FROM "2023-02-07 10:11:12"); + fetched rows / total rows = 1/1 + +--------------------------------------------------+ + | extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | + |--------------------------------------------------| + | 202302 | + +--------------------------------------------------+ FROM_DAYS --------- diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 6f7628b9aa..c14fbf4d1c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -487,6 +487,47 @@ public void testHourOfDayWithUnderscores() throws IOException { verifyDataRows(result, rows(17)); } + @Test + public void testExtractWithDatetime() throws IOException { + JSONObject datetimeResult = executeQuery( + String.format( + "SELECT extract(DAY_SECOND FROM datetime(cast(datetime0 AS STRING))) FROM %s LIMIT 1", + TEST_INDEX_CALCS)); + verifyDataRows(datetimeResult, rows(9101735)); + } + + @Test + public void testExtractWithTime() throws IOException { + JSONObject timeResult = executeQuery( + String.format( + "SELECT extract(HOUR_SECOND FROM cast(time0 AS TIME)) FROM %s LIMIT 1", + TEST_INDEX_CALCS)); + verifyDataRows(timeResult, rows(210732)); + + } + + @Test + public void testExtractWithDate() throws IOException { + JSONObject dateResult = executeQuery( + String.format( + "SELECT extract(YEAR_MONTH FROM cast(date0 AS DATE)) FROM %s LIMIT 1", + TEST_INDEX_CALCS)); + verifyDataRows(dateResult, rows(200404)); + } + + @Test + public void testExtractWithDifferentTypesReturnSameResult() throws IOException { + JSONObject dateResult = executeQuery( + String.format("SELECT extract(YEAR_MONTH FROM datetime0) FROM %s LIMIT 1", TEST_INDEX_CALCS)); + + JSONObject datetimeResult = executeQuery( + String.format( + "SELECT extract(YEAR_MONTH FROM date(datetime0)) FROM %s LIMIT 1", + TEST_INDEX_CALCS)); + + dateResult.getJSONArray("datarows").similar(datetimeResult.getJSONArray("datarows")); + } + @Test public void testHourFunctionAliasesReturnTheSameResults() throws IOException { JSONObject result1 = executeQuery("SELECT hour('11:30:00')"); diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 960a54e92e..438902cc1c 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -210,6 +210,7 @@ DEGREES: 'DEGREES'; E: 'E'; EXP: 'EXP'; EXPM1: 'EXPM1'; +EXTRACT: 'EXTRACT'; FLOOR: 'FLOOR'; FROM_DAYS: 'FROM_DAYS'; FROM_UNIXTIME: 'FROM_UNIXTIME'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index a8ddb5a884..a9ccb0c97b 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -309,6 +309,7 @@ functionCall | relevanceFunction #relevanceFunctionCall | highlightFunction #highlightFunctionCall | positionFunction #positionFunctionCall + | extractFunction #extractFunctionCall | getFormatFunction #getFormatFunctionCall ; @@ -323,6 +324,32 @@ getFormatType | TIMESTAMP ; +extractFunction + : EXTRACT LR_BRACKET datetimePart FROM functionArg RR_BRACKET + ; + +datetimePart + : MICROSECOND + | SECOND + | MINUTE + | HOUR + | DAY + | WEEK + | MONTH + | QUARTER + | YEAR + | SECOND_MICROSECOND + | MINUTE_MICROSECOND + | MINUTE_SECOND + | HOUR_MICROSECOND + | HOUR_SECOND + | HOUR_MINUTE + | DAY_MICROSECOND + | DAY_SECOND + | DAY_MINUTE + | DAY_HOUR + | YEAR_MONTH + ; highlightFunction : HIGHLIGHT LR_BRACKET relevanceField (COMMA highlightArg)* RR_BRACKET diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index c024d74f8c..d1afd90246 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -30,6 +30,7 @@ import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DataTypeFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DateLiteralContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DistinctCountFunctionCallContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ExtractFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.FilterClauseContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.FilteredAggregationFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.FunctionArgContext; @@ -487,6 +488,14 @@ private Function buildFunction(String functionName, ); } + @Override + public UnresolvedExpression visitExtractFunctionCall(ExtractFunctionCallContext ctx) { + return new Function( + ctx.extractFunction().EXTRACT().toString(), + getExtractFunctionArguments(ctx)); + } + + private QualifiedName visitIdentifiers(List identifiers) { return new QualifiedName( identifiers.stream() @@ -623,4 +632,13 @@ private List altMultiFieldRelevanceFunctionArguments( fillRelevanceArgs(ctx.relevanceArg(), builder); return builder.build(); } + + private List getExtractFunctionArguments( + ExtractFunctionCallContext ctx) { + List args = Arrays.asList( + new Literal(ctx.extractFunction().datetimePart().getText(), DataType.STRING), + visitFunctionArg(ctx.extractFunction().functionArg()) + ); + return args; + } } diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index ab4f48791b..4b87693037 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -175,6 +175,53 @@ private static Stream nowLikeFunctionsData() { ); } + private static Stream getPartForExtractFunction() { + return Stream.of( + Arguments.of("MICROSECOND"), + Arguments.of("SECOND"), + Arguments.of("MINUTE"), + Arguments.of("HOUR"), + Arguments.of("DAY"), + Arguments.of("WEEK"), + Arguments.of("MONTH"), + Arguments.of("QUARTER"), + Arguments.of("YEAR"), + Arguments.of("SECOND_MICROSECOND"), + Arguments.of("MINUTE_MICROSECOND"), + Arguments.of("MINUTE_SECOND"), + Arguments.of("HOUR_MICROSECOND"), + Arguments.of("HOUR_SECOND"), + Arguments.of("HOUR_MINUTE"), + Arguments.of("DAY_MICROSECOND"), + Arguments.of("DAY_SECOND"), + Arguments.of("DAY_MINUTE"), + Arguments.of("DAY_HOUR"), + Arguments.of("YEAR_MONTH") + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("getPartForExtractFunction") + public void can_parse_extract_function(String part) { + assertNotNull(parser.parse(String.format("SELECT extract(%s FROM \"2023-02-06\")", part))); + } + + private static Stream getInvalidPartForExtractFunction() { + return Stream.of( + Arguments.of("INVALID"), + Arguments.of("\"SECOND\""), + Arguments.of("123") + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("getInvalidPartForExtractFunction") + public void cannot_parse_extract_function_invalid_part(String part) { + assertThrows( + SyntaxCheckException.class, + () -> parser.parse(String.format("SELECT extract(%s FROM \"2023-02-06\")", part))); + } + @Test public void can_parse_weekday_function() { assertNotNull(parser.parse("SELECT weekday('2022-11-18')")); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index 80e7ddb8e5..f55b92dde7 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -192,6 +192,14 @@ public void canBuildFunctionCall() { ); } + @Test + public void canBuildExtractFunctionCall() { + assertEquals( + function("extract", stringLiteral("DAY"), dateLiteral("2023-02-09")).toString(), + buildExprAst("extract(DAY FROM \"2023-02-09\")").toString() + ); + } + @Test public void canBuildGetFormatFunctionCall() { assertEquals(