From 8aa2efcad360574406b20aa914f88026c60b6cb6 Mon Sep 17 00:00:00 2001 From: penghuo Date: Thu, 9 Jul 2020 22:04:29 -0700 Subject: [PATCH 1/3] add date and time support --- .../sql/analysis/ExpressionAnalyzer.java | 3 +- .../sql/ast/dsl/AstDSL.java | 12 ++ .../sql/ast/expression/DataType.java | 28 +++- .../sql/data/model/ExprDateValue.java | 74 ++++++++++ .../sql/data/model/ExprTimeValue.java | 67 +++++++++ .../sql/data/model/ExprTimestampValue.java | 75 ++++++++++ .../sql/data/model/ExprValueUtils.java | 30 ++++ .../sql/expression/DSL.java | 5 + .../expression/config/ExpressionConfig.java | 2 + .../expression/datetime/DateTimeFunction.java | 138 ++++++++++++++++++ .../function/BuiltinFunctionName.java | 5 + .../sql/data/model/DateTimeValueTest.java | 73 +++++++++ .../sql/data/model/ExprValueUtilsTest.java | 25 ++++ .../datetime/DateTimeFunctionTest.java | 74 ++++++++++ docs/user/dql/expressions.rst | 36 +++-- sql/src/main/antlr/OpenDistroSQLLexer.g4 | 1 + sql/src/main/antlr/OpenDistroSQLParser.g4 | 25 ++++ .../sql/sql/parser/AstExpressionBuilder.java | 17 +++ .../sql/parser/AstExpressionBuilderTest.java | 35 +++++ 19 files changed, 709 insertions(+), 16 deletions(-) create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDateValue.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimeValue.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java create mode 100644 core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java create mode 100644 core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunctionTest.java diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java index dd1081bf52..2f9710b7b6 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java @@ -78,7 +78,8 @@ public Expression visitEqualTo(EqualTo node, AnalysisContext context) { @Override public Expression visitLiteral(Literal node, AnalysisContext context) { - return DSL.literal(ExprValueUtils.fromObjectValue(node.getValue())); + return DSL + .literal(ExprValueUtils.fromObjectValue(node.getValue(), node.getType().getCoreType())); } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java index b5096b4e60..4c62c119d4 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java @@ -123,6 +123,18 @@ public static Literal intLiteral(Integer value) { return literal(value, DataType.INTEGER); } + public static Literal dateLiteral(String value) { + return literal(value, DataType.DATE); + } + + public static Literal timeLiteral(String value) { + return literal(value, DataType.TIME); + } + + public static Literal timestampLiteral(String value) { + return literal(value, DataType.TIMESTAMP); + } + public static Literal doubleLiteral(Double value) { return literal(value, DataType.DOUBLE); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java index 51662e6c53..4befa380c0 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java @@ -15,12 +15,28 @@ package com.amazon.opendistroforelasticsearch.sql.ast.expression; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * The DataType defintion in AST. + * Question, could we use {@link ExprCoreType} directly in AST? + */ +@RequiredArgsConstructor public enum DataType { - TYPE_ERROR, - NULL, + TYPE_ERROR(ExprCoreType.UNKNOWN), + NULL(ExprCoreType.UNKNOWN), + + INTEGER(ExprCoreType.INTEGER), + DOUBLE(ExprCoreType.DOUBLE), + STRING(ExprCoreType.STRING), + BOOLEAN(ExprCoreType.BOOLEAN), + + DATE(ExprCoreType.DATE), + TIME(ExprCoreType.TIME), + TIMESTAMP(ExprCoreType.TIMESTAMP); - INTEGER, - DOUBLE, - STRING, - BOOLEAN + @Getter + private final ExprCoreType coreType; } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDateValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDateValue.java new file mode 100644 index 0000000000..dc06b2cfb3 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDateValue.java @@ -0,0 +1,74 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + +/** + * Date Value. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +public class ExprDateValue implements ExprValue { + /** + * todo. only support UTC now. + */ + private static final ZoneId ZONE = ZoneId.of("UTC"); + private final Instant date; + + /** + * Constructor of ExprDateValue. + */ + public ExprDateValue(String date) { + try { + LocalDate localDate = LocalDate.parse(date); + this.date = localDate.atStartOfDay(ZONE).toInstant(); + } catch (DateTimeParseException e) { + throw new SemanticCheckException(String.format("date:%s in unsupported format, please use " + + "yyyy-MM-dd", date)); + } + } + + @Override + public String value() { + return DateTimeFormatter.ISO_LOCAL_DATE.withZone(ZONE).format(date); + } + + @Override + public ExprCoreType type() { + return ExprCoreType.DATE; + } + + @Override + public String toString() { + return String.format("DATE '%s'", value()); + } + + public ZonedDateTime getDate() { + return date.atZone(ZONE); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimeValue.java new file mode 100644 index 0000000000..82fd72a4f1 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimeValue.java @@ -0,0 +1,67 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + +/** + * Time Value. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +public class ExprTimeValue implements ExprValue { + /** + * todo. only support UTC now. + */ + private static final ZoneId ZONE = ZoneId.of("UTC"); + private final LocalTime time; + + /** + * Constructor. + */ + public ExprTimeValue(String time) { + try { + this.time = LocalTime.parse(time); + } catch (DateTimeParseException e) { + throw new SemanticCheckException(String.format("time:%s in unsupported format, please use " + + "HH:mm:ss", time)); + } + } + + @Override + public String value() { + return DateTimeFormatter.ISO_LOCAL_TIME.withZone(ZONE).format(time); + } + + @Override + public ExprCoreType type() { + return ExprCoreType.TIME; + } + + @Override + public String toString() { + return String.format("TIME '%s'", value()); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java new file mode 100644 index 0000000000..c6865bf981 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java @@ -0,0 +1,75 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoUnit; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + +/** + * Timestamp Value. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +public class ExprTimestampValue implements ExprValue { + /** + * todo. only support UTC now. + */ + private static final ZoneId ZONE = ZoneId.of("UTC"); + /** + * todo. only support timestamp in format yyyy-MM-dd HH:mm:ss. + */ + private static final DateTimeFormatter FORMATTER = DateTimeFormatter + .ofPattern("yyyy-MM-dd HH:mm:ss"); + private final Instant timestamp; + + /** + * Constructor. + */ + public ExprTimestampValue(String timestamp) { + try { + this.timestamp = LocalDateTime.parse(timestamp, FORMATTER).atZone(ZONE).toInstant(); + } catch (DateTimeParseException e) { + throw new SemanticCheckException(String.format("timestamp:%s in unsupported format, please " + + "use yyyy-MM-dd HH:mm:ss", timestamp)); + } + + } + + @Override + public String value() { + return FORMATTER.withZone(ZONE).format(timestamp.truncatedTo(ChronoUnit.SECONDS)); + } + + @Override + public ExprCoreType type() { + return ExprCoreType.TIMESTAMP; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", value()); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java index 1f601989fa..ab5ec7a9b6 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java @@ -23,6 +23,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; import com.google.common.annotations.VisibleForTesting; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -117,6 +118,22 @@ public static ExprValue fromObjectValue(Object o) { } } + /** + * Construct ExprValue from Object with ExprCoreType. + */ + public static ExprValue fromObjectValue(Object o, ExprCoreType type) { + switch (type) { + case TIMESTAMP: + return new ExprTimestampValue((String)o); + case DATE: + return new ExprDateValue((String)o); + case TIME: + return new ExprTimeValue((String)o); + default: + return fromObjectValue(o); + } + } + public static Integer getIntegerValue(ExprValue exprValue) { return getNumberValue(exprValue).intValue(); } @@ -149,6 +166,19 @@ public static Boolean getBooleanValue(ExprValue exprValue) { return convert(exprValue, BOOLEAN); } + /** + * Get {@link ZonedDateTime} from ExprValue of Date type. + */ + public static ZonedDateTime getDateValue(ExprValue exprValue) { + if (ExprCoreType.DATE == exprValue.type()) { + return ((ExprDateValue) exprValue).getDate(); + } else { + throw new ExpressionEvaluationException( + String.format("invalid to convert expression with type:%s to type:%s", exprValue.type(), + ExprCoreType.DATE)); + } + } + /** * Get Number Value from {@link ExprValue}. */ diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java index 7af871dc53..e6ccfa0486 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java @@ -81,6 +81,11 @@ public FunctionExpression multiply(Expression... expressions) { repository.compile(BuiltinFunctionName.MULTIPLY.getName(), Arrays.asList(expressions)); } + public FunctionExpression dayofmonth(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.DAYOFMONTH.getName(), Arrays.asList(expressions)); + } + public FunctionExpression divide(Expression... expressions) { return (FunctionExpression) repository.compile(BuiltinFunctionName.DIVIDE.getName(), Arrays.asList(expressions)); diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/config/ExpressionConfig.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/config/ExpressionConfig.java index da9d9ea1d1..f35d136049 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/config/ExpressionConfig.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/config/ExpressionConfig.java @@ -17,6 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AggregatorFunction; +import com.amazon.opendistroforelasticsearch.sql.expression.datetime.DateTimeFunction; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; import com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic.ArithmeticFunction; import com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic.UnaryFunction; @@ -43,6 +44,7 @@ public BuiltinFunctionRepository functionRepository() { UnaryFunction.register(builtinFunctionRepository); UnaryPredicateOperator.register(builtinFunctionRepository); AggregatorFunction.register(builtinFunctionRepository); + DateTimeFunction.register(builtinFunctionRepository); return builtinFunctionRepository; } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java new file mode 100644 index 0000000000..1257a70f1c --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java @@ -0,0 +1,138 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.datetime; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getDateValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.DAYOFMONTH; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature; +import java.util.Collections; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; + +/** + * The definition of date and time functions. + * todo, keep define and unaryImpl internally for now. there are two purpose for doing this + * 1) have the clear interface for function define. + * 2) the implementation should rely on ExprValue. + */ +@UtilityClass +public class DateTimeFunction { + public void register(BuiltinFunctionRepository repository) { + repository.register(dayOfMonth()); + } + + /** + * DAYOFMONTH(DATE). return the day of the month (1-31). + */ + private FunctionResolver dayOfMonth() { + return define(DAYOFMONTH.getName(), + unaryImpl(DateTimeFunction::exprDayOfMonth, INTEGER, DATE) + ); + } + + /** + * Define overloaded function with implementation. + * @param functionName function name. + * @param functions a list of function implementation. + * @return FunctionResolver. + */ + private FunctionResolver define(FunctionName functionName, + Function>... functions) { + + FunctionResolver.FunctionResolverBuilder builder = FunctionResolver.builder(); + builder.functionName(functionName); + for (Function> func : functions) { + Pair functionBuilder = func.apply(functionName); + builder.functionBundle(functionBuilder.getKey(), functionBuilder.getValue()); + } + return builder.build(); + } + + /** + * Unary Function Implementation. + * @param function {@link ExprValue} based unary function. + * @param returnType return type. + * @param argsType argument type. + * + * @return Unary Function Implementation. + */ + private Function> unaryImpl( + Function function, + ExprType returnType, + ExprType argsType) { + + return functionName -> { + FunctionSignature functionSignature = + new FunctionSignature(functionName, Collections.singletonList(argsType)); + FunctionBuilder functionBuilder = + arguments -> new FunctionExpression(functionName, arguments) { + @Override + public ExprValue valueOf(Environment valueEnv) { + ExprValue value = arguments.get(0).valueOf(valueEnv); + if (value.isMissing()) { + return ExprValueUtils.missingValue(); + } else if (value.isNull()) { + return ExprValueUtils.nullValue(); + } else { + return function.apply(value); + } + } + + @Override + public ExprType type() { + return returnType; + } + + @Override + public String toString() { + return String.format("%s(%s)", functionName, + arguments.stream() + .map(Object::toString) + .collect(Collectors.joining(", "))); + } + }; + return Pair.of(functionSignature, functionBuilder); + }; + } + + /** + * Day of Month implementation for ExprValue. + * @param date ExprValue of Date type. + * @return ExprValue. + */ + private ExprValue exprDayOfMonth(ExprValue date) { + return new ExprIntegerValue(getDateValue(date).getMonthValue()); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java index 24bb3aafa5..1ba7057305 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java @@ -39,6 +39,11 @@ public enum BuiltinFunctionName { GTE(FunctionName.of(">=")), LIKE(FunctionName.of("like")), + /** + * Date and Time Functions. + */ + DAYOFMONTH(FunctionName.of("dayofmonth")), + /** * Aggregation Function. */ diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java new file mode 100644 index 0000000000..08599e9b7a --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java @@ -0,0 +1,73 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import org.junit.jupiter.api.Test; + +public class DateTimeValueTest { + + @Test + public void timestampValueInterfaceTest() { + ExprTimeValue timeValue = new ExprTimeValue("01:01:01"); + + assertEquals(TIME, timeValue.type()); + assertEquals("01:01:01", timeValue.value()); + assertEquals("TIME '01:01:01'", timeValue.toString()); + } + + @Test + public void timeValueInterfaceTest() { + ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07 01:01:01"); + + assertEquals(TIMESTAMP, timestampValue.type()); + assertEquals("2020-07-07 01:01:01", timestampValue.value()); + assertEquals("TIMESTAMP '2020-07-07 01:01:01'", timestampValue.toString()); + } + + @Test + public void dateInUnsupportedFormat() { + SemanticCheckException exception = + assertThrows(SemanticCheckException.class, () -> new ExprDateValue("2020-07-07Z")); + assertEquals("date:2020-07-07Z in unsupported format, please use yyyy-MM-dd", + exception.getMessage()); + } + + @Test + public void timeInUnsupportedFormat() { + SemanticCheckException exception = + assertThrows(SemanticCheckException.class, () -> new ExprTimeValue("01:01:0")); + assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss", + exception.getMessage()); + } + + @Test + public void timestampInUnsupportedFormat() { + SemanticCheckException exception = + assertThrows(SemanticCheckException.class, + () -> new ExprTimestampValue("2020-07-07T01:01:01Z")); + assertEquals( + "timestamp:2020-07-07T01:01:01Z in unsupported format, please use yyyy-MM-dd HH:mm:ss", + exception.getMessage()); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java index d0814ff672..f9bd0ff36d 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java @@ -26,6 +26,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import java.time.LocalDate; +import java.time.ZoneId; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; @@ -187,4 +189,27 @@ public void bindingTuples() { } } } + + @Test + public void constructDateAndTimeValue() { + assertEquals(new ExprDateValue("2012-07-07"), + ExprValueUtils.fromObjectValue("2012-07-07", ExprCoreType.DATE)); + assertEquals(new ExprTimeValue("01:01:01"), + ExprValueUtils.fromObjectValue("01:01:01", ExprCoreType.TIME)); + assertEquals(new ExprTimestampValue("2012-07-07 01:01:01"), + ExprValueUtils.fromObjectValue("2012-07-07 01:01:01", ExprCoreType.TIMESTAMP)); + } + + @Test + public void getDateFromValue() { + assertEquals(LocalDate.parse("2012-07-07").atStartOfDay(ZoneId.of("UTC")), + ExprValueUtils.getDateValue(new ExprDateValue( + "2012-07-07"))); + + ExpressionEvaluationException exception = + assertThrows(ExpressionEvaluationException.class, + () -> ExprValueUtils.getDateValue(integerValue(1))); + assertEquals("invalid to convert expression with type:INTEGER to type:DATE", + exception.getMessage()); + } } \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunctionTest.java new file mode 100644 index 0000000000..32e4f9b257 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -0,0 +1,74 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.datetime; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.missingValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.nullValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDateValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; +import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class DateTimeFunctionTest extends ExpressionTestBase { + @Mock + Environment env; + + @Mock + Expression nullRef; + + @Mock + Expression missingRef; + + @BeforeEach + public void setup() { + when(nullRef.valueOf(env)).thenReturn(nullValue()); + when(missingRef.valueOf(env)).thenReturn(missingValue()); + } + + @Test + public void dayOfMonth() { + when(nullRef.type()).thenReturn(DATE); + when(missingRef.type()).thenReturn(DATE); + + FunctionExpression expression = dsl.dayofmonth(DSL.literal(new ExprDateValue("2020-07-07"))); + assertEquals(INTEGER, expression.type()); + assertEquals("dayofmonth(DATE '2020-07-07')", expression.toString()); + assertEquals(integerValue(7), eval(expression)); + assertEquals(nullValue(), eval(dsl.dayofmonth(nullRef))); + assertEquals(missingValue(), eval(dsl.dayofmonth(missingRef))); + } + + private ExprValue eval(Expression expression) { + return expression.valueOf(env); + } +} \ No newline at end of file diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index c81f6ee6f7..6f9f7436b3 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -27,28 +27,33 @@ A literal is a symbol that represents a value. The most common literal values in 1. Numeric literals: specify numeric values such as integer and floating-point numbers. 2. String literals: specify a string enclosed by single or double quotes. 3. Boolean literals: ``true`` or ``false``. +4. Date and Time literals +4.1. Date literals: DATE 'YYYY-MM-DD' represent the date. +4.2. Date literals: TIME 'hh:mm:ss' represent the time. +4.3. Date literals: TIMESTAMP 'YYYY-MM-DD hh:mm:ss' represent the date. Examples -------- Here is an example for different type of literals:: - od> SELECT 123, 'hello', false, -4.567; + od> SELECT 123, 'hello', false, -4.567, DATE '2020-07-07', TIME '01:01:01', TIMESTAMP '2020-07-07 01:01:01'; fetched rows / total rows = 1/1 - +-------+-----------+---------+----------+ - | 123 | "hello" | false | -4.567 | - |-------+-----------+---------+----------| - | 123 | hello | False | -4.567 | - +-------+-----------+---------+----------+ + +-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------+ + | 123 | "hello" | false | -4.567 | DATE '2020-07-07' | TIME '01:01:01' | TIMESTAMP '2020-07-07 01:01:01' | + |-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------| + | 123 | hello | False | -4.567 | 2020-07-07 | 01:01:01 | 2020-07-07 01:01:01 | + +-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------+ Limitations ----------- The current implementation has the following limitations at the moment: -1. Only literals of data types listed as above are supported for now. Other type of literals, such as date and NULL, will be added in future. +1. Only literals of data types listed as above are supported for now. Other type of literals, such as NULL, will be added in future. 2. Expression of literals, such as arithmetic expressions, will be supported later. 3. Standard ANSI ``VALUES`` clause is not supported, although the ``SELECT`` literal example above is implemented by a Values operator internally. +4. Date and Time literals only support ``date format`` listed above. Arithmetic Expressions ====================== @@ -111,8 +116,8 @@ Null Handling If any argument is missing or null, the final result of evaluation will be missing or null accordingly. -Examples --------- +Examples arithmetic function +---------------------------- Here is an example for different type of arithmetic expressions:: @@ -124,6 +129,19 @@ Here is an example for different type of arithmetic expressions:: | 1.234 | 5 | +---------------+---------------------+ +Examples date function +---------------------- + +Here is an example for different type of arithmetic expressions:: + + od> SELECT dayofmonth(DATE '2020-07-07'); + fetched rows / total rows = 1/1 + +---------------------------------+ + | dayofmonth(DATE '2020-07-07') | + |---------------------------------| + | 7 | + +---------------------------------+ + Limitations ----------- diff --git a/sql/src/main/antlr/OpenDistroSQLLexer.g4 b/sql/src/main/antlr/OpenDistroSQLLexer.g4 index 61c13a3a30..f8f6a6c2ac 100644 --- a/sql/src/main/antlr/OpenDistroSQLLexer.g4 +++ b/sql/src/main/antlr/OpenDistroSQLLexer.g4 @@ -186,6 +186,7 @@ SINH: 'SINH'; SQRT: 'SQRT'; SUBTRACT: 'SUBTRACT'; TAN: 'TAN'; +TIME: 'TIME'; TIMESTAMP: 'TIMESTAMP'; UPPER: 'UPPER'; diff --git a/sql/src/main/antlr/OpenDistroSQLParser.g4 b/sql/src/main/antlr/OpenDistroSQLParser.g4 index fb2d7eb83e..045fed1772 100644 --- a/sql/src/main/antlr/OpenDistroSQLParser.g4 +++ b/sql/src/main/antlr/OpenDistroSQLParser.g4 @@ -76,6 +76,7 @@ constant | sign? decimalLiteral #signedDecimal | sign? realLiteral #signedReal | booleanLiteral #boolean + | datetimeLiteral #datetime // Doesn't support the following types for now //| nullLiteral #null //| BIT_STRING @@ -114,6 +115,25 @@ nullLiteral : NULL_LITERAL ; +// Date and Time Literal, follow ANSI 92 +datetimeLiteral + : dateLiteral + | timeLiteral + | timestampLiteral + ; + +dateLiteral + : DATE date=stringLiteral + ; + +timeLiteral + : TIME time=stringLiteral + ; + +timestampLiteral + : TIMESTAMP timestamp=stringLiteral + ; + // Expressions, predicates // Simplified approach for expression @@ -142,6 +162,11 @@ functionCall scalarFunctionName : ABS + | dateTimeFunctionName + ; + +dateTimeFunctionName + : DAYOFMONTH ; functionArgs diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java index 057d2dbf2f..606813eb41 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java @@ -27,6 +27,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.NestedExpressionAtomContext; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParserBaseVisitor; import java.util.Arrays; @@ -82,4 +83,20 @@ public UnresolvedExpression visitBoolean(BooleanContext ctx) { return AstDSL.booleanLiteral(Boolean.valueOf(ctx.getText())); } + @Override + public UnresolvedExpression visitDateLiteral(OpenDistroSQLParser.DateLiteralContext ctx) { + return AstDSL.dateLiteral(unquoteIdentifier(ctx.date.getText())); + } + + @Override + public UnresolvedExpression visitTimeLiteral(OpenDistroSQLParser.TimeLiteralContext ctx) { + return AstDSL.timeLiteral(unquoteIdentifier(ctx.time.getText())); + } + + @Override + public UnresolvedExpression visitTimestampLiteral( + OpenDistroSQLParser.TimestampLiteralContext ctx) { + return AstDSL.timestampLiteral(unquoteIdentifier(ctx.timestamp.getText())); + } + } diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java index cd304b5d38..5dece1f09f 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -17,10 +17,13 @@ package com.amazon.opendistroforelasticsearch.sql.sql.parser; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.dateLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.doubleLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.function; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.timeLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.timestampLiteral; import static org.junit.jupiter.api.Assertions.assertEquals; import com.amazon.opendistroforelasticsearch.sql.ast.Node; @@ -67,6 +70,30 @@ public void canBuildBooleanLiteral() { ); } + @Test + public void canBuildDateLiteral() { + assertEquals( + dateLiteral("2020-07-07"), + buildExprAst("DATE '2020-07-07'") + ); + } + + @Test + public void canBuildTimeLiteral() { + assertEquals( + timeLiteral("11:30:45"), + buildExprAst("TIME '11:30:45'") + ); + } + + @Test + public void canBuildTimestampLiteral() { + assertEquals( + timestampLiteral("2020-07-07 11:30:45"), + buildExprAst("TIMESTAMP '2020-07-07 11:30:45'") + ); + } + @Test public void canBuildArithmeticExpression() { assertEquals( @@ -107,6 +134,14 @@ public void canBuildNestedFunctionCall() { ); } + @Test + public void canBuildDateAndTimeFunctionCall() { + assertEquals( + function("dayofmonth", dateLiteral("2020-07-07")), + buildExprAst("dayofmonth(DATE '2020-07-07')") + ); + } + private Node buildExprAst(String expr) { OpenDistroSQLLexer lexer = new OpenDistroSQLLexer(new CaseInsensitiveCharStream(expr)); OpenDistroSQLParser parser = new OpenDistroSQLParser(new CommonTokenStream(lexer)); From e6c6b6b54fdc766ab9c18ac46352c52dc4ac2af4 Mon Sep 17 00:00:00 2001 From: penghuo Date: Thu, 9 Jul 2020 22:14:02 -0700 Subject: [PATCH 2/3] update doc --- docs/user/dql/expressions.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 6f9f7436b3..780abb1f8d 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -28,9 +28,10 @@ A literal is a symbol that represents a value. The most common literal values in 2. String literals: specify a string enclosed by single or double quotes. 3. Boolean literals: ``true`` or ``false``. 4. Date and Time literals -4.1. Date literals: DATE 'YYYY-MM-DD' represent the date. -4.2. Date literals: TIME 'hh:mm:ss' represent the time. -4.3. Date literals: TIMESTAMP 'YYYY-MM-DD hh:mm:ss' represent the date. + + 4.1. Date literals: DATE 'YYYY-MM-DD' represent the date. + 4.2. Date literals: TIME 'hh:mm:ss' represent the time. + 4.3. Date literals: TIMESTAMP 'YYYY-MM-DD hh:mm:ss' represent the date. Examples -------- @@ -53,7 +54,7 @@ The current implementation has the following limitations at the moment: 1. Only literals of data types listed as above are supported for now. Other type of literals, such as NULL, will be added in future. 2. Expression of literals, such as arithmetic expressions, will be supported later. 3. Standard ANSI ``VALUES`` clause is not supported, although the ``SELECT`` literal example above is implemented by a Values operator internally. -4. Date and Time literals only support ``date format`` listed above. +4. Date and Time literals only support DATE_FORMAT listed above. Arithmetic Expressions ====================== @@ -116,7 +117,7 @@ Null Handling If any argument is missing or null, the final result of evaluation will be missing or null accordingly. -Examples arithmetic function +Arithmetic function examples ---------------------------- Here is an example for different type of arithmetic expressions:: @@ -129,7 +130,7 @@ Here is an example for different type of arithmetic expressions:: | 1.234 | 5 | +---------------+---------------------+ -Examples date function +Date function examples ---------------------- Here is an example for different type of arithmetic expressions:: From 698718868e6d32fc8739e3e2104e18e2c3092806 Mon Sep 17 00:00:00 2001 From: penghuo Date: Thu, 9 Jul 2020 22:15:13 -0700 Subject: [PATCH 3/3] update doc --- docs/user/dql/expressions.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 780abb1f8d..ea5d9160ba 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -27,11 +27,7 @@ A literal is a symbol that represents a value. The most common literal values in 1. Numeric literals: specify numeric values such as integer and floating-point numbers. 2. String literals: specify a string enclosed by single or double quotes. 3. Boolean literals: ``true`` or ``false``. -4. Date and Time literals - - 4.1. Date literals: DATE 'YYYY-MM-DD' represent the date. - 4.2. Date literals: TIME 'hh:mm:ss' represent the time. - 4.3. Date literals: TIMESTAMP 'YYYY-MM-DD hh:mm:ss' represent the date. +4. Date and Time literals: DATE 'YYYY-MM-DD' represent the date, TIME 'hh:mm:ss' represent the time, TIMESTAMP 'YYYY-MM-DD hh:mm:ss' represent the timestamp. Examples --------