diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java index 93571dabe4..82dee84297 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java @@ -39,6 +39,7 @@ public enum ScalarFunction implements TypeExpression { ASIN(func(T(NUMBER)).to(T)), ATAN(func(T(NUMBER)).to(T)), ATAN2(func(T(NUMBER), NUMBER).to(T)), + CAST(), CBRT(func(T(NUMBER)).to(T)), CEIL(func(T(NUMBER)).to(T)), CONCAT(), // TODO: varargs support required diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java index b0e0054547..be436b3e3a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java @@ -16,12 +16,10 @@ package com.amazon.opendistroforelasticsearch.sql.executor.format; import com.alibaba.druid.sql.ast.expr.SQLCaseExpr; -import com.alibaba.druid.sql.ast.expr.SQLCastExpr; import com.amazon.opendistroforelasticsearch.sql.domain.Field; import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; import com.amazon.opendistroforelasticsearch.sql.domain.Query; -import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; import com.amazon.opendistroforelasticsearch.sql.domain.Select; import com.amazon.opendistroforelasticsearch.sql.domain.TableOnJoinSelect; import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMapping; @@ -324,14 +322,10 @@ private Schema.Type fetchMethodReturnType(Field field) { // TODO: return type information is disconnected from the function definitions in SQLFunctions. // Refactor SQLFunctions to have functions self-explanatory (types, scripts) and pluggable // (similar to Strategy pattern) - if (field.getExpression() instanceof SQLCastExpr) { - return SQLFunctions.getCastFunctionReturnType( - ((SQLCastExpr) field.getExpression()).getDataType().getName()); - } else if (field.getExpression() instanceof SQLCaseExpr) { + if (field.getExpression() instanceof SQLCaseExpr) { return Schema.Type.TEXT; } - return SQLFunctions.getScriptFunctionReturnType( - ((ScriptMethodField) field).getFunctionName()); + return SQLFunctions.getScriptFunctionReturnType(field); } default: throw new UnsupportedOperationException( diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/CastParser.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/CastParser.java deleted file mode 100644 index 53661647a1..0000000000 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/CastParser.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.parser; - -import com.alibaba.druid.sql.ast.expr.SQLCastExpr; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; -import com.google.common.base.Joiner; - -import java.util.ArrayList; -import java.util.List; - -/** - * Created by leonlu on 2017/9/21. - */ -public class CastParser { - - private int fieldId = 0; - - private enum DataType { - INT, LONG, FLOAT, DOUBLE, STRING, DATETIME - } - - private SQLCastExpr castExpr; - private String alias; - private String tableAlias; - - public CastParser(SQLCastExpr castExpr, String alias, String tableAlias) { - this.castExpr = castExpr; - this.alias = alias; - this.tableAlias = tableAlias; - } - - public String parse(boolean isReturn) throws SqlParseException { - List result = new ArrayList<>(); - - String dataType = castExpr.getDataType().getName().toUpperCase(); - String fileName = String.format("doc['%s'].value", Util.expr2Object(castExpr.getExpr())); - String name = "cast_field" + (++fieldId); - - try { - if (DataType.valueOf(dataType) == DataType.INT) { - result.add(String.format("def %s = Double.parseDouble(%s.toString()).intValue()", name, fileName)); - } else if (DataType.valueOf(dataType) == DataType.LONG) { - result.add(String.format("def %s = Double.parseDouble(%s.toString()).longValue()", name, fileName)); - } else if (DataType.valueOf(dataType) == DataType.FLOAT) { - result.add(String.format("def %s = Double.parseDouble(%s.toString()).floatValue()", name, fileName)); - } else if (DataType.valueOf(dataType) == DataType.DOUBLE) { - result.add(String.format("def %s = Double.parseDouble(%s.toString()).doubleValue()", name, fileName)); - } else if (DataType.valueOf(dataType) == DataType.STRING) { - result.add(String.format("def %s = %s.toString()", name, fileName)); - } else if (DataType.valueOf(dataType) == DataType.DATETIME) { - result.add(String.format("def %s = new Date(Double.parseDouble(%s.toString()).longValue())", - name, fileName)); - } else { - throw new SqlParseException("not support cast to data type:" + dataType); - } - if (isReturn) { - result.add("return " + name); - } - - return Joiner.on("; ").join(result); - } catch (Exception ex) { - throw new SqlParseException(String.format("field cast to type: %s failed. error:%s", - dataType, ex.getMessage())); - } - } -} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java index f149f99d54..de2020daa6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java @@ -118,11 +118,9 @@ private Field makeFieldImpl(SQLExpr expr, String alias, String tableAlias) throw if (alias == null) { alias = "cast_" + castExpr.getExpr().toString(); } - String scriptCode = new CastParser(castExpr, alias, tableAlias).parse(true); - List methodParameters = new ArrayList<>(); - methodParameters.add(new KVValue(alias)); - methodParameters.add(new KVValue(scriptCode)); - return new MethodField("script", methodParameters, null, alias); + ArrayList methodParameters = new ArrayList<>(); + methodParameters.add(((SQLCastExpr) expr).getExpr()); + return makeMethodField("CAST", methodParameters, null, alias, tableAlias, true); } else if (expr instanceof SQLNumericLiteralExpr) { SQLMethodInvokeExpr methodInvokeExpr = new SQLMethodInvokeExpr("assign", null); methodInvokeExpr.addParameter(expr); @@ -344,7 +342,12 @@ public MethodField makeMethodField(String name, List arguments, SQLAggr String scriptCode = new CaseWhenParser((SQLCaseExpr) object, alias, tableAlias).parse(); paramers.add(new KVValue("script", new SQLCharExpr(scriptCode))); } else if (object instanceof SQLCastExpr) { - String scriptCode = new CastParser((SQLCastExpr) object, alias, tableAlias).parse(false); + String castName = sqlFunctions.nextId("cast"); + List methodParameters = new ArrayList<>(); + methodParameters.add(new KVValue(((SQLCastExpr) object).getExpr().toString())); + String castType = ((SQLCastExpr) object).getDataType().getName(); + String scriptCode = sqlFunctions.getCastScriptStatement(castName, castType, methodParameters); + methodParameters.add(new KVValue(scriptCode)); paramers.add(new KVValue("script", new SQLCharExpr(scriptCode))); } else if (object instanceof SQLAggregateExpr) { SQLObject parent = object.getParent(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/WhereParser.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/WhereParser.java index ea2981da32..762c3a2c80 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/WhereParser.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/WhereParser.java @@ -20,6 +20,7 @@ import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; import com.alibaba.druid.sql.ast.expr.SQLBooleanExpr; +import com.alibaba.druid.sql.ast.expr.SQLCastExpr; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLInListExpr; @@ -98,8 +99,6 @@ public Where findWhere() throws SqlParseException { } public void parseWhere(SQLExpr expr, Where where) throws SqlParseException { - - if (expr instanceof SQLBinaryOpExpr) { SQLBinaryOpExpr bExpr = (SQLBinaryOpExpr) expr; if (explainSpecialCondWithBothSidesAreLiterals(bExpr, where)) { @@ -199,7 +198,8 @@ private boolean isCond(SQLBinaryOpExpr expr) { } return leftSide instanceof SQLIdentifierExpr || leftSide instanceof SQLPropertyExpr - || leftSide instanceof SQLVariantRefExpr; + || leftSide instanceof SQLVariantRefExpr + || leftSide instanceof SQLCastExpr; } private boolean isAllowedMethodOnConditionLeft(SQLMethodInvokeExpr method, SQLBinaryOperator operator) { @@ -233,6 +233,7 @@ private void routeCond(SQLBinaryOpExpr bExpr, SQLExpr sub, Where where) throws S private void explainCond(String opear, SQLExpr expr, Where where) throws SqlParseException { if (expr instanceof SQLBinaryOpExpr) { SQLBinaryOpExpr soExpr = (SQLBinaryOpExpr) expr; + boolean methodAsOpear = false; boolean isNested = false; @@ -522,11 +523,23 @@ private MethodField parseSQLMethodInvokeExprWithFunctionInWhere(SQLMethodInvokeE return methodField; } + private MethodField parseSQLCastExprWithFunctionInWhere(SQLCastExpr soExpr) throws SqlParseException { + ArrayList parameters = new ArrayList<>(); + parameters.add(soExpr.getExpr()); + return fieldMaker.makeMethodField( + "CAST", + parameters, + null, + null, + query != null ? query.getFrom().getAlias() : null, + false + ); + } + private SQLMethodInvokeExpr parseSQLBinaryOpExprWhoIsConditionInWhere(SQLBinaryOpExpr soExpr) throws SqlParseException { - if (!(soExpr.getLeft() instanceof SQLMethodInvokeExpr - || soExpr.getRight() instanceof SQLMethodInvokeExpr)) { + if (bothSideAreNotFunction(soExpr) && bothSidesAreNotCast(soExpr)) { return null; } @@ -567,6 +580,13 @@ private SQLMethodInvokeExpr parseSQLBinaryOpExprWhoIsConditionInWhere(SQLBinaryO rightMethod = parseSQLMethodInvokeExprWithFunctionInWhere((SQLMethodInvokeExpr) soExpr.getRight()); } + if (soExpr.getLeft() instanceof SQLCastExpr) { + leftMethod = parseSQLCastExprWithFunctionInWhere((SQLCastExpr) soExpr.getLeft()); + } + if (soExpr.getRight() instanceof SQLCastExpr) { + rightMethod = parseSQLCastExprWithFunctionInWhere((SQLCastExpr) soExpr.getRight()); + } + String v1 = leftMethod.getParams().get(0).value.toString(); String v1Dec = leftMethod.getParams().size() == 2 ? leftMethod.getParams().get(1).value.toString() + ";" : ""; @@ -588,6 +608,14 @@ private SQLMethodInvokeExpr parseSQLBinaryOpExprWhoIsConditionInWhere(SQLBinaryO } + private Boolean bothSideAreNotFunction(SQLBinaryOpExpr soExpr) { + return !(soExpr.getLeft() instanceof SQLMethodInvokeExpr || soExpr.getRight() instanceof SQLMethodInvokeExpr); + } + + private Boolean bothSidesAreNotCast(SQLBinaryOpExpr soExpr) { + return !(soExpr.getLeft() instanceof SQLCastExpr || soExpr.getRight() instanceof SQLCastExpr); + } + private Object[] getMethodValuesWithSubQueries(SQLMethodInvokeExpr method) throws SqlParseException { List values = new ArrayList<>(); for (SQLExpr innerExpr : method.getParameters()) { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DefaultQueryAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DefaultQueryAction.java index 90030d081d..fa18ca2677 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DefaultQueryAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DefaultQueryAction.java @@ -23,7 +23,6 @@ import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; import com.amazon.opendistroforelasticsearch.sql.domain.Order; -import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; import com.amazon.opendistroforelasticsearch.sql.domain.Select; import com.amazon.opendistroforelasticsearch.sql.domain.Where; import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; @@ -273,8 +272,7 @@ private ScriptSortType getScriptSortType(Order order) { scriptFunctionReturnType = SQLFunctions.getCastFunctionReturnType( ((SQLCastExpr) order.getSortField().getExpression()).getDataType().getName()); } else { - ScriptMethodField smf = (ScriptMethodField) order.getSortField(); - scriptFunctionReturnType = SQLFunctions.getScriptFunctionReturnType(smf.getFunctionName()); + scriptFunctionReturnType = SQLFunctions.getScriptFunctionReturnType(order.getSortField()); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java index 7d62524644..33565db746 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java @@ -18,6 +18,7 @@ import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBooleanExpr; +import com.alibaba.druid.sql.ast.expr.SQLCastExpr; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLNullExpr; @@ -25,8 +26,11 @@ import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; import com.alibaba.druid.sql.ast.expr.SQLTextLiteralExpr; import com.alibaba.druid.sql.ast.expr.SQLVariantRefExpr; +import com.amazon.opendistroforelasticsearch.sql.domain.Field; import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; +import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; import com.google.common.base.Joiner; import com.google.common.base.Strings; @@ -84,7 +88,7 @@ public class SQLFunctions { "if", "ifnull", "isnull" ); - private static final Set utilityFunctions = Sets.newHashSet("field", "assign"); + private static final Set utilityFunctions = Sets.newHashSet("field", "assign", "cast"); public static final Set builtInFunctions = Stream.of( numberOperators, @@ -117,9 +121,15 @@ public static boolean isFunctionTranslatedToScript(String function) { } public Tuple function(String methodName, List paramers, String name, - boolean returnValue) { + boolean returnValue) throws SqlParseException { Tuple functionStr = null; switch (methodName.toLowerCase()) { + case "cast": { + SQLCastExpr castExpr = (SQLCastExpr) ((SQLIdentifierExpr) paramers.get(0).value).getParent(); + String typeName = castExpr.getDataType().getName(); + functionStr = cast(typeName, paramers); + break; + } case "lower": { functionStr = lower( (SQLExpr) paramers.get(0).value, @@ -401,6 +411,12 @@ public String getLocaleForCaseChangingFunction(List paramers) { return locale; } + public Tuple cast(String castType, List paramers) throws SqlParseException { + String name = nextId("cast"); + return new Tuple<>(name, getCastScriptStatement(name, castType, paramers)); + } + + public Tuple upper(SQLExpr field, String locale, String valueName) { String name = nextId("upper"); @@ -929,15 +945,40 @@ private Tuple isnull(SQLExpr expr) { return new Tuple<>(name, def(name, resultStr)); } + public String getCastScriptStatement(String name, String castType, List paramers) + throws SqlParseException { + String castFieldName = String.format("doc['%s'].value", paramers.get(0).toString()); + switch (StringUtils.toUpper(castType)) { + case "INT": + return String.format("def %s = Double.parseDouble(%s.toString()).intValue()", name, castFieldName); + case "LONG": + return String.format("def %s = Double.parseDouble(%s.toString()).longValue()", name, castFieldName); + case "FLOAT": + return String.format("def %s = Double.parseDouble(%s.toString()).floatValue()", name, castFieldName); + case "DOUBLE": + return String.format("def %s = Double.parseDouble(%s.toString()).doubleValue()", name, castFieldName); + case "STRING": + return String.format("def %s = %s.toString()", name, castFieldName); + case "DATETIME": + return String.format("def %s = DateTimeFormatter.ofPattern(\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\").format(" + + "DateTimeFormatter.ISO_DATE_TIME.parse(%s.toString()))", name, castFieldName); + default: + throw new SqlParseException("Unsupported cast type " + castType); + } + } + /** * Returns return type of script function. This is simple approach, that might be not the best solution in the long * term. For example - for JDBC, if the column type in index is INTEGER, and the query is "select column+5", current * approach will return type of result column as DOUBLE, although there is enough information to understand that * it might be safely treated as INTEGER. */ - public static Schema.Type getScriptFunctionReturnType(String functionName) { - functionName = functionName.toLowerCase(); - + public static Schema.Type getScriptFunctionReturnType(Field field) { + String functionName = ((ScriptMethodField) field).getFunctionName().toLowerCase(); + if (functionName.equals("cast")) { + String castType = ((SQLCastExpr) field.getExpression()).getDataType().getName(); + return getCastFunctionReturnType(castType); + } if (dateFunctions.contains(functionName) || stringOperators.contains(functionName)) { return Schema.Type.TEXT; } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java index a227b49c46..80e61cb381 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java @@ -30,7 +30,18 @@ import org.junit.Test; import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.TimeZone; import java.util.stream.IntStream; import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; @@ -38,6 +49,10 @@ import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvDouble; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvInt; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvString; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.closeTo; @@ -60,6 +75,8 @@ public class SQLFunctionsIT extends SQLIntegTestCase { @Override protected void init() throws Exception { loadIndex(Index.ACCOUNT); + loadIndex(Index.ONLINE); + loadIndex(Index.DATE); } @Test @@ -202,6 +219,7 @@ public void castIntFieldToStringWithAliasTest() throws IOException { for (int i = 0; i < hits.length; ++i) { Assert.assertThat(hits[i].getFields().get("cast_string_alias").getValue(), is("9838")); } + } @Test @@ -210,12 +228,11 @@ public void castIntFieldToFloatWithoutAliasJdbcFormatTest() { "SELECT CAST(balance AS FLOAT) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY balance DESC LIMIT 1"); - String float_type_cast = "{\"name\":\"cast_balance\",\"type\":\"float\"}"; - assertEquals(response.getJSONArray("schema").get(0).toString(), float_type_cast); - Assert.assertThat( - response.getJSONArray("datarows") - .getJSONArray(0).getFloat(0), - equalTo(49989.0F)); + verifySchema(response, + schema("cast_balance", null, "float")); + + verifyDataRows(response, + rows(49989)); } @Test @@ -224,13 +241,11 @@ public void castIntFieldToFloatWithAliasJdbcFormatTest() { "SELECT CAST(balance AS FLOAT) AS jdbc_float_alias " + "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY jdbc_float_alias LIMIT 1"); - String float_type_cast = "{\"name\":\"jdbc_float_alias\",\"type\":\"float\"}"; - assertEquals(response.getJSONArray("schema").get(0).toString(), float_type_cast); - Assert.assertThat( - response.getJSONArray("datarows") - .getJSONArray(0).getFloat(0), - equalTo(1011.0F)); + verifySchema(response, + schema("jdbc_float_alias", null, "float")); + verifyDataRows(response, + rows(1011)); } @Test @@ -254,6 +269,7 @@ public void castIntFieldToDoubleWithAliasOrderByTest() throws IOException { for (int i = 0; i < hits.length; ++i) { Assert.assertThat(hits[i].getFields().get("alias").getValue(), is(40.0)); } + } @Test @@ -262,18 +278,15 @@ public void castIntFieldToFloatWithoutAliasJdbcFormatGroupByTest() { "SELECT CAST(balance AS FLOAT) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY balance DESC LIMIT 5"); - assertEquals("CAST(balance AS FLOAT)", response.query("/schema/0/name")); - assertNull(response.query("/schema/0/alias")); - assertEquals("float", response.query("/schema/0/type")); - - Float[] expectedOutput = new Float[] {22026.0F, 23285.0F, 36038.0F, 39063.0F, 45493.0F}; - assertEquals(5, response.getJSONArray("datarows").length()); - for (int i = 0; i < response.getJSONArray("datarows").length(); ++i) { - Assert.assertThat( - response.getJSONArray("datarows") - .getJSONArray(i).getFloat(0), - equalTo(expectedOutput[i])); - } + verifySchema(response, + schema("CAST(balance AS FLOAT)", null, "float")); + + verifyDataRows(response, + rows(22026), + rows(23285), + rows(36038), + rows(39063), + rows(45493)); } @Test @@ -282,18 +295,15 @@ public void castIntFieldToFloatWithAliasJdbcFormatGroupByTest() { "SELECT CAST(balance AS FLOAT) AS jdbc_float_alias " + "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY jdbc_float_alias ASC LIMIT 5"); - assertEquals("jdbc_float_alias", response.query("/schema/0/name")); - assertEquals("jdbc_float_alias", response.query("/schema/0/alias")); - assertEquals("float", response.query("/schema/0/type")); - - Float[] expectedOutput = new Float[] {22026.0F, 23285.0F, 36038.0F, 39063.0F, 45493.0F}; - assertEquals(5, response.getJSONArray("datarows").length()); - for (int i = 0; i < response.getJSONArray("datarows").length(); ++i) { - Assert.assertThat( - response.getJSONArray("datarows") - .getJSONArray(i).getFloat(0), - equalTo(expectedOutput[i])); - } + verifySchema(response, + schema("jdbc_float_alias", "jdbc_float_alias", "float")); + + verifyDataRows(response, + rows("22026.0"), + rows("23285.0"), + rows("36038.0"), + rows("39063.0"), + rows("45493.0")); } @Test @@ -302,18 +312,136 @@ public void castIntFieldToDoubleWithAliasJdbcFormatGroupByTest() { "SELECT CAST(age AS DOUBLE) AS jdbc_double_alias " + "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY jdbc_double_alias DESC LIMIT 5"); - assertEquals("jdbc_double_alias", response.query("/schema/0/name")); - assertEquals("jdbc_double_alias", response.query("/schema/0/alias")); - assertEquals("double", response.query("/schema/0/type")); + verifySchema(response, + schema("jdbc_double_alias", "jdbc_double_alias", "double")); - Double[] expectedOutput = new Double[] {31.0, 39.0, 26.0, 32.0, 35.0}; - assertEquals(5, response.getJSONArray("datarows").length()); - for (int i = 0; i < response.getJSONArray("datarows").length(); ++i) { - Assert.assertThat( - response.getJSONArray("datarows") - .getJSONArray(i).getDouble(0), - equalTo(expectedOutput[i])); - } + verifyDataRows(response, + rows("31.0"), + rows("39.0"), + rows("26.0"), + rows("32.0"), + rows("35.0")); + } + + @Test + public void castKeywordFieldToDatetimeWithoutAliasJdbcFormatTest() { + JSONObject response = executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) FROM " + + TestsConstants.TEST_INDEX_DATE + " ORDER BY date_keyword"); + + verifySchema(response, schema("cast_date_keyword", null, "date")); + + verifyDataRows(response, + rows("2014-08-19T07:09:13.434Z"), + rows("2019-09-25T02:04:13.469Z")); + } + + @Test + public void castKeywordFieldToDatetimeWithAliasJdbcFormatTest() { + JSONObject response = executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) AS test_alias FROM " + + TestsConstants.TEST_INDEX_DATE + " ORDER BY date_keyword"); + + verifySchema(response, schema("test_alias", null, "date")); + + verifyDataRows(response, + rows("2014-08-19T07:09:13.434Z"), + rows("2019-09-25T02:04:13.469Z")); + } + + @Test + public void castFieldToDatetimeWithWhereClauseJdbcFormatTest() { + JSONObject response = executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) FROM " + + TestsConstants.TEST_INDEX_DATE + " WHERE date_keyword IS NOT NULL ORDER BY date_keyword"); + + verifySchema(response, schema("cast_date_keyword", null, "date")); + + verifyDataRows(response, + rows("2014-08-19T07:09:13.434Z"), + rows("2019-09-25T02:04:13.469Z")); + } + + @Test + public void castFieldToDatetimeWithGroupByJdbcFormatTest() { + JSONObject response = executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) AS test_alias FROM " + + TestsConstants.TEST_INDEX_DATE + " GROUP BY test_alias DESC"); + + verifySchema(response, schema("test_alias", "test_alias", "double")); + + verifyDataRows(response, + rows("2014-08-19T07:09:13.434Z"), + rows("2019-09-25T02:04:13.469Z")); + } + + @Test + public void castStatementInWhereClauseGreaterThanTest() { + JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT + + " WHERE (account_number < CAST(age AS DOUBLE)) ORDER BY balance LIMIT 5"); + + verifySchema(response, schema("balance", null, "long")); + + verifyDataRows(response, + rows(4180), + rows(5686), + rows(7004), + rows(7831), + rows(14127)); + } + + @Test + public void castStatementInWhereClauseLessThanTest() { + JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT + + " WHERE (account_number > CAST(age AS DOUBLE)) ORDER BY balance LIMIT 5"); + + verifySchema(response, schema("balance", null, "long")); + + verifyDataRows(response, + rows(1011), + rows(1031), + rows(1110), + rows(1133), + rows(1172)); + } + + @Test + public void castStatementInWhereClauseEqualToConstantTest() { + JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT + + " WHERE (CAST(age AS DOUBLE) = 36.0) ORDER BY balance LIMIT 5"); + + verifySchema(response, schema("balance", null, "long")); + verifyDataRows(response, + rows(1249), + rows(1463), + rows(3960), + rows(5686), + rows(6025)); + } + + @Test + public void castStatementInWhereClauseLessThanConstantTest() { + JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT + + " WHERE (CAST(age AS DOUBLE) < 36.0) ORDER BY balance LIMIT 5"); + + verifySchema(response, schema("balance", null, "long")); + + verifyDataRows(response, + rows(1011), + rows(1031), + rows(1110), + rows(1133), + rows(1172)); + } + + /** + * Testing compilation + * Result comparison is empty -> comparing different types (Date and keyword) + */ + @Test + public void castStatementInWhereClauseDatetimeCastTest() { + JSONObject response = executeJdbcRequest("SELECT date_keyword FROM " + + TestsConstants.TEST_INDEX_DATE + + " WHERE (CAST(date_keyword AS DATETIME) = \'2014-08-19T07:09:13.434Z\')"); + + String schema_result = "{\"name\":\"date_keyword\",\"type\":\"keyword\"}"; + assertEquals(response.getJSONArray("schema").get(0).toString(), schema_result); } @Test diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java index 0dcefad83c..5be2ec3a6a 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java @@ -335,7 +335,11 @@ public enum Index { WEBLOG(TestsConstants.TEST_INDEX_WEBLOG, "weblog", TestUtils.getWeblogsIndexMapping(), - "src/test/resources/weblogs.json"); + "src/test/resources/weblogs.json"), + DATE(TestsConstants.TEST_INDEX_DATE, + "dates", + TestUtils.getDateIndexMapping(), + "src/test/resources/dates.json"); private final String name; private final String type; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java index 8a85f7a14c..2adc9060cc 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java @@ -518,6 +518,18 @@ public static String getWeblogsIndexMapping() { "}"; } + public static String getDateIndexMapping() { + return "{ \"dates\": {" + + " \"properties\": {\n" + + " \"date_keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }"+ + " }"+ + " }" + + "}"; + } + public static void loadBulk(Client client, String jsonPath, String defaultIndex) throws Exception { System.out.println(String.format("Loading file %s into elasticsearch cluster", jsonPath)); String absJsonPath = getResourceFilePath(jsonPath); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java index beebbf7805..b4687c2daf 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java @@ -44,6 +44,7 @@ public class TestsConstants { public final static String TEST_INDEX_BANK_TWO = TEST_INDEX_BANK + "_two"; public final static String TEST_INDEX_ORDER = TEST_INDEX + "_order"; public final static String TEST_INDEX_WEBLOG = TEST_INDEX + "_weblog"; + public final static String TEST_INDEX_DATE = TEST_INDEX + "_date"; public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/SqlParserTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/SqlParserTest.java index 30d6c6d20d..9bf0d8ac4b 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/SqlParserTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/SqlParserTest.java @@ -61,7 +61,10 @@ import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DOG; import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_GAME_OF_THRONES; import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ODBC; +import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.getScriptFieldFromQuery; +import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.scriptContainsString; import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; public class SqlParserTest { @@ -1313,7 +1316,8 @@ public void castToDateTimeTest() throws Exception { String scriptCode = (String) methodField.getParams().get(1).value; Assert.assertEquals("cast_age",alias); Assert.assertTrue(scriptCode.contains("doc['age'].value")); - Assert.assertTrue(scriptCode.contains("new Date(Double.parseDouble(doc['age'].value.toString()).longValue())")); + Assert.assertTrue(scriptCode.contains("DateTimeFormatter.ofPattern(\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\").format(" + + "DateTimeFormatter.ISO_DATE_TIME.parse(doc['age'].value.toString()))")); } @Test diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java index d4a86d51da..74a67948ee 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java @@ -17,6 +17,7 @@ import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr; import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; +import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; import com.google.common.collect.ImmutableList; import org.elasticsearch.common.collect.Tuple; @@ -27,7 +28,7 @@ public class SQLFunctionsTest { @Test - public void testAssign() { + public void testAssign() throws SqlParseException { SQLFunctions sqlFunctions = new SQLFunctions(); final SQLIntegerExpr sqlIntegerExpr = new SQLIntegerExpr(10); diff --git a/src/test/resources/dates.json b/src/test/resources/dates.json new file mode 100644 index 0000000000..038a30a116 --- /dev/null +++ b/src/test/resources/dates.json @@ -0,0 +1,4 @@ +{"index":{"_type": "dates"}} +{"date_keyword": "2014-08-19T07:09:13.434Z"} +{"index":{"_type": "dates"}} +{"date_keyword": "2019-09-25T02:04:13.469Z"} \ No newline at end of file