From 44c0504252c25b7a887ae36bb000e5dc08ec26f4 Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 17 Jun 2020 12:06:57 -0700 Subject: [PATCH 01/14] Add new SQL module with support for SELECT literal --- .../sql/analysis/Analyzer.java | 22 +- .../sql/ast/AbstractNodeVisitor.java | 5 + .../sql/ast/dsl/AstDSL.java | 12 + .../sql/ast/tree/Values.java | 55 +++ .../sql/expression/DSL.java | 8 + .../sql/planner/DefaultImplementor.java | 108 ++++++ .../sql/planner/Planner.java | 16 +- .../sql/planner/logical/LogicalPlanDSL.java | 9 +- .../logical/LogicalPlanNodeVisitor.java | 4 + .../sql/planner/logical/LogicalProject.java | 4 +- .../sql/planner/logical/LogicalRemove.java | 4 +- .../sql/planner/logical/LogicalValues.java | 48 +++ .../sql/planner/physical/PhysicalPlanDSL.java | 11 +- .../physical/PhysicalPlanNodeVisitor.java | 5 + .../sql/planner/physical/ProjectOperator.java | 10 +- .../sql/planner/physical/RemoveOperator.java | 4 +- .../sql/planner/physical/ValuesOperator.java | 74 ++++ .../sql/analysis/AnalyzerTest.java | 23 +- .../sql/planner/PlannerTest.java | 23 ++ docs/category.json | 4 +- docs/user/dql/expressions.rst | 43 +++ .../storage/ElasticsearchIndex.java | 75 +--- .../sql/sql/LiteralValueIT.java | 72 ++++ legacy/build.gradle | 2 + .../sql/plugin/SQLPlugin.java | 4 +- .../sql/plugin/rest/RestSQLQueryAction.java | 135 +++++++ settings.gradle | 1 + sql/build.gradle | 81 ++++ sql/lombok.config | 3 + sql/src/main/antlr/OpenDistroSQLLexer.g4 | 345 ++++++++++++++++++ sql/src/main/antlr/OpenDistroSQLParser.g4 | 115 ++++++ .../sql/sql/SQLService.java | 99 +++++ .../sql/sql/antlr/SQLSyntaxParser.java | 43 +++ .../sql/sql/config/SQLServiceConfig.java | 59 +++ .../sql/sql/parser/AstBuilder.java | 64 ++++ .../sql/sql/parser/AstExpressionBuilder.java | 54 +++ .../sql/sql/parser/AstBuilderTest.java | 72 ++++ 37 files changed, 1621 insertions(+), 95 deletions(-) create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java create mode 100644 docs/user/dql/expressions.rst create mode 100644 integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/LiteralValueIT.java create mode 100644 plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestSQLQueryAction.java create mode 100644 sql/build.gradle create mode 100644 sql/lombok.config create mode 100644 sql/src/main/antlr/OpenDistroSQLLexer.g4 create mode 100644 sql/src/main/antlr/OpenDistroSQLParser.g4 create mode 100644 sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java create mode 100644 sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParser.java create mode 100644 sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java create mode 100644 sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java create mode 100644 sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java create mode 100644 sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java index 23afac1c4b..c3014246e5 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java @@ -19,6 +19,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Let; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Map; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation; @@ -31,10 +32,12 @@ import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprMissingValue; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; @@ -47,12 +50,14 @@ import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRemove; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalSort; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalValues; import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; import com.amazon.opendistroforelasticsearch.sql.storage.Table; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; @@ -145,9 +150,9 @@ public LogicalPlan visitAggregation(Aggregation node, AnalysisContext context) { @Override public LogicalPlan visitProject(Project node, AnalysisContext context) { LogicalPlan child = node.getChild().get(0).accept(this, context); - List referenceExpressions = + List referenceExpressions = node.getProjectList().stream() - .map(expr -> (ReferenceExpression) expressionAnalyzer.analyze(expr, context)) + .map(expr -> expressionAnalyzer.analyze(expr, context)) .collect(Collectors.toList()); if (node.hasArgument()) { Argument argument = node.getArgExprList().get(0); @@ -222,4 +227,17 @@ public LogicalPlan visitDedupe(Dedupe node, AnalysisContext context) { keepEmpty, consecutive); } + + @Override + public LogicalPlan visitValues(Values node, AnalysisContext context) { + List> values = node.getValues(); + List> valueExprs = new ArrayList<>(); + for (List value : values) { + valueExprs.add(value.stream() + .map(val -> (LiteralExpression) expressionAnalyzer.analyze(val, context)) + .collect(Collectors.toList())); + } + return new LogicalValues(valueExprs); + } + } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java index 7e3b161ec8..9474d33637 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java @@ -39,6 +39,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; /** * AST nodes visitor Defines the traverse path. @@ -168,4 +169,8 @@ public T visitSort(Sort node, C context) { public T visitDedupe(Dedupe node, C context) { return visitChildren(node, context); } + + public T visitValues(Values node, C context) { + return visitChildren(node, context); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java index 1e1cb911a2..c89b8bad42 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java @@ -41,6 +41,8 @@ import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; +import com.google.common.collect.ImmutableList; import java.util.Arrays; import java.util.List; import lombok.experimental.UtilityClass; @@ -85,6 +87,16 @@ public static UnresolvedPlan rename(UnresolvedPlan input, Map... maps) { return new Rename(Arrays.asList(maps), input); } + /** + * Initialize Values node by rows of literals. + * @param values tuple list + * @return Values node + */ + @SafeVarargs + public UnresolvedPlan values(List... values) { + return new Values(Arrays.asList(values)); + } + public static UnresolvedExpression qualifiedName(String... parts) { return new QualifiedName(Arrays.asList(parts)); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java new file mode 100644 index 0000000000..d1a6cdb35f --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.Node; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * AST node class for literal values. + */ +@ToString +@Getter +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Values extends UnresolvedPlan { + + private final List> values; + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + throw new UnsupportedOperationException("Values node is supposed to have no child node"); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitValues(this, context); + } + + @Override + public List getChild() { + return ImmutableList.of(); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java index 29f37eb92f..ca5e4e16f3 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java @@ -45,6 +45,14 @@ public static LiteralExpression literal(Double value) { return new LiteralExpression(ExprValueUtils.doubleValue(value)); } + public static LiteralExpression literal(String value) { + return new LiteralExpression(ExprValueUtils.stringValue(value)); + } + + public static LiteralExpression literal(Boolean value) { + return new LiteralExpression(ExprValueUtils.booleanValue(value)); + } + public static LiteralExpression literal(ExprValue value) { return new LiteralExpression(value); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java new file mode 100644 index 0000000000..b27b7fdd29 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner; + +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalDedupe; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalEval; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalProject; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRemove; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalSort; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalValues; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.AggregationOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.DedupeOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.EvalOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.FilterOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.ProjectOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.RemoveOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.RenameOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.SortOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.ValuesOperator; + +/** + * Default implementor for logical to physical translation. Default here means all logical operator + * will be translated to correspondent pipelining operator. + * Different storage can override methods here to optimize default pipelining operator, for example + * a storage has the flexibility to override visitFilter and visitRelation to push down filtering + * operation to an physical index scan operator. + * + * @param context type + */ +public class DefaultImplementor extends LogicalPlanNodeVisitor { + + @Override + public PhysicalPlan visitDedupe(LogicalDedupe node, C context) { + return new DedupeOperator( + visitChild(node, context), + node.getDedupeList(), + node.getAllowedDuplication(), + node.getKeepEmpty(), + node.getConsecutive()); + } + + @Override + public PhysicalPlan visitProject(LogicalProject node, C context) { + return new ProjectOperator(visitChild(node, context), node.getProjectList()); + } + + @Override + public PhysicalPlan visitRemove(LogicalRemove node, C context) { + return new RemoveOperator(visitChild(node, context), node.getRemoveList()); + } + + @Override + public PhysicalPlan visitEval(LogicalEval node, C context) { + return new EvalOperator(visitChild(node, context), node.getExpressions()); + } + + @Override + public PhysicalPlan visitSort(LogicalSort node, C context) { + return new SortOperator(visitChild(node, context), node.getCount(), node.getSortList()); + } + + @Override + public PhysicalPlan visitRename(LogicalRename node, C context) { + return new RenameOperator(visitChild(node, context), node.getRenameMap()); + } + + @Override + public PhysicalPlan visitAggregation(LogicalAggregation node, C context) { + return new AggregationOperator( + visitChild(node, context), node.getAggregatorList(), node.getGroupByList()); + } + + @Override + public PhysicalPlan visitFilter(LogicalFilter node, C context) { + return new FilterOperator(visitChild(node, context), node.getCondition()); + } + + @Override + public PhysicalPlan visitValues(LogicalValues node, C context) { + return new ValuesOperator(node.getValues()); + } + + protected PhysicalPlan visitChild(LogicalPlan node, C context) { + // Logical operators visited here can only have single child. + return node.getChild().get(0).accept(this, context); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java index fb96e02444..54e40970f1 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java @@ -22,6 +22,7 @@ import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import java.util.List; import lombok.RequiredArgsConstructor; /** @@ -36,7 +37,8 @@ public class Planner { private final StorageEngine storageEngine; /** - * Generate optimal physical plan for logical plan. + * Generate optimal physical plan for logical plan. If no table involved, + * translate logical plan to physical by default implementor. * TODO: for now just delegate entire logical plan to storage engine. * * @param plan logical plan @@ -44,6 +46,10 @@ public class Planner { */ public PhysicalPlan plan(LogicalPlan plan) { String tableName = findTableName(plan); + if (tableName.isEmpty()) { + return plan.accept(new DefaultImplementor<>(), null); + } + Table table = storageEngine.getTable(tableName); return table.implement(plan); } @@ -53,9 +59,11 @@ private String findTableName(LogicalPlan plan) { @Override protected String visitNode(LogicalPlan node, Object context) { - // So far all logical node has single child except LogicalRelation - // whose visitRelation() is already overridden. - return node.getChild().get(0).accept(this, context); + List children = node.getChild(); + if (children.isEmpty()) { + return ""; + } + return children.get(0).accept(this, context); } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java index 98df7dacf3..2f828ad9e6 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java @@ -17,6 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.google.common.collect.ImmutableSet; @@ -49,7 +50,7 @@ public static LogicalPlan rename( return new LogicalRename(input, renameMap); } - public static LogicalPlan project(LogicalPlan input, ReferenceExpression... fields) { + public static LogicalPlan project(LogicalPlan input, Expression... fields) { return new LogicalProject(input, Arrays.asList(fields)); } @@ -80,4 +81,10 @@ public static LogicalPlan dedupe( return new LogicalDedupe( input, Arrays.asList(fields), allowedDuplication, keepEmpty, consecutive); } + + @SafeVarargs + public LogicalPlan values(List... values) { + return new LogicalValues(Arrays.asList(values)); + } + } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java index 92629ff71b..bd63b84e5d 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java @@ -61,4 +61,8 @@ public R visitEval(LogicalEval plan, C context) { public R visitSort(LogicalSort plan, C context) { return visitNode(plan, context); } + + public R visitValues(LogicalValues plan, C context) { + return visitNode(plan, context); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java index a24e5be68a..edf179903c 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java @@ -15,7 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.planner.logical; -import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import java.util.Arrays; import java.util.List; import lombok.EqualsAndHashCode; @@ -32,7 +32,7 @@ public class LogicalProject extends LogicalPlan { private final LogicalPlan child; @Getter - private final List projectList; + private final List projectList; @Override public List getChild() { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java index 11d280cd9b..8037b09f4b 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java @@ -15,7 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.planner.logical; -import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import java.util.Arrays; import java.util.List; import java.util.Set; @@ -33,7 +33,7 @@ public class LogicalRemove extends LogicalPlan { private final LogicalPlan child; @Getter - private final Set removeList; + private final Set removeList; @Override public List getChild() { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java new file mode 100644 index 0000000000..79c64bf9c3 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Logical operator which is a sequence of literal list. + */ +@ToString +@Getter +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class LogicalValues extends LogicalPlan { + + private final List> values; + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitValues(this, context); + } + + @Override + public List getChild() { + return ImmutableList.of(); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java index 620a18d3a6..2e4246b54f 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java @@ -17,6 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.google.common.collect.ImmutableSet; @@ -46,11 +47,11 @@ public static RenameOperator rename( return new RenameOperator(input, renameMap); } - public static ProjectOperator project(PhysicalPlan input, ReferenceExpression... fields) { + public static ProjectOperator project(PhysicalPlan input, Expression... fields) { return new ProjectOperator(input, Arrays.asList(fields)); } - public static RemoveOperator remove(PhysicalPlan input, ReferenceExpression... fields) { + public static RemoveOperator remove(PhysicalPlan input, Expression... fields) { return new RemoveOperator(input, ImmutableSet.copyOf(fields)); } @@ -77,4 +78,10 @@ public static DedupeOperator dedupe( return new DedupeOperator( input, Arrays.asList(expressions), allowedDuplication, keepEmpty, consecutive); } + + @SafeVarargs + public ValuesOperator values(List... values) { + return new ValuesOperator(Arrays.asList(values)); + } + } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java index 7975aa07bf..fc05fff336 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -60,4 +60,9 @@ public R visitEval(EvalOperator node, C context) { public R visitDedupe(DedupeOperator node, C context) { return visitNode(node, context); } + + public R visitValues(ValuesOperator node, C context) { + return visitNode(node, context); + } + } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java index 019c7a7f0b..edb7c973a5 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java @@ -17,7 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; import java.util.Collections; @@ -34,7 +34,7 @@ @RequiredArgsConstructor public class ProjectOperator extends PhysicalPlan { private final PhysicalPlan input; - private final List projectList; + private final List projectList; @Override public R accept(PhysicalPlanNodeVisitor visitor, C context) { @@ -55,11 +55,11 @@ public boolean hasNext() { public ExprValue next() { ExprValue inputValue = input.next(); ImmutableMap.Builder mapBuilder = new Builder<>(); - for (ReferenceExpression ref : projectList) { - ExprValue exprValue = ref.valueOf(inputValue.bindingTuples()); + for (Expression expr : projectList) { + ExprValue exprValue = expr.valueOf(inputValue.bindingTuples()); // missing value is ignored. if (!exprValue.isMissing()) { - mapBuilder.put(ref.toString(), exprValue); + mapBuilder.put(expr.toString(), exprValue); } } return ExprTupleValue.fromExprValueMap(mapBuilder.build()); diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java index 810469de72..259e52a2b0 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java @@ -21,7 +21,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; -import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; import java.util.Collections; @@ -41,7 +41,7 @@ @RequiredArgsConstructor public class RemoveOperator extends PhysicalPlan { private final PhysicalPlan input; - private final Set removeList; + private final Set removeList; @Override public R accept(PhysicalPlanNodeVisitor visitor, C context) { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java new file mode 100644 index 0000000000..b93e5ca737 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprCollectionValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; +import com.google.common.collect.ImmutableList; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +/** + * Physical operator for Values. + */ +@ToString +@EqualsAndHashCode(callSuper = false, of = "values") +public class ValuesOperator extends PhysicalPlan { + + /** + * Original values list. + */ + private final List> values; + + /** + * Values iterator. + */ + private final Iterator> valuesIterator; + + public ValuesOperator(List> values) { + this.values = values; + this.valuesIterator = values.iterator(); + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitValues(this, context); + } + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public boolean hasNext() { + return valuesIterator.hasNext(); + } + + @Override + public ExprValue next() { + List values = valuesIterator.next().stream() + .map(expr -> expr.valueOf(null)) + .collect(Collectors.toList()); + return new ExprCollectionValue(values); + } + +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java index 0fe577bcbb..56c61599e5 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java @@ -23,10 +23,12 @@ import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.relation; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL; @@ -113,7 +115,7 @@ public void rename_to_invalid_expression() { AstDSL.agg( AstDSL.relation("schema"), AstDSL.exprList(AstDSL.aggregate("avg", field("integer_value"))), - Collections.emptyList(), + emptyList(), ImmutableList.of(), AstDSL.defaultStatsArgs()), AstDSL.map( @@ -165,4 +167,23 @@ public void project_source_change_type_env() { AstDSL.defaultFieldsArgs(), AstDSL.field("float_value")))); } + + @Test + public void project_values() { + assertAnalyzeEqual( + LogicalPlanDSL.project( + LogicalPlanDSL.values(emptyList()), + DSL.literal(123), + DSL.literal("hello"), + DSL.literal(false) + ), + AstDSL.project( + AstDSL.values(emptyList()), + AstDSL.intLiteral(123), + AstDSL.stringLiteral("hello"), + AstDSL.booleanLiteral(false) + ) + ); + } + } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java index a9b4a54e39..33bbd2c89f 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java @@ -15,6 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.planner; +import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -43,6 +44,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -86,6 +88,27 @@ public void planner_test() { ); } + @Test + public void plan_a_query_without_relation_involved() { + // Storage engine mock is not needed here since no relation involved. + Mockito.reset(storageEngine); + + assertPhysicalPlan( + PhysicalPlanDSL.project( + PhysicalPlanDSL.values(emptyList()), + DSL.literal(123), + DSL.literal("hello"), + DSL.literal(false) + ), + LogicalPlanDSL.project( + LogicalPlanDSL.values(emptyList()), + DSL.literal(123), + DSL.literal("hello"), + DSL.literal(false) + ) + ); + } + protected void assertPhysicalPlan(PhysicalPlan expected, LogicalPlan logicalPlan) { assertEquals(expected, analyze(logicalPlan)); } diff --git a/docs/category.json b/docs/category.json index ce28ee5d23..c77193d15e 100644 --- a/docs/category.json +++ b/docs/category.json @@ -13,5 +13,7 @@ "experiment/ppl/cmd/stats.rst", "experiment/ppl/cmd/where.rst" ], - "sql_cli": [] + "sql_cli": [ + "user/dql/expressions.rst" + ] } \ No newline at end of file diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst new file mode 100644 index 0000000000..7f0ef78372 --- /dev/null +++ b/docs/user/dql/expressions.rst @@ -0,0 +1,43 @@ +=========== +Expressions +=========== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Introduction +============ + +The expression are expressions that return a scalar value. It can be used in many statements, for example + +Literal Value +============= + +Description +----------- + +A literal is a symbol that represents a value. The most common literal values include: + + 1. Numeric literals: specify numeric values such as integer and floating-point numbers. + 2. String literals: specify a string enclosed by single or double quotes. + 3. Boolean literals: `true` or `false`. + +Examples +-------- + +Here are examples for different type of literals:: + + od> SELECT 1; + fetched rows / total rows = 2/2 + +------------------+ + | account_number | + |------------------+ + | 1 | + | 13 | + +------------------+ + + diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java index 2faeaf66c0..4ed71394e1 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java @@ -19,26 +19,10 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprType; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalDedupe; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalEval; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter; +import com.amazon.opendistroforelasticsearch.sql.planner.DefaultImplementor; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalProject; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRemove; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalSort; -import com.amazon.opendistroforelasticsearch.sql.planner.physical.AggregationOperator; -import com.amazon.opendistroforelasticsearch.sql.planner.physical.DedupeOperator; -import com.amazon.opendistroforelasticsearch.sql.planner.physical.EvalOperator; -import com.amazon.opendistroforelasticsearch.sql.planner.physical.FilterOperator; import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; -import com.amazon.opendistroforelasticsearch.sql.planner.physical.ProjectOperator; -import com.amazon.opendistroforelasticsearch.sql.planner.physical.RemoveOperator; -import com.amazon.opendistroforelasticsearch.sql.planner.physical.RenameOperator; -import com.amazon.opendistroforelasticsearch.sql.planner.physical.SortOperator; import com.amazon.opendistroforelasticsearch.sql.storage.Table; import com.google.common.collect.ImmutableMap; import java.util.HashMap; @@ -92,69 +76,16 @@ public Map getFieldTypes() { public PhysicalPlan implement(LogicalPlan plan) { ElasticsearchIndexScan indexScan = new ElasticsearchIndexScan(client, indexName); - /** + /* * Visit logical plan with index scan as context so logical operators visited, such as * aggregation, filter, will accumulate (push down) Elasticsearch query and aggregation DSL on * index scan. */ - return plan.accept( - new LogicalPlanNodeVisitor() { - @Override - public PhysicalPlan visitDedupe(LogicalDedupe node, ElasticsearchIndexScan context) { - return new DedupeOperator( - visitChild(node, context), - node.getDedupeList(), - node.getAllowedDuplication(), - node.getKeepEmpty(), - node.getConsecutive()); - } - - @Override - public PhysicalPlan visitProject(LogicalProject node, ElasticsearchIndexScan context) { - return new ProjectOperator(visitChild(node, context), node.getProjectList()); - } - - @Override - public PhysicalPlan visitRemove(LogicalRemove node, ElasticsearchIndexScan context) { - return new RemoveOperator(visitChild(node, context), node.getRemoveList()); - } - - @Override - public PhysicalPlan visitEval(LogicalEval node, ElasticsearchIndexScan context) { - return new EvalOperator(visitChild(node, context), node.getExpressions()); - } - - @Override - public PhysicalPlan visitSort(LogicalSort node, ElasticsearchIndexScan context) { - return new SortOperator(visitChild(node, context), node.getCount(), node.getSortList()); - } - - @Override - public PhysicalPlan visitRename(LogicalRename node, ElasticsearchIndexScan context) { - return new RenameOperator(visitChild(node, context), node.getRenameMap()); - } - - @Override - public PhysicalPlan visitAggregation( - LogicalAggregation node, ElasticsearchIndexScan context) { - return new AggregationOperator( - visitChild(node, context), node.getAggregatorList(), node.getGroupByList()); - } - - @Override - public PhysicalPlan visitFilter(LogicalFilter node, ElasticsearchIndexScan context) { - return new FilterOperator(visitChild(node, context), node.getCondition()); - } - + return plan.accept(new DefaultImplementor() { @Override public PhysicalPlan visitRelation(LogicalRelation node, ElasticsearchIndexScan context) { return indexScan; } - - private PhysicalPlan visitChild(LogicalPlan node, ElasticsearchIndexScan context) { - // Logical operators visited here can only have single child. - return node.getChild().get(0).accept(this, context); - } }, indexScan); } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/LiteralValueIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/LiteralValueIT.java new file mode 100644 index 0000000000..bb603898af --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/LiteralValueIT.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getResponseBody; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; + +import com.amazon.opendistroforelasticsearch.sql.legacy.RestIntegTestCase; +import java.io.IOException; +import java.util.Locale; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.junit.Assert; +import org.junit.Test; + +/** + * Integration test for different type of literal values such as integer, decimal, boolean etc. + */ +public class LiteralValueIT extends RestIntegTestCase { + + @Test + public void testIntegerLiteral() { + assertEquals( + "{\n" + + " \"schema\": [{\n" + + " \"name\": \"name\",\n" + + " \"type\": \"integer\"\n" + + " }],\n" + + " \"total\": 1,\n" + + " \"datarows\": [[\"123\"]],\n" + + " \"size\": 1\n" + + "}\n", + executeQuery("SELECT 123") + ); + } + + /** + * This is temporary and would be replaced by comparison test very soon. + */ + private String executeQuery(String query) { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + try { + Response response = client().performRequest(request); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + return getResponseBody(response, true); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + +} diff --git a/legacy/build.gradle b/legacy/build.gradle index 144c54cf94..7e401add79 100644 --- a/legacy/build.gradle +++ b/legacy/build.gradle @@ -64,6 +64,8 @@ dependencies { compile group: 'org.json', name: 'json', version:'20180813' compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' compile group: 'org.elasticsearch', name: 'elasticsearch', version: "${es_version}" + compile project(':sql') + compile project(':common') // ANTLR gradle plugin and runtime dependency antlr "org.antlr:antlr4:4.7.1" diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java index 5fb9fb6034..a20a4a33d1 100644 --- a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java @@ -23,6 +23,7 @@ import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlStatsAction; import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; import com.amazon.opendistroforelasticsearch.sql.plugin.rest.RestPPLQueryAction; +import com.amazon.opendistroforelasticsearch.sql.plugin.rest.RestSQLQueryAction; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -84,7 +85,8 @@ public List getRestHandlers(Settings settings, RestController restC return Arrays.asList( new RestPPLQueryAction(restController, clusterService), - new RestSqlAction(settings, restController), + //new RestSqlAction(settings, restController), + new RestSQLQueryAction(clusterService), new RestSqlStatsAction(settings, restController), new RestSqlSettingsAction(settings, restController) ); diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestSQLQueryAction.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestSQLQueryAction.java new file mode 100644 index 0000000000..8e8f48729c --- /dev/null +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestSQLQueryAction.java @@ -0,0 +1,135 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.plugin.rest; + +import static com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import static com.amazon.opendistroforelasticsearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; +import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; +import static org.elasticsearch.rest.RestStatus.OK; + +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.security.SecurityAccess; +import com.amazon.opendistroforelasticsearch.sql.protocol.response.QueryResult; +import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.SimpleJsonResponseFormatter; +import com.amazon.opendistroforelasticsearch.sql.sql.SQLService; +import com.amazon.opendistroforelasticsearch.sql.sql.config.SQLServiceConfig; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.Collections; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestStatus; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +/** + * New SQL REST action handler. + */ +public class RestSQLQueryAction extends BaseRestHandler { + + private static final Logger LOG = LogManager.getLogger(); + + public static final String SQL_QUERY_ENDPOINT = "_opendistro/_sql"; + private static final String SQL_QUERY_FIELD_NAME = "query"; + + private final ClusterService clusterService; + + public RestSQLQueryAction(ClusterService clusterService) { + super(); + this.clusterService = clusterService; + } + + @Override + public String getName() { + return "sql_query_action"; + } + + @Override + public List routes() { + return Collections.singletonList( + new Route(RestRequest.Method.POST, SQL_QUERY_ENDPOINT) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nodeClient) { + SQLService sqlService = createSQLService(nodeClient); + String query = parseQueryFromPayload(request); + return channel -> sqlService.execute(query, createListener(channel)); + } + + private String parseQueryFromPayload(RestRequest restRequest) { + String content = restRequest.content().utf8ToString(); + JSONObject jsonContent; + try { + jsonContent = new JSONObject(content); + } catch (JSONException e) { + throw new IllegalArgumentException("Failed to parse request payload", e); + } + return jsonContent.optString(SQL_QUERY_FIELD_NAME); + } + + private SQLService createSQLService(NodeClient client) { + return doPrivileged(() -> { + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + context.registerBean(ClusterService.class, () -> clusterService); + context.registerBean(NodeClient.class, () -> client); + context.register(ElasticsearchPluginConfig.class); + context.register(SQLServiceConfig.class); + context.refresh(); + return context.getBean(SQLService.class); + }); + } + + // TODO: many duplicate code here and for example SQLServiceConfig + private ResponseListener createListener(RestChannel channel) { + SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); + return new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + sendResponse(OK, formatter.format(new QueryResult(response.getResults()))); + } + + @Override + public void onFailure(Exception e) { + LOG.error("Error happened during query handling", e); + sendResponse(INTERNAL_SERVER_ERROR, formatter.format(e)); + } + + private void sendResponse(RestStatus status, String content) { + channel.sendResponse(new BytesRestResponse(status, "application/json; charset=UTF-8", content)); + } + }; + } + + private T doPrivileged(PrivilegedExceptionAction action) { + try { + return SecurityAccess.doPrivileged(action); + } catch (IOException e) { + throw new IllegalStateException("Failed to perform privileged action", e); + } + } + +} diff --git a/settings.gradle b/settings.gradle index 60a225d8a7..97aeab1de0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -24,4 +24,5 @@ include 'core' include 'protocol' include 'doctest' include 'legacy' +include 'sql' diff --git a/sql/build.gradle b/sql/build.gradle new file mode 100644 index 0000000000..56b18c010e --- /dev/null +++ b/sql/build.gradle @@ -0,0 +1,81 @@ +plugins { + id 'java' + id "io.freefair.lombok" + id 'jacoco' + id 'antlr' +} + +repositories { + mavenCentral() +} + +generateGrammarSource { + arguments += ['-visitor', '-package', 'com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser'] + source = sourceSets.main.antlr + outputDirectory = file("build/generated-src/antlr/main/com/amazon/opendistroforelasticsearch/sql/sql/antlr/parser") +} + +configurations { + compile { + extendsFrom = extendsFrom.findAll { it != configurations.antlr } + } +} + +dependencies { + antlr "org.antlr:antlr4:4.7.1" + + compile "org.antlr:antlr4-runtime:4.7.1" + compile group: 'com.google.guava', name: 'guava', version:'23.0' + compile group: 'org.json', name: 'json', version:'20180813' + compile group: 'org.springframework', name: 'spring-context', version: '5.2.5.RELEASE' + compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE' + compile project(':common') + compile project(':core') + compile project(':protocol') + + testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') + testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testCompile group: 'org.mockito', name: 'mockito-core', version: '3.3.3' + testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} + +jacoco { + toolVersion = "0.8.5" +} + +jacocoTestReport { + reports { + html.enabled true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/antlr/parser/**']) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = 1.0 + } + + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/antlr/parser/**']) + })) + } +} +check.dependsOn jacocoTestCoverageVerification diff --git a/sql/lombok.config b/sql/lombok.config new file mode 100644 index 0000000000..189c0bef98 --- /dev/null +++ b/sql/lombok.config @@ -0,0 +1,3 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true diff --git a/sql/src/main/antlr/OpenDistroSQLLexer.g4 b/sql/src/main/antlr/OpenDistroSQLLexer.g4 new file mode 100644 index 0000000000..61c13a3a30 --- /dev/null +++ b/sql/src/main/antlr/OpenDistroSQLLexer.g4 @@ -0,0 +1,345 @@ +/* +MySQL (Positive Technologies) grammar +The MIT License (MIT). +Copyright (c) 2015-2017, Ivan Kochurkin (kvanttt@gmail.com), Positive Technologies. +Copyright (c) 2017, Ivan Khudyashev (IHudyashov@ptsecurity.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +lexer grammar OpenDistroSQLLexer; + +channels { SQLCOMMENT, ERRORCHANNEL } + + +// SKIP + +SPACE: [ \t\r\n]+ -> channel(HIDDEN); +SPEC_SQL_COMMENT: '/*!' .+? '*/' -> channel(SQLCOMMENT); +COMMENT_INPUT: '/*' .*? '*/' -> channel(HIDDEN); +LINE_COMMENT: ( + ('-- ' | '#') ~[\r\n]* ('\r'? '\n' | EOF) + | '--' ('\r'? '\n' | EOF) + ) -> channel(HIDDEN); + + +// Keywords +// Common Keywords + +ALL: 'ALL'; +AND: 'AND'; +AS: 'AS'; +ASC: 'ASC'; +BETWEEN: 'BETWEEN'; +BY: 'BY'; +CASE: 'CASE'; +CAST: 'CAST'; +CROSS: 'CROSS'; +DATETIME: 'DATETIME'; +DELETE: 'DELETE'; +DESC: 'DESC'; +DESCRIBE: 'DESCRIBE'; +DISTINCT: 'DISTINCT'; +DOUBLE: 'DOUBLE'; +ELSE: 'ELSE'; +EXISTS: 'EXISTS'; +FALSE: 'FALSE'; +FLOAT: 'FLOAT'; +FROM: 'FROM'; +GROUP: 'GROUP'; +HAVING: 'HAVING'; +IN: 'IN'; +INNER: 'INNER'; +INT: 'INT'; +IS: 'IS'; +JOIN: 'JOIN'; +LEFT: 'LEFT'; +LIKE: 'LIKE'; +LIMIT: 'LIMIT'; +LONG: 'LONG'; +MATCH: 'MATCH'; +NATURAL: 'NATURAL'; +NOT: 'NOT'; +NULL_LITERAL: 'NULL'; +ON: 'ON'; +OR: 'OR'; +ORDER: 'ORDER'; +OUTER: 'OUTER'; +REGEXP: 'REGEXP'; +RIGHT: 'RIGHT'; +SELECT: 'SELECT'; +SHOW: 'SHOW'; +STRING: 'STRING'; +THEN: 'THEN'; +TRUE: 'TRUE'; +UNION: 'UNION'; +USING: 'USING'; +WHEN: 'WHEN'; +WHERE: 'WHERE'; + + +// OD SQL special keyword +MISSING: 'MISSING'; +EXCEPT: 'MINUS'; + + +// Group function Keywords + +AVG: 'AVG'; +COUNT: 'COUNT'; +MAX: 'MAX'; +MIN: 'MIN'; +SUM: 'SUM'; + + +// Common function Keywords + +SUBSTRING: 'SUBSTRING'; +TRIM: 'TRIM'; +YEAR: 'YEAR'; + + +// Keywords, but can be ID +// Common Keywords, but can be ID + +END: 'END'; +FULL: 'FULL'; +OFFSET: 'OFFSET'; + + +// PRIVILEGES + +TABLES: 'TABLES'; + + +// Common function names + +ABS: 'ABS'; +ACOS: 'ACOS'; +ADD: 'ADD'; +ASCII: 'ASCII'; +ASIN: 'ASIN'; +ATAN: 'ATAN'; +ATAN2: 'ATAN2'; +CBRT: 'CBRT'; +CEIL: 'CEIL'; +CONCAT: 'CONCAT'; +CONCAT_WS: 'CONCAT_WS'; +COS: 'COS'; +COSH: 'COSH'; +COT: 'COT'; +CURDATE: 'CURDATE'; +DATE: 'DATE'; +DATE_FORMAT: 'DATE_FORMAT'; +DAYOFMONTH: 'DAYOFMONTH'; +DEGREES: 'DEGREES'; +E: 'E'; +EXP: 'EXP'; +EXPM1: 'EXPM1'; +FLOOR: 'FLOOR'; +IF: 'IF'; +IFNULL: 'IFNULL'; +ISNULL: 'ISNULL'; +LENGTH: 'LENGTH'; +LN: 'LN'; +LOCATE: 'LOCATE'; +LOG: 'LOG'; +LOG10: 'LOG10'; +LOG2: 'LOG2'; +LOWER: 'LOWER'; +LTRIM: 'LTRIM'; +MAKETIME: 'MAKETIME'; +MODULUS: 'MODULUS'; +MONTH: 'MONTH'; +MONTHNAME: 'MONTHNAME'; +MULTIPLY: 'MULTIPLY'; +NOW: 'NOW'; +PI: 'PI'; +POW: 'POW'; +POWER: 'POWER'; +RADIANS: 'RADIANS'; +RAND: 'RAND'; +REPLACE: 'REPLACE'; +RINT: 'RINT'; +ROUND: 'ROUND'; +RTRIM: 'RTRIM'; +SIGN: 'SIGN'; +SIGNUM: 'SIGNUM'; +SIN: 'SIN'; +SINH: 'SINH'; +SQRT: 'SQRT'; +SUBTRACT: 'SUBTRACT'; +TAN: 'TAN'; +TIMESTAMP: 'TIMESTAMP'; +UPPER: 'UPPER'; + +D: 'D'; +T: 'T'; +TS: 'TS'; +LEFT_BRACE: '{'; +RIGHT_BRACE: '}'; + + +// OD SQL special functions +DATE_HISTOGRAM: 'DATE_HISTOGRAM'; +DAY_OF_MONTH: 'DAY_OF_MONTH'; +DAY_OF_YEAR: 'DAY_OF_YEAR'; +DAY_OF_WEEK: 'DAY_OF_WEEK'; +EXCLUDE: 'EXCLUDE'; +EXTENDED_STATS: 'EXTENDED_STATS'; +FIELD: 'FIELD'; +FILTER: 'FILTER'; +GEO_BOUNDING_BOX: 'GEO_BOUNDING_BOX'; +GEO_CELL: 'GEO_CELL'; +GEO_DISTANCE: 'GEO_DISTANCE'; +GEO_DISTANCE_RANGE: 'GEO_DISTANCE_RANGE'; +GEO_INTERSECTS: 'GEO_INTERSECTS'; +GEO_POLYGON: 'GEO_POLYGON'; +HISTOGRAM: 'HISTOGRAM'; +HOUR_OF_DAY: 'HOUR_OF_DAY'; +INCLUDE: 'INCLUDE'; +IN_TERMS: 'IN_TERMS'; +MATCHPHRASE: 'MATCHPHRASE'; +MATCH_PHRASE: 'MATCH_PHRASE'; +MATCHQUERY: 'MATCHQUERY'; +MATCH_QUERY: 'MATCH_QUERY'; +MINUTE_OF_DAY: 'MINUTE_OF_DAY'; +MINUTE_OF_HOUR: 'MINUTE_OF_HOUR'; +MONTH_OF_YEAR: 'MONTH_OF_YEAR'; +MULTIMATCH: 'MULTIMATCH'; +MULTI_MATCH: 'MULTI_MATCH'; +NESTED: 'NESTED'; +PERCENTILES: 'PERCENTILES'; +REGEXP_QUERY: 'REGEXP_QUERY'; +REVERSE_NESTED: 'REVERSE_NESTED'; +QUERY: 'QUERY'; +RANGE: 'RANGE'; +SCORE: 'SCORE'; +SECOND_OF_MINUTE: 'SECOND_OF_MINUTE'; +STATS: 'STATS'; +TERM: 'TERM'; +TERMS: 'TERMS'; +TOPHITS: 'TOPHITS'; +WEEK_OF_YEAR: 'WEEK_OF_YEAR'; +WILDCARDQUERY: 'WILDCARDQUERY'; +WILDCARD_QUERY: 'WILDCARD_QUERY'; + + +// Operators + +// Operators. Arithmetics + +STAR: '*'; +DIVIDE: '/'; +MODULE: '%'; +PLUS: '+'; +MINUS: '-'; +DIV: 'DIV'; +MOD: 'MOD'; + + +// Operators. Comparation + +EQUAL_SYMBOL: '='; +GREATER_SYMBOL: '>'; +LESS_SYMBOL: '<'; +EXCLAMATION_SYMBOL: '!'; + + +// Operators. Bit + +BIT_NOT_OP: '~'; +BIT_OR_OP: '|'; +BIT_AND_OP: '&'; +BIT_XOR_OP: '^'; + + +// Constructors symbols + +DOT: '.'; +LR_BRACKET: '('; +RR_BRACKET: ')'; +COMMA: ','; +SEMI: ';'; +AT_SIGN: '@'; +ZERO_DECIMAL: '0'; +ONE_DECIMAL: '1'; +TWO_DECIMAL: '2'; +SINGLE_QUOTE_SYMB: '\''; +DOUBLE_QUOTE_SYMB: '"'; +REVERSE_QUOTE_SYMB: '`'; +COLON_SYMB: ':'; + + +// Literal Primitives + +START_NATIONAL_STRING_LITERAL: 'N' SQUOTA_STRING; +STRING_LITERAL: DQUOTA_STRING | SQUOTA_STRING | BQUOTA_STRING; +DECIMAL_LITERAL: DEC_DIGIT+; +HEXADECIMAL_LITERAL: 'X' '\'' (HEX_DIGIT HEX_DIGIT)+ '\'' + | '0X' HEX_DIGIT+; + +REAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+ + | DEC_DIGIT+ '.' EXPONENT_NUM_PART + | (DEC_DIGIT+)? '.' (DEC_DIGIT+ EXPONENT_NUM_PART) + | DEC_DIGIT+ EXPONENT_NUM_PART; +NULL_SPEC_LITERAL: '\\' 'N'; +BIT_STRING: BIT_STRING_L; + + + +// Hack for dotID +// Prevent recognize string: .123somelatin AS ((.123), FLOAT_LITERAL), ((somelatin), ID) +// it must recoginze: .123somelatin AS ((.), DOT), (123somelatin, ID) + +DOT_ID: '.' ID_LITERAL; + + + +// Identifiers + +ID: ID_LITERAL; +// DOUBLE_QUOTE_ID: '"' ~'"'+ '"'; +REVERSE_QUOTE_ID: '`' ~'`'+ '`'; +STRING_USER_NAME: ( + SQUOTA_STRING | DQUOTA_STRING + | BQUOTA_STRING | ID_LITERAL + ) '@' + ( + SQUOTA_STRING | DQUOTA_STRING + | BQUOTA_STRING | ID_LITERAL + ); + + +// Fragments for Literal primitives + +fragment EXPONENT_NUM_PART: 'E' [-+]? DEC_DIGIT+; +fragment ID_LITERAL: [A-Z_$0-9@]*?[A-Z_$]+?[A-Z_$\-0-9]*; +fragment DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; +fragment SQUOTA_STRING: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\''; +fragment BQUOTA_STRING: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; +fragment HEX_DIGIT: [0-9A-F]; +fragment DEC_DIGIT: [0-9]; +fragment BIT_STRING_L: 'B' '\'' [01]+ '\''; + + + +// Last tokens must generate Errors + +ERROR_RECOGNITION: . -> channel(ERRORCHANNEL); diff --git a/sql/src/main/antlr/OpenDistroSQLParser.g4 b/sql/src/main/antlr/OpenDistroSQLParser.g4 new file mode 100644 index 0000000000..5a367d57c9 --- /dev/null +++ b/sql/src/main/antlr/OpenDistroSQLParser.g4 @@ -0,0 +1,115 @@ +/* +MySQL (Positive Technologies) grammar +The MIT License (MIT). +Copyright (c) 2015-2017, Ivan Kochurkin (kvanttt@gmail.com), Positive Technologies. +Copyright (c) 2017, Ivan Khudyashev (IHudyashov@ptsecurity.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +parser grammar OpenDistroSQLParser; + +options { tokenVocab=OpenDistroSQLLexer; } + + +// Top Level Description + +// Root rule +root + : sqlStatement? EOF + ; + +// Only SELECT +sqlStatement + : dmlStatement + ; + +dmlStatement + : selectStatement + ; + + +// Data Manipulation Language + +// Primary DML Statements + +selectStatement + : querySpecification #simpleSelect + ; + + +// Select Statement's Details + +querySpecification + : SELECT selectElements + ; + +selectElements + : selectElement (',' selectElement)* + ; + +selectElement + : expression #selectExpressionElement + ; + + +// Literals + +decimalLiteral + : DECIMAL_LITERAL | ZERO_DECIMAL | ONE_DECIMAL | TWO_DECIMAL + ; + +stringLiteral + : ( + STRING_LITERAL + | START_NATIONAL_STRING_LITERAL + ) STRING_LITERAL+ + | ( + STRING_LITERAL + | START_NATIONAL_STRING_LITERAL + ) + ; + +booleanLiteral + : TRUE | FALSE; + +constant + : stringLiteral | decimalLiteral + | '-' decimalLiteral + | booleanLiteral + | REAL_LITERAL | BIT_STRING + | NOT? nullLiteral=(NULL_LITERAL | NULL_SPEC_LITERAL) + | LEFT_BRACE dateType=(D | T | TS | DATE | TIME | TIMESTAMP) stringLiteral RIGHT_BRACE + ; + + +// Expressions, predicates + +// Simplified approach for expression +expression + : predicate #predicateExpression + ; + +predicate + : expressionAtom #expressionAtomPredicate + ; + +expressionAtom + : constant #constantExpressionAtom + ; diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java new file mode 100644 index 0000000000..1b1b8b975d --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql; + +import com.amazon.opendistroforelasticsearch.sql.analysis.AnalysisContext; +import com.amazon.opendistroforelasticsearch.sql.analysis.Analyzer; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import com.amazon.opendistroforelasticsearch.sql.planner.Planner; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.sql.parser.AstBuilder; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * SQL service. + */ +public class SQLService { + + private final SQLSyntaxParser parser; + + private final Analyzer analyzer; + + private final StorageEngine storageEngine; + + private final ExecutionEngine executionEngine; + + /** + * Initialize SQL service. + * @param parser SQL syntax parser + * @param analyzer AST analyzer + * @param storageEngine storage engine + * @param executionEngine execution engine + */ + public SQLService(SQLSyntaxParser parser, Analyzer analyzer, + StorageEngine storageEngine, ExecutionEngine executionEngine) { + this.parser = parser; + this.analyzer = analyzer; + this.storageEngine = storageEngine; + this.executionEngine = executionEngine; + } + + /** + * Parse, analyze, plan and execute the query. + * @param query SQL query + * @param listener callback listener + */ + public void execute(String query, ResponseListener listener) { + try { + executionEngine.execute( + plan( + analyze( + parse(query))), listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Parse query and convert parse tree (CST) to abstract syntax tree (AST). + */ + private UnresolvedPlan parse(String query) { + ParseTree cst = parser.parse(query); + return cst.accept(new AstBuilder()); + } + + /** + * Analyze abstract syntax to generate logical plan. + */ + private LogicalPlan analyze(UnresolvedPlan ast) { + return analyzer.analyze(ast, new AnalysisContext()); + } + + /** + * Generate optimal physical plan from logical plan. + */ + private PhysicalPlan plan(LogicalPlan logicalPlan) { + return new Planner(storageEngine).plan(logicalPlan); + } + +} diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParser.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParser.java new file mode 100644 index 0000000000..1a6ad372e1 --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParser.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.antlr; + +import com.amazon.opendistroforelasticsearch.sql.common.antlr.CaseInsensitiveCharStream; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxAnalysisErrorListener; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLLexer; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * SQL syntax parser which encapsulates an ANTLR parser. + */ +public class SQLSyntaxParser { + + /** + * Parse a SQL query by ANTLR parser. + * @param query a SQL query + * @return parse tree root + */ + public ParseTree parse(String query) { + OpenDistroSQLLexer lexer = new OpenDistroSQLLexer(new CaseInsensitiveCharStream(query)); + OpenDistroSQLParser parser = new OpenDistroSQLParser(new CommonTokenStream(lexer)); + parser.addErrorListener(new SyntaxAnalysisErrorListener()); + return parser.root(); + } + +} diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java new file mode 100644 index 0000000000..73da87af40 --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.config; + +import com.amazon.opendistroforelasticsearch.sql.analysis.Analyzer; +import com.amazon.opendistroforelasticsearch.sql.analysis.ExpressionAnalyzer; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.sql.SQLService; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * SQL service configuration. + */ +@Configuration +@Import({ExpressionConfig.class}) +public class SQLServiceConfig { + + @Autowired + private StorageEngine storageEngine; + + @Autowired + private ExecutionEngine executionEngine; + + @Autowired + private BuiltinFunctionRepository functionRepository; + + @Bean + public Analyzer analyzer() { + return new Analyzer(new ExpressionAnalyzer(functionRepository), storageEngine); + } + + @Bean + public SQLService pplService() { + return new SQLService(new SQLSyntaxParser(), analyzer(), storageEngine, executionEngine); + } + +} + diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java new file mode 100644 index 0000000000..94af0b5a8f --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.parser; + +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.SimpleSelectContext; + +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParserBaseVisitor; +import com.google.common.collect.ImmutableList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * Abstract syntax tree (AST) builder. + */ +public class AstBuilder extends OpenDistroSQLParserBaseVisitor { + + private final AstExpressionBuilder expressionBuilder = new AstExpressionBuilder(); + + @Override + public UnresolvedPlan visitSimpleSelect(SimpleSelectContext ctx) { + List selectElements = ctx.querySpecification().selectElements().children; + Project project = new Project(selectElements.stream() + .map(this::visitAstExpression) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + + // Attach an Values operator with only a empty row inside so that + // Project operator can have a chance to evaluate its expression + // though the evaluation doesn't have any dependency in the Values. + Values emptyValue = new Values(ImmutableList.of(Collections.emptyList())); + return project.attach(emptyValue); + } + + @Override + protected UnresolvedPlan aggregateResult(UnresolvedPlan aggregate, UnresolvedPlan nextResult) { + return nextResult != null ? nextResult : aggregate; + } + + private UnresolvedExpression visitAstExpression(ParseTree tree) { + return expressionBuilder.visit(tree); + } + +} diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java new file mode 100644 index 0000000000..ce95b6dbe0 --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.parser; + +import static com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils.unquoteIdentifier; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.BooleanLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.DecimalLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.StringLiteralContext; + +import com.amazon.opendistroforelasticsearch.sql.ast.expression.DataType; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParserBaseVisitor; + +/** + * Expression builder to parse text to expression in AST. + */ +public class AstExpressionBuilder extends OpenDistroSQLParserBaseVisitor { + + @Override + public UnresolvedExpression visitStringLiteral(StringLiteralContext ctx) { + return new Literal(unquoteIdentifier(ctx.getText()), DataType.STRING); + } + + @Override + public UnresolvedExpression visitDecimalLiteral(DecimalLiteralContext ctx) { + return new Literal(Integer.valueOf(ctx.getText()), DataType.INTEGER); + } + + @Override + public UnresolvedExpression visitBooleanLiteral(BooleanLiteralContext ctx) { + return new Literal(Boolean.valueOf(ctx.getText()), DataType.BOOLEAN); + } + + @Override + protected UnresolvedExpression aggregateResult(UnresolvedExpression aggregate, UnresolvedExpression nextResult) { + return nextResult != null ? nextResult : aggregate; + } + +} diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java new file mode 100644 index 0000000000..6782dca638 --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.parser; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.project; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.values; +import static java.util.Collections.emptyList; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; +import org.antlr.v4.runtime.tree.ParseTree; +import org.junit.jupiter.api.Test; + +class AstBuilderTest { + + /** + * SQL syntax parser that helps prepare parse tree as AstBuilder input. + */ + private final SQLSyntaxParser parser = new SQLSyntaxParser(); + + /** + * AST builder class that being tested. + */ + private final AstBuilder astBuilder = new AstBuilder(); + + @Test + public void buildAstForSelectIntegerLiteral() { + assertEquals( + project( + values(emptyList()), + intLiteral(123) + ), + buildAST("SELECT 123") + ); + } + + @Test + public void buildAstForSelectStringAndBooleanLiteral() { + assertEquals( + project( + values(emptyList()), + stringLiteral("hello"), + booleanLiteral(false) + ), + buildAST("SELECT 'hello', false") + ); + } + + private UnresolvedPlan buildAST(String query) { + ParseTree parseTree = parser.parse(query); + return parseTree.accept(astBuilder); + } + +} \ No newline at end of file From e6a4b88a9f5c8b9fffa4c80651f065602c834e60 Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 17 Jun 2020 15:44:16 -0700 Subject: [PATCH 02/14] Add support for more data types --- .../sql/ast/tree/Values.java | 2 +- .../sql/planner/DefaultImplementor.java | 7 +++ .../sql/sql/LiteralValueIT.java | 44 ++++++++++++++----- sql/src/main/antlr/OpenDistroSQLParser.g4 | 31 +++++++++---- .../sql/sql/parser/AstExpressionBuilder.java | 27 +++++++----- .../sql/sql/parser/AstBuilderTest.java | 21 +++------ 6 files changed, 86 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java index d1a6cdb35f..dc42644630 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java @@ -27,7 +27,7 @@ import lombok.ToString; /** - * AST node class for literal values. + * AST node class for a sequence of literal values. */ @ToString @Getter diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java index b27b7fdd29..5cf81a0ee0 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java @@ -23,6 +23,7 @@ import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalProject; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRemove; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalSort; @@ -100,6 +101,12 @@ public PhysicalPlan visitValues(LogicalValues node, C context) { return new ValuesOperator(node.getValues()); } + @Override + public PhysicalPlan visitRelation(LogicalRelation node, C context) { + throw new UnsupportedOperationException("Storage engine is responsible for " + + "implementing and optimizing logical plan with relation involved"); + } + protected PhysicalPlan visitChild(LogicalPlan node, C context) { // Logical operators visited here can only have single child. return node.getChild().get(0).accept(this, context); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/LiteralValueIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/LiteralValueIT.java index bb603898af..0a4a53b20b 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/LiteralValueIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/LiteralValueIT.java @@ -34,19 +34,39 @@ public class LiteralValueIT extends RestIntegTestCase { @Test - public void testIntegerLiteral() { - assertEquals( + public void testSelectLiterals() { + // TODO: Temporary manual assertion and will be replaced by comparison test soon. + String expected = "{\n" - + " \"schema\": [{\n" - + " \"name\": \"name\",\n" - + " \"type\": \"integer\"\n" - + " }],\n" - + " \"total\": 1,\n" - + " \"datarows\": [[\"123\"]],\n" - + " \"size\": 1\n" - + "}\n", - executeQuery("SELECT 123") - ); + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"123\",\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " {\n" + + " \"name\": \"\\\"hello\\\"\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"false\",\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " {\n" + + " \"name\": \"-4.567\",\n" + + " \"type\": \"double\"\n" + + " }\n" + + " ],\n" + + " \"total\": 1,\n" + + " \"datarows\": [[\n" + + " 123,\n" + + " \"hello\",\n" + + " false,\n" + + " -4.567\n" + + " ]],\n" + + " \"size\": 1\n" + + "}\n"; + + assertEquals(expected, executeQuery("SELECT 123, 'hello', false, -4.567")); } /** diff --git a/sql/src/main/antlr/OpenDistroSQLParser.g4 b/sql/src/main/antlr/OpenDistroSQLParser.g4 index 5a367d57c9..bd79fb104d 100644 --- a/sql/src/main/antlr/OpenDistroSQLParser.g4 +++ b/sql/src/main/antlr/OpenDistroSQLParser.g4 @@ -71,6 +71,18 @@ selectElement // Literals +constant + : stringLiteral #string + | sign? decimalLiteral #signedDecimal + | sign? realLiteral #signedReal + | booleanLiteral #boolean + // Doesn't support the following types for now + //| nullLiteral #null + //| BIT_STRING + //| NOT? nullLiteral=(NULL_LITERAL | NULL_SPEC_LITERAL) + //| LEFT_BRACE dateType=(D | T | TS | DATE | TIME | TIMESTAMP) stringLiteral RIGHT_BRACE + ; + decimalLiteral : DECIMAL_LITERAL | ZERO_DECIMAL | ONE_DECIMAL | TWO_DECIMAL ; @@ -87,17 +99,20 @@ stringLiteral ; booleanLiteral - : TRUE | FALSE; + : TRUE | FALSE + ; -constant - : stringLiteral | decimalLiteral - | '-' decimalLiteral - | booleanLiteral - | REAL_LITERAL | BIT_STRING - | NOT? nullLiteral=(NULL_LITERAL | NULL_SPEC_LITERAL) - | LEFT_BRACE dateType=(D | T | TS | DATE | TIME | TIMESTAMP) stringLiteral RIGHT_BRACE +realLiteral + : REAL_LITERAL + ; + +sign + : PLUS | MINUS ; +nullLiteral + : NULL_LITERAL + ; // Expressions, predicates diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java index ce95b6dbe0..42cf589bc2 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java @@ -17,12 +17,12 @@ package com.amazon.opendistroforelasticsearch.sql.sql.parser; import static com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils.unquoteIdentifier; -import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.BooleanLiteralContext; -import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.DecimalLiteralContext; -import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.StringLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.BooleanContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.SignedDecimalContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.SignedRealContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.StringContext; -import com.amazon.opendistroforelasticsearch.sql.ast.expression.DataType; -import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParserBaseVisitor; @@ -32,18 +32,23 @@ public class AstExpressionBuilder extends OpenDistroSQLParserBaseVisitor { @Override - public UnresolvedExpression visitStringLiteral(StringLiteralContext ctx) { - return new Literal(unquoteIdentifier(ctx.getText()), DataType.STRING); + public UnresolvedExpression visitString(StringContext ctx) { + return AstDSL.stringLiteral(unquoteIdentifier(ctx.getText())); } @Override - public UnresolvedExpression visitDecimalLiteral(DecimalLiteralContext ctx) { - return new Literal(Integer.valueOf(ctx.getText()), DataType.INTEGER); + public UnresolvedExpression visitSignedDecimal(SignedDecimalContext ctx) { + return AstDSL.intLiteral(Integer.valueOf(ctx.getText())); } @Override - public UnresolvedExpression visitBooleanLiteral(BooleanLiteralContext ctx) { - return new Literal(Boolean.valueOf(ctx.getText()), DataType.BOOLEAN); + public UnresolvedExpression visitSignedReal(SignedRealContext ctx) { + return AstDSL.doubleLiteral(Double.valueOf(ctx.getText())); + } + + @Override + public UnresolvedExpression visitBoolean(BooleanContext ctx) { + return AstDSL.booleanLiteral(Boolean.valueOf(ctx.getText())); } @Override diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java index 6782dca638..91e979fdb8 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java @@ -17,7 +17,9 @@ package com.amazon.opendistroforelasticsearch.sql.sql.parser; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.doubleLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.nullLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.project; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.values; @@ -42,25 +44,16 @@ class AstBuilderTest { private final AstBuilder astBuilder = new AstBuilder(); @Test - public void buildAstForSelectIntegerLiteral() { - assertEquals( - project( - values(emptyList()), - intLiteral(123) - ), - buildAST("SELECT 123") - ); - } - - @Test - public void buildAstForSelectStringAndBooleanLiteral() { + public void buildASTForSelectLiterals() { assertEquals( project( values(emptyList()), + intLiteral(123), stringLiteral("hello"), - booleanLiteral(false) + booleanLiteral(false), + doubleLiteral(-4.567) ), - buildAST("SELECT 'hello', false") + buildAST("SELECT 123, 'hello', false, -4.567") ); } From 364024cc9b717f1681450d90bc4bcc37467dcbaf Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 17 Jun 2020 20:25:32 -0700 Subject: [PATCH 03/14] Route request to new frontend --- legacy/build.gradle | 1 + .../legacy/plugin}/RestSQLQueryAction.java | 53 ++++++-- .../sql/legacy/plugin/RestSqlAction.java | 16 ++- .../sql/legacy/plugin/SQLQueryRequest.java | 97 +++++++++++++++ .../sql/legacy/plugin/SqlPlug.java | 116 ------------------ .../legacy/plugin/SQLQueryRequestTest.java | 116 ++++++++++++++++++ .../sql/plugin/SQLPlugin.java | 7 +- .../sql/sql/SQLService.java | 21 +++- 8 files changed, 294 insertions(+), 133 deletions(-) rename {plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest => legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin}/RestSQLQueryAction.java (69%) create mode 100644 legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequest.java delete mode 100644 legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlPlug.java create mode 100644 legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequestTest.java diff --git a/legacy/build.gradle b/legacy/build.gradle index 7e401add79..192d718cf9 100644 --- a/legacy/build.gradle +++ b/legacy/build.gradle @@ -66,6 +66,7 @@ dependencies { compile group: 'org.elasticsearch', name: 'elasticsearch', version: "${es_version}" compile project(':sql') compile project(':common') + compile project(':elasticsearch') // ANTLR gradle plugin and runtime dependency antlr "org.antlr:antlr4:4.7.1" diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestSQLQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java similarity index 69% rename from plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestSQLQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java index 8e8f48729c..18adca13d3 100644 --- a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestSQLQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -14,15 +14,20 @@ * */ -package com.amazon.opendistroforelasticsearch.sql.plugin.rest; +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; import static com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; import static com.amazon.opendistroforelasticsearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.elasticsearch.rest.RestStatus.OK; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchNodeClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.ElasticsearchExecutionEngine; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.security.SecurityAccess; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.ElasticsearchStorageEngine; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; import com.amazon.opendistroforelasticsearch.sql.protocol.response.QueryResult; import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.SimpleJsonResponseFormatter; import com.amazon.opendistroforelasticsearch.sql.sql.SQLService; @@ -45,13 +50,16 @@ import org.springframework.context.annotation.AnnotationConfigApplicationContext; /** - * New SQL REST action handler. + * New SQL REST action handler. This will not registered to Elasticsearch until all old + * functionalities migrated to new query engine and legacy REST handler removed. */ public class RestSQLQueryAction extends BaseRestHandler { private static final Logger LOG = LogManager.getLogger(); - public static final String SQL_QUERY_ENDPOINT = "_opendistro/_sql"; + public static final RestChannelConsumer NOT_SUPPORTED_YET = null; + + public static final String SQL_QUERY_ENDPOINT = "_opendistro/_newsql"; private static final String SQL_QUERY_FIELD_NAME = "query"; private final ClusterService clusterService; @@ -75,12 +83,36 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nodeClient) { + JSONObject jsonContent = parseJsonPayload(request); + String query = jsonContent.optString(SQL_QUERY_FIELD_NAME); + return prepareRequest( + new SQLQueryRequest(request, new SqlRequest(query, jsonContent)), + nodeClient + ); + } + + /** + * Prepare REST channel consumer for a SQL query request. + * @param request SQL request + * @param nodeClient node client + * @return channel consumer + */ + public RestChannelConsumer prepareRequest(SQLQueryRequest request, NodeClient nodeClient) { + if (!request.isSupported()) { + return NOT_SUPPORTED_YET; + } + SQLService sqlService = createSQLService(nodeClient); - String query = parseQueryFromPayload(request); - return channel -> sqlService.execute(query, createListener(channel)); + UnresolvedPlan ast; + try { + ast = sqlService.parse(request.getQuery()); + } catch (RuntimeException e) { //TODO: change to specific syntax exception + return NOT_SUPPORTED_YET; + } + return channel -> sqlService.execute(ast, createListener(channel)); } - private String parseQueryFromPayload(RestRequest restRequest) { + private JSONObject parseJsonPayload(RestRequest restRequest) { String content = restRequest.content().utf8ToString(); JSONObject jsonContent; try { @@ -88,7 +120,7 @@ private String parseQueryFromPayload(RestRequest restRequest) { } catch (JSONException e) { throw new IllegalArgumentException("Failed to parse request payload", e); } - return jsonContent.optString(SQL_QUERY_FIELD_NAME); + return jsonContent; } private SQLService createSQLService(NodeClient client) { @@ -96,7 +128,9 @@ private SQLService createSQLService(NodeClient client) { AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); context.registerBean(ClusterService.class, () -> clusterService); context.registerBean(NodeClient.class, () -> client); - context.register(ElasticsearchPluginConfig.class); + context.register(ElasticsearchNodeClient.class); + context.register(ElasticsearchStorageEngine.class); + context.register(ElasticsearchExecutionEngine.class); context.register(SQLServiceConfig.class); context.refresh(); return context.getBean(SQLService.class); @@ -119,7 +153,8 @@ public void onFailure(Exception e) { } private void sendResponse(RestStatus status, String content) { - channel.sendResponse(new BytesRestResponse(status, "application/json; charset=UTF-8", content)); + channel.sendResponse(new BytesRestResponse( + status, "application/json; charset=UTF-8", content)); } }; } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java index 965d015ddf..39fc040cc5 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java @@ -51,7 +51,6 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestChannel; -import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestStatus; @@ -89,9 +88,15 @@ public class RestSqlAction extends BaseRestHandler { public static final String EXPLAIN_API_ENDPOINT = QUERY_API_ENDPOINT + "/_explain"; public static final String CURSOR_CLOSE_ENDPOINT = QUERY_API_ENDPOINT + "/close"; - public RestSqlAction(Settings settings, RestController restController) { + /** + * New SQL query request handler. + */ + private final RestSQLQueryAction newSqlQueryHandler; + + public RestSqlAction(Settings settings, RestSQLQueryAction newSqlQueryHandler) { super(); this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings); + this.newSqlQueryHandler = newSqlQueryHandler; } @Override @@ -135,6 +140,13 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli LOG.info("[{}] Incoming request {}: {}", LogUtils.getRequestId(), request.uri(), QueryDataAnonymizer.anonymizeData(sqlRequest.getSql())); + + SQLQueryRequest newSqlRequest = new SQLQueryRequest(request, sqlRequest); + RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); + if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { + return result; + } + final QueryAction queryAction = explainRequest(client, sqlRequest, SqlRequestParam.getFormat(request.params())); return channel -> executeSqlRequest(request, queryAction, client, channel); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequest.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequest.java new file mode 100644 index 0000000000..b7bf75b075 --- /dev/null +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequest.java @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; + +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequestParam; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.elasticsearch.rest.RestRequest; +import org.json.JSONObject; + +/** + * SQL query request. + */ +@ToString +@Getter +@EqualsAndHashCode +public class SQLQueryRequest { + + /** + * JSON payload in REST request. + */ + private final JSONObject jsonContent; + + /** + * SQL query. + */ + private final String query; + + /** + * Request path. + */ + private final String path; + + /** + * Request format. + */ + private final Format format; + + /** + * Populate useful info from original REST request and SQL request. + * @param restRequest ES REST request + * @param sqlRequest Legacy SQL request + */ + public SQLQueryRequest(RestRequest restRequest, SqlRequest sqlRequest) { + this.jsonContent = sqlRequest.getJsonContent(); + this.query = sqlRequest.getSql(); + this.path = restRequest.path(); + this.format = SqlRequestParam.getFormat(restRequest.params()); + } + + /** + * Pre-check if the request can be supported by meeting the following criteria: + * 1.Not explain request + * 2.Only "query" field in payload. In other word, it's not a cursor request + * (with either "fetch_size" or "cursor" field) or request with extra field + * such as "filter". + * 3.Response format expected is default JDBC format. + * + * @return true if supported. + */ + public boolean isSupported() { + return !isExplainRequest() + && isOnlyQueryFieldInPayload() + && isDefaultFormat(); + } + + private boolean isExplainRequest() { + return path.endsWith("/_explain"); + } + + private boolean isOnlyQueryFieldInPayload() { + return jsonContent.keySet().size() == 1 + && jsonContent.has("query"); + } + + private boolean isDefaultFormat() { + return format == Format.JDBC; + } + +} diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlPlug.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlPlug.java deleted file mode 100644 index 603b5f8db0..0000000000 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlPlug.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; - -import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.legacy.executor.AsyncRestExecutor; -import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.IndexScopedSettings; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.plugins.ActionPlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.threadpool.ExecutorBuilder; -import org.elasticsearch.threadpool.FixedExecutorBuilder; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.function.Supplier; - -public class SqlPlug extends Plugin implements ActionPlugin { - - /** - * Sql plugin specific settings in ES cluster settings - */ - private final SqlSettings sqlSettings = new SqlSettings(); - - public SqlPlug() { - } - - - public String name() { - return "sql"; - } - - public String description() { - return "Use sql to query elasticsearch."; - } - - - @Override - public List getRestHandlers(Settings settings, RestController restController, - ClusterSettings clusterSettings, IndexScopedSettings indexScopedSettings, - SettingsFilter settingsFilter, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster) { - LocalClusterState.state().setResolver(indexNameExpressionResolver); - Metrics.getInstance().registerDefaultMetrics(); - return Arrays.asList( - new RestSqlAction(settings, restController), - new RestSqlStatsAction(settings, restController), - new RestSqlSettingsAction(settings, restController)); - } - - @Override - public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver) { - LocalClusterState.state().setClusterService(clusterService); - LocalClusterState.state().setSqlSettings(sqlSettings); - return super.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, - xContentRegistry, environment, nodeEnvironment, namedWriteableRegistry, indexNameExpressionResolver); - } - - @Override - public List> getExecutorBuilders(Settings settings) { - return Collections.singletonList( - new FixedExecutorBuilder( - settings, - AsyncRestExecutor.SQL_WORKER_THREAD_POOL_NAME, - EsExecutors.numberOfProcessors(settings), - 1000, - null - ) - ); - } - - @Override - public List> getSettings() { - return sqlSettings.getSettings(); - } -} diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequestTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequestTest.java new file mode 100644 index 0000000000..d7089274e1 --- /dev/null +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequestTest.java @@ -0,0 +1,116 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.rest.RestRequest; +import org.json.JSONObject; +import org.junit.Test; + +public class SQLQueryRequestTest { + + @Test + public void shouldSupportQueryWithDefaultFormat() { + SQLQueryRequest request = SQLQueryRequestBuilder.request("SELECT 1").build(); + assertTrue(request.isSupported()); + } + + @Test + public void shouldNotSupportExplain() { + SQLQueryRequest explainRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .path(EXPLAIN_API_ENDPOINT) + .build(); + assertFalse(explainRequest.isSupported()); + } + + @Test + public void shouldNotSupportCursorRequest() { + SQLQueryRequest fetchSizeRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .jsonContent("{\"query\": \"SELECT 1\", \"fetch_size\": 5}") + .build(); + assertFalse(fetchSizeRequest.isSupported()); + + SQLQueryRequest cursorRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .jsonContent("{\"cursor\": \"abcdefgh...\"}") + .build(); + assertFalse(cursorRequest.isSupported()); + } + + @Test + public void shouldNotSupportCSVFormat() { + SQLQueryRequest csvRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .format("csv") + .build(); + assertFalse(csvRequest.isSupported()); + } + + /** + * SQL query request build helper to improve test data setup readability. + */ + private static class SQLQueryRequestBuilder { + private String jsonContent; + private String query; + private String path = QUERY_API_ENDPOINT; + private String format = "jdbc"; + + static SQLQueryRequestBuilder request(String query) { + SQLQueryRequestBuilder builder = new SQLQueryRequestBuilder(); + builder.query = query; + return builder; + } + + SQLQueryRequestBuilder jsonContent(String jsonContent) { + this.jsonContent = jsonContent; + return this; + } + + SQLQueryRequestBuilder path(String path) { + this.path = path; + return this; + } + + SQLQueryRequestBuilder format(String format) { + this.format = format; + return this; + } + + SQLQueryRequest build() { + RestRequest restRequest = mock(RestRequest.class); + when(restRequest.path()).thenReturn(path); + when(restRequest.params()).thenReturn(ImmutableMap.of("format", format)); + + if (jsonContent == null) { + jsonContent = "{\"query\": \"" + query + "\"}"; + } + SqlRequest sqlRequest = new SqlRequest(query, new JSONObject(jsonContent)); + return new SQLQueryRequest(restRequest, sqlRequest); + } + } + +} \ No newline at end of file diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java index a20a4a33d1..58e53fce0f 100644 --- a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java @@ -18,12 +18,12 @@ import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import com.amazon.opendistroforelasticsearch.sql.legacy.executor.AsyncRestExecutor; import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSQLQueryAction; import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction; import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlSettingsAction; import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlStatsAction; import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; import com.amazon.opendistroforelasticsearch.sql.plugin.rest.RestPPLQueryAction; -import com.amazon.opendistroforelasticsearch.sql.plugin.rest.RestSQLQueryAction; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -83,10 +83,11 @@ public List getRestHandlers(Settings settings, RestController restC LocalClusterState.state().setResolver(indexNameExpressionResolver); Metrics.getInstance().registerDefaultMetrics(); + RestSQLQueryAction newSqlQueryHandler = new RestSQLQueryAction(clusterService); return Arrays.asList( new RestPPLQueryAction(restController, clusterService), - //new RestSqlAction(settings, restController), - new RestSQLQueryAction(clusterService), + newSqlQueryHandler, + new RestSqlAction(settings, newSqlQueryHandler), new RestSqlStatsAction(settings, restController), new RestSqlSettingsAction(settings, restController) ); diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java index 1b1b8b975d..b068ca73a8 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java @@ -77,7 +77,7 @@ public void execute(String query, ResponseListener listener) { /** * Parse query and convert parse tree (CST) to abstract syntax tree (AST). */ - private UnresolvedPlan parse(String query) { + public UnresolvedPlan parse(String query) { ParseTree cst = parser.parse(query); return cst.accept(new AstBuilder()); } @@ -85,15 +85,30 @@ private UnresolvedPlan parse(String query) { /** * Analyze abstract syntax to generate logical plan. */ - private LogicalPlan analyze(UnresolvedPlan ast) { + public LogicalPlan analyze(UnresolvedPlan ast) { return analyzer.analyze(ast, new AnalysisContext()); } /** * Generate optimal physical plan from logical plan. */ - private PhysicalPlan plan(LogicalPlan logicalPlan) { + public PhysicalPlan plan(LogicalPlan logicalPlan) { return new Planner(storageEngine).plan(logicalPlan); } + /** + * Given AST, run the remaining steps to execute it. + * @param ast AST + * @param listener callback listener + */ + public void execute(UnresolvedPlan ast, ResponseListener listener) { + try { + executionEngine.execute( + plan( + analyze(ast)), listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + } From 89a73a64fd191210adf4b30f341342e78a685da4 Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 17 Jun 2020 21:35:40 -0700 Subject: [PATCH 04/14] Add more UT --- .../physical/PhysicalPlanNodeVisitorTest.java | 5 ++ .../planner/physical/ValuesOperatorTest.java | 59 +++++++++++++++++++ .../sql/legacy/plugin/RestSQLQueryAction.java | 2 +- .../sql/legacy/plugin/RestSqlAction.java | 2 +- 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperatorTest.java diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java index 59f0ff4906..e36f1607b9 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java @@ -24,6 +24,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.util.Collections; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -105,6 +106,10 @@ public void test_PhysicalPlanVisitor_should_return_null() { PhysicalPlan dedupe = PhysicalPlanDSL.dedupe(plan, ref); assertNull(dedupe.accept(new PhysicalPlanNodeVisitor() { }, null)); + + PhysicalPlan values = PhysicalPlanDSL.values(Collections.emptyList()); + assertNull(values.accept(new PhysicalPlanNodeVisitor() { + }, null)); } public static class PhysicalPlanPrinter extends PhysicalPlanNodeVisitor { diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperatorTest.java new file mode 100644 index 0000000000..58f5acef76 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperatorTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.collectionValue; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; +import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.values; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +class ValuesOperatorTest { + + @Test + public void shouldHaveNoChild() { + ValuesOperator values = values(ImmutableList.of(literal(1))); + assertThat( + values.getChild(), + is(empty()) + ); + } + + @Test + public void iterateSingleRow() { + ValuesOperator values = values(ImmutableList.of(literal(1), literal("abc"))); + List results = new ArrayList<>(); + while (values.hasNext()) { + results.add(values.next()); + } + + assertThat( + results, + contains(collectionValue(Arrays.asList(1, "abc"))) + ); + } + +} \ No newline at end of file diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java index 18adca13d3..f27900ed0b 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -137,7 +137,7 @@ private SQLService createSQLService(NodeClient client) { }); } - // TODO: many duplicate code here and for example SQLServiceConfig + // TODO: duplicate code here as in RestPPLQueryAction private ResponseListener createListener(RestChannel channel) { SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); return new ResponseListener() { diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java index 39fc040cc5..8351187085 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java @@ -140,7 +140,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli LOG.info("[{}] Incoming request {}: {}", LogUtils.getRequestId(), request.uri(), QueryDataAnonymizer.anonymizeData(sqlRequest.getSql())); - + // Route request to new query engine if it's supported already SQLQueryRequest newSqlRequest = new SQLQueryRequest(request, sqlRequest); RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { From 607f2ca85c318c926e6666cc3a298dd32c87bf5e Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 17 Jun 2020 21:49:08 -0700 Subject: [PATCH 05/14] Fix doctest --- docs/user/dql/expressions.rst | 25 ++++++++++++++++--------- doctest/test_docs.py | 8 ++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 7f0ef78372..f2784ca68e 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -29,15 +29,22 @@ A literal is a symbol that represents a value. The most common literal values in Examples -------- -Here are examples for different type of literals:: +Here is an example for different type of literals:: - od> SELECT 1; - fetched rows / total rows = 2/2 - +------------------+ - | account_number | - |------------------+ - | 1 | - | 13 | - +------------------+ + od> SELECT 123, 'hello', false, -4.567; + fetched rows / total rows = 1/1 + +-------+-----------+---------+----------+ + | 123 | "hello" | false | -4.567 | + |-------+-----------+---------+----------| + | 123 | hello | False | -4.567 | + +-------+-----------+---------+----------+ +Limitations +----------- + +The current implementation has the following limitations at the moment: + + 1. Only literals of data types listed as above are supported for now. Other type of literals, such as date and NULL, will be added in future. + 2. Expression of literals, such as arithmetic expressions, will be supported later. + 3. Standard ANSI `VALUES` clause is not supported, although this is implemented by a Values operator internally. diff --git a/doctest/test_docs.py b/doctest/test_docs.py index a5a72f2c8e..57e10f9a21 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -170,6 +170,14 @@ def load_tests(loader, suite, ignore): # TODO: add until the migration to new architecture is done, then we have an artifact including ppl and sql both # for fn in doctest_files('sql/basics.rst'): # tests.append(docsuite(fn, setUp=set_up_accounts)) + for fn in doctest_files(sql_cli_docs): + tests.append( + docsuite( + fn, + parser=sql_cli_parser, + setUp=set_up_accounts + ) + ) # docs with ppl-cli based examples for fn in doctest_files(ppl_cli_docs): From 6b0197cf36c2ac4a6d48b51a3c62b08c0381f6d1 Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 17 Jun 2020 22:04:34 -0700 Subject: [PATCH 06/14] Add more UT --- .../sql/planner/DefaultImplementorTest.java | 125 ++++++++++++++++++ docs/user/index.rst | 2 + 2 files changed, 127 insertions(+) create mode 100644 core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java new file mode 100644 index 0000000000..e57e98b530 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java @@ -0,0 +1,125 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner; + +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.aggregation; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.eval; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.filter; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.project; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.remove; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.rename; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.sort; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.values; +import static java.util.Collections.emptyList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AvgAggregator; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; + +class DefaultImplementorTest { + + private final DefaultImplementor implementor = new DefaultImplementor<>(); + + @Test + public void visitShouldReturnDefaultPhysicalOperator() { + String indexName = "test"; + ReferenceExpression include = ref("age"); + ReferenceExpression exclude = ref("name"); + ReferenceExpression dedupeField = ref("name"); + Expression filterExpr = literal(ExprBooleanValue.ofTrue()); + List groupByExprs = Arrays.asList(ref("age")); + List aggregators = Arrays.asList(new AvgAggregator(groupByExprs, ExprType.DOUBLE)); + Map mappings = + ImmutableMap.of(ref("name"), ref("lastname")); + Pair newEvalField = + ImmutablePair.of(ref("name1"), ref("name")); + Integer sortCount = 100; + Pair sortField = + ImmutablePair.of(Sort.SortOption.PPL_ASC, ref("name1")); + + LogicalPlan plan = + project( + LogicalPlanDSL.dedupe( + sort( + eval( + remove( + rename( + aggregation( + filter(values(emptyList()), filterExpr), + aggregators, + groupByExprs), + mappings), + exclude), + newEvalField), + sortCount, + sortField), + dedupeField), + include); + + PhysicalPlan actual = plan.accept(implementor, null); + + assertEquals( + PhysicalPlanDSL.project( + PhysicalPlanDSL.dedupe( + PhysicalPlanDSL.sort( + PhysicalPlanDSL.eval( + PhysicalPlanDSL.remove( + PhysicalPlanDSL.rename( + PhysicalPlanDSL.agg( + PhysicalPlanDSL.filter( + PhysicalPlanDSL.values(emptyList()), + filterExpr), + aggregators, + groupByExprs), + mappings), + exclude), + newEvalField), + sortCount, + sortField), + dedupeField), + include), + actual); + + } + + @Test + public void visitRelationShouldThrowException() { + assertThrows(UnsupportedOperationException.class, + () -> new LogicalRelation("test").accept(implementor, null)); + } + +} \ No newline at end of file diff --git a/docs/user/index.rst b/docs/user/index.rst index 028e7b7e76..c3562c305c 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -19,6 +19,8 @@ Open Distro for Elasticsearch SQL enables you to extract insights out of Elastic * **Data Query Language** + - `Expressions `_ + - `Basic Queries `_ - `Complex Queries `_ From 5e977e3ad9696b863bb0038c6dbbb3f39002050b Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 18 Jun 2020 09:16:59 -0700 Subject: [PATCH 07/14] Add more UT --- .../sql/sql/SQLService.java | 7 +- .../sql/sql/config/SQLServiceConfig.java | 4 +- .../sql/sql/domain}/SQLQueryRequest.java | 26 +-- .../sql/sql/parser/AstExpressionBuilder.java | 5 - .../sql/sql/SQLServiceTest.java | 152 ++++++++++++++++++ .../sql/sql/antlr/SQLSyntaxParserTest.java | 40 +++++ .../sql/sql/config/SQLServiceConfigTest.java | 31 ++++ .../sql/sql/domain}/SQLQueryRequestTest.java | 30 ++-- .../sql/sql/parser/AstBuilderTest.java | 1 - .../sql/parser/AstExpressionBuilderTest.java | 73 +++++++++ 10 files changed, 318 insertions(+), 51 deletions(-) rename {legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin => sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain}/SQLQueryRequest.java (68%) create mode 100644 sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java create mode 100644 sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java create mode 100644 sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfigTest.java rename {legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin => sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain}/SQLQueryRequestTest.java (72%) create mode 100644 sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java index b068ca73a8..fb91e80602 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java @@ -26,6 +26,7 @@ import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; import com.amazon.opendistroforelasticsearch.sql.sql.parser.AstBuilder; import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; import org.antlr.v4.runtime.tree.ParseTree; @@ -60,15 +61,15 @@ public SQLService(SQLSyntaxParser parser, Analyzer analyzer, /** * Parse, analyze, plan and execute the query. - * @param query SQL query + * @param request SQL query request * @param listener callback listener */ - public void execute(String query, ResponseListener listener) { + public void execute(SQLQueryRequest request, ResponseListener listener) { try { executionEngine.execute( plan( analyze( - parse(query))), listener); + parse(request.getQuery()))), listener); } catch (Exception e) { listener.onFailure(e); } diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java index 73da87af40..e7584eccc9 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java @@ -30,7 +30,7 @@ import org.springframework.context.annotation.Import; /** - * SQL service configuration. + * SQL service configuration for Spring container initialization. */ @Configuration @Import({ExpressionConfig.class}) @@ -51,7 +51,7 @@ public Analyzer analyzer() { } @Bean - public SQLService pplService() { + public SQLService sqlService() { return new SQLService(new SQLSyntaxParser(), analyzer(), storageEngine, executionEngine); } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequest.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java similarity index 68% rename from legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequest.java rename to sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java index b7bf75b075..397d51f194 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequest.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java @@ -14,15 +14,14 @@ * */ -package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; +package com.amazon.opendistroforelasticsearch.sql.sql.domain; -import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; -import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequestParam; +import com.google.common.base.Strings; +import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.ToString; -import org.elasticsearch.rest.RestRequest; import org.json.JSONObject; /** @@ -31,6 +30,7 @@ @ToString @Getter @EqualsAndHashCode +@RequiredArgsConstructor public class SQLQueryRequest { /** @@ -51,19 +51,7 @@ public class SQLQueryRequest { /** * Request format. */ - private final Format format; - - /** - * Populate useful info from original REST request and SQL request. - * @param restRequest ES REST request - * @param sqlRequest Legacy SQL request - */ - public SQLQueryRequest(RestRequest restRequest, SqlRequest sqlRequest) { - this.jsonContent = sqlRequest.getJsonContent(); - this.query = sqlRequest.getSql(); - this.path = restRequest.path(); - this.format = SqlRequestParam.getFormat(restRequest.params()); - } + private final String format; /** * Pre-check if the request can be supported by meeting the following criteria: @@ -91,7 +79,7 @@ private boolean isOnlyQueryFieldInPayload() { } private boolean isDefaultFormat() { - return format == Format.JDBC; + return Strings.isNullOrEmpty(format) || "jdbc".equalsIgnoreCase(format); } } diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java index 42cf589bc2..be1df60115 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java @@ -51,9 +51,4 @@ public UnresolvedExpression visitBoolean(BooleanContext ctx) { return AstDSL.booleanLiteral(Boolean.valueOf(ctx.getText())); } - @Override - protected UnresolvedExpression aggregateResult(UnresolvedExpression aggregate, UnresolvedExpression nextResult) { - return nextResult != null ? nextResult : aggregate; - } - } diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java new file mode 100644 index 0000000000..ce563414ff --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java @@ -0,0 +1,152 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql; + +import static com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.sql.config.SQLServiceConfig; +import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; +import com.amazon.opendistroforelasticsearch.sql.sql.parser.AstBuilder; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import java.util.Collections; +import org.antlr.v4.runtime.tree.ParseTree; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +@ExtendWith(MockitoExtension.class) +class SQLServiceTest { + + private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + + private SQLService sqlService; + + @Mock + private StorageEngine storageEngine; + + @Mock + private ExecutionEngine executionEngine; + + @BeforeEach + public void setUp() { + context.registerBean(StorageEngine.class, () -> storageEngine); + context.registerBean(ExecutionEngine.class, () -> executionEngine); + context.register(SQLServiceConfig.class); + context.refresh(); + sqlService = context.getBean(SQLService.class); + } + + @Test + public void canExecuteSqlQuery() { + doAnswer(invocation -> { + ResponseListener listener = invocation.getArgument(1); + listener.onResponse(new QueryResponse(Collections.emptyList())); + return null; + }).when(executionEngine).execute(any(), any()); + + sqlService.execute( + new SQLQueryRequest(new JSONObject(), "SELECT 123", "_opendistro/_sql", "jdbc"), + new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + assertNotNull(response); + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + }); + } + + @Test + public void canExecuteFromAst() { + doAnswer(invocation -> { + ResponseListener listener = invocation.getArgument(1); + listener.onResponse(new QueryResponse(Collections.emptyList())); + return null; + }).when(executionEngine).execute(any(), any()); + + ParseTree parseTree = new SQLSyntaxParser().parse("SELECT 123"); + UnresolvedPlan ast = parseTree.accept(new AstBuilder()); + + sqlService.execute(ast, + new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + assertNotNull(response); + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + }); + } + + @Test + public void canCaptureErrorDuringExecution() { + sqlService.execute( + new SQLQueryRequest(new JSONObject(), "SELECT", "_opendistro/_sql", ""), + new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + fail(); + } + + @Override + public void onFailure(Exception e) { + assertNotNull(e); + } + }); + } + + @Test + public void canCaptureErrorDuringExecutionFromAst() { + doThrow(new RuntimeException()).when(executionEngine).execute(any(), any()); + + ParseTree parseTree = new SQLSyntaxParser().parse("SELECT 123"); + UnresolvedPlan ast = parseTree.accept(new AstBuilder()); + + sqlService.execute(ast, + new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + fail(); + } + + @Override + public void onFailure(Exception e) { + assertNotNull(e); + } + }); + } + +} \ No newline at end of file diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java new file mode 100644 index 0000000000..ffd97095c0 --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.antlr; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.antlr.v4.runtime.tree.ParseTree; +import org.junit.jupiter.api.Test; + +class SQLSyntaxParserTest { + + private final SQLSyntaxParser parser = new SQLSyntaxParser(); + + @Test + public void canParseSelectLiterals() { + ParseTree parseTree = parser.parse("SELECT 123, 'hello'"); + assertNotNull(parseTree); + } + + @Test + public void canNotParseInvalidSelect() { + assertThrows(RuntimeException.class, () -> parser.parse("SELECT ,")); + } + +} \ No newline at end of file diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfigTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfigTest.java new file mode 100644 index 0000000000..b254cbe312 --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfigTest.java @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.config; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.junit.jupiter.api.Test; + +class SQLServiceConfigTest { + + @Test + public void shouldReturnSQLService() { + SQLServiceConfig config = new SQLServiceConfig(); + assertNotNull(config.sqlService()); + } + +} \ No newline at end of file diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequestTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java similarity index 72% rename from legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequestTest.java rename to sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java index d7089274e1..58ef043363 100644 --- a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SQLQueryRequestTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java @@ -14,20 +14,13 @@ * */ -package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; - -import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; -import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; -import com.google.common.collect.ImmutableMap; -import org.elasticsearch.rest.RestRequest; +package com.amazon.opendistroforelasticsearch.sql.sql.domain; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.json.JSONObject; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class SQLQueryRequestTest { @@ -41,7 +34,7 @@ public void shouldSupportQueryWithDefaultFormat() { public void shouldNotSupportExplain() { SQLQueryRequest explainRequest = SQLQueryRequestBuilder.request("SELECT 1") - .path(EXPLAIN_API_ENDPOINT) + .path("_opendistro/_sql/_explain") .build(); assertFalse(explainRequest.isSupported()); } @@ -76,7 +69,7 @@ public void shouldNotSupportCSVFormat() { private static class SQLQueryRequestBuilder { private String jsonContent; private String query; - private String path = QUERY_API_ENDPOINT; + private String path = "_/opendistro/_sql"; private String format = "jdbc"; static SQLQueryRequestBuilder request(String query) { @@ -101,15 +94,10 @@ SQLQueryRequestBuilder format(String format) { } SQLQueryRequest build() { - RestRequest restRequest = mock(RestRequest.class); - when(restRequest.path()).thenReturn(path); - when(restRequest.params()).thenReturn(ImmutableMap.of("format", format)); - if (jsonContent == null) { jsonContent = "{\"query\": \"" + query + "\"}"; } - SqlRequest sqlRequest = new SqlRequest(query, new JSONObject(jsonContent)); - return new SQLQueryRequest(restRequest, sqlRequest); + return new SQLQueryRequest(new JSONObject(jsonContent), query, path, format); } } diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java index 91e979fdb8..a5546db97f 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java @@ -19,7 +19,6 @@ import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.doubleLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; -import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.nullLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.project; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.values; diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java new file mode 100644 index 0000000000..4dbc185995 --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.parser; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.ast.Node; +import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.CaseInsensitiveCharStream; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxAnalysisErrorListener; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLLexer; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser; +import org.antlr.v4.runtime.CommonTokenStream; +import org.junit.jupiter.api.Test; + +class AstExpressionBuilderTest { + + private final AstExpressionBuilder astExprBuilder = new AstExpressionBuilder(); + + @Test + public void canBuildExprAstForStringLiteral() { + assertEquals( + AstDSL.stringLiteral("hello"), + buildExprAst("'hello'") + ); + } + + @Test + public void canBuildExprAstForIntegerLiteral() { + assertEquals( + AstDSL.intLiteral(123), + buildExprAst("123") + ); + } + + @Test + public void canBuildExprAstForNegativeRealLiteral() { + assertEquals( + AstDSL.doubleLiteral(-4.567), + buildExprAst("-4.567") + ); + } + + @Test + public void canBuildExprAstForBooleanLiteral() { + assertEquals( + AstDSL.booleanLiteral(true), + buildExprAst("true") + ); + } + + private Node buildExprAst(String expr) { + OpenDistroSQLLexer lexer = new OpenDistroSQLLexer(new CaseInsensitiveCharStream(expr)); + OpenDistroSQLParser parser = new OpenDistroSQLParser(new CommonTokenStream(lexer)); + parser.addErrorListener(new SyntaxAnalysisErrorListener()); + return parser.constant().accept(astExprBuilder); + } + +} \ No newline at end of file From 616d34bad0673368a4a97c31b4b55efc6cd3431f Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 18 Jun 2020 10:18:38 -0700 Subject: [PATCH 08/14] Fix checkstyle and jacoco --- config/checkstyle/suppressions.xml | 1 + .../sql/analysis/AnalyzerTest.java | 5 ++-- .../sql/legacy/plugin/RestSQLQueryAction.java | 4 +-- .../sql/legacy/plugin/RestSqlAction.java | 9 ++++-- .../sql/sql/SQLService.java | 30 +++++++++---------- .../sql/sql/domain/SQLQueryRequestTest.java | 12 ++++++-- 6 files changed, 36 insertions(+), 25 deletions(-) diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml index 7bac5ae9d2..bb76a38b97 100644 --- a/config/checkstyle/suppressions.xml +++ b/config/checkstyle/suppressions.xml @@ -7,5 +7,6 @@ + \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java index 56c61599e5..9260527d01 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java @@ -28,7 +28,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL; @@ -172,13 +171,13 @@ public void project_source_change_type_env() { public void project_values() { assertAnalyzeEqual( LogicalPlanDSL.project( - LogicalPlanDSL.values(emptyList()), + LogicalPlanDSL.values(ImmutableList.of(DSL.literal(123))), DSL.literal(123), DSL.literal("hello"), DSL.literal(false) ), AstDSL.project( - AstDSL.values(emptyList()), + AstDSL.values(ImmutableList.of(AstDSL.intLiteral(123))), AstDSL.intLiteral(123), AstDSL.stringLiteral("hello"), AstDSL.booleanLiteral(false) diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java index f27900ed0b..b243ec4a27 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -27,11 +27,11 @@ import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.ElasticsearchExecutionEngine; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.security.SecurityAccess; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.ElasticsearchStorageEngine; -import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; import com.amazon.opendistroforelasticsearch.sql.protocol.response.QueryResult; import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.SimpleJsonResponseFormatter; import com.amazon.opendistroforelasticsearch.sql.sql.SQLService; import com.amazon.opendistroforelasticsearch.sql.sql.config.SQLServiceConfig; +import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.Collections; @@ -86,7 +86,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod JSONObject jsonContent = parseJsonPayload(request); String query = jsonContent.optString(SQL_QUERY_FIELD_NAME); return prepareRequest( - new SQLQueryRequest(request, new SqlRequest(query, jsonContent)), + new SQLQueryRequest(jsonContent, query, request.path(), ""), nodeClient ); } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java index 8351187085..6b096a8241 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java @@ -41,6 +41,7 @@ import com.amazon.opendistroforelasticsearch.sql.legacy.utils.JsonPrettyFormatter; import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils; import com.amazon.opendistroforelasticsearch.sql.legacy.utils.QueryDataAnonymizer; +import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -140,15 +141,17 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli LOG.info("[{}] Incoming request {}: {}", LogUtils.getRequestId(), request.uri(), QueryDataAnonymizer.anonymizeData(sqlRequest.getSql())); + Format format = SqlRequestParam.getFormat(request.params()); + // Route request to new query engine if it's supported already - SQLQueryRequest newSqlRequest = new SQLQueryRequest(request, sqlRequest); + SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), sqlRequest.getSql(), + request.path(), format.getFormatName()); RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { return result; } - final QueryAction queryAction = - explainRequest(client, sqlRequest, SqlRequestParam.getFormat(request.params())); + final QueryAction queryAction = explainRequest(client, sqlRequest, format); return channel -> executeSqlRequest(request, queryAction, client, channel); } catch (Exception e) { logAndPublishMetrics(e); diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java index fb91e80602..599d02bddb 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java @@ -75,6 +75,21 @@ public void execute(SQLQueryRequest request, ResponseListener lis } } + /** + * Given AST, run the remaining steps to execute it. + * @param ast AST + * @param listener callback listener + */ + public void execute(UnresolvedPlan ast, ResponseListener listener) { + try { + executionEngine.execute( + plan( + analyze(ast)), listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + /** * Parse query and convert parse tree (CST) to abstract syntax tree (AST). */ @@ -97,19 +112,4 @@ public PhysicalPlan plan(LogicalPlan logicalPlan) { return new Planner(storageEngine).plan(logicalPlan); } - /** - * Given AST, run the remaining steps to execute it. - * @param ast AST - * @param listener callback listener - */ - public void execute(UnresolvedPlan ast, ResponseListener listener) { - try { - executionEngine.execute( - plan( - analyze(ast)), listener); - } catch (Exception e) { - listener.onFailure(e); - } - } - } diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java index 58ef043363..1f906adf29 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java @@ -25,11 +25,19 @@ public class SQLQueryRequestTest { @Test - public void shouldSupportQueryWithDefaultFormat() { + public void shouldSupportQuery() { SQLQueryRequest request = SQLQueryRequestBuilder.request("SELECT 1").build(); assertTrue(request.isSupported()); } + @Test + public void shouldSupportQueryWithJDBCFormat() { + SQLQueryRequest request = SQLQueryRequestBuilder.request("SELECT 1") + .format("jdbc") + .build(); + assertTrue(request.isSupported()); + } + @Test public void shouldNotSupportExplain() { SQLQueryRequest explainRequest = @@ -70,7 +78,7 @@ private static class SQLQueryRequestBuilder { private String jsonContent; private String query; private String path = "_/opendistro/_sql"; - private String format = "jdbc"; + private String format; static SQLQueryRequestBuilder request(String query) { SQLQueryRequestBuilder builder = new SQLQueryRequestBuilder(); From 84f4da778df1c0c374fda3555c7dcd907ac56000 Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 18 Jun 2020 11:25:41 -0700 Subject: [PATCH 09/14] Address PR --- .../sql/analysis/Analyzer.java | 15 ++++++++++----- .../sql/ast/dsl/AstDSL.java | 2 +- .../sql/planner/DefaultImplementor.java | 9 +++++---- .../sql/planner/Planner.java | 4 +++- .../sql/planner/logical/LogicalRemove.java | 4 ++-- .../sql/planner/physical/PhysicalPlanDSL.java | 2 +- .../sql/planner/physical/RemoveOperator.java | 4 ++-- .../sql/planner/physical/ValuesOperator.java | 2 +- .../sql/analysis/AnalyzerTest.java | 3 +-- docs/user/dql/expressions.rst | 12 ++++++------ 10 files changed, 32 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java index c3014246e5..2ab24df7bf 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java @@ -150,18 +150,23 @@ public LogicalPlan visitAggregation(Aggregation node, AnalysisContext context) { @Override public LogicalPlan visitProject(Project node, AnalysisContext context) { LogicalPlan child = node.getChild().get(0).accept(this, context); - List referenceExpressions = - node.getProjectList().stream() - .map(expr -> expressionAnalyzer.analyze(expr, context)) - .collect(Collectors.toList()); + if (node.hasArgument()) { Argument argument = node.getArgExprList().get(0); Boolean exclude = (Boolean) argument.getValue().getValue(); if (exclude) { + List referenceExpressions = + node.getProjectList().stream() + .map(expr -> (ReferenceExpression) expressionAnalyzer.analyze(expr, context)) + .collect(Collectors.toList()); return new LogicalRemove(child, ImmutableSet.copyOf(referenceExpressions)); } } - return new LogicalProject(child, referenceExpressions); + + List expressions = node.getProjectList().stream() + .map(expr -> expressionAnalyzer.analyze(expr, context)) + .collect(Collectors.toList()); + return new LogicalProject(child, expressions); } /** diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java index c89b8bad42..d8d37c204c 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java @@ -89,7 +89,7 @@ public static UnresolvedPlan rename(UnresolvedPlan input, Map... maps) { /** * Initialize Values node by rows of literals. - * @param values tuple list + * @param values rows in which each row is a list of literal values * @return Values node */ @SafeVarargs diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java index 5cf81a0ee0..090cd0c297 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java @@ -40,11 +40,12 @@ import com.amazon.opendistroforelasticsearch.sql.planner.physical.ValuesOperator; /** - * Default implementor for logical to physical translation. Default here means all logical operator - * will be translated to correspondent pipelining operator. + * Default implementor for implementing logical to physical translation. "Default" here means all + * logical operator will be translated to correspondent physical operator to pipeline operations + * in post-processing style in memory. * Different storage can override methods here to optimize default pipelining operator, for example * a storage has the flexibility to override visitFilter and visitRelation to push down filtering - * operation to an physical index scan operator. + * operation and return a single physical index scan operator. * * @param context type */ @@ -108,7 +109,7 @@ public PhysicalPlan visitRelation(LogicalRelation node, C context) { } protected PhysicalPlan visitChild(LogicalPlan node, C context) { - // Logical operators visited here can only have single child. + // Logical operators visited here must have a single child return node.getChild().get(0).accept(this, context); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java index 54e40970f1..e47232bdb6 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java @@ -16,6 +16,8 @@ package com.amazon.opendistroforelasticsearch.sql.planner; +import static com.google.common.base.Strings.isNullOrEmpty; + import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; @@ -46,7 +48,7 @@ public class Planner { */ public PhysicalPlan plan(LogicalPlan plan) { String tableName = findTableName(plan); - if (tableName.isEmpty()) { + if (isNullOrEmpty(tableName)) { return plan.accept(new DefaultImplementor<>(), null); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java index 8037b09f4b..11d280cd9b 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java @@ -15,7 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.planner.logical; -import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import java.util.Arrays; import java.util.List; import java.util.Set; @@ -33,7 +33,7 @@ public class LogicalRemove extends LogicalPlan { private final LogicalPlan child; @Getter - private final Set removeList; + private final Set removeList; @Override public List getChild() { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java index 2e4246b54f..6dedd39e04 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java @@ -51,7 +51,7 @@ public static ProjectOperator project(PhysicalPlan input, Expression... fields) return new ProjectOperator(input, Arrays.asList(fields)); } - public static RemoveOperator remove(PhysicalPlan input, Expression... fields) { + public static RemoveOperator remove(PhysicalPlan input, ReferenceExpression... fields) { return new RemoveOperator(input, ImmutableSet.copyOf(fields)); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java index 259e52a2b0..810469de72 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java @@ -21,7 +21,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; -import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; import java.util.Collections; @@ -41,7 +41,7 @@ @RequiredArgsConstructor public class RemoveOperator extends PhysicalPlan { private final PhysicalPlan input; - private final Set removeList; + private final Set removeList; @Override public R accept(PhysicalPlanNodeVisitor visitor, C context) { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java index b93e5ca737..215589e5ef 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java @@ -34,7 +34,7 @@ public class ValuesOperator extends PhysicalPlan { /** - * Original values list. + * Original values list for print and equality check. */ private final List> values; diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java index 9260527d01..790e3f7457 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java @@ -23,7 +23,6 @@ import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.relation; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; -import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -114,7 +113,7 @@ public void rename_to_invalid_expression() { AstDSL.agg( AstDSL.relation("schema"), AstDSL.exprList(AstDSL.aggregate("avg", field("integer_value"))), - emptyList(), + Collections.emptyList(), ImmutableList.of(), AstDSL.defaultStatsArgs()), AstDSL.map( diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index f2784ca68e..1680d27e3c 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -22,9 +22,9 @@ Description A literal is a symbol that represents a value. The most common literal values include: - 1. Numeric literals: specify numeric values such as integer and floating-point numbers. - 2. String literals: specify a string enclosed by single or double quotes. - 3. Boolean literals: `true` or `false`. +1. Numeric literals: specify numeric values such as integer and floating-point numbers. +2. String literals: specify a string enclosed by single or double quotes. +3. Boolean literals: `true` or `false`. Examples -------- @@ -44,7 +44,7 @@ Limitations The current implementation has the following limitations at the moment: - 1. Only literals of data types listed as above are supported for now. Other type of literals, such as date and NULL, will be added in future. - 2. Expression of literals, such as arithmetic expressions, will be supported later. - 3. Standard ANSI `VALUES` clause is not supported, although this is implemented by a Values operator internally. +1. Only literals of data types listed as above are supported for now. Other type of literals, such as date and NULL, will be added in future. +2. Expression of literals, such as arithmetic expressions, will be supported later. +3. Standard ANSI `VALUES` clause is not supported, although this is implemented by a Values operator internally. From 66133355b9cff464c9f23d202e3ec2113a249c7d Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 18 Jun 2020 11:50:04 -0700 Subject: [PATCH 10/14] Add more docs --- docs/user/dql/expressions.rst | 10 +++++----- .../sql/sql/parser/AstBuilder.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 1680d27e3c..661322ea0d 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -12,10 +12,10 @@ Expressions Introduction ============ -The expression are expressions that return a scalar value. It can be used in many statements, for example +Expressions, particularly value expressions, are those which return a scalar value. Expressions have different types and forms. For example, there are literal values as atom expression and arithmetic, predicate and function expression built on top of them. And also expressions can be used in different clauses, such as using arithmetic expression in `SELECT`, `WHERE` or `HAVING` clause. -Literal Value -============= +Literal Values +============== Description ----------- @@ -24,7 +24,7 @@ A literal is a symbol that represents a value. The most common literal values in 1. Numeric literals: specify numeric values such as integer and floating-point numbers. 2. String literals: specify a string enclosed by single or double quotes. -3. Boolean literals: `true` or `false`. +3. Boolean literals: ``true`` or ``false``. Examples -------- @@ -46,5 +46,5 @@ The current implementation has the following limitations at the moment: 1. Only literals of data types listed as above are supported for now. Other type of literals, such as date and NULL, will be added in future. 2. Expression of literals, such as arithmetic expressions, will be supported later. -3. Standard ANSI `VALUES` clause is not supported, although this is implemented by a Values operator internally. +3. Standard ANSI ``VALUES`` clause is not supported, although the ``SELECT`` literal example above is implemented by a Values operator internally. diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java index 94af0b5a8f..40f42293d2 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java @@ -47,7 +47,7 @@ public UnresolvedPlan visitSimpleSelect(SimpleSelectContext ctx) { // Attach an Values operator with only a empty row inside so that // Project operator can have a chance to evaluate its expression - // though the evaluation doesn't have any dependency in the Values. + // though the evaluation doesn't have any dependency on what's in Values. Values emptyValue = new Values(ImmutableList.of(Collections.emptyList())); return project.attach(emptyValue); } From f8f922e85c3f80ff842345c6c9f71a8e83b20fe2 Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 18 Jun 2020 11:59:46 -0700 Subject: [PATCH 11/14] Use new syntax check exception --- .../sql/legacy/plugin/RestSQLQueryAction.java | 3 ++- .../sql/sql/antlr/SQLSyntaxParserTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java index b243ec4a27..e4efb4a3fa 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -22,6 +22,7 @@ import static org.elasticsearch.rest.RestStatus.OK; import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchNodeClient; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.ElasticsearchExecutionEngine; @@ -106,7 +107,7 @@ public RestChannelConsumer prepareRequest(SQLQueryRequest request, NodeClient no UnresolvedPlan ast; try { ast = sqlService.parse(request.getQuery()); - } catch (RuntimeException e) { //TODO: change to specific syntax exception + } catch (SyntaxCheckException e) { return NOT_SUPPORTED_YET; } return channel -> sqlService.execute(ast, createListener(channel)); diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java index ffd97095c0..9eafe6dafd 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; import org.antlr.v4.runtime.tree.ParseTree; import org.junit.jupiter.api.Test; @@ -34,7 +35,7 @@ public void canParseSelectLiterals() { @Test public void canNotParseInvalidSelect() { - assertThrows(RuntimeException.class, () -> parser.parse("SELECT ,")); + assertThrows(SyntaxCheckException.class, () -> parser.parse("SELECT ,")); } } \ No newline at end of file From dceff7332b77f3120ef86368448c8cdd2479e66a Mon Sep 17 00:00:00 2001 From: Dai Date: Fri, 19 Jun 2020 08:47:45 -0700 Subject: [PATCH 12/14] Unregister new handler and add UT --- docs/user/dql/expressions.rst | 2 +- .../sql/legacy/plugin/RestSQLQueryAction.java | 33 ++------ .../sql/legacy/plugin/RestSqlAction.java | 11 ++- .../legacy/plugin/RestSQLQueryActionTest.java | 79 +++++++++++++++++++ .../sql/plugin/SQLPlugin.java | 5 +- .../sql/sql/antlr/SQLSyntaxParserTest.java | 2 +- 6 files changed, 94 insertions(+), 38 deletions(-) create mode 100644 legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryActionTest.java diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 661322ea0d..d743485d83 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -12,7 +12,7 @@ Expressions Introduction ============ -Expressions, particularly value expressions, are those which return a scalar value. Expressions have different types and forms. For example, there are literal values as atom expression and arithmetic, predicate and function expression built on top of them. And also expressions can be used in different clauses, such as using arithmetic expression in `SELECT`, `WHERE` or `HAVING` clause. +Expressions, particularly value expressions, are those which return a scalar value. Expressions have different types and forms. For example, there are literal values as atom expression and arithmetic, predicate and function expression built on top of them. And also expressions can be used in different clauses, such as using arithmetic expression in ``SELECT``, ``WHERE`` or ``HAVING`` clause. Literal Values ============== diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java index e4efb4a3fa..97ab6d7245 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -35,7 +35,6 @@ import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; import java.io.IOException; import java.security.PrivilegedExceptionAction; -import java.util.Collections; import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -46,13 +45,12 @@ import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestStatus; -import org.json.JSONException; -import org.json.JSONObject; import org.springframework.context.annotation.AnnotationConfigApplicationContext; /** - * New SQL REST action handler. This will not registered to Elasticsearch until all old - * functionalities migrated to new query engine and legacy REST handler removed. + * New SQL REST action handler. This will not be registered to Elasticsearch unless: + * 1) we want to test new SQL engine; + * 2) all old functionalities migrated to new query engine and legacy REST handler removed. */ public class RestSQLQueryAction extends BaseRestHandler { @@ -60,9 +58,6 @@ public class RestSQLQueryAction extends BaseRestHandler { public static final RestChannelConsumer NOT_SUPPORTED_YET = null; - public static final String SQL_QUERY_ENDPOINT = "_opendistro/_newsql"; - private static final String SQL_QUERY_FIELD_NAME = "query"; - private final ClusterService clusterService; public RestSQLQueryAction(ClusterService clusterService) { @@ -77,19 +72,12 @@ public String getName() { @Override public List routes() { - return Collections.singletonList( - new Route(RestRequest.Method.POST, SQL_QUERY_ENDPOINT) - ); + throw new UnsupportedOperationException("New SQL handler is not ready yet"); } @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nodeClient) { - JSONObject jsonContent = parseJsonPayload(request); - String query = jsonContent.optString(SQL_QUERY_FIELD_NAME); - return prepareRequest( - new SQLQueryRequest(jsonContent, query, request.path(), ""), - nodeClient - ); + throw new UnsupportedOperationException("New SQL handler is not ready yet"); } /** @@ -113,17 +101,6 @@ public RestChannelConsumer prepareRequest(SQLQueryRequest request, NodeClient no return channel -> sqlService.execute(ast, createListener(channel)); } - private JSONObject parseJsonPayload(RestRequest restRequest) { - String content = restRequest.content().utf8ToString(); - JSONObject jsonContent; - try { - jsonContent = new JSONObject(content); - } catch (JSONException e) { - throw new IllegalArgumentException("Failed to parse request payload", e); - } - return jsonContent; - } - private SQLService createSQLService(NodeClient client) { return doPrivileged(() -> { AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java index 6b096a8241..a3e17b32b6 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java @@ -47,6 +47,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.rest.BaseRestHandler; @@ -94,10 +95,10 @@ public class RestSqlAction extends BaseRestHandler { */ private final RestSQLQueryAction newSqlQueryHandler; - public RestSqlAction(Settings settings, RestSQLQueryAction newSqlQueryHandler) { + public RestSqlAction(Settings settings, ClusterService clusterService) { super(); this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings); - this.newSqlQueryHandler = newSqlQueryHandler; + this.newSqlQueryHandler = new RestSQLQueryAction(clusterService); } @Override @@ -144,8 +145,10 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli Format format = SqlRequestParam.getFormat(request.params()); // Route request to new query engine if it's supported already - SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), sqlRequest.getSql(), - request.path(), format.getFormatName()); + SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), + sqlRequest.getSql(), + request.path(), + format.getFormatName()); RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { return result; diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryActionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryActionTest.java new file mode 100644 index 0000000000..cf3f4b6ff7 --- /dev/null +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryActionTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSQLQueryAction.NOT_SUPPORTED_YET; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; +import org.json.JSONObject; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class RestSQLQueryActionTest { + + @Mock + private ClusterService clusterService; + + @Mock + private NodeClient nodeClient; + + @Test + public void handleQueryThatCanSupport() { + SQLQueryRequest request = new SQLQueryRequest( + new JSONObject("{\"query\": \"SELECT -123\"}"), + "SELECT -123", + QUERY_API_ENDPOINT, + ""); + + RestSQLQueryAction queryAction = new RestSQLQueryAction(clusterService); + assertNotSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); + } + + @Test + public void skipExplainThatNotSupport() { + SQLQueryRequest request = new SQLQueryRequest( + new JSONObject("{\"query\": \"SELECT * FROM test\"}"), + "SELECT * FROM test", + EXPLAIN_API_ENDPOINT, + ""); + + RestSQLQueryAction queryAction = new RestSQLQueryAction(clusterService); + assertSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); + } + + @Test + public void skipQueryThatNotSupport() { + SQLQueryRequest request = new SQLQueryRequest( + new JSONObject("{\"query\": \"SELECT * FROM test\"}"), + "SELECT * FROM test", + QUERY_API_ENDPOINT, + ""); + + RestSQLQueryAction queryAction = new RestSQLQueryAction(clusterService); + assertSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); + } + +} \ No newline at end of file diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java index 58e53fce0f..8df9180480 100644 --- a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java @@ -18,7 +18,6 @@ import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import com.amazon.opendistroforelasticsearch.sql.legacy.executor.AsyncRestExecutor; import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSQLQueryAction; import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction; import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlSettingsAction; import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlStatsAction; @@ -83,11 +82,9 @@ public List getRestHandlers(Settings settings, RestController restC LocalClusterState.state().setResolver(indexNameExpressionResolver); Metrics.getInstance().registerDefaultMetrics(); - RestSQLQueryAction newSqlQueryHandler = new RestSQLQueryAction(clusterService); return Arrays.asList( new RestPPLQueryAction(restController, clusterService), - newSqlQueryHandler, - new RestSqlAction(settings, newSqlQueryHandler), + new RestSqlAction(settings, clusterService), new RestSqlStatsAction(settings, restController), new RestSqlSettingsAction(settings, restController) ); diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java index 9eafe6dafd..7bd8d93b2b 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -35,7 +35,7 @@ public void canParseSelectLiterals() { @Test public void canNotParseInvalidSelect() { - assertThrows(SyntaxCheckException.class, () -> parser.parse("SELECT ,")); + assertThrows(SyntaxCheckException.class, () -> parser.parse("SELECT * FROM test")); } } \ No newline at end of file From efb49a8da78286014272ca4f6eddae0ac59c40fa Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 25 Jun 2020 09:48:17 -0700 Subject: [PATCH 13/14] Support fetch size 0 which is default request by JDBC driver --- .../sql/sql/domain/SQLQueryRequest.java | 5 +++-- .../sql/sql/domain/SQLQueryRequestTest.java | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java index 397d51f194..5857a0933f 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java @@ -74,8 +74,9 @@ private boolean isExplainRequest() { } private boolean isOnlyQueryFieldInPayload() { - return jsonContent.keySet().size() == 1 - && jsonContent.has("query"); + return (jsonContent.keySet().size() == 1 && jsonContent.has("query")) + || (jsonContent.keySet().size() == 2 && jsonContent.has("query") + && jsonContent.has("fetch_size") && jsonContent.getInt("fetch_size") == 0); } private boolean isDefaultFormat() { diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java index 1f906adf29..6838b6be73 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java @@ -38,6 +38,15 @@ public void shouldSupportQueryWithJDBCFormat() { assertTrue(request.isSupported()); } + @Test + public void shouldSupportQueryWithZeroFetchSize() { + SQLQueryRequest request = + SQLQueryRequestBuilder.request("SELECT 1") + .jsonContent("{\"query\": \"SELECT 1\", \"fetch_size\": 0}") + .build(); + assertTrue(request.isSupported()); + } + @Test public void shouldNotSupportExplain() { SQLQueryRequest explainRequest = From 5a062ce62470ae1ae4f0515d8d58d09ce909aeb9 Mon Sep 17 00:00:00 2001 From: Dai Date: Fri, 26 Jun 2020 15:25:40 -0700 Subject: [PATCH 14/14] Address PR comments --- .../sql/planner/logical/LogicalValues.java | 14 +++++++++++++- .../sql/legacy/plugin/RestSqlAction.java | 4 ++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java index 79c64bf9c3..8ad790da7d 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java @@ -25,7 +25,19 @@ import lombok.ToString; /** - * Logical operator which is a sequence of literal list. + * Logical operator which is a sequence of literal rows (like a relation). + * Basically, Values operator is used to create rows of constant literals + * "out of nothing" which is corresponding with VALUES clause in SQL. + * Mostly all rows must have the same number of literals and each column should + * have same type or can be converted implicitly. + * In particular, typical use cases include: + * 1. Project without relation involved. + * 2. Defining query or insertion without a relation. + * Take the following logical plan for example: + * 
+ *  LogicalProject(expr=[log(2),true,1+2])
+ *   |_ LogicalValues([[]])  #an empty row so that Project can evaluate its expressions in next()
+ *
*/ @ToString @Getter diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java index a3e17b32b6..2af051a1dd 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java @@ -151,8 +151,12 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli format.getFormatName()); RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { + LOG.info("[{}] Request {} is handled by new SQL query engine", + LogUtils.getRequestId(), newSqlRequest); return result; } + LOG.debug("[{}] Request {} is not supported and falling back to old SQL engine", + LogUtils.getRequestId(), newSqlRequest); final QueryAction queryAction = explainRequest(client, sqlRequest, format); return channel -> executeSqlRequest(request, queryAction, client, channel);