From 4e5730c24c427b9768714ade53c19ee5b07da580 Mon Sep 17 00:00:00 2001 From: Fu Zhe Date: Tue, 3 Aug 2021 16:12:58 +0800 Subject: [PATCH] Remove uesless Analyzers (#2549) --- dbms/src/Analyzers/AnalyzeArrayJoins.h | 46 -- dbms/src/Analyzers/AnalyzeColumns.cpp | 455 --------------- dbms/src/Analyzers/AnalyzeColumns.h | 48 -- dbms/src/Analyzers/AnalyzeLambdas.cpp | 201 ------- dbms/src/Analyzers/AnalyzeLambdas.h | 35 -- dbms/src/Analyzers/AnalyzeResultOfQuery.cpp | 70 --- dbms/src/Analyzers/AnalyzeResultOfQuery.h | 32 -- dbms/src/Analyzers/CMakeLists.txt | 3 - dbms/src/Analyzers/CollectAliases.cpp | 109 ---- dbms/src/Analyzers/CollectAliases.h | 54 -- dbms/src/Analyzers/CollectTables.cpp | 186 ------ dbms/src/Analyzers/CollectTables.h | 52 -- dbms/src/Analyzers/ExecuteTableFunctions.cpp | 80 --- dbms/src/Analyzers/ExecuteTableFunctions.h | 35 -- .../Analyzers/OptimizeGroupOrderLimitBy.cpp | 264 --------- .../src/Analyzers/OptimizeGroupOrderLimitBy.h | 36 -- .../TranslatePositionalArguments.cpp | 86 --- .../Analyzers/TranslatePositionalArguments.h | 21 - .../Analyzers/TypeAndConstantInference.cpp | 529 ------------------ dbms/src/Analyzers/TypeAndConstantInference.h | 59 -- dbms/src/Analyzers/tests/CMakeLists.txt | 24 - dbms/src/Analyzers/tests/analyze_columns.cpp | 71 --- .../Analyzers/tests/analyze_columns.reference | 18 - dbms/src/Analyzers/tests/analyze_columns.sh | 7 - dbms/src/Analyzers/tests/analyze_lambdas.cpp | 44 -- .../Analyzers/tests/analyze_lambdas.reference | 4 - dbms/src/Analyzers/tests/analyze_lambdas.sh | 3 - .../tests/analyze_result_of_query.cpp | 53 -- .../tests/analyze_result_of_query.reference | 1 - .../tests/analyze_result_of_query.sh | 3 - dbms/src/Analyzers/tests/collect_aliases.cpp | 36 -- .../Analyzers/tests/collect_aliases.reference | 8 - dbms/src/Analyzers/tests/collect_aliases.sh | 3 - dbms/src/Analyzers/tests/collect_tables.cpp | 56 -- .../tests/optimize_group_order_limit_by.cpp | 80 --- .../optimize_group_order_limit_by.reference | 19 - .../tests/optimize_group_order_limit_by.sh | 5 - .../tests/translate_positional_arguments.cpp | 37 -- .../translate_positional_arguments.reference | 8 - .../tests/translate_positional_arguments.sh | 3 - .../tests/type_and_constant_inference.cpp | 78 --- .../type_and_constant_inference.reference | 34 -- .../tests/type_and_constant_inference.sh | 26 - dbms/src/CMakeLists.txt | 1 - .../Storages/DeltaMerge/Index/MinMaxIndex.cpp | 1 - 45 files changed, 3024 deletions(-) delete mode 100644 dbms/src/Analyzers/AnalyzeArrayJoins.h delete mode 100644 dbms/src/Analyzers/AnalyzeColumns.cpp delete mode 100644 dbms/src/Analyzers/AnalyzeColumns.h delete mode 100644 dbms/src/Analyzers/AnalyzeLambdas.cpp delete mode 100644 dbms/src/Analyzers/AnalyzeLambdas.h delete mode 100644 dbms/src/Analyzers/AnalyzeResultOfQuery.cpp delete mode 100644 dbms/src/Analyzers/AnalyzeResultOfQuery.h delete mode 100644 dbms/src/Analyzers/CMakeLists.txt delete mode 100644 dbms/src/Analyzers/CollectAliases.cpp delete mode 100644 dbms/src/Analyzers/CollectAliases.h delete mode 100644 dbms/src/Analyzers/CollectTables.cpp delete mode 100644 dbms/src/Analyzers/CollectTables.h delete mode 100644 dbms/src/Analyzers/ExecuteTableFunctions.cpp delete mode 100644 dbms/src/Analyzers/ExecuteTableFunctions.h delete mode 100644 dbms/src/Analyzers/OptimizeGroupOrderLimitBy.cpp delete mode 100644 dbms/src/Analyzers/OptimizeGroupOrderLimitBy.h delete mode 100644 dbms/src/Analyzers/TranslatePositionalArguments.cpp delete mode 100644 dbms/src/Analyzers/TranslatePositionalArguments.h delete mode 100644 dbms/src/Analyzers/TypeAndConstantInference.cpp delete mode 100644 dbms/src/Analyzers/TypeAndConstantInference.h delete mode 100644 dbms/src/Analyzers/tests/CMakeLists.txt delete mode 100644 dbms/src/Analyzers/tests/analyze_columns.cpp delete mode 100644 dbms/src/Analyzers/tests/analyze_columns.reference delete mode 100755 dbms/src/Analyzers/tests/analyze_columns.sh delete mode 100644 dbms/src/Analyzers/tests/analyze_lambdas.cpp delete mode 100644 dbms/src/Analyzers/tests/analyze_lambdas.reference delete mode 100755 dbms/src/Analyzers/tests/analyze_lambdas.sh delete mode 100644 dbms/src/Analyzers/tests/analyze_result_of_query.cpp delete mode 100644 dbms/src/Analyzers/tests/analyze_result_of_query.reference delete mode 100755 dbms/src/Analyzers/tests/analyze_result_of_query.sh delete mode 100644 dbms/src/Analyzers/tests/collect_aliases.cpp delete mode 100644 dbms/src/Analyzers/tests/collect_aliases.reference delete mode 100755 dbms/src/Analyzers/tests/collect_aliases.sh delete mode 100644 dbms/src/Analyzers/tests/collect_tables.cpp delete mode 100644 dbms/src/Analyzers/tests/optimize_group_order_limit_by.cpp delete mode 100644 dbms/src/Analyzers/tests/optimize_group_order_limit_by.reference delete mode 100755 dbms/src/Analyzers/tests/optimize_group_order_limit_by.sh delete mode 100644 dbms/src/Analyzers/tests/translate_positional_arguments.cpp delete mode 100644 dbms/src/Analyzers/tests/translate_positional_arguments.reference delete mode 100755 dbms/src/Analyzers/tests/translate_positional_arguments.sh delete mode 100644 dbms/src/Analyzers/tests/type_and_constant_inference.cpp delete mode 100644 dbms/src/Analyzers/tests/type_and_constant_inference.reference delete mode 100755 dbms/src/Analyzers/tests/type_and_constant_inference.sh diff --git a/dbms/src/Analyzers/AnalyzeArrayJoins.h b/dbms/src/Analyzers/AnalyzeArrayJoins.h deleted file mode 100644 index 4bc9c3fb72b..00000000000 --- a/dbms/src/Analyzers/AnalyzeArrayJoins.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ - -class WriteBuffer; -struct CollectAliases; -struct CollectTables; - - -/** For every ARRAY JOIN, collect a map: - * result alias -> source - * - * There could be several variants: - * - * SELECT elem FROM t ARRAY JOIN array AS elem elem -> array - * SELECT n.elem FROM t ARRAY JOIN nested AS n n -> nested - * SELECT array FROM t ARRAY JOIN array array -> array - * SELECT nested.elem FROM t ARRAY JOIN nested nested -> nested - * SELECT elem FROM t ARRAY JOIN [1, 2, 3] AS elem elem -> [1, 2, 3] - * - * Does not analyze arrayJoin functions. - */ -struct AnalyzeArrayJoins -{ - void process(const ASTPtr & ast); - - struct SourceInfo - { - String column_name; - ASTPtr node; - }; - - using ResultToSource = std::unordered_map; - using ArrayJoins = std::vector; - - /// Debug output - void dump(WriteBuffer & out) const; -}; - -} diff --git a/dbms/src/Analyzers/AnalyzeColumns.cpp b/dbms/src/Analyzers/AnalyzeColumns.cpp deleted file mode 100644 index 73760c4ff01..00000000000 --- a/dbms/src/Analyzers/AnalyzeColumns.cpp +++ /dev/null @@ -1,455 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int NOT_IMPLEMENTED; - extern const int AMBIGUOUS_TABLE_NAME; - extern const int AMBIGUOUS_COLUMN_NAME; - extern const int UNKNOWN_TABLE; - extern const int THERE_IS_NO_COLUMN; -} - - -namespace -{ - -/// Find by fully qualified name, like db.table.column -const CollectTables::TableInfo * findTableByDatabaseAndTableName( - const CollectTables & tables, const String & database_name, const String & table_name) -{ - for (const auto & table : tables.tables) - if (table.database_name == database_name && table.table_name == table_name) - return &table; - - return nullptr; -} - - -/** Find by single-qualified name, like table.column or alias.column. - * - * There are primary matches: - * when name is alias like - * SELECT name.column FROM (SELECT 1) AS name - * or name is table in current database like - * SELECT name.column FROM name - * - * And secondary matches: - * when name is name of table in explicitly specified database like - * SELECT name.column FROM db.name - * - * If there is only one primary match - return it. - * If there is many primary matches - ambiguity. - * If there is no primary matches and only one secondary match - return it. - * If there is no primary matches and many secondary matches - ambiguity. - * If there is no any matches - not found. - */ -const CollectTables::TableInfo * findTableByNameOrAlias( - const CollectTables & tables, const String & name) -{ - const CollectTables::TableInfo * primary_match = nullptr; - const CollectTables::TableInfo * secondary_match = nullptr; - - for (const auto & table : tables.tables) - { - if (table.alias == name - || (table.database_name.empty() && table.table_name == name)) - { - if (primary_match) - throw Exception("Table name " + backQuoteIfNeed(name) + " is ambiguous", ErrorCodes::AMBIGUOUS_TABLE_NAME); - primary_match = &table; - } - else if (!primary_match && table.table_name == name) - { - if (secondary_match) - throw Exception("Table name " + backQuoteIfNeed(name) + " is ambiguous", ErrorCodes::AMBIGUOUS_TABLE_NAME); - secondary_match = &table; - } - } - - if (primary_match) - return primary_match; - if (secondary_match) - return secondary_match; - return nullptr; -} - - -/** Find table in case when its name is not specified. Like just - * SELECT column FROM t1, t2 - * Select a table, where specified column exists. - * If more than one such table - ambiguity. - */ -const CollectTables::TableInfo * findTableWithUnqualifiedName(const CollectTables & tables, const String & column_name) -{ - const CollectTables::TableInfo * res = nullptr; - - for (const auto & table : tables.tables) - { - if (table.structure_of_subquery) - { - if (table.structure_of_subquery.has(column_name)) - { - if (res) - throw Exception("Ambiguous column name " + backQuoteIfNeed(column_name), ErrorCodes::AMBIGUOUS_COLUMN_NAME); - res = &table; - break; - } - } - else if (table.storage) - { - if (table.storage->hasColumn(column_name)) - { - if (res) - throw Exception("Ambiguous column name " + backQuoteIfNeed(column_name), ErrorCodes::AMBIGUOUS_COLUMN_NAME); - res = &table; - } - } - else - throw Exception("Logical error: no storage and no structure of subquery is specified for table", ErrorCodes::LOGICAL_ERROR); - } - - return res; -} - - -/// Create maximum-qualified identifier for column in table. -ASTPtr createASTIdentifierForColumnInTable(const String & column, const CollectTables::TableInfo & table) -{ - ASTPtr database_name_identifier_node; - if (!table.database_name.empty()) - database_name_identifier_node = std::make_shared(table.database_name, ASTIdentifier::Column); - - ASTPtr table_name_identifier_node; - String table_name_or_alias; - - if (!table.table_name.empty()) - table_name_or_alias = table.table_name; - else if (table.database_name.empty() && !table.alias.empty()) - table_name_or_alias = table.alias; - - if (!table_name_or_alias.empty()) - table_name_identifier_node = std::make_shared(table_name_or_alias, ASTIdentifier::Column); - - ASTPtr column_identifier_node = std::make_shared(column, ASTIdentifier::Column); - - String compound_name; - if (database_name_identifier_node) - compound_name += table.database_name + "."; - if (table_name_identifier_node) - compound_name += table_name_or_alias + "."; - compound_name += column; - - auto elem = std::make_shared(compound_name, ASTIdentifier::Column); - - if (database_name_identifier_node) - elem->children.emplace_back(std::move(database_name_identifier_node)); - if (table_name_identifier_node) - elem->children.emplace_back(std::move(table_name_identifier_node)); - if (!elem->children.empty()) - elem->children.emplace_back(std::move(column_identifier_node)); - - return elem; -} - - -void createASTsForAllColumnsInTable(const CollectTables::TableInfo & table, ASTs & res) -{ - if (table.storage) - for (const auto & name : table.storage->getColumns().getNamesOfPhysical()) - res.emplace_back(createASTIdentifierForColumnInTable(name, table)); - else - for (size_t i = 0, size = table.structure_of_subquery.columns(); i < size; ++i) - res.emplace_back(createASTIdentifierForColumnInTable(table.structure_of_subquery.getByPosition(i).name, table)); -} - - -ASTs expandUnqualifiedAsterisk(const CollectTables & tables) -{ - ASTs res; - for (const auto & table : tables.tables) - createASTsForAllColumnsInTable(table, res); - return res; -} - - -ASTs expandQualifiedAsterisk( - const IAST & ast, const CollectTables & tables) -{ - if (ast.children.size() != 1) - throw Exception("Logical error: AST node for qualified asterisk has number of children not equal to one", ErrorCodes::LOGICAL_ERROR); - - const ASTIdentifier & qualifier = static_cast(*ast.children[0]); - - const CollectTables::TableInfo * table = nullptr; - - if (qualifier.children.empty()) - table = findTableByNameOrAlias(tables, qualifier.name); - else if (qualifier.children.size() == 2) - table = findTableByDatabaseAndTableName(tables, - static_cast(*qualifier.children[0]).name, - static_cast(*qualifier.children[1]).name); - else - throw Exception("Unsupported number of components in asterisk qualifier", ErrorCodes::NOT_IMPLEMENTED); - - /// TODO Implement for case table.nested.* and database.table.nested.* - - if (!table) - throw Exception("There is no table " + qualifier.name + " in query", ErrorCodes::UNKNOWN_TABLE); - - ASTs res; - createASTsForAllColumnsInTable(*table, res); - return res; -} - - -void processIdentifier( - const ASTPtr & ast, AnalyzeColumns::Columns & columns, const CollectAliases & aliases, const CollectTables & tables) -{ - const ASTIdentifier & identifier = static_cast(*ast); - - if (aliases.aliases.count(identifier.name)) - return; - - if (columns.count(identifier.name)) - return; - - const CollectTables::TableInfo * table = nullptr; - String column_name; - - if (identifier.children.empty()) - { - /** Lambda parameters are not columns from table. Just skip them. - * This step requires AnalyzeLambdas to be done on AST. - */ - if (startsWith(identifier.name, "_lambda")) - return; - - table = findTableWithUnqualifiedName(tables, identifier.name); - if (table) - column_name = identifier.name; - } - else if (identifier.children.size() == 2) - { - const String & first = static_cast(*identifier.children[0]).name; - const String & second = static_cast(*identifier.children[1]).name; - - /// table.column - table = findTableByNameOrAlias(tables, first); - - if (table) - { - column_name = second; - } - else - { - /// column.nested - table = findTableWithUnqualifiedName(tables, identifier.name); - if (table) - column_name = identifier.name; - } - } - else if (identifier.children.size() == 3) - { - const String & first = static_cast(*identifier.children[0]).name; - const String & second = static_cast(*identifier.children[1]).name; - const String & third = static_cast(*identifier.children[2]).name; - - /// database.table.column - table = findTableByDatabaseAndTableName(tables, first, second); - - if (table) - { - column_name = third; - } - else - { - /// table.column.nested - table = findTableByNameOrAlias(tables, first); - - if (table) - { - column_name = second + "." + third; - } - else - { - /// column.nested.nested - table = findTableWithUnqualifiedName(tables, identifier.name); - if (table) - column_name = identifier.name; - } - } - } - - if (!table) - throw Exception("Cannot find column " + identifier.name, ErrorCodes::THERE_IS_NO_COLUMN); - - AnalyzeColumns::ColumnInfo info; - info.node = ast; - info.table = *table; - info.name_in_table = column_name; - - if (table->structure_of_subquery) - { - if (!table->structure_of_subquery.has(column_name)) - throw Exception("Cannot find column " + backQuoteIfNeed(column_name) + " in subquery", ErrorCodes::LOGICAL_ERROR); - - info.data_type = table->structure_of_subquery.getByName(column_name).type; - } - else if (table->storage) - { - info.data_type = table->storage->getColumn(column_name).type; - } - else - throw Exception("Logical error: no storage and no structure of subquery is specified for table", ErrorCodes::LOGICAL_ERROR); - - columns[identifier.name] = info; -} - - -void processImpl(ASTPtr & ast, AnalyzeColumns::Columns & columns, const CollectAliases & aliases, const CollectTables & tables) -{ - /// Don't go into subqueries and table-like expressions. - if (typeid_cast(ast.get()) - || typeid_cast(ast.get())) - { - return; - } - else if (const ASTFunction * func = typeid_cast(ast.get())) - { - String func_name_lowercase = Poco::toLower(func->name); - - /// As special case, treat count(*) as count(), not as count(list of all columns). - if (func_name_lowercase == "count" && func->arguments->children.size() == 1 - && typeid_cast(func->arguments->children[0].get())) - { - func->arguments->children.clear(); - } - } - else if (typeid_cast(ast.get())) - { - /// Replace asterisks to list of columns. - ASTs & asts = ast->children; - for (int i = static_cast(asts.size()) - 1; i >= 0; --i) - { - if (typeid_cast(asts[i].get())) - { - ASTs expanded = expandUnqualifiedAsterisk(tables); - asts.erase(asts.begin() + i); - asts.insert(asts.begin() + i, expanded.begin(), expanded.end()); - } - else if (ASTQualifiedAsterisk * asterisk = typeid_cast(asts[i].get())) - { - ASTs expanded = expandQualifiedAsterisk(*asterisk, tables); - asts.erase(asts.begin() + i); - asts.insert(asts.begin() + i, expanded.begin(), expanded.end()); - } - } - } - else if (typeid_cast(ast.get())) - { - processIdentifier(ast, columns, aliases, tables); - return; - } - - for (auto & child : ast->children) - processImpl(child, columns, aliases, tables); -} - -} - - -void AnalyzeColumns::process(ASTPtr & ast, const CollectAliases & aliases, const CollectTables & tables) -{ - /// If this is SELECT query, don't go into FORMAT and SETTINGS clauses - /// - they contain identifiers that are not columns. - const ASTSelectQuery * select = typeid_cast(ast.get()); - - for (auto & child : ast->children) - { - if (select && child.get() == select->settings.get()) - continue; - - processImpl(child, columns, aliases, tables); - } -} - - -void AnalyzeColumns::dump(WriteBuffer & out) const -{ - /// For need of tests, we need to dump result in some fixed order. - std::vector vec; - vec.reserve(columns.size()); - for (auto it = columns.begin(); it != columns.end(); ++it) - vec.emplace_back(it); - - std::sort(vec.begin(), vec.end(), [](const auto & a, const auto & b) { return a->first < b->first; }); - - for (const auto & it : vec) - { - writeString(it->first, out); - writeCString(" -> ", out); - - writeProbablyBackQuotedString(it->second.name_in_table, out); - writeCString(" ", out); - writeProbablyBackQuotedString(it->second.data_type->getName(), out); - - const auto & table = it->second.table; - - writeCString(". Database name: ", out); - if (table.database_name.empty()) - writeCString("(none)", out); - else - writeProbablyBackQuotedString(table.database_name, out); - - writeCString(". Table name: ", out); - if (table.table_name.empty()) - writeCString("(none)", out); - else - writeProbablyBackQuotedString(table.table_name, out); - - writeCString(". Alias: ", out); - if (table.alias.empty()) - writeCString("(none)", out); - else - writeProbablyBackQuotedString(table.alias, out); - - writeCString(". Storage: ", out); - if (!table.storage) - writeCString("(none)", out); - else - writeProbablyBackQuotedString(table.storage->getName(), out); - - writeCString(". AST: ", out); - if (it->second.node) - { - std::stringstream formatted_ast; - formatAST(*it->second.node, formatted_ast, false, true); - writeString(formatted_ast.str(), out); - } - else - writeCString("(none)", out); - - writeChar('\n', out); - } -} - - -} diff --git a/dbms/src/Analyzers/AnalyzeColumns.h b/dbms/src/Analyzers/AnalyzeColumns.h deleted file mode 100644 index cf93bbc5ed6..00000000000 --- a/dbms/src/Analyzers/AnalyzeColumns.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ - -class WriteBuffer; -struct CollectAliases; -struct CollectTables; - - -/** For every identifier, that is not an alias, - * determine from what table it comes, - * its original name in table, - * and its data type. - * - * Also: - * - expand asterisks (such as *, t.*, db.table.* and (TODO) even db.table.nested.*) to corresponding list of columns; - * - translate count(*) to count(); - * - TODO expand alias columns that come from table definition; - * - TODO replace column names to fully qualified names: identical columns will have same names. - * - * If column is not found or in case of ambiguity, throw an exception. - */ -struct AnalyzeColumns -{ - void process(ASTPtr & ast, const CollectAliases & aliases, const CollectTables & tables); - - struct ColumnInfo - { - ASTPtr node; - CollectTables::TableInfo table; - String name_in_table; - DataTypePtr data_type; - }; - - using Columns = std::unordered_map; - Columns columns; - - /// Debug output - void dump(WriteBuffer & out) const; -}; - -} diff --git a/dbms/src/Analyzers/AnalyzeLambdas.cpp b/dbms/src/Analyzers/AnalyzeLambdas.cpp deleted file mode 100644 index a060080c2a4..00000000000 --- a/dbms/src/Analyzers/AnalyzeLambdas.cpp +++ /dev/null @@ -1,201 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int BAD_LAMBDA; - extern const int RESERVED_IDENTIFIER_NAME; -} - - -AnalyzeLambdas::LambdaParameters AnalyzeLambdas::extractLambdaParameters(ASTPtr & ast) -{ - /// Lambda parameters could be specified in AST in two forms: - /// - just as single parameter: x -> x + 1 - /// - parameters in tuple: (x, y) -> x + 1 - -#define LAMBDA_ERROR_MESSAGE " There are two valid forms of lambda expressions: x -> ... and (x, y...) -> ..." - - if (!ast->tryGetAlias().empty()) - throw Exception("Lambda parameters cannot have aliases." - LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA); - - if (const ASTIdentifier * identifier = typeid_cast(ast.get())) - { - return { identifier->name }; - } - else if (const ASTFunction * function = typeid_cast(ast.get())) - { - if (function->name != "tuple") - throw Exception("Left hand side of '->' or first argument of 'lambda' is a function, but this function is not tuple." - LAMBDA_ERROR_MESSAGE " Found function '" + function->name + "' instead.", ErrorCodes::BAD_LAMBDA); - - if (!function->arguments || function->arguments->children.empty()) - throw Exception("Left hand side of '->' or first argument of 'lambda' is empty tuple." - LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA); - - LambdaParameters res; - res.reserve(function->arguments->children.size()); - - for (const ASTPtr & arg : function->arguments->children) - { - const ASTIdentifier * arg_identifier = typeid_cast(arg.get()); - - if (!arg_identifier) - throw Exception("Left hand side of '->' or first argument of 'lambda' contains something that is not just identifier." - LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA); - - if (!arg_identifier->children.empty()) - throw Exception("Left hand side of '->' or first argument of 'lambda' contains compound identifier." - LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA); - - if (!arg_identifier->alias.empty()) - throw Exception("Lambda parameters cannot have aliases." - LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA); - - res.emplace_back(arg_identifier->name); - } - - return res; - - } - else - throw Exception("Unexpected left hand side of '->' or first argument of 'lambda'." - LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA); - -#undef LAMBDA_ERROR_MESSAGE -} - - -namespace -{ - - -/// Currently visible parameters in all scopes of lambda expressions. -/// Lambda expressions could be nested: arrayMap(x -> arrayMap(y -> x[y], x), [[1], [2, 3]]) -using LambdaScopes = std::vector; - -void processIdentifier(ASTPtr & ast, LambdaScopes & lambda_scopes) -{ - ASTIdentifier & identifier = static_cast(*ast); - - if (identifier.children.empty()) - { - bool found = false; - - /// From most inner scope towards outer scopes. - for (ssize_t num_scopes = lambda_scopes.size(), scope_idx = num_scopes - 1; scope_idx >= 0; --scope_idx) - { - for (size_t arg_idx = 0, num_args = lambda_scopes[scope_idx].size(); arg_idx < num_args; ++arg_idx) - { - if (lambda_scopes[scope_idx][arg_idx] == identifier.name) - { - identifier.name = "_lambda" + toString(scope_idx) + "_arg" + toString(arg_idx); - - found = true; - break; - } - } - if (found) - break; - } - - if (!found && startsWith(identifier.name, "_lambda")) - throw Exception("Identifier names starting with '_lambda' are reserved for parameters of lambda expressions.", - ErrorCodes::RESERVED_IDENTIFIER_NAME); - } -} - - -void processImpl( - ASTPtr & ast, - LambdaScopes & lambda_scopes, - const ASTPtr & parent_function_for_this_argument, - AnalyzeLambdas::HigherOrderFunctions & higher_order_functions) -{ - /// Don't go into subqueries and table-like expressions. - if (typeid_cast(ast.get()) - || typeid_cast(ast.get())) - { - return; - } - else if (ASTFunction * func = typeid_cast(ast.get())) - { - /** We must memoize parameters from left hand side (x, y) and then analyze right hand side. - */ - if (func->name == "lambda") - { - auto num_arguments = func->arguments->children.size(); - if (num_arguments != 2) - throw Exception("Lambda expression ('->' or 'lambda' function) must have exactly two arguments." - " Found " + toString(num_arguments) + " instead.", ErrorCodes::BAD_LAMBDA); - - lambda_scopes.emplace_back(AnalyzeLambdas::extractLambdaParameters(func->arguments->children[0])); - for (size_t i = 0; i < num_arguments; ++i) - processImpl(func->arguments->children[i], lambda_scopes, nullptr, higher_order_functions); - lambda_scopes.pop_back(); - - if (!parent_function_for_this_argument) - throw Exception("Lambda expression ('->' or 'lambda' function) must be presented as an argument of higher-order function." - " Found standalone lambda expression instead.", ErrorCodes::BAD_LAMBDA); - - higher_order_functions.emplace_back(parent_function_for_this_argument); - } - else - { - /// When diving into function arguments, pass current ast node. - if (func->arguments) - for (auto & child : func->arguments->children) - processImpl(child, lambda_scopes, ast, higher_order_functions); - - if (func->parameters) - for (auto & child : func->parameters->children) - processImpl(child, lambda_scopes, nullptr, higher_order_functions); - } - - return; - } - else if (typeid_cast(ast.get())) - { - processIdentifier(ast, lambda_scopes); - return; - } - - for (auto & child : ast->children) - processImpl(child, lambda_scopes, nullptr, higher_order_functions); -} - -} - - -void AnalyzeLambdas::process(ASTPtr & ast) -{ - LambdaScopes lambda_scopes; - for (auto & child : ast->children) - processImpl(child, lambda_scopes, nullptr, higher_order_functions); -} - - -void AnalyzeLambdas::dump(WriteBuffer & out) const -{ - for (const auto & ast : higher_order_functions) - { - writeString(ast->getColumnName(), out); - writeChar('\n', out); - } -} - - -} diff --git a/dbms/src/Analyzers/AnalyzeLambdas.h b/dbms/src/Analyzers/AnalyzeLambdas.h deleted file mode 100644 index 0a693737295..00000000000 --- a/dbms/src/Analyzers/AnalyzeLambdas.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include -#include - - -namespace DB -{ - -class WriteBuffer; - - -/** For every lambda expression, rename its parameters to '_lambda0_arg0' form. - * Check correctness of lambda expressions. - * Find functions, that have lambda expressions as arguments (they are called "higher order" functions). - * - * This should be done before CollectAliases. - */ -struct AnalyzeLambdas -{ - void process(ASTPtr & ast); - - /// Parameters of lambda expressions. - using LambdaParameters = std::vector; - static LambdaParameters extractLambdaParameters(ASTPtr & ast); - - - using HigherOrderFunctions = std::vector; - HigherOrderFunctions higher_order_functions; - - /// Debug output - void dump(WriteBuffer & out) const; -}; - -} diff --git a/dbms/src/Analyzers/AnalyzeResultOfQuery.cpp b/dbms/src/Analyzers/AnalyzeResultOfQuery.cpp deleted file mode 100644 index fa8f3b52dce..00000000000 --- a/dbms/src/Analyzers/AnalyzeResultOfQuery.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int UNEXPECTED_AST_STRUCTURE; -} - - -void AnalyzeResultOfQuery::process(ASTPtr & ast, const Context & context, ExecuteTableFunctions & table_functions) -{ - const ASTSelectQuery * select = typeid_cast(ast.get()); - if (!select) - throw Exception("AnalyzeResultOfQuery::process was called for not a SELECT query", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - if (!select->select_expression_list) - throw Exception("SELECT query doesn't have select_expression_list", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - AnalyzeLambdas analyze_lambdas; - analyze_lambdas.process(ast); - - CollectAliases collect_aliases; - collect_aliases.process(ast); - - CollectTables collect_tables; - collect_tables.process(ast, context, collect_aliases, table_functions); - - AnalyzeColumns analyze_columns; - analyze_columns.process(ast, collect_aliases, collect_tables); - - TypeAndConstantInference inference; - inference.process(ast, context, collect_aliases, analyze_columns, analyze_lambdas, table_functions); - - for (const ASTPtr & child : select->select_expression_list->children) - { - auto it = inference.info.find(child->getColumnName()); - if (it == inference.info.end()) - throw Exception("Logical error: type information for result column of SELECT query was not inferred", ErrorCodes::LOGICAL_ERROR); - - String name = child->getAliasOrColumnName(); - const TypeAndConstantInference::ExpressionInfo & info = it->second; - - result.insert(ColumnWithTypeAndName( - info.is_constant_expression ? info.data_type->createColumnConst(1, info.value) : nullptr, - info.data_type, - std::move(name))); - } -} - - -void AnalyzeResultOfQuery::dump(WriteBuffer & out) const -{ - writeString(result.dumpStructure(), out); -} - - -} diff --git a/dbms/src/Analyzers/AnalyzeResultOfQuery.h b/dbms/src/Analyzers/AnalyzeResultOfQuery.h deleted file mode 100644 index 54a69c1758e..00000000000 --- a/dbms/src/Analyzers/AnalyzeResultOfQuery.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include -#include - - -namespace DB -{ - -class WriteBuffer; -class Context; -struct ExecuteTableFunctions; - - -/** For SELECT query, determine names and types of columns of result, - * and if some columns are constant expressions, calculate their values. - * - * NOTE It's possible to memoize calculations, that happens under the hood - * and could be duplicated in subsequent analysis of subqueries. - */ -struct AnalyzeResultOfQuery -{ - void process(ASTPtr & ast, const Context & context, ExecuteTableFunctions & table_functions); - - /// Block will have non-nullptr columns for constant expressions. - Block result; - - /// Debug output - void dump(WriteBuffer & out) const; -}; - -} diff --git a/dbms/src/Analyzers/CMakeLists.txt b/dbms/src/Analyzers/CMakeLists.txt deleted file mode 100644 index b2aec21574d..00000000000 --- a/dbms/src/Analyzers/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -if(ENABLE_TESTS) - add_subdirectory(tests EXCLUDE_FROM_ALL) -endif() diff --git a/dbms/src/Analyzers/CollectAliases.cpp b/dbms/src/Analyzers/CollectAliases.cpp deleted file mode 100644 index 5f29595c72e..00000000000 --- a/dbms/src/Analyzers/CollectAliases.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int MULTIPLE_EXPRESSIONS_FOR_ALIAS; -} - - -static void processImpl(const ASTPtr & ast, CollectAliases::Aliases & aliases, CollectAliases::Kind kind, size_t keep_kind_for_depth) -{ - String alias = ast->tryGetAlias(); - if (!alias.empty()) - { - auto it_inserted = aliases.emplace(alias, CollectAliases::AliasInfo(ast, kind)); - - if (!it_inserted.second && ast->getTreeHash() != it_inserted.first->second.node->getTreeHash()) - { - std::stringstream message; - message << "Different expressions with the same alias " << backQuoteIfNeed(alias) << ":\n"; - formatAST(*it_inserted.first->second.node, message, false, true); - message << "\nand\n"; - formatAST(*ast, message, false, true); - message << "\n"; - - throw Exception(message.str(), ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS); - } - } - - for (auto & child : ast->children) - { - if (typeid_cast(child.get())) - { - /// Don't go into subqueries. - } - else if (typeid_cast(child.get())) - { - processImpl(child, aliases, CollectAliases::Kind::Table, 1); - } - else if (typeid_cast(child.get())) - { - /// ASTArrayJoin -> ASTExpressionList -> element of expression AS alias - processImpl(child, aliases, CollectAliases::Kind::ArrayJoin, 3); - } - else if (keep_kind_for_depth > 0) - { - processImpl(child, aliases, kind, keep_kind_for_depth - 1); - } - else - { - processImpl(child, aliases, CollectAliases::Kind::Expression, 0); - } - } -} - - -void CollectAliases::process(const ASTPtr & ast) -{ - processImpl(ast, aliases, Kind::Expression, 0); -} - - -void CollectAliases::dump(WriteBuffer & out) const -{ - /// For need of tests, we need to dump result in some fixed order. - std::vector vec; - vec.reserve(aliases.size()); - for (auto it = aliases.begin(); it != aliases.end(); ++it) - vec.emplace_back(it); - - std::sort(vec.begin(), vec.end(), [](const auto & a, const auto & b) { return a->first < b->first; }); - - for (const auto & it : vec) - { - writeProbablyBackQuotedString(it->first, out); - writeCString(" -> ", out); - - switch (it->second.kind) - { - case Kind::Expression: - writeCString("(expression) ", out); - break; - case Kind::Table: - writeCString("(table) ", out); - break; - case Kind::ArrayJoin: - writeCString("(array join) ", out); - break; - } - - std::stringstream formatted_ast; - formatAST(*it->second.node, formatted_ast, false, true); - writeString(formatted_ast.str(), out); - - writeChar('\n', out); - } -} - - -} diff --git a/dbms/src/Analyzers/CollectAliases.h b/dbms/src/Analyzers/CollectAliases.h deleted file mode 100644 index 46ce74909f8..00000000000 --- a/dbms/src/Analyzers/CollectAliases.h +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once - -#include -#include - - -namespace DB -{ - -class WriteBuffer; - - -/** Build a map: alias -> AST node. - * - * Also fill information about what kind each alias has: - * - expression alias; - * - table alias; - * - ARRAY JOIN alias. - * - * As extension to standard SQL, aliases could be specified and used in any part of query. - * Example: SELECT a, (1 AS a) + 1 AS b FROM t GROUP BY a, b - * Alias could be used in query before it is defined. - * - * Aliases could not be redefined. Example: 1 AS a, a + 1 AS a - is prohibited. - * - * Don't descend into subqueries (as aliases are local inside them). - */ -struct CollectAliases -{ - void process(const ASTPtr & ast); - - enum class Kind - { - Expression, /// Example: SELECT a AS b, f(x) AS y - Table, /// Example: SELECT t.* FROM (SELECT 1) AS t - ArrayJoin /// Example: SELECT t.x.a FROM t ARRAY JOIN arr AS x - }; - - struct AliasInfo - { - ASTPtr node; - Kind kind; - - AliasInfo(const ASTPtr & node, Kind kind) : node(node), kind(kind) {} - }; - - using Aliases = std::unordered_map; - Aliases aliases; - - /// Debug output - void dump(WriteBuffer & out) const; -}; - -} diff --git a/dbms/src/Analyzers/CollectTables.cpp b/dbms/src/Analyzers/CollectTables.cpp deleted file mode 100644 index d7dce3277c9..00000000000 --- a/dbms/src/Analyzers/CollectTables.cpp +++ /dev/null @@ -1,186 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int UNEXPECTED_AST_STRUCTURE; - extern const int LOGICAL_ERROR; -} - - -static CollectTables::TableInfo processOrdinaryTable(const ASTPtr & ast_database_and_table_name, const Context & context) -{ - const ASTIdentifier & identifier = static_cast(*ast_database_and_table_name); - - CollectTables::TableInfo res; - res.node = ast_database_and_table_name; - res.alias = identifier.tryGetAlias(); - - if (ast_database_and_table_name->children.empty()) - { - res.table_name = identifier.name; - } - else - { - if (ast_database_and_table_name->children.size() != 2) - throw Exception("Logical error: number of components in table expression not equal to two", ErrorCodes::LOGICAL_ERROR); - - res.database_name = static_cast(*identifier.children[0]).name; - res.table_name = static_cast(*identifier.children[1]).name; - } - - res.storage = context.getTable(res.database_name, res.table_name); - return res; -} - - -static CollectTables::TableInfo processTableFunction(const ASTPtr & ast_table_function, const ExecuteTableFunctions & table_functions) -{ - const ASTFunction & function = typeid_cast(*ast_table_function); - - IAST::Hash ast_hash = ast_table_function->getTreeHash(); - - auto it = table_functions.tables.find(ast_hash); - if (table_functions.tables.end() == it) - throw Exception("Table function " + function.name + " was not executed in advance.", ErrorCodes::LOGICAL_ERROR); - - CollectTables::TableInfo res; - res.node = ast_table_function; - res.alias = function.tryGetAlias(); - res.storage = it->second; - return res; -} - - -static CollectTables::TableInfo processNoTables(const Context & context) -{ - /// No FROM section. Interpret it as FROM system.one. - CollectTables::TableInfo res; - res.database_name = "system"; - res.table_name = "one"; - res.storage = context.getTable(res.database_name, res.table_name); - return res; -} - - -static CollectTables::TableInfo processSubquery(ASTPtr & ast_subquery, const Context & context, ExecuteTableFunctions & table_functions) -{ - AnalyzeResultOfQuery analyzer; - analyzer.process(typeid_cast(*ast_subquery).children.at(0), context, table_functions); - - CollectTables::TableInfo res; - res.node = ast_subquery; - res.alias = ast_subquery->tryGetAlias(); - res.structure_of_subquery = analyzer.result; - return res; -} - - -void CollectTables::process(ASTPtr & ast, const Context & context, const CollectAliases & /*aliases*/, ExecuteTableFunctions & table_functions) -{ - const ASTSelectQuery * select = typeid_cast(ast.get()); - if (!select) - throw Exception("CollectTables::process was called for not a SELECT query", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - if (!select->tables) - { - tables.emplace_back(processNoTables(context)); - return; - } - - for (auto & child : select->tables->children) - { - ASTTablesInSelectQueryElement & element = static_cast(*child); - - if (!element.table_expression) /// This is ARRAY JOIN - continue; - - ASTTableExpression & table_expression = static_cast(*element.table_expression); - - if (table_expression.database_and_table_name) - { - tables.emplace_back(processOrdinaryTable(table_expression.database_and_table_name, context)); - - /// TODO It could be alias to another table expression. - } - else if (table_expression.table_function) - { - tables.emplace_back(processTableFunction(table_expression.table_function, table_functions)); - } - else if (table_expression.subquery) - { - tables.emplace_back(processSubquery(table_expression.subquery, context, table_functions)); - } - else - throw Exception("Logical error: no known elements in ASTTableExpression", ErrorCodes::LOGICAL_ERROR); - } - - /// TODO Control that tables don't have conflicting names. -} - - -void CollectTables::dump(WriteBuffer & out) const -{ - for (const auto & table : tables) - { - writeCString("Database name: ", out); - if (table.database_name.empty()) - writeCString("(none)", out); - else - writeProbablyBackQuotedString(table.database_name, out); - - writeCString(". Table name: ", out); - if (table.table_name.empty()) - writeCString("(none)", out); - else - writeProbablyBackQuotedString(table.table_name, out); - - writeCString(". Alias: ", out); - if (table.alias.empty()) - writeCString("(none)", out); - else - writeProbablyBackQuotedString(table.alias, out); - - writeCString(". Storage: ", out); - if (!table.storage) - writeCString("(none)", out); - else - writeProbablyBackQuotedString(table.storage->getName(), out); - - writeCString(". Structure of subquery: ", out); - if (!table.structure_of_subquery) - writeCString("(none)", out); - else - writeString(table.structure_of_subquery.dumpStructure(), out); - - writeCString(". AST: ", out); - if (!table.node) - writeCString("(none)", out); - else - { - std::stringstream formatted_ast; - formatAST(*table.node, formatted_ast, false, true); - writeString(formatted_ast.str(), out); - } - - writeChar('\n', out); - } -} - -} diff --git a/dbms/src/Analyzers/CollectTables.h b/dbms/src/Analyzers/CollectTables.h deleted file mode 100644 index dc31e296fe2..00000000000 --- a/dbms/src/Analyzers/CollectTables.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ - -class Context; -struct CollectAliases; -struct ExecuteTableFunctions; -class WriteBuffer; - - -/** Collect and analyze table-like expressions in section FROM in a query. - * For every expression, keep its alias. - * - * For ordinary tables, determine database and table name, obtain and keep StoragePtr. - * For subqueries, determine result structure. This requires analysis of subquery, such as type inference. - * For table functions, grab them from prepared ExecuteTableFunctions object. - */ -struct CollectTables -{ - void process(ASTPtr & ast, const Context & context, const CollectAliases & aliases, ExecuteTableFunctions & table_functions); - - enum class Kind - { - OrdinaryTable, - TableFunction, - Subquery - }; - - struct TableInfo - { - ASTPtr node; - String database_name; - String table_name; - String alias; - StoragePtr storage; - Block structure_of_subquery; - }; - - using Tables = std::vector; - Tables tables; - - /// Debug output - void dump(WriteBuffer & out) const; -}; - -} diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.cpp b/dbms/src/Analyzers/ExecuteTableFunctions.cpp deleted file mode 100644 index d79fa1481f0..00000000000 --- a/dbms/src/Analyzers/ExecuteTableFunctions.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int UNEXPECTED_AST_STRUCTURE; - extern const int LOGICAL_ERROR; -} - -/// Allows to execute exactly same table functions only once. -using ASTTreeToTable = std::map; - - -static void processTableFunction(const ASTPtr & ast_table_function, const Context & context, ExecuteTableFunctions::Tables & result_map) -{ - const ASTFunction & function = typeid_cast(*ast_table_function); - - /// If already executed. - IAST::Hash ast_hash = ast_table_function->getTreeHash(); - if (result_map.count(ast_hash)) - return; - - /// Obtain table function - TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(function.name, context); - /// Execute it and store result - StoragePtr table = table_function_ptr->execute(ast_table_function, context); - result_map[ast_hash] = table; -} - - -void ExecuteTableFunctions::process(ASTPtr & ast, const Context & context) -{ - const ASTSelectQuery * select = typeid_cast(ast.get()); - if (!select) - throw Exception("ExecuteTableFunctions::process was called for not a SELECT query", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - if (!select->tables) - return; - - for (auto & child : select->tables->children) - { - ASTTablesInSelectQueryElement & element = static_cast(*child); - - if (!element.table_expression) /// This is ARRAY JOIN - continue; - - ASTTableExpression & table_expression = static_cast(*element.table_expression); - - if (!table_expression.table_function) - continue; - - processTableFunction(table_expression.table_function, context, tables); - } -} - - -void ExecuteTableFunctions::dump(WriteBuffer & out) const -{ - for (const auto & table : tables) - { - writeString(table.second->getName(), out); - writeCString("\n\n", out); - writeString(table.second->getColumns().getAllPhysical().toString(), out); - writeCString("\n", out); - } -} - -} diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.h b/dbms/src/Analyzers/ExecuteTableFunctions.h deleted file mode 100644 index d5d70d6cacd..00000000000 --- a/dbms/src/Analyzers/ExecuteTableFunctions.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ - -class Context; -class WriteBuffer; - - -/** For every table function, found in first level of the query - * (don't go into subqueries) - * execute it and save corresponding StoragePtr. - * - * Execution of table functions must be done in a stage of query analysis, - * because otherwise we don't know table structure. So, it is assumed as cheap operation. - * - * Multiple occurences of table functions with same arguments will be executed only once. - */ -struct ExecuteTableFunctions -{ - void process(ASTPtr & ast, const Context & context); - - using Tables = std::map; - Tables tables; - - /// Debug output - void dump(WriteBuffer & out) const; -}; - -} diff --git a/dbms/src/Analyzers/OptimizeGroupOrderLimitBy.cpp b/dbms/src/Analyzers/OptimizeGroupOrderLimitBy.cpp deleted file mode 100644 index 5b34c6cede7..00000000000 --- a/dbms/src/Analyzers/OptimizeGroupOrderLimitBy.cpp +++ /dev/null @@ -1,264 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int UNEXPECTED_AST_STRUCTURE; -} - - -static bool isInjectiveFunction( - const ASTFunction * ast_function, - const TypeAndConstantInference::ExpressionInfo & function_info, - const TypeAndConstantInference::Info & all_info) -{ - if (!function_info.function) - return false; - - Block block_with_constants; - - const ASTs & children = ast_function->arguments->children; - for (const auto & child : children) - { - String child_name = child->getColumnName(); - const TypeAndConstantInference::ExpressionInfo & child_info = all_info.at(child_name); - - block_with_constants.insert(ColumnWithTypeAndName( - child_info.is_constant_expression ? child_info.data_type->createColumnConst(1, child_info.value) : nullptr, - child_info.data_type, - child_name)); - } - - return function_info.function->isInjective(block_with_constants); -} - - -static bool isDeterministicFunctionOfKeys( - const ASTFunction * ast_function, - const TypeAndConstantInference::ExpressionInfo & function_info, - const TypeAndConstantInference::Info & all_info, - const std::vector & keys) -{ - if (!function_info.function || !function_info.function->isDeterministicInScopeOfQuery()) - return false; - - for (const auto & child : ast_function->arguments->children) - { - String child_name = child->getColumnName(); - const TypeAndConstantInference::ExpressionInfo & child_info = all_info.at(child_name); - - /// Function argument is constant. - if (child_info.is_constant_expression) - continue; - - /// Function argument is one of keys. - if (keys.end() != std::find(keys.begin(), keys.end(), child_name)) - continue; - - /// Function argument is a function, that deterministically depend on keys. - if (const ASTFunction * child_function = typeid_cast(child.get())) - { - if (isDeterministicFunctionOfKeys(child_function, child_info, all_info, keys)) - continue; - } - - return false; - } - - return true; -} - - -static void processGroupByLikeList(ASTPtr & ast, TypeAndConstantInference & expression_info) -{ - if (!ast) - return; - - ASTs & elems = ast->children; - - std::unordered_set unique_keys; - size_t i = 0; - - auto restart = [&] - { - i = 0; - unique_keys.clear(); - }; - - /// Always leave last element in GROUP BY, even if it is constant. - while (i < elems.size() && elems.size() > 1) - { - ASTPtr & elem = elems[i]; - - String column_name = elem->getColumnName(); /// TODO canonicalization of names - auto it = expression_info.info.find(column_name); - if (it == expression_info.info.end()) - throw Exception("Type inference was not done for " + column_name, ErrorCodes::LOGICAL_ERROR); - const TypeAndConstantInference::ExpressionInfo & info = it->second; - - /// Removing constant expressions. - /// Removing duplicate keys. - if (info.is_constant_expression - || !unique_keys.emplace(column_name).second) - { - elems.erase(elems.begin() + i); - continue; - } - - if (info.function && !elem->children.empty()) - { - const ASTFunction * ast_function = typeid_cast(elem.get()); - if (!ast_function) - throw Exception("Column is marked as function during type inference, but corresponding AST node " - + column_name + " is not a function", ErrorCodes::LOGICAL_ERROR); - - /// Unwrap injective functions. - if (isInjectiveFunction(ast_function, info, expression_info.info)) - { - auto args = ast_function->arguments; - elems.erase(elems.begin() + i); - elems.insert(elems.begin() + i, args->children.begin(), args->children.end()); - - restart(); /// Previous keys may become deterministic function of newly added keys. - continue; - } - - /// Remove deterministic functions of another keys. - std::vector other_keys; - other_keys.reserve(elems.size() - 1); - for (size_t j = 0, size = elems.size(); j < size; ++j) - if (j != i) - other_keys.emplace_back(elems[j]->getColumnName()); - - if (isDeterministicFunctionOfKeys(ast_function, info, expression_info.info, other_keys)) - { - elems.erase(elems.begin() + i); - continue; - } - } - - ++i; - } -} - - -static void processOrderByList(ASTPtr & ast, TypeAndConstantInference & expression_info) -{ - if (!ast) - return; - - ASTs & elems = ast->children; - - /// sort column name and collation - std::set> unique_keys; - size_t i = 0; - while (i < elems.size()) - { - const ASTOrderByElement * order_by_elem = typeid_cast(elems[i].get()); - if (!order_by_elem) - throw Exception("Child of ORDER BY clause is not an ASTOrderByElement", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - /// It has ASC|DESC and COLLATE inplace, and expression as its only child. - if (order_by_elem->children.empty()) - throw Exception("ORDER BY element has no children", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - const ASTPtr & elem = order_by_elem->children[0]; - String collation; - if (order_by_elem->collation) - { - const ASTLiteral * lit = typeid_cast(order_by_elem->collation.get()); - if (!lit) - throw Exception("Collation in ORDER BY clause is not an ASTLiteral", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - if (lit->value.getType() != Field::Types::String) - throw Exception("Collation in ORDER BY clause is not a string literal", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - collation = lit->value.get(); - } - - String column_name = elem->getColumnName(); - auto it = expression_info.info.find(column_name); - if (it == expression_info.info.end()) - throw Exception("Type inference was not done for " + column_name, ErrorCodes::LOGICAL_ERROR); - const TypeAndConstantInference::ExpressionInfo & info = it->second; - - /// Removing constant expressions. - /// Removing duplicate keys. - if (info.is_constant_expression - /// Having same element but with empty collation. Empty collation is considered more "granular" than any special collation. - || unique_keys.count(std::make_pair(column_name, String())) - /// Having same element with same collation. - || !unique_keys.emplace(column_name, collation).second) - { - elems.erase(elems.begin() + i); - continue; - } - - if (i > 0 && collation.empty() && info.function && !elem->children.empty()) - { - const ASTFunction * ast_function = typeid_cast(elem.get()); - if (!ast_function) - throw Exception("Column is marked as function during type inference, but corresponding AST node " - + column_name + " is not a function", ErrorCodes::LOGICAL_ERROR); - - /// Remove deterministic functions of previous keys. Only consider keys without collation. - std::vector prev_keys; - prev_keys.reserve(i); - for (size_t j = 0; j < i; ++j) - if (!typeid_cast(*elems[j]).collation) - prev_keys.emplace_back(elems[j]->children.at(0)->getColumnName()); - - if (isDeterministicFunctionOfKeys(ast_function, info, expression_info.info, prev_keys)) - { - elems.erase(elems.begin() + i); - continue; - } - } - - ++i; - } -} - - -void OptimizeGroupOrderLimitBy::process(ASTPtr & ast, TypeAndConstantInference & expression_info) -{ - ASTSelectQuery * select = typeid_cast(ast.get()); - if (!select) - throw Exception("AnalyzeResultOfQuery::process was called for not a SELECT query", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - if (!select->select_expression_list) - throw Exception("SELECT query doesn't have select_expression_list", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - processGroupByLikeList(select->group_expression_list, expression_info); - processGroupByLikeList(select->limit_by_expression_list, expression_info); - - if (select->order_expression_list) - { - processOrderByList(select->order_expression_list, expression_info); - - /// ORDER BY could be completely eliminated - if (select->order_expression_list->children.empty()) - { - select->children.erase(std::remove( - select->children.begin(), select->children.end(), select->order_expression_list), select->children.end()); - select->order_expression_list.reset(); - } - } -} - - -} diff --git a/dbms/src/Analyzers/OptimizeGroupOrderLimitBy.h b/dbms/src/Analyzers/OptimizeGroupOrderLimitBy.h deleted file mode 100644 index 44d83193df4..00000000000 --- a/dbms/src/Analyzers/OptimizeGroupOrderLimitBy.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ - -class Context; -class WriteBuffer; -struct TypeAndConstantInference; - - -/** Transform GROUP BY, ORDER BY and LIMIT BY sections. - * (LIMIT BY is an extension to SQL language, do not be confused with ordinary LIMIT) - * - * Remove constant expressions (like ORDER BY concat('hello', 'world')). - * For GROUP BY, unwrap injective functions (like GROUP BY toString(x) -> GROUP BY x). - * For GROUP BY, remove deterministic functions of another keys (like GROUP BY x + 1, x -> GROUP BY x). - * TODO For ORDER BY, remove deterministic functions of previous keys (like ORDER BY num, toString(num) -> ORDER BY num), - * but only if no collation has specified. - * As a special case, remove duplicate keys. - * For LIMIT BY, apply all the same as for GROUP BY. - * - * TODO We should apply something similar for DISTINCT, - * but keys for DISTINCT are specified implicitly (as whole SELECT expression list). - * - * This should be run after CollectAliases, because some aliases will be lost from AST during this transformation. - * This should be run after TranslatePositionalArguments for positional arguments like ORDER BY 1, 2 not to be confused with constants. - */ -struct OptimizeGroupOrderLimitBy -{ - void process(ASTPtr & ast, TypeAndConstantInference & expression_info); -}; - -} diff --git a/dbms/src/Analyzers/TranslatePositionalArguments.cpp b/dbms/src/Analyzers/TranslatePositionalArguments.cpp deleted file mode 100644 index c4a0b8814ec..00000000000 --- a/dbms/src/Analyzers/TranslatePositionalArguments.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int UNEXPECTED_AST_STRUCTURE; - extern const int INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE; -} - - -static void processElement(ASTPtr & element, const ASTPtr & select_expression_list, const char * description) -{ - if (ASTLiteral * literal = typeid_cast(element.get())) - { - if (literal->value.getType() == Field::Types::UInt64) - { - UInt64 position = literal->value.get(); - - if (!literal->alias.empty()) - throw Exception("Unsigned numeric literal " + toString(position) + " in " + toString(description) - + " section is interpreted as positional argument, " - "but it has alias " + backQuoteIfNeed(literal->alias) + " that is not expected", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - if (position == 0) - throw Exception("Unsigned numeric literal 0 in " + toString(description) + " section is interpreted as positional argument, " - "but positional arguments are 1-based", ErrorCodes::INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE); - - if (position > select_expression_list->children.size()) - throw Exception("Unsigned numeric literal " + toString(position) + " in " + String(description) - + " section is interpreted as positional argument, " - "but it is greater than number of expressions in SELECT section (" - + toString(select_expression_list->children.size()) + ")", ErrorCodes::INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE); - - element = select_expression_list->children[position - 1]->clone(); - } - } -} - - -static void processClause(ASTPtr & ast, const ASTPtr & select_expression_list, const char * description, bool is_order_by) -{ - if (!ast) - return; - - for (auto & child : ast->children) - { - if (is_order_by) - { - if (!typeid_cast(child.get())) - throw Exception("Child of ORDER BY clause is not an ASTOrderByElement", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - /// It has ASC|DESC and COLLATE inplace, and expression as its only child. - if (child->children.empty()) - throw Exception("ORDER BY element has no children", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - processElement(child->children[0], select_expression_list, description); - } - else - processElement(child, select_expression_list, description); - } -} - - -void TranslatePositionalArguments::process(ASTPtr & ast) -{ - ASTSelectQuery * select = typeid_cast(ast.get()); - if (!select) - throw Exception("TranslatePositionalArguments::process was called for not a SELECT query", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - if (!select->select_expression_list) - throw Exception("SELECT query doesn't have select_expression_list", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - processClause(select->group_expression_list, select->select_expression_list, "GROUP BY", false); - processClause(select->order_expression_list, select->select_expression_list, "ORDER BY", true); - processClause(select->limit_by_expression_list, select->select_expression_list, "LIMIT BY", false); -} - -} diff --git a/dbms/src/Analyzers/TranslatePositionalArguments.h b/dbms/src/Analyzers/TranslatePositionalArguments.h deleted file mode 100644 index 6209685eaeb..00000000000 --- a/dbms/src/Analyzers/TranslatePositionalArguments.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ - -class Context; -class WriteBuffer; - - -/** Transform GROUP BY, ORDER BY and LIMIT BY sections. - * Replace positional arguments (like ORDER BY 1, 2) to corresponding columns. - */ -struct TranslatePositionalArguments -{ - void process(ASTPtr & ast); -}; - -} diff --git a/dbms/src/Analyzers/TypeAndConstantInference.cpp b/dbms/src/Analyzers/TypeAndConstantInference.cpp deleted file mode 100644 index bc62e0b2c29..00000000000 --- a/dbms/src/Analyzers/TypeAndConstantInference.cpp +++ /dev/null @@ -1,529 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int FUNCTION_CANNOT_HAVE_PARAMETERS; - extern const int UNEXPECTED_AST_STRUCTURE; - extern const int BAD_LAMBDA; -} - - -namespace -{ - -Field getValueFromConstantColumn(const ColumnPtr & column) -{ - if (!column->isColumnConst()) - throw Exception("Logical error: expected that column is constant", ErrorCodes::LOGICAL_ERROR); - if (column->size() != 1) - throw Exception("Logical error: expected that column with constant has single element", ErrorCodes::LOGICAL_ERROR); - return (*column)[0]; -} - - -/// Description of single parameter of lambda expression: name -> type. -/// Note, that after AnalyzeLambda step, names of lambda parameters are unique in single select query. -using LambdaParameters = std::unordered_map; - - -void processImpl( - ASTPtr & ast, const Context & context, - CollectAliases & aliases, const AnalyzeColumns & columns, - TypeAndConstantInference::Info & info, - const AnalyzeLambdas & lambdas, - ExecuteTableFunctions & table_functions); - - -void processLiteral(const String & column_name, const ASTPtr & ast, TypeAndConstantInference::Info & info) -{ - const ASTLiteral * literal = static_cast(ast.get()); - - TypeAndConstantInference::ExpressionInfo expression_info; - expression_info.node = ast; - expression_info.is_constant_expression = true; - expression_info.data_type = applyVisitor(FieldToDataType(), literal->value); - expression_info.value = convertFieldToType(literal->value, *expression_info.data_type); - info.emplace(column_name, std::move(expression_info)); -} - - -void processIdentifier(const String & column_name, const ASTPtr & ast, TypeAndConstantInference::Info & info, - const Context & context, CollectAliases & aliases, const AnalyzeColumns & columns, - const AnalyzeLambdas & lambdas, ExecuteTableFunctions & table_functions) -{ - /// Column from table - auto it = columns.columns.find(column_name); - if (it != columns.columns.end()) - { - TypeAndConstantInference::ExpressionInfo expression_info; - expression_info.node = ast; - expression_info.data_type = it->second.data_type; - - /// If it comes from subquery and we know, that it is constant expression. - const Block & structure_of_subquery = it->second.table.structure_of_subquery; - if (structure_of_subquery) - { - const ColumnWithTypeAndName & column_from_subquery = structure_of_subquery.getByName(it->second.name_in_table); - if (column_from_subquery.column) - { - expression_info.is_constant_expression = true; - expression_info.value = getValueFromConstantColumn(column_from_subquery.column); - } - } - - info.emplace(column_name, std::move(expression_info)); - } - else - { - /// Alias - auto it = aliases.aliases.find(column_name); - if (it != aliases.aliases.end()) - { - /// TODO Cyclic aliases. - - if (it->second.kind != CollectAliases::Kind::Expression) - throw Exception("Logical error: unexpected kind of alias", ErrorCodes::LOGICAL_ERROR); - - processImpl(it->second.node, context, aliases, columns, info, lambdas, table_functions); - info[column_name] = info[it->second.node->getColumnName()]; - } - } -} - - -void processFunction(const String & column_name, ASTPtr & ast, TypeAndConstantInference::Info & info, - const Context & context) -{ - ASTFunction * function = static_cast(ast.get()); - - /// Special case for lambda functions. Lambda function has special return type "Function". - /// We first create info with Function of unspecified arguments, and will specify them later. - if (function->name == "lambda") - { - size_t number_of_lambda_parameters = AnalyzeLambdas::extractLambdaParameters(function->arguments->children.at(0)).size(); - - TypeAndConstantInference::ExpressionInfo expression_info; - expression_info.node = ast; - expression_info.data_type = std::make_unique(DataTypes(number_of_lambda_parameters)); - info.emplace(column_name, std::move(expression_info)); - return; - } - - DataTypes argument_types; - ColumnsWithTypeAndName argument_columns; - - if (function->arguments) - { - for (const auto & child : function->arguments->children) - { - auto it = info.find(child->getColumnName()); - if (it == info.end()) - throw Exception("Logical error: type of function argument was not inferred during depth-first search", ErrorCodes::LOGICAL_ERROR); - - argument_types.emplace_back(it->second.data_type); - argument_columns.emplace_back(ColumnWithTypeAndName(nullptr, it->second.data_type, "")); - if (it->second.is_constant_expression) - argument_columns.back().column = it->second.data_type->createColumnConst(1, it->second.value); - } - } - - /// Special cases for COUNT(DISTINCT ...) function. - bool column_name_changed = false; - String func_name_lowercase = Poco::toLower(function->name); - if (func_name_lowercase == "countdistinct") /// It comes in that form from parser. - { - /// Select implementation of countDistinct based on settings. - /// Important that it is done as query rewrite. It means rewritten query - /// will be sent to remote servers during distributed query execution, - /// and on all remote servers, function implementation will be same. - function->name = context.getSettingsRef().count_distinct_implementation; - column_name_changed = true; - } - - /// Aggregate function. - Array parameters = (function->parameters) ? getAggregateFunctionParametersArray(function->parameters) : Array(); - if (AggregateFunctionPtr aggregate_function_ptr = AggregateFunctionFactory::instance().tryGet(function->name, argument_types, parameters)) - { - /// Note that aggregate function could never be constant expression. - - /// (?) Replace function name to canonical one. Because same function could be referenced by different names. - // function->name = aggregate_function_ptr->getName(); - - TypeAndConstantInference::ExpressionInfo expression_info; - expression_info.node = ast; - expression_info.data_type = aggregate_function_ptr->getReturnType(); - expression_info.aggregate_function = aggregate_function_ptr; - info.emplace(column_name_changed ? ast->getColumnName() : column_name, std::move(expression_info)); - return; - } - - /// Ordinary function. - if (function->parameters) - throw Exception("The only parametric functions (functions with two separate parenthesis pairs) are aggregate functions" - ", and '" + function->name + "' is not an aggregate function.", ErrorCodes::FUNCTION_CANNOT_HAVE_PARAMETERS); - - /// IN operator. This is special case, because subqueries in right hand side are not scalar subqueries. - if (function->name == "in" - || function->name == "notIn" - || function->name == "globalIn" - || function->name == "globalNotIn") - { - /// For simplicity reasons, do not consider this as constant expression. We may change it in future. - TypeAndConstantInference::ExpressionInfo expression_info; - expression_info.node = ast; - expression_info.data_type = std::make_shared(); - info.emplace(column_name, std::move(expression_info)); - return; - } - - const auto & function_builder_ptr = FunctionFactory::instance().get(function->name, context); - - /// (?) Replace function name to canonical one. Because same function could be referenced by different names. - // function->name = function_ptr->getName(); - - ColumnsWithTypeAndName columns_for_analysis; - columns_for_analysis.reserve(argument_types.size()); - - bool all_consts = true; - if (function->arguments) - { - for (const auto & child : function->arguments->children) - { - String child_name = child->getColumnName(); - const TypeAndConstantInference::ExpressionInfo & child_info = info.at(child_name); - columns_for_analysis.emplace_back( - child_info.is_constant_expression ? child_info.data_type->createColumnConst(1, child_info.value) : nullptr, - child_info.data_type, - child_name); - - if (!child_info.is_constant_expression) - all_consts = false; - } - } - - auto function_ptr = function_builder_ptr->build(argument_columns); - - TypeAndConstantInference::ExpressionInfo expression_info; - expression_info.node = ast; - expression_info.function = function_ptr; - expression_info.data_type = function_ptr->getReturnType(); - - if (all_consts && function_ptr->isSuitableForConstantFolding()) - { - Block block_with_constants(columns_for_analysis); - - ColumnNumbers argument_numbers(columns_for_analysis.size()); - for (size_t i = 0, size = argument_numbers.size(); i < size; ++i) - argument_numbers[i] = i; - - size_t result_position = argument_numbers.size(); - block_with_constants.insert({nullptr, expression_info.data_type, column_name}); - - function_ptr->execute(block_with_constants, argument_numbers, result_position); - - const auto & result_column = block_with_constants.getByPosition(result_position).column; - if (result_column->isColumnConst()) - { - expression_info.is_constant_expression = true; - expression_info.value = (*result_column)[0]; - } - } - - info.emplace(column_name, std::move(expression_info)); -} - - -void processScalarSubquery(const String & column_name, ASTPtr & ast, TypeAndConstantInference::Info & info, - const Context & context, ExecuteTableFunctions & table_functions) -{ - ASTSubquery * subquery = static_cast(ast.get()); - - AnalyzeResultOfQuery analyzer; - analyzer.process(subquery->children.at(0), context, table_functions); - - if (!analyzer.result) - throw Exception("Logical error: no columns returned from scalar subquery", ErrorCodes::LOGICAL_ERROR); - - TypeAndConstantInference::ExpressionInfo expression_info; - expression_info.node = ast; - - if (analyzer.result.columns() == 1) - { - const auto & elem = analyzer.result.getByPosition(0); - expression_info.data_type = elem.type; - - if (elem.column) - { - expression_info.is_constant_expression = true; - expression_info.value = getValueFromConstantColumn(elem.column); - } - } - else - { - /// Result of scalar subquery is interpreted as tuple. - size_t size = analyzer.result.columns(); - DataTypes types; - types.reserve(size); - bool all_consts = true; - for (size_t i = 0; i < size; ++i) - { - const auto & elem = analyzer.result.getByPosition(i); - types.emplace_back(elem.type); - if (!elem.column) - all_consts = false; - } - - expression_info.data_type = std::make_shared(types); - - if (all_consts) - { - TupleBackend value(size); - - for (size_t i = 0; i < size; ++i) - value[i] = getValueFromConstantColumn(analyzer.result.getByPosition(i).column); - - expression_info.is_constant_expression = true; - expression_info.value = Tuple(std::move(value)); - } - } - - info.emplace(column_name, std::move(expression_info)); -} - - -void processHigherOrderFunction( - ASTPtr & ast, const Context & context, - CollectAliases & aliases, const AnalyzeColumns & columns, - TypeAndConstantInference::Info & info, - const AnalyzeLambdas & lambdas, - ExecuteTableFunctions & table_functions) -{ - ASTFunction * function = static_cast(ast.get()); - - const auto & function_builder_ptr = FunctionFactory::instance().get(function->name, context); - - if (!function->arguments) - throw Exception("Unexpected AST for higher-order function", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - DataTypes types; - types.reserve(function->arguments->children.size()); - for (const auto & child : function->arguments->children) - { - String child_name = child->getColumnName(); - const TypeAndConstantInference::ExpressionInfo & child_info = info.at(child_name); - types.emplace_back(child_info.data_type); - } - - function_builder_ptr->getLambdaArgumentTypes(types); - - /// For every lambda expression, dive into it. - - if (types.size() != function->arguments->children.size()) - throw Exception("Logical error: size of types was changed after call to IFunction::getLambdaArgumentTypes", - ErrorCodes::LOGICAL_ERROR); - - for (size_t i = 0, size = function->arguments->children.size(); i < size; ++i) - { - const auto & child = function->arguments->children[i]; - const ASTFunction * lambda = typeid_cast(child.get()); - if (lambda && lambda->name == "lambda") - { - const auto * lambda_type = typeid_cast(types[i].get()); - - if (!lambda_type) - throw Exception("Logical error: IFunction::getLambdaArgumentTypes returned data type for lambda expression," - " that is not DataTypeFunction", ErrorCodes::LOGICAL_ERROR); - - if (!lambda->arguments || lambda->arguments->children.size() != 2) - throw Exception("Lambda function must have exactly two arguments (sides of arrow)", ErrorCodes::BAD_LAMBDA); - - /// Set types for every lambda parameter. - - AnalyzeLambdas::LambdaParameters parameters = AnalyzeLambdas::extractLambdaParameters(lambda->arguments->children[0]); - - const DataTypes & lambda_argument_types = lambda_type->getArgumentTypes(); - - if (parameters.size() != lambda_argument_types.size()) - throw Exception("DataTypeExpression for lambda function has different number of argument types than number of lambda parameters", - ErrorCodes::LOGICAL_ERROR); - - for (size_t param_idx = 0, num_params = parameters.size(); param_idx < num_params; ++param_idx) - { - TypeAndConstantInference::ExpressionInfo expression_info; - expression_info.node = typeid_cast(lambda->arguments->children[0].get()) - ? lambda->arguments->children[0] - : typeid_cast(*lambda->arguments->children[0]).arguments->children.at(param_idx); - - expression_info.data_type = lambda_argument_types[param_idx]; - - info.emplace(parameters[param_idx], std::move(expression_info)); - } - - /// Now dive into. - - processImpl(lambda->arguments->children[1], context, aliases, columns, info, lambdas, table_functions); - - /// Update Expression type (expression signature). - - info.at(lambda->getColumnName()).data_type = std::make_shared( - lambda_argument_types, info.at(lambda->arguments->children[1]->getColumnName()).data_type); - } - } -} - - -void processImpl( - ASTPtr & ast, const Context & context, - CollectAliases & aliases, const AnalyzeColumns & columns, - TypeAndConstantInference::Info & info, - const AnalyzeLambdas & lambdas, - ExecuteTableFunctions & table_functions) -{ - const ASTFunction * function = typeid_cast(ast.get()); - - /// Bottom-up. - - /// Don't go into components of compound identifiers. - if (!typeid_cast(ast.get())) - { - for (auto & child : ast->children) - { - /** Don't go into subqueries and table-like expressions. - * Also don't go into components of compound identifiers. - */ - if (typeid_cast(child.get()) - || typeid_cast(child.get())) - continue; - - /** Postpone diving into lambda expressions. - * We must first infer types of other arguments of higher-order function, - * and then process lambda expression. - * Example: - * arrayMap((x, y) -> x + y, arr1, arr2) - * First, infer type of 'arr1' and 'arr2'. - * Then, ask function arrayMap, what types will have 'x' and 'y'. - * And then, infer type of 'x + y'. - */ - if (function && function->name == "lambda") - continue; - - processImpl(child, context, aliases, columns, info, lambdas, table_functions); - } - } - - const ASTLiteral * literal = nullptr; - const ASTIdentifier * identifier = nullptr; - const ASTSubquery * subquery = nullptr; - - function - || (literal = typeid_cast(ast.get())) - || (identifier = typeid_cast(ast.get())) - || (subquery = typeid_cast(ast.get())); - - if (!literal && !identifier && !function && !subquery) - return; - - /// Same expression is already processed. - String column_name = ast->getColumnName(); - if (info.count(column_name)) - return; - - if (function) - { - /// If this is higher-order function, determine types of lambda arguments and infer types of subexpressions inside lambdas. - if (lambdas.higher_order_functions.end() != std::find(lambdas.higher_order_functions.begin(), lambdas.higher_order_functions.end(), ast)) - processHigherOrderFunction(ast, context, aliases, columns, info, lambdas, table_functions); - - processFunction(column_name, ast, info, context); - } - else if (literal) - processLiteral(column_name, ast, info); - else if (identifier) - processIdentifier(column_name, ast, info, context, aliases, columns, lambdas, table_functions); - else if (subquery) - processScalarSubquery(column_name, ast, info, context, table_functions); -} - -} - - -void TypeAndConstantInference::process(ASTPtr & ast, const Context & context, - CollectAliases & aliases, - const AnalyzeColumns & columns, - const AnalyzeLambdas & lambdas, - ExecuteTableFunctions & table_functions) -{ - processImpl(ast, context, aliases, columns, info, lambdas, table_functions); -} - - -void TypeAndConstantInference::dump(WriteBuffer & out) const -{ - /// For need of tests, we need to dump result in some fixed order. - std::vector vec; - vec.reserve(info.size()); - for (auto it = info.begin(); it != info.end(); ++it) - vec.emplace_back(it); - - std::sort(vec.begin(), vec.end(), [](const auto & a, const auto & b) { return a->first < b->first; }); - - for (const auto & it : vec) - { - writeString(it->first, out); - writeCString(" -> ", out); - writeString(it->second.data_type->getName(), out); - - if (it->second.is_constant_expression) - { - writeCString(" = ", out); - String value = applyVisitor(FieldVisitorToString(), it->second.value); - writeString(value, out); - } - - writeCString(". AST: ", out); - if (!it->second.node) - writeCString("(none)", out); - else - { - std::stringstream formatted_ast; - formatAST(*it->second.node, formatted_ast, false, true); - writeString(formatted_ast.str(), out); - } - - writeChar('\n', out); - } -} - -} diff --git a/dbms/src/Analyzers/TypeAndConstantInference.h b/dbms/src/Analyzers/TypeAndConstantInference.h deleted file mode 100644 index f41e4125d00..00000000000 --- a/dbms/src/Analyzers/TypeAndConstantInference.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace DB -{ - -class Context; -class WriteBuffer; -struct CollectAliases; -struct AnalyzeColumns; -struct AnalyzeLambdas; -struct ExecuteTableFunctions; -class IFunctionBase; -class IAggregateFunction; - - -/** For every expression, deduce its type, - * and if it is a constant expression, calculate its value. - * - * Types and constants inference goes together, - * because sometimes resulting type of a function depend on value of constant expression. - * Notable examples: tupleElement(tuple, N) and toFixedString(s, N) functions. - * - * Also creates and stores function objects. - * TODO (?) Also calculate ids for expressions, that will identify common subexpressions. - */ -struct TypeAndConstantInference -{ - void process(ASTPtr & ast, const Context & context, - CollectAliases & aliases, - const AnalyzeColumns & columns, - const AnalyzeLambdas & analyze_lambdas, - ExecuteTableFunctions & table_functions); - - struct ExpressionInfo - { - /// Must identify identical expressions. - /// For example following three expressions in query are the same: SELECT sum(x) AS a, SUM(t.x) AS b, a FROM t - UInt128 id {}; - ASTPtr node; - DataTypePtr data_type; - bool is_constant_expression = false; - Field value; /// Has meaning if is_constant_expression == true. - std::shared_ptr function; - std::shared_ptr aggregate_function; - }; - - /// Key is getColumnName of AST node. - using Info = std::unordered_map; - Info info; - - /// Debug output - void dump(WriteBuffer & out) const; -}; - -} diff --git a/dbms/src/Analyzers/tests/CMakeLists.txt b/dbms/src/Analyzers/tests/CMakeLists.txt deleted file mode 100644 index a4f331dbd3a..00000000000 --- a/dbms/src/Analyzers/tests/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -add_executable(collect_aliases collect_aliases.cpp) -target_link_libraries(collect_aliases dbms) - -add_executable(collect_tables collect_tables.cpp) -target_link_libraries(collect_tables dbms clickhouse_storages_system) - -add_executable(analyze_columns analyze_columns.cpp) -target_link_libraries(analyze_columns dbms clickhouse_storages_system) - -add_executable(type_and_constant_inference type_and_constant_inference.cpp) -target_link_libraries(type_and_constant_inference - clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions) - -add_executable(analyze_result_of_query analyze_result_of_query.cpp) -target_link_libraries(analyze_result_of_query dbms clickhouse_storages_system) - -add_executable(translate_positional_arguments translate_positional_arguments.cpp) -target_link_libraries(translate_positional_arguments dbms) - -add_executable(optimize_group_order_limit_by optimize_group_order_limit_by.cpp) -target_link_libraries(optimize_group_order_limit_by dbms clickhouse_storages_system) - -add_executable(analyze_lambdas analyze_lambdas.cpp) -target_link_libraries(analyze_lambdas dbms) diff --git a/dbms/src/Analyzers/tests/analyze_columns.cpp b/dbms/src/Analyzers/tests/analyze_columns.cpp deleted file mode 100644 index af9cef55108..00000000000 --- a/dbms/src/Analyzers/tests/analyze_columns.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -/// Parses query from stdin and print found columns and corresponding tables. - -int main(int, char **) -try -{ - using namespace DB; - - ReadBufferFromFileDescriptor in(STDIN_FILENO); - WriteBufferFromFileDescriptor out(STDOUT_FILENO); - - String query; - readStringUntilEOF(query, in); - - ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0); - - Context context = Context::createGlobal(); - - auto system_database = std::make_shared("system"); - context.addDatabase("system", system_database); - system_database->attachTable("one", StorageSystemOne::create("one")); - system_database->attachTable("numbers", StorageSystemNumbers::create("numbers", false)); - context.setCurrentDatabase("system"); - - AnalyzeLambdas analyze_lambdas; - analyze_lambdas.process(ast); - - CollectAliases collect_aliases; - collect_aliases.process(ast); - - ExecuteTableFunctions execute_table_functions; - execute_table_functions.process(ast, context); - - CollectTables collect_tables; - collect_tables.process(ast, context, collect_aliases, execute_table_functions); - - AnalyzeColumns analyze_columns; - analyze_columns.process(ast, collect_aliases, collect_tables); - - analyze_columns.dump(out); - out.next(); - - std::cout << "\n"; - formatAST(*ast, std::cout, false, true); - std::cout << "\n"; - - return 0; -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; - return 1; -} diff --git a/dbms/src/Analyzers/tests/analyze_columns.reference b/dbms/src/Analyzers/tests/analyze_columns.reference deleted file mode 100644 index 78370334334..00000000000 --- a/dbms/src/Analyzers/tests/analyze_columns.reference +++ /dev/null @@ -1,18 +0,0 @@ -dummy -> dummy UInt8. Database name: system. Table name: one. Alias: (none). Storage: SystemOne. AST: dummy -number -> number UInt64. Database name: system. Table name: numbers. Alias: t. Storage: SystemNumbers. AST: number -numbers.number -> number UInt64. Database name: system. Table name: numbers. Alias: t. Storage: SystemNumbers. AST: numbers.number -one.dummy -> dummy UInt8. Database name: system. Table name: one. Alias: (none). Storage: SystemOne. AST: one.dummy -system.numbers.number -> number UInt64. Database name: system. Table name: numbers. Alias: t. Storage: SystemNumbers. AST: system.numbers.number -system.one.dummy -> dummy UInt8. Database name: system. Table name: one. Alias: (none). Storage: SystemOne. AST: system.one.dummy -t.number -> number UInt64. Database name: system. Table name: numbers. Alias: t. Storage: SystemNumbers. AST: t.number - -SELECT dummy, number, one.dummy, numbers.number, system.one.dummy, system.numbers.number, system.one.dummy, system.numbers.number, system.one.dummy, system.numbers.number, system.one.dummy, system.numbers.number, system.numbers.number, t.number FROM system.one , system.numbers AS t - -c -> c UInt8. Database name: (none). Table name: (none). Alias: (none). Storage: (none). AST: c - -SELECT arrayMap((_lambda0_arg0, _lambda0_arg1) -> arrayMap((_lambda1_arg0, _lambda1_arg1) -> _lambda0_arg0[_lambda1_arg0], _lambda0_arg0, c), [[1], [2, 3]]) FROM (SELECT 1 AS c, 2 AS d) - -c -> c UInt8. Database name: (none). Table name: (none). Alias: (none). Storage: (none). AST: c -x -> x UInt8. Database name: (none). Table name: (none). Alias: (none). Storage: (none). AST: x - -SELECT x, arrayMap((_lambda0_arg0, _lambda0_arg1) -> (_lambda0_arg0 + _lambda0_arg1), x, c) FROM (SELECT 1 AS x, 2 AS c) diff --git a/dbms/src/Analyzers/tests/analyze_columns.sh b/dbms/src/Analyzers/tests/analyze_columns.sh deleted file mode 100755 index 4fb139ff37b..00000000000 --- a/dbms/src/Analyzers/tests/analyze_columns.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -echo "SELECT dummy, number, one.dummy, numbers.number, system.one.dummy, system.numbers.number, one.*, numbers.*, system.one.*, system.numbers.*, *, t.*, t.number FROM system.one, system.numbers AS t" | ./analyze_columns -echo -echo "SELECT arrayMap((x, y) -> arrayMap((y, z) -> x[y], x, c), [[1], [2, 3]]) FROM (SELECT 1 AS c, 2 AS d)" | ./analyze_columns -echo -echo "SELECT x, arrayMap((x, y) -> x + y, x, c) FROM (SELECT 1 AS x, 2 AS c)" | ./analyze_columns diff --git a/dbms/src/Analyzers/tests/analyze_lambdas.cpp b/dbms/src/Analyzers/tests/analyze_lambdas.cpp deleted file mode 100644 index c82219e4722..00000000000 --- a/dbms/src/Analyzers/tests/analyze_lambdas.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -/// Parses query from stdin and print found higher order functions and query with rewritten names of lambda parameters. - -int main(int, char **) -try -{ - using namespace DB; - - ReadBufferFromFileDescriptor in(STDIN_FILENO); - WriteBufferFromFileDescriptor out(STDOUT_FILENO); - - String query; - readStringUntilEOF(query, in); - - ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0); - - AnalyzeLambdas analyzer; - analyzer.process(ast); - - analyzer.dump(out); - out.next(); - - std::cout << "\n"; - formatAST(*ast, std::cout, false, true); - std::cout << "\n"; - - return 0; -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; - return 1; -} diff --git a/dbms/src/Analyzers/tests/analyze_lambdas.reference b/dbms/src/Analyzers/tests/analyze_lambdas.reference deleted file mode 100644 index 0dab2e44265..00000000000 --- a/dbms/src/Analyzers/tests/analyze_lambdas.reference +++ /dev/null @@ -1,4 +0,0 @@ -arrayMap(lambda(tuple(_lambda1_arg0), arrayElement(_lambda1_arg0, _lambda0_arg1)), arr3) -arrayMap(lambda(tuple(_lambda0_arg0, _lambda0_arg1), plus(_lambda0_arg0, arrayMap(lambda(tuple(_lambda1_arg0), arrayElement(_lambda1_arg0, _lambda0_arg1)), arr3))), arr1, arr2) - -SELECT arrayMap((_lambda0_arg0, _lambda0_arg1) -> (_lambda0_arg0 + arrayMap(_lambda1_arg0 -> _lambda1_arg0[_lambda0_arg1], arr3)), arr1, arr2) diff --git a/dbms/src/Analyzers/tests/analyze_lambdas.sh b/dbms/src/Analyzers/tests/analyze_lambdas.sh deleted file mode 100755 index 2ee03340bae..00000000000 --- a/dbms/src/Analyzers/tests/analyze_lambdas.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -echo "SELECT arrayMap((x, y) -> x + arrayMap(x -> x[y], arr3), arr1, arr2)" | ./analyze_lambdas diff --git a/dbms/src/Analyzers/tests/analyze_result_of_query.cpp b/dbms/src/Analyzers/tests/analyze_result_of_query.cpp deleted file mode 100644 index 16d26bddf4b..00000000000 --- a/dbms/src/Analyzers/tests/analyze_result_of_query.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -/// Parses query from stdin and print names and types of result columns. - -int main(int, char **) -try -{ - using namespace DB; - - ReadBufferFromFileDescriptor in(STDIN_FILENO); - WriteBufferFromFileDescriptor out(STDOUT_FILENO); - - String query; - readStringUntilEOF(query, in); - - ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0); - - Context context = Context::createGlobal(); - - auto system_database = std::make_shared("system"); - context.addDatabase("system", system_database); - context.setCurrentDatabase("system"); - system_database->attachTable("one", StorageSystemOne::create("one")); - system_database->attachTable("numbers", StorageSystemNumbers::create("numbers", false)); - - ExecuteTableFunctions execute_table_functions; - execute_table_functions.process(ast, context); - - AnalyzeResultOfQuery analyzer; - analyzer.process(ast, context, execute_table_functions); - - analyzer.dump(out); - - return 0; -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; - return 1; -} diff --git a/dbms/src/Analyzers/tests/analyze_result_of_query.reference b/dbms/src/Analyzers/tests/analyze_result_of_query.reference deleted file mode 100644 index 42eb3325db5..00000000000 --- a/dbms/src/Analyzers/tests/analyze_result_of_query.reference +++ /dev/null @@ -1 +0,0 @@ -1 UInt8 ColumnConst 1, plus(2, 3) UInt16 ColumnConst 1, x FixedString(20) ColumnConst 1, y String ColumnConst 1, z Int16 nullptr \ No newline at end of file diff --git a/dbms/src/Analyzers/tests/analyze_result_of_query.sh b/dbms/src/Analyzers/tests/analyze_result_of_query.sh deleted file mode 100755 index 03e4da15605..00000000000 --- a/dbms/src/Analyzers/tests/analyze_result_of_query.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -echo "SELECT 1, 2 + 3, toFixedString('Hello, world', 20) AS x, ('Hello', 1).1 AS y, z FROM system.numbers WHERE (arrayJoin([-1, 1]) AS z) = 1" | ./analyze_result_of_query diff --git a/dbms/src/Analyzers/tests/collect_aliases.cpp b/dbms/src/Analyzers/tests/collect_aliases.cpp deleted file mode 100644 index 3666793ce61..00000000000 --- a/dbms/src/Analyzers/tests/collect_aliases.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - - -/// Parses query from stdin and print found aliases. - -int main(int, char **) -try -{ - using namespace DB; - - ReadBufferFromFileDescriptor in(STDIN_FILENO); - WriteBufferFromFileDescriptor out(STDOUT_FILENO); - - String query; - readStringUntilEOF(query, in); - - ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0); - - CollectAliases collector; - collector.process(ast); - collector.dump(out); - - return 0; -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; - return 1; -} diff --git a/dbms/src/Analyzers/tests/collect_aliases.reference b/dbms/src/Analyzers/tests/collect_aliases.reference deleted file mode 100644 index 014a598bc49..00000000000 --- a/dbms/src/Analyzers/tests/collect_aliases.reference +++ /dev/null @@ -1,8 +0,0 @@ -a -> (array join) arr AS a -b -> (array join) (arr1 AS z) + 1 AS b -ghi -> (expression) def AS ghi -t -> (table) d.t AS t -u -> (table) (SELECT 1 AS xxx) AS u -x -> (expression) 1 AS x -y -> (expression) x + 2 AS y -z -> (expression) arr1 AS z diff --git a/dbms/src/Analyzers/tests/collect_aliases.sh b/dbms/src/Analyzers/tests/collect_aliases.sh deleted file mode 100755 index 7473d22b8fc..00000000000 --- a/dbms/src/Analyzers/tests/collect_aliases.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -echo "SELECT 1 AS x, x + 2 AS y FROM d.t AS t INNER JOIN (SELECT 1 AS xxx) AS u USING (abc, def AS ghi) ARRAY JOIN arr AS a, (arr1 AS z) + 1 AS b" | ./collect_aliases diff --git a/dbms/src/Analyzers/tests/collect_tables.cpp b/dbms/src/Analyzers/tests/collect_tables.cpp deleted file mode 100644 index 201880e7e54..00000000000 --- a/dbms/src/Analyzers/tests/collect_tables.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -/// Parses query from stdin and print found tables. - -int main(int, char **) -try -{ - using namespace DB; - - ReadBufferFromFileDescriptor in(STDIN_FILENO); - WriteBufferFromFileDescriptor out(STDOUT_FILENO); - - String query; - readStringUntilEOF(query, in); - - ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0); - - Context context = Context::createGlobal(); - - auto system_database = std::make_shared("system"); - context.addDatabase("system", system_database); - context.setCurrentDatabase("system"); - system_database->attachTable("one", StorageSystemOne::create("one")); - system_database->attachTable("numbers", StorageSystemNumbers::create("numbers", false)); - - CollectAliases collect_aliases; - collect_aliases.process(ast); - - ExecuteTableFunctions execute_table_functions; - execute_table_functions.process(ast, context); - - CollectTables collect_tables; - collect_tables.process(ast, context, collect_aliases, execute_table_functions); - collect_tables.dump(out); - - return 0; -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; - return 1; -} diff --git a/dbms/src/Analyzers/tests/optimize_group_order_limit_by.cpp b/dbms/src/Analyzers/tests/optimize_group_order_limit_by.cpp deleted file mode 100644 index 5680f1de5c0..00000000000 --- a/dbms/src/Analyzers/tests/optimize_group_order_limit_by.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -/// Parses query from stdin and print same query with optimized GROUP BY, ORDER BY, LIMIT BY. - -int main(int, char **) -try -{ - using namespace DB; - - ReadBufferFromFileDescriptor in(STDIN_FILENO); - WriteBufferFromFileDescriptor out(STDOUT_FILENO); - - String query; - readStringUntilEOF(query, in); - - ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0); - - Context context = Context::createGlobal(); - - auto system_database = std::make_shared("system"); - context.addDatabase("system", system_database); - system_database->attachTable("one", StorageSystemOne::create("one")); - system_database->attachTable("numbers", StorageSystemNumbers::create("numbers", false)); - context.setCurrentDatabase("system"); - - AnalyzeLambdas analyze_lambdas; - analyze_lambdas.process(ast); - - CollectAliases collect_aliases; - collect_aliases.process(ast); - - ExecuteTableFunctions execute_table_functions; - execute_table_functions.process(ast, context); - - CollectTables collect_tables; - collect_tables.process(ast, context, collect_aliases, execute_table_functions); - - AnalyzeColumns analyze_columns; - analyze_columns.process(ast, collect_aliases, collect_tables); - - TypeAndConstantInference inference; - inference.process(ast, context, collect_aliases, analyze_columns, analyze_lambdas, execute_table_functions); - - TranslatePositionalArguments translation; - translation.process(ast); - - OptimizeGroupOrderLimitBy optimizer; - optimizer.process(ast, inference); - - formatAST(*ast, std::cout, false); - std::cout << "\n"; - return 0; - - return 0; -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; - return 1; -} diff --git a/dbms/src/Analyzers/tests/optimize_group_order_limit_by.reference b/dbms/src/Analyzers/tests/optimize_group_order_limit_by.reference deleted file mode 100644 index 3dc91598468..00000000000 --- a/dbms/src/Analyzers/tests/optimize_group_order_limit_by.reference +++ /dev/null @@ -1,19 +0,0 @@ -SELECT - number, - materialize('abc') -FROM -( - SELECT - number, - 10 AS b - FROM system.numbers -) -GROUP BY number - -SELECT number -FROM system.numbers -GROUP BY number -ORDER BY - number ASC, - rand() ASC, - toString(rand()) ASC COLLATE 'tr' diff --git a/dbms/src/Analyzers/tests/optimize_group_order_limit_by.sh b/dbms/src/Analyzers/tests/optimize_group_order_limit_by.sh deleted file mode 100755 index 37219cd164f..00000000000 --- a/dbms/src/Analyzers/tests/optimize_group_order_limit_by.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -echo "SELECT number, materialize('abc') FROM (SELECT number, 10 AS b FROM system.numbers) GROUP BY number, toString(number + 1), number + number, 1, 2, 'Hello', b" | ./optimize_group_order_limit_by -echo -echo "SELECT number FROM system.numbers GROUP BY 1 ORDER BY number, 'hello' DESC COLLATE 'tr', number + 1, rand(), identity(number * 2, rand()), toString(rand()) COLLATE 'tr'" | ./optimize_group_order_limit_by diff --git a/dbms/src/Analyzers/tests/translate_positional_arguments.cpp b/dbms/src/Analyzers/tests/translate_positional_arguments.cpp deleted file mode 100644 index 8d1b9ab6c60..00000000000 --- a/dbms/src/Analyzers/tests/translate_positional_arguments.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - - -/// Parses query from stdin and print same query with translated positional arguments. - -int main(int, char **) -try -{ - using namespace DB; - - ReadBufferFromFileDescriptor in(STDIN_FILENO); - - String query; - readStringUntilEOF(query, in); - - ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0); - - TranslatePositionalArguments translator; - translator.process(ast); - - formatAST(*ast, std::cout, false); - std::cout << "\n"; - return 0; -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; - return 1; -} diff --git a/dbms/src/Analyzers/tests/translate_positional_arguments.reference b/dbms/src/Analyzers/tests/translate_positional_arguments.reference deleted file mode 100644 index dc1e92a4f3d..00000000000 --- a/dbms/src/Analyzers/tests/translate_positional_arguments.reference +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - abc, - def + 1, - count() -GROUP BY - abc, - def + 1 -ORDER BY abc DESC diff --git a/dbms/src/Analyzers/tests/translate_positional_arguments.sh b/dbms/src/Analyzers/tests/translate_positional_arguments.sh deleted file mode 100755 index f9f1d5a12bc..00000000000 --- a/dbms/src/Analyzers/tests/translate_positional_arguments.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -echo "SELECT abc, def + 1, count() GROUP BY 1, 2 ORDER BY 1 DESC" | ./translate_positional_arguments diff --git a/dbms/src/Analyzers/tests/type_and_constant_inference.cpp b/dbms/src/Analyzers/tests/type_and_constant_inference.cpp deleted file mode 100644 index 372c7240ec3..00000000000 --- a/dbms/src/Analyzers/tests/type_and_constant_inference.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -/// Parses query from stdin and print data types of expressions; and for constant expressions, print its values. - -int main(int, char **) -try -{ - using namespace DB; - - registerFunctions(); - registerAggregateFunctions(); - registerTableFunctions(); - - ReadBufferFromFileDescriptor in(STDIN_FILENO); - WriteBufferFromFileDescriptor out(STDOUT_FILENO); - - String query; - readStringUntilEOF(query, in); - - ParserSelectQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0); - - Context context = Context::createGlobal(); - - auto system_database = std::make_shared("system"); - context.addDatabase("system", system_database); - system_database->attachTable("one", StorageSystemOne::create("one")); - system_database->attachTable("numbers", StorageSystemNumbers::create("numbers", false)); - context.setCurrentDatabase("system"); - - AnalyzeLambdas analyze_lambdas; - analyze_lambdas.process(ast); - - CollectAliases collect_aliases; - collect_aliases.process(ast); - - ExecuteTableFunctions execute_table_functions; - execute_table_functions.process(ast, context); - - CollectTables collect_tables; - collect_tables.process(ast, context, collect_aliases, execute_table_functions); - - AnalyzeColumns analyze_columns; - analyze_columns.process(ast, collect_aliases, collect_tables); - - TypeAndConstantInference inference; - inference.process(ast, context, collect_aliases, analyze_columns, analyze_lambdas, execute_table_functions); - - inference.dump(out); - out.next(); - - return 0; -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; - return 1; -} diff --git a/dbms/src/Analyzers/tests/type_and_constant_inference.reference b/dbms/src/Analyzers/tests/type_and_constant_inference.reference deleted file mode 100644 index 882fbdc4e7e..00000000000 --- a/dbms/src/Analyzers/tests/type_and_constant_inference.reference +++ /dev/null @@ -1,34 +0,0 @@ -' ' -> String = ' '. AST: ' ' -'Hello' -> String = 'Hello'. AST: 'Hello' -'Hello, world' -> String = 'Hello, world'. AST: 'Hello, world' -'World' -> String = 'World'. AST: 'World' -0.5 -> Float64 = 0.5. AST: 0.5 -1 -> UInt8 = 1. AST: 1 -20 -> UInt8 = 20. AST: 20 -[1, -1, 0.1] -> Array(Float64) = [1., -1., 0.1]. AST: [1, -1, 0.1] -concat('Hello', ' ', 'World') -> String = 'Hello World'. AST: concat('Hello', ' ', 'World') AS world -count() -> UInt64. AST: count() -e -> Float64 = 2.718281828459045. AST: e() AS e -e() -> Float64 = 2.718281828459045. AST: e() AS e -pi() -> Float64 = 3.141592653589793. AST: pi() -plus(e, pi()) -> Float64 = 5.859874482048838. AST: e + pi() -quantileTiming(0.5)(1) -> Float32. AST: quantileTiming(0.5)(1) -rand() -> UInt32. AST: rand() -sleep(1) -> UInt8. AST: sleep(1) -sum(e) -> Float64. AST: sum(e) -system.numbers.number -> UInt64. AST: system.numbers.number -toFixedString('Hello, world', 20) -> FixedString(20) = 'Hello, world\0\0\0\0\0\0\0\0'. AST: toFixedString('Hello, world', 20) -tuple('Hello', 1) -> Tuple(String, UInt8) = ('Hello', 1). AST: ('Hello', 1) -tupleElement(tuple('Hello', 1), 1) -> String = 'Hello'. AST: ('Hello', 1).1 - -t.x -> UInt8 = 1. AST: t.x - -x -> UInt8 = 1. AST: x - -1 -> UInt8 = 1. AST: 1 -t.x -> UInt8 = 1. AST: t.x -x -> UInt8 = 1. AST: x - -x -> Tuple(UInt8, UInt8) = (1, 2). AST: x -y -> Tuple(UInt8, UInt8) = (3, 4). AST: y -z -> String = 'Hello, world'. AST: z diff --git a/dbms/src/Analyzers/tests/type_and_constant_inference.sh b/dbms/src/Analyzers/tests/type_and_constant_inference.sh deleted file mode 100755 index 58af97ebf45..00000000000 --- a/dbms/src/Analyzers/tests/type_and_constant_inference.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -echo "SELECT - 1, - *, - sleep(1), - rand(), - 'Hello' || ' ' || 'World' AS world, - e() AS e, - e + pi(), - count(*), - sum(e), - quantileTiming(0.5)(1), - [1, -1, 0.1], - toFixedString('Hello, world', 20), - ('Hello', 1).1 -FROM system.numbers" | ./type_and_constant_inference - -echo -echo "SELECT t.x FROM (SELECT 1 AS x) AS t" | ./type_and_constant_inference -echo -echo "SELECT x FROM (SELECT 1 AS x)" | ./type_and_constant_inference -echo -echo "SELECT t.x, x, 1 FROM (SELECT 1 AS x) AS t" | ./type_and_constant_inference -echo -echo "SELECT *, z FROM (SELECT (1, 2) AS x, (SELECT 3, 4) AS y), (SELECT 'Hello, world' AS z)" | ./type_and_constant_inference diff --git a/dbms/src/CMakeLists.txt b/dbms/src/CMakeLists.txt index 8c1c48a5420..bfc318d9ac8 100644 --- a/dbms/src/CMakeLists.txt +++ b/dbms/src/CMakeLists.txt @@ -13,6 +13,5 @@ add_subdirectory (AggregateFunctions) add_subdirectory (Server) add_subdirectory (Client) add_subdirectory (TableFunctions) -add_subdirectory (Analyzers) add_subdirectory (Flash) add_subdirectory (Encryption) diff --git a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp index c6d2f4922a2..7e7e3e0b8a9 100644 --- a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include