From 8e1762241d825a40e9485810a51117459186df7b Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 27 Apr 2022 15:40:10 +0800 Subject: [PATCH 01/42] init. --- dbms/src/Flash/Coprocessor/DAGContext.h | 11 +++ .../Coprocessor/DAGQueryBlockInterpreter.cpp | 68 +++++++++++++++++-- .../Coprocessor/DAGQueryBlockInterpreter.h | 2 + dbms/src/Flash/Coprocessor/InterpreterDAG.cpp | 2 - dbms/src/Flash/tests/gtest_interpreter.cpp | 66 ++++++++++++++++++ dbms/src/TestUtils/InterpreterTestUtils.cpp | 56 +++++++++++++-- dbms/src/TestUtils/InterpreterTestUtils.h | 17 +++-- .../TestUtils/tests/gtest_mock_executors.cpp | 26 +++++-- 8 files changed, 223 insertions(+), 25 deletions(-) create mode 100644 dbms/src/Flash/tests/gtest_interpreter.cpp diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index 18ad73ec207..a849221f0b6 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -275,6 +275,14 @@ class DAGContext return sql_mode & f; } + void setIsInterpreterTest(bool is_interpreter_test_) { is_interpreter_test = is_interpreter_test_; } + + bool isInterpreterTest() const { return is_interpreter_test; } + + void setMockTableScanStreams(UInt32 streams) { mock_table_scan_streams = streams; } + + UInt32 mockTableScanStreams() const { return mock_table_scan_streams; } + void cancelAllExchangeReceiver(); void initExchangeReceiverIfMPP(Context & context, size_t max_streams); @@ -345,6 +353,9 @@ class DAGContext /// vector of SubqueriesForSets(such as join build subquery). /// The order of the vector is also the order of the subquery. std::vector subqueries; + + bool is_interpreter_test = false; /// switch for interpreter unit test, do not use it in production. + UInt32 mock_table_scan_streams = 0; /// Used in interpreter unit test. Determine how many MockTableScanBlockInpustreams to generate. }; } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 561011e2d95..565e8ac47d9 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include namespace DB @@ -300,6 +301,63 @@ void DAGQueryBlockInterpreter::handleTableScan(const TiDBTableScan & table_scan, } } +std::pair, std::vector> getColumnsForTableScan( + const TiDBTableScan & table_scan) +{ + std::vector columms; + std::vector names_and_types; + TiDB::TableInfo table_info; + for (Int32 i = 0; i < table_scan.getColumnSize(); ++i) + { + auto const & ci = table_scan.getColumns()[i]; + TiDB::ColumnInfo column_info; + column_info.id = ci.column_id(); + column_info.tp = TiDB::TypeString; + table_info.columns.push_back(column_info); + } + for (const auto & column : table_info.columns) + { + auto type = getDataTypeByColumnInfoForComputingLayer(column); + names_and_types.push_back({"test", type}); + columms.push_back(ColumnWithTypeAndName(type, "test")); + } + return {columms, names_and_types}; +} + +// ywq todo +void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline) +{ + // has table to read + // construct pushed down filter conditions. + std::vector conditions; + if (query_block.selection) + { + for (const auto & condition : query_block.selection->selection().conditions()) + conditions.push_back(&condition); + } + + auto columns = getColumnsForTableScan(table_scan); + analyzer = std::make_unique(std::move(columns.second), context); + for (size_t i = 0; i < context.getDAGContext()->mockTableScanStreams(); ++i) + { + auto mock_table_scan_stream = std::make_shared(std::get<0>(columns), context.getSettingsRef().max_block_size); + pipeline.streams.emplace_back(mock_table_scan_stream); + } + /// Set the limits and quota for reading data, the speed and time of the query. + setQuotaAndLimitsOnTableScan(context, pipeline); + FAIL_POINT_PAUSE(FailPoints::pause_after_copr_streams_acquired); + /// handle timezone/duration cast for local and remote table scan. ywq todo check.. + DAGStorageInterpreter storage_interpreter(context, query_block, table_scan, conditions, max_streams); + executeCastAfterTableScan(table_scan, storage_interpreter.is_need_add_cast_column, context.getDAGContext()->mockTableScanStreams(), pipeline); + recordProfileStreams(pipeline, query_block.source_name); + /// handle pushed down filter for local and remote table scan. + if (query_block.selection) + { + executePushedDownFilter(conditions, 1, pipeline); + recordProfileStreams(pipeline, query_block.selection_name); + } +} + void DAGQueryBlockInterpreter::executePushedDownFilter( const std::vector & conditions, size_t remote_read_streams_start_index, @@ -1042,7 +1100,10 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) else if (query_block.isTableScanSource()) { TiDBTableScan table_scan(query_block.source, dagContext()); - handleTableScan(table_scan, pipeline); + if (context.getDAGContext()->isInterpreterTest()) + handleMockTableScan(table_scan, pipeline); + else + handleTableScan(table_scan, pipeline); dagContext().table_scan_executor_id = query_block.source_name; } else if (query_block.source->tp() == tipb::ExecType::TypeWindow) @@ -1083,20 +1144,17 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) query_block.qb_column_prefix, pipeline.streams.size()); dagContext().final_concurrency = std::min(std::max(dagContext().final_concurrency, pipeline.streams.size()), max_streams); - if (res.before_aggregation) { // execute aggregation executeAggregation(pipeline, res.before_aggregation, res.aggregation_keys, res.aggregation_collators, res.aggregate_descriptions, res.is_final_agg); } - if (res.before_having) { // execute having executeWhere(pipeline, res.before_having, res.having_column_name); recordProfileStreams(pipeline, query_block.having_name); } - if (res.before_order_and_select) { executeExpression(pipeline, res.before_order_and_select); @@ -1111,14 +1169,12 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) // execute final project action executeProject(pipeline, final_project); - // execute limit if (query_block.limit_or_topn && query_block.limit_or_topn->tp() == tipb::TypeLimit) { executeLimit(pipeline); recordProfileStreams(pipeline, query_block.limit_or_topn_name); } - restorePipelineConcurrency(pipeline); // execute exchange_sender diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h index 8e6908dec80..177ea89efaf 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +58,7 @@ class DAGQueryBlockInterpreter private: #endif void executeImpl(DAGPipeline & pipeline); + void handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline); void handleTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline); void executeCastAfterTableScan( const TiDBTableScan & table_scan, diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index 6b118f1dd40..ca66f923b68 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -79,7 +79,6 @@ BlockIO InterpreterDAG::execute() BlockInputStreams streams = executeQueryBlock(*dag.getRootQueryBlock()); DAGPipeline pipeline; pipeline.streams = streams; - /// add union to run in parallel if needed if (dagContext().isMPPTask()) /// MPPTask do not need the returned blocks. @@ -95,7 +94,6 @@ BlockIO InterpreterDAG::execute() SizeLimits(settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode), dagContext().log->identifier()); } - BlockIO res; res.in = pipeline.firstStream(); return res; diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp new file mode 100644 index 00000000000..62630584fa1 --- /dev/null +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -0,0 +1,66 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +namespace DB +{ +namespace tests +{ +class InterpreterExecuteTest : public DB::tests::InterpreterTest +{ +public: + void initializeContext() override + { + InterpreterTest::initializeContext(); + + context.addMockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}); + context.addMockTable({"test_db", "test_table_1"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}); + context.addMockTable({"test_db", "r_table"}, {{"r_a", TiDB::TP::TypeLong}, {"r_b", TiDB::TP::TypeString}, {"join_c", TiDB::TP::TypeString}}); + context.addMockTable({"test_db", "l_table"}, {{"l_a", TiDB::TP::TypeLong}, {"l_b", TiDB::TP::TypeString}, {"join_c", TiDB::TP::TypeString}}); + context.addExchangeRelationSchema("sender_1", {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}); + } +}; + +TEST_F(InterpreterExecuteTest, SingleQueryBlock) +try +{ + auto request = context.scan("test_db", "test_table_1") + .filter(eq(col("s2"), col("s3"))) + .aggregation({Max(col("s1"))}, {col("s2"), col("s3")}) + .filter(eq(col("s2"), col("s3"))) + .topN("s2", false, 10) + .build(context); + { + String expected = "Expression\n" + " MergeSorting\n" + " PartialSorting\n" + " Expression\n" + " Filter\n" + " ParallelAggregating\n" + " Expression\n" + " Expression\n" + " Filter\n" + " MockTableScan\n" + " Expression x 9\n" + " MockTableScan\n"; + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + } +} +CATCH + + +} // namespace tests +} // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp index 737978a8bc4..614908883e3 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.cpp +++ b/dbms/src/TestUtils/InterpreterTestUtils.cpp @@ -13,24 +13,42 @@ // limitations under the License. #include +#include +#include #include #include - namespace DB::tests { -DAGContext & MockExecutorTest::getDAGContext() +// not necessary... +// void executeInterpreter(const std::shared_ptr & request, Context & context) +// { +// DAGContext dag_context(*request); +// dag_context.log = Logger::get("interpreterTest"); +// dag_context.setIsInterpreterTest(true); +// dag_context.setMockTableScanStreams(10); +// context.setDAGContext(&dag_context); +// // Currently, don't care about regions information in interpreter tests. +// DAGQuerySource dag(context); +// auto res = executeQuery(dag, context, false, QueryProcessingStage::Complete); +// FmtBuffer fb; +// res.in->dumpTree(fb); +// std::cout << fb.toString() << std::endl; +// } + +DAGContext & InterpreterTest::getDAGContext() { assert(dag_context_ptr != nullptr); return *dag_context_ptr; } -void MockExecutorTest::initializeContext() +void InterpreterTest::initializeContext() { dag_context_ptr = std::make_unique(1024); context = MockDAGRequestContext(TiFlashTestEnv::getContext()); + dag_context_ptr->log = Logger::get("interpreterTest"); } -void MockExecutorTest::SetUpTestCase() +void InterpreterTest::SetUpTestCase() { try { @@ -43,8 +61,34 @@ void MockExecutorTest::SetUpTestCase() } } -void MockExecutorTest::dagRequestEqual(String & expected_string, const std::shared_ptr & actual) +void InterpreterTest::initializeClientInfo() +{ + context.context.setCurrentQueryId("test"); + ClientInfo & client_info = context.context.getClientInfo(); + client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + client_info.interface = ClientInfo::Interface::GRPC; +} + +void InterpreterTest::executeInterpreter(const String & expected_string, const std::shared_ptr & request) { - ASSERT_EQ(Poco::trimInPlace(expected_string), Poco::trim(ExecutorSerializer().serialize(actual.get()))); + DAGContext dag_context(*request); + dag_context.log = Logger::get("interpreterTest"); + dag_context.setIsInterpreterTest(true); + dag_context.setMockTableScanStreams(10); + // todo change the code here. + context.context.setDAGContext(&dag_context); + // Currently, don't care about regions information in interpreter tests. + DAGQuerySource dag(context.context); + auto res = executeQuery(dag, context.context, false, QueryProcessingStage::Complete); + FmtBuffer fb; + res.in->dumpTree(fb); + std::cout << fb.toString() << std::endl; + ASSERT_EQ(Poco::trim(expected_string), Poco::trim(fb.toString())); } + +void InterpreterTest::dagRequestEqual(const String & expected_string, const std::shared_ptr & actual) +{ + ASSERT_EQ(Poco::trim(expected_string), Poco::trim(ExecutorSerializer().serialize(actual.get()))); +} + } // namespace DB::tests diff --git a/dbms/src/TestUtils/InterpreterTestUtils.h b/dbms/src/TestUtils/InterpreterTestUtils.h index 074c65da6f0..96eba2cdabd 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.h +++ b/dbms/src/TestUtils/InterpreterTestUtils.h @@ -26,30 +26,39 @@ #include namespace DB::tests { -class MockExecutorTest : public ::testing::Test +void executeInterpreter(const std::shared_ptr & request, Context & context); +class InterpreterTest : public ::testing::Test { protected: void SetUp() override { initializeContext(); + initializeClientInfo(); } public: - MockExecutorTest() + InterpreterTest() : context(TiFlashTestEnv::getContext()) {} static void SetUpTestCase(); virtual void initializeContext(); + void initializeClientInfo(); + DAGContext & getDAGContext(); - static void dagRequestEqual(String & expected_string, const std::shared_ptr & actual); + static void dagRequestEqual(const String & expected_string, const std::shared_ptr & actual); + + void executeInterpreter(const String & expected_string, const std::shared_ptr & request); protected: MockDAGRequestContext context; std::unique_ptr dag_context_ptr; }; -#define ASSERT_DAGREQUEST_EQAUL(str, request) dagRequestEqual(str, request); +#define ASSERT_DAGREQUEST_EQAUL(str, request) dagRequestEqual((str), (request)); +#define ASSERT_BLOCKINPUTSTREAM_EQAUL(str, request) executeInterpreter((str), (request)) + +// #define xxxx } // namespace DB::tests \ No newline at end of file diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index 5c7d77c399a..02f420dacc3 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -12,22 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include -#include namespace DB { namespace tests { -class MockDAGRequestTest : public DB::tests::MockExecutorTest +class MockDAGRequestTest : public DB::tests::InterpreterTest { public: void initializeContext() override { - dag_context_ptr = std::make_unique(1024); - context = MockDAGRequestContext(TiFlashTestEnv::getContext()); + InterpreterTest::initializeContext(); context.addMockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}); context.addMockTable({"test_db", "test_table_1"}, {{"s1", TiDB::TP::TypeLong}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}); @@ -45,11 +42,14 @@ try String expected = "table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); } + // runInterpreter(request); + request = context.scan("test_db", "test_table_1").build(context); { String expected = "table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); } + // runInterpreter(request); } CATCH @@ -61,14 +61,16 @@ try String expected = "selection_1 | equals(<0, String>, <1, String>)}\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } request = context.scan("test_db", "test_table_1") - .filter(And(eq(col("s1"), col("s2")), lt(col("s2"), lt(col("s1"), col("s2"))))) + .filter(And(eq(col("s1"), col("s2")), lt(col("s2"), col("s2")))) // type in lt must be same .build(context); { - String expected = "selection_1 | equals(<0, Long>, <1, String>) and less(<1, String>, less(<0, Long>, <1, String>))}\n" + String expected = "selection_1 | equals(<0, Long>, <1, String>) and less(<1, String>, <1, String>)}\n" " table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } } CATCH @@ -83,6 +85,7 @@ try String expected = "project_1 | {<0, String>}\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } request = context.scan("test_db", "test_table_1") .project({col("s3"), eq(col("s1"), col("s2"))}) @@ -91,6 +94,7 @@ try String expected = "project_1 | {<2, String>, equals(<0, Long>, <1, String>)}\n" " table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } request = context.scan("test_db", "test_table_1") .project({"s1", "s2"}) @@ -99,6 +103,7 @@ try String expected = "project_1 | {<0, Long>, <1, String>}\n" " table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } } CATCH @@ -113,6 +118,7 @@ try String expected = "limit_1 | 10\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } request = context.scan("test_db", "test_table_1") .limit(lit(Field(static_cast(10)))) @@ -121,6 +127,7 @@ try String expected = "limit_1 | 10\n" " table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } } CATCH @@ -135,6 +142,7 @@ try String expected = "topn_1 | order_by: {(<0, String>, desc: false), (<1, String>, desc: true)}, limit: 10\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } request = context.scan("test_db", "test_table") .topN("s1", false, 10) @@ -143,6 +151,7 @@ try String expected = "topn_1 | order_by: {(<0, String>, desc: false)}, limit: 10\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // executeInterpreter(request, context.context); } } CATCH @@ -157,6 +166,7 @@ try String expected = "aggregation_1 | group_by: {<1, String>}, agg_func: {max(<0, String>)}\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } request = context.scan("test_db", "test_table") @@ -166,6 +176,7 @@ try String expected = "aggregation_1 | group_by: {<1, String>, less(<0, String>, <1, String>)}, agg_func: {max(<0, String>)}\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } } CATCH @@ -195,6 +206,7 @@ try " selection_1 | equals(<0, Long>, <1, String>) and equals(<0, Long>, <1, String>)}\n" " table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); + // runInterpreter(request); } } CATCH From c8fd369036de980d580a320bbb572bfa1330256d Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 27 Apr 2022 16:10:51 +0800 Subject: [PATCH 02/42] fix. --- dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp | 7 ++++--- dbms/src/Flash/tests/gtest_interpreter.cpp | 6 ++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 565e8ac47d9..d9211ade66b 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -338,7 +338,8 @@ void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_s auto columns = getColumnsForTableScan(table_scan); analyzer = std::make_unique(std::move(columns.second), context); - for (size_t i = 0; i < context.getDAGContext()->mockTableScanStreams(); ++i) + auto mock_table_scan_streams = context.getDAGContext()->mockTableScanStreams(); + for (size_t i = 0; i < mock_table_scan_streams; ++i) { auto mock_table_scan_stream = std::make_shared(std::get<0>(columns), context.getSettingsRef().max_block_size); pipeline.streams.emplace_back(mock_table_scan_stream); @@ -348,12 +349,12 @@ void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_s FAIL_POINT_PAUSE(FailPoints::pause_after_copr_streams_acquired); /// handle timezone/duration cast for local and remote table scan. ywq todo check.. DAGStorageInterpreter storage_interpreter(context, query_block, table_scan, conditions, max_streams); - executeCastAfterTableScan(table_scan, storage_interpreter.is_need_add_cast_column, context.getDAGContext()->mockTableScanStreams(), pipeline); + executeCastAfterTableScan(table_scan, storage_interpreter.is_need_add_cast_column, mock_table_scan_streams, pipeline); recordProfileStreams(pipeline, query_block.source_name); /// handle pushed down filter for local and remote table scan. if (query_block.selection) { - executePushedDownFilter(conditions, 1, pipeline); + executePushedDownFilter(conditions, mock_table_scan_streams, pipeline); recordProfileStreams(pipeline, query_block.selection_name); } } diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 62630584fa1..acfdfe9cdfa 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -50,12 +50,10 @@ try " Expression\n" " Filter\n" " ParallelAggregating\n" - " Expression\n" + " Expression x 10\n" " Expression\n" " Filter\n" - " MockTableScan\n" - " Expression x 9\n" - " MockTableScan\n"; + " MockTableScan\n"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } } From 491dafd40b3da2bdaa0d332b0e0ce7616f1872a6 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 28 Apr 2022 12:02:06 +0800 Subject: [PATCH 03/42] fix restoreConcurrency. --- .../Coprocessor/DAGQueryBlockInterpreter.cpp | 4 +++- dbms/src/Flash/tests/gtest_interpreter.cpp | 24 +++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index d9211ade66b..49b7e9fc694 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -339,6 +339,7 @@ void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_s auto columns = getColumnsForTableScan(table_scan); analyzer = std::make_unique(std::move(columns.second), context); auto mock_table_scan_streams = context.getDAGContext()->mockTableScanStreams(); + max_streams = mock_table_scan_streams; for (size_t i = 0; i < mock_table_scan_streams; ++i) { auto mock_table_scan_stream = std::make_shared(std::get<0>(columns), context.getSettingsRef().max_block_size); @@ -831,7 +832,7 @@ void DAGQueryBlockInterpreter::executeAggregation( collators, aggregate_descriptions, is_final_agg); - + /// If there are several sources, then we perform parallel aggregation if (pipeline.streams.size() > 1) { @@ -849,6 +850,7 @@ void DAGQueryBlockInterpreter::executeAggregation( pipeline.streams.resize(1); // should record for agg before restore concurrency. See #3804. recordProfileStreams(pipeline, query_block.aggregation_name); + std::cout << "can restore: " << query_block.can_restore_pipeline_concurrency << std::endl; restorePipelineConcurrency(pipeline); } else diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index acfdfe9cdfa..09470088aaf 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -44,16 +44,20 @@ try .topN("s2", false, 10) .build(context); { - String expected = "Expression\n" - " MergeSorting\n" - " PartialSorting\n" - " Expression\n" - " Filter\n" - " ParallelAggregating\n" - " Expression x 10\n" - " Expression\n" - " Filter\n" - " MockTableScan\n"; + String expected = "Union\n" + " SharedQuery x 10\n" + " Expression\n" + " MergeSorting\n" + " Union\n" + " PartialSorting x 10\n" + " Expression\n" + " Filter\n" + " SharedQuery\n" + " ParallelAggregating\n" + " Expression x 10\n" + " Expression\n" + " Filter\n" + " MockTableScan\n"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } } From fbd66bbe912d95807e8d1874a4ae8b427b39c06d Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 28 Apr 2022 12:58:13 +0800 Subject: [PATCH 04/42] some tests --- dbms/src/Flash/tests/gtest_interpreter.cpp | 71 ++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 09470088aaf..50328a29641 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -60,6 +60,77 @@ try " MockTableScan\n"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } + + request = context.scan("test_db", "test_table_1") + .filter(eq(col("s2"), col("s3"))) + .aggregation({Max(col("s1"))}, {col("s2"), col("s3")}) + .filter(eq(col("s2"), col("s3"))) + .limit(10) + .build(context); + + { + String expected = "Union\n" + " SharedQuery x 10\n" + " Limit\n" + " Union\n" + " Limit x 10\n" + " Expression\n" + " Expression\n" + " Filter\n" + " SharedQuery\n" + " ParallelAggregating\n" + " Expression x 10\n" + " Expression\n" + " Filter\n" + " MockTableScan\n"; + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + } +} +CATCH + +TEST_F(InterpreterExecuteTest, MultipleQueryBlockWithSource) +try +{ + auto request = context.scan("test_db", "test_table_1") + .project({"s1", "s2", "s3"}) + .project({"s1", "s2"}) + .project("s1") + .build(context); + { + String expected = "Union\n" + " Expression x 10\n" + " Expression\n" + " Expression\n" + " Expression\n" + " Expression\n" + " Expression\n" + " Expression\n" + " Expression\n" + " Expression\n" + " Expression\n" + " MockTableScan\n"; + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + } + + request = context.scan("test_db", "test_table_1") + .project({"s1", "s2", "s3"}) + .topN({{"s1", true}, {"s2", false}}, 10) + .project({"s1", "s2"}) + .build(context); + { + String expected = "Expression\n" + " Expression\n" + " Expression\n" + " Expression\n" + " MergeSorting\n" + " Union\n" + " PartialSorting x 10\n" + " Expression\n" + " Expression\n" + " Expression\n" + " MockTableScan\n"; + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + } } CATCH From b5e5614c26d78816abb58f5e24f170bd27622c24 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 28 Apr 2022 12:58:40 +0800 Subject: [PATCH 05/42] format. --- .../Coprocessor/DAGQueryBlockInterpreter.cpp | 2 +- dbms/src/Flash/tests/gtest_interpreter.cpp | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 49b7e9fc694..9427d14fe7a 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -832,7 +832,7 @@ void DAGQueryBlockInterpreter::executeAggregation( collators, aggregate_descriptions, is_final_agg); - + /// If there are several sources, then we perform parallel aggregation if (pipeline.streams.size() > 1) { diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 50328a29641..6aed6ef3447 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -62,12 +62,12 @@ try } request = context.scan("test_db", "test_table_1") - .filter(eq(col("s2"), col("s3"))) - .aggregation({Max(col("s1"))}, {col("s2"), col("s3")}) - .filter(eq(col("s2"), col("s3"))) - .limit(10) - .build(context); - + .filter(eq(col("s2"), col("s3"))) + .aggregation({Max(col("s1"))}, {col("s2"), col("s3")}) + .filter(eq(col("s2"), col("s3"))) + .limit(10) + .build(context); + { String expected = "Union\n" " SharedQuery x 10\n" @@ -113,10 +113,10 @@ try } request = context.scan("test_db", "test_table_1") - .project({"s1", "s2", "s3"}) - .topN({{"s1", true}, {"s2", false}}, 10) - .project({"s1", "s2"}) - .build(context); + .project({"s1", "s2", "s3"}) + .topN({{"s1", true}, {"s2", false}}, 10) + .project({"s1", "s2"}) + .build(context); { String expected = "Expression\n" " Expression\n" From 99830e3584291b2ef36e56703393b04010b9aabd Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 28 Apr 2022 14:09:16 +0800 Subject: [PATCH 06/42] fix build fail. --- .../MockTableScanBlockInputStream.cpp | 2 +- .../{TestUtils => DataStreams}/MockTableScanBlockInputStream.h | 0 dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp | 2 +- dbms/src/WindowFunctions/tests/gtest_window_functions.cpp | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename dbms/src/{TestUtils => DataStreams}/MockTableScanBlockInputStream.cpp (96%) rename dbms/src/{TestUtils => DataStreams}/MockTableScanBlockInputStream.h (100%) diff --git a/dbms/src/TestUtils/MockTableScanBlockInputStream.cpp b/dbms/src/DataStreams/MockTableScanBlockInputStream.cpp similarity index 96% rename from dbms/src/TestUtils/MockTableScanBlockInputStream.cpp rename to dbms/src/DataStreams/MockTableScanBlockInputStream.cpp index 316c7487a63..d8c9df666a6 100644 --- a/dbms/src/TestUtils/MockTableScanBlockInputStream.cpp +++ b/dbms/src/DataStreams/MockTableScanBlockInputStream.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include namespace DB { diff --git a/dbms/src/TestUtils/MockTableScanBlockInputStream.h b/dbms/src/DataStreams/MockTableScanBlockInputStream.h similarity index 100% rename from dbms/src/TestUtils/MockTableScanBlockInputStream.h rename to dbms/src/DataStreams/MockTableScanBlockInputStream.h diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 9427d14fe7a..24efbd59602 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -46,7 +46,7 @@ #include #include #include -#include +#include #include namespace DB diff --git a/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp b/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp index f94a20c1a65..38c9e2804e9 100644 --- a/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp +++ b/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include From 347bf8a5ab8c59fb1967271aa93618ca57e8831c Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 28 Apr 2022 15:01:10 +0800 Subject: [PATCH 07/42] format, more tests. --- .../Coprocessor/DAGQueryBlockInterpreter.cpp | 2 +- dbms/src/Flash/tests/gtest_interpreter.cpp | 216 +++++++++++++----- .../tests/gtest_window_functions.cpp | 2 +- 3 files changed, 166 insertions(+), 54 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 24efbd59602..383185625b6 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -46,7 +47,6 @@ #include #include #include -#include #include namespace DB diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 6aed6ef3447..02792c05020 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -30,7 +30,6 @@ class InterpreterExecuteTest : public DB::tests::InterpreterTest context.addMockTable({"test_db", "test_table_1"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}); context.addMockTable({"test_db", "r_table"}, {{"r_a", TiDB::TP::TypeLong}, {"r_b", TiDB::TP::TypeString}, {"join_c", TiDB::TP::TypeString}}); context.addMockTable({"test_db", "l_table"}, {{"l_a", TiDB::TP::TypeLong}, {"l_b", TiDB::TP::TypeString}, {"join_c", TiDB::TP::TypeString}}); - context.addExchangeRelationSchema("sender_1", {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}); } }; @@ -44,20 +43,21 @@ try .topN("s2", false, 10) .build(context); { - String expected = "Union\n" - " SharedQuery x 10\n" - " Expression\n" - " MergeSorting\n" - " Union\n" - " PartialSorting x 10\n" - " Expression\n" - " Filter\n" - " SharedQuery\n" - " ParallelAggregating\n" - " Expression x 10\n" - " Expression\n" - " Filter\n" - " MockTableScan\n"; + String expected = R"( +Union + SharedQuery x 10 + Expression + MergeSorting + Union + PartialSorting x 10 + Expression + Filter + SharedQuery + ParallelAggregating + Expression x 10 + Expression + Filter + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } @@ -69,20 +69,21 @@ try .build(context); { - String expected = "Union\n" - " SharedQuery x 10\n" - " Limit\n" - " Union\n" - " Limit x 10\n" - " Expression\n" - " Expression\n" - " Filter\n" - " SharedQuery\n" - " ParallelAggregating\n" - " Expression x 10\n" - " Expression\n" - " Filter\n" - " MockTableScan\n"; + String expected = R"( +Union + SharedQuery x 10 + Limit + Union + Limit x 10 + Expression + Expression + Filter + SharedQuery + ParallelAggregating + Expression x 10 + Expression + Filter + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } } @@ -97,18 +98,19 @@ try .project("s1") .build(context); { - String expected = "Union\n" - " Expression x 10\n" - " Expression\n" - " Expression\n" - " Expression\n" - " Expression\n" - " Expression\n" - " Expression\n" - " Expression\n" - " Expression\n" - " Expression\n" - " MockTableScan\n"; + String expected = R"( +Union + Expression x 10 + Expression + Expression + Expression + Expression + Expression + Expression + Expression + Expression + Expression + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } @@ -118,17 +120,127 @@ try .project({"s1", "s2"}) .build(context); { - String expected = "Expression\n" - " Expression\n" - " Expression\n" - " Expression\n" - " MergeSorting\n" - " Union\n" - " PartialSorting x 10\n" - " Expression\n" - " Expression\n" - " Expression\n" - " MockTableScan\n"; + String expected = R"( +Expression + Expression + Expression + Expression + MergeSorting + Union + PartialSorting x 10 + Expression + Expression + Expression + MockTableScan)"; + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + } + + request = context.scan("test_db", "test_table_1") + .project({"s1", "s2", "s3"}) + .topN({{"s1", true}, {"s2", false}}, 10) + .project({"s1", "s2"}) + .aggregation({Max(col("s1"))}, {col("s1"), col("s2")}) + .project({"max(s1)", "s1", "s2"}) + .build(context); + { + String expected = R"( +Expression + Expression + Expression + Expression + Aggregating + Concat + Expression + Expression + Expression + Expression + MergeSorting + Union + PartialSorting x 10 + Expression + Expression + Expression + MockTableScan)"; + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + } + + request = context.scan("test_db", "test_table_1") + .project({"s1", "s2", "s3"}) + .topN({{"s1", true}, {"s2", false}}, 10) + .project({"s1", "s2"}) + .aggregation({Max(col("s1"))}, {col("s1"), col("s2")}) + .project({"max(s1)", "s1", "s2"}) + .filter(eq(col("s1"), col("s2"))) + .project({"max(s1)", "s1"}) + .limit(10) + .build(context); + { + String expected = R"( +Limit + Expression + Expression + Expression + Expression + Expression + Filter + Expression + Expression + Expression + Aggregating + Concat + Expression + Expression + Expression + Expression + MergeSorting + Union + PartialSorting x 10 + Expression + Expression + Expression + MockTableScan)"; + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + } + + // Join Source. + DAGRequestBuilder table1 = context.scan("test_db", "r_table"); + DAGRequestBuilder table2 = context.scan("test_db", "l_table"); + DAGRequestBuilder table3 = context.scan("test_db", "r_table"); + DAGRequestBuilder table4 = context.scan("test_db", "l_table"); + + request = table1.join( + table2.join( + table3.join(table4, + {col("join_c")}, + ASTTableJoin::Kind::Left), + {col("join_c")}, + ASTTableJoin::Kind::Left), + {col("join_c")}, + ASTTableJoin::Kind::Left) + .build(context); + { + String expected = R"( +CreatingSets + Union + HashJoinBuildBlockInputStream x 10 + Expression + Expression + MockTableScan + Union x 2 + HashJoinBuildBlockInputStream x 10 + Expression + Expression + Expression + HashJoinProbe + Expression + MockTableScan + Union + Expression x 10 + Expression + Expression + HashJoinProbe + Expression + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } } diff --git a/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp b/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp index 38c9e2804e9..e4205f6f938 100644 --- a/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp +++ b/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp @@ -14,10 +14,10 @@ #include #include +#include #include #include #include -#include #include #include From a7d46025a33137b236a387e45fe91c0dad5f893b Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 28 Apr 2022 15:05:56 +0800 Subject: [PATCH 08/42] clean --- .../Coprocessor/DAGQueryBlockInterpreter.cpp | 1 - dbms/src/TestUtils/InterpreterTestUtils.cpp | 16 ---------------- 2 files changed, 17 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 383185625b6..49116a51b43 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -850,7 +850,6 @@ void DAGQueryBlockInterpreter::executeAggregation( pipeline.streams.resize(1); // should record for agg before restore concurrency. See #3804. recordProfileStreams(pipeline, query_block.aggregation_name); - std::cout << "can restore: " << query_block.can_restore_pipeline_concurrency << std::endl; restorePipelineConcurrency(pipeline); } else diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp index 614908883e3..ebf51502860 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.cpp +++ b/dbms/src/TestUtils/InterpreterTestUtils.cpp @@ -19,22 +19,6 @@ #include namespace DB::tests { -// not necessary... -// void executeInterpreter(const std::shared_ptr & request, Context & context) -// { -// DAGContext dag_context(*request); -// dag_context.log = Logger::get("interpreterTest"); -// dag_context.setIsInterpreterTest(true); -// dag_context.setMockTableScanStreams(10); -// context.setDAGContext(&dag_context); -// // Currently, don't care about regions information in interpreter tests. -// DAGQuerySource dag(context); -// auto res = executeQuery(dag, context, false, QueryProcessingStage::Complete); -// FmtBuffer fb; -// res.in->dumpTree(fb); -// std::cout << fb.toString() << std::endl; -// } - DAGContext & InterpreterTest::getDAGContext() { assert(dag_context_ptr != nullptr); From 9a7b988aebfdf7ed10920b98e2411f61d63441de Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 28 Apr 2022 19:18:36 +0800 Subject: [PATCH 09/42] refactor --- .../MockTableScanBlockInputStream.cpp | 2 +- .../MockTableScanBlockInputStream.h | 2 +- dbms/src/Flash/Coprocessor/DAGContext.h | 35 +++++--- .../Coprocessor/DAGQueryBlockInterpreter.cpp | 56 +----------- dbms/src/Flash/Coprocessor/InterpreterDAG.cpp | 2 + dbms/src/Flash/Coprocessor/TiDBTableScan.h | 5 ++ dbms/src/Flash/tests/gtest_interpreter.cpp | 87 ++++++++++--------- dbms/src/Interpreters/executeQuery.cpp | 4 +- dbms/src/Storages/Transaction/TiDB.cpp | 27 ++++++ dbms/src/Storages/Transaction/TiDB.h | 5 ++ dbms/src/TestUtils/InterpreterTestUtils.cpp | 5 +- .../TestUtils/tests/gtest_mock_executors.cpp | 14 +-- 12 files changed, 121 insertions(+), 123 deletions(-) diff --git a/dbms/src/DataStreams/MockTableScanBlockInputStream.cpp b/dbms/src/DataStreams/MockTableScanBlockInputStream.cpp index d8c9df666a6..0405e8082db 100644 --- a/dbms/src/DataStreams/MockTableScanBlockInputStream.cpp +++ b/dbms/src/DataStreams/MockTableScanBlockInputStream.cpp @@ -32,7 +32,7 @@ MockTableScanBlockInputStream::MockTableScanBlockInputStream(ColumnsWithTypeAndN } } -ColumnPtr MockTableScanBlockInputStream::makeColumn(ColumnWithTypeAndName elem) +ColumnPtr MockTableScanBlockInputStream::makeColumn(ColumnWithTypeAndName elem) const { auto column = elem.type->createColumn(); size_t row_count = 0; diff --git a/dbms/src/DataStreams/MockTableScanBlockInputStream.h b/dbms/src/DataStreams/MockTableScanBlockInputStream.h index d148d7f3ac1..624afc195ee 100644 --- a/dbms/src/DataStreams/MockTableScanBlockInputStream.h +++ b/dbms/src/DataStreams/MockTableScanBlockInputStream.h @@ -34,7 +34,7 @@ class MockTableScanBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; - ColumnPtr makeColumn(ColumnWithTypeAndName elem); + ColumnPtr makeColumn(ColumnWithTypeAndName elem) const; }; } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index a849221f0b6..0b063cde0e0 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -14,6 +14,7 @@ #pragma once +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #ifdef __clang__ @@ -154,7 +155,6 @@ class DAGContext initExecutorIdToJoinIdMap(); initOutputInfo(); } - // for test explicit DAGContext(UInt64 max_error_count_) : dag_request(nullptr) @@ -169,6 +169,27 @@ class DAGContext , warning_count(0) {} + // for tests need to run query tasks. + explicit DAGContext(const tipb::DAGRequest & dag_request_, String log_identifier, size_t concurrency) + : dag_request(&dag_request_) + , initialize_concurrency(concurrency) + , is_mpp_task(false) + , is_root_mpp_task(false) + , tunnel_set(nullptr) + , log(Logger::get(log_identifier)) + , flags(dag_request->flags()) + , sql_mode(dag_request->sql_mode()) + , max_recorded_error_count(getMaxErrorCount(*dag_request)) + , warnings(max_recorded_error_count) + , warning_count(0) + , is_test(true) + { + assert(dag_request->has_root_executor() || dag_request->executors_size() > 0); + return_executor_id = dag_request->root_executor().has_executor_id() || dag_request->executors(0).has_executor_id(); + + initOutputInfo(); + } + void attachBlockIO(const BlockIO & io_); std::unordered_map & getProfileStreamsMap(); @@ -275,13 +296,7 @@ class DAGContext return sql_mode & f; } - void setIsInterpreterTest(bool is_interpreter_test_) { is_interpreter_test = is_interpreter_test_; } - - bool isInterpreterTest() const { return is_interpreter_test; } - - void setMockTableScanStreams(UInt32 streams) { mock_table_scan_streams = streams; } - - UInt32 mockTableScanStreams() const { return mock_table_scan_streams; } + bool isTest() const { return is_test; } void cancelAllExchangeReceiver(); @@ -295,6 +310,7 @@ class DAGContext const tipb::DAGRequest * dag_request; Int64 compile_time_ns = 0; size_t final_concurrency = 1; + size_t initialize_concurrency = 1; bool has_read_wait_index = false; Clock::time_point read_wait_index_start_timestamp{Clock::duration::zero()}; Clock::time_point read_wait_index_end_timestamp{Clock::duration::zero()}; @@ -354,8 +370,7 @@ class DAGContext /// The order of the vector is also the order of the subquery. std::vector subqueries; - bool is_interpreter_test = false; /// switch for interpreter unit test, do not use it in production. - UInt32 mock_table_scan_streams = 0; /// Used in interpreter unit test. Determine how many MockTableScanBlockInpustreams to generate. + bool is_test = false; /// switch for test, do not use it in production. }; } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 49116a51b43..a0fe69beb94 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -115,7 +115,7 @@ AnalysisResult analyzeExpressions( AnalysisResult res; ExpressionActionsChain chain; // selection on table scan had been executed in handleTableScan - if (query_block.selection && !query_block.isTableScanSource()) + if (query_block.selection && (!query_block.isTableScanSource() || context.getDAGContext()->isTest())) { std::vector where_conditions; for (const auto & c : query_block.selection->selection().conditions()) @@ -301,63 +301,15 @@ void DAGQueryBlockInterpreter::handleTableScan(const TiDBTableScan & table_scan, } } -std::pair, std::vector> getColumnsForTableScan( - const TiDBTableScan & table_scan) -{ - std::vector columms; - std::vector names_and_types; - TiDB::TableInfo table_info; - for (Int32 i = 0; i < table_scan.getColumnSize(); ++i) - { - auto const & ci = table_scan.getColumns()[i]; - TiDB::ColumnInfo column_info; - column_info.id = ci.column_id(); - column_info.tp = TiDB::TypeString; - table_info.columns.push_back(column_info); - } - for (const auto & column : table_info.columns) - { - auto type = getDataTypeByColumnInfoForComputingLayer(column); - names_and_types.push_back({"test", type}); - columms.push_back(ColumnWithTypeAndName(type, "test")); - } - return {columms, names_and_types}; -} - -// ywq todo void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline) { - // has table to read - // construct pushed down filter conditions. - std::vector conditions; - if (query_block.selection) - { - for (const auto & condition : query_block.selection->selection().conditions()) - conditions.push_back(&condition); - } - - auto columns = getColumnsForTableScan(table_scan); + auto columns = getColumnsForTableScan(table_scan.getTableScan()); analyzer = std::make_unique(std::move(columns.second), context); - auto mock_table_scan_streams = context.getDAGContext()->mockTableScanStreams(); - max_streams = mock_table_scan_streams; - for (size_t i = 0; i < mock_table_scan_streams; ++i) + for (size_t i = 0; i < context.getDAGContext()->final_concurrency; ++i) { auto mock_table_scan_stream = std::make_shared(std::get<0>(columns), context.getSettingsRef().max_block_size); pipeline.streams.emplace_back(mock_table_scan_stream); } - /// Set the limits and quota for reading data, the speed and time of the query. - setQuotaAndLimitsOnTableScan(context, pipeline); - FAIL_POINT_PAUSE(FailPoints::pause_after_copr_streams_acquired); - /// handle timezone/duration cast for local and remote table scan. ywq todo check.. - DAGStorageInterpreter storage_interpreter(context, query_block, table_scan, conditions, max_streams); - executeCastAfterTableScan(table_scan, storage_interpreter.is_need_add_cast_column, mock_table_scan_streams, pipeline); - recordProfileStreams(pipeline, query_block.source_name); - /// handle pushed down filter for local and remote table scan. - if (query_block.selection) - { - executePushedDownFilter(conditions, mock_table_scan_streams, pipeline); - recordProfileStreams(pipeline, query_block.selection_name); - } } void DAGQueryBlockInterpreter::executePushedDownFilter( @@ -1102,7 +1054,7 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) else if (query_block.isTableScanSource()) { TiDBTableScan table_scan(query_block.source, dagContext()); - if (context.getDAGContext()->isInterpreterTest()) + if (context.getDAGContext()->isTest()) handleMockTableScan(table_scan, pipeline); else handleTableScan(table_scan, pipeline); diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index ca66f923b68..c0edb2335e0 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -30,6 +30,8 @@ InterpreterDAG::InterpreterDAG(Context & context_, const DAGQuerySource & dag_) max_streams = settings.max_threads; else max_streams = 1; + if (dagContext().isTest()) + max_streams = dagContext().initialize_concurrency; if (max_streams > 1) { max_streams *= settings.max_streams_to_max_threads_ratio; diff --git a/dbms/src/Flash/Coprocessor/TiDBTableScan.h b/dbms/src/Flash/Coprocessor/TiDBTableScan.h index 31c145709a6..dd5863c7450 100644 --- a/dbms/src/Flash/Coprocessor/TiDBTableScan.h +++ b/dbms/src/Flash/Coprocessor/TiDBTableScan.h @@ -17,6 +17,7 @@ #include #include +#include "tipb/executor.pb.h" namespace DB { @@ -50,6 +51,10 @@ class TiDBTableScan { return table_scan->executor_id(); } + const tipb::Executor * getTableScan() const + { + return table_scan; + } private: const tipb::Executor * table_scan; diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 02792c05020..4755d642afa 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -55,7 +55,6 @@ Union SharedQuery ParallelAggregating Expression x 10 - Expression Filter MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); @@ -81,9 +80,8 @@ Union SharedQuery ParallelAggregating Expression x 10 - Expression - Filter - MockTableScan)"; + Filter + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } } @@ -121,17 +119,19 @@ Union .build(context); { String expected = R"( -Expression - Expression +Union + Expression x 10 Expression Expression - MergeSorting - Union - PartialSorting x 10 - Expression - Expression + SharedQuery + Expression + MergeSorting + Union + PartialSorting x 10 Expression - MockTableScan)"; + Expression + Expression + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } @@ -144,23 +144,25 @@ Expression .build(context); { String expected = R"( -Expression - Expression +Union + Expression x 10 Expression Expression - Aggregating - Concat - Expression - Expression + Expression + SharedQuery + ParallelAggregating + Expression x 10 Expression Expression - MergeSorting - Union - PartialSorting x 10 - Expression - Expression + SharedQuery + Expression + MergeSorting + Union + PartialSorting x 10 Expression - MockTableScan)"; + Expression + Expression + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } @@ -176,29 +178,34 @@ Expression .build(context); { String expected = R"( -Limit - Expression - Expression - Expression - Expression +Union + SharedQuery x 10 + Limit + Union + Limit x 10 Expression - Filter + Expression Expression Expression Expression - Aggregating - Concat + Filter + Expression Expression Expression - Expression - Expression - MergeSorting - Union - PartialSorting x 10 - Expression + SharedQuery + ParallelAggregating + Expression x 10 + Expression + Expression + SharedQuery Expression - Expression - MockTableScan)"; + MergeSorting + Union + PartialSorting x 10 + Expression + Expression + Expression + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 6c96e7c22ad..7299cd721fd 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -202,7 +202,7 @@ std::tuple executeQueryImpl( try { - if (!internal) + if (!internal && !context.getDAGContext()->isTest()) logQuery(query.substr(0, settings.log_queries_cut_to_length), context, execute_query_logger); /// Check the limits. @@ -384,7 +384,7 @@ std::tuple executeQueryImpl( } }; - if (!internal && res.in) + if (!internal && res.in && context.getDAGContext()->isTest()) { auto pipeline_log_str = [&res]() { FmtBuffer log_buffer; diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index 763dcac39fc..a6cd6881d71 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -25,6 +25,8 @@ #include #include #include +#include "Storages/Transaction/TypeMapping.h" +#include "tipb/executor.pb.h" namespace DB { @@ -76,6 +78,31 @@ Field GenDefaultField(const TiDB::ColumnInfo & col_info) throw Exception("Not implemented codec flag: " + std::to_string(col_info.getCodecFlag()), ErrorCodes::LOGICAL_ERROR); } } + +std::pair, std::vector> getColumnsForTableScan( + const tipb::Executor * table_scan) +{ + std::vector columms; + std::vector names_and_types; + TiDB::TableInfo table_info; + const auto& tipb_table_scan = table_scan->tbl_scan(); + + for (Int32 i = 0; i < tipb_table_scan.columns_size(); ++i) + { + auto const & ci = tipb_table_scan.columns(i); + TiDB::ColumnInfo column_info; + column_info.id = ci.column_id(); + column_info.tp = TiDB::TypeString; + table_info.columns.push_back(column_info); + } + for (const auto & column : table_info.columns) + { + auto type = getDataTypeByColumnInfoForComputingLayer(column); + names_and_types.push_back({"test", type}); + columms.push_back(DB::ColumnWithTypeAndName(type, "test")); + } + return {columms, names_and_types}; +} } // namespace DB namespace TiDB diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index f67bfb332c7..a8339885fe4 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -20,8 +20,11 @@ #include #include #include +#include +#include #include +#include "tipb/executor.pb.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -40,6 +43,8 @@ extern const int LOGICAL_ERROR; namespace DB { struct SchemaNameMapper; +std::pair, std::vector> getColumnsForTableScan( + const tipb::Executor * table_scan); } namespace TiDB diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp index ebf51502860..a783ff2805e 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.cpp +++ b/dbms/src/TestUtils/InterpreterTestUtils.cpp @@ -55,10 +55,7 @@ void InterpreterTest::initializeClientInfo() void InterpreterTest::executeInterpreter(const String & expected_string, const std::shared_ptr & request) { - DAGContext dag_context(*request); - dag_context.log = Logger::get("interpreterTest"); - dag_context.setIsInterpreterTest(true); - dag_context.setMockTableScanStreams(10); + DAGContext dag_context(*request, "interpreter_test", 10); // todo change the code here. context.context.setDAGContext(&dag_context); // Currently, don't care about regions information in interpreter tests. diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index 02f420dacc3..a7a7fd22b8c 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -42,14 +42,13 @@ try String expected = "table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); } - // runInterpreter(request); + request = context.scan("test_db", "test_table_1").build(context); { String expected = "table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); } - // runInterpreter(request); } CATCH @@ -61,7 +60,6 @@ try String expected = "selection_1 | equals(<0, String>, <1, String>)}\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } request = context.scan("test_db", "test_table_1") .filter(And(eq(col("s1"), col("s2")), lt(col("s2"), col("s2")))) // type in lt must be same @@ -70,7 +68,6 @@ try String expected = "selection_1 | equals(<0, Long>, <1, String>) and less(<1, String>, <1, String>)}\n" " table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } } CATCH @@ -85,7 +82,6 @@ try String expected = "project_1 | {<0, String>}\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } request = context.scan("test_db", "test_table_1") .project({col("s3"), eq(col("s1"), col("s2"))}) @@ -94,7 +90,6 @@ try String expected = "project_1 | {<2, String>, equals(<0, Long>, <1, String>)}\n" " table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } request = context.scan("test_db", "test_table_1") .project({"s1", "s2"}) @@ -103,7 +98,6 @@ try String expected = "project_1 | {<0, Long>, <1, String>}\n" " table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } } CATCH @@ -118,7 +112,6 @@ try String expected = "limit_1 | 10\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } request = context.scan("test_db", "test_table_1") .limit(lit(Field(static_cast(10)))) @@ -127,7 +120,6 @@ try String expected = "limit_1 | 10\n" " table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } } CATCH @@ -142,7 +134,6 @@ try String expected = "topn_1 | order_by: {(<0, String>, desc: false), (<1, String>, desc: true)}, limit: 10\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } request = context.scan("test_db", "test_table") .topN("s1", false, 10) @@ -166,7 +157,6 @@ try String expected = "aggregation_1 | group_by: {<1, String>}, agg_func: {max(<0, String>)}\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } request = context.scan("test_db", "test_table") @@ -176,7 +166,6 @@ try String expected = "aggregation_1 | group_by: {<1, String>, less(<0, String>, <1, String>)}, agg_func: {max(<0, String>)}\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } } CATCH @@ -206,7 +195,6 @@ try " selection_1 | equals(<0, Long>, <1, String>) and equals(<0, Long>, <1, String>)}\n" " table_scan_0 | {<0, Long>, <1, String>, <2, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // runInterpreter(request); } } CATCH From 781489efa8eb4d1c71cefca1a65e1a97651ea6b0 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 28 Apr 2022 20:50:02 +0800 Subject: [PATCH 10/42] getColumnsFromTableScan utils and clean code. --- .../Coprocessor/DAGQueryBlockInterpreter.cpp | 5 +- dbms/src/Flash/Coprocessor/TiDBTableScan.h | 3 - dbms/src/Flash/tests/gtest_interpreter.cpp | 11 ++- dbms/src/Storages/Transaction/TiDB.cpp | 86 +++++++++++++------ dbms/src/Storages/Transaction/TiDB.h | 15 ++-- 5 files changed, 77 insertions(+), 43 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index a0fe69beb94..5c4b4b9fdab 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include namespace DB @@ -303,9 +304,9 @@ void DAGQueryBlockInterpreter::handleTableScan(const TiDBTableScan & table_scan, void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline) { - auto columns = getColumnsForTableScan(table_scan.getTableScan()); + auto columns = TiDB::getColumnsFromTableScan(table_scan.getTableScan()->tbl_scan()); analyzer = std::make_unique(std::move(columns.second), context); - for (size_t i = 0; i < context.getDAGContext()->final_concurrency; ++i) + for (size_t i = 0; i < max_streams; ++i) { auto mock_table_scan_stream = std::make_shared(std::get<0>(columns), context.getSettingsRef().max_block_size); pipeline.streams.emplace_back(mock_table_scan_stream); diff --git a/dbms/src/Flash/Coprocessor/TiDBTableScan.h b/dbms/src/Flash/Coprocessor/TiDBTableScan.h index dd5863c7450..b7cbb54ebce 100644 --- a/dbms/src/Flash/Coprocessor/TiDBTableScan.h +++ b/dbms/src/Flash/Coprocessor/TiDBTableScan.h @@ -16,9 +16,6 @@ #include -#include -#include "tipb/executor.pb.h" - namespace DB { /// TiDBTableScan is a wrap to hide the difference of `TableScan` and `PartitionTableScan` diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 4755d642afa..2a445d93150 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -55,8 +55,8 @@ Union SharedQuery ParallelAggregating Expression x 10 - Filter - MockTableScan)"; + Filter + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } @@ -244,10 +244,9 @@ CreatingSets Union Expression x 10 Expression - Expression - HashJoinProbe - Expression - MockTableScan)"; + HashJoinProbe + Expression + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); } } diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index a6cd6881d71..fede8ff7c68 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -25,8 +25,7 @@ #include #include #include -#include "Storages/Transaction/TypeMapping.h" -#include "tipb/executor.pb.h" +#include namespace DB { @@ -78,31 +77,6 @@ Field GenDefaultField(const TiDB::ColumnInfo & col_info) throw Exception("Not implemented codec flag: " + std::to_string(col_info.getCodecFlag()), ErrorCodes::LOGICAL_ERROR); } } - -std::pair, std::vector> getColumnsForTableScan( - const tipb::Executor * table_scan) -{ - std::vector columms; - std::vector names_and_types; - TiDB::TableInfo table_info; - const auto& tipb_table_scan = table_scan->tbl_scan(); - - for (Int32 i = 0; i < tipb_table_scan.columns_size(); ++i) - { - auto const & ci = tipb_table_scan.columns(i); - TiDB::ColumnInfo column_info; - column_info.id = ci.column_id(); - column_info.tp = TiDB::TypeString; - table_info.columns.push_back(column_info); - } - for (const auto & column : table_info.columns) - { - auto type = getDataTypeByColumnInfoForComputingLayer(column); - names_and_types.push_back({"test", type}); - columms.push_back(DB::ColumnWithTypeAndName(type, "test")); - } - return {columms, names_and_types}; -} } // namespace DB namespace TiDB @@ -1130,4 +1104,62 @@ ColumnInfo fieldTypeToColumnInfo(const tipb::FieldType & field_type) return ret; } +ColumnsWithTypeAndName getColumnWithTypeAndNameFromTableScan(const tipb::TableScan & table_scan) +{ + std::vector column_with_type_and_names; + column_with_type_and_names.reserve(table_scan.columns_size()); + for (Int32 i = 0; i < table_scan.columns_size(); ++i) + { + String name = "mock_table_scan_" + std::to_string(i); + auto const & ci = table_scan.columns(i); + TiDB::ColumnInfo column_info; + column_info.id = ci.column_id(); + column_info.tp = static_cast(ci.tp()); + column_info.name = name; + auto type = DB::getDataTypeByColumnInfoForComputingLayer(column_info); + column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(type, name)); + } + return column_with_type_and_names; +} + +NamesAndTypes getNamesAndTypeFromTableScan(const tipb::TableScan & table_scan) +{ + NamesAndTypes names_and_types; + names_and_types.reserve(table_scan.columns_size()); + + for (Int32 i = 0; i < table_scan.columns_size(); ++i) + { + String name = "mock_table_scan_" + std::to_string(i); + TiDB::ColumnInfo column_info; + auto const & ci = table_scan.columns(i); + column_info.tp = static_cast(ci.tp()); + column_info.id = ci.column_id(); + auto type = DB::getDataTypeByColumnInfoForComputingLayer(column_info); + names_and_types.push_back({name, type}); + } + return names_and_types; +} + +std::pair getColumnsFromTableScan( + const tipb::TableScan & table_scan) +{ + ColumnsWithTypeAndName column_with_type_and_names; + NamesAndTypes names_and_types; + column_with_type_and_names.reserve(table_scan.columns_size()); + names_and_types.reserve(table_scan.columns_size()); + + for (Int32 i = 0; i < table_scan.columns_size(); ++i) + { + String name = "mock_table_scan_" + std::to_string(i); + auto const & ci = table_scan.columns(i); + TiDB::ColumnInfo column_info; + column_info.id = ci.column_id(); + column_info.tp = static_cast(ci.tp()); + auto type = DB::getDataTypeByColumnInfoForComputingLayer(column_info); + names_and_types.push_back({name, type}); + column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(type, name)); + } + return {column_with_type_and_names, names_and_types}; +} + } // namespace TiDB diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index a8339885fe4..962b2e6695c 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -14,17 +14,17 @@ #pragma once +#include #include +#include #include #include #include #include #include -#include -#include +#include #include -#include "tipb/executor.pb.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -43,8 +43,6 @@ extern const int LOGICAL_ERROR; namespace DB { struct SchemaNameMapper; -std::pair, std::vector> getColumnsForTableScan( - const tipb::Executor * table_scan); } namespace TiDB @@ -403,4 +401,11 @@ String genJsonNull(); tipb::FieldType columnInfoToFieldType(const ColumnInfo & ci); ColumnInfo fieldTypeToColumnInfo(const tipb::FieldType & field_type); +using ColumnsWithTypeAndName = DB::ColumnsWithTypeAndName; +using NamesAndTypes = DB::NamesAndTypes; +std::pair getColumnsFromTableScan( + const tipb::TableScan & table_scan); +NamesAndTypes getNamesAndTypeFromTableScan(const tipb::TableScan & table_scan); +ColumnsWithTypeAndName getColumnWithTypeAndNameFromTableScan(const tipb::TableScan & table_scan); + } // namespace TiDB From 283639eaa9eb9f7f73216449d936de97ac6d5999 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 29 Apr 2022 10:59:59 +0800 Subject: [PATCH 11/42] refactor gencolumn related functions. --- dbms/src/Flash/Coprocessor/DAGContext.h | 1 - .../Coprocessor/DAGQueryBlockInterpreter.cpp | 7 +-- dbms/src/Interpreters/executeQuery.cpp | 2 +- dbms/src/Storages/Transaction/TiDB.cpp | 50 +++++++++---------- dbms/src/Storages/Transaction/TiDB.h | 11 ++-- 5 files changed, 37 insertions(+), 34 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index 0b063cde0e0..ca59eca6122 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -14,7 +14,6 @@ #pragma once -#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #ifdef __clang__ diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 5c4b4b9fdab..02b0ebdd5b6 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -304,11 +304,12 @@ void DAGQueryBlockInterpreter::handleTableScan(const TiDBTableScan & table_scan, void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline) { - auto columns = TiDB::getColumnsFromTableScan(table_scan.getTableScan()->tbl_scan()); - analyzer = std::make_unique(std::move(columns.second), context); + auto names_and_types = TiDB::genNamesAndTypesFromTableScan(table_scan.getTableScan()->tbl_scan()); + auto columns_with_type_and_name = TiDB::getColumnWithTypeAndName(names_and_types); + analyzer = std::make_unique(std::move(names_and_types), context); for (size_t i = 0; i < max_streams; ++i) { - auto mock_table_scan_stream = std::make_shared(std::get<0>(columns), context.getSettingsRef().max_block_size); + auto mock_table_scan_stream = std::make_shared(columns_with_type_and_name, context.getSettingsRef().max_block_size); pipeline.streams.emplace_back(mock_table_scan_stream); } } diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 7299cd721fd..e3414770f45 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -384,7 +384,7 @@ std::tuple executeQueryImpl( } }; - if (!internal && res.in && context.getDAGContext()->isTest()) + if (!internal && res.in && !context.getDAGContext()->isTest()) { auto pipeline_log_str = [&res]() { FmtBuffer log_buffer; diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index fede8ff7c68..0ab394f0f57 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -1104,25 +1104,24 @@ ColumnInfo fieldTypeToColumnInfo(const tipb::FieldType & field_type) return ret; } -ColumnsWithTypeAndName getColumnWithTypeAndNameFromTableScan(const tipb::TableScan & table_scan) +DAGSchema genSchemaFromTableScan(const tipb::TableScan & table_scan) { - std::vector column_with_type_and_names; - column_with_type_and_names.reserve(table_scan.columns_size()); + DAGSchema schema; + schema.reserve(table_scan.columns_size()); for (Int32 i = 0; i < table_scan.columns_size(); ++i) { String name = "mock_table_scan_" + std::to_string(i); - auto const & ci = table_scan.columns(i); TiDB::ColumnInfo column_info; - column_info.id = ci.column_id(); + auto const & ci = table_scan.columns(i); column_info.tp = static_cast(ci.tp()); + column_info.id = ci.column_id(); column_info.name = name; - auto type = DB::getDataTypeByColumnInfoForComputingLayer(column_info); - column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(type, name)); + schema.push_back({name, column_info}); } - return column_with_type_and_names; + return schema; } -NamesAndTypes getNamesAndTypeFromTableScan(const tipb::TableScan & table_scan) +NamesAndTypes genNamesAndTypesFromTableScan(const tipb::TableScan & table_scan) { NamesAndTypes names_and_types; names_and_types.reserve(table_scan.columns_size()); @@ -1140,26 +1139,27 @@ NamesAndTypes getNamesAndTypeFromTableScan(const tipb::TableScan & table_scan) return names_and_types; } -std::pair getColumnsFromTableScan( - const tipb::TableScan & table_scan) +ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema) { - ColumnsWithTypeAndName column_with_type_and_names; - NamesAndTypes names_and_types; - column_with_type_and_names.reserve(table_scan.columns_size()); - names_and_types.reserve(table_scan.columns_size()); + std::vector column_with_type_and_names; + column_with_type_and_names.reserve(schema.size()); + for (const auto & col : schema) + { + auto type = DB::getDataTypeByColumnInfoForComputingLayer(col.second); + column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(type, col.first)); + } + return column_with_type_and_names; +} - for (Int32 i = 0; i < table_scan.columns_size(); ++i) +ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types) +{ + std::vector column_with_type_and_names; + column_with_type_and_names.reserve(names_and_types.size()); + for (const auto & col : names_and_types) { - String name = "mock_table_scan_" + std::to_string(i); - auto const & ci = table_scan.columns(i); - TiDB::ColumnInfo column_info; - column_info.id = ci.column_id(); - column_info.tp = static_cast(ci.tp()); - auto type = DB::getDataTypeByColumnInfoForComputingLayer(column_info); - names_and_types.push_back({name, type}); - column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(type, name)); + column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(col.type, col.name)); } - return {column_with_type_and_names, names_and_types}; + return column_with_type_and_names; } } // namespace TiDB diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index 962b2e6695c..6207ba62b4f 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -403,9 +403,12 @@ ColumnInfo fieldTypeToColumnInfo(const tipb::FieldType & field_type); using ColumnsWithTypeAndName = DB::ColumnsWithTypeAndName; using NamesAndTypes = DB::NamesAndTypes; -std::pair getColumnsFromTableScan( - const tipb::TableScan & table_scan); -NamesAndTypes getNamesAndTypeFromTableScan(const tipb::TableScan & table_scan); -ColumnsWithTypeAndName getColumnWithTypeAndNameFromTableScan(const tipb::TableScan & table_scan); +using DAGColumnInfo = std::pair; +using DAGSchema = std::vector; + +DAGSchema genSchemaFromTableScan(const tipb::TableScan & table_scan); +NamesAndTypes genNamesAndTypesFromTableScan(const tipb::TableScan & table_scan); +ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema); +ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types); } // namespace TiDB From 7dbd5c577ce546d4e0c4bbf8fe645332ec9fa957 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 29 Apr 2022 11:01:20 +0800 Subject: [PATCH 12/42] remove useless cout. --- dbms/src/TestUtils/InterpreterTestUtils.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp index a783ff2805e..6ce72014788 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.cpp +++ b/dbms/src/TestUtils/InterpreterTestUtils.cpp @@ -63,7 +63,6 @@ void InterpreterTest::executeInterpreter(const String & expected_string, const s auto res = executeQuery(dag, context.context, false, QueryProcessingStage::Complete); FmtBuffer fb; res.in->dumpTree(fb); - std::cout << fb.toString() << std::endl; ASSERT_EQ(Poco::trim(expected_string), Poco::trim(fb.toString())); } From 120a983d139315930237faadc0ff3fdb9e23f527 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 29 Apr 2022 11:10:18 +0800 Subject: [PATCH 13/42] all user choose stream size. --- dbms/src/Flash/tests/gtest_interpreter.cpp | 14 +++++++------- dbms/src/TestUtils/InterpreterTestUtils.cpp | 4 ++-- dbms/src/TestUtils/InterpreterTestUtils.h | 6 ++---- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 2a445d93150..961ab525be8 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -57,7 +57,7 @@ Union Expression x 10 Filter MockTableScan)"; - ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } request = context.scan("test_db", "test_table_1") @@ -82,7 +82,7 @@ Union Expression x 10 Filter MockTableScan)"; - ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } } CATCH @@ -109,7 +109,7 @@ Union Expression Expression MockTableScan)"; - ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } request = context.scan("test_db", "test_table_1") @@ -132,7 +132,7 @@ Union Expression Expression MockTableScan)"; - ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } request = context.scan("test_db", "test_table_1") @@ -163,7 +163,7 @@ Union Expression Expression MockTableScan)"; - ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } request = context.scan("test_db", "test_table_1") @@ -206,7 +206,7 @@ Union Expression Expression MockTableScan)"; - ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } // Join Source. @@ -247,7 +247,7 @@ CreatingSets HashJoinProbe Expression MockTableScan)"; - ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request); + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } } CATCH diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp index 6ce72014788..e74c02d0cbf 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.cpp +++ b/dbms/src/TestUtils/InterpreterTestUtils.cpp @@ -53,9 +53,9 @@ void InterpreterTest::initializeClientInfo() client_info.interface = ClientInfo::Interface::GRPC; } -void InterpreterTest::executeInterpreter(const String & expected_string, const std::shared_ptr & request) +void InterpreterTest::executeInterpreter(const String & expected_string, const std::shared_ptr & request, size_t concurrency) { - DAGContext dag_context(*request, "interpreter_test", 10); + DAGContext dag_context(*request, "interpreter_test", concurrency); // todo change the code here. context.context.setDAGContext(&dag_context); // Currently, don't care about regions information in interpreter tests. diff --git a/dbms/src/TestUtils/InterpreterTestUtils.h b/dbms/src/TestUtils/InterpreterTestUtils.h index 96eba2cdabd..28d44d3a5f2 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.h +++ b/dbms/src/TestUtils/InterpreterTestUtils.h @@ -50,7 +50,7 @@ class InterpreterTest : public ::testing::Test static void dagRequestEqual(const String & expected_string, const std::shared_ptr & actual); - void executeInterpreter(const String & expected_string, const std::shared_ptr & request); + void executeInterpreter(const String & expected_string, const std::shared_ptr & request, size_t concurrency); protected: MockDAGRequestContext context; @@ -58,7 +58,5 @@ class InterpreterTest : public ::testing::Test }; #define ASSERT_DAGREQUEST_EQAUL(str, request) dagRequestEqual((str), (request)); -#define ASSERT_BLOCKINPUTSTREAM_EQAUL(str, request) executeInterpreter((str), (request)) - -// #define xxxx +#define ASSERT_BLOCKINPUTSTREAM_EQAUL(str, request, concurrency) executeInterpreter((str), (request), (concurrency)) } // namespace DB::tests \ No newline at end of file From 0ce36686acf9a2d6fef3a978d1617ffa7360b4c0 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 29 Apr 2022 15:12:19 +0800 Subject: [PATCH 14/42] address comments. --- dbms/src/Core/ColumnsWithTypeAndName.h | 2 - dbms/src/Flash/Coprocessor/DAGContext.h | 1 + .../Coprocessor/DAGQueryBlockInterpreter.cpp | 6 +- dbms/src/Flash/Coprocessor/GenSchema.cpp | 76 +++++++++++++++++++ dbms/src/Flash/Coprocessor/GenSchema.h | 30 ++++++++ dbms/src/Flash/Coprocessor/InterpreterDAG.cpp | 5 +- dbms/src/Storages/Transaction/TiDB.cpp | 59 -------------- dbms/src/Storages/Transaction/TiDB.h | 14 ---- .../TestUtils/tests/gtest_mock_executors.cpp | 1 - 9 files changed, 114 insertions(+), 80 deletions(-) create mode 100644 dbms/src/Flash/Coprocessor/GenSchema.cpp create mode 100644 dbms/src/Flash/Coprocessor/GenSchema.h diff --git a/dbms/src/Core/ColumnsWithTypeAndName.h b/dbms/src/Core/ColumnsWithTypeAndName.h index 61c77cf161e..71a1a7da33c 100644 --- a/dbms/src/Core/ColumnsWithTypeAndName.h +++ b/dbms/src/Core/ColumnsWithTypeAndName.h @@ -18,9 +18,7 @@ #include - namespace DB { using ColumnsWithTypeAndName = std::vector; - } diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index ca59eca6122..4874641abb8 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -166,6 +166,7 @@ class DAGContext , max_recorded_error_count(max_error_count_) , warnings(max_recorded_error_count) , warning_count(0) + , is_test(true) {} // for tests need to run query tasks. diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 02b0ebdd5b6..a9bed8c48f6 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -116,6 +117,7 @@ AnalysisResult analyzeExpressions( AnalysisResult res; ExpressionActionsChain chain; // selection on table scan had been executed in handleTableScan + // In test mode, filter is not pushed down to table scan if (query_block.selection && (!query_block.isTableScanSource() || context.getDAGContext()->isTest())) { std::vector where_conditions; @@ -304,8 +306,8 @@ void DAGQueryBlockInterpreter::handleTableScan(const TiDBTableScan & table_scan, void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline) { - auto names_and_types = TiDB::genNamesAndTypesFromTableScan(table_scan.getTableScan()->tbl_scan()); - auto columns_with_type_and_name = TiDB::getColumnWithTypeAndName(names_and_types); + auto names_and_types = genNamesAndTypesFromTableScan(table_scan.getTableScan()->tbl_scan()); + auto columns_with_type_and_name = getColumnWithTypeAndName(names_and_types); analyzer = std::make_unique(std::move(names_and_types), context); for (size_t i = 0; i < max_streams; ++i) { diff --git a/dbms/src/Flash/Coprocessor/GenSchema.cpp b/dbms/src/Flash/Coprocessor/GenSchema.cpp new file mode 100644 index 00000000000..4216d1bd75a --- /dev/null +++ b/dbms/src/Flash/Coprocessor/GenSchema.cpp @@ -0,0 +1,76 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include + +namespace DB +{ +ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema) +{ + std::vector column_with_type_and_names; + column_with_type_and_names.reserve(schema.size()); + for (const auto & col : schema) + { + auto type = getDataTypeByColumnInfoForComputingLayer(col.second); + column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(type, col.first)); + } + return column_with_type_and_names; +} + +ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types) +{ + std::vector column_with_type_and_names; + column_with_type_and_names.reserve(names_and_types.size()); + for (const auto & col : names_and_types) + { + column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(col.type, col.name)); + } + return column_with_type_and_names; +} + +DAGSchema genSchemaFromTableScan(const tipb::TableScan & table_scan) +{ + DAGSchema schema; + schema.reserve(table_scan.columns_size()); + for (Int32 i = 0; i < table_scan.columns_size(); ++i) + { + String name = "mock_table_scan_" + std::to_string(i); + TiDB::ColumnInfo column_info; + auto const & ci = table_scan.columns(i); + column_info.tp = static_cast(ci.tp()); + column_info.id = ci.column_id(); + column_info.name = name; + schema.push_back({name, column_info}); + } + return schema; +} + +NamesAndTypes genNamesAndTypesFromTableScan(const tipb::TableScan & table_scan) +{ + NamesAndTypes names_and_types; + names_and_types.reserve(table_scan.columns_size()); + + for (Int32 i = 0; i < table_scan.columns_size(); ++i) + { + String name = "mock_table_scan_" + std::to_string(i); + TiDB::ColumnInfo column_info; + auto const & ci = table_scan.columns(i); + column_info.tp = static_cast(ci.tp()); + column_info.id = ci.column_id(); + auto type = getDataTypeByColumnInfoForComputingLayer(column_info); + names_and_types.push_back({name, type}); + } + return names_and_types; +} +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Flash/Coprocessor/GenSchema.h b/dbms/src/Flash/Coprocessor/GenSchema.h new file mode 100644 index 00000000000..64e25f6db27 --- /dev/null +++ b/dbms/src/Flash/Coprocessor/GenSchema.h @@ -0,0 +1,30 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +namespace DB +{ +using DAGColumnInfo = std::pair; +using DAGSchema = std::vector; +NamesAndTypes genNamesAndTypesFromTableScan(const tipb::TableScan & table_scan); +DAGSchema genSchemaFromTableScan(const tipb::TableScan & table_scan); +ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema); +ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types); +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index c0edb2335e0..b7c75c06e67 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -28,10 +28,11 @@ InterpreterDAG::InterpreterDAG(Context & context_, const DAGQuerySource & dag_) const Settings & settings = context.getSettingsRef(); if (dagContext().isBatchCop() || dagContext().isMPPTask()) max_streams = settings.max_threads; + else if (dagContext().isTest()) + max_streams = dagContext().initialize_concurrency; else max_streams = 1; - if (dagContext().isTest()) - max_streams = dagContext().initialize_concurrency; + if (max_streams > 1) { max_streams *= settings.max_streams_to_max_threads_ratio; diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index 0ab394f0f57..6ca5885b1f7 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -1103,63 +1103,4 @@ ColumnInfo fieldTypeToColumnInfo(const tipb::FieldType & field_type) } return ret; } - -DAGSchema genSchemaFromTableScan(const tipb::TableScan & table_scan) -{ - DAGSchema schema; - schema.reserve(table_scan.columns_size()); - for (Int32 i = 0; i < table_scan.columns_size(); ++i) - { - String name = "mock_table_scan_" + std::to_string(i); - TiDB::ColumnInfo column_info; - auto const & ci = table_scan.columns(i); - column_info.tp = static_cast(ci.tp()); - column_info.id = ci.column_id(); - column_info.name = name; - schema.push_back({name, column_info}); - } - return schema; -} - -NamesAndTypes genNamesAndTypesFromTableScan(const tipb::TableScan & table_scan) -{ - NamesAndTypes names_and_types; - names_and_types.reserve(table_scan.columns_size()); - - for (Int32 i = 0; i < table_scan.columns_size(); ++i) - { - String name = "mock_table_scan_" + std::to_string(i); - TiDB::ColumnInfo column_info; - auto const & ci = table_scan.columns(i); - column_info.tp = static_cast(ci.tp()); - column_info.id = ci.column_id(); - auto type = DB::getDataTypeByColumnInfoForComputingLayer(column_info); - names_and_types.push_back({name, type}); - } - return names_and_types; -} - -ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema) -{ - std::vector column_with_type_and_names; - column_with_type_and_names.reserve(schema.size()); - for (const auto & col : schema) - { - auto type = DB::getDataTypeByColumnInfoForComputingLayer(col.second); - column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(type, col.first)); - } - return column_with_type_and_names; -} - -ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types) -{ - std::vector column_with_type_and_names; - column_with_type_and_names.reserve(names_and_types.size()); - for (const auto & col : names_and_types) - { - column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(col.type, col.name)); - } - return column_with_type_and_names; -} - } // namespace TiDB diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index 6207ba62b4f..df3d4bb41eb 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -14,15 +14,12 @@ #pragma once -#include #include -#include #include #include #include #include #include -#include #include @@ -400,15 +397,4 @@ String genJsonNull(); tipb::FieldType columnInfoToFieldType(const ColumnInfo & ci); ColumnInfo fieldTypeToColumnInfo(const tipb::FieldType & field_type); - -using ColumnsWithTypeAndName = DB::ColumnsWithTypeAndName; -using NamesAndTypes = DB::NamesAndTypes; -using DAGColumnInfo = std::pair; -using DAGSchema = std::vector; - -DAGSchema genSchemaFromTableScan(const tipb::TableScan & table_scan); -NamesAndTypes genNamesAndTypesFromTableScan(const tipb::TableScan & table_scan); -ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema); -ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types); - } // namespace TiDB diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index a7a7fd22b8c..6dbf791669f 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -142,7 +142,6 @@ try String expected = "topn_1 | order_by: {(<0, String>, desc: false)}, limit: 10\n" " table_scan_0 | {<0, String>, <1, String>}\n"; ASSERT_DAGREQUEST_EQAUL(expected, request); - // executeInterpreter(request, context.context); } } CATCH From 02f3509b017e06d04c9a622006b2623525fe1d94 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 29 Apr 2022 16:29:40 +0800 Subject: [PATCH 15/42] rename. --- dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp | 2 +- .../Flash/Coprocessor/{GenSchema.cpp => GenSchemaAndColumn.cpp} | 2 +- .../src/Flash/Coprocessor/{GenSchema.h => GenSchemaAndColumn.h} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename dbms/src/Flash/Coprocessor/{GenSchema.cpp => GenSchemaAndColumn.cpp} (98%) rename dbms/src/Flash/Coprocessor/{GenSchema.h => GenSchemaAndColumn.h} (100%) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index a9bed8c48f6..07afd471ed0 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -39,7 +39,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Flash/Coprocessor/GenSchema.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp similarity index 98% rename from dbms/src/Flash/Coprocessor/GenSchema.cpp rename to dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index 4216d1bd75a..c57c7860bd4 100644 --- a/dbms/src/Flash/Coprocessor/GenSchema.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include namespace DB diff --git a/dbms/src/Flash/Coprocessor/GenSchema.h b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h similarity index 100% rename from dbms/src/Flash/Coprocessor/GenSchema.h rename to dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h From 3dad3726bf6bc19d576094a9322073425020c940 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 5 May 2022 10:38:13 +0800 Subject: [PATCH 16/42] rename. --- .../Coprocessor/DAGQueryBlockInterpreter.cpp | 2 +- .../Flash/Coprocessor/GenSchemaAndColumn.cpp | 80 ++++++++++++++----- .../Flash/Coprocessor/GenSchemaAndColumn.h | 7 +- dbms/src/Storages/Transaction/TiDB.cpp | 2 +- dbms/src/Storages/Transaction/TiDB.h | 1 + 5 files changed, 66 insertions(+), 26 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 07afd471ed0..f99f76b4411 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -306,7 +306,7 @@ void DAGQueryBlockInterpreter::handleTableScan(const TiDBTableScan & table_scan, void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline) { - auto names_and_types = genNamesAndTypesFromTableScan(table_scan.getTableScan()->tbl_scan()); + auto names_and_types = genNamesAndTypes(table_scan.getTableScan()->tbl_scan()); auto columns_with_type_and_name = getColumnWithTypeAndName(names_and_types); analyzer = std::make_unique(std::move(names_and_types), context); for (size_t i = 0; i < max_streams; ++i) diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index c57c7860bd4..d2e2ed165b3 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -13,41 +13,54 @@ // limitations under the License. #include #include - namespace DB { -ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema) +// unused +DAGSchema genSchema(const tipb::TableScan & table_scan) { - std::vector column_with_type_and_names; - column_with_type_and_names.reserve(schema.size()); - for (const auto & col : schema) + DAGSchema schema; + schema.reserve(table_scan.columns_size()); + for (Int32 i = 0; i < table_scan.columns_size(); ++i) { - auto type = getDataTypeByColumnInfoForComputingLayer(col.second); - column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(type, col.first)); + String name = "mock_table_scan_" + std::to_string(i); + TiDB::ColumnInfo column_info; + auto const & ci = table_scan.columns(i); + column_info.tp = static_cast(ci.tp()); + column_info.id = ci.column_id(); + column_info.name = name; + schema.push_back({name, column_info}); } - return column_with_type_and_names; + return schema; } -ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types) +NamesAndTypes genNamesAndTypes(const tipb::TableScan & table_scan) { - std::vector column_with_type_and_names; - column_with_type_and_names.reserve(names_and_types.size()); - for (const auto & col : names_and_types) + NamesAndTypes names_and_types; + names_and_types.reserve(table_scan.columns_size()); + + for (Int32 i = 0; i < table_scan.columns_size(); ++i) { - column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(col.type, col.name)); + String name = "mock_table_scan_" + std::to_string(i); + TiDB::ColumnInfo column_info; + table_scan.columns(); + auto const & ci = table_scan.columns(i); + column_info.tp = static_cast(ci.tp()); + column_info.id = ci.column_id(); + auto type = getDataTypeByColumnInfoForComputingLayer(column_info); + names_and_types.push_back({name, type}); } - return column_with_type_and_names; + return names_and_types; } -DAGSchema genSchemaFromTableScan(const tipb::TableScan & table_scan) +DAGSchema genSchema(::google::protobuf::RepeatedPtrField<::tipb::ColumnInfo> column_infos) { DAGSchema schema; - schema.reserve(table_scan.columns_size()); - for (Int32 i = 0; i < table_scan.columns_size(); ++i) + schema.reserve(column_infos.size()); + for (Int32 i = 0; i < column_infos.size(); ++i) { String name = "mock_table_scan_" + std::to_string(i); TiDB::ColumnInfo column_info; - auto const & ci = table_scan.columns(i); + auto const & ci = column_infos[i]; column_info.tp = static_cast(ci.tp()); column_info.id = ci.column_id(); column_info.name = name; @@ -56,16 +69,16 @@ DAGSchema genSchemaFromTableScan(const tipb::TableScan & table_scan) return schema; } -NamesAndTypes genNamesAndTypesFromTableScan(const tipb::TableScan & table_scan) +NamesAndTypes genNamesAndTypes(::google::protobuf::RepeatedPtrField<::tipb::ColumnInfo> column_infos) { NamesAndTypes names_and_types; - names_and_types.reserve(table_scan.columns_size()); + names_and_types.reserve(column_infos.size()); - for (Int32 i = 0; i < table_scan.columns_size(); ++i) + for (Int32 i = 0; i < column_infos.size(); ++i) { String name = "mock_table_scan_" + std::to_string(i); TiDB::ColumnInfo column_info; - auto const & ci = table_scan.columns(i); + auto const & ci = column_infos[i]; column_info.tp = static_cast(ci.tp()); column_info.id = ci.column_id(); auto type = getDataTypeByColumnInfoForComputingLayer(column_info); @@ -73,4 +86,27 @@ NamesAndTypes genNamesAndTypesFromTableScan(const tipb::TableScan & table_scan) } return names_and_types; } + +ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema) +{ + std::vector column_with_type_and_names; + column_with_type_and_names.reserve(schema.size()); + for (const auto & col : schema) + { + auto type = getDataTypeByColumnInfoForComputingLayer(col.second); + column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(type, col.first)); + } + return column_with_type_and_names; +} + +ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types) +{ + std::vector column_with_type_and_names; + column_with_type_and_names.reserve(names_and_types.size()); + for (const auto & col : names_and_types) + { + column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(col.type, col.name)); + } + return column_with_type_and_names; +} } // namespace DB \ No newline at end of file diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h index 64e25f6db27..e8c4ff9c25e 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h @@ -23,8 +23,11 @@ namespace DB { using DAGColumnInfo = std::pair; using DAGSchema = std::vector; -NamesAndTypes genNamesAndTypesFromTableScan(const tipb::TableScan & table_scan); -DAGSchema genSchemaFromTableScan(const tipb::TableScan & table_scan); +NamesAndTypes genNamesAndTypes(const tipb::TableScan & table_scan); +DAGSchema genSchema(const tipb::TableScan & table_scan); +NamesAndTypes genNamesAndTypes(::google::protobuf::RepeatedPtrField<::tipb::ColumnInfo> & column_infos); +DAGSchema genSchema(::google::protobuf::RepeatedPtrField<::tipb::ColumnInfo> & column_infos); ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema); ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types); + } // namespace DB \ No newline at end of file diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index 6ca5885b1f7..763dcac39fc 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -25,7 +25,6 @@ #include #include #include -#include namespace DB { @@ -1103,4 +1102,5 @@ ColumnInfo fieldTypeToColumnInfo(const tipb::FieldType & field_type) } return ret; } + } // namespace TiDB diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index df3d4bb41eb..f67bfb332c7 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -397,4 +397,5 @@ String genJsonNull(); tipb::FieldType columnInfoToFieldType(const ColumnInfo & ci); ColumnInfo fieldTypeToColumnInfo(const tipb::FieldType & field_type); + } // namespace TiDB From 4910d32a7c78b0113394fe41d5b3d99090395dbe Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 5 May 2022 11:12:31 +0800 Subject: [PATCH 17/42] address comments. --- dbms/src/Core/ColumnsWithTypeAndName.h | 1 + .../Coprocessor/DAGQueryBlockInterpreter.cpp | 5 +- .../Flash/Coprocessor/GenSchemaAndColumn.cpp | 58 ++++--------------- .../Flash/Coprocessor/GenSchemaAndColumn.h | 8 +-- dbms/src/Flash/Coprocessor/TiDBTableScan.h | 4 -- dbms/src/Interpreters/executeQuery.cpp | 4 +- 6 files changed, 20 insertions(+), 60 deletions(-) diff --git a/dbms/src/Core/ColumnsWithTypeAndName.h b/dbms/src/Core/ColumnsWithTypeAndName.h index 71a1a7da33c..7bcf8649589 100644 --- a/dbms/src/Core/ColumnsWithTypeAndName.h +++ b/dbms/src/Core/ColumnsWithTypeAndName.h @@ -18,6 +18,7 @@ #include + namespace DB { using ColumnsWithTypeAndName = std::vector; diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index f99f76b4411..1d19123bdf1 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -304,9 +304,10 @@ void DAGQueryBlockInterpreter::handleTableScan(const TiDBTableScan & table_scan, } } +// for tests, we need to mock tableScan blockInputStream as the source stream. void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline) { - auto names_and_types = genNamesAndTypes(table_scan.getTableScan()->tbl_scan()); + auto names_and_types = genNamesAndTypes(table_scan); auto columns_with_type_and_name = getColumnWithTypeAndName(names_and_types); analyzer = std::make_unique(std::move(names_and_types), context); for (size_t i = 0; i < max_streams; ++i) @@ -1058,7 +1059,7 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) else if (query_block.isTableScanSource()) { TiDBTableScan table_scan(query_block.source, dagContext()); - if (context.getDAGContext()->isTest()) + if (dagContext().isTest()) handleMockTableScan(table_scan, pipeline); else handleTableScan(table_scan, pipeline); diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index d2e2ed165b3..04e772b334a 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -13,18 +13,18 @@ // limitations under the License. #include #include +#include namespace DB { -// unused -DAGSchema genSchema(const tipb::TableScan & table_scan) +DAGSchema genSchema(const TiDBTableScan & table_scan) { DAGSchema schema; - schema.reserve(table_scan.columns_size()); - for (Int32 i = 0; i < table_scan.columns_size(); ++i) + schema.reserve(table_scan.getColumnSize()); + for (Int32 i = 0; i < table_scan.getColumnSize(); ++i) { - String name = "mock_table_scan_" + std::to_string(i); + String name = fmt::format("mock_table_scan_{}", i); TiDB::ColumnInfo column_info; - auto const & ci = table_scan.columns(i); + auto const & ci = table_scan.getColumns()[i]; column_info.tp = static_cast(ci.tp()); column_info.id = ci.column_id(); column_info.name = name; @@ -33,52 +33,16 @@ DAGSchema genSchema(const tipb::TableScan & table_scan) return schema; } -NamesAndTypes genNamesAndTypes(const tipb::TableScan & table_scan) +NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan) { NamesAndTypes names_and_types; - names_and_types.reserve(table_scan.columns_size()); + names_and_types.reserve(table_scan.getColumnSize()); - for (Int32 i = 0; i < table_scan.columns_size(); ++i) + for (Int32 i = 0; i < table_scan.getColumnSize(); ++i) { - String name = "mock_table_scan_" + std::to_string(i); + String name = fmt::format("mock_table_scan_{}", i); TiDB::ColumnInfo column_info; - table_scan.columns(); - auto const & ci = table_scan.columns(i); - column_info.tp = static_cast(ci.tp()); - column_info.id = ci.column_id(); - auto type = getDataTypeByColumnInfoForComputingLayer(column_info); - names_and_types.push_back({name, type}); - } - return names_and_types; -} - -DAGSchema genSchema(::google::protobuf::RepeatedPtrField<::tipb::ColumnInfo> column_infos) -{ - DAGSchema schema; - schema.reserve(column_infos.size()); - for (Int32 i = 0; i < column_infos.size(); ++i) - { - String name = "mock_table_scan_" + std::to_string(i); - TiDB::ColumnInfo column_info; - auto const & ci = column_infos[i]; - column_info.tp = static_cast(ci.tp()); - column_info.id = ci.column_id(); - column_info.name = name; - schema.push_back({name, column_info}); - } - return schema; -} - -NamesAndTypes genNamesAndTypes(::google::protobuf::RepeatedPtrField<::tipb::ColumnInfo> column_infos) -{ - NamesAndTypes names_and_types; - names_and_types.reserve(column_infos.size()); - - for (Int32 i = 0; i < column_infos.size(); ++i) - { - String name = "mock_table_scan_" + std::to_string(i); - TiDB::ColumnInfo column_info; - auto const & ci = column_infos[i]; + auto const & ci = table_scan.getColumns()[i]; column_info.tp = static_cast(ci.tp()); column_info.id = ci.column_id(); auto type = getDataTypeByColumnInfoForComputingLayer(column_info); diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h index e8c4ff9c25e..72bb6540f5b 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h @@ -16,17 +16,15 @@ #include #include +#include #include -#include namespace DB { using DAGColumnInfo = std::pair; using DAGSchema = std::vector; -NamesAndTypes genNamesAndTypes(const tipb::TableScan & table_scan); -DAGSchema genSchema(const tipb::TableScan & table_scan); -NamesAndTypes genNamesAndTypes(::google::protobuf::RepeatedPtrField<::tipb::ColumnInfo> & column_infos); -DAGSchema genSchema(::google::protobuf::RepeatedPtrField<::tipb::ColumnInfo> & column_infos); +NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan); +DAGSchema genSchema(const TiDBTableScan & table_scan); ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema); ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types); diff --git a/dbms/src/Flash/Coprocessor/TiDBTableScan.h b/dbms/src/Flash/Coprocessor/TiDBTableScan.h index b7cbb54ebce..22d2f6bc929 100644 --- a/dbms/src/Flash/Coprocessor/TiDBTableScan.h +++ b/dbms/src/Flash/Coprocessor/TiDBTableScan.h @@ -48,10 +48,6 @@ class TiDBTableScan { return table_scan->executor_id(); } - const tipb::Executor * getTableScan() const - { - return table_scan; - } private: const tipb::Executor * table_scan; diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index e3414770f45..6c96e7c22ad 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -202,7 +202,7 @@ std::tuple executeQueryImpl( try { - if (!internal && !context.getDAGContext()->isTest()) + if (!internal) logQuery(query.substr(0, settings.log_queries_cut_to_length), context, execute_query_logger); /// Check the limits. @@ -384,7 +384,7 @@ std::tuple executeQueryImpl( } }; - if (!internal && res.in && !context.getDAGContext()->isTest()) + if (!internal && res.in) { auto pipeline_log_str = [&res]() { FmtBuffer log_buffer; From cc81990cdd4c4334435d34cd8a94d15ea8037204 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 5 May 2022 11:23:31 +0800 Subject: [PATCH 18/42] clean --- dbms/src/Core/ColumnsWithTypeAndName.h | 1 + dbms/src/Flash/Coprocessor/DAGContext.h | 1 + dbms/src/TestUtils/InterpreterTestUtils.cpp | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Core/ColumnsWithTypeAndName.h b/dbms/src/Core/ColumnsWithTypeAndName.h index 7bcf8649589..61c77cf161e 100644 --- a/dbms/src/Core/ColumnsWithTypeAndName.h +++ b/dbms/src/Core/ColumnsWithTypeAndName.h @@ -22,4 +22,5 @@ namespace DB { using ColumnsWithTypeAndName = std::vector; + } diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index 4874641abb8..d031ff103ff 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -154,6 +154,7 @@ class DAGContext initExecutorIdToJoinIdMap(); initOutputInfo(); } + // for test explicit DAGContext(UInt64 max_error_count_) : dag_request(nullptr) diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp index e74c02d0cbf..2cc096d4095 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.cpp +++ b/dbms/src/TestUtils/InterpreterTestUtils.cpp @@ -56,7 +56,6 @@ void InterpreterTest::initializeClientInfo() void InterpreterTest::executeInterpreter(const String & expected_string, const std::shared_ptr & request, size_t concurrency) { DAGContext dag_context(*request, "interpreter_test", concurrency); - // todo change the code here. context.context.setDAGContext(&dag_context); // Currently, don't care about regions information in interpreter tests. DAGQuerySource dag(context.context); From fc46cd52338411323098e0b274681ca75fbb8a13 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 5 May 2022 14:49:19 +0800 Subject: [PATCH 19/42] address comments. --- .../Flash/Coprocessor/GenSchemaAndColumn.cpp | 55 +++++++------------ .../Flash/Coprocessor/GenSchemaAndColumn.h | 2 - dbms/src/Flash/tests/gtest_interpreter.cpp | 7 ++- 3 files changed, 25 insertions(+), 39 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index 04e772b334a..817245507c2 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -12,57 +12,44 @@ // See the License for the specific language governing permissions and // limitations under the License. #include -#include -#include +#include + namespace DB { -DAGSchema genSchema(const TiDBTableScan & table_scan) -{ - DAGSchema schema; - schema.reserve(table_scan.getColumnSize()); - for (Int32 i = 0; i < table_scan.getColumnSize(); ++i) - { - String name = fmt::format("mock_table_scan_{}", i); - TiDB::ColumnInfo column_info; - auto const & ci = table_scan.getColumns()[i]; - column_info.tp = static_cast(ci.tp()); - column_info.id = ci.column_id(); - column_info.name = name; - schema.push_back({name, column_info}); - } - return schema; -} - NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan) { NamesAndTypes names_and_types; names_and_types.reserve(table_scan.getColumnSize()); + String handle_column_name = MutableSupport::tidb_pk_column_name; + for (Int32 i = 0; i < table_scan.getColumnSize(); ++i) { String name = fmt::format("mock_table_scan_{}", i); TiDB::ColumnInfo column_info; auto const & ci = table_scan.getColumns()[i]; column_info.tp = static_cast(ci.tp()); - column_info.id = ci.column_id(); - auto type = getDataTypeByColumnInfoForComputingLayer(column_info); - names_and_types.push_back({name, type}); + ColumnID cid = ci.column_id(); + // Column ID -1 return the handle column + if (cid == TiDBPkColumnID) + name = handle_column_name; + else if (cid == ExtraTableIDColumnID) + name = MutableSupport::extra_table_id_column_name; + if (cid == ExtraTableIDColumnID) + { + NameAndTypePair extra_table_id_column_pair = {name, MutableSupport::extra_table_id_column_type}; + names_and_types.push_back(std::move(extra_table_id_column_pair)); + } + else + { + auto type = getDataTypeByColumnInfoForComputingLayer(column_info); + NameAndTypePair pair = {name, type}; + names_and_types.push_back(std::move(pair)); + } } return names_and_types; } -ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema) -{ - std::vector column_with_type_and_names; - column_with_type_and_names.reserve(schema.size()); - for (const auto & col : schema) - { - auto type = getDataTypeByColumnInfoForComputingLayer(col.second); - column_with_type_and_names.push_back(DB::ColumnWithTypeAndName(type, col.first)); - } - return column_with_type_and_names; -} - ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types) { std::vector column_with_type_and_names; diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h index 72bb6540f5b..6cd9a9a789e 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h @@ -24,8 +24,6 @@ namespace DB using DAGColumnInfo = std::pair; using DAGSchema = std::vector; NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan); -DAGSchema genSchema(const TiDBTableScan & table_scan); -ColumnsWithTypeAndName getColumnWithTypeAndName(const DAGSchema & schema); ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types); } // namespace DB \ No newline at end of file diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 961ab525be8..2c33b015e2d 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -244,9 +244,10 @@ CreatingSets Union Expression x 10 Expression - HashJoinProbe - Expression - MockTableScan)"; + Expression + HashJoinProbe + Expression + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } } From b23f51498410e1937f16ab6f6f0ca82be1f90b7d Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 5 May 2022 15:45:25 +0800 Subject: [PATCH 20/42] address comments --- .../Flash/Coprocessor/GenSchemaAndColumn.cpp | 32 +++++++------------ dbms/src/Flash/tests/gtest_interpreter.cpp | 7 ++-- dbms/src/TestUtils/mockExecutor.cpp | 2 ++ 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index 817245507c2..7689b68f3f0 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -20,31 +20,23 @@ NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan) { NamesAndTypes names_and_types; names_and_types.reserve(table_scan.getColumnSize()); - - String handle_column_name = MutableSupport::tidb_pk_column_name; - for (Int32 i = 0; i < table_scan.getColumnSize(); ++i) { - String name = fmt::format("mock_table_scan_{}", i); TiDB::ColumnInfo column_info; - auto const & ci = table_scan.getColumns()[i]; + const auto & ci = table_scan.getColumns()[i]; column_info.tp = static_cast(ci.tp()); - ColumnID cid = ci.column_id(); - // Column ID -1 return the handle column - if (cid == TiDBPkColumnID) - name = handle_column_name; - else if (cid == ExtraTableIDColumnID) - name = MutableSupport::extra_table_id_column_name; - if (cid == ExtraTableIDColumnID) - { - NameAndTypePair extra_table_id_column_pair = {name, MutableSupport::extra_table_id_column_type}; - names_and_types.push_back(std::move(extra_table_id_column_pair)); - } - else + column_info.id = ci.column_id(); + + switch (column_info.id) { - auto type = getDataTypeByColumnInfoForComputingLayer(column_info); - NameAndTypePair pair = {name, type}; - names_and_types.push_back(std::move(pair)); + case TiDBPkColumnID: + names_and_types.emplace_back(MutableSupport::tidb_pk_column_name, getDataTypeByColumnInfoForComputingLayer(column_info)); + break; + case ExtraTableIDColumnID: + names_and_types.emplace_back(MutableSupport::extra_table_id_column_name, MutableSupport::extra_table_id_column_type); + break; + default: + names_and_types.emplace_back(fmt::format("mock_table_scan_{}", i), getDataTypeByColumnInfoForComputingLayer(column_info)); } } return names_and_types; diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 2c33b015e2d..961ab525be8 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -244,10 +244,9 @@ CreatingSets Union Expression x 10 Expression - Expression - HashJoinProbe - Expression - MockTableScan)"; + HashJoinProbe + Expression + MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } } diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index c862c7deec8..3313aae6a93 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -88,11 +88,13 @@ DAGRequestBuilder & DAGRequestBuilder::mockTable(const String & db, const String assert(!columns.empty()); TableInfo table_info; table_info.name = db + "." + table; + int i = 0; for (const auto & column : columns) { TiDB::ColumnInfo ret; ret.tp = column.second; ret.name = column.first; + ret.id = i++; table_info.columns.push_back(std::move(ret)); } String empty_alias; From 927234c8be50bc677b76ba8e1310c5127528ec5f Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 5 May 2022 17:09:48 +0800 Subject: [PATCH 21/42] address comments. --- dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp | 1 + dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 1d19123bdf1..21f8ebcf7a9 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -1103,6 +1103,7 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) query_block.qb_column_prefix, pipeline.streams.size()); dagContext().final_concurrency = std::min(std::max(dagContext().final_concurrency, pipeline.streams.size()), max_streams); + if (res.before_aggregation) { // execute aggregation diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h index 6cd9a9a789e..617f69de925 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.h @@ -21,9 +21,6 @@ namespace DB { -using DAGColumnInfo = std::pair; -using DAGSchema = std::vector; NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan); ColumnsWithTypeAndName getColumnWithTypeAndName(const NamesAndTypes & names_and_types); - } // namespace DB \ No newline at end of file From 2639b7f51679ddf64511954f3564f9f797c31e04 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 5 May 2022 17:34:52 +0800 Subject: [PATCH 22/42] format. --- dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 21f8ebcf7a9..138cde7af6a 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -1103,7 +1103,7 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) query_block.qb_column_prefix, pipeline.streams.size()); dagContext().final_concurrency = std::min(std::max(dagContext().final_concurrency, pipeline.streams.size()), max_streams); - + if (res.before_aggregation) { // execute aggregation From eca8746b0701d662b920efe9116d6c36bd53633a Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 5 May 2022 19:44:21 +0800 Subject: [PATCH 23/42] address comment. --- dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index 7689b68f3f0..88f61ab6b1f 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -30,6 +30,7 @@ NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan) switch (column_info.id) { case TiDBPkColumnID: + // todo: need to check if the type of pk_handle_columns matches the type that used in delta merge tree. names_and_types.emplace_back(MutableSupport::tidb_pk_column_name, getDataTypeByColumnInfoForComputingLayer(column_info)); break; case ExtraTableIDColumnID: From 589263d164e3dacac05de945e67ee3b16a747c97 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 6 May 2022 10:29:52 +0800 Subject: [PATCH 24/42] fix. --- .../Flash/Coprocessor/DAGQueryBlockInterpreter.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 4341b8d4285..c11c5bd75a2 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -157,6 +157,19 @@ AnalysisResult analyzeExpressions( } } // namespace +// for tests, we need to mock tableScan blockInputStream as the source stream. +void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline) +{ + auto names_and_types = genNamesAndTypes(table_scan); + auto columns_with_type_and_name = getColumnWithTypeAndName(names_and_types); + analyzer = std::make_unique(std::move(names_and_types), context); + for (size_t i = 0; i < max_streams; ++i) + { + auto mock_table_scan_stream = std::make_shared(columns_with_type_and_name, context.getSettingsRef().max_block_size); + pipeline.streams.emplace_back(mock_table_scan_stream); + } +} + void DAGQueryBlockInterpreter::handleTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline) { const auto push_down_filter = PushDownFilter::toPushDownFilter(query_block.selection); From caf6b73d82c0c379344e1c1bf3f5789754815818 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Mon, 9 May 2022 16:40:28 +0800 Subject: [PATCH 25/42] init. --- .../ExpressionBlockInputStream.cpp | 6 ++ .../DataStreams/ExpressionBlockInputStream.h | 2 + .../DataStreams/FilterBlockInputStream.cpp | 23 ++++++++ dbms/src/DataStreams/FilterBlockInputStream.h | 2 + .../HashJoinBuildBlockInputStream.cpp | 20 +++++++ .../HashJoinBuildBlockInputStream.h | 1 + dbms/src/DataStreams/IBlockInputStream.cpp | 15 +++-- dbms/src/DataStreams/IBlockInputStream.h | 1 + .../src/DataStreams/LimitBlockInputStream.cpp | 6 ++ dbms/src/DataStreams/LimitBlockInputStream.h | 2 + .../MergeSortingBlockInputStream.cpp | 13 +++++ .../MergeSortingBlockInputStream.h | 1 + .../ParallelAggregatingBlockInputStream.cpp | 17 ++++++ .../ParallelAggregatingBlockInputStream.h | 3 + .../PartialSortingBlockInputStream.cpp | 21 +++++-- .../PartialSortingBlockInputStream.h | 1 + .../DataStreams/SharedQueryBlockInputStream.h | 6 ++ .../DataStreams/TiRemoteBlockInputStream.h | 23 ++++++-- dbms/src/Flash/tests/gtest_interpreter.cpp | 2 - dbms/src/Interpreters/ExpressionActions.cpp | 57 ++++++++++++++++++- dbms/src/Interpreters/ExpressionActions.h | 6 +- 21 files changed, 210 insertions(+), 18 deletions(-) diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp index c21f0cbdff5..9532f33b503 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp @@ -54,4 +54,10 @@ Block ExpressionBlockInputStream::readImpl() return res; } +void ExpressionBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const +{ + IProfilingBlockInputStream::print(buffer, indent, multiplier); + buffer.append(": "); + expression->dumpActions(buffer); +} } // namespace DB diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.h b/dbms/src/DataStreams/ExpressionBlockInputStream.h index 1d1e059d51d..6b4f3a4ee69 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.h +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.h @@ -15,6 +15,7 @@ #pragma once #include +#include namespace DB @@ -44,6 +45,7 @@ class ExpressionBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; + void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; private: ExpressionActionsPtr expression; diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index 3739b57d82d..62fce41126b 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -219,5 +219,28 @@ Block FilterBlockInputStream::readImpl() } } +// ywq todo complicate... +void FilterBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const +{ + IProfilingBlockInputStream::print(buffer, indent, multiplier); + buffer.append(": {"); + const auto & actions = expression->getActions(); + buffer.joinStr( + actions.begin(), + actions.end(), + [](const auto & arg, FmtBuffer & fb) { + arg.dumpAction(fb); + }, + ", "); + buffer.append("}"); + // ostr << "} input: {"; + // const auto & input_columns = expression->getRequiredColumnsWithTypes(); + // dumpIter(input_columns.cbegin(), input_columns.cend(), ostr, [](const auto & s, std::ostream & os) { os << s.name << '(' << s.type->getName() << ')'; }); + // ostr << "} output: {"; + // const auto & output = expression->getSampleBlock(); + // dumpIter(output.cbegin(), output.cend(), ostr, [](const auto & s, std::ostream & os) { os << s.name << '(' << s.type->getName() << ')'; }); + // ostr << "}]"; +} + } // namespace DB diff --git a/dbms/src/DataStreams/FilterBlockInputStream.h b/dbms/src/DataStreams/FilterBlockInputStream.h index 3d7ec131f47..2f0f8edf690 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.h +++ b/dbms/src/DataStreams/FilterBlockInputStream.h @@ -47,6 +47,8 @@ class FilterBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; + void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; + private: ExpressionActionsPtr expression; diff --git a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp index 91fd34bfff4..1ab5a46906c 100644 --- a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp +++ b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp @@ -13,6 +13,7 @@ // limitations under the License. +#include #include namespace DB { @@ -25,4 +26,23 @@ Block HashJoinBuildBlockInputStream::readImpl() return block; } +void HashJoinBuildBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const +{ + IProfilingBlockInputStream::print(buffer, indent, multiplier); + static const std::unordered_map join_type_map{ + {ASTTableJoin::Kind::Inner, "Inner"}, + {ASTTableJoin::Kind::Left, "Left"}, + {ASTTableJoin::Kind::Right, "Right"}, + {ASTTableJoin::Kind::Full, "Full"}, + {ASTTableJoin::Kind::Cross, "Cross"}, + {ASTTableJoin::Kind::Comma, "Comma"}, + {ASTTableJoin::Kind::Anti, "Anti"}, + {ASTTableJoin::Kind::Cross_Left, "Cross_Left"}, + {ASTTableJoin::Kind::Cross_Right, "Cross_Right"}, + {ASTTableJoin::Kind::Cross_Anti, "Cross_Anti"}}; + auto join_type_it = join_type_map.find(join->getKind()); + if (join_type_it == join_type_map.end()) + throw TiFlashException("Unknown join type", Errors::Coprocessor::Internal); + buffer.fmtAppend(": build_concurrency{{{}}}, join_kind{{{}}}", join->getBuildConcurrency(), join_type_it->second); +} } // namespace DB diff --git a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.h b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.h index 57b505f5237..52498a6834c 100644 --- a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.h +++ b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.h @@ -41,6 +41,7 @@ class HashJoinBuildBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; + void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; private: JoinPtr join; diff --git a/dbms/src/DataStreams/IBlockInputStream.cpp b/dbms/src/DataStreams/IBlockInputStream.cpp index 57dbe0e6ad0..804f626587f 100644 --- a/dbms/src/DataStreams/IBlockInputStream.cpp +++ b/dbms/src/DataStreams/IBlockInputStream.cpp @@ -16,6 +16,8 @@ #include #include +#include + namespace DB { @@ -77,15 +79,20 @@ size_t IBlockInputStream::checkDepthImpl(size_t max_depth, size_t level) const return res + 1; } - -void IBlockInputStream::dumpTree(FmtBuffer & buffer, size_t indent, size_t multiplier) +void IBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const { - // todo append getHeader().dumpStructure() buffer.fmtAppend( - "{}{}{}\n", + "{}{}{}", String(indent, ' '), getName(), multiplier > 1 ? fmt::format(" x {}", multiplier) : ""); +} + +void IBlockInputStream::dumpTree(FmtBuffer & buffer, size_t indent, size_t multiplier) +{ + // todo append getHeader().dumpStructure() + print(buffer, indent, multiplier); + buffer.append("\n"); ++indent; /// If the subtree is repeated several times, then we output it once with the multiplier. diff --git a/dbms/src/DataStreams/IBlockInputStream.h b/dbms/src/DataStreams/IBlockInputStream.h index 75fdffb3d29..fc0630fd2b1 100644 --- a/dbms/src/DataStreams/IBlockInputStream.h +++ b/dbms/src/DataStreams/IBlockInputStream.h @@ -180,6 +180,7 @@ class IBlockInputStream : private boost::noncopyable BlockInputStreams children; mutable std::shared_mutex children_mutex; bool collected = false; // a flag to avoid duplicated collecting, since some InputStream is shared by multiple inputStreams + virtual void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const; private: TableLockHolders table_locks; diff --git a/dbms/src/DataStreams/LimitBlockInputStream.cpp b/dbms/src/DataStreams/LimitBlockInputStream.cpp index 81c31fc5d77..c29871a3c67 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.cpp +++ b/dbms/src/DataStreams/LimitBlockInputStream.cpp @@ -83,4 +83,10 @@ Block LimitBlockInputStream::readImpl() return res; } +void LimitBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const +{ + IProfilingBlockInputStream::print(buffer, indent, multiplier); + buffer.fmtAppend(": limit = {}, offset = {}, pos = {}", limit, offset, pos); +} + } // namespace DB diff --git a/dbms/src/DataStreams/LimitBlockInputStream.h b/dbms/src/DataStreams/LimitBlockInputStream.h index 21978773daf..e4338288efc 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.h +++ b/dbms/src/DataStreams/LimitBlockInputStream.h @@ -43,6 +43,8 @@ class LimitBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; + void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; + private: size_t limit; diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp index 0975ace963a..83e9bbb871f 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -287,5 +287,18 @@ Block MergeSortingBlocksBlockInputStream::mergeImpl(std::priority_queue", des.column_name, des.direction == 1 ? "ASC" : "DESC"); + }, + ", "); + buffer.append("}"); +} } // namespace DB diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.h b/dbms/src/DataStreams/MergeSortingBlockInputStream.h index 003390c2525..5af47d8f3ba 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.h +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.h @@ -109,6 +109,7 @@ class MergeSortingBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; + void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; private: SortDescription description; diff --git a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 3163975108f..5954527f6be 100644 --- a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -13,6 +13,9 @@ // limitations under the License. #include +#include +#include +#include #include #include #include @@ -274,5 +277,19 @@ void ParallelAggregatingBlockInputStream::execute() threads_data[0].local_delta_memory, no_more_keys); } +// ywq todo +// add aggregate keys. +void ParallelAggregatingBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const +{ + IProfilingBlockInputStream::print(buffer, indent, multiplier); + buffer.fmtAppend(": max_threads: {}, final: {}, agg_funcs: {{", max_threads, final ? "true" : "false"); + const auto & aggregates = params.aggregates; + buffer.joinStr( + aggregates.cbegin(), + aggregates.cend(), + [](const auto & agg, FmtBuffer & fb) { fb.append(agg.column_name); }, + ", "); + buffer.append("}"); +} } // namespace DB diff --git a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h index 398c3d35bbc..72f800c4baa 100644 --- a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h +++ b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h @@ -19,6 +19,7 @@ #include #include #include +#include namespace DB { @@ -62,6 +63,8 @@ class ParallelAggregatingBlockInputStream : public IProfilingBlockInputStream } Block readImpl() override; + void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; + private: const LoggerPtr log; diff --git a/dbms/src/DataStreams/PartialSortingBlockInputStream.cpp b/dbms/src/DataStreams/PartialSortingBlockInputStream.cpp index 30f520fdec3..c1a2861ba19 100644 --- a/dbms/src/DataStreams/PartialSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/PartialSortingBlockInputStream.cpp @@ -12,15 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - +#include #include +#include namespace DB { - - Block PartialSortingBlockInputStream::readImpl() { Block res = children.back()->read(); @@ -28,5 +26,18 @@ Block PartialSortingBlockInputStream::readImpl() return res; } - +void PartialSortingBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const +{ + IProfilingBlockInputStream::print(buffer, indent, multiplier); + buffer.fmtAppend(": limit = {}", limit); + // buffer.fmtAppend(": limit = {}, columns {{", limit); + // buffer.joinStr( + // description.cbegin(), + // description.cend(), + // [](const auto & des, FmtBuffer & fb) { + // fb.fmtAppend("<{}, {}>", des.column_name, des.direction == 1 ? "ASC" : "DESC"); + // }, + // ", "); + // buffer.append("}"); } +} // namespace DB diff --git a/dbms/src/DataStreams/PartialSortingBlockInputStream.h b/dbms/src/DataStreams/PartialSortingBlockInputStream.h index 4a7a62474df..b1c8d4341da 100644 --- a/dbms/src/DataStreams/PartialSortingBlockInputStream.h +++ b/dbms/src/DataStreams/PartialSortingBlockInputStream.h @@ -50,6 +50,7 @@ class PartialSortingBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; + void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; private: SortDescription description; diff --git a/dbms/src/DataStreams/SharedQueryBlockInputStream.h b/dbms/src/DataStreams/SharedQueryBlockInputStream.h index e7cece67f0b..aa057fc29e1 100644 --- a/dbms/src/DataStreams/SharedQueryBlockInputStream.h +++ b/dbms/src/DataStreams/SharedQueryBlockInputStream.h @@ -169,6 +169,12 @@ class SharedQueryBlockInputStream : public IProfilingBlockInputStream throw Exception(exception_msg); } + // void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override + // { + // IProfilingBlockInputStream::print(buffer, indent, multiplier); + // buffer.fmtAppend(": clients: {}", queue.size()); + // } + private: MPMCQueue queue; diff --git a/dbms/src/DataStreams/TiRemoteBlockInputStream.h b/dbms/src/DataStreams/TiRemoteBlockInputStream.h index c1e29617586..e4d271fa18a 100644 --- a/dbms/src/DataStreams/TiRemoteBlockInputStream.h +++ b/dbms/src/DataStreams/TiRemoteBlockInputStream.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -60,11 +61,11 @@ class TiRemoteBlockInputStream : public IProfilingBlockInputStream void initRemoteExecutionSummaries(tipb::SelectResponse & resp, size_t index) { - for (auto & execution_summary : resp.execution_summaries()) + for (const auto & execution_summary : resp.execution_summaries()) { if (execution_summary.has_executor_id()) { - auto & executor_id = execution_summary.executor_id(); + const auto & executor_id = execution_summary.executor_id(); execution_summaries[index][executor_id].time_processed_ns = execution_summary.time_processed_ns(); execution_summaries[index][executor_id].num_produced_rows = execution_summary.num_produced_rows(); execution_summaries[index][executor_id].num_iterations = execution_summary.num_iterations(); @@ -84,11 +85,11 @@ class TiRemoteBlockInputStream : public IProfilingBlockInputStream return; } auto & execution_summaries_map = execution_summaries[index]; - for (auto & execution_summary : resp.execution_summaries()) + for (const auto & execution_summary : resp.execution_summaries()) { if (execution_summary.has_executor_id()) { - auto & executor_id = execution_summary.executor_id(); + const auto & executor_id = execution_summary.executor_id(); if (unlikely(execution_summaries_map.find(executor_id) == execution_summaries_map.end())) { LOG_FMT_WARNING(log, "execution {} not found in execution_summaries, this should not happen", executor_id); @@ -244,6 +245,20 @@ class TiRemoteBlockInputStream : public IProfilingBlockInputStream LOG_FMT_DEBUG(log, "finish read {} rows from remote", total_rows); remote_reader->close(); } + + void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override + { + IProfilingBlockInputStream::print(buffer, indent, multiplier); + buffer.append(": schema: {"); + buffer.joinStr( + sample_block.begin(), + sample_block.end(), + [](const auto & arg, FmtBuffer & fb) { + fb.fmtAppend("<{}, {}>", arg.name, arg.type->getName()); + }, + ", "); + buffer.append("}"); + } }; using ExchangeReceiverInputStream = TiRemoteBlockInputStream; diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 961ab525be8..5207136b77e 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -251,7 +251,5 @@ CreatingSets } } CATCH - - } // namespace tests } // namespace DB \ No newline at end of file diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 45d5293d584..3d6af2ebb76 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -461,8 +461,56 @@ void ExpressionAction::executeOnTotals(Block & block) const join->joinTotals(block); } +// ywq todo +void ExpressionAction::dumpAction(FmtBuffer & fb) const +{ + switch (type) + { + case ADD_COLUMN: + fb.append("Add column"); + // fb.fmtAppend("Add {} {} {}", result_name, + // (result_type ? result_type->getName() : "(no type)"), + // (added_column ? added_column->getName() : "(no column)")); + break; + case REMOVE_COLUMN: + fb.append("Remove column"); + // fb.fmtAppend("Remove {}", source_name); + break; + case COPY_COLUMN: + fb.append("Copy column"); + // fb.fmtAppend("Copy {} = {}", result_name, source_name); + break; + case APPLY_FUNCTION: + fb.fmtAppend("{}", (function ? function->getName() : "(no function)")); + // fb.joinStr( + // argument_names.cbegin(), + // argument_names.cend(), + // [](const auto & s, FmtBuffer & fb) { + // fb.append(s); + // }, + // ", "); + break; + case JOIN: + fb.append("join"); + // fb.joinStr(columns_added_by_join.cbegin(), columns_added_by_join.cend(), [](const auto & c, FmtBuffer &fb){ + // fb.append(c.name); + // }, ", "); + break; + case PROJECT: + fb.append("project"); + // fb.joinStr(projections.cbegin(), projections.cend(), [](const auto & proj, FmtBuffer & fb) { + // fb.append(proj.first); + // // if (!proj.second.empty() && proj.first != proj.second) + // // fb.fmtAppend(" as {}", proj.second); + // }, ", "); + // fb.append("}"); + break; + default: + break; + } +} -std::string ExpressionAction::toString() const +String ExpressionAction::toString() const { std::stringstream ss; switch (type) @@ -529,7 +577,6 @@ std::string ExpressionAction::toString() const default: throw Exception("Unexpected Action type", ErrorCodes::LOGICAL_ERROR); } - return ss.str(); } @@ -937,6 +984,12 @@ std::string ExpressionActions::dumpActions() const return ss.str(); } +void ExpressionActions::dumpActions(FmtBuffer & fb) const +{ + for (const auto & action : actions) + action.dumpAction(fb); +} + void ExpressionActions::optimize() { optimizeArrayJoin(); diff --git a/dbms/src/Interpreters/ExpressionActions.h b/dbms/src/Interpreters/ExpressionActions.h index 7ec54a1a8ae..e98960a1d33 100644 --- a/dbms/src/Interpreters/ExpressionActions.h +++ b/dbms/src/Interpreters/ExpressionActions.h @@ -121,6 +121,8 @@ struct ExpressionAction std::string toString() const; + void dumpAction(FmtBuffer & fb) const; + private: friend class ExpressionActions; @@ -212,6 +214,8 @@ class ExpressionActions std::string dumpActions() const; + void dumpActions(FmtBuffer & fb) const; + static std::string getSmallestColumn(const NamesAndTypesList & columns); BlockInputStreamPtr createStreamWithNonJoinedDataIfFullOrRightJoin(const Block & source_header, size_t index, size_t step, size_t max_block_size) const; @@ -251,7 +255,7 @@ struct ExpressionActionsChain ExpressionActionsPtr actions; Names required_output; - Step(const ExpressionActionsPtr & actions_ = nullptr, const Names & required_output_ = Names()) + explicit Step(const ExpressionActionsPtr & actions_ = nullptr, const Names & required_output_ = Names()) : actions(actions_) , required_output(required_output_) {} From 0874ad5db80ea6478b42f289856dd0d47c80113e Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Mon, 9 May 2022 17:58:29 +0800 Subject: [PATCH 26/42] update. --- dbms/src/DataStreams/FilterBlockInputStream.cpp | 8 ++------ dbms/src/DataStreams/LimitBlockInputStream.cpp | 2 +- .../ParallelAggregatingBlockInputStream.cpp | 16 ++++++++-------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index 62fce41126b..2f7e7b1f8dd 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -20,6 +20,8 @@ #include #include +#include "Common/FmtUtils.h" + namespace DB { namespace ErrorCodes @@ -233,12 +235,6 @@ void FilterBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t mul }, ", "); buffer.append("}"); - // ostr << "} input: {"; - // const auto & input_columns = expression->getRequiredColumnsWithTypes(); - // dumpIter(input_columns.cbegin(), input_columns.cend(), ostr, [](const auto & s, std::ostream & os) { os << s.name << '(' << s.type->getName() << ')'; }); - // ostr << "} output: {"; - // const auto & output = expression->getSampleBlock(); - // dumpIter(output.cbegin(), output.cend(), ostr, [](const auto & s, std::ostream & os) { os << s.name << '(' << s.type->getName() << ')'; }); // ostr << "}]"; } diff --git a/dbms/src/DataStreams/LimitBlockInputStream.cpp b/dbms/src/DataStreams/LimitBlockInputStream.cpp index c29871a3c67..03510cdf6e4 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.cpp +++ b/dbms/src/DataStreams/LimitBlockInputStream.cpp @@ -86,7 +86,7 @@ Block LimitBlockInputStream::readImpl() void LimitBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const { IProfilingBlockInputStream::print(buffer, indent, multiplier); - buffer.fmtAppend(": limit = {}, offset = {}, pos = {}", limit, offset, pos); + buffer.fmtAppend(": limit = {} always_read_till_end = {}", limit, always_read_till_end); } } // namespace DB diff --git a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 5954527f6be..e88525a4bb9 100644 --- a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -282,14 +282,14 @@ void ParallelAggregatingBlockInputStream::execute() void ParallelAggregatingBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const { IProfilingBlockInputStream::print(buffer, indent, multiplier); - buffer.fmtAppend(": max_threads: {}, final: {}, agg_funcs: {{", max_threads, final ? "true" : "false"); - const auto & aggregates = params.aggregates; - buffer.joinStr( - aggregates.cbegin(), - aggregates.cend(), - [](const auto & agg, FmtBuffer & fb) { fb.append(agg.column_name); }, - ", "); - buffer.append("}"); + buffer.fmtAppend(": max_threads: {}, final: {}", max_threads, final ? "true" : "false"); + // const auto & aggregates = params.aggregates; + // buffer.joinStr( + // aggregates.cbegin(), + // aggregates.cend(), + // [](const auto & agg, FmtBuffer & fb) { fb.append(agg.column_name); }, + // ", "); + // buffer.append("}"); } } // namespace DB From dc82bcb68b3d42d176a69e587f8a1f128d153153 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 17 May 2022 15:06:28 +0800 Subject: [PATCH 27/42] update. --- dbms/src/DataStreams/IBlockInputStream.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/DataStreams/IBlockInputStream.cpp b/dbms/src/DataStreams/IBlockInputStream.cpp index 804f626587f..2eeebb8f378 100644 --- a/dbms/src/DataStreams/IBlockInputStream.cpp +++ b/dbms/src/DataStreams/IBlockInputStream.cpp @@ -16,8 +16,6 @@ #include #include -#include - namespace DB { From beb504c690324658992a72a0b6b7041b10244073 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 17 May 2022 16:10:11 +0800 Subject: [PATCH 28/42] update --- dbms/src/DataStreams/IBlockInputStream.cpp | 2 + dbms/src/DataStreams/IBlockInputStream.h | 5 ++ .../Coprocessor/DAGQueryBlockInterpreter.cpp | 67 +++++++++++++------ .../Coprocessor/DAGQueryBlockInterpreter.h | 6 +- .../Flash/Coprocessor/InterpreterUtils.cpp | 4 +- dbms/src/Flash/Coprocessor/InterpreterUtils.h | 3 +- 6 files changed, 60 insertions(+), 27 deletions(-) diff --git a/dbms/src/DataStreams/IBlockInputStream.cpp b/dbms/src/DataStreams/IBlockInputStream.cpp index 2eeebb8f378..ad56c81e05c 100644 --- a/dbms/src/DataStreams/IBlockInputStream.cpp +++ b/dbms/src/DataStreams/IBlockInputStream.cpp @@ -84,6 +84,8 @@ void IBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multipli String(indent, ' '), getName(), multiplier > 1 ? fmt::format(" x {}", multiplier) : ""); + if (!extra_info.empty()) + buffer.fmtAppend(": {}", extra_info); } void IBlockInputStream::dumpTree(FmtBuffer & buffer, size_t indent, size_t multiplier) diff --git a/dbms/src/DataStreams/IBlockInputStream.h b/dbms/src/DataStreams/IBlockInputStream.h index fc0630fd2b1..0b4af904f79 100644 --- a/dbms/src/DataStreams/IBlockInputStream.h +++ b/dbms/src/DataStreams/IBlockInputStream.h @@ -135,6 +135,8 @@ class IBlockInputStream : private boost::noncopyable */ void addTableLock(const TableLockHolder & lock) { table_locks.push_back(lock); } + void setExtraInfo(String info) { extra_info = info; } + template void forEachChild(F && f) @@ -189,6 +191,9 @@ class IBlockInputStream : private boost::noncopyable mutable std::mutex tree_id_mutex; mutable String tree_id; + /// The info that hints why the inputStream is needed to run. + String extra_info; + /// Get text with names of this source and the entire subtree, this function should only be called after the /// InputStream tree is constructed. String getTreeID() const; diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index cef1fe1b596..b41635a40ff 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -197,7 +197,10 @@ void DAGQueryBlockInterpreter::prepareJoin( ExpressionActionsChain chain; if (dag_analyzer.appendJoinKeyAndJoinFilters(chain, keys, key_types, key_names, left, is_right_out_join, filters, filter_column_name)) { - pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, chain.getLastActions(), log->identifier()); }); + pipeline.transform([&](auto & stream) { + stream = std::make_shared(stream, chain.getLastActions(), log->identifier()); + stream->setExtraInfo("Append Join key and join Filters"); + }); } } @@ -472,8 +475,9 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & }; right_pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, join_ptr, get_concurrency_build_index(), log->identifier()); + stream->setExtraInfo("Right join build"); }); - executeUnion(right_pipeline, max_streams, log, /*ignore_block=*/true); + executeUnion(right_pipeline, max_streams, log, /*ignore_block=*/true, "for join"); right_query.source = right_pipeline.firstStream(); right_query.join = join_ptr; @@ -490,19 +494,22 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & if (is_tiflash_right_join) { auto & join_execute_info = dagContext().getJoinExecuteInfoMap()[query_block.source_name]; - for (size_t i = 0; i < join_build_concurrency; i++) + for (size_t i = 0; i < join_build_concurrency; ++i) { auto non_joined_stream = chain.getLastActions()->createStreamWithNonJoinedDataIfFullOrRightJoin( pipeline.firstStream()->getHeader(), i, join_build_concurrency, settings.max_block_size); + non_joined_stream->setExtraInfo("Add Stream With NonJoinedData If FullOrRightJoin"); pipeline.streams_with_non_joined_data.push_back(non_joined_stream); join_execute_info.non_joined_streams.push_back(non_joined_stream); } } for (auto & stream : pipeline.streams) + { stream = std::make_shared(stream, chain.getLastActions(), log->identifier()); + } /// add a project to remove all the useless column NamesWithAliases project_cols; @@ -512,7 +519,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & /// it is guaranteed by its children query block project_cols.emplace_back(c.name, c.name); } - executeProject(pipeline, project_cols); + executeProject(pipeline, project_cols, "remove useless column after join"); analyzer = std::make_unique(std::move(join_output_columns), context); } @@ -526,19 +533,22 @@ void DAGQueryBlockInterpreter::recordJoinExecuteInfo(size_t build_side_index, co dagContext().getJoinExecuteInfoMap()[query_block.source_name] = std::move(join_execute_info); } -void DAGQueryBlockInterpreter::executeWhere(DAGPipeline & pipeline, const ExpressionActionsPtr & expr, String & filter_column) +void DAGQueryBlockInterpreter::executeWhere(DAGPipeline & pipeline, const ExpressionActionsPtr & expr, String & filter_column, String extra_info) { - pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, expr, filter_column, log->identifier()); }); + pipeline.transform([&](auto & stream) { + stream = std::make_shared(stream, expr, filter_column, log->identifier()); + stream->setExtraInfo(extra_info); + }); } void DAGQueryBlockInterpreter::executeWindow( DAGPipeline & pipeline, WindowDescription & window_description) { - executeExpression(pipeline, window_description.before_window); + executeExpression(pipeline, window_description.before_window, "before window"); /// If there are several streams, we merge them into one - executeUnion(pipeline, max_streams, log); + executeUnion(pipeline, max_streams, log, "for window"); assert(pipeline.streams.size() == 1); pipeline.firstStream() = std::make_shared(pipeline.firstStream(), window_description, log->identifier()); } @@ -551,7 +561,10 @@ void DAGQueryBlockInterpreter::executeAggregation( AggregateDescriptions & aggregate_descriptions, bool is_final_agg) { - pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, expression_actions_ptr, log->identifier()); }); + pipeline.transform([&](auto & stream) { + stream = std::make_shared(stream, expression_actions_ptr, log->identifier()); + stream->setExtraInfo("before aggregation"); + }); Block before_agg_header = pipeline.firstStream()->getHeader(); @@ -604,11 +617,14 @@ void DAGQueryBlockInterpreter::executeAggregation( } } -void DAGQueryBlockInterpreter::executeExpression(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr) +void DAGQueryBlockInterpreter::executeExpression(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, String extra_info) { if (!expressionActionsPtr->getActions().empty()) { - pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, expressionActionsPtr, log->identifier()); }); + pipeline.transform([&](auto & stream) { + stream = std::make_shared(stream, expressionActionsPtr, log->identifier()); + stream->setExtraInfo(extra_info); + }); } } @@ -640,7 +656,7 @@ void DAGQueryBlockInterpreter::orderStreams(DAGPipeline & pipeline, SortDescript }); /// If there are several streams, we merge them into one - executeUnion(pipeline, max_streams, log); + executeUnion(pipeline, max_streams, log, "for order"); /// Merge the sorted blocks. pipeline.firstStream() = std::make_shared( @@ -671,6 +687,7 @@ void DAGQueryBlockInterpreter::handleExchangeReceiver(DAGPipeline & pipeline) BlockInputStreamPtr stream = std::make_shared(it->second, log->identifier(), query_block.source_name); exchange_receiver_io_input_streams.push_back(stream); stream = std::make_shared(stream, 8192, 0, log->identifier()); + stream->setExtraInfo("after exchange receiver"); pipeline.streams.push_back(stream); } std::vector source_columns; @@ -703,7 +720,10 @@ void DAGQueryBlockInterpreter::handleProjection(DAGPipeline & pipeline, const ti output_columns.emplace_back(alias, col.type); project_cols.emplace_back(col.name, alias); } - pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, chain.getLastActions(), log->identifier()); }); + pipeline.transform([&](auto & stream) { + stream = std::make_shared(stream, chain.getLastActions(), log->identifier()); + stream->setExtraInfo("before project"); + }); executeProject(pipeline, project_cols); analyzer = std::make_unique(std::move(output_columns), context); } @@ -718,7 +738,7 @@ void DAGQueryBlockInterpreter::handleWindow(DAGPipeline & pipeline, const tipb:: DAGExpressionAnalyzer dag_analyzer(input_columns, context); WindowDescription window_description = dag_analyzer.buildWindowDescription(window); executeWindow(pipeline, window_description); - executeExpression(pipeline, window_description.after_window); + executeExpression(pipeline, window_description.after_window, "after window"); analyzer = std::make_unique(window_description.after_window_columns, context); } @@ -802,7 +822,7 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) if (res.before_where) { // execute where - executeWhere(pipeline, res.before_where, res.filter_column_name); + executeWhere(pipeline, res.before_where, res.filter_column_name, "execute where"); recordProfileStreams(pipeline, query_block.selection_name); } @@ -822,12 +842,12 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) if (res.before_having) { // execute having - executeWhere(pipeline, res.before_having, res.having_column_name); + executeWhere(pipeline, res.before_having, res.having_column_name, "execute having"); recordProfileStreams(pipeline, query_block.having_name); } if (res.before_order_and_select) { - executeExpression(pipeline, res.before_order_and_select); + executeExpression(pipeline, res.before_order_and_select, "before order and select"); } if (!res.order_columns.empty()) @@ -838,7 +858,7 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) } // execute final project action - executeProject(pipeline, final_project); + executeProject(pipeline, final_project, "final projection"); // execute limit if (query_block.limit_or_topn && query_block.limit_or_topn->tp() == tipb::TypeLimit) { @@ -855,12 +875,15 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) } } -void DAGQueryBlockInterpreter::executeProject(DAGPipeline & pipeline, NamesWithAliases & project_cols) +void DAGQueryBlockInterpreter::executeProject(DAGPipeline & pipeline, NamesWithAliases & project_cols, String extra_info) { if (project_cols.empty()) return; ExpressionActionsPtr project = generateProjectExpressionActions(pipeline.firstStream(), context, project_cols); - pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, project, log->identifier()); }); + pipeline.transform([&](auto & stream) { + stream = std::make_shared(stream, project, log->identifier()); + stream->setExtraInfo(extra_info); + }); } void DAGQueryBlockInterpreter::executeLimit(DAGPipeline & pipeline) @@ -873,7 +896,7 @@ void DAGQueryBlockInterpreter::executeLimit(DAGPipeline & pipeline) pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, limit, 0, log->identifier(), false); }); if (pipeline.hasMoreThanOneStream()) { - executeUnion(pipeline, max_streams, log); + executeUnion(pipeline, max_streams, log, "for limit"); pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, limit, 0, log->identifier(), false); }); } } @@ -913,7 +936,7 @@ BlockInputStreams DAGQueryBlockInterpreter::execute() executeImpl(pipeline); if (!pipeline.streams_with_non_joined_data.empty()) { - executeUnion(pipeline, max_streams, log); + executeUnion(pipeline, max_streams, log, "final union"); restorePipelineConcurrency(pipeline); } diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h index f5a8d2b5ce5..e6a1cee49ff 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h @@ -79,8 +79,8 @@ class DAGQueryBlockInterpreter std::vector & source_columns, String & filter_column_for_other_condition, String & filter_column_for_other_eq_condition); - void executeWhere(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, String & filter_column); - void executeExpression(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr); + void executeWhere(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, String & filter_column, String extra_info = ""); + void executeExpression(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, String extra_info = ""); void executeWindowOrder(DAGPipeline & pipeline, SortDescription sort_desc); void orderStreams(DAGPipeline & pipeline, SortDescription order_descr, Int64 limit); void executeOrder(DAGPipeline & pipeline, const std::vector & order_columns); @@ -95,7 +95,7 @@ class DAGQueryBlockInterpreter const TiDB::TiDBCollators & collators, AggregateDescriptions & aggregate_descriptions, bool is_final_agg); - void executeProject(DAGPipeline & pipeline, NamesWithAliases & project_cols); + void executeProject(DAGPipeline & pipeline, NamesWithAliases & project_cols, String extra_info = ""); void handleExchangeSender(DAGPipeline & pipeline); void recordProfileStreams(DAGPipeline & pipeline, const String & key); diff --git a/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp b/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp index 69060071997..4de96b72c42 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp @@ -62,7 +62,8 @@ void executeUnion( DAGPipeline & pipeline, size_t max_streams, const LoggerPtr & log, - bool ignore_block) + bool ignore_block, + String extra_info) { if (pipeline.streams.size() == 1 && pipeline.streams_with_non_joined_data.empty()) return; @@ -73,6 +74,7 @@ void executeUnion( pipeline.firstStream() = std::make_shared(pipeline.streams, non_joined_data_stream, max_streams, log->identifier()); else pipeline.firstStream() = std::make_shared(pipeline.streams, non_joined_data_stream, max_streams, log->identifier()); + pipeline.firstStream()->setExtraInfo(extra_info); pipeline.streams.resize(1); } else if (non_joined_data_stream != nullptr) diff --git a/dbms/src/Flash/Coprocessor/InterpreterUtils.h b/dbms/src/Flash/Coprocessor/InterpreterUtils.h index 91e6d483220..daffe6143b4 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterUtils.h +++ b/dbms/src/Flash/Coprocessor/InterpreterUtils.h @@ -37,7 +37,8 @@ void executeUnion( DAGPipeline & pipeline, size_t max_streams, const LoggerPtr & log, - bool ignore_block = false); + bool ignore_block = false, + String extra_info = ""); ExpressionActionsPtr generateProjectExpressionActions( const BlockInputStreamPtr & stream, From a5cc5842910f18b3f260791118135d5f90b1b954 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 17 May 2022 21:13:57 +0800 Subject: [PATCH 29/42] refine. --- .../ExpressionBlockInputStream.cpp | 7 - .../DataStreams/ExpressionBlockInputStream.h | 2 +- .../HashJoinBuildBlockInputStream.cpp | 2 +- dbms/src/DataStreams/IBlockInputStream.cpp | 3 +- .../src/DataStreams/LimitBlockInputStream.cpp | 2 +- .../MergeSortingBlockInputStream.cpp | 2 +- .../ParallelAggregatingBlockInputStream.cpp | 18 +- .../PartialSortingBlockInputStream.cpp | 19 +- .../DataStreams/SharedQueryBlockInputStream.h | 2 + dbms/src/Flash/tests/gtest_interpreter.cpp | 222 +++++++++--------- 10 files changed, 136 insertions(+), 143 deletions(-) diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp index 9532f33b503..c8180288b84 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp @@ -53,11 +53,4 @@ Block ExpressionBlockInputStream::readImpl() expression->execute(res); return res; } - -void ExpressionBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const -{ - IProfilingBlockInputStream::print(buffer, indent, multiplier); - buffer.append(": "); - expression->dumpActions(buffer); -} } // namespace DB diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.h b/dbms/src/DataStreams/ExpressionBlockInputStream.h index 6b4f3a4ee69..58223d8ba39 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.h +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.h @@ -45,7 +45,7 @@ class ExpressionBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; - void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; + // void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; private: ExpressionActionsPtr expression; diff --git a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp index 1ab5a46906c..8729398a27b 100644 --- a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp +++ b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp @@ -43,6 +43,6 @@ void HashJoinBuildBlockInputStream::print(FmtBuffer & buffer, size_t indent, siz auto join_type_it = join_type_map.find(join->getKind()); if (join_type_it == join_type_map.end()) throw TiFlashException("Unknown join type", Errors::Coprocessor::Internal); - buffer.fmtAppend(": build_concurrency{{{}}}, join_kind{{{}}}", join->getBuildConcurrency(), join_type_it->second); + buffer.fmtAppend(", build_concurrency{{{}}}, join_kind{{{}}}", join->getBuildConcurrency(), join_type_it->second); } } // namespace DB diff --git a/dbms/src/DataStreams/IBlockInputStream.cpp b/dbms/src/DataStreams/IBlockInputStream.cpp index ad56c81e05c..458574a7605 100644 --- a/dbms/src/DataStreams/IBlockInputStream.cpp +++ b/dbms/src/DataStreams/IBlockInputStream.cpp @@ -85,12 +85,11 @@ void IBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multipli getName(), multiplier > 1 ? fmt::format(" x {}", multiplier) : ""); if (!extra_info.empty()) - buffer.fmtAppend(": {}", extra_info); + buffer.fmtAppend(": <{}>", extra_info); } void IBlockInputStream::dumpTree(FmtBuffer & buffer, size_t indent, size_t multiplier) { - // todo append getHeader().dumpStructure() print(buffer, indent, multiplier); buffer.append("\n"); ++indent; diff --git a/dbms/src/DataStreams/LimitBlockInputStream.cpp b/dbms/src/DataStreams/LimitBlockInputStream.cpp index 03510cdf6e4..99ec1bed705 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.cpp +++ b/dbms/src/DataStreams/LimitBlockInputStream.cpp @@ -86,7 +86,7 @@ Block LimitBlockInputStream::readImpl() void LimitBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const { IProfilingBlockInputStream::print(buffer, indent, multiplier); - buffer.fmtAppend(": limit = {} always_read_till_end = {}", limit, always_read_till_end); + buffer.fmtAppend(", limit = {} always_read_till_end = {}", limit, always_read_till_end); } } // namespace DB diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp index 83e9bbb871f..4ee965fd408 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -290,7 +290,7 @@ Block MergeSortingBlocksBlockInputStream::mergeImpl(std::priority_queue", des.column_name, des.direction == 1 ? "ASC" : "DESC"); - // }, - // ", "); - // buffer.append("}"); + buffer.fmtAppend(": limit = {}, columns {{", limit); + buffer.joinStr( + description.cbegin(), + description.cend(), + [](const auto & des, FmtBuffer & fb) { + fb.fmtAppend("<{}, {}>", des.column_name, des.direction == 1 ? "ASC" : "DESC"); + }, + ", "); + buffer.append("}"); } } // namespace DB diff --git a/dbms/src/DataStreams/SharedQueryBlockInputStream.h b/dbms/src/DataStreams/SharedQueryBlockInputStream.h index aa057fc29e1..d2a1f05341c 100644 --- a/dbms/src/DataStreams/SharedQueryBlockInputStream.h +++ b/dbms/src/DataStreams/SharedQueryBlockInputStream.h @@ -169,6 +169,8 @@ class SharedQueryBlockInputStream : public IProfilingBlockInputStream throw Exception(exception_msg); } + + // ywq todo // void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override // { // IProfilingBlockInputStream::print(buffer, indent, multiplier); diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 0d07159b8fc..a349622b68b 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -49,16 +49,16 @@ try String expected = R"( Union SharedQuery x 10 - Expression - MergeSorting + Expression: + MergeSorting, limit = 10, columns {} Union - PartialSorting x 10 - Expression - Filter + PartialSorting x 10: limit = 10, columns {} + Expression: + Filter: : {equals} SharedQuery - ParallelAggregating - Expression x 10 - Filter + ParallelAggregating, max_threads: 10, final: true, columns: {max(mock_table_scan_0)_collator_0 } + Expression x 10: + Filter: : {equals} MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -74,16 +74,16 @@ Union String expected = R"( Union SharedQuery x 10 - Limit + Limit, limit = 10 always_read_till_end = false Union - Limit x 10 - Expression - Expression - Filter + Limit x 10, limit = 10 always_read_till_end = false + Expression: + Expression: + Filter: : {equals} SharedQuery - ParallelAggregating - Expression x 10 - Filter + ParallelAggregating, max_threads: 10, final: true, columns: {max(mock_table_scan_0)_collator_0 } + Expression x 10: + Filter: : {equals} MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -101,16 +101,16 @@ try { String expected = R"( Union - Expression x 10 + Expression x 10: Expression - Expression - Expression + Expression: + Expression: Expression - Expression - Expression + Expression: + Expression: Expression - Expression - Expression + Expression: + Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -123,17 +123,17 @@ Union { String expected = R"( Union - Expression x 10 + Expression x 10: Expression - Expression + Expression: SharedQuery - Expression - MergeSorting + Expression: + MergeSorting, limit = 10, columns {<__QB_3_mock_table_scan_0, DESC>, <__QB_3_mock_table_scan_1, ASC>} Union - PartialSorting x 10 + PartialSorting x 10: limit = 10, columns {<__QB_3_mock_table_scan_0, DESC>, <__QB_3_mock_table_scan_1, ASC>} Expression - Expression - Expression + Expression: + Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -148,23 +148,23 @@ Union { String expected = R"( Union - Expression x 10 + Expression x 10: Expression - Expression - Expression + Expression: + Expression: SharedQuery - ParallelAggregating - Expression x 10 + ParallelAggregating, max_threads: 10, final: true, columns: {max(__QB_3___QB_4_mock_table_scan_0)_collator_0 } + Expression x 10: Expression - Expression + Expression: SharedQuery - Expression - MergeSorting + Expression: + MergeSorting, limit = 10, columns {<__QB_4_mock_table_scan_0, DESC>, <__QB_4_mock_table_scan_1, ASC>} Union - PartialSorting x 10 + PartialSorting x 10: limit = 10, columns {<__QB_4_mock_table_scan_0, DESC>, <__QB_4_mock_table_scan_1, ASC>} Expression - Expression - Expression + Expression: + Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -183,31 +183,31 @@ Union String expected = R"( Union SharedQuery x 10 - Limit + Limit, limit = 10 always_read_till_end = false Union - Limit x 10 - Expression + Limit x 10, limit = 10 always_read_till_end = false + Expression: Expression - Expression - Expression - Expression - Filter + Expression: + Expression: + Expression: + Filter: : {equals} Expression - Expression - Expression + Expression: + Expression: SharedQuery - ParallelAggregating - Expression x 10 + ParallelAggregating, max_threads: 10, final: true, columns: {max(__QB_4___QB_5_mock_table_scan_0)_collator_0 } + Expression x 10: Expression - Expression + Expression: SharedQuery - Expression - MergeSorting + Expression: + MergeSorting, limit = 10, columns {<__QB_5_mock_table_scan_0, DESC>, <__QB_5_mock_table_scan_1, ASC>} Union - PartialSorting x 10 + PartialSorting x 10: limit = 10, columns {<__QB_5_mock_table_scan_0, DESC>, <__QB_5_mock_table_scan_1, ASC>} Expression - Expression - Expression + Expression: + Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -231,24 +231,24 @@ Union { String expected = R"( CreatingSets - Union - HashJoinBuildBlockInputStream x 10 - Expression - Expression + Union: + HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + Expression: + Expression: MockTableScan - Union x 2 - HashJoinBuildBlockInputStream x 10 - Expression - Expression - Expression + Union x 2: + HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + Expression: + Expression: + Expression: HashJoinProbe - Expression + Expression: MockTableScan Union - Expression x 10 - Expression + Expression x 10: + Expression: HashJoinProbe - Expression + Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -261,16 +261,16 @@ CreatingSets { String expected = R"( Union - Expression x 10 + Expression x 10: Expression - Expression - Expression + Expression: + Expression: Expression - Expression - Expression + Expression: + Expression: Expression - Expression - Expression + Expression: + Expression: MockExchangeReceiver)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -285,16 +285,16 @@ Union String expected = R"( Union MockExchangeSender x 10 - Expression + Expression: Expression - Expression - Expression + Expression: + Expression: Expression - Expression - Expression + Expression: + Expression: Expression - Expression - Expression + Expression: + Expression: MockExchangeReceiver)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -318,24 +318,24 @@ Union { String expected = R"( CreatingSets - Union - HashJoinBuildBlockInputStream x 10 - Expression - Expression + Union: + HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + Expression: + Expression: MockExchangeReceiver - Union x 2 - HashJoinBuildBlockInputStream x 10 - Expression - Expression - Expression + Union x 2: + HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + Expression: + Expression: + Expression: HashJoinProbe - Expression + Expression: MockExchangeReceiver Union - Expression x 10 - Expression + Expression x 10: + Expression: HashJoinProbe - Expression + Expression: MockExchangeReceiver)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -360,25 +360,25 @@ CreatingSets { String expected = R"( CreatingSets - Union - HashJoinBuildBlockInputStream x 10 - Expression - Expression + Union: + HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + Expression: + Expression: MockExchangeReceiver - Union x 2 - HashJoinBuildBlockInputStream x 10 - Expression - Expression - Expression + Union x 2: + HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + Expression: + Expression: + Expression: HashJoinProbe - Expression + Expression: MockExchangeReceiver Union MockExchangeSender x 10 - Expression - Expression + Expression: + Expression: HashJoinProbe - Expression + Expression: MockExchangeReceiver)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } From 948d42d5fc6be09a64e7aca95ecc70614d73719b Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 17 May 2022 21:17:23 +0800 Subject: [PATCH 30/42] tiny refine. --- dbms/src/DataStreams/ExpressionBlockInputStream.cpp | 1 + dbms/src/DataStreams/ExpressionBlockInputStream.h | 1 - dbms/src/DataStreams/FilterBlockInputStream.cpp | 5 +---- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp index c8180288b84..c21f0cbdff5 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp @@ -53,4 +53,5 @@ Block ExpressionBlockInputStream::readImpl() expression->execute(res); return res; } + } // namespace DB diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.h b/dbms/src/DataStreams/ExpressionBlockInputStream.h index 58223d8ba39..d6ee6d81036 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.h +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.h @@ -45,7 +45,6 @@ class ExpressionBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; - // void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; private: ExpressionActionsPtr expression; diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index 2f7e7b1f8dd..8d8d5d35a3b 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -16,12 +16,11 @@ #include #include #include +#include #include #include #include -#include "Common/FmtUtils.h" - namespace DB { namespace ErrorCodes @@ -235,8 +234,6 @@ void FilterBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t mul }, ", "); buffer.append("}"); - // ostr << "}]"; } - } // namespace DB From 3263384d88579c498d545375e5b7366aba84d006 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 18 May 2022 15:24:51 +0800 Subject: [PATCH 31/42] update. --- .../DataStreams/FilterBlockInputStream.cpp | 1 - .../HashJoinBuildBlockInputStream.cpp | 2 +- .../MergeSortingBlockInputStream.cpp | 10 +- .../ParallelAggregatingBlockInputStream.cpp | 10 +- .../PartialSortingBlockInputStream.cpp | 10 +- .../DataStreams/SharedQueryBlockInputStream.h | 8 - dbms/src/Flash/Coprocessor/DAGContext.h | 2 +- .../Coprocessor/DAGQueryBlockInterpreter.cpp | 20 +- .../Coprocessor/DAGQueryBlockInterpreter.h | 2 +- .../Coprocessor/DAGStorageInterpreter.cpp | 4 + dbms/src/Flash/Coprocessor/InterpreterDAG.cpp | 6 +- .../Flash/Coprocessor/InterpreterUtils.cpp | 10 + dbms/src/Flash/tests/gtest_interpreter.cpp | 174 +++++++++--------- dbms/src/Interpreters/ExpressionActions.cpp | 22 --- 14 files changed, 121 insertions(+), 160 deletions(-) diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index 8d8d5d35a3b..0624aea0eca 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -220,7 +220,6 @@ Block FilterBlockInputStream::readImpl() } } -// ywq todo complicate... void FilterBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const { IProfilingBlockInputStream::print(buffer, indent, multiplier); diff --git a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp index 8729398a27b..4e4a724b023 100644 --- a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp +++ b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp @@ -43,6 +43,6 @@ void HashJoinBuildBlockInputStream::print(FmtBuffer & buffer, size_t indent, siz auto join_type_it = join_type_map.find(join->getKind()); if (join_type_it == join_type_map.end()) throw TiFlashException("Unknown join type", Errors::Coprocessor::Internal); - buffer.fmtAppend(", build_concurrency{{{}}}, join_kind{{{}}}", join->getBuildConcurrency(), join_type_it->second); + buffer.fmtAppend(", join_kind = {}", join_type_it->second); } } // namespace DB diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp index 4ee965fd408..2e42726e27c 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -290,15 +290,7 @@ Block MergeSortingBlocksBlockInputStream::mergeImpl(std::priority_queue", des.column_name, des.direction == 1 ? "ASC" : "DESC"); - }, - ", "); - buffer.append("}"); + buffer.fmtAppend(", limit = {}", limit); } } // namespace DB diff --git a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 53d63da7464..2a97a71ad5b 100644 --- a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -278,18 +278,10 @@ void ParallelAggregatingBlockInputStream::execute() no_more_keys); } -// ywq todo void ParallelAggregatingBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const { IProfilingBlockInputStream::print(buffer, indent, multiplier); - buffer.fmtAppend(", max_threads: {}, final: {}, columns: {{", max_threads, final ? "true" : "false"); - const auto & aggregates = params.aggregates; - buffer.joinStr( - aggregates.cbegin(), - aggregates.cend(), - [](const auto & agg, FmtBuffer & fb) { fb.append(agg.column_name); }, - ", "); - buffer.append("}"); + buffer.fmtAppend(", max_threads: {}, final: {}", max_threads, final ? "true" : "false"); } } // namespace DB diff --git a/dbms/src/DataStreams/PartialSortingBlockInputStream.cpp b/dbms/src/DataStreams/PartialSortingBlockInputStream.cpp index 0bf792bb4e3..6625baa7066 100644 --- a/dbms/src/DataStreams/PartialSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/PartialSortingBlockInputStream.cpp @@ -29,14 +29,6 @@ Block PartialSortingBlockInputStream::readImpl() void PartialSortingBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const { IProfilingBlockInputStream::print(buffer, indent, multiplier); - buffer.fmtAppend(": limit = {}, columns {{", limit); - buffer.joinStr( - description.cbegin(), - description.cend(), - [](const auto & des, FmtBuffer & fb) { - fb.fmtAppend("<{}, {}>", des.column_name, des.direction == 1 ? "ASC" : "DESC"); - }, - ", "); - buffer.append("}"); + buffer.fmtAppend(": limit = {}", limit); } } // namespace DB diff --git a/dbms/src/DataStreams/SharedQueryBlockInputStream.h b/dbms/src/DataStreams/SharedQueryBlockInputStream.h index d2a1f05341c..e7cece67f0b 100644 --- a/dbms/src/DataStreams/SharedQueryBlockInputStream.h +++ b/dbms/src/DataStreams/SharedQueryBlockInputStream.h @@ -169,14 +169,6 @@ class SharedQueryBlockInputStream : public IProfilingBlockInputStream throw Exception(exception_msg); } - - // ywq todo - // void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override - // { - // IProfilingBlockInputStream::print(buffer, indent, multiplier); - // buffer.fmtAppend(": clients: {}", queue.size()); - // } - private: MPMCQueue queue; diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index d031ff103ff..e3e5efdcbc6 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -174,7 +174,7 @@ class DAGContext explicit DAGContext(const tipb::DAGRequest & dag_request_, String log_identifier, size_t concurrency) : dag_request(&dag_request_) , initialize_concurrency(concurrency) - , is_mpp_task(false) + , is_mpp_task(true) , is_root_mpp_task(false) , tunnel_set(nullptr) , log(Logger::get(log_identifier)) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 80905914ca4..a953e10f3fa 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -257,7 +257,7 @@ void getJoinKeyTypes(const tipb::Join & join, DataTypes & key_types) } } -void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & pipeline, SubqueryForSet & right_query) +void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & pipeline, SubqueryForSet & right_query, const String & executor_id) { // build static const std::unordered_map equal_join_type_map{ @@ -479,7 +479,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & }; right_pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, join_ptr, get_concurrency_build_index(), log->identifier()); - stream->setExtraInfo("Right join build"); + stream->setExtraInfo(fmt::format("Right join build, executor_id = {}", executor_id)); }); executeUnion(right_pipeline, max_streams, log, /*ignore_block=*/true, "for join"); @@ -513,6 +513,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & for (auto & stream : pipeline.streams) { stream = std::make_shared(stream, chain.getLastActions(), log->identifier()); + stream->setExtraInfo(fmt::format("executor_id = {}", executor_id)); } /// add a project to remove all the useless column @@ -552,7 +553,7 @@ void DAGQueryBlockInterpreter::executeWindow( executeExpression(pipeline, window_description.before_window, "before window"); /// If there are several streams, we merge them into one - executeUnion(pipeline, max_streams, log, "for window"); + executeUnion(pipeline, max_streams, log, false, "for window"); assert(pipeline.streams.size() == 1); pipeline.firstStream() = std::make_shared(pipeline.firstStream(), window_description, log->identifier()); } @@ -623,6 +624,7 @@ void DAGQueryBlockInterpreter::executeAggregation( void DAGQueryBlockInterpreter::executeExpression(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, String extra_info) { + // std::cout << "ywq test--------------------------------------- " << extra_info << std::endl; if (!expressionActionsPtr->getActions().empty()) { pipeline.transform([&](auto & stream) { @@ -660,7 +662,7 @@ void DAGQueryBlockInterpreter::orderStreams(DAGPipeline & pipeline, SortDescript }); /// If there are several streams, we merge them into one - executeUnion(pipeline, max_streams, log, "for order"); + executeUnion(pipeline, max_streams, log, false, "for order"); /// Merge the sorted blocks. pipeline.firstStream() = std::make_shared( @@ -740,9 +742,9 @@ void DAGQueryBlockInterpreter::handleProjection(DAGPipeline & pipeline, const ti } pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, chain.getLastActions(), log->identifier()); - stream->setExtraInfo("before project"); + stream->setExtraInfo("before projection"); }); - executeProject(pipeline, project_cols); + executeProject(pipeline, project_cols, "projection"); analyzer = std::make_unique(std::move(output_columns), context); } @@ -790,7 +792,7 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) if (query_block.source->tp() == tipb::ExecType::TypeJoin) { SubqueryForSet right_query; - handleJoin(query_block.source->join(), pipeline, right_query); + handleJoin(query_block.source->join(), pipeline, right_query, query_block.source->executor_id()); recordProfileStreams(pipeline, query_block.source_name); dagContext().addSubquery(query_block.source_name, std::move(right_query)); } @@ -920,7 +922,7 @@ void DAGQueryBlockInterpreter::executeLimit(DAGPipeline & pipeline) pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, limit, 0, log->identifier(), false); }); if (pipeline.hasMoreThanOneStream()) { - executeUnion(pipeline, max_streams, log, "for limit"); + executeUnion(pipeline, max_streams, log, false, "for limit"); pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, limit, 0, log->identifier(), false); }); } } @@ -967,7 +969,7 @@ BlockInputStreams DAGQueryBlockInterpreter::execute() executeImpl(pipeline); if (!pipeline.streams_with_non_joined_data.empty()) { - executeUnion(pipeline, max_streams, log, "final union"); + executeUnion(pipeline, max_streams, log, false, "final union"); restorePipelineConcurrency(pipeline); } diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h index d98829a9448..7cbeeec9720 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h @@ -60,7 +60,7 @@ class DAGQueryBlockInterpreter void executeImpl(DAGPipeline & pipeline); void handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline); void handleTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline); - void handleJoin(const tipb::Join & join, DAGPipeline & pipeline, SubqueryForSet & right_query); + void handleJoin(const tipb::Join & join, DAGPipeline & pipeline, SubqueryForSet & right_query, const String & executor_id); void prepareJoin( const google::protobuf::RepeatedPtrField & keys, const DataTypes & key_types, diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index f5354994b44..2247281898a 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -373,8 +373,10 @@ void DAGStorageInterpreter::executePushedDownFilter( { auto & stream = pipeline.streams[i]; stream = std::make_shared(stream, before_where, filter_column_name, log->identifier()); + stream->setExtraInfo("push down filter"); // after filter, do project action to keep the schema of local streams and remote streams the same. stream = std::make_shared(stream, project_after_where, log->identifier()); + stream->setExtraInfo("push down filter"); } } @@ -412,6 +414,7 @@ void DAGStorageInterpreter::executeCastAfterTableScan( { auto & stream = pipeline.streams[i++]; stream = std::make_shared(stream, extra_cast, log->identifier()); + stream->setExtraInfo("cast after tableScan"); } // remote streams if (i < pipeline.streams.size()) @@ -424,6 +427,7 @@ void DAGStorageInterpreter::executeCastAfterTableScan( { auto & stream = pipeline.streams[i++]; stream = std::make_shared(stream, project_for_cop_read, log->identifier()); + stream->setExtraInfo("cast after tableScan"); } } } diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index b7c75c06e67..741aa7b5e26 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -26,7 +26,7 @@ InterpreterDAG::InterpreterDAG(Context & context_, const DAGQuerySource & dag_) , dag(dag_) { const Settings & settings = context.getSettingsRef(); - if (dagContext().isBatchCop() || dagContext().isMPPTask()) + if (dagContext().isBatchCop() || (dagContext().isMPPTask() && !dagContext().isTest())) max_streams = settings.max_threads; else if (dagContext().isTest()) max_streams = dagContext().initialize_concurrency; @@ -85,9 +85,9 @@ BlockIO InterpreterDAG::execute() /// add union to run in parallel if needed if (dagContext().isMPPTask()) /// MPPTask do not need the returned blocks. - executeUnion(pipeline, max_streams, dagContext().log, /*ignore_block=*/true); + executeUnion(pipeline, max_streams, dagContext().log, /*ignore_block=*/true, "for mpp"); else - executeUnion(pipeline, max_streams, dagContext().log); + executeUnion(pipeline, max_streams, dagContext().log, false, "for non mpp"); if (dagContext().hasSubquery()) { const Settings & settings = context.getSettingsRef(); diff --git a/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp b/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp index 4de96b72c42..17112d4860a 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp @@ -34,6 +34,7 @@ void restoreConcurrency( { BlockInputStreamPtr shared_query_block_input_stream = std::make_shared(concurrency * 5, pipeline.firstStream(), log->identifier()); + shared_query_block_input_stream->setExtraInfo("restore concurrency"); pipeline.streams.assign(concurrency, shared_query_block_input_stream); } } @@ -46,13 +47,22 @@ BlockInputStreamPtr combinedNonJoinedDataStream( { BlockInputStreamPtr ret = nullptr; if (pipeline.streams_with_non_joined_data.size() == 1) + { ret = pipeline.streams_with_non_joined_data.at(0); + ret->setExtraInfo("non joined data"); + } else if (pipeline.streams_with_non_joined_data.size() > 1) { if (ignore_block) + { ret = std::make_shared(pipeline.streams_with_non_joined_data, nullptr, max_threads, log->identifier()); + ret->setExtraInfo("combine non joined(ignore block)"); + } else + { ret = std::make_shared(pipeline.streams_with_non_joined_data, nullptr, max_threads, log->identifier()); + ret->setExtraInfo("combine non joined"); + } } pipeline.streams_with_non_joined_data.clear(); return ret; diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index a349622b68b..7232eb34742 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -47,16 +47,16 @@ try .build(context); { String expected = R"( -Union - SharedQuery x 10 +Union: + SharedQuery x 10: Expression: - MergeSorting, limit = 10, columns {} - Union - PartialSorting x 10: limit = 10, columns {} + MergeSorting, limit = 10 + Union: + PartialSorting x 10: limit = 10 Expression: Filter: : {equals} - SharedQuery - ParallelAggregating, max_threads: 10, final: true, columns: {max(mock_table_scan_0)_collator_0 } + SharedQuery: + ParallelAggregating, max_threads: 10, final: true Expression x 10: Filter: : {equals} MockTableScan)"; @@ -72,16 +72,16 @@ Union { String expected = R"( -Union - SharedQuery x 10 +Union: + SharedQuery x 10: Limit, limit = 10 always_read_till_end = false - Union + Union: Limit x 10, limit = 10 always_read_till_end = false Expression: Expression: Filter: : {equals} - SharedQuery - ParallelAggregating, max_threads: 10, final: true, columns: {max(mock_table_scan_0)_collator_0 } + SharedQuery: + ParallelAggregating, max_threads: 10, final: true Expression x 10: Filter: : {equals} MockTableScan)"; @@ -100,16 +100,16 @@ try .build(context); { String expected = R"( -Union +Union: Expression x 10: - Expression - Expression: + Expression: + Expression: Expression: - Expression - Expression: + Expression: + Expression: Expression: - Expression - Expression: + Expression: + Expression: Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); @@ -122,17 +122,17 @@ Union .build(context); { String expected = R"( -Union +Union: Expression x 10: - Expression - Expression: - SharedQuery + Expression: + Expression: + SharedQuery: Expression: - MergeSorting, limit = 10, columns {<__QB_3_mock_table_scan_0, DESC>, <__QB_3_mock_table_scan_1, ASC>} - Union - PartialSorting x 10: limit = 10, columns {<__QB_3_mock_table_scan_0, DESC>, <__QB_3_mock_table_scan_1, ASC>} - Expression - Expression: + MergeSorting, limit = 10 + Union: + PartialSorting x 10: limit = 10 + Expression: + Expression: Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); @@ -147,23 +147,23 @@ Union .build(context); { String expected = R"( -Union +Union: Expression x 10: - Expression - Expression: + Expression: + Expression: Expression: - SharedQuery - ParallelAggregating, max_threads: 10, final: true, columns: {max(__QB_3___QB_4_mock_table_scan_0)_collator_0 } + SharedQuery: + ParallelAggregating, max_threads: 10, final: true Expression x 10: - Expression - Expression: - SharedQuery + Expression: + Expression: + SharedQuery: Expression: - MergeSorting, limit = 10, columns {<__QB_4_mock_table_scan_0, DESC>, <__QB_4_mock_table_scan_1, ASC>} - Union - PartialSorting x 10: limit = 10, columns {<__QB_4_mock_table_scan_0, DESC>, <__QB_4_mock_table_scan_1, ASC>} - Expression - Expression: + MergeSorting, limit = 10 + Union: + PartialSorting x 10: limit = 10 + Expression: + Expression: Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); @@ -181,32 +181,32 @@ Union .build(context); { String expected = R"( -Union - SharedQuery x 10 +Union: + SharedQuery x 10: Limit, limit = 10 always_read_till_end = false - Union + Union: Limit x 10, limit = 10 always_read_till_end = false Expression: - Expression - Expression: + Expression: + Expression: Expression: Expression: Filter: : {equals} - Expression - Expression: + Expression: + Expression: Expression: - SharedQuery - ParallelAggregating, max_threads: 10, final: true, columns: {max(__QB_4___QB_5_mock_table_scan_0)_collator_0 } + SharedQuery: + ParallelAggregating, max_threads: 10, final: true Expression x 10: - Expression - Expression: - SharedQuery + Expression: + Expression: + SharedQuery: Expression: - MergeSorting, limit = 10, columns {<__QB_5_mock_table_scan_0, DESC>, <__QB_5_mock_table_scan_1, ASC>} - Union - PartialSorting x 10: limit = 10, columns {<__QB_5_mock_table_scan_0, DESC>, <__QB_5_mock_table_scan_1, ASC>} - Expression - Expression: + MergeSorting, limit = 10 + Union: + PartialSorting x 10: limit = 10 + Expression: + Expression: Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); @@ -232,22 +232,22 @@ Union String expected = R"( CreatingSets Union: - HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: MockTableScan Union x 2: - HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: Expression: - HashJoinProbe + HashJoinProbe: Expression: MockTableScan - Union + Union: Expression x 10: Expression: - HashJoinProbe + HashJoinProbe: Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); @@ -260,16 +260,16 @@ CreatingSets .build(context); { String expected = R"( -Union +Union: Expression x 10: - Expression - Expression: + Expression: + Expression: Expression: - Expression - Expression: + Expression: + Expression: Expression: - Expression - Expression: + Expression: + Expression: Expression: MockExchangeReceiver)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); @@ -283,17 +283,17 @@ Union .build(context); { String expected = R"( -Union +Union: MockExchangeSender x 10 Expression: - Expression - Expression: + Expression: + Expression: Expression: - Expression - Expression: + Expression: + Expression: Expression: - Expression - Expression: + Expression: + Expression: Expression: MockExchangeReceiver)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); @@ -319,22 +319,22 @@ Union String expected = R"( CreatingSets Union: - HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: MockExchangeReceiver Union x 2: - HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: Expression: - HashJoinProbe + HashJoinProbe: Expression: MockExchangeReceiver - Union + Union: Expression x 10: Expression: - HashJoinProbe + HashJoinProbe: Expression: MockExchangeReceiver)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); @@ -361,23 +361,23 @@ CreatingSets String expected = R"( CreatingSets Union: - HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: MockExchangeReceiver Union x 2: - HashJoinBuildBlockInputStream x 10: , build_concurrency{10}, join_kind{Left} + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: Expression: - HashJoinProbe + HashJoinProbe: Expression: MockExchangeReceiver - Union + Union: MockExchangeSender x 10 Expression: Expression: - HashJoinProbe + HashJoinProbe: Expression: MockExchangeReceiver)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 3d6af2ebb76..14ce8c44904 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -461,49 +461,27 @@ void ExpressionAction::executeOnTotals(Block & block) const join->joinTotals(block); } -// ywq todo void ExpressionAction::dumpAction(FmtBuffer & fb) const { switch (type) { case ADD_COLUMN: fb.append("Add column"); - // fb.fmtAppend("Add {} {} {}", result_name, - // (result_type ? result_type->getName() : "(no type)"), - // (added_column ? added_column->getName() : "(no column)")); break; case REMOVE_COLUMN: fb.append("Remove column"); - // fb.fmtAppend("Remove {}", source_name); break; case COPY_COLUMN: fb.append("Copy column"); - // fb.fmtAppend("Copy {} = {}", result_name, source_name); break; case APPLY_FUNCTION: fb.fmtAppend("{}", (function ? function->getName() : "(no function)")); - // fb.joinStr( - // argument_names.cbegin(), - // argument_names.cend(), - // [](const auto & s, FmtBuffer & fb) { - // fb.append(s); - // }, - // ", "); break; case JOIN: fb.append("join"); - // fb.joinStr(columns_added_by_join.cbegin(), columns_added_by_join.cend(), [](const auto & c, FmtBuffer &fb){ - // fb.append(c.name); - // }, ", "); break; case PROJECT: fb.append("project"); - // fb.joinStr(projections.cbegin(), projections.cend(), [](const auto & proj, FmtBuffer & fb) { - // fb.append(proj.first); - // // if (!proj.second.empty() && proj.first != proj.second) - // // fb.fmtAppend(" as {}", proj.second); - // }, ", "); - // fb.append("}"); break; default: break; From 6364a96c776d6add6976c3a4e1e4b2d6da668f3f Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 18 May 2022 15:29:15 +0800 Subject: [PATCH 32/42] tiny refine. --- dbms/src/DataStreams/ExpressionBlockInputStream.h | 2 -- dbms/src/DataStreams/IBlockInputStream.h | 1 - dbms/src/DataStreams/LimitBlockInputStream.cpp | 1 - dbms/src/DataStreams/LimitBlockInputStream.h | 1 - dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h | 1 - dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp | 1 - dbms/src/Interpreters/executeQuery.cpp | 6 +++--- 7 files changed, 3 insertions(+), 10 deletions(-) diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.h b/dbms/src/DataStreams/ExpressionBlockInputStream.h index d6ee6d81036..eb08ad6eda3 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.h +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.h @@ -15,8 +15,6 @@ #pragma once #include -#include - namespace DB { diff --git a/dbms/src/DataStreams/IBlockInputStream.h b/dbms/src/DataStreams/IBlockInputStream.h index 0b4af904f79..88ea5c10fad 100644 --- a/dbms/src/DataStreams/IBlockInputStream.h +++ b/dbms/src/DataStreams/IBlockInputStream.h @@ -137,7 +137,6 @@ class IBlockInputStream : private boost::noncopyable void setExtraInfo(String info) { extra_info = info; } - template void forEachChild(F && f) { diff --git a/dbms/src/DataStreams/LimitBlockInputStream.cpp b/dbms/src/DataStreams/LimitBlockInputStream.cpp index 99ec1bed705..26f05f6baef 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.cpp +++ b/dbms/src/DataStreams/LimitBlockInputStream.cpp @@ -88,5 +88,4 @@ void LimitBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t mult IProfilingBlockInputStream::print(buffer, indent, multiplier); buffer.fmtAppend(", limit = {} always_read_till_end = {}", limit, always_read_till_end); } - } // namespace DB diff --git a/dbms/src/DataStreams/LimitBlockInputStream.h b/dbms/src/DataStreams/LimitBlockInputStream.h index e4338288efc..237907c6fc3 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.h +++ b/dbms/src/DataStreams/LimitBlockInputStream.h @@ -45,7 +45,6 @@ class LimitBlockInputStream : public IProfilingBlockInputStream Block readImpl() override; void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; - private: size_t limit; size_t offset; diff --git a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h index 72f800c4baa..774193d78dc 100644 --- a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h +++ b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.h @@ -19,7 +19,6 @@ #include #include #include -#include namespace DB { diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index a953e10f3fa..42d4d366be0 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -624,7 +624,6 @@ void DAGQueryBlockInterpreter::executeAggregation( void DAGQueryBlockInterpreter::executeExpression(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, String extra_info) { - // std::cout << "ywq test--------------------------------------- " << extra_info << std::endl; if (!expressionActionsPtr->getActions().empty()) { pipeline.transform([&](auto & stream) { diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 6c96e7c22ad..1c0b19b199f 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -187,7 +187,7 @@ std::tuple executeQueryImpl( } catch (...) { - if (!internal) + if (!internal && !context.getDAGContext()->isTest()) { /// Anyway log the query. String str = query_src.str(max_query_size); @@ -202,7 +202,7 @@ std::tuple executeQueryImpl( try { - if (!internal) + if (!internal && !context.getDAGContext()->isTest()) logQuery(query.substr(0, settings.log_queries_cut_to_length), context, execute_query_logger); /// Check the limits. @@ -384,7 +384,7 @@ std::tuple executeQueryImpl( } }; - if (!internal && res.in) + if (!internal && res.in && !context.getDAGContext()->isTest()) { auto pipeline_log_str = [&res]() { FmtBuffer log_buffer; From ec10a476670bc193d2e57f3aa976c95b18fa9167 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 19 May 2022 19:49:37 +0800 Subject: [PATCH 33/42] update. --- dbms/src/Flash/tests/gtest_interpreter.cpp | 10 ++++----- dbms/src/Interpreters/ExpressionActions.cpp | 25 ++------------------- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 7232eb34742..db31f3ec761 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -54,11 +54,11 @@ Union: Union: PartialSorting x 10: limit = 10 Expression: - Filter: : {equals} + Filter: : {equals->Nullable(UInt8)} SharedQuery: ParallelAggregating, max_threads: 10, final: true Expression x 10: - Filter: : {equals} + Filter: : {equals->Nullable(UInt8)} MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -79,11 +79,11 @@ Union: Limit x 10, limit = 10 always_read_till_end = false Expression: Expression: - Filter: : {equals} + Filter: : {equals->Nullable(UInt8)} SharedQuery: ParallelAggregating, max_threads: 10, final: true Expression x 10: - Filter: : {equals} + Filter: : {equals->Nullable(UInt8)} MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -191,7 +191,7 @@ Union: Expression: Expression: Expression: - Filter: : {equals} + Filter: : {equals->Nullable(UInt8)} Expression: Expression: Expression: diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 14ce8c44904..fd9e2447afd 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -463,29 +463,8 @@ void ExpressionAction::executeOnTotals(Block & block) const void ExpressionAction::dumpAction(FmtBuffer & fb) const { - switch (type) - { - case ADD_COLUMN: - fb.append("Add column"); - break; - case REMOVE_COLUMN: - fb.append("Remove column"); - break; - case COPY_COLUMN: - fb.append("Copy column"); - break; - case APPLY_FUNCTION: - fb.fmtAppend("{}", (function ? function->getName() : "(no function)")); - break; - case JOIN: - fb.append("join"); - break; - case PROJECT: - fb.append("project"); - break; - default: - break; - } + if (type == APPLY_FUNCTION) + fb.fmtAppend("{}{}", (function ? function->getName() : "(no function)"), (result_type ? "->" + result_type->getName() : "")); } String ExpressionAction::toString() const From 5876a9bc28797e67be507fa8b337f66793c11a6d Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 19 May 2022 19:53:47 +0800 Subject: [PATCH 34/42] little refine. --- dbms/src/DataStreams/ExpressionBlockInputStream.h | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.h b/dbms/src/DataStreams/ExpressionBlockInputStream.h index eb08ad6eda3..1d1e059d51d 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.h +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.h @@ -16,6 +16,7 @@ #include + namespace DB { class ExpressionActions; From 428f14769dbd71bf268ddc2f74c08cc427c20968 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Mon, 23 May 2022 20:21:09 +0800 Subject: [PATCH 35/42] address comments. --- .../DataStreams/FilterBlockInputStream.cpp | 4 +-- dbms/src/DataStreams/FilterBlockInputStream.h | 2 +- .../HashJoinBuildBlockInputStream.cpp | 3 +- .../HashJoinBuildBlockInputStream.h | 2 +- dbms/src/DataStreams/IBlockInputStream.cpp | 8 ++--- dbms/src/DataStreams/IBlockInputStream.h | 3 +- .../src/DataStreams/LimitBlockInputStream.cpp | 3 +- dbms/src/DataStreams/LimitBlockInputStream.h | 2 +- .../MergeSortingBlockInputStream.cpp | 3 +- .../MergeSortingBlockInputStream.h | 2 +- .../ParallelAggregatingBlockInputStream.cpp | 3 +- .../ParallelAggregatingBlockInputStream.h | 4 +-- .../PartialSortingBlockInputStream.cpp | 3 +- .../PartialSortingBlockInputStream.h | 2 +- .../DataStreams/TiRemoteBlockInputStream.h | 9 +++-- .../Coprocessor/DAGQueryBlockInterpreter.cpp | 17 ++++----- .../Flash/Coprocessor/InterpreterUtils.cpp | 1 - dbms/src/Flash/tests/gtest_interpreter.cpp | 36 +++++++++---------- dbms/src/Interpreters/ExpressionActions.cpp | 18 +++++----- 19 files changed, 58 insertions(+), 67 deletions(-) diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index 0624aea0eca..2f0f78555ee 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -220,9 +220,9 @@ Block FilterBlockInputStream::readImpl() } } -void FilterBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const + +void FilterBlockInputStream::appendInfo(FmtBuffer & buffer) const { - IProfilingBlockInputStream::print(buffer, indent, multiplier); buffer.append(": {"); const auto & actions = expression->getActions(); buffer.joinStr( diff --git a/dbms/src/DataStreams/FilterBlockInputStream.h b/dbms/src/DataStreams/FilterBlockInputStream.h index 2f0f8edf690..1aef7fda7bb 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.h +++ b/dbms/src/DataStreams/FilterBlockInputStream.h @@ -47,7 +47,7 @@ class FilterBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; - void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; + void appendInfo(FmtBuffer & buffer) const override; private: diff --git a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp index 4e4a724b023..ad4ebe65d02 100644 --- a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp +++ b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp @@ -26,9 +26,8 @@ Block HashJoinBuildBlockInputStream::readImpl() return block; } -void HashJoinBuildBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const +void HashJoinBuildBlockInputStream::appendInfo(FmtBuffer & buffer) const { - IProfilingBlockInputStream::print(buffer, indent, multiplier); static const std::unordered_map join_type_map{ {ASTTableJoin::Kind::Inner, "Inner"}, {ASTTableJoin::Kind::Left, "Left"}, diff --git a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.h b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.h index 52498a6834c..dbfc7f30310 100644 --- a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.h +++ b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.h @@ -41,7 +41,7 @@ class HashJoinBuildBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; - void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; + void appendInfo(FmtBuffer & buffer) const override; private: JoinPtr join; diff --git a/dbms/src/DataStreams/IBlockInputStream.cpp b/dbms/src/DataStreams/IBlockInputStream.cpp index 458574a7605..a05fbf83c96 100644 --- a/dbms/src/DataStreams/IBlockInputStream.cpp +++ b/dbms/src/DataStreams/IBlockInputStream.cpp @@ -77,7 +77,7 @@ size_t IBlockInputStream::checkDepthImpl(size_t max_depth, size_t level) const return res + 1; } -void IBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const +void IBlockInputStream::dumpTree(FmtBuffer & buffer, size_t indent, size_t multiplier) { buffer.fmtAppend( "{}{}{}", @@ -86,11 +86,7 @@ void IBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multipli multiplier > 1 ? fmt::format(" x {}", multiplier) : ""); if (!extra_info.empty()) buffer.fmtAppend(": <{}>", extra_info); -} - -void IBlockInputStream::dumpTree(FmtBuffer & buffer, size_t indent, size_t multiplier) -{ - print(buffer, indent, multiplier); + appendInfo(buffer); buffer.append("\n"); ++indent; diff --git a/dbms/src/DataStreams/IBlockInputStream.h b/dbms/src/DataStreams/IBlockInputStream.h index 88ea5c10fad..2c2fa5724a5 100644 --- a/dbms/src/DataStreams/IBlockInputStream.h +++ b/dbms/src/DataStreams/IBlockInputStream.h @@ -177,11 +177,12 @@ class IBlockInputStream : private boost::noncopyable } } + virtual void appendInfo(FmtBuffer & buffer [[maybe_unused]]) const {}; + protected: BlockInputStreams children; mutable std::shared_mutex children_mutex; bool collected = false; // a flag to avoid duplicated collecting, since some InputStream is shared by multiple inputStreams - virtual void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const; private: TableLockHolders table_locks; diff --git a/dbms/src/DataStreams/LimitBlockInputStream.cpp b/dbms/src/DataStreams/LimitBlockInputStream.cpp index 26f05f6baef..87ac9067e1c 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.cpp +++ b/dbms/src/DataStreams/LimitBlockInputStream.cpp @@ -83,9 +83,8 @@ Block LimitBlockInputStream::readImpl() return res; } -void LimitBlockInputStream::print(FmtBuffer & buffer, size_t indent, size_t multiplier) const +void LimitBlockInputStream::appendInfo(FmtBuffer & buffer) const { - IProfilingBlockInputStream::print(buffer, indent, multiplier); buffer.fmtAppend(", limit = {} always_read_till_end = {}", limit, always_read_till_end); } } // namespace DB diff --git a/dbms/src/DataStreams/LimitBlockInputStream.h b/dbms/src/DataStreams/LimitBlockInputStream.h index 237907c6fc3..e6a7013210b 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.h +++ b/dbms/src/DataStreams/LimitBlockInputStream.h @@ -43,7 +43,7 @@ class LimitBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; - void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override; + void appendInfo(FmtBuffer & buffer) const override; private: size_t limit; diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp index 2e42726e27c..e79426f686e 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -287,9 +287,8 @@ Block MergeSortingBlocksBlockInputStream::mergeImpl(std::priority_queue & getConnectionProfileInfos() const { return connection_profile_infos; } - virtual void collectNewThreadCountOfThisLevel(int & cnt) override + void collectNewThreadCountOfThisLevel(int & cnt) override { remote_reader->collectNewThreadCount(cnt); } - virtual void resetNewThreadCountCompute() override + void resetNewThreadCountCompute() override { if (collected) { @@ -240,15 +240,14 @@ class TiRemoteBlockInputStream : public IProfilingBlockInputStream } protected: - virtual void readSuffixImpl() override + void readSuffixImpl() override { LOG_FMT_DEBUG(log, "finish read {} rows from remote", total_rows); remote_reader->close(); } - void print(FmtBuffer & buffer, size_t indent, size_t multiplier) const override + void appendInfo(FmtBuffer & buffer) const override { - IProfilingBlockInputStream::print(buffer, indent, multiplier); buffer.append(": schema: {"); buffer.joinStr( sample_block.begin(), diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 42d4d366be0..f383d0437c1 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -479,7 +479,8 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & }; right_pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, join_ptr, get_concurrency_build_index(), log->identifier()); - stream->setExtraInfo(fmt::format("Right join build, executor_id = {}", executor_id)); + stream->setExtraInfo( + fmt::format("join build, root_executor_id = {}", dagContext().getJoinExecuteInfoMap()[query_block.source_name].build_side_root_executor_id)); }); executeUnion(right_pipeline, max_streams, log, /*ignore_block=*/true, "for join"); @@ -513,7 +514,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & for (auto & stream : pipeline.streams) { stream = std::make_shared(stream, chain.getLastActions(), log->identifier()); - stream->setExtraInfo(fmt::format("executor_id = {}", executor_id)); + stream->setExtraInfo(fmt::format("join probe, join_executor_id = {}", executor_id)); } /// add a project to remove all the useless column @@ -553,7 +554,7 @@ void DAGQueryBlockInterpreter::executeWindow( executeExpression(pipeline, window_description.before_window, "before window"); /// If there are several streams, we merge them into one - executeUnion(pipeline, max_streams, log, false, "for window"); + executeUnion(pipeline, max_streams, log, false, "merge into one for window input"); assert(pipeline.streams.size() == 1); pipeline.firstStream() = std::make_shared(pipeline.firstStream(), window_description, log->identifier()); } @@ -661,7 +662,7 @@ void DAGQueryBlockInterpreter::orderStreams(DAGPipeline & pipeline, SortDescript }); /// If there are several streams, we merge them into one - executeUnion(pipeline, max_streams, log, false, "for order"); + executeUnion(pipeline, max_streams, log, false, "for partial order"); /// Merge the sorted blocks. pipeline.firstStream() = std::make_shared( @@ -692,7 +693,7 @@ void DAGQueryBlockInterpreter::handleExchangeReceiver(DAGPipeline & pipeline) BlockInputStreamPtr stream = std::make_shared(it->second, log->identifier(), query_block.source_name); exchange_receiver_io_input_streams.push_back(stream); stream = std::make_shared(stream, 8192, 0, log->identifier()); - stream->setExtraInfo("after exchange receiver"); + stream->setExtraInfo("squashing after exchange receiver"); pipeline.streams.push_back(stream); } NamesAndTypes source_columns; @@ -757,7 +758,7 @@ void DAGQueryBlockInterpreter::handleWindow(DAGPipeline & pipeline, const tipb:: DAGExpressionAnalyzer dag_analyzer(input_columns, context); WindowDescription window_description = dag_analyzer.buildWindowDescription(window); executeWindow(pipeline, window_description); - executeExpression(pipeline, window_description.after_window, "after window"); + executeExpression(pipeline, window_description.after_window, "cast after window"); analyzer = std::make_unique(window_description.after_window_columns, context); } @@ -921,7 +922,7 @@ void DAGQueryBlockInterpreter::executeLimit(DAGPipeline & pipeline) pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, limit, 0, log->identifier(), false); }); if (pipeline.hasMoreThanOneStream()) { - executeUnion(pipeline, max_streams, log, false, "for limit"); + executeUnion(pipeline, max_streams, log, false, "for partial limit"); pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, limit, 0, log->identifier(), false); }); } } @@ -968,7 +969,7 @@ BlockInputStreams DAGQueryBlockInterpreter::execute() executeImpl(pipeline); if (!pipeline.streams_with_non_joined_data.empty()) { - executeUnion(pipeline, max_streams, log, false, "final union"); + executeUnion(pipeline, max_streams, log, false, "final union for non_joined_data"); restorePipelineConcurrency(pipeline); } diff --git a/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp b/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp index 17112d4860a..da5aaca9aa6 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp @@ -49,7 +49,6 @@ BlockInputStreamPtr combinedNonJoinedDataStream( if (pipeline.streams_with_non_joined_data.size() == 1) { ret = pipeline.streams_with_non_joined_data.at(0); - ret->setExtraInfo("non joined data"); } else if (pipeline.streams_with_non_joined_data.size() > 1) { diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index db31f3ec761..016a1cb1af6 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -51,7 +51,7 @@ Union: SharedQuery x 10: Expression: MergeSorting, limit = 10 - Union: + Union: PartialSorting x 10: limit = 10 Expression: Filter: : {equals->Nullable(UInt8)} @@ -75,7 +75,7 @@ Union: Union: SharedQuery x 10: Limit, limit = 10 always_read_till_end = false - Union: + Union: Limit x 10, limit = 10 always_read_till_end = false Expression: Expression: @@ -129,7 +129,7 @@ Union: SharedQuery: Expression: MergeSorting, limit = 10 - Union: + Union: PartialSorting x 10: limit = 10 Expression: Expression: @@ -160,7 +160,7 @@ Union: SharedQuery: Expression: MergeSorting, limit = 10 - Union: + Union: PartialSorting x 10: limit = 10 Expression: Expression: @@ -184,7 +184,7 @@ Union: Union: SharedQuery x 10: Limit, limit = 10 always_read_till_end = false - Union: + Union: Limit x 10, limit = 10 always_read_till_end = false Expression: Expression: @@ -203,7 +203,7 @@ Union: SharedQuery: Expression: MergeSorting, limit = 10 - Union: + Union: PartialSorting x 10: limit = 10 Expression: Expression: @@ -232,22 +232,22 @@ Union: String expected = R"( CreatingSets Union: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: MockTableScan Union x 2: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: Expression: - HashJoinProbe: + HashJoinProbe: Expression: MockTableScan Union: Expression x 10: Expression: - HashJoinProbe: + HashJoinProbe: Expression: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); @@ -319,22 +319,22 @@ Union: String expected = R"( CreatingSets Union: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: MockExchangeReceiver Union x 2: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: Expression: - HashJoinProbe: + HashJoinProbe: Expression: MockExchangeReceiver Union: Expression x 10: Expression: - HashJoinProbe: + HashJoinProbe: Expression: MockExchangeReceiver)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); @@ -361,23 +361,23 @@ CreatingSets String expected = R"( CreatingSets Union: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: MockExchangeReceiver Union x 2: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: Expression: - HashJoinProbe: + HashJoinProbe: Expression: MockExchangeReceiver Union: MockExchangeSender x 10 Expression: Expression: - HashJoinProbe: + HashJoinProbe: Expression: MockExchangeReceiver)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index fd9e2447afd..0a3887ed84a 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -225,7 +225,7 @@ void ExpressionAction::prepare(Block & sample_block) for (const auto & name : array_joined_columns) { ColumnWithTypeAndName & current = sample_block.getByName(name); - const DataTypeArray * array_type = typeid_cast(&*current.type); + const auto * array_type = typeid_cast(&*current.type); if (!array_type) throw Exception("ARRAY JOIN requires array argument", ErrorCodes::TYPE_MISMATCH); current.type = array_type->getNestedType(); @@ -354,7 +354,7 @@ void ExpressionAction::execute(Block & block) const if (ColumnPtr converted = any_array_ptr->convertToFullColumnIfConst()) any_array_ptr = converted; - const ColumnArray * any_array = typeid_cast(&*any_array_ptr); + const auto * any_array = typeid_cast(&*any_array_ptr); if (!any_array) throw Exception("ARRAY JOIN of not array: " + *array_joined_columns.begin(), ErrorCodes::TYPE_MISMATCH); @@ -464,7 +464,7 @@ void ExpressionAction::executeOnTotals(Block & block) const void ExpressionAction::dumpAction(FmtBuffer & fb) const { if (type == APPLY_FUNCTION) - fb.fmtAppend("{}{}", (function ? function->getName() : "(no function)"), (result_type ? "->" + result_type->getName() : "")); + fb.fmtAppend("{}", (function ? function->getName() : "(no function)")); } String ExpressionAction::toString() const @@ -501,7 +501,7 @@ String ExpressionAction::toString() const case ARRAY_JOIN: ss << (array_join_is_left ? "LEFT " : "") << "ARRAY JOIN "; - for (NameSet::const_iterator it = array_joined_columns.begin(); it != array_joined_columns.end(); ++it) + for (auto it = array_joined_columns.begin(); it != array_joined_columns.end(); ++it) { if (it != array_joined_columns.begin()) ss << ", "; @@ -511,7 +511,7 @@ String ExpressionAction::toString() const case JOIN: ss << "JOIN "; - for (NamesAndTypesList::const_iterator it = columns_added_by_join.begin(); it != columns_added_by_join.end(); ++it) + for (auto it = columns_added_by_join.begin(); it != columns_added_by_join.end(); ++it) { if (it != columns_added_by_join.begin()) ss << ", "; @@ -846,9 +846,9 @@ void ExpressionActions::finalize(const Names & output_columns) if (final_columns.empty() && !input_columns.empty()) final_columns.insert(getSmallestColumn(input_columns)); - for (NamesAndTypesList::iterator it = input_columns.begin(); it != input_columns.end();) + for (auto it = input_columns.begin(); it != input_columns.end();) { - NamesAndTypesList::iterator it0 = it; + auto it0 = it; ++it; if (!needed_columns.count(it0->name)) { @@ -935,8 +935,8 @@ std::string ExpressionActions::dumpActions() const ss << "\noutput:\n"; NamesAndTypesList output_columns = sample_block.getNamesAndTypesList(); - for (NamesAndTypesList::const_iterator it = output_columns.begin(); it != output_columns.end(); ++it) - ss << it->name << " " << it->type->getName() << "\n"; + for (auto & output_column : output_columns) + ss << output_column.name << " " << output_column.type->getName() << "\n"; return ss.str(); } From 2c569c4227a5819618bf5c83a2f57b36d38bfbb7 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 24 May 2022 10:24:13 +0800 Subject: [PATCH 36/42] fix clang-tidy. --- dbms/src/DataStreams/FilterBlockInputStream.cpp | 2 +- dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp | 2 +- dbms/src/Interpreters/executeQuery.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index 2f0f78555ee..0ee85ffecf3 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -61,7 +61,7 @@ FilterBlockInputStream::FilterBlockInputStream( Block FilterBlockInputStream::getTotals() { - if (IProfilingBlockInputStream * child = dynamic_cast(&*children.back())) + if (auto * child = dynamic_cast(&*children.back())) { totals = child->getTotals(); expression->executeOnTotals(totals); diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index 2247281898a..8af4b509842 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -186,7 +186,7 @@ void setQuotaAndLimitsOnTableScan(Context & context, DAGPipeline & pipeline) QuotaForIntervals & quota = context.getQuota(); pipeline.transform([&](auto & stream) { - if (IProfilingBlockInputStream * p_stream = dynamic_cast(stream.get())) + if (auto * p_stream = dynamic_cast(stream.get())) { p_stream->setLimits(limits); p_stream->setQuota(quota); diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 1c0b19b199f..4769f00cdd8 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -480,7 +480,7 @@ void executeQuery( if (streams.in) { - const ASTQueryWithOutput * ast_query_with_output = dynamic_cast(ast.get()); + const auto * ast_query_with_output = dynamic_cast(ast.get()); WriteBuffer * out_buf = &ostr; std::optional out_file_buf; From c537d6e60e4378e627470e34b2e48bd1213689d5 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 24 May 2022 10:54:58 +0800 Subject: [PATCH 37/42] fix clang-tidy. --- dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index f383d0437c1..f75b73f83ac 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -91,7 +91,7 @@ struct AnalysisResult Names aggregation_keys; TiDB::TiDBCollators aggregation_collators; AggregateDescriptions aggregate_descriptions; - bool is_final_agg; + bool is_final_agg{}; }; AnalysisResult analyzeExpressions( From 6f7f79c584d45ebcb0d45730e02bfcc29a6ce173 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 24 May 2022 19:50:57 +0800 Subject: [PATCH 38/42] address comments. --- .../src/DataStreams/FilterBlockInputStream.cpp | 15 --------------- dbms/src/DataStreams/FilterBlockInputStream.h | 2 -- dbms/src/DataStreams/IBlockInputStream.h | 2 +- dbms/src/DataStreams/LimitBlockInputStream.cpp | 2 +- .../Coprocessor/DAGQueryBlockInterpreter.cpp | 12 ++++++------ .../Coprocessor/DAGQueryBlockInterpreter.h | 2 +- dbms/src/Flash/tests/gtest_interpreter.cpp | 18 +++++++++--------- dbms/src/Interpreters/ExpressionActions.cpp | 12 ------------ dbms/src/Interpreters/ExpressionActions.h | 4 ---- 9 files changed, 18 insertions(+), 51 deletions(-) diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index 0ee85ffecf3..fb6f644b2fa 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -220,19 +220,4 @@ Block FilterBlockInputStream::readImpl() } } - -void FilterBlockInputStream::appendInfo(FmtBuffer & buffer) const -{ - buffer.append(": {"); - const auto & actions = expression->getActions(); - buffer.joinStr( - actions.begin(), - actions.end(), - [](const auto & arg, FmtBuffer & fb) { - arg.dumpAction(fb); - }, - ", "); - buffer.append("}"); -} - } // namespace DB diff --git a/dbms/src/DataStreams/FilterBlockInputStream.h b/dbms/src/DataStreams/FilterBlockInputStream.h index 1aef7fda7bb..3d7ec131f47 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.h +++ b/dbms/src/DataStreams/FilterBlockInputStream.h @@ -47,8 +47,6 @@ class FilterBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; - void appendInfo(FmtBuffer & buffer) const override; - private: ExpressionActionsPtr expression; diff --git a/dbms/src/DataStreams/IBlockInputStream.h b/dbms/src/DataStreams/IBlockInputStream.h index 2c2fa5724a5..472eac282d4 100644 --- a/dbms/src/DataStreams/IBlockInputStream.h +++ b/dbms/src/DataStreams/IBlockInputStream.h @@ -177,7 +177,7 @@ class IBlockInputStream : private boost::noncopyable } } - virtual void appendInfo(FmtBuffer & buffer [[maybe_unused]]) const {}; + virtual void appendInfo(FmtBuffer & /*buffer*/) const {}; protected: BlockInputStreams children; diff --git a/dbms/src/DataStreams/LimitBlockInputStream.cpp b/dbms/src/DataStreams/LimitBlockInputStream.cpp index 87ac9067e1c..4ec6157257c 100644 --- a/dbms/src/DataStreams/LimitBlockInputStream.cpp +++ b/dbms/src/DataStreams/LimitBlockInputStream.cpp @@ -85,6 +85,6 @@ Block LimitBlockInputStream::readImpl() void LimitBlockInputStream::appendInfo(FmtBuffer & buffer) const { - buffer.fmtAppend(", limit = {} always_read_till_end = {}", limit, always_read_till_end); + buffer.fmtAppend(", limit = {}", limit); } } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 04fb8213c18..f9ed379b4e9 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -179,7 +179,7 @@ void DAGQueryBlockInterpreter::handleTableScan(const TiDBTableScan & table_scan, analyzer = std::move(storage_interpreter.analyzer); } -void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & pipeline, SubqueryForSet & right_query, const String & executor_id) +void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & pipeline, SubqueryForSet & right_query) { if (unlikely(input_streams_vec.size() != 2)) { @@ -258,7 +258,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & size_t join_build_concurrency = settings.join_concurrent_build ? std::min(max_streams, build_pipeline.streams.size()) : 1; /// build side streams - executeExpression(build_pipeline, build_side_prepare_actions); + executeExpression(build_pipeline, build_side_prepare_actions, "Append Join key and join Filters"); // add a HashJoinBuildBlockInputStream to build a shared hash table auto get_concurrency_build_index = JoinInterpreterHelper::concurrencyBuildIndexGenerator(join_build_concurrency); build_pipeline.transform([&](auto & stream) { @@ -273,7 +273,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & join_ptr->init(right_query.source->getHeader(), join_build_concurrency); /// probe side streams - executeExpression(probe_pipeline, probe_side_prepare_actions); + executeExpression(probe_pipeline, probe_side_prepare_actions, "probe side prepare actions"); NamesAndTypes source_columns; for (const auto & p : probe_pipeline.firstStream()->getHeader()) source_columns.emplace_back(p.name, p.type); @@ -293,7 +293,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & i, not_joined_concurrency, settings.max_block_size); - non_joined_stream->setExtraInfo("Add Stream With NonJoinedData If FullOrRightJoin"); + non_joined_stream->setExtraInfo("add stream with non_joined_data if full_or_right_join"); pipeline.streams_with_non_joined_data.push_back(non_joined_stream); join_execute_info.non_joined_streams.push_back(non_joined_stream); } @@ -301,7 +301,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & for (auto & stream : pipeline.streams) { stream = std::make_shared(stream, chain.getLastActions(), log->identifier()); - stream->setExtraInfo(fmt::format("join probe, join_executor_id = {}", executor_id)); + stream->setExtraInfo(fmt::format("join probe, join_executor_id = {}", query_block.source_name)); } /// add a project to remove all the useless column @@ -579,7 +579,7 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) if (query_block.source->tp() == tipb::ExecType::TypeJoin) { SubqueryForSet right_query; - handleJoin(query_block.source->join(), pipeline, right_query, query_block.source->executor_id()); + handleJoin(query_block.source->join(), pipeline, right_query); recordProfileStreams(pipeline, query_block.source_name); dagContext().addSubquery(query_block.source_name, std::move(right_query)); } diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h index 87ee03f41cd..2350b0ffd40 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h @@ -60,7 +60,7 @@ class DAGQueryBlockInterpreter void executeImpl(DAGPipeline & pipeline); void handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline); void handleTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline); - void handleJoin(const tipb::Join & join, DAGPipeline & pipeline, SubqueryForSet & right_query, const String & executor_id); + void handleJoin(const tipb::Join & join, DAGPipeline & pipeline, SubqueryForSet & right_query); void handleExchangeReceiver(DAGPipeline & pipeline); void handleMockExchangeReceiver(DAGPipeline & pipeline); void handleProjection(DAGPipeline & pipeline, const tipb::Projection & projection); diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 016a1cb1af6..097bfc76355 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -54,11 +54,11 @@ Union: Union: PartialSorting x 10: limit = 10 Expression: - Filter: : {equals->Nullable(UInt8)} + Filter: SharedQuery: ParallelAggregating, max_threads: 10, final: true Expression x 10: - Filter: : {equals->Nullable(UInt8)} + Filter: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -74,16 +74,16 @@ Union: String expected = R"( Union: SharedQuery x 10: - Limit, limit = 10 always_read_till_end = false + Limit, limit = 10 Union: - Limit x 10, limit = 10 always_read_till_end = false + Limit x 10, limit = 10 Expression: Expression: - Filter: : {equals->Nullable(UInt8)} + Filter: SharedQuery: ParallelAggregating, max_threads: 10, final: true Expression x 10: - Filter: : {equals->Nullable(UInt8)} + Filter: MockTableScan)"; ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); } @@ -183,15 +183,15 @@ Union: String expected = R"( Union: SharedQuery x 10: - Limit, limit = 10 always_read_till_end = false + Limit, limit = 10 Union: - Limit x 10, limit = 10 always_read_till_end = false + Limit x 10, limit = 10 Expression: Expression: Expression: Expression: Expression: - Filter: : {equals->Nullable(UInt8)} + Filter: Expression: Expression: Expression: diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 0a3887ed84a..dbdfd69fbbe 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -461,12 +461,6 @@ void ExpressionAction::executeOnTotals(Block & block) const join->joinTotals(block); } -void ExpressionAction::dumpAction(FmtBuffer & fb) const -{ - if (type == APPLY_FUNCTION) - fb.fmtAppend("{}", (function ? function->getName() : "(no function)")); -} - String ExpressionAction::toString() const { std::stringstream ss; @@ -941,12 +935,6 @@ std::string ExpressionActions::dumpActions() const return ss.str(); } -void ExpressionActions::dumpActions(FmtBuffer & fb) const -{ - for (const auto & action : actions) - action.dumpAction(fb); -} - void ExpressionActions::optimize() { optimizeArrayJoin(); diff --git a/dbms/src/Interpreters/ExpressionActions.h b/dbms/src/Interpreters/ExpressionActions.h index e98960a1d33..e8fb48f4e3f 100644 --- a/dbms/src/Interpreters/ExpressionActions.h +++ b/dbms/src/Interpreters/ExpressionActions.h @@ -121,8 +121,6 @@ struct ExpressionAction std::string toString() const; - void dumpAction(FmtBuffer & fb) const; - private: friend class ExpressionActions; @@ -214,8 +212,6 @@ class ExpressionActions std::string dumpActions() const; - void dumpActions(FmtBuffer & fb) const; - static std::string getSmallestColumn(const NamesAndTypesList & columns); BlockInputStreamPtr createStreamWithNonJoinedDataIfFullOrRightJoin(const Block & source_header, size_t index, size_t step, size_t max_block_size) const; From 985be0f6692abe7fdaa2a03521f699c8cb3eecad Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 25 May 2022 10:38:44 +0800 Subject: [PATCH 39/42] address comments --- .../Flash/Coprocessor/DAGQueryBlockInterpreter.cpp | 2 +- dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp | 6 +++--- dbms/src/Flash/tests/gtest_interpreter.cpp | 12 ++++++------ dbms/src/Interpreters/ExpressionActions.cpp | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index f9ed379b4e9..b35e23703c2 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -264,7 +264,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & build_pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, join_ptr, get_concurrency_build_index(), log->identifier()); stream->setExtraInfo( - fmt::format("join build, root_executor_id = {}", dagContext().getJoinExecuteInfoMap()[query_block.source_name].build_side_root_executor_id)); + fmt::format("join build, build_side_root_executor_id = {}", dagContext().getJoinExecuteInfoMap()[query_block.source_name].build_side_root_executor_id)); }); executeUnion(build_pipeline, max_streams, log, /*ignore_block=*/true, "for join"); diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index 8af4b509842..d91b18254f6 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -376,7 +376,7 @@ void DAGStorageInterpreter::executePushedDownFilter( stream->setExtraInfo("push down filter"); // after filter, do project action to keep the schema of local streams and remote streams the same. stream = std::make_shared(stream, project_after_where, log->identifier()); - stream->setExtraInfo("push down filter"); + stream->setExtraInfo("projection after push down filter"); } } @@ -414,7 +414,7 @@ void DAGStorageInterpreter::executeCastAfterTableScan( { auto & stream = pipeline.streams[i++]; stream = std::make_shared(stream, extra_cast, log->identifier()); - stream->setExtraInfo("cast after tableScan"); + stream->setExtraInfo("cast after local tableScan"); } // remote streams if (i < pipeline.streams.size()) @@ -427,7 +427,7 @@ void DAGStorageInterpreter::executeCastAfterTableScan( { auto & stream = pipeline.streams[i++]; stream = std::make_shared(stream, project_for_cop_read, log->identifier()); - stream->setExtraInfo("cast after tableScan"); + stream->setExtraInfo("cast after remote tableScan"); } } } diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index 097bfc76355..c11f0655860 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -232,12 +232,12 @@ Union: String expected = R"( CreatingSets Union: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: MockTableScan Union x 2: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: Expression: @@ -319,12 +319,12 @@ Union: String expected = R"( CreatingSets Union: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: MockExchangeReceiver Union x 2: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: Expression: @@ -361,12 +361,12 @@ CreatingSets String expected = R"( CreatingSets Union: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: MockExchangeReceiver Union x 2: - HashJoinBuildBlockInputStream x 10: , join_kind = Left + HashJoinBuildBlockInputStream x 10: , join_kind = Left Expression: Expression: Expression: diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index dbdfd69fbbe..8e75a64427c 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -929,7 +929,7 @@ std::string ExpressionActions::dumpActions() const ss << "\noutput:\n"; NamesAndTypesList output_columns = sample_block.getNamesAndTypesList(); - for (auto & output_column : output_columns) + for (const auto & output_column : output_columns) ss << output_column.name << " " << output_column.type->getName() << "\n"; return ss.str(); From 7c24b381d21d485e41f92144c4f578e7f226b644 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 25 May 2022 10:41:56 +0800 Subject: [PATCH 40/42] refine. --- dbms/src/DataStreams/FilterBlockInputStream.cpp | 1 + dbms/src/Flash/Coprocessor/InterpreterUtils.cpp | 2 -- dbms/src/Interpreters/executeQuery.cpp | 6 +++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index fb6f644b2fa..0415533317f 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -220,4 +220,5 @@ Block FilterBlockInputStream::readImpl() } } + } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp b/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp index da5aaca9aa6..f394ad07119 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp @@ -47,9 +47,7 @@ BlockInputStreamPtr combinedNonJoinedDataStream( { BlockInputStreamPtr ret = nullptr; if (pipeline.streams_with_non_joined_data.size() == 1) - { ret = pipeline.streams_with_non_joined_data.at(0); - } else if (pipeline.streams_with_non_joined_data.size() > 1) { if (ignore_block) diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 4769f00cdd8..96cfc0a58ae 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -187,7 +187,7 @@ std::tuple executeQueryImpl( } catch (...) { - if (!internal && !context.getDAGContext()->isTest()) + if (!internal) { /// Anyway log the query. String str = query_src.str(max_query_size); @@ -202,7 +202,7 @@ std::tuple executeQueryImpl( try { - if (!internal && !context.getDAGContext()->isTest()) + if (!internal) logQuery(query.substr(0, settings.log_queries_cut_to_length), context, execute_query_logger); /// Check the limits. @@ -384,7 +384,7 @@ std::tuple executeQueryImpl( } }; - if (!internal && res.in && !context.getDAGContext()->isTest()) + if (!internal && res.in) { auto pipeline_log_str = [&res]() { FmtBuffer log_buffer; From 1a913f06e9abf82594166748f44db2f7d3f47400 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 25 May 2022 11:39:21 +0800 Subject: [PATCH 41/42] address comments. --- dbms/src/DataStreams/FilterBlockInputStream.cpp | 1 - .../Flash/Coprocessor/DAGQueryBlockInterpreter.cpp | 10 +++++----- .../src/Flash/Coprocessor/DAGQueryBlockInterpreter.h | 6 +++--- dbms/src/Flash/Coprocessor/InterpreterUtils.cpp | 2 +- dbms/src/Flash/Coprocessor/InterpreterUtils.h | 2 +- dbms/src/Flash/tests/gtest_interpreter.cpp | 12 ++++++------ 6 files changed, 16 insertions(+), 17 deletions(-) diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index 0415533317f..a7ecf12d8cb 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index b35e23703c2..5fac49faaed 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -258,7 +258,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & size_t join_build_concurrency = settings.join_concurrent_build ? std::min(max_streams, build_pipeline.streams.size()) : 1; /// build side streams - executeExpression(build_pipeline, build_side_prepare_actions, "Append Join key and join Filters"); + executeExpression(build_pipeline, build_side_prepare_actions, "append join key and join filters for build side"); // add a HashJoinBuildBlockInputStream to build a shared hash table auto get_concurrency_build_index = JoinInterpreterHelper::concurrencyBuildIndexGenerator(join_build_concurrency); build_pipeline.transform([&](auto & stream) { @@ -273,7 +273,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & join_ptr->init(right_query.source->getHeader(), join_build_concurrency); /// probe side streams - executeExpression(probe_pipeline, probe_side_prepare_actions, "probe side prepare actions"); + executeExpression(probe_pipeline, probe_side_prepare_actions, "append join key and join filters for probe side"); NamesAndTypes source_columns; for (const auto & p : probe_pipeline.firstStream()->getHeader()) source_columns.emplace_back(p.name, p.type); @@ -326,7 +326,7 @@ void DAGQueryBlockInterpreter::recordJoinExecuteInfo(size_t build_side_index, co dagContext().getJoinExecuteInfoMap()[query_block.source_name] = std::move(join_execute_info); } -void DAGQueryBlockInterpreter::executeWhere(DAGPipeline & pipeline, const ExpressionActionsPtr & expr, String & filter_column, String extra_info) +void DAGQueryBlockInterpreter::executeWhere(DAGPipeline & pipeline, const ExpressionActionsPtr & expr, String & filter_column, const String & extra_info) { pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, expr, filter_column, log->identifier()); @@ -410,7 +410,7 @@ void DAGQueryBlockInterpreter::executeAggregation( } } -void DAGQueryBlockInterpreter::executeExpression(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, String extra_info) +void DAGQueryBlockInterpreter::executeExpression(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, const String & extra_info) { if (!expressionActionsPtr->getActions().empty()) { @@ -688,7 +688,7 @@ void DAGQueryBlockInterpreter::executeImpl(DAGPipeline & pipeline) } } -void DAGQueryBlockInterpreter::executeProject(DAGPipeline & pipeline, NamesWithAliases & project_cols, String extra_info) +void DAGQueryBlockInterpreter::executeProject(DAGPipeline & pipeline, NamesWithAliases & project_cols, const String & extra_info) { if (project_cols.empty()) return; diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h index 2350b0ffd40..e68c4f91cee 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h @@ -66,8 +66,8 @@ class DAGQueryBlockInterpreter void handleProjection(DAGPipeline & pipeline, const tipb::Projection & projection); void handleWindow(DAGPipeline & pipeline, const tipb::Window & window); void handleWindowOrder(DAGPipeline & pipeline, const tipb::Sort & window_sort); - void executeWhere(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, String & filter_column, String extra_info = ""); - void executeExpression(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, String extra_info = ""); + void executeWhere(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, String & filter_column, const String & extra_info = ""); + void executeExpression(DAGPipeline & pipeline, const ExpressionActionsPtr & expressionActionsPtr, const String & extra_info = ""); void executeWindowOrder(DAGPipeline & pipeline, SortDescription sort_desc); void orderStreams(DAGPipeline & pipeline, SortDescription order_descr, Int64 limit); void executeOrder(DAGPipeline & pipeline, const NamesAndTypes & order_columns); @@ -82,7 +82,7 @@ class DAGQueryBlockInterpreter const TiDB::TiDBCollators & collators, AggregateDescriptions & aggregate_descriptions, bool is_final_agg); - void executeProject(DAGPipeline & pipeline, NamesWithAliases & project_cols, String extra_info = ""); + void executeProject(DAGPipeline & pipeline, NamesWithAliases & project_cols, const String & extra_info = ""); void handleExchangeSender(DAGPipeline & pipeline); void handleMockExchangeSender(DAGPipeline & pipeline); diff --git a/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp b/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp index f394ad07119..c9810454218 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterUtils.cpp @@ -70,7 +70,7 @@ void executeUnion( size_t max_streams, const LoggerPtr & log, bool ignore_block, - String extra_info) + const String & extra_info) { if (pipeline.streams.size() == 1 && pipeline.streams_with_non_joined_data.empty()) return; diff --git a/dbms/src/Flash/Coprocessor/InterpreterUtils.h b/dbms/src/Flash/Coprocessor/InterpreterUtils.h index daffe6143b4..5c4d4721d5e 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterUtils.h +++ b/dbms/src/Flash/Coprocessor/InterpreterUtils.h @@ -38,7 +38,7 @@ void executeUnion( size_t max_streams, const LoggerPtr & log, bool ignore_block = false, - String extra_info = ""); + const String & extra_info = ""); ExpressionActionsPtr generateProjectExpressionActions( const BlockInputStreamPtr & stream, diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index c11f0655860..aed9d9e90f9 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -233,12 +233,12 @@ Union: CreatingSets Union: HashJoinBuildBlockInputStream x 10: , join_kind = Left - Expression: + Expression: Expression: MockTableScan Union x 2: HashJoinBuildBlockInputStream x 10: , join_kind = Left - Expression: + Expression: Expression: Expression: HashJoinProbe: @@ -320,12 +320,12 @@ Union: CreatingSets Union: HashJoinBuildBlockInputStream x 10: , join_kind = Left - Expression: + Expression: Expression: MockExchangeReceiver Union x 2: HashJoinBuildBlockInputStream x 10: , join_kind = Left - Expression: + Expression: Expression: Expression: HashJoinProbe: @@ -362,12 +362,12 @@ CreatingSets CreatingSets Union: HashJoinBuildBlockInputStream x 10: , join_kind = Left - Expression: + Expression: Expression: MockExchangeReceiver Union x 2: HashJoinBuildBlockInputStream x 10: , join_kind = Left - Expression: + Expression: Expression: Expression: HashJoinProbe: From e8fda6f82273d084a50f520736d910b75ffd2015 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 25 May 2022 16:34:09 +0800 Subject: [PATCH 42/42] fix bug. --- dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp index ad4ebe65d02..61808b48c50 100644 --- a/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp +++ b/dbms/src/DataStreams/HashJoinBuildBlockInputStream.cpp @@ -36,9 +36,13 @@ void HashJoinBuildBlockInputStream::appendInfo(FmtBuffer & buffer) const {ASTTableJoin::Kind::Cross, "Cross"}, {ASTTableJoin::Kind::Comma, "Comma"}, {ASTTableJoin::Kind::Anti, "Anti"}, + {ASTTableJoin::Kind::LeftSemi, "Left_Semi"}, + {ASTTableJoin::Kind::LeftAnti, "Left_Anti"}, {ASTTableJoin::Kind::Cross_Left, "Cross_Left"}, {ASTTableJoin::Kind::Cross_Right, "Cross_Right"}, - {ASTTableJoin::Kind::Cross_Anti, "Cross_Anti"}}; + {ASTTableJoin::Kind::Cross_Anti, "Cross_Anti"}, + {ASTTableJoin::Kind::Cross_LeftSemi, "Cross_LeftSemi"}, + {ASTTableJoin::Kind::Cross_LeftAnti, "Cross_LeftAnti"}}; auto join_type_it = join_type_map.find(join->getKind()); if (join_type_it == join_type_map.end()) throw TiFlashException("Unknown join type", Errors::Coprocessor::Internal);