From 13cecf81e2f5723be2b6cfe8c291d13f66013f15 Mon Sep 17 00:00:00 2001 From: Calvin Neo Date: Mon, 15 Aug 2022 15:18:50 +0800 Subject: [PATCH 1/5] Avoid hacks in Proxy's src/server.rs (#5622) ref pingcap/tiflash#5170 --- contrib/tiflash-proxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/tiflash-proxy b/contrib/tiflash-proxy index 56761b710e5..70de5355e12 160000 --- a/contrib/tiflash-proxy +++ b/contrib/tiflash-proxy @@ -1 +1 @@ -Subproject commit 56761b710e56a2abc6fead1c16e705134be08e6d +Subproject commit 70de5355e12daca1663df33db21dee6ac5bc45ae From cdf347800bf53537977632576bc75f1c9c00b447 Mon Sep 17 00:00:00 2001 From: SeaRise Date: Mon, 15 Aug 2022 16:24:50 +0800 Subject: [PATCH 2/5] disable profile info of `SharedQueryBlockInputStream.h` (#5621) close pingcap/tiflash#5302 --- dbms/src/DataStreams/IProfilingBlockInputStream.h | 2 +- dbms/src/DataStreams/SharedQueryBlockInputStream.h | 9 ++++++++- tests/sanitize/tsan.suppression | 1 - 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/dbms/src/DataStreams/IProfilingBlockInputStream.h b/dbms/src/DataStreams/IProfilingBlockInputStream.h index 2ce328c94e9..2b78ced4df0 100644 --- a/dbms/src/DataStreams/IProfilingBlockInputStream.h +++ b/dbms/src/DataStreams/IProfilingBlockInputStream.h @@ -54,7 +54,7 @@ class IProfilingBlockInputStream : public IBlockInputStream Block read() override final; - Block read(FilterPtr & res_filter, bool return_filter) override final; + Block read(FilterPtr & res_filter, bool return_filter) override; /** The default implementation calls readPrefixImpl() on itself, and then readPrefix() recursively for all children. * There are cases when you do not want `readPrefix` of children to be called synchronously, in this function, diff --git a/dbms/src/DataStreams/SharedQueryBlockInputStream.h b/dbms/src/DataStreams/SharedQueryBlockInputStream.h index d7c0707b5aa..7882c8cf5c9 100644 --- a/dbms/src/DataStreams/SharedQueryBlockInputStream.h +++ b/dbms/src/DataStreams/SharedQueryBlockInputStream.h @@ -112,7 +112,10 @@ class SharedQueryBlockInputStream : public IProfilingBlockInputStream } protected: - Block readImpl() override + /// The BlockStreamProfileInfo of SharedQuery is useless, + /// and it will trigger tsan UT fail because of data race. + /// So overriding method `read` here. + Block read(FilterPtr &, bool) override { std::unique_lock lock(mutex); @@ -134,6 +137,10 @@ class SharedQueryBlockInputStream : public IProfilingBlockInputStream return block; } + Block readImpl() override + { + throw Exception("Unsupport"); + } void fetchBlocks() { diff --git a/tests/sanitize/tsan.suppression b/tests/sanitize/tsan.suppression index d24afc30209..73824caa2b9 100644 --- a/tests/sanitize/tsan.suppression +++ b/tests/sanitize/tsan.suppression @@ -1,2 +1 @@ race:dbms/src/Common/TiFlashMetrics.h -race:dbms/src/DataStreams/BlockStreamProfileInfo.h \ No newline at end of file From 577aadae6b4cb7a4db43cb5fddf330d4d254e641 Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Mon, 15 Aug 2022 17:12:51 +0800 Subject: [PATCH 3/5] support CastTimeAsDuration in TiFlash (#5548) close pingcap/tiflash#5306 --- dbms/src/Flash/Coprocessor/DAGUtils.cpp | 2 +- dbms/src/Functions/FunctionsTiDBConversion.h | 38 +++++- .../Functions/tests/gtest_tidb_conversion.cpp | 113 +++++++++++++++++- .../fullstack-test/expr/cast_as_duration.test | 95 +++++++++++++++ 4 files changed, 245 insertions(+), 3 deletions(-) mode change 100644 => 100755 dbms/src/Flash/Coprocessor/DAGUtils.cpp mode change 100644 => 100755 dbms/src/Functions/FunctionsTiDBConversion.h create mode 100644 tests/fullstack-test/expr/cast_as_duration.test diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp old mode 100644 new mode 100755 index 4b1d3546406..75eb75ecbb0 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -107,7 +107,7 @@ const std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::CastTimeAsString, "tidb_cast"}, {tipb::ScalarFuncSig::CastTimeAsDecimal, "tidb_cast"}, {tipb::ScalarFuncSig::CastTimeAsTime, "tidb_cast"}, - //{tipb::ScalarFuncSig::CastTimeAsDuration, "cast"}, + {tipb::ScalarFuncSig::CastTimeAsDuration, "tidb_cast"}, //{tipb::ScalarFuncSig::CastTimeAsJson, "cast"}, //{tipb::ScalarFuncSig::CastDurationAsInt, "cast"}, diff --git a/dbms/src/Functions/FunctionsTiDBConversion.h b/dbms/src/Functions/FunctionsTiDBConversion.h old mode 100644 new mode 100755 index 036b04fa8ca..d649191ad3e --- a/dbms/src/Functions/FunctionsTiDBConversion.h +++ b/dbms/src/Functions/FunctionsTiDBConversion.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -1549,7 +1550,7 @@ struct TiDBConvertToTime } }; -/// cast duration as duration +/// cast time/duration as duration /// TODO: support more types convert to duration template struct TiDBConvertToDuration @@ -1601,6 +1602,41 @@ struct TiDBConvertToDuration block.getByPosition(result).column = std::move(to_col); } } + else if constexpr (std::is_same_v) + { + // cast date as duration + const auto & to_type = checkAndGetDataType(removeNullable(block.getByPosition(result).type).get()); + block.getByPosition(result).column = to_type->createColumnConst(size, toField(0)); // The DATE type is used for values with a date part but no time part. The value of Duration is always zero. + } + else if constexpr (std::is_same_v) + { + // cast time as duration + const auto * col_from = checkAndGetColumn(block.getByPosition(arguments[0]).column.get()); + const ColumnUInt64::Container & from_vec = col_from->getData(); + const auto & from_type = checkAndGetDataType(block.getByPosition(arguments[0]).type.get()); + int from_fsp = from_type->getFraction(); + + auto to_col = ColumnVector::create(); + auto & vec_to = to_col->getData(); + vec_to.resize(size); + const auto & to_type = checkAndGetDataType(removeNullable(block.getByPosition(result).type).get()); + int to_fsp = to_type->getFsp(); + + for (size_t i = 0; i < size; ++i) + { + MyDateTime datetime(from_vec[i]); + MyDuration duration(1 /*neg*/, datetime.hour, datetime.minute, datetime.second, datetime.micro_second, from_fsp); + if (to_fsp < from_fsp) + { + vec_to[i] = round(duration.nanoSecond(), (6 - to_fsp) + 3); + } + else + { + vec_to[i] = duration.nanoSecond(); + } + } + block.getByPosition(result).column = std::move(to_col); + } else { throw Exception( diff --git a/dbms/src/Functions/tests/gtest_tidb_conversion.cpp b/dbms/src/Functions/tests/gtest_tidb_conversion.cpp index 387d479c680..51ddcc81f1e 100644 --- a/dbms/src/Functions/tests/gtest_tidb_conversion.cpp +++ b/dbms/src/Functions/tests/gtest_tidb_conversion.cpp @@ -1313,7 +1313,7 @@ try const auto from_type = std::make_shared(3); const auto to_type_1 = std::make_shared(5); // from_fsp < to_fsp const auto to_type_2 = std::make_shared(3); // from_fsp == to_fsp - const auto to_type_3 = std::make_shared(2); // from_fsp < to_fsp + const auto to_type_3 = std::make_shared(2); // from_fsp > to_fsp ColumnWithTypeAndName input( createColumn({(20 * 3600 + 20 * 60 + 20) * 1000000000L + 555000000L, @@ -1766,5 +1766,116 @@ try } CATCH +TEST_F(TestTidbConversion, castTimeAsDuration) +try +{ + const auto to_type_1 = std::make_shared(5); // from_fsp < to_fsp + const auto to_type_2 = std::make_shared(4); // from_fsp == to_fsp + const auto to_type_3 = std::make_shared(2); // from_fsp > to_fsp + // cast datetime to duration + const auto datetime_type_ptr = std::make_shared(4); + MyDateTime date(2021, 10, 26, 0, 0, 0, 0); + MyDateTime datetime(2021, 10, 26, 11, 11, 11, 0); + MyDateTime datetime_frac1(2021, 10, 26, 11, 11, 11, 111100); + MyDateTime datetime_frac2(2021, 10, 26, 11, 11, 11, 123500); + MyDateTime datetime_frac3(2021, 10, 26, 11, 11, 11, 999900); + + auto col_datetime = ColumnUInt64::create(); + col_datetime->insert(Field(date.toPackedUInt())); + col_datetime->insert(Field(datetime.toPackedUInt())); + col_datetime->insert(Field(datetime_frac1.toPackedUInt())); + col_datetime->insert(Field(datetime_frac2.toPackedUInt())); + col_datetime->insert(Field(datetime_frac3.toPackedUInt())); + + auto ctn_datetime = ColumnWithTypeAndName(std::move(col_datetime), datetime_type_ptr, "datetime"); + ColumnWithTypeAndName datetime_output1( + createColumn({(0 * 3600 + 0 * 60 + 0) * 1000000000L + 000000000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 000000000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 111100000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 123500000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 999900000L}) + .column, + to_type_1, + "datetime_output1"); + ColumnWithTypeAndName datetime_output2( + createColumn({(0 * 3600 + 0 * 60 + 0) * 1000000000L + 000000000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 000000000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 111100000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 123500000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 999900000L}) + .column, + to_type_2, + "datetime_output2"); + + ColumnWithTypeAndName datetime_output3( + createColumn({(0 * 3600 + 0 * 60 + 0) * 1000000000L + 000000000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 000000000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 110000000L, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 120000000L, + (11 * 3600 + 11 * 60 + 12) * 1000000000L + 000000000L}) + .column, + to_type_3, + "datetime_output3"); + + + ASSERT_COLUMN_EQ(datetime_output1, executeFunction(func_name, {ctn_datetime, createCastTypeConstColumn(to_type_1->getName())})); + ASSERT_COLUMN_EQ(datetime_output2, executeFunction(func_name, {ctn_datetime, createCastTypeConstColumn(to_type_2->getName())})); + ASSERT_COLUMN_EQ(datetime_output3, executeFunction(func_name, {ctn_datetime, createCastTypeConstColumn(to_type_3->getName())})); + + + // Test Const + ColumnWithTypeAndName input_const(createConstColumn(1, datetime_frac2.toPackedUInt()).column, datetime_type_ptr, "input_const"); + ColumnWithTypeAndName output1_const(createConstColumn(1, (11 * 3600 + 11 * 60 + 11) * 1000000000L + 123500000L).column, to_type_1, "output1_const"); + ColumnWithTypeAndName output2_const(createConstColumn(1, (11 * 3600 + 11 * 60 + 11) * 1000000000L + 123500000L).column, to_type_2, "output2_const"); + ColumnWithTypeAndName output3_const(createConstColumn(1, (11 * 3600 + 11 * 60 + 11) * 1000000000L + 120000000L).column, to_type_3, "output3_const"); + + ASSERT_COLUMN_EQ(output1_const, executeFunction(func_name, {input_const, createCastTypeConstColumn(to_type_1->getName())})); + ASSERT_COLUMN_EQ(output2_const, executeFunction(func_name, {input_const, createCastTypeConstColumn(to_type_2->getName())})); + ASSERT_COLUMN_EQ(output3_const, executeFunction(func_name, {input_const, createCastTypeConstColumn(to_type_3->getName())})); + + // Test Nullable + ColumnWithTypeAndName input_nullable( + createColumn>({datetime_frac1.toPackedUInt(), + {}, + datetime_frac2.toPackedUInt(), + {}, + datetime_frac3.toPackedUInt()}) + .column, + makeNullable(datetime_type_ptr), + "input_nullable"); + ColumnWithTypeAndName output1_nullable( + createColumn>({(11 * 3600 + 11 * 60 + 11) * 1000000000L + 111100000L, + {}, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 123500000L, + {}, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 999900000L}) + .column, + makeNullable(to_type_1), + "output1_output"); + ColumnWithTypeAndName output2_nullable( + createColumn>({(11 * 3600 + 11 * 60 + 11) * 1000000000L + 111100000L, + {}, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 123500000L, + {}, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 999900000L}) + .column, + makeNullable(to_type_2), + "output2_output"); + ColumnWithTypeAndName output3_nullable( + createColumn>({(11 * 3600 + 11 * 60 + 11) * 1000000000L + 110000000L, + {}, + (11 * 3600 + 11 * 60 + 11) * 1000000000L + 120000000L, + {}, + (11 * 3600 + 11 * 60 + 12) * 1000000000L + 000000000L}) + .column, + makeNullable(to_type_3), + "output3_output"); + + ASSERT_COLUMN_EQ(output1_nullable, executeFunction(func_name, {input_nullable, createCastTypeConstColumn(makeNullable(to_type_1)->getName())})); + ASSERT_COLUMN_EQ(output2_nullable, executeFunction(func_name, {input_nullable, createCastTypeConstColumn(makeNullable(to_type_2)->getName())})); + ASSERT_COLUMN_EQ(output3_nullable, executeFunction(func_name, {input_nullable, createCastTypeConstColumn(makeNullable(to_type_3)->getName())})); +} +CATCH + } // namespace } // namespace DB::tests diff --git a/tests/fullstack-test/expr/cast_as_duration.test b/tests/fullstack-test/expr/cast_as_duration.test new file mode 100644 index 00000000000..ad2afcee92c --- /dev/null +++ b/tests/fullstack-test/expr/cast_as_duration.test @@ -0,0 +1,95 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +mysql> drop table if exists test.t; +mysql> create table test.t(c1 date, c2 datetime(4)); +mysql> insert into test.t values('2022-01-01','2022-09-20 11:11:11.0000'); +mysql> insert into test.t values('2022-01-01','2022-09-20 11:11:11.1111'); +mysql> insert into test.t values('2022-01-01','2022-09-20 11:11:11.1234'); +mysql> insert into test.t values('2022-01-01','2022-09-20 11:11:11.1255'); +mysql> insert into test.t values('2022-01-01','2022-09-20 11:11:11.9999'); +mysql> insert into test.t values(null,null); + +mysql> alter table test.t set tiflash replica 1; + +func> wait_table test t + +mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select cast(test.t.c1 as time(2)) from test.t; ++----------------------------+ +| cast(test.t.c1 as time(2)) | ++----------------------------+ +| 00:00:00.00 | +| 00:00:00.00 | +| 00:00:00.00 | +| 00:00:00.00 | +| 00:00:00.00 | +| NULL | ++----------------------------+ +mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select cast(test.t.c1 as time(4)) from test.t; ++----------------------------+ +| cast(test.t.c1 as time(4)) | ++----------------------------+ +| 00:00:00.0000 | +| 00:00:00.0000 | +| 00:00:00.0000 | +| 00:00:00.0000 | +| 00:00:00.0000 | +| NULL | ++----------------------------+ +mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select cast(test.t.c1 as time(5)) from test.t; ++----------------------------+ +| cast(test.t.c1 as time(5)) | ++----------------------------+ +| 00:00:00.00000 | +| 00:00:00.00000 | +| 00:00:00.00000 | +| 00:00:00.00000 | +| 00:00:00.00000 | +| NULL | ++----------------------------+ +mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select cast(test.t.c2 as time(2)) from test.t; ++----------------------------+ +| cast(test.t.c2 as time(2)) | ++----------------------------+ +| 11:11:11.00 | +| 11:11:11.11 | +| 11:11:11.12 | +| 11:11:11.13 | +| 11:11:12.00 | +| NULL | ++----------------------------+ +mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select cast(test.t.c2 as time(4)) from test.t; ++----------------------------+ +| cast(test.t.c2 as time(4)) | ++----------------------------+ +| 11:11:11.0000 | +| 11:11:11.1111 | +| 11:11:11.1234 | +| 11:11:11.1255 | +| 11:11:11.9999 | +| NULL | ++----------------------------+ +mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select cast(test.t.c2 as time(5)) from test.t; ++----------------------------+ +| cast(test.t.c2 as time(5)) | ++----------------------------+ +| 11:11:11.00000 | +| 11:11:11.11110 | +| 11:11:11.12340 | +| 11:11:11.12550 | +| 11:11:11.99990 | +| NULL | ++----------------------------+ +mysql> drop table if exists test.t; From 4ab1564a17c2ed8bd78b5255f721def878fdd296 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 16 Aug 2022 12:24:51 +0800 Subject: [PATCH 4/5] Optimize equality comparison for small str with fixed size (#5569) ref pingcap/tiflash#5294 --- .../Functions/CollationOperatorOptimized.h | 114 ++++++++-- dbms/src/Storages/Transaction/CollatorUtils.h | 2 - libs/libcommon/include/common/defines.h | 4 + libs/libcommon/include/common/fixed_mem_eq.h | 203 ++++++++++++++++++ 4 files changed, 306 insertions(+), 17 deletions(-) create mode 100644 libs/libcommon/include/common/fixed_mem_eq.h diff --git a/dbms/src/Functions/CollationOperatorOptimized.h b/dbms/src/Functions/CollationOperatorOptimized.h index 4651e8981bc..3eed2ec2965 100644 --- a/dbms/src/Functions/CollationOperatorOptimized.h +++ b/dbms/src/Functions/CollationOperatorOptimized.h @@ -20,11 +20,11 @@ #include #include #include +#include #include #include - namespace DB { @@ -50,7 +50,7 @@ struct IsEqualRelated> // Loop columns and invoke callback for each pair. // Remove last zero byte. template -__attribute__((flatten, always_inline)) inline void LoopTwoColumns( +FLATTEN_INLINE inline void LoopTwoColumns( const ColumnString::Chars_t & a_data, const ColumnString::Offsets & a_offsets, const ColumnString::Chars_t & b_data, @@ -79,7 +79,7 @@ __attribute__((flatten, always_inline)) inline void LoopTwoColumns( // Loop one column and invoke callback for each pair. // Remove last zero byte. template -__attribute__((flatten, always_inline)) inline void LoopOneColumn( +FLATTEN_INLINE inline void LoopOneColumn( const ColumnString::Chars_t & a_data, const ColumnString::Offsets & a_offsets, size_t size, @@ -97,6 +97,23 @@ __attribute__((flatten, always_inline)) inline void LoopOneColumn( } } +template +FLATTEN_INLINE inline void LoopOneColumnCmpEqFixedStr( + const ColumnString::Chars_t & a_data, + const ColumnString::Offsets & a_offsets, + const char * src, + Result & c) +{ + LoopOneColumn(a_data, a_offsets, a_offsets.size(), [&](std::string_view view, size_t i) { + if constexpr (trim) + view = RightTrim(view); + auto res = 1; + if (view.size() == n) + res = mem_utils::memcmp_eq_fixed_size(view.data(), src) ? 0 : 1; + c[i] = Op::apply(res, 0); + }); +} + // Handle str-column compare str-column. // - Optimize bin collator // - Check if columns do NOT contain tail space @@ -175,8 +192,6 @@ ALWAYS_INLINE inline bool CompareStringVectorConstant( const TiDB::TiDBCollatorPtr & collator, Result & c) { - bool use_optimized_path = false; - switch (collator->getCollatorType()) { case TiDB::ITiDBCollator::CollatorType::UTF8MB4_BIN: @@ -184,11 +199,46 @@ ALWAYS_INLINE inline bool CompareStringVectorConstant( case TiDB::ITiDBCollator::CollatorType::LATIN1_BIN: case TiDB::ITiDBCollator::CollatorType::ASCII_BIN: { - size_t size = a_offsets.size(); - std::string_view tar_str_view = RightTrim(b); // right trim const-str first - LoopOneColumn(a_data, a_offsets, size, [&c, &tar_str_view](const std::string_view & view, size_t i) { + if constexpr (IsEqualRelated::value) + { +#ifdef M + static_assert(false, "`M` is defined"); +#endif +#define M(k) \ + case k: \ + { \ + LoopOneColumnCmpEqFixedStr(a_data, a_offsets, tar_str_view.data(), c); \ + return true; \ + } + + switch (tar_str_view.size()) + { + M(0); + M(1); + M(2); + M(3); + M(4); + M(5); + M(6); + M(7); + M(8); + M(9); + M(10); + M(11); + M(12); + M(13); + M(14); + M(15); + M(16); + default: + break; + } +#undef M + } + + LoopOneColumn(a_data, a_offsets, a_offsets.size(), [&c, &tar_str_view](const std::string_view & view, size_t i) { if constexpr (IsEqualRelated::value) { c[i] = Op::apply(RawStrEqualCompare(RightTrim(view), tar_str_view), 0); @@ -199,13 +249,48 @@ ALWAYS_INLINE inline bool CompareStringVectorConstant( } }); - use_optimized_path = true; - break; + return true; } case TiDB::ITiDBCollator::CollatorType::BINARY: { - size_t size = a_offsets.size(); - LoopOneColumn(a_data, a_offsets, size, [&c, &b](const std::string_view & view, size_t i) { + if constexpr (IsEqualRelated::value) + { +#ifdef M + static_assert(false, "`M` is defined"); +#endif +#define M(k) \ + case k: \ + { \ + LoopOneColumnCmpEqFixedStr(a_data, a_offsets, b.data(), c); \ + return true; \ + } + + switch (b.size()) + { + M(0); + M(1); + M(2); + M(3); + M(4); + M(5); + M(6); + M(7); + M(8); + M(9); + M(10); + M(11); + M(12); + M(13); + M(14); + M(15); + M(16); + default: + break; + } +#undef M + } + + LoopOneColumn(a_data, a_offsets, a_offsets.size(), [&c, &b](const std::string_view & view, size_t i) { if constexpr (IsEqualRelated::value) { c[i] = Op::apply(RawStrEqualCompare((view), b), 0); @@ -216,13 +301,12 @@ ALWAYS_INLINE inline bool CompareStringVectorConstant( } }); - use_optimized_path = true; - break; + return true; } default: break; } - return use_optimized_path; + return false; } } // namespace DB diff --git a/dbms/src/Storages/Transaction/CollatorUtils.h b/dbms/src/Storages/Transaction/CollatorUtils.h index 3f318a5b700..c757ade8043 100644 --- a/dbms/src/Storages/Transaction/CollatorUtils.h +++ b/dbms/src/Storages/Transaction/CollatorUtils.h @@ -19,8 +19,6 @@ #include -#define FLATTEN_INLINE_PURE __attribute__((flatten, always_inline, pure)) - namespace DB { diff --git a/libs/libcommon/include/common/defines.h b/libs/libcommon/include/common/defines.h index ff79a4d2077..7996bb7a579 100644 --- a/libs/libcommon/include/common/defines.h +++ b/libs/libcommon/include/common/defines.h @@ -50,10 +50,14 @@ # define ALWAYS_INLINE __forceinline # define NO_INLINE static __declspec(noinline) # define MAY_ALIAS +# define FLATTEN_INLINE_PURE +# define FLATTEN_INLINE #else # define ALWAYS_INLINE __attribute__((__always_inline__)) # define NO_INLINE __attribute__((__noinline__)) # define MAY_ALIAS __attribute__((__may_alias__)) +# define FLATTEN_INLINE_PURE __attribute__((flatten, always_inline, pure)) +# define FLATTEN_INLINE __attribute__((flatten, always_inline)) #endif #if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__) diff --git a/libs/libcommon/include/common/fixed_mem_eq.h b/libs/libcommon/include/common/fixed_mem_eq.h new file mode 100644 index 00000000000..85691008c6d --- /dev/null +++ b/libs/libcommon/include/common/fixed_mem_eq.h @@ -0,0 +1,203 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include + +namespace mem_utils +{ + +FLATTEN_INLINE_PURE +constexpr inline bool memcmp_eq0(const char *, const char *) +{ + return true; +} + +FLATTEN_INLINE_PURE +inline bool memcmp_eq1(const char * a, const char * b) +{ + return a[0] == b[0]; +} + +FLATTEN_INLINE_PURE +inline bool memcmp_eq2(const char * a, const char * b) +{ + const uint16_t ax = *reinterpret_cast(a); + const uint16_t bx = *reinterpret_cast(b); + return ax == bx; +} + +/* +If use `&&` and cpu failed to predict `cmp -> jne`, the pipeline will be broken. + + movzx eax, word ptr [rdi] + cmp ax, word ptr [rsi] + jne ... + movzx eax, word ptr [rdi + 1] + cmp ax, word ptr [rsi + 1] + sete al + ret + +Use `&` to reduce unnecessary branch. Instructions like (1) and (2) are independent(same for (3) and (4)), it's friendly for parallelism. + + movzx eax, word ptr [rdi] // (1) + movzx ecx, word ptr [rdi + 1] // (2) + xor ax, word ptr [rsi] // (3) + xor cx, word ptr [rsi + 1] // (4) + or cx, ax + sete al + ret +*/ +FLATTEN_INLINE_PURE +inline bool memcmp_eq3(const char * a, const char * b) +{ + return memcmp_eq2(a, b) & memcmp_eq2(a + 1, b + 1); +} + +FLATTEN_INLINE_PURE +inline bool memcmp_eq4(const char * a, const char * b) +{ + const uint32_t ax = *reinterpret_cast(a); + const uint32_t bx = *reinterpret_cast(b); + return ax == bx; +} + +FLATTEN_INLINE_PURE +inline bool memcmp_eq8(const char * a, const char * b) +{ + const uint64_t ax = *reinterpret_cast(a); + const uint64_t bx = *reinterpret_cast(b); + return ax == bx; +} + +// check memory equal of two pointers in fixed size +template +ALWAYS_INLINE inline bool memcmp_eq_fixed_size(const char * a, const char * b) +{ +#ifdef M + static_assert(false, "`M` is defined"); +#else +#define M(s) \ + if constexpr (k == (s)) \ + { \ + return memcmp_eq##s(a, b); \ + } +#endif + + static_assert(k >= 0); + + if constexpr (k >= 16) + { + /* + For x86-64 clang 13.0.0 with options `-O3 -msse4.2`, `std::memcmp(.. , .. , 16)` will be translated to + + movdqu xmm0, xmmword ptr [rdi] + movdqu xmm1, xmmword ptr [rsi] + pxor xmm1, xmm0 + ptest xmm1, xmm1 + sete al + ret + + with options `-O3 -mavx2`, it will be + + vmovdqu xmm0, xmmword ptr [rdi] + vpxor xmm0, xmm0, xmmword ptr [rsi] + vptest xmm0, xmm0 + sete al + ret + + */ + return std::memcmp(a, b, k) == 0; + } + else if constexpr (k > 8) + { + /* + if use `std::memcmp(.. , .. , 9)`, it will be + x86-64 + mov rax, qword ptr [rdi] + xor rax, qword ptr [rsi] + mov cl, byte ptr [rdi + 8] + xor cl, byte ptr [rsi + 8] + movzx ecx, cl + or rcx, rax + sete al + ret + + arm-64 v8 + ldr x8, [x0] + ldr x9, [x1] + ldrb w10, [x0, #8] + ldrb w11, [x1, #8] + eor x8, x8, x9 + eor w9, w10, w11 + and x9, x9, #0xff + orr x8, x8, x9 + cmp x8, #0 + cset w0, eq + ret + + + Make operator fetch same size of memory to reduce instructions and get better parallelism. + + x86-64 + mov rax, qword ptr [rdi] + mov rcx, qword ptr [rdi + 1] + xor rax, qword ptr [rsi] + xor rcx, qword ptr [rsi + 1] + or rcx, rax + sete al + ret + + arm-64 v8 + ldr x8, [x0] + ldr x9, [x1] + ldur x10, [x0, #1] + ldur x11, [x1, #1] + cmp x8, x9 + cset w8, eq + cmp x10, x11 + cset w9, eq + and w0, w8, w9 + ret + + */ + return memcmp_eq8(a, b) & memcmp_eq8(a + k - 8, b + k - 8); + } + else if constexpr (k > 4) + { + if constexpr (k == 8) + return memcmp_eq8(a, b); + else + return memcmp_eq4(a, b) & memcmp_eq4(a + k - 4, b + k - 4); + } + else if constexpr (k > 2) + { + M(3); + M(4); + } + else + { + M(1); + M(2); + M(0); + } +#undef M +} + +} // namespace mem_utils From a89c8b6cadce86878c16299355b4dc528844529b Mon Sep 17 00:00:00 2001 From: yanweiqi <592838129@qq.com> Date: Tue, 16 Aug 2022 17:12:50 +0800 Subject: [PATCH 5/5] *: Remove embeddedDictionary and ExternalDictionary (#5627) ref pingcap/tiflash#5628 --- dbms/src/Databases/DatabaseDictionary.cpp | 239 --- dbms/src/Databases/DatabaseDictionary.h | 119 -- dbms/src/Databases/DatabaseFactory.cpp | 13 +- .../Embedded/GeoDictionariesLoader.cpp | 45 - .../Embedded/GeoDictionariesLoader.h | 29 - .../Embedded/GeodataProviders/Entries.h | 33 - .../GeodataProviders/HierarchiesProvider.cpp | 90 - .../GeodataProviders/HierarchiesProvider.h | 73 - .../HierarchyFormatReader.cpp | 72 - .../GeodataProviders/HierarchyFormatReader.h | 35 - .../GeodataProviders/IHierarchiesProvider.h | 64 - .../GeodataProviders/INamesProvider.h | 68 - .../GeodataProviders/NamesFormatReader.cpp | 41 - .../GeodataProviders/NamesFormatReader.h | 34 - .../GeodataProviders/NamesProvider.cpp | 63 - .../Embedded/GeodataProviders/NamesProvider.h | 65 - .../Embedded/GeodataProviders/Types.h | 32 - .../Embedded/IGeoDictionariesLoader.h | 37 - .../Embedded/RegionsHierarchies.cpp | 34 - .../Embedded/RegionsHierarchies.h | 56 - .../Embedded/RegionsHierarchy.cpp | 162 -- .../Dictionaries/Embedded/RegionsHierarchy.h | 145 -- .../Dictionaries/Embedded/RegionsNames.cpp | 107 -- dbms/src/Dictionaries/Embedded/RegionsNames.h | 133 -- .../Dictionaries/Embedded/TechDataHierarchy.h | 90 - .../FunctionsEmbeddedDictionaries.cpp | 34 - .../Functions/FunctionsEmbeddedDictionaries.h | 683 ------- .../FunctionsExternalDictionaries.cpp | 55 - .../Functions/FunctionsExternalDictionaries.h | 1703 ----------------- dbms/src/Functions/registerFunctions.cpp | 4 - dbms/src/Interpreters/Context.cpp | 78 - dbms/src/Interpreters/Context.h | 10 - .../src/Interpreters/EmbeddedDictionaries.cpp | 163 -- dbms/src/Interpreters/EmbeddedDictionaries.h | 119 -- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 27 +- .../src/Interpreters/ExternalDictionaries.cpp | 64 - dbms/src/Interpreters/ExternalDictionaries.h | 65 - .../Interpreters/IRuntimeComponentsFactory.h | 3 - .../Interpreters/RuntimeComponentsFactory.h | 6 - dbms/src/Server/Server.cpp | 15 - dbms/src/Storages/StorageDictionary.cpp | 122 -- dbms/src/Storages/StorageDictionary.h | 89 - .../System/StorageSystemDictionaries.cpp | 123 -- .../System/StorageSystemDictionaries.h | 49 - .../Storages/System/attachSystemTables.cpp | 2 - dbms/src/Storages/registerStorages.cpp | 2 - 46 files changed, 8 insertions(+), 5287 deletions(-) delete mode 100644 dbms/src/Databases/DatabaseDictionary.cpp delete mode 100644 dbms/src/Databases/DatabaseDictionary.h delete mode 100644 dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.cpp delete mode 100644 dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.h delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.cpp delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.cpp delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h delete mode 100644 dbms/src/Dictionaries/Embedded/GeodataProviders/Types.h delete mode 100644 dbms/src/Dictionaries/Embedded/IGeoDictionariesLoader.h delete mode 100644 dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp delete mode 100644 dbms/src/Dictionaries/Embedded/RegionsHierarchies.h delete mode 100644 dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp delete mode 100644 dbms/src/Dictionaries/Embedded/RegionsHierarchy.h delete mode 100644 dbms/src/Dictionaries/Embedded/RegionsNames.cpp delete mode 100644 dbms/src/Dictionaries/Embedded/RegionsNames.h delete mode 100644 dbms/src/Dictionaries/Embedded/TechDataHierarchy.h delete mode 100644 dbms/src/Functions/FunctionsEmbeddedDictionaries.cpp delete mode 100644 dbms/src/Functions/FunctionsEmbeddedDictionaries.h delete mode 100644 dbms/src/Functions/FunctionsExternalDictionaries.cpp delete mode 100644 dbms/src/Functions/FunctionsExternalDictionaries.h delete mode 100644 dbms/src/Interpreters/EmbeddedDictionaries.cpp delete mode 100644 dbms/src/Interpreters/EmbeddedDictionaries.h delete mode 100644 dbms/src/Interpreters/ExternalDictionaries.cpp delete mode 100644 dbms/src/Interpreters/ExternalDictionaries.h delete mode 100644 dbms/src/Storages/StorageDictionary.cpp delete mode 100644 dbms/src/Storages/StorageDictionary.h delete mode 100644 dbms/src/Storages/System/StorageSystemDictionaries.cpp delete mode 100644 dbms/src/Storages/System/StorageSystemDictionaries.h diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp deleted file mode 100644 index 0950468e95b..00000000000 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ -extern const int TABLE_ALREADY_EXISTS; -extern const int UNKNOWN_TABLE; -extern const int LOGICAL_ERROR; -extern const int CANNOT_GET_CREATE_TABLE_QUERY; -extern const int SYNTAX_ERROR; -} // namespace ErrorCodes - -DatabaseDictionary::DatabaseDictionary(const String & name_, const Context & context) - : name(name_) - , external_dictionaries(context.getExternalDictionaries()) - , log(&Poco::Logger::get("DatabaseDictionary(" + name + ")")) -{ -} - -void DatabaseDictionary::loadTables(Context &, ThreadPool *, bool) -{ -} - -Tables DatabaseDictionary::loadTables() -{ - auto objects_map = external_dictionaries.getObjectsMap(); - const auto & dictionaries = objects_map.get(); - - Tables tables; - for (const auto & pair : dictionaries) - { - const std::string & name = pair.first; - if (deleted_tables.count(name)) - continue; - auto dict_ptr = std::static_pointer_cast(pair.second.loadable); - if (dict_ptr) - { - const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); - auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - tables[name] = StorageDictionary::create(name, ColumnsDescription{columns}, dictionary_structure, name); - } - } - - return tables; -} - -bool DatabaseDictionary::isTableExist( - const Context & /*context*/, - const String & table_name) const -{ - auto objects_map = external_dictionaries.getObjectsMap(); - const auto & dictionaries = objects_map.get(); - return dictionaries.count(table_name) && !deleted_tables.count(table_name); -} - -StoragePtr DatabaseDictionary::tryGetTable( - const Context & /*context*/, - const String & table_name) const -{ - auto objects_map = external_dictionaries.getObjectsMap(); - const auto & dictionaries = objects_map.get(); - - if (deleted_tables.count(table_name)) - return {}; - { - auto it = dictionaries.find(table_name); - if (it != dictionaries.end()) - { - const auto & dict_ptr = std::static_pointer_cast(it->second.loadable); - if (dict_ptr) - { - const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); - auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - return StorageDictionary::create(table_name, ColumnsDescription{columns}, dictionary_structure, table_name); - } - } - } - - return {}; -} - -DatabaseIteratorPtr DatabaseDictionary::getIterator(const Context & /*context*/) -{ - return std::make_unique(loadTables()); -} - -bool DatabaseDictionary::empty(const Context & /*context*/) const -{ - auto objects_map = external_dictionaries.getObjectsMap(); - const auto & dictionaries = objects_map.get(); - for (const auto & pair : dictionaries) - if (pair.second.loadable && !deleted_tables.count(pair.first)) - return false; - return true; -} - -StoragePtr DatabaseDictionary::detachTable(const String & /*table_name*/) -{ - throw Exception("DatabaseDictionary: detachTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); -} - -void DatabaseDictionary::attachTable(const String & /*table_name*/, const StoragePtr & /*table*/) -{ - throw Exception("DatabaseDictionary: attachTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); -} - -void DatabaseDictionary::createTable( - const Context & /*context*/, - const String & /*table_name*/, - const StoragePtr & /*table*/, - const ASTPtr & /*query*/) -{ - throw Exception("DatabaseDictionary: createTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); -} - -void DatabaseDictionary::removeTable( - const Context & context, - const String & table_name) -{ - if (!isTableExist(context, table_name)) - throw Exception("Table " + name + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); - - auto objects_map = external_dictionaries.getObjectsMap(); - deleted_tables.insert(table_name); -} - -void DatabaseDictionary::renameTable( - const Context &, - const String &, - IDatabase &, - const String &) -{ - throw Exception("DatabaseDictionary: renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); -} - -void DatabaseDictionary::alterTable( - const Context &, - const String &, - const ColumnsDescription &, - const ASTModifier &) -{ - throw Exception("DatabaseDictionary: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); -} - -time_t DatabaseDictionary::getTableMetadataModificationTime( - const Context &, - const String &) -{ - return static_cast(0); -} - -ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context, - const String & table_name, - bool throw_on_error) const -{ - String query; - { - WriteBufferFromString buffer(query); - - const auto & dictionaries = context.getExternalDictionaries(); - auto dictionary = throw_on_error ? dictionaries.getDictionary(table_name) - : dictionaries.tryGetDictionary(table_name); - - auto names_and_types = StorageDictionary::getNamesAndTypes(dictionary->getStructure()); - buffer << "CREATE TABLE " << backQuoteIfNeed(name) << '.' << backQuoteIfNeed(table_name) << " ("; - buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types.begin(), names_and_types.end()); - buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")"; - } - - ParserCreateQuery parser; - const char * pos = query.data(); - std::string error_message; - auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, - /* hilite = */ false, - "", - /* allow_multi_statements = */ false, - 0); - - if (!ast && throw_on_error) - throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); - - return ast; -} - -ASTPtr DatabaseDictionary::getCreateTableQuery(const Context & context, const String & table_name) const -{ - return getCreateTableQueryImpl(context, table_name, true); -} - -ASTPtr DatabaseDictionary::tryGetCreateTableQuery(const Context & context, const String & table_name) const -{ - return getCreateTableQueryImpl(context, table_name, false); -} - -ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const -{ - String query; - { - WriteBufferFromString buffer(query); - buffer << "CREATE DATABASE " << backQuoteIfNeed(name) << " ENGINE = Dictionary"; - } - ParserCreateQuery parser; - return parseQuery(parser, query.data(), query.data() + query.size(), "", 0); -} - -void DatabaseDictionary::shutdown() -{ -} - -void DatabaseDictionary::drop(const Context & /*context*/) -{ - /// Additional actions to delete database are not required. -} - -} // namespace DB diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h deleted file mode 100644 index dfb20a2dce5..00000000000 --- a/dbms/src/Databases/DatabaseDictionary.h +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include - - -namespace Poco -{ - class Logger; -} - - -namespace DB -{ -class ExternalDictionaries; - -/* Database to store StorageDictionary tables - * automatically creates tables for all dictionaries - */ -class DatabaseDictionary : public IDatabase -{ -public: - DatabaseDictionary(const String & name_, const Context & context); - - String getEngineName() const override - { - return "Dictionary"; - } - - void loadTables( - Context & context, - ThreadPool * thread_pool, - bool has_force_restore_data_flag) override; - - bool isTableExist( - const Context & context, - const String & table_name) const override; - - StoragePtr tryGetTable( - const Context & context, - const String & table_name) const override; - - DatabaseIteratorPtr getIterator(const Context & context) override; - - bool empty(const Context & context) const override; - - void createTable( - const Context & context, - const String & table_name, - const StoragePtr & table, - const ASTPtr & query) override; - - void removeTable( - const Context & context, - const String & table_name) override; - - void attachTable(const String & table_name, const StoragePtr & table) override; - StoragePtr detachTable(const String & table_name) override; - - void renameTable( - const Context & context, - const String & table_name, - IDatabase & to_database, - const String & to_table_name) override; - - void alterTable( - const Context & context, - const String & name, - const ColumnsDescription & columns, - const ASTModifier & engine_modifier) override; - - time_t getTableMetadataModificationTime( - const Context & context, - const String & table_name) override; - - ASTPtr getCreateTableQuery( - const Context & context, - const String & table_name) const override; - - ASTPtr tryGetCreateTableQuery( - const Context & context, - const String & table_name) const override; - - ASTPtr getCreateDatabaseQuery(const Context & context) const override; - - void shutdown() override; - void drop(const Context & context) override; - -private: - const String name; - mutable std::mutex mutex; - const ExternalDictionaries & external_dictionaries; - std::unordered_set deleted_tables; - - Poco::Logger * log; - - Tables loadTables(); - - ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const; -}; - -} diff --git a/dbms/src/Databases/DatabaseFactory.cpp b/dbms/src/Databases/DatabaseFactory.cpp index b00e98f4f51..46026eaa1e1 100644 --- a/dbms/src/Databases/DatabaseFactory.cpp +++ b/dbms/src/Databases/DatabaseFactory.cpp @@ -13,7 +13,6 @@ // limitations under the License. #include -#include #include #include #include @@ -37,13 +36,17 @@ static inline ValueType safeGetLiteralValue(const ASTPtr & ast, const String & e { if (!ast || !typeid_cast(ast.get())) throw Exception( - "Database engine " + engine_name + " requested literal argument at index " + DB::toString(index), ErrorCodes::BAD_ARGUMENTS); + "Database engine " + engine_name + " requested literal argument at index " + DB::toString(index), + ErrorCodes::BAD_ARGUMENTS); return typeid_cast(ast.get())->value.safeGet(); } DatabasePtr DatabaseFactory::get( - const String & database_name, const String & metadata_path, const ASTStorage * engine_define, Context & context) + const String & database_name, + const String & metadata_path, + const ASTStorage * engine_define, + Context & context) { String engine_name = engine_define->engine->name; if (engine_name == "TiFlash") @@ -57,7 +60,7 @@ DatabasePtr DatabaseFactory::get( if (engine && engine->arguments) { const auto & arguments = engine->arguments->children; - if (arguments.size() >= 1) + if (!arguments.empty()) { const auto db_info_json = safeGetLiteralValue(arguments[0], engine_name, 0); if (!db_info_json.empty()) @@ -81,8 +84,6 @@ DatabasePtr DatabaseFactory::get( return std::make_shared(database_name, metadata_path, context); else if (engine_name == "Memory") return std::make_shared(database_name); - else if (engine_name == "Dictionary") - return std::make_shared(database_name, context); throw Exception("Unknown database engine: " + engine_name, ErrorCodes::UNKNOWN_DATABASE_ENGINE); } diff --git a/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.cpp b/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.cpp deleted file mode 100644 index 42c374f87a6..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include - - -std::unique_ptr GeoDictionariesLoader::reloadRegionsHierarchies( - const Poco::Util::AbstractConfiguration & config) -{ - static constexpr auto config_key = "path_to_regions_hierarchy_file"; - - if (!config.has(config_key)) - return {}; - - const auto default_hierarchy_file = config.getString(config_key); - auto data_provider = std::make_unique(default_hierarchy_file); - return std::make_unique(std::move(data_provider)); -} - -std::unique_ptr GeoDictionariesLoader::reloadRegionsNames( - const Poco::Util::AbstractConfiguration & config) -{ - static constexpr auto config_key = "path_to_regions_names_files"; - - if (!config.has(config_key)) - return {}; - - const auto directory = config.getString(config_key); - auto data_provider = std::make_unique(directory); - return std::make_unique(std::move(data_provider)); -} diff --git a/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.h b/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.h deleted file mode 100644 index 35264371952..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - - -// Default implementation of geo dictionaries loader used by native server application -class GeoDictionariesLoader : public IGeoDictionariesLoader -{ -public: - std::unique_ptr reloadRegionsHierarchies( - const Poco::Util::AbstractConfiguration & config) override; - - std::unique_ptr reloadRegionsNames( - const Poco::Util::AbstractConfiguration & config) override; -}; diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h b/dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h deleted file mode 100644 index 8f7f14ec183..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -struct RegionEntry -{ - RegionID id; - RegionID parent_id; - RegionType type; - RegionDepth depth; - RegionPopulation population; -}; - -struct RegionNameEntry -{ - RegionID id; - std::string name; -}; - diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp deleted file mode 100644 index 47dadeabc33..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include - -#include -#include -#include - - -bool RegionsHierarchyDataSource::isModified() const -{ - return updates_tracker.isModified(); -} - -IRegionsHierarchyReaderPtr RegionsHierarchyDataSource::createReader() -{ - updates_tracker.fixCurrentVersion(); - auto file_reader = std::make_shared(path); - return std::make_unique(std::move(file_reader)); -} - - -RegionsHierarchiesDataProvider::RegionsHierarchiesDataProvider(const std::string & path) - : path(path) -{ - discoverFilesWithCustomHierarchies(); -} - -void RegionsHierarchiesDataProvider::discoverFilesWithCustomHierarchies() -{ - std::string basename = Poco::Path(path).getBaseName(); - - Poco::Path dir_path = Poco::Path(path).absolute().parent(); - - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(dir_path); dir_it != dir_end; ++dir_it) - { - std::string candidate_basename = dir_it.path().getBaseName(); - - if ((0 == candidate_basename.compare(0, basename.size(), basename)) && - (candidate_basename.size() > basename.size() + 1) && - (candidate_basename[basename.size()] == '_')) - { - const std::string suffix = candidate_basename.substr(basename.size() + 1); - hierarchy_files.emplace(suffix, dir_it->path()); - } - } -} - -std::vector RegionsHierarchiesDataProvider::listCustomHierarchies() const -{ - std::vector names; - names.reserve(hierarchy_files.size()); - for (const auto & it : hierarchy_files) - names.push_back(it.first); - return names; -} - -IRegionsHierarchyDataSourcePtr RegionsHierarchiesDataProvider::getDefaultHierarchySource() const -{ - return std::make_shared(path); -} - -IRegionsHierarchyDataSourcePtr RegionsHierarchiesDataProvider::getHierarchySource(const std::string & name) const -{ - auto found = hierarchy_files.find(name); - - if (found != hierarchy_files.end()) - { - const auto & hierarchy_file_path = found->second; - return std::make_shared(hierarchy_file_path); - } - - throw Poco::Exception("Regions hierarchy `" + name + "` not found"); -} diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h deleted file mode 100644 index d0d5cda5117..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include - -#include - - -// Represents local file with regions hierarchy dump -class RegionsHierarchyDataSource - : public IRegionsHierarchyDataSource -{ -private: - std::string path; - FileUpdatesTracker updates_tracker; - -public: - RegionsHierarchyDataSource(const std::string & path_) - : path(path_) - , updates_tracker(path_) - {} - - bool isModified() const override; - - IRegionsHierarchyReaderPtr createReader() override; -}; - - -// Provides access to directory with multiple data source files: one file per regions hierarchy -class RegionsHierarchiesDataProvider - : public IRegionsHierarchiesDataProvider -{ -private: - // path to file with default regions hierarchy - std::string path; - - using HierarchyFiles = std::unordered_map; - HierarchyFiles hierarchy_files; - -public: - /** path must point to the file with the hierarchy of regions "by default". It will be accessible by an empty key. - * In addition, a number of files are searched for, the name of which (before the extension, if any) is added arbitrary _suffix. - * Such files are loaded, and the hierarchy of regions is put on the `suffix` key. - * - * For example, if /opt/geo/regions_hierarchy.txt is specified, - * then the /opt/geo/regions_hierarchy_ua.txt file will also be loaded, if any, it will be accessible by the `ua` key. - */ - RegionsHierarchiesDataProvider(const std::string & path); - - std::vector listCustomHierarchies() const override; - - IRegionsHierarchyDataSourcePtr getDefaultHierarchySource() const override; - IRegionsHierarchyDataSourcePtr getHierarchySource(const std::string & name) const override; - -private: - void discoverFilesWithCustomHierarchies(); -}; - diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.cpp b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.cpp deleted file mode 100644 index b3015a8d987..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.cpp +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include - - -bool RegionsHierarchyFormatReader::readNext(RegionEntry & entry) -{ - while (!input->eof()) - { - /** Our internal geobase has negative numbers, - * that means "this is garbage, ignore this row". - */ - Int32 read_region_id = 0; - Int32 read_parent_id = 0; - Int8 read_type = 0; - - DB::readIntText(read_region_id, *input); - DB::assertChar('\t', *input); - DB::readIntText(read_parent_id, *input); - DB::assertChar('\t', *input); - DB::readIntText(read_type, *input); - - /** Then there can be a newline (old version) - * or tab, the region's population, line feed (new version). - */ - RegionPopulation population = 0; - if (!input->eof() && *input->position() == '\t') - { - ++input->position(); - UInt64 population_big = 0; - DB::readIntText(population_big, *input); - population = population_big > std::numeric_limits::max() - ? std::numeric_limits::max() - : population_big; - } - DB::assertChar('\n', *input); - - if (read_region_id <= 0 || read_type < 0) - continue; - - RegionID region_id = read_region_id; - RegionID parent_id = 0; - - if (read_parent_id >= 0) - parent_id = read_parent_id; - - RegionType type = static_cast(read_type); - - entry.id = region_id; - entry.parent_id = parent_id; - entry.type = type; - entry.population = population; - return true; - } - - return false; -} diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h deleted file mode 100644 index f2716b1b8cc..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include - - -// Reads regions hierarchy in geoexport format -class RegionsHierarchyFormatReader : public IRegionsHierarchyReader -{ -private: - DB::ReadBufferPtr input; - -public: - RegionsHierarchyFormatReader(DB::ReadBufferPtr input_) - : input(std::move(input_)) - {} - - bool readNext(RegionEntry & entry) override; -}; - diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h b/dbms/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h deleted file mode 100644 index 0f289390344..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include -#include -#include - - -// Iterates over all regions in data source -class IRegionsHierarchyReader -{ -public: - virtual bool readNext(RegionEntry & entry) = 0; - - virtual ~IRegionsHierarchyReader() {} -}; - -using IRegionsHierarchyReaderPtr = std::unique_ptr; - - -// Data source for single regions hierarchy -class IRegionsHierarchyDataSource -{ -public: - // data modified since last createReader invocation - virtual bool isModified() const = 0; - - virtual IRegionsHierarchyReaderPtr createReader() = 0; - - virtual ~IRegionsHierarchyDataSource() {} -}; - -using IRegionsHierarchyDataSourcePtr = std::shared_ptr; - - -// Provides data sources for different regions hierarchies -class IRegionsHierarchiesDataProvider -{ -public: - virtual std::vector listCustomHierarchies() const = 0; - - virtual IRegionsHierarchyDataSourcePtr getDefaultHierarchySource() const = 0; - virtual IRegionsHierarchyDataSourcePtr getHierarchySource(const std::string & name) const = 0; - - virtual ~IRegionsHierarchiesDataProvider() {}; -}; - -using IRegionsHierarchiesDataProviderPtr = std::shared_ptr; - diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h b/dbms/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h deleted file mode 100644 index 65e123a9396..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include - - -// Iterates over all name entries in data source -class ILanguageRegionsNamesReader -{ -public: - virtual bool readNext(RegionNameEntry & entry) = 0; - - virtual ~ILanguageRegionsNamesReader() {} -}; - -using ILanguageRegionsNamesReaderPtr = std::unique_ptr; - - -// Regions names data source for one language -class ILanguageRegionsNamesDataSource -{ -public: - // data modified since last createReader invocation - virtual bool isModified() const = 0; - - // Upper bound on total length of all names - virtual size_t estimateTotalSize() const = 0; - - virtual ILanguageRegionsNamesReaderPtr createReader() = 0; - - virtual std::string getLanguage() const = 0; - - virtual std::string getSourceName() const = 0; - - virtual ~ILanguageRegionsNamesDataSource() {} -}; - -using ILanguageRegionsNamesDataSourcePtr = std::unique_ptr; - - -// Provides regions names data sources for different languages -class IRegionsNamesDataProvider -{ -public: - virtual ILanguageRegionsNamesDataSourcePtr getLanguageRegionsNamesSource( - const std::string& language) const = 0; - ; - - virtual ~IRegionsNamesDataProvider() {} -}; - -using IRegionsNamesDataProviderPtr = std::unique_ptr; - diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.cpp b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.cpp deleted file mode 100644 index 94a00a7c73c..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include - - -bool LanguageRegionsNamesFormatReader::readNext(RegionNameEntry & entry) -{ - while (!input->eof()) - { - Int32 read_region_id; - std::string region_name; - - DB::readIntText(read_region_id, *input); - DB::assertChar('\t', *input); - DB::readString(region_name, *input); - DB::assertChar('\n', *input); - - if (read_region_id <= 0) - continue; - - entry.id = read_region_id; - entry.name = region_name; - return true; - } - - return false; -} diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h deleted file mode 100644 index 2648d3a2e94..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include - - -// Reads regions names list in geoexport format -class LanguageRegionsNamesFormatReader : public ILanguageRegionsNamesReader -{ -private: - DB::ReadBufferPtr input; - -public: - LanguageRegionsNamesFormatReader(DB::ReadBufferPtr input_) - : input(std::move(input_)) - {} - - bool readNext(RegionNameEntry & entry) override; -}; diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp deleted file mode 100644 index 944fd83abec..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include - - -bool LanguageRegionsNamesDataSource::isModified() const -{ - return updates_tracker.isModified(); -} - -size_t LanguageRegionsNamesDataSource::estimateTotalSize() const -{ - return Poco::File(path).getSize(); -} - -ILanguageRegionsNamesReaderPtr LanguageRegionsNamesDataSource::createReader() -{ - updates_tracker.fixCurrentVersion(); - auto file_reader = std::make_shared(path); - return std::make_unique(std::move(file_reader)); -} - -std::string LanguageRegionsNamesDataSource::getLanguage() const -{ - return language; -} - -std::string LanguageRegionsNamesDataSource::getSourceName() const -{ - return path; -} - - -RegionsNamesDataProvider::RegionsNamesDataProvider(const std::string & directory_) - : directory(directory_) -{} - -ILanguageRegionsNamesDataSourcePtr RegionsNamesDataProvider::getLanguageRegionsNamesSource( - const std::string & language) const -{ - const auto data_file = getDataFilePath(language); - return std::make_unique(data_file, language); -} - -std::string RegionsNamesDataProvider::getDataFilePath(const std::string & language) const -{ - return directory + "/regions_names_" + language + ".txt"; -} diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h deleted file mode 100644 index 3a28ee6673a..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include - - -// Represents local file with list of regions ids / names -class LanguageRegionsNamesDataSource : public ILanguageRegionsNamesDataSource -{ -private: - std::string path; - FileUpdatesTracker updates_tracker; - std::string language; - -public: - LanguageRegionsNamesDataSource(const std::string & path_, const std::string & language_) - : path(path_) - , updates_tracker(path_) - , language(language_) - {} - - bool isModified() const override; - - size_t estimateTotalSize() const override; - - ILanguageRegionsNamesReaderPtr createReader() override; - - std::string getLanguage() const override; - - std::string getSourceName() const override; -}; - -using ILanguageRegionsNamesDataSourcePtr = std::unique_ptr; - - -// Provides access to directory with multiple data source files: one file per language -class RegionsNamesDataProvider : public IRegionsNamesDataProvider -{ -private: - std::string directory; - -public: - RegionsNamesDataProvider(const std::string & directory_); - - ILanguageRegionsNamesDataSourcePtr getLanguageRegionsNamesSource( - const std::string& language) const override; - -private: - std::string getDataFilePath(const std::string & language) const; -}; diff --git a/dbms/src/Dictionaries/Embedded/GeodataProviders/Types.h b/dbms/src/Dictionaries/Embedded/GeodataProviders/Types.h deleted file mode 100644 index a8a94b8700e..00000000000 --- a/dbms/src/Dictionaries/Embedded/GeodataProviders/Types.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - - -using RegionID = UInt32; -using RegionDepth = UInt8; -using RegionPopulation = UInt32; - -enum class RegionType : Int8 -{ - Hidden = -1, - Continent = 1, - Country = 3, - District = 4, - Area = 5, - City = 6, -}; diff --git a/dbms/src/Dictionaries/Embedded/IGeoDictionariesLoader.h b/dbms/src/Dictionaries/Embedded/IGeoDictionariesLoader.h deleted file mode 100644 index 7f7d18ed87e..00000000000 --- a/dbms/src/Dictionaries/Embedded/IGeoDictionariesLoader.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include - -#include - - -// Provides actual versions of geo dictionaries (regions hierarchies, regions names) -// Bind data structures (RegionsHierarchies, RegionsNames) with data providers -class IGeoDictionariesLoader -{ -public: - virtual std::unique_ptr reloadRegionsHierarchies( - const Poco::Util::AbstractConfiguration & config) = 0; - - virtual std::unique_ptr reloadRegionsNames( - const Poco::Util::AbstractConfiguration & config) = 0; - - virtual ~IGeoDictionariesLoader() {} -}; diff --git a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp b/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp deleted file mode 100644 index aa225a030b4..00000000000 --- a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - - -RegionsHierarchies::RegionsHierarchies(IRegionsHierarchiesDataProviderPtr data_provider) -{ - Poco::Logger * log = &Poco::Logger::get("RegionsHierarchies"); - - LOG_DEBUG(log, "Adding default regions hierarchy"); - data.emplace("", data_provider->getDefaultHierarchySource()); - - for (const auto & name : data_provider->listCustomHierarchies()) - { - LOG_FMT_DEBUG(log, "Adding regions hierarchy for {}", name); - data.emplace(name, data_provider->getHierarchySource(name)); - } - - reload(); -} diff --git a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h b/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h deleted file mode 100644 index 0481604b701..00000000000 --- a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include - -#include - - -/** Contains several hierarchies of regions. - * Used to support several different perspectives on the ownership of regions by countries. - * First of all, for the Crimea (Russian and Ukrainian points of view). - */ -class RegionsHierarchies -{ -private: - using Container = std::unordered_map; - Container data; - -public: - RegionsHierarchies(IRegionsHierarchiesDataProviderPtr data_provider); - - /** Reloads, if necessary, all hierarchies of regions. - */ - void reload() - { - for (auto & elem : data) - elem.second.reload(); - } - - - const RegionsHierarchy & get(const std::string & key) const - { - auto it = data.find(key); - - if (data.end() == it) - throw Poco::Exception("There is no regions hierarchy for key " + key); - - return it->second; - } -}; diff --git a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp deleted file mode 100644 index 9156d7148eb..00000000000 --- a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include - -#include - - -RegionsHierarchy::RegionsHierarchy(IRegionsHierarchyDataSourcePtr data_source_) - : data_source(data_source_) -{ -} - - -void RegionsHierarchy::reload() -{ - Poco::Logger * log = &Poco::Logger::get("RegionsHierarchy"); - - if (!data_source->isModified()) - return; - - LOG_DEBUG(log, "Reloading regions hierarchy"); - - const size_t initial_size = 10000; - const size_t max_size = 15000000; - - RegionParents new_parents(initial_size); - RegionParents new_city(initial_size); - RegionParents new_country(initial_size); - RegionParents new_area(initial_size); - RegionParents new_district(initial_size); - RegionParents new_continent(initial_size); - RegionParents new_top_continent(initial_size); - RegionPopulations new_populations(initial_size); - RegionDepths new_depths(initial_size); - RegionTypes types(initial_size); - - RegionID max_region_id = 0; - - - auto regions_reader = data_source->createReader(); - - RegionEntry region_entry; - while (regions_reader->readNext(region_entry)) - { - if (region_entry.id > max_region_id) - { - if (region_entry.id > max_size) - throw DB::Exception("Region id is too large: " + DB::toString(region_entry.id) + ", should be not more than " + DB::toString(max_size)); - - max_region_id = region_entry.id; - - while (region_entry.id >= new_parents.size()) - { - new_parents.resize(new_parents.size() * 2); - new_populations.resize(new_parents.size()); - types.resize(new_parents.size()); - } - } - - new_parents[region_entry.id] = region_entry.parent_id; - new_populations[region_entry.id] = region_entry.population; - types[region_entry.id] = region_entry.type; - } - - new_parents.resize(max_region_id + 1); - new_city.resize(max_region_id + 1); - new_country.resize(max_region_id + 1); - new_area.resize(max_region_id + 1); - new_district.resize(max_region_id + 1); - new_continent.resize(max_region_id + 1); - new_top_continent.resize(max_region_id + 1); - new_populations.resize(max_region_id + 1); - new_depths.resize(max_region_id + 1); - types.resize(max_region_id + 1); - - /// prescribe the cities and countries for the regions - for (RegionID i = 0; i <= max_region_id; ++i) - { - if (types[i] == RegionType::City) - new_city[i] = i; - - if (types[i] == RegionType::Area) - new_area[i] = i; - - if (types[i] == RegionType::District) - new_district[i] = i; - - if (types[i] == RegionType::Country) - new_country[i] = i; - - if (types[i] == RegionType::Continent) - { - new_continent[i] = i; - new_top_continent[i] = i; - } - - RegionDepth depth = 0; - RegionID current = i; - while (true) - { - ++depth; - - if (depth == std::numeric_limits::max()) - throw Poco::Exception("Logical error in regions hierarchy: region " + DB::toString(current) + " possible is inside infinite loop"); - - current = new_parents[current]; - if (current == 0) - break; - - if (current > max_region_id) - throw Poco::Exception("Logical error in regions hierarchy: region " + DB::toString(current) + " (specified as parent) doesn't exist"); - - if (types[current] == RegionType::City) - new_city[i] = current; - - if (types[current] == RegionType::Area) - new_area[i] = current; - - if (types[current] == RegionType::District) - new_district[i] = current; - - if (types[current] == RegionType::Country) - new_country[i] = current; - - if (types[current] == RegionType::Continent) - { - if (!new_continent[i]) - new_continent[i] = current; - new_top_continent[i] = current; - } - } - - new_depths[i] = depth; - } - - parents.swap(new_parents); - country.swap(new_country); - city.swap(new_city); - area.swap(new_area); - district.swap(new_district); - continent.swap(new_continent); - top_continent.swap(new_top_continent); - populations.swap(new_populations); - depths.swap(new_depths); -} diff --git a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h deleted file mode 100644 index 8970d39f14b..00000000000 --- a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include -#include -#include - - -class IRegionsHierarchyDataProvider; - -/** A class that lets you know if a region belongs to one RegionID region with another RegionID. - * Information about the hierarchy of regions is downloaded from a text file. - * Can on request update the data. - */ -class RegionsHierarchy : private boost::noncopyable -{ -private: - /// Relationship parent; 0, if there are no parents, the usual lookup table. - using RegionParents = std::vector; - /// type of region - using RegionTypes = std::vector; - /// depth in the tree, starting from the country (country: 1, root: 0) - using RegionDepths = std::vector; - /// population of the region. If more than 2 ^ 32 - 1, then it is equated to this maximum. - using RegionPopulations = std::vector; - - /// region -> parent region - RegionParents parents; - /// region -> city, including it or 0, if there is none - RegionParents city; - /// region -> country including it or 0, if there is none - RegionParents country; - /// region -> area that includes it or 0, if not - RegionParents area; - /// region -> district, including it or 0, if there is none - RegionParents district; - /// region -> the continent (the first when climbing in the hierarchy of regions), including it or 0, if there is none - RegionParents continent; - /// region -> the continent (the latter when you climb the hierarchy of regions), including it or 0, if there is none - RegionParents top_continent; - - /// region -> population or 0, if unknown. - RegionPopulations populations; - - /// region - depth in the tree - RegionDepths depths; - - IRegionsHierarchyDataSourcePtr data_source; - -public: - RegionsHierarchy(IRegionsHierarchyDataSourcePtr data_source_); - - /// Reloads, if necessary, the hierarchy of regions. Not threadsafe. - void reload(); - - - bool in(RegionID lhs, RegionID rhs) const - { - if (lhs >= parents.size()) - return false; - - while (lhs != 0 && lhs != rhs) - lhs = parents[lhs]; - - return lhs != 0; - } - - RegionID toCity(RegionID region) const - { - if (region >= city.size()) - return 0; - return city[region]; - } - - RegionID toCountry(RegionID region) const - { - if (region >= country.size()) - return 0; - return country[region]; - } - - RegionID toArea(RegionID region) const - { - if (region >= area.size()) - return 0; - return area[region]; - } - - RegionID toDistrict(RegionID region) const - { - if (region >= district.size()) - return 0; - return district[region]; - } - - RegionID toContinent(RegionID region) const - { - if (region >= continent.size()) - return 0; - return continent[region]; - } - - RegionID toTopContinent(RegionID region) const - { - if (region >= top_continent.size()) - return 0; - return top_continent[region]; - } - - RegionID toParent(RegionID region) const - { - if (region >= parents.size()) - return 0; - return parents[region]; - } - - RegionDepth getDepth(RegionID region) const - { - if (region >= depths.size()) - return 0; - return depths[region]; - } - - RegionPopulation getPopulation(RegionID region) const - { - if (region >= populations.size()) - return 0; - return populations[region]; - } -}; diff --git a/dbms/src/Dictionaries/Embedded/RegionsNames.cpp b/dbms/src/Dictionaries/Embedded/RegionsNames.cpp deleted file mode 100644 index 503838ce92b..00000000000 --- a/dbms/src/Dictionaries/Embedded/RegionsNames.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include - - -RegionsNames::RegionsNames(IRegionsNamesDataProviderPtr data_provider) -{ - for (size_t language_id = 0; language_id < SUPPORTED_LANGUAGES_COUNT; ++language_id) - { - const std::string & language = getSupportedLanguages()[language_id]; - names_sources[language_id] = data_provider->getLanguageRegionsNamesSource(language); - } - - reload(); -} - -std::string RegionsNames::dumpSupportedLanguagesNames() -{ - std::string res; - for (size_t i = 0; i < LANGUAGE_ALIASES_COUNT; ++i) - { - if (i > 0) - res += ", "; - res += '\''; - res += getLanguageAliases()[i].name; - res += '\''; - } - return res; -} - -void RegionsNames::reload() -{ - Poco::Logger * log = &Poco::Logger::get("RegionsNames"); - LOG_DEBUG(log, "Reloading regions names"); - - RegionID max_region_id = 0; - for (size_t language_id = 0; language_id < SUPPORTED_LANGUAGES_COUNT; ++language_id) - { - const std::string & language = getSupportedLanguages()[language_id]; - - auto names_source = names_sources[language_id]; - - if (!names_source->isModified()) - continue; - - LOG_FMT_DEBUG(log, "Reloading regions names for language: {}", language); - - auto names_reader = names_source->createReader(); - - const size_t initial_size = 10000; - const size_t max_size = 15000000; - - Chars new_chars; - StringRefs new_names_refs(initial_size, StringRef("", 0)); - - /// Allocate a continuous slice of memory, which is enough to store all names. - new_chars.reserve(names_source->estimateTotalSize()); - - RegionNameEntry name_entry; - while (names_reader->readNext(name_entry)) - { - size_t old_size = new_chars.size(); - - if (new_chars.capacity() < old_size + name_entry.name.length() + 1) - throw Poco::Exception("Logical error. Maybe size estimate of " + names_source->getSourceName() + " is wrong."); - - new_chars.resize(old_size + name_entry.name.length() + 1); - memcpy(&new_chars[old_size], name_entry.name.c_str(), name_entry.name.length() + 1); - - if (name_entry.id > max_region_id) - { - max_region_id = name_entry.id; - - if (name_entry.id > max_size) - throw DB::Exception("Region id is too large: " + DB::toString(name_entry.id) + ", should be not more than " + DB::toString(max_size)); - } - - while (name_entry.id >= new_names_refs.size()) - new_names_refs.resize(new_names_refs.size() * 2, StringRef("", 0)); - - new_names_refs[name_entry.id] = StringRef(&new_chars[old_size], name_entry.name.length()); - } - - chars[language_id].swap(new_chars); - names_refs[language_id].swap(new_names_refs); - } - - for (size_t language_id = 0; language_id < SUPPORTED_LANGUAGES_COUNT; ++language_id) - names_refs[language_id].resize(max_region_id + 1, StringRef("", 0)); -} diff --git a/dbms/src/Dictionaries/Embedded/RegionsNames.h b/dbms/src/Dictionaries/Embedded/RegionsNames.h deleted file mode 100644 index cf8bae56bea..00000000000 --- a/dbms/src/Dictionaries/Embedded/RegionsNames.h +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include - -#include -#include - -#include -#include - - -/** A class that allows you to recognize by region id its text name in one of the supported languages: ru, en, ua, by, kz, tr. - * - * Information about region names loads from text files with the following format names: - * regions_names_xx.txt, - * where xx is one of the two letters of the following supported languages: - * ru, en, ua, by, kz, tr. - * - * Can on request update the data. - */ -class RegionsNames -{ -public: - enum class Language - { - RU = 0, - EN, - UA, - BY, - KZ, - TR, - }; - -private: - static const size_t ROOT_LANGUAGE = 0; - static const size_t SUPPORTED_LANGUAGES_COUNT = 6; - static const size_t LANGUAGE_ALIASES_COUNT = 7; - - static const char ** getSupportedLanguages() - { - static const char * res[] { "ru", "en", "ua", "by", "kz", "tr" }; - return res; - } - - struct language_alias { const char * const name; const Language lang; }; - static const language_alias * getLanguageAliases() - { - static constexpr const language_alias language_aliases[] - { - { "ru", Language::RU }, - { "en", Language::EN }, - { "ua", Language::UA }, - { "uk", Language::UA }, - { "by", Language::BY }, - { "kz", Language::KZ }, - { "tr", Language::TR } - }; - - return language_aliases; - } - - using NamesSources = std::vector>; - - using Chars = std::vector; - using CharsForLanguageID = std::vector; - using StringRefs = std::vector; /// Lookup table RegionID -> StringRef - using StringRefsForLanguageID = std::vector; - -public: - RegionsNames(IRegionsNamesDataProviderPtr data_provider); - - StringRef getRegionName(RegionID region_id, Language language = Language::RU) const - { - size_t language_id = static_cast(language); - - if (region_id > names_refs[language_id].size()) - return StringRef("", 0); - - StringRef ref = names_refs[language_id][region_id]; - - while (ref.size == 0 && language_id != ROOT_LANGUAGE) - { - static const size_t FALLBACK[] = { 0, 0, 0, 0, 0, 1 }; - language_id = FALLBACK[language_id]; - ref = names_refs[language_id][region_id]; - } - - return ref; - } - - static Language getLanguageEnum(const std::string & language) - { - if (language.size() == 2) - { - for (size_t i = 0; i < LANGUAGE_ALIASES_COUNT; ++i) - { - const auto & alias = getLanguageAliases()[i]; - if (language[0] == alias.name[0] && language[1] == alias.name[1]) - return alias.lang; - } - } - throw Poco::Exception("Unsupported language for region name. Supported languages are: " + dumpSupportedLanguagesNames() + "."); - } - - void reload(); - -private: - static std::string dumpSupportedLanguagesNames(); - - NamesSources names_sources = NamesSources(SUPPORTED_LANGUAGES_COUNT); - - /// Bytes of names for each language, laid out in a row, separated by zeros - CharsForLanguageID chars = CharsForLanguageID(SUPPORTED_LANGUAGES_COUNT); - - /// Mapping for each language from the region id into a pointer to the byte range of the name - StringRefsForLanguageID names_refs = StringRefsForLanguageID(SUPPORTED_LANGUAGES_COUNT); -}; diff --git a/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h b/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h deleted file mode 100644 index 6f037606fc2..00000000000 --- a/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include - -#include - - -/** @brief Class that lets you know if a search engine or operating system belongs - * another search engine or operating system, respectively. - * Information about the hierarchy of regions is downloaded from the database. - */ -class TechDataHierarchy -{ -private: - UInt8 os_parent[256]{}; - UInt8 se_parent[256]{}; - -public: - void reload(); - - /// Has corresponding section in configuration file. - static bool isConfigured(const Poco::Util::AbstractConfiguration & config); - - - /// The "belongs" relation. - bool isOSIn(UInt8 lhs, UInt8 rhs) const - { - while (lhs != rhs && os_parent[lhs]) - lhs = os_parent[lhs]; - - return lhs == rhs; - } - - bool isSEIn(UInt8 lhs, UInt8 rhs) const - { - while (lhs != rhs && se_parent[lhs]) - lhs = se_parent[lhs]; - - return lhs == rhs; - } - - - UInt8 OSToParent(UInt8 x) const - { - return os_parent[x]; - } - - UInt8 SEToParent(UInt8 x) const - { - return se_parent[x]; - } - - - /// To the topmost ancestor. - UInt8 OSToMostAncestor(UInt8 x) const - { - while (os_parent[x]) - x = os_parent[x]; - return x; - } - - UInt8 SEToMostAncestor(UInt8 x) const - { - while (se_parent[x]) - x = se_parent[x]; - return x; - } -}; - - -class TechDataHierarchySingleton : public ext::Singleton - , public TechDataHierarchy -{ -}; diff --git a/dbms/src/Functions/FunctionsEmbeddedDictionaries.cpp b/dbms/src/Functions/FunctionsEmbeddedDictionaries.cpp deleted file mode 100644 index 869901ae601..00000000000 --- a/dbms/src/Functions/FunctionsEmbeddedDictionaries.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -namespace DB -{ -void registerFunctionsEmbeddedDictionaries(FunctionFactory & factory) -{ - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); -} - -} // namespace DB diff --git a/dbms/src/Functions/FunctionsEmbeddedDictionaries.h b/dbms/src/Functions/FunctionsEmbeddedDictionaries.h deleted file mode 100644 index 011f5b1ccf0..00000000000 --- a/dbms/src/Functions/FunctionsEmbeddedDictionaries.h +++ /dev/null @@ -1,683 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ -extern const int DICTIONARIES_WAS_NOT_LOADED; -extern const int BAD_ARGUMENTS; -extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} // namespace ErrorCodes - -/** Functions using Yandex.Metrica dictionaries - * - dictionaries of regions, operating systems, search engines. - * - * Climb up the tree to a certain level. - * regionToCity, regionToArea, regionToCountry, ... - * - * Convert values of a column - * regionToName - * - * Whether the first identifier is a descendant of the second. - * regionIn - * - * Get an array of region identifiers, consisting of the source and the parents chain. Order implementation defined. - * regionHierarchy - */ - - -struct RegionToCityImpl -{ - static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toCity(x); } -}; - -struct RegionToAreaImpl -{ - static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toArea(x); } -}; - -struct RegionToDistrictImpl -{ - static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toDistrict(x); } -}; - -struct RegionToCountryImpl -{ - static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toCountry(x); } -}; - -struct RegionToContinentImpl -{ - static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toContinent(x); } -}; - -struct RegionToTopContinentImpl -{ - static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toTopContinent(x); } -}; - -struct RegionToPopulationImpl -{ - static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.getPopulation(x); } -}; - -struct RegionInImpl -{ - static bool apply(UInt32 x, UInt32 y, const RegionsHierarchy & hierarchy) { return hierarchy.in(x, y); } -}; - -struct RegionHierarchyImpl -{ - static UInt32 toParent(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toParent(x); } -}; - - -/** Auxiliary thing, allowing to get from the dictionary a specific dictionary, corresponding to the point of view - * (the dictionary key passed as function argument). - * Example: when calling regionToCountry(x, 'ua'), a dictionary can be used, in which Crimea refers to Ukraine. - */ -struct RegionsHierarchyGetter -{ - using Src = const RegionsHierarchies; - using Dst = const RegionsHierarchy; - - static Dst & get(Src & src, const std::string & key) - { - return src.get(key); - } -}; - -/** For dictionaries without key support. Doing nothing. - */ -template -struct IdentityDictionaryGetter -{ - using Src = const Dict; - using Dst = const Dict; - - static Dst & get(Src & src, const std::string & key) - { - if (key.empty()) - return src; - else - throw Exception("Dictionary doesn't support 'point of view' keys.", ErrorCodes::BAD_ARGUMENTS); - } -}; - - -/// Converts an identifier using a dictionary. -template -class FunctionTransformWithDictionary : public IFunction -{ -public: - static constexpr auto name = Name::name; - using base_type = FunctionTransformWithDictionary; - -private: - const std::shared_ptr owned_dict; - -public: - FunctionTransformWithDictionary(const std::shared_ptr & owned_dict_) - : owned_dict(owned_dict_) - { - if (!owned_dict) - throw Exception("Dictionaries was not loaded. You need to check configuration file.", ErrorCodes::DICTIONARIES_WAS_NOT_LOADED); - } - - String getName() const override - { - return name; - } - - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.size() != 1 && arguments.size() != 2) - throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " - + toString(arguments.size()) + ", should be 1 or 2.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - if (arguments[0]->getName() != TypeName::get()) - throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName() - + " (must be " + String(TypeName::get()) + ")", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (arguments.size() == 2 && arguments[1]->getName() != TypeName::get()) - throw Exception("Illegal type " + arguments[1]->getName() + " of the second ('point of view') argument of function " + getName() - + " (must be " + String(TypeName::get()) + ")", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - return arguments[0]; - } - - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override - { - /// The dictionary key that defines the "point of view". - std::string dict_key; - - if (arguments.size() == 2) - { - const ColumnConst * key_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - - if (!key_col) - throw Exception("Illegal column " + block.getByPosition(arguments[1]).column->getName() - + " of second ('point of view') argument of function " + name - + ". Must be constant string.", - ErrorCodes::ILLEGAL_COLUMN); - - dict_key = key_col->getValue(); - } - - const typename DictGetter::Dst & dict = DictGetter::get(*owned_dict, dict_key); - - if (const ColumnVector * col_from = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get())) - { - auto col_to = ColumnVector::create(); - - const typename ColumnVector::Container & vec_from = col_from->getData(); - typename ColumnVector::Container & vec_to = col_to->getData(); - size_t size = vec_from.size(); - vec_to.resize(size); - - for (size_t i = 0; i < size; ++i) - vec_to[i] = Transform::apply(vec_from[i], dict); - - block.getByPosition(result).column = std::move(col_to); - } - else - throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName() - + " of first argument of function " + name, - ErrorCodes::ILLEGAL_COLUMN); - } -}; - - -/// Checks belonging using a dictionary. -template -class FunctionIsInWithDictionary : public IFunction -{ -public: - static constexpr auto name = Name::name; - using base_type = FunctionIsInWithDictionary; - -private: - const std::shared_ptr owned_dict; - -public: - FunctionIsInWithDictionary(const std::shared_ptr & owned_dict_) - : owned_dict(owned_dict_) - { - if (!owned_dict) - throw Exception("Dictionaries was not loaded. You need to check configuration file.", ErrorCodes::DICTIONARIES_WAS_NOT_LOADED); - } - - String getName() const override - { - return name; - } - - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.size() != 2 && arguments.size() != 3) - throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " - + toString(arguments.size()) + ", should be 2 or 3.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - if (arguments[0]->getName() != TypeName::get()) - throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + " (must be " + String(TypeName::get()) + ")", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (arguments[1]->getName() != TypeName::get()) - throw Exception("Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + " (must be " + String(TypeName::get()) + ")", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (arguments.size() == 3 && arguments[2]->getName() != TypeName::get()) - throw Exception("Illegal type " + arguments[2]->getName() + " of the third ('point of view') argument of function " + getName() - + " (must be " + String(TypeName::get()) + ")", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override - { - /// The dictionary key that defines the "point of view". - std::string dict_key; - - if (arguments.size() == 3) - { - const ColumnConst * key_col = checkAndGetColumnConst(block.getByPosition(arguments[2]).column.get()); - - if (!key_col) - throw Exception("Illegal column " + block.getByPosition(arguments[2]).column->getName() - + " of third ('point of view') argument of function " + name - + ". Must be constant string.", - ErrorCodes::ILLEGAL_COLUMN); - - dict_key = key_col->getValue(); - } - - const typename DictGetter::Dst & dict = DictGetter::get(*owned_dict, dict_key); - - const ColumnVector * col_vec1 = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get()); - const ColumnVector * col_vec2 = checkAndGetColumn>(block.getByPosition(arguments[1]).column.get()); - const ColumnConst * col_const1 = checkAndGetColumnConst>(block.getByPosition(arguments[0]).column.get()); - const ColumnConst * col_const2 = checkAndGetColumnConst>(block.getByPosition(arguments[1]).column.get()); - - if (col_vec1 && col_vec2) - { - auto col_to = ColumnUInt8::create(); - - const typename ColumnVector::Container & vec_from1 = col_vec1->getData(); - const typename ColumnVector::Container & vec_from2 = col_vec2->getData(); - typename ColumnUInt8::Container & vec_to = col_to->getData(); - size_t size = vec_from1.size(); - vec_to.resize(size); - - for (size_t i = 0; i < size; ++i) - vec_to[i] = Transform::apply(vec_from1[i], vec_from2[i], dict); - - block.getByPosition(result).column = std::move(col_to); - } - else if (col_vec1 && col_const2) - { - auto col_to = ColumnUInt8::create(); - - const typename ColumnVector::Container & vec_from1 = col_vec1->getData(); - const T const_from2 = col_const2->template getValue(); - typename ColumnUInt8::Container & vec_to = col_to->getData(); - size_t size = vec_from1.size(); - vec_to.resize(size); - - for (size_t i = 0; i < size; ++i) - vec_to[i] = Transform::apply(vec_from1[i], const_from2, dict); - - block.getByPosition(result).column = std::move(col_to); - } - else if (col_const1 && col_vec2) - { - auto col_to = ColumnUInt8::create(); - - const T const_from1 = col_const1->template getValue(); - const typename ColumnVector::Container & vec_from2 = col_vec2->getData(); - typename ColumnUInt8::Container & vec_to = col_to->getData(); - size_t size = vec_from2.size(); - vec_to.resize(size); - - for (size_t i = 0; i < size; ++i) - vec_to[i] = Transform::apply(const_from1, vec_from2[i], dict); - - block.getByPosition(result).column = std::move(col_to); - } - else if (col_const1 && col_const2) - { - block.getByPosition(result).column = DataTypeUInt8().createColumnConst(col_const1->size(), - toField(Transform::apply(col_const1->template getValue(), col_const2->template getValue(), dict))); - } - else - throw Exception("Illegal columns " + block.getByPosition(arguments[0]).column->getName() - + " and " + block.getByPosition(arguments[1]).column->getName() - + " of arguments of function " + name, - ErrorCodes::ILLEGAL_COLUMN); - } -}; - - -/// Gets an array of identifiers consisting of the source and the parents chain. -template -class FunctionHierarchyWithDictionary : public IFunction -{ -public: - static constexpr auto name = Name::name; - using base_type = FunctionHierarchyWithDictionary; - -private: - const std::shared_ptr owned_dict; - -public: - FunctionHierarchyWithDictionary(const std::shared_ptr & owned_dict_) - : owned_dict(owned_dict_) - { - if (!owned_dict) - throw Exception("Dictionaries was not loaded. You need to check configuration file.", ErrorCodes::DICTIONARIES_WAS_NOT_LOADED); - } - - String getName() const override - { - return name; - } - - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.size() != 1 && arguments.size() != 2) - throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " - + toString(arguments.size()) + ", should be 1 or 2.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - if (arguments[0]->getName() != TypeName::get()) - throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName() - + " (must be " + String(TypeName::get()) + ")", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (arguments.size() == 2 && arguments[1]->getName() != TypeName::get()) - throw Exception("Illegal type " + arguments[1]->getName() + " of the second ('point of view') argument of function " + getName() - + " (must be " + String(TypeName::get()) + ")", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - return std::make_shared(arguments[0]); - } - - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override - { - /// The dictionary key that defines the "point of view". - std::string dict_key; - - if (arguments.size() == 2) - { - const ColumnConst * key_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - - if (!key_col) - throw Exception("Illegal column " + block.getByPosition(arguments[1]).column->getName() - + " of second ('point of view') argument of function " + name - + ". Must be constant string.", - ErrorCodes::ILLEGAL_COLUMN); - - dict_key = key_col->getValue(); - } - - const typename DictGetter::Dst & dict = DictGetter::get(*owned_dict, dict_key); - - if (const ColumnVector * col_from = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get())) - { - auto col_values = ColumnVector::create(); - auto col_offsets = ColumnArray::ColumnOffsets::create(); - - auto & res_offsets = col_offsets->getData(); - auto & res_values = col_values->getData(); - - const typename ColumnVector::Container & vec_from = col_from->getData(); - size_t size = vec_from.size(); - res_offsets.resize(size); - res_values.reserve(size * 4); - - for (size_t i = 0; i < size; ++i) - { - T cur = vec_from[i]; - while (cur) - { - res_values.push_back(cur); - cur = Transform::toParent(cur, dict); - } - res_offsets[i] = res_values.size(); - } - - block.getByPosition(result).column = ColumnArray::create(std::move(col_values), std::move(col_offsets)); - } - else - throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName() - + " of first argument of function " + name, - ErrorCodes::ILLEGAL_COLUMN); - } -}; - - -struct NameRegionToCity -{ - static constexpr auto name = "regionToCity"; -}; -struct NameRegionToArea -{ - static constexpr auto name = "regionToArea"; -}; -struct NameRegionToDistrict -{ - static constexpr auto name = "regionToDistrict"; -}; -struct NameRegionToCountry -{ - static constexpr auto name = "regionToCountry"; -}; -struct NameRegionToContinent -{ - static constexpr auto name = "regionToContinent"; -}; -struct NameRegionToTopContinent -{ - static constexpr auto name = "regionToTopContinent"; -}; -struct NameRegionToPopulation -{ - static constexpr auto name = "regionToPopulation"; -}; -struct NameRegionHierarchy -{ - static constexpr auto name = "regionHierarchy"; -}; -struct NameRegionIn -{ - static constexpr auto name = "regionIn"; -}; - -struct FunctionRegionToCity : public FunctionTransformWithDictionary -{ - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getEmbeddedDictionaries().getRegionsHierarchies()); - } -}; - -struct FunctionRegionToArea : public FunctionTransformWithDictionary -{ - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getEmbeddedDictionaries().getRegionsHierarchies()); - } -}; - -struct FunctionRegionToDistrict : public FunctionTransformWithDictionary -{ - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getEmbeddedDictionaries().getRegionsHierarchies()); - } -}; - -struct FunctionRegionToCountry : public FunctionTransformWithDictionary -{ - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getEmbeddedDictionaries().getRegionsHierarchies()); - } -}; - -struct FunctionRegionToContinent : public FunctionTransformWithDictionary -{ - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getEmbeddedDictionaries().getRegionsHierarchies()); - } -}; - -struct FunctionRegionToTopContinent : public FunctionTransformWithDictionary -{ - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getEmbeddedDictionaries().getRegionsHierarchies()); - } -}; - -struct FunctionRegionToPopulation : public FunctionTransformWithDictionary -{ - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getEmbeddedDictionaries().getRegionsHierarchies()); - } -}; - -struct FunctionRegionIn : public FunctionIsInWithDictionary -{ - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getEmbeddedDictionaries().getRegionsHierarchies()); - } -}; - -struct FunctionRegionHierarchy : public FunctionHierarchyWithDictionary -{ - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getEmbeddedDictionaries().getRegionsHierarchies()); - } -}; - -/// Converts a region's numeric identifier to a name in the specified language using a dictionary. -class FunctionRegionToName : public IFunction -{ -public: - static constexpr auto name = "regionToName"; - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getEmbeddedDictionaries().getRegionsNames()); - } - -private: - const MultiVersion::Version owned_dict; - -public: - FunctionRegionToName(const MultiVersion::Version & owned_dict_) - : owned_dict(owned_dict_) - { - if (!owned_dict) - throw Exception("Dictionaries was not loaded. You need to check configuration file.", ErrorCodes::DICTIONARIES_WAS_NOT_LOADED); - } - - String getName() const override - { - return name; - } - - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - - /// For the purpose of query optimization, we assume this function to be injective - /// even in face of fact that there are many different cities named Moscow. - bool isInjective(const Block &) const override { return true; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.size() != 1 && arguments.size() != 2) - throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " - + toString(arguments.size()) + ", should be 1 or 2.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - if (arguments[0]->getName() != TypeName::get()) - throw Exception("Illegal type " + arguments[0]->getName() + " of the first argument of function " + getName() - + " (must be " + String(TypeName::get()) + ")", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (arguments.size() == 2 && arguments[1]->getName() != TypeName::get()) - throw Exception("Illegal type " + arguments[0]->getName() + " of the second argument of function " + getName() - + " (must be " + String(TypeName::get()) + ")", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - return std::make_shared(); - } - - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override - { - RegionsNames::Language language = RegionsNames::Language::RU; - - /// If the result language is specified - if (arguments.size() == 2) - { - if (const ColumnConst * col_language = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get())) - language = RegionsNames::getLanguageEnum(col_language->getValue()); - else - throw Exception("Illegal column " + block.getByPosition(arguments[1]).column->getName() - + " of the second argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - } - - const RegionsNames & dict = *owned_dict; - - if (const ColumnUInt32 * col_from = typeid_cast(block.getByPosition(arguments[0]).column.get())) - { - auto col_to = ColumnString::create(); - - const ColumnUInt32::Container & region_ids = col_from->getData(); - - for (size_t i = 0; i < region_ids.size(); ++i) - { - const StringRef & name_ref = dict.getRegionName(region_ids[i], language); - col_to->insertDataWithTerminatingZero(name_ref.data, name_ref.size + 1); - } - - block.getByPosition(result).column = std::move(col_to); - } - else - throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName() - + " of the first argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - } -}; - -}; // namespace DB diff --git a/dbms/src/Functions/FunctionsExternalDictionaries.cpp b/dbms/src/Functions/FunctionsExternalDictionaries.cpp deleted file mode 100644 index 64bbb6b0c1e..00000000000 --- a/dbms/src/Functions/FunctionsExternalDictionaries.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -namespace DB -{ -void registerFunctionsExternalDictionaries(FunctionFactory & factory) -{ - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); -} - -} // namespace DB diff --git a/dbms/src/Functions/FunctionsExternalDictionaries.h b/dbms/src/Functions/FunctionsExternalDictionaries.h deleted file mode 100644 index f91d0b9d8b7..00000000000 --- a/dbms/src/Functions/FunctionsExternalDictionaries.h +++ /dev/null @@ -1,1703 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - - -namespace DB -{ -namespace ErrorCodes -{ -extern const int DICTIONARIES_WAS_NOT_LOADED; -extern const int UNSUPPORTED_METHOD; -extern const int UNKNOWN_TYPE; -extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} // namespace ErrorCodes - -/** Functions that use plug-ins (external) dictionaries. - * - * Get the value of the attribute of the specified type. - * dictGetType(dictionary, attribute, id), - * Type - placeholder for the type name, any numeric and string types are currently supported. - * The type must match the actual attribute type with which it was declared in the dictionary structure. - * - * Get an array of identifiers, consisting of the source and parents chain. - * dictGetHierarchy(dictionary, id). - * - * Is the first identifier the child of the second. - * dictIsIn(dictionary, child_id, parent_id). - */ - - -class FunctionDictHas final : public IFunction -{ -public: - static constexpr auto name = "dictHas"; - - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getExternalDictionaries()); - } - - FunctionDictHas(const ExternalDictionaries & dictionaries) - : dictionaries(dictionaries) - {} - - String getName() const override { return name; } - -private: - size_t getNumberOfArguments() const override { return 2; } - - bool useDefaultImplementationForConstants() const final { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (!arguments[0]->isString()) - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + ", expected a string.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!checkDataType(arguments[1].get()) && !checkDataType(arguments[1].get())) - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + ", must be UInt64 or tuple(...).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, const size_t result) const override - { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - /** Do not require existence of the dictionary if the function is called for empty block. - * This is needed to allow successful query analysis on a server, - * that is the initiator of a distributed query, - * in the case when the function will be invoked for real data only at the remote servers. - * This feature is controversial and implemented specially - * for backward compatibility with the case in Yandex Banner System. - */ - if (block.rows() == 0) - { - auto & elem = block.getByPosition(result); - elem.column = elem.type->createColumn(); - return; - } - - auto dict = dictionaries.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - - if (!executeDispatchSimple(block, arguments, result, dict_ptr) && !executeDispatchSimple(block, arguments, result, dict_ptr) && !executeDispatchSimple(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; - } - - template - bool executeDispatchSimple( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - const auto id_col_untyped = block.getByPosition(arguments[1]).column.get(); - if (const auto id_col = checkAndGetColumn(id_col_untyped)) - { - const auto & ids = id_col->getData(); - - auto out = ColumnUInt8::create(ext::size(ids)); - dict->has(ids, out->getData()); - block.getByPosition(result).column = std::move(out); - } - else - throw Exception{"Second argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; - - return true; - } - - template - bool executeDispatchComplex( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - const ColumnWithTypeAndName & key_col_with_type = block.getByPosition(arguments[1]); - const ColumnPtr & key_col = key_col_with_type.column; - - if (checkColumn(key_col.get())) - { - const auto & key_columns = static_cast(*key_col).getColumns(); - const auto & key_types = static_cast(*key_col_with_type.type).getElements(); - - auto out = ColumnUInt8::create(key_col_with_type.column->size()); - dict->has(key_columns, key_types, out->getData()); - block.getByPosition(result).column = std::move(out); - } - else - throw Exception{"Second argument of function " + getName() + " must be " + dict->getKeyDescription(), ErrorCodes::TYPE_MISMATCH}; - - return true; - } - - const ExternalDictionaries & dictionaries; -}; - - -static bool isDictGetFunctionInjective(const ExternalDictionaries & dictionaries, const Block & sample_block) -{ - if (sample_block.columns() != 3 && sample_block.columns() != 4) - throw Exception{"Function dictGet... takes 3 or 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto dict_name_col = checkAndGetColumnConst(sample_block.getByPosition(0).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - const auto attr_name_col = checkAndGetColumnConst(sample_block.getByPosition(1).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - return dictionaries.getDictionary(dict_name_col->getValue())->isInjective(attr_name_col->getValue()); -} - - -class FunctionDictGetString final : public IFunction -{ -public: - static constexpr auto name = "dictGetString"; - - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getExternalDictionaries()); - } - - FunctionDictGetString(const ExternalDictionaries & dictionaries) - : dictionaries(dictionaries) - {} - - String getName() const override { return name; } - -private: - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - - bool useDefaultImplementationForConstants() const final { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - - bool isInjective(const Block & sample_block) const override - { - return isDictGetFunctionInjective(dictionaries, sample_block); - } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.size() != 3 && arguments.size() != 4) - throw Exception{"Number of arguments for function " + getName() + " doesn't match: passed " - + toString(arguments.size()) + ", should be 3 or 4.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - if (!arguments[0]->isString()) - { - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + ", expected a string.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - } - - if (!arguments[1]->isString()) - { - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + ", expected a string.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - } - - if (!checkDataType(arguments[2].get()) && !checkDataType(arguments[2].get())) - { - throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName() - + ", must be UInt64 or tuple(...).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - } - - /// This is for the case of range dictionaries. - if (arguments.size() == 4 && !checkDataType(arguments[3].get())) - { - throw Exception{"Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName() - + ", must be Date.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - } - - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, const size_t result) const override - { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - if (block.rows() == 0) - { - auto & elem = block.getByPosition(result); - elem.column = elem.type->createColumn(); - return; - } - - auto dict = dictionaries.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - - if (!executeDispatch(block, arguments, result, dict_ptr) && !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchRange(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; - } - - template - bool executeDispatch( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - if (arguments.size() != 3) - throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() + " requires exactly 3 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto attr_name_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - String attr_name = attr_name_col->getValue(); - - const auto id_col_untyped = block.getByPosition(arguments[2]).column.get(); - if (const auto id_col = checkAndGetColumn(id_col_untyped)) - { - auto out = ColumnString::create(); - dict->getString(attr_name, id_col->getData(), out.get()); - block.getByPosition(result).column = std::move(out); - } - else - throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; - - return true; - } - - template - bool executeDispatchComplex( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - if (arguments.size() != 3) - throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() + " requires exactly 3 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto attr_name_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - String attr_name = attr_name_col->getValue(); - - const ColumnWithTypeAndName & key_col_with_type = block.getByPosition(arguments[2]); - ColumnPtr key_col = key_col_with_type.column; - - /// Functions in external dictionaries only support full-value (not constant) columns with keys. - if (ColumnPtr key_col_materialized = key_col_with_type.column->convertToFullColumnIfConst()) - key_col = key_col_materialized; - - if (checkColumn(key_col.get())) - { - const auto & key_columns = static_cast(*key_col).getColumns(); - const auto & key_types = static_cast(*key_col_with_type.type).getElements(); - - auto out = ColumnString::create(); - dict->getString(attr_name, key_columns, key_types, out.get()); - block.getByPosition(result).column = std::move(out); - } - else - throw Exception{"Third argument of function " + getName() + " must be " + dict->getKeyDescription(), ErrorCodes::TYPE_MISMATCH}; - - return true; - } - - template - bool executeDispatchRange( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - if (arguments.size() != 4) - throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() + " requires exactly 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto attr_name_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - String attr_name = attr_name_col->getValue(); - - const auto id_col_untyped = block.getByPosition(arguments[2]).column.get(); - const auto date_col_untyped = block.getByPosition(arguments[3]).column.get(); - if (const auto id_col = checkAndGetColumn(id_col_untyped)) - executeRange(block, result, dict, attr_name, id_col, date_col_untyped); - else if (const auto id_col = checkAndGetColumnConst>(id_col_untyped)) - executeRange(block, result, dict, attr_name, id_col, date_col_untyped); - else - { - throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; - } - - return true; - } - - template - void executeRange( - Block & block, - const size_t result, - const DictionaryType * dictionary, - const std::string & attr_name, - const ColumnUInt64 * id_col, - const IColumn * date_col_untyped) const - { - if (const auto date_col = checkAndGetColumn(date_col_untyped)) - { - auto out = ColumnString::create(); - dictionary->getString(attr_name, id_col->getData(), date_col->getData(), out.get()); - block.getByPosition(result).column = std::move(out); - } - else if (const auto date_col = checkAndGetColumnConst>(date_col_untyped)) - { - auto out = ColumnString::create(); - const PaddedPODArray dates(id_col->size(), date_col->getValue()); - dictionary->getString(attr_name, id_col->getData(), dates, out.get()); - block.getByPosition(result).column = std::move(out); - } - else - { - throw Exception{"Fourth argument of function " + getName() + " must be Date", ErrorCodes::ILLEGAL_COLUMN}; - } - } - - template - void executeRange( - Block & block, - const size_t result, - const DictionaryType * dictionary, - const std::string & attr_name, - const ColumnConst * id_col, - const IColumn * date_col_untyped) const - { - if (const auto date_col = checkAndGetColumn(date_col_untyped)) - { - auto out = ColumnString::create(); - const PaddedPODArray ids(date_col->size(), id_col->getValue()); - dictionary->getString(attr_name, ids, date_col->getData(), out.get()); - block.getByPosition(result).column = std::move(out); - } - else if (const auto date_col = checkAndGetColumnConst>(date_col_untyped)) - { - const PaddedPODArray ids(1, id_col->getValue()); - const PaddedPODArray dates(1, date_col->getValue()); - - auto out = ColumnString::create(); - dictionary->getString(attr_name, ids, dates, out.get()); - block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString()); - } - else - { - throw Exception{"Fourth argument of function " + getName() + " must be Date", ErrorCodes::ILLEGAL_COLUMN}; - } - } - - const ExternalDictionaries & dictionaries; -}; - - -class FunctionDictGetStringOrDefault final : public IFunction -{ -public: - static constexpr auto name = "dictGetStringOrDefault"; - - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getExternalDictionaries()); - } - - FunctionDictGetStringOrDefault(const ExternalDictionaries & dictionaries) - : dictionaries(dictionaries) - {} - - String getName() const override { return name; } - -private: - size_t getNumberOfArguments() const override { return 4; } - - bool useDefaultImplementationForConstants() const final { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (!arguments[0]->isString()) - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!arguments[1]->isString()) - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!checkDataType(arguments[2].get()) && !checkDataType(arguments[2].get())) - { - throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName() - + ", must be UInt64 or tuple(...).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - } - - if (!arguments[3]->isString()) - throw Exception{"Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName() + ", must be String.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, const size_t result) const override - { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - if (block.rows() == 0) - { - auto & elem = block.getByPosition(result); - elem.column = elem.type->createColumn(); - return; - } - - auto dict = dictionaries.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - - if (!executeDispatch(block, arguments, result, dict_ptr) && !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; - } - - template - bool executeDispatch( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - const auto attr_name_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - String attr_name = attr_name_col->getValue(); - - const auto id_col_untyped = block.getByPosition(arguments[2]).column.get(); - if (const auto id_col = checkAndGetColumn(id_col_untyped)) - executeDispatch(block, arguments, result, dict, attr_name, id_col); - else if (const auto id_col = checkAndGetColumnConst>(id_col_untyped)) - executeDispatch(block, arguments, result, dict, attr_name, id_col); - else - throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; - - return true; - } - - template - void executeDispatch( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const DictionaryType * dictionary, - const std::string & attr_name, - const ColumnUInt64 * id_col) const - { - const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); - - if (const auto default_col = checkAndGetColumn(default_col_untyped)) - { - /// vector ids, vector defaults - auto out = ColumnString::create(); - const auto & ids = id_col->getData(); - dictionary->getString(attr_name, ids, default_col, out.get()); - block.getByPosition(result).column = std::move(out); - } - else if (const auto default_col = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) - { - /// vector ids, const defaults - auto out = ColumnString::create(); - const auto & ids = id_col->getData(); - String def = default_col->getValue(); - dictionary->getString(attr_name, ids, def, out.get()); - block.getByPosition(result).column = std::move(out); - } - else - throw Exception{"Fourth argument of function " + getName() + " must be String", ErrorCodes::ILLEGAL_COLUMN}; - } - - template - void executeDispatch( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const DictionaryType * dictionary, - const std::string & attr_name, - const ColumnConst * id_col) const - { - const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); - - if (const auto default_col = checkAndGetColumn(default_col_untyped)) - { - /// const ids, vector defaults - /// @todo avoid materialization - const PaddedPODArray ids(id_col->size(), id_col->getValue()); - auto out = ColumnString::create(); - dictionary->getString(attr_name, ids, default_col, out.get()); - block.getByPosition(result).column = std::move(out); - } - else if (const auto default_col = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) - { - /// const ids, const defaults - const PaddedPODArray ids(1, id_col->getValue()); - auto out = ColumnString::create(); - String def = default_col->getValue(); - dictionary->getString(attr_name, ids, def, out.get()); - block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString()); - } - else - throw Exception{"Fourth argument of function " + getName() + " must be String", ErrorCodes::ILLEGAL_COLUMN}; - } - - template - bool executeDispatchComplex( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - const auto attr_name_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - String attr_name = attr_name_col->getValue(); - - const ColumnWithTypeAndName & key_col_with_type = block.getByPosition(arguments[2]); - ColumnPtr key_col = key_col_with_type.column; - - /// Functions in external dictionaries only support full-value (not constant) columns with keys. - if (ColumnPtr key_col_materialized = key_col_with_type.column->convertToFullColumnIfConst()) - key_col = key_col_materialized; - - const auto & key_columns = typeid_cast(*key_col).getColumns(); - const auto & key_types = static_cast(*key_col_with_type.type).getElements(); - - auto out = ColumnString::create(); - - const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); - if (const auto default_col = checkAndGetColumn(default_col_untyped)) - { - dict->getString(attr_name, key_columns, key_types, default_col, out.get()); - } - else if (const auto default_col = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) - { - String def = default_col->getValue(); - dict->getString(attr_name, key_columns, key_types, def, out.get()); - } - else - throw Exception{"Fourth argument of function " + getName() + " must be String", ErrorCodes::ILLEGAL_COLUMN}; - - block.getByPosition(result).column = std::move(out); - return true; - } - - const ExternalDictionaries & dictionaries; -}; - - -template -struct DictGetTraits; -#define DECLARE_DICT_GET_TRAITS(TYPE, DATA_TYPE) \ - template <> \ - struct DictGetTraits \ - { \ - template \ - static void get( \ - const DictionaryType * dict, \ - const std::string & name, \ - const PaddedPODArray & ids, \ - PaddedPODArray & out) \ - { \ - dict->get##TYPE(name, ids, out); \ - } \ - template \ - static void get( \ - const DictionaryType * dict, \ - const std::string & name, \ - const Columns & key_columns, \ - const DataTypes & key_types, \ - PaddedPODArray & out) \ - { \ - dict->get##TYPE(name, key_columns, key_types, out); \ - } \ - template \ - static void get( \ - const DictionaryType * dict, \ - const std::string & name, \ - const PaddedPODArray & ids, \ - const PaddedPODArray & dates, \ - PaddedPODArray & out) \ - { \ - dict->get##TYPE(name, ids, dates, out); \ - } \ - template \ - static void getOrDefault( \ - const DictionaryType * dict, \ - const std::string & name, \ - const PaddedPODArray & ids, \ - const DefaultsType & def, \ - PaddedPODArray & out) \ - { \ - dict->get##TYPE(name, ids, def, out); \ - } \ - template \ - static void getOrDefault( \ - const DictionaryType * dict, \ - const std::string & name, \ - const Columns & key_columns, \ - const DataTypes & key_types, \ - const DefaultsType & def, \ - PaddedPODArray & out) \ - { \ - dict->get##TYPE(name, key_columns, key_types, def, out); \ - } \ - }; -DECLARE_DICT_GET_TRAITS(UInt8, DataTypeUInt8) -DECLARE_DICT_GET_TRAITS(UInt16, DataTypeUInt16) -DECLARE_DICT_GET_TRAITS(UInt32, DataTypeUInt32) -DECLARE_DICT_GET_TRAITS(UInt64, DataTypeUInt64) -DECLARE_DICT_GET_TRAITS(Int8, DataTypeInt8) -DECLARE_DICT_GET_TRAITS(Int16, DataTypeInt16) -DECLARE_DICT_GET_TRAITS(Int32, DataTypeInt32) -DECLARE_DICT_GET_TRAITS(Int64, DataTypeInt64) -DECLARE_DICT_GET_TRAITS(Float32, DataTypeFloat32) -DECLARE_DICT_GET_TRAITS(Float64, DataTypeFloat64) -DECLARE_DICT_GET_TRAITS(UInt16, DataTypeDate) -DECLARE_DICT_GET_TRAITS(UInt32, DataTypeDateTime) -DECLARE_DICT_GET_TRAITS(UInt128, DataTypeUUID) -#undef DECLARE_DICT_GET_TRAITS - - -template -class FunctionDictGet final : public IFunction -{ - using Type = typename DataType::FieldType; - -public: - static constexpr auto name = Name::name; - - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getExternalDictionaries()); - } - - FunctionDictGet(const ExternalDictionaries & dictionaries) - : dictionaries(dictionaries) - {} - - String getName() const override { return name; } - -private: - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - - bool useDefaultImplementationForConstants() const final { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - - bool isInjective(const Block & sample_block) const override - { - return isDictGetFunctionInjective(dictionaries, sample_block); - } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.size() != 3 && arguments.size() != 4) - throw Exception{"Function " + getName() + " takes 3 or 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - if (!arguments[0]->isString()) - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + ", expected a string.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!arguments[1]->isString()) - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + ", expected a string.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!checkDataType(arguments[2].get()) && !checkDataType(arguments[2].get())) - throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName() - + ", must be UInt64 or tuple(...).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (arguments.size() == 4 && !checkDataType(arguments[3].get())) - throw Exception{"Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName() - + ", must be Date.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, const size_t result) const override - { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - if (block.rows() == 0) - { - auto & elem = block.getByPosition(result); - elem.column = elem.type->createColumn(); - return; - } - - auto dict = dictionaries.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - - if (!executeDispatch(block, arguments, result, dict_ptr) && !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchRange(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; - } - - template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - if (arguments.size() != 3) - throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() + " requires exactly 3 arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto attr_name_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - String attr_name = attr_name_col->getValue(); - - const auto id_col_untyped = block.getByPosition(arguments[2]).column.get(); - if (const auto id_col = checkAndGetColumn(id_col_untyped)) - { - auto out = ColumnVector::create(id_col->size()); - const auto & ids = id_col->getData(); - auto & data = out->getData(); - DictGetTraits::get(dict, attr_name, ids, data); - block.getByPosition(result).column = std::move(out); - } - else if (const auto id_col = checkAndGetColumnConst>(id_col_untyped)) - { - const PaddedPODArray ids(1, id_col->getValue()); - PaddedPODArray data(1); - DictGetTraits::get(dict, attr_name, ids, data); - block.getByPosition(result).column = DataTypeNumber().createColumnConst(id_col->size(), toField(data.front())); - } - else - throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; - - return true; - } - - template - bool executeDispatchComplex( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - if (arguments.size() != 3) - throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() + " requires exactly 3 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto attr_name_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - String attr_name = attr_name_col->getValue(); - - const ColumnWithTypeAndName & key_col_with_type = block.getByPosition(arguments[2]); - ColumnPtr key_col = key_col_with_type.column; - - /// Functions in external dictionaries only support full-value (not constant) columns with keys. - if (ColumnPtr key_col_materialized = key_col_with_type.column->convertToFullColumnIfConst()) - key_col = key_col_materialized; - - if (checkColumn(key_col.get())) - { - const auto & key_columns = static_cast(*key_col).getColumns(); - const auto & key_types = static_cast(*key_col_with_type.type).getElements(); - - auto out = ColumnVector::create(key_columns.front()->size()); - auto & data = out->getData(); - DictGetTraits::get(dict, attr_name, key_columns, key_types, data); - block.getByPosition(result).column = std::move(out); - } - else - throw Exception{"Third argument of function " + getName() + " must be " + dict->getKeyDescription(), ErrorCodes::TYPE_MISMATCH}; - - return true; - } - - template - bool executeDispatchRange( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - if (arguments.size() != 4) - throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() + " requires exactly 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto attr_name_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - String attr_name = attr_name_col->getValue(); - - const auto id_col_untyped = block.getByPosition(arguments[2]).column.get(); - const auto date_col_untyped = block.getByPosition(arguments[3]).column.get(); - if (const auto id_col = checkAndGetColumn(id_col_untyped)) - executeRange(block, result, dict, attr_name, id_col, date_col_untyped); - else if (const auto id_col = checkAndGetColumnConst>(id_col_untyped)) - executeRange(block, result, dict, attr_name, id_col, date_col_untyped); - else - throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; - - return true; - } - - template - void executeRange( - Block & block, - const size_t result, - const DictionaryType * dictionary, - const std::string & attr_name, - const ColumnUInt64 * id_col, - const IColumn * date_col_untyped) const - { - if (const auto date_col = checkAndGetColumn(date_col_untyped)) - { - const auto size = id_col->size(); - const auto & ids = id_col->getData(); - const auto & dates = date_col->getData(); - - auto out = ColumnVector::create(size); - auto & data = out->getData(); - DictGetTraits::get(dictionary, attr_name, ids, dates, data); - block.getByPosition(result).column = std::move(out); - } - else if (const auto date_col = checkAndGetColumnConst>(date_col_untyped)) - { - const auto size = id_col->size(); - const auto & ids = id_col->getData(); - const PaddedPODArray dates(size, date_col->getValue()); - - auto out = ColumnVector::create(size); - auto & data = out->getData(); - DictGetTraits::get(dictionary, attr_name, ids, dates, data); - block.getByPosition(result).column = std::move(out); - } - else - throw Exception{"Fourth argument of function " + getName() + " must be Date", ErrorCodes::ILLEGAL_COLUMN}; - } - - template - void executeRange( - Block & block, - const size_t result, - const DictionaryType * dictionary, - const std::string & attr_name, - const ColumnConst * id_col, - const IColumn * date_col_untyped) const - { - if (const auto date_col = checkAndGetColumn(date_col_untyped)) - { - const auto size = date_col->size(); - const PaddedPODArray ids(size, id_col->getValue()); - const auto & dates = date_col->getData(); - - auto out = ColumnVector::create(size); - auto & data = out->getData(); - DictGetTraits::get(dictionary, attr_name, ids, dates, data); - block.getByPosition(result).column = std::move(out); - } - else if (const auto date_col = checkAndGetColumnConst>(date_col_untyped)) - { - const PaddedPODArray ids(1, id_col->getValue()); - const PaddedPODArray dates(1, date_col->getValue()); - PaddedPODArray data(1); - DictGetTraits::get(dictionary, attr_name, ids, dates, data); - block.getByPosition(result).column = DataTypeNumber().createColumnConst(id_col->size(), toField(data.front())); - } - else - throw Exception{"Fourth argument of function " + getName() + " must be Date", ErrorCodes::ILLEGAL_COLUMN}; - } - - const ExternalDictionaries & dictionaries; -}; - -struct NameDictGetUInt8 -{ - static constexpr auto name = "dictGetUInt8"; -}; -struct NameDictGetUInt16 -{ - static constexpr auto name = "dictGetUInt16"; -}; -struct NameDictGetUInt32 -{ - static constexpr auto name = "dictGetUInt32"; -}; -struct NameDictGetUInt64 -{ - static constexpr auto name = "dictGetUInt64"; -}; -struct NameDictGetInt8 -{ - static constexpr auto name = "dictGetInt8"; -}; -struct NameDictGetInt16 -{ - static constexpr auto name = "dictGetInt16"; -}; -struct NameDictGetInt32 -{ - static constexpr auto name = "dictGetInt32"; -}; -struct NameDictGetInt64 -{ - static constexpr auto name = "dictGetInt64"; -}; -struct NameDictGetFloat32 -{ - static constexpr auto name = "dictGetFloat32"; -}; -struct NameDictGetFloat64 -{ - static constexpr auto name = "dictGetFloat64"; -}; -struct NameDictGetDate -{ - static constexpr auto name = "dictGetDate"; -}; -struct NameDictGetDateTime -{ - static constexpr auto name = "dictGetDateTime"; -}; -struct NameDictGetUUID -{ - static constexpr auto name = "dictGetUUID"; -}; - -using FunctionDictGetUInt8 = FunctionDictGet; -using FunctionDictGetUInt16 = FunctionDictGet; -using FunctionDictGetUInt32 = FunctionDictGet; -using FunctionDictGetUInt64 = FunctionDictGet; -using FunctionDictGetInt8 = FunctionDictGet; -using FunctionDictGetInt16 = FunctionDictGet; -using FunctionDictGetInt32 = FunctionDictGet; -using FunctionDictGetInt64 = FunctionDictGet; -using FunctionDictGetFloat32 = FunctionDictGet; -using FunctionDictGetFloat64 = FunctionDictGet; -using FunctionDictGetDate = FunctionDictGet; -using FunctionDictGetDateTime = FunctionDictGet; -using FunctionDictGetUUID = FunctionDictGet; - - -template -class FunctionDictGetOrDefault final : public IFunction -{ - using Type = typename DataType::FieldType; - -public: - static constexpr auto name = Name::name; - - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getExternalDictionaries()); - } - - FunctionDictGetOrDefault(const ExternalDictionaries & dictionaries) - : dictionaries(dictionaries) - {} - - String getName() const override { return name; } - -private: - size_t getNumberOfArguments() const override { return 4; } - - bool useDefaultImplementationForConstants() const final { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (!arguments[0]->isString()) - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + ", expected a string.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!arguments[1]->isString()) - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + ", expected a string.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!checkDataType(arguments[2].get()) && !checkDataType(arguments[2].get())) - throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName() - + ", must be UInt64 or tuple(...).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!checkDataType(arguments[3].get())) - throw Exception{"Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName() - + ", must be " + String(DataType{}.getFamilyName()) + ".", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, const size_t result) const override - { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - if (block.rows() == 0) - { - auto & elem = block.getByPosition(result); - elem.column = elem.type->createColumn(); - return; - } - - auto dict = dictionaries.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - - if (!executeDispatch(block, arguments, result, dict_ptr) && !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr) && !executeDispatchComplex(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; - } - - template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - const auto attr_name_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - String attr_name = attr_name_col->getValue(); - - const auto id_col_untyped = block.getByPosition(arguments[2]).column.get(); - if (const auto id_col = checkAndGetColumn(id_col_untyped)) - executeDispatch(block, arguments, result, dict, attr_name, id_col); - else if (const auto id_col = checkAndGetColumnConst>(id_col_untyped)) - executeDispatch(block, arguments, result, dict, attr_name, id_col); - else - throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; - - return true; - } - - template - void executeDispatch( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const DictionaryType * dictionary, - const std::string & attr_name, - const ColumnUInt64 * id_col) const - { - const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); - - if (const auto default_col = checkAndGetColumn>(default_col_untyped)) - { - /// vector ids, vector defaults - auto out = ColumnVector::create(id_col->size()); - const auto & ids = id_col->getData(); - auto & data = out->getData(); - const auto & defs = default_col->getData(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, defs, data); - block.getByPosition(result).column = std::move(out); - } - else if (const auto default_col = checkAndGetColumnConst>(default_col_untyped)) - { - /// vector ids, const defaults - auto out = ColumnVector::create(id_col->size()); - const auto & ids = id_col->getData(); - auto & data = out->getData(); - const auto def = default_col->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); - block.getByPosition(result).column = std::move(out); - } - else - throw Exception{"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()), ErrorCodes::ILLEGAL_COLUMN}; - } - - template - void executeDispatch( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const DictionaryType * dictionary, - const std::string & attr_name, - const ColumnConst * id_col) const - { - const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); - - if (const auto default_col = checkAndGetColumn>(default_col_untyped)) - { - /// const ids, vector defaults - /// @todo avoid materialization - const PaddedPODArray ids(id_col->size(), id_col->getValue()); - - auto out = ColumnVector::create(id_col->size()); - auto & data = out->getData(); - const auto & defs = default_col->getData(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, defs, data); - block.getByPosition(result).column = std::move(out); - } - else if (const auto default_col = checkAndGetColumnConst>(default_col_untyped)) - { - /// const ids, const defaults - const PaddedPODArray ids(1, id_col->getValue()); - PaddedPODArray data(1); - const auto & def = default_col->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); - block.getByPosition(result).column = DataTypeNumber().createColumnConst(id_col->size(), toField(data.front())); - } - else - throw Exception{"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()), ErrorCodes::ILLEGAL_COLUMN}; - } - - template - bool executeDispatchComplex( - Block & block, - const ColumnNumbers & arguments, - const size_t result, - const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - const auto attr_name_col = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - String attr_name = attr_name_col->getValue(); - - const ColumnWithTypeAndName & key_col_with_type = block.getByPosition(arguments[2]); - ColumnPtr key_col = key_col_with_type.column; - - /// Functions in external dictionaries only support full-value (not constant) columns with keys. - if (ColumnPtr key_col_materialized = key_col_with_type.column->convertToFullColumnIfConst()) - key_col = key_col_materialized; - - const auto & key_columns = typeid_cast(*key_col).getColumns(); - const auto & key_types = static_cast(*key_col_with_type.type).getElements(); - - /// @todo detect when all key columns are constant - const auto rows = key_col->size(); - auto out = ColumnVector::create(rows); - auto & data = out->getData(); - - const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); - if (const auto default_col = checkAndGetColumn>(default_col_untyped)) - { - /// const defaults - const auto & defs = default_col->getData(); - - DictGetTraits::getOrDefault(dict, attr_name, key_columns, key_types, defs, data); - } - else if (const auto default_col = checkAndGetColumnConst>(default_col_untyped)) - { - const auto def = default_col->template getValue(); - - DictGetTraits::getOrDefault(dict, attr_name, key_columns, key_types, def, data); - } - else - throw Exception{"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()), ErrorCodes::ILLEGAL_COLUMN}; - - block.getByPosition(result).column = std::move(out); - return true; - } - - const ExternalDictionaries & dictionaries; -}; - -struct NameDictGetUInt8OrDefault -{ - static constexpr auto name = "dictGetUInt8OrDefault"; -}; -struct NameDictGetUInt16OrDefault -{ - static constexpr auto name = "dictGetUInt16OrDefault"; -}; -struct NameDictGetUInt32OrDefault -{ - static constexpr auto name = "dictGetUInt32OrDefault"; -}; -struct NameDictGetUInt64OrDefault -{ - static constexpr auto name = "dictGetUInt64OrDefault"; -}; -struct NameDictGetInt8OrDefault -{ - static constexpr auto name = "dictGetInt8OrDefault"; -}; -struct NameDictGetInt16OrDefault -{ - static constexpr auto name = "dictGetInt16OrDefault"; -}; -struct NameDictGetInt32OrDefault -{ - static constexpr auto name = "dictGetInt32OrDefault"; -}; -struct NameDictGetInt64OrDefault -{ - static constexpr auto name = "dictGetInt64OrDefault"; -}; -struct NameDictGetFloat32OrDefault -{ - static constexpr auto name = "dictGetFloat32OrDefault"; -}; -struct NameDictGetFloat64OrDefault -{ - static constexpr auto name = "dictGetFloat64OrDefault"; -}; -struct NameDictGetDateOrDefault -{ - static constexpr auto name = "dictGetDateOrDefault"; -}; -struct NameDictGetDateTimeOrDefault -{ - static constexpr auto name = "dictGetDateTimeOrDefault"; -}; -struct NameDictGetUUIDOrDefault -{ - static constexpr auto name = "dictGetUUIDOrDefault"; -}; - -using FunctionDictGetUInt8OrDefault = FunctionDictGetOrDefault; -using FunctionDictGetUInt16OrDefault = FunctionDictGetOrDefault; -using FunctionDictGetUInt32OrDefault = FunctionDictGetOrDefault; -using FunctionDictGetUInt64OrDefault = FunctionDictGetOrDefault; -using FunctionDictGetInt8OrDefault = FunctionDictGetOrDefault; -using FunctionDictGetInt16OrDefault = FunctionDictGetOrDefault; -using FunctionDictGetInt32OrDefault = FunctionDictGetOrDefault; -using FunctionDictGetInt64OrDefault = FunctionDictGetOrDefault; -using FunctionDictGetFloat32OrDefault = FunctionDictGetOrDefault; -using FunctionDictGetFloat64OrDefault = FunctionDictGetOrDefault; -using FunctionDictGetDateOrDefault = FunctionDictGetOrDefault; -using FunctionDictGetDateTimeOrDefault = FunctionDictGetOrDefault; -using FunctionDictGetUUIDOrDefault = FunctionDictGetOrDefault; - - -/// Functions to work with hierarchies. - -class FunctionDictGetHierarchy final : public IFunction -{ -public: - static constexpr auto name = "dictGetHierarchy"; - - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getExternalDictionaries()); - } - - FunctionDictGetHierarchy(const ExternalDictionaries & dictionaries) - : dictionaries(dictionaries) - {} - - String getName() const override { return name; } - -private: - size_t getNumberOfArguments() const override { return 2; } - bool isInjective(const Block & /*sample_block*/) const override { return true; } - - bool useDefaultImplementationForConstants() const final { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (!arguments[0]->isString()) - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + ", expected a string.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!checkDataType(arguments[1].get())) - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + ", must be UInt64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - return std::make_shared(std::make_shared()); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, const size_t result) const override - { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - if (block.rows() == 0) - { - auto & elem = block.getByPosition(result); - elem.column = elem.type->createColumn(); - return; - } - - auto dict = dictionaries.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - - if (!executeDispatch(block, arguments, result, dict_ptr) && !executeDispatch(block, arguments, result, dict_ptr) && !executeDispatch(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; - } - - template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - if (!dict->hasHierarchy()) - throw Exception{"Dictionary does not have a hierarchy", ErrorCodes::UNSUPPORTED_METHOD}; - - const auto get_hierarchies = [&](const PaddedPODArray & in, PaddedPODArray & out, PaddedPODArray & offsets) { - const auto size = in.size(); - - /// copy of `in` array - auto in_array = std::make_unique>(std::begin(in), std::end(in)); - /// used for storing and handling result of ::toParent call - auto out_array = std::make_unique>(size); - /// resulting hierarchies - std::vector> hierarchies(size); /// TODO Bad code, poor performance. - - /// total number of non-zero elements, used for allocating all the required memory upfront - size_t total_count = 0; - - while (true) - { - auto all_zeroes = true; - - /// erase zeroed identifiers, store non-zeroed ones - for (const auto i : ext::range(0, size)) - { - const auto id = (*in_array)[i]; - if (0 == id) - continue; - - - auto & hierarchy = hierarchies[i]; - - /// Checking for loop - if (std::find(std::begin(hierarchy), std::end(hierarchy), id) != std::end(hierarchy)) - continue; - - all_zeroes = false; - /// place id at it's corresponding place - hierarchy.push_back(id); - - ++total_count; - } - - if (all_zeroes) - break; - - /// translate all non-zero identifiers at once - dict->toParent(*in_array, *out_array); - - /// we're going to use the `in_array` from this iteration as `out_array` on the next one - std::swap(in_array, out_array); - } - - out.reserve(total_count); - offsets.resize(size); - - for (const auto i : ext::range(0, size)) - { - const auto & ids = hierarchies[i]; - out.insert_assume_reserved(std::begin(ids), std::end(ids)); - offsets[i] = out.size(); - } - }; - - const auto id_col_untyped = block.getByPosition(arguments[1]).column.get(); - if (const auto id_col = checkAndGetColumn(id_col_untyped)) - { - const auto & in = id_col->getData(); - auto backend = ColumnUInt64::create(); - auto offsets = ColumnArray::ColumnOffsets::create(); - get_hierarchies(in, backend->getData(), offsets->getData()); - block.getByPosition(result).column = ColumnArray::create(std::move(backend), std::move(offsets)); - } - else if (const auto id_col = checkAndGetColumnConst>(id_col_untyped)) - { - const PaddedPODArray in(1, id_col->getValue()); - auto backend = ColumnUInt64::create(); - auto offsets = ColumnArray::ColumnOffsets::create(); - get_hierarchies(in, backend->getData(), offsets->getData()); - auto array = ColumnArray::create(std::move(backend), std::move(offsets)); - block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(id_col->size(), (*array)[0].get()); - } - else - throw Exception{"Second argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN}; - - return true; - } - - const ExternalDictionaries & dictionaries; -}; - - -class FunctionDictIsIn final : public IFunction -{ -public: - static constexpr auto name = "dictIsIn"; - - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getExternalDictionaries()); - } - - FunctionDictIsIn(const ExternalDictionaries & dictionaries) - : dictionaries(dictionaries) - {} - - String getName() const override { return name; } - -private: - size_t getNumberOfArguments() const override { return 3; } - - bool useDefaultImplementationForConstants() const final { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (!arguments[0]->isString()) - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + ", expected a string.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!checkDataType(arguments[1].get())) - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + ", must be UInt64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - if (!checkDataType(arguments[2].get())) - throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName() - + ", must be UInt64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, const size_t result) const override - { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - if (block.rows() == 0) - { - auto & elem = block.getByPosition(result); - elem.column = elem.type->createColumn(); - return; - } - - auto dict = dictionaries.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - - if (!executeDispatch(block, arguments, result, dict_ptr) - && !executeDispatch(block, arguments, result, dict_ptr) - && !executeDispatch(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; - } - - template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) const - { - const auto dict = typeid_cast(dictionary); - if (!dict) - return false; - - if (!dict->hasHierarchy()) - throw Exception{"Dictionary does not have a hierarchy", ErrorCodes::UNSUPPORTED_METHOD}; - - const auto child_id_col_untyped = block.getByPosition(arguments[1]).column.get(); - const auto ancestor_id_col_untyped = block.getByPosition(arguments[2]).column.get(); - - if (const auto child_id_col = checkAndGetColumn(child_id_col_untyped)) - execute(block, result, dict, child_id_col, ancestor_id_col_untyped); - else if (const auto child_id_col = checkAndGetColumnConst>(child_id_col_untyped)) - execute(block, result, dict, child_id_col, ancestor_id_col_untyped); - else - throw Exception{"Illegal column " + child_id_col_untyped->getName() - + " of second argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN}; - - return true; - } - - template - bool execute(Block & block, const size_t result, const DictionaryType * dictionary, const ColumnUInt64 * child_id_col, const IColumn * ancestor_id_col_untyped) const - { - if (const auto ancestor_id_col = checkAndGetColumn(ancestor_id_col_untyped)) - { - auto out = ColumnUInt8::create(); - - const auto & child_ids = child_id_col->getData(); - const auto & ancestor_ids = ancestor_id_col->getData(); - auto & data = out->getData(); - const auto size = child_id_col->size(); - data.resize(size); - - dictionary->isInVectorVector(child_ids, ancestor_ids, data); - block.getByPosition(result).column = std::move(out); - } - else if (const auto ancestor_id_col = checkAndGetColumnConst>(ancestor_id_col_untyped)) - { - auto out = ColumnUInt8::create(); - - const auto & child_ids = child_id_col->getData(); - const auto ancestor_id = ancestor_id_col->getValue(); - auto & data = out->getData(); - const auto size = child_id_col->size(); - data.resize(size); - - dictionary->isInVectorConstant(child_ids, ancestor_id, data); - block.getByPosition(result).column = std::move(out); - } - else - { - throw Exception{"Illegal column " + ancestor_id_col_untyped->getName() - + " of third argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN}; - } - - return true; - } - - template - bool execute(Block & block, const size_t result, const DictionaryType * dictionary, const ColumnConst * child_id_col, const IColumn * ancestor_id_col_untyped) const - { - if (const auto ancestor_id_col = checkAndGetColumn(ancestor_id_col_untyped)) - { - auto out = ColumnUInt8::create(); - - const auto child_id = child_id_col->getValue(); - const auto & ancestor_ids = ancestor_id_col->getData(); - auto & data = out->getData(); - const auto size = child_id_col->size(); - data.resize(size); - - dictionary->isInConstantVector(child_id, ancestor_ids, data); - block.getByPosition(result).column = std::move(out); - } - else if (const auto ancestor_id_col = checkAndGetColumnConst>(ancestor_id_col_untyped)) - { - const auto child_id = child_id_col->getValue(); - const auto ancestor_id = ancestor_id_col->getValue(); - UInt8 res = 0; - - dictionary->isInConstantConstant(child_id, ancestor_id, res); - block.getByPosition(result).column = DataTypeUInt8().createColumnConst(child_id_col->size(), UInt64(res)); - } - else - throw Exception{"Illegal column " + ancestor_id_col_untyped->getName() - + " of third argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN}; - - return true; - } - - const ExternalDictionaries & dictionaries; -}; - - -}; // namespace DB diff --git a/dbms/src/Functions/registerFunctions.cpp b/dbms/src/Functions/registerFunctions.cpp index 743b2bad4c3..aef1b5c71e4 100644 --- a/dbms/src/Functions/registerFunctions.cpp +++ b/dbms/src/Functions/registerFunctions.cpp @@ -30,8 +30,6 @@ void registerFunctionsConditional(FunctionFactory &); void registerFunctionsConversion(FunctionFactory &); void registerFunctionsTiDBConversion(FunctionFactory &); void registerFunctionsDateTime(FunctionFactory &); -void registerFunctionsEmbeddedDictionaries(FunctionFactory &); -void registerFunctionsExternalDictionaries(FunctionFactory &); void registerFunctionsExternalModels(FunctionFactory &); void registerFunctionsFormatting(FunctionFactory &); void registerFunctionsHashing(FunctionFactory &); @@ -70,8 +68,6 @@ void registerFunctions() registerFunctionsConversion(factory); registerFunctionsTiDBConversion(factory); registerFunctionsDateTime(factory); - registerFunctionsEmbeddedDictionaries(factory); - registerFunctionsExternalDictionaries(factory); registerFunctionsExternalModels(factory); registerFunctionsFormatting(factory); registerFunctionsHashing(factory); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 83e678101db..beffc7d0928 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -32,8 +32,6 @@ #include #include #include -#include -#include #include #include #include @@ -134,8 +132,6 @@ struct ContextShared Databases databases; /// List of databases and tables in them. FormatFactory format_factory; /// Formats. - mutable std::shared_ptr embedded_dictionaries; /// Metrica's dictionaeis. Have lazy initialization. - mutable std::shared_ptr external_dictionaries; mutable std::shared_ptr external_models; String default_profile_name; /// Default profile name used for default values. String system_profile_name; /// Profile used by system processes @@ -1173,29 +1169,6 @@ Context & Context::getGlobalContext() return *global_context; } - -const EmbeddedDictionaries & Context::getEmbeddedDictionaries() const -{ - return getEmbeddedDictionariesImpl(false); -} - -EmbeddedDictionaries & Context::getEmbeddedDictionaries() -{ - return getEmbeddedDictionariesImpl(false); -} - - -const ExternalDictionaries & Context::getExternalDictionaries() const -{ - return getExternalDictionariesImpl(false); -} - -ExternalDictionaries & Context::getExternalDictionaries() -{ - return getExternalDictionariesImpl(false); -} - - const ExternalModels & Context::getExternalModels() const { return getExternalModelsImpl(false); @@ -1206,45 +1179,6 @@ ExternalModels & Context::getExternalModels() return getExternalModelsImpl(false); } - -EmbeddedDictionaries & Context::getEmbeddedDictionariesImpl(const bool throw_on_error) const -{ - std::lock_guard lock(shared->embedded_dictionaries_mutex); - - if (!shared->embedded_dictionaries) - { - auto geo_dictionaries_loader = runtime_components_factory->createGeoDictionariesLoader(); - - shared->embedded_dictionaries = std::make_shared( - std::move(geo_dictionaries_loader), - *this->global_context, - throw_on_error); - } - - return *shared->embedded_dictionaries; -} - - -ExternalDictionaries & Context::getExternalDictionariesImpl(const bool throw_on_error) const -{ - std::lock_guard lock(shared->external_dictionaries_mutex); - - if (!shared->external_dictionaries) - { - if (!this->global_context) - throw Exception("Logical error: there is no global context", ErrorCodes::LOGICAL_ERROR); - - auto config_repository = runtime_components_factory->createExternalDictionariesConfigRepository(); - - shared->external_dictionaries = std::make_shared( - std::move(config_repository), - *this->global_context, - throw_on_error); - } - - return *shared->external_dictionaries; -} - ExternalModels & Context::getExternalModelsImpl(bool throw_on_error) const { std::lock_guard lock(shared->external_models_mutex); @@ -1265,18 +1199,6 @@ ExternalModels & Context::getExternalModelsImpl(bool throw_on_error) const return *shared->external_models; } -void Context::tryCreateEmbeddedDictionaries() const -{ - static_cast(getEmbeddedDictionariesImpl(true)); -} - - -void Context::tryCreateExternalDictionaries() const -{ - static_cast(getExternalDictionariesImpl(true)); -} - - void Context::tryCreateExternalModels() const { static_cast(getExternalModelsImpl(true)); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index af49c37c041..66942ea709b 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -48,8 +48,6 @@ namespace DB struct ContextShared; class IRuntimeComponentsFactory; class QuotaForIntervals; -class EmbeddedDictionaries; -class ExternalDictionaries; class ExternalModels; class BackgroundProcessingPool; class MergeList; @@ -281,14 +279,8 @@ class Context /// Set a setting by name. Read the value in text form from a string (for example, from a config, or from a URL parameter). void setSetting(const String & name, const std::string & value); - const EmbeddedDictionaries & getEmbeddedDictionaries() const; - const ExternalDictionaries & getExternalDictionaries() const; const ExternalModels & getExternalModels() const; - EmbeddedDictionaries & getEmbeddedDictionaries(); - ExternalDictionaries & getExternalDictionaries(); ExternalModels & getExternalModels(); - void tryCreateEmbeddedDictionaries() const; - void tryCreateExternalDictionaries() const; void tryCreateExternalModels() const; /// I/O formats. @@ -491,8 +483,6 @@ class Context */ void checkDatabaseAccessRightsImpl(const std::string & database_name) const; - EmbeddedDictionaries & getEmbeddedDictionariesImpl(bool throw_on_error) const; - ExternalDictionaries & getExternalDictionariesImpl(bool throw_on_error) const; ExternalModels & getExternalModelsImpl(bool throw_on_error) const; StoragePtr getTableImpl(const String & database_name, const String & table_name, Exception * exception) const; diff --git a/dbms/src/Interpreters/EmbeddedDictionaries.cpp b/dbms/src/Interpreters/EmbeddedDictionaries.cpp deleted file mode 100644 index 3a6bbd161bb..00000000000 --- a/dbms/src/Interpreters/EmbeddedDictionaries.cpp +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ -extern const int UNFINISHED; -} - -void EmbeddedDictionaries::handleException(const bool throw_on_error) const -{ - const auto exception_ptr = std::current_exception(); - - tryLogCurrentException(log, "Cannot load dictionary! You must resolve this manually."); - - if (throw_on_error) - std::rethrow_exception(exception_ptr); -} - - -template -bool EmbeddedDictionaries::reloadDictionary( - MultiVersion & dictionary, - DictionaryReloader reload_dictionary, - const bool throw_on_error, - const bool force_reload) -{ - const auto & config = context.getConfigRef(); - - bool not_initialized = dictionary.get() == nullptr; - - if (force_reload || !is_fast_start_stage || not_initialized) - { - try - { - auto new_dictionary = reload_dictionary(config); - if (new_dictionary) - dictionary.set(std::move(new_dictionary)); - } - catch (...) - { - handleException(throw_on_error); - return false; - } - } - - return true; -} - - -bool EmbeddedDictionaries::reloadImpl(const bool throw_on_error, const bool force_reload) -{ - std::unique_lock lock(mutex); - - /** If you can not update the directories, then despite this, do not throw an exception (use the old directories). - * If there are no old correct directories, then when using functions that depend on them, - * will throw an exception. - * An attempt is made to load each directory separately. - */ - - LOG_INFO(log, "Loading dictionaries."); - - bool was_exception = false; - - DictionaryReloader reload_regions_hierarchies = [=](const Poco::Util::AbstractConfiguration & config) { - return geo_dictionaries_loader->reloadRegionsHierarchies(config); - }; - - if (!reloadDictionary(regions_hierarchies, std::move(reload_regions_hierarchies), throw_on_error, force_reload)) - was_exception = true; - - DictionaryReloader reload_regions_names = [=](const Poco::Util::AbstractConfiguration & config) { - return geo_dictionaries_loader->reloadRegionsNames(config); - }; - - if (!reloadDictionary(regions_names, std::move(reload_regions_names), throw_on_error, force_reload)) - was_exception = true; - - if (!was_exception) - LOG_INFO(log, "Loaded dictionaries."); - - return !was_exception; -} - - -void EmbeddedDictionaries::reloadPeriodically() -{ - setThreadName("DictReload"); - - while (true) - { - if (destroy.tryWait(cur_reload_period * 1000)) - return; - - if (reloadImpl(false)) - { - /// Success - cur_reload_period = reload_period; - is_fast_start_stage = false; - } - - if (is_fast_start_stage) - { - cur_reload_period = std::min(reload_period, 2 * cur_reload_period); /// exponentially increase delay - is_fast_start_stage = cur_reload_period < reload_period; /// leave fast start state - } - } -} - - -EmbeddedDictionaries::EmbeddedDictionaries( - std::unique_ptr geo_dictionaries_loader_, - Context & context_, - const bool throw_on_error) - : log(&Poco::Logger::get("EmbeddedDictionaries")) - , context(context_) - , geo_dictionaries_loader(std::move(geo_dictionaries_loader_)) - , reload_period(context_.getConfigRef().getInt("builtin_dictionaries_reload_interval", 3600)) -{ - reloadImpl(throw_on_error); - reloading_thread = std::thread([this] { reloadPeriodically(); }); -} - - -EmbeddedDictionaries::~EmbeddedDictionaries() -{ - destroy.set(); - reloading_thread.join(); -} - -void EmbeddedDictionaries::reload() -{ - if (!reloadImpl(true, true)) - throw Exception("Some embedded dictionaries were not successfully reloaded", ErrorCodes::UNFINISHED); -} - - -} // namespace DB diff --git a/dbms/src/Interpreters/EmbeddedDictionaries.h b/dbms/src/Interpreters/EmbeddedDictionaries.h deleted file mode 100644 index d7a5975036d..00000000000 --- a/dbms/src/Interpreters/EmbeddedDictionaries.h +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include -#include - - -namespace Poco -{ -class Logger; -namespace Util -{ -class AbstractConfiguration; -} -} // namespace Poco - -class RegionsHierarchies; -class TechDataHierarchy; -class RegionsNames; -class IGeoDictionariesLoader; - - -namespace DB -{ -class Context; - - -/// Metrica's Dictionaries which can be used in functions. - -class EmbeddedDictionaries -{ -private: - Poco::Logger * log; - Context & context; - - MultiVersion regions_hierarchies; - MultiVersion tech_data_hierarchy; - MultiVersion regions_names; - - std::unique_ptr geo_dictionaries_loader; - - /// Directories' updating periodicity (in seconds). - int reload_period; - int cur_reload_period = 1; - bool is_fast_start_stage = true; - - mutable std::mutex mutex; - - std::thread reloading_thread; - Poco::Event destroy; - - - void handleException(const bool throw_on_error) const; - - /** Updates directories (dictionaries) every reload_period seconds. - * If all dictionaries are not loaded at least once, try reload them with exponential delay (1, 2, ... reload_period). - * If all dictionaries are loaded, update them using constant reload_period delay. - */ - void reloadPeriodically(); - - /// Updates dictionaries. - bool reloadImpl(const bool throw_on_error, const bool force_reload = false); - - template - using DictionaryReloader = std::function(const Poco::Util::AbstractConfiguration & config)>; - - template - bool reloadDictionary( - MultiVersion & dictionary, - DictionaryReloader reload_dictionary, - const bool throw_on_error, - const bool force_reload); - -public: - /// Every reload_period seconds directories are updated inside a separate thread. - EmbeddedDictionaries( - std::unique_ptr geo_dictionaries_loader, - Context & context, - const bool throw_on_error); - - /// Forcibly reloads all dictionaries. - void reload(); - - ~EmbeddedDictionaries(); - - - MultiVersion::Version getRegionsHierarchies() const - { - return regions_hierarchies.get(); - } - - MultiVersion::Version getTechDataHierarchy() const - { - return tech_data_hierarchy.get(); - } - - MultiVersion::Version getRegionsNames() const - { - return regions_names.get(); - } -}; - -} // namespace DB diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index a532ed8a8e0..0cd00350943 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -1329,31 +1328,7 @@ void ExpressionAnalyzer::optimizeGroupBy() { if (const auto * function = typeid_cast(group_exprs[i].get())) { - /// assert function is injective - if (possibly_injective_function_names.count(function->name)) - { - /// do not handle semantic errors here - if (function->arguments->children.size() < 2) - { - ++i; - continue; - } - - const auto & dict_name = typeid_cast(*function->arguments->children[0]) - .value.safeGet(); - - const auto & dict_ptr = context.getExternalDictionaries().getDictionary(dict_name); - - const auto & attr_name = typeid_cast(*function->arguments->children[1]) - .value.safeGet(); - - if (!dict_ptr->isInjective(attr_name)) - { - ++i; - continue; - } - } - else if (!injective_function_names.count(function->name)) + if (!injective_function_names.count(function->name)) { ++i; continue; diff --git a/dbms/src/Interpreters/ExternalDictionaries.cpp b/dbms/src/Interpreters/ExternalDictionaries.cpp deleted file mode 100644 index 48bbd013008..00000000000 --- a/dbms/src/Interpreters/ExternalDictionaries.cpp +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -namespace DB -{ -namespace -{ -const ExternalLoaderUpdateSettings externalDictionariesUpdateSettings{}; - -const ExternalLoaderConfigSettings & getExternalDictionariesConfigSettings() -{ - static ExternalLoaderConfigSettings settings; - static std::once_flag flag; - - std::call_once(flag, [] { - settings.external_config = "dictionary"; - settings.external_name = "name"; - settings.path_setting_name = "dictionaries_config"; - }); - - return settings; -} -} // namespace - - -ExternalDictionaries::ExternalDictionaries( - std::unique_ptr config_repository, - Context & context, - bool throw_on_error) - : ExternalLoader(context.getConfigRef(), - externalDictionariesUpdateSettings, - getExternalDictionariesConfigSettings(), - std::move(config_repository), - &Poco::Logger::get("ExternalDictionaries"), - "external dictionary") - , context(context) -{ - init(throw_on_error); -} - -std::unique_ptr ExternalDictionaries::create( - const std::string & name, - const Configuration & config, - const std::string & config_prefix) -{ - return DictionaryFactory::instance().create(name, config, config_prefix, context); -} - -} // namespace DB diff --git a/dbms/src/Interpreters/ExternalDictionaries.h b/dbms/src/Interpreters/ExternalDictionaries.h deleted file mode 100644 index 830033e7e88..00000000000 --- a/dbms/src/Interpreters/ExternalDictionaries.h +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include - -#include - - -namespace DB -{ -class Context; - -/// Manages user-defined dictionaries. -class ExternalDictionaries : public ExternalLoader -{ -public: - using DictPtr = std::shared_ptr; - - /// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. - ExternalDictionaries( - std::unique_ptr config_repository, - Context & context, - bool throw_on_error); - - /// Forcibly reloads specified dictionary. - void reloadDictionary(const std::string & name) { reload(name); } - - DictPtr getDictionary(const std::string & name) const - { - return std::static_pointer_cast(getLoadable(name)); - } - - DictPtr tryGetDictionary(const std::string & name) const - { - return std::static_pointer_cast(tryGetLoadable(name)); - } - -protected: - std::unique_ptr create(const std::string & name, const Configuration & config, const std::string & config_prefix) override; - - using ExternalLoader::getObjectsMap; - - friend class StorageSystemDictionaries; - friend class DatabaseDictionary; - -private: - Context & context; -}; - -} // namespace DB diff --git a/dbms/src/Interpreters/IRuntimeComponentsFactory.h b/dbms/src/Interpreters/IRuntimeComponentsFactory.h index 7a382c28a13..e51579ec1d7 100644 --- a/dbms/src/Interpreters/IRuntimeComponentsFactory.h +++ b/dbms/src/Interpreters/IRuntimeComponentsFactory.h @@ -14,7 +14,6 @@ #pragma once -#include #include #include @@ -31,8 +30,6 @@ class IRuntimeComponentsFactory public: virtual std::unique_ptr createSecurityManager() = 0; - virtual std::unique_ptr createGeoDictionariesLoader() = 0; - // Repositories with configurations of user-defined objects (dictionaries, models) virtual std::unique_ptr createExternalDictionariesConfigRepository() = 0; diff --git a/dbms/src/Interpreters/RuntimeComponentsFactory.h b/dbms/src/Interpreters/RuntimeComponentsFactory.h index ff600d24aaf..b8f0317ed5d 100644 --- a/dbms/src/Interpreters/RuntimeComponentsFactory.h +++ b/dbms/src/Interpreters/RuntimeComponentsFactory.h @@ -14,7 +14,6 @@ #pragma once -#include #include #include #include @@ -32,11 +31,6 @@ class RuntimeComponentsFactory : public IRuntimeComponentsFactory return std::make_unique(); } - std::unique_ptr createGeoDictionariesLoader() override - { - return std::make_unique(); - } - std::unique_ptr createExternalDictionariesConfigRepository() override { return std::make_unique(); diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 2b146fb44fb..70bc1d3f26d 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -1279,21 +1279,6 @@ int Server::main(const std::vector & /*args*/) users_config_reloader.reset(); }); - /// try to load dictionaries immediately, throw on error and die - try - { - if (!config().getBool("dictionaries_lazy_load", true)) - { - global_context->tryCreateEmbeddedDictionaries(); - global_context->tryCreateExternalDictionaries(); - } - } - catch (...) - { - LOG_FMT_ERROR(log, "Caught exception while loading dictionaries."); - throw; - } - /// This object will periodically calculate some metrics. /// should init after `createTMTContext` cause we collect some data from the TiFlash context object. AsynchronousMetrics async_metrics(*global_context); diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp deleted file mode 100644 index 9efff31e720..00000000000 --- a/dbms/src/Storages/StorageDictionary.cpp +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - - -StorageDictionary::StorageDictionary( - const String & table_name_, - const ColumnsDescription & columns_, - const DictionaryStructure & dictionary_structure_, - const String & dictionary_name_) - : IStorage{columns_}, table_name(table_name_), - dictionary_name(dictionary_name_), - logger(&Poco::Logger::get("StorageDictionary")) -{ - checkNamesAndTypesCompatibleWithDictionary(dictionary_structure_); -} - -BlockInputStreams StorageDictionary::read( - const Names & column_names, - const SelectQueryInfo & /*query_info*/, - const Context & context, - QueryProcessingStage::Enum & processed_stage, - const size_t max_block_size, - const unsigned /*threads*/) -{ - processed_stage = QueryProcessingStage::FetchColumns; - auto dictionary = context.getExternalDictionaries().getDictionary(dictionary_name); - return BlockInputStreams{dictionary->getBlockInputStream(column_names, max_block_size)}; -} - -NamesAndTypesList StorageDictionary::getNamesAndTypes(const DictionaryStructure & dictionary_structure) -{ - NamesAndTypesList dictionary_names_and_types; - - if (dictionary_structure.id) - dictionary_names_and_types.emplace_back(dictionary_structure.id->name, std::make_shared()); - if (dictionary_structure.range_min) - dictionary_names_and_types.emplace_back(dictionary_structure.range_min->name, std::make_shared()); - if (dictionary_structure.range_max) - dictionary_names_and_types.emplace_back(dictionary_structure.range_max->name, std::make_shared()); - if (dictionary_structure.key) - for (const auto & attribute : *dictionary_structure.key) - dictionary_names_and_types.emplace_back(attribute.name, attribute.type); - - for (const auto & attribute : dictionary_structure.attributes) - dictionary_names_and_types.emplace_back(attribute.name, attribute.type); - - return dictionary_names_and_types; -} - -void StorageDictionary::checkNamesAndTypesCompatibleWithDictionary(const DictionaryStructure & dictionary_structure) const -{ - auto dictionary_names_and_types = getNamesAndTypes(dictionary_structure); - std::set namesAndTypesSet(dictionary_names_and_types.begin(), dictionary_names_and_types.end()); - - for (auto & column : getColumns().ordinary) - { - if (namesAndTypesSet.find(column) == namesAndTypesSet.end()) - { - std::string message = "Not found column "; - message += column.name + " " + column.type->getName(); - message += " in dictionary " + dictionary_name + ". "; - message += "There are only columns "; - message += generateNamesAndTypesDescription(dictionary_names_and_types.begin(), dictionary_names_and_types.end()); - throw Exception(message); - } - } -} - -void registerStorageDictionary(StorageFactory & factory) -{ - factory.registerStorage("Dictionary", [](const StorageFactory::Arguments & args) - { - if (args.engine_args.size() != 1) - throw Exception("Storage Dictionary requires single parameter: name of dictionary", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - args.engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args.engine_args[0], args.local_context); - String dictionary_name = typeid_cast(*args.engine_args[0]).value.safeGet(); - - const auto & dictionary = args.context.getExternalDictionaries().getDictionary(dictionary_name); - const DictionaryStructure & dictionary_structure = dictionary->getStructure(); - - return StorageDictionary::create( - args.table_name, args.columns, dictionary_structure, dictionary_name); - }); -} - -} diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h deleted file mode 100644 index 8d8d4dec6ca..00000000000 --- a/dbms/src/Storages/StorageDictionary.h +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include - - -namespace Poco -{ -class Logger; -} - -namespace DB -{ -struct DictionaryStructure; -struct IDictionaryBase; -class ExternalDictionaries; - -class StorageDictionary : public ext::SharedPtrHelper - , public IStorage -{ -public: - std::string getName() const override { return "Dictionary"; } - std::string getTableName() const override { return table_name; } - BlockInputStreams read(const Names & column_names, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum & processed_stage, - size_t max_block_size = DEFAULT_BLOCK_SIZE, - unsigned threads = 1) override; - - void drop() override {} - static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure); - - template - static std::string generateNamesAndTypesDescription(ForwardIterator begin, ForwardIterator end) - { - std::string description; - { - WriteBufferFromString buffer(description); - bool first = true; - for (; begin != end; ++begin) - { - if (!first) - buffer << ", "; - first = false; - - buffer << begin->name << ' ' << begin->type->getName(); - } - } - - return description; - } - -private: - using Ptr = MultiVersion::Version; - - String table_name; - String dictionary_name; - Poco::Logger * logger; - - void checkNamesAndTypesCompatibleWithDictionary(const DictionaryStructure & dictionary_structure) const; - -protected: - StorageDictionary(const String & table_name_, - const ColumnsDescription & columns_, - const DictionaryStructure & dictionary_structure_, - const String & dictionary_name_); -}; - -} // namespace DB diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.cpp b/dbms/src/Storages/System/StorageSystemDictionaries.cpp deleted file mode 100644 index e22106918f7..00000000000 --- a/dbms/src/Storages/System/StorageSystemDictionaries.cpp +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name) - : name{name} -{ - setColumns(ColumnsDescription({ - { "name", std::make_shared() }, - { "origin", std::make_shared() }, - { "type", std::make_shared() }, - { "key", std::make_shared() }, - { "attribute.names", std::make_shared(std::make_shared()) }, - { "attribute.types", std::make_shared(std::make_shared()) }, - { "bytes_allocated", std::make_shared() }, - { "query_count", std::make_shared() }, - { "hit_rate", std::make_shared() }, - { "element_count", std::make_shared() }, - { "load_factor", std::make_shared() }, - { "creation_time", std::make_shared() }, - { "source", std::make_shared() }, - { "last_exception", std::make_shared() }, - })); -} - - -BlockInputStreams StorageSystemDictionaries::read( - const Names & column_names, - const SelectQueryInfo &, - const Context & context, - QueryProcessingStage::Enum & processed_stage, - const size_t, - const unsigned) -{ - check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; - - const auto & external_dictionaries = context.getExternalDictionaries(); - auto objects_map = external_dictionaries.getObjectsMap(); - const auto & dictionaries = objects_map.get(); - - MutableColumns res_columns = getSampleBlock().cloneEmptyColumns(); - - for (const auto & dict_info : dictionaries) - { - size_t i = 0; - - res_columns[i++]->insert(dict_info.first); - res_columns[i++]->insert(dict_info.second.origin); - - if (dict_info.second.loadable) - { - const auto dict_ptr = std::static_pointer_cast(dict_info.second.loadable); - - res_columns[i++]->insert(dict_ptr->getTypeName()); - - const auto & dict_struct = dict_ptr->getStructure(); - res_columns[i++]->insert(dict_struct.getKeyDescription()); - res_columns[i++]->insert(ext::map(dict_struct.attributes, [] (auto & attr) { return attr.name; })); - res_columns[i++]->insert(ext::map(dict_struct.attributes, [] (auto & attr) { return attr.type->getName(); })); - res_columns[i++]->insert(static_cast(dict_ptr->getBytesAllocated())); - res_columns[i++]->insert(static_cast(dict_ptr->getQueryCount())); - res_columns[i++]->insert(dict_ptr->getHitRate()); - res_columns[i++]->insert(static_cast(dict_ptr->getElementCount())); - res_columns[i++]->insert(dict_ptr->getLoadFactor()); - res_columns[i++]->insert(static_cast(std::chrono::system_clock::to_time_t(dict_ptr->getCreationTime()))); - res_columns[i++]->insert(dict_ptr->getSource()->toString()); - } - else - { - while (i < 13) - res_columns[i++]->insertDefault(); - } - - if (dict_info.second.exception) - { - try - { - std::rethrow_exception(dict_info.second.exception); - } - catch (...) - { - res_columns[i++]->insert(getCurrentExceptionMessage(false)); - } - } - else - res_columns[i++]->insertDefault(); - } - - return BlockInputStreams(1, std::make_shared(getSampleBlock().cloneWithColumns(std::move(res_columns)))); -} - -} diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.h b/dbms/src/Storages/System/StorageSystemDictionaries.h deleted file mode 100644 index de0b7154266..00000000000 --- a/dbms/src/Storages/System/StorageSystemDictionaries.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include - - -namespace DB -{ -class Context; - - -class StorageSystemDictionaries : public ext::SharedPtrHelper - , public IStorage -{ -public: - std::string getName() const override { return "SystemDictionaries"; } - std::string getTableName() const override { return name; } - - BlockInputStreams read( - const Names & column_names, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum & processed_stage, - size_t max_block_size, - unsigned num_streams) override; - -private: - const std::string name; - -protected: - StorageSystemDictionaries(const std::string & name); -}; - -} // namespace DB diff --git a/dbms/src/Storages/System/attachSystemTables.cpp b/dbms/src/Storages/System/attachSystemTables.cpp index eb278e39bd1..4ddf1f1e244 100644 --- a/dbms/src/Storages/System/attachSystemTables.cpp +++ b/dbms/src/Storages/System/attachSystemTables.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -57,7 +56,6 @@ void attachSystemTablesServer(IDatabase & system_database) attachSystemTablesLocal(system_database); system_database.attachTable("processes", StorageSystemProcesses::create("processes")); system_database.attachTable("metrics", StorageSystemMetrics::create("metrics")); - system_database.attachTable("dictionaries", StorageSystemDictionaries::create("dictionaries")); system_database.attachTable("models", StorageSystemModels::create("models")); system_database.attachTable("graphite_retentions", StorageSystemGraphite::create("graphite_retentions")); system_database.attachTable("macros", StorageSystemMacros::create("macros")); diff --git a/dbms/src/Storages/registerStorages.cpp b/dbms/src/Storages/registerStorages.cpp index ddf815316ab..d17dfc911e0 100644 --- a/dbms/src/Storages/registerStorages.cpp +++ b/dbms/src/Storages/registerStorages.cpp @@ -27,7 +27,6 @@ void registerStorageNull(StorageFactory & factory); void registerStorageMerge(StorageFactory & factory); void registerStorageBuffer(StorageFactory & factory); void registerStorageMemory(StorageFactory & factory); -void registerStorageDictionary(StorageFactory & factory); void registerStorageSet(StorageFactory & factory); void registerStorageJoin(StorageFactory & factory); void registerStorageView(StorageFactory & factory); @@ -46,7 +45,6 @@ void registerStorages() registerStorageMerge(factory); registerStorageBuffer(factory); registerStorageMemory(factory); - registerStorageDictionary(factory); registerStorageSet(factory); registerStorageJoin(factory); registerStorageView(factory);