Merge branch 'master' into fix_ddl_crash

JaySon-Huang · Aug 5, 2021 · 78cd377 · 78cd377
2 parents 120a2e7 + 1593d34
commit 78cd377
Show file tree

Hide file tree

Showing 57 changed files with 581 additions and 264 deletions.
diff --git a/contrib/client-c b/contrib/client-c
diff --git a/contrib/tiflash-proxy b/contrib/tiflash-proxy
diff --git a/contrib/tipb b/contrib/tipb
diff --git a/dbms/src/Columns/ColumnAggregateFunction.cpp b/dbms/src/Columns/ColumnAggregateFunction.cpp
@@ -180,7 +180,7 @@ void ColumnAggregateFunction::updateHashWithValues(IColumn::HashValues & hash_va
     }
 }
 
-void ColumnAggregateFunction::updateWeakHash32(WeakHash32 & hash) const
+void ColumnAggregateFunction::updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const
 {
     auto s = data.size();
     if (hash.getData().size() != data.size())

diff --git a/dbms/src/Columns/ColumnAggregateFunction.h b/dbms/src/Columns/ColumnAggregateFunction.h
@@ -146,7 +146,7 @@ class ColumnAggregateFunction final : public COWPtrHelper<IColumn, ColumnAggrega
 
     void updateHashWithValues(IColumn::HashValues & hash_values, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
 
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    void updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
 
     size_t byteSize() const override;
 

diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp
@@ -222,15 +222,15 @@ void ColumnArray::updateHashWithValues(IColumn::HashValues & hash_values, const
     }
 }
 
-void ColumnArray::updateWeakHash32(WeakHash32 & hash) const
+void ColumnArray::updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> & collator, String & sort_key_container) const
 {
     auto s = offsets->size();
     if (hash.getData().size() != s)
         throw Exception("Size of WeakHash32 does not match size of column: column size is " + std::to_string(s) +
                         ", hash size is " + std::to_string(hash.getData().size()), ErrorCodes::LOGICAL_ERROR);
 
     WeakHash32 internal_hash(data->size());
-    data->updateWeakHash32(internal_hash);
+    data->updateWeakHash32(internal_hash, collator, sort_key_container);
 
     Offset prev_offset = 0;
     const auto & offsets_data = getOffsets();

diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h
@@ -65,7 +65,7 @@ class ColumnArray final : public COWPtrHelper<IColumn, ColumnArray>
     const char * deserializeAndInsertFromArena(const char * pos, std::shared_ptr<TiDB::ITiDBCollator>) override;
     void updateHashWithValue(size_t n, SipHash & hash, std::shared_ptr<TiDB::ITiDBCollator>, String &) const override;
     void updateHashWithValues(IColumn::HashValues & hash_values, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    void updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
     void insert(const Field & x) override;
     void insertFrom(const IColumn & src_, size_t n) override;

diff --git a/dbms/src/Columns/ColumnConst.cpp b/dbms/src/Columns/ColumnConst.cpp
@@ -85,14 +85,14 @@ void ColumnConst::getPermutation(bool /*reverse*/, size_t /*limit*/, int /*nan_d
         res[i] = i;
 }
 
-void ColumnConst::updateWeakHash32(WeakHash32 & hash) const
+void ColumnConst::updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> & collator, String & sort_key_container) const
 {
     if (hash.getData().size() != s)
         throw Exception("Size of WeakHash32 does not match size of column: column size is " + std::to_string(s) +
                         ", hash size is " + std::to_string(hash.getData().size()), ErrorCodes::LOGICAL_ERROR);
 
     WeakHash32 element_hash(1);
-    data->updateWeakHash32(element_hash);
+    data->updateWeakHash32(element_hash, collator, sort_key_container);
     size_t data_hash = element_hash.getData()[0];
 
     for (auto & value : hash.getData())

diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h
@@ -153,7 +153,7 @@ class ColumnConst final : public COWPtrHelper<IColumn, ColumnConst>
         }
     }
 
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    void updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
 
     ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
     ColumnPtr replicate(const Offsets & offsets) const override;

diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp
@@ -124,7 +124,7 @@ void ColumnDecimal<T>::updateHashWithValues(IColumn::HashValues & hash_values, c
 }
 
 template <typename T>
-void ColumnDecimal<T>::updateWeakHash32(WeakHash32 & hash) const
+void ColumnDecimal<T>::updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const
 {
     auto s = data.size();
 

diff --git a/dbms/src/Columns/ColumnDecimal.h b/dbms/src/Columns/ColumnDecimal.h
@@ -124,7 +124,7 @@ class ColumnDecimal final : public COWPtrHelper<ColumnVectorHelper, ColumnDecima
     const char * deserializeAndInsertFromArena(const char * pos, std::shared_ptr<TiDB::ITiDBCollator>) override;
     void updateHashWithValue(size_t n, SipHash & hash, std::shared_ptr<TiDB::ITiDBCollator>, String &) const override;
     void updateHashWithValues(IColumn::HashValues & hash_values, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    void updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
     void getPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res) const override;
 

diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp
@@ -107,7 +107,7 @@ void ColumnFixedString::updateHashWithValues(IColumn::HashValues & hash_values,
     }
 }
 
-void ColumnFixedString::updateWeakHash32(WeakHash32 & hash) const
+void ColumnFixedString::updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const
 {
     auto s = size();
 

diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h
@@ -101,7 +101,7 @@ class ColumnFixedString final : public COWPtrHelper<IColumn, ColumnFixedString>
 
     void updateHashWithValues(IColumn::HashValues & hash_values, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
 
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    void updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
 
     int compareAt(size_t p1, size_t p2, const IColumn & rhs_, int /*nan_direction_hint*/) const override
     {

diff --git a/dbms/src/Columns/ColumnFunction.h b/dbms/src/Columns/ColumnFunction.h
@@ -95,7 +95,7 @@ class ColumnFunction final : public COWPtrHelper<IColumn, ColumnFunction>
         throw Exception("updateHashWithValues is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
     }
 
-    void updateWeakHash32(WeakHash32 &) const override
+    void updateWeakHash32(WeakHash32 &, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override
     {
         throw Exception("updateWeakHash32 is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
     }

diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp
@@ -81,7 +81,7 @@ void ColumnNullable::updateHashWithValues(
     }
 }
 
-void ColumnNullable::updateWeakHash32(WeakHash32 & hash) const
+void ColumnNullable::updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> & collator, String & sort_key_container) const
 {
     auto s = size();
 
@@ -90,7 +90,7 @@ void ColumnNullable::updateWeakHash32(WeakHash32 & hash) const
                         ", hash size is " + std::to_string(hash.getData().size()), ErrorCodes::LOGICAL_ERROR);
 
     WeakHash32 old_hash = hash;
-    nested_column->updateWeakHash32(hash);
+    nested_column->updateWeakHash32(hash, collator, sort_key_container);
 
     const auto & null_map_data = getNullMapData();
     auto & hash_data = hash.getData();

diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h
@@ -77,7 +77,7 @@ class ColumnNullable final : public COWPtrHelper<IColumn, ColumnNullable>
     ColumnPtr replicate(const Offsets & replicate_offsets) const override;
     void updateHashWithValue(size_t n, SipHash & hash, std::shared_ptr<TiDB::ITiDBCollator>, String &) const override;
     void updateHashWithValues(IColumn::HashValues & hash_values, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    void updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
     void getExtremes(Field & min, Field & max) const override;
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override

diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp
@@ -347,7 +347,7 @@ void ColumnString::getPermutationWithCollationImpl(const ICollator & collator, b
     }
 }
 
-void ColumnString::updateWeakHash32(WeakHash32 & hash) const
+void ColumnString::updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> & collator, String & sort_key_container) const
 {
     auto s = offsets.size();
 
@@ -359,15 +359,32 @@ void ColumnString::updateWeakHash32(WeakHash32 & hash) const
     UInt32 * hash_data = hash.getData().data();
     Offset prev_offset = 0;
 
-    for (const auto & offset : offsets)
+    if (collator != nullptr)
     {
-        auto str_size = offset - prev_offset;
-        /// Skip last zero byte.
-        *hash_data = ::updateWeakHash32(pos, str_size - 1, *hash_data);
+        for (const auto & offset : offsets)
+        {
+            auto str_size = offset - prev_offset;
+            /// Skip last zero byte.
+            auto sort_key = collator->sortKey(reinterpret_cast<const char *>(pos), str_size - 1, sort_key_container);
+            *hash_data = ::updateWeakHash32(reinterpret_cast<const UInt8 *>(sort_key.data), sort_key.size, *hash_data);
+
+            pos += str_size;
+            prev_offset = offset;
+            ++hash_data;
+        }
+    }
+    else
+    {
+        for (const auto & offset : offsets)
+        {
+            auto str_size = offset - prev_offset;
+            /// Skip last zero byte.
+            *hash_data = ::updateWeakHash32(pos, str_size - 1, *hash_data);
 
-        pos += str_size;
-        prev_offset = offset;
-        ++hash_data;
+            pos += str_size;
+            prev_offset = offset;
+            ++hash_data;
+        }
     }
 }
 

diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h
@@ -184,18 +184,26 @@ class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
         offsets.resize_assume_reserved(offsets.size() - n);
     }
 
-    StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin, std::shared_ptr<TiDB::ITiDBCollator>, String &) const override
+    StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin, std::shared_ptr<TiDB::ITiDBCollator> collator, String & sort_key_container) const override
     {
         size_t string_size = sizeAt(n);
         size_t offset = offsetAt(n);
+        const void * src = &chars[offset];
 
         StringRef res;
+
+        if (collator != nullptr)
+        {
+            /// Skip last zero byte.
+            auto sort_key = collator->sortKey(reinterpret_cast<const char *>(src), string_size - 1, sort_key_container);
+            string_size = sort_key.size;
+            src = sort_key.data;
+        }
         res.size = sizeof(string_size) + string_size;
         char * pos = arena.allocContinue(res.size, begin);
         memcpy(pos, &string_size, sizeof(string_size));
-        memcpy(pos + sizeof(string_size), &chars[offset], string_size);
+        memcpy(pos + sizeof(string_size), src, string_size);
         res.data = pos;
-
         return res;
     }
 
@@ -259,7 +267,7 @@ class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
         }
     }
 
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    void updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
 
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 

diff --git a/dbms/src/Columns/ColumnTuple.cpp b/dbms/src/Columns/ColumnTuple.cpp
@@ -154,7 +154,7 @@ void ColumnTuple::updateHashWithValues(IColumn::HashValues & hash_values, const
         column->updateHashWithValues(hash_values, collator, sort_key_container);
 }
 
-void ColumnTuple::updateWeakHash32(WeakHash32 & hash) const
+void ColumnTuple::updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> & collator, String & sort_key_container) const
 {
     auto s = size();
 
@@ -163,7 +163,7 @@ void ColumnTuple::updateWeakHash32(WeakHash32 & hash) const
                         ", hash size is " + std::to_string(hash.getData().size()), ErrorCodes::LOGICAL_ERROR);
 
     for (const auto & column : columns)
-        column->updateWeakHash32(hash);
+        column->updateWeakHash32(hash, collator, sort_key_container);
 }
 
 void ColumnTuple::insertRangeFrom(const IColumn & src, size_t start, size_t length)

diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h
@@ -58,7 +58,7 @@ class ColumnTuple final : public COWPtrHelper<IColumn, ColumnTuple>
     const char * deserializeAndInsertFromArena(const char * pos, std::shared_ptr<TiDB::ITiDBCollator>) override;
     void updateHashWithValue(size_t n, SipHash & hash, std::shared_ptr<TiDB::ITiDBCollator>, String &) const override;
     void updateHashWithValues(IColumn::HashValues & hash_values, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    void updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
     ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
     ColumnPtr permute(const Permutation & perm, size_t limit) const override;

diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
@@ -62,7 +62,7 @@ void ColumnVector<T>::updateHashWithValues(IColumn::HashValues & hash_values, co
 }
 
 template <typename T>
-void ColumnVector<T>::updateWeakHash32(WeakHash32 & hash) const
+void ColumnVector<T>::updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const
 {
     auto s = data.size();
 

diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h
@@ -183,7 +183,7 @@ class ColumnVector final : public COWPtrHelper<ColumnVectorHelper, ColumnVector<
 
     void updateHashWithValue(size_t n, SipHash & hash, std::shared_ptr<TiDB::ITiDBCollator>, String &) const override;
     void updateHashWithValues(IColumn::HashValues & hash_values, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    void updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override;
 
     size_t byteSize() const override
     {

diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h
@@ -181,7 +181,7 @@ class IColumn : public COWPtr<IColumn>
     /// Update hash function value. Hash is calculated for each element.
     /// It's a fast weak hash function. Mainly need to scatter data between threads.
     /// WeakHash32 must have the same size as column.
-    virtual void updateWeakHash32(WeakHash32 & hash) const = 0;
+    virtual void updateWeakHash32(WeakHash32 & hash, const std::shared_ptr<TiDB::ITiDBCollator> & collator = nullptr, String & sort_key_container = TiDB::dummy_sort_key_contaner) const = 0;
 
     /** Removes elements that don't match the filter.
       * Is used in WHERE and HAVING operations.

diff --git a/dbms/src/Columns/IColumnDummy.h b/dbms/src/Columns/IColumnDummy.h
@@ -68,7 +68,7 @@ class IColumnDummy : public IColumn
     {
     }
 
-    void updateWeakHash32(WeakHash32 &) const override
+    void updateWeakHash32(WeakHash32 &, const std::shared_ptr<TiDB::ITiDBCollator> &, String &) const override
     {
     }
 

diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h
@@ -342,15 +342,15 @@ struct HashMethodSerialized
     HashMethodSerialized(const ColumnRawPtrs & key_columns_, const Sizes & /*key_sizes*/, const TiDB::TiDBCollators & collators_)
         : key_columns(key_columns_), keys_size(key_columns_.size()), collators(collators_) {}
 
-protected:
-    friend class columns_hashing_impl::HashMethodBase<Self, Value, Mapped, false>;
-
     ALWAYS_INLINE SerializedKeyHolder getKeyHolder(size_t row, Arena * pool, std::vector<String> & sort_key_containers) const
     {
         return SerializedKeyHolder{
             serializeKeysToPoolContiguous(row, keys_size, key_columns, collators, sort_key_containers, *pool),
             *pool};
     }
+
+protected:
+    friend class columns_hashing_impl::HashMethodBase<Self, Value, Mapped, false>;
 };
 
 /// For the case when there is one string key.

diff --git a/dbms/src/Core/SIMD.h b/dbms/src/Core/SIMD.h
@@ -1,6 +1,8 @@
 #pragma once
 #if defined(__aarch64__)
+#if __has_include(<asm/hwcap.h>)
 #include <asm/hwcap.h>
+#endif
 #include <sys/auxv.h>
 #endif
 namespace DB
@@ -75,6 +77,11 @@ enum class SIMDFeature
 
 static inline bool SIMDRuntimeSupport(SIMDFeature feature)
 {
+    /// Notice that we do not detect support for Darwin/arm64 since
+    /// it does not have HWCAP support. However, if such feature is
+    /// ever needed in the future, a good reference can be:
+    /// https://github.com/golang/sys/pull/114
+#if __has_include(<asm/hwcap.h>)
     unsigned long hwcap;
     switch (feature)
     {
@@ -94,8 +101,9 @@ static inline bool SIMDRuntimeSupport(SIMDFeature feature)
             return hwcap & HWCAP2_SVE2;
 #else
             return false;
-#endif
+#endif // HWCAP2_SVE2
     }
+#endif // __has_include(<asm/hwcap.h>)
     return false;
 }
 #endif