Merge branch 'master' into wal-replace-writablefile

pingcap · Mar 1, 2022 · 5379c6e · 5379c6e
2 parents 727705d + 4954a24
commit 5379c6e
Show file tree

Hide file tree

Showing 145 changed files with 8,328 additions and 2,233 deletions.
diff --git a/.ci/integration_test.groovy b/.ci/integration_test.groovy
@@ -35,8 +35,8 @@ catchError {
                     sh  """
                         rm -rf ${curws}/tics/contrib
                         rm -rf ${curws}/tics/.git
-                    
-                        COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} TAR_PATH=${curws}/tics/tests/.build bash -e ${curws}/tics/release-centos7-llvm/scripts/fetch-ci-build.sh
+
+                        COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} bash -e ${curws}/tics/release-centos7-llvm/scripts/fetch-ci-build.sh
                         """
                 }
             }

diff --git a/.ci/util.groovy b/.ci/util.groovy
@@ -142,7 +142,7 @@ def runTest(label, name, testPath, tidbBranch) {
                     container("docker") {
                         sh """
                         pwd
-                        DOWNLOAD_TAR=true COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} TAR_PATH=./tests/.build bash -e release-centos7-llvm/scripts/fetch-ci-build.sh
+                        DOWNLOAD_TAR=true COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} bash -e release-centos7-llvm/scripts/fetch-ci-build.sh
                         """
                     }
                 }

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,18 +1,16 @@
 project (TiFlash)
 cmake_minimum_required (VERSION 2.8)
 
+set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${TiFlash_SOURCE_DIR}/cmake/Modules/")
 set(CMAKE_MACOSX_RPATH 1)
 
-option(TIFLASH_ENABLE_RUNTIME_RPATH "Add $ORIGIN/ to tiflash installation rpath" OFF)
 option(TIFLASH_ENABLE_LLVM_DEVELOPMENT "enable facilities for development with LLVM" OFF)
 
-if(TIFLASH_ENABLE_LLVM_DEVELOPMENT)
-    # enable RPATHs to sysroot to make it easier for LLVM development
-    set(CMAKE_BUILD_RPATH_USE_ORIGIN TRUE)
-    set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-    set(ENV{LD_LIBRARY_PATH} "${CMAKE_PREFIX_PATH}/lib:${CMAKE_PREFIX_PATH}/lib/x86_64-unknown-linux-gnu/:${CMAKE_PREFIX_PATH}/lib/aarch64-unknown-linux-gnu/")
-    set(CMAKE_INSTALL_RPATH "$ORIGIN/;${CMAKE_PREFIX_PATH}/lib;${CMAKE_PREFIX_PATH}/lib/x86_64-unknown-linux-gnu/;${CMAKE_PREFIX_PATH}/lib/aarch64-unknown-linux-gnu/")
+if(CMAKE_PREFIX_PATH)
+    # append paths for cmake to check libs
+    set(ENV{LD_LIBRARY_PATH}
+            "${CMAKE_PREFIX_PATH}/lib:${CMAKE_PREFIX_PATH}/lib/x86_64-unknown-linux-gnu/:${CMAKE_PREFIX_PATH}/lib/aarch64-unknown-linux-gnu/")
 endif()
 
 message (STATUS "Using CXX=${CMAKE_CXX_COMPILER}, ver=${CMAKE_CXX_COMPILER_VERSION};CC=${CMAKE_C_COMPILER}, ver=${CMAKE_C_COMPILER_VERSION}")
@@ -142,6 +140,22 @@ set (COMPILER_FLAGS                      "${COMPILER_FLAGS} ${CXX11_ABI_FLAGS}")
 
 string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER})
 
+find_program (LLVM_OBJCOPY_PATH NAMES "llvm-objcopy${COMPILER_POSTFIX}" "llvm-objcopy")
+find_program (LLVM_AR_PATH NAMES "llvm-ar${COMPILER_POSTFIX}" "llvm-ar")
+find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib${COMPILER_POSTFIX}" "llvm-ranlib")
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND LLVM_OBJCOPY_PATH)
+    set(CMAKE_OBJCOPY ${LLVM_OBJCOPY_PATH})
+endif()
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND LLVM_AR_PATH)
+    set(CMAKE_AR ${LLVM_AR_PATH})
+endif()
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND LLVM_RANLIB_PATH)
+    set(CMAKE_RANLIB ${LLVM_RANLIB_PATH})
+endif()
+
 find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld")
 find_program (GOLD_PATH NAMES "gold")
 

diff --git a/README.md b/README.md
@@ -195,7 +195,7 @@ Normally a CMake-based IDE, e.g., Clion and VSCode, should be able to open TiFla
 If your toolchain is set up using [TiFlash Env](#tiflash-env), and you may not want to add those libs to your system loader config, you can pass the following CMake options to your IDE:
 
 ```
--DTIFLASH_ENABLE_LLVM_DEVELOPMENT=ON -DCMAKE_PREFIX_PATH=$TIFLASH_ENV
+-DCMAKE_PREFIX_PATH=$TIFLASH_ENV
 ```
 
 Remember that `$TIFLASH_ENV` is a placeholder mentioned in [TiFlash Env](#tiflash-env).

diff --git a/cluster_manage/flash_tools.py b/cluster_manage/flash_tools.py
@@ -14,7 +14,6 @@ def __init__(self, file_path):
             self.http_port = self.conf_toml['http_port']
         else:
             self.http_port = self.conf_toml['https_port']
-        tmp_path = self.conf_toml['tmp_path']
 
         p = self.conf_toml['flash']
         service_addr = p['service_addr']
@@ -32,7 +31,7 @@ def __init__(self, file_path):
         self.cluster_refresh_interval = min(
             int(flash_cluster.get('refresh_interval', 20)), self.cluster_master_ttl)
         self.update_rule_interval = int(flash_cluster.get('update_rule_interval', 10))
-        self.log_path = flash_cluster.get('log', '{}/flash_cluster_manager.log'.format(tmp_path))
+        self.log_path = flash_cluster.get('log', '{}/flash_cluster_manager.log'.format(self.conf_toml.get('tmp_path', '/tmp')))
         self.max_time_out = self.cluster_master_ttl
 
         self.enable_tls = False

diff --git a/cmake/tiflash_linux_post_install.cmake b/cmake/tiflash_linux_post_install.cmake
@@ -0,0 +1,11 @@
+find_program (OBJCOPY_PATH NAMES "objcopy")
+find_program (LLVM_OBJCOPY_PATH NAMES "llvm-objcopy${COMPILER_POSTFIX}" "llvm-objcopy")
+
+if (LLVM_OBJCOPY_PATH)
+    set(CMAKE_OBJCOPY ${LLVM_OBJCOPY_PATH})
+else ()
+    set(CMAKE_OBJCOPY ${OBJCOPY_PATH})
+endif ()
+
+message(STATUS "executing: ${CMAKE_OBJCOPY} --compress-debug-sections=zlib-gnu ${CMAKE_INSTALL_PREFIX}/tiflash")
+execute_process(COMMAND ${CMAKE_OBJCOPY} --compress-debug-sections=zlib-gnu ${CMAKE_INSTALL_PREFIX}/tiflash)
diff --git a/contrib/client-c b/contrib/client-c
diff --git a/contrib/tiflash-proxy b/contrib/tiflash-proxy
diff --git a/contrib/tiflash-proxy-cmake/CMakeLists.txt b/contrib/tiflash-proxy-cmake/CMakeLists.txt
@@ -3,8 +3,6 @@ set(_TIFLASH_PROXY_LIBRARY "${_TIFLASH_PROXY_SOURCE_DIR}/target/release/${CMAKE_
 file(GLOB_RECURSE _TIFLASH_PROXY_SRCS "${_TIFLASH_PROXY_SOURCE_DIR}/*.rs")
 list(FILTER _TIFLASH_PROXY_SRCS EXCLUDE REGEX ${_TIFLASH_PROXY_SOURCE_DIR}/target/.*)
 
-option(ENABLE_PROXY_RUNTIME_RELATIVE_PATH "Make proxy dynamic lib to be linked by relative path" OFF)
-
 if(TIFLASH_LLVM_TOOLCHAIN AND USE_LIBCXX)
     set(TIFLASH_RUST_LINKER ${CMAKE_CURRENT_BINARY_DIR}/tiflash-linker)
     set(TIFLASH_RUST_LINKER_TMP ${CMAKE_CURRENT_BINARY_DIR}/tmp/tiflash-linker)
@@ -13,16 +11,13 @@ if(TIFLASH_LLVM_TOOLCHAIN AND USE_LIBCXX)
     file(COPY ${TIFLASH_RUST_LINKER_TMP}
             DESTINATION ${CMAKE_CURRENT_BINARY_DIR}
             FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE)
-    set(TIFLASH_RUSTFLAGS "-C linker=${TIFLASH_RUST_LINKER}")
+    set(TIFLASH_RUSTFLAGS "-C rpath=yes -C linker=${TIFLASH_RUST_LINKER}")
     if(ARCH_AARCH64)
         set(TIFLASH_RUSTFLAGS "-C link-arg=-Wl,-Bsymbolic ${TIFLASH_RUSTFLAGS}")
     endif()
     if(LINKER_NAME)
         set(TIFLASH_RUSTFLAGS "-C link-arg=-fuse-ld=${LINKER_NAME} ${TIFLASH_RUSTFLAGS}")
     endif()
-    if(TIFLASH_ENABLE_RUNTIME_RPATH)
-        set(TIFLASH_RUSTFLAGS "-C rpath=yes ${TIFLASH_RUSTFLAGS}")
-    endif()
     set(TIFLASH_RUST_ENV CXXSTDLIB=c++ CMAKE=${CMAKE_COMMAND} CFLAGS=-w CXXFLAGS=-w RUSTFLAGS=${TIFLASH_RUSTFLAGS})
     message(STATUS "enforce LLVM toolchain for rust: ${TIFLASH_RUST_ENV}")
 endif()
@@ -35,14 +30,6 @@ add_custom_command(OUTPUT ${_TIFLASH_PROXY_LIBRARY}
     WORKING_DIRECTORY ${_TIFLASH_PROXY_SOURCE_DIR}
     DEPENDS "${_TIFLASH_PROXY_SRCS}" "${_TIFLASH_PROXY_SOURCE_DIR}/Cargo.lock" "${_TIFLASH_PROXY_SOURCE_DIR}/rust-toolchain")
 
-if(APPLE AND ENABLE_PROXY_RUNTIME_RELATIVE_PATH)
-    # For MacOS, we need to explicitly adjust install name for tiflash_proxy library, as it is copied from raftstore_proxy library and the install name is kept as "raftstore_proxy".
-    add_custom_command(OUTPUT ${_TIFLASH_PROXY_LIBRARY}
-        APPEND
-        COMMENT "Changing install name for tiflash proxy library on MacOS"
-        COMMAND ${CMAKE_INSTALL_NAME_TOOL} -id @executable_path/libtiflash_proxy.dylib ${_TIFLASH_PROXY_LIBRARY})
-endif()
-
 add_custom_target(tiflash_proxy ALL
     DEPENDS ${_TIFLASH_PROXY_LIBRARY})
 

diff --git a/dbms/src/Common/MPMCQueue.h b/dbms/src/Common/MPMCQueue.h
@@ -121,14 +121,30 @@ class MPMCQueue
     /// Finish a NORMAL queue will wake up all blocking readers and writers.
     /// After `finish()` the queue can't be pushed any more while `pop` is allowed
     /// the queue is empty.
-    void finish()
+    /// Return true if the previous status is NORMAL.
+    bool finish()
     {
         std::unique_lock lock(mu);
         if (isNormal())
         {
             status = Status::FINISHED;
             notifyAll();
+            return true;
         }
+        else
+            return false;
+    }
+
+    bool isNextPopNonBlocking() const
+    {
+        std::unique_lock lock(mu);
+        return read_pos < write_pos || !isNormal();
+    }
+
+    bool isNextPushNonBlocking() const
+    {
+        std::unique_lock lock(mu);
+        return write_pos - read_pos < capacity || !isNormal();
     }
 
     MPMCQueueStatus getStatus() const

diff --git a/dbms/src/Common/MyDuration.h b/dbms/src/Common/MyDuration.h
@@ -71,6 +71,7 @@ class MyDuration
     Int32 minutes() const;
     Int32 seconds() const;
     Int32 microSecond() const;
+    Int64 nanoSecond() const { return nanos; };
 
     String toString() const;
 };

diff --git a/dbms/src/Common/MyTime.cpp b/dbms/src/Common/MyTime.cpp
@@ -501,6 +501,22 @@ int MyTimeBase::weekDay() const
     return diff;
 }
 
+const String & MyTimeBase::weekDayName() const
+{
+    static const String invalid_weekday;
+    if (month == 0 || day == 0)
+        return invalid_weekday;
+    return weekday_names[weekDay()];
+}
+
+const String & MyTimeBase::monthName() const
+{
+    static const String invalid_month_name;
+    if (month <= 0 || month > 12)
+        return invalid_month_name;
+    return month_names[month - 1];
+}
+
 bool checkTimeValid(Int32 year, Int32 month, Int32 day, Int32 hour, Int32 minute, Int32 second)
 {
     if (year > 9999 || month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59)
@@ -581,51 +597,59 @@ std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t
         {
         case 14: // YYYYMMDDHHMMSS
         {
-            std::sscanf(seps[0].c_str(), "%4d%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute, &second); //NOLINT
+            int ret = std::sscanf(seps[0].c_str(), "%4d%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute, &second); //NOLINT(cert-err34-c): check conversion error manually
+            truncated_or_incorrect = (ret != 6);
             hhmmss = true;
             break;
         }
         case 12: // YYMMDDHHMMSS
         {
-            std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute, &second); //NOLINT
+            int ret = std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute, &second); //NOLINT(cert-err34-c): check conversion error manually
+            truncated_or_incorrect = (ret != 6);
             year = adjustYear(year);
             hhmmss = true;
             break;
         }
         case 11: // YYMMDDHHMMS
         {
-            std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d%1d", &year, &month, &day, &hour, &minute, &second); //NOLINT
+            int ret = std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d%1d", &year, &month, &day, &hour, &minute, &second); //NOLINT(cert-err34-c): check conversion error manually
+            truncated_or_incorrect = (ret != 6);
             year = adjustYear(year);
             hhmmss = true;
             break;
         }
         case 10: // YYMMDDHHMM
         {
-            std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute); //NOLINT
+            int ret = std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute); //NOLINT(cert-err34-c): check conversion error manually
+            truncated_or_incorrect = (ret != 5);
             year = adjustYear(year);
             break;
         }
         case 9: // YYMMDDHHM
         {
-            std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%1d", &year, &month, &day, &hour, &minute); //NOLINT
+            int ret = std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%1d", &year, &month, &day, &hour, &minute); //NOLINT(cert-err34-c): check conversion error manually
+            truncated_or_incorrect = (ret != 5);
             year = adjustYear(year);
             break;
         }
         case 8: // YYYYMMDD
         {
-            std::sscanf(seps[0].c_str(), "%4d%2d%2d", &year, &month, &day); //NOLINT
+            int ret = std::sscanf(seps[0].c_str(), "%4d%2d%2d", &year, &month, &day); //NOLINT(cert-err34-c): check conversion error manually
+            truncated_or_incorrect = (ret != 3);
             break;
         }
         case 7: // YYMMDDH
         {
-            std::sscanf(seps[0].c_str(), "%2d%2d%2d%1d", &year, &month, &day, &hour); //NOLINT
+            int ret = std::sscanf(seps[0].c_str(), "%2d%2d%2d%1d", &year, &month, &day, &hour); //NOLINT(cert-err34-c): check conversion error manually
+            truncated_or_incorrect = (ret != 4);
             year = adjustYear(year);
             break;
         }
         case 6: // YYMMDD
         case 5: // YYMMD
         {
-            std::sscanf(seps[0].c_str(), "%2d%2d%2d", &year, &month, &day); //NOLINT
+            int ret = std::sscanf(seps[0].c_str(), "%2d%2d%2d", &year, &month, &day); //NOLINT(cert-err34-c): check conversion error manually
+            truncated_or_incorrect = (ret != 3);
             year = adjustYear(year);
             break;
         }
@@ -644,28 +668,29 @@ std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t
             switch (frac_str.size())
             {
             case 0:
-                ret = 1;
                 is_date = true;
                 break;
             case 1:
             case 2:
             {
-                ret = std::sscanf(frac_str.c_str(), "%2d ", &hour); //NOLINT
+                ret = std::sscanf(frac_str.c_str(), "%2d ", &hour); //NOLINT(cert-err34-c): check conversion error manually
+                truncated_or_incorrect = (ret != 1);
                 break;
             }
             case 3:
             case 4:
             {
-                ret = std::sscanf(frac_str.c_str(), "%2d%2d ", &hour, &minute); //NOLINT
+                ret = std::sscanf(frac_str.c_str(), "%2d%2d ", &hour, &minute); //NOLINT(cert-err34-c): check conversion error manually
+                truncated_or_incorrect = (ret != 2);
                 break;
             }
             default:
             {
-                ret = std::sscanf(frac_str.c_str(), "%2d%2d%2d ", &hour, &minute, &second); //NOLINT
+                ret = std::sscanf(frac_str.c_str(), "%2d%2d%2d ", &hour, &minute, &second); //NOLINT(cert-err34-c): check conversion error manually
+                truncated_or_incorrect = (ret != 3);
                 break;
             }
             }
-            truncated_or_incorrect = (ret == 0);
         }
         if (l == 9 || l == 10)
         {
@@ -885,6 +910,7 @@ void convertTimeZoneImpl(UInt64 from_time, UInt64 & to_time, const DateLUTImpl &
         }
         else
         {
+            /// For time earlier than 1970-01-01 00:00:00 UTC, return 0, aligned with mysql and tidb
             to_time = 0;
             return;
         }
@@ -937,7 +963,7 @@ std::pair<time_t, UInt32> roundTimeByFsp(time_t second, UInt64 nano_second, UInt
 // the implementation is the same as TiDB
 int calcDayNum(int year, int month, int day)
 {
-    if (year == 0 || month == 0)
+    if (year == 0 && month == 0)
         return 0;
     int delsum = 365 * year + 31 * (month - 1) + day;
     if (month <= 2)
@@ -1045,11 +1071,15 @@ void MyTimeBase::check(bool allow_zero_in_date, bool allow_invalid_date) const
     UInt8 max_day = 31;
     if (!allow_invalid_date)
     {
+        if (month < 1)
+        {
+            throw TiFlashException(fmt::format("Incorrect time value: month {}", month), Errors::Types::WrongValue);
+        }
         constexpr static UInt8 max_days_in_month[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
         static auto is_leap_year = [](UInt16 _year) {
             return ((_year % 4 == 0) && (_year % 100 != 0)) || (_year % 400 == 0);
         };
-        max_day = max_days_in_month[month - 1]; // NOLINT
+        max_day = max_days_in_month[month - 1];
         if (month == 2 && is_leap_year(year))
         {
             max_day = 29;

diff --git a/dbms/src/Common/MyTime.h b/dbms/src/Common/MyTime.h
@@ -83,6 +83,9 @@ struct MyTimeBase
 
     // returns the week day of current date(0 as sunday)
     int weekDay() const;
+    // returns the week day name of current date, return empty string if invalid
+    const String & weekDayName() const;
+    const String & monthName() const;
     // the following methods are port from TiDB
     int yearDay() const;
 
@@ -176,7 +179,6 @@ int calcDayNum(int year, int month, int day);
 
 size_t maxFormattedDateTimeStringLength(const String & format);
 
-/// For time earlier than 1970-01-01 00:00:00 UTC, return 0, aligned with mysql and tidb
 inline time_t getEpochSecond(const MyDateTime & my_time, const DateLUTImpl & time_zone)
 {
     return time_zone.makeDateTime(my_time.year, my_time.month, my_time.day, my_time.hour, my_time.minute, my_time.second);

diff --git a/dbms/src/Common/OptimizedRegularExpression.h b/dbms/src/Common/OptimizedRegularExpression.h
@@ -48,7 +48,8 @@ class OptimizedRegularExpressionImpl
     {
         RE_CASELESS = 0x00000001,
         RE_NO_CAPTURE = 0x00000010,
-        RE_DOT_NL = 0x00000100
+        RE_DOT_NL = 0x00000100,
+        RE_NO_OPTIMIZE = 0x00001000
     };
 
     using Match = OptimizedRegularExpressionDetails::Match;