Merge branch 'stonedb-5.7-dev' into load-uneffect

.github/workflows/pull_requests.yml

@@ -133,8 +133,10 @@ jobs:
 
       - name: create coverage.info
         run: |
+          cat codecov.yml | curl --data-binary @- https://codecov.io/validate
           cd build
-          lcov --capture --directory . --output-file  coverage.info --test-name coverage
+          lcov --capture --directory . --output-file coverage.info --test-name coverage
 
       - name: Codecov
-        uses: codecov/codecov-action@v3.1.1
+        uses: codecov/codecov-action@v3.1.4
+

codecov.yml

@@ -0,0 +1,3 @@
+ignore:
+- "include/boost_1_66_0"
+- "extra"

mysql-test/suite/tianmu/r/issue1616.result

@@ -0,0 +1,61 @@
+DROP DATABASE IF EXISTS issue1616_test;
+CREATE DATABASE issue1616_test;
+USE issue1616_test;
+CREATE TABLE T1  (id int(11) NOT NULL auto_increment,  parent_id int(11) DEFAULT '0' NOT NULL,  level tinyint(4)
+DEFAULT '0' NOT NULL, PRIMARY KEY (id)) engine=tianmu;
+INSERT INTO T1 VALUES (3,1,1),(4,1,1);
+INSERT INTO T1 VALUES (3,1,1),(4,1,1);
+ERROR 23000: Duplicate entry '3' for key 'PRIMARY'
+SELECT * FROM T1;
+id	parent_id	level
+3	1	1
+4	1	1
+UPDATE IGNORE T1 SET id=id+1;
+SELECT * FROM T1;
+id	parent_id	level
+3	1	1
+5	1	1
+UPDATE T1 SET id =10;
+ERROR 23000: Duplicate entry '10' for key 'PRIMARY'
+SELECT * FROM T1;
+id	parent_id	level
+3	1	1
+5	1	1
+UPDATE T1 SET ID=5 WHERE ID=3;
+ERROR 23000: Duplicate entry '5' for key 'PRIMARY'
+SELECT * FROM T1;
+id	parent_id	level
+3	1	1
+5	1	1
+DROP TABLE T1;
+CREATE TABLE T2 (dt datetime, val int, primary key(dt)) ENGINE =tianmu;
+INSERT INTO T2 VALUES ('2017-11-05 20:29:36',1), ('2027-11-05 20:29:36', 2);
+UPDATE T2 SET dt ='2027-11-05 20:29:36' WHERE val =1;
+ERROR 23000: Duplicate entry '2027-11-05 20:29:36' for key 'PRIMARY'
+SELECT * FROM T2;
+dt	val
+2017-11-05 20:29:36	1
+2027-11-05 20:29:36	2
+DROP TABLE T2;
+CREATE TABLE T3  (id int(11) NOT NULL auto_increment,  parent_id int(11) DEFAULT '0' NOT NULL,  level tinyint(4)
+DEFAULT '0' NOT NULL, PRIMARY KEY (id, parent_id)) engine=tianmu;
+INSERT INTO T3 VALUES (3,1,1),(4,1,1);
+INSERT INTO T3 VALUES (3,1,1),(4,1,1);
+ERROR 23000: Duplicate entry '3-1' for key 'PRIMARY'
+UPDATE IGNORE T3 SET id=id+1;
+SELECT * FROM T3;
+id	parent_id	level
+4	1	1
+5	1	1
+DROP TABLE T3;
+CREATE TABLE T4  (id int(11) NOT NULL auto_increment,  parent_id int(11) DEFAULT '0' NOT NULL,  level tinyint(4)
+DEFAULT '0' NOT NULL, PRIMARY KEY (id)) engine=innodb;
+INSERT INTO T4 VALUES (3,1,1),(4,1,1);
+UPDATE T4 SET id =10;
+ERROR 23000: Duplicate entry '10' for key 'PRIMARY'
+SELECT * FROM T4;
+id	parent_id	level
+3	1	1
+4	1	1
+DROP TABLE T4;
+DROP DATABASE issue1616_test;

mysql-test/suite/tianmu/t/issue1616.test

@@ -0,0 +1,64 @@
+--source include/have_tianmu.inc
+--disable_warnings
+DROP DATABASE IF EXISTS issue1616_test;
+CREATE DATABASE issue1616_test;
+USE issue1616_test;
+--enable_warnings
+
+CREATE TABLE T1  (id int(11) NOT NULL auto_increment,  parent_id int(11) DEFAULT '0' NOT NULL,  level tinyint(4)
+                  DEFAULT '0' NOT NULL, PRIMARY KEY (id)) engine=tianmu;
+
+INSERT INTO T1 VALUES (3,1,1),(4,1,1);
+--ERROR 1062
+INSERT INTO T1 VALUES (3,1,1),(4,1,1);
+
+SELECT * FROM T1;
+
+UPDATE IGNORE T1 SET id=id+1;
+
+SELECT * FROM T1;
+
+--ERROR 1062
+UPDATE T1 SET id =10;
+
+SELECT * FROM T1;
+
+--ERROR 1062
+UPDATE T1 SET ID=5 WHERE ID=3; 
+SELECT * FROM T1;
+
+DROP TABLE T1;
+
+
+CREATE TABLE T2 (dt datetime, val int, primary key(dt)) ENGINE =tianmu;
+INSERT INTO T2 VALUES ('2017-11-05 20:29:36',1), ('2027-11-05 20:29:36', 2);
+--ERROR 1062
+UPDATE T2 SET dt ='2027-11-05 20:29:36' WHERE val =1;
+
+SELECT * FROM T2;
+DROP TABLE T2;
+
+#multi-keys
+CREATE TABLE T3  (id int(11) NOT NULL auto_increment,  parent_id int(11) DEFAULT '0' NOT NULL,  level tinyint(4)
+                  DEFAULT '0' NOT NULL, PRIMARY KEY (id, parent_id)) engine=tianmu;
+
+INSERT INTO T3 VALUES (3,1,1),(4,1,1);
+--ERROR 1062
+INSERT INTO T3 VALUES (3,1,1),(4,1,1);
+
+UPDATE IGNORE T3 SET id=id+1;
+SELECT * FROM T3;
+
+DROP TABLE T3;
+
+CREATE TABLE T4  (id int(11) NOT NULL auto_increment,  parent_id int(11) DEFAULT '0' NOT NULL,  level tinyint(4)
+                  DEFAULT '0' NOT NULL, PRIMARY KEY (id)) engine=innodb;
+
+INSERT INTO T4 VALUES (3,1,1),(4,1,1);
+--ERROR 1062
+UPDATE T4 SET id =10;
+
+SELECT * FROM T4;
+DROP TABLE T4;
+
+DROP DATABASE issue1616_test;

storage/tianmu/core/engine.cpp

@@ -1826,7 +1826,7 @@ int Engine::InsertRow(const std::string &table_path, [[maybe_unused]] Transactio
 
 int Engine::UpdateRow(const std::string &table_path, TABLE *table, std::shared_ptr<TableShare> &share, uint64_t row_id,
                       const uchar *old_data, uchar *new_data) {
-  DBUG_ENTER("Engine::UpdateRow");
+  // DBUG_ENTER("Engine::UpdateRow");
   int ret = 0;
   if (tianmu_sysvar_insert_delayed && table->s->tmp_table == NO_TMP_TABLE && tianmu_sysvar_enable_rowstore) {
     UpdateToDelta(table_path, share, table, row_id, old_data, new_data);
@@ -1835,13 +1835,13 @@ int Engine::UpdateRow(const std::string &table_path, TABLE *table, std::shared_p
     auto tm_table = current_txn_->GetTableByPath(table_path);
     ret = tm_table->Update(table, row_id, old_data, new_data);
   }
-
-  DBUG_RETURN(ret);
+  return ret;
+  // DBUG_RETURN(ret);
 }
 
 int Engine::DeleteRow(const std::string &table_path, TABLE *table, std::shared_ptr<TableShare> &share,
                       uint64_t row_id) {
-  DBUG_ENTER("Engine::UpdateRow");
+  DBUG_ENTER("Engine::DeleteRow");
   int ret = 0;
   if (tianmu_sysvar_insert_delayed && table->s->tmp_table == NO_TMP_TABLE && tianmu_sysvar_enable_rowstore) {
     DeleteToDelta(share, table, row_id);

storage/tianmu/core/engine.h

@@ -179,6 +179,9 @@ class Engine final {
   static const char *StrToFiled(const char *ptr, Field *field, DeltaRecordHead *deltaRecord, int col_num);
   static char *FiledToStr(char *ptr, Field *field, DeltaRecordHead *deltaRecord, int col_num, THD *thd);
 
+  void setExtra(ha_extra_function extra) { extra_info = extra; }
+  ha_extra_function getExtra() { return extra_info; }
+
  private:
   void AddTx(Transaction *tx);
   void RemoveTx(Transaction *tx);
@@ -288,6 +291,8 @@ class Engine final {
   uint64_t m_mem_available_ = 0;
   uint64_t m_swap_used_ = 0;
   index::KVStore *store_;
+
+  ha_extra_function extra_info;
 };
 
 class ResultSender {

storage/tianmu/core/table_share.cpp

@@ -26,6 +26,7 @@ namespace Tianmu {
 namespace core {
 TableShare::TableShare(const fs::path &table_path, const TABLE_SHARE *table_share)
     : no_cols(table_share->fields), table_path(table_path) {
+  s = const_cast<TABLE_SHARE *>(table_share);
   try {
     system::TianmuFile ftbl;
     ftbl.OpenReadOnly(table_path / common::TABLE_DESC_FILE);

storage/tianmu/core/table_share.h

@@ -67,6 +67,9 @@ class TableShare final {
   // MySQL lock
   THR_LOCK thr_lock;
 
+  // TABLE share in sql.
+  TABLE_SHARE *s;
+
  private:
   TABLE_META meta;
   size_t no_cols;

storage/tianmu/core/tianmu_table.cpp

@@ -1016,7 +1016,6 @@ int TianmuTable::Insert(TABLE *table) {
 }
 
 int TianmuTable::Update(TABLE *table, uint64_t row_id, const uchar *old_data, uchar *new_data) {
-  // todo(dfx): move to before for loop, need test
   my_bitmap_map *org_bitmap2 = dbug_tmp_use_all_columns(table, table->read_set);
   std::shared_ptr<void> defer(nullptr,
                               [org_bitmap2, table](...) { dbug_tmp_restore_column_map(table->read_set, org_bitmap2); });
@@ -1025,6 +1024,9 @@ int TianmuTable::Update(TABLE *table, uint64_t row_id, const uchar *old_data, uc
   core::Engine *eng = reinterpret_cast<core::Engine *>(tianmu_hton->data);
   assert(eng);
 
+  // uinsg check_unique_constraint(table) to check whether it has unique constr on this table;
+  // now, tianmu only support PK, the unique constraint is not supported now.
+  // the vfield is not in our consideration. and should not has any triggers on it.
   for (uint col_id = 0; col_id < table->s->fields; col_id++) {
     if (!bitmap_is_set(table->write_set, col_id)) {
       continue;

storage/tianmu/handler/ha_tianmu.cpp

@@ -419,7 +419,7 @@ int ha_tianmu::write_row([[maybe_unused]] uchar *buf) {
  Called from sql_select.cc, sql_acl.cc, sql_update.cc, and sql_insert.cc.
  */
 int ha_tianmu::update_row(const uchar *old_data, uchar *new_data) {
-  DBUG_ENTER(__PRETTY_FUNCTION__);
+  DBUG_ENTER("ha_tianmu::update_row");
 
   core::Engine *eng = reinterpret_cast<core::Engine *>(tianmu_hton->data);
   assert(eng);
@@ -1176,6 +1176,15 @@ int ha_tianmu::extra(enum ha_extra_function operation) {
     cq_.reset();
     query_.reset();
   }
+
+  extra_info = operation;
+  if (operation == HA_EXTRA_IGNORE_DUP_KEY || operation == HA_EXTRA_NO_IGNORE_DUP_KEY) {
+    core::Engine *eng = reinterpret_cast<core::Engine *>(tianmu_hton->data);
+    ASSERT(eng);
+    if (eng)
+      eng->setExtra(extra_info);
+  }
+
   DBUG_RETURN(0);
 }
 

storage/tianmu/handler/ha_tianmu.h

@@ -155,6 +155,9 @@ class ha_tianmu final : public handler {
   int fill_row_by_id(uchar *buf, uint64_t rowid);
   void key_convert(const uchar *key, uint key_len, std::vector<uint> cols, std::vector<std::string> &keys);
 
+  void setExtra(ha_extra_function extra) { extra_info = extra; }
+  ha_extra_function getExtra() { return extra_info; }
+
  public:
   static const Alter_inplace_info::HA_ALTER_FLAGS TIANMU_SUPPORTED_ALTER_ADD_DROP_ORDER;
   static const Alter_inplace_info::HA_ALTER_FLAGS TIANMU_SUPPORTED_ALTER_COLUMN_NAME;
@@ -182,6 +185,8 @@ class ha_tianmu final : public handler {
   std::unique_ptr<core::CompiledQuery> cq_;
   bool result_ = false;
   std::vector<std::vector<uchar>> blob_buffers_;
+
+  ha_extra_function extra_info; /*!< TRX_DUP_IGNORE | TRX_DUP_REPLACE */
 };
 
 }  // namespace DBHandler

storage/tianmu/index/tianmu_table_index.cpp

@@ -27,6 +27,8 @@
 namespace Tianmu {
 namespace index {
 
+// Here, In future, tianmu maybe has uniquekey, secondary key, fulltext, etc. Therefore, a type of index should be
+// added. But, now. we only have only one index: PK.
 TianmuTableIndex::TianmuTableIndex(const std::string &name, TABLE *table) {
   std::string fullname;
   // normalize the table name.
@@ -168,26 +170,26 @@ common::ErrorCode TianmuTableIndex::CheckUniqueness(core::Transaction *tx, const
     s = tx->KVTrans().Get(rocksdb_key_->get_cf(), pk_slice, &retrieved_value);
   }
 
-  if (!s.ok() && !s.IsNotFound()) {
-    TIANMU_LOG(LogCtl_Level::ERROR, "RockDb read fail:%s", s.getState());
-    return common::ErrorCode::FAILED;
-  }
-
-  if (s.ok()) {
+  if (s.ok()) {  // means that there's  another key existed.
     return common::ErrorCode::DUPP_KEY;
-  }
-
-  return common::ErrorCode::SUCCESS;
+  } else if (s.IsNotFound()) {  // not exist a key.
+    return common::ErrorCode::NOT_FOUND_KEY;
+  } else  // failed.
+    return common::ErrorCode::FAILED;
 }
 
 common::ErrorCode TianmuTableIndex::InsertIndex(core::Transaction *tx, std::vector<std::string> &fields, uint64_t row) {
   StringWriter value, key;
 
   rocksdb_key_->pack_key(key, fields, value);
 
+  // 1)inserts into a PK; 2) insert into without a PK. the return value of `CheckUniqueness` is as following:
+  // 1: common::ErrorCode::DUPP_KEY; 2: common::ErrorCode::NOT_FOUND_KEY; 3:common::ErrorCode::FAILED
   common::ErrorCode err_code = CheckUniqueness(tx, {(const char *)key.ptr(), key.length()});
-  if (err_code != common::ErrorCode::SUCCESS)
+  if (idx_type_ == IndexType::INDEX_TYPE_PRIMARY && err_code == common::ErrorCode::DUPP_KEY) {  // PK
     return err_code;
+  } else if (idx_type_ == IndexType::INDEX_TYPE_SECONDARY) {  // do not need uniqueness constrain
+  }
 
   value.write_uint64(row);
   const auto cf = rocksdb_key_->get_cf();
@@ -196,27 +198,37 @@ common::ErrorCode TianmuTableIndex::InsertIndex(core::Transaction *tx, std::vect
   if (!s.ok()) {
     TIANMU_LOG(LogCtl_Level::ERROR, "RocksDB: insert key fail!");
     err_code = common::ErrorCode::FAILED;
-  }
+  } else if (s.ok())
+    err_code = common::ErrorCode::SUCCESS;
+
   return err_code;
 }
 
 common::ErrorCode TianmuTableIndex::UpdateIndex(core::Transaction *tx, std::string &nkey, std::string &okey,
                                                 uint64_t row) {
-  StringWriter value, packkey;
-  std::vector<std::string> ofields, nfields;
+  StringWriter new_value, old_value, new_packke, old_packkey;
+  std::vector<std::string> old_fields, new_fields;
+
+  old_fields.emplace_back(okey);
+  new_fields.emplace_back(nkey);
+
+  rocksdb_key_->pack_key(new_packke, new_fields, new_value);
+  rocksdb_key_->pack_key(old_packkey, old_fields, old_value);
 
-  ofields.emplace_back(okey);
-  nfields.emplace_back(nkey);
+  common::ErrorCode err_code = CheckUniqueness(tx, {(const char *)new_packke.ptr(), new_packke.length()});
 
-  rocksdb_key_->pack_key(packkey, ofields, value);
-  common::ErrorCode err_code = CheckUniqueness(tx, {(const char *)packkey.ptr(), packkey.length()});
   if (err_code == common::ErrorCode::DUPP_KEY) {
-    const auto cf = rocksdb_key_->get_cf();
-    tx->KVTrans().Delete(cf, {(const char *)packkey.ptr(), packkey.length()});
-  } else {
-    TIANMU_LOG(LogCtl_Level::WARN, "RockDb: don't have the key for update!");
+  } else if (err_code == common::ErrorCode::NOT_FOUND_KEY || err_code == common::ErrorCode::SUCCESS) {
+    // gets the old index. then update it.
+    common::ErrorCode code = CheckUniqueness(tx, {(const char *)old_packkey.ptr(), old_packkey.length()});
+    if (code != common::ErrorCode::SUCCESS) {
+      const auto cf = rocksdb_key_->get_cf();
+      // delete old index then insert a new one.
+      rocksdb::Status ret_del = tx->KVTrans().Delete(cf, {(const char *)old_packkey.ptr(), old_packkey.length()});
+      err_code = InsertIndex(tx, new_fields, row);
+    }
   }
-  err_code = InsertIndex(tx, nfields, row);
+
   return err_code;
 }
 

storage/tianmu/index/tianmu_table_index.h

@@ -23,6 +23,7 @@
 #include "rocksdb/db.h"
 
 #include "common/common_definitions.h"
+#include "index/rdb_meta_manager.h"
 #include "index/rdb_utils.h"
 
 namespace Tianmu {
@@ -58,6 +59,7 @@ class TianmuTableIndex final {
   common::ErrorCode UpdateIndex(core::Transaction *tx, std::string &nkey, std::string &okey, uint64_t row);
   common::ErrorCode DeleteIndex(core::Transaction *tx, std::vector<std::string> &fields, uint64_t row);
   common::ErrorCode GetRowByKey(core::Transaction *tx, std::vector<std::string> &fields, uint64_t &row);
+  index::IndexType type() { return idx_type_; }
 
  public:
   std::shared_ptr<RdbTable> rocksdb_tbl_;
@@ -67,6 +69,8 @@ class TianmuTableIndex final {
   uint keyid_ = 0;
 
  private:
+  index::IndexType idx_type_ = index::IndexType::INDEX_TYPE_PRIMARY;
+
   common::ErrorCode CheckUniqueness(core::Transaction *tx, const rocksdb::Slice &pk_slice);
 };