add delay clean table lock (pingcap#905)

config/tidb.toml

@@ -28,6 +28,9 @@ lease = "10s"
 # turn off this option if there will be a large number of tables created.
 split-table = true
 
+# delay-clean-table-lock is used to control whether delayed-release the table lock in the abnormal situation. (Milliseconds)
+delay-clean-table-lock = 60000
+
 # The limit of concurrent executed sessions.
 # token-limit = 1000
 

core/src/main/scala/com/pingcap/tispark/TiBatchWrite.scala

@@ -41,6 +41,11 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 object TiBatchWrite {
+  // Milliseconds
+  private val MIN_DELAY_CLEAN_TABLE_LOCK = 60000
+  private val DELAY_CLEAN_TABLE_LOCK_AND_COMMIT_BACKOFF_DELTA = 30000
+  private val PRIMARY_KEY_COMMIT_BACKOFF = MIN_DELAY_CLEAN_TABLE_LOCK - DELAY_CLEAN_TABLE_LOCK_AND_COMMIT_BACKOFF_DELTA
+
   type SparkRow = org.apache.spark.sql.Row
   type TiRow = com.pingcap.tikv.row.Row
   type TiDataType = com.pingcap.tikv.types.DataType
@@ -146,7 +151,26 @@ class TiBatchWrite(@transient val df: DataFrame,
 
     // lock table
     tiDBJDBCClient = new TiDBJDBCClient(TiDBUtils.createConnectionFactory(options.url)())
-    isEnableTableLock = tiDBJDBCClient.isEnableTableLock
+    isEnableTableLock = {
+      if (tiDBJDBCClient.isEnableTableLock) {
+        if (tiDBJDBCClient.getDelayCleanTableLock >= MIN_DELAY_CLEAN_TABLE_LOCK) {
+          true
+        } else {
+          logger.warn(
+            s"table lock disabled! to enable table lock, please set tidb config: delay-clean-table-lock >= $MIN_DELAY_CLEAN_TABLE_LOCK"
+          )
+          false
+        }
+      } else {
+        false
+      }
+    }
+    if (!isEnableTableLock) {
+      logger.warn(
+        s"table lock disabled! to enable table lock, please set tidb config: enable-table-lock = true"
+      )
+    }
+
     isEnableSplitRegion = tiDBJDBCClient.isEnableSplitTable
     lockTable()
 
@@ -324,7 +348,7 @@ class TiBatchWrite(@transient val df: DataFrame,
         s"invalid transaction tso with startTs=$startTs, commitTs=$commitTs"
       )
     }
-    val commitPrimaryBackoff = ConcreteBackOffer.newCustomBackOff(BackOffer.BATCH_COMMIT_BACKOFF)
+    val commitPrimaryBackoff = ConcreteBackOffer.newCustomBackOff(PRIMARY_KEY_COMMIT_BACKOFF)
 
     if (connectionLost()) {
       throw new TiBatchWriteException("tidb's jdbc connection is lost!")

tikv-client/src/main/java/com/pingcap/tikv/TiDBJDBCClient.java

@@ -31,6 +31,8 @@ public class TiDBJDBCClient implements AutoCloseable {
   private static final String SELECT_TIDB_CONFIG_SQL = "select @@tidb_config";
   private static final String ENABLE_TABLE_LOCK_KEY = "enable-table-lock";
   private static final Boolean ENABLE_TABLE_LOCK_DEFAULT = false;
+  private static final String DELAY_CLEAN_TABLE_LOCK = "delay-clean-table-lock";
+  private static final int DELAY_CLEAN_TABLE_LOCK_DEFAULT = 0;
   private static final String ENABLE_SPLIT_TABLE_KEY = "split-table";
   private static final Boolean ENABLE_SPLIT_TABLE_DEFAULT = false;
 
@@ -45,6 +47,20 @@ public boolean isEnableTableLock() throws IOException, SQLException {
     return (Boolean) enableTableLock;
   }
 
+  /**
+   * get enable-table-lock config from tidb
+   *
+   * @return Milliseconds
+   * @throws IOException
+   * @throws SQLException
+   */
+  public int getDelayCleanTableLock() throws IOException, SQLException {
+    Map<String, Object> configMap = readConfMapFromTiDB();
+    Object enableTableLock =
+        configMap.getOrDefault(DELAY_CLEAN_TABLE_LOCK, DELAY_CLEAN_TABLE_LOCK_DEFAULT);
+    return (int) enableTableLock;
+  }
+
   public boolean lockTableWriteLocal(String databaseName, String tableName) throws SQLException {
     try (Statement tidbStmt = connection.createStatement()) {
       String sql = "lock tables `" + databaseName + "`.`" + tableName + "` write local";