From acefdc6b82f94fe4f409b1fbe78241247b6b9632 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Thu, 19 May 2022 09:34:46 -0600 Subject: [PATCH 1/8] Update tidb-configuration and system variables for 6.1 --- .../three-nodes-hybrid-deployment.md | 1 - command-line-flags-for-tidb-configuration.md | 7 +- configure-memory-usage.md | 39 ++---- dashboard/dashboard-faq.md | 2 +- enable-tls-between-clients-and-servers.md | 2 +- grafana-tidb-dashboard.md | 2 +- optimizer-hints.md | 2 +- sql-prepared-plan-cache.md | 2 +- system-variables.md | 117 +++++++++++++++--- tidb-configuration-file.md | 89 +------------ 10 files changed, 116 insertions(+), 147 deletions(-) diff --git a/best-practices/three-nodes-hybrid-deployment.md b/best-practices/three-nodes-hybrid-deployment.md index 6795834cf8ee2..61d23578500ad 100644 --- a/best-practices/three-nodes-hybrid-deployment.md +++ b/best-practices/three-nodes-hybrid-deployment.md @@ -44,7 +44,6 @@ tikv: rocksdb.rate-bytes-per-sec: “200M” tidb: - performance.committer-concurrency: 4 performance.max-procs: 8 ``` diff --git a/command-line-flags-for-tidb-configuration.md b/command-line-flags-for-tidb-configuration.md index bf62af02eb0dd..dbf39c6ebfcaf 100644 --- a/command-line-flags-for-tidb-configuration.md +++ b/command-line-flags-for-tidb-configuration.md @@ -188,9 +188,4 @@ When you start the TiDB cluster, you can use command-line options or environment ## `--repair-list` + The names of the tables to be repaired in the repair mode. -+ Default: `""` - -## `--require-secure-transport` - -+ Determines whether to require the client to use the secure mode for data transport. -+ Default: `false` \ No newline at end of file ++ Default: `""` \ No newline at end of file diff --git a/configure-memory-usage.md b/configure-memory-usage.md index 9e7dc2e0339e4..0b104275bae11 100644 --- a/configure-memory-usage.md +++ b/configure-memory-usage.md @@ -6,38 +6,16 @@ aliases: ['/docs/dev/configure-memory-usage/','/docs/dev/how-to/configure/memory # TiDB Memory Control -Currently, TiDB can track the memory quota of a single SQL query and take actions to prevent OOM (out of memory) or troubleshoot OOM when the memory usage exceeds a specific threshold value. In the TiDB configuration file, you can configure the options as below to control TiDB behaviors when the memory quota exceeds the threshold value: +Currently, TiDB can track the memory quota of a single SQL query and take actions to prevent OOM (out of memory) or troubleshoot OOM when the memory usage exceeds a specific threshold value. -``` -# Valid options: ["log", "cancel"] -oom-action = "cancel" -``` +The system variable `tidb_mem_oom_action` specifies the action to take when a query reaches the memory limit: -- If the configuration item above uses "log", when the memory quota of a single SQL query exceeds the threshold value which is controlled by the `tidb_mem_quota_query` variable, TiDB prints an entry of log. Then the SQL query continues to be executed. If OOM occurs, you can find the corresponding SQL query in the log. -- If the configuration item above uses "cancel", when the memory quota of a single SQL query exceeds the threshold value, TiDB stops executing the SQL query immediately and returns an error to the client. The error information clearly shows the memory usage of each physical execution operator that consumes much memory in the SQL execution process. +- A value of `LOG` means that queries will continue to execute when the `tidb_mem_quota_query` limit is reached, but TiDB will print an entry to the log. +- A value of `CANCEL` means TiDB stops executing the SQL query immediately after the `tidb_mem_quota_query` limit is reached, and returns an error to the client. The error information clearly shows the memory usage of each physical execution operator that consumes much memory in the SQL execution process. ## Configure the memory quota of a query -In the configuration file, you can set the default Memory Quota for each Query. The following example sets it to 32GB: - -``` -mem-quota-query = 34359738368 -``` - -In addition, you can control the memory quota of a query using the following session variables. Generally, you only need to configure `tidb_mem_quota_query`. Other variables are used for advanced configuration which most users do not need to care about. - -| Variable Name | Description | Unit | Default Value | -| -------------------------------- | ------------------------------------------------- | ---- | ------------- | -| tidb_mem_quota_query | Control the memory quota of a query | Byte | 1 << 30 (1 GB) | -| tidb_mem_quota_hashjoin | Control the memory quota of "HashJoinExec" | Byte | 32 << 30 | -| tidb_mem_quota_mergejoin | Control the memory quota of "MergeJoinExec" | Byte | 32 << 30 | -| tidb_mem_quota_sort | Control the memory quota of "SortExec" | Byte | 32 << 30 | -| tidb_mem_quota_topn | Control the memory quota of "TopNExec" | Byte | 32 << 30 | -| tidb_mem_quota_indexlookupreader | Control the memory quota of "IndexLookUpExecutor" | Byte | 32 << 30 | -| tidb_mem_quota_indexlookupjoin | Control the memory quota of "IndexLookUpJoin" | Byte | 32 << 30 | -| tidb_mem_quota_nestedloopapply | Control the memory quota of "NestedLoopApplyExec" | Byte | 32 << 30 | - -Some usage examples: +The system variable `tidb_mem_quota_query` sets the limit for a query in bytes. Some usage examples: {{< copyable "sql" >}} @@ -93,7 +71,6 @@ The following example constructs a memory-intensive SQL statement that triggers {{< copyable "" >}} ```toml - mem-quota-query = 34359738368 // Increases the memory limit of each query to construct SQL statements that take up larger memory. [performance] memory-usage-alarm-ratio = 0.8 ``` @@ -132,7 +109,7 @@ The following example constructs a memory-intensive SQL statement that triggers TiDB supports disk spill for execution operators. When the memory usage of a SQL execution exceeds the memory quota, tidb-server can spill the intermediate data of execution operators to the disk to relieve memory pressure. Operators supporting disk spill include Sort, MergeJoin, HashJoin, and HashAgg. -- The disk spill behavior is jointly controlled by the [`mem-quota-query`](/tidb-configuration-file.md#mem-quota-query), [`oom-use-tmp-storage`](/tidb-configuration-file.md#oom-use-tmp-storage), [`tmp-storage-path`](/tidb-configuration-file.md#tmp-storage-path), and [`tmp-storage-quota`](/tidb-configuration-file.md#tmp-storage-quota) parameters. +- The disk spill behavior is jointly controlled by `tidb_mem_quota_query`, [`oom-use-tmp-storage`](/tidb-configuration-file.md#oom-use-tmp-storage), [`tmp-storage-path`](/tidb-configuration-file.md#tmp-storage-path), and [`tmp-storage-quota`](/tidb-configuration-file.md#tmp-storage-quota) parameters. - When the disk spill is triggered, TiDB outputs a log containing the keywords `memory exceeds quota, spill to disk now` or `memory exceeds quota, set aggregate mode to spill-mode`. - Disk spill for the Sort, MergeJoin, and HashJoin operator is introduced in v4.0.0; disk spill for the HashAgg operator is introduced in v5.2.0. - When the SQL executions containing Sort, MergeJoin, or HashJoin cause OOM, TiDB triggers disk spill by default. When SQL executions containing HashAgg cause OOM, TiDB does not trigger disk spill by default. You can configure the system variable `tidb_executor_concurrency = 1` to trigger disk spill for HashAgg. @@ -148,7 +125,7 @@ The following example uses a memory-consuming SQL statement to demonstrate the d {{< copyable "sql" >}} ```sql - set tidb_mem_quota_query = 1 << 30; + SET tidb_mem_quota_query = 1 << 30; ``` 2. Create a single table `CREATE TABLE t(a int);` and insert 256 rows of different data. @@ -172,7 +149,7 @@ The following example uses a memory-consuming SQL statement to demonstrate the d {{< copyable "sql" >}} ```sql - set tidb_executor_concurrency = 1; + SET tidb_executor_concurrency = 1; ``` 5. Execute the same SQL statement. You can find that this time, the statement is successfully executed and no error message is returned. From the following detailed execution plan, you can see that HashAgg has used 600 MB of hard disk space. diff --git a/dashboard/dashboard-faq.md b/dashboard/dashboard-faq.md index d14318be60e71..b976e9663eb1d 100644 --- a/dashboard/dashboard-faq.md +++ b/dashboard/dashboard-faq.md @@ -57,7 +57,7 @@ If your deployment tool is TiUP, take the following steps to solve this problem. ### An `invalid connection` error is shown on the **Slow Queries** page -The possible reason is that you have enabled the `prepared-plan-cache` feature of TiDB. As an experimental feature, when enabled, `prepared-plan-cache` might not function properly in specific TiDB versions, which could cause this problem in TiDB Dashboard (and other applications). You can disable `prepared-plan-cache` by updating [TiDB Configuration file](/tidb-configuration-file.md#prepared-plan-cache) to solve this problem. +The possible reason is that you have enabled the Prepared Plan Cache feature of TiDB. As an experimental feature, when enabled, Prepared Plan Cache might not function properly in specific TiDB versions, which could cause this problem in TiDB Dashboard (and other applications). You can disable Prepared Plan Cache by setting the system variable `tidb_enable_prepared_plan_cache = OFF`. ### A `required component NgMonitoring is not started` error is shown diff --git a/enable-tls-between-clients-and-servers.md b/enable-tls-between-clients-and-servers.md index 353c1a926bc14..a68404a124f68 100644 --- a/enable-tls-between-clients-and-servers.md +++ b/enable-tls-between-clients-and-servers.md @@ -20,7 +20,7 @@ To use connections secured with TLS, you first need to configure the TiDB server Similar to MySQL, TiDB allows TLS and non-TLS connections on the same TCP port. For a TiDB server with TLS enabled, you can choose to securely connect to the TiDB server through an encrypted connection, or to use an unencrypted connection. You can use the following ways to require the use of secure connections: -+ Configure the launch parameter `--require-secure-transport` to require secure connections to the TiDB server for all users. ++ Configure the system variable `require_secure_transport` to require secure connections to the TiDB server for all users. + Specify `REQUIRE SSL` when you create a user (`create user`), or modify an existing user (`alter user`), which is to specify that specified users must use the encrypted connection to access TiDB. The following is an example of creating a user: {{< copyable "sql" >}} diff --git a/grafana-tidb-dashboard.md b/grafana-tidb-dashboard.md index 41afa9706721a..f539a536df1bc 100644 --- a/grafana-tidb-dashboard.md +++ b/grafana-tidb-dashboard.md @@ -59,7 +59,7 @@ To understand the key metrics displayed on the TiDB dashboard, check the followi - Transaction Statement Num: the number of SQL statements in a transaction - Transaction Retry Num: the number of times that a transaction retries - Session Retry Error OPS: the number of errors encountered during the transaction retry per second. This metric includes two error types: retry failure and exceeding the maximum number of retries - - Commit Token Wait Duration: the wait duration in the flow control queue during the transaction commit. If the wait duration is long, it means that the transaction to commit is too large and the flow is controlled. If the system still has resources available, you can speed up the commit process by increasing the `committer-concurrency` value in the TiDB configuration file + - Commit Token Wait Duration: the wait duration in the flow control queue during the transaction commit. If the wait duration is long, it means that the transaction to commit is too large and the flow is controlled. If the system still has resources available, you can speed up the commit process by increasing the system variable `tidb_committer_concurrency`. - KV Transaction OPS: the number of transactions executed per second within each TiDB instance - A user transaction might trigger multiple transaction executions in TiDB, including reading internal metadata, atomic retries of the user transaction, and so on - TiDB's internally scheduled tasks also operate on the database through transactions, which are also included in this panel diff --git a/optimizer-hints.md b/optimizer-hints.md index dc76f497db97f..34b89c7f8eccf 100644 --- a/optimizer-hints.md +++ b/optimizer-hints.md @@ -337,7 +337,7 @@ In addition to this hint, setting the `tidb_replica_read` environment variable t The `IGNORE_PLAN_CACHE()` hint reminds the optimizer not to use the Plan Cache when handling the current `prepare` statement. -This hint is used to temporarily disable the Plan Cache for a certain type of queries when [prepare-plan-cache](/tidb-configuration-file.md#prepared-plan-cache) is enabled. +This hint is used to temporarily disable the Plan Cache for a certain type of queries when [prepare-plan-cache](/sql-prepared-plan-cache.md) is enabled. In the following example, the Plan Cache is forcibly disabled when executing the `prepare` statement. diff --git a/sql-prepared-plan-cache.md b/sql-prepared-plan-cache.md index e3f679e7d1a95..90f676abae6a3 100644 --- a/sql-prepared-plan-cache.md +++ b/sql-prepared-plan-cache.md @@ -53,7 +53,7 @@ There are several points worth noting about execution plan caching and query per - Considering that the parameters of `Execute` are different, the execution plan cache prohibits some aggressive query optimization methods that are closely related to specific parameter values to ensure adaptability. This causes that the query plan may not be optimal for certain parameter values. For example, the filter condition of the query is `where a > ? And a < ?`, the parameters of the first `Execute` statement are `2` and `1` respectively. Considering that these two parameters maybe be `1` and `2` in the next execution time, the optimizer does not generate the optimal `TableDual` execution plan that is specific to current parameter values; - If cache invalidation and elimination are not considered, an execution plan cache is applied to various parameter values, which in theory also results in non-optimal execution plans for certain values. For example, if the filter condition is `where a < ?` and the parameter value used for the first execution is `1`, then the optimizer generates the optimal `IndexScan` execution plan and puts it into the cache. In the subsequent executions, if the value becomes `10000`, the `TableScan` plan might be the better one. But due to the execution plan cache, the previously generated `IndexScan` is used for execution. Therefore, the execution plan cache is more suitable for application scenarios where the query is simple (the ratio of compilation is high) and the execution plan is relatively fixed. -Currently, the execution plan cache is disabled by default. You can enable this feature by enabling the [`prepared-plan-cache`](/tidb-configuration-file.md#prepared-plan-cache) in the TiDB configuration file. +Currently, the execution plan cache is disabled by default. You can enable this feature by setting the system variable `tidb_enable_prepared_plan_cache` to `ON`. > **Note：** > diff --git a/system-variables.md b/system-variables.md index d1a71ca0cbfe1..9e6149f5c06e9 100644 --- a/system-variables.md +++ b/system-variables.md @@ -181,7 +181,8 @@ mysql> SELECT * FROM t1; ### ddl_slow_threshold -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: `300` - Unit: Milliseconds - Log DDL operations whose execution time exceeds the threshold value. @@ -320,13 +321,15 @@ This variable is an alias for `last_insert_id`. ### plugin_dir -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: "" - Indicates the directory to load plugins as specified by a command-line flag. ### plugin_load -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: "" - Indicates the plugins to load when TiDB is started. These plugins are specified by a command-line flag and separated by commas. @@ -353,6 +356,15 @@ This variable is an alias for `last_insert_id`. - This variable is used to seed the random value generator used in the `RAND()` SQL function. - The behavior of this variable is MySQL compatible. +### require_secure_transport New in v6.1.0 + +- Scope: GLOBAL +- Persists to cluster: Yes +- Default value: `OFF` +- This variable ensures that all connections to TiDB are either on a local socket, or using TLS. See [Enable TLS between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) for additional details. +- Setting this variable to `ON` requires you to connect to TiDB from a session that has TLS enabled. This helps prevent lock-out scenarios when TLS is not configured correctly. +- This setting was previously a tidb.toml option (`security.require-secure-transport`), but changed to a system variable starting from TiDB 6.1. + ### skip_name_resolve New in v5.2.0 - Scope: GLOBAL @@ -495,7 +507,7 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a > **Note:** > -> Only when the `run-auto-analyze` option is enabled in the starting configuration file of TiDB, the `auto_analyze` feature can be triggered. +> This feature requires the system variable `tidb_enable_auto_analyze` set to `ON`. ### tidb_auto_analyze_start_time @@ -568,7 +580,8 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a ### tidb_check_mb4_value_in_utf8 -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: `ON` - This variable is used to enforce that the `utf8` character set only stores values from the [Basic Multilingual Plane (BMP)](https://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane). To store characters outside the BMP, it is recommended to use the `utf8mb4` character set. - You might need to disable this option when upgrading your cluster from an earlier version of TiDB where the `utf8` checking was more relaxed. For details, see [FAQs After Upgrade](/faq/upgrade-faq.md). @@ -581,6 +594,16 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a - This variable is used to set the scan index concurrency of executing the `ADMIN CHECKSUM TABLE` statement. - When the variable is set to a larger value, the execution performance of other queries is affected. +### tidb_committer_concurrency New in v6.1.0 + +- Scope: GLOBAL +- Persists to cluster: Yes +- Default value: `128` +- Range: `[1, 10000]` +- The number of goroutines for requests related to executing commit in the commit phase of the single transaction. +- If the transaction to commit is too large, the waiting time for the flow control queue when the transaction is committed might be too long. In this situation, you can increase the configuration value to speed up the commit. +- This setting was previously a tidb.toml option (`performance.committer-concurrency`), but changed to a system variable starting from TiDB 6.1. + ### tidb_config - Scope: SESSION @@ -737,6 +760,14 @@ Constraint checking is always performed in place for pessimistic transactions (d > - If you have enabled TiDB Binlog, enabling this variable cannot improve the performance. To improve the performance, it is recommended to use [TiCDC](/ticdc/ticdc-overview.md) instead. > - Enabling this parameter only means that Async Commit becomes an optional mode of transaction commit. In fact, the most suitable mode of transaction commit is determined by TiDB. +### tidb_enable_auto_analyze New in v6.1.0 + +- Scope: GLOBAL +- Persists to cluster: Yes +- Default value: `ON` +- Determines whether TiDB automatically updates table statistics as a background operation +- This setting was previously a tidb.toml option (`performance.run-auto-analyze`), but changed to a system variable starting from TiDB 6.1. + ### tidb_enable_auto_increment_in_generated - Scope: SESSION | GLOBAL @@ -744,6 +775,14 @@ Constraint checking is always performed in place for pessimistic transactions (d - Default value: `OFF` - This variable is used to determine whether to include the `AUTO_INCREMENT` columns when creating a generated column or an expression index. +### tidb_enable_batch_dml New in v6.1.0 + +- Scope: GLOBAL +- Persists to cluster: Yes +- Default value: `OFF` +- Determines if TiDB permits non-transactional 'batched' statements. Only the value of `OFF` can be considered safe, as batch DML does not provide ACID guarantees. +- This setting was previously a tidb.toml option (`enable-batch-dml`), but changed to a system variable starting from TiDB 6.1. + ### tidb_enable_cascades_planner > **Warning:** @@ -774,7 +813,8 @@ Constraint checking is always performed in place for pessimistic transactions (d ### tidb_enable_collect_execution_info -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: `ON` - This variable controls whether to record the execution information of each operator in the slow query log. @@ -888,6 +928,14 @@ Constraint checking is always performed in place for pessimistic transactions (d - Default value: `OFF` - This variable controls whether to enable concurrency for the `Apply` operator. The number of concurrencies is controlled by the `tidb_executor_concurrency` variable. The `Apply` operator processes correlated subqueries and has no concurrency by default, so the execution speed is slow. Setting this variable value to `1` can increase concurrency and speed up execution. Currently, concurrency for `Apply` is disabled by default. +### tidb_enable_prepared_plan_cache New in v6.1.0 + +- Scope: GLOBAL +- Persists to cluster: Yes +- Default value: `ON` +- Determines whether to enable Plan Cache of the `PREPARE` statement. +- This setting was previously a tidb.toml option (`prepared-plan-cache.enabled`), but changed to a system variable starting from TiDB 6.1. + ### tidb_enable_pseudo_for_outdated_stats New in v5.3.0 - Scope: SESSION | GLOBAL @@ -908,7 +956,8 @@ Constraint checking is always performed in place for pessimistic transactions (d ### tidb_enable_slow_log -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: `ON` - This variable is used to control whether to enable the slow log feature. @@ -1078,7 +1127,8 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified ### tidb_expensive_query_time_threshold -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: `60` - Range: `[10, 2147483647]` - Unit: Seconds @@ -1088,7 +1138,8 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified ### tidb_force_priority -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: `NO_PRIORITY` - This variable is used to change the default priority for statements executed on a TiDB server. A use case is to ensure that a particular user that is performing OLAP queries receives lower priority than users performing OLTP queries. - You can set the value of this variable to `NO_PRIORITY`, `LOW_PRIORITY`, `DELAYED` or `HIGH_PRIORITY`. @@ -1320,6 +1371,16 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Range: `[100, 16384]` - This variable is used to set the maximum number of schema versions (the table IDs modified for corresponding versions) allowed to be cached. The value range is 100 ~ 16384. +### tidb_mem_oom_action New in v6.1.0 + +- Scope: GLOBAL +- Persists to cluster: Yes +- Default value: `CANCEL` +- Possible values: `CANCEL`, `LOG` +- Specifies what operation TiDB performs when a single SQL statement exceeds the memory quota specified by `tidb_mem_quota_query` and cannot be spilled over to disk. See [TiDB Memory Control](/configure-memory-usage.md) for details. +- The default value is `CANCEL`, but in TiDB v4.0.2 and earlier versions, the default value is `LOG`. +- This setting was previously a tidb.toml option (`oom-action`), but changed to a system variable starting from TiDB 6.1. + ### tidb_mem_quota_apply_cache New in v5.0 - Scope: SESSION | GLOBAL @@ -1342,7 +1403,8 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified ### tidb_mem_quota_query -- Scope: SESSION +- Scope: SESSION | GLOBAL +- Persists to cluster: Yes - Default value: `1073741824` (1 GiB) - Range: `[-1, 9223372036854775807]` - Unit: Bytes @@ -1351,7 +1413,8 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified ### tidb_memory_usage_alarm_ratio -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: `0.8` - TiDB triggers an alarm when the percentage of the memory it takes exceeds a certain threshold. For the detailed usage description of this feature, see [`memory-usage-alarm-ratio`](/tidb-configuration-file.md#memory-usage-alarm-ratio-new-in-v409). - You can set the initial value of this variable by configuring [`memory-usage-alarm-ratio`](/tidb-configuration-file.md#memory-usage-alarm-ratio-new-in-v409). @@ -1566,11 +1629,29 @@ explain select * from t where age=5; ### tidb_pprof_sql_cpu New in v4.0 -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: `0` - Range: `[0, 1]` - This variable is used to control whether to mark the corresponding SQL statement in the profile output to identify and troubleshoot performance issues. +### tidb_prepared_plan_cache_memory_guard_ratio New in v6.1.0 + +- Scope: GLOBAL +- Persists to cluster: Yes +- Default value: `0.1` +- This setting is used to prevent the tidb.toml option `performance.max-memory` from being exceeded. When `max-memory` * (1 - `tidb_prepared_plan_cache_memory_guard_ratio`) is exceeded, the elements in the LRU are removed. +- This setting was previously a tidb.toml option (`prepared-plan-cache.memory-guard-ratio`), but changed to a system variable starting from TiDB 6.1. + +### tidb_prepared_plan_cache_size New in v6.1.0 + +- Scope: GLOBAL +- Persists to cluster: Yes +- Default value: `100` +- Range: `[1, 100000]` +- The maximum number of statements that can be cached in the prepared plan cache. +- This setting was previously a tidb.toml option (`prepared-plan-cache.capacity`), but changed to a system variable starting from TiDB 6.1. + ### tidb_projection_concurrency > **Warning:** @@ -1587,9 +1668,10 @@ explain select * from t where age=5; ### tidb_query_log_max_len -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: Yes - Default value: `4096` (4 KiB) -- Range: `[-1, 9223372036854775807]` +- Range: `[0, 1073741824]` - Unit: Bytes - The maximum length of the SQL statement output. When the output length of a statement is larger than the `tidb_query-log-max-len` value, the statement is truncated to output. @@ -1598,6 +1680,7 @@ Usage example: ```sql SET tidb_query_log_max_len = 20 ``` +- This setting was previously also available a tidb.toml option (`log.query-log-max-len`), but is only a system variable starting from TiDB 6.1. ### tidb_rc_read_check_ts New in v6.0.0 @@ -1621,7 +1704,8 @@ SET tidb_query_log_max_len = 20 ### tidb_record_plan_in_slow_log -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: `ON` - This variable is used to control whether to include the execution plan of slow queries in the slow log. @@ -1729,7 +1813,8 @@ Query OK, 0 rows affected, 1 warning (0.00 sec) ### tidb_slow_log_threshold -- Scope: INSTANCE +- Scope: GLOBAL +- Persists to cluster: No - Default value: `300` - Range: `[-1, 9223372036854775807]` - Unit: Milliseconds diff --git a/tidb-configuration-file.md b/tidb-configuration-file.md index 5223d1629cf35..eb3619c6092a2 100644 --- a/tidb-configuration-file.md +++ b/tidb-configuration-file.md @@ -26,14 +26,6 @@ The TiDB configuration file supports more options than command-line parameters. + Maximum Value (64-bit platforms): `18446744073709551615` + Maximum Value (32-bit platforms): `4294967295` -### `mem-quota-query` - -- The maximum memory available for a single SQL statement. -- Default value: `1073741824` (in bytes) -- Note: When you upgrade the cluster from v2.0.x or v3.0.x to v4.0.9 or later versions, the default value of this configuration is `34359738368`. -- Requests that require more memory than this value are handled based on the behavior defined by `oom-action`. -- This value is the initial value of the system variable [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query). - ### `oom-use-tmp-storage` + Controls whether to enable the temporary storage for some operators when a single SQL statement exceeds the memory quota specified by `mem-quota-query`. @@ -53,22 +45,6 @@ The TiDB configuration file supports more options than command-line parameters. + Default value: `-1` + When the remaining available storage in `tmp-storage-path` is lower than the value defined by `tmp-storage-quota`, the TiDB server reports an error when it is started, and exits. -### `oom-action` - -- Specifies what operation TiDB performs when a single SQL statement exceeds the memory quota specified by `mem-quota-query` and cannot be spilled over to disk. -- Default value: `"cancel"` (In TiDB v4.0.2 and earlier versions, the default value is `"log"`) -- The valid options are `"log"` and `"cancel"`. When `oom-action="log"`, it prints the log only. When `oom-action="cancel"`, it cancels the operation and outputs the log. - -### `lower-case-table-names` - -- Configures the value of the `lower_case_table_names` system variable. -- Default value: `2` -- For details, see the [MySQL description](https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_lower_case_table_names) of this variable. - - > **Note:** - > - > Currently, TiDB only supports setting the value of this option to `2`. This means it is case-sensitive when you save a table name, but case-insensitive when you compare table names. The comparison is based on the lower case. - ### `lease` + The timeout of the DDL lease. @@ -236,12 +212,6 @@ Configuration items related to log. - Default value: `10000` - When the number of query rows (including the intermediate results based on statistics) is larger than this value, it is an `expensive` operation and outputs log with the `[EXPENSIVE_QUERY]` prefix. -### `query-log-max-len` - -- The maximum length of SQL output. -- Default value: `4096` -- When the length of the statement is longer than `query-log-max-len`, the statement is truncated to output. - ## log.file Configuration items related to log files. @@ -275,11 +245,6 @@ Configuration items related to log files. Configuration items related to security. -### `require-secure-transport` - -- Determines whether to require the client to use the secure mode for data transport. -- Default value: `false` - ### `enable-sem` - Enables the Security Enhanced Mode (SEM). @@ -372,12 +337,6 @@ Configuration items related to performance. - Unit: Millisecond - The transaction that holds locks longer than this time can only be committed or rolled back. The commit might not be successful. -### `committer-concurrency` - -+ The number of goroutines for requests related to executing commit in the commit phase of the single transaction. -+ Default value: `128` -+ If the transaction to commit is too large, the waiting time for the flow control queue when the transaction is committed might be too long. In this situation, you can increase the configuration value to speed up the commit. - ### `stmt-count-limit` - The maximum number of statements allowed in a single TiDB transaction. @@ -427,11 +386,6 @@ Configuration items related to performance. - `mysql.stats_histograms`/`mysql.stats_buckets` and `mysql.stats_top_n`: TiDB no longer automatically analyzes and proactively updates statistics. - `mysql.stats_feedback`: TiDB no longer updates the statistics of the tables and indexes according to a part of statistics returned by the queried data. -### `run-auto-analyze` - -- Determines whether TiDB executes automatic analysis. -- Default value: `true` - ### `feedback-probability` - The probability that TiDB collects the feedback statistics of each query. @@ -467,27 +421,6 @@ Configuration items related to performance. + Default value: `false` + This configuration item controls the initial value of [`tidb_enforce_mpp`](/system-variables.md#tidb_enforce_mpp-new-in-v51). For example, when this configuration item is set to `true`, the default value of `tidb_enforce_mpp` is `ON`. -## prepared-plan-cache - -The [`plan cache`](/sql-prepared-plan-cache.md) configuration of the `PREPARE` statement. - -### `enabled` - -- Determines whether to enable Plan Cache of the `PREPARE` statement. -- Default value: `false` - -### `capacity` - -- The number of cached statements. -- Default value: `1000` -- The type is `UINT`. Values less than `0` are converted to large integers. - -### `memory-guard-ratio` - -- It is used to prevent `performance.max-memory` from being exceeded. When `max-memory * (1 - prepared-plan-cache.memory-guard-ratio)` is exceeded, the elements in the LRU are removed. -- Default value: `0.1` -- The minimum value is `0`; the maximum value is `1`. - ## opentracing Configuration items related to opentracing. @@ -717,24 +650,4 @@ The `experimental` section, introduced in v3.1.0, describes the configurations r ### `allow-expression-index` New in v4.0.0 + Controls whether an expression index can be created. Since TiDB v5.2.0, if the function in an expression is safe, you can create an expression index directly based on this function without enabling this configuration. If you want to create an expression index based on other functions, you can enable this configuration, but correctness issues might exist. By querying the `tidb_allow_function_for_expression_index` variable, you can get the functions that are safe to be directly used for creating an expression. -+ Default value: `false` - -### `stats-load-concurrency` New in v5.4.0 - -> **WARNING:** -> -> Currently, synchronously loading statistics is an experimental feature. It is not recommended that you use it in production environments. - -+ The maximum number of columns that the TiDB synchronously loading statistics feature can process concurrently. -+ Default value: `5` -+ Currently, the valid value range is `[1, 128]`. - -### `stats-load-queue-size` New in v5.4.0 - -> **WARNING:** -> -> Currently, synchronously loading statistics is an experimental feature. It is not recommended that you use it in production environments. - -+ The maximum number of column requests that the TiDB synchronously loading statistics feature can cache. -+ Default value: `1000` -+ Currently, the valid value range is `[1, 100000]`. ++ Default value: `false` \ No newline at end of file From e3c94429dff1070ce2cae51b927e715c67ca45f4 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 25 May 2022 08:38:34 -0600 Subject: [PATCH 2/8] Address PR feedback --- identify-expensive-queries.md | 2 +- releases/release-5.0.0.md | 2 +- releases/release-5.0.1.md | 2 +- releases/release-5.1.0.md | 2 +- sql-prepared-plan-cache.md | 2 +- system-variables.md | 22 ++++++++++++---------- tidb-configuration-file.md | 20 ++++++++++++++++++++ 7 files changed, 37 insertions(+), 15 deletions(-) diff --git a/identify-expensive-queries.md b/identify-expensive-queries.md index e68f346f87698..b9ea61935f970 100644 --- a/identify-expensive-queries.md +++ b/identify-expensive-queries.md @@ -5,7 +5,7 @@ aliases: ['/docs/dev/identify-expensive-queries/','/docs/dev/how-to/maintain/ide # Identify Expensive Queries -TiDB allows you to identify expensive queries during SQL execution, so you can diagnose and improve the performance of SQL execution. Specifically, TiDB prints the information about statements whose execution time exceeds [`tidb_expensive_query_time_threshold`](/system-variables.md#tidb_expensive_query_time_threshold) (60 seconds by default) or memory usage exceeds [`mem-quota-query`](/tidb-configuration-file.md#mem-quota-query) (1 GB by default) to the [tidb-server log file](/tidb-configuration-file.md#logfile) ("tidb.log" by default). +TiDB allows you to identify expensive queries during SQL execution, so you can diagnose and improve the performance of SQL execution. Specifically, TiDB prints the information about statements whose execution time exceeds [`tidb_expensive_query_time_threshold`](/system-variables.md#tidb_expensive_query_time_threshold) (60 seconds by default) or memory usage exceeds [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) (1 GB by default) to the [tidb-server log file](/tidb-configuration-file.md#logfile) ("tidb.log" by default). > **Note:** > diff --git a/releases/release-5.0.0.md b/releases/release-5.0.0.md index 4720885fff4b4..b1ecdb83f072c 100644 --- a/releases/release-5.0.0.md +++ b/releases/release-5.0.0.md @@ -368,7 +368,7 @@ This feature is enabled by default. You can disable it by running the `pd-ctl co ### Optimize the memory management module to reduce system OOM risks -Track the memory usage of aggregate functions. This feature is enabled by default. When SQL statements with aggregate functions are executed, if the total memory usage of the current query exceeds the threshold set by [`mem-quota-query`](/tidb-configuration-file.md#mem-quota-query), the system automatically performs operations defined by [`oom-action`](/tidb-configuration-file.md#oom-action). +Track the memory usage of aggregate functions. This feature is enabled by default. When SQL statements with aggregate functions are executed, if the total memory usage of the current query exceeds the threshold set by `mem-quota-query`, the system automatically performs operations defined by `oom-action`. ### Improve the system availability during network partition diff --git a/releases/release-5.0.1.md b/releases/release-5.0.1.md index 67de4fb9f7e88..69b1dba224d93 100644 --- a/releases/release-5.0.1.md +++ b/releases/release-5.0.1.md @@ -10,7 +10,7 @@ TiDB version: 5.0.1 ## Compatibility change -- The default value of the [`committer-concurrency`](/tidb-configuration-file.md#committer-concurrency) configuration item is changed from `16` to `128`. +- The default value of the `committer-concurrency` configuration item is changed from `16` to `128`. ## Improvements diff --git a/releases/release-5.1.0.md b/releases/release-5.1.0.md index 634c033f0b853..010832c424231 100644 --- a/releases/release-5.1.0.md +++ b/releases/release-5.1.0.md @@ -40,7 +40,7 @@ In v5.1, the key new features or improvements are as follows: | Configuration file | Configuration item | Change type | Description | |:----------|:-----------|:-----------|:-----------| | TiDB configuration file | [`security.enable-sem`](/tidb-configuration-file.md#enable-sem) | Newly added | Controls whether to enable the Security Enhanced Mode (SEM). The default value of this configuration item is `false`, which means the SEM is disabled. | -| TiDB configuration file | [`performance.committer-concurrency`](/tidb-configuration-file.md#committer-concurrency) | Modified | Controls the concurrency number for requests related to commit operations in the commit phase of a single transaction. The default value is changed from `16` to `128`. | +| TiDB configuration file | `performance.committer-concurrency` | Modified | Controls the concurrency number for requests related to commit operations in the commit phase of a single transaction. The default value is changed from `16` to `128`. | | TiDB configuration file | [`performance.tcp-no-delay`](/tidb-configuration-file.md#tcp-no-delay) | Newly added | Determines whether to enable TCP_NODELAY at the TCP layer. The default value is `true`, which means TCP_NODELAY is enabled. | | TiDB configuration file | [`performance.enforce-mpp`](/tidb-configuration-file.md#enforce-mpp) | Newly added | Controls whether TiDB ignores cost estimates of Optimizer at the instance level and enforces the MPP mode. The default value is `false`. This configuration item controls the initial value of the system variable [`tidb_enforce_mpp`](/system-variables.md#tidb_enforce_mpp-new-in-v51). | | TiDB configuration file | [`pessimistic-txn.deadlock-history-capacity`](/tidb-configuration-file.md#deadlock-history-capacity) | Newly added | Sets the maximum number of deadlock events that can be recorded in the [`INFORMATION_SCHEMA.DEADLOCKS`](/information-schema/information-schema-deadlocks.md) table of a single TiDB server. The default value is `10`. | diff --git a/sql-prepared-plan-cache.md b/sql-prepared-plan-cache.md index 90f676abae6a3..cebfd810f1e63 100644 --- a/sql-prepared-plan-cache.md +++ b/sql-prepared-plan-cache.md @@ -53,7 +53,7 @@ There are several points worth noting about execution plan caching and query per - Considering that the parameters of `Execute` are different, the execution plan cache prohibits some aggressive query optimization methods that are closely related to specific parameter values to ensure adaptability. This causes that the query plan may not be optimal for certain parameter values. For example, the filter condition of the query is `where a > ? And a < ?`, the parameters of the first `Execute` statement are `2` and `1` respectively. Considering that these two parameters maybe be `1` and `2` in the next execution time, the optimizer does not generate the optimal `TableDual` execution plan that is specific to current parameter values; - If cache invalidation and elimination are not considered, an execution plan cache is applied to various parameter values, which in theory also results in non-optimal execution plans for certain values. For example, if the filter condition is `where a < ?` and the parameter value used for the first execution is `1`, then the optimizer generates the optimal `IndexScan` execution plan and puts it into the cache. In the subsequent executions, if the value becomes `10000`, the `TableScan` plan might be the better one. But due to the execution plan cache, the previously generated `IndexScan` is used for execution. Therefore, the execution plan cache is more suitable for application scenarios where the query is simple (the ratio of compilation is high) and the execution plan is relatively fixed. -Currently, the execution plan cache is disabled by default. You can enable this feature by setting the system variable `tidb_enable_prepared_plan_cache` to `ON`. +Since v6.1.0 the execution plan cache is enabled by default. You can control prepared plan cache via the system variable `tidb_enable_prepared_plan_cache`. > **Note：** > diff --git a/system-variables.md b/system-variables.md index 485ca2516739a..722b90ae1f2a2 100644 --- a/system-variables.md +++ b/system-variables.md @@ -363,7 +363,7 @@ This variable is an alias for `last_insert_id`. - Default value: `OFF` - This variable ensures that all connections to TiDB are either on a local socket, or using TLS. See [Enable TLS between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) for additional details. - Setting this variable to `ON` requires you to connect to TiDB from a session that has TLS enabled. This helps prevent lock-out scenarios when TLS is not configured correctly. -- This setting was previously a tidb.toml option (`security.require-secure-transport`), but changed to a system variable starting from TiDB 6.1. +- This setting was previously a `tidb.toml` option (`security.require-secure-transport`), but changed to a system variable starting from TiDB 6.1.0. ### skip_name_resolve New in v5.2.0 @@ -602,7 +602,7 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a - Range: `[1, 10000]` - The number of goroutines for requests related to executing commit in the commit phase of the single transaction. - If the transaction to commit is too large, the waiting time for the flow control queue when the transaction is committed might be too long. In this situation, you can increase the configuration value to speed up the commit. -- This setting was previously a tidb.toml option (`performance.committer-concurrency`), but changed to a system variable starting from TiDB 6.1. +- This setting was previously a `tidb.toml` option (`performance.committer-concurrency`), but changed to a system variable starting from TiDB 6.1.0. ### tidb_config @@ -766,7 +766,7 @@ Constraint checking is always performed in place for pessimistic transactions (d - Persists to cluster: Yes - Default value: `ON` - Determines whether TiDB automatically updates table statistics as a background operation -- This setting was previously a tidb.toml option (`performance.run-auto-analyze`), but changed to a system variable starting from TiDB 6.1. +- This setting was previously a `tidb.toml` option (`performance.run-auto-analyze`), but changed to a system variable starting from TiDB 6.1.0. ### tidb_enable_auto_increment_in_generated @@ -781,7 +781,7 @@ Constraint checking is always performed in place for pessimistic transactions (d - Persists to cluster: Yes - Default value: `OFF` - Determines if TiDB permits non-transactional 'batched' statements. Only the value of `OFF` can be considered safe, as batch DML does not provide ACID guarantees. -- This setting was previously a tidb.toml option (`enable-batch-dml`), but changed to a system variable starting from TiDB 6.1. +- This setting was previously a `tidb.toml` option (`enable-batch-dml`), but changed to a system variable starting from TiDB 6.1.0. ### tidb_enable_cascades_planner @@ -932,8 +932,8 @@ Constraint checking is always performed in place for pessimistic transactions (d - Scope: GLOBAL - Persists to cluster: Yes - Default value: `ON` -- Determines whether to enable Plan Cache of the `PREPARE` statement. -- This setting was previously a tidb.toml option (`prepared-plan-cache.enabled`), but changed to a system variable starting from TiDB 6.1. +- Determines whether to enable [Prepared Plan Cache](/sql-prepared-plan-cache.md). When it is enabled, the execution plans of `Prepare` and `Execute` are cached so that the subsequent executions skip optimizing the execution plans, which brings performance improvement. +- This setting was previously a `tidb.toml` option (`prepared-plan-cache.enabled`), but changed to a system variable starting from TiDB 6.1.0. ### tidb_enable_pseudo_for_outdated_stats New in v5.3.0 @@ -1378,7 +1378,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Possible values: `CANCEL`, `LOG` - Specifies what operation TiDB performs when a single SQL statement exceeds the memory quota specified by `tidb_mem_quota_query` and cannot be spilled over to disk. See [TiDB Memory Control](/configure-memory-usage.md) for details. - The default value is `CANCEL`, but in TiDB v4.0.2 and earlier versions, the default value is `LOG`. -- This setting was previously a tidb.toml option (`oom-action`), but changed to a system variable starting from TiDB 6.1. +- This setting was previously a `tidb.toml` option (`oom-action`), but changed to a system variable starting from TiDB 6.1.0. ### tidb_mem_quota_apply_cache New in v5.0 @@ -1408,7 +1408,8 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Range: `[-1, 9223372036854775807]` - Unit: Bytes - This variable is used to set the threshold value of memory quota for a query. -- If the memory quota of a query during execution exceeds the threshold value, TiDB performs the operation designated by the OOMAction option in the configuration file. The initial value of this variable is configured by [`mem-quota-query`](/tidb-configuration-file.md#mem-quota-query). +- If the memory quota of a query during execution exceeds the threshold value, TiDB performs the operation designated by `tidb_mem_oom_action`. +- This setting was previously session scoped and used the value of `mem-quota-query` from `tidb.toml` as an initial value. Starting from v6.1.0 `tidb_mem_quota_query` is now a `SESSION | GLOBAL` scoped variable. ### tidb_memory_usage_alarm_ratio @@ -1639,8 +1640,9 @@ explain select * from t where age=5; - Scope: GLOBAL - Persists to cluster: Yes - Default value: `0.1` +- Range: `[0, 1]` - This setting is used to prevent the tidb.toml option `performance.max-memory` from being exceeded. When `max-memory` * (1 - `tidb_prepared_plan_cache_memory_guard_ratio`) is exceeded, the elements in the LRU are removed. -- This setting was previously a tidb.toml option (`prepared-plan-cache.memory-guard-ratio`), but changed to a system variable starting from TiDB 6.1. +- This setting was previously a `tidb.toml` option (`prepared-plan-cache.memory-guard-ratio`), but changed to a system variable starting from TiDB 6.1.0. ### tidb_prepared_plan_cache_size New in v6.1.0 @@ -1649,7 +1651,7 @@ explain select * from t where age=5; - Default value: `100` - Range: `[1, 100000]` - The maximum number of statements that can be cached in the prepared plan cache. -- This setting was previously a tidb.toml option (`prepared-plan-cache.capacity`), but changed to a system variable starting from TiDB 6.1. +- This setting was previously a `tidb.toml` option (`prepared-plan-cache.capacity`), but changed to a system variable starting from TiDB 6.1.0. ### tidb_projection_concurrency diff --git a/tidb-configuration-file.md b/tidb-configuration-file.md index eb3619c6092a2..c20c895bce19c 100644 --- a/tidb-configuration-file.md +++ b/tidb-configuration-file.md @@ -421,6 +421,26 @@ Configuration items related to performance. + Default value: `false` + This configuration item controls the initial value of [`tidb_enforce_mpp`](/system-variables.md#tidb_enforce_mpp-new-in-v51). For example, when this configuration item is set to `true`, the default value of `tidb_enforce_mpp` is `ON`. +### `stats-load-concurrency` New in v5.4.0 + +> **WARNING:** +> +> Currently, synchronously loading statistics is an experimental feature. It is not recommended that you use it in production environments. + ++ The maximum number of columns that the TiDB synchronously loading statistics feature can process concurrently. ++ Default value: `5` ++ Currently, the valid value range is `[1, 128]`. + +### `stats-load-queue-size` New in v5.4.0 + +> **WARNING:** +> +> Currently, synchronously loading statistics is an experimental feature. It is not recommended that you use it in production environments. + ++ The maximum number of column requests that the TiDB synchronously loading statistics feature can cache. ++ Default value: `1000` ++ Currently, the valid value range is `[1, 100000]`. + ## opentracing Configuration items related to opentracing. From 518bc91fde3ab8f220f24672ed3d6d694a1136c2 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 25 May 2022 09:21:23 -0600 Subject: [PATCH 3/8] Fix line endings --- configure-memory-usage.md | 6 +- releases/release-5.0.0.md | 932 +++++++++++++++++++------------------- 2 files changed, 468 insertions(+), 470 deletions(-) diff --git a/configure-memory-usage.md b/configure-memory-usage.md index 0b104275bae11..5301458e5d5fc 100644 --- a/configure-memory-usage.md +++ b/configure-memory-usage.md @@ -6,12 +6,10 @@ aliases: ['/docs/dev/configure-memory-usage/','/docs/dev/how-to/configure/memory # TiDB Memory Control -Currently, TiDB can track the memory quota of a single SQL query and take actions to prevent OOM (out of memory) or troubleshoot OOM when the memory usage exceeds a specific threshold value. - -The system variable `tidb_mem_oom_action` specifies the action to take when a query reaches the memory limit: +Currently, TiDB can track the memory quota of a single SQL query and take actions to prevent OOM (out of memory) or troubleshoot OOM when the memory usage exceeds a specific threshold value. The system variable `tidb_mem_oom_action` specifies the action to take when a query reaches the memory limit: - A value of `LOG` means that queries will continue to execute when the `tidb_mem_quota_query` limit is reached, but TiDB will print an entry to the log. -- A value of `CANCEL` means TiDB stops executing the SQL query immediately after the `tidb_mem_quota_query` limit is reached, and returns an error to the client. The error information clearly shows the memory usage of each physical execution operator that consumes much memory in the SQL execution process. +- A value of `CANCEL` means TiDB stops executing the SQL query immediately after the `tidb_mem_quota_query` limit is reached, and returns an error to the client. The error information clearly shows the memory usage of each physical execution operator that consumes memory in the SQL execution process. ## Configure the memory quota of a query diff --git a/releases/release-5.0.0.md b/releases/release-5.0.0.md index b1ecdb83f072c..aebaa3b9fcd00 100644 --- a/releases/release-5.0.0.md +++ b/releases/release-5.0.0.md @@ -1,466 +1,466 @@ ---- -title: What's New in TiDB 5.0 ---- - -# What's New in TiDB 5.0 - -Release date: April 7, 2021 - -TiDB version: 5.0.0 - -In v5.0, PingCAP is dedicated to helping enterprises quickly build applications based on TiDB, freeing them from worries about database performance, performance jitter, security, high availability, disaster recovery, troubleshooting SQL performance, and so on. - -In v5.0, the key new features or improvements are as follows: - -+ Introduce Massively Parallel Processing (MPP) architecture through TiFlash nodes, which shares the execution workloads of large join queries among TiFlash nodes. When the MPP mode is enabled, TiDB, based on cost, determines whether to use the MPP framework to perform the calculation. In the MPP mode, the join keys are redistributed through the `Exchange` operation while being calculated, which distributes the calculation pressure to each TiFlash node and speeds up the calculation. According to the benchmark, with the same cluster resource, TiDB 5.0 MPP shows 2 to 3 times of speedup over Greenplum 6.15.0 and Apache Spark 3.1.1, and some queries have 8 times better performance. -+ Introduce the clustered index feature to improve database performance. For example, in the TPC-C tpmC test, the performance of TiDB, with clustered index enabled, improves by 39%. -+ Enable the async commit feature to reduce the write latency. For example, in the 64-thread Sysbench test, the average latency of updating indexes, with async commit enabled, is reduced by 41.7%, from 12.04 ms to 7.01 ms. -+ Reduce jitters. This is achieved by improving the optimizer stability and by limiting system tasks' usages of I/O, network, CPU, and memory resources. For example, in the 8-hour performance test, the standard deviation of TPC-C tpmC does not exceed 2%. -+ Enhance system stability by improving scheduling and by keeping execution plans stable as much as possible. -+ Introduces Raft Joint Consensus algorithm, which ensures the system availability during the Region membership change. -+ Optimize `EXPLAIN` features and invisible index, which helps Database Administrators (DBAs) debug SQL statements more efficiently. -+ Guarantee reliability for enterprise data. You can back up data from TiDB to Amazon S3 storage and Google Cloud GCS, or restore data from these cloud storage platforms. -+ Improve performance of data import from or data export to Amazon S3 storage or TiDB/MySQL, which helps enterprises quickly build applications on the cloud. For example, in the TPC-C test, the performance of importing 1 TiB data improves by 40%, from 254 GiB/h to 366 GiB/h. - -## Compatibility changes - -### System variables - -+ Add the [`tidb_executor_concurrency`](/system-variables.md#tidb_executor_concurrency-new-in-v50) system variable to control the concurrency of multiple operators. The previous `tidb_*_concurrency` settings (such as `tidb_projection_concurrency`) still take effect but with a warning when you use them. -+ Add the [`tidb_skip_ascii_check`](/system-variables.md#tidb_skip_ascii_check-new-in-v50) system variable to specify whether to skip the ASCII validation check when the ASCII character set is written. This default value is `OFF`. -+ Add the [`tidb_enable_strict_double_type_check`](/system-variables.md#tidb_enable_strict_double_type_check-new-in-v50) system variable to determine whether the syntax like `double(N)` can be defined in the table schema. This default value is `OFF`. -+ Change the default value of [`tidb_dml_batch_size`](/system-variables.md#tidb_dml_batch_size) from `20000` to `0`. This means that batch DML statements are no longer used by default in `LOAD`/`INSERT INTO SELECT ...`. Instead, large transactions are used to comply with the strict ACID semantics. - - > **Note:** - > - > The scope of the variable is changed from session to global, and the default value is changed from `20000` to `0`. If the application relies on the original default value, you need to use the `set global` statement to modify the variable to the original value after the upgrade. - -+ Control temporary tables’ syntax compatibility using the [`tidb_enable_noop_functions`](/system-variables.md#tidb_enable_noop_functions-new-in-v40) system variable. When this variable value is `OFF`, the `CREATE TEMPORARY TABLE` syntax returns an error. -+ Add the following system variables to directly control the garbage collection-related parameters: - - [`tidb_gc_concurrency`](/system-variables.md#tidb_gc_concurrency-new-in-v50) - - [`tidb_gc_enable`](/system-variables.md#tidb_gc_enable-new-in-v50) - - [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) - - [`tidb_gc_run_interval`](/system-variables.md#tidb_gc_run_interval-new-in-v50) - - [`tidb_gc_scan_lock_mode`](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) -+ Change the default value of [`enable-joint-consensus`](/pd-configuration-file.md#enable-joint-consensus-new-in-v50) from `false` to `true`, which enables the Joint Consensus feature by default. -+ Change the value of [`tidb_enable_amend_pessimistic_txn`](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407) from `0` or `1` to `ON` or `OFF`. -+ Change the default value of [`tidb_enable_clustered_index`](/system-variables.md#tidb_enable_clustered_index-new-in-v50) from `OFF` to `INT_ONLY` with the following new meanings: - + `ON`: clustered index is enabled. Adding or deleting non-clustered indexes is supported. - + `OFF`: clustered index is disabled. Adding or deleting non-clustered indexes is supported. - + `INT_ONLY`: the default value. The behavior is consistent with that before v5.0. You can control whether to enable clustered index for the INT type together with `alter-primary-key = false`. - - > **Note:** - > - > The `INT_ONLY` value of `tidb_enable_clustered_index` in 5.0 GA has the same meaning as the `OFF` value in 5.0 RC. After upgrading from a 5.0 RC cluster with the `OFF` setting to 5.0 GA, it will be displayed as `INT_ONLY`. - -### Configuration file parameters - -+ Add the [`index-limit`](/tidb-configuration-file.md#index-limit-new-in-v50) configuration item for TiDB. Its value defaults to `64` and ranges between `[64,512]`. A MySQL table supports 64 indexes at most. If its value exceeds the default setting and more than 64 indexes are created for a table, when the table schema is re-imported into MySQL, an error will be reported. -+ Add the [`enable-enum-length-limit`](/tidb-configuration-file.md#enable-enum-length-limit-new-in-v50) configuration item for TiDB to be compatible and consistent with MySQL’s ENUM/SET length (ENUM length < 255). The default value is `true`. -+ Replace the `pessimistic-txn.enable` configuration item with the [`tidb_txn_mode`](/system-variables.md#tidb_txn_mode) environment variable. -+ Replace the `performance.max-memory` configuration item with [`performance.server-memory-quota`](/tidb-configuration-file.md#server-memory-quota-new-in-v409) -+ Replace the `tikv-client.copr-cache.enable` configuration item with [`tikv-client.copr-cache.capacity-mb`](/tidb-configuration-file.md#capacity-mb). If the item’s value is `0.0`, this feature is disabled. If the item’s value is greater than `0.0`, this feature is enabled. Its default value is `1000.0`. -+ Replace the `rocksdb.auto-tuned` configuration item with [`rocksdb.rate-limiter-auto-tuned`](/tikv-configuration-file.md#rate-limiter-auto-tuned-new-in-v50). -+ Delete the `raftstore.sync-log` configuration item. By default, written data is forcibly spilled to the disk. Before v5.0, you can explicitly disable `raftstore.sync-log`. Since v5.0, the configuration value is forcibly set to `true`. -+ Change the default value of the `gc.enable-compaction-filter` configuration item from `false` to `true`. -+ Change the default value of the `enable-cross-table-merge` configuration item from `false` to `true`. -+ Change the default value of the [`rate-limiter-auto-tuned`](/tikv-configuration-file.md#rate-limiter-auto-tuned-new-in-v50) configuration item from `false` to `true`. - -### Others - -+ Before the upgrade, check the value of the TiDB configuration [`feedback-probability`](/tidb-configuration-file.md#feedback-probability). If the value is not 0, the "panic in the recoverable goroutine" error will occur after the upgrade, but this error does not affect the upgrade. -+ Forbid conversion between `VARCHAR` type and `CHAR` type during the column type change to avoid data correctness issues. - -## New features - -### SQL - -#### List partitioning (**Experimental**) - -[User document](/partitioned-table.md#list-partitioning) - -With the list partitioning feature, you can effectively query and maintain tables with a large amount of data. - -With this feature enabled, partitions and how data is distributed among partitions are defined according to the `PARTITION BY LIST(expr) PARTITION part_name VALUES IN (...)` expression. The partitioned tables’ data set supports at most 1024 distinct integer values. You can define the values using the `PARTITION ... VALUES IN (...)` clause. - -To enable list partitioning, set the session variable [`tidb_enable_list_partition`](/system-variables.md#tidb_enable_list_partition-new-in-v50) to `ON`. - -#### List COLUMNS partitioning (**Experimental**) - -[User document](/partitioned-table.md#list-columns-partitioning) - -List COLUMNS partitioning is a variant of list partitioning. You can use multiple columns as partition keys. Besides the integer data type, you can also use the columns in the string, `DATE`, and `DATETIME` data types as partition columns. - -To enable List COLUMNS partitioning, set the session variable [`tidb_enable_list_partition`](/system-variables.md#tidb_enable_list_partition-new-in-v50) to `ON`. - -#### Invisible indexes - -[User document](/sql-statements/sql-statement-alter-index.md), [#9246](https://github.com/pingcap/tidb/issues/9246) - -When you tune performance or select optimal indexes, you can set an index to be `Visible` or `Invisible` by using SQL statements. This setting can avoid performing resource-consuming operations, such as `DROP INDEX` and `ADD INDEX`. - -To modify the visibility of an index, use the `ALTER INDEX` statement. After the modification, the optimizer decides whether to add this index to the index list based on the index visibility. - -#### `EXCEPT` and `INTERSECT` operators - -[User document](/functions-and-operators/set-operators.md), [#18031](https://github.com/pingcap/tidb/issues/18031) - -The `INTERSECT` operator is a set operator, which returns the intersection of the result sets of two or more queries. To some extent, it is an alternative to the `Inner Join` operator. - -The `EXCEPT` operator is a set operator, which combines the result sets of two queries and returns elements that are in the first query result but not in the second. - -### Transaction - -[User document](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407), [#18005](https://github.com/pingcap/tidb/issues/18005) - -In the pessimistic transaction mode, if the tables involved in a transaction contain concurrent DDL operations or `SCHEMA VERSION` changes, the system automatically updates the transaction's `SCHEMA VERSION` to the latest to ensure the successful transaction commit, and to avoid that the client receives the `Information schema is changed` error when the transaction is interrupted by DDL operations or `SCHEMA VERSION` changes. - -This feature is disabled by default. To enable the feature, modify the value of [`tidb_enable_amend_pessimistic_txn`](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407) system variable. This feature is introduced in v4.0.7 and has the following issues fixed in v5.0: - -+ The compatibility issue that occurs when TiDB Binlog executes `Add Column` operations -+ The data inconsistency issue that occurs when using the feature together with the unique index -+ The data inconsistency issue that occurs when using the feature together with the added index - -Currently, this feature still has the following incompatibility issues: - -+ Transaction’s semantics might change when there are concurrent transactions -+ Known compatibility issue that occurs when using the feature together with TiDB Binlog -+ Incompatibility with `Change Column` - -### Character set and collation - -- Support the `utf8mb4_unicode_ci` and `utf8_unicode_ci` collations. [User document](/character-set-and-collation.md#new-framework-for-collations), [#17596](https://github.com/pingcap/tidb/issues/17596) -- Support the case-insensitive comparison sort for collations - -### Security - -[User document](/log-redaction.md), [#18566](https://github.com/pingcap/tidb/issues/18566) - -To meet security compliance requirements (such as *General Data Protection Regulation*, or GDPR), the system supports desensitizing information (such as ID and credit card number) in the output error messages and logs, which can avoid leaking sensitive information. - -TiDB supports desensitizing the output log information. To enable this feature, use the following switches: - -+ The global variable [`tidb_redact_log`](/system-variables.md#tidb_redact_log). Its default value is `0`, which means that desensitization is disabled. To enable desensitization for tidb-server logs, set the variable value to `1`. -+ The configuration item `security.redact-info-log`. Its default value is `false`, which means that desensitization is disabled. To enable desensitization for tikv-server logs, set the variable value to `true`. -+ The configuration item `security.redact-info-log`. Its default value is `false`, which means that desensitization is disabled. To enable desensitization for pd-server logs, set the variable value to `true`. -+ The configuration item `security.redact_info_log` for tiflash-server and `security.redact-info-log` for tiflash-learner. Their default values are both `false`, which means that desensitization is disabled. To enable desensitization for tiflash-server and tiflash-learner logs, set the values of both variables to `true`. - -This feature is introduced in v5.0. To use the feature, enable the system variable and all configuration items above. - -## Performance optimization - -### MPP architecture - -[User document](/tiflash/use-tiflash.md) - -TiDB introduces the MPP architecture through TiFlash nodes. This architecture allows multiple TiFlash nodes to share the execution workload of large join queries. - -When the MPP mode is on, TiDB determines whether to send a query to the MPP engine for computation based on the calculation cost. In the MPP mode, TiDB distributes the computation of table joins to each running TiFlash node by redistributing the join key during data calculation (`Exchange` operation), and thus accelerates the calculation. Furthermore, with the aggregation computing feature that TiFlash has already supported, TiDB can pushdown the computation of a query to the TiFlash MPP cluster. Then the distributed environment can help accelerate the entire execution process and dramatically increase the speed of analytic queries. - -In the TPC-H 100 benchmark test, TiFlash MPP delivers significant processing speed over analytic engines of traditional analytic databases and SQL on Hadoop. With this architecture, you can perform large-scale analytic queries directly on the latest transaction data, with a higher performance than traditional offline analytic solutions. According to the benchmark, with the same cluster resource, TiDB 5.0 MPP shows 2 to 3 times of speedup over Greenplum 6.15.0 and Apache Spark 3.1.1, and some queries have 8 times better performance. - -Currently, the main features that the MPP mode does not support are as follows (For details, refer to [Use TiFlash](/tiflash/use-tiflash.md)): - -+ Table partitioning -+ Window Function -+ Collation -+ Some built-in functions -+ Reading data from TiKV -+ OOM spill -+ Union -+ Full Outer Join - -### Clustered index - -[User document](/clustered-indexes.md), [#4841](https://github.com/pingcap/tidb/issues/4841) - -When you are designing table structures or analyzing database behaviors, it is recommended to use the clustered index feature if you find that some columns with primary keys are often grouped and sorted, queries on these columns often return a certain range of data or a small amount of data with different values, and the corresponding data does not cause read or write hotspot issues. - -Clustered indexes, also known as _index-organized tables_ in some database management systems, is a storage structure associated with the data of a table. When creating a clustered index, you can specify one or more columns from the table as the keys for the index. TiDB stores these keys in a specific structure, which allows TiDB to quickly and efficiently find the rows associated with the keys, thus improves the performance of querying and writing data. - -When the clustered index feature is enabled, the TiDB performance improves significantly (for example in the TPC-C tpmC test, the performance of TiDB, with clustered index enabled, improves by 39%) in the following cases: - -+ When data is inserted, the clustered index reduces one write of the index data from the network. -+ When a query with an equivalent condition only involves the primary key, the clustered index reduces one read of index data from the network. -+ When a query with a range condition only involves the primary key, the clustered index reduces multiple reads of index data from the network. -+ When a query with an equivalent or range condition involves the primary key prefix, the clustered index reduces multiple reads of index data from the network. - -Each table can either use a clustered or non-clustered index to sort and store data. The differences of these two storage structures are as follows: - -+ When creating a clustered index, you can specify one or more columns in the table as the key value of the index. A clustered index sorts and stores the data of a table according to the key value. Each table can have only one clustered index. If a table has a clustered index, it is called a clustered index table. Otherwise, it is called a non-clustered index table. -+ When you create a non-clustered index, the data in the table is stored in an unordered structure. You do not need to explicitly specify the key value of the non-clustered index, because TiDB automatically assigns a unique ROWID to each row of data. During a query, the ROWID is used to locate the corresponding row. Because there are at least two network I/O operations when you query or insert data, the performance is degraded compared with clustered indexes. - -When table data is modified, the database system automatically maintains clustered indexes and non-clustered indexes for you. - -All primary keys are created as non-clustered indexes by default. You can create a primary key as a clustered index or non-clustered index in either of the following two ways: - -+ Specify the keyword `CLUSTERED | NONCLUSTERED` in the statement when creating a table, then the system creates the table in the specified way. The syntax is as follows: - -```sql -CREATE TABLE `t` (`a` VARCHAR(255), `b` INT, PRIMARY KEY (`a`, `b`) CLUSTERED); -``` - -Or - -```sql -CREATE TABLE `t` (`a` VARCHAR(255) PRIMARY KEY CLUSTERED, `b` INT); -``` - -You can execute the statement `SHOW INDEX FROM tbl-name` to query whether a table has a clustered index. - -+ Configure the system variable `tidb_enable_clustered_index` to control the clustered index feature. Supported values are `ON`, `OFF`, and `INT_ONLY`. - + `ON`: Indicates that the clustered index feature is enabled for all types of primary keys. Adding and dropping non-clustered indexes are supported. - + `OFF`: Indicates that the clustered index feature is disabled for all types of primary keys. Adding and dropping non-clustered indexes are supported. - + `INT_ONLY`: The default value. If the variable is set to `INT_ONLY` and `alter-primary-key` is set to `false`, the primary keys which consist of single integer columns are created as clustered indexes by default. The behavior is consistent with that of TiDB v5.0 and earlier versions. - -If a `CREATE TABLE` statement contains the keyword `CLUSTERED | NONCLUSTERED`, the statement overrides the configuration of the system variable and the configuration item. - -You are recommended to use the clustered index feature by specifying the keyword `CLUSTERED | NONCLUSTERED` in statements. In this way, it is more flexible for TiDB to use all data types of clustered and non-clustered indexes in the system at the same time as required. - -It is not recommended to use `tidb_enable_clustered_index = INT_ONLY`, because `INT_ONLY` is temporarily used to make this feature compatible and will be deprecated in the future. - -Limitations for the clustered index are as follows: - -+ Mutual conversion between clustered indexes and non-clustered indexes is not supported. -+ Dropping clustered indexes is not supported. -+ Adding, dropping, and altering clustered indexes using `ALTER TABLE` statements are not supported. -+ Reorganizing and re-creating a clustered index is not supported. -+ Enabling or disabling indexes is not supported, which means the invisible index feature is not effective for clustered indexes. -+ Creating a `UNIQUE KEY` as a clustered index is not supported. -+ Using the clustered index feature together with TiDB Binlog is not supported. After TiDB Binlog is enabled, TiDB only supports creating a single integer primary key as a clustered index. TiDB Binlog does not replicate data changes of existing tables with clustered indexes to the downstream. -+ Using the clustered index feature together with the attributes `SHARD_ROW_ID_BITS` and `PRE_SPLIT_REGIONS` is not supported. -+ If the cluster is upgraded to a later version then rolls back, you need to downgrade newly-added tables by exporting table data before the rollback and importing the data after the rollback. Other tables are not affected. - -### Async Commit - -[User document](/system-variables.md#tidb_enable_async_commit-new-in-v50), [#8316](https://github.com/tikv/tikv/issues/8316) - -The client of the database will wait for the database system to complete the transaction commit in two phases (2PC) synchronously. The transaction returns the result to the client after the first phase commit is successful, and the system executes the second phase commit operation in the background asynchronously to reduce the transaction commit latency. If the transaction write involves only one Region, the second phase is omitted directly, and the transaction becomes a one-phase commit. - -After the Async Commit feature is enabled, with the same hardware and configuration, when Sysbench is set to test the Update index with 64 threads, the average latency decreases by 41.7% from 12.04ms to 7.01ms. - -When Async Commit feature is enabled, to reduce one network interaction latency and improve the performance of data writes, database application developers are recommended to consider reducing the consistency of transactions from linear consistency to [causal consistency](/transaction-overview.md#causal-consistency). The SQL statement to enable causal consistency is `START TRANSACTION WITH CAUSAL CONSISTENCY`. - -After the causal consistency is enabled, with the same hardware and configuration, when Sysbench is set to test oltp_write_only with 64 threads, the average latency decreased by 5.6% from 11.86ms to 11.19ms. - -After the consistency of transactions is reduced from the linear consistency to causal consistency, if there is no interdependence between multiple transactions in the application, the transactions do not have a globally consistent order. - -**The Async Commit feature is enabled by default for newly created v5.0 clusters.** - -This feature is disabled by default for clusters upgraded from earlier versions to v5.0. You can enable this feature by executing the `set global tidb_enable_async_commit = ON;` and `set global tidb_enable_1pc = ON;` statements. - -The limitation for the Async Commit feature is as follows: - -+ Direct downgrade is not supported. - -### Enable the Coprocessor cache feature by default - -[User document](/tidb-configuration-file.md#tikv-clientcopr-cache-new-in-v400), [#18028](https://github.com/pingcap/tidb/issues/18028) - -In 5.0 GA, the Coprocessor cache feature is enabled by default. After this feature is enabled, to reduce the latency of reading data, TiDB caches the calculation results of the operators pushed down to tikv-server in tidb-server. - -To disable the Coprocessor cache feature, you can modify the `capacity-mb` configuration item of `tikv-client.copr-cache` to `0.0`. - -### Improve the execution performance of `delete from table where id **Note:** + > + > The scope of the variable is changed from session to global, and the default value is changed from `20000` to `0`. If the application relies on the original default value, you need to use the `set global` statement to modify the variable to the original value after the upgrade. + ++ Control temporary tables’ syntax compatibility using the [`tidb_enable_noop_functions`](/system-variables.md#tidb_enable_noop_functions-new-in-v40) system variable. When this variable value is `OFF`, the `CREATE TEMPORARY TABLE` syntax returns an error. ++ Add the following system variables to directly control the garbage collection-related parameters: + - [`tidb_gc_concurrency`](/system-variables.md#tidb_gc_concurrency-new-in-v50) + - [`tidb_gc_enable`](/system-variables.md#tidb_gc_enable-new-in-v50) + - [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) + - [`tidb_gc_run_interval`](/system-variables.md#tidb_gc_run_interval-new-in-v50) + - [`tidb_gc_scan_lock_mode`](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) ++ Change the default value of [`enable-joint-consensus`](/pd-configuration-file.md#enable-joint-consensus-new-in-v50) from `false` to `true`, which enables the Joint Consensus feature by default. ++ Change the value of [`tidb_enable_amend_pessimistic_txn`](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407) from `0` or `1` to `ON` or `OFF`. ++ Change the default value of [`tidb_enable_clustered_index`](/system-variables.md#tidb_enable_clustered_index-new-in-v50) from `OFF` to `INT_ONLY` with the following new meanings: + + `ON`: clustered index is enabled. Adding or deleting non-clustered indexes is supported. + + `OFF`: clustered index is disabled. Adding or deleting non-clustered indexes is supported. + + `INT_ONLY`: the default value. The behavior is consistent with that before v5.0. You can control whether to enable clustered index for the INT type together with `alter-primary-key = false`. + + > **Note:** + > + > The `INT_ONLY` value of `tidb_enable_clustered_index` in 5.0 GA has the same meaning as the `OFF` value in 5.0 RC. After upgrading from a 5.0 RC cluster with the `OFF` setting to 5.0 GA, it will be displayed as `INT_ONLY`. + +### Configuration file parameters + ++ Add the [`index-limit`](/tidb-configuration-file.md#index-limit-new-in-v50) configuration item for TiDB. Its value defaults to `64` and ranges between `[64,512]`. A MySQL table supports 64 indexes at most. If its value exceeds the default setting and more than 64 indexes are created for a table, when the table schema is re-imported into MySQL, an error will be reported. ++ Add the [`enable-enum-length-limit`](/tidb-configuration-file.md#enable-enum-length-limit-new-in-v50) configuration item for TiDB to be compatible and consistent with MySQL’s ENUM/SET length (ENUM length < 255). The default value is `true`. ++ Replace the `pessimistic-txn.enable` configuration item with the [`tidb_txn_mode`](/system-variables.md#tidb_txn_mode) environment variable. ++ Replace the `performance.max-memory` configuration item with [`performance.server-memory-quota`](/tidb-configuration-file.md#server-memory-quota-new-in-v409) ++ Replace the `tikv-client.copr-cache.enable` configuration item with [`tikv-client.copr-cache.capacity-mb`](/tidb-configuration-file.md#capacity-mb). If the item’s value is `0.0`, this feature is disabled. If the item’s value is greater than `0.0`, this feature is enabled. Its default value is `1000.0`. ++ Replace the `rocksdb.auto-tuned` configuration item with [`rocksdb.rate-limiter-auto-tuned`](/tikv-configuration-file.md#rate-limiter-auto-tuned-new-in-v50). ++ Delete the `raftstore.sync-log` configuration item. By default, written data is forcibly spilled to the disk. Before v5.0, you can explicitly disable `raftstore.sync-log`. Since v5.0, the configuration value is forcibly set to `true`. ++ Change the default value of the `gc.enable-compaction-filter` configuration item from `false` to `true`. ++ Change the default value of the `enable-cross-table-merge` configuration item from `false` to `true`. ++ Change the default value of the [`rate-limiter-auto-tuned`](/tikv-configuration-file.md#rate-limiter-auto-tuned-new-in-v50) configuration item from `false` to `true`. + +### Others + ++ Before the upgrade, check the value of the TiDB configuration [`feedback-probability`](/tidb-configuration-file.md#feedback-probability). If the value is not 0, the "panic in the recoverable goroutine" error will occur after the upgrade, but this error does not affect the upgrade. ++ Forbid conversion between `VARCHAR` type and `CHAR` type during the column type change to avoid data correctness issues. + +## New features + +### SQL + +#### List partitioning (**Experimental**) + +[User document](/partitioned-table.md#list-partitioning) + +With the list partitioning feature, you can effectively query and maintain tables with a large amount of data. + +With this feature enabled, partitions and how data is distributed among partitions are defined according to the `PARTITION BY LIST(expr) PARTITION part_name VALUES IN (...)` expression. The partitioned tables’ data set supports at most 1024 distinct integer values. You can define the values using the `PARTITION ... VALUES IN (...)` clause. + +To enable list partitioning, set the session variable [`tidb_enable_list_partition`](/system-variables.md#tidb_enable_list_partition-new-in-v50) to `ON`. + +#### List COLUMNS partitioning (**Experimental**) + +[User document](/partitioned-table.md#list-columns-partitioning) + +List COLUMNS partitioning is a variant of list partitioning. You can use multiple columns as partition keys. Besides the integer data type, you can also use the columns in the string, `DATE`, and `DATETIME` data types as partition columns. + +To enable List COLUMNS partitioning, set the session variable [`tidb_enable_list_partition`](/system-variables.md#tidb_enable_list_partition-new-in-v50) to `ON`. + +#### Invisible indexes + +[User document](/sql-statements/sql-statement-alter-index.md), [#9246](https://github.com/pingcap/tidb/issues/9246) + +When you tune performance or select optimal indexes, you can set an index to be `Visible` or `Invisible` by using SQL statements. This setting can avoid performing resource-consuming operations, such as `DROP INDEX` and `ADD INDEX`. + +To modify the visibility of an index, use the `ALTER INDEX` statement. After the modification, the optimizer decides whether to add this index to the index list based on the index visibility. + +#### `EXCEPT` and `INTERSECT` operators + +[User document](/functions-and-operators/set-operators.md), [#18031](https://github.com/pingcap/tidb/issues/18031) + +The `INTERSECT` operator is a set operator, which returns the intersection of the result sets of two or more queries. To some extent, it is an alternative to the `Inner Join` operator. + +The `EXCEPT` operator is a set operator, which combines the result sets of two queries and returns elements that are in the first query result but not in the second. + +### Transaction + +[User document](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407), [#18005](https://github.com/pingcap/tidb/issues/18005) + +In the pessimistic transaction mode, if the tables involved in a transaction contain concurrent DDL operations or `SCHEMA VERSION` changes, the system automatically updates the transaction's `SCHEMA VERSION` to the latest to ensure the successful transaction commit, and to avoid that the client receives the `Information schema is changed` error when the transaction is interrupted by DDL operations or `SCHEMA VERSION` changes. + +This feature is disabled by default. To enable the feature, modify the value of [`tidb_enable_amend_pessimistic_txn`](/system-variables.md#tidb_enable_amend_pessimistic_txn-new-in-v407) system variable. This feature is introduced in v4.0.7 and has the following issues fixed in v5.0: + ++ The compatibility issue that occurs when TiDB Binlog executes `Add Column` operations ++ The data inconsistency issue that occurs when using the feature together with the unique index ++ The data inconsistency issue that occurs when using the feature together with the added index + +Currently, this feature still has the following incompatibility issues: + ++ Transaction’s semantics might change when there are concurrent transactions ++ Known compatibility issue that occurs when using the feature together with TiDB Binlog ++ Incompatibility with `Change Column` + +### Character set and collation + +- Support the `utf8mb4_unicode_ci` and `utf8_unicode_ci` collations. [User document](/character-set-and-collation.md#new-framework-for-collations), [#17596](https://github.com/pingcap/tidb/issues/17596) +- Support the case-insensitive comparison sort for collations + +### Security + +[User document](/log-redaction.md), [#18566](https://github.com/pingcap/tidb/issues/18566) + +To meet security compliance requirements (such as *General Data Protection Regulation*, or GDPR), the system supports desensitizing information (such as ID and credit card number) in the output error messages and logs, which can avoid leaking sensitive information. + +TiDB supports desensitizing the output log information. To enable this feature, use the following switches: + ++ The global variable [`tidb_redact_log`](/system-variables.md#tidb_redact_log). Its default value is `0`, which means that desensitization is disabled. To enable desensitization for tidb-server logs, set the variable value to `1`. ++ The configuration item `security.redact-info-log`. Its default value is `false`, which means that desensitization is disabled. To enable desensitization for tikv-server logs, set the variable value to `true`. ++ The configuration item `security.redact-info-log`. Its default value is `false`, which means that desensitization is disabled. To enable desensitization for pd-server logs, set the variable value to `true`. ++ The configuration item `security.redact_info_log` for tiflash-server and `security.redact-info-log` for tiflash-learner. Their default values are both `false`, which means that desensitization is disabled. To enable desensitization for tiflash-server and tiflash-learner logs, set the values of both variables to `true`. + +This feature is introduced in v5.0. To use the feature, enable the system variable and all configuration items above. + +## Performance optimization + +### MPP architecture + +[User document](/tiflash/use-tiflash.md) + +TiDB introduces the MPP architecture through TiFlash nodes. This architecture allows multiple TiFlash nodes to share the execution workload of large join queries. + +When the MPP mode is on, TiDB determines whether to send a query to the MPP engine for computation based on the calculation cost. In the MPP mode, TiDB distributes the computation of table joins to each running TiFlash node by redistributing the join key during data calculation (`Exchange` operation), and thus accelerates the calculation. Furthermore, with the aggregation computing feature that TiFlash has already supported, TiDB can pushdown the computation of a query to the TiFlash MPP cluster. Then the distributed environment can help accelerate the entire execution process and dramatically increase the speed of analytic queries. + +In the TPC-H 100 benchmark test, TiFlash MPP delivers significant processing speed over analytic engines of traditional analytic databases and SQL on Hadoop. With this architecture, you can perform large-scale analytic queries directly on the latest transaction data, with a higher performance than traditional offline analytic solutions. According to the benchmark, with the same cluster resource, TiDB 5.0 MPP shows 2 to 3 times of speedup over Greenplum 6.15.0 and Apache Spark 3.1.1, and some queries have 8 times better performance. + +Currently, the main features that the MPP mode does not support are as follows (For details, refer to [Use TiFlash](/tiflash/use-tiflash.md)): + ++ Table partitioning ++ Window Function ++ Collation ++ Some built-in functions ++ Reading data from TiKV ++ OOM spill ++ Union ++ Full Outer Join + +### Clustered index + +[User document](/clustered-indexes.md), [#4841](https://github.com/pingcap/tidb/issues/4841) + +When you are designing table structures or analyzing database behaviors, it is recommended to use the clustered index feature if you find that some columns with primary keys are often grouped and sorted, queries on these columns often return a certain range of data or a small amount of data with different values, and the corresponding data does not cause read or write hotspot issues. + +Clustered indexes, also known as _index-organized tables_ in some database management systems, is a storage structure associated with the data of a table. When creating a clustered index, you can specify one or more columns from the table as the keys for the index. TiDB stores these keys in a specific structure, which allows TiDB to quickly and efficiently find the rows associated with the keys, thus improves the performance of querying and writing data. + +When the clustered index feature is enabled, the TiDB performance improves significantly (for example in the TPC-C tpmC test, the performance of TiDB, with clustered index enabled, improves by 39%) in the following cases: + ++ When data is inserted, the clustered index reduces one write of the index data from the network. ++ When a query with an equivalent condition only involves the primary key, the clustered index reduces one read of index data from the network. ++ When a query with a range condition only involves the primary key, the clustered index reduces multiple reads of index data from the network. ++ When a query with an equivalent or range condition involves the primary key prefix, the clustered index reduces multiple reads of index data from the network. + +Each table can either use a clustered or non-clustered index to sort and store data. The differences of these two storage structures are as follows: + ++ When creating a clustered index, you can specify one or more columns in the table as the key value of the index. A clustered index sorts and stores the data of a table according to the key value. Each table can have only one clustered index. If a table has a clustered index, it is called a clustered index table. Otherwise, it is called a non-clustered index table. ++ When you create a non-clustered index, the data in the table is stored in an unordered structure. You do not need to explicitly specify the key value of the non-clustered index, because TiDB automatically assigns a unique ROWID to each row of data. During a query, the ROWID is used to locate the corresponding row. Because there are at least two network I/O operations when you query or insert data, the performance is degraded compared with clustered indexes. + +When table data is modified, the database system automatically maintains clustered indexes and non-clustered indexes for you. + +All primary keys are created as non-clustered indexes by default. You can create a primary key as a clustered index or non-clustered index in either of the following two ways: + ++ Specify the keyword `CLUSTERED | NONCLUSTERED` in the statement when creating a table, then the system creates the table in the specified way. The syntax is as follows: + +```sql +CREATE TABLE `t` (`a` VARCHAR(255), `b` INT, PRIMARY KEY (`a`, `b`) CLUSTERED); +``` + +Or + +```sql +CREATE TABLE `t` (`a` VARCHAR(255) PRIMARY KEY CLUSTERED, `b` INT); +``` + +You can execute the statement `SHOW INDEX FROM tbl-name` to query whether a table has a clustered index. + ++ Configure the system variable `tidb_enable_clustered_index` to control the clustered index feature. Supported values are `ON`, `OFF`, and `INT_ONLY`. + + `ON`: Indicates that the clustered index feature is enabled for all types of primary keys. Adding and dropping non-clustered indexes are supported. + + `OFF`: Indicates that the clustered index feature is disabled for all types of primary keys. Adding and dropping non-clustered indexes are supported. + + `INT_ONLY`: The default value. If the variable is set to `INT_ONLY` and `alter-primary-key` is set to `false`, the primary keys which consist of single integer columns are created as clustered indexes by default. The behavior is consistent with that of TiDB v5.0 and earlier versions. + +If a `CREATE TABLE` statement contains the keyword `CLUSTERED | NONCLUSTERED`, the statement overrides the configuration of the system variable and the configuration item. + +You are recommended to use the clustered index feature by specifying the keyword `CLUSTERED | NONCLUSTERED` in statements. In this way, it is more flexible for TiDB to use all data types of clustered and non-clustered indexes in the system at the same time as required. + +It is not recommended to use `tidb_enable_clustered_index = INT_ONLY`, because `INT_ONLY` is temporarily used to make this feature compatible and will be deprecated in the future. + +Limitations for the clustered index are as follows: + ++ Mutual conversion between clustered indexes and non-clustered indexes is not supported. ++ Dropping clustered indexes is not supported. ++ Adding, dropping, and altering clustered indexes using `ALTER TABLE` statements are not supported. ++ Reorganizing and re-creating a clustered index is not supported. ++ Enabling or disabling indexes is not supported, which means the invisible index feature is not effective for clustered indexes. ++ Creating a `UNIQUE KEY` as a clustered index is not supported. ++ Using the clustered index feature together with TiDB Binlog is not supported. After TiDB Binlog is enabled, TiDB only supports creating a single integer primary key as a clustered index. TiDB Binlog does not replicate data changes of existing tables with clustered indexes to the downstream. ++ Using the clustered index feature together with the attributes `SHARD_ROW_ID_BITS` and `PRE_SPLIT_REGIONS` is not supported. ++ If the cluster is upgraded to a later version then rolls back, you need to downgrade newly-added tables by exporting table data before the rollback and importing the data after the rollback. Other tables are not affected. + +### Async Commit + +[User document](/system-variables.md#tidb_enable_async_commit-new-in-v50), [#8316](https://github.com/tikv/tikv/issues/8316) + +The client of the database will wait for the database system to complete the transaction commit in two phases (2PC) synchronously. The transaction returns the result to the client after the first phase commit is successful, and the system executes the second phase commit operation in the background asynchronously to reduce the transaction commit latency. If the transaction write involves only one Region, the second phase is omitted directly, and the transaction becomes a one-phase commit. + +After the Async Commit feature is enabled, with the same hardware and configuration, when Sysbench is set to test the Update index with 64 threads, the average latency decreases by 41.7% from 12.04ms to 7.01ms. + +When Async Commit feature is enabled, to reduce one network interaction latency and improve the performance of data writes, database application developers are recommended to consider reducing the consistency of transactions from linear consistency to [causal consistency](/transaction-overview.md#causal-consistency). The SQL statement to enable causal consistency is `START TRANSACTION WITH CAUSAL CONSISTENCY`. + +After the causal consistency is enabled, with the same hardware and configuration, when Sysbench is set to test oltp_write_only with 64 threads, the average latency decreased by 5.6% from 11.86ms to 11.19ms. + +After the consistency of transactions is reduced from the linear consistency to causal consistency, if there is no interdependence between multiple transactions in the application, the transactions do not have a globally consistent order. + +**The Async Commit feature is enabled by default for newly created v5.0 clusters.** + +This feature is disabled by default for clusters upgraded from earlier versions to v5.0. You can enable this feature by executing the `set global tidb_enable_async_commit = ON;` and `set global tidb_enable_1pc = ON;` statements. + +The limitation for the Async Commit feature is as follows: + ++ Direct downgrade is not supported. + +### Enable the Coprocessor cache feature by default + +[User document](/tidb-configuration-file.md#tikv-clientcopr-cache-new-in-v400), [#18028](https://github.com/pingcap/tidb/issues/18028) + +In 5.0 GA, the Coprocessor cache feature is enabled by default. After this feature is enabled, to reduce the latency of reading data, TiDB caches the calculation results of the operators pushed down to tikv-server in tidb-server. + +To disable the Coprocessor cache feature, you can modify the `capacity-mb` configuration item of `tikv-client.copr-cache` to `0.0`. + +### Improve the execution performance of `delete from table where id Date: Wed, 25 May 2022 09:32:55 -0600 Subject: [PATCH 4/8] fix linter --- system-variables.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/system-variables.md b/system-variables.md index 722b90ae1f2a2..d7da83c9f9387 100644 --- a/system-variables.md +++ b/system-variables.md @@ -1675,12 +1675,6 @@ explain select * from t where age=5; - Range: `[0, 1073741824]` - Unit: Bytes - The maximum length of the SQL statement output. When the output length of a statement is larger than the `tidb_query-log-max-len` value, the statement is truncated to output. - -Usage example: - -```sql -SET tidb_query_log_max_len = 20 -``` - This setting was previously also available a tidb.toml option (`log.query-log-max-len`), but is only a system variable starting from TiDB 6.1. ### tidb_rc_read_check_ts New in v6.0.0 From 54e50a12c3ad9db966172d0a6dbe2bfab4005ed3 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 25 May 2022 09:59:34 -0600 Subject: [PATCH 5/8] Fix additional references --- benchmark/benchmark-tidb-using-sysbench.md | 3 +-- dynamic-config.md | 1 - enable-disk-spill-encrypt.md | 2 +- error-codes.md | 2 +- faq/sql-faq.md | 2 ++ system-variables.md | 2 +- tidb-configuration-file.md | 4 ++-- tidb-troubleshooting-map.md | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmark/benchmark-tidb-using-sysbench.md b/benchmark/benchmark-tidb-using-sysbench.md index 29a88ad2dde9e..6af763298e0d1 100644 --- a/benchmark/benchmark-tidb-using-sysbench.md +++ b/benchmark/benchmark-tidb-using-sysbench.md @@ -11,13 +11,12 @@ It is recommended to use Sysbench 1.0 or later, which can be [downloaded here](h ### TiDB configuration -Higher log level means fewer logs to be printed and thus positively influences TiDB performance. Enable `prepared plan cache` in the TiDB configuration to lower the cost of optimizing execution plan. Specifically, you can add the following command in the TiUP configuration file: +Higher log level means fewer logs to be printed and thus positively influences TiDB performance. Specifically, you can add the following command in the TiUP configuration file: ```yaml server_configs: tidb: log.level: "error" - prepared-plan-cache.enabled: true ``` ### TiKV configuration diff --git a/dynamic-config.md b/dynamic-config.md index 5f3c23d3f8443..1d222df7b6b58 100644 --- a/dynamic-config.md +++ b/dynamic-config.md @@ -318,7 +318,6 @@ The following TiDB configuration items can be modified online: | Configuration item | SQL variable | Description | | :--- | :--- | -| `mem-quota-query` | `tidb_mem_quota_query` | The memory usage limit of a query | | `log.enable-slow-log` | `tidb_enable_slow_log` | Whether to enable slow log | | `log.slow-threshold` | `tidb_slow_log_threshold` | The threshold of slow log | | `log.expensive-threshold` | `tidb_expensive_query_time_threshold` | The threshold of a expensive query | diff --git a/enable-disk-spill-encrypt.md b/enable-disk-spill-encrypt.md index 005d6f63394a9..5b8f3af2ec0f4 100644 --- a/enable-disk-spill-encrypt.md +++ b/enable-disk-spill-encrypt.md @@ -5,7 +5,7 @@ summary: Learn how to enable encryption for disk spill in TiDB. # Enable Encryption for Disk Spill -When the `oom-use-tmp-storage` configuration item is set to `true`, if the memory usage of a single SQL statement exceeds the limit of `mem-quota-query` setting, some operators can save the intermediate results during execution as a temporary file to the disk and delete the file after the query is completed. +When the `oom-use-tmp-storage` configuration item is set to `true`, if the memory usage of a single SQL statement exceeds the limit of the system variable `tidb_mem_quota_query`, some operators can save the intermediate results during execution as a temporary file to the disk and delete the file after the query is completed. You can enable encryption for disk spill to prevent attackers from accessing data by reading these temporary files. diff --git a/error-codes.md b/error-codes.md index 0e30be1b4093f..b00b98bd86da0 100644 --- a/error-codes.md +++ b/error-codes.md @@ -22,7 +22,7 @@ TiDB is compatible with the error codes in MySQL, and in most cases returns the The memory used by the request exceeds the threshold limit for the TiDB memory usage. - Increase the memory limit for a single SQL statement by configuring `mem-quota-query`. + Increase the memory limit for a single SQL statement by configuring the system variable `tidb_mem_quota_query`. * Error Number: 8002 diff --git a/faq/sql-faq.md b/faq/sql-faq.md index 9145ec54d5b66..3c7dc0e773852 100644 --- a/faq/sql-faq.md +++ b/faq/sql-faq.md @@ -169,6 +169,8 @@ Trigger strategy: `auto analyze` is automatically triggered when the number of r When the modified number or the current total row number is larger than `tidb_auto_analyze_ratio`, the `analyze` statement is automatically triggered. The default value of `tidb_auto_analyze_ratio` is 0.5, indicating that this feature is enabled by default. To ensure safety, its minimum value is 0.3 when the feature is enabled, and it must be smaller than `pseudo-estimate-ratio` whose default value is 0.8, otherwise pseudo statistics will be used for a period of time. It is recommended to set `tidb_auto_analyze_ratio` to 0.5. +Auto analyze can be disabled with the system variable `tidb_enable_auto_analyze`. + ## Can I use hints to override the optimizer behavior? TiDB supports multiple ways to override the default query optimizer behavior, including [hints](/optimizer-hints.md) and [SQL Plan Management](/sql-plan-management.md). The basic usage is similar to MySQL, with several TiDB specific extensions: diff --git a/system-variables.md b/system-variables.md index d7da83c9f9387..05fbd1a7560d2 100644 --- a/system-variables.md +++ b/system-variables.md @@ -1674,7 +1674,7 @@ explain select * from t where age=5; - Default value: `4096` (4 KiB) - Range: `[0, 1073741824]` - Unit: Bytes -- The maximum length of the SQL statement output. When the output length of a statement is larger than the `tidb_query-log-max-len` value, the statement is truncated to output. +- The maximum length of the SQL statement output. When the output length of a statement is larger than the `tidb_query_log_max_len` value, the statement is truncated to output. - This setting was previously also available a tidb.toml option (`log.query-log-max-len`), but is only a system variable starting from TiDB 6.1. ### tidb_rc_read_check_ts New in v6.0.0 diff --git a/tidb-configuration-file.md b/tidb-configuration-file.md index c20c895bce19c..55f29927816ec 100644 --- a/tidb-configuration-file.md +++ b/tidb-configuration-file.md @@ -28,12 +28,12 @@ The TiDB configuration file supports more options than command-line parameters. ### `oom-use-tmp-storage` -+ Controls whether to enable the temporary storage for some operators when a single SQL statement exceeds the memory quota specified by `mem-quota-query`. ++ Controls whether to enable the temporary storage for some operators when a single SQL statement exceeds the memory quota specified by the system variable `tidb_mem_quota_query`. + Default value: `true` ### `tmp-storage-path` -+ Specifies the temporary storage path for some operators when a single SQL statement exceeds the memory quota specified by `mem-quota-query`. ++ Specifies the temporary storage path for some operators when a single SQL statement exceeds the memory quota specified by the system variable `tidb_mem_quota_query`. + Default value: `/_tidb/MC4wLjAuMDo0MDAwLzAuMC4wLjA6MTAwODA=/tmp-storage`. `MC4wLjAuMDo0MDAwLzAuMC4wLjA6MTAwODA=` is the `Base64` encoding result of `:/:`. + This configuration takes effect only when `oom-use-tmp-storage` is `true`. diff --git a/tidb-troubleshooting-map.md b/tidb-troubleshooting-map.md index 1eb3bcb86f9f6..15cf54dfca5ea 100644 --- a/tidb-troubleshooting-map.md +++ b/tidb-troubleshooting-map.md @@ -126,7 +126,7 @@ Refer to [5 PD issues](#5-pd-issues). > **Note:** > - > The default threshold for a single SQL memory usage is `1GB` (in bytes, scope:`SESSION`). You can set this parameter by configuring `tidb_mem_quota_query`. You can also modify the `mem-quota-query` item (in bytes) in the configuration file by hot loading the configuration items. + > The default threshold for a single SQL memory usage is `1GB`. You can set this parameter by configuring the system variable `tidb_mem_quota_query`. - 3.2.3 Mitigate OOM issues From c14cd46e4b1370f221e6150e23039d4cd3c2f79c Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Thu, 26 May 2022 16:44:23 -0600 Subject: [PATCH 6/8] Update configure-memory-usage.md Co-authored-by: bb7133 --- configure-memory-usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure-memory-usage.md b/configure-memory-usage.md index 5301458e5d5fc..2289e2e7b1760 100644 --- a/configure-memory-usage.md +++ b/configure-memory-usage.md @@ -8,7 +8,7 @@ aliases: ['/docs/dev/configure-memory-usage/','/docs/dev/how-to/configure/memory Currently, TiDB can track the memory quota of a single SQL query and take actions to prevent OOM (out of memory) or troubleshoot OOM when the memory usage exceeds a specific threshold value. The system variable `tidb_mem_oom_action` specifies the action to take when a query reaches the memory limit: -- A value of `LOG` means that queries will continue to execute when the `tidb_mem_quota_query` limit is reached, but TiDB will print an entry to the log. +- A value of `LOG` means that queries will continue to execute when the [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) limit is reached, but TiDB will print an entry to the log. - A value of `CANCEL` means TiDB stops executing the SQL query immediately after the `tidb_mem_quota_query` limit is reached, and returns an error to the client. The error information clearly shows the memory usage of each physical execution operator that consumes memory in the SQL execution process. ## Configure the memory quota of a query From 9ddd7db69d5122bba11c3e0586c1b6a81e9eddc3 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Thu, 26 May 2022 16:50:53 -0600 Subject: [PATCH 7/8] Add links to sysvar page for mem-quota-query --- configure-memory-usage.md | 14 +++++++------- enable-disk-spill-encrypt.md | 2 +- error-codes.md | 2 +- optimizer-hints.md | 2 +- releases/release-4.0.10.md | 2 +- tidb-configuration-file.md | 4 ++-- tidb-troubleshooting-map.md | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/configure-memory-usage.md b/configure-memory-usage.md index 2289e2e7b1760..49a7257624676 100644 --- a/configure-memory-usage.md +++ b/configure-memory-usage.md @@ -6,34 +6,34 @@ aliases: ['/docs/dev/configure-memory-usage/','/docs/dev/how-to/configure/memory # TiDB Memory Control -Currently, TiDB can track the memory quota of a single SQL query and take actions to prevent OOM (out of memory) or troubleshoot OOM when the memory usage exceeds a specific threshold value. The system variable `tidb_mem_oom_action` specifies the action to take when a query reaches the memory limit: +Currently, TiDB can track the memory quota of a single SQL query and take actions to prevent OOM (out of memory) or troubleshoot OOM when the memory usage exceeds a specific threshold value. The system variable [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) specifies the action to take when a query reaches the memory limit: - A value of `LOG` means that queries will continue to execute when the [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) limit is reached, but TiDB will print an entry to the log. -- A value of `CANCEL` means TiDB stops executing the SQL query immediately after the `tidb_mem_quota_query` limit is reached, and returns an error to the client. The error information clearly shows the memory usage of each physical execution operator that consumes memory in the SQL execution process. +- A value of `CANCEL` means TiDB stops executing the SQL query immediately after the [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) limit is reached, and returns an error to the client. The error information clearly shows the memory usage of each physical execution operator that consumes memory in the SQL execution process. ## Configure the memory quota of a query -The system variable `tidb_mem_quota_query` sets the limit for a query in bytes. Some usage examples: +The system variable [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) sets the limit for a query in bytes. Some usage examples: {{< copyable "sql" >}} ```sql -- Set the threshold value of memory quota for a single SQL query to 8GB: -set @@tidb_mem_quota_query = 8 << 30; +SET tidb_mem_quota_query = 8 << 30; ``` {{< copyable "sql" >}} ```sql -- Set the threshold value of memory quota for a single SQL query to 8MB: -set @@tidb_mem_quota_query = 8 << 20; +SET tidb_mem_quota_query = 8 << 20; ``` {{< copyable "sql" >}} ```sql -- Set the threshold value of memory quota for a single SQL query to 8KB: -set @@tidb_mem_quota_query = 8 << 10; +SET tidb_mem_quota_query = 8 << 10; ``` ## Configure the memory usage threshold of a tidb-server instance @@ -107,7 +107,7 @@ The following example constructs a memory-intensive SQL statement that triggers TiDB supports disk spill for execution operators. When the memory usage of a SQL execution exceeds the memory quota, tidb-server can spill the intermediate data of execution operators to the disk to relieve memory pressure. Operators supporting disk spill include Sort, MergeJoin, HashJoin, and HashAgg. -- The disk spill behavior is jointly controlled by `tidb_mem_quota_query`, [`oom-use-tmp-storage`](/tidb-configuration-file.md#oom-use-tmp-storage), [`tmp-storage-path`](/tidb-configuration-file.md#tmp-storage-path), and [`tmp-storage-quota`](/tidb-configuration-file.md#tmp-storage-quota) parameters. +- The disk spill behavior is jointly controlled by [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query), [`oom-use-tmp-storage`](/tidb-configuration-file.md#oom-use-tmp-storage), [`tmp-storage-path`](/tidb-configuration-file.md#tmp-storage-path), and [`tmp-storage-quota`](/tidb-configuration-file.md#tmp-storage-quota) parameters. - When the disk spill is triggered, TiDB outputs a log containing the keywords `memory exceeds quota, spill to disk now` or `memory exceeds quota, set aggregate mode to spill-mode`. - Disk spill for the Sort, MergeJoin, and HashJoin operator is introduced in v4.0.0; disk spill for the HashAgg operator is introduced in v5.2.0. - When the SQL executions containing Sort, MergeJoin, or HashJoin cause OOM, TiDB triggers disk spill by default. When SQL executions containing HashAgg cause OOM, TiDB does not trigger disk spill by default. You can configure the system variable `tidb_executor_concurrency = 1` to trigger disk spill for HashAgg. diff --git a/enable-disk-spill-encrypt.md b/enable-disk-spill-encrypt.md index 5b8f3af2ec0f4..575c20405d1de 100644 --- a/enable-disk-spill-encrypt.md +++ b/enable-disk-spill-encrypt.md @@ -5,7 +5,7 @@ summary: Learn how to enable encryption for disk spill in TiDB. # Enable Encryption for Disk Spill -When the `oom-use-tmp-storage` configuration item is set to `true`, if the memory usage of a single SQL statement exceeds the limit of the system variable `tidb_mem_quota_query`, some operators can save the intermediate results during execution as a temporary file to the disk and delete the file after the query is completed. +When the `oom-use-tmp-storage` configuration item is set to `true`, if the memory usage of a single SQL statement exceeds the limit of the system variable [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query), some operators can save the intermediate results during execution as a temporary file to the disk and delete the file after the query is completed. You can enable encryption for disk spill to prevent attackers from accessing data by reading these temporary files. diff --git a/error-codes.md b/error-codes.md index b00b98bd86da0..8ee2ecc33ef4c 100644 --- a/error-codes.md +++ b/error-codes.md @@ -22,7 +22,7 @@ TiDB is compatible with the error codes in MySQL, and in most cases returns the The memory used by the request exceeds the threshold limit for the TiDB memory usage. - Increase the memory limit for a single SQL statement by configuring the system variable `tidb_mem_quota_query`. + Increase the memory limit for a single SQL statement by configuring the system variable [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query). * Error Number: 8002 diff --git a/optimizer-hints.md b/optimizer-hints.md index 34b89c7f8eccf..84ab465681dd6 100644 --- a/optimizer-hints.md +++ b/optimizer-hints.md @@ -319,7 +319,7 @@ In the following hint, `MEMORY_QUOTA(1024 MB)` means that the memory usage is li select /*+ MEMORY_QUOTA(1024 MB) */ * from t; ``` -In addition to this hint, the `tidb_mem_quota_query` system variable can also limit the memory usage of a statement. +In addition to this hint, the [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) system variable can also limit the memory usage of a statement. ### READ_CONSISTENT_REPLICA() diff --git a/releases/release-4.0.10.md b/releases/release-4.0.10.md index bbadf4b61ab29..2269f123bd87f 100644 --- a/releases/release-4.0.10.md +++ b/releases/release-4.0.10.md @@ -92,7 +92,7 @@ TiDB version: 4.0.10 + Dumpling - - Avoid TiDB out of memory (OOM) by setting the default behavior of `tidb_mem_quota_query` [#233](https://github.com/pingcap/dumpling/pull/233) + - Avoid TiDB out of memory (OOM) by setting the default behavior of [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) [#233](https://github.com/pingcap/dumpling/pull/233) + Backup & Restore (BR) diff --git a/tidb-configuration-file.md b/tidb-configuration-file.md index 55f29927816ec..1d6255c744f99 100644 --- a/tidb-configuration-file.md +++ b/tidb-configuration-file.md @@ -28,12 +28,12 @@ The TiDB configuration file supports more options than command-line parameters. ### `oom-use-tmp-storage` -+ Controls whether to enable the temporary storage for some operators when a single SQL statement exceeds the memory quota specified by the system variable `tidb_mem_quota_query`. ++ Controls whether to enable the temporary storage for some operators when a single SQL statement exceeds the memory quota specified by the system variable [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query). + Default value: `true` ### `tmp-storage-path` -+ Specifies the temporary storage path for some operators when a single SQL statement exceeds the memory quota specified by the system variable `tidb_mem_quota_query`. ++ Specifies the temporary storage path for some operators when a single SQL statement exceeds the memory quota specified by the system variable [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query). + Default value: `/_tidb/MC4wLjAuMDo0MDAwLzAuMC4wLjA6MTAwODA=/tmp-storage`. `MC4wLjAuMDo0MDAwLzAuMC4wLjA6MTAwODA=` is the `Base64` encoding result of `:/:`. + This configuration takes effect only when `oom-use-tmp-storage` is `true`. diff --git a/tidb-troubleshooting-map.md b/tidb-troubleshooting-map.md index 15cf54dfca5ea..a97467af0427c 100644 --- a/tidb-troubleshooting-map.md +++ b/tidb-troubleshooting-map.md @@ -126,7 +126,7 @@ Refer to [5 PD issues](#5-pd-issues). > **Note:** > - > The default threshold for a single SQL memory usage is `1GB`. You can set this parameter by configuring the system variable `tidb_mem_quota_query`. + > The default threshold for a single SQL memory usage is `1GB`. You can set this parameter by configuring the system variable [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query). - 3.2.3 Mitigate OOM issues From 5e5b61c10aa6ebac2006b9fa85c85b6c5fdf6574 Mon Sep 17 00:00:00 2001 From: Ran Date: Fri, 27 May 2022 12:38:26 +0800 Subject: [PATCH 8/8] unify versioning: TiDB 6.1 -> TiDB v6.1.0 Signed-off-by: Ran --- system-variables.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/system-variables.md b/system-variables.md index 05fbd1a7560d2..98b46fa70cdf7 100644 --- a/system-variables.md +++ b/system-variables.md @@ -363,7 +363,7 @@ This variable is an alias for `last_insert_id`. - Default value: `OFF` - This variable ensures that all connections to TiDB are either on a local socket, or using TLS. See [Enable TLS between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) for additional details. - Setting this variable to `ON` requires you to connect to TiDB from a session that has TLS enabled. This helps prevent lock-out scenarios when TLS is not configured correctly. -- This setting was previously a `tidb.toml` option (`security.require-secure-transport`), but changed to a system variable starting from TiDB 6.1.0. +- This setting was previously a `tidb.toml` option (`security.require-secure-transport`), but changed to a system variable starting from TiDB v6.1.0. ### skip_name_resolve New in v5.2.0 @@ -602,7 +602,7 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a - Range: `[1, 10000]` - The number of goroutines for requests related to executing commit in the commit phase of the single transaction. - If the transaction to commit is too large, the waiting time for the flow control queue when the transaction is committed might be too long. In this situation, you can increase the configuration value to speed up the commit. -- This setting was previously a `tidb.toml` option (`performance.committer-concurrency`), but changed to a system variable starting from TiDB 6.1.0. +- This setting was previously a `tidb.toml` option (`performance.committer-concurrency`), but changed to a system variable starting from TiDB v6.1.0. ### tidb_config @@ -765,8 +765,8 @@ Constraint checking is always performed in place for pessimistic transactions (d - Scope: GLOBAL - Persists to cluster: Yes - Default value: `ON` -- Determines whether TiDB automatically updates table statistics as a background operation -- This setting was previously a `tidb.toml` option (`performance.run-auto-analyze`), but changed to a system variable starting from TiDB 6.1.0. +- Determines whether TiDB automatically updates table statistics as a background operation. +- This setting was previously a `tidb.toml` option (`performance.run-auto-analyze`), but changed to a system variable starting from TiDB v6.1.0. ### tidb_enable_auto_increment_in_generated @@ -781,7 +781,7 @@ Constraint checking is always performed in place for pessimistic transactions (d - Persists to cluster: Yes - Default value: `OFF` - Determines if TiDB permits non-transactional 'batched' statements. Only the value of `OFF` can be considered safe, as batch DML does not provide ACID guarantees. -- This setting was previously a `tidb.toml` option (`enable-batch-dml`), but changed to a system variable starting from TiDB 6.1.0. +- This setting was previously a `tidb.toml` option (`enable-batch-dml`), but changed to a system variable starting from TiDB v6.1.0. ### tidb_enable_cascades_planner @@ -933,7 +933,7 @@ Constraint checking is always performed in place for pessimistic transactions (d - Persists to cluster: Yes - Default value: `ON` - Determines whether to enable [Prepared Plan Cache](/sql-prepared-plan-cache.md). When it is enabled, the execution plans of `Prepare` and `Execute` are cached so that the subsequent executions skip optimizing the execution plans, which brings performance improvement. -- This setting was previously a `tidb.toml` option (`prepared-plan-cache.enabled`), but changed to a system variable starting from TiDB 6.1.0. +- This setting was previously a `tidb.toml` option (`prepared-plan-cache.enabled`), but changed to a system variable starting from TiDB v6.1.0. ### tidb_enable_pseudo_for_outdated_stats New in v5.3.0 @@ -1378,7 +1378,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Possible values: `CANCEL`, `LOG` - Specifies what operation TiDB performs when a single SQL statement exceeds the memory quota specified by `tidb_mem_quota_query` and cannot be spilled over to disk. See [TiDB Memory Control](/configure-memory-usage.md) for details. - The default value is `CANCEL`, but in TiDB v4.0.2 and earlier versions, the default value is `LOG`. -- This setting was previously a `tidb.toml` option (`oom-action`), but changed to a system variable starting from TiDB 6.1.0. +- This setting was previously a `tidb.toml` option (`oom-action`), but changed to a system variable starting from TiDB v6.1.0. ### tidb_mem_quota_apply_cache New in v5.0 @@ -1642,7 +1642,7 @@ explain select * from t where age=5; - Default value: `0.1` - Range: `[0, 1]` - This setting is used to prevent the tidb.toml option `performance.max-memory` from being exceeded. When `max-memory` * (1 - `tidb_prepared_plan_cache_memory_guard_ratio`) is exceeded, the elements in the LRU are removed. -- This setting was previously a `tidb.toml` option (`prepared-plan-cache.memory-guard-ratio`), but changed to a system variable starting from TiDB 6.1.0. +- This setting was previously a `tidb.toml` option (`prepared-plan-cache.memory-guard-ratio`), but changed to a system variable starting from TiDB v6.1.0. ### tidb_prepared_plan_cache_size New in v6.1.0 @@ -1651,7 +1651,7 @@ explain select * from t where age=5; - Default value: `100` - Range: `[1, 100000]` - The maximum number of statements that can be cached in the prepared plan cache. -- This setting was previously a `tidb.toml` option (`prepared-plan-cache.capacity`), but changed to a system variable starting from TiDB 6.1.0. +- This setting was previously a `tidb.toml` option (`prepared-plan-cache.capacity`), but changed to a system variable starting from TiDB v6.1.0. ### tidb_projection_concurrency @@ -1675,7 +1675,7 @@ explain select * from t where age=5; - Range: `[0, 1073741824]` - Unit: Bytes - The maximum length of the SQL statement output. When the output length of a statement is larger than the `tidb_query_log_max_len` value, the statement is truncated to output. -- This setting was previously also available a tidb.toml option (`log.query-log-max-len`), but is only a system variable starting from TiDB 6.1. +- This setting was previously also available a tidb.toml option (`log.query-log-max-len`), but is only a system variable starting from TiDB v6.1.0. ### tidb_rc_read_check_ts New in v6.0.0