diff --git a/DEPS.bzl b/DEPS.bzl index 8d09e06ff0d67..bffae277e3f2c 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -5780,13 +5780,13 @@ def go_deps(): name = "com_github_pingcap_tipb", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/tipb", - sha256 = "bb5d3cf24245a26acb1c8f8c110eb3313f21a655a8b49280f2772082bb29dc22", - strip_prefix = "github.com/pingcap/tipb@v0.0.0-20230607071926-bda24015c2d6", + sha256 = "74b710db49c16c0400c44d14e7d2e23257d094d7cf39c71e5715bb5ac59fd57a", + strip_prefix = "github.com/pingcap/tipb@v0.0.0-20230802082933-256c189cd860", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230607071926-bda24015c2d6.zip", - "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230607071926-bda24015c2d6.zip", - "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230607071926-bda24015c2d6.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230607071926-bda24015c2d6.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230802082933-256c189cd860.zip", + "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230802082933-256c189cd860.zip", + "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230802082933-256c189cd860.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230802082933-256c189cd860.zip", ], ) go_repository( diff --git a/go.mod b/go.mod index 98779f07949f9..0435008a14b50 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e - github.com/pingcap/tipb v0.0.0-20230607071926-bda24015c2d6 + github.com/pingcap/tipb v0.0.0-20230802082933-256c189cd860 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 github.com/prometheus/client_model v0.4.0 diff --git a/go.sum b/go.sum index bbd8149d978e6..2347a3307ee2f 100644 --- a/go.sum +++ b/go.sum @@ -816,8 +816,8 @@ github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 h1:2SOzvGvE8beiC1Y4g github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tipb v0.0.0-20230607071926-bda24015c2d6 h1:D79RE4RVhq2ic8sqDSv7QdL0tT5aZV3CaCXUAT41iWc= -github.com/pingcap/tipb v0.0.0-20230607071926-bda24015c2d6/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/tipb v0.0.0-20230802082933-256c189cd860 h1:2DXvgx8MGCJitU7/IaUC+2onDAuqB7B/KMyxxiX3Y5I= +github.com/pingcap/tipb v0.0.0-20230802082933-256c189cd860/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/util/execdetails/execdetails.go b/util/execdetails/execdetails.go index ed5fec68e4047..29c53132bf9ea 100644 --- a/util/execdetails/execdetails.go +++ b/util/execdetails/execdetails.go @@ -583,15 +583,18 @@ func (crs *CopRuntimeStats) RecordOneCopTask(address string, summary *tipb.Execu storeType: crs.storeType, BasicRuntimeStats: BasicRuntimeStats{ tiflashScanContext: TiFlashScanContext{ - totalDmfileScannedPacks: summary.GetTiflashScanContext().GetTotalDmfileScannedPacks(), - totalDmfileSkippedPacks: summary.GetTiflashScanContext().GetTotalDmfileSkippedPacks(), - totalDmfileScannedRows: summary.GetTiflashScanContext().GetTotalDmfileScannedRows(), - totalDmfileSkippedRows: summary.GetTiflashScanContext().GetTotalDmfileSkippedRows(), - totalDmfileRoughSetIndexLoadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileRoughSetIndexLoadTimeMs(), - totalDmfileReadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileReadTimeMs(), - totalCreateSnapshotTimeMs: summary.GetTiflashScanContext().GetTotalCreateSnapshotTimeMs(), - totalLocalRegionNum: summary.GetTiflashScanContext().GetTotalLocalRegionNum(), - totalRemoteRegionNum: summary.GetTiflashScanContext().GetTotalRemoteRegionNum()}}, threads: int32(summary.GetConcurrency()), + totalDmfileScannedPacks: summary.GetTiflashScanContext().GetTotalDmfileScannedPacks(), + totalDmfileSkippedPacks: summary.GetTiflashScanContext().GetTotalDmfileSkippedPacks(), + totalDmfileScannedRows: summary.GetTiflashScanContext().GetTotalDmfileScannedRows(), + totalDmfileSkippedRows: summary.GetTiflashScanContext().GetTotalDmfileSkippedRows(), + totalDmfileRoughSetIndexCheckTimeMs: summary.GetTiflashScanContext().GetTotalDmfileRoughSetIndexCheckTimeMs(), + totalDmfileReadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileReadTimeMs(), + totalCreateSnapshotTimeMs: summary.GetTiflashScanContext().GetTotalCreateSnapshotTimeMs(), + totalLocalRegionNum: summary.GetTiflashScanContext().GetTotalLocalRegionNum(), + totalRemoteRegionNum: summary.GetTiflashScanContext().GetTotalRemoteRegionNum(), + totalLearnerReadMs: summary.GetTiflashScanContext().GetTotalLearnerReadMs(), + totalDisaggReadCacheHitSize: summary.GetTiflashScanContext().GetTotalDisaggReadCacheHitSize(), + totalDisaggReadCacheMissSize: summary.GetTiflashScanContext().GetTotalDisaggReadCacheMissSize()}}, threads: int32(summary.GetConcurrency()), totalTasks: 1, } data.BasicRuntimeStats.loop.Store(int32(*summary.NumIterations)) @@ -716,33 +719,39 @@ type RuntimeStats interface { // TiFlashScanContext is used to express the table scan information in tiflash type TiFlashScanContext struct { - totalDmfileScannedPacks uint64 - totalDmfileScannedRows uint64 - totalDmfileSkippedPacks uint64 - totalDmfileSkippedRows uint64 - totalDmfileRoughSetIndexLoadTimeMs uint64 - totalDmfileReadTimeMs uint64 - totalCreateSnapshotTimeMs uint64 - totalLocalRegionNum uint64 - totalRemoteRegionNum uint64 + totalDmfileScannedPacks uint64 + totalDmfileScannedRows uint64 + totalDmfileSkippedPacks uint64 + totalDmfileSkippedRows uint64 + totalDmfileRoughSetIndexCheckTimeMs uint64 + totalDmfileReadTimeMs uint64 + totalCreateSnapshotTimeMs uint64 + totalLocalRegionNum uint64 + totalRemoteRegionNum uint64 + totalLearnerReadMs uint64 + totalDisaggReadCacheHitSize uint64 + totalDisaggReadCacheMissSize uint64 } // Clone implements the deep copy of * TiFlashshScanContext func (context *TiFlashScanContext) Clone() TiFlashScanContext { return TiFlashScanContext{ - totalDmfileScannedPacks: context.totalDmfileScannedPacks, - totalDmfileScannedRows: context.totalDmfileScannedRows, - totalDmfileSkippedPacks: context.totalDmfileSkippedPacks, - totalDmfileSkippedRows: context.totalDmfileSkippedRows, - totalDmfileRoughSetIndexLoadTimeMs: context.totalDmfileRoughSetIndexLoadTimeMs, - totalDmfileReadTimeMs: context.totalDmfileReadTimeMs, - totalCreateSnapshotTimeMs: context.totalCreateSnapshotTimeMs, - totalLocalRegionNum: context.totalLocalRegionNum, - totalRemoteRegionNum: context.totalRemoteRegionNum, + totalDmfileScannedPacks: context.totalDmfileScannedPacks, + totalDmfileScannedRows: context.totalDmfileScannedRows, + totalDmfileSkippedPacks: context.totalDmfileSkippedPacks, + totalDmfileSkippedRows: context.totalDmfileSkippedRows, + totalDmfileRoughSetIndexCheckTimeMs: context.totalDmfileRoughSetIndexCheckTimeMs, + totalDmfileReadTimeMs: context.totalDmfileReadTimeMs, + totalCreateSnapshotTimeMs: context.totalCreateSnapshotTimeMs, + totalLocalRegionNum: context.totalLocalRegionNum, + totalRemoteRegionNum: context.totalRemoteRegionNum, + totalLearnerReadMs: context.totalLearnerReadMs, + totalDisaggReadCacheHitSize: context.totalDisaggReadCacheHitSize, + totalDisaggReadCacheMissSize: context.totalDisaggReadCacheMissSize, } } func (context *TiFlashScanContext) String() string { - return fmt.Sprintf("tiflash_scan:{dtfile:{total_scanned_packs:%d, total_skipped_packs:%d, total_scanned_rows:%d, total_skipped_rows:%d, total_rs_index_load_time: %dms, total_read_time: %dms}, total_create_snapshot_time: %dms, total_local_region_num: %d, total_remote_region_num: %d}", context.totalDmfileScannedPacks, context.totalDmfileSkippedPacks, context.totalDmfileScannedRows, context.totalDmfileSkippedRows, context.totalDmfileRoughSetIndexLoadTimeMs, context.totalDmfileReadTimeMs, context.totalCreateSnapshotTimeMs, context.totalLocalRegionNum, context.totalRemoteRegionNum) + return fmt.Sprintf("tiflash_scan:{dtfile:{total_scanned_packs:%d, total_skipped_packs:%d, total_scanned_rows:%d, total_skipped_rows:%d, total_rs_index_check_time: %dms, total_read_time: %dms, total_disagg_read_cache_hit_size: %d, total_disagg_read_cache_miss_size: %d}, total_create_snapshot_time: %dms, total_local_region_num: %d, total_remote_region_num: %d, total_learner_read_time: %dms}", context.totalDmfileScannedPacks, context.totalDmfileSkippedPacks, context.totalDmfileScannedRows, context.totalDmfileSkippedRows, context.totalDmfileRoughSetIndexCheckTimeMs, context.totalDmfileReadTimeMs, context.totalDisaggReadCacheHitSize, context.totalDisaggReadCacheMissSize, context.totalCreateSnapshotTimeMs, context.totalLocalRegionNum, context.totalRemoteRegionNum, context.totalLearnerReadMs) } // Merge make sum to merge the information in TiFlashScanContext @@ -751,11 +760,14 @@ func (context *TiFlashScanContext) Merge(other TiFlashScanContext) { context.totalDmfileScannedRows += other.totalDmfileScannedRows context.totalDmfileSkippedPacks += other.totalDmfileSkippedPacks context.totalDmfileSkippedRows += other.totalDmfileSkippedRows - context.totalDmfileRoughSetIndexLoadTimeMs += other.totalDmfileRoughSetIndexLoadTimeMs + context.totalDmfileRoughSetIndexCheckTimeMs += other.totalDmfileRoughSetIndexCheckTimeMs context.totalDmfileReadTimeMs += other.totalDmfileReadTimeMs context.totalCreateSnapshotTimeMs += other.totalCreateSnapshotTimeMs context.totalLocalRegionNum += other.totalLocalRegionNum context.totalRemoteRegionNum += other.totalRemoteRegionNum + context.totalLearnerReadMs += other.totalLearnerReadMs + context.totalDisaggReadCacheHitSize += other.totalDisaggReadCacheHitSize + context.totalDisaggReadCacheMissSize += other.totalDisaggReadCacheMissSize } // Empty check whether TiFlashScanContext is Empty, if scan no pack and skip no pack, we regard it as empty diff --git a/util/execdetails/execdetails_test.go b/util/execdetails/execdetails_test.go index 811a236a0ba61..faac7100815fc 100644 --- a/util/execdetails/execdetails_test.go +++ b/util/execdetails/execdetails_test.go @@ -15,6 +15,7 @@ package execdetails import ( + "fmt" "strconv" "sync" "testing" @@ -135,18 +136,22 @@ func mockExecutorExecutionSummary(TimeProcessedNs, NumProducedRows, NumIteration NumIterations: &NumIterations, XXX_unrecognized: nil} } -func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, totalDmfileScannedPacks, totalDmfileScannedRows, totalDmfileSkippedPacks, totalDmfileSkippedRows, totalDmfileRoughSetIndexLoadTimeMs, totalDmfileReadTimeMs, totalCreateSnapshotTimeMs uint64, totalLocalRegionNum uint64, totalRemoteRegionNum uint64, ExecutorID string) *tipb.ExecutorExecutionSummary { +func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, totalDmfileScannedPacks, totalDmfileScannedRows, totalDmfileSkippedPacks, totalDmfileSkippedRows, totalDmfileRoughSetIndexCheckTimeMs, totalDmfileReadTimeMs, totalCreateSnapshotTimeMs, totalLocalRegionNum, totalRemoteRegionNum, totalLearnerReadMs, totalDisaggReadCacheHitSize, totalDisaggReadCacheMissSize uint64, ExecutorID string) *tipb.ExecutorExecutionSummary { tiflashScanContext := tipb.TiFlashScanContext{ - TotalDmfileScannedPacks: &totalDmfileScannedPacks, - TotalDmfileSkippedPacks: &totalDmfileSkippedPacks, - TotalDmfileScannedRows: &totalDmfileScannedRows, - TotalDmfileSkippedRows: &totalDmfileSkippedRows, - TotalDmfileRoughSetIndexLoadTimeMs: &totalDmfileRoughSetIndexLoadTimeMs, - TotalDmfileReadTimeMs: &totalDmfileReadTimeMs, - TotalCreateSnapshotTimeMs: &totalCreateSnapshotTimeMs, - TotalLocalRegionNum: &totalLocalRegionNum, - TotalRemoteRegionNum: &totalRemoteRegionNum, + TotalDmfileScannedPacks: &totalDmfileScannedPacks, + TotalDmfileSkippedPacks: &totalDmfileSkippedPacks, + TotalDmfileScannedRows: &totalDmfileScannedRows, + TotalDmfileSkippedRows: &totalDmfileSkippedRows, + TotalDmfileRoughSetIndexCheckTimeMs: &totalDmfileRoughSetIndexCheckTimeMs, + TotalDmfileReadTimeMs: &totalDmfileReadTimeMs, + TotalCreateSnapshotTimeMs: &totalCreateSnapshotTimeMs, + TotalLocalRegionNum: &totalLocalRegionNum, + TotalRemoteRegionNum: &totalRemoteRegionNum, + TotalLearnerReadMs: &totalLearnerReadMs, + TotalDisaggReadCacheHitSize: &totalDisaggReadCacheHitSize, + TotalDisaggReadCacheMissSize: &totalDisaggReadCacheMissSize, } + fmt.Println("tiflashScanContext is ", tiflashScanContext.String()) return &tipb.ExecutorExecutionSummary{TimeProcessedNs: &TimeProcessedNs, NumProducedRows: &NumProducedRows, NumIterations: &NumIterations, Concurrency: &Concurrency, ExecutorId: &ExecutorID, DetailInfo: &tipb.ExecutorExecutionSummary_TiflashScanContext{TiflashScanContext: &tiflashScanContext}, XXX_unrecognized: nil} } @@ -210,10 +215,10 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) { tableScanID := 1 aggID := 2 tableReaderID := 3 - stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 1, 8192, 0, 0, 15, 200, 40, 10, 4, "tablescan_"+strconv.Itoa(tableScanID))) - stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, "tablescan_"+strconv.Itoa(tableScanID))) - stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 2, 12000, 1, 6000, 60, 1000, 20, 5, 1, "aggregation_"+strconv.Itoa(aggID))) - stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 1, 8192, 10, 80000, 40, 2000, 30, 1, 1, "aggregation_"+strconv.Itoa(aggID))) + stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 1, 8192, 0, 0, 15, 200, 40, 10, 4, 1, 100, 50, "tablescan_"+strconv.Itoa(tableScanID))) + stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, "tablescan_"+strconv.Itoa(tableScanID))) + stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 2, 12000, 1, 6000, 60, 1000, 20, 5, 1, 0, 20, 0, "aggregation_"+strconv.Itoa(aggID))) + stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 1, 8192, 10, 80000, 40, 2000, 30, 1, 1, 0, 0, 0, "aggregation_"+strconv.Itoa(aggID))) scanDetail := &util.ScanDetail{ TotalKeys: 10, ProcessedKeys: 10, @@ -227,15 +232,15 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) { require.True(t, stats.ExistsCopStats(tableScanID)) cop := stats.GetOrCreateCopStats(tableScanID, "tiflash") - require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 202ms}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4}", cop.String()) + require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_check_time: 15ms, total_read_time: 202ms, total_disagg_read_cache_hit_size: 100, total_disagg_read_cache_miss_size: 50}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4, total_learner_read_time: 1ms}", cop.String()) copStats := cop.stats["8.8.8.8"] require.NotNil(t, copStats) copStats.SetRowNum(10) copStats.Record(time.Second, 10) - require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 200ms}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4}", copStats.String()) - expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:3, total_skipped_packs:11, total_scanned_rows:20192, total_skipped_rows:86000, total_rs_index_load_time: 100ms, total_read_time: 3000ms}, total_create_snapshot_time: 50ms, total_local_region_num: 6, total_remote_region_num: 2}" + require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_check_time: 15ms, total_read_time: 200ms, total_disagg_read_cache_hit_size: 100, total_disagg_read_cache_miss_size: 50}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4, total_learner_read_time: 1ms}", copStats.String()) + expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:3, total_skipped_packs:11, total_scanned_rows:20192, total_skipped_rows:86000, total_rs_index_check_time: 100ms, total_read_time: 3000ms, total_disagg_read_cache_hit_size: 20, total_disagg_read_cache_miss_size: 0}, total_create_snapshot_time: 50ms, total_local_region_num: 6, total_remote_region_num: 2, total_learner_read_time: 0ms}" require.Equal(t, expected, stats.GetOrCreateCopStats(aggID, "tiflash").String()) rootStats := stats.GetRootStats(tableReaderID)