Skip to content

Commit

Permalink
Explain : Add new item in TiFlashScanContext to record region num inf…
Browse files Browse the repository at this point in the history 
…ormation (#40666)

ref #39273
  • Loading branch information
@hongyunyan
hongyunyan authored Jan 20, 2023
1 parent 38a7c62 commit c8dc908
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 10 deletions.
12 changes: 10 additions & 2 deletions util/execdetails/execdetails.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,9 @@ func (crs *CopRuntimeStats) RecordOneCopTask(address string, summary *tipb.Execu
totalDmfileSkippedRows: summary.GetTiflashScanContext().GetTotalDmfileSkippedRows(),
totalDmfileRoughSetIndexLoadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileRoughSetIndexLoadTimeMs(),
totalDmfileReadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileReadTimeMs(),
totalCreateSnapshotTimeMs: summary.GetTiflashScanContext().GetTotalCreateSnapshotTimeMs()}}, threads: int32(summary.GetConcurrency()),
totalCreateSnapshotTimeMs: summary.GetTiflashScanContext().GetTotalCreateSnapshotTimeMs(),
totalLocalRegionNum: summary.GetTiflashScanContext().GetTotalLocalRegionNum(),
totalRemoteRegionNum: summary.GetTiflashScanContext().GetTotalRemoteRegionNum()}}, threads: int32(summary.GetConcurrency()),
totalTasks: 1,
})
}
Expand Down Expand Up @@ -538,6 +540,8 @@ type TiFlashScanContext struct {
totalDmfileRoughSetIndexLoadTimeMs uint64
totalDmfileReadTimeMs uint64
totalCreateSnapshotTimeMs uint64
totalLocalRegionNum uint64
totalRemoteRegionNum uint64
}

// Clone implements the deep copy of * TiFlashshScanContext
Expand All @@ -550,10 +554,12 @@ func (context *TiFlashScanContext) Clone() TiFlashScanContext {
totalDmfileRoughSetIndexLoadTimeMs: context.totalDmfileRoughSetIndexLoadTimeMs,
totalDmfileReadTimeMs: context.totalDmfileReadTimeMs,
totalCreateSnapshotTimeMs: context.totalCreateSnapshotTimeMs,
totalLocalRegionNum: context.totalLocalRegionNum,
totalRemoteRegionNum: context.totalRemoteRegionNum,
}
}
func (context *TiFlashScanContext) String() string {
return fmt.Sprintf("tiflash_scan:{dtfile:{total_scanned_packs:%d, total_skipped_packs:%d, total_scanned_rows:%d, total_skipped_rows:%d, total_rs_index_load_time: %dms, total_read_time: %dms}, total_create_snapshot_time: %dms}", context.totalDmfileScannedPacks, context.totalDmfileSkippedPacks, context.totalDmfileScannedRows, context.totalDmfileSkippedRows, context.totalDmfileRoughSetIndexLoadTimeMs, context.totalDmfileReadTimeMs, context.totalCreateSnapshotTimeMs)
return fmt.Sprintf("tiflash_scan:{dtfile:{total_scanned_packs:%d, total_skipped_packs:%d, total_scanned_rows:%d, total_skipped_rows:%d, total_rs_index_load_time: %dms, total_read_time: %dms}, total_create_snapshot_time: %dms, total_local_region_num: %d, total_remote_region_num: %d}", context.totalDmfileScannedPacks, context.totalDmfileSkippedPacks, context.totalDmfileScannedRows, context.totalDmfileSkippedRows, context.totalDmfileRoughSetIndexLoadTimeMs, context.totalDmfileReadTimeMs, context.totalCreateSnapshotTimeMs, context.totalLocalRegionNum, context.totalRemoteRegionNum)
}

// Merge make sum to merge the information in TiFlashScanContext
Expand All @@ -565,6 +571,8 @@ func (context *TiFlashScanContext) Merge(other TiFlashScanContext) {
context.totalDmfileRoughSetIndexLoadTimeMs += other.totalDmfileRoughSetIndexLoadTimeMs
context.totalDmfileReadTimeMs += other.totalDmfileReadTimeMs
context.totalCreateSnapshotTimeMs += other.totalCreateSnapshotTimeMs
context.totalLocalRegionNum += other.totalLocalRegionNum
context.totalRemoteRegionNum += other.totalRemoteRegionNum
}

// Empty check whether TiFlashScanContext is Empty, if scan no pack and skip no pack, we regard it as empty
Expand Down
18 changes: 10 additions & 8 deletions util/execdetails/execdetails_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func mockExecutorExecutionSummary(TimeProcessedNs, NumProducedRows, NumIteration
NumIterations: &NumIterations, XXX_unrecognized: nil}
}

func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, totalDmfileScannedPacks, totalDmfileScannedRows, totalDmfileSkippedPacks, totalDmfileSkippedRows, totalDmfileRoughSetIndexLoadTimeMs, totalDmfileReadTimeMs, totalCreateSnapshotTimeMs uint64, ExecutorID string) *tipb.ExecutorExecutionSummary {
func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, totalDmfileScannedPacks, totalDmfileScannedRows, totalDmfileSkippedPacks, totalDmfileSkippedRows, totalDmfileRoughSetIndexLoadTimeMs, totalDmfileReadTimeMs, totalCreateSnapshotTimeMs uint64, totalLocalRegionNum uint64, totalRemoteRegionNum uint64, ExecutorID string) *tipb.ExecutorExecutionSummary {
tiflashScanContext := tipb.TiFlashScanContext{
TotalDmfileScannedPacks: &totalDmfileScannedPacks,
TotalDmfileSkippedPacks: &totalDmfileSkippedPacks,
Expand All @@ -144,6 +144,8 @@ func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, Nu
TotalDmfileRoughSetIndexLoadTimeMs: &totalDmfileRoughSetIndexLoadTimeMs,
TotalDmfileReadTimeMs: &totalDmfileReadTimeMs,
TotalCreateSnapshotTimeMs: &totalCreateSnapshotTimeMs,
TotalLocalRegionNum: &totalLocalRegionNum,
TotalRemoteRegionNum: &totalRemoteRegionNum,
}
return &tipb.ExecutorExecutionSummary{TimeProcessedNs: &TimeProcessedNs, NumProducedRows: &NumProducedRows,
NumIterations: &NumIterations, Concurrency: &Concurrency, ExecutorId: &ExecutorID, DetailInfo: &tipb.ExecutorExecutionSummary_TiflashScanContext{TiflashScanContext: &tiflashScanContext}, XXX_unrecognized: nil}
Expand Down Expand Up @@ -208,10 +210,10 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) {
tableScanID := 1
aggID := 2
tableReaderID := 3
stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 1, 8192, 0, 0, 15, 200, 40, "tablescan_"+strconv.Itoa(tableScanID)))
stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, "tablescan_"+strconv.Itoa(tableScanID)))
stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 2, 12000, 1, 6000, 60, 1000, 20, "aggregation_"+strconv.Itoa(aggID)))
stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 1, 8192, 10, 80000, 40, 2000, 30, "aggregation_"+strconv.Itoa(aggID)))
stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 1, 8192, 0, 0, 15, 200, 40, 10, 4, "tablescan_"+strconv.Itoa(tableScanID)))
stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, "tablescan_"+strconv.Itoa(tableScanID)))
stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 2, 12000, 1, 6000, 60, 1000, 20, 5, 1, "aggregation_"+strconv.Itoa(aggID)))
stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 1, 8192, 10, 80000, 40, 2000, 30, 1, 1, "aggregation_"+strconv.Itoa(aggID)))
scanDetail := &util.ScanDetail{
TotalKeys: 10,
ProcessedKeys: 10,
Expand All @@ -225,15 +227,15 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) {
require.True(t, stats.ExistsCopStats(tableScanID))

cop := stats.GetOrCreateCopStats(tableScanID, "tiflash")
require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 202ms}, total_create_snapshot_time: 40ms}", cop.String())
require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 202ms}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4}", cop.String())

copStats := cop.stats["8.8.8.8"]
require.NotNil(t, copStats)

copStats.SetRowNum(10)
copStats.Record(time.Second, 10)
require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 200ms}, total_create_snapshot_time: 40ms}", copStats.String())
expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:3, total_skipped_packs:11, total_scanned_rows:20192, total_skipped_rows:86000, total_rs_index_load_time: 100ms, total_read_time: 3000ms}, total_create_snapshot_time: 50ms}"
require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 200ms}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4}", copStats.String())
expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:3, total_skipped_packs:11, total_scanned_rows:20192, total_skipped_rows:86000, total_rs_index_load_time: 100ms, total_read_time: 3000ms}, total_create_snapshot_time: 50ms, total_local_region_num: 6, total_remote_region_num: 2}"
require.Equal(t, expected, stats.GetOrCreateCopStats(aggID, "tiflash").String())

rootStats := stats.GetRootStats(tableReaderID)
Expand Down

0 comments on commit c8dc908

Please sign in to comment.