diff --git a/tests/pdctl/hot/hot_test.go b/tests/pdctl/hot/hot_test.go index cc4e76b1e9b5..4c07c8fdc14c 100644 --- a/tests/pdctl/hot/hot_test.go +++ b/tests/pdctl/hot/hot_test.go @@ -16,6 +16,7 @@ package hot_test import ( "context" "encoding/json" + "strconv" "testing" "time" @@ -219,3 +220,83 @@ func (s *hotTestSuite) TestHotWithStoreID(c *C) { c.Assert(hotRegion.AsLeader[1].TotalBytesRate, Equals, float64(200000000)) c.Assert(hotRegion.AsLeader[2].TotalBytesRate, Equals, float64(100000000)) } + +func (s *hotTestSuite) TestHistoryHotRegions(c *C) { + statistics.Denoising = false + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cluster, err := tests.NewTestCluster(ctx, 1, + func(cfg *config.Config, serverName string) { + cfg.Schedule.HotRegionCacheHitsThreshold = 0 + cfg.Schedule.HotRegionsWriteInterval.Duration = 1000 * time.Millisecond + cfg.Schedule.HotRegionsResevervedDays = 1 + }, + ) + c.Assert(err, IsNil) + err = cluster.RunInitialServers() + c.Assert(err, IsNil) + cluster.WaitLeader() + pdAddr := cluster.GetConfig().GetClientURL() + cmd := pdctlCmd.GetRootCmd() + + stores := []*metapb.Store{ + { + Id: 1, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + { + Id: 2, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + { + Id: 3, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + } + + leaderServer := cluster.GetServer(cluster.GetLeader()) + c.Assert(leaderServer.BootstrapCluster(), IsNil) + for _, store := range stores { + pdctl.MustPutStore(c, leaderServer.GetServer(), store) + } + defer cluster.Destroy() + startTime := time.Now().UnixNano() / int64(time.Millisecond) + pdctl.MustPutRegion(c, cluster, 1, 1, []byte("a"), []byte("b"), core.SetWrittenBytes(3000000000), core.SetReportInterval(statistics.WriteReportInterval)) + pdctl.MustPutRegion(c, cluster, 2, 2, []byte("c"), []byte("d"), core.SetWrittenBytes(6000000000), core.SetReportInterval(statistics.WriteReportInterval)) + pdctl.MustPutRegion(c, cluster, 3, 1, []byte("e"), []byte("f"), core.SetWrittenBytes(9000000000), core.SetReportInterval(statistics.WriteReportInterval)) + pdctl.MustPutRegion(c, cluster, 4, 3, []byte("g"), []byte("h"), core.SetWrittenBytes(9000000000), core.SetReportInterval(statistics.WriteReportInterval)) + // wait hot scheduler starts + time.Sleep(5000 * time.Millisecond) + endTime := time.Now().UnixNano() / int64(time.Millisecond) + start := strconv.FormatInt(startTime, 10) + end := strconv.FormatInt(endTime, 10) + args := []string{"-u", pdAddr, "hot", "history", + start, end, + "hot_region_type", "write", + "region_ids", "1,2,3", + "store_ids", "1,4", + } + output, e := pdctl.ExecuteCommand(cmd, args...) + hotRegions := core.HistoryHotRegions{} + c.Assert(e, IsNil) + c.Assert(json.Unmarshal(output, &hotRegions), IsNil) + regions := hotRegions.HistoryHotRegion + c.Assert(len(regions), Equals, 2) + c.Assert(regions[0].RegionID, Equals, uint64(1)) + c.Assert(regions[0].StoreID, Equals, uint64(1)) + c.Assert(regions[0].HotRegionType, Equals, "write") + c.Assert(regions[1].RegionID, Equals, uint64(3)) + c.Assert(regions[1].StoreID, Equals, uint64(1)) + c.Assert(regions[1].HotRegionType, Equals, "write") + args = []string{"-u", pdAddr, "hot", "history", + start, end, + "is_leader", "false", + } + output, e = pdctl.ExecuteCommand(cmd, args...) + c.Assert(e, IsNil) + c.Assert(json.Unmarshal(output, &hotRegions), IsNil) + c.Assert(len(hotRegions.HistoryHotRegion), Equals, 0) +} diff --git a/tools/pd-ctl/pdctl/command/global.go b/tools/pd-ctl/pdctl/command/global.go index bc60fea1068a..9c9787fd3d0d 100644 --- a/tools/pd-ctl/pdctl/command/global.go +++ b/tools/pd-ctl/pdctl/command/global.go @@ -79,26 +79,21 @@ func doRequest(cmd *cobra.Command, prefix string, method string, endpoints := getEndpoints(cmd) err := tryURLs(cmd, endpoints, func(endpoint string) error { - var err error - url := endpoint + "/" + prefix - if method == "" { - method = http.MethodGet - } - var req *http.Request + return doGet(endpoint, prefix, method, &resp, b) + }) + return resp, err +} - req, err = http.NewRequest(method, url, b.body) - if err != nil { - return err - } - if b.contentType != "" { - req.Header.Set("Content-Type", b.contentType) - } - // the resp would be returned by the outer function - resp, err = dial(req) - if err != nil { - return err - } - return nil +func doRequestSingleEndpoint(cmd *cobra.Command, endpoint, prefix, method string, + opts ...BodyOption) (string, error) { + b := &bodyOption{} + for _, o := range opts { + o(b) + } + var resp string + + err := requestURL(cmd, endpoint, func(endpoint string) error { + return doGet(endpoint, prefix, method, &resp, b) }) return resp, err } @@ -133,20 +128,11 @@ type DoFunc func(endpoint string) error func tryURLs(cmd *cobra.Command, endpoints []string, f DoFunc) error { var err error for _, endpoint := range endpoints { - var u *url.URL - u, err = url.Parse(endpoint) + endpoint, err := checkURL(endpoint) if err != nil { - cmd.Println("address format is wrong, should like 'http://127.0.0.1:2379' or '127.0.0.1:2379'") + cmd.Println(err.Error()) os.Exit(1) } - // tolerate some schemes that will be used by users, the TiKV SDK - // use 'tikv' as the scheme, it is really confused if we do not - // support it by pd-ctl - if u.Scheme == "" || u.Scheme == "pd" || u.Scheme == "tikv" { - u.Scheme = "http" - } - - endpoint = u.String() err = f(endpoint) if err != nil { continue @@ -159,6 +145,15 @@ func tryURLs(cmd *cobra.Command, endpoints []string, f DoFunc) error { return err } +func requestURL(cmd *cobra.Command, endpoint string, f DoFunc) error { + endpoint, err := checkURL(endpoint) + if err != nil { + cmd.Println(err.Error()) + os.Exit(1) + } + return f(endpoint) +} + func getEndpoints(cmd *cobra.Command) []string { addrs, err := cmd.Flags().GetString("pd") if err != nil { @@ -206,3 +201,43 @@ func postJSON(cmd *cobra.Command, prefix string, input map[string]interface{}) { } cmd.Println("Success!") } + +// doGet send a get request to server. +func doGet(endpoint, prefix, method string, resp *string, b *bodyOption) error { + var err error + url := endpoint + "/" + prefix + if method == "" { + method = http.MethodGet + } + var req *http.Request + + req, err = http.NewRequest(method, url, b.body) + if err != nil { + return err + } + if b.contentType != "" { + req.Header.Set("Content-Type", b.contentType) + } + // the resp would be returned by the outer function + *resp, err = dial(req) + if err != nil { + return err + } + return nil +} + +func checkURL(endpoint string) (string, error) { + var u *url.URL + u, err := url.Parse(endpoint) + if err != nil { + return "", errors.Errorf("address format is wrong, should like 'http://127.0.0.1:2379' or '127.0.0.1:2379'") + } + // tolerate some schemes that will be used by users, the TiKV SDK + // use 'tikv' as the scheme, it is really confused if we do not + // support it by pd-ctl + if u.Scheme == "" || u.Scheme == "pd" || u.Scheme == "tikv" { + u.Scheme = "http" + } + + return u.String(), nil +} diff --git a/tools/pd-ctl/pdctl/command/hot_command.go b/tools/pd-ctl/pdctl/command/hot_command.go index 17e2e8e6c745..37ef9942445b 100644 --- a/tools/pd-ctl/pdctl/command/hot_command.go +++ b/tools/pd-ctl/pdctl/command/hot_command.go @@ -14,17 +14,23 @@ package command import ( + "bytes" + "encoding/json" "net/http" + "sort" "strconv" + "strings" "github.com/pingcap/errors" "github.com/spf13/cobra" + "github.com/tikv/pd/server/core" ) const ( - hotReadRegionsPrefix = "pd/api/v1/hotspot/regions/read" - hotWriteRegionsPrefix = "pd/api/v1/hotspot/regions/write" - hotStoresPrefix = "pd/api/v1/hotspot/stores" + hotReadRegionsPrefix = "pd/api/v1/hotspot/regions/read" + hotWriteRegionsPrefix = "pd/api/v1/hotspot/regions/write" + hotStoresPrefix = "pd/api/v1/hotspot/stores" + hotRegionsHistoryPrefix = "pd/api/v1/hotspot/regions/history" ) // NewHotSpotCommand return a hot subcommand of rootCmd @@ -36,6 +42,7 @@ func NewHotSpotCommand() *cobra.Command { cmd.AddCommand(NewHotWriteRegionCommand()) cmd.AddCommand(NewHotReadRegionCommand()) cmd.AddCommand(NewHotStoreCommand()) + cmd.AddCommand(NewHotRegionsHistoryCommand()) return cmd } @@ -106,6 +113,55 @@ func showHotStoresCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } +// NewHotRegionsHistoryCommand return a hot history regions subcommand of hotSpotCmd +func NewHotRegionsHistoryCommand() *cobra.Command { + cmd := &cobra.Command{ + // TODO + // Need a better description. + Use: "history [ ]", + Short: "show the hot history regions", + Run: showHotRegionsHistoryCommandFunc, + } + return cmd +} + +func showHotRegionsHistoryCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 2 || len(args)%2 != 0 { + cmd.Println(cmd.UsageString()) + } + input, err := parseHotRegionsHistoryArgs(args) + if err != nil { + cmd.Printf("Failed to get history hotspot: %s\n", err) + } + data, _ := json.Marshal(input) + endpoints := getEndpoints(cmd) + hotRegions := &core.HistoryHotRegions{} + for _, endpoint := range endpoints { + tempHotRegions := core.HistoryHotRegions{} + resp, err := doRequestSingleEndpoint(cmd, endpoint, hotRegionsHistoryPrefix, + http.MethodGet, WithBody("application/json", bytes.NewBuffer(data))) + if err != nil { + cmd.Printf("Failed to get history hotspot: %s\n", err) + return + } + err = json.Unmarshal([]byte(resp), &tempHotRegions) + if err != nil { + cmd.Printf("Failed to get history hotspot: %s\n", err) + return + } + hotRegions.HistoryHotRegion = append(hotRegions.HistoryHotRegion, tempHotRegions.HistoryHotRegion...) + } + sort.SliceStable(hotRegions.HistoryHotRegion, func(i, j int) bool { + return hotRegions.HistoryHotRegion[i].UpdateTime > hotRegions.HistoryHotRegion[j].UpdateTime + }) + resp, err := json.Marshal(hotRegions) + if err != nil { + cmd.Printf("Failed to get history hotspot: %s\n", err) + return + } + cmd.Println(string(resp)) +} + func parseOptionalArgs(cmd *cobra.Command, prefix string, args []string) (string, error) { argsLen := len(args) if argsLen > 0 { @@ -123,3 +179,81 @@ func parseOptionalArgs(cmd *cobra.Command, prefix string, args []string) (string } return prefix, nil } + +func parseHotRegionsHistoryArgs(args []string) (map[string]interface{}, error) { + startTime, err := strconv.ParseInt(args[0], 10, 64) + if err != nil { + return nil, errors.Errorf("start_time should be a number,but got %s", args[0]) + } + endTime, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + return nil, errors.Errorf("end_time should be a number,but got %s", args[1]) + } + input := map[string]interface{}{ + "start_time": startTime, + "end_time": endTime, + } + stringToIntSlice := func(s string) ([]int64, error) { + results := make([]int64, 0) + args := strings.Split(s, ",") + for _, arg := range args { + result, err := strconv.ParseInt(arg, 10, 64) + if err != nil { + return nil, err + } + results = append(results, result) + } + return results, nil + } + for index := 2; index < len(args); index += 2 { + switch args[index] { + case "hot_region_type": + input["hot_region_type"] = []string{args[index+1]} + case "region_ids": + results, err := stringToIntSlice(args[index+1]) + if err != nil { + return nil, errors.Errorf("region_ids should be a number slice,but got %s", args[index+1]) + } + input["region_ids"] = results + case "store_ids": + results, err := stringToIntSlice(args[index+1]) + if err != nil { + return nil, errors.Errorf("store_ids should be a number slice,but got %s", args[index+1]) + } + input["store_ids"] = results + case "peer_ids": + results, err := stringToIntSlice(args[index+1]) + if err != nil { + return nil, errors.Errorf("peer_ids should be a number slice,but got %s", args[index+1]) + } + input["peer_ids"] = results + case "is_leader": + isLeader, err := strconv.ParseBool(args[index+1]) + if err != nil { + return nil, errors.Errorf("is_leader should be a bool,but got %s", args[index+1]) + } + input["is_leaders"] = []bool{isLeader} + case "is_learner": + isLearner, err := strconv.ParseBool(args[index+1]) + if err != nil { + return nil, errors.Errorf("is_learners should be a bool,but got %s", args[index+1]) + } + input["is_learners"] = []bool{isLearner} + default: + return nil, errors.Errorf("key should be one of hot_region_type,region_ids,store_ids,peer_ids,is_leaders,is_learners") + } + } + if _, ok := input["is_leaders"]; !ok { + input["is_leaders"] = []bool{ + true, + false, + } + } + if _, ok := input["is_learners"]; !ok { + input["is_learners"] = []bool{ + true, + false, + } + } + return input, nil +}