Skip to content

Commit

Permalink
command:add ctl command to get hot history regions(ref #25281) (#4103)
Browse files Browse the repository at this point in the history 
* command:add hot-history-regions command(ref #pingcap/tidb/issues/25281)

Signed-off-by: qidi1 <1083369179@qq.com>
  • Loading branch information
@qidi1
qidi1 authored Dec 21, 2021
1 parent af174e6 commit 447fbd4
Show file tree
Hide file tree
Showing 3 changed files with 320 additions and 33 deletions.
112 changes: 112 additions & 0 deletions tests/pdctl/hot/hot_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package hot_test
import (
"context"
"encoding/json"
"strconv"
"testing"
"time"

Expand Down Expand Up @@ -233,3 +234,114 @@ func (s *hotTestSuite) TestHotWithStoreID(c *C) {
c.Assert(hotRegion.AsLeader[1].TotalBytesRate, Equals, float64(200000000))
c.Assert(hotRegion.AsLeader[2].TotalBytesRate, Equals, float64(100000000))
}

func (s *hotTestSuite) TestHistoryHotRegions(c *C) {
statistics.Denoising = false
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cluster, err := tests.NewTestCluster(ctx, 1,
func(cfg *config.Config, serverName string) {
cfg.Schedule.HotRegionCacheHitsThreshold = 0
cfg.Schedule.HotRegionsWriteInterval.Duration = 1000 * time.Millisecond
cfg.Schedule.HotRegionsReservedDays = 1
},
)
c.Assert(err, IsNil)
err = cluster.RunInitialServers()
c.Assert(err, IsNil)
cluster.WaitLeader()
pdAddr := cluster.GetConfig().GetClientURL()
cmd := pdctlCmd.GetRootCmd()

stores := []*metapb.Store{
{
Id: 1,
State: metapb.StoreState_Up,
LastHeartbeat: time.Now().UnixNano(),
},
{
Id: 2,
State: metapb.StoreState_Up,
LastHeartbeat: time.Now().UnixNano(),
},
{
Id: 3,
State: metapb.StoreState_Up,
LastHeartbeat: time.Now().UnixNano(),
},
}

leaderServer := cluster.GetServer(cluster.GetLeader())
c.Assert(leaderServer.BootstrapCluster(), IsNil)
for _, store := range stores {
pdctl.MustPutStore(c, leaderServer.GetServer(), store)
}
defer cluster.Destroy()
startTime := time.Now().UnixNano() / int64(time.Millisecond)
pdctl.MustPutRegion(c, cluster, 1, 1, []byte("a"), []byte("b"), core.SetWrittenBytes(3000000000), core.SetReportInterval(statistics.WriteReportInterval))
pdctl.MustPutRegion(c, cluster, 2, 2, []byte("c"), []byte("d"), core.SetWrittenBytes(6000000000), core.SetReportInterval(statistics.WriteReportInterval))
pdctl.MustPutRegion(c, cluster, 3, 1, []byte("e"), []byte("f"), core.SetWrittenBytes(9000000000), core.SetReportInterval(statistics.WriteReportInterval))
pdctl.MustPutRegion(c, cluster, 4, 3, []byte("g"), []byte("h"), core.SetWrittenBytes(9000000000), core.SetReportInterval(statistics.WriteReportInterval))
// wait hot scheduler starts
time.Sleep(5000 * time.Millisecond)
endTime := time.Now().UnixNano() / int64(time.Millisecond)
start := strconv.FormatInt(startTime, 10)
end := strconv.FormatInt(endTime, 10)
args := []string{"-u", pdAddr, "hot", "history",
start, end,
"hot_region_type", "write",
"region_id", "1,2",
"store_id", "1,4",
"is_learner", "false",
}
output, e := pdctl.ExecuteCommand(cmd, args...)
hotRegions := core.HistoryHotRegions{}
c.Assert(e, IsNil)
c.Assert(json.Unmarshal(output, &hotRegions), IsNil)
regions := hotRegions.HistoryHotRegion
c.Assert(len(regions), Equals, 1)
c.Assert(regions[0].RegionID, Equals, uint64(1))
c.Assert(regions[0].StoreID, Equals, uint64(1))
c.Assert(regions[0].HotRegionType, Equals, "write")
args = []string{"-u", pdAddr, "hot", "history",
start, end,
"hot_region_type", "write",
"region_id", "1,2",
"store_id", "1,2",
}
output, e = pdctl.ExecuteCommand(cmd, args...)
c.Assert(e, IsNil)
c.Assert(json.Unmarshal(output, &hotRegions), IsNil)
regions = hotRegions.HistoryHotRegion
c.Assert(len(regions), Equals, 2)
isSort := regions[0].UpdateTime > regions[1].UpdateTime || regions[0].RegionID < regions[1].RegionID
c.Assert(isSort, Equals, true)
args = []string{"-u", pdAddr, "hot", "history",
start, end,
"hot_region_type", "read",
"is_leader", "false",
"peer_id", "12",
}
output, e = pdctl.ExecuteCommand(cmd, args...)
c.Assert(e, IsNil)
c.Assert(json.Unmarshal(output, &hotRegions), IsNil)
c.Assert(len(hotRegions.HistoryHotRegion), Equals, 0)
args = []string{"-u", pdAddr, "hot", "history"}
output, e = pdctl.ExecuteCommand(cmd, args...)
c.Assert(e, IsNil)
c.Assert(json.Unmarshal(output, &hotRegions), NotNil)
args = []string{"-u", pdAddr, "hot", "history",
start, end,
"region_id", "dada",
}
output, e = pdctl.ExecuteCommand(cmd, args...)
c.Assert(e, IsNil)
c.Assert(json.Unmarshal(output, &hotRegions), NotNil)
args = []string{"-u", pdAddr, "hot", "history",
start, end,
"region_ids", "12323",
}
output, e = pdctl.ExecuteCommand(cmd, args...)
c.Assert(e, IsNil)
c.Assert(json.Unmarshal(output, &hotRegions), NotNil)
}
95 changes: 65 additions & 30 deletions tools/pd-ctl/pdctl/command/global.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,26 +80,21 @@ func doRequest(cmd *cobra.Command, prefix string, method string,

endpoints := getEndpoints(cmd)
err := tryURLs(cmd, endpoints, func(endpoint string) error {
var err error
url := endpoint + "/" + prefix
if method == "" {
method = http.MethodGet
}
var req *http.Request
return doGet(endpoint, prefix, method, &resp, b)
})
return resp, err
}

req, err = http.NewRequest(method, url, b.body)
if err != nil {
return err
}
if b.contentType != "" {
req.Header.Set("Content-Type", b.contentType)
}
// the resp would be returned by the outer function
resp, err = dial(req)
if err != nil {
return err
}
return nil
func doRequestSingleEndpoint(cmd *cobra.Command, endpoint, prefix, method string,
opts ...BodyOption) (string, error) {
b := &bodyOption{}
for _, o := range opts {
o(b)
}
var resp string

err := requestURL(cmd, endpoint, func(endpoint string) error {
return doGet(endpoint, prefix, method, &resp, b)
})
return resp, err
}
Expand Down Expand Up @@ -134,20 +129,11 @@ type DoFunc func(endpoint string) error
func tryURLs(cmd *cobra.Command, endpoints []string, f DoFunc) error {
var err error
for _, endpoint := range endpoints {
var u *url.URL
u, err = url.Parse(endpoint)
endpoint, err = checkURL(endpoint)
if err != nil {
cmd.Println("address format is wrong, should like 'http://127.0.0.1:2379' or '127.0.0.1:2379'")
cmd.Println(err.Error())
os.Exit(1)
}
// tolerate some schemes that will be used by users, the TiKV SDK
// use 'tikv' as the scheme, it is really confused if we do not
// support it by pd-ctl
if u.Scheme == "" || u.Scheme == "pd" || u.Scheme == "tikv" {
u.Scheme = "http"
}

endpoint = u.String()
err = f(endpoint)
if err != nil {
continue
Expand All @@ -160,6 +146,15 @@ func tryURLs(cmd *cobra.Command, endpoints []string, f DoFunc) error {
return err
}

func requestURL(cmd *cobra.Command, endpoint string, f DoFunc) error {
endpoint, err := checkURL(endpoint)
if err != nil {
cmd.Println(err.Error())
os.Exit(1)
}
return f(endpoint)
}

func getEndpoints(cmd *cobra.Command) []string {
addrs, err := cmd.Flags().GetString("pd")
if err != nil {
Expand Down Expand Up @@ -207,3 +202,43 @@ func postJSON(cmd *cobra.Command, prefix string, input map[string]interface{}) {
}
cmd.Println("Success!")
}

// doGet send a get request to server.
func doGet(endpoint, prefix, method string, resp *string, b *bodyOption) error {
var err error
url := endpoint + "/" + prefix
if method == "" {
method = http.MethodGet
}
var req *http.Request

req, err = http.NewRequest(method, url, b.body)
if err != nil {
return err
}
if b.contentType != "" {
req.Header.Set("Content-Type", b.contentType)
}
// the resp would be returned by the outer function
*resp, err = dial(req)
if err != nil {
return err
}
return nil
}

func checkURL(endpoint string) (string, error) {
var u *url.URL
u, err := url.Parse(endpoint)
if err != nil {
return "", errors.Errorf("address format is wrong, should like 'http://127.0.0.1:2379' or '127.0.0.1:2379'")
}
// tolerate some schemes that will be used by users, the TiKV SDK
// use 'tikv' as the scheme, it is really confused if we do not
// support it by pd-ctl
if u.Scheme == "" || u.Scheme == "pd" || u.Scheme == "tikv" {
u.Scheme = "http"
}

return u.String(), nil
}
Loading

0 comments on commit 447fbd4

Please sign in to comment.