Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

infosync: fully integrated PD HTTP client into InfoSyncer #49006

Merged
merged 5 commits into from
Dec 1, 2023
Merged

infosync: fully integrated PD HTTP client into InfoSyncer #49006

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
72f3c32
Fully integrated PD HTTP client into InfoSyncer
JmPotato Nov 29, 2023
296a147
Fix the InfoSyncer test
JmPotato Nov 29, 2023
326330c
Reduce the code
JmPotato Nov 29, 2023
77e05cd
Update the PD HTTP client version
JmPotato Nov 30, 2023
f8cf570
Update the client-go version
JmPotato Dec 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions DEPS.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7006,26 +7006,26 @@ def go_deps():
name = "com_github_tikv_client_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/client-go/v2",
sha256 = "548df2ca5c27559e3318b97b4cb91703d5c253410e7f9fa0eb926e2d3aa28b59",
strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20231116051730-1c2351c28173",
sha256 = "c6c5bfe8a84d0b680266035d1ac87af6beca4cc552693d6753d4a9f18610c30d",
strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20231201024404-0ff16620f6c0",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231116051730-1c2351c28173.zip",
"http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231116051730-1c2351c28173.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231116051730-1c2351c28173.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231116051730-1c2351c28173.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231201024404-0ff16620f6c0.zip",
"http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231201024404-0ff16620f6c0.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231201024404-0ff16620f6c0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231201024404-0ff16620f6c0.zip",
],
)
go_repository(
name = "com_github_tikv_pd_client",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/pd/client",
sha256 = "5232ba0bba677a6d4614ae2cc102554d59cd00d473d9138739508d6f25169f02",
strip_prefix = "github.com/tikv/pd/client@v0.0.0-20231127075044-9f4803d8bd05",
sha256 = "a6dcccfd436232d847d2ba2669d67d9708d7bc97f7571a4e57da18fdefeed852",
strip_prefix = "github.com/tikv/pd/client@v0.0.0-20231130081618-862eee18738e",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231127075044-9f4803d8bd05.zip",
"http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231127075044-9f4803d8bd05.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231127075044-9f4803d8bd05.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231127075044-9f4803d8bd05.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231130081618-862eee18738e.zip",
"http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231130081618-862eee18738e.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231130081618-862eee18738e.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20231130081618-862eee18738e.zip",
],
)
go_repository(
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/lightning/backend/local/local_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -668,7 +668,7 @@ func (c *mockPdClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOpti
return c.stores, nil
}

func (c *mockPdClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*pd.Region, error) {
func (c *mockPdClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int, opts ...pd.GetRegionOption) ([]*pd.Region, error) {
return c.regions, nil
}

Expand Down
4 changes: 3 additions & 1 deletion br/pkg/pdutil/pd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,9 @@ func (p *PdController) getRegionCountWith(
}
var err error
for _, addr := range p.getAllPDAddrs() {
v, e := get(ctx, addr, pdhttp.RegionStatsByKeyRange(pdhttp.NewKeyRange(start, end)), p.cli, http.MethodGet, nil)
v, e := get(ctx, addr,
pdhttp.RegionStatsByKeyRange(pdhttp.NewKeyRange(start, end), false),
p.cli, http.MethodGet, nil)
if e != nil {
err = e
continue
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/streamhelper/basic_lib_for_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,7 @@ type mockPDClient struct {
fakeRegions []*region
}

func (p *mockPDClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*pd.Region, error) {
func (p *mockPDClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int, _ ...pd.GetRegionOption) ([]*pd.Region, error) {
sort.Slice(p.fakeRegions, func(i, j int) bool {
return bytes.Compare(p.fakeRegions[i].rng.StartKey, p.fakeRegions[j].rng.StartKey) < 0
})
Expand Down
3 changes: 2 additions & 1 deletion br/tests/br_key_locked/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,14 @@ func (c *codecPDClient) ScanRegions(
startKey []byte,
endKey []byte,
limit int,
opts ...pd.GetRegionOption,
) ([]*pd.Region, error) {
startKey = codec.EncodeBytes(nil, startKey)
if len(endKey) > 0 {
endKey = codec.EncodeBytes(nil, endKey)
}

regions, err := c.Client.ScanRegions(ctx, startKey, endKey, limit)
regions, err := c.Client.ScanRegions(ctx, startKey, endKey, limit, opts...)
if err != nil {
return nil, errors.Trace(err)
}
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,8 @@ require (
github.com/stretchr/testify v1.8.4
github.com/tdakkota/asciicheck v0.2.0
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
github.com/tikv/client-go/v2 v2.0.8-0.20231116051730-1c2351c28173
github.com/tikv/pd/client v0.0.0-20231127075044-9f4803d8bd05
github.com/tikv/client-go/v2 v2.0.8-0.20231201024404-0ff16620f6c0
github.com/tikv/pd/client v0.0.0-20231130081618-862eee18738e
github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966
github.com/twmb/murmur3 v1.1.6
github.com/uber/jaeger-client-go v2.22.1+incompatible
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -851,10 +851,10 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJf
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
github.com/tikv/client-go/v2 v2.0.8-0.20231116051730-1c2351c28173 h1:lmJzX0kqrV7kO21wrZPbtjkidzwbDCfXeQrhDWEi5dE=
github.com/tikv/client-go/v2 v2.0.8-0.20231116051730-1c2351c28173/go.mod h1:BOGTSZtbMHEnGC4HOpbONdnTQF+E9nb2Io7c3P9sb7g=
github.com/tikv/pd/client v0.0.0-20231127075044-9f4803d8bd05 h1:87NPUfzaVrO5MTBwVCPQ/FlJGpFnHi6WFYHDYD3n3Zc=
github.com/tikv/pd/client v0.0.0-20231127075044-9f4803d8bd05/go.mod h1:cd6zBqRM9aogxf26K8NnFRPVtq9BnRE59tKEpX8IaWQ=
github.com/tikv/client-go/v2 v2.0.8-0.20231201024404-0ff16620f6c0 h1:am8ME/PpDyhIM6oFg7QLq1FswcuL2DHi4PRBzkSDgGY=
github.com/tikv/client-go/v2 v2.0.8-0.20231201024404-0ff16620f6c0/go.mod h1:M+i6/DR7y1J7mG3SOdrmdkuLZe1gEqkdiV8J+mX1DiI=
github.com/tikv/pd/client v0.0.0-20231130081618-862eee18738e h1:11cWLLmEreKof/VJi6LLQ+Jkav5ZqPJgeI+KX4pc/DE=
github.com/tikv/pd/client v0.0.0-20231130081618-862eee18738e/go.mod h1:cd6zBqRM9aogxf26K8NnFRPVtq9BnRE59tKEpX8IaWQ=
github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 h1:quvGphlmUVU+nhpFa4gg4yJyTRJ13reZMDHrKwYw53M=
github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966/go.mod h1:27bSVNWSBOHm+qRp1T9qzaIpsWEP6TbUnei/43HK+PQ=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
Expand Down
99 changes: 0 additions & 99 deletions pkg/ddl/placement/rule.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,10 @@
package placement

import (
"encoding/hex"
"encoding/json"
"fmt"
"regexp"
"strings"

"github.com/pingcap/tidb/pkg/util/codec"
pd "github.com/tikv/pd/client/http"
"gopkg.in/yaml.v2"
)
Expand All @@ -32,102 +29,6 @@ const (
attributeEvictLeader = "evict-leader"
)

// RuleGroupConfig defines basic config of rule group
type RuleGroupConfig struct {
ID string `json:"id"`
Index int `json:"index"`
Override bool `json:"override"`
}

var (
_ json.Marshaler = (*TiFlashRule)(nil)
_ json.Unmarshaler = (*TiFlashRule)(nil)
)

// TiFlashRule extends Rule with other necessary fields.
type TiFlashRule struct {
GroupID string
ID string
Index int
Override bool
Role pd.PeerRoleType
Count int
Constraints []pd.LabelConstraint
LocationLabels []string
IsolationLevel string
StartKey []byte
EndKey []byte
}

type tiFlashRule struct {
GroupID string `json:"group_id"`
ID string `json:"id"`
Index int `json:"index,omitempty"`
Override bool `json:"override,omitempty"`
Role pd.PeerRoleType `json:"role"`
Count int `json:"count"`
Constraints []pd.LabelConstraint `json:"label_constraints,omitempty"`
LocationLabels []string `json:"location_labels,omitempty"`
IsolationLevel string `json:"isolation_level,omitempty"`
StartKeyHex string `json:"start_key"`
EndKeyHex string `json:"end_key"`
}

// MarshalJSON implements json.Marshaler interface for TiFlashRule.
func (r *TiFlashRule) MarshalJSON() ([]byte, error) {
return json.Marshal(&tiFlashRule{
GroupID: r.GroupID,
ID: r.ID,
Index: r.Index,
Override: r.Override,
Role: r.Role,
Count: r.Count,
Constraints: r.Constraints,
LocationLabels: r.LocationLabels,
IsolationLevel: r.IsolationLevel,
StartKeyHex: hex.EncodeToString(codec.EncodeBytes(nil, r.StartKey)),
EndKeyHex: hex.EncodeToString(codec.EncodeBytes(nil, r.EndKey)),
})
}

// UnmarshalJSON implements json.Unmarshaler interface for TiFlashRule.
func (r *TiFlashRule) UnmarshalJSON(bytes []byte) error {
var rule tiFlashRule
if err := json.Unmarshal(bytes, &rule); err != nil {
return err
}
*r = TiFlashRule{
GroupID: rule.GroupID,
ID: rule.ID,
Index: rule.Index,
Override: rule.Override,
Role: rule.Role,
Count: rule.Count,
Constraints: rule.Constraints,
LocationLabels: rule.LocationLabels,
IsolationLevel: rule.IsolationLevel,
}

startKey, err := hex.DecodeString(rule.StartKeyHex)
if err != nil {
return err
}

endKey, err := hex.DecodeString(rule.EndKeyHex)
if err != nil {
return err
}

_, r.StartKey, err = codec.DecodeBytes(startKey, nil)
if err != nil {
return err
}

_, r.EndKey, err = codec.DecodeBytes(endKey, nil)

return err
}

// RuleBuilder is used to build the Rules from a constraint string.
type RuleBuilder struct {
role pd.PeerRoleType
Expand Down
1 change: 1 addition & 0 deletions pkg/domain/domain_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ func TestInfo(t *testing.T) {
t.Skip("ETCD use ip:port as unix socket address, skip when it is unavailable.")
}

// NOTICE: this failpoint has been REMOVED, be aware of this if you want to reopen this test.
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/domain/infosync/FailPlacement", `return(true)`))

s, err := mockstore.NewMockStore()
Expand Down
2 changes: 1 addition & 1 deletion pkg/domain/infosync/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ go_test(
srcs = ["info_test.go"],
embed = [":infosync"],
flaky = True,
shard_count = 4,
shard_count = 3,
deps = [
"//pkg/ddl/placement",
"//pkg/ddl/util",
Expand Down
Loading
Loading