diff --git a/.github/workflows/check-dashboard.yaml b/.github/workflows/check-dashboard.yaml deleted file mode 100644 index d8a7fafb541f..000000000000 --- a/.github/workflows/check-dashboard.yaml +++ /dev/null @@ -1,53 +0,0 @@ -on: - push: - branches: - - master - - release-4.0 - - release-5.* - paths: - - "go.mod" - pull_request: - branches: - - master - - release-4.0 - - release-5.* - paths: - - "go.mod" - -name: TiDB Dashboard - -jobs: - build: - name: Check UI up-to-date - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@master - - uses: actions/setup-go@v1 - with: - go-version: "1.16.4" - - uses: actions/cache@v1 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Fetch Go module - run: | - go mod download - go mod tidy - DASHBOARD_DIR=$(go list -f "{{.Dir}}" -m github.com/pingcap/tidb-dashboard) - DASHBOARD_RELEASE_VERSION=$(grep -v '^#' "${DASHBOARD_DIR}/release-version") - echo DASHBOARD_DIR="$DASHBOARD_DIR" >> $GITHUB_ENV - echo DASHBOARD_RELEASE_VERSION="$DASHBOARD_RELEASE_VERSION" >> $GITHUB_ENV - - name: Fetch UI source specified by release version - run: | - rm -rf /tmp/dashboard - mkdir -p /tmp/dashboard - cd /tmp/dashboard - DOWNLOAD_URL="https://github.com/pingcap/tidb-dashboard/archive/v${DASHBOARD_RELEASE_VERSION}.zip" - curl -L "${DOWNLOAD_URL}" --fail --output source.zip - unzip source.zip - - name: Check diff - run: | - diff -qr ${DASHBOARD_DIR}/ui/ /tmp/dashboard/tidb-dashboard-${DASHBOARD_RELEASE_VERSION}/ui/ diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index f9700e9f5f8e..cfc113cfbd44 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -16,7 +16,7 @@ jobs: ~/go/pkg/mod ~/.cache/go-build **/.tools - **/.dashboard_asset_cache + **/.dashboard_download_cache key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-golang diff --git a/.github/workflows/pd-tests.yaml b/.github/workflows/pd-tests.yaml index d0c4717a7729..c29a68ec11a2 100644 --- a/.github/workflows/pd-tests.yaml +++ b/.github/workflows/pd-tests.yaml @@ -32,7 +32,7 @@ jobs: ~/go/pkg/mod ~/.cache/go-build **/.tools - **/.dashboard_asset_cache + **/.dashboard_download_cache key: ${{ runner.os }}-go-${{ matrix.worker_id }}-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go-${{ matrix.worker_id }} diff --git a/.gitignore b/.gitignore index db35cd73a5ab..fbe6a8595a8d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ vendor default* *.bak .vscode/ -/.dashboard_asset_cache +/.dashboard_download_cache /.dashboard_build_temp /.tools/bin docs/swagger/* diff --git a/Makefile b/Makefile index 33452be558e5..c5c752a11595 100644 --- a/Makefile +++ b/Makefile @@ -271,7 +271,7 @@ clean-test: clean-build: # Cleaning building files... - rm -rf .dashboard_asset_cache/ + rm -rf .dashboard_download_cache/ rm -rf $(BUILD_BIN_PATH) rm -rf $(GO_TOOLS_BIN_PATH) diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go index 4f97e92fc5b2..19ea3fd9e368 100644 --- a/cmd/pd-server/main.go +++ b/cmd/pd-server/main.go @@ -74,12 +74,6 @@ func main() { // Flushing any buffered log entries defer log.Sync() - // The old logger - err = logutil.InitLogger(&cfg.Log) - if err != nil { - log.Fatal("initialize logger error", errs.ZapError(err)) - } - server.LogPDInfo() for _, msg := range cfg.WarningMsgs { diff --git a/go.mod b/go.mod index 72f5b5324da1..dc20da2b03a5 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/coreos/go-semver v0.3.0 - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f github.com/docker/go-units v0.4.0 github.com/go-echarts/go-echarts v1.0.0 github.com/gogo/protobuf v1.3.1 @@ -25,20 +24,19 @@ require ( github.com/mgechev/revive v1.0.2 github.com/montanaflynn/stats v0.5.0 github.com/opentracing/opentracing-go v1.1.0 - github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect + github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce github.com/pingcap/kvproto v0.0.0-20211213085605-3329b3c5404c - github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7 + github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d - github.com/pingcap/tidb-dashboard v0.0.0-20211206031355-bcc43a01d537 + github.com/pingcap/tidb-dashboard v0.0.0-20211231024458-18087fcc06d7 github.com/prometheus/client_golang v1.1.0 github.com/prometheus/common v0.6.0 github.com/sasha-s/go-deadlock v0.2.0 - github.com/sirupsen/logrus v1.2.0 github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba @@ -50,9 +48,8 @@ require ( go.etcd.io/bbolt v1.3.5 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 go.uber.org/goleak v1.1.12 - go.uber.org/zap v1.16.0 + go.uber.org/zap v1.19.0 golang.org/x/tools v0.1.5 google.golang.org/grpc v1.26.0 - gopkg.in/natefinch/lumberjack.v2 v2.0.0 gotest.tools/gotestsum v1.7.0 ) diff --git a/go.sum b/go.sum index 28dc33c13d7d..f228ea2fdd1d 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,13 @@ github.com/AlekSi/gocov-xml v1.0.0 h1:4QctJBgXEkbzeKz6PJy6bt3JSPNSN4I2mITYW+eKUo github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -15,8 +20,8 @@ github.com/ReneKroon/ttlcache/v2 v2.3.0 h1:qZnUjRKIrbKHH6vF5T7Y9Izn5ObfTZfyYpGhv github.com/ReneKroon/ttlcache/v2 v2.3.0/go.mod h1:zbo6Pv/28e21Z8CzzqgYRArQYGYtjONRxaAKGxzQvG4= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd h1:59Whn6shj5MTVjTf2OX6+7iMcmY6h5CK0kTWwRaplL4= -github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0= +github.com/VividCortex/mysqlerr v1.0.0 h1:5pZ2TZA+YnzPgzBfiUWGqWmKDVNBdrkf9g+DNe1Tiq8= +github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -25,8 +30,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= -github.com/appleboy/gin-jwt/v2 v2.6.3 h1:aK4E3DjihWEBUTjEeRnGkA5nUkmwJPL1CPonMa2usRs= -github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0= +github.com/antonmedv/expr v1.9.0/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -34,11 +38,15 @@ github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54 github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/axw/gocov v1.0.0 h1:YsqYR66hUmilVr23tu8USgnJIJvnwh3n7j5zRn7x4LU= github.com/axw/gocov v1.0.0/go.mod h1:LvQpEYiwwIb2nYkXY2fDWhg9/AsYqkhmrCshjlUJECE= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch h1:KLE/YeX+9FNaGVW5MtImRVPhjDpfpgJhvkuYWBmOYbo= +github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch/go.mod h1:KjBLriHXe7L6fGceqWzTod8HUB/TP1WWDtfuSYtYXaI= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= @@ -52,6 +60,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -60,8 +69,9 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -71,9 +81,11 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -99,6 +111,8 @@ github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzP github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= +github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc= @@ -109,8 +123,9 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/gin-gonic/gin v1.5.0 h1:fi+bqFAx/oLK54somfCtEZs9HeH1LHVoEPUgARpTqyc= -github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gin-gonic/gin v1.7.4 h1:QmUZXrvJ9qZ3GfWvQ+2wnW/1ePrTEJqPKMYEU3lD/DM= +github.com/gin-gonic/gin v1.7.4/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-echarts/go-echarts v1.0.0 h1:n181E4iXwj4zrU9VYmdM2m8dyhERt2w9k9YhHqdp6A8= @@ -137,24 +152,32 @@ github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc= -github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c/go.mod h1:UqxAgEOt89sCiXlrc/ycnx00LVvUO/eS8tMUkWX4R7w= -github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= -github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/goccy/go-graphviz v0.0.5 h1:qcjgvNiYbLyfLAq9LvyYBJ7sNMbQh9w4FoAzBDrYhYw= -github.com/goccy/go-graphviz v0.0.5/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= +github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ= +github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -167,6 +190,7 @@ github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -179,10 +203,9 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9 h1:K+lX49/3eURCE1IjlaZN//u6c+9nfDAMnyQ9E2dsJbY= -github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20211122183932-1daafda22083 h1:c8EUapQFi+kjzedr4c6WqbwMdmB95+oDBWZ5XFHFYxY= +github.com/google/pprof v0.0.0-20211122183932-1daafda22083/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= @@ -205,14 +228,46 @@ github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/henrylee2cn/ameda v1.4.10/go.mod h1:liZulR8DgHxdK+MEwvZIylGnmcjzQ6N6f2PlWe7nEO4= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d h1:uGg2frlt3IcT7kbV6LEp5ONv4vmoO2FW4qSO+my/aoM= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/ZzIY= github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik= +github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oAlxAg= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= +github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= @@ -223,8 +278,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= @@ -232,35 +287,47 @@ github.com/joomcode/errorx v1.0.1 h1:CalpDWz14ZHd68fIqluJasJosAewpz2TFaJALrUxjrk github.com/joomcode/errorx v1.0.1/go.mod h1:kgco15ekB6cs+4Xjzo7SPeXzx38PbJzBwbnu9qfVNHQ= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8= -github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= +github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= @@ -268,12 +335,15 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0= +github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= +github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= +github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 h1:QASJXOGm2RZ5Ardbc86qNFvby9AqkLDibfChMtAg5QM= @@ -296,7 +366,6 @@ github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM= github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= @@ -312,9 +381,8 @@ github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsq github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= +github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= +github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= @@ -330,7 +398,6 @@ github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 h1:LllgC9eGfqzkfubMgjKIDyZYaa609nNWAyNZtpy2B3M= github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI= -github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= @@ -339,18 +406,18 @@ github.com/pingcap/kvproto v0.0.0-20211213085605-3329b3c5404c h1:jrPg+QFqQ7VyI30 github.com/pingcap/kvproto v0.0.0-20211213085605-3329b3c5404c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7 h1:k2BbABz9+TNpYRwsCCFS8pEEnFVOdbgEjL/kTlLuzZQ= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= -github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3/go.mod h1:tckvA041UWP+NqYzrJ3fMgC/Hw9wnmQ/tUkp/JaHly8= +github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 h1:SvWCbCPh1YeHd9yQLksvJYAgft6wLTY1aNG81tpyscQ= +github.com/pingcap/log v0.0.0-20210906054005-afc726e70354/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d h1:k3/APKZjXOyJrFy8VyYwRlZhMelpD3qBLJNsw3bPl/g= github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaWTao2LHOyMlsc2Dg1vW+mDY9dEbPzVyOlaeM= -github.com/pingcap/tidb-dashboard v0.0.0-20211206031355-bcc43a01d537 h1:qoKNQJY2hQ9/+q+aD6SxrNX5cZ8A9XZyw6rQg7MbwlE= -github.com/pingcap/tidb-dashboard v0.0.0-20211206031355-bcc43a01d537/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ= +github.com/pingcap/tidb-dashboard v0.0.0-20211231024458-18087fcc06d7 h1:JduqSbZR2Nf9Sy2a5HpOvRWzoktmW9aCmc+XJ/x9W+E= +github.com/pingcap/tidb-dashboard v0.0.0-20211231024458-18087fcc06d7/go.mod h1:4hk/3owVGWdvI9Kx6yCqqvM1T5PVgwyQNyMQxD3rwfc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -373,30 +440,36 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4= github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v3.21.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shhdgit/testfixtures/v3 v3.6.2-0.20211219171712-c4f264d673d3/go.mod h1:Z0OLtuFJ7Y4yLsVijHK8uq95NjGFlYJy+I00ElAEtUQ= github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -413,7 +486,9 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -435,8 +510,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/thoas/go-funk v0.8.0 h1:JP9tKSvnpFVclYgDM0Is7FD9M4fhPvqA0s0BsXmzSRQ= github.com/thoas/go-funk v0.8.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= -github.com/tidwall/gjson v1.3.5 h1:2oW9FBNu8qt9jy5URgrzsVx/T/KSn3qn/smJQ0crlDQ= -github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= +github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= @@ -464,17 +539,16 @@ github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= -github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1 h1:d71/KA0LhvkrJ/Ok+Wx9qK7bU8meKA1Hk0jpVI5kJjk= -github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI= -github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY= -github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= @@ -485,12 +559,14 @@ go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXw go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/dig v1.8.0 h1:1rR6hnL/bu1EVcjnRDN5kx1vbIjEJDTGhSQ2B3ddpcI= -go.uber.org/dig v1.8.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= -go.uber.org/fx v1.10.0 h1:S2K/H8oNied0Je/mLKdWzEWKZfv9jtxSDm8CnwK+5Fg= -go.uber.org/fx v1.10.0/go.mod h1:vLRicqpG/qQEzno4SYU86iCwfT95EZza+Eba0ItuxqY= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/dig v1.9.0 h1:pJTDXKEhRqBI8W7rU7kwT5EgyRZuSMVSFcZolOvKK9U= +go.uber.org/dig v1.9.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= +go.uber.org/fx v1.12.0 h1:+1+3Cz9M0dFMPy9SW9XUIUHye8bnPUm7q7DroNGWYG4= +go.uber.org/fx v1.12.0/go.mod h1:egT3Kyg1JFYQkvKLZ3EsykxkNrZxgXS+gKoKo7abERY= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= @@ -498,23 +574,29 @@ go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -544,7 +626,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -552,7 +633,6 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -573,17 +653,20 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -591,8 +674,9 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -609,6 +693,7 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -617,6 +702,7 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -630,6 +716,8 @@ golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -639,9 +727,8 @@ gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= @@ -656,9 +743,7 @@ google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRn google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -666,11 +751,9 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc= -gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -680,11 +763,12 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.0.6 h1:mA0XRPjIKi4bkE9nv+NKs6qj6QWOchqUSdWOcpd3x1E= gorm.io/driver/mysql v1.0.6/go.mod h1:KdrTanmfLPPyAOeYGyG+UpDys7/7eeWT1zCq+oekYnU= gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= @@ -698,7 +782,6 @@ gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= moul.io/zapgorm2 v1.1.0 h1:qwAlMBYf+qJkJ7PAzJl4oCe6eS6QGiKAXUPeis0+RBE= diff --git a/pkg/apiutil/apiutil.go b/pkg/apiutil/apiutil.go index 2c61ed45f288..0a904dcc7c18 100644 --- a/pkg/apiutil/apiutil.go +++ b/pkg/apiutil/apiutil.go @@ -27,6 +27,14 @@ import ( "github.com/unrolled/render" ) +var ( + // componentSignatureKey is used for http request header key + // to identify component signature + componentSignatureKey = "component" + // componentAnonymousValue identifies anonymous request source + componentAnonymousValue = "anonymous" +) + // DeferClose captures the error returned from closing (if an error occurs). // This is designed to be used in a defer statement. func DeferClose(c io.Closer, err *error) { @@ -127,3 +135,34 @@ func ErrorResp(rd *render.Render, w http.ResponseWriter, err error) { rd.JSON(w, http.StatusInternalServerError, err.Error()) } } + +// GetComponentNameOnHTTP returns component name from Request Header +func GetComponentNameOnHTTP(r *http.Request) string { + componentName := r.Header.Get(componentSignatureKey) + if len(componentName) == 0 { + componentName = componentAnonymousValue + } + return componentName +} + +// ComponentSignatureRoundTripper is used to add component signature in HTTP header +type ComponentSignatureRoundTripper struct { + proxied http.RoundTripper + component string +} + +// NewComponentSignatureRoundTripper returns a new ComponentSignatureRoundTripper. +func NewComponentSignatureRoundTripper(roundTripper http.RoundTripper, componentName string) *ComponentSignatureRoundTripper { + return &ComponentSignatureRoundTripper{ + proxied: roundTripper, + component: componentName, + } +} + +// RoundTrip is used to implement RoundTripper +func (rt *ComponentSignatureRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) { + req.Header.Add(componentSignatureKey, rt.component) + // Send the request, get the response and the error + resp, err = rt.proxied.RoundTrip(req) + return +} diff --git a/pkg/dashboard/adapter/config.go b/pkg/dashboard/adapter/config.go index 27bf1f7a4ccc..63c900acf776 100644 --- a/pkg/dashboard/adapter/config.go +++ b/pkg/dashboard/adapter/config.go @@ -35,7 +35,6 @@ func GenDashboardConfig(srv *server.Server) (*config.Config, error) { dashboardCfg.PublicPathPrefix = cfg.Dashboard.PublicPathPrefix dashboardCfg.EnableTelemetry = cfg.Dashboard.EnableTelemetry dashboardCfg.EnableExperimental = cfg.Dashboard.EnableExperimental - dashboardCfg.EnableNonRootLogin = true if dashboardCfg.ClusterTLSConfig, err = cfg.Security.ToTLSConfig(); err != nil { return nil, err } diff --git a/pkg/dashboard/dashboard.go b/pkg/dashboard/dashboard.go index e752f1dedd92..59bfcf74160a 100644 --- a/pkg/dashboard/dashboard.go +++ b/pkg/dashboard/dashboard.go @@ -27,8 +27,7 @@ import ( "github.com/pingcap/tidb-dashboard/pkg/uiserver" "github.com/tikv/pd/pkg/dashboard/adapter" - // inject distro information to dashboard - _ "github.com/tikv/pd/pkg/dashboard/distro" + "github.com/tikv/pd/pkg/dashboard/distroutil" "github.com/tikv/pd/pkg/dashboard/keyvisual" ui "github.com/tikv/pd/pkg/dashboard/uiserver" "github.com/tikv/pd/server" @@ -70,6 +69,8 @@ func GetServiceBuilders() []server.HandlerBuilder { return []server.HandlerBuilder{ // Dashboard API Service func(ctx context.Context, srv *server.Server) (http.Handler, server.ServiceGroup, error) { + distroutil.MustLoadAndReplaceStrings() + if cfg, err = adapter.GenDashboardConfig(srv); err != nil { return nil, apiServiceGroup, err } diff --git a/pkg/dashboard/distro/.gitignore b/pkg/dashboard/distro/.gitignore deleted file mode 100644 index 0c93aa327fe3..000000000000 --- a/pkg/dashboard/distro/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/distro_info.go diff --git a/pkg/dashboard/distro/distro.go b/pkg/dashboard/distro/distro.go deleted file mode 100644 index dff0ac47086d..000000000000 --- a/pkg/dashboard/distro/distro.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2021 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build dashboard_distro -// +build dashboard_distro - -package distro - -import ( - "github.com/pingcap/tidb-dashboard/pkg/utils/distro" -) - -func init() { - distro.Replace(Resource) -} diff --git a/pkg/dashboard/distro/placeholder.go b/pkg/dashboard/distro/placeholder.go deleted file mode 100644 index 3cf9b18b1e3a..000000000000 --- a/pkg/dashboard/distro/placeholder.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2021 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package distro diff --git a/pkg/dashboard/distroutil/distro.go b/pkg/dashboard/distroutil/distro.go new file mode 100644 index 000000000000..bc18c018355c --- /dev/null +++ b/pkg/dashboard/distroutil/distro.go @@ -0,0 +1,53 @@ +// Copyright 2021 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package distroutil + +import ( + "os" + "path" + + "github.com/pingcap/log" + "github.com/pingcap/tidb-dashboard/util/distro" + "go.uber.org/zap" +) + +const ( + resFolderName string = "distro-res" + stringsFileName string = "strings.json" +) + +// MustGetResPath returns the default path that a distribution resource should be placed, +// which is the ${BinaryPath}/distro-res. +func MustGetResPath() string { + exePath, err := os.Executable() + if err != nil { + log.Fatal("Failed to read the execution path", zap.Error(err)) + return "" + } + return path.Join(path.Dir(exePath), resFolderName) +} + +// MustLoadAndReplaceStrings loads the distro strings from ${BinaryPath}/distro-res/strings.json +// and replace the strings in the TiDB Dashboard. If the strings file does not exist, the default +// distro string will be used. +func MustLoadAndReplaceStrings() { + resPath := MustGetResPath() + strings, err := distro.ReadResourceStringsFromFile(path.Join(resPath, stringsFileName)) + if err != nil { + log.Fatal("Failed to load distro strings", zap.Error(err)) + } + log.Info("Using distribution strings", zap.Any("strings", strings)) + distro.ReplaceGlobal(strings) +} diff --git a/pkg/dashboard/uiserver/embedded_assets_rewriter.go b/pkg/dashboard/uiserver/embedded_assets_rewriter.go index 4474dfb28ef7..2a5b4a5b3b6c 100644 --- a/pkg/dashboard/uiserver/embedded_assets_rewriter.go +++ b/pkg/dashboard/uiserver/embedded_assets_rewriter.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb-dashboard/pkg/config" "github.com/pingcap/tidb-dashboard/pkg/uiserver" + "github.com/tikv/pd/pkg/dashboard/distroutil" ) var once sync.Once @@ -29,7 +30,8 @@ var once sync.Once // Assets returns the Assets FileSystem of the dashboard UI func Assets(cfg *config.Config) http.FileSystem { once.Do(func() { - uiserver.RewriteAssets(assets, cfg, func(fs http.FileSystem, f http.File, path, newContent string, bs []byte) { + resPath := distroutil.MustGetResPath() + uiserver.RewriteAssets(assets, cfg, resPath, func(fs http.FileSystem, f http.File, path, newContent string, bs []byte) { m := fs.(vfsgenÛ°FS) fi := f.(os.FileInfo) m[path] = &vfsgenÛ°CompressedFileInfo{ diff --git a/pkg/logutil/log.go b/pkg/logutil/log.go index aa290a7102c9..d98b9bc2e6cb 100644 --- a/pkg/logutil/log.go +++ b/pkg/logutil/log.go @@ -15,31 +15,13 @@ package logutil import ( - "bytes" "fmt" - "os" - "path" - "runtime" "strings" - "sync" "sync/atomic" - "github.com/coreos/pkg/capnslog" - zaplog "github.com/pingcap/log" - log "github.com/sirupsen/logrus" - "github.com/tikv/pd/pkg/errs" - "go.etcd.io/etcd/raft" + "github.com/pingcap/log" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "google.golang.org/grpc/grpclog" - "gopkg.in/natefinch/lumberjack.v2" -) - -const ( - defaultLogTimeFormat = "2006/01/02 15:04:05.000" - defaultLogMaxSize = 300 // MB - defaultLogFormat = "text" - defaultLogLevel = log.InfoLevel ) // FileLogConfig serializes file log related config in toml/json. @@ -66,86 +48,6 @@ type LogConfig struct { File FileLogConfig `toml:"file" json:"file"` } -// redirectFormatter will redirect etcd logs to logrus logs. -type redirectFormatter struct{} - -// Format implements capnslog.Formatter hook. -func (rf *redirectFormatter) Format(pkg string, level capnslog.LogLevel, depth int, entries ...interface{}) { - if pkg != "" { - pkg = fmt.Sprint(pkg, ": ") - } - - logStr := fmt.Sprint(pkg, entries) - - switch level { - case capnslog.CRITICAL: - log.Fatalf(logStr) - case capnslog.ERROR: - log.Errorf(logStr) - case capnslog.WARNING: - log.Warningf(logStr) - case capnslog.NOTICE: - log.Infof(logStr) - case capnslog.INFO: - log.Infof(logStr) - case capnslog.DEBUG, capnslog.TRACE: - log.Debugf(logStr) - } -} - -// Flush only for implementing Formatter. -func (rf *redirectFormatter) Flush() {} - -// isSKippedPackageName tests whether path name is on log library calling stack. -func isSkippedPackageName(name string) bool { - return strings.Contains(name, "github.com/sirupsen/logrus") || - strings.Contains(name, "github.com/coreos/pkg/capnslog") -} - -// modifyHook injects file name and line pos into log entry. -type contextHook struct{} - -// Fire implements logrus.Hook interface -// https://github.com/sirupsen/logrus/issues/63 -func (hook *contextHook) Fire(entry *log.Entry) error { - pc := make([]uintptr, 4) - cnt := runtime.Callers(6, pc) - - for i := 0; i < cnt; i++ { - fu := runtime.FuncForPC(pc[i] - 1) - name := fu.Name() - if !isSkippedPackageName(name) { - file, line := fu.FileLine(pc[i] - 1) - entry.Data["file"] = path.Base(file) - entry.Data["line"] = line - break - } - } - return nil -} - -// Levels implements logrus.Hook interface. -func (hook *contextHook) Levels() []log.Level { - return log.AllLevels -} - -// StringToLogLevel translates log level string to log level. -func StringToLogLevel(level string) log.Level { - switch strings.ToLower(level) { - case "fatal": - return log.FatalLevel - case "error": - return log.ErrorLevel - case "warn", "warning": - return log.WarnLevel - case "debug": - return log.DebugLevel - case "info": - return log.InfoLevel - } - return defaultLogLevel -} - // StringToZapLogLevel translates log level string to log level. func StringToZapLogLevel(level string) zapcore.Level { switch strings.ToLower(level) { @@ -163,135 +65,11 @@ func StringToZapLogLevel(level string) zapcore.Level { return zapcore.InfoLevel } -// textFormatter is for compatibility with ngaut/log -type textFormatter struct { - DisableTimestamp bool -} - -// Format implements logrus.Formatter -func (f *textFormatter) Format(entry *log.Entry) ([]byte, error) { - var b *bytes.Buffer - if entry.Buffer != nil { - b = entry.Buffer - } else { - b = &bytes.Buffer{} - } - if !f.DisableTimestamp { - fmt.Fprintf(b, "%s ", entry.Time.Format(defaultLogTimeFormat)) - } - if file, ok := entry.Data["file"]; ok { - fmt.Fprintf(b, "%s:%v:", file, entry.Data["line"]) - } - fmt.Fprintf(b, " [%s] %s", entry.Level.String(), entry.Message) - for k, v := range entry.Data { - if k != "file" && k != "line" { - fmt.Fprintf(b, " %v=%v", k, v) - } - } - b.WriteByte('\n') - return b.Bytes(), nil -} - -// StringToLogFormatter uses the different log formatter according to a given format name. -func StringToLogFormatter(format string, disableTimestamp bool) log.Formatter { - switch strings.ToLower(format) { - case "text": - return &textFormatter{ - DisableTimestamp: disableTimestamp, - } - case "json": - return &log.JSONFormatter{ - TimestampFormat: defaultLogTimeFormat, - DisableTimestamp: disableTimestamp, - } - case "console": - return &log.TextFormatter{ - FullTimestamp: true, - TimestampFormat: defaultLogTimeFormat, - DisableTimestamp: disableTimestamp, - } - default: - return &textFormatter{} - } -} - -// InitFileLog initializes file based logging options. -func InitFileLog(cfg *zaplog.FileLogConfig) error { - if st, err := os.Stat(cfg.Filename); err == nil { - if st.IsDir() { - return errs.ErrInitFileLog.FastGenByArgs("can't use directory as log file name") - } - } - if cfg.MaxSize == 0 { - cfg.MaxSize = defaultLogMaxSize - } - - // use lumberjack to logrotate - output := &lumberjack.Logger{ - Filename: cfg.Filename, - MaxSize: cfg.MaxSize, - MaxBackups: cfg.MaxBackups, - MaxAge: cfg.MaxDays, - LocalTime: true, - } - - log.SetOutput(output) - return nil -} - -type wrapLogrus struct { - *log.Logger -} - -// V provides the functionality that returns whether a particular log level is at -// least l - this is needed to meet the LoggerV2 interface. GRPC's logging levels -// are: https://github.com/grpc/grpc-go/blob/master/grpclog/loggerv2.go#L71 -// 0=info, 1=warning, 2=error, 3=fatal -// logrus' are: https://github.com/sirupsen/logrus/blob/master/logrus.go -// 0=panic, 1=fatal, 2=error, 3=warn, 4=info, 5=debug -func (lg *wrapLogrus) V(l int) bool { - // translate to logrus level - logrusLevel := 4 - l - return int(lg.Logger.Level) <= logrusLevel -} - -var once sync.Once - -// InitLogger initializes PD's logger. -func InitLogger(cfg *zaplog.Config) error { - var err error - - once.Do(func() { - log.SetLevel(StringToLogLevel(cfg.Level)) - log.AddHook(&contextHook{}) - - if cfg.Format == "" { - cfg.Format = defaultLogFormat - } - log.SetFormatter(StringToLogFormatter(cfg.Format, cfg.DisableTimestamp)) - - // etcd log - capnslog.SetFormatter(&redirectFormatter{}) - // grpc log - lg := &wrapLogrus{log.StandardLogger()} - grpclog.SetLoggerV2(lg) - // raft log - raft.SetLogger(lg) - - if len(cfg.File.Filename) == 0 { - return - } - - err = InitFileLog(&cfg.File) - }) - return err -} - // LogPanic logs the panic reason and stack, then exit the process. // Commonly used with a `defer`. func LogPanic() { if e := recover(); e != nil { - zaplog.Fatal("panic", zap.Reflect("recover", e)) + log.Fatal("panic", zap.Reflect("recover", e)) } } diff --git a/pkg/logutil/log_test.go b/pkg/logutil/log_test.go index 21ed81867a65..33b0320b6f81 100644 --- a/pkg/logutil/log_test.go +++ b/pkg/logutil/log_test.go @@ -15,45 +15,20 @@ package logutil import ( - "bytes" "fmt" - "strings" "testing" - "github.com/coreos/pkg/capnslog" . "github.com/pingcap/check" - zaplog "github.com/pingcap/log" - log "github.com/sirupsen/logrus" "go.uber.org/zap/zapcore" ) -const ( - logPattern = `\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d\.\d\d\d ([\w_%!$@.,+~-]+|\\.)+:\d+: \[(fatal|error|warning|info|debug)\] .*?\n` -) - func Test(t *testing.T) { TestingT(t) } var _ = Suite(&testLogSuite{}) -type testLogSuite struct { - buf *bytes.Buffer -} - -func (s *testLogSuite) SetUpSuite(c *C) { - s.buf = &bytes.Buffer{} -} - -func (s *testLogSuite) TestStringToLogLevel(c *C) { - c.Assert(StringToLogLevel("fatal"), Equals, log.FatalLevel) - c.Assert(StringToLogLevel("ERROR"), Equals, log.ErrorLevel) - c.Assert(StringToLogLevel("warn"), Equals, log.WarnLevel) - c.Assert(StringToLogLevel("warning"), Equals, log.WarnLevel) - c.Assert(StringToLogLevel("debug"), Equals, log.DebugLevel) - c.Assert(StringToLogLevel("info"), Equals, log.InfoLevel) - c.Assert(StringToLogLevel("whatever"), Equals, log.InfoLevel) -} +type testLogSuite struct{} func (s *testLogSuite) TestStringToZapLogLevel(c *C) { c.Assert(StringToZapLogLevel("fatal"), Equals, zapcore.FatalLevel) @@ -65,53 +40,6 @@ func (s *testLogSuite) TestStringToZapLogLevel(c *C) { c.Assert(StringToZapLogLevel("whatever"), Equals, zapcore.InfoLevel) } -func (s *testLogSuite) TestStringToLogFormatter(c *C) { - c.Assert(StringToLogFormatter("text", true), DeepEquals, &textFormatter{ - DisableTimestamp: true, - }) - c.Assert(StringToLogFormatter("json", true), DeepEquals, &log.JSONFormatter{ - DisableTimestamp: true, - TimestampFormat: defaultLogTimeFormat, - }) - c.Assert(StringToLogFormatter("console", true), DeepEquals, &log.TextFormatter{ - DisableTimestamp: true, - FullTimestamp: true, - TimestampFormat: defaultLogTimeFormat, - }) - c.Assert(StringToLogFormatter("", true), DeepEquals, &textFormatter{}) -} - -// TestLogging assure log format and log redirection works. -func (s *testLogSuite) TestLogging(c *C) { - conf := &zaplog.Config{Level: "warn", File: zaplog.FileLogConfig{}} - c.Assert(InitLogger(conf), IsNil) - - log.SetOutput(s.buf) - - tlog := capnslog.NewPackageLogger("github.com/tikv/pd/pkg/logutil", "test") - - tlog.Infof("[this message should not be sent to buf]") - c.Assert(s.buf.Len(), Equals, 0) - - tlog.Warningf("[this message should be sent to buf]") - entry, err := s.buf.ReadString('\n') - c.Assert(err, IsNil) - c.Assert(entry, Matches, logPattern) - // All capnslog log will be trigered in logutil/log.go - c.Assert(strings.Contains(entry, "log.go"), IsTrue) - - log.Warnf("this message comes from logrus") - entry, err = s.buf.ReadString('\n') - c.Assert(err, IsNil) - c.Assert(entry, Matches, logPattern) - c.Assert(strings.Contains(entry, "log_test.go"), IsTrue) -} - -func (s *testLogSuite) TestFileLog(c *C) { - c.Assert(InitFileLog(&zaplog.FileLogConfig{Filename: "/tmp"}), NotNil) - c.Assert(InitFileLog(&zaplog.FileLogConfig{Filename: "/tmp/test_file_log", MaxSize: 0}), IsNil) -} - func (s *testLogSuite) TestRedactLog(c *C) { testcases := []struct { name string diff --git a/pkg/mock/mockcluster/mockcluster.go b/pkg/mock/mockcluster/mockcluster.go index ce8c79652ab4..0864e9e525c3 100644 --- a/pkg/mock/mockcluster/mockcluster.go +++ b/pkg/mock/mockcluster/mockcluster.go @@ -130,7 +130,7 @@ func (mc *Cluster) RegionWriteStats() map[uint64][]*statistics.HotPeerStat { // HotRegionsFromStore picks hot regions in specify store. func (mc *Cluster) HotRegionsFromStore(store uint64, kind statistics.RWType) []*core.RegionInfo { - stats := mc.HotCache.HotRegionsFromStore(store, kind, mc.GetHotRegionCacheHitsThreshold()) + stats := hotRegionsFromStore(mc.HotCache, store, kind, mc.GetHotRegionCacheHitsThreshold()) regions := make([]*core.RegionInfo, 0, len(stats)) for _, stat := range stats { region := mc.GetRegion(stat.RegionID) @@ -141,6 +141,14 @@ func (mc *Cluster) HotRegionsFromStore(store uint64, kind statistics.RWType) []* return regions } +// hotRegionsFromStore picks hot region in specify store. +func hotRegionsFromStore(w *statistics.HotCache, storeID uint64, kind statistics.RWType, minHotDegree int) []*statistics.HotPeerStat { + if stats, ok := w.RegionStats(kind, minHotDegree)[storeID]; ok && len(stats) > 0 { + return stats + } + return nil +} + // AllocPeer allocs a new peer on a store. func (mc *Cluster) AllocPeer(storeID uint64) (*metapb.Peer, error) { peerID, err := mc.AllocID() diff --git a/pkg/mock/mockhbstream/mockhbstream.go b/pkg/mock/mockhbstream/mockhbstream.go index 6b210368069f..54e563b95617 100644 --- a/pkg/mock/mockhbstream/mockhbstream.go +++ b/pkg/mock/mockhbstream/mockhbstream.go @@ -20,7 +20,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/tikv/pd/server/core" - "github.com/tikv/pd/server/schedule/opt" + "github.com/tikv/pd/server/schedule/hbstream" ) // HeartbeatStream is used to mock HeartbeatStream for test use. @@ -49,7 +49,7 @@ func (s HeartbeatStream) Send(m *pdpb.RegionHeartbeatResponse) error { func (s HeartbeatStream) SendMsg(region *core.RegionInfo, msg *pdpb.RegionHeartbeatResponse) {} // BindStream mock method. -func (s HeartbeatStream) BindStream(storeID uint64, stream opt.HeartbeatStream) {} +func (s HeartbeatStream) BindStream(storeID uint64, stream hbstream.HeartbeatStream) {} // Recv mocks method. func (s HeartbeatStream) Recv() *pdpb.RegionHeartbeatResponse { diff --git a/server/schedule/hbstream/heartbeat_streams_test.go b/pkg/mock/mockhbstream/mockhbstream_test.go similarity index 92% rename from server/schedule/hbstream/heartbeat_streams_test.go rename to pkg/mock/mockhbstream/mockhbstream_test.go index 1a4ac1d14be8..0e5cd02d7ea9 100644 --- a/server/schedule/hbstream/heartbeat_streams_test.go +++ b/pkg/mock/mockhbstream/mockhbstream_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package hbstream +package mockhbstream import ( "context" @@ -24,9 +24,9 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/tikv/pd/pkg/mock/mockcluster" - "github.com/tikv/pd/pkg/mock/mockhbstream" "github.com/tikv/pd/pkg/testutil" "github.com/tikv/pd/server/config" + "github.com/tikv/pd/server/schedule/hbstream" ) func TestHeaertbeatStreams(t *testing.T) { @@ -61,8 +61,8 @@ func (s *testHeartbeatStreamSuite) TestActivity(c *C) { ChangePeer: &pdpb.ChangePeer{Peer: &metapb.Peer{Id: 2, StoreId: 2}, ChangeType: eraftpb.ConfChangeType_AddLearnerNode}, } - hbs := NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true) - stream1, stream2 := mockhbstream.NewHeartbeatStream(), mockhbstream.NewHeartbeatStream() + hbs := hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true) + stream1, stream2 := NewHeartbeatStream(), NewHeartbeatStream() // Active stream is stream1. hbs.BindStream(1, stream1) diff --git a/plugin/scheduler_example/evict_leader.go b/plugin/scheduler_example/evict_leader.go index bc6e5d9bd775..40ad78a7944f 100644 --- a/plugin/scheduler_example/evict_leader.go +++ b/plugin/scheduler_example/evict_leader.go @@ -217,7 +217,7 @@ func (s *evictLeaderScheduler) Schedule(cluster opt.Cluster) []*operator.Operato s.conf.mu.RLock() defer s.conf.mu.RUnlock() for id, ranges := range s.conf.StoreIDWitRanges { - region := cluster.RandLeaderRegion(id, ranges, opt.IsRegionHealthy) + region := cluster.RandLeaderRegion(id, ranges, schedule.IsRegionHealthy) if region == nil { continue } diff --git a/scripts/embed-dashboard-ui.ps1 b/scripts/embed-dashboard-ui.ps1 index 62fc6662283a..53eb8453d6ee 100644 --- a/scripts/embed-dashboard-ui.ps1 +++ b/scripts/embed-dashboard-ui.ps1 @@ -1,6 +1,6 @@ $DIR = Split-Path -Parent $MyInvocation.MyCommand.Definition $BASE_DIR = (get-item $DIR).parent.FullName -$CACHE_DIR = Join-Path($BASE_DIR) "\.dashboard_asset_cache" +$CACHE_DIR = Join-Path($BASE_DIR) "\.dashboard_download_cache" echo '+ Create asset cache directory' diff --git a/scripts/embed-dashboard-ui.sh b/scripts/embed-dashboard-ui.sh index 838218310119..80c90bb17889 100755 --- a/scripts/embed-dashboard-ui.sh +++ b/scripts/embed-dashboard-ui.sh @@ -35,18 +35,16 @@ go mod tidy DASHBOARD_DIR=$(go list -f "{{.Dir}}" -m github.com/pingcap/tidb-dashboard) echo " - TiDB Dashboard directory: ${DASHBOARD_DIR}" +CACHE_DIR=${BASE_DIR}/.dashboard_download_cache +echo "+ Create download cache directory: ${CACHE_DIR}" +mkdir -p "${CACHE_DIR}" function download_embed_asset { - CACHE_DIR=${BASE_DIR}/.dashboard_asset_cache - - echo '+ Create asset cache directory' - mkdir -p "${CACHE_DIR}" - echo '+ Discover TiDB Dashboard release version' DASHBOARD_RELEASE_VERSION=$(grep -v '^#' "${DASHBOARD_DIR}/release-version") echo " - TiDB Dashboard release version: ${DASHBOARD_RELEASE_VERSION}" - echo '+ Check embedded assets exists in cache' + echo '+ Check whether pre-built assets are available' CACHE_FILE=${CACHE_DIR}/embedded-assets-golang-${DASHBOARD_RELEASE_VERSION}.zip if [[ -f "$CACHE_FILE" ]]; then echo " - Cached archive exists: ${CACHE_FILE}" @@ -86,7 +84,8 @@ function download_embed_asset { function compile_asset { BUILD_DIR=${BASE_DIR}/.dashboard_build_temp - echo '+ Clean up TiDB Dashboard build directory' + echo '+ Compiling TiDB Dashboard UI from source code' + echo '+ Clean up build directory' echo " - Build directory: ${DASHBOARD_DIR}" if [ -d "${BUILD_DIR}/ui/node_modules" ]; then echo " - Build dependency exists, keep dependency cache" @@ -99,22 +98,47 @@ function compile_asset { mkdir -p "${BUILD_DIR}/ui" fi - echo '+ Copy TiDB Dashboard source code to build directory' - echo " - Src: ${DASHBOARD_DIR}" + echo '+ Discover referenced TiDB Dashboard commit' + DASHBOARD_SHA=$(echo "${DASHBOARD_DIR}" | awk -F '-' '{print $NF}') + CACHE_FILE=${CACHE_DIR}/tidb-dashboard-${DASHBOARD_SHA}.zip + echo " - TiDB Dashboard commit: ${DASHBOARD_SHA}" + + echo '+ Check whether UI source code was downloaded' + if [[ -f "$CACHE_FILE" ]]; then + echo " - Source code archive exists: ${CACHE_FILE}" + else + echo ' - Source code archive does not exist' + echo ' - Download source code archive from GitHub' + + DOWNLOAD_URL="https://github.com/pingcap/tidb-dashboard/archive/${DASHBOARD_SHA}.zip" + DOWNLOAD_FILE=${CACHE_DIR}/tidb-dashboard.zip + echo " - Download ${DOWNLOAD_URL}" + if ! curl -L "${DOWNLOAD_URL}" --fail --output "${DOWNLOAD_FILE}"; then + echo + echo -e "${RED}Error: Failed to download TiDB Dashboard source code archive.${NC}" + exit 1 + fi + + mv "${DOWNLOAD_FILE}" "${CACHE_FILE}" + echo " - Source code archive downloaded to: ${CACHE_FILE}" + fi + + echo '+ Unpack source code archive' + DASHBOARD_FULL_SHA=$(unzip -l "${CACHE_FILE}" | sed -n '2p') + DASHBOARD_UNZIP_PATH=/tmp/tidb-dashboard-${DASHBOARD_FULL_SHA} + unzip -o "${CACHE_FILE}" -d /tmp + + echo '+ Copy unpacked source code to build directory' + echo " - Src: ${DASHBOARD_UNZIP_PATH}" echo " - Dest: ${BUILD_DIR}" - cp -r "${DASHBOARD_DIR}/." "${BUILD_DIR}/" + cp -r "${DASHBOARD_UNZIP_PATH}/." "${BUILD_DIR}/" + chmod -R u+w "${BUILD_DIR}" chmod u+x "${BUILD_DIR}"/scripts/*.sh echo '+ Build UI' cd "${BUILD_DIR}" - - if [ -n "${DISTRIBUTION_DIR}" ]; then - DISTRIBUTION_DIR=${DISTRIBUTION_DIR} scripts/replace_distro_resource.sh - DISTRO_BUILD_TAG=1 make ui - else - make ui - fi + make ui echo '+ Generating UI assets' echo ' - Generating...' diff --git a/server/api/region.go b/server/api/region.go index 910c128e5f91..468c90ddb9e0 100644 --- a/server/api/region.go +++ b/server/api/region.go @@ -28,12 +28,12 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/kvproto/pkg/replication_modepb" - log "github.com/sirupsen/logrus" + "github.com/pingcap/log" "github.com/tikv/pd/pkg/apiutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/core" + "github.com/tikv/pd/server/schedule" "github.com/tikv/pd/server/schedule/operator" - "github.com/tikv/pd/server/schedule/opt" "github.com/tikv/pd/server/statistics" "github.com/unrolled/render" "go.uber.org/zap" @@ -237,11 +237,11 @@ func (h *regionHandler) GetRegionByKey(w http.ResponseWriter, r *http.Request) { } // @Tags region -// @Summary Check if regions in the given key ranges are replicated. +// @Summary Check if regions in the given key ranges are replicated. Returns 'REPLICATED', 'INPROGRESS', or 'PENDING'. 'PENDING' means that there is at least one region pending for scheduling. Similarly, 'INPROGRESS' means there is at least one region in scheduling. // @Param startKey query string true "Regions start key, hex encoded" // @Param endKey query string true "Regions end key, hex encoded" // @Produce plain -// @Success 200 {string} string "true" +// @Success 200 {string} string "INPROGRESS" // @Failure 400 {string} string "The input is invalid." // @Router /regions/replicated [get] func (h *regionsHandler) CheckRegionsReplicated(w http.ResponseWriter, r *http.Request) { @@ -262,14 +262,26 @@ func (h *regionsHandler) CheckRegionsReplicated(w http.ResponseWriter, r *http.R } regions := rc.ScanRegions(startKey, endKey, -1) - replicated := true + state := "REPLICATED" for _, region := range regions { - if !opt.IsRegionReplicated(rc, region) { - replicated = false + if !schedule.IsRegionReplicated(rc, region) { + state = "INPROGRESS" + for _, item := range rc.GetCoordinator().GetWaitingRegions() { + if item.Key == region.GetID() { + state = "PENDING" + break + } + } break } } - h.rd.JSON(w, http.StatusOK, replicated) + failpoint.Inject("mockPending", func(val failpoint.Value) { + aok, ok := val.(bool) + if ok && aok { + state = "PENDING" + } + }) + h.rd.JSON(w, http.StatusOK, state) } type regionsHandler struct { diff --git a/server/api/region_test.go b/server/api/region_test.go index b6ad175534f4..3b2896820bb7 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -598,7 +598,7 @@ func (s *testRegionsReplicatedSuite) TestCheckRegionsReplicated(c *C) { }, } - status := false + status := "" // invalid url url := fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, s.urlPrefix, "_", "t") @@ -620,7 +620,13 @@ func (s *testRegionsReplicatedSuite) TestCheckRegionsReplicated(c *C) { err = readJSON(testDialClient, url, &status) c.Assert(err, IsNil) - c.Assert(status, Equals, true) + c.Assert(status, Equals, "REPLICATED") + + c.Assert(failpoint.Enable("github.com/tikv/pd/server/api/mockPending", "return(true)"), IsNil) + err = readJSON(testDialClient, url, &status) + c.Assert(err, IsNil) + c.Assert(status, Equals, "PENDING") + c.Assert(failpoint.Disable("github.com/tikv/pd/server/api/mockPending"), IsNil) // test multiple rules r1 = newTestRegionInfo(2, 1, []byte("a"), []byte("b")) @@ -637,7 +643,7 @@ func (s *testRegionsReplicatedSuite) TestCheckRegionsReplicated(c *C) { err = readJSON(testDialClient, url, &status) c.Assert(err, IsNil) - c.Assert(status, Equals, true) + c.Assert(status, Equals, "REPLICATED") // test multiple bundles bundle = append(bundle, placement.GroupBundle{ @@ -656,7 +662,7 @@ func (s *testRegionsReplicatedSuite) TestCheckRegionsReplicated(c *C) { err = readJSON(testDialClient, url, &status) c.Assert(err, IsNil) - c.Assert(status, Equals, false) + c.Assert(status, Equals, "INPROGRESS") r1 = newTestRegionInfo(2, 1, []byte("a"), []byte("b")) r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 1}, &metapb.Peer{Id: 6, StoreId: 1}, &metapb.Peer{Id: 7, StoreId: 1}) @@ -664,7 +670,7 @@ func (s *testRegionsReplicatedSuite) TestCheckRegionsReplicated(c *C) { err = readJSON(testDialClient, url, &status) c.Assert(err, IsNil) - c.Assert(status, Equals, true) + c.Assert(status, Equals, "REPLICATED") } // Create n regions (0..n) of n stores (0..n). diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 9e4d3a4f1fbf..64aea938a112 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -596,11 +596,11 @@ func (c *RaftCluster) HandleStoreHeartbeat(stats *pdpb.StoreStats) error { c.limiter.Collect(newStore.GetStoreStats()) } - regionIDs := make(map[uint64]struct{}, len(stats.GetPeerStats())) + regions := make(map[uint64]*core.RegionInfo, len(stats.GetPeerStats())) for _, peerStat := range stats.GetPeerStats() { regionID := peerStat.GetRegionId() - regionIDs[regionID] = struct{}{} region := c.GetRegion(regionID) + regions[regionID] = region if region == nil { log.Warn("discard hot peer stat for unknown region", zap.Uint64("region-id", regionID), @@ -626,7 +626,7 @@ func (c *RaftCluster) HandleStoreHeartbeat(stats *pdpb.StoreStats) error { peerInfo := core.NewPeerInfo(peer, loads, interval) c.hotStat.CheckReadAsync(statistics.NewCheckPeerTask(peerInfo, region)) } - c.hotStat.CheckReadAsync(statistics.NewCollectUnReportedPeerTask(storeID, regionIDs, interval)) + c.hotStat.CheckReadAsync(statistics.NewCollectUnReportedPeerTask(storeID, regions, interval)) return nil } diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index fb50c8689424..81986a5ed3ab 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -33,8 +33,8 @@ import ( "github.com/tikv/pd/server/core" "github.com/tikv/pd/server/id" "github.com/tikv/pd/server/kv" + "github.com/tikv/pd/server/schedule" "github.com/tikv/pd/server/schedule/labeler" - "github.com/tikv/pd/server/schedule/opt" "github.com/tikv/pd/server/schedule/placement" "github.com/tikv/pd/server/statistics" "github.com/tikv/pd/server/versioninfo" @@ -974,10 +974,10 @@ func (s *testRegionsInfoSuite) Test(c *C) { } for i := uint64(0); i < n; i++ { - region := tc.RandLeaderRegion(i, []core.KeyRange{core.NewKeyRange("", "")}, opt.IsRegionHealthy) + region := tc.RandLeaderRegion(i, []core.KeyRange{core.NewKeyRange("", "")}, schedule.IsRegionHealthy) c.Assert(region.GetLeader().GetStoreId(), Equals, i) - region = tc.RandFollowerRegion(i, []core.KeyRange{core.NewKeyRange("", "")}, opt.IsRegionHealthy) + region = tc.RandFollowerRegion(i, []core.KeyRange{core.NewKeyRange("", "")}, schedule.IsRegionHealthy) c.Assert(region.GetLeader().GetStoreId(), Not(Equals), i) c.Assert(region.GetStorePeer(i), NotNil) @@ -993,14 +993,14 @@ func (s *testRegionsInfoSuite) Test(c *C) { // All regions will be filtered out if they have pending peers. for i := uint64(0); i < n; i++ { for j := 0; j < cache.GetStoreLeaderCount(i); j++ { - region := tc.RandLeaderRegion(i, []core.KeyRange{core.NewKeyRange("", "")}, opt.IsRegionHealthy) + region := tc.RandLeaderRegion(i, []core.KeyRange{core.NewKeyRange("", "")}, schedule.IsRegionHealthy) newRegion := region.Clone(core.WithPendingPeers(region.GetPeers())) cache.SetRegion(newRegion) } - c.Assert(tc.RandLeaderRegion(i, []core.KeyRange{core.NewKeyRange("", "")}, opt.IsRegionHealthy), IsNil) + c.Assert(tc.RandLeaderRegion(i, []core.KeyRange{core.NewKeyRange("", "")}, schedule.IsRegionHealthy), IsNil) } for i := uint64(0); i < n; i++ { - c.Assert(tc.RandFollowerRegion(i, []core.KeyRange{core.NewKeyRange("", "")}, opt.IsRegionHealthy), IsNil) + c.Assert(tc.RandFollowerRegion(i, []core.KeyRange{core.NewKeyRange("", "")}, schedule.IsRegionHealthy), IsNil) } } diff --git a/server/cluster/coordinator.go b/server/cluster/coordinator.go index 7cdbc60628a6..a64df4e5f791 100644 --- a/server/cluster/coordinator.go +++ b/server/cluster/coordinator.go @@ -26,12 +26,14 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/log" + "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/logutil" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/core" "github.com/tikv/pd/server/kv" "github.com/tikv/pd/server/schedule" + "github.com/tikv/pd/server/schedule/checker" "github.com/tikv/pd/server/schedule/hbstream" "github.com/tikv/pd/server/schedule/operator" "github.com/tikv/pd/server/schedulers" @@ -65,7 +67,7 @@ type coordinator struct { ctx context.Context cancel context.CancelFunc cluster *RaftCluster - checkers *schedule.CheckerController + checkers *checker.Controller regionScatterer *schedule.RegionScatterer regionSplitter *schedule.RegionSplitter schedulers map[string]*scheduleController @@ -82,7 +84,7 @@ func newCoordinator(ctx context.Context, cluster *RaftCluster, hbStreams *hbstre ctx: ctx, cancel: cancel, cluster: cluster, - checkers: schedule.NewCheckerController(ctx, cluster, cluster.ruleManager, cluster.regionLabeler, opController), + checkers: checker.NewController(ctx, cluster, cluster.ruleManager, cluster.regionLabeler, opController), regionScatterer: schedule.NewRegionScatterer(ctx, cluster), regionSplitter: schedule.NewRegionSplitter(cluster, schedule.NewSplitRegionsHandler(cluster, opController)), schedulers: make(map[string]*scheduleController), @@ -92,6 +94,10 @@ func newCoordinator(ctx context.Context, cluster *RaftCluster, hbStreams *hbstre } } +func (c *coordinator) GetWaitingRegions() []*cache.Item { + return c.checkers.GetWaitingRegions() +} + // patrolRegions is used to scan regions. // The checkers will check these regions to decide if they need to do some operations. func (c *coordinator) patrolRegions() { diff --git a/server/cluster/coordinator_test.go b/server/cluster/coordinator_test.go index 34d38fd2db76..9beb3e8a2d90 100644 --- a/server/cluster/coordinator_test.go +++ b/server/cluster/coordinator_test.go @@ -239,7 +239,7 @@ func (s *testCoordinatorSuite) TestDispatch(c *C) { waitNoResponse(c, stream) } -func dispatchHeartbeat(co *coordinator, region *core.RegionInfo, stream opt.HeartbeatStream) error { +func dispatchHeartbeat(co *coordinator, region *core.RegionInfo, stream hbstream.HeartbeatStream) error { co.hbStreams.BindStream(region.GetLeader().GetStoreId(), stream) if err := co.cluster.putRegion(region.Clone()); err != nil { return err diff --git a/server/config/config.go b/server/config/config.go index 930d459b0f34..0ce34cc42e32 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -187,7 +187,7 @@ func NewConfig() *Config { fs.StringVar(&cfg.Metric.PushAddress, "metrics-addr", "", "prometheus pushgateway address, leaves it empty will disable prometheus push") - fs.StringVar(&cfg.Log.Level, "L", "", "log level: debug, info, warn, error, fatal (default 'info')") + fs.StringVar(&cfg.Log.Level, "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") fs.StringVar(&cfg.Log.File.Filename, "log-file", "", "log file path") fs.StringVar(&cfg.Security.CAPath, "cacert", "", "path of file that contains list of trusted TLS CAs") @@ -786,7 +786,7 @@ const ( defaultEnableJointConsensus = true defaultEnableCrossTableMerge = true defaultHotRegionsWriteInterval = 10 * time.Minute - defaultHotRegionsReservedDays = 0 + defaultHotRegionsReservedDays = 7 ) func (c *ScheduleConfig) adjust(meta *configMetaData, reloading bool) error { diff --git a/server/config/config_test.go b/server/config/config_test.go index a0353828689c..cb3c29eb3f39 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -194,6 +194,7 @@ leader-schedule-limit = 0 c.Assert(cfg.Schedule.LeaderScheduleLimit, Equals, uint64(0)) // When undefined, use default values. c.Assert(cfg.PreVote, IsTrue) + c.Assert(cfg.Log.Level, Equals, "info") c.Assert(cfg.Schedule.MaxMergeRegionKeys, Equals, uint64(defaultMaxMergeRegionKeys)) c.Assert(cfg.PDServerCfg.MetricStorage, Equals, "http://127.0.0.1:9090") @@ -464,7 +465,7 @@ wait-store-timeout = "120s" c.Assert(cfg.ReplicationMode.ReplicationMode, Equals, "majority") } -func (s *testConfigSuite) TestHotRegionConfig(c *C) { +func (s *testConfigSuite) TestHotHistoryRegionConfig(c *C) { cfgData := ` [schedule] hot-regions-reserved-days= 30 @@ -475,8 +476,14 @@ hot-regions-write-interval= "30m" c.Assert(err, IsNil) err = cfg.Adjust(&meta, false) c.Assert(err, IsNil) - c.Assert(cfg.Schedule.HotRegionsWriteInterval.Duration, Equals, time.Minute*30) + c.Assert(cfg.Schedule.HotRegionsWriteInterval.Duration, Equals, 30*time.Minute) c.Assert(cfg.Schedule.HotRegionsReservedDays, Equals, int64(30)) + // Verify default value + cfg = NewConfig() + err = cfg.Adjust(nil, false) + c.Assert(err, IsNil) + c.Assert(cfg.Schedule.HotRegionsWriteInterval.Duration, Equals, 10*time.Minute) + c.Assert(cfg.Schedule.HotRegionsReservedDays, Equals, int64(7)) } func (s *testConfigSuite) TestConfigClone(c *C) { diff --git a/server/handler.go b/server/handler.go index 7b16ba2f827d..ddc4c71322ca 100644 --- a/server/handler.go +++ b/server/handler.go @@ -37,7 +37,6 @@ import ( "github.com/tikv/pd/server/core/storelimit" "github.com/tikv/pd/server/schedule" "github.com/tikv/pd/server/schedule/operator" - "github.com/tikv/pd/server/schedule/opt" "github.com/tikv/pd/server/schedule/placement" "github.com/tikv/pd/server/schedulers" "github.com/tikv/pd/server/statistics" @@ -717,11 +716,11 @@ func (h *Handler) AddMergeRegionOperator(regionID uint64, targetID uint64) error return ErrRegionNotFound(targetID) } - if !opt.IsRegionHealthy(region) || !opt.IsRegionReplicated(c, region) { + if !schedule.IsRegionHealthy(region) || !schedule.IsRegionReplicated(c, region) { return ErrRegionAbnormalPeer(regionID) } - if !opt.IsRegionHealthy(target) || !opt.IsRegionReplicated(c, target) { + if !schedule.IsRegionHealthy(target) || !schedule.IsRegionReplicated(c, target) { return ErrRegionAbnormalPeer(targetID) } @@ -990,7 +989,7 @@ func (h *Handler) packHotRegions(hotPeersStat statistics.StoreHotPeersStat, hotR } } stat := core.HistoryHotRegion{ - // store in ms. + // store in ms. UpdateTime: hotPeerStat.LastUpdateTime.UnixNano() / int64(time.Millisecond), RegionID: hotPeerStat.RegionID, StoreID: hotPeerStat.StoreID, diff --git a/server/schedule/checker_controller.go b/server/schedule/checker/checker_controller.go similarity index 70% rename from server/schedule/checker_controller.go rename to server/schedule/checker/checker_controller.go index cde06dbde294..1cffdb60b7e4 100644 --- a/server/schedule/checker_controller.go +++ b/server/schedule/checker/checker_controller.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package checker import ( "context" @@ -21,7 +21,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/core" - "github.com/tikv/pd/server/schedule/checker" + "github.com/tikv/pd/server/schedule" "github.com/tikv/pd/server/schedule/labeler" "github.com/tikv/pd/server/schedule/operator" "github.com/tikv/pd/server/schedule/opt" @@ -31,42 +31,42 @@ import ( // DefaultCacheSize is the default length of waiting list. const DefaultCacheSize = 1000 -// CheckerController is used to manage all checkers. -type CheckerController struct { +// Controller is used to manage all checkers. +type Controller struct { cluster opt.Cluster opts *config.PersistOptions - opController *OperatorController - learnerChecker *checker.LearnerChecker - replicaChecker *checker.ReplicaChecker - ruleChecker *checker.RuleChecker - splitChecker *checker.SplitChecker - mergeChecker *checker.MergeChecker - jointStateChecker *checker.JointStateChecker - priorityInspector *checker.PriorityInspector + opController *schedule.OperatorController + learnerChecker *LearnerChecker + replicaChecker *ReplicaChecker + ruleChecker *RuleChecker + splitChecker *SplitChecker + mergeChecker *MergeChecker + jointStateChecker *JointStateChecker + priorityInspector *PriorityInspector regionWaitingList cache.Cache } -// NewCheckerController create a new CheckerController. +// NewController create a new Controller. // TODO: isSupportMerge should be removed. -func NewCheckerController(ctx context.Context, cluster opt.Cluster, ruleManager *placement.RuleManager, labeler *labeler.RegionLabeler, opController *OperatorController) *CheckerController { +func NewController(ctx context.Context, cluster opt.Cluster, ruleManager *placement.RuleManager, labeler *labeler.RegionLabeler, opController *schedule.OperatorController) *Controller { regionWaitingList := cache.NewDefaultCache(DefaultCacheSize) - return &CheckerController{ + return &Controller{ cluster: cluster, opts: cluster.GetOpts(), opController: opController, - learnerChecker: checker.NewLearnerChecker(cluster), - replicaChecker: checker.NewReplicaChecker(cluster, regionWaitingList), - ruleChecker: checker.NewRuleChecker(cluster, ruleManager, regionWaitingList), - splitChecker: checker.NewSplitChecker(cluster, ruleManager, labeler), - mergeChecker: checker.NewMergeChecker(ctx, cluster), - jointStateChecker: checker.NewJointStateChecker(cluster), - priorityInspector: checker.NewPriorityInspector(cluster), + learnerChecker: NewLearnerChecker(cluster), + replicaChecker: NewReplicaChecker(cluster, regionWaitingList), + ruleChecker: NewRuleChecker(cluster, ruleManager, regionWaitingList), + splitChecker: NewSplitChecker(cluster, ruleManager, labeler), + mergeChecker: NewMergeChecker(ctx, cluster), + jointStateChecker: NewJointStateChecker(cluster), + priorityInspector: NewPriorityInspector(cluster), regionWaitingList: regionWaitingList, } } // CheckRegion will check the region and add a new operator if needed. -func (c *CheckerController) CheckRegion(region *core.RegionInfo) []*operator.Operator { +func (c *Controller) CheckRegion(region *core.RegionInfo) []*operator.Operator { // If PD has restarted, it need to check learners added before and promote them. // Don't check isRaftLearnerEnabled cause it maybe disable learner feature but there are still some learners to promote. opController := c.opController @@ -114,42 +114,42 @@ func (c *CheckerController) CheckRegion(region *core.RegionInfo) []*operator.Ope } // GetMergeChecker returns the merge checker. -func (c *CheckerController) GetMergeChecker() *checker.MergeChecker { +func (c *Controller) GetMergeChecker() *MergeChecker { return c.mergeChecker } // GetRuleChecker returns the rule checker. -func (c *CheckerController) GetRuleChecker() *checker.RuleChecker { +func (c *Controller) GetRuleChecker() *RuleChecker { return c.ruleChecker } // GetWaitingRegions returns the regions in the waiting list. -func (c *CheckerController) GetWaitingRegions() []*cache.Item { +func (c *Controller) GetWaitingRegions() []*cache.Item { return c.regionWaitingList.Elems() } // AddWaitingRegion returns the regions in the waiting list. -func (c *CheckerController) AddWaitingRegion(region *core.RegionInfo) { +func (c *Controller) AddWaitingRegion(region *core.RegionInfo) { c.regionWaitingList.Put(region.GetID(), nil) } // RemoveWaitingRegion removes the region from the waiting list. -func (c *CheckerController) RemoveWaitingRegion(id uint64) { +func (c *Controller) RemoveWaitingRegion(id uint64) { c.regionWaitingList.Remove(id) } // GetPriorityRegions returns the region in priority queue -func (c *CheckerController) GetPriorityRegions() []uint64 { +func (c *Controller) GetPriorityRegions() []uint64 { return c.priorityInspector.GetPriorityRegions() } // RemovePriorityRegions removes priority region from priority queue -func (c *CheckerController) RemovePriorityRegions(id uint64) { +func (c *Controller) RemovePriorityRegions(id uint64) { c.priorityInspector.RemovePriorityRegion(id) } // GetPauseController returns pause controller of the checker -func (c *CheckerController) GetPauseController(name string) (*checker.PauseController, error) { +func (c *Controller) GetPauseController(name string) (*PauseController, error) { switch name { case "learner": return &c.learnerChecker.PauseController, nil diff --git a/server/schedule/checker/merge_checker.go b/server/schedule/checker/merge_checker.go index 15175b2c5384..fdd287eb2316 100644 --- a/server/schedule/checker/merge_checker.go +++ b/server/schedule/checker/merge_checker.go @@ -26,6 +26,7 @@ import ( "github.com/tikv/pd/pkg/logutil" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/core" + "github.com/tikv/pd/server/schedule" "github.com/tikv/pd/server/schedule/labeler" "github.com/tikv/pd/server/schedule/operator" "github.com/tikv/pd/server/schedule/opt" @@ -112,12 +113,12 @@ func (m *MergeChecker) Check(region *core.RegionInfo) []*operator.Operator { } // skip region has down peers or pending peers - if !opt.IsRegionHealthy(region) { + if !schedule.IsRegionHealthy(region) { checkerCounter.WithLabelValues("merge_checker", "special-peer").Inc() return nil } - if !opt.IsRegionReplicated(m.cluster, region) { + if !schedule.IsRegionReplicated(m.cluster, region) { checkerCounter.WithLabelValues("merge_checker", "abnormal-replica").Inc() return nil } @@ -169,7 +170,7 @@ func (m *MergeChecker) Check(region *core.RegionInfo) []*operator.Operator { func (m *MergeChecker) checkTarget(region, adjacent *core.RegionInfo) bool { return adjacent != nil && !m.splitCache.Exists(adjacent.GetID()) && !m.cluster.IsRegionHot(adjacent) && AllowMerge(m.cluster, region, adjacent) && checkPeerStore(m.cluster, region, adjacent) && - opt.IsRegionHealthy(adjacent) && opt.IsRegionReplicated(m.cluster, adjacent) + schedule.IsRegionHealthy(adjacent) && schedule.IsRegionReplicated(m.cluster, adjacent) } // AllowMerge returns true if two regions can be merged according to the key type. diff --git a/server/schedule/checker/merge_checker_test.go b/server/schedule/checker/merge_checker_test.go index 30dac3a67030..d461dd368159 100644 --- a/server/schedule/checker/merge_checker_test.go +++ b/server/schedule/checker/merge_checker_test.go @@ -26,6 +26,9 @@ import ( "github.com/tikv/pd/pkg/testutil" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/core" + "github.com/tikv/pd/server/core/storelimit" + "github.com/tikv/pd/server/schedule" + "github.com/tikv/pd/server/schedule/hbstream" "github.com/tikv/pd/server/schedule/labeler" "github.com/tikv/pd/server/schedule/operator" "github.com/tikv/pd/server/schedule/placement" @@ -77,63 +80,10 @@ func (s *testMergeCheckerSuite) SetUpTest(c *C) { s.cluster.PutStoreWithLabels(storeID, labels...) } s.regions = []*core.RegionInfo{ - core.NewRegionInfo( - &metapb.Region{ - Id: 1, - StartKey: []byte(""), - EndKey: []byte("a"), - Peers: []*metapb.Peer{ - {Id: 101, StoreId: 1}, - {Id: 102, StoreId: 2}, - }, - }, - &metapb.Peer{Id: 101, StoreId: 1}, - core.SetApproximateSize(1), - core.SetApproximateKeys(1), - ), - core.NewRegionInfo( - &metapb.Region{ - Id: 2, - StartKey: []byte("a"), - EndKey: []byte("t"), - Peers: []*metapb.Peer{ - {Id: 103, StoreId: 1}, - {Id: 104, StoreId: 4}, - {Id: 105, StoreId: 5}, - }, - }, - &metapb.Peer{Id: 104, StoreId: 4}, - core.SetApproximateSize(200), - core.SetApproximateKeys(200), - ), - core.NewRegionInfo( - &metapb.Region{ - Id: 3, - StartKey: []byte("t"), - EndKey: []byte("x"), - Peers: []*metapb.Peer{ - {Id: 106, StoreId: 2}, - {Id: 107, StoreId: 5}, - {Id: 108, StoreId: 6}, - }, - }, - &metapb.Peer{Id: 108, StoreId: 6}, - core.SetApproximateSize(1), - core.SetApproximateKeys(1), - ), - core.NewRegionInfo( - &metapb.Region{ - Id: 4, - StartKey: []byte("x"), - EndKey: []byte(""), - Peers: []*metapb.Peer{ - {Id: 109, StoreId: 4}, - }, - }, - &metapb.Peer{Id: 109, StoreId: 4}, - core.SetApproximateSize(1), - core.SetApproximateKeys(1), - ), + newRegionInfo(1, "", "a", 1, 1, []uint64{101, 1}, []uint64{101, 1}, []uint64{102, 2}), + newRegionInfo(2, "a", "t", 200, 200, []uint64{104, 4}, []uint64{103, 1}, []uint64{104, 4}, []uint64{105, 5}), + newRegionInfo(3, "t", "x", 1, 1, []uint64{108, 6}, []uint64{106, 2}, []uint64{107, 5}, []uint64{108, 6}), + newRegionInfo(4, "x", "", 1, 1, []uint64{109, 4}, []uint64{109, 4}), } for _, region := range s.regions { @@ -496,9 +446,73 @@ func (s *testMergeCheckerSuite) TestMatchPeers(c *C) { }) } -var _ = Suite(&testSplitMergeSuite{}) +func (s *testMergeCheckerSuite) TestStoreLimitWithMerge(c *C) { + cfg := config.NewTestOptions() + tc := mockcluster.NewCluster(s.ctx, cfg) + tc.SetMaxMergeRegionSize(2) + tc.SetMaxMergeRegionKeys(2) + tc.SetSplitMergeInterval(0) + regions := []*core.RegionInfo{ + newRegionInfo(1, "", "a", 1, 1, []uint64{101, 1}, []uint64{101, 1}, []uint64{102, 2}), + newRegionInfo(2, "a", "t", 200, 200, []uint64{104, 4}, []uint64{103, 1}, []uint64{104, 4}, []uint64{105, 5}), + newRegionInfo(3, "t", "x", 1, 1, []uint64{108, 6}, []uint64{106, 2}, []uint64{107, 5}, []uint64{108, 6}), + newRegionInfo(4, "x", "", 10, 10, []uint64{109, 4}, []uint64{109, 4}), + } + + for i := uint64(1); i <= 6; i++ { + tc.AddLeaderStore(i, 10) + } + + for _, region := range regions { + tc.PutRegion(region) + } -type testSplitMergeSuite struct{} + mc := NewMergeChecker(s.ctx, tc) + stream := hbstream.NewTestHeartbeatStreams(s.ctx, tc.ID, tc, false /* no need to run */) + oc := schedule.NewOperatorController(s.ctx, tc, stream) + + regions[2] = regions[2].Clone( + core.SetPeers([]*metapb.Peer{ + {Id: 109, StoreId: 2}, + {Id: 110, StoreId: 3}, + {Id: 111, StoreId: 6}, + }), + core.WithLeader(&metapb.Peer{Id: 109, StoreId: 2}), + ) + + // set to a small rate to reduce unstable possibility. + tc.SetAllStoresLimit(storelimit.AddPeer, 0.0000001) + tc.SetAllStoresLimit(storelimit.RemovePeer, 0.0000001) + tc.PutRegion(regions[2]) + // The size of Region is less or equal than 1MB. + for i := 0; i < 50; i++ { + ops := mc.Check(regions[2]) + c.Assert(ops, NotNil) + c.Assert(oc.AddOperator(ops...), IsTrue) + for _, op := range ops { + oc.RemoveOperator(op) + } + } + regions[2] = regions[2].Clone( + core.SetApproximateSize(2), + core.SetApproximateKeys(2), + ) + tc.PutRegion(regions[2]) + // The size of Region is more than 1MB but no more than 20MB. + for i := 0; i < 5; i++ { + ops := mc.Check(regions[2]) + c.Assert(ops, NotNil) + c.Assert(oc.AddOperator(ops...), IsTrue) + for _, op := range ops { + oc.RemoveOperator(op) + } + } + { + ops := mc.Check(regions[2]) + c.Assert(ops, NotNil) + c.Assert(oc.AddOperator(ops...), IsFalse) + } +} func (s *testMergeCheckerSuite) TestCache(c *C) { cfg := config.NewTestOptions() @@ -514,36 +528,8 @@ func (s *testMergeCheckerSuite) TestCache(c *C) { s.cluster.PutStoreWithLabels(storeID, labels...) } s.regions = []*core.RegionInfo{ - core.NewRegionInfo( - &metapb.Region{ - Id: 2, - StartKey: []byte("a"), - EndKey: []byte("t"), - Peers: []*metapb.Peer{ - {Id: 103, StoreId: 1}, - {Id: 104, StoreId: 4}, - {Id: 105, StoreId: 5}, - }, - }, - &metapb.Peer{Id: 104, StoreId: 4}, - core.SetApproximateSize(200), - core.SetApproximateKeys(200), - ), - core.NewRegionInfo( - &metapb.Region{ - Id: 3, - StartKey: []byte("t"), - EndKey: []byte("x"), - Peers: []*metapb.Peer{ - {Id: 106, StoreId: 2}, - {Id: 107, StoreId: 5}, - {Id: 108, StoreId: 6}, - }, - }, - &metapb.Peer{Id: 108, StoreId: 6}, - core.SetApproximateSize(1), - core.SetApproximateKeys(1), - ), + newRegionInfo(2, "a", "t", 200, 200, []uint64{104, 4}, []uint64{103, 1}, []uint64{104, 4}, []uint64{105, 5}), + newRegionInfo(3, "t", "x", 1, 1, []uint64{108, 6}, []uint64{106, 2}, []uint64{107, 5}, []uint64{108, 6}), } for _, region := range s.regions { @@ -567,3 +553,21 @@ func makeKeyRanges(keys ...string) []interface{} { } return res } + +func newRegionInfo(id uint64, startKey, endKey string, size, keys int64, leader []uint64, peers ...[]uint64) *core.RegionInfo { + prs := make([]*metapb.Peer, 0, len(peers)) + for _, peer := range peers { + prs = append(prs, &metapb.Peer{Id: peer[0], StoreId: peer[1]}) + } + return core.NewRegionInfo( + &metapb.Region{ + Id: id, + StartKey: []byte(startKey), + EndKey: []byte(endKey), + Peers: prs, + }, + &metapb.Peer{Id: leader[0], StoreId: leader[1]}, + core.SetApproximateSize(size), + core.SetApproximateKeys(keys), + ) +} diff --git a/server/schedule/hbstream/heartbeat_streams.go b/server/schedule/hbstream/heartbeat_streams.go index 204ede99abc7..1fe0a92c59ff 100644 --- a/server/schedule/hbstream/heartbeat_streams.go +++ b/server/schedule/hbstream/heartbeat_streams.go @@ -27,10 +27,14 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/logutil" "github.com/tikv/pd/server/core" - "github.com/tikv/pd/server/schedule/opt" "go.uber.org/zap" ) +// HeartbeatStream is an interface. +type HeartbeatStream interface { + Send(*pdpb.RegionHeartbeatResponse) error +} + const ( heartbeatStreamKeepAliveInterval = time.Minute heartbeatChanCapacity = 1024 @@ -38,7 +42,7 @@ const ( type streamUpdate struct { storeID uint64 - stream opt.HeartbeatStream + stream HeartbeatStream } // HeartbeatStreams is the bridge of communication with TIKV instance. @@ -47,7 +51,7 @@ type HeartbeatStreams struct { hbStreamCtx context.Context hbStreamCancel context.CancelFunc clusterID uint64 - streams map[uint64]opt.HeartbeatStream + streams map[uint64]HeartbeatStream msgCh chan *pdpb.RegionHeartbeatResponse streamCh chan streamUpdate storeInformer core.StoreSetInformer @@ -71,7 +75,7 @@ func newHbStreams(ctx context.Context, clusterID uint64, storeInformer core.Stor hbStreamCtx: hbStreamCtx, hbStreamCancel: hbStreamCancel, clusterID: clusterID, - streams: make(map[uint64]opt.HeartbeatStream), + streams: make(map[uint64]HeartbeatStream), msgCh: make(chan *pdpb.RegionHeartbeatResponse, heartbeatChanCapacity), streamCh: make(chan streamUpdate, 1), storeInformer: storeInformer, @@ -158,7 +162,7 @@ func (s *HeartbeatStreams) Close() { } // BindStream binds a stream with a specified store. -func (s *HeartbeatStreams) BindStream(storeID uint64, stream opt.HeartbeatStream) { +func (s *HeartbeatStreams) BindStream(storeID uint64, stream HeartbeatStream) { update := streamUpdate{ storeID: storeID, stream: stream, diff --git a/server/schedule/opt/healthy.go b/server/schedule/healthy.go similarity index 67% rename from server/schedule/opt/healthy.go rename to server/schedule/healthy.go index dead904c6467..a8333f9f0e6d 100644 --- a/server/schedule/opt/healthy.go +++ b/server/schedule/healthy.go @@ -12,15 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package opt +package schedule import ( "github.com/tikv/pd/server/core" + "github.com/tikv/pd/server/schedule/opt" ) -// BalanceEmptyRegionThreshold is a threshold which allow balance the empty region if the region number is less than this threshold. -var balanceEmptyRegionThreshold = 50 - // IsRegionHealthy checks if a region is healthy for scheduling. It requires the // region does not have any down or pending peers. func IsRegionHealthy(region *core.RegionInfo) bool { @@ -33,20 +31,10 @@ func IsRegionHealthyAllowPending(region *core.RegionInfo) bool { return len(region.GetDownPeers()) == 0 } -// IsEmptyRegionAllowBalance checks if a region is an empty region and can be balanced. -func IsEmptyRegionAllowBalance(cluster Cluster, region *core.RegionInfo) bool { - return region.GetApproximateSize() > core.EmptyRegionApproximateSize || cluster.GetRegionCount() < balanceEmptyRegionThreshold -} - -// AllowBalanceEmptyRegion returns a function that checks if a region is an empty region and can be balanced. -func AllowBalanceEmptyRegion(cluster Cluster) func(*core.RegionInfo) bool { - return func(region *core.RegionInfo) bool { return IsEmptyRegionAllowBalance(cluster, region) } -} - // IsRegionReplicated checks if a region is fully replicated. When placement // rules is enabled, its peers should fit corresponding rules. When placement // rules is disabled, it should have enough replicas and no any learner peer. -func IsRegionReplicated(cluster Cluster, region *core.RegionInfo) bool { +func IsRegionReplicated(cluster opt.Cluster, region *core.RegionInfo) bool { if cluster.GetOpts().IsPlacementRulesEnabled() { return cluster.GetRuleManager().FitRegion(cluster, region).IsSatisfied() } @@ -54,6 +42,6 @@ func IsRegionReplicated(cluster Cluster, region *core.RegionInfo) bool { } // ReplicatedRegion returns a function that checks if a region is fully replicated. -func ReplicatedRegion(cluster Cluster) func(*core.RegionInfo) bool { +func ReplicatedRegion(cluster opt.Cluster) func(*core.RegionInfo) bool { return func(region *core.RegionInfo) bool { return IsRegionReplicated(cluster, region) } } diff --git a/server/schedule/opt/healthy_test.go b/server/schedule/healthy_test.go similarity index 97% rename from server/schedule/opt/healthy_test.go rename to server/schedule/healthy_test.go index 8f2e7feba047..8adb69d9ede0 100644 --- a/server/schedule/opt/healthy_test.go +++ b/server/schedule/healthy_test.go @@ -12,11 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package opt +package schedule import ( "context" - "testing" . "github.com/pingcap/check" "github.com/pingcap/kvproto/pkg/metapb" @@ -26,10 +25,6 @@ import ( "github.com/tikv/pd/server/core" ) -func TestOpt(t *testing.T) { - TestingT(t) -} - var _ = Suite(&testRegionHealthySuite{}) type testRegionHealthySuite struct { diff --git a/server/schedule/operator_controller_test.go b/server/schedule/operator_controller_test.go index 1ac798494f66..03ca334d3b1e 100644 --- a/server/schedule/operator_controller_test.go +++ b/server/schedule/operator_controller_test.go @@ -30,7 +30,6 @@ import ( "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/core" "github.com/tikv/pd/server/core/storelimit" - "github.com/tikv/pd/server/schedule/checker" "github.com/tikv/pd/server/schedule/hbstream" "github.com/tikv/pd/server/schedule/operator" ) @@ -592,74 +591,6 @@ func (t *testOperatorControllerSuite) TestDispatchUnfinishedStep(c *C) { } } -func (t *testOperatorControllerSuite) TestStoreLimitWithMerge(c *C) { - cfg := config.NewTestOptions() - tc := mockcluster.NewCluster(t.ctx, cfg) - tc.SetMaxMergeRegionSize(2) - tc.SetMaxMergeRegionKeys(2) - tc.SetSplitMergeInterval(0) - regions := []*core.RegionInfo{ - newRegionInfo(1, "", "a", 1, 1, []uint64{101, 1}, []uint64{101, 1}, []uint64{102, 2}), - newRegionInfo(2, "a", "t", 200, 200, []uint64{104, 4}, []uint64{103, 1}, []uint64{104, 4}, []uint64{105, 5}), - newRegionInfo(3, "t", "x", 1, 1, []uint64{108, 6}, []uint64{106, 2}, []uint64{107, 5}, []uint64{108, 6}), - newRegionInfo(4, "x", "", 10, 10, []uint64{109, 4}, []uint64{109, 4}), - } - - for i := uint64(1); i <= 6; i++ { - tc.AddLeaderStore(i, 10) - } - - for _, region := range regions { - tc.PutRegion(region) - } - - mc := checker.NewMergeChecker(t.ctx, tc) - stream := hbstream.NewTestHeartbeatStreams(t.ctx, tc.ID, tc, false /* no need to run */) - oc := NewOperatorController(t.ctx, tc, stream) - - regions[2] = regions[2].Clone( - core.SetPeers([]*metapb.Peer{ - {Id: 109, StoreId: 2}, - {Id: 110, StoreId: 3}, - {Id: 111, StoreId: 6}, - }), - core.WithLeader(&metapb.Peer{Id: 109, StoreId: 2}), - ) - - // set to a small rate to reduce unstable possibility. - tc.SetAllStoresLimit(storelimit.AddPeer, 0.0000001) - tc.SetAllStoresLimit(storelimit.RemovePeer, 0.0000001) - tc.PutRegion(regions[2]) - // The size of Region is less or equal than 1MB. - for i := 0; i < 50; i++ { - ops := mc.Check(regions[2]) - c.Assert(ops, NotNil) - c.Assert(oc.AddOperator(ops...), IsTrue) - for _, op := range ops { - oc.RemoveOperator(op) - } - } - regions[2] = regions[2].Clone( - core.SetApproximateSize(2), - core.SetApproximateKeys(2), - ) - tc.PutRegion(regions[2]) - // The size of Region is more than 1MB but no more than 20MB. - for i := 0; i < 5; i++ { - ops := mc.Check(regions[2]) - c.Assert(ops, NotNil) - c.Assert(oc.AddOperator(ops...), IsTrue) - for _, op := range ops { - oc.RemoveOperator(op) - } - } - { - ops := mc.Check(regions[2]) - c.Assert(ops, NotNil) - c.Assert(oc.AddOperator(ops...), IsFalse) - } -} - func newRegionInfo(id uint64, startKey, endKey string, size, keys int64, leader []uint64, peers ...[]uint64) *core.RegionInfo { prs := make([]*metapb.Peer, 0, len(peers)) for _, peer := range peers { diff --git a/server/schedule/opt/opts.go b/server/schedule/opt/opts.go index 6e92f5126217..3dfbfdfa7b7d 100644 --- a/server/schedule/opt/opts.go +++ b/server/schedule/opt/opts.go @@ -15,7 +15,6 @@ package opt import ( - "github.com/pingcap/kvproto/pkg/pdpb" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/core" "github.com/tikv/pd/server/schedule/placement" @@ -41,8 +40,3 @@ type Cluster interface { AddSuspectRegions(ids ...uint64) GetBasicCluster() *core.BasicCluster } - -// HeartbeatStream is an interface. -type HeartbeatStream interface { - Send(*pdpb.RegionHeartbeatResponse) error -} diff --git a/server/schedule/region_scatterer.go b/server/schedule/region_scatterer.go index 408df09e6970..c07facf2876d 100644 --- a/server/schedule/region_scatterer.go +++ b/server/schedule/region_scatterer.go @@ -250,7 +250,7 @@ func (r *RegionScatterer) ScatterRegions(regions map[uint64]*core.RegionInfo, fa // Scatter relocates the region. If the group is defined, the regions' leader with the same group would be scattered // in a group level instead of cluster level. func (r *RegionScatterer) Scatter(region *core.RegionInfo, group string) (*operator.Operator, error) { - if !opt.IsRegionReplicated(r.cluster, region) { + if !IsRegionReplicated(r.cluster, region) { r.cluster.AddSuspectRegions(region.GetID()) scatterCounter.WithLabelValues("skip", "not-replicated").Inc() log.Warn("region not replicated during scatter", zap.Uint64("region-id", region.GetID())) diff --git a/server/schedule/region_splitter.go b/server/schedule/region_splitter.go index ce4d84e31dcc..30ac286111fb 100644 --- a/server/schedule/region_splitter.go +++ b/server/schedule/region_splitter.go @@ -166,7 +166,7 @@ func (r *RegionSplitter) checkRegionValid(region *core.RegionInfo) bool { if r.cluster.IsRegionHot(region) { return false } - if !opt.IsRegionReplicated(r.cluster, region) { + if !IsRegionReplicated(r.cluster, region) { r.cluster.AddSuspectRegions(region.GetID()) return false } diff --git a/server/schedulers/balance_leader.go b/server/schedulers/balance_leader.go index 705082a1762a..288d594c89df 100644 --- a/server/schedulers/balance_leader.go +++ b/server/schedulers/balance_leader.go @@ -203,7 +203,7 @@ func (l *balanceLeaderScheduler) Schedule(cluster opt.Cluster) []*operator.Opera // It randomly selects a health region from the source store, then picks // the best follower peer and transfers the leader. func (l *balanceLeaderScheduler) transferLeaderOut(plan *balancePlan) []*operator.Operator { - plan.region = plan.cluster.RandLeaderRegion(plan.SourceStoreID(), l.conf.Ranges, opt.IsRegionHealthy) + plan.region = plan.cluster.RandLeaderRegion(plan.SourceStoreID(), l.conf.Ranges, schedule.IsRegionHealthy) if plan.region == nil { log.Debug("store has no leader", zap.String("scheduler", l.GetName()), zap.Uint64("store-id", plan.SourceStoreID())) schedulerCounter.WithLabelValues(l.GetName(), "no-leader-region").Inc() @@ -235,7 +235,7 @@ func (l *balanceLeaderScheduler) transferLeaderOut(plan *balancePlan) []*operato // It randomly selects a health region from the target store, then picks // the worst follower peer and transfers the leader. func (l *balanceLeaderScheduler) transferLeaderIn(plan *balancePlan) []*operator.Operator { - plan.region = plan.cluster.RandFollowerRegion(plan.TargetStoreID(), l.conf.Ranges, opt.IsRegionHealthy) + plan.region = plan.cluster.RandFollowerRegion(plan.TargetStoreID(), l.conf.Ranges, schedule.IsRegionHealthy) if plan.region == nil { log.Debug("store has no follower", zap.String("scheduler", l.GetName()), zap.Uint64("store-id", plan.TargetStoreID())) schedulerCounter.WithLabelValues(l.GetName(), "no-follower-region").Inc() diff --git a/server/schedulers/balance_region.go b/server/schedulers/balance_region.go index 135dcf72e24f..cc9e8cec8793 100644 --- a/server/schedulers/balance_region.go +++ b/server/schedulers/balance_region.go @@ -62,6 +62,8 @@ const ( BalanceRegionName = "balance-region-scheduler" // BalanceRegionType is balance region scheduler type. BalanceRegionType = "balance-region" + // BalanceEmptyRegionThreshold is a threshold which allow balance the empty region if the region number is less than this threshold. + balanceEmptyRegionThreshold = 50 ) type balanceRegionSchedulerConfig struct { @@ -160,7 +162,7 @@ func (s *balanceRegionScheduler) Schedule(cluster opt.Cluster) []*operator.Opera // allow empty region to be scheduled in range cluster allowBalanceEmptyRegion = func(region *core.RegionInfo) bool { return true } default: - allowBalanceEmptyRegion = opt.AllowBalanceEmptyRegion(cluster) + allowBalanceEmptyRegion = isAllowBalanceEmptyRegion(cluster) } for _, plan.source = range stores { @@ -169,18 +171,18 @@ func (s *balanceRegionScheduler) Schedule(cluster opt.Cluster) []*operator.Opera schedulerCounter.WithLabelValues(s.GetName(), "total").Inc() // Priority pick the region that has a pending peer. // Pending region may means the disk is overload, remove the pending region firstly. - plan.region = cluster.RandPendingRegion(plan.SourceStoreID(), s.conf.Ranges, opt.IsRegionHealthyAllowPending, opt.ReplicatedRegion(cluster), allowBalanceEmptyRegion) + plan.region = cluster.RandPendingRegion(plan.SourceStoreID(), s.conf.Ranges, schedule.IsRegionHealthyAllowPending, schedule.ReplicatedRegion(cluster), allowBalanceEmptyRegion) if plan.region == nil { // Then pick the region that has a follower in the source store. - plan.region = cluster.RandFollowerRegion(plan.SourceStoreID(), s.conf.Ranges, opt.IsRegionHealthy, opt.ReplicatedRegion(cluster), allowBalanceEmptyRegion) + plan.region = cluster.RandFollowerRegion(plan.SourceStoreID(), s.conf.Ranges, schedule.IsRegionHealthy, schedule.ReplicatedRegion(cluster), allowBalanceEmptyRegion) } if plan.region == nil { // Then pick the region has the leader in the source store. - plan.region = cluster.RandLeaderRegion(plan.SourceStoreID(), s.conf.Ranges, opt.IsRegionHealthy, opt.ReplicatedRegion(cluster), allowBalanceEmptyRegion) + plan.region = cluster.RandLeaderRegion(plan.SourceStoreID(), s.conf.Ranges, schedule.IsRegionHealthy, schedule.ReplicatedRegion(cluster), allowBalanceEmptyRegion) } if plan.region == nil { // Finally pick learner. - plan.region = cluster.RandLearnerRegion(plan.SourceStoreID(), s.conf.Ranges, opt.IsRegionHealthy, opt.ReplicatedRegion(cluster), allowBalanceEmptyRegion) + plan.region = cluster.RandLearnerRegion(plan.SourceStoreID(), s.conf.Ranges, schedule.IsRegionHealthy, schedule.ReplicatedRegion(cluster), allowBalanceEmptyRegion) } if plan.region == nil { schedulerCounter.WithLabelValues(s.GetName(), "no-region").Inc() @@ -260,3 +262,13 @@ func (s *balanceRegionScheduler) transferPeer(plan *balancePlan) *operator.Opera schedulerCounter.WithLabelValues(s.GetName(), "no-replacement").Inc() return nil } + +// isEmptyRegionAllowBalance checks if a region is an empty region and can be balanced. +func isEmptyRegionAllowBalance(cluster opt.Cluster, region *core.RegionInfo) bool { + return region.GetApproximateSize() > core.EmptyRegionApproximateSize || cluster.GetRegionCount() < balanceEmptyRegionThreshold +} + +// isAllowBalanceEmptyRegion returns a function that checks if a region is an empty region and can be balanced. +func isAllowBalanceEmptyRegion(cluster opt.Cluster) func(*core.RegionInfo) bool { + return func(region *core.RegionInfo) bool { return isEmptyRegionAllowBalance(cluster, region) } +} diff --git a/server/schedulers/evict_leader.go b/server/schedulers/evict_leader.go index c09d615cf7c9..1862c337de8b 100644 --- a/server/schedulers/evict_leader.go +++ b/server/schedulers/evict_leader.go @@ -279,7 +279,7 @@ func scheduleEvictLeaderOnce(name, typ string, cluster opt.Cluster, storeRanges ops := make([]*operator.Operator, 0, len(storeRanges)) for id, ranges := range storeRanges { var filters []filter.Filter - region := cluster.RandLeaderRegion(id, ranges, opt.IsRegionHealthy) + region := cluster.RandLeaderRegion(id, ranges, schedule.IsRegionHealthy) if region == nil { // try to pick unhealthy region region = cluster.RandLeaderRegion(id, ranges) diff --git a/server/schedulers/grant_leader.go b/server/schedulers/grant_leader.go index dabb299995a8..997407d24bea 100644 --- a/server/schedulers/grant_leader.go +++ b/server/schedulers/grant_leader.go @@ -232,7 +232,7 @@ func (s *grantLeaderScheduler) Schedule(cluster opt.Cluster) []*operator.Operato defer s.conf.mu.RUnlock() ops := make([]*operator.Operator, 0, len(s.conf.StoreIDWithRanges)) for id, ranges := range s.conf.StoreIDWithRanges { - region := cluster.RandFollowerRegion(id, ranges, opt.IsRegionHealthy) + region := cluster.RandFollowerRegion(id, ranges, schedule.IsRegionHealthy) if region == nil { schedulerCounter.WithLabelValues(s.GetName(), "no-follower").Inc() continue diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index e9e4ecc4cbe3..176cce602c55 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -653,12 +653,12 @@ func (bs *balanceSolver) isRegionAvailable(region *core.RegionInfo) bool { } } - if !opt.IsRegionHealthyAllowPending(region) { + if !schedule.IsRegionHealthyAllowPending(region) { schedulerCounter.WithLabelValues(bs.sche.GetName(), "unhealthy-replica").Inc() return false } - if !opt.IsRegionReplicated(bs.cluster, region) { + if !schedule.IsRegionReplicated(bs.cluster, region) { log.Debug("region has abnormal replica count", zap.String("scheduler", bs.sche.GetName()), zap.Uint64("region-id", region.GetID())) schedulerCounter.WithLabelValues(bs.sche.GetName(), "abnormal-replica").Inc() return false diff --git a/server/schedulers/hot_region_test.go b/server/schedulers/hot_region_test.go index dd7192cba55e..39328ceac744 100644 --- a/server/schedulers/hot_region_test.go +++ b/server/schedulers/hot_region_test.go @@ -1552,7 +1552,7 @@ func (s *testHotCacheSuite) TestCheckRegionFlow(c *C) { c.Check(len(items), Greater, 0) for _, item := range items { if item.StoreID == 3 { - c.Check(item.IsNeedDelete(), IsTrue) + c.Check(item.GetActionType(), Equals, statistics.Remove) continue } c.Check(item.HotDegree, Equals, testcase.DegreeAfterTransferLeader+2) @@ -1586,9 +1586,9 @@ func (s *testHotCacheSuite) TestCheckRegionFlowWithDifferentThreshold(c *C) { items = tc.AddLeaderRegionWithWriteInfo(201, 1, rate*statistics.WriteReportInterval, 0, 0, statistics.WriteReportInterval, []uint64{3, 4}, 1) for _, item := range items { if item.StoreID < 4 { - c.Check(item.IsNeedDelete(), IsTrue) + c.Check(item.GetActionType(), Equals, statistics.Remove) } else { - c.Check(item.IsNeedDelete(), IsFalse) + c.Check(item.GetActionType(), Equals, statistics.Update) } } } diff --git a/server/schedulers/random_merge.go b/server/schedulers/random_merge.go index 9643387fdfa7..fff6cdc9a828 100644 --- a/server/schedulers/random_merge.go +++ b/server/schedulers/random_merge.go @@ -109,7 +109,7 @@ func (s *randomMergeScheduler) Schedule(cluster opt.Cluster) []*operator.Operato schedulerCounter.WithLabelValues(s.GetName(), "no-source-store").Inc() return nil } - region := cluster.RandLeaderRegion(store.GetID(), s.conf.Ranges, opt.IsRegionHealthy) + region := cluster.RandLeaderRegion(store.GetID(), s.conf.Ranges, schedule.IsRegionHealthy) if region == nil { schedulerCounter.WithLabelValues(s.GetName(), "no-region").Inc() return nil @@ -139,10 +139,10 @@ func (s *randomMergeScheduler) Schedule(cluster opt.Cluster) []*operator.Operato } func (s *randomMergeScheduler) allowMerge(cluster opt.Cluster, region, target *core.RegionInfo) bool { - if !opt.IsRegionHealthy(region) || !opt.IsRegionHealthy(target) { + if !schedule.IsRegionHealthy(region) || !schedule.IsRegionHealthy(target) { return false } - if !opt.IsRegionReplicated(cluster, region) || !opt.IsRegionReplicated(cluster, target) { + if !schedule.IsRegionReplicated(cluster, region) || !schedule.IsRegionReplicated(cluster, target) { return false } if cluster.IsRegionHot(region) || cluster.IsRegionHot(target) { diff --git a/server/schedulers/shuffle_leader.go b/server/schedulers/shuffle_leader.go index eb09af8bdc01..2c22be2da199 100644 --- a/server/schedulers/shuffle_leader.go +++ b/server/schedulers/shuffle_leader.go @@ -115,7 +115,7 @@ func (s *shuffleLeaderScheduler) Schedule(cluster opt.Cluster) []*operator.Opera schedulerCounter.WithLabelValues(s.GetName(), "no-target-store").Inc() return nil } - region := cluster.RandFollowerRegion(targetStore.GetID(), s.conf.Ranges, opt.IsRegionHealthy) + region := cluster.RandFollowerRegion(targetStore.GetID(), s.conf.Ranges, schedule.IsRegionHealthy) if region == nil { schedulerCounter.WithLabelValues(s.GetName(), "no-follower").Inc() return nil diff --git a/server/schedulers/shuffle_region.go b/server/schedulers/shuffle_region.go index 7cab03ed55a5..ea46312de24d 100644 --- a/server/schedulers/shuffle_region.go +++ b/server/schedulers/shuffle_region.go @@ -135,13 +135,13 @@ func (s *shuffleRegionScheduler) scheduleRemovePeer(cluster opt.Cluster) (*core. for _, source := range candidates.Stores { var region *core.RegionInfo if s.conf.IsRoleAllow(roleFollower) { - region = cluster.RandFollowerRegion(source.GetID(), s.conf.GetRanges(), opt.IsRegionHealthy, opt.ReplicatedRegion(cluster)) + region = cluster.RandFollowerRegion(source.GetID(), s.conf.GetRanges(), schedule.IsRegionHealthy, schedule.ReplicatedRegion(cluster)) } if region == nil && s.conf.IsRoleAllow(roleLeader) { - region = cluster.RandLeaderRegion(source.GetID(), s.conf.GetRanges(), opt.IsRegionHealthy, opt.ReplicatedRegion(cluster)) + region = cluster.RandLeaderRegion(source.GetID(), s.conf.GetRanges(), schedule.IsRegionHealthy, schedule.ReplicatedRegion(cluster)) } if region == nil && s.conf.IsRoleAllow(roleLearner) { - region = cluster.RandLearnerRegion(source.GetID(), s.conf.GetRanges(), opt.IsRegionHealthy, opt.ReplicatedRegion(cluster)) + region = cluster.RandLearnerRegion(source.GetID(), s.conf.GetRanges(), schedule.IsRegionHealthy, schedule.ReplicatedRegion(cluster)) } if region != nil { return region, region.GetStorePeer(source.GetID()) diff --git a/server/statistics/hot_cache.go b/server/statistics/hot_cache.go index 125b5f6206e8..ece8932c959c 100644 --- a/server/statistics/hot_cache.go +++ b/server/statistics/hot_cache.go @@ -28,43 +28,39 @@ const queueCap = 20000 // HotCache is a cache hold hot regions. type HotCache struct { - ctx context.Context - readFlowQueue chan FlowItemTask - writeFlowQueue chan FlowItemTask - writeFlow *hotPeerCache - readFlow *hotPeerCache + ctx context.Context + writeCache *hotPeerCache + readCache *hotPeerCache } // NewHotCache creates a new hot spot cache. func NewHotCache(ctx context.Context) *HotCache { w := &HotCache{ - ctx: ctx, - readFlowQueue: make(chan FlowItemTask, queueCap), - writeFlowQueue: make(chan FlowItemTask, queueCap), - writeFlow: NewHotPeerCache(Write), - readFlow: NewHotPeerCache(Read), + ctx: ctx, + writeCache: NewHotPeerCache(Write), + readCache: NewHotPeerCache(Read), } - go w.updateItems(w.readFlowQueue, w.runReadTask) - go w.updateItems(w.writeFlowQueue, w.runWriteTask) + go w.updateItems(w.readCache.taskQueue, w.runReadTask) + go w.updateItems(w.writeCache.taskQueue, w.runWriteTask) return w } // CheckWritePeerSync checks the write status, returns update items. // This is used for mockcluster. func (w *HotCache) CheckWritePeerSync(peer *core.PeerInfo, region *core.RegionInfo) *HotPeerStat { - return w.writeFlow.CheckPeerFlow(peer, region) + return w.writeCache.checkPeerFlow(peer, region) } // CheckReadPeerSync checks the read status, returns update items. // This is used for mockcluster. func (w *HotCache) CheckReadPeerSync(peer *core.PeerInfo, region *core.RegionInfo) *HotPeerStat { - return w.readFlow.CheckPeerFlow(peer, region) + return w.readCache.checkPeerFlow(peer, region) } // CheckWriteAsync puts the flowItem into queue, and check it asynchronously -func (w *HotCache) CheckWriteAsync(task FlowItemTask) bool { +func (w *HotCache) CheckWriteAsync(task flowItemTask) bool { select { - case w.writeFlowQueue <- task: + case w.writeCache.taskQueue <- task: return true default: return false @@ -72,9 +68,9 @@ func (w *HotCache) CheckWriteAsync(task FlowItemTask) bool { } // CheckReadAsync puts the flowItem into queue, and check it asynchronously -func (w *HotCache) CheckReadAsync(task FlowItemTask) bool { +func (w *HotCache) CheckReadAsync(task flowItemTask) bool { select { - case w.readFlowQueue <- task: + case w.readCache.taskQueue <- task: return true default: return false @@ -86,39 +82,26 @@ func (w *HotCache) CheckReadAsync(task FlowItemTask) bool { func (w *HotCache) Update(item *HotPeerStat) { switch item.Kind { case Write: - update(item, w.writeFlow) + updateStat(w.writeCache, item) case Read: - update(item, w.readFlow) + updateStat(w.readCache, item) } } // RegionStats returns hot items according to kind func (w *HotCache) RegionStats(kind RWType, minHotDegree int) map[uint64][]*HotPeerStat { + task := newCollectRegionStatsTask(minHotDegree) + var succ bool switch kind { case Write: - task := newCollectRegionStatsTask(minHotDegree) - succ := w.CheckWriteAsync(task) - if !succ { - return nil - } - return task.waitRet(w.ctx) + succ = w.CheckWriteAsync(task) case Read: - task := newCollectRegionStatsTask(minHotDegree) - succ := w.CheckReadAsync(task) - if !succ { - return nil - } - return task.waitRet(w.ctx) + succ = w.CheckReadAsync(task) } - return nil -} - -// HotRegionsFromStore picks hot region in specify store. -func (w *HotCache) HotRegionsFromStore(storeID uint64, kind RWType, minHotDegree int) []*HotPeerStat { - if stats, ok := w.RegionStats(kind, minHotDegree)[storeID]; ok && len(stats) > 0 { - return stats + if !succ { + return nil } - return nil + return task.waitRet(w.ctx) } // IsRegionHot checks if the region is hot. @@ -149,13 +132,13 @@ func (w *HotCache) ResetMetrics() { // ExpiredReadItems returns the read items which are already expired. // This is used for mockcluster. func (w *HotCache) ExpiredReadItems(region *core.RegionInfo) []*HotPeerStat { - return w.readFlow.CollectExpiredItems(region) + return w.readCache.collectExpiredItems(region) } // ExpiredWriteItems returns the write items which are already expired. // This is used for mockcluster. func (w *HotCache) ExpiredWriteItems(region *core.RegionInfo) []*HotPeerStat { - return w.writeFlow.CollectExpiredItems(region) + return w.writeCache.collectExpiredItems(region) } func incMetrics(name string, storeID uint64, kind RWType) { @@ -172,14 +155,14 @@ func incMetrics(name string, storeID uint64, kind RWType) { func (w *HotCache) GetFilledPeriod(kind RWType) int { switch kind { case Write: - return w.writeFlow.getDefaultTimeMedian().GetFilledPeriod() + return w.writeCache.getDefaultTimeMedian().GetFilledPeriod() case Read: - return w.readFlow.getDefaultTimeMedian().GetFilledPeriod() + return w.readCache.getDefaultTimeMedian().GetFilledPeriod() } return 0 } -func (w *HotCache) updateItems(queue <-chan FlowItemTask, runTask func(task FlowItemTask)) { +func (w *HotCache) updateItems(queue <-chan flowItemTask, runTask func(task flowItemTask)) { for { select { case <-w.ctx.Done(): @@ -190,29 +173,30 @@ func (w *HotCache) updateItems(queue <-chan FlowItemTask, runTask func(task Flow } } -func (w *HotCache) runReadTask(task FlowItemTask) { +func (w *HotCache) runReadTask(task flowItemTask) { if task != nil { - // TODO: do we need a run-task timeout to protect the queue won't be stucked by a task? - task.runTask(w.readFlow) - hotCacheFlowQueueStatusGauge.WithLabelValues(Read.String()).Set(float64(len(w.readFlowQueue))) + // TODO: do we need a run-task timeout to protect the queue won't be stuck by a task? + task.runTask(w.readCache) + hotCacheFlowQueueStatusGauge.WithLabelValues(Read.String()).Set(float64(len(w.readCache.taskQueue))) } } -func (w *HotCache) runWriteTask(task FlowItemTask) { +func (w *HotCache) runWriteTask(task flowItemTask) { if task != nil { - // TODO: do we need a run-task timeout to protect the queue won't be stucked by a task? - task.runTask(w.writeFlow) - hotCacheFlowQueueStatusGauge.WithLabelValues(Write.String()).Set(float64(len(w.writeFlowQueue))) + // TODO: do we need a run-task timeout to protect the queue won't be stuck by a task? + task.runTask(w.writeCache) + hotCacheFlowQueueStatusGauge.WithLabelValues(Write.String()).Set(float64(len(w.writeCache.taskQueue))) } } -func update(item *HotPeerStat, flow *hotPeerCache) { - flow.Update(item) - if item.IsNeedDelete() { - incMetrics("remove_item", item.StoreID, item.Kind) - } else if item.IsNew() { +func updateStat(cache *hotPeerCache, item *HotPeerStat) { + cache.update(item) + switch item.actionType { + case Add: incMetrics("add_item", item.StoreID, item.Kind) - } else { + case Remove: + incMetrics("remove_item", item.StoreID, item.Kind) + case Update: incMetrics("update_item", item.StoreID, item.Kind) } } diff --git a/server/statistics/hot_cache_task.go b/server/statistics/hot_cache_task.go index 672306b35d06..2f71c5ecee56 100644 --- a/server/statistics/hot_cache_task.go +++ b/server/statistics/hot_cache_task.go @@ -31,10 +31,10 @@ const ( collectMetricsTaskType ) -// FlowItemTask indicates the task in flowItem queue -type FlowItemTask interface { +// flowItemTask indicates the task in flowItem queue +type flowItemTask interface { taskType() flowItemTaskKind - runTask(flow *hotPeerCache) + runTask(cache *hotPeerCache) } type checkPeerTask struct { @@ -43,7 +43,7 @@ type checkPeerTask struct { } // NewCheckPeerTask creates task to update peerInfo -func NewCheckPeerTask(peerInfo *core.PeerInfo, regionInfo *core.RegionInfo) FlowItemTask { +func NewCheckPeerTask(peerInfo *core.PeerInfo, regionInfo *core.RegionInfo) flowItemTask { return &checkPeerTask{ peerInfo: peerInfo, regionInfo: regionInfo, @@ -54,10 +54,10 @@ func (t *checkPeerTask) taskType() flowItemTaskKind { return checkPeerTaskType } -func (t *checkPeerTask) runTask(flow *hotPeerCache) { - stat := flow.CheckPeerFlow(t.peerInfo, t.regionInfo) +func (t *checkPeerTask) runTask(cache *hotPeerCache) { + stat := cache.checkPeerFlow(t.peerInfo, t.regionInfo) if stat != nil { - update(stat, flow) + updateStat(cache, stat) } } @@ -66,7 +66,7 @@ type checkExpiredTask struct { } // NewCheckExpiredItemTask creates task to collect expired items -func NewCheckExpiredItemTask(region *core.RegionInfo) FlowItemTask { +func NewCheckExpiredItemTask(region *core.RegionInfo) flowItemTask { return &checkExpiredTask{ region: region, } @@ -76,25 +76,25 @@ func (t *checkExpiredTask) taskType() flowItemTaskKind { return checkExpiredTaskType } -func (t *checkExpiredTask) runTask(flow *hotPeerCache) { - expiredStats := flow.CollectExpiredItems(t.region) +func (t *checkExpiredTask) runTask(cache *hotPeerCache) { + expiredStats := cache.collectExpiredItems(t.region) for _, stat := range expiredStats { - update(stat, flow) + updateStat(cache, stat) } } type collectUnReportedPeerTask struct { - storeID uint64 - regionIDs map[uint64]struct{} - interval uint64 + storeID uint64 + regions map[uint64]*core.RegionInfo + interval uint64 } // NewCollectUnReportedPeerTask creates task to collect unreported peers -func NewCollectUnReportedPeerTask(storeID uint64, regionIDs map[uint64]struct{}, interval uint64) FlowItemTask { +func NewCollectUnReportedPeerTask(storeID uint64, regions map[uint64]*core.RegionInfo, interval uint64) flowItemTask { return &collectUnReportedPeerTask{ - storeID: storeID, - regionIDs: regionIDs, - interval: interval, + storeID: storeID, + regions: regions, + interval: interval, } } @@ -102,10 +102,10 @@ func (t *collectUnReportedPeerTask) taskType() flowItemTaskKind { return collectUnReportedPeerTaskType } -func (t *collectUnReportedPeerTask) runTask(flow *hotPeerCache) { - stats := flow.CheckColdPeer(t.storeID, t.regionIDs, t.interval) +func (t *collectUnReportedPeerTask) runTask(cache *hotPeerCache) { + stats := cache.checkColdPeer(t.storeID, t.regions, t.interval) for _, stat := range stats { - update(stat, flow) + updateStat(cache, stat) } } @@ -125,8 +125,8 @@ func (t *collectRegionStatsTask) taskType() flowItemTaskKind { return collectRegionStatsTaskType } -func (t *collectRegionStatsTask) runTask(flow *hotPeerCache) { - t.ret <- flow.RegionStats(t.minDegree) +func (t *collectRegionStatsTask) runTask(cache *hotPeerCache) { + t.ret <- cache.RegionStats(t.minDegree) } // TODO: do we need a wait-return timeout? @@ -157,8 +157,8 @@ func (t *isRegionHotTask) taskType() flowItemTaskKind { return isRegionHotTaskType } -func (t *isRegionHotTask) runTask(flow *hotPeerCache) { - t.ret <- flow.isRegionHotWithAnyPeers(t.region, t.minHotDegree) +func (t *isRegionHotTask) runTask(cache *hotPeerCache) { + t.ret <- cache.isRegionHotWithAnyPeers(t.region, t.minHotDegree) } // TODO: do we need a wait-return timeout? @@ -185,6 +185,6 @@ func (t *collectMetricsTask) taskType() flowItemTaskKind { return collectMetricsTaskType } -func (t *collectMetricsTask) runTask(flow *hotPeerCache) { - flow.CollectMetrics(t.typ) +func (t *collectMetricsTask) runTask(cache *hotPeerCache) { + cache.collectMetrics(t.typ) } diff --git a/server/statistics/hot_peer.go b/server/statistics/hot_peer.go index e7055a38af30..0bebaaa7a5f3 100644 --- a/server/statistics/hot_peer.go +++ b/server/statistics/hot_peer.go @@ -33,48 +33,48 @@ const ( type dimStat struct { typ RegionStatKind - Rolling *movingaverage.TimeMedian // it's used to statistic hot degree and average speed. - LastAverage *movingaverage.AvgOverTime // it's used to obtain the average speed in last second as instantaneous speed. + rolling *movingaverage.TimeMedian // it's used to statistic hot degree and average speed. + lastAverage *movingaverage.AvgOverTime // it's used to obtain the average speed in last second as instantaneous speed. } func newDimStat(typ RegionStatKind, reportInterval time.Duration) *dimStat { return &dimStat{ typ: typ, - Rolling: movingaverage.NewTimeMedian(DefaultAotSize, rollingWindowsSize, reportInterval), - LastAverage: movingaverage.NewAvgOverTime(reportInterval), + rolling: movingaverage.NewTimeMedian(DefaultAotSize, rollingWindowsSize, reportInterval), + lastAverage: movingaverage.NewAvgOverTime(reportInterval), } } func (d *dimStat) Add(delta float64, interval time.Duration) { - d.LastAverage.Add(delta, interval) - d.Rolling.Add(delta, interval) + d.lastAverage.Add(delta, interval) + d.rolling.Add(delta, interval) } func (d *dimStat) isLastAverageHot(threshold float64) bool { - return d.LastAverage.Get() >= threshold + return d.lastAverage.Get() >= threshold } func (d *dimStat) isHot(threshold float64) bool { - return d.Rolling.Get() >= threshold + return d.rolling.Get() >= threshold } func (d *dimStat) isFull() bool { - return d.LastAverage.IsFull() + return d.lastAverage.IsFull() } func (d *dimStat) clearLastAverage() { - d.LastAverage.Clear() + d.lastAverage.Clear() } func (d *dimStat) Get() float64 { - return d.Rolling.Get() + return d.rolling.Get() } func (d *dimStat) Clone() *dimStat { return &dimStat{ typ: d.typ, - Rolling: d.Rolling.Clone(), - LastAverage: d.LastAverage.Clone(), + rolling: d.rolling.Clone(), + lastAverage: d.lastAverage.Clone(), } } @@ -97,11 +97,8 @@ type HotPeerStat struct { // LastUpdateTime used to calculate average write LastUpdateTime time.Time `json:"last_update_time"` - needDelete bool - isLeader bool - isNew bool - // TODO: remove it when we send peer stat by store info - justTransferLeader bool + actionType ActionType + isLeader bool interval uint64 thresholds []float64 peers []uint64 @@ -140,10 +137,9 @@ func (stat *HotPeerStat) Log(str string, level func(msg string, fields ...zap.Fi zap.Int("hot-degree", stat.HotDegree), zap.Int("hot-anti-count", stat.AntiCount), zap.Duration("sum-interval", stat.getIntervalSum()), - zap.Bool("need-delete", stat.IsNeedDelete()), zap.String("source", stat.source.String()), zap.Bool("allow-adopt", stat.allowAdopt), - zap.Bool("just-transfer-leader", stat.justTransferLeader), + zap.String("action-type", stat.actionType.String()), zap.Time("last-transfer-leader-time", stat.lastTransferLeaderTime)) } @@ -152,22 +148,17 @@ func (stat *HotPeerStat) IsNeedCoolDownTransferLeader(minHotDegree int) bool { return time.Since(stat.lastTransferLeaderTime).Seconds() < float64(minHotDegree*stat.hotStatReportInterval()) } -// IsNeedDelete to delete the item in cache. -func (stat *HotPeerStat) IsNeedDelete() bool { - return stat.needDelete -} - // IsLeader indicates the item belong to the leader. func (stat *HotPeerStat) IsLeader() bool { return stat.isLeader } -// IsNew indicates the item is first update in the cache of the region. -func (stat *HotPeerStat) IsNew() bool { - return stat.isNew +// GetActionType returns the item action type. +func (stat *HotPeerStat) GetActionType() ActionType { + return stat.actionType } -// GetLoad returns denoised load if possible. +// GetLoad returns denoising load if possible. func (stat *HotPeerStat) GetLoad(k RegionStatKind) float64 { if len(stat.rollingLoads) > int(k) { return math.Round(stat.rollingLoads[int(k)].Get()) @@ -175,7 +166,7 @@ func (stat *HotPeerStat) GetLoad(k RegionStatKind) float64 { return math.Round(stat.Loads[int(k)]) } -// GetLoads returns denoised load if possible. +// GetLoads returns denoising load if possible. func (stat *HotPeerStat) GetLoads() []float64 { regionStats := stat.Kind.RegionStats() loads := make([]float64, len(regionStats)) @@ -185,7 +176,8 @@ func (stat *HotPeerStat) GetLoads() []float64 { return loads } -// GetThresholds returns thresholds +// GetThresholds returns thresholds. +// Only for test purpose. func (stat *HotPeerStat) GetThresholds() []float64 { return stat.thresholds } @@ -201,9 +193,9 @@ func (stat *HotPeerStat) Clone() *HotPeerStat { return &ret } -func (stat *HotPeerStat) isFullAndHot() bool { +func (stat *HotPeerStat) isHot() bool { return slice.AnyOf(stat.rollingLoads, func(i int) bool { - return stat.rollingLoads[i].isFull() && stat.rollingLoads[i].isLastAverageHot(stat.thresholds[i]) + return stat.rollingLoads[i].isLastAverageHot(stat.thresholds[i]) }) } @@ -224,5 +216,5 @@ func (stat *HotPeerStat) getIntervalSum() time.Duration { if len(stat.rollingLoads) == 0 || stat.rollingLoads[0] == nil { return 0 } - return stat.rollingLoads[0].LastAverage.GetIntervalSum() + return stat.rollingLoads[0].lastAverage.GetIntervalSum() } diff --git a/server/statistics/hot_peer_cache.go b/server/statistics/hot_peer_cache.go index c5a08db5dd8d..7de9a8ceaca3 100644 --- a/server/statistics/hot_peer_cache.go +++ b/server/statistics/hot_peer_cache.go @@ -61,6 +61,7 @@ type hotPeerCache struct { inheritItem map[uint64]*HotPeerStat // regionID -> HotPeerStat topNTTL time.Duration reportIntervalSecs int + taskQueue chan flowItemTask } // NewHotPeerCache creates a hotPeerCache @@ -71,6 +72,7 @@ func NewHotPeerCache(kind RWType) *hotPeerCache { storesOfRegion: make(map[uint64]map[uint64]struct{}), regionsOfStore: make(map[uint64]map[uint64]struct{}), inheritItem: make(map[uint64]*HotPeerStat), + taskQueue: make(chan flowItemTask, queueCap), } if kind == Write { c.reportIntervalSecs = WriteReportInterval @@ -98,14 +100,14 @@ func (f *hotPeerCache) RegionStats(minHotDegree int) map[uint64][]*HotPeerStat { return res } -// Update updates the items in statistics. -func (f *hotPeerCache) Update(item *HotPeerStat) { - if item.IsNeedDelete() { +// update updates the items in statistics. +func (f *hotPeerCache) update(item *HotPeerStat) { + if item.actionType == Remove { if item.AntiCount > 0 { // means it's deleted because expired rather than cold f.putInheritItem(item) } f.removeItem(item) - item.Log("region heartbeat delete from cache", log.Debug) + item.Log("region heartbeat remove from cache", log.Debug) } else { f.putItem(item) item.Log("region heartbeat update", log.Debug) @@ -136,15 +138,15 @@ func (f *hotPeerCache) collectPeerMetrics(loads []float64, interval uint64) { } } -// CollectExpiredItems collects expired items, mark them as needDelete and puts them into inherit items -func (f *hotPeerCache) CollectExpiredItems(region *core.RegionInfo) []*HotPeerStat { +// collectExpiredItems collects expired items, mark them as needDelete and puts them into inherit items +func (f *hotPeerCache) collectExpiredItems(region *core.RegionInfo) []*HotPeerStat { regionID := region.GetID() items := make([]*HotPeerStat, 0) for _, storeID := range f.getAllStoreIDs(region) { if region.GetStorePeer(storeID) == nil { item := f.getOldHotPeerStat(regionID, storeID) if item != nil { - item.needDelete = true + item.actionType = Remove items = append(items, item) } } @@ -152,10 +154,10 @@ func (f *hotPeerCache) CollectExpiredItems(region *core.RegionInfo) []*HotPeerSt return items } -// CheckPeerFlow checks the flow information of a peer. -// Notice: CheckPeerFlow couldn't be used concurrently. -// CheckPeerFlow will update oldItem's rollingLoads into newItem, thus we should use write lock here. -func (f *hotPeerCache) CheckPeerFlow(peer *core.PeerInfo, region *core.RegionInfo) *HotPeerStat { +// checkPeerFlow checks the flow information of a peer. +// Notice: checkPeerFlow couldn't be used concurrently. +// checkPeerFlow will update oldItem's rollingLoads into newItem, thus we should use write lock here. +func (f *hotPeerCache) checkPeerFlow(peer *core.PeerInfo, region *core.RegionInfo) *HotPeerStat { interval := peer.GetInterval() if Denoising && interval < HotRegionReportMinInterval { return nil @@ -167,7 +169,6 @@ func (f *hotPeerCache) CheckPeerFlow(peer *core.PeerInfo, region *core.RegionInf for i := range deltaLoads { loads[i] = deltaLoads[i] / float64(interval) } - justTransferLeader := f.justTransferLeader(region) oldItem := f.getOldHotPeerStat(region.GetID(), storeID) thresholds := f.calcHotThresholds(storeID) regionPeers := region.GetPeers() @@ -176,18 +177,17 @@ func (f *hotPeerCache) CheckPeerFlow(peer *core.PeerInfo, region *core.RegionInf peers = append(peers, peer.StoreId) } newItem := &HotPeerStat{ - StoreID: storeID, - RegionID: region.GetID(), - Kind: f.kind, - Loads: loads, - LastUpdateTime: time.Now(), - needDelete: false, - isLeader: region.GetLeader().GetStoreId() == storeID, - justTransferLeader: justTransferLeader, - interval: interval, - peers: peers, - thresholds: thresholds, - source: direct, + StoreID: storeID, + RegionID: region.GetID(), + Kind: f.kind, + Loads: loads, + LastUpdateTime: time.Now(), + isLeader: region.GetLeader().GetStoreId() == storeID, + interval: interval, + peers: peers, + actionType: Update, + thresholds: thresholds, + source: direct, } if oldItem == nil { @@ -205,11 +205,11 @@ func (f *hotPeerCache) CheckPeerFlow(peer *core.PeerInfo, region *core.RegionInf } } } - return f.updateHotPeerStat(newItem, oldItem, deltaLoads, time.Duration(interval)*time.Second) + return f.updateHotPeerStat(region, newItem, oldItem, deltaLoads, time.Duration(interval)*time.Second) } -// CheckColdPeer checks the collect the un-heartbeat peer and maintain it. -func (f *hotPeerCache) CheckColdPeer(storeID uint64, reportRegions map[uint64]struct{}, interval uint64) (ret []*HotPeerStat) { +// checkColdPeer checks the collect the un-heartbeat peer and maintain it. +func (f *hotPeerCache) checkColdPeer(storeID uint64, reportRegions map[uint64]*core.RegionInfo, interval uint64) (ret []*HotPeerStat) { if Denoising && interval < HotRegionReportMinInterval { return } @@ -218,7 +218,7 @@ func (f *hotPeerCache) CheckColdPeer(storeID uint64, reportRegions map[uint64]st return } for regionID := range previousHotStat { - if _, ok := reportRegions[regionID]; !ok { + if region, ok := reportRegions[regionID]; !ok { oldItem := f.getOldHotPeerStat(regionID, storeID) if oldItem == nil { continue @@ -228,21 +228,20 @@ func (f *hotPeerCache) CheckColdPeer(storeID uint64, reportRegions map[uint64]st RegionID: regionID, Kind: f.kind, // use oldItem.thresholds to make the newItem won't affect the threshold - Loads: oldItem.thresholds, - LastUpdateTime: time.Now(), - needDelete: false, - isLeader: oldItem.isLeader, - justTransferLeader: oldItem.justTransferLeader, - interval: interval, - peers: oldItem.peers, - thresholds: oldItem.thresholds, - inCold: true, + Loads: oldItem.thresholds, + LastUpdateTime: time.Now(), + isLeader: oldItem.isLeader, + interval: interval, + peers: oldItem.peers, + actionType: Update, + thresholds: oldItem.thresholds, + inCold: true, } deltaLoads := make([]float64, RegionStatCount) for i, loads := range oldItem.thresholds { deltaLoads[i] = loads * float64(interval) } - stat := f.updateHotPeerStat(newItem, oldItem, deltaLoads, time.Duration(interval)*time.Second) + stat := f.updateHotPeerStat(region, newItem, oldItem, deltaLoads, time.Duration(interval)*time.Second) if stat != nil { ret = append(ret, stat) } @@ -251,7 +250,7 @@ func (f *hotPeerCache) CheckColdPeer(storeID uint64, reportRegions map[uint64]st return } -func (f *hotPeerCache) CollectMetrics(typ string) { +func (f *hotPeerCache) collectMetrics(typ string) { for storeID, peers := range f.peersOfStore { store := storeTag(storeID) thresholds := f.calcHotThresholds(storeID) @@ -338,6 +337,9 @@ func (f *hotPeerCache) isOldColdPeer(oldItem *HotPeerStat, storeID uint64) bool } func (f *hotPeerCache) justTransferLeader(region *core.RegionInfo) bool { + if region == nil { + return false + } ids, ok := f.storesOfRegion[region.GetID()] if ok { for storeID := range ids { @@ -379,10 +381,10 @@ func (f *hotPeerCache) getDefaultTimeMedian() *movingaverage.TimeMedian { return movingaverage.NewTimeMedian(DefaultAotSize, rollingWindowsSize, time.Duration(f.reportIntervalSecs)*time.Second) } -func (f *hotPeerCache) updateHotPeerStat(newItem, oldItem *HotPeerStat, deltaLoads []float64, interval time.Duration) *HotPeerStat { +func (f *hotPeerCache) updateHotPeerStat(region *core.RegionInfo, newItem, oldItem *HotPeerStat, deltaLoads []float64, interval time.Duration) *HotPeerStat { regionStats := f.kind.RegionStats() if oldItem == nil { - return f.updateNewHotPeerStat(newItem, deltaLoads, interval) + return f.updateNewHotPeerStat(regionStats, newItem, deltaLoads, interval) } if newItem.source == adopt { @@ -395,14 +397,14 @@ func (f *hotPeerCache) updateHotPeerStat(newItem, oldItem *HotPeerStat, deltaLoa newItem.allowAdopt = oldItem.allowAdopt } - if newItem.justTransferLeader { + if f.justTransferLeader(region) { newItem.lastTransferLeaderTime = time.Now() // skip the first heartbeat flow statistic after transfer leader, because its statistics are calculated by the last leader in this store and are inaccurate // maintain anticount and hotdegree to avoid store threshold and hot peer are unstable. // For write stat, as the stat is send by region heartbeat, the first heartbeat will be skipped. // For read stat, as the stat is send by store heartbeat, the first heartbeat won't be skipped. if newItem.Kind == Write { - inheritItemDegree(newItem, oldItem) + inheritItem(newItem, oldItem) return newItem } } else { @@ -416,7 +418,7 @@ func (f *hotPeerCache) updateHotPeerStat(newItem, oldItem *HotPeerStat, deltaLoa isFull := newItem.rollingLoads[0].isFull() // The intervals of dims are the same, so it is only necessary to determine whether any of them if !isFull { // not update hot degree and anti count - inheritItemDegree(newItem, oldItem) + inheritItem(newItem, oldItem) } else { // If item is inCold, it means the pd didn't recv this item in the store heartbeat, // thus we make it colder @@ -424,13 +426,13 @@ func (f *hotPeerCache) updateHotPeerStat(newItem, oldItem *HotPeerStat, deltaLoa coldItem(newItem, oldItem) } else { if f.isOldColdPeer(oldItem, newItem.StoreID) { - if newItem.isFullAndHot() { - initItemDegree(newItem) + if newItem.isHot() { + initItem(newItem) } else { - newItem.needDelete = true + newItem.actionType = Remove } } else { - if newItem.isFullAndHot() { + if newItem.isHot() { hotItem(newItem, oldItem) } else { coldItem(newItem, oldItem) @@ -442,8 +444,7 @@ func (f *hotPeerCache) updateHotPeerStat(newItem, oldItem *HotPeerStat, deltaLoa return newItem } -func (f *hotPeerCache) updateNewHotPeerStat(newItem *HotPeerStat, deltaLoads []float64, interval time.Duration) *HotPeerStat { - regionStats := f.kind.RegionStats() +func (f *hotPeerCache) updateNewHotPeerStat(regionStats []RegionStatKind, newItem *HotPeerStat, deltaLoads []float64, interval time.Duration) *HotPeerStat { if interval == 0 { return nil } @@ -454,9 +455,9 @@ func (f *hotPeerCache) updateNewHotPeerStat(newItem *HotPeerStat, deltaLoads []f return nil } if interval.Seconds() >= float64(f.reportIntervalSecs) { - initItemDegree(newItem) + initItem(newItem) } - newItem.isNew = true + newItem.actionType = Add newItem.rollingLoads = make([]*dimStat, len(regionStats)) for i, k := range regionStats { ds := newDimStat(k, time.Duration(newItem.hotStatReportInterval())*time.Second) @@ -522,7 +523,7 @@ func coldItem(newItem, oldItem *HotPeerStat) { newItem.HotDegree = oldItem.HotDegree - 1 newItem.AntiCount = oldItem.AntiCount - 1 if newItem.AntiCount <= 0 { - newItem.needDelete = true + newItem.actionType = Remove } else { newItem.allowAdopt = true } @@ -537,7 +538,7 @@ func hotItem(newItem, oldItem *HotPeerStat) { } } -func initItemDegree(item *HotPeerStat) { +func initItem(item *HotPeerStat) { item.HotDegree = 1 item.AntiCount = hotRegionAntiCount item.allowAdopt = true @@ -546,7 +547,7 @@ func initItemDegree(item *HotPeerStat) { } } -func inheritItemDegree(newItem, oldItem *HotPeerStat) { +func inheritItem(newItem, oldItem *HotPeerStat) { newItem.HotDegree = oldItem.HotDegree newItem.AntiCount = oldItem.AntiCount } diff --git a/server/statistics/hot_peer_cache_test.go b/server/statistics/hot_peer_cache_test.go index c3ce13f5fb5b..3ef1c03ed126 100644 --- a/server/statistics/hot_peer_cache_test.go +++ b/server/statistics/hot_peer_cache_test.go @@ -72,17 +72,17 @@ type testCacheCase struct { kind RWType operator operator expect int - needDelete bool + actionType ActionType } func (t *testHotPeerCache) TestCache(c *C) { tests := []*testCacheCase{ - {Read, transferLeader, 3, false}, - {Read, movePeer, 4, true}, - {Read, addReplica, 4, false}, - {Write, transferLeader, 3, true}, - {Write, movePeer, 4, true}, - {Write, addReplica, 4, true}, + {Read, transferLeader, 3, Update}, + {Read, movePeer, 4, Remove}, + {Read, addReplica, 4, Update}, + {Write, transferLeader, 3, Remove}, + {Write, movePeer, 4, Remove}, + {Write, addReplica, 4, Remove}, } for _, t := range tests { testCache(c, t) @@ -97,13 +97,13 @@ func testCache(c *C, t *testCacheCase) { cache := NewHotPeerCache(t.kind) region := buildRegion(t.kind, 3, 60) checkAndUpdate(c, cache, region, defaultSize[t.kind]) - checkHit(c, cache, region, t.kind, false) // all peers are new + checkHit(c, cache, region, t.kind, Add) // all peers are new srcStore, region := schedule(c, t.operator, region, 10) res := checkAndUpdate(c, cache, region, t.expect) - checkHit(c, cache, region, t.kind, true) // hit cache + checkHit(c, cache, region, t.kind, Update) // hit cache if t.expect != defaultSize[t.kind] { - checkNeedDelete(c, res, srcStore, t.needDelete) + checkOp(c, res, srcStore, t.actionType) } } @@ -122,10 +122,10 @@ func orderingPeers(cache *hotPeerCache, region *core.RegionInfo) []*metapb.Peer func checkFlow(cache *hotPeerCache, region *core.RegionInfo, peers []*metapb.Peer) (res []*HotPeerStat) { reportInterval := region.GetInterval() interval := reportInterval.GetEndTimestamp() - reportInterval.GetStartTimestamp() - res = append(res, cache.CollectExpiredItems(region)...) + res = append(res, cache.collectExpiredItems(region)...) for _, peer := range peers { peerInfo := core.NewPeerInfo(peer, region.GetLoads(), interval) - item := cache.CheckPeerFlow(peerInfo, region) + item := cache.checkPeerFlow(peerInfo, region) if item != nil { res = append(res, item) } @@ -135,7 +135,7 @@ func checkFlow(cache *hotPeerCache, region *core.RegionInfo, peers []*metapb.Pee func updateFlow(cache *hotPeerCache, res []*HotPeerStat) []*HotPeerStat { for _, p := range res { - cache.Update(p) + cache.update(p) } return res } @@ -168,7 +168,7 @@ func checkAndUpdateSkipOne(c *C, cache *hotPeerCache, region *core.RegionInfo, e return updateFlow(cache, res) } -func checkHit(c *C, cache *hotPeerCache, region *core.RegionInfo, kind RWType, isHit bool) { +func checkHit(c *C, cache *hotPeerCache, region *core.RegionInfo, kind RWType, actionType ActionType) { var peers []*metapb.Peer if kind == Read { peers = []*metapb.Peer{region.GetLeader()} @@ -178,14 +178,14 @@ func checkHit(c *C, cache *hotPeerCache, region *core.RegionInfo, kind RWType, i for _, peer := range peers { item := cache.getOldHotPeerStat(region.GetID(), peer.StoreId) c.Assert(item, NotNil) - c.Assert(item.isNew, Equals, !isHit) + c.Assert(item.actionType, Equals, actionType) } } -func checkNeedDelete(c *C, ret []*HotPeerStat, storeID uint64, needDelete bool) { +func checkOp(c *C, ret []*HotPeerStat, storeID uint64, actionType ActionType) { for _, item := range ret { if item.StoreID == storeID { - c.Assert(item.needDelete, Equals, needDelete) + c.Assert(item.actionType, Equals, actionType) return } } @@ -296,55 +296,55 @@ func (t *testHotPeerCache) TestUpdateHotPeerStat(c *C) { m := RegionHeartBeatReportInterval / StoreHeartBeatReportInterval // skip interval=0 - newItem := &HotPeerStat{needDelete: false, thresholds: []float64{0.0, 0.0, 0.0}, Kind: Read} - newItem = cache.updateHotPeerStat(newItem, nil, []float64{0.0, 0.0, 0.0}, 0) + newItem := &HotPeerStat{actionType: Update, thresholds: []float64{0.0, 0.0, 0.0}, Kind: Read} + newItem = cache.updateHotPeerStat(nil, newItem, nil, []float64{0.0, 0.0, 0.0}, 0) c.Check(newItem, IsNil) // new peer, interval is larger than report interval, but no hot - newItem = &HotPeerStat{needDelete: false, thresholds: []float64{1.0, 1.0, 1.0}, Kind: Read} - newItem = cache.updateHotPeerStat(newItem, nil, []float64{0.0, 0.0, 0.0}, 10*time.Second) + newItem = &HotPeerStat{actionType: Update, thresholds: []float64{1.0, 1.0, 1.0}, Kind: Read} + newItem = cache.updateHotPeerStat(nil, newItem, nil, []float64{0.0, 0.0, 0.0}, 10*time.Second) c.Check(newItem, IsNil) // new peer, interval is less than report interval - newItem = &HotPeerStat{needDelete: false, thresholds: []float64{0.0, 0.0, 0.0}, Kind: Read} - newItem = cache.updateHotPeerStat(newItem, nil, []float64{60.0, 60.0, 60.0}, 4*time.Second) + newItem = &HotPeerStat{actionType: Update, thresholds: []float64{0.0, 0.0, 0.0}, Kind: Read} + newItem = cache.updateHotPeerStat(nil, newItem, nil, []float64{60.0, 60.0, 60.0}, 4*time.Second) c.Check(newItem, NotNil) c.Check(newItem.HotDegree, Equals, 0) c.Check(newItem.AntiCount, Equals, 0) // sum of interval is less than report interval oldItem := newItem - newItem = cache.updateHotPeerStat(newItem, oldItem, []float64{60.0, 60.0, 60.0}, 4*time.Second) + newItem = cache.updateHotPeerStat(nil, newItem, oldItem, []float64{60.0, 60.0, 60.0}, 4*time.Second) c.Check(newItem.HotDegree, Equals, 0) c.Check(newItem.AntiCount, Equals, 0) // sum of interval is larger than report interval, and hot oldItem = newItem - newItem = cache.updateHotPeerStat(newItem, oldItem, []float64{60.0, 60.0, 60.0}, 4*time.Second) + newItem = cache.updateHotPeerStat(nil, newItem, oldItem, []float64{60.0, 60.0, 60.0}, 4*time.Second) c.Check(newItem.HotDegree, Equals, 1) c.Check(newItem.AntiCount, Equals, 2*m) // sum of interval is less than report interval oldItem = newItem - newItem = cache.updateHotPeerStat(newItem, oldItem, []float64{60.0, 60.0, 60.0}, 4*time.Second) + newItem = cache.updateHotPeerStat(nil, newItem, oldItem, []float64{60.0, 60.0, 60.0}, 4*time.Second) c.Check(newItem.HotDegree, Equals, 1) c.Check(newItem.AntiCount, Equals, 2*m) // sum of interval is larger than report interval, and hot oldItem = newItem - newItem = cache.updateHotPeerStat(newItem, oldItem, []float64{60.0, 60.0, 60.0}, 10*time.Second) + newItem = cache.updateHotPeerStat(nil, newItem, oldItem, []float64{60.0, 60.0, 60.0}, 10*time.Second) c.Check(newItem.HotDegree, Equals, 2) c.Check(newItem.AntiCount, Equals, 2*m) // sum of interval is larger than report interval, and cold oldItem = newItem newItem.thresholds = []float64{10.0, 10.0, 10.0} - newItem = cache.updateHotPeerStat(newItem, oldItem, []float64{60.0, 60.0, 60.0}, 10*time.Second) + newItem = cache.updateHotPeerStat(nil, newItem, oldItem, []float64{60.0, 60.0, 60.0}, 10*time.Second) c.Check(newItem.HotDegree, Equals, 1) c.Check(newItem.AntiCount, Equals, 2*m-1) // sum of interval is larger than report interval, and cold for i := 0; i < 2*m-1; i++ { oldItem = newItem - newItem = cache.updateHotPeerStat(newItem, oldItem, []float64{60.0, 60.0, 60.0}, 10*time.Second) + newItem = cache.updateHotPeerStat(nil, newItem, oldItem, []float64{60.0, 60.0, 60.0}, 10*time.Second) } c.Check(newItem.HotDegree, Less, 0) c.Check(newItem.AntiCount, Equals, 0) - c.Check(newItem.needDelete, IsTrue) + c.Check(newItem.actionType, Equals, Remove) } func (t *testHotPeerCache) TestThresholdWithUpdateHotPeerStat(c *C) { @@ -369,7 +369,7 @@ func (t *testHotPeerCache) testMetrics(c *C, interval, byteRate, expectThreshold Kind: cache.kind, StoreID: storeID, RegionID: i, - needDelete: false, + actionType: Update, thresholds: thresholds, Loads: make([]float64, DimLen), } @@ -379,8 +379,8 @@ func (t *testHotPeerCache) testMetrics(c *C, interval, byteRate, expectThreshold if oldItem != nil && oldItem.rollingLoads[RegionReadBytes].isHot(thresholds[RegionReadBytes]) == true { break } - item := cache.updateHotPeerStat(newItem, oldItem, []float64{byteRate * interval, 0.0, 0.0}, time.Duration(interval)*time.Second) - cache.Update(item) + item := cache.updateHotPeerStat(nil, newItem, oldItem, []float64{byteRate * interval, 0.0, 0.0}, time.Duration(interval)*time.Second) + cache.update(item) } thresholds := cache.calcHotThresholds(storeID) if i < TopNN { @@ -493,13 +493,13 @@ func BenchmarkCheckRegionFlow(b *testing.B) { for i := 0; i < b.N; i++ { items := make([]*HotPeerStat, 0) for _, peerInfo := range peerInfos { - item := cache.CheckPeerFlow(peerInfo, region) + item := cache.checkPeerFlow(peerInfo, region) if item != nil { items = append(items, item) } } for _, ret := range items { - cache.Update(ret) + cache.update(ret) } } } diff --git a/server/statistics/kind.go b/server/statistics/kind.go index e5965fcbfcbd..65ec300f7ac6 100644 --- a/server/statistics/kind.go +++ b/server/statistics/kind.go @@ -147,3 +147,25 @@ func (k RWType) RegionStats() []RegionStatKind { } return nil } + +// ActionType indicates the action type for the stat item. +type ActionType int + +// Flags for action type. +const ( + Add ActionType = iota + Remove + Update +) + +func (t ActionType) String() string { + switch t { + case Add: + return "add" + case Remove: + return "remove" + case Update: + return "update" + } + return "unimplemented" +} diff --git a/tests/pdctl/global_test.go b/tests/pdctl/global_test.go new file mode 100644 index 000000000000..bb14eeafac25 --- /dev/null +++ b/tests/pdctl/global_test.go @@ -0,0 +1,79 @@ +// Copyright 2021 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pdctl + +import ( + "context" + "fmt" + "net/http" + "testing" + + . "github.com/pingcap/check" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/apiutil" + "github.com/tikv/pd/pkg/testutil" + "github.com/tikv/pd/server" + cmd "github.com/tikv/pd/tools/pd-ctl/pdctl" + "go.uber.org/zap" +) + +func Test(t *testing.T) { + TestingT(t) +} + +var _ = Suite(&globalTestSuite{}) + +type globalTestSuite struct{} + +func (s *globalTestSuite) SetUpSuite(c *C) { + server.EnableZap = true +} + +func (s *globalTestSuite) TestSendAndGetComponent(c *C) { + handler := func(ctx context.Context, s *server.Server) (http.Handler, server.ServiceGroup, error) { + mux := http.NewServeMux() + mux.HandleFunc("/pd/api/v1/health", func(w http.ResponseWriter, r *http.Request) { + component := apiutil.GetComponentNameOnHTTP(r) + for k := range r.Header { + log.Info("header", zap.String("key", k)) + } + log.Info("component", zap.String("component", component)) + c.Assert(component, Equals, "pdctl") + fmt.Fprint(w, component) + }) + info := server.ServiceGroup{ + IsCore: true, + } + return mux, info, nil + } + cfg := server.NewTestSingleConfig(checkerWithNilAssert(c)) + ctx, cancel := context.WithCancel(context.Background()) + svr, err := server.CreateServer(ctx, cfg, handler) + c.Assert(err, IsNil) + err = svr.Run() + c.Assert(err, IsNil) + pdAddr := svr.GetAddr() + defer func() { + cancel() + svr.Close() + testutil.CleanServer(svr.GetConfig().DataDir) + }() + + cmd := cmd.GetRootCmd() + args := []string{"-u", pdAddr, "health"} + output, err := ExecuteCommand(cmd, args...) + c.Assert(err, IsNil) + c.Assert(string(output), Equals, "pdctl\n") +} diff --git a/tests/pdctl/helper.go b/tests/pdctl/helper.go index 66e675edb7f7..75e7d19c288b 100644 --- a/tests/pdctl/helper.go +++ b/tests/pdctl/helper.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/spf13/cobra" + "github.com/tikv/pd/pkg/assertutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/api" "github.com/tikv/pd/server/core" @@ -113,3 +114,11 @@ func MustPutRegion(c *check.C, cluster *tests.TestCluster, regionID, storeID uin c.Assert(err, check.IsNil) return r } + +func checkerWithNilAssert(c *check.C) *assertutil.Checker { + checker := assertutil.NewChecker(c.FailNow) + checker.IsNil = func(obtained interface{}) { + c.Assert(obtained, check.IsNil) + } + return checker +} diff --git a/tools/pd-ctl/pdctl/command/global.go b/tools/pd-ctl/pdctl/command/global.go index bc6dcaa82eac..8f679daf5169 100644 --- a/tools/pd-ctl/pdctl/command/global.go +++ b/tools/pd-ctl/pdctl/command/global.go @@ -25,11 +25,15 @@ import ( "github.com/pingcap/errors" "github.com/spf13/cobra" + "github.com/tikv/pd/pkg/apiutil" "go.etcd.io/etcd/pkg/transport" ) var ( - dialClient = &http.Client{} + pdControllerComponentName = "pdctl" + dialClient = &http.Client{ + Transport: apiutil.NewComponentSignatureRoundTripper(http.DefaultTransport, pdControllerComponentName), + } pingPrefix = "pd/api/v1/ping" ) @@ -46,9 +50,8 @@ func InitHTTPSClient(caPath, certPath, keyPath string) error { } dialClient = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: tlsConfig, - }, + Transport: apiutil.NewComponentSignatureRoundTripper( + &http.Transport{TLSClientConfig: tlsConfig}, pdControllerComponentName), } return nil diff --git a/tools/pd-simulator/main.go b/tools/pd-simulator/main.go index e6fecd624545..12729f7f772c 100644 --- a/tools/pd-simulator/main.go +++ b/tools/pd-simulator/main.go @@ -25,7 +25,6 @@ import ( "github.com/BurntSushi/toml" "github.com/pingcap/log" - "github.com/tikv/pd/pkg/logutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/api" "github.com/tikv/pd/server/config" @@ -114,7 +113,7 @@ func NewSingleServer(ctx context.Context, simConfig *simulator.SimConfig) (*serv log.Fatal("setup logger error", zap.Error(err)) } - err = logutil.InitLogger(&simConfig.ServerConfig.Log) + simConfig.ServerConfig.SetupLogger() if err != nil { log.Fatal("initialize logger error", zap.Error(err)) }