Support new Dashboard versioning (#2623)

.github/workflows/check-dashboard.yaml

@@ -28,21 +28,21 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
-      - name: Fetch Dashboard Go module
+      - name: Fetch Go module
         run: |
           go mod download
           DASHBOARD_DIR=$(go list -f "{{.Dir}}" -m github.com/pingcap-incubator/tidb-dashboard)
-          DASHBOARD_UI_VERSION=$(grep -v '^#' "${DASHBOARD_DIR}/ui/.github_release_version")
+          DASHBOARD_RELEASE_VERSION=$(grep -v '^#' "${DASHBOARD_DIR}/release-version")
           echo ::set-env name=DASHBOARD_DIR::"$DASHBOARD_DIR"
-          echo ::set-env name=DASHBOARD_UI_VERSION::"$DASHBOARD_UI_VERSION"
-      - name: Fetch UI source specified by version
+          echo ::set-env name=DASHBOARD_RELEASE_VERSION::"$DASHBOARD_RELEASE_VERSION"
+      - name: Fetch UI source specified by release version
         run: |
           rm -rf /tmp/dashboard
           mkdir -p /tmp/dashboard
           cd /tmp/dashboard
-          DOWNLOAD_URL="https://github.com/pingcap-incubator/tidb-dashboard/archive/ui_release_${DASHBOARD_UI_VERSION}.zip"
-          curl -L "${DOWNLOAD_URL}" > source.zip
+          DOWNLOAD_URL="https://github.com/pingcap-incubator/tidb-dashboard/archive/v${DASHBOARD_RELEASE_VERSION}.zip"
+          curl -L "${DOWNLOAD_URL}" --fail --output source.zip
           unzip source.zip
       - name: Check diff
         run: |
-          diff -qr ${DASHBOARD_DIR}/ui/ /tmp/dashboard/tidb-dashboard-ui_release_${DASHBOARD_UI_VERSION}/ui/
+          diff -qr ${DASHBOARD_DIR}/ui/ /tmp/dashboard/tidb-dashboard-${DASHBOARD_RELEASE_VERSION}/ui/

.gitignore

@@ -12,6 +12,7 @@ default*
 *.bak
 .vscode/
 /.dashboard_asset_cache
+/.dashboard_build_temp
 /.tools/bin
 docs/swagger/*
 !docs/swagger/placeholder.go

Makefile

@@ -12,8 +12,8 @@ GOCHECKER := awk '{ print } END { if (NR > 0) { exit 1 } }'
 OVERALLS := overalls
 
 GO_TOOLS_BIN_PATH := $(shell pwd)/.tools/bin
-export GOBIN := $(GO_TOOLS_BIN_PATH)
-export PATH := $(GO_TOOLS_BIN_PATH):$(PATH)
+PATH := $(GO_TOOLS_BIN_PATH):$(PATH)
+SHELL := env PATH=$(PATH) GOBIN=$(GO_TOOLS_BIN_PATH) /bin/bash
 
 FAILPOINT_ENABLE  := $$(find $$PWD/ -type d | grep -vE "\.git" | xargs failpoint-ctl enable)
 FAILPOINT_DISABLE := $$(find $$PWD/ -type d | grep -vE "\.git" | xargs failpoint-ctl disable)
@@ -57,6 +57,15 @@ LDFLAGS += -X "$(PD_PKG)/server.PDGitHash=$(shell git rev-parse HEAD)"
 LDFLAGS += -X "$(PD_PKG)/server.PDGitBranch=$(shell git rev-parse --abbrev-ref HEAD)"
 LDFLAGS += -X "$(PD_PKG)/server.PDEdition=$(PD_EDITION)"
 
+ifneq ($(DASHBOARD), 0)
+	# Note: LDFLAGS must be evaluated lazily for these scripts to work correctly
+	LDFLAGS += -X "github.com/pingcap-incubator/tidb-dashboard/pkg/utils/version.InternalVersion=$(shell scripts/describe-dashboard.sh internal-version)"
+	LDFLAGS += -X "github.com/pingcap-incubator/tidb-dashboard/pkg/utils/version.Standalone=No"
+	LDFLAGS += -X "github.com/pingcap-incubator/tidb-dashboard/pkg/utils/version.PDVersion=$(shell git describe --tags --dirty)"
+	LDFLAGS += -X "github.com/pingcap-incubator/tidb-dashboard/pkg/utils/version.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
+	LDFLAGS += -X "github.com/pingcap-incubator/tidb-dashboard/pkg/utils/version.BuildGitHash=$(shell scripts/describe-dashboard.sh git-hash)"
+endif
+
 GOVER_MAJOR := $(shell go version | sed -E -e "s/.*go([0-9]+)[.]([0-9]+).*/\1/")
 GOVER_MINOR := $(shell go version | sed -E -e "s/.*go([0-9]+)[.]([0-9]+).*/\2/")
 GO111 := $(shell [ $(GOVER_MAJOR) -gt 1 ] || [ $(GOVER_MAJOR) -eq 1 ] && [ $(GOVER_MINOR) -ge 11 ]; echo $$?)
@@ -76,14 +85,16 @@ build: pd-server pd-ctl pd-recover
 
 tools: pd-tso-bench pd-analysis pd-heartbeat-bench
 
-pd-server: export GO111MODULE=on
-pd-server:
+PD_SERVER_DEP :=
 ifneq ($(SWAGGER), 0)
-	make swagger-spec
+	PD_SERVER_DEP += swagger-spec
 endif
 ifneq ($(DASHBOARD), 0)
-	make dashboard-ui
+	PD_SERVER_DEP += dashboard-ui
 endif
+
+pd-server: export GO111MODULE=on
+pd-server: ${PD_SERVER_DEP}
 	CGO_ENABLED=$(BUILD_CGO_ENABLED) go build $(BUILD_FLAGS) -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -tags "$(BUILD_TAGS)" -o bin/pd-server cmd/pd-server/main.go
 
 pd-server-basic:

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/opentracing/opentracing-go v1.0.2
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d
-	github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200619091231-6843d6e0b7aa
+	github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200710045508-523e95bc5ec9
 	github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12
 	github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9
 	github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d

go.sum

@@ -274,8 +274,8 @@ github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
 github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g=
 github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8=
-github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200619091231-6843d6e0b7aa h1:FWFt6j3DcCPxmKoVvHzqc/P1SjLkHYvNqCJPqP7+kOM=
-github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200619091231-6843d6e0b7aa/go.mod h1:9yaAM77sPfa5/f6sdxr3jSkKfIz463KRHyiFHiGjdes=
+github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200710045508-523e95bc5ec9 h1:OVkInFrwXoqz39+8MTr7gxFzluagrtSsxOVOJRBKz2Q=
+github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200710045508-523e95bc5ec9/go.mod h1:9yaAM77sPfa5/f6sdxr3jSkKfIz463KRHyiFHiGjdes=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
 github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0=

scripts/describe-dashboard.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ "$#" -ne 1 ]; then
+  exit 1
+fi
+
+if [ "${DASHBOARD-}" == "0" ]; then
+  exit 0
+fi
+
+DASHBOARD_DIR=$(go list -f "{{.Dir}}" -m github.com/pingcap-incubator/tidb-dashboard)
+
+if [ "$1" = "git-hash" ]; then
+  echo "${DASHBOARD_DIR}" | awk -F- '{print $NF}'
+elif [ "$1" = "internal-version" ]; then
+  grep -v '^#' "${DASHBOARD_DIR}/release-version"
+else
+  exit 1
+fi

scripts/embed-dashboard-ui.sh

@@ -1,44 +1,121 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+
+if [ "${DASHBOARD-}" == "0" ] || [ "${DASHBOARD-}" = "SKIP" ]; then
+  # DASHBOARD=0 will completely exclude TiDB Dashboard in building when calling from Makefile
+  # while DASHBOARD=SKIP will keep current asset file unchanged and include it in building
+  echo '+ Skip TiDB Dashboard'
+  exit 0
+fi
+
+
+RED='\033[1;31m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 BASE_DIR="$(dirname "$DIR")"
-CACHE_DIR=${BASE_DIR}/.dashboard_asset_cache
+ASSET_FILE_NAME=embedded_assets_handler.go
+ASSET_DEST_PATH=${BASE_DIR}/pkg/dashboard/uiserver/${ASSET_FILE_NAME}
+
 
-echo '+ Create cache directory'
-mkdir -p "${CACHE_DIR}"
+echo '+ Clean up existing asset file'
+rm -f ASSET_DEST_PATH
 
-echo '+ Fetch Dashboard Go module'
+echo '+ Fetch TiDB Dashboard Go module'
 go mod download
+DASHBOARD_DIR=$(go list -f "{{.Dir}}" -m github.com/pingcap-incubator/tidb-dashboard)
+echo "  - TiDB Dashboard directory: ${DASHBOARD_DIR}"
 
-echo '+ Discover Dashboard UI version'
 
-DASHBOARD_DIR=$(go list -f "{{.Dir}}" -m github.com/pingcap-incubator/tidb-dashboard)
-echo "  - Dashboard directory: ${DASHBOARD_DIR}"
+function download_embed_asset {
+  CACHE_DIR=${BASE_DIR}/.dashboard_asset_cache
 
-DASHBOARD_UI_VERSION=$(grep -v '^#' "${DASHBOARD_DIR}/ui/.github_release_version")
-echo "  - Dashboard ui version: ${DASHBOARD_UI_VERSION}"
+  echo '+ Create asset cache directory'
+  mkdir -p "${CACHE_DIR}"
 
-echo '+ Check embedded assets exists in cache'
-CACHE_FILE=${CACHE_DIR}/embedded-assets-golang-${DASHBOARD_UI_VERSION}.zip
-if [[ -f "$CACHE_FILE" ]]; then
-  echo "  - Cached archive exists: ${CACHE_FILE}"
-else
-  echo '  - Cached archive does not exist'
-  echo '  - Download pre-built embedded assets from GitHub release'
+  echo '+ Discover TiDB Dashboard release version'
+  DASHBOARD_RELEASE_VERSION=$(grep -v '^#' "${DASHBOARD_DIR}/release-version")
+  echo "  - TiDB Dashboard release version: ${DASHBOARD_RELEASE_VERSION}"
 
-  DOWNLOAD_URL="https://github.com/pingcap-incubator/tidb-dashboard/releases/download/ui_release_${DASHBOARD_UI_VERSION}/embedded-assets-golang.zip"
-  echo "  - Download ${DOWNLOAD_URL}"
-  curl -L "${DOWNLOAD_URL}" > embedded-assets-golang.zip
+  echo '+ Check embedded assets exists in cache'
+  CACHE_FILE=${CACHE_DIR}/embedded-assets-golang-${DASHBOARD_RELEASE_VERSION}.zip
+  if [[ -f "$CACHE_FILE" ]]; then
+    echo "  - Cached archive exists: ${CACHE_FILE}"
+  else
+    echo '  - Cached archive does not exist'
+    echo '  - Download pre-built embedded assets from GitHub release'
 
-  echo "  - Save archive to cache: ${CACHE_FILE}"
-  mv embedded-assets-golang.zip "${CACHE_FILE}"
-fi
+    DOWNLOAD_URL="https://github.com/pingcap-incubator/tidb-dashboard/releases/download/v${DASHBOARD_RELEASE_VERSION}/embedded-assets-golang.zip"
+    echo "  - Download ${DOWNLOAD_URL}"
+    if ! curl -L "${DOWNLOAD_URL}" --fail --output embedded-assets-golang.zip; then
+      echo
+      echo -e "${RED}Error: Failed to download assets of TiDB Dashboard release version ${DASHBOARD_RELEASE_VERSION}.${NC}"
+      if [ "${DASHBOARD_RELEASE_VERSION}" == "nightly" ]; then
+        echo 'This project is using the nightly version of TiDB Dashboard, which does not have any release.'
+      else
+        echo 'This may be caused by using a non-release version of TiDB Dashboard, or the release is still in progress.'
+      fi
+      echo
+      echo -e "To compile PD without TiDB Dashboard:                       ${YELLOW}DASHBOARD=0 make${NC}"
+      echo -e "To compile PD by building TiDB Dashboard assets on-the-fly: ${YELLOW}DASHBOARD=COMPILE make${NC}  or  ${YELLOW}NO_MINIMIZE=1 DASHBOARD=COMPILE make${NC}"
+      exit 1
+    fi
+
+    echo "  - Save archive to cache: ${CACHE_FILE}"
+    mv embedded-assets-golang.zip "${CACHE_FILE}"
+  fi
 
-echo '+ Unpack embedded asset from archive'
-unzip -o "${CACHE_FILE}"
-MOVE_FILE=embedded_assets_handler.go
-gofmt -s -w embedded_assets_handler.go
-MOVE_DEST=pkg/dashboard/uiserver/${MOVE_FILE}
-mv ${MOVE_FILE} ${MOVE_DEST}
-echo "  - Unpacked ${MOVE_DEST}"
+  echo '+ Unpack embedded asset from archive'
+  unzip -o "${CACHE_FILE}"
+  gofmt -s -w ${ASSET_FILE_NAME}
+  mv "${ASSET_FILE_NAME}" "${ASSET_DEST_PATH}"
+  echo "  - Unpacked ${ASSET_DEST_PATH}"
+}
+
+
+function compile_asset {
+  BUILD_DIR=${BASE_DIR}/.dashboard_build_temp
+
+  echo '+ Clean up TiDB Dashboard build directory'
+  echo "  - Build directory: ${DASHBOARD_DIR}"
+  if [ -d "${BUILD_DIR}/ui/node_modules" ]; then
+    echo "  - Build dependency exists, keep dependency cache"
+    mv "${BUILD_DIR}/ui/node_modules" ./
+    rm -rf "${BUILD_DIR}"
+    mkdir -p "${BUILD_DIR}/ui"
+    mv ./node_modules "${BUILD_DIR}/ui/"
+  else
+    rm -rf "${BUILD_DIR}"
+    mkdir -p "${BUILD_DIR}/ui"
+  fi
+
+  echo '+ Copy TiDB Dashboard source code to build directory'
+  echo "  - Src:  ${DASHBOARD_DIR}"
+  echo "  - Dest: ${BUILD_DIR}"
+  cp -r "${DASHBOARD_DIR}/." "${BUILD_DIR}/"
+  chmod -R u+w "${BUILD_DIR}"
+  chmod u+x "${BUILD_DIR}"/scripts/*.sh
+
+  echo '+ Build UI'
+  cd "${BUILD_DIR}"
+  make ui
+
+  echo '+ Generating UI assets'
+  echo '  - Generating...'
+  NO_ASSET_BUILD_TAG=1 scripts/embed_ui_assets.sh
+  echo '  - Writing...'
+  cp "pkg/uiserver/${ASSET_FILE_NAME}" "${ASSET_DEST_PATH}"
+  cd -
+  echo "  - Wrote ${ASSET_DEST_PATH}"
+
+  echo '+ Build UI complete'
+}
+
+
+if [ "${DASHBOARD-}" == "COMPILE" ]; then
+  compile_asset
+else
+  download_embed_asset
+fi