From da1f1dac5dbd66b89cc6cce087f168c3caee3612 Mon Sep 17 00:00:00 2001 From: TommyLike Date: Mon, 22 Apr 2019 14:20:16 +0800 Subject: [PATCH] Update cluster script --- Makefile | 6 ---- test/e2e/kube-batch/util.go | 3 +- test/kubemark/README.md | 53 +++++++++++++++++++++++++++++++++ test/kubemark/kube-batch.yaml | 2 +- test/kubemark/start-kubemark.sh | 43 +++++++++++++++----------- test/kubemark/stop-kubemark.sh | 11 +++++++ 6 files changed, 92 insertions(+), 26 deletions(-) create mode 100644 test/kubemark/README.md diff --git a/Makefile b/Makefile index 551ed7789..23c7f1082 100644 --- a/Makefile +++ b/Makefile @@ -74,12 +74,6 @@ e2e-kind: vkctl images coverage: KUBE_COVER=y hack/make-rules/test.sh $(WHAT) $(TESTS) -benchmark: - #NOTE: !Only GCE platform is supported now - test/kubemark/start-kubemark.sh - go test ./test/e2e/kube-batch -v -timeout 30m --ginkgo.focus="Feature:Performance" - test/kubemark/stop-kubemark.sh - clean: rm -rf _output/ rm -f kube-batch diff --git a/test/e2e/kube-batch/util.go b/test/e2e/kube-batch/util.go index ef3d41e55..809517df1 100644 --- a/test/e2e/kube-batch/util.go +++ b/test/e2e/kube-batch/util.go @@ -830,7 +830,8 @@ func getkubemarkConfigPath() string { if err != nil { return "" } - configPath := filepath.Join(wd, "../kubemark/kubeconfig.kubemark") + //TODO: Please update this path as well once the whole tests are being moved into root test folder. + configPath := filepath.Join(wd, "../../kubemark/kubeconfig.kubemark") exist, err := file.FileExists(configPath) if err != nil || !exist { return "" diff --git a/test/kubemark/README.md b/test/kubemark/README.md new file mode 100644 index 000000000..1af23eaa7 --- /dev/null +++ b/test/kubemark/README.md @@ -0,0 +1,53 @@ +# Guide for running kubemark with kube-batch scheduler + +## 1.Pre-requisites +There are several requirements before installing kubemark cluster for test. +1. **Docker**: Building kubernetes releases with command``make quick-release`` require having a docker environment. +2. **Google cloud SDKs**: Kubemark utilize GCE to create its master node in default, therefore you need ensure + google cloud SDKs are installed on your environment. **NOTES**: ``gcloud beta`` is required in addition. +3. **External Kubernetes cluster**: Kubemark utilize external kubernetes cluster to hold its ``Hollo Nodes``, before + creating kubemark cluster, please ensure your environment can successfully execute command: ``kubectl get nodes`` +4. **Python**: There are some commands which use python binaries to perform specific tasks, please ensure it's installed + on your environment. +5. **Google Container Registry API**: Google container register API should be enabled to upload images during setup cluster + process. + +## 2. Setup kubemark clusters +All GCE related configure options are located in file +``vendor/k8s.io/kubernetes/cluster/kubemark/gce/config-default.sh``, it's usually required to update some of them according +to your own GCP environment before executing. For instance: +```$xslt +ZONE= +NETWORK= +``` +then execute command in root folder: +```$xslt +./test/kubemark/start-kubemark.sh +``` +Setup script will clone kubernetes project to temp folder and build its releases every time, +In order to speed up the performance, it's better to clone & build the kubernetes yourself. +Then, try the commands in root folder: +```$xslt +SOURCE_OUTPUT= ./test/kubemark/start-kubemark.sh +``` + +After script successfully executed, you can verify your kubemark cluster with command in kube-batch root folder: +```$xslt +➜ kube-batch git: kubectl --kubeconfig test/kubemark/kubeconfig.kubemark get nodes +NAME STATUS ROLES AGE VERSION +hollow-node-2wgdq Ready 5m8s v1.15.0-alpha.1.162+b3847120241251 +hollow-node-8kksj Ready 5m8s v1.15.0-alpha.1.162+b3847120241251 +...... +``` +## 3. Running performance tests: +Command as below: +```$xslt +go test ./test/e2e/kube-batch -v -timeout 30m --ginkgo.focus="Feature:Performance" +``` +When finished, there would be a metric json file with timestamp generated in folder ```test/e2e``` + +## 4. Cleanup kubemark cluster +Command as below: +```$xslt +./test/kubemark/stop-kubemark.sh +``` diff --git a/test/kubemark/kube-batch.yaml b/test/kubemark/kube-batch.yaml index 0365c302d..ed0ebd1cc 100755 --- a/test/kubemark/kube-batch.yaml +++ b/test/kubemark/kube-batch.yaml @@ -8,7 +8,7 @@ spec: nodeName: kubernetes-master containers: - name: kube-batch - image: kubesigs/kube-batch:v0.4.2 + image: kubesigs/kube-batch:{{RELEASE_VER}} imagePullPolicy: IfNotPresent resources: requests: diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 55dfb1009..67a25b49e 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -1,11 +1,22 @@ #!/usr/bin/env bash -TMP_ROOT="$(dirname "${BASH_SOURCE}")/../../vendor/k8s.io/kubernetes" +VK_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../.. +TMP_ROOT="${VK_ROOT}/vendor/k8s.io/kubernetes" KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}") KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" -CRD_DIRECTORY="${KUBE_ROOT}/../../../deployment/kube-batch/templates" -QUEUE_DIR="${KUBE_ROOT}/../../../config/queue" +CRD_DIRECTORY="${VK_ROOT}/deployment/kube-batch/templates" +QUEUE_DIR="${VK_ROOT}/config/queue" + +#Release version for kube batch +RELEASE_VER=v0.4.2 + +#Ensure external cluster exists and kubectl binary works +kubectl get nodes +if [[ $? != 0 ]]; then + echo "External kubernetes cluster required for kubemark" + exit 1 +fi #Build kubernetes Binary and copy to _output folder if [ ! -d "$KUBE_ROOT/_output" ]; then @@ -24,20 +35,25 @@ if [ ! -d "$KUBE_ROOT/_output" ]; then fi fi - +#Update kubemark script to install kube-batch when starting master +echo "Modify kubemark master script" +cp "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh" "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh.bak" #Appending lines to start kube-batch src="start-kubemaster-component \"kube-scheduler\"" -dest="start-kubemaster-component \"kube-scheduler\" \ncp \${KUBE_ROOT}/kubeconfig.kubemark /etc/srv/kubernetes \nstart-kubemaster-component \"kube-batch\"" +dest="start-kubemaster-component \"kube-scheduler\" \ncp \${KUBE_ROOT}/kubeconfig.kubemark /etc/srv/kubernetes \ncp \${KUBE_ROOT}/kube-batch.yaml /etc/kubernetes/manifests\n" sed -i "s@${src}@${dest}@g" "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh" #Appending lines to copy kube-batch.yaml +cp "${KUBEMARK_DIRECTORY}/start-kubemark.sh" "${KUBEMARK_DIRECTORY}/start-kubemark.sh.bak" src1="\"\${SERVER_BINARY_TAR}\" \\\\" dest1="\"\${SERVER_BINARY_TAR}\" \\\\\n \"\${RESOURCE_DIRECTORY}/kube-batch.yaml\" \\\\" sed -i "s@${src1}@${dest1}@g" "${KUBEMARK_DIRECTORY}/start-kubemark.sh" +#Update kube-batch yaml and copy it to resource folder +cp "${VK_ROOT}/test/kubemark/kube-batch.yaml" ${RESOURCE_DIRECTORY} +sed -i "s@{{RELEASE_VER}}@${RELEASE_VER}@g" "${RESOURCE_DIRECTORY}/kube-batch.yaml" -cp kube-batch.yaml ${RESOURCE_DIRECTORY} - +#Start the cluster bash -x ${KUBEMARK_DIRECTORY}/start-kubemark.sh #creating the CRD Queue and PodGroup @@ -49,14 +65,5 @@ kubectl --kubeconfig="${RESOURCE_DIRECTORY}"/kubeconfig.kubemark apply -f "${CR kubectl --kubeconfig="${RESOURCE_DIRECTORY}"/kubeconfig.kubemark apply -f "${QUEUE_DIR}"/default.yaml #copy the kubemark config -cp ${RESOURCE_DIRECTORY}/kubeconfig.kubemark ./ - -#Reverting the script changes in the vendor and tmp -data="kube-batch.yaml" -#sed -i "/${data}/d" "${KUBEMARK_DIRECTORY}/start-kubemark.sh" -data1="kube-batch" -data2="kubeconfig.kubemark" -#sed -i "/${data1}/d" "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh" -#sed -i "/${data2}/d" "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh" -#rm -rf ${RESOURCE_DIRECTORY}/kube-batch.yaml -rm -rf /tmp/src/ +cp ${RESOURCE_DIRECTORY}/kubeconfig.kubemark "${VK_ROOT}/test/kubemark" + diff --git a/test/kubemark/stop-kubemark.sh b/test/kubemark/stop-kubemark.sh index 58271516f..07e106aef 100755 --- a/test/kubemark/stop-kubemark.sh +++ b/test/kubemark/stop-kubemark.sh @@ -3,5 +3,16 @@ TMP_ROOT="$(dirname "${BASH_SOURCE}")/../../vendor/k8s.io/kubernetes" KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}") KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" +RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" bash -x ${KUBEMARK_DIRECTORY}/stop-kubemark.sh + +if [[ -f "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh.bak" ]]; then + mv "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh.bak" "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh" +fi + +if [[ -f "${KUBEMARK_DIRECTORY}/start-kubemark.sh.bak" ]]; then + mv "${KUBEMARK_DIRECTORY}/start-kubemark.sh.bak" "${KUBEMARK_DIRECTORY}/start-kubemark.sh" +fi +rm -rf ${RESOURCE_DIRECTORY}/kube-batch.yaml +rm -rf /tmp/src/k8s.io