Merge pull request kubernetes-retired#817 from TommyLike/feature/fix_…

…kubemark_issue

Fix kubemark issue & Add document for kubemark test

test/e2e/kube-batch/util.go

@@ -830,7 +830,8 @@ func getkubemarkConfigPath() string {
 	if err != nil {
 		return ""
 	}
-	configPath := filepath.Join(wd, "../kubemark/kubeconfig.kubemark")
+	//TODO: Please update this path as well once the whole tests are being moved into root test folder.
+	configPath := filepath.Join(wd, "../../kubemark/kubeconfig.kubemark")
 	exist, err := file.FileExists(configPath)
 	if err != nil || !exist {
 		return ""

test/kubemark/README.md

@@ -0,0 +1,53 @@
+# Guide for running kubemark with kube-batch scheduler
+
+## 1.Pre-requisites
+There are several requirements before installing kubemark cluster for test.
+1. **Docker**: Building kubernetes releases with command``make quick-release`` require having a docker environment.
+2. **Google cloud SDKs**: Kubemark utilize GCE to create its master node in default, therefore you need ensure
+   google cloud SDKs are installed on your environment. **NOTES**: ``gcloud beta`` is required in addition.
+3. **External Kubernetes cluster**: Kubemark utilize external kubernetes cluster to hold its ``Hollow Nodes``, before
+   creating kubemark cluster, please ensure your environment can successfully execute command: ``kubectl get nodes``
+4. **Python**: There are some commands which use python binaries to perform specific tasks, please ensure it's installed
+   on your environment.
+5. **Google Container Registry API**: Google container register API should be enabled to upload images during setup cluster
+   process.
+
+## 2. Setup kubemark clusters
+All GCE related configure options are located in file 
+``vendor/k8s.io/kubernetes/cluster/kubemark/gce/config-default.sh``, it's usually required to update some of them according
+to your own GCP environment before executing. For instance:
+```$xslt
+ZONE=<specify your own GCP zone>
+NETWORK=<specify your own network resource name>
+```
+then execute command in root folder:
+```$xslt
+./test/kubemark/start-kubemark.sh
+```
+Setup script will clone kubernetes project to temp folder and build its releases every time, 
+In order to speed up the performance, it's better to clone & build the kubernetes yourself.
+Then, try the commands in root folder:
+```$xslt
+SOURCE_OUTPUT=<path to kubernetes `_output` folder> ./test/kubemark/start-kubemark.sh
+```
+
+After script successfully executed, you can verify your kubemark cluster with command in kube-batch root folder:
+```$xslt
+➜  kube-batch git: kubectl --kubeconfig test/kubemark/kubeconfig.kubemark get nodes
+NAME                STATUS   ROLES    AGE    VERSION
+hollow-node-2wgdq   Ready    <none>   5m8s   v1.15.0-alpha.1.162+b3847120241251
+hollow-node-8kksj   Ready    <none>   5m8s   v1.15.0-alpha.1.162+b3847120241251
+......
+```
+## 3. Running performance tests:
+Command as below:
+```$xslt
+go test ./test/e2e/kube-batch -v -timeout 30m --ginkgo.focus="Feature:Performance"
+```
+When finished, there would be a metric json file with timestamp generated in folder ```test/e2e```
+
+## 4. Cleanup kubemark cluster
+Command as below:
+```$xslt
+./test/kubemark/stop-kubemark.sh
+```

test/kubemark/kube-batch.yaml

@@ -8,7 +8,7 @@ spec:
   nodeName: kubernetes-master
   containers:
   - name: kube-batch
-    image: kubesigs/kube-batch:v0.4.2
+    image: kubesigs/kube-batch:{{RELEASE_VER}}
     imagePullPolicy: IfNotPresent
     resources:
       requests:

test/kubemark/start-kubemark.sh

@@ -1,55 +1,69 @@
 #!/usr/bin/env bash
 
-TMP_ROOT="$(dirname "${BASH_SOURCE}")/../../vendor/k8s.io/kubernetes"
+VK_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../..
+TMP_ROOT="${VK_ROOT}/vendor/k8s.io/kubernetes"
 KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}")
-KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
 KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
 RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
-CRD_DIRECTORY="${KUBE_ROOT}/../../../deployment/kube-batch/templates"
-QUEUE_DIR="${KUBE_ROOT}/../../../config/queue"
+CRD_DIRECTORY="${VK_ROOT}/deployment/kube-batch/templates"
+QUEUE_DIR="${VK_ROOT}/config/queue"
+
+#Release version for kube batch
+RELEASE_VER=v0.4.2
+
+#Ensure external cluster exists and kubectl binary works
+kubectl get nodes
+if [[ $? != 0 ]]; then
+  echo "External kubernetes cluster required for kubemark"
+  exit 1
+fi
 
 #Build kubernetes Binary and copy to _output folder
 if [ ! -d "$KUBE_ROOT/_output" ]; then
-  mkdir -p /tmp/src/k8s.io
-  cd /tmp/src/k8s.io
-  git clone https://github.com/kubernetes/kubernetes.git
-  cd kubernetes
-  make quick-release
-  mv _output/  $KUBE_ROOT
+  #If source folder is specified, overwrite the _output folder
+  if [[ "${SOURCE_OUTPUT}xxx" != "xxx" ]]; then
+    echo "Copying release files into kubernetes output folder from ${SOURCE_OUTPUT}"
+    cp -r ${SOURCE_OUTPUT} $KUBE_ROOT
+  else
+    echo "Building kubernetes in temp folder /tmp/src/k8s.io and copying release files."
+    mkdir -p /tmp/src/k8s.io
+    cd /tmp/src/k8s.io
+    git clone https://github.com/kubernetes/kubernetes.git
+    cd kubernetes
+    make quick-release
+    mv _output/  $KUBE_ROOT
+  fi
 fi
 
-
+#Update kubemark script to install kube-batch when starting master
+echo "Modify kubemark master script"
+cp "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh" "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh.bak"
 #Appending lines to start kube-batch
 src="start-kubemaster-component \"kube-scheduler\""
-dest="start-kubemaster-component \"kube-scheduler\" \ncp \${KUBE_ROOT}/kubeconfig.kubemark /etc/srv/kubernetes \nstart-kubemaster-component \"kube-batch\""
+dest="start-kubemaster-component \"kube-scheduler\" \ncp \${KUBE_ROOT}/kubeconfig.kubemark /etc/srv/kubernetes \ncp \${KUBE_ROOT}/kube-batch.yaml /etc/kubernetes/manifests\n"
 sed -i "s@${src}@${dest}@g" "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh"
 
 #Appending lines to copy kube-batch.yaml
+cp "${KUBEMARK_DIRECTORY}/start-kubemark.sh" "${KUBEMARK_DIRECTORY}/start-kubemark.sh.bak"
 src1="\"\${SERVER_BINARY_TAR}\" \\\\"
 dest1="\"\${SERVER_BINARY_TAR}\" \\\\\n    \"\${RESOURCE_DIRECTORY}/kube-batch.yaml\" \\\\"
 sed -i "s@${src1}@${dest1}@g" "${KUBEMARK_DIRECTORY}/start-kubemark.sh"
 
+#Update kube-batch yaml and copy it to resource folder
+cp "${VK_ROOT}/test/kubemark/kube-batch.yaml"  ${RESOURCE_DIRECTORY}
+sed -i "s@{{RELEASE_VER}}@${RELEASE_VER}@g" "${RESOURCE_DIRECTORY}/kube-batch.yaml"
 
-cp kube-batch.yaml  ${RESOURCE_DIRECTORY} 
-
+#Start the cluster
 bash -x ${KUBEMARK_DIRECTORY}/start-kubemark.sh
 
 #creating the CRD Queue and PodGroup
-podgroup=$("${KUBECTL}" --kubeconfig="${RESOURCE_DIRECTORY}"/kubeconfig.kubemark create -f  "${CRD_DIRECTORY}"/scheduling_v1alpha1_queue.yaml 2> /dev/null) || true
-queue=$("${KUBECTL}" --kubeconfig="${RESOURCE_DIRECTORY}"/kubeconfig.kubemark create -f  "${CRD_DIRECTORY}"/scheduling_v1alpha1_podgroup.yaml 2> /dev/null) || true
+echo "Creating kube batch resource in cluster via folder ${CRD_DIRECTORY}."
+kubectl --kubeconfig="${RESOURCE_DIRECTORY}"/kubeconfig.kubemark apply -f  "${CRD_DIRECTORY}"/scheduling_v1alpha1_queue.yaml
+kubectl --kubeconfig="${RESOURCE_DIRECTORY}"/kubeconfig.kubemark apply -f  "${CRD_DIRECTORY}"/scheduling_v1alpha1_podgroup.yaml
 
 #creating default queue
-defaultqueue=$("${KUBECTL}" --kubeconfig="${RESOURCE_DIRECTORY}"/kubeconfig.kubemark create -f  "${QUEUE_DIR}"/default.yaml 2> /dev/null) || true
-
-#copy the kubemark config 
-cp ${RESOURCE_DIRECTORY}/kubeconfig.kubemark  ./
-
-#Reverting the script changes in the vendor and tmp
-data="kube-batch.yaml"
-sed -i "/${data}/d" "${KUBEMARK_DIRECTORY}/start-kubemark.sh"
-data1="kube-batch"
-data2="kubeconfig.kubemark"
-sed -i "/${data1}/d" "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh"
-sed -i "/${data2}/d" "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh"
-rm -rf ${RESOURCE_DIRECTORY}/kube-batch.yaml
-rm -rf /tmp/src/
+kubectl --kubeconfig="${RESOURCE_DIRECTORY}"/kubeconfig.kubemark apply -f  "${QUEUE_DIR}"/default.yaml
+
+#copy the kubemark config
+cp ${RESOURCE_DIRECTORY}/kubeconfig.kubemark  "${VK_ROOT}/test/kubemark"
+

test/kubemark/stop-kubemark.sh

@@ -3,5 +3,16 @@
 TMP_ROOT="$(dirname "${BASH_SOURCE}")/../../vendor/k8s.io/kubernetes"
 KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}")
 KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
+RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
 
 bash -x ${KUBEMARK_DIRECTORY}/stop-kubemark.sh
+
+if [[ -f "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh.bak" ]]; then
+    mv "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh.bak" "${KUBEMARK_DIRECTORY}/resources/start-kubemark-master.sh"
+fi
+
+if [[ -f "${KUBEMARK_DIRECTORY}/start-kubemark.sh.bak" ]]; then
+    mv "${KUBEMARK_DIRECTORY}/start-kubemark.sh.bak" "${KUBEMARK_DIRECTORY}/start-kubemark.sh"
+fi
+rm -rf ${RESOURCE_DIRECTORY}/kube-batch.yaml
+rm -rf /tmp/src/k8s.io