Skip to content

Commit

Permalink
Merge pull request volcano-sh#5 from volcano-sh/codes/support_helm_chart
Browse files Browse the repository at this point in the history
  • Loading branch information
TommyLike authored Mar 15, 2019
2 parents 6591909 + d069a83 commit 00e22f6
Show file tree
Hide file tree
Showing 22 changed files with 796 additions and 29 deletions.
4 changes: 2 additions & 2 deletions cmd/controllers/app/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,10 @@ func Run(opt *options.ServerOption) error {
return err
}

JobController := job.NewJobController(config)
jobController := job.NewJobController(config)

run := func(ctx context.Context) {
JobController.Run(ctx.Done())
jobController.Run(ctx.Done())
<-ctx.Done()
}

Expand Down
44 changes: 44 additions & 0 deletions hack/e2e-admission-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
apiVersion: admissionregistration.k8s.io/v1beta1
kind: ValidatingWebhookConfiguration
metadata:
name: validate-volcano-job
webhooks:
- clientConfig:
caBundle: {{CA_BUNDLE}}

# the url should agree with webhook service
url: https://{{host}}:{{hostPort}}/jobs
failurePolicy: Ignore
name: validatejob.volcano.sh
rules:
- apiGroups:
- "batch.volcano.sh"
apiVersions:
- "v1alpha1"
operations:
- CREATE
- UPDATE
resources:
- jobs
---
apiVersion: admissionregistration.k8s.io/v1beta1
kind: MutatingWebhookConfiguration
metadata:
name: mutate-volcano-job
webhooks:
- clientConfig:
caBundle: {{CA_BUNDLE}}

# the url should agree with webhook service
url: https://{{host}}:{{hostPort}}/mutating-jobs
failurePolicy: Ignore
name: mutatejob.volcano.sh
rules:
- apiGroups:
- "batch.volcano.sh"
apiVersions:
- "v1alpha1"
operations:
- CREATE
resources:
- jobs
27 changes: 27 additions & 0 deletions hack/e2e-kind-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# this config file contains all config fields with comments
kind: Config
apiVersion: kind.sigs.k8s.io/v1alpha2
# 1 control plane node and 3 workers
nodes:
# the control plane node config
- role: control-plane
# patch the generated kubeadm config with some extra settings
kubeadmConfigPatches:
- |
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
networking:
serviceSubnet: 10.0.0.0/16
# patch it further using a JSON 6902 patch
kubeadmConfigPatchesJson6902:
- group: kubeadm.k8s.io
version: v1beta1
kind: ClusterConfiguration
patch: |
- op: add
path: /apiServer/certSANs/-
value: my-hostname
# the three workers
- role: worker
# replicas specifes the number of nodes to create with this configuration
replicas: 3
135 changes: 135 additions & 0 deletions hack/run-e2e-kind.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
#!/bin/bash

export VK_ROOT=$(dirname "${BASH_SOURCE}")/..
export VK_BIN=${VK_ROOT}/_output/bin
export LOG_LEVEL=3
export SHOW_VOLCANO_LOGS=${SHOW_VOLCANO_LOGS:-1}

if [ "${CLUSTER_NAME}xxx" != "xxx" ];then
export CLUSTER_CONTEXT="--name ${CLUSTER_NAME}"
fi

export KIND_OPT=${KIND_OPT:="--image kindest/node:v1.13.2-huawei --config ${VK_ROOT}/hack/e2e-kind-config.yaml"}

export KIND_IMAGE=$(echo ${KIND_OPT} |grep -E -o "image \w+\/[^ ]*" | sed "s/image //")

# check if kind installed
function check-prerequisites {
echo "checking prerequisites"
which kind >/dev/null 2>&1
if [ $? -ne 0 ]; then
echo "kind not installed, exiting."
exit 1
else
echo -n "found kind, version: " && kind version
fi

which kubectl >/dev/null 2>&1
if [ $? -ne 0 ]; then
echo "kubectl not installed, exiting."
exit 1
else
echo -n "found kubectl, " && kubectl version --short --client
fi
}

# check if the images that kind use exists.
function check-kind-image {
docker images | awk '{print $1":"$2}' | grep -q "${KIND_IMAGE}"
if [ $? -ne 0 ]; then
echo "image: ${KIND_IMAGE} not found."
exit 1
fi
}

# spin up cluster with kind command
function kind-up-cluster {
check-prerequisites
check-kind-image
echo "Running kind: [kind create cluster ${CLUSTER_CONTEXT} ${KIND_OPT}]"
kind create cluster ${CLUSTER_CONTEXT} ${KIND_OPT}
}

function install-volcano {
kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml
kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml
kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml
kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml

# TODO: make vk-controllers and vk-scheduler run in container / in k8s
# start controller
nohup ${VK_BIN}/vk-controllers --kubeconfig ${KUBECONFIG} --logtostderr --v ${LOG_LEVEL} > controller.log 2>&1 &
echo $! > vk-controllers.pid

# start scheduler
nohup ${VK_BIN}/vk-scheduler --kubeconfig ${KUBECONFIG} --scheduler-conf=example/kube-batch-conf.yaml --logtostderr --v ${LOG_LEVEL} > scheduler.log 2>&1 &
echo $! > vk-scheduler.pid
}

function uninstall-volcano {
kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml
kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml
kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml
kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml

kill -9 $(cat vk-controllers.pid)
kill -9 $(cat vk-scheduler.pid)
rm vk-controllers.pid vk-scheduler.pid
}

# clean up
function cleanup {
uninstall-volcano

echo "Running kind: [kind delete cluster ${CLUSTER_CONTEXT}]"
kind delete cluster ${CLUSTER_CONTEXT}

if [ ${SHOW_VOLCANO_LOGS} -eq 1 ]; then
echo "===================================================================================="
echo "=============================>>>>> Scheduler Logs <<<<<============================="
echo "===================================================================================="

cat scheduler.log

echo "===================================================================================="
echo "=============================>>>>> Controller Logs <<<<<============================"
echo "===================================================================================="

cat controller.log
fi
}

echo $* | grep -E -q "\-\-help|\-h"
if [ $? -eq 0 ]; then
echo "Customize the kind-cluster name:
export CLUSTER_NAME=<custom cluster name>
Customize kind options other than --name:
export KIND_OPT=<kind options>
Disable displaying volcano component logs:
export SHOW_VOLCANO_LOGS=0
If you don't have kindest/node:v1.13.2-huawei on the host, checkout the following url to build.
http://code-cbu.huawei.com/CBU-PaaS/Community/K8S/kind/tags/v0.1.0-huawei
"
exit 0
fi


trap cleanup EXIT


kind-up-cluster

KUBECONFIG="$(kind get kubeconfig-path ${CLUSTER_CONTEXT})"

install-volcano

# Run e2e test
cd ${VK_ROOT}
KUBECONFIG=${KUBECONFIG} go test ./test/e2e -v -timeout 30m
64 changes: 39 additions & 25 deletions hack/run-e2e.sh
Original file line number Diff line number Diff line change
@@ -1,48 +1,62 @@
#!/bin/bash

export PATH="${HOME}/.kubeadm-dind-cluster:${PATH}"
export VK_BIN=_output/bin
export LOG_LEVEL=3
export MASTER="http://127.0.0.1:8080"
export VK_BIN=$PWD/_output/bin
export LOG_LEVEL=2
export NUM_NODES=3
export CERT_PATH=/etc/kubernetes/pki
export HOST=localhost
export HOSTPORT=32222

dind_url=https://cdn.rawgit.com/kubernetes-sigs/kubeadm-dind-cluster/master/fixed/dind-cluster-v1.12.sh
dind_dest=./hack/dind-cluster-v1.12.sh
kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml
kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml
kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml
kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml

# start k8s dind cluster
curl ${dind_url} --output ${dind_dest}
chmod +x ${dind_dest}
${dind_dest} up
# config admission-controller TODO: make it easier to deploy
CA_BUNDLE=`kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n'`
sed -i "s|{{CA_BUNDLE}}|$CA_BUNDLE|g" hack/e2e-admission-config.yaml
sed -i "s|{{host}}|${HOST}|g" hack/e2e-admission-config.yaml
sed -i "s|{{hostPort}}|${HOSTPORT}|g" hack/e2e-admission-config.yaml

kubectl create -f config/crds/scheduling_v1alpha1_podgroup.yaml
kubectl create -f config/crds/scheduling_v1alpha1_queue.yaml
kubectl create -f config/crds/batch_v1alpha1_job.yaml
kubectl create -f config/crds/bus_v1alpha1_command.yaml
kubectl create -f hack/e2e-admission-config.yaml

# start controller
nohup ${VK_BIN}/vk-controllers --kubeconfig ${HOME}/.kube/config --logtostderr --v ${LOG_LEVEL} > controller.log 2>&1 &
nohup ${VK_BIN}/vk-controllers --kubeconfig ${HOME}/.kube/config --master=${MASTER} --logtostderr --v ${LOG_LEVEL} > controller.log 2>&1 &

# start scheduler
nohup ${VK_BIN}/vk-scheduler --kubeconfig ${HOME}/.kube/config --scheduler-conf=example/kube-batch-conf.yaml --logtostderr --v ${LOG_LEVEL} > scheduler.log 2>&1 &
nohup ${VK_BIN}/vk-scheduler --kubeconfig ${HOME}/.kube/config --scheduler-conf=example/kube-batch-conf.yaml --master=${MASTER} --logtostderr --v ${LOG_LEVEL} > scheduler.log 2>&1 &

# start admission-controller
nohup ${VK_BIN}/vk-admission --tls-cert-file=${CERT_PATH}/apiserver.crt --tls-private-key-file=${CERT_PATH}/apiserver.key --kubeconfig ${HOME}/.kube/config --port ${HOSTPORT} --logtostderr --v ${LOG_LEVEL} > admission.log 2>&1 &

# clean up
function cleanup {
killall -9 vk-scheduler vk-controllers
./hack/dind-cluster-v1.12.sh down

echo "===================================================================================="
echo "=============================>>>>> Scheduler Logs <<<<<============================="
echo "===================================================================================="

cat scheduler.log
killall -9 -r vk-scheduler -r vk-controllers -r vk-admission

if [[ -f scheduler.log ]] ; then
echo "===================================================================================="
echo "=============================>>>>> Scheduler Logs <<<<<============================="
echo "===================================================================================="
cat scheduler.log
fi

if [[ -f controller.log ]] ; then
echo "===================================================================================="
echo "=============================>>>>> Controller Logs <<<<<============================"
echo "===================================================================================="
cat controller.log
fi

echo "===================================================================================="
echo "=============================>>>>> Controller Logs <<<<<============================"
echo "=============================>>>>> admission Logs <<<<<============================"
echo "===================================================================================="

cat controller.log
cat admission.log
}

trap cleanup EXIT

# Run e2e test
go test ./test/e2e -v
go test ./test/e2e -v -timeout 30m
4 changes: 4 additions & 0 deletions installer/chart/volcano-init/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
name: volcano-init
version: 0.0.1
description: volcano crds and admission-controller config
apiVersion: v1
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@ spec:
type: object
type: array
schedulerName:
description: SchedulerName is the default value of `taskSpecs.template.spec.schedulerName`.
description: SchedulerName is the default value of `tasks.template.spec.schedulerName`.
type: string
taskSpecs:
tasks:
description: Tasks specifies the task specification of Job
items:
properties:
Expand Down Expand Up @@ -139,6 +139,10 @@ spec:
description: The number of running pods.
format: int32
type: integer
version:
description: Job's current version
format: int32
type: integer
state:
description: Current state of Job.
properties:
Expand All @@ -156,6 +160,8 @@ spec:
type: object
type: object
version: v1alpha1
subresources:
status: {}
status:
acceptedNames:
kind: ""
Expand Down
File renamed without changes.
File renamed without changes.
2 changes: 2 additions & 0 deletions installer/chart/volcano-init/values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
basic:
namespace: default
4 changes: 4 additions & 0 deletions installer/chart/volcano/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
name: volcano
version: 0.0.1
description: volcano
apiVersion: v1
Loading

0 comments on commit 00e22f6

Please sign in to comment.