diff --git a/cmd/controllers/app/server.go b/cmd/controllers/app/server.go index c7566b7bce..989b8c617c 100644 --- a/cmd/controllers/app/server.go +++ b/cmd/controllers/app/server.go @@ -60,10 +60,10 @@ func Run(opt *options.ServerOption) error { return err } - JobController := job.NewJobController(config) + jobController := job.NewJobController(config) run := func(ctx context.Context) { - JobController.Run(ctx.Done()) + jobController.Run(ctx.Done()) <-ctx.Done() } diff --git a/hack/e2e-admission-config.yaml b/hack/e2e-admission-config.yaml new file mode 100644 index 0000000000..663bf1acc1 --- /dev/null +++ b/hack/e2e-admission-config.yaml @@ -0,0 +1,44 @@ +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: validate-volcano-job +webhooks: + - clientConfig: + caBundle: {{CA_BUNDLE}} + + # the url should agree with webhook service + url: https://{{host}}:{{hostPort}}/jobs + failurePolicy: Ignore + name: validatejob.volcano.sh + rules: + - apiGroups: + - "batch.volcano.sh" + apiVersions: + - "v1alpha1" + operations: + - CREATE + - UPDATE + resources: + - jobs +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: mutate-volcano-job +webhooks: + - clientConfig: + caBundle: {{CA_BUNDLE}} + + # the url should agree with webhook service + url: https://{{host}}:{{hostPort}}/mutating-jobs + failurePolicy: Ignore + name: mutatejob.volcano.sh + rules: + - apiGroups: + - "batch.volcano.sh" + apiVersions: + - "v1alpha1" + operations: + - CREATE + resources: + - jobs diff --git a/hack/e2e-kind-config.yaml b/hack/e2e-kind-config.yaml new file mode 100644 index 0000000000..d4f3c99da6 --- /dev/null +++ b/hack/e2e-kind-config.yaml @@ -0,0 +1,27 @@ +# this config file contains all config fields with comments +kind: Config +apiVersion: kind.sigs.k8s.io/v1alpha2 +# 1 control plane node and 3 workers +nodes: + # the control plane node config + - role: control-plane + # patch the generated kubeadm config with some extra settings + kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta1 + kind: ClusterConfiguration + networking: + serviceSubnet: 10.0.0.0/16 + # patch it further using a JSON 6902 patch + kubeadmConfigPatchesJson6902: + - group: kubeadm.k8s.io + version: v1beta1 + kind: ClusterConfiguration + patch: | + - op: add + path: /apiServer/certSANs/- + value: my-hostname + # the three workers + - role: worker + # replicas specifes the number of nodes to create with this configuration + replicas: 3 \ No newline at end of file diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh new file mode 100755 index 0000000000..f8dd0a34d9 --- /dev/null +++ b/hack/run-e2e-kind.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +export VK_ROOT=$(dirname "${BASH_SOURCE}")/.. +export VK_BIN=${VK_ROOT}/_output/bin +export LOG_LEVEL=3 +export SHOW_VOLCANO_LOGS=${SHOW_VOLCANO_LOGS:-1} + +if [ "${CLUSTER_NAME}xxx" != "xxx" ];then + export CLUSTER_CONTEXT="--name ${CLUSTER_NAME}" +fi + +export KIND_OPT=${KIND_OPT:="--image kindest/node:v1.13.2-huawei --config ${VK_ROOT}/hack/e2e-kind-config.yaml"} + +export KIND_IMAGE=$(echo ${KIND_OPT} |grep -E -o "image \w+\/[^ ]*" | sed "s/image //") + +# check if kind installed +function check-prerequisites { + echo "checking prerequisites" + which kind >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "kind not installed, exiting." + exit 1 + else + echo -n "found kind, version: " && kind version + fi + + which kubectl >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "kubectl not installed, exiting." + exit 1 + else + echo -n "found kubectl, " && kubectl version --short --client + fi +} + +# check if the images that kind use exists. +function check-kind-image { + docker images | awk '{print $1":"$2}' | grep -q "${KIND_IMAGE}" + if [ $? -ne 0 ]; then + echo "image: ${KIND_IMAGE} not found." + exit 1 + fi +} + +# spin up cluster with kind command +function kind-up-cluster { + check-prerequisites + check-kind-image + echo "Running kind: [kind create cluster ${CLUSTER_CONTEXT} ${KIND_OPT}]" + kind create cluster ${CLUSTER_CONTEXT} ${KIND_OPT} +} + +function install-volcano { + kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml + kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml + kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml + kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml + + # TODO: make vk-controllers and vk-scheduler run in container / in k8s + # start controller + nohup ${VK_BIN}/vk-controllers --kubeconfig ${KUBECONFIG} --logtostderr --v ${LOG_LEVEL} > controller.log 2>&1 & + echo $! > vk-controllers.pid + + # start scheduler + nohup ${VK_BIN}/vk-scheduler --kubeconfig ${KUBECONFIG} --scheduler-conf=example/kube-batch-conf.yaml --logtostderr --v ${LOG_LEVEL} > scheduler.log 2>&1 & + echo $! > vk-scheduler.pid +} + +function uninstall-volcano { + kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml + kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml + kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml + kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml + + kill -9 $(cat vk-controllers.pid) + kill -9 $(cat vk-scheduler.pid) + rm vk-controllers.pid vk-scheduler.pid +} + +# clean up +function cleanup { + uninstall-volcano + + echo "Running kind: [kind delete cluster ${CLUSTER_CONTEXT}]" + kind delete cluster ${CLUSTER_CONTEXT} + + if [ ${SHOW_VOLCANO_LOGS} -eq 1 ]; then + echo "====================================================================================" + echo "=============================>>>>> Scheduler Logs <<<<<=============================" + echo "====================================================================================" + + cat scheduler.log + + echo "====================================================================================" + echo "=============================>>>>> Controller Logs <<<<<============================" + echo "====================================================================================" + + cat controller.log + fi +} + +echo $* | grep -E -q "\-\-help|\-h" +if [ $? -eq 0 ]; then + echo "Customize the kind-cluster name: + + export CLUSTER_NAME= + +Customize kind options other than --name: + + export KIND_OPT= + +Disable displaying volcano component logs: + + export SHOW_VOLCANO_LOGS=0 + +If you don't have kindest/node:v1.13.2-huawei on the host, checkout the following url to build. + + http://code-cbu.huawei.com/CBU-PaaS/Community/K8S/kind/tags/v0.1.0-huawei +" + exit 0 +fi + + +trap cleanup EXIT + + +kind-up-cluster + +KUBECONFIG="$(kind get kubeconfig-path ${CLUSTER_CONTEXT})" + +install-volcano + +# Run e2e test +cd ${VK_ROOT} +KUBECONFIG=${KUBECONFIG} go test ./test/e2e -v -timeout 30m diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh index d27dc60c67..a952c90798 100755 --- a/hack/run-e2e.sh +++ b/hack/run-e2e.sh @@ -1,48 +1,62 @@ #!/bin/bash export PATH="${HOME}/.kubeadm-dind-cluster:${PATH}" -export VK_BIN=_output/bin -export LOG_LEVEL=3 +export MASTER="http://127.0.0.1:8080" +export VK_BIN=$PWD/_output/bin +export LOG_LEVEL=2 export NUM_NODES=3 +export CERT_PATH=/etc/kubernetes/pki +export HOST=localhost +export HOSTPORT=32222 -dind_url=https://cdn.rawgit.com/kubernetes-sigs/kubeadm-dind-cluster/master/fixed/dind-cluster-v1.12.sh -dind_dest=./hack/dind-cluster-v1.12.sh +kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml +kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml +kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml +kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml -# start k8s dind cluster -curl ${dind_url} --output ${dind_dest} -chmod +x ${dind_dest} -${dind_dest} up +# config admission-controller TODO: make it easier to deploy +CA_BUNDLE=`kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n'` +sed -i "s|{{CA_BUNDLE}}|$CA_BUNDLE|g" hack/e2e-admission-config.yaml +sed -i "s|{{host}}|${HOST}|g" hack/e2e-admission-config.yaml +sed -i "s|{{hostPort}}|${HOSTPORT}|g" hack/e2e-admission-config.yaml -kubectl create -f config/crds/scheduling_v1alpha1_podgroup.yaml -kubectl create -f config/crds/scheduling_v1alpha1_queue.yaml -kubectl create -f config/crds/batch_v1alpha1_job.yaml -kubectl create -f config/crds/bus_v1alpha1_command.yaml +kubectl create -f hack/e2e-admission-config.yaml # start controller -nohup ${VK_BIN}/vk-controllers --kubeconfig ${HOME}/.kube/config --logtostderr --v ${LOG_LEVEL} > controller.log 2>&1 & +nohup ${VK_BIN}/vk-controllers --kubeconfig ${HOME}/.kube/config --master=${MASTER} --logtostderr --v ${LOG_LEVEL} > controller.log 2>&1 & # start scheduler -nohup ${VK_BIN}/vk-scheduler --kubeconfig ${HOME}/.kube/config --scheduler-conf=example/kube-batch-conf.yaml --logtostderr --v ${LOG_LEVEL} > scheduler.log 2>&1 & +nohup ${VK_BIN}/vk-scheduler --kubeconfig ${HOME}/.kube/config --scheduler-conf=example/kube-batch-conf.yaml --master=${MASTER} --logtostderr --v ${LOG_LEVEL} > scheduler.log 2>&1 & + +# start admission-controller +nohup ${VK_BIN}/vk-admission --tls-cert-file=${CERT_PATH}/apiserver.crt --tls-private-key-file=${CERT_PATH}/apiserver.key --kubeconfig ${HOME}/.kube/config --port ${HOSTPORT} --logtostderr --v ${LOG_LEVEL} > admission.log 2>&1 & # clean up function cleanup { - killall -9 vk-scheduler vk-controllers - ./hack/dind-cluster-v1.12.sh down - - echo "====================================================================================" - echo "=============================>>>>> Scheduler Logs <<<<<=============================" - echo "====================================================================================" - - cat scheduler.log + killall -9 -r vk-scheduler -r vk-controllers -r vk-admission + + if [[ -f scheduler.log ]] ; then + echo "====================================================================================" + echo "=============================>>>>> Scheduler Logs <<<<<=============================" + echo "====================================================================================" + cat scheduler.log + fi + + if [[ -f controller.log ]] ; then + echo "====================================================================================" + echo "=============================>>>>> Controller Logs <<<<<============================" + echo "====================================================================================" + cat controller.log + fi echo "====================================================================================" - echo "=============================>>>>> Controller Logs <<<<<============================" + echo "=============================>>>>> admission Logs <<<<<============================" echo "====================================================================================" - cat controller.log + cat admission.log } trap cleanup EXIT # Run e2e test -go test ./test/e2e -v +go test ./test/e2e -v -timeout 30m diff --git a/installer/chart/volcano-init/Chart.yaml b/installer/chart/volcano-init/Chart.yaml new file mode 100644 index 0000000000..ef2fa97f21 --- /dev/null +++ b/installer/chart/volcano-init/Chart.yaml @@ -0,0 +1,4 @@ +name: volcano-init +version: 0.0.1 +description: volcano crds and admission-controller config +apiVersion: v1 \ No newline at end of file diff --git a/config/crds/batch_v1alpha1_job.yaml b/installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml similarity index 96% rename from config/crds/batch_v1alpha1_job.yaml rename to installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml index e069176bcf..c47e4c82be 100644 --- a/config/crds/batch_v1alpha1_job.yaml +++ b/installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml @@ -76,9 +76,9 @@ spec: type: object type: array schedulerName: - description: SchedulerName is the default value of `taskSpecs.template.spec.schedulerName`. + description: SchedulerName is the default value of `tasks.template.spec.schedulerName`. type: string - taskSpecs: + tasks: description: Tasks specifies the task specification of Job items: properties: @@ -139,6 +139,10 @@ spec: description: The number of running pods. format: int32 type: integer + version: + description: Job's current version + format: int32 + type: integer state: description: Current state of Job. properties: @@ -156,6 +160,8 @@ spec: type: object type: object version: v1alpha1 + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/config/crds/bus_v1alpha1_command.yaml b/installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml similarity index 100% rename from config/crds/bus_v1alpha1_command.yaml rename to installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml diff --git a/config/crds/scheduling_v1alpha1_podgroup.yaml b/installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml similarity index 100% rename from config/crds/scheduling_v1alpha1_podgroup.yaml rename to installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml diff --git a/config/crds/scheduling_v1alpha1_queue.yaml b/installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml similarity index 100% rename from config/crds/scheduling_v1alpha1_queue.yaml rename to installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml diff --git a/installer/chart/volcano-init/values.yaml b/installer/chart/volcano-init/values.yaml new file mode 100644 index 0000000000..13097a0c59 --- /dev/null +++ b/installer/chart/volcano-init/values.yaml @@ -0,0 +1,2 @@ +basic: + namespace: default \ No newline at end of file diff --git a/installer/chart/volcano/Chart.yaml b/installer/chart/volcano/Chart.yaml new file mode 100644 index 0000000000..fbc85a71a2 --- /dev/null +++ b/installer/chart/volcano/Chart.yaml @@ -0,0 +1,4 @@ +name: volcano +version: 0.0.1 +description: volcano +apiVersion: v1 diff --git a/installer/chart/volcano/plugins/gen-admission-secret/gen-admission-secret.sh b/installer/chart/volcano/plugins/gen-admission-secret/gen-admission-secret.sh new file mode 100644 index 0000000000..3aae09c908 --- /dev/null +++ b/installer/chart/volcano/plugins/gen-admission-secret/gen-admission-secret.sh @@ -0,0 +1,128 @@ +#!/bin/bash +#TODO: this file is used for release, should not place it here +set -e + +usage() { + cat <> ${tmpdir}/csr.conf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = ${service} +DNS.2 = ${service}.${namespace} +DNS.3 = ${service}.${namespace}.svc +EOF + +openssl genrsa -out ${tmpdir}/server-key.pem 2048 +openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf + +# clean-up any previously created CSR for our service. Ignore errors if not present. +kubectl delete csr ${csrName} 2>/dev/null || true + +# create server cert/key CSR and send to k8s API +cat <&2 + exit 1 +fi +echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem + +# ca cert +kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' >> ${tmpdir}/ca-cert.pem + +# create the secret with CA cert and server cert/key +kubectl create secret generic ${secret} \ + --from-file=tls.key=${tmpdir}/server-key.pem \ + --from-file=tls.crt=${tmpdir}/server-cert.pem \ + --from-file=ca.crt=${tmpdir}/ca-cert.pem \ + --dry-run -o yaml | + kubectl -n ${namespace} apply -f - diff --git a/installer/chart/volcano/plugins/gen-admission-secret/plugin.yaml b/installer/chart/volcano/plugins/gen-admission-secret/plugin.yaml new file mode 100644 index 0000000000..e2f8698d59 --- /dev/null +++ b/installer/chart/volcano/plugins/gen-admission-secret/plugin.yaml @@ -0,0 +1,7 @@ +name: "gen-admission-secret" +version: "1.0.0" +usage: "Integrate Keybase.io tools with Helm" +description: This plugin provides signed cert to admission server. +ignoreFlags: false +useTunnel: false +command: "$HELM_PLUGIN_DIR/gen-admission-secret.sh" \ No newline at end of file diff --git a/installer/chart/volcano/templates/admission-config.yaml b/installer/chart/volcano/templates/admission-config.yaml new file mode 100644 index 0000000000..4181dbec8e --- /dev/null +++ b/installer/chart/volcano/templates/admission-config.yaml @@ -0,0 +1,52 @@ +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ .Release.Name }}-validate-job + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-delete +webhooks: + - clientConfig: + caBundle: "" + service: + name: {{ .Release.Name }}-admission-service + namespace: {{.Values.basic.namespace}} + path: /jobs + failurePolicy: Ignore + name: validatejob.volcano.sh + namespaceSelector: {} + rules: + - apiGroups: + - "batch.volcano.sh" + apiVersions: + - "v1alpha1" + operations: + - CREATE + - UPDATE + resources: + - jobs +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ .Release.Name }}-mutate-job + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-delete +webhooks: + - clientConfig: + caBundle: "" + service: + name: {{ .Release.Name }}-admission-service + namespace: {{.Values.basic.namespace}} + path: /mutating-jobs + failurePolicy: Ignore + name: mutatejob.volcano.sh + namespaceSelector: {} + rules: + - apiGroups: + - "batch.volcano.sh" + apiVersions: + - "v1alpha1" + operations: + - CREATE + resources: + - jobs diff --git a/installer/chart/volcano/templates/admission.yaml b/installer/chart/volcano/templates/admission.yaml new file mode 100644 index 0000000000..678b45db01 --- /dev/null +++ b/installer/chart/volcano/templates/admission.yaml @@ -0,0 +1,100 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-admission + namespace: {{.Values.basic.namespace}} + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-admission + namespace: {{.Values.basic.namespace}} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations"] + verbs: ["get", "list", "watch", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-admission-role + namespace: {{.Values.basic.namespace}} +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-admission + namespace: {{.Values.basic.namespace}} +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-admission + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: admission + admission: "true" + name: {{ .Release.Name }}-admission + namespace: {{.Values.basic.namespace}} +spec: + replicas: 1 + selector: + matchLabels: + app: admission + admission: "true" + template: + metadata: + labels: + app: admission + admission: "true" + spec: + serviceAccount: {{ .Release.Name }}-admission + containers: + - args: + - --tls-cert-file=/admission.local.config/certificates/tls.crt + - --tls-private-key-file=/admission.local.config/certificates/tls.key + - --ca-cert-file=/admission.local.config/certificates/ca.crt + - --mutate-webhook-config-name={{ .Release.Name }}-mutate-job + - --validate-webhook-config-name={{ .Release.Name }}-validate-job + - --alsologtostderr + - --port=443 + - -v=4 + - 2>&1 + image: {{.Values.basic.admission_image_name}}:{{.Values.basic.image_tag_version}} + imagePullPolicy: IfNotPresent + name: admission + volumeMounts: + - mountPath: /admission.local.config/certificates + name: admission-certs + readOnly: true + volumes: + - name: admission-certs + secret: + defaultMode: 420 + secretName: {{.Values.basic.admission_secret_name}} + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + test: admission + name: {{ .Release.Name }}-admission-service + namespace: {{.Values.basic.namespace}} +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 443 + selector: + admission: "true" + sessionAffinity: None diff --git a/installer/chart/volcano/templates/controllers.yaml b/installer/chart/volcano/templates/controllers.yaml new file mode 100644 index 0000000000..c2c39a54f5 --- /dev/null +++ b/installer/chart/volcano/templates/controllers.yaml @@ -0,0 +1,85 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-controllers + namespace: {{.Values.basic.namespace}} + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-controllers + namespace: {{.Values.basic.namespace}} +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch", "delete", "update"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "update", "delete"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs/status"] + verbs: ["update", "patch"] + - apiGroups: ["bus.volcano.sh"] + resources: ["commands"] + verbs: ["get", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["scheduling.incubator.k8s.io"] + resources: ["podgroups"] + verbs: ["get", "list", "watch", "create", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-controllers-role + namespace: {{.Values.basic.namespace}} +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-controllers + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-controllers + apiGroup: rbac.authorization.k8s.io + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-controllers + namespace: {{.Values.basic.namespace}} +spec: + replicas: 1 + selector: + matchLabels: + vk-controllers: test + template: + metadata: + labels: + vk-controllers: test + spec: + serviceAccount: {{ .Release.Name }}-controllers + containers: + - name: {{ .Release.Name }}-controllers + image: {{.Values.basic.controller_image_name}}:{{.Values.basic.image_tag_version}} + args: + - --alsologtostderr + - -v=4 + - 2>&1 + imagePullPolicy: "IfNotPresent" diff --git a/installer/chart/volcano/templates/scheduler.yaml b/installer/chart/volcano/templates/scheduler.yaml new file mode 100644 index 0000000000..60d41a6d20 --- /dev/null +++ b/installer/chart/volcano/templates/scheduler.yaml @@ -0,0 +1,95 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-scheduler + namespace: {{.Values.basic.namespace}} + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-scheduler + namespace: {{.Values.basic.namespace}} +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "update", "delete"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "update", "bind", "updateStatus", "delete"] + - apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["list", "watch"] + - apiGroups: ["scheduling.incubator.k8s.io"] + resources: ["podgroups"] + verbs: ["list", "watch", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["list", "watch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-scheduler-role + namespace: {{.Values.basic.namespace}} +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-scheduler + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-scheduler + apiGroup: rbac.authorization.k8s.io + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-scheduler + namespace: {{.Values.basic.namespace}} +spec: + replicas: 1 + selector: + matchLabels: + vk-scheduler: test + template: + metadata: + labels: + vk-scheduler: test + spec: + serviceAccount: {{ .Release.Name }}-scheduler + containers: + - name: {{ .Release.Name }}-scheduler + image: {{.Values.basic.scheduler_image_name}}:{{.Values.basic.image_tag_version}} + args: + - --alsologtostderr + - -v=2 + - 2>&1 + imagePullPolicy: "IfNotPresent" + diff --git a/installer/chart/volcano/values.yaml b/installer/chart/volcano/values.yaml new file mode 100644 index 0000000000..f7094bd109 --- /dev/null +++ b/installer/chart/volcano/values.yaml @@ -0,0 +1,7 @@ +basic: + image_tag_version: "1.0" + namespace: default + controller_image_name: "volcano-controllers" + scheduler_image_name: "volcano-scheduler" + admission_image_name: "volcano-admission" + admission_secret_name: "volcano-admission-secret" \ No newline at end of file diff --git a/installer/dockerfile/admission/Dockerfile b/installer/dockerfile/admission/Dockerfile new file mode 100644 index 0000000000..1827514ef0 --- /dev/null +++ b/installer/dockerfile/admission/Dockerfile @@ -0,0 +1,19 @@ +# Copyright 2019 The Volcano Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +FROM alpine:latest + +ADD vk-admission /vk-admission +ENTRYPOINT ["/vk-admission"] diff --git a/installer/dockerfile/controllers/Dockerfile b/installer/dockerfile/controllers/Dockerfile new file mode 100644 index 0000000000..d9c7b3eef7 --- /dev/null +++ b/installer/dockerfile/controllers/Dockerfile @@ -0,0 +1,19 @@ +# Copyright 2019 The Volcano Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +FROM alpine:latest + +ADD vk-controllers /vk-controllers +ENTRYPOINT ["/vk-controllers"] diff --git a/installer/dockerfile/scheduler/Dockerfile b/installer/dockerfile/scheduler/Dockerfile new file mode 100644 index 0000000000..8b8d6d640d --- /dev/null +++ b/installer/dockerfile/scheduler/Dockerfile @@ -0,0 +1,19 @@ +# Copyright 2019 The Volcano Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +FROM alpine:latest + +ADD vk-scheduler /vk-scheduler +ENTRYPOINT ["/vk-scheduler"]