From 466d441446405f09502065056019d9e995820227 Mon Sep 17 00:00:00 2001 From: kunming Date: Mon, 8 Jul 2019 11:50:01 -0700 Subject: [PATCH 1/9] update release script; fix post submit --- submit_release_job.sh | 12 +++++----- test/workflows/app.yaml | 7 ++---- test/workflows/components/workflows.libsonnet | 1 + .../environments/test/params.libsonnet | 22 +++++++++++++------ 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/submit_release_job.sh b/submit_release_job.sh index 6dc155d2e..0c7bce78b 100644 --- a/submit_release_job.sh +++ b/submit_release_job.sh @@ -17,11 +17,11 @@ BUILD_NUMBER=$(uuidgen) BUILD_NUMBER=${BUILD_NUMBER:0:4} REPO_OWNER=kubeflow REPO_NAME=pytorch-operator -ENV=releasing +ENV=test DATE=`date +%Y%m%d` PULL_BASE_SHA=${COMMIT:0:8} VERSION_TAG="v${DATE}-${PULL_BASE_SHA}" - +BUILD_NUMBER_LOWER=$(echo "$BUILD_NUMBER" | tr '[:upper:]' '[:lower:]') PROW_VAR="JOB_NAME=${JOB_NAME},JOB_TYPE=${JOB_TYPE},REPO_NAME=${REPO_NAME}" PROW_VAR="${PROW_VAR},REPO_OWNER=${REPO_OWNER},BUILD_NUMBER=${BUILD_NUMBER}" @@ -29,8 +29,10 @@ PROW_VAR="${PROW_VAR},PULL_BASE_SHA=${PULL_BASE_SHA}" cd ${ROOT}/test/workflows -ks param set --env=${ENV} workflows namespace kubeflow-releasing -ks param set --env=${ENV} workflows name "${USER}-${JOB_NAME}-${PULL_BASE_SHA}-${BUILD_NUMBER}" +ks param set --env=${ENV} workflows namespace kubeflow-test-infra +ks param set --env=${ENV} workflows name "${JOB_NAME}-${PULL_BASE_SHA}-${BUILD_NUMBER_LOWER}-${USER}" ks param set --env=${ENV} workflows prow_env "${PROW_VAR}" ks param set --env=${ENV} workflows versionTag "${VERSION_TAG}" -ks apply ${ENV} -c workflows +ks param set --env=${ENV} workflows registry gcr.io/kubeflow-images-public +ks param set --env=${ENV} workflows bucket kubeflow-releasing-artifacts +# ks apply ${ENV} -c workflows diff --git a/test/workflows/app.yaml b/test/workflows/app.yaml index 9b55ab221..8ea4684db 100644 --- a/test/workflows/app.yaml +++ b/test/workflows/app.yaml @@ -1,4 +1,4 @@ -apiVersion: 0.1.0 +apiVersion: 0.3.0 environments: releasing: destination: @@ -9,16 +9,13 @@ environments: test: destination: namespace: kubeflow-test-infra - server: https://35.196.185.88 + server: https://35.196.213.148 k8sVersion: v1.7.0 path: test kind: ksonnet.io/app name: worfklows-app registries: incubator: - gitVersion: - commitSha: 422d521c05aa905df949868143b26445f5e4eda5 - refSpec: master protocol: github uri: github.com/ksonnet/parts/tree/master/incubator version: 0.0.1 diff --git a/test/workflows/components/workflows.libsonnet b/test/workflows/components/workflows.libsonnet index b4ea5bd1d..77b25bf05 100644 --- a/test/workflows/components/workflows.libsonnet +++ b/test/workflows/components/workflows.libsonnet @@ -97,6 +97,7 @@ // command: List to pass as the container command. buildTemplate(step_name, image, command):: { name: step_name, + activeDeadlineSeconds: 2100, container: { command: command, image: image, diff --git a/test/workflows/environments/test/params.libsonnet b/test/workflows/environments/test/params.libsonnet index d195c2a71..6565aa8a9 100644 --- a/test/workflows/environments/test/params.libsonnet +++ b/test/workflows/environments/test/params.libsonnet @@ -1,10 +1,18 @@ -local params = import "../../components/params.libsonnet"; +local params = import '../../components/params.libsonnet'; + params { components+: { - // Insert component parameter overrides here. Ex: - // guestbook +: { - // name: "guestbook-dev", - // replicas: params.global.replicas, - // }, + workflows+: { + namespace: 'kubeflow-test-infra', + name: 'pytorch-operator-release-6aa39a41-6985-kunming', + prow_env: 'JOB_NAME=pytorch-operator-release,JOB_TYPE=pytorch-operator-release,REPO_NAME=pytorch-operator,REPO_OWNER=kubeflow,BUILD_NUMBER=6985,PULL_BASE_SHA=6aa39a41', + versionTag: 'v20190703-6aa39a41', + registry: 'gcr.io/kubeflow-images-public', + bucket: 'kubeflow-releasing-artifacts', + }, + "workflows-v1alpha2"+: { + registry: 'gcr.io/kubeflow-images-public', + bucket: 'kubeflow-releasing-artifacts', + }, }, -} +} \ No newline at end of file From 1255e9daa11606a197ed44cee049f8d9b0750382 Mon Sep 17 00:00:00 2001 From: Johnu George Date: Tue, 16 Jul 2019 09:19:03 +0530 Subject: [PATCH 2/9] Changing k8s cluster version --- scripts/create-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create-cluster.sh b/scripts/create-cluster.sh index cf5b3a143..6826c6efc 100755 --- a/scripts/create-cluster.sh +++ b/scripts/create-cluster.sh @@ -33,7 +33,7 @@ echo "Creating GPU cluster" gcloud --project ${PROJECT} beta container clusters create ${CLUSTER_NAME} \ --zone ${ZONE} \ --accelerator type=nvidia-tesla-k80,count=1 \ - --cluster-version 1.11 + --cluster-version 1.12 echo "Configuring kubectl" gcloud --project ${PROJECT} container clusters get-credentials ${CLUSTER_NAME} \ --zone ${ZONE} From 2438a64798f28d899055eebe6a69af70805dc310 Mon Sep 17 00:00:00 2001 From: Johnu George Date: Tue, 16 Jul 2019 09:43:28 +0530 Subject: [PATCH 3/9] Changing k8s version --- test/workflows/app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/workflows/app.yaml b/test/workflows/app.yaml index 8ea4684db..6bf4f28e9 100644 --- a/test/workflows/app.yaml +++ b/test/workflows/app.yaml @@ -10,7 +10,7 @@ environments: destination: namespace: kubeflow-test-infra server: https://35.196.213.148 - k8sVersion: v1.7.0 + k8sVersion: v1.11.10 path: test kind: ksonnet.io/app name: worfklows-app From 836a2ce6baaded33826111b66714cddbdf0090d2 Mon Sep 17 00:00:00 2001 From: Johnu George Date: Tue, 16 Jul 2019 10:11:47 +0530 Subject: [PATCH 4/9] app.yaml changes --- test/workflows/app.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/workflows/app.yaml b/test/workflows/app.yaml index 6bf4f28e9..3e22c9111 100644 --- a/test/workflows/app.yaml +++ b/test/workflows/app.yaml @@ -9,13 +9,16 @@ environments: test: destination: namespace: kubeflow-test-infra - server: https://35.196.213.148 - k8sVersion: v1.11.10 + server: https://35.196.185.88 + k8sVersion: v1.7.0 path: test kind: ksonnet.io/app name: worfklows-app registries: incubator: + gitVersion: + commitSha: 422d521c05aa905df949868143b26445f5e4eda5 + refSpec: master protocol: github uri: github.com/ksonnet/parts/tree/master/incubator version: 0.0.1 From 7d814c323a4a322dfa35ec35d88c34b5bf890d5d Mon Sep 17 00:00:00 2001 From: Johnu George Date: Tue, 16 Jul 2019 11:30:35 +0530 Subject: [PATCH 5/9] Deploy the existing app --- scripts/setup-kubeflow.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/setup-kubeflow.sh b/scripts/setup-kubeflow.sh index 6276df6e3..8319abb89 100755 --- a/scripts/setup-kubeflow.sh +++ b/scripts/setup-kubeflow.sh @@ -31,7 +31,7 @@ VERSION=$(git describe --tags --always --dirty) GO_DIR=${GOPATH}/src/github.com/${REPO_OWNER}/${REPO_NAME} APP_NAME=test-app KUBEFLOW_VERSION=master -KF_ENV=pytorch +KF_ENV=test echo "Activating service-account" gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} @@ -44,9 +44,10 @@ echo "Setting account ${ACCOUNT}" kubectl create clusterrolebinding default-admin --clusterrole=cluster-admin --user=${ACCOUNT} echo "Install ksonnet app in namespace ${NAMESPACE}" -/usr/local/bin/ks init ${APP_NAME} -cd ${APP_NAME} -/usr/local/bin/ks env add ${KF_ENV} +#/usr/local/bin/ks init ${APP_NAME} +#cd ${APP_NAME} +#/usr/local/bin/ks env add ${KF_ENV} +cd ${GO_DIR}/test/workflows /usr/local/bin/ks env set ${KF_ENV} --namespace ${NAMESPACE} /usr/local/bin/ks registry add kubeflow github.com/kubeflow/kubeflow/tree/${KUBEFLOW_VERSION}/kubeflow @@ -55,6 +56,7 @@ echo "Install PyTorch ksonnet package" echo "Install PyTorch operator" /usr/local/bin/ks generate pytorch-operator pytorch-operator --pytorchJobImage=${REGISTRY}/${REPO_NAME}:${VERSION} +/usr/local/bin/ks show ${KF_ENV} -c pytorch-operator /usr/local/bin/ks apply ${KF_ENV} -c pytorch-operator TIMEOUT=30 From ab3a52397bd4696722076d772055933906837dd8 Mon Sep 17 00:00:00 2001 From: Johnu George Date: Tue, 16 Jul 2019 11:54:00 +0530 Subject: [PATCH 6/9] Changing worker image --- test/workflows/components/workflows.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/workflows/components/workflows.libsonnet b/test/workflows/components/workflows.libsonnet index 77b25bf05..84937c18b 100644 --- a/test/workflows/components/workflows.libsonnet +++ b/test/workflows/components/workflows.libsonnet @@ -54,7 +54,7 @@ local srcRootDir = testDir + "/src"; // The directory containing the kubeflow/pytorch-operator repo local srcDir = srcRootDir + "/kubeflow/pytorch-operator"; - local testWorkerImage = "gcr.io/kubeflow-ci/test-worker"; + local testWorkerImage = "gcr.io/kubeflow-ci/test-worker:v20190421-fba47fe-e3b0c4"; local golangImage = "golang:1.9.4-stretch"; // TODO(jose5918) Build our own helm image local helmImage = "volumecontroller/golang:1.9.2"; From aac604897514ed845289fafc44a32ab7064e7f3a Mon Sep 17 00:00:00 2001 From: Johnu George Date: Tue, 16 Jul 2019 12:27:22 +0530 Subject: [PATCH 7/9] debug version --- scripts/setup-kubeflow.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setup-kubeflow.sh b/scripts/setup-kubeflow.sh index 8319abb89..bf0adb26a 100755 --- a/scripts/setup-kubeflow.sh +++ b/scripts/setup-kubeflow.sh @@ -47,6 +47,7 @@ echo "Install ksonnet app in namespace ${NAMESPACE}" #/usr/local/bin/ks init ${APP_NAME} #cd ${APP_NAME} #/usr/local/bin/ks env add ${KF_ENV} +/usr/local/bin/ks version cd ${GO_DIR}/test/workflows /usr/local/bin/ks env set ${KF_ENV} --namespace ${NAMESPACE} /usr/local/bin/ks registry add kubeflow github.com/kubeflow/kubeflow/tree/${KUBEFLOW_VERSION}/kubeflow From fa4288c9e368f585f521fdf006f83810c72c1dd8 Mon Sep 17 00:00:00 2001 From: Johnu George Date: Tue, 16 Jul 2019 13:16:01 +0530 Subject: [PATCH 8/9] Moving to ks 0.13 --- scripts/setup-kubeflow.sh | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/scripts/setup-kubeflow.sh b/scripts/setup-kubeflow.sh index bf0adb26a..0af6d62d4 100755 --- a/scripts/setup-kubeflow.sh +++ b/scripts/setup-kubeflow.sh @@ -31,7 +31,7 @@ VERSION=$(git describe --tags --always --dirty) GO_DIR=${GOPATH}/src/github.com/${REPO_OWNER}/${REPO_NAME} APP_NAME=test-app KUBEFLOW_VERSION=master -KF_ENV=test +KF_ENV=pytorch echo "Activating service-account" gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} @@ -44,21 +44,18 @@ echo "Setting account ${ACCOUNT}" kubectl create clusterrolebinding default-admin --clusterrole=cluster-admin --user=${ACCOUNT} echo "Install ksonnet app in namespace ${NAMESPACE}" -#/usr/local/bin/ks init ${APP_NAME} -#cd ${APP_NAME} -#/usr/local/bin/ks env add ${KF_ENV} -/usr/local/bin/ks version -cd ${GO_DIR}/test/workflows -/usr/local/bin/ks env set ${KF_ENV} --namespace ${NAMESPACE} -/usr/local/bin/ks registry add kubeflow github.com/kubeflow/kubeflow/tree/${KUBEFLOW_VERSION}/kubeflow +/usr/local/bin/ks-13 init ${APP_NAME} +cd ${APP_NAME} +/usr/local/bin/ks-13 env add ${KF_ENV} +/usr/local/bin/ks-13 env set ${KF_ENV} --namespace ${NAMESPACE} +/usr/local/bin/ks-13 registry add kubeflow github.com/kubeflow/kubeflow/tree/${KUBEFLOW_VERSION}/kubeflow echo "Install PyTorch ksonnet package" -/usr/local/bin/ks pkg install kubeflow/pytorch-job@${KUBEFLOW_VERSION} +/usr/local/bin/ks-13 pkg install kubeflow/pytorch-job@${KUBEFLOW_VERSION} echo "Install PyTorch operator" -/usr/local/bin/ks generate pytorch-operator pytorch-operator --pytorchJobImage=${REGISTRY}/${REPO_NAME}:${VERSION} -/usr/local/bin/ks show ${KF_ENV} -c pytorch-operator -/usr/local/bin/ks apply ${KF_ENV} -c pytorch-operator +/usr/local/bin/ks-13 generate pytorch-operator pytorch-operator --pytorchJobImage=${REGISTRY}/${REPO_NAME}:${VERSION} +/usr/local/bin/ks-13 apply ${KF_ENV} -c pytorch-operator TIMEOUT=30 until kubectl get pods -n ${NAMESPACE} | grep pytorch-operator | grep 1/1 || [[ $TIMEOUT -eq 1 ]]; do From 157959bdaf20044c5444d6be567e8e33c5ec73bf Mon Sep 17 00:00:00 2001 From: Johnu George Date: Tue, 16 Jul 2019 13:46:11 +0530 Subject: [PATCH 9/9] Removing specific commit from app.yaml --- test/workflows/app.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/workflows/app.yaml b/test/workflows/app.yaml index 3e22c9111..490988c15 100644 --- a/test/workflows/app.yaml +++ b/test/workflows/app.yaml @@ -16,9 +16,6 @@ kind: ksonnet.io/app name: worfklows-app registries: incubator: - gitVersion: - commitSha: 422d521c05aa905df949868143b26445f5e4eda5 - refSpec: master protocol: github uri: github.com/ksonnet/parts/tree/master/incubator version: 0.0.1