update release script; fix post submit (#189)

* update release script; fix post submit

* Changing k8s cluster version

* Changing k8s version

* app.yaml changes

* Deploy the existing app

* Changing worker image

* debug version

* Moving to ks 0.13

* Removing specific commit from app.yaml

scripts/create-cluster.sh

@@ -33,7 +33,7 @@ echo "Creating GPU cluster"
 gcloud --project ${PROJECT} beta container clusters create ${CLUSTER_NAME} \
     --zone ${ZONE} \
     --accelerator type=nvidia-tesla-k80,count=1 \
-    --cluster-version 1.11
+    --cluster-version 1.12
 echo "Configuring kubectl"
 gcloud --project ${PROJECT} container clusters get-credentials ${CLUSTER_NAME} \
     --zone ${ZONE}

scripts/setup-kubeflow.sh

@@ -44,18 +44,18 @@ echo "Setting account ${ACCOUNT}"
 kubectl create clusterrolebinding default-admin --clusterrole=cluster-admin --user=${ACCOUNT}
 
 echo "Install ksonnet app in namespace ${NAMESPACE}"
-/usr/local/bin/ks init ${APP_NAME}
+/usr/local/bin/ks-13 init ${APP_NAME}
 cd ${APP_NAME}
-/usr/local/bin/ks env add ${KF_ENV}
-/usr/local/bin/ks env set ${KF_ENV} --namespace ${NAMESPACE}
-/usr/local/bin/ks registry add kubeflow github.com/kubeflow/kubeflow/tree/${KUBEFLOW_VERSION}/kubeflow
+/usr/local/bin/ks-13 env add ${KF_ENV}
+/usr/local/bin/ks-13 env set ${KF_ENV} --namespace ${NAMESPACE}
+/usr/local/bin/ks-13 registry add kubeflow github.com/kubeflow/kubeflow/tree/${KUBEFLOW_VERSION}/kubeflow
 
 echo "Install PyTorch ksonnet package"
-/usr/local/bin/ks pkg install kubeflow/pytorch-job@${KUBEFLOW_VERSION}
+/usr/local/bin/ks-13 pkg install kubeflow/pytorch-job@${KUBEFLOW_VERSION}
 
 echo "Install PyTorch operator"
-/usr/local/bin/ks generate pytorch-operator pytorch-operator --pytorchJobImage=${REGISTRY}/${REPO_NAME}:${VERSION}
-/usr/local/bin/ks apply ${KF_ENV} -c pytorch-operator
+/usr/local/bin/ks-13 generate pytorch-operator pytorch-operator --pytorchJobImage=${REGISTRY}/${REPO_NAME}:${VERSION}
+/usr/local/bin/ks-13 apply ${KF_ENV} -c pytorch-operator
 
 TIMEOUT=30
 until kubectl get pods -n ${NAMESPACE} | grep pytorch-operator | grep 1/1 || [[ $TIMEOUT -eq 1 ]]; do

submit_release_job.sh

@@ -17,20 +17,22 @@ BUILD_NUMBER=$(uuidgen)
 BUILD_NUMBER=${BUILD_NUMBER:0:4}
 REPO_OWNER=kubeflow
 REPO_NAME=pytorch-operator
-ENV=releasing
+ENV=test
 DATE=`date +%Y%m%d`
 PULL_BASE_SHA=${COMMIT:0:8}
 VERSION_TAG="v${DATE}-${PULL_BASE_SHA}"
-
+BUILD_NUMBER_LOWER=$(echo "$BUILD_NUMBER" | tr '[:upper:]' '[:lower:]')
 
 PROW_VAR="JOB_NAME=${JOB_NAME},JOB_TYPE=${JOB_TYPE},REPO_NAME=${REPO_NAME}"
 PROW_VAR="${PROW_VAR},REPO_OWNER=${REPO_OWNER},BUILD_NUMBER=${BUILD_NUMBER}" 
 PROW_VAR="${PROW_VAR},PULL_BASE_SHA=${PULL_BASE_SHA}" 
 
 cd ${ROOT}/test/workflows
 
-ks param set --env=${ENV} workflows namespace kubeflow-releasing
-ks param set --env=${ENV} workflows name "${USER}-${JOB_NAME}-${PULL_BASE_SHA}-${BUILD_NUMBER}"
+ks param set --env=${ENV} workflows namespace kubeflow-test-infra
+ks param set --env=${ENV} workflows name "${JOB_NAME}-${PULL_BASE_SHA}-${BUILD_NUMBER_LOWER}-${USER}"
 ks param set --env=${ENV} workflows prow_env "${PROW_VAR}"
 ks param set --env=${ENV} workflows versionTag "${VERSION_TAG}"
-ks apply ${ENV} -c workflows
+ks param set --env=${ENV} workflows registry gcr.io/kubeflow-images-public
+ks param set --env=${ENV} workflows bucket kubeflow-releasing-artifacts
+# ks apply ${ENV} -c workflows

test/workflows/app.yaml

@@ -1,4 +1,4 @@
-apiVersion: 0.1.0
+apiVersion: 0.3.0
 environments:
   releasing:
     destination:
@@ -16,9 +16,6 @@ kind: ksonnet.io/app
 name: worfklows-app
 registries:
   incubator:
-    gitVersion:
-      commitSha: 422d521c05aa905df949868143b26445f5e4eda5
-      refSpec: master
     protocol: github
     uri: github.com/ksonnet/parts/tree/master/incubator
 version: 0.0.1

test/workflows/components/workflows.libsonnet

@@ -54,7 +54,7 @@
       local srcRootDir = testDir + "/src";
       // The directory containing the kubeflow/pytorch-operator repo
       local srcDir = srcRootDir + "/kubeflow/pytorch-operator";
-      local testWorkerImage = "gcr.io/kubeflow-ci/test-worker";
+      local testWorkerImage = "gcr.io/kubeflow-ci/test-worker:v20190421-fba47fe-e3b0c4";
       local golangImage = "golang:1.9.4-stretch";
       // TODO(jose5918) Build our own helm image
       local helmImage = "volumecontroller/golang:1.9.2";
@@ -97,6 +97,7 @@
         // command: List to pass as the container command.
         buildTemplate(step_name, image, command):: {
           name: step_name,
+          activeDeadlineSeconds: 2100,
           container: {
             command: command,
             image: image,

test/workflows/environments/test/params.libsonnet

@@ -1,10 +1,18 @@
-local params = import "../../components/params.libsonnet";
+local params = import '../../components/params.libsonnet';
+
 params {
   components+: {
-    // Insert component parameter overrides here. Ex:
-    // guestbook +: {
-    //   name: "guestbook-dev",
-    //   replicas: params.global.replicas,
-    // },
+    workflows+: {
+      namespace: 'kubeflow-test-infra',
+      name: 'pytorch-operator-release-6aa39a41-6985-kunming',
+      prow_env: 'JOB_NAME=pytorch-operator-release,JOB_TYPE=pytorch-operator-release,REPO_NAME=pytorch-operator,REPO_OWNER=kubeflow,BUILD_NUMBER=6985,PULL_BASE_SHA=6aa39a41',
+      versionTag: 'v20190703-6aa39a41',
+      registry: 'gcr.io/kubeflow-images-public',
+      bucket: 'kubeflow-releasing-artifacts',
+    },
+    "workflows-v1alpha2"+: {
+      registry: 'gcr.io/kubeflow-images-public',
+      bucket: 'kubeflow-releasing-artifacts',
+    },
   },
-}
+}