update from master

.cloudbuild.yaml

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Test before submit:
+# gcloud builds submit --config=.cloudbuild.yaml --substitutions=COMMIT_SHA="$(git rev-parse HEAD)" --project=ml-pipeline-test
+
 steps:
 
 # Build the Python SDK
@@ -52,14 +55,16 @@ steps:
   waitFor: ["-"]
 - name: 'gcr.io/cloud-builders/docker'
   args: ['build', '-t', 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA',
-         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
-         '/workspace/frontend/Dockerfile', '/workspace']
+         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA',
+         '--build-arg', 'TAG_NAME=$TAG_NAME',
+         '-f', '/workspace/frontend/Dockerfile', '/workspace']
   id:   'buildFrontend'
   waitFor: ['prepareFrontend']
 - name: 'gcr.io/cloud-builders/docker'
   args: ['build', '-t', 'gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA',
-         '--build-arg', 'COMMIT_SHA=$COMMIT_SHA', '-f',
-         '/workspace/backend/Dockerfile', '/workspace']
+         '--build-arg', 'COMMIT_SHA=$COMMIT_SHA',
+         '--build-arg', 'TAG_NAME=$TAG_NAME',
+         '-f', '/workspace/backend/Dockerfile', '/workspace']
   id:   'buildApiServer'
   waitFor: ['copyPythonSDK']
 
@@ -88,6 +93,32 @@ steps:
          '/workspace/backend/Dockerfile.visualization', '/workspace']
   id: 'buildVisualizationServer'
   waitFor: ["-"]
+- id: 'buildMetadataWriter'
+  name: 'gcr.io/cloud-builders/docker'
+  args: ['build', '-t', 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA',
+         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
+         '/workspace/backend/metadata_writer/Dockerfile', '/workspace']
+  waitFor: ["-"]
+- id: 'buildCacheServer'
+  name: 'gcr.io/cloud-builders/docker'
+  args: ['build', '-t', 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA',
+          '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
+          '/workspace/backend/Dockerfile.cacheserver', '/workspace']
+  waitFor: ["-"]
+- id: 'buildCacheDeployer'
+  name: 'gcr.io/cloud-builders/docker'
+  args: ['build', '-t', 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA',
+          '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
+          '/workspace/backend/src/cache/deployer/Dockerfile', '/workspace']
+  waitFor: ["-"]
+
+# Build marketplace deployer
+- id: 'buildMarketplaceDeployer'
+  name: 'gcr.io/cloud-builders/docker'
+  args: ['build', '-t', 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA',
+         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
+         '/workspace/manifests/gcp_marketplace/deployer/Dockerfile', '/workspace/manifests/gcp_marketplace']
+  waitFor: ["-"]
 
 # Build the Kubeflow-based pipeline component images
 - name: 'gcr.io/cloud-builders/docker'
@@ -100,6 +131,11 @@ steps:
   args: ['-c', 'cd /workspace/components/kubeflow/launcher && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
   id: 'buildLauncher'
   waitFor: ["-"]
+- id: 'buildCpuTrainer'
+  name: 'gcr.io/cloud-builders/docker'
+  entrypoint: '/bin/bash'
+  args: ['-c', 'cd /workspace/components/kubeflow/dnntrainer && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA -l ml-pipeline-kubeflow-tf-trainer -b 1.6.0']
+  waitFor: ["-"]
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: '/bin/bash'
   args: ['-c', 'cd /workspace/components/kubeflow/dnntrainer && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA -l ml-pipeline-kubeflow-tf-trainer-gpu -b 1.6.0-gpu']
@@ -163,6 +199,51 @@ steps:
          '/workspace/third_party/metadata_envoy/Dockerfile', '/workspace']
   id:   'buildMetadataEnvoy'
 
+# Pull third_party images
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:0.21.1']
+  id:   'pullMetadataServer'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance']
+  id:   'pullMinio'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/mysql:5.6']
+  id:   'pullMysql'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.14']
+  id:   'pullCloudsqlProxy'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance']
+  id:   'pullArgoExecutor'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance']
+  id:   'pullArgoWorkflowController'
+
+# Tag for Hosted - SemVersion to Major.Minor parsing
+- id: "parseMajorMinorVersion"
+  waitFor: ["-"]
+  name: gcr.io/cloud-builders/docker
+  entrypoint: /bin/bash
+  args:
+    - -ceux
+    - |
+      # Parse major minor version and save to a file for reusing in other steps.
+      cat /workspace/VERSION | sed -e "s#[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)#\1.\2#" > /workspace/mm.ver
+
+# Tag for Hosted - Tag to hosted folder with MKP friendly name
+- id:   'tagForHosted'
+  waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildScheduledWorkflow',            
+            'buildViewerCrdController', 'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
+            'buildMetadataWriter', 'buildCacheServer', 'buildCacheDeployer', 'buildMetadataEnvoy',
+            'buildMarketplaceDeployer', 'pullMetadataServer', 'pullMinio', 'pullMysql', 'pullCloudsqlProxy',
+            'pullArgoExecutor', 'pullArgoWorkflowController']
+  name: 'gcr.io/cloud-builders/docker'
+  entrypoint: bash
+  args:
+  - -ceux
+  - |
+    /workspace/test/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)
+
 images:
 # Images for the pipeline system itself
 - 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA'
@@ -172,12 +253,18 @@ images:
 - 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA'
 - 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA'
 - 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA'
+- 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA'
+- 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA'
+- 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA'
+
+# Images for Marketplace
+- 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA'
 
 # Images for the Kubeflow-based pipeline components
 - 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA'
 - 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA'
 - 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer-gpu:$COMMIT_SHA'
-- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf:$COMMIT_SHA'
+- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tfjob:$COMMIT_SHA'
 
 # Images for the Dataproc-based pipeline components
 - 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-analyze:$COMMIT_SHA'

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -4,13 +4,33 @@ about: Report a bug encountered with Kubeflow Pipelines
 
 ---
 
-**What happened:**
+### What steps did you take:
+[A clear and concise description of what the bug is.]
 
-**What did you expect to happen:**
+### What happened:
+
+### What did you expect to happen:
+
+### Environment:
+<!-- Please fill in those that seem relevant. -->
+
+How did you deploy Kubeflow Pipelines (KFP)?
+<!-- If you are not sure, here's [an introduction of all options](https://www.kubeflow.org/docs/pipelines/installation/overview/). -->
+
+KFP version: <!-- If you are not sure, build commit shows on bottom of KFP UI left sidenav. -->
+
+KFP SDK version: <!-- Please attach the output of this shell command: $pip list | grep kfp -->
 
-**What steps did you take:**
-[A clear and concise description of what the bug is.]
 
-**Anything else you would like to add:**
+### Anything else you would like to add:
 [Miscellaneous information that will assist in solving the issue.]
 
+/kind bug
+<!-- Please include labels by uncommenting them to help us better triage issues, choose from the following -->
+<!--
+// /area frontend
+// /area backend
+// /area sdk
+// /area testing
+// /area engprod
+-->

.gitignore

@@ -14,7 +14,7 @@ bower_components/
 dist
 
 # Web server
-frontend/server/*.js
+frontend/server/dist
 
 # Python built package
 *.egg-info
@@ -39,13 +39,17 @@ frontend/test/ui/visual-regression/screenshots/screen
 
 *.pyc
 .DS_Store
+build
 
 .ipynb_checkpoints
 *.egg-info
 
 # go vendor
 vendor
 
+# Go module cache
+backend/pkg/mod/cache
+
 # Bazel output artifacts
 bazel-*
 
@@ -61,3 +65,4 @@ _artifacts
 
 # Generated Python SDK documentation
 docs/_build
+