diff --git a/frontend/server/k8s-helper.ts b/frontend/server/k8s-helper.ts index 1eb8ff89031..a3bd2dffaf2 100644 --- a/frontend/server/k8s-helper.ts +++ b/frontend/server/k8s-helper.ts @@ -44,33 +44,7 @@ const workflowPlural = 'workflows'; /** Default pod template spec used to create tensorboard viewer. */ export const defaultPodTemplateSpec = { spec: { - containers: [ - { - env: [ - { - name: 'GOOGLE_APPLICATION_CREDENTIALS', - value: '/secret/gcp-credentials/user-gcp-sa.json', - }, - ], - volumeMounts: [ - { - name: 'gcp-credentials', - mountPath: '/secret/gcp-credentials/user-gcp-sa.json', - readOnly: true, - }, - ], - }, - ], - volumes: [ - { - name: 'gcp-credentials', - volumeSource: { - secret: { - secretName: 'user-gcp-sa', - }, - }, - }, - ], + containers: [{}], }, }; diff --git a/manifests/kustomize/gcp-workload-identity-setup.sh b/manifests/kustomize/gcp-workload-identity-setup.sh new file mode 100755 index 00000000000..8f034a7e32b --- /dev/null +++ b/manifests/kustomize/gcp-workload-identity-setup.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Google service Account (GSA) +SYSTEM_GSA=${SYSTEM_GSA:-$CLUSTER_NAME-kfp-system} +USER_GSA=${USER_GSA:-$CLUSTER_NAME-kfp-user} + +# Kubernetes Service Account (KSA) +SYSTEM_KSA=(ml-pipeline-ui) +USER_KSA=(pipeline-runner default) # default service account is used for container building, TODO: give it a specific name + +cat < CLUSTER_NAME= NAMESPACE= ./gcp-workload-identity-setup.sh +``` + +PROJECT_ID: GCP project ID your cluster belongs to. +CLUSTER_NAME: your GKE cluster's name. +NAMESPACE: Kubernetes namespace your Kubeflow Pipelines standalone deployment belongs to (default is kubeflow). +EOF +} +if [ -z "$PROJECT_ID" ]; then + usage + echo + echo "Error: PROJECT_ID env variable is empty!" + exit 1 +fi +if [ -z "$CLUSTER_NAME" ]; then + usage + echo + echo "Error: CLUSTER_NAME env variable is empty!" + exit 1 +fi +echo "Env variables set:" +echo "* PROJECT_ID=$PROJECT_ID" +echo "* CLUSTER_NAME=$CLUSTER_NAME" +echo "* NAMESPACE=$NAMESPACE" +echo + +read -p "Continue? (Y/n) " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 0 +fi + +echo "Creating Google service accounts..." +function create_gsa_if_not_present { + local name=${1} + local already_present=$(gcloud iam service-accounts list --filter='name:'$name'' --format='value(name)') + if [ -n "$already_present" ]; then + echo "Service account $name already exists" + else + gcloud iam service-accounts create $name + fi +} +create_gsa_if_not_present $SYSTEM_GSA +create_gsa_if_not_present $USER_GSA + +# You can optionally choose to add iam policy bindings to grant project permissions to these GSAs. +# You can also set these up later. +# gcloud projects add-iam-policy-binding $PROJECT_ID \ +# --member="serviceAccount:$SYSTEM_GSA@$PROJECT_ID.iam.gserviceaccount.com" \ +# --role="roles/editor" +# gcloud projects add-iam-policy-binding $PROJECT_ID \ +# --member="serviceAccount:$USER_GSA@$PROJECT_ID.iam.gserviceaccount.com" \ +# --role="roles/editor" + +# Bind KSA to GSA through workload identity. +# Documentation: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity +function bind_gsa_and_ksa { + local gsa=${1} + local ksa=${2} + + gcloud iam service-accounts add-iam-policy-binding $gsa@$PROJECT_ID.iam.gserviceaccount.com \ + --member="serviceAccount:$PROJECT_ID.svc.id.goog[$NAMESPACE/$ksa]" \ + --role="roles/iam.workloadIdentityUser" \ + > /dev/null # hide verbose output + kubectl annotate serviceaccount \ + --namespace $NAMESPACE \ + --overwrite \ + $ksa \ + iam.gke.io/gcp-service-account=$gsa@$PROJECT_ID.iam.gserviceaccount.com + echo "* Bound KSA $ksa to GSA $gsa" +} + +echo "Binding each kfp system KSA to $SYSTEM_GSA" +for ksa in ${SYSTEM_KSA[@]}; do + bind_gsa_and_ksa $SYSTEM_GSA $ksa +done + +echo "Binding each kfp user KSA to $USER_GSA" +for ksa in ${USER_KSA[@]}; do + bind_gsa_and_ksa $USER_GSA $ksa +done diff --git a/manifests/kustomize/wi-utils.sh b/manifests/kustomize/wi-utils.sh new file mode 100644 index 00000000000..f5a06db927d --- /dev/null +++ b/manifests/kustomize/wi-utils.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function create_gsa_if_not_present { + local name=${1} + local already_present=$(gcloud iam service-accounts list --filter='name:'$name'' --format='value(name)') + if [ -n "$already_present" ]; then + echo "Service account $name already exists" + else + gcloud iam service-accounts create $name + fi +} + +# Bind KSA to GSA through workload identity. +# Documentation: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity +function bind_gsa_and_ksa { + local gsa=${1} + local ksa=${2} + local project=${3:-$PROJECT_ID} + local gsa_full="$gsa@$project.iam.gserviceaccount.com" + local namespace=${4:-$NAMESPACE} + + gcloud iam service-accounts add-iam-policy-binding $gsa_full \ + --member="serviceAccount:$project.svc.id.goog[$namespace/$ksa]" \ + --role="roles/iam.workloadIdentityUser" \ + > /dev/null # hide verbose output + kubectl annotate serviceaccount \ + --namespace $namespace \ + --overwrite \ + $ksa \ + iam.gke.io/gcp-service-account=$gsa_full + echo "* Bound KSA $ksa in namespace $namespace to GSA $gsa_full" +} + +# This can be used to programmatically verify workload identity binding grants corresponding GSA +# permissions successfully. +# Usage: verify_workload_identity_binding $KSA $NAMESPACE +# +# If you want to verify manually, use the following command instead: +# kubectl run test-$RANDOM --rm -it --restart=Never \ +# --image=google/cloud-sdk:slim \ +# --serviceaccount $ksa \ +# --namespace $namespace \ +# -- /bin/bash +# It connects you to a pod using specified KSA running an image with gcloud and gsutil CLI tools. +function verify_workload_identity_binding { + local ksa=${1} + local namespace=${2} + local max_attempts=10 + local workload_identity_is_ready=false + for i in $(seq 1 ${max_attempts}) + do + workload_identity_is_ready=true + kubectl run test-$RANDOM --rm -i --restart=Never \ + --image=google/cloud-sdk:slim \ + --serviceaccount $ksa \ + --namespace $namespace \ + -- gcloud auth list || workload_identity_is_ready=false + kubectl run test-$RANDOM --rm -i --restart=Never \ + --image=google/cloud-sdk:slim \ + --serviceaccount $ksa \ + --namespace $namespace \ + -- gsutil ls gs:// || workload_identity_is_ready=false + if [ "$workload_identity_is_ready" = true ]; then + break + fi + done + if [ ! "$workload_identity_is_ready" = true ]; then + echo "Workload identity bindings are not ready after $max_attempts attempts" + return 1 + fi +} diff --git a/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.yaml b/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.yaml new file mode 100644 index 00000000000..b2aa75b01c4 --- /dev/null +++ b/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.yaml @@ -0,0 +1,974 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + annotations: + pipelines.kubeflow.org/pipeline_spec: '{"description": "Constructs a Kubeflow + pipeline.\n\n Creates Kubeflow ContainerOps for each TFX component encountered + in the\n logical pipeline definition.\n ", "inputs": [{"default": + "gs://ml-pipeline-playground/tfx_taxi_simple/data", "name": "data-root"}, {"default": + "gs://ml-pipeline-playground/tfx_taxi_simple/modules/taxi_utils.py", "name": + "module-file"}, {"default": "gs://your-bucket/tfx_taxi_simple/{{workflow.uid}}", + "name": "pipeline-root"}], "name": "parameterized_tfx_oss"}' + generateName: parameterized-tfx-oss- +spec: + arguments: + parameters: + - name: data-root + value: gs://ml-pipeline-playground/tfx_taxi_simple/data + - name: module-file + value: gs://ml-pipeline-playground/tfx_taxi_simple/modules/taxi_utils.py + - name: pipeline-root + value: gs://your-bucket/tfx_taxi_simple/{{workflow.uid}} + entrypoint: parameterized-tfx-oss + serviceAccountName: pipeline-runner + templates: + - container: + args: + - --pipeline_name + - parameterized_tfx_oss + - --pipeline_root + - '{{inputs.parameters.pipeline-root}}' + - --kubeflow_metadata_config + - "{\n \"mysqlDbServiceHost\": {\n \"environmentVariable\": \"mysql_host\"\ + \n },\n \"mysqlDbServicePort\": {\n \"environmentVariable\": \"mysql_port\"\ + \n },\n \"mysqlDbName\": {\n \"environmentVariable\": \"mysql_database\"\ + \n },\n \"mysqlDbUser\": {\n \"environmentVariable\": \"username\"\n\ + \ },\n \"mysqlDbPassword\": {\n \"environmentVariable\": \"password\"\ + \n }\n}" + - --beam_pipeline_args + - '[]' + - --additional_pipeline_args + - '{}' + - --component_launcher_class_path + - tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - --serialized_component + - '{"__class__": "CsvExampleGen", "__module__": "tfx.components.example_gen.csv_example_gen.component", + "__tfx_object_type__": "jsonable", "_instance_name": null, "driver_class": + {"__class__": "Driver", "__module__": "tfx.components.example_gen.driver", + "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", + "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": + "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.example_gen.csv_example_gen.executor", + "__tfx_object_type__": "class"}}, "spec": {"__class__": "FileBasedExampleGenSpec", + "__module__": "tfx.types.standard_component_specs", "__tfx_object_type__": + "jsonable", "exec_properties": {"custom_config": null, "input_config": "{\n \"splits\": + [\n {\n \"name\": \"single_split\",\n \"pattern\": \"*\"\n }\n ]\n}", + "output_config": "{\n \"splitConfig\": {\n \"splits\": [\n {\n \"hashBuckets\": + 2,\n \"name\": \"train\"\n },\n {\n \"hashBuckets\": + 1,\n \"name\": \"eval\"\n }\n ]\n }\n}"}, "inputs": {"__class__": + "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": + "jsonable", "_compat_aliases": {"input": "input_base"}, "_data": {"input_base": + {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": + "jsonable", "_artifacts": [{"__class__": "ExternalArtifact", "__module__": + "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": + {"properties": {"split": {"stringValue": ""}, "type_name": {"stringValue": + "ExternalPath"}}, "uri": "{{inputs.parameters.data-root}}"}, "artifact_type": + {"name": "ExternalPath", "properties": {"name": "STRING", "pipeline_name": + "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", + "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExternalPath"}}}, + "outputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", + "__tfx_object_type__": "jsonable", "_compat_aliases": {}, "_data": {"examples": + {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": + "jsonable", "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "train"}, "type_name": + {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}, {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "eval"}, "type_name": + {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}], "type_name": "ExamplesPath"}}}}}' + - --component_config + - 'null' + - --enable_cache + command: + - python + - /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + env: + - name: WORKFLOW_ID + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + envFrom: + - configMapRef: + name: metadata-configmap + optional: true + - secretRef: + name: mysql-credential + optional: true + image: tensorflow/tfx:0.16.0.dev20191101 + inputs: + parameters: + - name: data-root + - name: pipeline-root + name: csvexamplegen + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + - container: + args: + - --pipeline_name + - parameterized_tfx_oss + - --pipeline_root + - '{{inputs.parameters.pipeline-root}}' + - --kubeflow_metadata_config + - "{\n \"mysqlDbServiceHost\": {\n \"environmentVariable\": \"mysql_host\"\ + \n },\n \"mysqlDbServicePort\": {\n \"environmentVariable\": \"mysql_port\"\ + \n },\n \"mysqlDbName\": {\n \"environmentVariable\": \"mysql_database\"\ + \n },\n \"mysqlDbUser\": {\n \"environmentVariable\": \"username\"\n\ + \ },\n \"mysqlDbPassword\": {\n \"environmentVariable\": \"password\"\ + \n }\n}" + - --beam_pipeline_args + - '[]' + - --additional_pipeline_args + - '{}' + - --component_launcher_class_path + - tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - --serialized_component + - '{"__class__": "Evaluator", "__module__": "tfx.components.evaluator.component", + "__tfx_object_type__": "jsonable", "_instance_name": null, "driver_class": + {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", + "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", + "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": + "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.evaluator.executor", + "__tfx_object_type__": "class"}}, "spec": {"__class__": "EvaluatorSpec", "__module__": + "tfx.types.standard_component_specs", "__tfx_object_type__": "jsonable", "exec_properties": + {"fairness_indicator_thresholds": null, "feature_slicing_spec": "{\n \"specs\": + [\n {\n \"columnForSlicing\": [\n \"trip_start_hour\"\n ]\n }\n ]\n}"}, + "inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", + "__tfx_object_type__": "jsonable", "_compat_aliases": {"model": "model_exports"}, + "_data": {"examples": {"__class__": "Channel", "__module__": "tfx.types.channel", + "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Examples", + "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", + "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": + {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": + "CsvExampleGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": + "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, + {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": + "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, + "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "eval"}, "type_name": + {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}], "type_name": "ExamplesPath"}, "model_exports": {"__class__": + "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", + "_artifacts": [{"__class__": "Model", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "Trainer"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "ModelExportPath"}}}, "artifact_type": {"name": "ModelExportPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], + "type_name": "ModelExportPath"}}}, "outputs": {"__class__": "_PropertyDictWrapper", + "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", + "_compat_aliases": {"evaluation": "output"}, "_data": {"output": {"__class__": + "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", + "_artifacts": [{"__class__": "ModelEvaluation", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "Evaluator"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "ModelEvalPath"}}}, "artifact_type": {"name": "ModelEvalPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], + "type_name": "ModelEvalPath"}}}}}' + - --component_config + - 'null' + - --enable_cache + command: + - python + - /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + env: + - name: WORKFLOW_ID + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + envFrom: + - configMapRef: + name: metadata-configmap + optional: true + - secretRef: + name: mysql-credential + optional: true + image: tensorflow/tfx:0.16.0.dev20191101 + inputs: + parameters: + - name: pipeline-root + name: evaluator + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + - container: + args: + - --pipeline_name + - parameterized_tfx_oss + - --pipeline_root + - '{{inputs.parameters.pipeline-root}}' + - --kubeflow_metadata_config + - "{\n \"mysqlDbServiceHost\": {\n \"environmentVariable\": \"mysql_host\"\ + \n },\n \"mysqlDbServicePort\": {\n \"environmentVariable\": \"mysql_port\"\ + \n },\n \"mysqlDbName\": {\n \"environmentVariable\": \"mysql_database\"\ + \n },\n \"mysqlDbUser\": {\n \"environmentVariable\": \"username\"\n\ + \ },\n \"mysqlDbPassword\": {\n \"environmentVariable\": \"password\"\ + \n }\n}" + - --beam_pipeline_args + - '[]' + - --additional_pipeline_args + - '{}' + - --component_launcher_class_path + - tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - --serialized_component + - '{"__class__": "ExampleValidator", "__module__": "tfx.components.example_validator.component", + "__tfx_object_type__": "jsonable", "_instance_name": null, "driver_class": + {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", + "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", + "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": + "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.example_validator.executor", + "__tfx_object_type__": "class"}}, "spec": {"__class__": "ExampleValidatorSpec", + "__module__": "tfx.types.standard_component_specs", "__tfx_object_type__": + "jsonable", "exec_properties": {}, "inputs": {"__class__": "_PropertyDictWrapper", + "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", + "_compat_aliases": {"statistics": "stats"}, "_data": {"schema": {"__class__": + "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", + "_artifacts": [{"__class__": "Schema", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "SchemaGen"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "SchemaPath"}}}, "artifact_type": {"name": "SchemaPath", "properties": {"name": + "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": + "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": + "SchemaPath"}, "stats": {"__class__": "Channel", "__module__": "tfx.types.channel", + "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "ExampleStatistics", + "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", + "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": + {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": + "StatisticsGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": + "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}, {"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "StatisticsGen"}, "split": {"stringValue": "eval"}, "type_name": + {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}], "type_name": "ExampleStatisticsPath"}}}, "outputs": {"__class__": + "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": + "jsonable", "_compat_aliases": {"anomalies": "output"}, "_data": {"output": + {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": + "jsonable", "_artifacts": [{"__class__": "ExampleAnomalies", "__module__": + "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": + {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": + "parameterized_tfx_oss"}, "producer_component": {"stringValue": "ExampleValidator"}, + "split": {"stringValue": ""}, "type_name": {"stringValue": "ExampleValidationPath"}}}, + "artifact_type": {"name": "ExampleValidationPath", "properties": {"name": + "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": + "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": + "ExampleValidationPath"}}}}}' + - --component_config + - 'null' + - --enable_cache + command: + - python + - /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + env: + - name: WORKFLOW_ID + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + envFrom: + - configMapRef: + name: metadata-configmap + optional: true + - secretRef: + name: mysql-credential + optional: true + image: tensorflow/tfx:0.16.0.dev20191101 + inputs: + parameters: + - name: pipeline-root + name: examplevalidator + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + - container: + args: + - --pipeline_name + - parameterized_tfx_oss + - --pipeline_root + - '{{inputs.parameters.pipeline-root}}' + - --kubeflow_metadata_config + - "{\n \"mysqlDbServiceHost\": {\n \"environmentVariable\": \"mysql_host\"\ + \n },\n \"mysqlDbServicePort\": {\n \"environmentVariable\": \"mysql_port\"\ + \n },\n \"mysqlDbName\": {\n \"environmentVariable\": \"mysql_database\"\ + \n },\n \"mysqlDbUser\": {\n \"environmentVariable\": \"username\"\n\ + \ },\n \"mysqlDbPassword\": {\n \"environmentVariable\": \"password\"\ + \n }\n}" + - --beam_pipeline_args + - '[]' + - --additional_pipeline_args + - '{}' + - --component_launcher_class_path + - tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - --serialized_component + - '{"__class__": "ModelValidator", "__module__": "tfx.components.model_validator.component", + "__tfx_object_type__": "jsonable", "_instance_name": null, "driver_class": + {"__class__": "Driver", "__module__": "tfx.components.model_validator.driver", + "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", + "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": + "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.model_validator.executor", + "__tfx_object_type__": "class"}}, "spec": {"__class__": "ModelValidatorSpec", + "__module__": "tfx.types.standard_component_specs", "__tfx_object_type__": + "jsonable", "exec_properties": {}, "inputs": {"__class__": "_PropertyDictWrapper", + "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", + "_compat_aliases": {}, "_data": {"examples": {"__class__": "Channel", "__module__": + "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": + "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": + "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, + "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "train"}, "type_name": + {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}, {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "eval"}, "type_name": + {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}], "type_name": "ExamplesPath"}, "model": {"__class__": "Channel", + "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": + [{"__class__": "Model", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": + "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, + "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "Trainer"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "ModelExportPath"}}}, "artifact_type": {"name": "ModelExportPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], + "type_name": "ModelExportPath"}}}, "outputs": {"__class__": "_PropertyDictWrapper", + "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", + "_compat_aliases": {}, "_data": {"blessing": {"__class__": "Channel", "__module__": + "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": + "ModelBlessing", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": + "jsonable", "artifact": {"properties": {"name": {"stringValue": "blessing"}, + "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "ModelValidator"}, "split": {"stringValue": ""}, "type_name": + {"stringValue": "ModelBlessingPath"}}}, "artifact_type": {"name": "ModelBlessingPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}], "type_name": "ModelBlessingPath"}}}}}' + - --component_config + - 'null' + - --enable_cache + command: + - python + - /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + env: + - name: WORKFLOW_ID + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + envFrom: + - configMapRef: + name: metadata-configmap + optional: true + - secretRef: + name: mysql-credential + optional: true + image: tensorflow/tfx:0.16.0.dev20191101 + inputs: + parameters: + - name: pipeline-root + name: modelvalidator + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + - dag: + tasks: + - arguments: + parameters: + - name: data-root + value: '{{inputs.parameters.data-root}}' + - name: pipeline-root + value: '{{inputs.parameters.pipeline-root}}' + name: csvexamplegen + template: csvexamplegen + - arguments: + parameters: + - name: pipeline-root + value: '{{inputs.parameters.pipeline-root}}' + dependencies: + - csvexamplegen + - trainer + name: evaluator + template: evaluator + - arguments: + parameters: + - name: pipeline-root + value: '{{inputs.parameters.pipeline-root}}' + dependencies: + - schemagen + - statisticsgen + name: examplevalidator + template: examplevalidator + - arguments: + parameters: + - name: pipeline-root + value: '{{inputs.parameters.pipeline-root}}' + dependencies: + - csvexamplegen + - trainer + name: modelvalidator + template: modelvalidator + - arguments: + parameters: + - name: pipeline-root + value: '{{inputs.parameters.pipeline-root}}' + dependencies: + - modelvalidator + - trainer + name: pusher + template: pusher + - arguments: + parameters: + - name: pipeline-root + value: '{{inputs.parameters.pipeline-root}}' + dependencies: + - statisticsgen + name: schemagen + template: schemagen + - arguments: + parameters: + - name: pipeline-root + value: '{{inputs.parameters.pipeline-root}}' + dependencies: + - csvexamplegen + name: statisticsgen + template: statisticsgen + - arguments: + parameters: + - name: module-file + value: '{{inputs.parameters.module-file}}' + - name: pipeline-root + value: '{{inputs.parameters.pipeline-root}}' + dependencies: + - schemagen + - transform + name: trainer + template: trainer + - arguments: + parameters: + - name: module-file + value: '{{inputs.parameters.module-file}}' + - name: pipeline-root + value: '{{inputs.parameters.pipeline-root}}' + dependencies: + - csvexamplegen + - schemagen + name: transform + template: transform + inputs: + parameters: + - name: data-root + - name: module-file + - name: pipeline-root + name: parameterized-tfx-oss + - container: + args: + - --pipeline_name + - parameterized_tfx_oss + - --pipeline_root + - '{{inputs.parameters.pipeline-root}}' + - --kubeflow_metadata_config + - "{\n \"mysqlDbServiceHost\": {\n \"environmentVariable\": \"mysql_host\"\ + \n },\n \"mysqlDbServicePort\": {\n \"environmentVariable\": \"mysql_port\"\ + \n },\n \"mysqlDbName\": {\n \"environmentVariable\": \"mysql_database\"\ + \n },\n \"mysqlDbUser\": {\n \"environmentVariable\": \"username\"\n\ + \ },\n \"mysqlDbPassword\": {\n \"environmentVariable\": \"password\"\ + \n }\n}" + - --beam_pipeline_args + - '[]' + - --additional_pipeline_args + - '{}' + - --component_launcher_class_path + - tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - --serialized_component + - '{"__class__": "Pusher", "__module__": "tfx.components.pusher.component", + "__tfx_object_type__": "jsonable", "_instance_name": null, "driver_class": + {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", + "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", + "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": + "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.pusher.executor", + "__tfx_object_type__": "class"}}, "spec": {"__class__": "PusherSpec", "__module__": + "tfx.types.standard_component_specs", "__tfx_object_type__": "jsonable", "exec_properties": + {"custom_config": null, "push_destination": "{\n \"filesystem\": {\n \"baseDirectory\": + \"{{inputs.parameters.pipeline-root}}/model_serving\"\n }\n}"}, "inputs": + {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", + "__tfx_object_type__": "jsonable", "_compat_aliases": {"model": "model_export"}, + "_data": {"model_blessing": {"__class__": "Channel", "__module__": "tfx.types.channel", + "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "ModelBlessing", + "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", + "artifact": {"properties": {"name": {"stringValue": "blessing"}, "pipeline_name": + {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": + "ModelValidator"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "ModelBlessingPath"}}}, "artifact_type": {"name": "ModelBlessingPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], + "type_name": "ModelBlessingPath"}, "model_export": {"__class__": "Channel", + "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": + [{"__class__": "Model", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": + "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, + "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "Trainer"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "ModelExportPath"}}}, "artifact_type": {"name": "ModelExportPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], + "type_name": "ModelExportPath"}}}, "outputs": {"__class__": "_PropertyDictWrapper", + "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", + "_compat_aliases": {"pushed_model": "model_push"}, "_data": {"model_push": + {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": + "jsonable", "_artifacts": [{"__class__": "PushedModel", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "model_push"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, + "producer_component": {"stringValue": "Pusher"}, "split": {"stringValue": + ""}, "type_name": {"stringValue": "ModelPushPath"}}}, "artifact_type": {"name": + "ModelPushPath", "properties": {"name": "STRING", "pipeline_name": "STRING", + "producer_component": "STRING", "span": "INT", "split": "STRING", "state": + "STRING", "type_name": "STRING"}}}], "type_name": "ModelPushPath"}}}}}' + - --component_config + - 'null' + - --enable_cache + command: + - python + - /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + env: + - name: WORKFLOW_ID + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + envFrom: + - configMapRef: + name: metadata-configmap + optional: true + - secretRef: + name: mysql-credential + optional: true + image: tensorflow/tfx:0.16.0.dev20191101 + inputs: + parameters: + - name: pipeline-root + name: pusher + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + - container: + args: + - --pipeline_name + - parameterized_tfx_oss + - --pipeline_root + - '{{inputs.parameters.pipeline-root}}' + - --kubeflow_metadata_config + - "{\n \"mysqlDbServiceHost\": {\n \"environmentVariable\": \"mysql_host\"\ + \n },\n \"mysqlDbServicePort\": {\n \"environmentVariable\": \"mysql_port\"\ + \n },\n \"mysqlDbName\": {\n \"environmentVariable\": \"mysql_database\"\ + \n },\n \"mysqlDbUser\": {\n \"environmentVariable\": \"username\"\n\ + \ },\n \"mysqlDbPassword\": {\n \"environmentVariable\": \"password\"\ + \n }\n}" + - --beam_pipeline_args + - '[]' + - --additional_pipeline_args + - '{}' + - --component_launcher_class_path + - tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - --serialized_component + - '{"__class__": "SchemaGen", "__module__": "tfx.components.schema_gen.component", + "__tfx_object_type__": "jsonable", "_instance_name": null, "driver_class": + {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", + "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", + "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": + "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.schema_gen.executor", + "__tfx_object_type__": "class"}}, "spec": {"__class__": "SchemaGenSpec", "__module__": + "tfx.types.standard_component_specs", "__tfx_object_type__": "jsonable", "exec_properties": + {"infer_feature_shape": false}, "inputs": {"__class__": "_PropertyDictWrapper", + "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", + "_compat_aliases": {"statistics": "stats"}, "_data": {"stats": {"__class__": + "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", + "_artifacts": [{"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "StatisticsGen"}, "split": {"stringValue": "train"}, "type_name": + {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}, {"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "StatisticsGen"}, "split": {"stringValue": "eval"}, "type_name": + {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}], "type_name": "ExampleStatisticsPath"}}}, "outputs": {"__class__": + "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": + "jsonable", "_compat_aliases": {"schema": "output"}, "_data": {"output": {"__class__": + "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", + "_artifacts": [{"__class__": "Schema", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "SchemaGen"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "SchemaPath"}}}, "artifact_type": {"name": "SchemaPath", "properties": {"name": + "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": + "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": + "SchemaPath"}}}}}' + - --component_config + - 'null' + - --enable_cache + command: + - python + - /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + env: + - name: WORKFLOW_ID + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + envFrom: + - configMapRef: + name: metadata-configmap + optional: true + - secretRef: + name: mysql-credential + optional: true + image: tensorflow/tfx:0.16.0.dev20191101 + inputs: + parameters: + - name: pipeline-root + name: schemagen + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + - container: + args: + - --pipeline_name + - parameterized_tfx_oss + - --pipeline_root + - '{{inputs.parameters.pipeline-root}}' + - --kubeflow_metadata_config + - "{\n \"mysqlDbServiceHost\": {\n \"environmentVariable\": \"mysql_host\"\ + \n },\n \"mysqlDbServicePort\": {\n \"environmentVariable\": \"mysql_port\"\ + \n },\n \"mysqlDbName\": {\n \"environmentVariable\": \"mysql_database\"\ + \n },\n \"mysqlDbUser\": {\n \"environmentVariable\": \"username\"\n\ + \ },\n \"mysqlDbPassword\": {\n \"environmentVariable\": \"password\"\ + \n }\n}" + - --beam_pipeline_args + - '[]' + - --additional_pipeline_args + - '{}' + - --component_launcher_class_path + - tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - --serialized_component + - '{"__class__": "StatisticsGen", "__module__": "tfx.components.statistics_gen.component", + "__tfx_object_type__": "jsonable", "_instance_name": null, "driver_class": + {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", + "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", + "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": + "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.statistics_gen.executor", + "__tfx_object_type__": "class"}}, "spec": {"__class__": "StatisticsGenSpec", + "__module__": "tfx.types.standard_component_specs", "__tfx_object_type__": + "jsonable", "exec_properties": {}, "inputs": {"__class__": "_PropertyDictWrapper", + "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", + "_compat_aliases": {"examples": "input_data"}, "_data": {"input_data": {"__class__": + "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", + "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "train"}, "type_name": + {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}, {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "eval"}, "type_name": + {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}], "type_name": "ExamplesPath"}}}, "outputs": {"__class__": "_PropertyDictWrapper", + "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", + "_compat_aliases": {"statistics": "output"}, "_data": {"output": {"__class__": + "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", + "_artifacts": [{"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "StatisticsGen"}, "split": {"stringValue": "train"}, "type_name": + {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}, {"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "StatisticsGen"}, "split": {"stringValue": "eval"}, "type_name": + {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}], "type_name": "ExampleStatisticsPath"}}}}}' + - --component_config + - 'null' + - --enable_cache + command: + - python + - /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + env: + - name: WORKFLOW_ID + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + envFrom: + - configMapRef: + name: metadata-configmap + optional: true + - secretRef: + name: mysql-credential + optional: true + image: tensorflow/tfx:0.16.0.dev20191101 + inputs: + parameters: + - name: pipeline-root + name: statisticsgen + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + - container: + args: + - --pipeline_name + - parameterized_tfx_oss + - --pipeline_root + - '{{inputs.parameters.pipeline-root}}' + - --kubeflow_metadata_config + - "{\n \"mysqlDbServiceHost\": {\n \"environmentVariable\": \"mysql_host\"\ + \n },\n \"mysqlDbServicePort\": {\n \"environmentVariable\": \"mysql_port\"\ + \n },\n \"mysqlDbName\": {\n \"environmentVariable\": \"mysql_database\"\ + \n },\n \"mysqlDbUser\": {\n \"environmentVariable\": \"username\"\n\ + \ },\n \"mysqlDbPassword\": {\n \"environmentVariable\": \"password\"\ + \n }\n}" + - --beam_pipeline_args + - '[]' + - --additional_pipeline_args + - '{}' + - --component_launcher_class_path + - tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - --serialized_component + - '{"__class__": "Trainer", "__module__": "tfx.components.trainer.component", + "__tfx_object_type__": "jsonable", "_instance_name": null, "driver_class": + {"__class__": "Driver", "__module__": "tfx.components.trainer.driver", "__tfx_object_type__": + "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", "__module__": + "tfx.components.base.executor_spec", "__tfx_object_type__": "jsonable", "executor_class": + {"__class__": "Executor", "__module__": "tfx.components.trainer.executor", + "__tfx_object_type__": "class"}}, "spec": {"__class__": "TrainerSpec", "__module__": + "tfx.types.standard_component_specs", "__tfx_object_type__": "jsonable", "exec_properties": + {"custom_config": null, "eval_args": "{\n \"numSteps\": 5\n}", "module_file": + "{{inputs.parameters.module-file}}", "train_args": "{\n \"numSteps\": 10\n}", + "trainer_fn": null}, "inputs": {"__class__": "_PropertyDictWrapper", "__module__": + "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": + {"transform_graph": "transform_output"}, "_data": {"examples": {"__class__": + "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", + "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "transformed_examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, + "producer_component": {"stringValue": "Transform"}, "split": {"stringValue": + "train"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": + {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": + "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", + "state": "STRING", "type_name": "STRING"}}}, {"__class__": "Examples", "__module__": + "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": + {"properties": {"name": {"stringValue": "transformed_examples"}, "pipeline_name": + {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": + "Transform"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": + "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], + "type_name": "ExamplesPath"}, "schema": {"__class__": "Channel", "__module__": + "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": + "Schema", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": + "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, + "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "SchemaGen"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "SchemaPath"}}}, "artifact_type": {"name": "SchemaPath", "properties": {"name": + "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": + "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": + "SchemaPath"}, "transform_output": {"__class__": "Channel", "__module__": + "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": + "TransformGraph", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": + "jsonable", "artifact": {"properties": {"name": {"stringValue": "transform_output"}, + "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "Transform"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "TransformPath"}}}, "artifact_type": {"name": "TransformPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], + "type_name": "TransformPath"}}}, "outputs": {"__class__": "_PropertyDictWrapper", + "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", + "_compat_aliases": {"model": "output"}, "_data": {"output": {"__class__": + "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", + "_artifacts": [{"__class__": "Model", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "Trainer"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "ModelExportPath"}}}, "artifact_type": {"name": "ModelExportPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], + "type_name": "ModelExportPath"}}}}}' + - --component_config + - 'null' + - --enable_cache + command: + - python + - /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + env: + - name: WORKFLOW_ID + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + envFrom: + - configMapRef: + name: metadata-configmap + optional: true + - secretRef: + name: mysql-credential + optional: true + image: tensorflow/tfx:0.16.0.dev20191101 + inputs: + parameters: + - name: module-file + - name: pipeline-root + name: trainer + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + - container: + args: + - --pipeline_name + - parameterized_tfx_oss + - --pipeline_root + - '{{inputs.parameters.pipeline-root}}' + - --kubeflow_metadata_config + - "{\n \"mysqlDbServiceHost\": {\n \"environmentVariable\": \"mysql_host\"\ + \n },\n \"mysqlDbServicePort\": {\n \"environmentVariable\": \"mysql_port\"\ + \n },\n \"mysqlDbName\": {\n \"environmentVariable\": \"mysql_database\"\ + \n },\n \"mysqlDbUser\": {\n \"environmentVariable\": \"username\"\n\ + \ },\n \"mysqlDbPassword\": {\n \"environmentVariable\": \"password\"\ + \n }\n}" + - --beam_pipeline_args + - '[]' + - --additional_pipeline_args + - '{}' + - --component_launcher_class_path + - tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - --serialized_component + - '{"__class__": "Transform", "__module__": "tfx.components.transform.component", + "__tfx_object_type__": "jsonable", "_instance_name": null, "driver_class": + {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", + "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", + "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": + "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.transform.executor", + "__tfx_object_type__": "class"}}, "spec": {"__class__": "TransformSpec", "__module__": + "tfx.types.standard_component_specs", "__tfx_object_type__": "jsonable", "exec_properties": + {"module_file": "{{inputs.parameters.module-file}}", "preprocessing_fn": null}, + "inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", + "__tfx_object_type__": "jsonable", "_compat_aliases": {"examples": "input_data"}, + "_data": {"input_data": {"__class__": "Channel", "__module__": "tfx.types.channel", + "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Examples", + "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", + "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": + {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": + "CsvExampleGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": + "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, + {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": + "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, + "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "eval"}, "type_name": + {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", + "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": + "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": + "STRING"}}}], "type_name": "ExamplesPath"}, "schema": {"__class__": "Channel", + "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": + [{"__class__": "Schema", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": + "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, + "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": + {"stringValue": "SchemaGen"}, "split": {"stringValue": ""}, "type_name": {"stringValue": + "SchemaPath"}}}, "artifact_type": {"name": "SchemaPath", "properties": {"name": + "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": + "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": + "SchemaPath"}}}, "outputs": {"__class__": "_PropertyDictWrapper", "__module__": + "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": + {"transform_graph": "transform_output"}, "_data": {"transform_output": {"__class__": + "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", + "_artifacts": [{"__class__": "TransformGraph", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "transform_output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, + "producer_component": {"stringValue": "Transform"}, "split": {"stringValue": + ""}, "type_name": {"stringValue": "TransformPath"}}}, "artifact_type": {"name": + "TransformPath", "properties": {"name": "STRING", "pipeline_name": "STRING", + "producer_component": "STRING", "span": "INT", "split": "STRING", "state": + "STRING", "type_name": "STRING"}}}], "type_name": "TransformPath"}, "transformed_examples": + {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": + "jsonable", "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", + "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": + "transformed_examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, + "producer_component": {"stringValue": "Transform"}, "split": {"stringValue": + "train"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": + {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": + "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", + "state": "STRING", "type_name": "STRING"}}}, {"__class__": "Examples", "__module__": + "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": + {"properties": {"name": {"stringValue": "transformed_examples"}, "pipeline_name": + {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": + "Transform"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": + "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": + {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", + "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], + "type_name": "ExamplesPath"}}}}}' + - --component_config + - 'null' + - --enable_cache + command: + - python + - /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + env: + - name: WORKFLOW_ID + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + envFrom: + - configMapRef: + name: metadata-configmap + optional: true + - secretRef: + name: mysql-credential + optional: true + image: tensorflow/tfx:0.16.0.dev20191101 + inputs: + parameters: + - name: module-file + - name: pipeline-root + name: transform + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json diff --git a/samples/core/ai_platform/ai_platform.ipynb b/samples/core/ai_platform/ai_platform.ipynb index f212ef46491..e3ac76aed97 100644 --- a/samples/core/ai_platform/ai_platform.ipynb +++ b/samples/core/ai_platform/ai_platform.ipynb @@ -46,7 +46,6 @@ "import kfp\n", "import kfp.components as comp\n", "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "\n", "import pandas as pd\n", "\n", @@ -133,8 +132,6 @@ " query=QUERY,\n", " project_id=project_id,\n", " output_gcs_path=data_gcs_path\n", - " ).apply(\n", - " gcp.use_gcp_secret('user-gcp-sa') \n", " )" ] }, @@ -173,7 +170,7 @@ " args=trainer_args,\n", " job_dir=trainer_output_gcs_path,\n", " runtime_version=runtime_version\n", - " ).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " )" ] }, { @@ -208,7 +205,7 @@ " version_id=model_version, \n", " runtime_version=runtime_version, \n", " replace_existing_version=True, \n", - " set_default=True).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " set_default=True)" ] }, { @@ -348,4 +345,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/samples/core/dataflow/dataflow.ipynb b/samples/core/dataflow/dataflow.ipynb index 0e8f87ce866..25488c34403 100644 --- a/samples/core/dataflow/dataflow.ipynb +++ b/samples/core/dataflow/dataflow.ipynb @@ -42,7 +42,7 @@ "- Cloud Dataflow API is enabled.\n", "- The component is running under a secret Kubeflow user service account in a Kubeflow Pipeline cluster. For example:\n", "```\n", - "component_op(...).apply(gcp.use_gcp_secret('user-gcp-sa'))\n", + "component_op(...)\n", "```\n", "The Kubeflow user service account is a member of:\n", "- `roles/dataflow.developer` role of the project.\n", @@ -302,7 +302,6 @@ "source": [ "import kfp\n", "import kfp.dsl as dsl\n", - "import kfp.gcp as gcp\n", "import json\n", "output_file = '{}/wc/wordcount.out'.format(output)\n", "@dsl.pipeline(\n", @@ -325,7 +324,7 @@ " staging_dir = staging_dir, \n", " requirements_file_path = requirements_file_path, \n", " args = args,\n", - " wait_interval = wait_interval).apply(gcp.use_gcp_secret('user-gcp-sa'))" + " wait_interval = wait_interval)" ] }, { diff --git a/samples/core/exit_handler/exit_handler.py b/samples/core/exit_handler/exit_handler.py index ed52d357999..9c2b04c84a7 100755 --- a/samples/core/exit_handler/exit_handler.py +++ b/samples/core/exit_handler/exit_handler.py @@ -21,7 +21,7 @@ def gcs_download_op(url): return dsl.ContainerOp( name='GCS - Download', - image='google/cloud-sdk:216.0.0', + image='google/cloud-sdk:272.0.0', command=['sh', '-c'], arguments=['gsutil cat $0 | tee $1', url, '/tmp/results.txt'], file_outputs={ diff --git a/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb b/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb index 9dfb3e62826..cce39764e01 100644 --- a/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb +++ b/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb @@ -285,7 +285,6 @@ "source": [ "import kfp\n", "import kfp.dsl as dsl\n", - "from kfp.gcp import use_gcp_secret\n", "\n", "# The pipeline definition\n", "@dsl.pipeline(\n", @@ -293,7 +292,7 @@ " description='Sample for deploying models using KFP model serving component'\n", ")\n", "def model_server():\n", - " deploy = kubeflow_deploy_op().apply(use_gcp_secret('user-gcp-sa'))" + " deploy = kubeflow_deploy_op()" ] }, { @@ -349,4 +348,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/samples/core/parallel_join/parallel_join.py b/samples/core/parallel_join/parallel_join.py index 247842ca943..dc061e8f174 100755 --- a/samples/core/parallel_join/parallel_join.py +++ b/samples/core/parallel_join/parallel_join.py @@ -20,7 +20,7 @@ def gcs_download_op(url): return dsl.ContainerOp( name='GCS - Download', - image='google/cloud-sdk:216.0.0', + image='google/cloud-sdk:272.0.0', command=['sh', '-c'], arguments=['gsutil cat $0 | tee $1', url, '/tmp/results.txt'], file_outputs={ diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py index cd6ca9d23ba..ddcedc82d70 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py @@ -149,6 +149,10 @@ def _create_test_pipeline( config = kubeflow_dag_runner.KubeflowDagRunnerConfig( kubeflow_metadata_config=kubeflow_dag_runner. get_default_kubeflow_metadata_config(), + # TODO: remove this override when KubeflowDagRunnerConfig doesn't default to use_gcp_secret op. + pipeline_operator_funcs=list(filter( + lambda operator: operator.__name__.find('gcp_secret') == -1, + kubeflow_dag_runner.get_default_pipeline_operator_funcs())), tfx_image='tensorflow/tfx:0.15.0', ) kfp_runner = kubeflow_dag_runner.KubeflowDagRunner( diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py.yaml b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py.yaml new file mode 100644 index 00000000000..28847505ea4 --- /dev/null +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py.yaml @@ -0,0 +1,1131 @@ +"apiVersion": |- + argoproj.io/v1alpha1 +"kind": |- + Workflow +"metadata": + "annotations": + "pipelines.kubeflow.org/pipeline_spec": |- + {"description": "Constructs a Kubeflow pipeline.\n\n Creates Kubeflow ContainerOps for each TFX component encountered in the\n logical pipeline definition.\n ", "inputs": [{"default": "gs://ml-pipeline-playground/tfx_taxi_simple/data", "name": "data-root"}, {"default": "gs://ml-pipeline-playground/tfx_taxi_simple/modules/taxi_utils.py", "name": "module-file"}, {"default": "gs://your-bucket/tfx_taxi_simple/{{workflow.uid}}", "name": "pipeline-root"}], "name": "parameterized_tfx_oss"} + "generateName": |- + parameterized-tfx-oss- +"spec": + "arguments": + "parameters": + - "name": |- + data-root + "value": |- + gs://ml-pipeline-playground/tfx_taxi_simple/data + - "name": |- + module-file + "value": |- + gs://ml-pipeline-playground/tfx_taxi_simple/modules/taxi_utils.py + - "name": |- + pipeline-root + "value": |- + gs://your-bucket/tfx_taxi_simple/{{workflow.uid}} + "entrypoint": |- + parameterized-tfx-oss + "serviceAccountName": |- + pipeline-runner + "templates": + - "container": + "args": + - |- + --pipeline_name + - |- + parameterized_tfx_oss + - |- + --pipeline_root + - |- + {{inputs.parameters.pipeline-root}} + - |- + --kubeflow_metadata_config + - |- + { + "mysqlDbServiceHost": { + "environmentVariable": "mysql_host" + }, + "mysqlDbServicePort": { + "environmentVariable": "mysql_port" + }, + "mysqlDbName": { + "environmentVariable": "mysql_database" + }, + "mysqlDbUser": { + "environmentVariable": "username" + }, + "mysqlDbPassword": { + "environmentVariable": "password" + } + } + - |- + --beam_pipeline_args + - |- + [] + - |- + --additional_pipeline_args + - |- + {} + - |- + --component_launcher_class_path + - |- + tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - |- + --serialized_component + - |- + {"__class__": "NodeWrapper", "__module__": "tfx.orchestration.kubeflow.node_wrapper", "__tfx_object_type__": "jsonable", "_exec_properties": {"custom_config": null, "input_config": "{\n \"splits\": [\n {\n \"name\": \"single_split\",\n \"pattern\": \"*\"\n }\n ]\n}", "output_config": "{\n \"splitConfig\": {\n \"splits\": [\n {\n \"hashBuckets\": 2,\n \"name\": \"train\"\n },\n {\n \"hashBuckets\": 1,\n \"name\": \"eval\"\n }\n ]\n }\n}"}, "_id": "CsvExampleGen", "_inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"input": "input_base"}, "_data": {"input_base": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "ExternalArtifact", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"split": {"stringValue": ""}, "type_name": {"stringValue": "ExternalPath"}}, "uri": "{{inputs.parameters.data-root}}"}, "artifact_type": {"name": "ExternalPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExternalPath"}}}, "_outputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {}, "_data": {"examples": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExamplesPath"}}}, "_type": "tfx.components.example_gen.csv_example_gen.component.CsvExampleGen", "driver_class": {"__class__": "Driver", "__module__": "tfx.components.example_gen.driver", "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.example_gen.csv_example_gen.executor", "__tfx_object_type__": "class"}}} + - |- + --component_config + - |- + null + - |- + --enable_cache + "command": + - |- + python + - |- + /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + "env": + - "name": |- + WORKFLOW_ID + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.labels['workflows.argoproj.io/workflow'] + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "envFrom": + - "configMapRef": + "name": |- + metadata-configmap + "optional": !!bool |- + true + - "secretRef": + "name": |- + mysql-credential + "optional": !!bool |- + true + "image": |- + tensorflow/tfx:0.15.0 + "inputs": + "parameters": + - "name": |- + data-root + - "name": |- + pipeline-root + "metadata": + "labels": + "add-pod-env": |- + true + "name": |- + csvexamplegen + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "container": + "args": + - |- + --pipeline_name + - |- + parameterized_tfx_oss + - |- + --pipeline_root + - |- + {{inputs.parameters.pipeline-root}} + - |- + --kubeflow_metadata_config + - |- + { + "mysqlDbServiceHost": { + "environmentVariable": "mysql_host" + }, + "mysqlDbServicePort": { + "environmentVariable": "mysql_port" + }, + "mysqlDbName": { + "environmentVariable": "mysql_database" + }, + "mysqlDbUser": { + "environmentVariable": "username" + }, + "mysqlDbPassword": { + "environmentVariable": "password" + } + } + - |- + --beam_pipeline_args + - |- + [] + - |- + --additional_pipeline_args + - |- + {} + - |- + --component_launcher_class_path + - |- + tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - |- + --serialized_component + - |- + {"__class__": "NodeWrapper", "__module__": "tfx.orchestration.kubeflow.node_wrapper", "__tfx_object_type__": "jsonable", "_exec_properties": {"fairness_indicator_thresholds": null, "feature_slicing_spec": "{\n \"specs\": [\n {\n \"columnForSlicing\": [\n \"trip_start_hour\"\n ]\n }\n ]\n}"}, "_id": "Evaluator", "_inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"model": "model_exports"}, "_data": {"examples": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExamplesPath"}, "model_exports": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Model", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Trainer"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "ModelExportPath"}}}, "artifact_type": {"name": "ModelExportPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ModelExportPath"}}}, "_outputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"evaluation": "output"}, "_data": {"output": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "ModelEvaluation", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Evaluator"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "ModelEvalPath"}}}, "artifact_type": {"name": "ModelEvalPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ModelEvalPath"}}}, "_type": "tfx.components.evaluator.component.Evaluator", "driver_class": {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.evaluator.executor", "__tfx_object_type__": "class"}}} + - |- + --component_config + - |- + null + - |- + --enable_cache + "command": + - |- + python + - |- + /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + "env": + - "name": |- + WORKFLOW_ID + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.labels['workflows.argoproj.io/workflow'] + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "envFrom": + - "configMapRef": + "name": |- + metadata-configmap + "optional": !!bool |- + true + - "secretRef": + "name": |- + mysql-credential + "optional": !!bool |- + true + "image": |- + tensorflow/tfx:0.15.0 + "inputs": + "parameters": + - "name": |- + pipeline-root + "metadata": + "labels": + "add-pod-env": |- + true + "name": |- + evaluator + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "container": + "args": + - |- + --pipeline_name + - |- + parameterized_tfx_oss + - |- + --pipeline_root + - |- + {{inputs.parameters.pipeline-root}} + - |- + --kubeflow_metadata_config + - |- + { + "mysqlDbServiceHost": { + "environmentVariable": "mysql_host" + }, + "mysqlDbServicePort": { + "environmentVariable": "mysql_port" + }, + "mysqlDbName": { + "environmentVariable": "mysql_database" + }, + "mysqlDbUser": { + "environmentVariable": "username" + }, + "mysqlDbPassword": { + "environmentVariable": "password" + } + } + - |- + --beam_pipeline_args + - |- + [] + - |- + --additional_pipeline_args + - |- + {} + - |- + --component_launcher_class_path + - |- + tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - |- + --serialized_component + - |- + {"__class__": "NodeWrapper", "__module__": "tfx.orchestration.kubeflow.node_wrapper", "__tfx_object_type__": "jsonable", "_exec_properties": {}, "_id": "ExampleValidator", "_inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"statistics": "stats"}, "_data": {"schema": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Schema", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "SchemaGen"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "SchemaPath"}}}, "artifact_type": {"name": "SchemaPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "SchemaPath"}, "stats": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "StatisticsGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, {"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "StatisticsGen"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExampleStatisticsPath"}}}, "_outputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"anomalies": "output"}, "_data": {"output": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "ExampleAnomalies", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "ExampleValidator"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "ExampleValidationPath"}}}, "artifact_type": {"name": "ExampleValidationPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExampleValidationPath"}}}, "_type": "tfx.components.example_validator.component.ExampleValidator", "driver_class": {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.example_validator.executor", "__tfx_object_type__": "class"}}} + - |- + --component_config + - |- + null + - |- + --enable_cache + "command": + - |- + python + - |- + /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + "env": + - "name": |- + WORKFLOW_ID + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.labels['workflows.argoproj.io/workflow'] + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "envFrom": + - "configMapRef": + "name": |- + metadata-configmap + "optional": !!bool |- + true + - "secretRef": + "name": |- + mysql-credential + "optional": !!bool |- + true + "image": |- + tensorflow/tfx:0.15.0 + "inputs": + "parameters": + - "name": |- + pipeline-root + "metadata": + "labels": + "add-pod-env": |- + true + "name": |- + examplevalidator + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "container": + "args": + - |- + --pipeline_name + - |- + parameterized_tfx_oss + - |- + --pipeline_root + - |- + {{inputs.parameters.pipeline-root}} + - |- + --kubeflow_metadata_config + - |- + { + "mysqlDbServiceHost": { + "environmentVariable": "mysql_host" + }, + "mysqlDbServicePort": { + "environmentVariable": "mysql_port" + }, + "mysqlDbName": { + "environmentVariable": "mysql_database" + }, + "mysqlDbUser": { + "environmentVariable": "username" + }, + "mysqlDbPassword": { + "environmentVariable": "password" + } + } + - |- + --beam_pipeline_args + - |- + [] + - |- + --additional_pipeline_args + - |- + {} + - |- + --component_launcher_class_path + - |- + tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - |- + --serialized_component + - |- + {"__class__": "NodeWrapper", "__module__": "tfx.orchestration.kubeflow.node_wrapper", "__tfx_object_type__": "jsonable", "_exec_properties": {}, "_id": "ModelValidator", "_inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {}, "_data": {"examples": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExamplesPath"}, "model": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Model", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Trainer"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "ModelExportPath"}}}, "artifact_type": {"name": "ModelExportPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ModelExportPath"}}}, "_outputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {}, "_data": {"blessing": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "ModelBlessing", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "blessing"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "ModelValidator"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "ModelBlessingPath"}}}, "artifact_type": {"name": "ModelBlessingPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ModelBlessingPath"}}}, "_type": "tfx.components.model_validator.component.ModelValidator", "driver_class": {"__class__": "Driver", "__module__": "tfx.components.model_validator.driver", "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.model_validator.executor", "__tfx_object_type__": "class"}}} + - |- + --component_config + - |- + null + - |- + --enable_cache + "command": + - |- + python + - |- + /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + "env": + - "name": |- + WORKFLOW_ID + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.labels['workflows.argoproj.io/workflow'] + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "envFrom": + - "configMapRef": + "name": |- + metadata-configmap + "optional": !!bool |- + true + - "secretRef": + "name": |- + mysql-credential + "optional": !!bool |- + true + "image": |- + tensorflow/tfx:0.15.0 + "inputs": + "parameters": + - "name": |- + pipeline-root + "metadata": + "labels": + "add-pod-env": |- + true + "name": |- + modelvalidator + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "dag": + "tasks": + - "arguments": + "parameters": + - "name": |- + data-root + "value": |- + {{inputs.parameters.data-root}} + - "name": |- + pipeline-root + "value": |- + {{inputs.parameters.pipeline-root}} + "name": |- + csvexamplegen + "template": |- + csvexamplegen + - "arguments": + "parameters": + - "name": |- + pipeline-root + "value": |- + {{inputs.parameters.pipeline-root}} + "dependencies": + - |- + csvexamplegen + - |- + trainer + "name": |- + evaluator + "template": |- + evaluator + - "arguments": + "parameters": + - "name": |- + pipeline-root + "value": |- + {{inputs.parameters.pipeline-root}} + "dependencies": + - |- + schemagen + - |- + statisticsgen + "name": |- + examplevalidator + "template": |- + examplevalidator + - "arguments": + "parameters": + - "name": |- + pipeline-root + "value": |- + {{inputs.parameters.pipeline-root}} + "dependencies": + - |- + csvexamplegen + - |- + trainer + "name": |- + modelvalidator + "template": |- + modelvalidator + - "arguments": + "parameters": + - "name": |- + pipeline-root + "value": |- + {{inputs.parameters.pipeline-root}} + "dependencies": + - |- + modelvalidator + - |- + trainer + "name": |- + pusher + "template": |- + pusher + - "arguments": + "parameters": + - "name": |- + pipeline-root + "value": |- + {{inputs.parameters.pipeline-root}} + "dependencies": + - |- + statisticsgen + "name": |- + schemagen + "template": |- + schemagen + - "arguments": + "parameters": + - "name": |- + pipeline-root + "value": |- + {{inputs.parameters.pipeline-root}} + "dependencies": + - |- + csvexamplegen + "name": |- + statisticsgen + "template": |- + statisticsgen + - "arguments": + "parameters": + - "name": |- + module-file + "value": |- + {{inputs.parameters.module-file}} + - "name": |- + pipeline-root + "value": |- + {{inputs.parameters.pipeline-root}} + "dependencies": + - |- + schemagen + - |- + transform + "name": |- + trainer + "template": |- + trainer + - "arguments": + "parameters": + - "name": |- + module-file + "value": |- + {{inputs.parameters.module-file}} + - "name": |- + pipeline-root + "value": |- + {{inputs.parameters.pipeline-root}} + "dependencies": + - |- + csvexamplegen + - |- + schemagen + "name": |- + transform + "template": |- + transform + "inputs": + "parameters": + - "name": |- + data-root + - "name": |- + module-file + - "name": |- + pipeline-root + "name": |- + parameterized-tfx-oss + - "container": + "args": + - |- + --pipeline_name + - |- + parameterized_tfx_oss + - |- + --pipeline_root + - |- + {{inputs.parameters.pipeline-root}} + - |- + --kubeflow_metadata_config + - |- + { + "mysqlDbServiceHost": { + "environmentVariable": "mysql_host" + }, + "mysqlDbServicePort": { + "environmentVariable": "mysql_port" + }, + "mysqlDbName": { + "environmentVariable": "mysql_database" + }, + "mysqlDbUser": { + "environmentVariable": "username" + }, + "mysqlDbPassword": { + "environmentVariable": "password" + } + } + - |- + --beam_pipeline_args + - |- + [] + - |- + --additional_pipeline_args + - |- + {} + - |- + --component_launcher_class_path + - |- + tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - |- + --serialized_component + - |- + {"__class__": "NodeWrapper", "__module__": "tfx.orchestration.kubeflow.node_wrapper", "__tfx_object_type__": "jsonable", "_exec_properties": {"custom_config": null, "push_destination": "{\n \"filesystem\": {\n \"baseDirectory\": \"{{inputs.parameters.pipeline-root}}/model_serving\"\n }\n}"}, "_id": "Pusher", "_inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"model": "model_export"}, "_data": {"model_blessing": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "ModelBlessing", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "blessing"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "ModelValidator"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "ModelBlessingPath"}}}, "artifact_type": {"name": "ModelBlessingPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ModelBlessingPath"}, "model_export": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Model", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Trainer"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "ModelExportPath"}}}, "artifact_type": {"name": "ModelExportPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ModelExportPath"}}}, "_outputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"pushed_model": "model_push"}, "_data": {"model_push": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "PushedModel", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "model_push"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Pusher"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "ModelPushPath"}}}, "artifact_type": {"name": "ModelPushPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ModelPushPath"}}}, "_type": "tfx.components.pusher.component.Pusher", "driver_class": {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.pusher.executor", "__tfx_object_type__": "class"}}} + - |- + --component_config + - |- + null + - |- + --enable_cache + "command": + - |- + python + - |- + /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + "env": + - "name": |- + WORKFLOW_ID + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.labels['workflows.argoproj.io/workflow'] + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "envFrom": + - "configMapRef": + "name": |- + metadata-configmap + "optional": !!bool |- + true + - "secretRef": + "name": |- + mysql-credential + "optional": !!bool |- + true + "image": |- + tensorflow/tfx:0.15.0 + "inputs": + "parameters": + - "name": |- + pipeline-root + "metadata": + "labels": + "add-pod-env": |- + true + "name": |- + pusher + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "container": + "args": + - |- + --pipeline_name + - |- + parameterized_tfx_oss + - |- + --pipeline_root + - |- + {{inputs.parameters.pipeline-root}} + - |- + --kubeflow_metadata_config + - |- + { + "mysqlDbServiceHost": { + "environmentVariable": "mysql_host" + }, + "mysqlDbServicePort": { + "environmentVariable": "mysql_port" + }, + "mysqlDbName": { + "environmentVariable": "mysql_database" + }, + "mysqlDbUser": { + "environmentVariable": "username" + }, + "mysqlDbPassword": { + "environmentVariable": "password" + } + } + - |- + --beam_pipeline_args + - |- + [] + - |- + --additional_pipeline_args + - |- + {} + - |- + --component_launcher_class_path + - |- + tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - |- + --serialized_component + - |- + {"__class__": "NodeWrapper", "__module__": "tfx.orchestration.kubeflow.node_wrapper", "__tfx_object_type__": "jsonable", "_exec_properties": {"infer_feature_shape": false}, "_id": "SchemaGen", "_inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"statistics": "stats"}, "_data": {"stats": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "StatisticsGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, {"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "StatisticsGen"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExampleStatisticsPath"}}}, "_outputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"schema": "output"}, "_data": {"output": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Schema", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "SchemaGen"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "SchemaPath"}}}, "artifact_type": {"name": "SchemaPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "SchemaPath"}}}, "_type": "tfx.components.schema_gen.component.SchemaGen", "driver_class": {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.schema_gen.executor", "__tfx_object_type__": "class"}}} + - |- + --component_config + - |- + null + - |- + --enable_cache + "command": + - |- + python + - |- + /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + "env": + - "name": |- + WORKFLOW_ID + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.labels['workflows.argoproj.io/workflow'] + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "envFrom": + - "configMapRef": + "name": |- + metadata-configmap + "optional": !!bool |- + true + - "secretRef": + "name": |- + mysql-credential + "optional": !!bool |- + true + "image": |- + tensorflow/tfx:0.15.0 + "inputs": + "parameters": + - "name": |- + pipeline-root + "metadata": + "labels": + "add-pod-env": |- + true + "name": |- + schemagen + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "container": + "args": + - |- + --pipeline_name + - |- + parameterized_tfx_oss + - |- + --pipeline_root + - |- + {{inputs.parameters.pipeline-root}} + - |- + --kubeflow_metadata_config + - |- + { + "mysqlDbServiceHost": { + "environmentVariable": "mysql_host" + }, + "mysqlDbServicePort": { + "environmentVariable": "mysql_port" + }, + "mysqlDbName": { + "environmentVariable": "mysql_database" + }, + "mysqlDbUser": { + "environmentVariable": "username" + }, + "mysqlDbPassword": { + "environmentVariable": "password" + } + } + - |- + --beam_pipeline_args + - |- + [] + - |- + --additional_pipeline_args + - |- + {} + - |- + --component_launcher_class_path + - |- + tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - |- + --serialized_component + - |- + {"__class__": "NodeWrapper", "__module__": "tfx.orchestration.kubeflow.node_wrapper", "__tfx_object_type__": "jsonable", "_exec_properties": {}, "_id": "StatisticsGen", "_inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"examples": "input_data"}, "_data": {"input_data": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExamplesPath"}}}, "_outputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"statistics": "output"}, "_data": {"output": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "StatisticsGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, {"__class__": "ExampleStatistics", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "StatisticsGen"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": "ExampleStatisticsPath"}}}, "artifact_type": {"name": "ExampleStatisticsPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExampleStatisticsPath"}}}, "_type": "tfx.components.statistics_gen.component.StatisticsGen", "driver_class": {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.statistics_gen.executor", "__tfx_object_type__": "class"}}} + - |- + --component_config + - |- + null + - |- + --enable_cache + "command": + - |- + python + - |- + /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + "env": + - "name": |- + WORKFLOW_ID + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.labels['workflows.argoproj.io/workflow'] + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "envFrom": + - "configMapRef": + "name": |- + metadata-configmap + "optional": !!bool |- + true + - "secretRef": + "name": |- + mysql-credential + "optional": !!bool |- + true + "image": |- + tensorflow/tfx:0.15.0 + "inputs": + "parameters": + - "name": |- + pipeline-root + "metadata": + "labels": + "add-pod-env": |- + true + "name": |- + statisticsgen + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "container": + "args": + - |- + --pipeline_name + - |- + parameterized_tfx_oss + - |- + --pipeline_root + - |- + {{inputs.parameters.pipeline-root}} + - |- + --kubeflow_metadata_config + - |- + { + "mysqlDbServiceHost": { + "environmentVariable": "mysql_host" + }, + "mysqlDbServicePort": { + "environmentVariable": "mysql_port" + }, + "mysqlDbName": { + "environmentVariable": "mysql_database" + }, + "mysqlDbUser": { + "environmentVariable": "username" + }, + "mysqlDbPassword": { + "environmentVariable": "password" + } + } + - |- + --beam_pipeline_args + - |- + [] + - |- + --additional_pipeline_args + - |- + {} + - |- + --component_launcher_class_path + - |- + tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - |- + --serialized_component + - |- + {"__class__": "NodeWrapper", "__module__": "tfx.orchestration.kubeflow.node_wrapper", "__tfx_object_type__": "jsonable", "_exec_properties": {"custom_config": null, "eval_args": "{\n \"numSteps\": 5\n}", "module_file": "{{inputs.parameters.module-file}}", "train_args": "{\n \"numSteps\": 10\n}", "trainer_fn": null}, "_id": "Trainer", "_inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"transform_graph": "transform_output"}, "_data": {"examples": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "transformed_examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Transform"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "transformed_examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Transform"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExamplesPath"}, "schema": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Schema", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "SchemaGen"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "SchemaPath"}}}, "artifact_type": {"name": "SchemaPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "SchemaPath"}, "transform_output": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "TransformGraph", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "transform_output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Transform"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "TransformPath"}}}, "artifact_type": {"name": "TransformPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "TransformPath"}}}, "_outputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"model": "output"}, "_data": {"output": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Model", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Trainer"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "ModelExportPath"}}}, "artifact_type": {"name": "ModelExportPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ModelExportPath"}}}, "_type": "tfx.components.trainer.component.Trainer", "driver_class": {"__class__": "Driver", "__module__": "tfx.components.trainer.driver", "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.trainer.executor", "__tfx_object_type__": "class"}}} + - |- + --component_config + - |- + null + - |- + --enable_cache + "command": + - |- + python + - |- + /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + "env": + - "name": |- + WORKFLOW_ID + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.labels['workflows.argoproj.io/workflow'] + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "envFrom": + - "configMapRef": + "name": |- + metadata-configmap + "optional": !!bool |- + true + - "secretRef": + "name": |- + mysql-credential + "optional": !!bool |- + true + "image": |- + tensorflow/tfx:0.15.0 + "inputs": + "parameters": + - "name": |- + module-file + - "name": |- + pipeline-root + "metadata": + "labels": + "add-pod-env": |- + true + "name": |- + trainer + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "container": + "args": + - |- + --pipeline_name + - |- + parameterized_tfx_oss + - |- + --pipeline_root + - |- + {{inputs.parameters.pipeline-root}} + - |- + --kubeflow_metadata_config + - |- + { + "mysqlDbServiceHost": { + "environmentVariable": "mysql_host" + }, + "mysqlDbServicePort": { + "environmentVariable": "mysql_port" + }, + "mysqlDbName": { + "environmentVariable": "mysql_database" + }, + "mysqlDbUser": { + "environmentVariable": "username" + }, + "mysqlDbPassword": { + "environmentVariable": "password" + } + } + - |- + --beam_pipeline_args + - |- + [] + - |- + --additional_pipeline_args + - |- + {} + - |- + --component_launcher_class_path + - |- + tfx.orchestration.launcher.in_process_component_launcher.InProcessComponentLauncher + - |- + --serialized_component + - |- + {"__class__": "NodeWrapper", "__module__": "tfx.orchestration.kubeflow.node_wrapper", "__tfx_object_type__": "jsonable", "_exec_properties": {"module_file": "{{inputs.parameters.module-file}}", "preprocessing_fn": null}, "_id": "Transform", "_inputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"examples": "input_data"}, "_data": {"input_data": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "CsvExampleGen"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExamplesPath"}, "schema": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Schema", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "SchemaGen"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "SchemaPath"}}}, "artifact_type": {"name": "SchemaPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "SchemaPath"}}}, "_outputs": {"__class__": "_PropertyDictWrapper", "__module__": "tfx.types.node_common", "__tfx_object_type__": "jsonable", "_compat_aliases": {"transform_graph": "transform_output"}, "_data": {"transform_output": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "TransformGraph", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "transform_output"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Transform"}, "split": {"stringValue": ""}, "type_name": {"stringValue": "TransformPath"}}}, "artifact_type": {"name": "TransformPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "TransformPath"}, "transformed_examples": {"__class__": "Channel", "__module__": "tfx.types.channel", "__tfx_object_type__": "jsonable", "_artifacts": [{"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "transformed_examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Transform"}, "split": {"stringValue": "train"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}, {"__class__": "Examples", "__module__": "tfx.types.standard_artifacts", "__tfx_object_type__": "jsonable", "artifact": {"properties": {"name": {"stringValue": "transformed_examples"}, "pipeline_name": {"stringValue": "parameterized_tfx_oss"}, "producer_component": {"stringValue": "Transform"}, "split": {"stringValue": "eval"}, "type_name": {"stringValue": "ExamplesPath"}}}, "artifact_type": {"name": "ExamplesPath", "properties": {"name": "STRING", "pipeline_name": "STRING", "producer_component": "STRING", "span": "INT", "split": "STRING", "state": "STRING", "type_name": "STRING"}}}], "type_name": "ExamplesPath"}}}, "_type": "tfx.components.transform.component.Transform", "driver_class": {"__class__": "BaseDriver", "__module__": "tfx.components.base.base_driver", "__tfx_object_type__": "class"}, "executor_spec": {"__class__": "ExecutorClassSpec", "__module__": "tfx.components.base.executor_spec", "__tfx_object_type__": "jsonable", "executor_class": {"__class__": "Executor", "__module__": "tfx.components.transform.executor", "__tfx_object_type__": "class"}}} + - |- + --component_config + - |- + null + - |- + --enable_cache + "command": + - |- + python + - |- + /tfx-src/tfx/orchestration/kubeflow/container_entrypoint.py + "env": + - "name": |- + WORKFLOW_ID + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.labels['workflows.argoproj.io/workflow'] + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "envFrom": + - "configMapRef": + "name": |- + metadata-configmap + "optional": !!bool |- + true + - "secretRef": + "name": |- + mysql-credential + "optional": !!bool |- + true + "image": |- + tensorflow/tfx:0.15.0 + "inputs": + "parameters": + - "name": |- + module-file + - "name": |- + pipeline-root + "metadata": + "labels": + "add-pod-env": |- + true + "name": |- + transform + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json diff --git a/samples/core/secret/secret.py b/samples/core/secret/secret.py index 0a93310dfc9..ab611f712b5 100644 --- a/samples/core/secret/secret.py +++ b/samples/core/secret/secret.py @@ -16,13 +16,12 @@ import kfp from kfp import dsl -from kfp.gcp import use_gcp_secret def gcs_read_op(url): return dsl.ContainerOp( name='Access GCS using auth token', - image='google/cloud-sdk:latest', + image='google/cloud-sdk:272.0.0', command=['sh', '-c'], arguments=[ 'gsutil ls "$0" && echo "$1"', @@ -35,7 +34,7 @@ def gcs_read_op(url): def use_gcp_api_op(): return dsl.ContainerOp( name='Using Google Cloud APIs with Auth', - image='google/cloud-sdk:latest', + image='google/cloud-sdk:272.0.0', command=[ 'sh', '-c', 'pip install google-cloud-storage && "$0" "$*"', @@ -57,11 +56,8 @@ def use_gcp_api_op(): def secret_op_pipeline(url='gs://ml-pipeline-playground/shakespeare1.txt'): """A pipeline that uses secret to access cloud hosted resouces.""" - gcs_read_task = gcs_read_op(url).apply( - use_gcp_secret('user-gcp-sa')) - use_gcp_api_task = use_gcp_api_op().apply( - use_gcp_secret('user-gcp-sa')) - + gcs_read_task = gcs_read_op(url) + use_gcp_api_task = use_gcp_api_op() if __name__ == '__main__': - kfp.compiler.Compiler().compile(secret_op_pipeline, __file__ + '.yaml') \ No newline at end of file + kfp.compiler.Compiler().compile(secret_op_pipeline, __file__ + '.yaml') diff --git a/samples/core/sequential/sequential.py b/samples/core/sequential/sequential.py index 764cd52c183..36b5d3edb86 100755 --- a/samples/core/sequential/sequential.py +++ b/samples/core/sequential/sequential.py @@ -21,7 +21,7 @@ def gcs_download_op(url): return dsl.ContainerOp( name='GCS - Download', - image='google/cloud-sdk:216.0.0', + image='google/cloud-sdk:272.0.0', command=['sh', '-c'], arguments=['gsutil cat $0 | tee $1', url, '/tmp/results.txt'], file_outputs={ diff --git a/samples/core/volume_snapshot_ops/volume_snapshot_ops.py b/samples/core/volume_snapshot_ops/volume_snapshot_ops.py index 543620694a2..5fc2f811266 100644 --- a/samples/core/volume_snapshot_ops/volume_snapshot_ops.py +++ b/samples/core/volume_snapshot_ops/volume_snapshot_ops.py @@ -30,7 +30,7 @@ def volume_snapshotop_sequential(url): step1 = dsl.ContainerOp( name="step1_ingest", - image="google/cloud-sdk:216.0.0", + image="google/cloud-sdk:272.0.0", command=["sh", "-c"], arguments=["mkdir /data/step1 && " "gsutil cat %s | gzip -c >/data/step1/file1.gz" % url], @@ -81,4 +81,4 @@ def volume_snapshotop_sequential(url): ) if __name__ == '__main__': - kfp.compiler.Compiler().compile(volume_snapshotop_sequential, __file__ + '.yaml') \ No newline at end of file + kfp.compiler.Compiler().compile(volume_snapshotop_sequential, __file__ + '.yaml') diff --git a/samples/core/xgboost_training_cm/xgboost_training_cm.py b/samples/core/xgboost_training_cm/xgboost_training_cm.py index 933791ea8d9..01f82416882 100755 --- a/samples/core/xgboost_training_cm/xgboost_training_cm.py +++ b/samples/core/xgboost_training_cm/xgboost_training_cm.py @@ -18,7 +18,6 @@ import kfp from kfp import components from kfp import dsl -from kfp import gcp import os import subprocess @@ -297,8 +296,5 @@ def xgb_train_pipeline( output_dir=output_template ).after(_predict_op) - dsl.get_pipeline_conf().add_op_transformer( - gcp.use_gcp_secret('user-gcp-sa')) - if __name__ == '__main__': kfp.compiler.Compiler().compile(xgb_train_pipeline, __file__ + '.yaml') diff --git a/samples/core/xgboost_training_cm/xgboost_training_cm.yaml b/samples/core/xgboost_training_cm/xgboost_training_cm.yaml new file mode 100644 index 00000000000..930f3417e77 --- /dev/null +++ b/samples/core/xgboost_training_cm/xgboost_training_cm.yaml @@ -0,0 +1,963 @@ +"apiVersion": |- + argoproj.io/v1alpha1 +"kind": |- + Workflow +"metadata": + "annotations": + "pipelines.kubeflow.org/pipeline_spec": |- + {"description": "A trainer that does end-to-end distributed training for XGBoost models.", "inputs": [{"default": "gs://your-gcs-bucket", "name": "output"}, {"default": "your-gcp-project", "name": "project"}, {"default": "xgb-{{workflow.uid}}", "name": "cluster_name"}, {"default": "us-central1", "name": "region"}, {"default": "gs://ml-pipeline-playground/sfpd/train.csv", "name": "train_data"}, {"default": "gs://ml-pipeline-playground/sfpd/eval.csv", "name": "eval_data"}, {"default": "gs://ml-pipeline-playground/sfpd/schema.json", "name": "schema"}, {"default": "resolution", "name": "target"}, {"default": "200", "name": "rounds"}, {"default": "2", "name": "workers"}, {"default": "ACTION", "name": "true_label"}], "name": "XGBoost Trainer"} + "generateName": |- + xgboost-trainer- +"spec": + "arguments": + "parameters": + - "name": |- + output + "value": |- + gs://your-gcs-bucket + - "name": |- + project + "value": |- + your-gcp-project + - "name": |- + cluster_name + "value": |- + xgb-{{workflow.uid}} + - "name": |- + region + "value": |- + us-central1 + - "name": |- + train_data + "value": |- + gs://ml-pipeline-playground/sfpd/train.csv + - "name": |- + eval_data + "value": |- + gs://ml-pipeline-playground/sfpd/eval.csv + - "name": |- + schema + "value": |- + gs://ml-pipeline-playground/sfpd/schema.json + - "name": |- + target + "value": |- + resolution + - "name": |- + rounds + "value": |- + 200 + - "name": |- + workers + "value": |- + 2 + - "name": |- + true_label + "value": |- + ACTION + "entrypoint": |- + xgboost-trainer + "onExit": |- + dataproc-delete-cluster + "serviceAccountName": |- + pipeline-runner + "templates": + - "container": + "args": + - |- + --predictions + - |- + {{inputs.parameters.output}}/{{workflow.uid}}/data/predict_output/part-*.csv + - |- + --target_lambda + - "" + - |- + --output + - |- + {{inputs.parameters.output}}/{{workflow.uid}}/data + "command": + - |- + python2 + - |- + /ml/confusion_matrix.py + "image": |- + gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:9670cc1aadfbbed9c52b84ea859ea97aa81213ad + "inputs": + "parameters": + - "name": |- + output + "metadata": + "annotations": + "pipelines.kubeflow.org/component_spec": |- + {"description": "Calculates confusion matrix", "inputs": [{"description": "GCS path of prediction file pattern.", "name": "Predictions", "type": "GCSPath"}, {"default": "", "description": "Text of Python lambda function which computes target value. For example, \"lambda x: x['a'] + x['b']\". If not set, the input must include a \"target\" column.", "name": "Target lambda", "type": "String"}, {"description": "GCS path of the output directory.", "name": "Output dir", "type": "GCSPath"}], "name": "Confusion matrix", "outputs": [{"name": "MLPipeline UI metadata", "type": "UI metadata"}, {"name": "MLPipeline Metrics", "type": "Metrics"}]} + "name": |- + confusion-matrix + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "name": |- + mlpipeline-metrics + "path": |- + /mlpipeline-metrics.json + - "container": + "args": + - |- + kfp_component.google.dataproc + - |- + create_cluster + - |- + --project_id + - |- + {{inputs.parameters.project}} + - |- + --region + - |- + {{inputs.parameters.region}} + - |- + --name + - |- + {{inputs.parameters.cluster_name}} + - |- + --name_prefix + - "" + - |- + --initialization_actions + - |- + ["gs://ml-pipeline-playground/dataproc-example/initialization_actions.sh"] + - |- + --config_bucket + - "" + - |- + --image_version + - |- + 1.2 + - |- + --cluster + - "" + - |- + --wait_interval + - |- + 30 + "command": [] + "env": + - "name": |- + KFP_POD_NAME + "value": |- + {{pod.name}} + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "image": |- + gcr.io/ml-pipeline/ml-pipeline-gcp:9670cc1aadfbbed9c52b84ea859ea97aa81213ad + "inputs": + "parameters": + - "name": |- + cluster_name + - "name": |- + project + - "name": |- + region + "metadata": + "annotations": + "pipelines.kubeflow.org/component_spec": |- + {"description": "Creates a DataProc cluster under a project.\n", "inputs": [{"description": "Required. The ID of the Google Cloud Platform project that the cluster belongs to.", "name": "project_id", "type": "GCPProjectID"}, {"description": "Required. The Cloud Dataproc region in which to handle the request.", "name": "region", "type": "GCPRegion"}, {"default": "", "description": "Optional. The cluster name. Cluster names within a project must be unique. Names of deleted clusters can be reused", "name": "name", "type": "String"}, {"default": "", "description": "Optional. The prefix of the cluster name.", "name": "name_prefix", "type": "String"}, {"default": "", "description": "Optional. List of GCS URIs of executables to execute on each node after config is completed. By default, executables are run on master and all worker nodes.", "name": "initialization_actions", "type": "List"}, {"default": "", "description": "Optional. A Google Cloud Storage bucket used to stage job dependencies, config files, and job driver console output.", "name": "config_bucket", "type": "GCSPath"}, {"default": "", "description": "Optional. The version of software inside the cluster.", "name": "image_version", "type": "String"}, {"default": "", "description": "Optional. The full cluster config. See [full details](https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.clusters#Cluster)", "name": "cluster", "type": "Dict"}, {"default": "30", "description": "Optional. The wait seconds between polling the operation. Defaults to 30.", "name": "wait_interval", "type": "Integer"}], "metadata": {"labels": {"add-pod-env": "true"}}, "name": "dataproc_create_cluster", "outputs": [{"description": "The cluster name of the created cluster.", "name": "cluster_name", "type": "String"}, {"name": "MLPipeline UI metadata", "type": "UI metadata"}]} + "labels": + "add-pod-env": |- + true + "name": |- + dataproc-create-cluster + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "name": |- + dataproc-create-cluster-cluster_name + "path": |- + /tmp/kfp/output/dataproc/cluster_name.txt + - "container": + "args": + - |- + kfp_component.google.dataproc + - |- + delete_cluster + - |- + --project_id + - |- + {{inputs.parameters.project}} + - |- + --region + - |- + {{inputs.parameters.region}} + - |- + --name + - |- + {{inputs.parameters.cluster_name}} + - |- + --wait_interval + - |- + 30 + "command": [] + "env": + - "name": |- + KFP_POD_NAME + "value": |- + {{pod.name}} + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "image": |- + gcr.io/ml-pipeline/ml-pipeline-gcp:9670cc1aadfbbed9c52b84ea859ea97aa81213ad + "inputs": + "parameters": + - "name": |- + cluster_name + - "name": |- + project + - "name": |- + region + "metadata": + "annotations": + "pipelines.kubeflow.org/component_spec": |- + {"description": "Deletes a DataProc cluster.\n", "inputs": [{"description": "Required. The ID of the Google Cloud Platform project that the cluster belongs to.", "name": "project_id", "type": "GCPProjectID"}, {"description": "Required. The Cloud Dataproc region in which to handle the request.", "name": "region", "type": "GCPRegion"}, {"description": "Required. The cluster name to delete.", "name": "name", "type": "String"}, {"default": "30", "description": "Optional. The wait seconds between polling the operation. Defaults to 30.", "name": "wait_interval", "type": "Integer"}], "metadata": {"labels": {"add-pod-env": "true"}}, "name": "dataproc_delete_cluster"} + "labels": + "add-pod-env": |- + true + "name": |- + dataproc-delete-cluster + - "container": + "args": + - |- + kfp_component.google.dataproc + - |- + submit_pyspark_job + - |- + --project_id + - |- + {{inputs.parameters.project}} + - |- + --region + - |- + {{inputs.parameters.region}} + - |- + --cluster_name + - |- + {{inputs.parameters.cluster_name}} + - |- + --main_python_file_uri + - |- + gs://ml-pipeline-playground/dataproc-example/analyze_run.py + - |- + --args + - |- + ["--output", "{{inputs.parameters.output}}/{{workflow.uid}}/data", "--train", "{{inputs.parameters.train_data}}", "--schema", "{{inputs.parameters.schema}}"] + - |- + --pyspark_job + - "" + - |- + --job + - "" + - |- + --wait_interval + - |- + 30 + "command": [] + "env": + - "name": |- + KFP_POD_NAME + "value": |- + {{pod.name}} + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "image": |- + gcr.io/ml-pipeline/ml-pipeline-gcp:9670cc1aadfbbed9c52b84ea859ea97aa81213ad + "inputs": + "parameters": + - "name": |- + cluster_name + - "name": |- + output + - "name": |- + project + - "name": |- + region + - "name": |- + schema + - "name": |- + train_data + "metadata": + "annotations": + "pipelines.kubeflow.org/component_spec": |- + {"description": "Submits a Cloud Dataproc job for running Apache PySpark applications on YARN.", "inputs": [{"description": "Required. The ID of the Google Cloud Platform project that the cluster belongs to.", "name": "project_id", "type": "GCPProjectID"}, {"description": "Required. The Cloud Dataproc region in which to handle the request.", "name": "region", "type": "GCPRegion"}, {"description": "Required. The cluster to run the job.", "name": "cluster_name", "type": "String"}, {"description": "Required. The HCFS URI of the main Python file to use as the driver. Must be a .py file.", "name": "main_python_file_uri", "type": "GCSPath"}, {"default": "", "description": "Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.", "name": "args", "type": "List"}, {"default": "", "description": "Optional. The full payload of a [PySparkJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/PySparkJob).", "name": "pyspark_job", "type": "Dict"}, {"default": "", "description": "Optional. The full payload of a [Dataproc job](https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs).", "name": "job", "type": "Dict"}, {"default": "30", "description": "Optional. The wait seconds between polling the operation. Defaults to 30.", "name": "wait_interval", "type": "Integer"}], "metadata": {"labels": {"add-pod-env": "true"}}, "name": "dataproc_submit_pyspark_job", "outputs": [{"description": "The ID of the created job.", "name": "job_id", "type": "String"}, {"name": "MLPipeline UI metadata", "type": "UI metadata"}]} + "pipelines.kubeflow.org/task_display_name": |- + Analyzer + "labels": + "add-pod-env": |- + true + "name": |- + dataproc-submit-pyspark-job + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "name": |- + dataproc-submit-pyspark-job-job_id + "path": |- + /tmp/kfp/output/dataproc/job_id.txt + - "container": + "args": + - |- + kfp_component.google.dataproc + - |- + submit_pyspark_job + - |- + --project_id + - |- + {{inputs.parameters.project}} + - |- + --region + - |- + {{inputs.parameters.region}} + - |- + --cluster_name + - |- + {{inputs.parameters.cluster_name}} + - |- + --main_python_file_uri + - |- + gs://ml-pipeline-playground/dataproc-example/transform_run.py + - |- + --args + - |- + ["--output", "{{inputs.parameters.output}}/{{workflow.uid}}/data", "--analysis", "{{inputs.parameters.output}}/{{workflow.uid}}/data", "--target", "{{inputs.parameters.target}}", "--train", "{{inputs.parameters.train_data}}", "--eval", "{{inputs.parameters.eval_data}}"] + - |- + --pyspark_job + - "" + - |- + --job + - "" + - |- + --wait_interval + - |- + 30 + "command": [] + "env": + - "name": |- + KFP_POD_NAME + "value": |- + {{pod.name}} + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "image": |- + gcr.io/ml-pipeline/ml-pipeline-gcp:9670cc1aadfbbed9c52b84ea859ea97aa81213ad + "inputs": + "parameters": + - "name": |- + cluster_name + - "name": |- + eval_data + - "name": |- + output + - "name": |- + project + - "name": |- + region + - "name": |- + target + - "name": |- + train_data + "metadata": + "annotations": + "pipelines.kubeflow.org/component_spec": |- + {"description": "Submits a Cloud Dataproc job for running Apache PySpark applications on YARN.", "inputs": [{"description": "Required. The ID of the Google Cloud Platform project that the cluster belongs to.", "name": "project_id", "type": "GCPProjectID"}, {"description": "Required. The Cloud Dataproc region in which to handle the request.", "name": "region", "type": "GCPRegion"}, {"description": "Required. The cluster to run the job.", "name": "cluster_name", "type": "String"}, {"description": "Required. The HCFS URI of the main Python file to use as the driver. Must be a .py file.", "name": "main_python_file_uri", "type": "GCSPath"}, {"default": "", "description": "Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.", "name": "args", "type": "List"}, {"default": "", "description": "Optional. The full payload of a [PySparkJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/PySparkJob).", "name": "pyspark_job", "type": "Dict"}, {"default": "", "description": "Optional. The full payload of a [Dataproc job](https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs).", "name": "job", "type": "Dict"}, {"default": "30", "description": "Optional. The wait seconds between polling the operation. Defaults to 30.", "name": "wait_interval", "type": "Integer"}], "metadata": {"labels": {"add-pod-env": "true"}}, "name": "dataproc_submit_pyspark_job", "outputs": [{"description": "The ID of the created job.", "name": "job_id", "type": "String"}, {"name": "MLPipeline UI metadata", "type": "UI metadata"}]} + "pipelines.kubeflow.org/task_display_name": |- + Transformer + "labels": + "add-pod-env": |- + true + "name": |- + dataproc-submit-pyspark-job-2 + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "name": |- + dataproc-submit-pyspark-job-2-job_id + "path": |- + /tmp/kfp/output/dataproc/job_id.txt + - "container": + "args": + - |- + kfp_component.google.dataproc + - |- + submit_spark_job + - |- + --project_id + - |- + {{inputs.parameters.project}} + - |- + --region + - |- + {{inputs.parameters.region}} + - |- + --cluster_name + - |- + {{inputs.parameters.cluster_name}} + - |- + --main_jar_file_uri + - "" + - |- + --main_class + - |- + ml.dmlc.xgboost4j.scala.example.spark.XGBoostTrainer + - |- + --args + - |- + ["gs://ml-pipeline-playground/trainconfcla.json", "{{inputs.parameters.rounds}}", "{{inputs.parameters.workers}}", "{{inputs.parameters.output}}/{{workflow.uid}}/data", "{{inputs.parameters.target}}", "{{inputs.parameters.output}}/{{workflow.uid}}/data/train/part-*", "{{inputs.parameters.output}}/{{workflow.uid}}/data/eval/part-*", "{{inputs.parameters.output}}/{{workflow.uid}}/data/train_output"] + - |- + --spark_job + - |- + {"jarFileUris": ["gs://ml-pipeline-playground/xgboost4j-example-0.8-SNAPSHOT-jar-with-dependencies.jar"]} + - |- + --job + - "" + - |- + --wait_interval + - |- + 30 + "command": [] + "env": + - "name": |- + KFP_POD_NAME + "value": |- + {{pod.name}} + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "image": |- + gcr.io/ml-pipeline/ml-pipeline-gcp:9670cc1aadfbbed9c52b84ea859ea97aa81213ad + "inputs": + "parameters": + - "name": |- + cluster_name + - "name": |- + output + - "name": |- + project + - "name": |- + region + - "name": |- + rounds + - "name": |- + target + - "name": |- + workers + "metadata": + "annotations": + "pipelines.kubeflow.org/component_spec": |- + {"description": "Submits a Cloud Dataproc job for running Apache Spark applications on YARN.", "inputs": [{"description": "Required. The ID of the Google Cloud Platform project that the cluster belongs to.", "name": "project_id", "type": "GCPProjectID"}, {"description": "Required. The Cloud Dataproc region in which to handle the request.", "name": "region", "type": "GCPRegion"}, {"description": "Required. The cluster to run the job.", "name": "cluster_name", "type": "String"}, {"default": "", "description": "The HCFS URI of the jar file that contains the main class.", "name": "main_jar_file_uri", "type": "GCSPath"}, {"default": "", "description": "The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jarFileUris.", "name": "main_class", "type": "String"}, {"default": "", "description": "Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.", "name": "args", "type": "List"}, {"default": "", "description": "Optional. The full payload of a [SparkJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/SparkJob).", "name": "spark_job", "type": "Dict"}, {"default": "", "description": "Optional. The full payload of a [Dataproc job](https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs).", "name": "job", "type": "Dict"}, {"default": "30", "description": "Optional. The wait seconds between polling the operation. Defaults to 30.", "name": "wait_interval", "type": "Integer"}], "metadata": {"labels": {"add-pod-env": "true"}}, "name": "dataproc_submit_spark_job", "outputs": [{"description": "The ID of the created job.", "name": "job_id", "type": "String"}, {"name": "MLPipeline UI metadata", "type": "UI metadata"}]} + "pipelines.kubeflow.org/task_display_name": |- + Trainer + "labels": + "add-pod-env": |- + true + "name": |- + dataproc-submit-spark-job + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "name": |- + dataproc-submit-spark-job-job_id + "path": |- + /tmp/kfp/output/dataproc/job_id.txt + - "container": + "args": + - |- + kfp_component.google.dataproc + - |- + submit_spark_job + - |- + --project_id + - |- + {{inputs.parameters.project}} + - |- + --region + - |- + {{inputs.parameters.region}} + - |- + --cluster_name + - |- + {{inputs.parameters.cluster_name}} + - |- + --main_jar_file_uri + - "" + - |- + --main_class + - |- + ml.dmlc.xgboost4j.scala.example.spark.XGBoostPredictor + - |- + --args + - |- + ["{{inputs.parameters.output}}/{{workflow.uid}}/data/train_output", "{{inputs.parameters.output}}/{{workflow.uid}}/data/eval/part-*", "{{inputs.parameters.output}}/{{workflow.uid}}/data", "{{inputs.parameters.target}}", "{{inputs.parameters.output}}/{{workflow.uid}}/data/predict_output"] + - |- + --spark_job + - |- + {"jarFileUris": ["gs://ml-pipeline-playground/xgboost4j-example-0.8-SNAPSHOT-jar-with-dependencies.jar"]} + - |- + --job + - "" + - |- + --wait_interval + - |- + 30 + "command": [] + "env": + - "name": |- + KFP_POD_NAME + "value": |- + {{pod.name}} + - "name": |- + KFP_POD_NAME + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.name + - "name": |- + KFP_NAMESPACE + "valueFrom": + "fieldRef": + "fieldPath": |- + metadata.namespace + "image": |- + gcr.io/ml-pipeline/ml-pipeline-gcp:9670cc1aadfbbed9c52b84ea859ea97aa81213ad + "inputs": + "parameters": + - "name": |- + cluster_name + - "name": |- + output + - "name": |- + project + - "name": |- + region + - "name": |- + target + "metadata": + "annotations": + "pipelines.kubeflow.org/component_spec": |- + {"description": "Submits a Cloud Dataproc job for running Apache Spark applications on YARN.", "inputs": [{"description": "Required. The ID of the Google Cloud Platform project that the cluster belongs to.", "name": "project_id", "type": "GCPProjectID"}, {"description": "Required. The Cloud Dataproc region in which to handle the request.", "name": "region", "type": "GCPRegion"}, {"description": "Required. The cluster to run the job.", "name": "cluster_name", "type": "String"}, {"default": "", "description": "The HCFS URI of the jar file that contains the main class.", "name": "main_jar_file_uri", "type": "GCSPath"}, {"default": "", "description": "The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jarFileUris.", "name": "main_class", "type": "String"}, {"default": "", "description": "Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.", "name": "args", "type": "List"}, {"default": "", "description": "Optional. The full payload of a [SparkJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/SparkJob).", "name": "spark_job", "type": "Dict"}, {"default": "", "description": "Optional. The full payload of a [Dataproc job](https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs).", "name": "job", "type": "Dict"}, {"default": "30", "description": "Optional. The wait seconds between polling the operation. Defaults to 30.", "name": "wait_interval", "type": "Integer"}], "metadata": {"labels": {"add-pod-env": "true"}}, "name": "dataproc_submit_spark_job", "outputs": [{"description": "The ID of the created job.", "name": "job_id", "type": "String"}, {"name": "MLPipeline UI metadata", "type": "UI metadata"}]} + "pipelines.kubeflow.org/task_display_name": |- + Predictor + "labels": + "add-pod-env": |- + true + "name": |- + dataproc-submit-spark-job-2 + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "name": |- + dataproc-submit-spark-job-2-job_id + "path": |- + /tmp/kfp/output/dataproc/job_id.txt + - "dag": + "tasks": + - "arguments": + "parameters": + - "name": |- + output + "value": |- + {{inputs.parameters.output}} + "dependencies": + - |- + dataproc-submit-spark-job-2 + "name": |- + confusion-matrix + "template": |- + confusion-matrix + - "arguments": + "parameters": + - "name": |- + cluster_name + "value": |- + {{inputs.parameters.cluster_name}} + - "name": |- + project + "value": |- + {{inputs.parameters.project}} + - "name": |- + region + "value": |- + {{inputs.parameters.region}} + "name": |- + dataproc-create-cluster + "template": |- + dataproc-create-cluster + - "arguments": + "parameters": + - "name": |- + cluster_name + "value": |- + {{inputs.parameters.cluster_name}} + - "name": |- + output + "value": |- + {{inputs.parameters.output}} + - "name": |- + project + "value": |- + {{inputs.parameters.project}} + - "name": |- + region + "value": |- + {{inputs.parameters.region}} + - "name": |- + schema + "value": |- + {{inputs.parameters.schema}} + - "name": |- + train_data + "value": |- + {{inputs.parameters.train_data}} + "dependencies": + - |- + dataproc-create-cluster + "name": |- + dataproc-submit-pyspark-job + "template": |- + dataproc-submit-pyspark-job + - "arguments": + "parameters": + - "name": |- + cluster_name + "value": |- + {{inputs.parameters.cluster_name}} + - "name": |- + eval_data + "value": |- + {{inputs.parameters.eval_data}} + - "name": |- + output + "value": |- + {{inputs.parameters.output}} + - "name": |- + project + "value": |- + {{inputs.parameters.project}} + - "name": |- + region + "value": |- + {{inputs.parameters.region}} + - "name": |- + target + "value": |- + {{inputs.parameters.target}} + - "name": |- + train_data + "value": |- + {{inputs.parameters.train_data}} + "dependencies": + - |- + dataproc-submit-pyspark-job + "name": |- + dataproc-submit-pyspark-job-2 + "template": |- + dataproc-submit-pyspark-job-2 + - "arguments": + "parameters": + - "name": |- + cluster_name + "value": |- + {{inputs.parameters.cluster_name}} + - "name": |- + output + "value": |- + {{inputs.parameters.output}} + - "name": |- + project + "value": |- + {{inputs.parameters.project}} + - "name": |- + region + "value": |- + {{inputs.parameters.region}} + - "name": |- + rounds + "value": |- + {{inputs.parameters.rounds}} + - "name": |- + target + "value": |- + {{inputs.parameters.target}} + - "name": |- + workers + "value": |- + {{inputs.parameters.workers}} + "dependencies": + - |- + dataproc-submit-pyspark-job-2 + "name": |- + dataproc-submit-spark-job + "template": |- + dataproc-submit-spark-job + - "arguments": + "parameters": + - "name": |- + cluster_name + "value": |- + {{inputs.parameters.cluster_name}} + - "name": |- + output + "value": |- + {{inputs.parameters.output}} + - "name": |- + project + "value": |- + {{inputs.parameters.project}} + - "name": |- + region + "value": |- + {{inputs.parameters.region}} + - "name": |- + target + "value": |- + {{inputs.parameters.target}} + "dependencies": + - |- + dataproc-submit-spark-job + "name": |- + dataproc-submit-spark-job-2 + "template": |- + dataproc-submit-spark-job-2 + - "arguments": + "parameters": + - "name": |- + output + "value": |- + {{inputs.parameters.output}} + - "name": |- + true_label + "value": |- + {{inputs.parameters.true_label}} + "dependencies": + - |- + dataproc-submit-spark-job-2 + "name": |- + roc-curve + "template": |- + roc-curve + "inputs": + "parameters": + - "name": |- + cluster_name + - "name": |- + eval_data + - "name": |- + output + - "name": |- + project + - "name": |- + region + - "name": |- + rounds + - "name": |- + schema + - "name": |- + target + - "name": |- + train_data + - "name": |- + true_label + - "name": |- + workers + "name": |- + exit-handler-1 + - "container": + "args": + - |- + --predictions + - |- + {{inputs.parameters.output}}/{{workflow.uid}}/data/predict_output/part-*.csv + - |- + --trueclass + - |- + {{inputs.parameters.true_label}} + - |- + --true_score_column + - |- + {{inputs.parameters.true_label}} + - |- + --target_lambda + - "" + - |- + --output + - |- + {{inputs.parameters.output}}/{{workflow.uid}}/data + "command": + - |- + python2 + - |- + /ml/roc.py + "image": |- + gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:9670cc1aadfbbed9c52b84ea859ea97aa81213ad + "inputs": + "parameters": + - "name": |- + output + - "name": |- + true_label + "metadata": + "annotations": + "pipelines.kubeflow.org/component_spec": |- + {"description": "Calculates Receiver Operating Characteristic curve. See https://en.wikipedia.org/wiki/Receiver_operating_characteristic", "inputs": [{"description": "GCS path of prediction file pattern.", "name": "Predictions dir", "type": "GCSPath"}, {"default": "true", "description": "The true class label for the sample. Default is \"true\".", "name": "True class", "type": "String"}, {"default": "true", "description": "The name of the column for positive probability.", "name": "True score column", "type": "String"}, {"default": "", "description": "Text of Python lambda function which returns boolean value indicating whether the classification result is correct.\\nFor example, \"lambda x: x['a'] and x['b']\". If missing, input must have a \"target\" column.", "name": "Target lambda", "type": "String"}, {"description": "GCS path of the output directory.", "name": "Output dir", "type": "GCSPath"}], "name": "ROC curve", "outputs": [{"name": "MLPipeline UI metadata", "type": "UI metadata"}, {"name": "MLPipeline Metrics", "type": "Metrics"}]} + "name": |- + roc-curve + "outputs": + "artifacts": + - "name": |- + mlpipeline-ui-metadata + "path": |- + /mlpipeline-ui-metadata.json + - "name": |- + mlpipeline-metrics + "path": |- + /mlpipeline-metrics.json + - "dag": + "tasks": + - "arguments": + "parameters": + - "name": |- + cluster_name + "value": |- + {{inputs.parameters.cluster_name}} + - "name": |- + eval_data + "value": |- + {{inputs.parameters.eval_data}} + - "name": |- + output + "value": |- + {{inputs.parameters.output}} + - "name": |- + project + "value": |- + {{inputs.parameters.project}} + - "name": |- + region + "value": |- + {{inputs.parameters.region}} + - "name": |- + rounds + "value": |- + {{inputs.parameters.rounds}} + - "name": |- + schema + "value": |- + {{inputs.parameters.schema}} + - "name": |- + target + "value": |- + {{inputs.parameters.target}} + - "name": |- + train_data + "value": |- + {{inputs.parameters.train_data}} + - "name": |- + true_label + "value": |- + {{inputs.parameters.true_label}} + - "name": |- + workers + "value": |- + {{inputs.parameters.workers}} + "name": |- + exit-handler-1 + "template": |- + exit-handler-1 + "inputs": + "parameters": + - "name": |- + cluster_name + - "name": |- + eval_data + - "name": |- + output + - "name": |- + project + - "name": |- + region + - "name": |- + rounds + - "name": |- + schema + - "name": |- + target + - "name": |- + train_data + - "name": |- + true_label + - "name": |- + workers + "name": |- + xgboost-trainer diff --git a/sdk/python/kfp/containers/_container_builder.py b/sdk/python/kfp/containers/_container_builder.py index 3e3ee925281..4df0e4990c0 100644 --- a/sdk/python/kfp/containers/_container_builder.py +++ b/sdk/python/kfp/containers/_container_builder.py @@ -133,20 +133,6 @@ def _generate_kaniko_spec(self, context, docker_filename, target_image): '--digest-file=/dev/termination-log', # This is suggested by the Kaniko devs as a way to return the image digest from Kaniko Pod. See https://github.com/GoogleContainerTools/kaniko#--digest-file ], 'image': 'gcr.io/kaniko-project/executor@sha256:78d44ec4e9cb5545d7f85c1924695c89503ded86a59f92c7ae658afa3cff5400', - 'env': [{ - 'name': 'GOOGLE_APPLICATION_CREDENTIALS', - 'value': '/secret/gcp-credentials/user-gcp-sa.json' - }], - 'volumeMounts': [{ - 'mountPath': '/secret/gcp-credentials', - 'name': 'gcp-credentials', - }], - }], - 'volumes': [{ - 'name': 'gcp-credentials', - 'secret': { - 'secretName': 'user-gcp-sa', - }, }], 'serviceAccountName': 'default'} } diff --git a/sdk/python/kfp/containers/_k8s_job_helper.py b/sdk/python/kfp/containers/_k8s_job_helper.py index 46a5be9f949..73bc487c8f9 100644 --- a/sdk/python/kfp/containers/_k8s_job_helper.py +++ b/sdk/python/kfp/containers/_k8s_job_helper.py @@ -48,24 +48,10 @@ def _create_k8s_job(self, yaml_spec): annotations=yaml_spec['metadata']['annotations'])) container = k8s_client.V1Container(name = yaml_spec['spec']['containers'][0]['name'], image = yaml_spec['spec']['containers'][0]['image'], - args = yaml_spec['spec']['containers'][0]['args'], - volume_mounts = [k8s_client.V1VolumeMount( - name=yaml_spec['spec']['containers'][0]['volumeMounts'][0]['name'], - mount_path=yaml_spec['spec']['containers'][0]['volumeMounts'][0]['mountPath'], - )], - env = [k8s_client.V1EnvVar( - name=yaml_spec['spec']['containers'][0]['env'][0]['name'], - value=yaml_spec['spec']['containers'][0]['env'][0]['value'], - )]) + args = yaml_spec['spec']['containers'][0]['args']) pod.spec = k8s_client.V1PodSpec(restart_policy=yaml_spec['spec']['restartPolicy'], containers = [container], - service_account_name=yaml_spec['spec']['serviceAccountName'], - volumes=[k8s_client.V1Volume( - name=yaml_spec['spec']['volumes'][0]['name'], - secret=k8s_client.V1SecretVolumeSource( - secret_name=yaml_spec['spec']['volumes'][0]['secret']['secretName'], - ) - )]) + service_account_name=yaml_spec['spec']['serviceAccountName']) try: api_response = self._corev1.create_namespaced_pod(yaml_spec['metadata']['namespace'], pod) return api_response.metadata.name, True diff --git a/sdk/python/tests/compiler/testdata/kaniko.basic.yaml b/sdk/python/tests/compiler/testdata/kaniko.basic.yaml index 149f137c9bf..7f0b760f480 100644 --- a/sdk/python/tests/compiler/testdata/kaniko.basic.yaml +++ b/sdk/python/tests/compiler/testdata/kaniko.basic.yaml @@ -32,13 +32,3 @@ spec: "--destination=gcr.io/mlpipeline/kaniko_image:latest", "--digest-file=/dev/termination-log", ] - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - mountPath: /secret/gcp-credentials - name: gcp-credentials - volumes: - - name: gcp-credentials - secret: - secretName: user-gcp-sa \ No newline at end of file diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py index 2b8500ec963..ca0173a06da 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py @@ -30,7 +30,7 @@ def volume_snapshotop_sequential(url): step1 = dsl.ContainerOp( name="step1_ingest", - image="google/cloud-sdk:216.0.0", + image="google/cloud-sdk:272.0.0", command=["sh", "-c"], arguments=["mkdir /data/step1 && " "gsutil cat %s | gzip -c >/data/step1/file1.gz" % url], diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml index 4b23521adeb..3ad51af4893 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml @@ -35,7 +35,7 @@ spec: command: - sh - -c - image: google/cloud-sdk:216.0.0 + image: google/cloud-sdk:272.0.0 volumeMounts: - mountPath: /data name: create-volume diff --git a/test/build_image.yaml b/test/build_image.yaml index b4684aeecb1..223f63637cc 100644 --- a/test/build_image.yaml +++ b/test/build_image.yaml @@ -18,15 +18,11 @@ metadata: generateName: build-images- spec: entrypoint: build-images - volumes: - - name: gcp-credentials - secret: - secretName: user-gcp-sa arguments: parameters: - name: image-build-context-gcs-uri - name: image-builder-image - value: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4 + value: gcr.io/ml-pipeline-test/image-builder:v20191122-0.1.25-582-g7908980c - name: api-image - name: frontend-image - name: scheduledworkflow-image @@ -123,11 +119,6 @@ spec: env: - name: DOCKER_HOST value: 127.0.0.1 - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - name: gcp-credentials - mountPath: /secret/gcp-credentials sidecars: - name: dind image: docker:17.10-dind diff --git a/test/component_test.yaml b/test/component_test.yaml index bdf9fb0aeea..a742a2df6c3 100644 --- a/test/component_test.yaml +++ b/test/component_test.yaml @@ -18,15 +18,11 @@ metadata: generateName: sample-test- spec: entrypoint: sample-test - volumes: - - name: gcp-credentials - secret: - secretName: user-gcp-sa arguments: parameters: - name: image-build-context-gcs-uri - name: image-builder-image - value: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4 + value: gcr.io/ml-pipeline-test/image-builder:v20191122-0.1.25-582-g7908980c - name: commit-sha - name: component-image-prefix - name: target-image-prefix @@ -79,6 +75,8 @@ spec: # Build and push image - name: build-image-by-dockerfile + retryStrategy: + limit: 1 inputs: parameters: # GCS URI prefix pointing to a .tar.gz archive of Docker build context @@ -107,11 +105,6 @@ spec: env: - name: DOCKER_HOST value: 127.0.0.1 - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - name: gcp-credentials - mountPath: /secret/gcp-credentials sidecars: - name: dind image: docker:17.10-dind @@ -142,10 +135,3 @@ spec: "--namespace", "{{inputs.parameters.namespace}}", "--test-name", "{{inputs.parameters.test-name}}", ] - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - name: gcp-credentials - mountPath: /secret/gcp-credentials - diff --git a/test/deploy-cluster.sh b/test/deploy-cluster.sh index aac48f5177c..b96bf55aa08 100755 --- a/test/deploy-cluster.sh +++ b/test/deploy-cluster.sh @@ -22,6 +22,7 @@ set -ex TEST_CLUSTER_PREFIX=${WORKFLOW_FILE%.*} TEST_CLUSTER_DEFAULT=$(echo $TEST_CLUSTER_PREFIX | cut -d _ -f 1)-${COMMIT_SHA:0:7}-${RANDOM} TEST_CLUSTER=${TEST_CLUSTER:-${TEST_CLUSTER_DEFAULT}} +ENABLE_WORKLOAD_IDENTITY=${ENABLE_WORKLOAD_IDENTITY:-false} SHOULD_CLEANUP_CLUSTER=false function clean_up { @@ -57,14 +58,20 @@ if gcloud container clusters describe ${TEST_CLUSTER} &>/dev/null; then else echo "Creating a new test cluster: ${TEST_CLUSTER}" SHOULD_CLEANUP_CLUSTER=true - # "storage-rw" is needed to allow VMs to push to gcr.io - # reference: https://cloud.google.com/compute/docs/access/service-accounts#accesscopesiam - SCOPE_ARG="--scopes=storage-rw,cloud-platform" # Machine type and cluster size is the same as kubeflow deployment to # easily compare performance. We can reduce usage later. NODE_POOL_CONFIG_ARG="--num-nodes=2 --machine-type=n1-standard-8 \ --enable-autoscaling --max-nodes=8 --min-nodes=2" - gcloud container clusters create ${TEST_CLUSTER} ${SCOPE_ARG} ${NODE_POOL_CONFIG_ARG} + if [ "$ENABLE_WORKLOAD_IDENTITY" = true ]; then + WI_ARG="--identity-namespace=$PROJECT.svc.id.goog" + SCOPE_ARG= + else + WI_ARG= + # "storage-rw" is needed to allow VMs to push to gcr.io when using default GCE service account. + # reference: https://cloud.google.com/compute/docs/access/service-accounts#accesscopesiam + SCOPE_ARG="--scopes=storage-rw,cloud-platform" + fi + gcloud beta container clusters create ${TEST_CLUSTER} ${SCOPE_ARG} ${NODE_POOL_CONFIG_ARG} ${WI_ARG} fi gcloud container clusters get-credentials ${TEST_CLUSTER} @@ -74,13 +81,15 @@ gcloud container clusters get-credentials ${TEST_CLUSTER} kubectl delete namespace ${NAMESPACE} --wait || echo "No need to delete ${NAMESPACE} namespace. It doesn't exist." kubectl create namespace ${NAMESPACE} --dry-run -o yaml | kubectl apply -f - -if [ -z $SA_KEY_FILE ]; then - SA_KEY_FILE=${DIR}/key.json - # The service account key is for default VM service account. - # ref: https://cloud.google.com/compute/docs/access/service-accounts#compute_engine_default_service_account - # It was generated by the following command - # `gcloud iam service-accounts keys create $SA_KEY_FILE --iam-account ${VM_SERVICE_ACCOUNT}` - # Because there's a limit of 10 keys per service account, we are reusing the same key stored in the following bucket. - gsutil cp "gs://ml-pipeline-test-keys/ml-pipeline-test-sa-key.json" $SA_KEY_FILE +if [ "$ENABLE_WORKLOAD_IDENTITY" != true ]; then + if [ -z $SA_KEY_FILE ]; then + SA_KEY_FILE=${DIR}/key.json + # The service account key is for default VM service account. + # ref: https://cloud.google.com/compute/docs/access/service-accounts#compute_engine_default_service_account + # It was generated by the following command + # `gcloud iam service-accounts keys create $SA_KEY_FILE --iam-account ${VM_SERVICE_ACCOUNT}` + # Because there's a limit of 10 keys per service account, we are reusing the same key stored in the following bucket. + gsutil cp "gs://ml-pipeline-test-keys/ml-pipeline-test-sa-key.json" $SA_KEY_FILE + fi + kubectl create secret -n ${NAMESPACE} generic user-gcp-sa --from-file=user-gcp-sa.json=$SA_KEY_FILE --dry-run -o yaml | kubectl apply -f - fi -kubectl create secret -n ${NAMESPACE} generic user-gcp-sa --from-file=user-gcp-sa.json=$SA_KEY_FILE --dry-run -o yaml | kubectl apply -f - diff --git a/test/deploy-pipeline-lite.sh b/test/deploy-pipeline-lite.sh index 1c73341423e..e7292a71b31 100755 --- a/test/deploy-pipeline-lite.sh +++ b/test/deploy-pipeline-lite.sh @@ -19,7 +19,9 @@ set -ex # Env inputs: # * $GCR_IMAGE_BASE_DIR # * $GCR_IMAGE_TAG +# * ENABLE_WORKLOAD_IDENTITY GCR_IMAGE_TAG=${GCR_IMAGE_TAG:-latest} +ENABLE_WORKLOAD_IDENTITY=${ENABLE_WORKLOAD_IDENTITY:-false} if ! which kustomize; then # Download kustomize cli tool @@ -49,8 +51,28 @@ kustomize edit set image gcr.io/ml-pipeline/inverse-proxy-agent=${GCR_IMAGE_BASE cat kustomization.yaml kustomize build . | kubectl apply -f - + # show current info echo "Status of pods after kubectl apply" kubectl get pods -n ${NAMESPACE} +if [ "$ENABLE_WORKLOAD_IDENTITY" = true ]; then + # Use static GSAs for testing, so we don't need to GC them. + export SYSTEM_GSA="test-kfp-system" + export USER_GSA="test-kfp-user" + + yes | PROJECT_ID=$PROJECT CLUSTER_NAME=$TEST_CLUSTER NAMESPACE=$NAMESPACE \ + ${DIR}/../manifests/kustomize/gcp-workload-identity-setup.sh + + gcloud projects add-iam-policy-binding $PROJECT \ + --member="serviceAccount:$SYSTEM_GSA@$PROJECT.iam.gserviceaccount.com" \ + --role="roles/editor" + gcloud projects add-iam-policy-binding $PROJECT \ + --member="serviceAccount:$USER_GSA@$PROJECT.iam.gserviceaccount.com" \ + --role="roles/editor" + + source "$DIR/../manifests/kustomize/wi-utils.sh" + verify_workload_identity_binding "pipeline-runner" $NAMESPACE +fi + popd diff --git a/test/e2e_test_gke_v2.yaml b/test/e2e_test_gke_v2.yaml index f90b1ed6cfd..2a86b76e0e9 100644 --- a/test/e2e_test_gke_v2.yaml +++ b/test/e2e_test_gke_v2.yaml @@ -18,15 +18,11 @@ metadata: generateName: integration-test- spec: entrypoint: integration-test - volumes: - - name: gcp-credentials - secret: - secretName: user-gcp-sa arguments: parameters: - name: image-build-context-gcs-uri - name: image-builder-image - value: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4 + value: gcr.io/ml-pipeline-test/image-builder:v20191122-0.1.25-582-g7908980c - name: target-image-prefix - name: test-results-gcs-dir - name: initialization-test-image-suffix @@ -140,6 +136,8 @@ spec: # Build and push image - name: build-image + retryStrategy: + limit: 1 inputs: parameters: # GCS URI prefix pointing to a .tar.gz archive of Docker build context @@ -168,11 +166,6 @@ spec: env: - name: DOCKER_HOST value: 127.0.0.1 - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - name: gcp-credentials - mountPath: /secret/gcp-credentials sidecars: - name: dind image: docker:17.10-dind @@ -190,12 +183,6 @@ spec: args: [ "--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}", ] - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - name: gcp-credentials - mountPath: /secret/gcp-credentials - name: run-api-integration-tests inputs: @@ -207,12 +194,6 @@ spec: args: [ "--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}", ] - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - name: gcp-credentials - mountPath: /secret/gcp-credentials - name: run-frontend-integration-tests inputs: @@ -224,12 +205,6 @@ spec: args: [ "--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}", ] - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - name: gcp-credentials - mountPath: /secret/gcp-credentials - name: run-basic-e2e-tests inputs: @@ -246,9 +221,3 @@ spec: "--namespace", "{{inputs.parameters.namespace}}", "--test-name", "{{inputs.parameters.test-name}}", ] - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - name: gcp-credentials - mountPath: /secret/gcp-credentials diff --git a/test/install-argo.sh b/test/install-argo.sh index 1eca2a3b9ce..d876d0d87ff 100755 --- a/test/install-argo.sh +++ b/test/install-argo.sh @@ -16,6 +16,8 @@ set -ex +ENABLE_WORKLOAD_IDENTITY=${ENABLE_WORKLOAD_IDENTITY:-false} + # Tests work without these lines. TODO: Verify and remove these lines kubectl config set-context $(kubectl config current-context) --namespace=default echo "Add necessary cluster role bindings" @@ -38,8 +40,25 @@ fi # kubectl create ns argo --dry-run -o yaml | kubectl apply -f - # kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/$ARGO_VERSION/manifests/install.yaml +ARGO_KSA="test-runner" + # Some workflows are deployed to the non-default namespace where the GCP credential secret is stored # In this case, the default service account in that namespace doesn't have enough permission echo "add service account for running the test workflow" -kubectl create serviceaccount test-runner -n ${NAMESPACE} --dry-run -o yaml | kubectl apply -f - -kubectl create clusterrolebinding test-admin-binding --clusterrole=cluster-admin --serviceaccount=${NAMESPACE}:test-runner --dry-run -o yaml | kubectl apply -f - +kubectl create serviceaccount ${ARGO_KSA} -n ${NAMESPACE} --dry-run -o yaml | kubectl apply -f - +kubectl create clusterrolebinding test-admin-binding --clusterrole=cluster-admin --serviceaccount=${NAMESPACE}:${ARGO_KSA} --dry-run -o yaml | kubectl apply -f - + +if [ "$ENABLE_WORKLOAD_IDENTITY" = true ]; then + ARGO_GSA="test-argo" + # Util library including create_gsa_if_not_present and bind_gsa_and_ksa functions. + source "$DIR/../manifests/kustomize/wi-utils.sh" + create_gsa_if_not_present $ARGO_GSA + + gcloud projects add-iam-policy-binding $PROJECT \ + --member="serviceAccount:$ARGO_GSA@$PROJECT.iam.gserviceaccount.com" \ + --role="roles/editor" \ + > /dev/null # hide verbose output + bind_gsa_and_ksa $ARGO_GSA $ARGO_KSA $PROJECT $NAMESPACE + + verify_workload_identity_binding $ARGO_KSA $NAMESPACE +fi diff --git a/test/postsubmit-tests-with-pipeline-deployment.sh b/test/postsubmit-tests-with-pipeline-deployment.sh index 2cbf653e64f..34fecb1c06d 100755 --- a/test/postsubmit-tests-with-pipeline-deployment.sh +++ b/test/postsubmit-tests-with-pipeline-deployment.sh @@ -35,6 +35,7 @@ GCR_IMAGE_BASE_DIR=gcr.io/ml-pipeline-test TARGET_IMAGE_BASE_DIR=gcr.io/ml-pipeline-test/${PULL_BASE_SHA} TIMEOUT_SECONDS=1800 NAMESPACE=kubeflow +ENABLE_WORKLOAD_IDENTITY=true COMMIT_SHA="$PULL_BASE_SHA" while [ "$1" != "" ]; do diff --git a/test/presubmit-tests-with-pipeline-deployment.sh b/test/presubmit-tests-with-pipeline-deployment.sh index 33292da6f46..d3602046fea 100755 --- a/test/presubmit-tests-with-pipeline-deployment.sh +++ b/test/presubmit-tests-with-pipeline-deployment.sh @@ -33,6 +33,7 @@ PROJECT=ml-pipeline-test TEST_RESULT_BUCKET=ml-pipeline-test TIMEOUT_SECONDS=1800 NAMESPACE=kubeflow +ENABLE_WORKLOAD_IDENTITY=true while [ "$1" != "" ]; do case $1 in diff --git a/test/sample-test/Dockerfile b/test/sample-test/Dockerfile index 662e9fbf3f1..0b789b99e52 100644 --- a/test/sample-test/Dockerfile +++ b/test/sample-test/Dockerfile @@ -1,5 +1,6 @@ # This image has the script to kick off the ML pipeline sample e2e test, +# TODO: use google/cloud-sdk:272.0.0, currently some python dependencies break if we use the new version FROM google/cloud-sdk:236.0.0 RUN apt-get update -y && \ diff --git a/test/sample_test.yaml b/test/sample_test.yaml index bef7d2011a2..c39c3e72af2 100644 --- a/test/sample_test.yaml +++ b/test/sample_test.yaml @@ -18,15 +18,11 @@ metadata: generateName: sample-test- spec: entrypoint: sample-test - volumes: - - name: gcp-credentials - secret: - secretName: user-gcp-sa arguments: parameters: - name: image-build-context-gcs-uri - name: image-builder-image - value: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4 + value: gcr.io/ml-pipeline-test/image-builder:v20191122-0.1.25-582-g7908980c - name: target-image-prefix - name: test-results-gcs-dir - name: sample-tests-image-suffix @@ -91,6 +87,8 @@ spec: - parameterized_tfx_oss # Build and push image - name: build-image-by-dockerfile + retryStrategy: + limit: 1 inputs: parameters: # GCS URI prefix pointing to a .tar.gz archive of Docker build context @@ -119,11 +117,6 @@ spec: env: - name: DOCKER_HOST value: 127.0.0.1 - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - name: gcp-credentials - mountPath: /secret/gcp-credentials sidecars: - name: dind image: docker:17.10-dind @@ -148,10 +141,3 @@ spec: "--target-image-prefix", "{{inputs.parameters.target-image-prefix}}", "--test-name", "{{inputs.parameters.test-name}}", ] - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - volumeMounts: - - name: gcp-credentials - mountPath: /secret/gcp-credentials - diff --git a/test/test-prep.sh b/test/test-prep.sh index 5a941e488e8..5e95dd36cde 100755 --- a/test/test-prep.sh +++ b/test/test-prep.sh @@ -16,11 +16,12 @@ set -x +ZONE=${ZONE:-us-east1-b} if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then # activating the service account gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" fi -gcloud config set compute/zone us-east1-b +gcloud config set compute/zone ${ZONE} gcloud config set core/project ${PROJECT} #Uploading the source code to GCS: