diff --git a/components/gcp/bigquery/query/component.yaml b/components/gcp/bigquery/query/component.yaml index 3dc4e2386c2..2d56141185c 100644 --- a/components/gcp/bigquery/query/component.yaml +++ b/components/gcp/bigquery/query/component.yaml @@ -68,10 +68,10 @@ implementation: --table_id, {inputValue: table_id}, --dataset_location, {inputValue: dataset_location}, --output_gcs_path, {inputValue: output_gcs_path}, - --job_config, {inputValue: job_config} + --job_config, {inputValue: job_config}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: output_gcs_path: /tmp/kfp/output/bigquery/query-output-path.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/container/component_sdk/python/kfp_component/launcher/__main__.py b/components/gcp/container/component_sdk/python/kfp_component/launcher/__main__.py index 7d9937935c9..0e767679add 100644 --- a/components/gcp/container/component_sdk/python/kfp_component/launcher/__main__.py +++ b/components/gcp/container/component_sdk/python/kfp_component/launcher/__main__.py @@ -15,6 +15,7 @@ import argparse import fire import importlib +import os import sys import logging from .launcher import launch @@ -26,8 +27,14 @@ def main(): description='Launch a python module or file.') parser.add_argument('file_or_module', type=str, help='Either a python file path or a module name.') + parser.add_argument('--ui-metadata-path', type=str, + help='Path for the file where the mlpipeline-ui-metadata.json data should be written.') parser.add_argument('args', nargs=argparse.REMAINDER) args = parser.parse_args() + + if args.ui_metadata_path: + os.environ['KFP_UI_METADATA_PATH'] = args.ui_metadata_path + launch(args.file_or_module, args.args) if __name__ == '__main__': diff --git a/components/gcp/dataflow/launch_python/component.yaml b/components/gcp/dataflow/launch_python/component.yaml index cc86a09a413..00940de5338 100644 --- a/components/gcp/dataflow/launch_python/component.yaml +++ b/components/gcp/dataflow/launch_python/component.yaml @@ -61,10 +61,10 @@ implementation: --staging_dir, {inputValue: staging_dir}, --requirements_file_path, {inputValue: requirements_file_path}, --args, {inputValue: args}, - --wait_interval, {inputValue: wait_interval} + --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataflow/job_id.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataflow/launch_template/component.yaml b/components/gcp/dataflow/launch_template/component.yaml index 6da76449fa8..0d0c6b75eab 100644 --- a/components/gcp/dataflow/launch_template/component.yaml +++ b/components/gcp/dataflow/launch_template/component.yaml @@ -73,9 +73,9 @@ implementation: --validate_only, {inputValue: validate_only}, --staging_dir, {inputValue: staging_dir}, --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataflow/job_id.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/create_cluster/component.yaml b/components/gcp/dataproc/create_cluster/component.yaml index f232e4c155f..585a71a8079 100644 --- a/components/gcp/dataproc/create_cluster/component.yaml +++ b/components/gcp/dataproc/create_cluster/component.yaml @@ -81,10 +81,10 @@ implementation: --config_bucket, {inputValue: config_bucket}, --image_version, {inputValue: image_version}, --cluster, {inputValue: cluster}, - --wait_interval, {inputValue: wait_interval} + --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: cluster_name: /tmp/kfp/output/dataproc/cluster_name.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_hadoop_job/component.yaml b/components/gcp/dataproc/submit_hadoop_job/component.yaml index 93b7cadb5c0..a783fc56a0c 100644 --- a/components/gcp/dataproc/submit_hadoop_job/component.yaml +++ b/components/gcp/dataproc/submit_hadoop_job/component.yaml @@ -91,10 +91,10 @@ implementation: --args, {inputValue: args}, --hadoop_job, {inputValue: hadoop_job}, --job, {inputValue: job}, - --wait_interval, {inputValue: wait_interval} + --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_hive_job/component.yaml b/components/gcp/dataproc/submit_hive_job/component.yaml index 2464d08c50f..5041aebda26 100644 --- a/components/gcp/dataproc/submit_hive_job/component.yaml +++ b/components/gcp/dataproc/submit_hive_job/component.yaml @@ -86,10 +86,10 @@ implementation: --script_variables, {inputValue: script_variables}, --hive_job, {inputValue: hive_job}, --job, {inputValue: job}, - --wait_interval, {inputValue: wait_interval} + --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_pig_job/component.yaml b/components/gcp/dataproc/submit_pig_job/component.yaml index cb2ffe8dff3..586dc6c1144 100644 --- a/components/gcp/dataproc/submit_pig_job/component.yaml +++ b/components/gcp/dataproc/submit_pig_job/component.yaml @@ -86,10 +86,10 @@ implementation: --script_variables, {inputValue: script_variables}, --pig_job, {inputValue: pig_job}, --job, {inputValue: job}, - --wait_interval, {inputValue: wait_interval} + --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_pyspark_job/component.yaml b/components/gcp/dataproc/submit_pyspark_job/component.yaml index 4f256b002b7..d81b9346163 100644 --- a/components/gcp/dataproc/submit_pyspark_job/component.yaml +++ b/components/gcp/dataproc/submit_pyspark_job/component.yaml @@ -79,10 +79,10 @@ implementation: --args, {inputValue: args}, --pyspark_job, {inputValue: pyspark_job}, --job, {inputValue: job}, - --wait_interval, {inputValue: wait_interval} + --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_spark_job/component.yaml b/components/gcp/dataproc/submit_spark_job/component.yaml index f915b7bb2ef..af86ca066c9 100644 --- a/components/gcp/dataproc/submit_spark_job/component.yaml +++ b/components/gcp/dataproc/submit_spark_job/component.yaml @@ -87,10 +87,10 @@ implementation: --args, {inputValue: args}, --spark_job, {inputValue: spark_job}, --job, {inputValue: job}, - --wait_interval, {inputValue: wait_interval} + --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_sparksql_job/component.yaml b/components/gcp/dataproc/submit_sparksql_job/component.yaml index 89dd2fe948c..52d6c091918 100644 --- a/components/gcp/dataproc/submit_sparksql_job/component.yaml +++ b/components/gcp/dataproc/submit_sparksql_job/component.yaml @@ -86,10 +86,10 @@ implementation: --script_variables, {inputValue: script_variables}, --sparksql_job, {inputValue: sparksql_job}, --job, {inputValue: job}, - --wait_interval, {inputValue: wait_interval} + --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/ml_engine/batch_predict/component.yaml b/components/gcp/ml_engine/batch_predict/component.yaml index c6aa4ce2460..c66ef5e40b7 100644 --- a/components/gcp/ml_engine/batch_predict/component.yaml +++ b/components/gcp/ml_engine/batch_predict/component.yaml @@ -81,10 +81,10 @@ implementation: --output_data_format, {inputValue: output_data_format}, --prediction_input, {inputValue: prediction_input}, --job_id_prefix, {inputValue: job_id_prefix}, - --wait_interval, {inputValue: wait_interval} + --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/ml_engine/job_id.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/ml_engine/deploy/component.yaml b/components/gcp/ml_engine/deploy/component.yaml index 13b981fed53..30eae108a72 100644 --- a/components/gcp/ml_engine/deploy/component.yaml +++ b/components/gcp/ml_engine/deploy/component.yaml @@ -109,6 +109,7 @@ implementation: --replace_existing_version, {inputValue: replace_existing_version}, --set_default, {inputValue: set_default}, --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" @@ -116,4 +117,3 @@ implementation: model_uri: /tmp/kfp/output/ml_engine/model_uri.txt model_name: /tmp/kfp/output/ml_engine/model_name.txt version_name: /tmp/kfp/output/ml_engine/version_name.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/ml_engine/train/component.yaml b/components/gcp/ml_engine/train/component.yaml index a48f1730a22..29222b0f5c7 100644 --- a/components/gcp/ml_engine/train/component.yaml +++ b/components/gcp/ml_engine/train/component.yaml @@ -118,11 +118,11 @@ implementation: --worker_image_uri, {inputValue: worker_image_uri}, --training_input, {inputValue: training_input}, --job_id_prefix, {inputValue: job_id_prefix}, - --wait_interval, {inputValue: wait_interval} + --wait_interval, {inputValue: wait_interval}, + --ui-metadata-path, {outputPath: MLPipeline UI metadata}, ] env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/ml_engine/job_id.txt job_dir: /tmp/kfp/output/ml_engine/job_dir.txt - MLPipeline UI metadata: /mlpipeline-ui-metadata.json