From 5fa24e75cb000a703ea5c66d5a6eabb81c6d402f Mon Sep 17 00:00:00 2001 From: KevinGrantLee Date: Tue, 13 Aug 2024 10:34:52 -0700 Subject: [PATCH 01/54] temp title: change title Signed-off-by: KevinGrantLee --- sdk/python/kfp/compiler/compiler_test.py | 24 +- .../kfp/compiler/pipeline_spec_builder.py | 44 ++- sdk/python/kfp/dsl/pipeline_task.py | 99 +++---- sdk/python/kfp/dsl/pipeline_task_test.py | 74 ++--- sdk/python/kfp/dsl/structures.py | 10 +- .../pipeline_with_resource_spec.yaml | 14 +- .../pipeline_with_string_machine_fields.py | 21 ++ .../pipeline_with_string_machine_fields.yaml | 73 +++++ ...th_string_machine_fields_pipeline_input.py | 26 ++ ..._string_machine_fields_pipeline_input.yaml | 116 ++++++++ ..._with_string_machine_fields_task_output.py | 42 +++ ...ith_string_machine_fields_task_output.yaml | 265 ++++++++++++++++++ 12 files changed, 680 insertions(+), 128 deletions(-) create mode 100644 sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.py create mode 100644 sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.yaml create mode 100644 sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.py create mode 100644 sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.yaml create mode 100644 sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.py create mode 100644 sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.yaml diff --git a/sdk/python/kfp/compiler/compiler_test.py b/sdk/python/kfp/compiler/compiler_test.py index d417d9eec19..d7312d196d5 100644 --- a/sdk/python/kfp/compiler/compiler_test.py +++ b/sdk/python/kfp/compiler/compiler_test.py @@ -3382,31 +3382,31 @@ def simple_pipeline(): ['exec-return-1']['container']) self.assertEqual( - 5, dict_format['deploymentSpec']['executors']['exec-return-1-2'] - ['container']['resources']['cpuLimit']) + '5', dict_format['deploymentSpec']['executors']['exec-return-1-2'] + ['container']['resources']['resourceCpuLimit']) self.assertNotIn( 'memoryLimit', dict_format['deploymentSpec']['executors'] ['exec-return-1-2']['container']['resources']) self.assertEqual( - 50, dict_format['deploymentSpec']['executors']['exec-return-1-3'] - ['container']['resources']['memoryLimit']) + '50G', dict_format['deploymentSpec']['executors']['exec-return-1-3'] + ['container']['resources']['resourceMemoryLimit']) self.assertNotIn( 'cpuLimit', dict_format['deploymentSpec']['executors'] ['exec-return-1-3']['container']['resources']) self.assertEqual( - 2, dict_format['deploymentSpec']['executors']['exec-return-1-4'] - ['container']['resources']['cpuRequest']) + '2', dict_format['deploymentSpec']['executors']['exec-return-1-4'] + ['container']['resources']['resourceCpuRequest']) self.assertEqual( - 5, dict_format['deploymentSpec']['executors']['exec-return-1-4'] - ['container']['resources']['cpuLimit']) + '5', dict_format['deploymentSpec']['executors']['exec-return-1-4'] + ['container']['resources']['resourceCpuLimit']) self.assertEqual( - 4, dict_format['deploymentSpec']['executors']['exec-return-1-4'] - ['container']['resources']['memoryRequest']) + '4G', dict_format['deploymentSpec']['executors']['exec-return-1-4'] + ['container']['resources']['resourceMemoryRequest']) self.assertEqual( - 50, dict_format['deploymentSpec']['executors']['exec-return-1-4'] - ['container']['resources']['memoryLimit']) + '50G', dict_format['deploymentSpec']['executors']['exec-return-1-4'] + ['container']['resources']['resourceMemoryLimit']) class TestPlatformConfig(unittest.TestCase): diff --git a/sdk/python/kfp/compiler/pipeline_spec_builder.py b/sdk/python/kfp/compiler/pipeline_spec_builder.py index 6d2a0cfa9d2..2d37bcea9a8 100644 --- a/sdk/python/kfp/compiler/pipeline_spec_builder.py +++ b/sdk/python/kfp/compiler/pipeline_spec_builder.py @@ -127,6 +127,12 @@ def build_task_spec_for_task( if task._task_spec.retry_policy is not None: pipeline_task_spec.retry_policy.CopyFrom( task._task_spec.retry_policy.to_proto()) + + # Inject resource fields into inputs + if task.container_spec and task.container_spec.resources: + for key, val in task.container_spec.resources.__dict__.items(): + if val and pipeline_channel.extract_pipeline_channels_from_any(val): + task.inputs[key] = val for input_name, input_value in task.inputs.items(): # Since LoopParameterArgument and LoopArtifactArgument and LoopArgumentVariable are narrower @@ -607,6 +613,24 @@ def build_container_spec_for_task( Returns: A PipelineContainerSpec object for the task. """ + def convert_to_placeholder(input_value: str) -> str: + """Checks if input is a pipeline channel and if so, converts to + compiler injected input name.""" + pipeline_channels = ( + pipeline_channel.extract_pipeline_channels_from_any(input_value) + ) + if pipeline_channels: + assert len(pipeline_channels) == 1 + channel = pipeline_channels[0] + additional_input_name = ( + compiler_utils.additional_input_name_for_pipeline_channel( + channel)) + additional_input_placeholder = placeholders.InputValuePlaceholder( + additional_input_name)._to_string() + input_value = input_value.replace( + channel.pattern, additional_input_placeholder) + return input_value + container_spec = ( pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec( image=task.container_spec.image, @@ -620,23 +644,23 @@ def build_container_spec_for_task( if task.container_spec.resources is not None: if task.container_spec.resources.cpu_request is not None: - container_spec.resources.cpu_request = ( - task.container_spec.resources.cpu_request) + container_spec.resources.resource_cpu_request = ( + convert_to_placeholder(task.container_spec.resources.cpu_request)) if task.container_spec.resources.cpu_limit is not None: - container_spec.resources.cpu_limit = ( - task.container_spec.resources.cpu_limit) + container_spec.resources.resource_cpu_limit = ( + convert_to_placeholder(task.container_spec.resources.cpu_limit)) if task.container_spec.resources.memory_request is not None: - container_spec.resources.memory_request = ( - task.container_spec.resources.memory_request) + container_spec.resources.resource_memory_request = ( + convert_to_placeholder(task.container_spec.resources.memory_request)) if task.container_spec.resources.memory_limit is not None: - container_spec.resources.memory_limit = ( - task.container_spec.resources.memory_limit) + container_spec.resources.resource_memory_limit = ( + convert_to_placeholder(task.container_spec.resources.memory_limit)) if task.container_spec.resources.accelerator_count is not None: container_spec.resources.accelerator.CopyFrom( pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec .ResourceSpec.AcceleratorConfig( - type=task.container_spec.resources.accelerator_type, - count=task.container_spec.resources.accelerator_count, + resource_type=convert_to_placeholder(task.container_spec.resources.accelerator_type), + resource_count=convert_to_placeholder(task.container_spec.resources.accelerator_count), )) return container_spec diff --git a/sdk/python/kfp/dsl/pipeline_task.py b/sdk/python/kfp/dsl/pipeline_task.py index 2e82d23378a..90dbce63b41 100644 --- a/sdk/python/kfp/dsl/pipeline_task.py +++ b/sdk/python/kfp/dsl/pipeline_task.py @@ -27,6 +27,7 @@ from kfp.dsl import placeholders from kfp.dsl import structures from kfp.dsl import utils +from kfp.dsl import pipeline_channel from kfp.dsl.types import type_utils from kfp.local import pipeline_orchestrator from kfp.pipeline_spec import pipeline_spec_pb2 @@ -321,9 +322,9 @@ def _ensure_container_spec_exists(self) -> None: f'{caller_method_name} can only be used on single-step components, not pipelines used as components, or special components like importers.' ) - def _validate_cpu_request_limit(self, cpu: str) -> float: + def _validate_cpu_request_limit(self, cpu: str) -> str: """Validates cpu request/limit string and converts to its numeric - value. + string value. Args: cpu: CPU requests or limits. This string should be a number or a @@ -335,17 +336,19 @@ def _validate_cpu_request_limit(self, cpu: str) -> float: ValueError if the cpu request/limit string value is invalid. Returns: - The numeric value (float) of the cpu request/limit. + The numeric string of the cpu request/limit. """ - if re.match(r'([0-9]*[.])?[0-9]+m?$', cpu) is None: - raise ValueError( - 'Invalid cpu string. Should be float or integer, or integer' - ' followed by "m".') - - return float(cpu[:-1]) / 1000 if cpu.endswith('m') else float(cpu) + if isinstance(cpu, pipeline_channel.PipelineChannel): + cpu = str(cpu) + else: + if re.match(r'([0-9]*[.])?[0-9]+m?$', cpu) is None: + raise ValueError( + 'Invalid cpu string. Should be float or integer, or integer' + ' followed by "m".') + return cpu @block_if_final() - def set_cpu_request(self, cpu: str) -> 'PipelineTask': + def set_cpu_request(self, cpu: Union[str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets CPU request (minimum) for the task. Args: @@ -370,7 +373,7 @@ def set_cpu_request(self, cpu: str) -> 'PipelineTask': return self @block_if_final() - def set_cpu_limit(self, cpu: str) -> 'PipelineTask': + def set_cpu_limit(self, cpu: Union[str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets CPU limit (maximum) for the task. Args: @@ -395,7 +398,7 @@ def set_cpu_limit(self, cpu: str) -> 'PipelineTask': return self @block_if_final() - def set_accelerator_limit(self, limit: int) -> 'PipelineTask': + def set_accelerator_limit(self, limit: Union[int, str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets accelerator limit (maximum) for the task. Only applies if accelerator type is also set via .set_accelerator_type(). @@ -406,11 +409,13 @@ def set_accelerator_limit(self, limit: int) -> 'PipelineTask': Self return to allow chained setting calls. """ self._ensure_container_spec_exists() - - if isinstance(limit, str): - if re.match(r'[1-9]\d*$', limit) is None: - raise ValueError(f'{"limit"!r} must be positive integer.') - limit = int(limit) + if isinstance(limit, pipeline_channel.PipelineChannel): + limit = str(limit) + else: + if isinstance(limit, int): + limit = str(limit) + if isinstance(limit, str) and re.match(r'^0$|^1$|^2$|^4$|^8$|^16$', limit) is None: + raise ValueError(f'{"limit"!r} must be one of 0, 1, 2, 4, 8, 16.') if self.container_spec.resources is not None: self.container_spec.resources.accelerator_count = limit @@ -438,9 +443,9 @@ def set_gpu_limit(self, gpu: str) -> 'PipelineTask': category=DeprecationWarning) return self.set_accelerator_limit(gpu) - def _validate_memory_request_limit(self, memory: str) -> float: + def _validate_memory_request_limit(self, memory: str) -> str: """Validates memory request/limit string and converts to its numeric - value. + string value. Args: memory: Memory requests or limits. This string should be a number or @@ -451,47 +456,21 @@ def _validate_memory_request_limit(self, memory: str) -> float: ValueError if the memory request/limit string value is invalid. Returns: - The numeric value (float) of the memory request/limit. + The numeric string value of the memory request/limit. """ - if re.match(r'^[0-9]+(E|Ei|P|Pi|T|Ti|G|Gi|M|Mi|K|Ki){0,1}$', - memory) is None: - raise ValueError( - 'Invalid memory string. Should be a number or a number ' - 'followed by one of "E", "Ei", "P", "Pi", "T", "Ti", "G", ' - '"Gi", "M", "Mi", "K", "Ki".') - - if memory.endswith('E'): - memory = float(memory[:-1]) * constants._E / constants._G - elif memory.endswith('Ei'): - memory = float(memory[:-2]) * constants._EI / constants._G - elif memory.endswith('P'): - memory = float(memory[:-1]) * constants._P / constants._G - elif memory.endswith('Pi'): - memory = float(memory[:-2]) * constants._PI / constants._G - elif memory.endswith('T'): - memory = float(memory[:-1]) * constants._T / constants._G - elif memory.endswith('Ti'): - memory = float(memory[:-2]) * constants._TI / constants._G - elif memory.endswith('G'): - memory = float(memory[:-1]) - elif memory.endswith('Gi'): - memory = float(memory[:-2]) * constants._GI / constants._G - elif memory.endswith('M'): - memory = float(memory[:-1]) * constants._M / constants._G - elif memory.endswith('Mi'): - memory = float(memory[:-2]) * constants._MI / constants._G - elif memory.endswith('K'): - memory = float(memory[:-1]) * constants._K / constants._G - elif memory.endswith('Ki'): - memory = float(memory[:-2]) * constants._KI / constants._G + if isinstance(memory, pipeline_channel.PipelineChannel): + memory = str(memory) else: - # By default interpret as a plain integer, in the unit of Bytes. - memory = float(memory) / constants._G - + if re.match(r'^[0-9]+(E|Ei|P|Pi|T|Ti|G|Gi|M|Mi|K|Ki){0,1}$', + memory) is None: + raise ValueError( + 'Invalid memory string. Should be a number or a number ' + 'followed by one of "E", "Ei", "P", "Pi", "T", "Ti", "G", ' + '"Gi", "M", "Mi", "K", "Ki".') return memory @block_if_final() - def set_memory_request(self, memory: str) -> 'PipelineTask': + def set_memory_request(self, memory: Union[str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets memory request (minimum) for the task. Args: @@ -515,7 +494,7 @@ def set_memory_request(self, memory: str) -> 'PipelineTask': return self @block_if_final() - def set_memory_limit(self, memory: str) -> 'PipelineTask': + def set_memory_limit(self, memory: Union[str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets memory limit (maximum) for the task. Args: @@ -579,7 +558,7 @@ def add_node_selector_constraint(self, accelerator: str) -> 'PipelineTask': return self.set_accelerator_type(accelerator) @block_if_final() - def set_accelerator_type(self, accelerator: str) -> 'PipelineTask': + def set_accelerator_type(self, accelerator: Union[str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets accelerator type to use when executing this task. Args: @@ -589,14 +568,16 @@ def set_accelerator_type(self, accelerator: str) -> 'PipelineTask': Self return to allow chained setting calls. """ self._ensure_container_spec_exists() + if isinstance(accelerator, pipeline_channel.PipelineChannel): + accelerator = str(accelerator) if self.container_spec.resources is not None: self.container_spec.resources.accelerator_type = accelerator if self.container_spec.resources.accelerator_count is None: - self.container_spec.resources.accelerator_count = 1 + self.container_spec.resources.accelerator_count = '1' else: self.container_spec.resources = structures.ResourceSpec( - accelerator_count=1, accelerator_type=accelerator) + accelerator_count='1', accelerator_type=accelerator) return self diff --git a/sdk/python/kfp/dsl/pipeline_task_test.py b/sdk/python/kfp/dsl/pipeline_task_test.py index c5bedc38111..217568750f7 100644 --- a/sdk/python/kfp/dsl/pipeline_task_test.py +++ b/sdk/python/kfp/dsl/pipeline_task_test.py @@ -145,42 +145,42 @@ def test_set_caching_options(self): @parameterized.parameters( { 'cpu': '123', - 'expected_cpu_number': 123, + 'expected_cpu': '123', }, { 'cpu': '123m', - 'expected_cpu_number': 0.123, + 'expected_cpu': '123m', }, { 'cpu': '123.0', - 'expected_cpu_number': 123, + 'expected_cpu': '123.0', }, { 'cpu': '123.0m', - 'expected_cpu_number': 0.123, + 'expected_cpu': '123.0m', }, ) def test_set_valid_cpu_request_limit(self, cpu: str, - expected_cpu_number: float): + expected_cpu: str): task = pipeline_task.PipelineTask( component_spec=structures.ComponentSpec.from_yaml_documents( V2_YAML), args={'input1': 'value'}, ) task.set_cpu_request(cpu) - self.assertEqual(expected_cpu_number, + self.assertEqual(expected_cpu, task.container_spec.resources.cpu_request) task.set_cpu_limit(cpu) - self.assertEqual(expected_cpu_number, + self.assertEqual(expected_cpu, task.container_spec.resources.cpu_limit) @parameterized.parameters( { - 'gpu_limit': '123', - 'expected_gpu_number': 123, + 'gpu_limit': '1', + 'expected_gpu_number': '1', },) def test_set_valid_gpu_limit(self, gpu_limit: str, - expected_gpu_number: int): + expected_gpu_number: str): task = pipeline_task.PipelineTask( component_spec=structures.ComponentSpec.from_yaml_documents( V2_YAML), @@ -210,15 +210,19 @@ def test_add_valid_node_selector_constraint(self): @parameterized.parameters( { - 'limit': '123', - 'expected': 123, + 'limit': '1', + 'expected_limit': '1', }, { - 'limit': 123, - 'expected': 123, + 'limit': 1, + 'expected_limit': '1', + }, + { + 'limit': 16, + 'expected_limit': '16', }, ) - def test_set_accelerator_limit(self, limit, expected): + def test_set_accelerator_limit(self, limit, expected_limit): task = pipeline_task.PipelineTask( component_spec=structures.ComponentSpec.from_yaml_documents( V2_YAML), @@ -226,74 +230,74 @@ def test_set_accelerator_limit(self, limit, expected): ) task.set_accelerator_limit(limit) - self.assertEqual(expected, + self.assertEqual(expected_limit, task.container_spec.resources.accelerator_count) @parameterized.parameters( { 'memory': '1E', - 'expected_memory_number': 1000000000, + 'expected_memory': '1E', }, { 'memory': '15Ei', - 'expected_memory_number': 17293822569.102703, + 'expected_memory': '15Ei', }, { 'memory': '2P', - 'expected_memory_number': 2000000, + 'expected_memory': '2P', }, { 'memory': '25Pi', - 'expected_memory_number': 28147497.6710656, + 'expected_memory': '25Pi', }, { 'memory': '3T', - 'expected_memory_number': 3000, + 'expected_memory': '3T', }, { 'memory': '35Ti', - 'expected_memory_number': 38482.90697216, + 'expected_memory': '35Ti', }, { 'memory': '4G', - 'expected_memory_number': 4, + 'expected_memory': '4G', }, { 'memory': '45Gi', - 'expected_memory_number': 48.31838208, + 'expected_memory': '45Gi', }, { 'memory': '5M', - 'expected_memory_number': 0.005, + 'expected_memory': '5M', }, { 'memory': '55Mi', - 'expected_memory_number': 0.05767168, + 'expected_memory': '55Mi', }, { 'memory': '6K', - 'expected_memory_number': 0.000006, + 'expected_memory': '6K', }, { 'memory': '65Ki', - 'expected_memory_number': 0.00006656, + 'expected_memory': '65Ki', }, { 'memory': '7000', - 'expected_memory_number': 0.000007, + 'expected_memory': '7000', }, ) - def test_set_memory_limit(self, memory: str, expected_memory_number: int): + def test_set_memory_limit(self, memory: str, expected_memory: str): task = pipeline_task.PipelineTask( component_spec=structures.ComponentSpec.from_yaml_documents( V2_YAML), args={'input1': 'value'}, ) task.set_memory_request(memory) - self.assertEqual(expected_memory_number, + self.assertEqual(expected_memory, task.container_spec.resources.memory_request) task.set_memory_limit(memory) - self.assertEqual(expected_memory_number, + self.assertEqual(expected_memory, task.container_spec.resources.memory_limit) def test_set_accelerator_type_with_type_only(self): @@ -305,7 +309,7 @@ def test_set_accelerator_type_with_type_only(self): task.set_accelerator_type('NVIDIA_TESLA_K80') self.assertEqual( structures.ResourceSpec( - accelerator_type='NVIDIA_TESLA_K80', accelerator_count=1), + accelerator_type='NVIDIA_TESLA_K80', accelerator_count='1'), task.container_spec.resources) def test_set_accelerator_type_with_accelerator_count(self): @@ -314,10 +318,10 @@ def test_set_accelerator_type_with_accelerator_count(self): V2_YAML), args={'input1': 'value'}, ) - task.set_accelerator_limit('5').set_accelerator_type('TPU_V3') + task.set_accelerator_limit('4').set_accelerator_type('TPU_V3') self.assertEqual( structures.ResourceSpec( - accelerator_type='TPU_V3', accelerator_count=5), + accelerator_type='TPU_V3', accelerator_count='4'), task.container_spec.resources) def test_set_env_variable(self): diff --git a/sdk/python/kfp/dsl/structures.py b/sdk/python/kfp/dsl/structures.py index 440f9a3940a..5a73d93b35c 100644 --- a/sdk/python/kfp/dsl/structures.py +++ b/sdk/python/kfp/dsl/structures.py @@ -239,12 +239,12 @@ class ResourceSpec: container. accelerator_count (optional): the number of accelerators attached. """ - cpu_request: Optional[float] = None - cpu_limit: Optional[float] = None - memory_request: Optional[float] = None - memory_limit: Optional[float] = None + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None accelerator_type: Optional[str] = None - accelerator_count: Optional[int] = None + accelerator_count: Optional[str] = None @dataclasses.dataclass diff --git a/sdk/python/test_data/pipelines/pipeline_with_resource_spec.yaml b/sdk/python/test_data/pipelines/pipeline_with_resource_spec.yaml index a9d93bc4de1..07a4ca82975 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_resource_spec.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_resource_spec.yaml @@ -61,12 +61,12 @@ deploymentSpec: image: gcr.io/my-project/my-fancy-trainer resources: accelerator: - count: '1' - type: tpu-v3 - cpuLimit: 4.0 - cpuRequest: 2.0 - memoryLimit: 15.032385536 - memoryRequest: 4.294967296 + resourceCount: '1' + resourceType: tpu-v3 + resourceCpuLimit: '4' + resourceCpuRequest: '2' + resourceMemoryLimit: 14Gi + resourceMemoryRequest: 4Gi pipelineInfo: description: A linear two-step pipeline with resource specification. name: two-step-pipeline-with-resource-spec @@ -119,4 +119,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.7.0 +sdkVersion: kfp-2.8.0 diff --git a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.py b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.py new file mode 100644 index 00000000000..05bddfb66fb --- /dev/null +++ b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.py @@ -0,0 +1,21 @@ +from kfp import dsl + + +@dsl.component +def sum_numbers(a: int, b:int) -> int: + return a + b + + +@dsl.pipeline +def pipeline(): + sum_numbers_task = sum_numbers(a=1, b=2) + sum_numbers_task.set_cpu_limit('4000m') + sum_numbers_task.set_memory_limit('15G') + sum_numbers_task.set_accelerator_type('NVIDIA_TESLA_P4') + sum_numbers_task.set_accelerator_limit('1') + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile( + pipeline_func=pipeline, package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.yaml b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.yaml new file mode 100644 index 00000000000..7278695a618 --- /dev/null +++ b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.yaml @@ -0,0 +1,73 @@ +# PIPELINE DEFINITION +# Name: pipeline +components: + comp-sum-numbers: + executorLabel: exec-sum-numbers + inputDefinitions: + parameters: + a: + parameterType: NUMBER_INTEGER + b: + parameterType: NUMBER_INTEGER + outputDefinitions: + parameters: + Output: + parameterType: NUMBER_INTEGER +deploymentSpec: + executors: + exec-sum-numbers: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - sum_numbers + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef sum_numbers(a: int, b:int) -> int:\n return a + b\n\n" + image: python:3.8 + resources: + accelerator: + resourceCount: '1' + resourceType: NVIDIA_TESLA_P4 + resourceCpuLimit: 4000m + resourceMemoryLimit: 15G +pipelineInfo: + name: pipeline +root: + dag: + tasks: + sum-numbers: + cachingOptions: + enableCache: true + componentRef: + name: comp-sum-numbers + inputs: + parameters: + a: + runtimeValue: + constant: 1.0 + b: + runtimeValue: + constant: 2.0 + taskInfo: + name: sum-numbers +schemaVersion: 2.1.0 +sdkVersion: kfp-2.8.0 diff --git a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.py b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.py new file mode 100644 index 00000000000..d44977a1340 --- /dev/null +++ b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.py @@ -0,0 +1,26 @@ +from kfp import dsl + + +@dsl.component +def sum_numbers(a: int, b:int) -> int: + return a + b + + +@dsl.pipeline +def pipeline( + cpu_limit: str = '4000m', + memory_limit: str = '15G', + accelerator_type: str = 'NVIDIA_TESLA_P4', + accelerator_limit: str = '1', +): + sum_numbers_task = sum_numbers(a=1, b=2) + sum_numbers_task.set_cpu_limit(cpu_limit) + sum_numbers_task.set_memory_limit(memory_limit) + sum_numbers_task.set_accelerator_type(accelerator_type) + sum_numbers_task.set_accelerator_limit(accelerator_limit) + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile( + pipeline_func=pipeline, package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.yaml b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.yaml new file mode 100644 index 00000000000..65815886409 --- /dev/null +++ b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.yaml @@ -0,0 +1,116 @@ +# PIPELINE DEFINITION +# Name: pipeline +# Inputs: +# accelerator_limit: str [Default: '1'] +# accelerator_type: str [Default: 'NVIDIA_TESLA_P4'] +# cpu_limit: str [Default: '2000m'] +# memory_limit: str [Default: '10G'] +components: + comp-sum-numbers: + executorLabel: exec-sum-numbers + inputDefinitions: + parameters: + a: + parameterType: NUMBER_INTEGER + b: + parameterType: NUMBER_INTEGER + outputDefinitions: + parameters: + Output: + parameterType: NUMBER_INTEGER +deploymentSpec: + executors: + exec-sum-numbers: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - sum_numbers + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef sum_numbers(a: int, b:int) -> int:\n return a + b\n\n" + image: python:3.8 + resources: + accelerator: + resourceCount: '{{$.inputs.parameters[''pipelinechannel--accelerator_limit'']}}' + resourceType: '{{$.inputs.parameters[''pipelinechannel--accelerator_type'']}}' + resourceCpuLimit: '{{$.inputs.parameters[''pipelinechannel--cpu_limit'']}}' + resourceMemoryLimit: '{{$.inputs.parameters[''pipelinechannel--memory_limit'']}}' +pipelineInfo: + name: pipeline +root: + dag: + tasks: + sum-numbers: + cachingOptions: + enableCache: true + componentRef: + name: comp-sum-numbers + inputs: + parameters: + a: + runtimeValue: + constant: 1.0 + accelerator_count: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--accelerator_limit'']}}' + accelerator_type: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--accelerator_type'']}}' + b: + runtimeValue: + constant: 2.0 + cpu_limit: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--cpu_limit'']}}' + memory_limit: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--memory_limit'']}}' + pipelinechannel--accelerator_limit: + componentInputParameter: accelerator_limit + pipelinechannel--accelerator_type: + componentInputParameter: accelerator_type + pipelinechannel--cpu_limit: + componentInputParameter: cpu_limit + pipelinechannel--memory_limit: + componentInputParameter: memory_limit + taskInfo: + name: sum-numbers + inputDefinitions: + parameters: + accelerator_limit: + defaultValue: '1' + isOptional: true + parameterType: STRING + accelerator_type: + defaultValue: NVIDIA_TESLA_P4 + isOptional: true + parameterType: STRING + cpu_limit: + defaultValue: 2000m + isOptional: true + parameterType: STRING + memory_limit: + defaultValue: 10G + isOptional: true + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.8.0 diff --git a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.py b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.py new file mode 100644 index 00000000000..5c58508b2e4 --- /dev/null +++ b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.py @@ -0,0 +1,42 @@ +from kfp import dsl + + +@dsl.component +def cpu_limit() -> str: + return '4000m' + + +@dsl.component +def memory_limit() -> str: + return '15G' + + +@dsl.component +def accelerator_type() -> str: + return 'NVIDIA_TESLA_P4' + + +@dsl.component +def accelerator_limit() -> str: + return '1' + + +@dsl.component +def sum_numbers(a: int, b:int) -> int: + return a + b + + +@dsl.pipeline +def pipeline( +): + sum_numbers_task = sum_numbers(a=1, b=2) + sum_numbers_task.set_cpu_limit(cpu_limit().output) + sum_numbers_task.set_memory_limit(memory_limit().output) + sum_numbers_task.set_accelerator_type(accelerator_type().output) + sum_numbers_task.set_accelerator_limit(accelerator_limit().output) + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile( + pipeline_func=pipeline, package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.yaml b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.yaml new file mode 100644 index 00000000000..13f80133f8b --- /dev/null +++ b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.yaml @@ -0,0 +1,265 @@ +# PIPELINE DEFINITION +# Name: pipeline +components: + comp-accelerator-limit: + executorLabel: exec-accelerator-limit + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-accelerator-type: + executorLabel: exec-accelerator-type + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-cpu-limit: + executorLabel: exec-cpu-limit + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-memory-limit: + executorLabel: exec-memory-limit + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-sum-numbers: + executorLabel: exec-sum-numbers + inputDefinitions: + parameters: + a: + parameterType: NUMBER_INTEGER + b: + parameterType: NUMBER_INTEGER + outputDefinitions: + parameters: + Output: + parameterType: NUMBER_INTEGER +deploymentSpec: + executors: + exec-accelerator-limit: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - accelerator_limit + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef accelerator_limit() -> str:\n return '1'\n\n" + image: python:3.8 + exec-accelerator-type: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - accelerator_type + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef accelerator_type() -> str:\n return 'NVIDIA_TESLA_P4'\n\n" + image: python:3.8 + exec-cpu-limit: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - cpu_limit + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef cpu_limit() -> str:\n return '4000m'\n\n" + image: python:3.8 + exec-memory-limit: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - memory_limit + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef memory_limit() -> str:\n return '15G'\n\n" + image: python:3.8 + exec-sum-numbers: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - sum_numbers + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.8.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef sum_numbers(a: int, b:int) -> int:\n return a + b\n\n" + image: python:3.8 + resources: + accelerator: + resourceCount: '{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}' + resourceType: '{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}' + resourceCpuLimit: '{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}' + resourceMemoryLimit: '{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}' +pipelineInfo: + name: pipeline +root: + dag: + tasks: + accelerator-limit: + cachingOptions: + enableCache: true + componentRef: + name: comp-accelerator-limit + taskInfo: + name: accelerator-limit + accelerator-type: + cachingOptions: + enableCache: true + componentRef: + name: comp-accelerator-type + taskInfo: + name: accelerator-type + cpu-limit: + cachingOptions: + enableCache: true + componentRef: + name: comp-cpu-limit + taskInfo: + name: cpu-limit + memory-limit: + cachingOptions: + enableCache: true + componentRef: + name: comp-memory-limit + taskInfo: + name: memory-limit + sum-numbers: + cachingOptions: + enableCache: true + componentRef: + name: comp-sum-numbers + inputs: + parameters: + a: + runtimeValue: + constant: 1.0 + accelerator_count: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}' + accelerator_type: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}' + b: + runtimeValue: + constant: 2.0 + cpu_limit: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}' + memory_limit: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}' + pipelinechannel--accelerator-limit-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: accelerator-limit + pipelinechannel--accelerator-type-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: accelerator-type + pipelinechannel--cpu-limit-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: cpu-limit + pipelinechannel--memory-limit-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: memory-limit + taskInfo: + name: sum-numbers +schemaVersion: 2.1.0 +sdkVersion: kfp-2.8.0 From b8a1d720025602cb7968ef184ef1968a09275e2f Mon Sep 17 00:00:00 2001 From: KevinGrantLee Date: Tue, 13 Aug 2024 10:40:10 -0700 Subject: [PATCH 02/54] add release notes Signed-off-by: KevinGrantLee --- sdk/RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/RELEASE.md b/sdk/RELEASE.md index 1f64612dcbd..fc0dc0d5487 100644 --- a/sdk/RELEASE.md +++ b/sdk/RELEASE.md @@ -1,6 +1,7 @@ # Current Version (in development) ## Features +* Support dynamic machine type parameters in pipeline task setters. [\#11097](https://github.com/kubeflow/pipelines/pull/11097) ## Breaking changes From 513d46b37f7439ddda7a44a205c31c57a0e46e10 Mon Sep 17 00:00:00 2001 From: KevinGrantLee Date: Tue, 13 Aug 2024 10:54:51 -0700 Subject: [PATCH 03/54] formatting Signed-off-by: KevinGrantLee --- .../kfp/compiler/pipeline_spec_builder.py | 31 +++++++++------- sdk/python/kfp/dsl/pipeline_task.py | 37 ++++++++++++++----- sdk/python/kfp/dsl/pipeline_task_test.py | 6 +-- .../pipeline_with_string_machine_fields.py | 2 +- ...th_string_machine_fields_pipeline_input.py | 2 +- ..._with_string_machine_fields_task_output.py | 5 +-- 6 files changed, 51 insertions(+), 32 deletions(-) diff --git a/sdk/python/kfp/compiler/pipeline_spec_builder.py b/sdk/python/kfp/compiler/pipeline_spec_builder.py index 2d37bcea9a8..6e4bc4e8690 100644 --- a/sdk/python/kfp/compiler/pipeline_spec_builder.py +++ b/sdk/python/kfp/compiler/pipeline_spec_builder.py @@ -127,7 +127,7 @@ def build_task_spec_for_task( if task._task_spec.retry_policy is not None: pipeline_task_spec.retry_policy.CopyFrom( task._task_spec.retry_policy.to_proto()) - + # Inject resource fields into inputs if task.container_spec and task.container_spec.resources: for key, val in task.container_spec.resources.__dict__.items(): @@ -613,22 +613,22 @@ def build_container_spec_for_task( Returns: A PipelineContainerSpec object for the task. """ + def convert_to_placeholder(input_value: str) -> str: """Checks if input is a pipeline channel and if so, converts to compiler injected input name.""" pipeline_channels = ( - pipeline_channel.extract_pipeline_channels_from_any(input_value) - ) + pipeline_channel.extract_pipeline_channels_from_any(input_value)) if pipeline_channels: assert len(pipeline_channels) == 1 channel = pipeline_channels[0] additional_input_name = ( - compiler_utils.additional_input_name_for_pipeline_channel( - channel)) + compiler_utils.additional_input_name_for_pipeline_channel( + channel)) additional_input_placeholder = placeholders.InputValuePlaceholder( - additional_input_name)._to_string() - input_value = input_value.replace( - channel.pattern, additional_input_placeholder) + additional_input_name)._to_string() + input_value = input_value.replace(channel.pattern, + additional_input_placeholder) return input_value container_spec = ( @@ -645,22 +645,27 @@ def convert_to_placeholder(input_value: str) -> str: if task.container_spec.resources is not None: if task.container_spec.resources.cpu_request is not None: container_spec.resources.resource_cpu_request = ( - convert_to_placeholder(task.container_spec.resources.cpu_request)) + convert_to_placeholder( + task.container_spec.resources.cpu_request)) if task.container_spec.resources.cpu_limit is not None: container_spec.resources.resource_cpu_limit = ( convert_to_placeholder(task.container_spec.resources.cpu_limit)) if task.container_spec.resources.memory_request is not None: container_spec.resources.resource_memory_request = ( - convert_to_placeholder(task.container_spec.resources.memory_request)) + convert_to_placeholder( + task.container_spec.resources.memory_request)) if task.container_spec.resources.memory_limit is not None: container_spec.resources.resource_memory_limit = ( - convert_to_placeholder(task.container_spec.resources.memory_limit)) + convert_to_placeholder( + task.container_spec.resources.memory_limit)) if task.container_spec.resources.accelerator_count is not None: container_spec.resources.accelerator.CopyFrom( pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec .ResourceSpec.AcceleratorConfig( - resource_type=convert_to_placeholder(task.container_spec.resources.accelerator_type), - resource_count=convert_to_placeholder(task.container_spec.resources.accelerator_count), + resource_type=convert_to_placeholder( + task.container_spec.resources.accelerator_type), + resource_count=convert_to_placeholder( + task.container_spec.resources.accelerator_count), )) return container_spec diff --git a/sdk/python/kfp/dsl/pipeline_task.py b/sdk/python/kfp/dsl/pipeline_task.py index 90dbce63b41..773fb1e0676 100644 --- a/sdk/python/kfp/dsl/pipeline_task.py +++ b/sdk/python/kfp/dsl/pipeline_task.py @@ -27,7 +27,6 @@ from kfp.dsl import placeholders from kfp.dsl import structures from kfp.dsl import utils -from kfp.dsl import pipeline_channel from kfp.dsl.types import type_utils from kfp.local import pipeline_orchestrator from kfp.pipeline_spec import pipeline_spec_pb2 @@ -348,7 +347,10 @@ def _validate_cpu_request_limit(self, cpu: str) -> str: return cpu @block_if_final() - def set_cpu_request(self, cpu: Union[str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': + def set_cpu_request( + self, + cpu: Union[str, + pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets CPU request (minimum) for the task. Args: @@ -373,7 +375,10 @@ def set_cpu_request(self, cpu: Union[str, pipeline_channel.PipelineChannel]) -> return self @block_if_final() - def set_cpu_limit(self, cpu: Union[str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': + def set_cpu_limit( + self, + cpu: Union[str, + pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets CPU limit (maximum) for the task. Args: @@ -398,7 +403,9 @@ def set_cpu_limit(self, cpu: Union[str, pipeline_channel.PipelineChannel]) -> 'P return self @block_if_final() - def set_accelerator_limit(self, limit: Union[int, str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': + def set_accelerator_limit( + self, limit: Union[int, str, + pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets accelerator limit (maximum) for the task. Only applies if accelerator type is also set via .set_accelerator_type(). @@ -414,8 +421,10 @@ def set_accelerator_limit(self, limit: Union[int, str, pipeline_channel.Pipeline else: if isinstance(limit, int): limit = str(limit) - if isinstance(limit, str) and re.match(r'^0$|^1$|^2$|^4$|^8$|^16$', limit) is None: - raise ValueError(f'{"limit"!r} must be one of 0, 1, 2, 4, 8, 16.') + if isinstance(limit, str) and re.match(r'^0$|^1$|^2$|^4$|^8$|^16$', + limit) is None: + raise ValueError( + f'{"limit"!r} must be one of 0, 1, 2, 4, 8, 16.') if self.container_spec.resources is not None: self.container_spec.resources.accelerator_count = limit @@ -462,7 +471,7 @@ def _validate_memory_request_limit(self, memory: str) -> str: memory = str(memory) else: if re.match(r'^[0-9]+(E|Ei|P|Pi|T|Ti|G|Gi|M|Mi|K|Ki){0,1}$', - memory) is None: + memory) is None: raise ValueError( 'Invalid memory string. Should be a number or a number ' 'followed by one of "E", "Ei", "P", "Pi", "T", "Ti", "G", ' @@ -470,7 +479,10 @@ def _validate_memory_request_limit(self, memory: str) -> str: return memory @block_if_final() - def set_memory_request(self, memory: Union[str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': + def set_memory_request( + self, + memory: Union[str, + pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets memory request (minimum) for the task. Args: @@ -494,7 +506,10 @@ def set_memory_request(self, memory: Union[str, pipeline_channel.PipelineChannel return self @block_if_final() - def set_memory_limit(self, memory: Union[str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': + def set_memory_limit( + self, + memory: Union[str, + pipeline_channel.PipelineChannel]) -> 'PipelineTask': """Sets memory limit (maximum) for the task. Args: @@ -558,7 +573,9 @@ def add_node_selector_constraint(self, accelerator: str) -> 'PipelineTask': return self.set_accelerator_type(accelerator) @block_if_final() - def set_accelerator_type(self, accelerator: Union[str, pipeline_channel.PipelineChannel]) -> 'PipelineTask': + def set_accelerator_type( + self, accelerator: Union[str, pipeline_channel.PipelineChannel] + ) -> 'PipelineTask': """Sets accelerator type to use when executing this task. Args: diff --git a/sdk/python/kfp/dsl/pipeline_task_test.py b/sdk/python/kfp/dsl/pipeline_task_test.py index 217568750f7..8543058b826 100644 --- a/sdk/python/kfp/dsl/pipeline_task_test.py +++ b/sdk/python/kfp/dsl/pipeline_task_test.py @@ -160,8 +160,7 @@ def test_set_caching_options(self): 'expected_cpu': '123.0m', }, ) - def test_set_valid_cpu_request_limit(self, cpu: str, - expected_cpu: str): + def test_set_valid_cpu_request_limit(self, cpu: str, expected_cpu: str): task = pipeline_task.PipelineTask( component_spec=structures.ComponentSpec.from_yaml_documents( V2_YAML), @@ -171,8 +170,7 @@ def test_set_valid_cpu_request_limit(self, cpu: str, self.assertEqual(expected_cpu, task.container_spec.resources.cpu_request) task.set_cpu_limit(cpu) - self.assertEqual(expected_cpu, - task.container_spec.resources.cpu_limit) + self.assertEqual(expected_cpu, task.container_spec.resources.cpu_limit) @parameterized.parameters( { diff --git a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.py b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.py index 05bddfb66fb..84965bd88b4 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.py +++ b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields.py @@ -2,7 +2,7 @@ @dsl.component -def sum_numbers(a: int, b:int) -> int: +def sum_numbers(a: int, b: int) -> int: return a + b diff --git a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.py b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.py index d44977a1340..eb8e1f79fe2 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.py +++ b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_pipeline_input.py @@ -2,7 +2,7 @@ @dsl.component -def sum_numbers(a: int, b:int) -> int: +def sum_numbers(a: int, b: int) -> int: return a + b diff --git a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.py b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.py index 5c58508b2e4..7795bb8a0f4 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.py +++ b/sdk/python/test_data/pipelines/pipeline_with_string_machine_fields_task_output.py @@ -22,13 +22,12 @@ def accelerator_limit() -> str: @dsl.component -def sum_numbers(a: int, b:int) -> int: +def sum_numbers(a: int, b: int) -> int: return a + b @dsl.pipeline -def pipeline( -): +def pipeline(): sum_numbers_task = sum_numbers(a=1, b=2) sum_numbers_task.set_cpu_limit(cpu_limit().output) sum_numbers_task.set_memory_limit(memory_limit().output) From e55c330a8917cb615f30a0ce5d9b17d8f9563a65 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Aug 2024 12:44:41 -0400 Subject: [PATCH 04/54] feat(backend): move comp logic to workflow params (#10979) * feat(backend): move comp logic to workflow params Signed-off-by: zazulam Co-authored-by: droctothorpe Co-authored-by: andreafehrman Co-authored-by: MonicaZhang1 Co-authored-by: kylekaminky Co-authored-by: CarterFendley Signed-off-by: zazulam * address pr comments Signed-off-by: zazulam * Use function name instead of base name and address edge cases Signed-off-by: droctothorpe Co-authored-by: zazulam * Improve logic and update tests Signed-off-by: droctothorpe Co-authored-by: zazulam * POC hashing command and args Signed-off-by: droctothorpe Co-authored-by: zazulam * Add comments to clarify the logic Signed-off-by: droctothorpe Co-authored-by: zazulam * Hash entire PipelineContainerSpec Signed-off-by: droctothorpe Co-authored-by: zazulam --------- Signed-off-by: zazulam Signed-off-by: droctothorpe Co-authored-by: droctothorpe Co-authored-by: andreafehrman Co-authored-by: MonicaZhang1 Co-authored-by: kylekaminky Co-authored-by: CarterFendley Signed-off-by: KevinGrantLee --- backend/src/v2/compiler/argocompiler/argo.go | 124 ++++++++++++++---- .../src/v2/compiler/argocompiler/container.go | 6 +- .../create_mount_delete_dynamic_pvc.yaml | 88 ++++++------- .../testdata/create_pod_metadata.yaml | 90 +++++++------ .../argocompiler/testdata/hello_world.yaml | 29 ++-- .../argocompiler/testdata/importer.yaml | 19 +-- 6 files changed, 220 insertions(+), 136 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index a5cfed5faef..1f1c19ed3ec 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -15,12 +15,16 @@ package argocompiler import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" "fmt" "strings" wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/v2/compiler" + log "github.com/sirupsen/logrus" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" k8score "k8s.io/api/core/v1" @@ -63,7 +67,7 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S if err != nil { return nil, err } - // fill root component default paramters to PipelineJob + // fill root component default parameters to PipelineJob specParams := spec.GetRoot().GetInputDefinitions().GetParameters() for name, param := range specParams { _, ok := job.RuntimeConfig.ParameterValues[name] @@ -108,6 +112,9 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S "pipelines.kubeflow.org/v2_component": "true", }, }, + Arguments: wfapi.Arguments{ + Parameters: []wfapi.Parameter{}, + }, ServiceAccountName: "pipeline-runner", Entrypoint: tmplEntrypoint, }, @@ -180,69 +187,134 @@ func (c *workflowCompiler) templateName(componentName string) string { return componentName } -// WIP: store component spec, task spec and executor spec in annotations - const ( - annotationComponents = "pipelines.kubeflow.org/components-" - annotationContainers = "pipelines.kubeflow.org/implementations-" - annotationKubernetesSpec = "pipelines.kubeflow.org/kubernetes-" + argumentsComponents = "components-" + argumentsContainers = "implementations-" + argumentsKubernetesSpec = "kubernetes-" ) func (c *workflowCompiler) saveComponentSpec(name string, spec *pipelinespec.ComponentSpec) error { - return c.saveProtoToAnnotation(annotationComponents+name, spec) + hashedComponent := c.hashComponentContainer(name) + + return c.saveProtoToArguments(argumentsComponents+hashedComponent, spec) } // useComponentSpec returns a placeholder we can refer to the component spec // in argo workflow fields. func (c *workflowCompiler) useComponentSpec(name string) (string, error) { - return c.annotationPlaceholder(annotationComponents + name) + hashedComponent := c.hashComponentContainer(name) + + return c.argumentsPlaceholder(argumentsComponents + hashedComponent) } func (c *workflowCompiler) saveComponentImpl(name string, msg proto.Message) error { - return c.saveProtoToAnnotation(annotationContainers+name, msg) + hashedComponent := c.hashComponentContainer(name) + + return c.saveProtoToArguments(argumentsContainers+hashedComponent, msg) } func (c *workflowCompiler) useComponentImpl(name string) (string, error) { - return c.annotationPlaceholder(annotationContainers + name) + hashedComponent := c.hashComponentContainer(name) + + return c.argumentsPlaceholder(argumentsContainers + hashedComponent) } func (c *workflowCompiler) saveKubernetesSpec(name string, spec *structpb.Struct) error { - return c.saveProtoToAnnotation(annotationKubernetesSpec+name, spec) + return c.saveProtoToArguments(argumentsKubernetesSpec+name, spec) } func (c *workflowCompiler) useKubernetesImpl(name string) (string, error) { - return c.annotationPlaceholder(annotationKubernetesSpec + name) + return c.argumentsPlaceholder(argumentsKubernetesSpec + name) } -// TODO(Bobgy): sanitize component name -func (c *workflowCompiler) saveProtoToAnnotation(name string, msg proto.Message) error { +// saveProtoToArguments saves a proto message to the workflow arguments. The +// message is serialized to JSON and stored in the workflow arguments and then +// referenced by the workflow templates using AWF templating syntax. The reason +// for storing it in the workflow arguments is because there is a 1-many +// relationship between components and tasks that reference them. The workflow +// arguments allow us to deduplicate the component logic (implementation & spec +// in IR), significantly reducing the size of the argo workflow manifest. +func (c *workflowCompiler) saveProtoToArguments(componentName string, msg proto.Message) error { if c == nil { return fmt.Errorf("compiler is nil") } - if c.wf.Annotations == nil { - c.wf.Annotations = make(map[string]string) + if c.wf.Spec.Arguments.Parameters == nil { + c.wf.Spec.Arguments = wfapi.Arguments{Parameters: []wfapi.Parameter{}} } - if _, alreadyExists := c.wf.Annotations[name]; alreadyExists { - return fmt.Errorf("annotation %q already exists", name) + if c.wf.Spec.Arguments.GetParameterByName(componentName) != nil { + return nil } json, err := stablyMarshalJSON(msg) if err != nil { - return fmt.Errorf("saving component spec of %q to annotations: %w", name, err) + return fmt.Errorf("saving component spec of %q to arguments: %w", componentName, err) } - // TODO(Bobgy): verify name adheres to Kubernetes annotation restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set - c.wf.Annotations[name] = json + c.wf.Spec.Arguments.Parameters = append(c.wf.Spec.Arguments.Parameters, wfapi.Parameter{ + Name: componentName, + Value: wfapi.AnyStringPtr(json), + }) return nil } -func (c *workflowCompiler) annotationPlaceholder(name string) (string, error) { +// argumentsPlaceholder checks for the unique component name within the workflow +// arguments and returns a template tag that references the component in the +// workflow arguments. +func (c *workflowCompiler) argumentsPlaceholder(componentName string) (string, error) { if c == nil { return "", fmt.Errorf("compiler is nil") } - if _, exists := c.wf.Annotations[name]; !exists { - return "", fmt.Errorf("using component spec: failed to find annotation %q", name) + if c.wf.Spec.Arguments.GetParameterByName(componentName) == nil { + return "", fmt.Errorf("using component spec: failed to find workflow parameter %q", componentName) + } + + return workflowParameter(componentName), nil +} + +// hashComponentContainer serializes and hashes the container field of a given +// component. +func (c *workflowCompiler) hashComponentContainer(componentName string) string { + log.Debug("componentName: ", componentName) + // Return early for root component since it has no command and args. + if componentName == "root" { + return componentName } - // Reference: https://argoproj.github.io/argo-workflows/variables/ - return fmt.Sprintf("{{workflow.annotations.%s}}", name), nil + if c.executors != nil { // Don't bother if there are no executors in the pipeline spec. + // Look up the executorLabel for the component in question. + executorLabel := c.spec.Components[componentName].GetExecutorLabel() + log.Debug("executorLabel: ", executorLabel) + // Iterate through the list of executors. + for executorName, executorValue := range c.executors { + log.Debug("executorName: ", executorName) + // If one of them matches the executorLabel we extracted earlier... + if executorName == executorLabel { + // Get the corresponding container. + container := executorValue.GetContainer() + if container != nil { + containerHash, err := hashValue(container) + if err != nil { + // Do not bubble up since this is not a breaking error + // and we can just return the componentName in full. + log.Debug("Error hashing container: ", err) + } + + return containerHash + } + } + } + } + + return componentName +} + +// hashValue serializes and hashes a provided value. +func hashValue(value interface{}) (string, error) { + bytes, err := json.Marshal(value) + if err != nil { + return "", err + } + h := sha256.New() + h.Write([]byte(bytes)) + + return hex.EncodeToString(h.Sum(nil)), nil } const ( diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index 14ed4f70679..b04adc58f8a 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -358,9 +358,11 @@ func (c *workflowCompiler) addContainerExecutorTemplate(refName string) string { }, } // Update pod metadata if it defined in the Kubernetes Spec - if kubernetesConfigString, ok := c.wf.Annotations[annotationKubernetesSpec+refName]; ok { + kubernetesConfigParam := c.wf.Spec.Arguments.GetParameterByName(argumentsKubernetesSpec + refName) + + if kubernetesConfigParam != nil { k8sExecCfg := &kubernetesplatform.KubernetesExecutorConfig{} - if err := jsonpb.UnmarshalString(kubernetesConfigString, k8sExecCfg); err == nil { + if err := jsonpb.UnmarshalString(string(*kubernetesConfigParam.Value), k8sExecCfg); err == nil { extendPodMetadata(&executor.Metadata, k8sExecCfg) } } diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml index e3b427d2455..7a5565595ed 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml @@ -1,36 +1,36 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - annotations: - pipelines.kubeflow.org/components-comp-comp: '{"executorLabel":"exec-comp"}' - pipelines.kubeflow.org/components-comp-comp-2: '{"executorLabel":"exec-comp-2"}' - pipelines.kubeflow.org/components-comp-createpvc: '{"executorLabel":"exec-createpvc","inputDefinitions":{"parameters":{"access_modes":{"parameterType":"LIST"},"annotations":{"isOptional":true,"parameterType":"STRUCT"},"pvc_name":{"isOptional":true,"parameterType":"STRING"},"pvc_name_suffix":{"isOptional":true,"parameterType":"STRING"},"size":{"parameterType":"STRING"},"storage_class_name":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"volume_name":{"isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"name":{"parameterType":"STRING"}}}}' - pipelines.kubeflow.org/components-comp-deletepvc: '{"executorLabel":"exec-deletepvc","inputDefinitions":{"parameters":{"pvc_name":{"parameterType":"STRING"}}}}' - pipelines.kubeflow.org/components-root: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}},"comp-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}},"createpvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}},"deletepvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}}}}' - pipelines.kubeflow.org/implementations-comp-comp: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m - ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.0.0-beta.16'' - \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d) printf \"%s\" - \"$0\" \u003e \"$program_path/ephemeral_component.py\" python3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\" - ","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import - *\n\ndef comp():\n pass\n\n"],"image":"python:3.7"}' - pipelines.kubeflow.org/implementations-comp-comp-2: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m - ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.0.0-beta.16'' - \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d) printf \"%s\" - \"$0\" \u003e \"$program_path/ephemeral_component.py\" python3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\" - ","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import - *\n\ndef comp():\n pass\n\n"],"image":"python:3.7"}' - pipelines.kubeflow.org/implementations-comp-createpvc: '{"image":"argostub/createpvc"}' - pipelines.kubeflow.org/implementations-comp-deletepvc: '{"image":"argostub/deletepvc"}' - pipelines.kubeflow.org/kubernetes-comp-comp: '{"pvcMount":[{"mountPath":"/data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - pipelines.kubeflow.org/kubernetes-comp-comp-2: '{"pvcMount":[{"mountPath":"/reused_data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' creationTimestamp: null generateName: my-pipeline- spec: - arguments: {} + arguments: + parameters: + - name: kubernetes-comp-comp + value: '{"pvcMount":[{"mountPath":"/data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' + - name: components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080 + value: '{"executorLabel":"exec-comp"}' + - name: implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080 + value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif + ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 + -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 + python3 -m pip install --quiet --no-warn-script-location ''kfp==2.0.0-beta.16'' + \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d) printf + \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\" python3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\" + ","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import + *\n\ndef comp():\n pass\n\n"],"image":"python:3.7"}' + - name: kubernetes-comp-comp-2 + value: '{"pvcMount":[{"mountPath":"/reused_data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' + - name: components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767 + value: '{"executorLabel":"exec-createpvc","inputDefinitions":{"parameters":{"access_modes":{"parameterType":"LIST"},"annotations":{"isOptional":true,"parameterType":"STRUCT"},"pvc_name":{"isOptional":true,"parameterType":"STRING"},"pvc_name_suffix":{"isOptional":true,"parameterType":"STRING"},"size":{"parameterType":"STRING"},"storage_class_name":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"volume_name":{"isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"name":{"parameterType":"STRING"}}}}' + - name: implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767 + value: '{"image":"argostub/createpvc"}' + - name: components-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725 + value: '{"executorLabel":"exec-deletepvc","inputDefinitions":{"parameters":{"pvc_name":{"parameterType":"STRING"}}}}' + - name: implementations-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725 + value: '{"image":"argostub/deletepvc"}' + - name: components-root + value: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}},"comp-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}},"createpvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}},"deletepvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -180,32 +180,32 @@ spec: volumes: - emptyDir: {} name: kfp-launcher - - emptyDir: { } + - emptyDir: {} name: gcs-scratch - - emptyDir: { } + - emptyDir: {} name: s3-scratch - - emptyDir: { } + - emptyDir: {} name: minio-scratch - - emptyDir: { } + - emptyDir: {} name: dot-local-scratch - - emptyDir: { } + - emptyDir: {} name: dot-cache-scratch - - emptyDir: { } + - emptyDir: {} name: dot-config-scratch - dag: tasks: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-comp}}' + value: '{{workflow.parameters.components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-comp}}' + value: '{{workflow.parameters.implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config - value: '{{workflow.annotations.pipelines.kubeflow.org/kubernetes-comp-comp}}' + value: '{{workflow.parameters.kubernetes-comp-comp}}' depends: createpvc.Succeeded name: comp-driver template: system-container-driver @@ -222,15 +222,15 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-comp-2}}' + value: '{{workflow.parameters.components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-comp-2}}' + value: '{{workflow.parameters.implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config - value: '{{workflow.annotations.pipelines.kubeflow.org/kubernetes-comp-comp-2}}' + value: '{{workflow.parameters.kubernetes-comp-comp-2}}' depends: comp.Succeeded && createpvc.Succeeded name: comp-2-driver template: system-container-driver @@ -247,11 +247,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-createpvc}}' + value: '{{workflow.parameters.components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-createpvc}}' + value: '{{workflow.parameters.implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: createpvc @@ -259,11 +259,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-deletepvc}}' + value: '{{workflow.parameters.components-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-deletepvc}}' + value: '{{workflow.parameters.implementations-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' depends: comp-2.Succeeded && createpvc.Succeeded @@ -343,7 +343,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-root}}' + value: '{{workflow.parameters.components-root}}' - name: runtime-config value: '{}' - name: driver-type diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml index 450aed6aeca..67308ba33dd 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml @@ -1,20 +1,24 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - annotations: - pipelines.kubeflow.org/components-comp-hello-world: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - pipelines.kubeflow.org/components-root: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - pipelines.kubeflow.org/implementations-comp-hello-world: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf - \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def - hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser - = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", - dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}' - pipelines.kubeflow.org/kubernetes-comp-hello-world: '{"podMetadata":{"annotations":{"experiment_id":"234567","run_id":"123456"},"labels":{"kubeflow.com/common":"test","kubeflow.com/kfp":"pipeline-node"}}}' creationTimestamp: null generateName: hello-world- spec: - arguments: {} + arguments: + parameters: + - name: kubernetes-comp-hello-world + value: '{"podMetadata":{"annotations":{"experiment_id":"234567","run_id":"123456"},"labels":{"kubeflow.com/common":"test","kubeflow.com/kfp":"pipeline-node"}}}' + - name: components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 + value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' + - name: implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 + value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf + \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def + hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser + = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", + dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args + = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}' + - name: components-root + value: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -124,20 +128,20 @@ spec: name: "" resources: {} volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch initContainers: - command: - launcher-v2 @@ -168,34 +172,34 @@ spec: outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: { } - name: gcs-scratch - - emptyDir: { } - name: s3-scratch - - emptyDir: { } - name: minio-scratch - - emptyDir: { } - name: dot-local-scratch - - emptyDir: { } - name: dot-cache-scratch - - emptyDir: { } - name: dot-config-scratch + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch - dag: tasks: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-hello-world}}' + value: '{{workflow.parameters.components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-hello-world}}' + value: '{{workflow.parameters.implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config - value: '{{workflow.annotations.pipelines.kubeflow.org/kubernetes-comp-hello-world}}' + value: '{{workflow.parameters.kubernetes-comp-hello-world}}' name: hello-world-driver template: system-container-driver - arguments: @@ -282,7 +286,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-root}}' + value: '{{workflow.parameters.components-root}}' - name: runtime-config value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - name: driver-type diff --git a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml index 5685ece5de5..6bc59366d8d 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml @@ -1,19 +1,22 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - annotations: - pipelines.kubeflow.org/components-comp-hello-world: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - pipelines.kubeflow.org/components-root: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - pipelines.kubeflow.org/implementations-comp-hello-world: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf - \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def - hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser - = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", - dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}' creationTimestamp: null generateName: hello-world- spec: - arguments: {} + arguments: + parameters: + - name: components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 + value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' + - name: implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 + value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf + \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def + hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser + = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", + dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args + = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}' + - name: components-root + value: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -180,11 +183,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-hello-world}}' + value: '{{workflow.parameters.components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-hello-world}}' + value: '{{workflow.parameters.implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: hello-world-driver @@ -273,7 +276,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-root}}' + value: '{{workflow.parameters.components-root}}' - name: runtime-config value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - name: driver-type diff --git a/backend/src/v2/compiler/argocompiler/testdata/importer.yaml b/backend/src/v2/compiler/argocompiler/testdata/importer.yaml index d0e6ef6eaae..23ec461c307 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/importer.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/importer.yaml @@ -1,14 +1,17 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - annotations: - pipelines.kubeflow.org/components-comp-importer: '{"executorLabel":"exec-importer","inputDefinitions":{"parameters":{"uri":{"type":"STRING"}}},"outputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Dataset"}}}}}' - pipelines.kubeflow.org/components-root: '{"dag":{"tasks":{"importer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}}}}},"taskInfo":{"name":"importer"}}}},"inputDefinitions":{"parameters":{"dataset2":{"type":"STRING"}}}}' - pipelines.kubeflow.org/implementations-comp-importer: '{"artifactUri":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}},"typeSchema":{"schemaTitle":"system.Dataset"}}' creationTimestamp: null generateName: pipeline-with-importer- spec: - arguments: {} + arguments: + parameters: + - name: components-comp-importer + value: '{"executorLabel":"exec-importer","inputDefinitions":{"parameters":{"uri":{"type":"STRING"}}},"outputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Dataset"}}}}}' + - name: implementations-comp-importer + value: '{"artifactUri":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}},"typeSchema":{"schemaTitle":"system.Dataset"}}' + - name: components-root + value: '{"dag":{"tasks":{"importer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}}}}},"taskInfo":{"name":"importer"}}}},"inputDefinitions":{"parameters":{"dataset2":{"type":"STRING"}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -81,9 +84,9 @@ spec: - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}}}}},"taskInfo":{"name":"importer"}}' - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-importer}}' + value: '{{workflow.parameters.components-comp-importer}}' - name: importer - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-importer}}' + value: '{{workflow.parameters.implementations-comp-importer}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: importer @@ -162,7 +165,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-root}}' + value: '{{workflow.parameters.components-root}}' - name: runtime-config value: '{}' - name: driver-type From a503083e3c589d168018a828b4a76ea679ddd3f4 Mon Sep 17 00:00:00 2001 From: Googler Date: Thu, 8 Aug 2024 14:37:05 -0700 Subject: [PATCH 05/54] feat(component): internal Signed-off-by: Googler PiperOrigin-RevId: 660985413 Signed-off-by: KevinGrantLee --- components/google-cloud/RELEASE.md | 1 - .../evaluation_llm_text_generation_pipeline.py | 17 ----------------- 2 files changed, 18 deletions(-) diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md index aa174de2589..7c600b8d044 100644 --- a/components/google-cloud/RELEASE.md +++ b/components/google-cloud/RELEASE.md @@ -10,7 +10,6 @@ * Bump image for Structured Data pipelines. * Add check that component in preview.custom_job.utils.create_custom_training_job_from_component doesn't have any parameters that share names with any custom job fields * Add dynamic machine spec support for `preview.custom_job.utils.create_custom_training_job_from_component`. -* Add preflight validations for LLM text generation pipeline. * Apply latest GCPC image vulnerability resolutions (base OS and software updates). ## Release 2.15.0 diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py index d00fb809852..9b613ee8eb3 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py @@ -15,12 +15,10 @@ from typing import Dict, List, NamedTuple -from google_cloud_pipeline_components import google_template_metadata from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationPreprocessorOp from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationTextGenerationOp from google_cloud_pipeline_components._implementation.model_evaluation import ModelNamePreprocessorOp from google_cloud_pipeline_components.preview.model_evaluation.model_evaluation_import_component import model_evaluation_import as ModelImportEvaluationOp -from google_cloud_pipeline_components.proto import template_metadata_pb2 from google_cloud_pipeline_components.types.artifact_types import VertexModel from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp from kfp import dsl @@ -31,21 +29,6 @@ _PIPELINE_NAME = 'evaluation-llm-text-generation-pipeline' -output_gcs_validation = template_metadata_pb2.GoogleCloudStorageValidation( - gcs_uri='{{$.parameter.batch_predict_gcs_destination_output_uri}}', - is_input=False, - default_service_account='{{$.pipeline_google_cloud_project_number}}-compute@developer.gserviceaccount.com', - override_placeholder='{{$.parameter.service_account}}', -) - - -@google_template_metadata.set_template_metadata( - template_metadata=template_metadata_pb2.TemplateMetadata( - preflight_validations=template_metadata_pb2.ValidationItems( - gcs_validations=[output_gcs_validation] - ) - ) -) @dsl.pipeline(name=_PIPELINE_NAME) def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-default-value project: str, From 0aa54ee26af0e9f452ce7b045f314f04af198d57 Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 9 Aug 2024 11:06:15 -0700 Subject: [PATCH 06/54] feat(components): internal Signed-off-by: Googler PiperOrigin-RevId: 661332120 Signed-off-by: KevinGrantLee --- .../preview/compiler/__init__.py | 7 - .../preview/compiler/compiler.py | 189 ------------------ 2 files changed, 196 deletions(-) delete mode 100644 components/google-cloud/google_cloud_pipeline_components/preview/compiler/__init__.py delete mode 100644 components/google-cloud/google_cloud_pipeline_components/preview/compiler/compiler.py diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/compiler/__init__.py b/components/google-cloud/google_cloud_pipeline_components/preview/compiler/__init__.py deleted file mode 100644 index a12db98c92e..00000000000 --- a/components/google-cloud/google_cloud_pipeline_components/preview/compiler/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""KFP DSL compiler with Vertex Specific Features.""" - -__all__ = [ - 'Compiler', -] - -from google_cloud_pipeline_components.preview.compiler.compiler import Compiler diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/compiler/compiler.py b/components/google-cloud/google_cloud_pipeline_components/preview/compiler/compiler.py deleted file mode 100644 index b8a24ee2578..00000000000 --- a/components/google-cloud/google_cloud_pipeline_components/preview/compiler/compiler.py +++ /dev/null @@ -1,189 +0,0 @@ -"""KFP DSL compiler with Vertex Specific Features. - -This compiler is intended to compile PipelineSpec with Vertex Specifc features. - -To ensure full compatibility with Vertex specifc functionalities, -Google first party pipelines should utilize this version of compiler. -""" - -import os -from os import path -from typing import Any, Dict, Optional - -from absl import logging -from google.protobuf import json_format -from google_cloud_pipeline_components.proto import template_metadata_pb2 -from kfp import compiler as kfp_compiler -from kfp.dsl import base_component -from kfp.pipeline_spec import pipeline_spec_pb2 -import yaml - - -class Compiler: - """Compiles pipelines composed using the KFP SDK DSL to a YAML pipeline definition. - - The pipeline definition is `PipelineSpec IR - `_, - the protobuf message that defines a pipeline. - - Example: - :: - - @dsl.pipeline( - name='name', - ) - def my_pipeline(a: int, b: str = 'default value'): - ... - - compiler.Compiler().compile( - pipeline_func=my_pipeline, - package_path='path/to/pipeline.yaml', - pipeline_parameters={'a': 1}, - ) - """ - - def merge_template_metadata_to_pipeline_spec_proto( - self, - template_metadata: Optional[template_metadata_pb2.TemplateMetadata], - pipeline_spec_proto: pipeline_spec_pb2.PipelineSpec, - ) -> pipeline_spec_pb2.PipelineSpec: - """Merges TemplateMetadata into a PipelineSpec for execution on Google Cloud. - - This function prepares a PipelineSpec for execution on Google Cloud by - incorporating TemplateMetadata into the platform-specific configuration. The - metadata is converted to JSON and embedded within the 'google_cloud' - platform - configuration. - - Args: - template_metadata: A TemplateMetadata object containing component - metadata. - pipeline_spec_proto: A PipelineSpec protobuf object representing the - pipeline specification. - - Returns: - A modified PipelineSpec protobuf object with the TemplateMetadata merged - into the 'google_cloud' PlatformSpec configuration or the original - PlatformSpec proto if the template_metadata is none. - """ - if template_metadata is None: - return pipeline_spec_proto - template_metadata_json = json_format.MessageToJson(template_metadata) - platform_spec_proto = pipeline_spec_pb2.PlatformSpec() - platform_spec_proto.platform = "google_cloud" - json_format.Parse(template_metadata_json, platform_spec_proto.config) - pipeline_spec_proto.root.platform_specs.append(platform_spec_proto) - return pipeline_spec_proto - - def parse_pipeline_spec_yaml( - self, - pipeline_spec_yaml_file: str, - ) -> pipeline_spec_pb2.PipelineSpec: - """Parse pipeline spec yaml parses to the proto. - - Args: - pipeline_spec_yaml_file: Path to the pipeline spec yaml file. - - Returns: - Proto parsed. - - Raises: - ValueError: When the PipelineSpec is invalid. - """ - with open(pipeline_spec_yaml_file, "r") as f: - pipeline_spec_yaml = f.read() - pipeline_spec_dict = yaml.safe_load(pipeline_spec_yaml) - pipeline_spec_proto = pipeline_spec_pb2.PipelineSpec() - try: - json_format.ParseDict(pipeline_spec_dict, pipeline_spec_proto) - except json_format.ParseError as e: - raise ValueError( - "Failed to parse %s . Please check if that is a valid YAML file" - " parsing a pipelineSpec proto." % pipeline_spec_yaml_file - ) from e - if not pipeline_spec_proto.HasField("pipeline_info"): - raise ValueError( - "PipelineInfo field not found in the pipeline spec YAML file %s." - % pipeline_spec_yaml_file - ) - if not pipeline_spec_proto.pipeline_info.display_name: - logging.warning( - ( - "PipelineInfo.displayName field is empty in pipeline spec YAML" - " file %s." - ), - pipeline_spec_yaml_file, - ) - if not pipeline_spec_proto.pipeline_info.description: - logging.warning( - ( - "PipelineInfo.description field is empty in pipeline spec YAML" - " file %s." - ), - pipeline_spec_yaml_file, - ) - return pipeline_spec_proto - - def compile( - self, - pipeline_func: base_component.BaseComponent, - package_path: str, - pipeline_name: Optional[str] = None, - pipeline_parameters: Optional[Dict[str, Any]] = None, - type_check: bool = True, - includ_vertex_specifc_features=True, - ) -> None: - """Compiles the pipeline or component function into IR YAML. - - By default, this compiler will compile any Vertex Specifc Features. - - Args: - pipeline_func: Pipeline function constructed with the ``@dsl.pipeline`` - or component constructed with the ``@dsl.component`` decorator. - package_path: Output YAML file path. For example, - ``'~/my_pipeline.yaml'`` or ``'~/my_component.yaml'``. - pipeline_name: Name of the pipeline. - pipeline_parameters: Map of parameter names to argument values. - type_check: Whether to enable type checking of component interfaces - during compilation. - includ_vertex_specifc_features: Whether to enable compiling Vertex - Specific Features. - """ - if not includ_vertex_specifc_features: - kfp_compiler.Compiler().compile( - pipeline_func=pipeline_func, - package_path=package_path, - pipeline_name=pipeline_name, - pipeline_parameters=pipeline_parameters, - type_check=type_check, - ) - return - - local_temp_output_dir = path.join(path.dirname(package_path), "tmp.yaml") - - kfp_compiler.Compiler().compile( - pipeline_func=pipeline_func, - package_path=local_temp_output_dir, - pipeline_name=pipeline_name, - pipeline_parameters=pipeline_parameters, - type_check=type_check, - ) - - original_pipeline_spec = self.parse_pipeline_spec_yaml( - local_temp_output_dir - ) - template_metadata = getattr(pipeline_func, "template_metadata", None) - updated_pipeline_spec = self.merge_template_metadata_to_pipeline_spec_proto( - template_metadata, original_pipeline_spec - ) - updated_pipeline_spec_dict = json_format.MessageToDict( - updated_pipeline_spec - ) - - with open( - package_path, - "w", - ) as f: - yaml.dump(updated_pipeline_spec_dict, f) - - os.remove(local_temp_output_dir) From eb4755969deb8f5426b9ea2bde8645796c3f41f8 Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 9 Aug 2024 18:23:46 -0700 Subject: [PATCH 07/54] fix(components): Fix to model batch explanation component for Structured Data pipelines Signed-off-by: Googler PiperOrigin-RevId: 661475667 Signed-off-by: KevinGrantLee --- components/google-cloud/RELEASE.md | 2 +- .../forecasting/forecasting_ensemble.py | 2 +- .../forecasting/forecasting_stage_1_tuner.py | 4 +- .../forecasting/forecasting_stage_2_tuner.py | 4 +- .../learn_to_learn_forecasting_pipeline.yaml | 76 +++++++-------- ...ence_to_sequence_forecasting_pipeline.yaml | 76 +++++++-------- ...sion_transformer_forecasting_pipeline.yaml | 76 +++++++-------- ...es_dense_encoder_forecasting_pipeline.yaml | 76 +++++++-------- .../tabular/auto_feature_engineering.py | 2 +- ...ml_tabular_feature_selection_pipeline.yaml | 90 +++++++++--------- .../tabular/automl_tabular_v2_pipeline.yaml | 94 +++++++++---------- ...illation_stage_feature_transform_engine.py | 4 +- .../automl/tabular/feature_selection.py | 4 +- .../tabular/feature_selection_pipeline.yaml | 8 +- .../tabular/feature_transform_engine.py | 6 +- .../tabnet_hyperparameter_tuning_job.py | 4 +- ...et_hyperparameter_tuning_job_pipeline.yaml | 30 +++--- .../preview/automl/tabular/tabnet_trainer.py | 4 +- .../tabular/tabnet_trainer_pipeline.yaml | 26 ++--- ...wide_and_deep_hyperparameter_tuning_job.py | 4 +- ...ep_hyperparameter_tuning_job_pipeline.yaml | 28 +++--- .../automl/tabular/wide_and_deep_trainer.py | 4 +- .../wide_and_deep_trainer_pipeline.yaml | 26 ++--- ...st_hyperparameter_tuning_job_pipeline.yaml | 28 +++--- .../tabular/xgboost_trainer_pipeline.yaml | 26 ++--- .../bqml_arima_predict_pipeline.yaml | 20 ++-- .../bqml_arima_train_pipeline.yaml | 62 ++++++------ .../forecasting/prophet_predict_pipeline.yaml | 26 ++--- .../v1/automl/forecasting/prophet_trainer.py | 6 +- .../forecasting/prophet_trainer_pipeline.yaml | 28 +++--- .../tabular/automl_tabular_pipeline.yaml | 86 ++++++++--------- .../v1/automl/tabular/cv_trainer.py | 4 +- .../v1/automl/tabular/ensemble.py | 4 +- .../v1/automl/tabular/finalizer.py | 2 +- .../v1/automl/tabular/infra_validator.py | 2 +- .../automl/tabular/split_materialized_data.py | 2 +- .../v1/automl/tabular/stage_1_tuner.py | 4 +- .../automl/tabular/stats_and_example_gen.py | 4 +- .../training_configurator_and_validator.py | 2 +- .../v1/automl/tabular/transform.py | 4 +- 40 files changed, 480 insertions(+), 480 deletions(-) diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md index 7c600b8d044..af292c0ab47 100644 --- a/components/google-cloud/RELEASE.md +++ b/components/google-cloud/RELEASE.md @@ -1,5 +1,5 @@ ## Upcoming release -* Bump image for Structured Data pipelines. +* Fix to model batch explanation component for Structured Data pipelines; image bump. ## Release 2.16.0 * Updated the Starry Net pipeline's template gallery description, and added dataprep_nan_threshold and dataprep_zero_threshold args to the Starry Net pipeline. diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_ensemble.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_ensemble.py index dda131659b8..0c4a2aa6834 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_ensemble.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_ensemble.py @@ -72,7 +72,7 @@ def automl_forecasting_ensemble( # fmt: on job_id = dsl.PIPELINE_JOB_ID_PLACEHOLDER task_id = dsl.PIPELINE_TASK_ID_PLACEHOLDER - image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625' + image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625' display_name = f'automl-forecasting-ensemble-{job_id}-{task_id}' error_file_path = f'{root_dir}/{job_id}/{task_id}/error.pb' diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_1_tuner.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_1_tuner.py index 95b9e406016..cc2a4bf1a63 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_1_tuner.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_1_tuner.py @@ -99,14 +99,14 @@ def automl_forecasting_stage_1_tuner( ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' ' "container_spec": {"image_uri":"' ), - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625', '", "args": ["forecasting_mp_l2l_stage_1_tuner', '", "--region=', location, '", "--transform_output_path=', transform_output.uri, '", "--training_docker_uri=', - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625', '", "--reduce_search_space_mode=', reduce_search_space_mode, f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_2_tuner.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_2_tuner.py index adb0d643e6e..2da6c0d22b7 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_2_tuner.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_2_tuner.py @@ -97,14 +97,14 @@ def automl_forecasting_stage_2_tuner( ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' ' "container_spec": {"image_uri":"' ), - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625', '", "args": ["forecasting_mp_l2l_stage_2_tuner', '", "--region=', location, '", "--transform_output_path=', transform_output.uri, '", "--training_docker_uri=', - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625', f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}', '", "--training_base_dir=', root_dir, diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/learn_to_learn_forecasting_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/learn_to_learn_forecasting_pipeline.yaml index 6b49dc77e04..97f07356b62 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/learn_to_learn_forecasting_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/learn_to_learn_forecasting_pipeline.yaml @@ -5577,7 +5577,7 @@ deploymentSpec: - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": - {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", @@ -5611,7 +5611,7 @@ deploymentSpec: - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": - {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", @@ -5646,11 +5646,11 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"args\": [\"forecasting_mp_l2l_stage_1_tuner", "\", \"--region=", "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", @@ -5689,11 +5689,11 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"args\": [\"forecasting_mp_l2l_stage_2_tuner", "\", \"--region=", "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", @@ -5732,7 +5732,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -5797,7 +5797,7 @@ deploymentSpec: \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-calculate-training-parameters-2: container: args: @@ -5853,7 +5853,7 @@ deploymentSpec: \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-attribution: container: args: @@ -6044,8 +6044,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -6062,7 +6062,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -6093,7 +6093,7 @@ deploymentSpec: \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ \ ),\n )(forecasting_type, quantiles)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-finalize-eval-quantile-parameters-2: container: args: @@ -6121,7 +6121,7 @@ deploymentSpec: \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ \ ),\n )(forecasting_type, quantiles)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-or-create-model-description: container: args: @@ -6150,7 +6150,7 @@ deploymentSpec: \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-or-create-model-description-2: container: args: @@ -6179,7 +6179,7 @@ deploymentSpec: \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-prediction-image-uri: container: args: @@ -6202,14 +6202,14 @@ deploymentSpec: Returns the prediction image corresponding to the given model type.\"\"\"\ \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ \ must be hardcoded without any breaks in the code so string\n # replacement\ - \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240804_0625',\n\ - \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240804_0625',\n\ - \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240804_0625',\n\ - \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240804_0625',\n\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240808_0625',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240808_0625',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240808_0625',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240808_0625',\n\ \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ \ )\n return images[model_type]\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-prediction-image-uri-2: container: args: @@ -6232,14 +6232,14 @@ deploymentSpec: Returns the prediction image corresponding to the given model type.\"\"\"\ \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ \ must be hardcoded without any breaks in the code so string\n # replacement\ - \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240804_0625',\n\ - \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240804_0625',\n\ - \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240804_0625',\n\ - \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240804_0625',\n\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240808_0625',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240808_0625',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240808_0625',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240808_0625',\n\ \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ \ )\n return images[model_type]\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-predictions-column: container: args: @@ -6262,7 +6262,7 @@ deploymentSpec: \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ \ return f'predicted_{target_column}.value'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-predictions-column-2: container: args: @@ -6285,7 +6285,7 @@ deploymentSpec: \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ \ return f'predicted_{target_column}.value'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-importer: importer: artifactUri: @@ -6334,7 +6334,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-explanation-2: container: args: @@ -6376,7 +6376,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-predict: container: args: @@ -6731,7 +6731,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-upload-2: container: args: @@ -6760,7 +6760,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-set-optional-inputs: container: args: @@ -6817,7 +6817,7 @@ deploymentSpec: \ 'model_display_name',\n 'transformations',\n ],\n\ \ )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ model_display_name,\n transformations,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -6863,7 +6863,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-string-not-empty: container: args: @@ -6887,7 +6887,7 @@ deploymentSpec: \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ \ \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri: container: args: @@ -6917,7 +6917,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri-2: container: args: @@ -6947,7 +6947,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-training-configurator-and-validator: container: args: @@ -6992,7 +6992,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 pipelineInfo: description: The AutoML Forecasting pipeline. name: learn-to-learn-forecasting diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/sequence_to_sequence_forecasting_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/sequence_to_sequence_forecasting_pipeline.yaml index 6fece21f998..65bc6863845 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/sequence_to_sequence_forecasting_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/sequence_to_sequence_forecasting_pipeline.yaml @@ -5559,7 +5559,7 @@ deploymentSpec: - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": - {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", @@ -5593,7 +5593,7 @@ deploymentSpec: - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": - {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", @@ -5628,11 +5628,11 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"args\": [\"forecasting_mp_l2l_stage_1_tuner", "\", \"--region=", "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", @@ -5671,11 +5671,11 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"args\": [\"forecasting_mp_l2l_stage_2_tuner", "\", \"--region=", "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", @@ -5714,7 +5714,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -5779,7 +5779,7 @@ deploymentSpec: \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-calculate-training-parameters-2: container: args: @@ -5835,7 +5835,7 @@ deploymentSpec: \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-attribution: container: args: @@ -6026,8 +6026,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -6044,7 +6044,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -6075,7 +6075,7 @@ deploymentSpec: \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ \ ),\n )(forecasting_type, quantiles)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-finalize-eval-quantile-parameters-2: container: args: @@ -6103,7 +6103,7 @@ deploymentSpec: \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ \ ),\n )(forecasting_type, quantiles)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-or-create-model-description: container: args: @@ -6132,7 +6132,7 @@ deploymentSpec: \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-or-create-model-description-2: container: args: @@ -6161,7 +6161,7 @@ deploymentSpec: \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-prediction-image-uri: container: args: @@ -6184,14 +6184,14 @@ deploymentSpec: Returns the prediction image corresponding to the given model type.\"\"\"\ \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ \ must be hardcoded without any breaks in the code so string\n # replacement\ - \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240804_0625',\n\ - \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240804_0625',\n\ - \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240804_0625',\n\ - \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240804_0625',\n\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240808_0625',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240808_0625',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240808_0625',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240808_0625',\n\ \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ \ )\n return images[model_type]\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-prediction-image-uri-2: container: args: @@ -6214,14 +6214,14 @@ deploymentSpec: Returns the prediction image corresponding to the given model type.\"\"\"\ \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ \ must be hardcoded without any breaks in the code so string\n # replacement\ - \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240804_0625',\n\ - \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240804_0625',\n\ - \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240804_0625',\n\ - \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240804_0625',\n\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240808_0625',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240808_0625',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240808_0625',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240808_0625',\n\ \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ \ )\n return images[model_type]\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-predictions-column: container: args: @@ -6244,7 +6244,7 @@ deploymentSpec: \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ \ return f'predicted_{target_column}.value'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-predictions-column-2: container: args: @@ -6267,7 +6267,7 @@ deploymentSpec: \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ \ return f'predicted_{target_column}.value'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-importer: importer: artifactUri: @@ -6316,7 +6316,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-explanation-2: container: args: @@ -6358,7 +6358,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-predict: container: args: @@ -6713,7 +6713,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-upload-2: container: args: @@ -6742,7 +6742,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-set-optional-inputs: container: args: @@ -6799,7 +6799,7 @@ deploymentSpec: \ 'model_display_name',\n 'transformations',\n ],\n\ \ )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ model_display_name,\n transformations,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -6845,7 +6845,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-string-not-empty: container: args: @@ -6869,7 +6869,7 @@ deploymentSpec: \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ \ \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri: container: args: @@ -6899,7 +6899,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri-2: container: args: @@ -6929,7 +6929,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-training-configurator-and-validator: container: args: @@ -6974,7 +6974,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 pipelineInfo: description: The Sequence to Sequence (Seq2Seq) Forecasting pipeline. name: sequence-to-sequence-forecasting diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/temporal_fusion_transformer_forecasting_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/temporal_fusion_transformer_forecasting_pipeline.yaml index 566b4ef1e2b..b5c9d95da37 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/temporal_fusion_transformer_forecasting_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/temporal_fusion_transformer_forecasting_pipeline.yaml @@ -5552,7 +5552,7 @@ deploymentSpec: - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": - {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", @@ -5586,7 +5586,7 @@ deploymentSpec: - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": - {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", @@ -5621,11 +5621,11 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"args\": [\"forecasting_mp_l2l_stage_1_tuner", "\", \"--region=", "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", @@ -5664,11 +5664,11 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"args\": [\"forecasting_mp_l2l_stage_2_tuner", "\", \"--region=", "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", @@ -5707,7 +5707,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -5772,7 +5772,7 @@ deploymentSpec: \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-calculate-training-parameters-2: container: args: @@ -5828,7 +5828,7 @@ deploymentSpec: \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-attribution: container: args: @@ -6019,8 +6019,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -6037,7 +6037,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -6068,7 +6068,7 @@ deploymentSpec: \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ \ ),\n )(forecasting_type, quantiles)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-finalize-eval-quantile-parameters-2: container: args: @@ -6096,7 +6096,7 @@ deploymentSpec: \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ \ ),\n )(forecasting_type, quantiles)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-or-create-model-description: container: args: @@ -6125,7 +6125,7 @@ deploymentSpec: \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-or-create-model-description-2: container: args: @@ -6154,7 +6154,7 @@ deploymentSpec: \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-prediction-image-uri: container: args: @@ -6177,14 +6177,14 @@ deploymentSpec: Returns the prediction image corresponding to the given model type.\"\"\"\ \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ \ must be hardcoded without any breaks in the code so string\n # replacement\ - \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240804_0625',\n\ - \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240804_0625',\n\ - \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240804_0625',\n\ - \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240804_0625',\n\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240808_0625',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240808_0625',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240808_0625',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240808_0625',\n\ \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ \ )\n return images[model_type]\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-prediction-image-uri-2: container: args: @@ -6207,14 +6207,14 @@ deploymentSpec: Returns the prediction image corresponding to the given model type.\"\"\"\ \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ \ must be hardcoded without any breaks in the code so string\n # replacement\ - \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240804_0625',\n\ - \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240804_0625',\n\ - \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240804_0625',\n\ - \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240804_0625',\n\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240808_0625',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240808_0625',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240808_0625',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240808_0625',\n\ \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ \ )\n return images[model_type]\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-predictions-column: container: args: @@ -6237,7 +6237,7 @@ deploymentSpec: \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ \ return f'predicted_{target_column}.value'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-predictions-column-2: container: args: @@ -6260,7 +6260,7 @@ deploymentSpec: \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ \ return f'predicted_{target_column}.value'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-importer: importer: artifactUri: @@ -6309,7 +6309,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-explanation-2: container: args: @@ -6351,7 +6351,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-predict: container: args: @@ -6706,7 +6706,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-upload-2: container: args: @@ -6735,7 +6735,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-set-optional-inputs: container: args: @@ -6792,7 +6792,7 @@ deploymentSpec: \ 'model_display_name',\n 'transformations',\n ],\n\ \ )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ model_display_name,\n transformations,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -6838,7 +6838,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-string-not-empty: container: args: @@ -6862,7 +6862,7 @@ deploymentSpec: \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ \ \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri: container: args: @@ -6892,7 +6892,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri-2: container: args: @@ -6922,7 +6922,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-training-configurator-and-validator: container: args: @@ -6967,7 +6967,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 pipelineInfo: description: The Temporal Fusion Transformer (TFT) Forecasting pipeline. name: temporal-fusion-transformer-forecasting diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/time_series_dense_encoder_forecasting_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/time_series_dense_encoder_forecasting_pipeline.yaml index f3a9b21cc19..f6a485fd4e3 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/time_series_dense_encoder_forecasting_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/time_series_dense_encoder_forecasting_pipeline.yaml @@ -5577,7 +5577,7 @@ deploymentSpec: - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": - {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", @@ -5611,7 +5611,7 @@ deploymentSpec: - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": - {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", @@ -5646,11 +5646,11 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"args\": [\"forecasting_mp_l2l_stage_1_tuner", "\", \"--region=", "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", @@ -5689,11 +5689,11 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"args\": [\"forecasting_mp_l2l_stage_2_tuner", "\", \"--region=", "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625", "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", @@ -5732,7 +5732,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -5797,7 +5797,7 @@ deploymentSpec: \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-calculate-training-parameters-2: container: args: @@ -5853,7 +5853,7 @@ deploymentSpec: \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-attribution: container: args: @@ -6044,8 +6044,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -6062,7 +6062,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -6093,7 +6093,7 @@ deploymentSpec: \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ \ ),\n )(forecasting_type, quantiles)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-finalize-eval-quantile-parameters-2: container: args: @@ -6121,7 +6121,7 @@ deploymentSpec: \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ \ ),\n )(forecasting_type, quantiles)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-or-create-model-description: container: args: @@ -6150,7 +6150,7 @@ deploymentSpec: \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-or-create-model-description-2: container: args: @@ -6179,7 +6179,7 @@ deploymentSpec: \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-prediction-image-uri: container: args: @@ -6202,14 +6202,14 @@ deploymentSpec: Returns the prediction image corresponding to the given model type.\"\"\"\ \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ \ must be hardcoded without any breaks in the code so string\n # replacement\ - \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240804_0625',\n\ - \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240804_0625',\n\ - \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240804_0625',\n\ - \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240804_0625',\n\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240808_0625',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240808_0625',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240808_0625',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240808_0625',\n\ \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ \ )\n return images[model_type]\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-prediction-image-uri-2: container: args: @@ -6232,14 +6232,14 @@ deploymentSpec: Returns the prediction image corresponding to the given model type.\"\"\"\ \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ \ must be hardcoded without any breaks in the code so string\n # replacement\ - \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240804_0625',\n\ - \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240804_0625',\n\ - \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240804_0625',\n\ - \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240804_0625',\n\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240808_0625',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240808_0625',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240808_0625',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240808_0625',\n\ \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ \ )\n return images[model_type]\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-predictions-column: container: args: @@ -6262,7 +6262,7 @@ deploymentSpec: \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ \ return f'predicted_{target_column}.value'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-predictions-column-2: container: args: @@ -6285,7 +6285,7 @@ deploymentSpec: \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ \ return f'predicted_{target_column}.value'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-importer: importer: artifactUri: @@ -6334,7 +6334,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-explanation-2: container: args: @@ -6376,7 +6376,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-predict: container: args: @@ -6731,7 +6731,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-upload-2: container: args: @@ -6760,7 +6760,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-set-optional-inputs: container: args: @@ -6817,7 +6817,7 @@ deploymentSpec: \ 'model_display_name',\n 'transformations',\n ],\n\ \ )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ model_display_name,\n transformations,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -6863,7 +6863,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-string-not-empty: container: args: @@ -6887,7 +6887,7 @@ deploymentSpec: \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ \ \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri: container: args: @@ -6917,7 +6917,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri-2: container: args: @@ -6947,7 +6947,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-training-configurator-and-validator: container: args: @@ -6992,7 +6992,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 pipelineInfo: description: The Timeseries Dense Encoder (TiDE) Forecasting pipeline. name: time-series-dense-encoder-forecasting diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/auto_feature_engineering.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/auto_feature_engineering.py index 7b0de04f55f..361d3d1b776 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/auto_feature_engineering.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/auto_feature_engineering.py @@ -65,7 +65,7 @@ def automated_feature_engineering( ' 1, "machine_spec": {"machine_type": "n1-standard-16"},' ' "container_spec": {"image_uri":"' ), - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625', '", "args": ["feature_engineering", "--project=', project, '", "--location=', location, '", "--data_source_bigquery_table_path=', data_source_bigquery_table_path, diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_feature_selection_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_feature_selection_pipeline.yaml index 1a40bca0d0c..fe2b8ca0c42 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_feature_selection_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_feature_selection_pipeline.yaml @@ -8622,9 +8622,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", @@ -8665,9 +8665,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", @@ -8708,7 +8708,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", @@ -8720,7 +8720,7 @@ deploymentSpec: "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", @@ -8749,7 +8749,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", @@ -8761,7 +8761,7 @@ deploymentSpec: "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", @@ -8790,7 +8790,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", @@ -8802,7 +8802,7 @@ deploymentSpec: "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", @@ -8831,7 +8831,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -8846,7 +8846,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -8855,7 +8855,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -8864,7 +8864,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -8884,9 +8884,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", @@ -8931,9 +8931,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", @@ -8978,7 +8978,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"transform\", \"--is_mp=true\", \"--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}", "\", \"--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", @@ -8999,7 +8999,7 @@ deploymentSpec: \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", - "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625", "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", @@ -9030,7 +9030,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"transform\", \"--is_mp=true\", \"--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}", "\", \"--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", @@ -9051,7 +9051,7 @@ deploymentSpec: \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", - "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625", "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", @@ -9087,7 +9087,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bool-identity-2: container: args: @@ -9109,7 +9109,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bool-identity-3: container: args: @@ -9131,7 +9131,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-calculate-training-parameters: container: args: @@ -9223,7 +9223,7 @@ deploymentSpec: \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ \ reduce_search_space_mode,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-calculate-training-parameters-2: container: args: @@ -9315,7 +9315,7 @@ deploymentSpec: \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ \ reduce_search_space_mode,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-check-if-binary-classification: container: args: @@ -9343,7 +9343,7 @@ deploymentSpec: \ with open(example_gen_metadata, 'r') as f:\n metadata_path = f.read()\n\ \ metadata = json.loads(metadata_path)\n return str(metadata['objective']\ \ == 'binary_classification').lower()\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-attribution: container: args: @@ -9536,7 +9536,7 @@ deploymentSpec: \ 'r') as f:\n split_0_content = f.read()\n with open(split_1, 'r')\ \ as f:\n split_1_content = f.read()\n with open(splits, 'w') as f:\n\ \ f.write(','.join([split_0_content, split_1_content]))\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-model-batch-explanation: container: args: @@ -9578,7 +9578,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-explanation-2: container: args: @@ -9620,7 +9620,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-explanation-3: container: args: @@ -9662,7 +9662,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-predict: container: args: @@ -10275,7 +10275,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-upload-2: container: args: @@ -10304,7 +10304,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-upload-3: container: args: @@ -10333,7 +10333,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-purge-unused-features: container: args: @@ -10383,7 +10383,7 @@ deploymentSpec: \n train_spec['transformations'] = purged_transformation_list\n metadata['train_spec']\ \ = train_spec\n\n with open(output_metadata, 'w') as f:\n f.write(json.dumps(metadata))\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-read-input-uri: container: args: @@ -10411,7 +10411,7 @@ deploymentSpec: \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ \ with open(split_uri, 'r') as f:\n data_source = json.loads(f.read())\n\ \ return data_source['tf_record_data_source']['file_patterns']\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-read-input-uri-2: container: args: @@ -10439,7 +10439,7 @@ deploymentSpec: \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ \ with open(split_uri, 'r') as f:\n data_source = json.loads(f.read())\n\ \ return data_source['tf_record_data_source']['file_patterns']\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-string-not-empty: container: args: @@ -10463,7 +10463,7 @@ deploymentSpec: \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ \ \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-tabular-feature-ranking-and-selection: container: args: @@ -10480,7 +10480,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"feature_selection\", \"--data_source=", "{{$.inputs.artifacts[''data_source''].uri}}", "\", \"--target_column=", "{{$.inputs.parameters[''target_column_name'']}}", "\", \"--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}", @@ -10493,7 +10493,7 @@ deploymentSpec: \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", - "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625", "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", @@ -10526,7 +10526,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"stats_generator\",", "\"--train_spec={\\\"prediction_type\\\": \\\"", "{{$.inputs.parameters[''prediction_type'']}}", "\\\", \\\"target_column\\\": \\\"", "{{$.inputs.parameters[''target_column_name'']}}", "\\\", \\\"optimization_objective\\\": @@ -10559,7 +10559,7 @@ deploymentSpec: \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", - "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625", "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", @@ -10614,7 +10614,7 @@ deploymentSpec: \ f'{directory}/prediction.results-*',\n ],\n 'coder':\ \ 'PROTO_VALUE',\n },\n }\n with open(result, 'w') as f:\n f.write(json.dumps(data_source))\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-write-bp-result-path-2: container: args: @@ -10644,7 +10644,7 @@ deploymentSpec: \ f'{directory}/prediction.results-*',\n ],\n 'coder':\ \ 'PROTO_VALUE',\n },\n }\n with open(result, 'w') as f:\n f.write(json.dumps(data_source))\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 pipelineInfo: description: The AutoML Tabular pipeline. name: automl-tabular-feature-selection-pipeline diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_v2_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_v2_pipeline.yaml index 5b37c8f7f56..87f7bafce82 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_v2_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_v2_pipeline.yaml @@ -9452,9 +9452,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", @@ -9495,9 +9495,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", @@ -9538,7 +9538,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", @@ -9550,7 +9550,7 @@ deploymentSpec: "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", @@ -9579,7 +9579,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", @@ -9591,7 +9591,7 @@ deploymentSpec: "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", @@ -9620,7 +9620,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", @@ -9632,7 +9632,7 @@ deploymentSpec: "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", @@ -9661,7 +9661,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -9676,7 +9676,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -9685,7 +9685,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -9694,7 +9694,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -9714,9 +9714,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", @@ -9761,9 +9761,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", @@ -9813,7 +9813,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bool-identity-2: container: args: @@ -9835,7 +9835,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bool-identity-3: container: args: @@ -9857,7 +9857,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-calculate-training-parameters: container: args: @@ -9949,7 +9949,7 @@ deploymentSpec: \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ \ reduce_search_space_mode,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-calculate-training-parameters-2: container: args: @@ -10041,7 +10041,7 @@ deploymentSpec: \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ \ reduce_search_space_mode,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-distillation-stage-feature-transform-engine: container: args: @@ -10075,14 +10075,14 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -10329,8 +10329,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -10347,7 +10347,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -10382,7 +10382,7 @@ deploymentSpec: \ collections.namedtuple(\n 'Outputs',\n [\n 'bigquery_destination_output_uri',\n\ \ ],\n )(\n f'{bigquery_staging_dataset_uri}.{table_prefix}{model_display_name}{curr_time}',\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-bigquery-destination-output-uri-2: container: args: @@ -10414,7 +10414,7 @@ deploymentSpec: \ collections.namedtuple(\n 'Outputs',\n [\n 'bigquery_destination_output_uri',\n\ \ ],\n )(\n f'{bigquery_staging_dataset_uri}.{table_prefix}{model_display_name}{curr_time}',\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-bp-bq-output-table: container: args: @@ -10442,7 +10442,7 @@ deploymentSpec: \n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'bq_output_table_uri',\n ],\n )(\n f\"{bp_job.metadata['bigqueryOutputDataset']}.{bp_job.metadata['bigqueryOutputTable']}\"\ ,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-bp-bq-output-table-2: container: args: @@ -10470,7 +10470,7 @@ deploymentSpec: \n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'bq_output_table_uri',\n ],\n )(\n f\"{bp_job.metadata['bigqueryOutputDataset']}.{bp_job.metadata['bigqueryOutputTable']}\"\ ,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-model-display-name: container: args: @@ -10497,7 +10497,7 @@ deploymentSpec: \n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'model_display_name',\n ],\n )(\n model_display_name,\n )\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-transform-config-path: container: args: @@ -10530,7 +10530,7 @@ deploymentSpec: \ )\n\n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'transform_config_path',\n ],\n )(\n transform_config_path,\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-importer: importer: artifactUri: @@ -10564,7 +10564,7 @@ deploymentSpec: \ 'r') as f:\n split_0_content = f.read()\n with open(split_1, 'r')\ \ as f:\n split_1_content = f.read()\n with open(splits, 'w') as f:\n\ \ f.write(','.join([split_0_content, split_1_content]))\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-model-batch-explanation: container: args: @@ -10606,7 +10606,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-explanation-2: container: args: @@ -10648,7 +10648,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-explanation-3: container: args: @@ -10690,7 +10690,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-predict: container: args: @@ -11303,7 +11303,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-upload-2: container: args: @@ -11332,7 +11332,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-upload-3: container: args: @@ -11361,7 +11361,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-set-optional-inputs: container: args: @@ -11409,7 +11409,7 @@ deploymentSpec: \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ \ ],\n )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -11455,7 +11455,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-split-materialized-data-2: container: args: @@ -11501,7 +11501,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-string-not-empty: container: args: @@ -11525,7 +11525,7 @@ deploymentSpec: \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ \ \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-training-configurator-and-validator: container: args: @@ -11570,7 +11570,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 exec-training-configurator-and-validator-2: container: args: @@ -11615,7 +11615,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 pipelineInfo: description: The AutoML Tabular pipeline v2. name: automl-tabular-v2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/distillation_stage_feature_transform_engine.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/distillation_stage_feature_transform_engine.py index 225f997a137..aa895f7a23e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/distillation_stage_feature_transform_engine.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/distillation_stage_feature_transform_engine.py @@ -77,7 +77,7 @@ def distillation_stage_feature_transform_engine( # fmt: on return dsl.ContainerSpec( - image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625', + image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625', command=[], args=[ 'distillation_stage_feature_transform_engine', @@ -185,7 +185,7 @@ def distillation_stage_feature_transform_engine( dataflow_machine_type, ] ), - '--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625', + '--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625', dsl.ConcatPlaceholder( items=[ '--dataflow_disk_size_gb=', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection.py index caf61cbb790..f84a13bc78b 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection.py @@ -100,7 +100,7 @@ def tabular_feature_ranking_and_selection( ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' ' "container_spec": {"image_uri":"' ), - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625', '", "args": ["feature_selection", "--data_source=', data_source.uri, '", "--target_column=', @@ -137,7 +137,7 @@ def tabular_feature_ranking_and_selection( ), dataflow_max_num_workers, '", "--dataflow_worker_container_image=', - 'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625', '", "--dataflow_machine_type=', dataflow_machine_type, '", "--dataflow_disk_size_gb=', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection_pipeline.yaml index b21083d2c42..5dd99653a0e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection_pipeline.yaml @@ -983,8 +983,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -1001,7 +1001,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -1049,7 +1049,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 pipelineInfo: description: Defines pipeline for feature transform engine component. name: feature-selection diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_transform_engine.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_transform_engine.py index 0a1174299b4..08d2a178bfb 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_transform_engine.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_transform_engine.py @@ -308,7 +308,7 @@ def feature_transform_engine( # fmt: on return dsl.ContainerSpec( - image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625', + image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625', command=[], args=[ 'feature_transform_engine', @@ -637,8 +637,8 @@ def feature_transform_engine( dsl.ConcatPlaceholder( items=['--dataflow_machine_type=', dataflow_machine_type] ), - '--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625', - '--feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625', + '--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625', + '--feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625', dsl.ConcatPlaceholder( items=['--dataflow_disk_size_gb=', dataflow_disk_size_gb] ), diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job.py index 7f1194939d1..e6996cc22c2 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job.py @@ -158,7 +158,7 @@ def tabnet_hyperparameter_tuning_job( ', "disk_spec": ', training_disk_spec, ', "container_spec": {"image_uri":"', - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240808_0625', '", "args": ["--target_column=', target_column, '", "--weight_column=', @@ -166,7 +166,7 @@ def tabnet_hyperparameter_tuning_job( '", "--model_type=', prediction_type, '", "--prediction_docker_uri=', - 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625', '", "--prediction_docker_uri_artifact_path=', prediction_docker_uri_output, '", "--baseline_path=', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job_pipeline.yaml index ff3749cdb47..df7363e9158 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job_pipeline.yaml @@ -2826,7 +2826,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -2841,7 +2841,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -2866,7 +2866,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-transform-engine: container: args: @@ -2951,8 +2951,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -2969,7 +2969,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -3037,7 +3037,7 @@ deploymentSpec: \ = {\n 'instanceSchemaUri': instance_schema_uri,\n 'predictionSchemaUri':\ \ prediction_schema_uri,\n }\n unmanaged_container_model.uri = os.path.join(\n\ \ trials_dir, 'trial_{}'.format(best_trial['id']), 'model'\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-model-display-name: container: args: @@ -3064,7 +3064,7 @@ deploymentSpec: \n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'model_display_name',\n ],\n )(\n model_display_name,\n )\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-tabnet-study-spec-parameters: container: args: @@ -3580,7 +3580,7 @@ deploymentSpec: \ = ', '.join(extra_overrides)\n warnings.warn(\n f'The overrides\ \ {extra_override_str} were not found in the params and '\n 'will\ \ be ignored.'\n )\n\n return study_spec_parameters\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-model-batch-predict: container: args: @@ -3821,7 +3821,7 @@ deploymentSpec: \ 'training_disk_spec',\n 'eval_machine_spec',\n 'eval_replica_count',\n\ \ ],\n )(\n training_machine_spec,\n training_disk_spec,\n\ \ eval_machine_spec,\n eval_replica_count,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-set-optional-inputs: container: args: @@ -3869,7 +3869,7 @@ deploymentSpec: \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ \ ],\n )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -3915,7 +3915,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-tabnet-hyperparameter-tuning-job: container: args: @@ -3943,11 +3943,11 @@ deploymentSpec: ", \"trial_job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"", "1", "\", \"machine_spec\": ", "{{$.inputs.parameters[''training_machine_spec'']}}", ", \"disk_spec\": ", "{{$.inputs.parameters[''training_disk_spec'']}}", - ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240804_0625", + ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240808_0625", "\", \"args\": [\"--target_column=", "{{$.inputs.parameters[''target_column'']}}", "\", \"--weight_column=", "{{$.inputs.parameters[''weight_column'']}}", "\", \"--model_type=", "{{$.inputs.parameters[''prediction_type'']}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--prediction_docker_uri_artifact_path=", "{{$.outputs.parameters[''prediction_docker_uri_output''].output_file}}", "\", \"--baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", @@ -4016,7 +4016,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 pipelineInfo: description: The TabNet built-in algorithm HyperparameterTuningJob pipeline. name: automl-tabular-tabnet-hyperparameter-tuning-job diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer.py index b9d0ee0fa1f..9ed5b39714a 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer.py @@ -165,7 +165,7 @@ def tabnet_trainer( ', "disk_spec": ', training_disk_spec, ', "container_spec": {"image_uri":"', - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240808_0625', '", "args": ["--target_column=', target_column, '", "--weight_column=', @@ -173,7 +173,7 @@ def tabnet_trainer( '", "--model_type=', prediction_type, '", "--prediction_docker_uri=', - 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625', '", "--baseline_path=', instance_baseline.uri, '", "--metadata_path=', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer_pipeline.yaml index b6d53dc3c37..b7a1ef0278e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer_pipeline.yaml @@ -2875,7 +2875,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -2890,7 +2890,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -2915,7 +2915,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-transform-engine: container: args: @@ -3000,8 +3000,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -3018,7 +3018,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -3048,7 +3048,7 @@ deploymentSpec: \n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'model_display_name',\n ],\n )(\n model_display_name,\n )\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-model-batch-predict: container: args: @@ -3289,7 +3289,7 @@ deploymentSpec: \ 'training_disk_spec',\n 'eval_machine_spec',\n 'eval_replica_count',\n\ \ ],\n )(\n training_machine_spec,\n training_disk_spec,\n\ \ eval_machine_spec,\n eval_replica_count,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-set-optional-inputs: container: args: @@ -3337,7 +3337,7 @@ deploymentSpec: \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ \ ],\n )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -3383,7 +3383,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-tabnet-trainer: container: args: @@ -3401,11 +3401,11 @@ deploymentSpec: "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"", "1", "\", \"machine_spec\": ", "{{$.inputs.parameters[''training_machine_spec'']}}", ", \"disk_spec\": ", "{{$.inputs.parameters[''training_disk_spec'']}}", - ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240804_0625", + ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240808_0625", "\", \"args\": [\"--target_column=", "{{$.inputs.parameters[''target_column'']}}", "\", \"--weight_column=", "{{$.inputs.parameters[''weight_column'']}}", "\", \"--model_type=", "{{$.inputs.parameters[''prediction_type'']}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", @@ -3492,7 +3492,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 pipelineInfo: description: 'Train a model using the Tabular Workflow for TabNet pipelines. diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job.py index c6b855aa4fc..df7a03d25a8 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job.py @@ -158,7 +158,7 @@ def wide_and_deep_hyperparameter_tuning_job( ', "disk_spec": ', training_disk_spec, ', "container_spec": {"image_uri":"', - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20240808_0625', '", "args": ["--target_column=', target_column, '", "--weight_column=', @@ -166,7 +166,7 @@ def wide_and_deep_hyperparameter_tuning_job( '", "--model_type=', prediction_type, '", "--prediction_docker_uri=', - 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625', '", "--prediction_docker_uri_artifact_path=', prediction_docker_uri_output, '", "--baseline_path=', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job_pipeline.yaml index 97832a556cb..38b4c4df5c4 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job_pipeline.yaml @@ -2632,7 +2632,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -2647,7 +2647,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -2672,7 +2672,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-transform-engine: container: args: @@ -2757,8 +2757,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -2775,7 +2775,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -2843,7 +2843,7 @@ deploymentSpec: \ = {\n 'instanceSchemaUri': instance_schema_uri,\n 'predictionSchemaUri':\ \ prediction_schema_uri,\n }\n unmanaged_container_model.uri = os.path.join(\n\ \ trials_dir, 'trial_{}'.format(best_trial['id']), 'model'\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-model-display-name: container: args: @@ -2870,7 +2870,7 @@ deploymentSpec: \n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'model_display_name',\n ],\n )(\n model_display_name,\n )\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-wide-and-deep-study-spec-parameters: container: args: @@ -3147,7 +3147,7 @@ deploymentSpec: \ 'training_disk_spec',\n 'eval_machine_spec',\n 'eval_replica_count',\n\ \ ],\n )(\n training_machine_spec,\n training_disk_spec,\n\ \ eval_machine_spec,\n eval_replica_count,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-set-optional-inputs: container: args: @@ -3195,7 +3195,7 @@ deploymentSpec: \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ \ ],\n )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -3241,7 +3241,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-training-configurator-and-validator: container: args: @@ -3286,7 +3286,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 exec-wide-and-deep-hyperparameter-tuning-job: container: args: @@ -3314,11 +3314,11 @@ deploymentSpec: ", \"trial_job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"", "1", "\", \"machine_spec\": ", "{{$.inputs.parameters[''training_machine_spec'']}}", ", \"disk_spec\": ", "{{$.inputs.parameters[''training_disk_spec'']}}", - ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20240804_0625", + ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20240808_0625", "\", \"args\": [\"--target_column=", "{{$.inputs.parameters[''target_column'']}}", "\", \"--weight_column=", "{{$.inputs.parameters[''weight_column'']}}", "\", \"--model_type=", "{{$.inputs.parameters[''prediction_type'']}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--prediction_docker_uri_artifact_path=", "{{$.outputs.parameters[''prediction_docker_uri_output''].output_file}}", "\", \"--baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer.py index ca4b8e25bb6..262f58b74dc 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer.py @@ -161,7 +161,7 @@ def wide_and_deep_trainer( ', "disk_spec": ', training_disk_spec, ', "container_spec": {"image_uri":"', - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20240808_0625', '", "args": ["--target_column=', target_column, '", "--weight_column=', @@ -169,7 +169,7 @@ def wide_and_deep_trainer( '", "--model_type=', prediction_type, '", "--prediction_docker_uri=', - 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625', '", "--baseline_path=', instance_baseline.uri, '", "--metadata_path=', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer_pipeline.yaml index a69bc88d161..245933c307e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer_pipeline.yaml @@ -2674,7 +2674,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -2689,7 +2689,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -2714,7 +2714,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-transform-engine: container: args: @@ -2799,8 +2799,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -2817,7 +2817,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -2847,7 +2847,7 @@ deploymentSpec: \n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'model_display_name',\n ],\n )(\n model_display_name,\n )\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-model-batch-predict: container: args: @@ -3040,7 +3040,7 @@ deploymentSpec: \ 'training_disk_spec',\n 'eval_machine_spec',\n 'eval_replica_count',\n\ \ ],\n )(\n training_machine_spec,\n training_disk_spec,\n\ \ eval_machine_spec,\n eval_replica_count,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-set-optional-inputs: container: args: @@ -3088,7 +3088,7 @@ deploymentSpec: \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ \ ],\n )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -3134,7 +3134,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-training-configurator-and-validator: container: args: @@ -3179,7 +3179,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 exec-wide-and-deep-trainer: container: args: @@ -3197,11 +3197,11 @@ deploymentSpec: "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"", "1", "\", \"machine_spec\": ", "{{$.inputs.parameters[''training_machine_spec'']}}", ", \"disk_spec\": ", "{{$.inputs.parameters[''training_disk_spec'']}}", - ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20240804_0625", + ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20240808_0625", "\", \"args\": [\"--target_column=", "{{$.inputs.parameters[''target_column'']}}", "\", \"--weight_column=", "{{$.inputs.parameters[''weight_column'']}}", "\", \"--model_type=", "{{$.inputs.parameters[''prediction_type'']}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job_pipeline.yaml index 029ec487b56..ce1cd67845e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job_pipeline.yaml @@ -2620,7 +2620,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -2651,7 +2651,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-transform-engine: container: args: @@ -2736,8 +2736,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -2754,7 +2754,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -2818,7 +2818,7 @@ deploymentSpec: \ return re.sub(r'^/gcs/', r'gs://', path)\n\n master_worker_pool_spec\ \ = {\n 'replica_count': 1,\n 'machine_spec': {\n 'machine_type':\ \ machine_type,\n },\n 'container_spec': {\n 'image_uri':\ - \ 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/xgboost-training:20240804_0625',\n\ + \ 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/xgboost-training:20240808_0625',\n\ \ 'args': [\n f'--job_dir={get_gcs_path(job_dir)}',\n\ \ f'--instance_schema_path={get_gcs_path(instance_schema_uri)}',\n\ \ f'--prediction_schema_path={get_gcs_path(prediction_schema_uri)}',\n\ @@ -2831,7 +2831,7 @@ deploymentSpec: \ f'--baseline_path={get_gcs_path(instance_baseline)}',\n \ \ f'--eval_metric={eval_metric}',\n f'--disable_default_eval_metric={disable_default_eval_metric}',\n\ \ f'--seed={seed}',\n f'--seed_per_iteration={seed_per_iteration}',\n\ - \ '--prediction_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/xgboost-prediction-server:20240804_0625',\n\ + \ '--prediction_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/xgboost-prediction-server:20240808_0625',\n\ \ ],\n },\n }\n\n # Add optional arguments if set\n if\ \ weight_column:\n master_worker_pool_spec['container_spec']['args'].append(\n\ \ f'--weight_column={weight_column}'\n )\n\n # Add accelerator_type\ @@ -2850,7 +2850,7 @@ deploymentSpec: \ ],\n )(\n worker_pool_specs_lst,\n get_gcs_path(instance_schema_uri),\n\ \ get_gcs_path(prediction_schema_uri),\n get_gcs_path(trials),\n\ \ get_gcs_path(prediction_docker_uri_output),\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-best-hyperparameter-tuning-job-trial: container: args: @@ -2915,7 +2915,7 @@ deploymentSpec: \ = {\n 'instanceSchemaUri': instance_schema_uri,\n 'predictionSchemaUri':\ \ prediction_schema_uri,\n }\n unmanaged_container_model.uri = os.path.join(\n\ \ trials_dir, 'trial_{}'.format(best_trial['id']), 'model'\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-model-display-name: container: args: @@ -2942,7 +2942,7 @@ deploymentSpec: \n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'model_display_name',\n ],\n )(\n model_display_name,\n )\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-prediction-type-for-xgboost: container: args: @@ -2971,7 +2971,7 @@ deploymentSpec: \ Must be one of'\n ' [reg:squarederror, reg:squaredlogerror, reg:logistic,\ \ reg:gamma,'\n ' reg:tweedie, reg:pseudohubererror, binary:logistic,'\n\ \ ' multi:softprob].'\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-xgboost-study-spec-parameters: container: args: @@ -3546,7 +3546,7 @@ deploymentSpec: \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ \ ],\n )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -3592,7 +3592,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-training-configurator-and-validator: container: args: @@ -3637,7 +3637,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 exec-xgboost-hyperparameter-tuning-job: container: args: diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer_pipeline.yaml index 99b93f5def1..247c45853d3 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer_pipeline.yaml @@ -2844,7 +2844,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -2875,7 +2875,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-transform-engine: container: args: @@ -2960,8 +2960,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -2978,7 +2978,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 30.0 @@ -3098,10 +3098,10 @@ deploymentSpec: \ worker pool specs.\n \"\"\"\n import copy\n import collections\n import\ \ os\n import re\n\n def get_gcs_path(path):\n return re.sub(r'/gcs/',\ \ 'gs://', path)\n\n formatted_job_dir = get_gcs_path(job_dir)\n prediction_docker_uri\ - \ = (\n 'us-docker.pkg.dev/vertex-ai/automl-tabular/xgboost-prediction-server:20240804_0625'\n\ + \ = (\n 'us-docker.pkg.dev/vertex-ai/automl-tabular/xgboost-prediction-server:20240808_0625'\n\ \ )\n master_worker_pool_spec = {\n 'replica_count': 1,\n 'machine_spec':\ \ {\n 'machine_type': machine_type,\n },\n 'container_spec':\ - \ {\n 'image_uri': 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/xgboost-training:20240804_0625',\n\ + \ {\n 'image_uri': 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/xgboost-training:20240808_0625',\n\ \ 'args': [\n f'--job_dir={formatted_job_dir}',\n\ \ f'--target_column={target_column}',\n f'--objective={objective}',\n\ \ f'--training_data_path={get_gcs_path(materialized_train_split)}',\n\ @@ -3159,7 +3159,7 @@ deploymentSpec: \ 'predictionSchemaUri': os.path.join(model_dir, 'prediction_schema.yaml'),\n\ \ }\n unmanaged_container_model.uri = model_dir\n\n return collections.namedtuple('Outputs',\ \ ['worker_pool_specs'])(\n worker_pool_specs_lst\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-model-display-name: container: args: @@ -3186,7 +3186,7 @@ deploymentSpec: \n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'model_display_name',\n ],\n )(\n model_display_name,\n )\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-prediction-type-for-xgboost: container: args: @@ -3215,7 +3215,7 @@ deploymentSpec: \ Must be one of'\n ' [reg:squarederror, reg:squaredlogerror, reg:logistic,\ \ reg:gamma,'\n ' reg:tweedie, reg:pseudohubererror, binary:logistic,'\n\ \ ' multi:softprob].'\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-model-batch-predict: container: args: @@ -3407,7 +3407,7 @@ deploymentSpec: \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ \ ],\n )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-split-materialized-data: container: args: @@ -3453,7 +3453,7 @@ deploymentSpec: \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ \ 'w') as f:\n f.write(file_patterns[2])\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 exec-training-configurator-and-validator: container: args: @@ -3498,7 +3498,7 @@ deploymentSpec: ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 exec-xgboost-trainer: container: args: diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_predict_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_predict_pipeline.yaml index 4f5e5aea686..abd52bb4e7f 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_predict_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_predict_pipeline.yaml @@ -658,7 +658,7 @@ deploymentSpec: \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ \ ref.project, ref.dataset_id)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-create-dataset-2: container: args: @@ -693,7 +693,7 @@ deploymentSpec: \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ \ ref.project, ref.dataset_id)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-delete-dataset-with-prefix: container: args: @@ -727,7 +727,7 @@ deploymentSpec: \ if dataset.dataset_id.startswith(dataset_prefix):\n client.delete_dataset(\n\ \ dataset=dataset.dataset_id,\n delete_contents=delete_contents)\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-query-job: container: args: @@ -788,7 +788,7 @@ deploymentSpec: \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ \ return config\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-first-valid: container: args: @@ -812,7 +812,7 @@ deploymentSpec: \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ \n for value in json.loads(values):\n if value:\n return value\n\ \ raise ValueError('No valid values.')\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-model-metadata: container: args: @@ -851,7 +851,7 @@ deploymentSpec: \ 'forecast_horizon',\n ],\n )(\n options.time_series_timestamp_column,\n\ \ options.time_series_id_column,\n options.time_series_data_column,\n\ \ options.horizon,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-table-location: container: args: @@ -887,7 +887,7 @@ deploymentSpec: \ if table.startswith('bq://'):\n table = table[len('bq://'):]\n elif\ \ table.startswith('bigquery://'):\n table = table[len('bigquery://'):]\n\ \ return client.get_table(table).location\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-load-table-from-uri: container: args: @@ -928,7 +928,7 @@ deploymentSpec: \ source_format=source_format)\n client.load_table_from_uri(\n source_uris=csv_list,\n\ \ destination=destination,\n project=project,\n location=location,\n\ \ job_config=job_config).result()\n return destination\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-maybe-replace-with-default: container: args: @@ -950,7 +950,7 @@ deploymentSpec: \ *\n\ndef maybe_replace_with_default(value: str, default: str = '') ->\ \ str:\n \"\"\"Replaces string with another value if it is a dash.\"\"\"\ \n return default if not value else value\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-validate-inputs: container: args: @@ -1046,7 +1046,7 @@ deploymentSpec: \ raise ValueError(\n 'Granularity unit should be one of the\ \ following: '\n f'{valid_data_granularity_units}, got: {data_granularity_unit}.')\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 pipelineInfo: description: Forecasts using a BQML ARIMA_PLUS model. name: automl-tabular-bqml-arima-prediction diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_train_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_train_pipeline.yaml index 552ed4f4046..bdb07f0ef34 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_train_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_train_pipeline.yaml @@ -3399,7 +3399,7 @@ deploymentSpec: \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ \ ref.project, ref.dataset_id)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-create-dataset-2: container: args: @@ -3434,7 +3434,7 @@ deploymentSpec: \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ \ ref.project, ref.dataset_id)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-create-model-job: container: args: @@ -3494,7 +3494,7 @@ deploymentSpec: \ if dataset.dataset_id.startswith(dataset_prefix):\n client.delete_dataset(\n\ \ dataset=dataset.dataset_id,\n delete_contents=delete_contents)\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-list-rows: container: args: @@ -3532,7 +3532,7 @@ deploymentSpec: \ metadata['datasetId'], metadata['tableId']]))\n result = []\n for row\ \ in rows:\n result.append({col: str(value) for col, value in dict(row).items()})\n\ \ return result\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-list-rows-2: container: args: @@ -3570,7 +3570,7 @@ deploymentSpec: \ metadata['datasetId'], metadata['tableId']]))\n result = []\n for row\ \ in rows:\n result.append({col: str(value) for col, value in dict(row).items()})\n\ \ return result\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-query-job: container: args: @@ -3739,7 +3739,7 @@ deploymentSpec: \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ \ return config\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-build-job-configuration-query-2: container: args: @@ -3773,7 +3773,7 @@ deploymentSpec: \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ \ return config\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-build-job-configuration-query-3: container: args: @@ -3807,7 +3807,7 @@ deploymentSpec: \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ \ return config\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-build-job-configuration-query-4: container: args: @@ -3841,7 +3841,7 @@ deploymentSpec: \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ \ return config\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-build-job-configuration-query-5: container: args: @@ -3875,7 +3875,7 @@ deploymentSpec: \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ \ return config\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-build-job-configuration-query-6: container: args: @@ -3909,7 +3909,7 @@ deploymentSpec: \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ \ return config\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-build-serialized-query-parameters: container: args: @@ -3980,7 +3980,7 @@ deploymentSpec: \ 'name': 'start_time',\n 'parameterType': {\n 'type':\ \ 'TIMESTAMP'\n },\n 'parameterValue': {\n 'value': start_time\n\ \ },\n })\n return query_parameters\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-build-serialized-query-parameters-2: container: args: @@ -4051,7 +4051,7 @@ deploymentSpec: \ 'name': 'start_time',\n 'parameterType': {\n 'type':\ \ 'TIMESTAMP'\n },\n 'parameterValue': {\n 'value': start_time\n\ \ },\n })\n return query_parameters\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-build-serialized-query-parameters-3: container: args: @@ -4122,7 +4122,7 @@ deploymentSpec: \ 'name': 'start_time',\n 'parameterType': {\n 'type':\ \ 'TIMESTAMP'\n },\n 'parameterValue': {\n 'value': start_time\n\ \ },\n })\n return query_parameters\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-cond: container: args: @@ -4144,7 +4144,7 @@ deploymentSpec: \ *\n\ndef cond(predicate: bool, true_str: str, false_str: str) -> str:\n\ \ \"\"\"Returns true_str if predicate is true, else false_str.\"\"\"\n\ \ return true_str if predicate else false_str\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-create-metrics-artifact: container: args: @@ -4170,7 +4170,7 @@ deploymentSpec: \ 'MAPE': 'meanAbsolutePercentageError',\n }\n metrics = {metric_name_map[k]:\ \ v for k, v in dict(metrics_rows[0]).items()}\n evaluation_metrics.metadata\ \ = metrics\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-transform-engine: container: args: @@ -4255,8 +4255,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -4273,7 +4273,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 exec-get-fte-suffix: container: args: @@ -4301,7 +4301,7 @@ deploymentSpec: \ table.table_id.startswith(fte_table):\n return table.table_id[len(fte_table)\ \ + 1:]\n raise ValueError(\n f'No FTE output tables found in {bigquery_staging_full_dataset_id}.')\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-table-location: container: args: @@ -4337,7 +4337,7 @@ deploymentSpec: \ if table.startswith('bq://'):\n table = table[len('bq://'):]\n elif\ \ table.startswith('bigquery://'):\n table = table[len('bigquery://'):]\n\ \ return client.get_table(table).location\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-value: container: args: @@ -4358,7 +4358,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef get_value(d: Dict[str, str], key: str) -> str:\n return d[key]\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-window-query-priority: container: args: @@ -4382,7 +4382,7 @@ deploymentSpec: \ depending on the window number.\"\"\"\n if int(window['window_number'])\ \ <= max_interactive:\n return 'INTERACTIVE'\n else:\n return 'BATCH'\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-maybe-replace-with-default: container: args: @@ -4404,7 +4404,7 @@ deploymentSpec: \ *\n\ndef maybe_replace_with_default(value: str, default: str = '') ->\ \ str:\n \"\"\"Replaces string with another value if it is a dash.\"\"\"\ \n return default if not value else value\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-query-with-retry: container: args: @@ -4458,7 +4458,7 @@ deploymentSpec: \ 'Query failed with %s. Retrying after %d seconds.', e, wait_time)\n\ \ time.sleep(wait_time)\n retry_count += 1\n return destination_uri\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-query-with-retry-2: container: args: @@ -4512,7 +4512,7 @@ deploymentSpec: \ 'Query failed with %s. Retrying after %d seconds.', e, wait_time)\n\ \ time.sleep(wait_time)\n retry_count += 1\n return destination_uri\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-query-with-retry-3: container: args: @@ -4566,7 +4566,7 @@ deploymentSpec: \ 'Query failed with %s. Retrying after %d seconds.', e, wait_time)\n\ \ time.sleep(wait_time)\n retry_count += 1\n return destination_uri\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri: container: args: @@ -4596,7 +4596,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri-2: container: args: @@ -4626,7 +4626,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-validate-inputs: container: args: @@ -4722,7 +4722,7 @@ deploymentSpec: \ raise ValueError(\n 'Granularity unit should be one of the\ \ following: '\n f'{valid_data_granularity_units}, got: {data_granularity_unit}.')\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-wrapped-in-list: container: args: @@ -4743,7 +4743,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef wrapped_in_list(value: str) -> List[str]:\n \"\"\"Wraps a string\ \ in a list.\"\"\"\n return [value]\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 pipelineInfo: description: Trains a BQML ARIMA_PLUS model. name: automl-tabular-bqml-arima-train diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_predict_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_predict_pipeline.yaml index 5f16149ee9e..16950ba3d69 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_predict_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_predict_pipeline.yaml @@ -1461,7 +1461,7 @@ deploymentSpec: \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ \ ref.project, ref.dataset_id)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-delete-dataset-with-prefix: container: args: @@ -1495,7 +1495,7 @@ deploymentSpec: \ if dataset.dataset_id.startswith(dataset_prefix):\n client.delete_dataset(\n\ \ dataset=dataset.dataset_id,\n delete_contents=delete_contents)\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-query-job: container: args: @@ -1583,7 +1583,7 @@ deploymentSpec: \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ \ return config\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-build-job-configuration-query-2: container: args: @@ -1617,7 +1617,7 @@ deploymentSpec: \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ \ return config\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-first-valid: container: args: @@ -1641,7 +1641,7 @@ deploymentSpec: \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ \n for value in json.loads(values):\n if value:\n return value\n\ \ raise ValueError('No valid values.')\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-table-location: container: args: @@ -1677,7 +1677,7 @@ deploymentSpec: \ if table.startswith('bq://'):\n table = table[len('bq://'):]\n elif\ \ table.startswith('bigquery://'):\n table = table[len('bigquery://'):]\n\ \ return client.get_table(table).location\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-table-location-2: container: args: @@ -1713,7 +1713,7 @@ deploymentSpec: \ if table.startswith('bq://'):\n table = table[len('bq://'):]\n elif\ \ table.startswith('bigquery://'):\n table = table[len('bigquery://'):]\n\ \ return client.get_table(table).location\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-load-table-from-uri: container: args: @@ -1754,7 +1754,7 @@ deploymentSpec: \ source_format=source_format)\n client.load_table_from_uri(\n source_uris=csv_list,\n\ \ destination=destination,\n project=project,\n location=location,\n\ \ job_config=job_config).result()\n return destination\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-make-vertex-model-artifact: container: args: @@ -1778,7 +1778,7 @@ deploymentSpec: Creates a google.VertexModel artifact.\"\"\"\n vertex_model.metadata =\ \ {'resourceName': model_resource_name}\n vertex_model.uri = (f'https://{location}-aiplatform.googleapis.com'\n\ \ f'/v1/{model_resource_name}')\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-maybe-replace-with-default: container: args: @@ -1800,7 +1800,7 @@ deploymentSpec: \ *\n\ndef maybe_replace_with_default(value: str, default: str = '') ->\ \ str:\n \"\"\"Replaces string with another value if it is a dash.\"\"\"\ \n return default if not value else value\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-model-batch-predict: container: args: @@ -1879,7 +1879,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-table-to-uri-2: container: args: @@ -1909,7 +1909,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-validate-inputs: container: args: @@ -2005,7 +2005,7 @@ deploymentSpec: \ raise ValueError(\n 'Granularity unit should be one of the\ \ following: '\n f'{valid_data_granularity_units}, got: {data_granularity_unit}.')\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 pipelineInfo: description: Creates a batch prediction using a Prophet model. name: prophet-predict diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer.py index e58d8846487..cf7c55d0b90 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer.py @@ -108,17 +108,17 @@ def prophet_trainer( '"machine_spec": {"machine_type": "n1-standard-4"}, ', ( '"container_spec":' - ' {"image_uri":"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", ' + ' {"image_uri":"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", ' ), '"args": ["prophet_trainer", "', ( f'--job_name=dataflow-{dsl.PIPELINE_JOB_NAME_PLACEHOLDER}", "' ), ( - '--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625", "' + '--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625", "' ), ( - '--prediction_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/fte-prediction-server:20240804_0625", "' + '--prediction_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/fte-prediction-server:20240808_0625", "' ), '--artifacts_dir=', root_dir, diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer_pipeline.yaml index e805d39b991..d4ab4b922e8 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer_pipeline.yaml @@ -2021,7 +2021,7 @@ deploymentSpec: \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ \ ref.project, ref.dataset_id)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-delete-dataset-with-prefix: container: args: @@ -2055,7 +2055,7 @@ deploymentSpec: \ if dataset.dataset_id.startswith(dataset_prefix):\n client.delete_dataset(\n\ \ dataset=dataset.dataset_id,\n delete_contents=delete_contents)\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bigquery-query-job: container: args: @@ -2116,7 +2116,7 @@ deploymentSpec: \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ \ return config\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-transform-engine: container: args: @@ -2201,8 +2201,8 @@ deploymentSpec: "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' - - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625 - - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' @@ -2219,7 +2219,7 @@ deploymentSpec: - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625 exec-get-fte-suffix: container: args: @@ -2247,7 +2247,7 @@ deploymentSpec: \ table.table_id.startswith(fte_table):\n return table.table_id[len(fte_table)\ \ + 1:]\n raise ValueError(\n f'No FTE output tables found in {bigquery_staging_full_dataset_id}.')\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-get-table-location: container: args: @@ -2283,7 +2283,7 @@ deploymentSpec: \ if table.startswith('bq://'):\n table = table[len('bq://'):]\n elif\ \ table.startswith('bigquery://'):\n table = table[len('bigquery://'):]\n\ \ return client.get_table(table).location\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-model-evaluation-regression: container: args: @@ -2394,10 +2394,10 @@ deploymentSpec: ", "\"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, ", "\"job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"1\", ", "\"machine_spec\": {\"machine_type\": \"n1-standard-4\"}, ", "\"container_spec\": - {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625\", + {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625\", ", "\"args\": [\"prophet_trainer\", \"", "--job_name=dataflow-{{$.pipeline_job_name}}\", - \"", "--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625\", - \"", "--prediction_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/fte-prediction-server:20240804_0625\", + \"", "--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625\", + \"", "--prediction_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/fte-prediction-server:20240808_0625\", \"", "--artifacts_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/model/\", \"", "--evaluated_examples_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/eval/\", \"", "--region=", "{{$.inputs.parameters[''location'']}}", @@ -2458,7 +2458,7 @@ deploymentSpec: \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-validate-inputs: container: args: @@ -2554,7 +2554,7 @@ deploymentSpec: \ raise ValueError(\n 'Granularity unit should be one of the\ \ following: '\n f'{valid_data_granularity_units}, got: {data_granularity_unit}.')\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-wrapped-in-list: container: args: @@ -2575,7 +2575,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef wrapped_in_list(value: str) -> List[str]:\n \"\"\"Wraps a string\ \ in a list.\"\"\"\n return [value]\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 pipelineInfo: description: Trains one Prophet model per time series. name: prophet-train diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/automl_tabular_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/automl_tabular_pipeline.yaml index f1882393976..cda7f0bc85e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/automl_tabular_pipeline.yaml +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/automl_tabular_pipeline.yaml @@ -8420,9 +8420,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", @@ -8463,9 +8463,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", @@ -8506,7 +8506,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", @@ -8518,7 +8518,7 @@ deploymentSpec: "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", @@ -8547,7 +8547,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", @@ -8559,7 +8559,7 @@ deploymentSpec: "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", @@ -8588,7 +8588,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", @@ -8600,7 +8600,7 @@ deploymentSpec: "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", - \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625", "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", @@ -8629,7 +8629,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' @@ -8644,7 +8644,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -8653,7 +8653,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -8662,7 +8662,7 @@ deploymentSpec: args: - --executor_input - '{{$}}' - image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625 resources: cpuLimit: 8.0 memoryLimit: 52.0 @@ -8682,9 +8682,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", @@ -8729,9 +8729,9 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", - "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", @@ -8776,7 +8776,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"transform\", \"--is_mp=true\", \"--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}", "\", \"--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", @@ -8797,7 +8797,7 @@ deploymentSpec: \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", - "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625", "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", @@ -8828,7 +8828,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"transform\", \"--is_mp=true\", \"--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}", "\", \"--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", @@ -8849,7 +8849,7 @@ deploymentSpec: \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", - "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625", "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", @@ -8885,7 +8885,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bool-identity-2: container: args: @@ -8907,7 +8907,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-bool-identity-3: container: args: @@ -8929,7 +8929,7 @@ deploymentSpec: \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-calculate-training-parameters: container: args: @@ -9021,7 +9021,7 @@ deploymentSpec: \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ \ reduce_search_space_mode,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-calculate-training-parameters-2: container: args: @@ -9113,7 +9113,7 @@ deploymentSpec: \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ \ reduce_search_space_mode,\n )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-feature-attribution: container: args: @@ -9299,7 +9299,7 @@ deploymentSpec: \n return collections.namedtuple(\n 'Outputs',\n [\n \ \ 'model_display_name',\n ],\n )(\n model_display_name,\n )\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-importer: importer: artifactUri: @@ -9333,7 +9333,7 @@ deploymentSpec: \ 'r') as f:\n split_0_content = f.read()\n with open(split_1, 'r')\ \ as f:\n split_1_content = f.read()\n with open(splits, 'w') as f:\n\ \ f.write(','.join([split_0_content, split_1_content]))\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-model-batch-explanation: container: args: @@ -9375,7 +9375,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-explanation-2: container: args: @@ -9417,7 +9417,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-explanation-3: container: args: @@ -9459,7 +9459,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-batch-predict: container: args: @@ -10072,7 +10072,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-upload-2: container: args: @@ -10101,7 +10101,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-model-upload-3: container: args: @@ -10130,7 +10130,7 @@ deploymentSpec: - -u - -m - launcher - image: gcr.io/ml-pipeline/automl-tables-private:1.0.17 + image: gcr.io/ml-pipeline/automl-tables-private:1.0.18 exec-read-input-uri: container: args: @@ -10158,7 +10158,7 @@ deploymentSpec: \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ \ with open(split_uri, 'r') as f:\n data_source = json.loads(f.read())\n\ \ return data_source['tf_record_data_source']['file_patterns']\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-read-input-uri-2: container: args: @@ -10186,7 +10186,7 @@ deploymentSpec: \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ \ with open(split_uri, 'r') as f:\n data_source = json.loads(f.read())\n\ \ return data_source['tf_record_data_source']['file_patterns']\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-set-optional-inputs: container: args: @@ -10234,7 +10234,7 @@ deploymentSpec: \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ \ ],\n )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ \ )\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-string-not-empty: container: args: @@ -10258,7 +10258,7 @@ deploymentSpec: \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ \ \"\"\"\n return 'true' if value else 'false'\n\n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-tabular-stats-and-example-gen: container: args: @@ -10275,7 +10275,7 @@ deploymentSpec: \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", - "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\", \"args\": [\"stats_generator\",", "\"--train_spec={\\\"prediction_type\\\": \\\"", "{{$.inputs.parameters[''prediction_type'']}}", "\\\", \\\"target_column\\\": \\\"", "{{$.inputs.parameters[''target_column_name'']}}", "\\\", \\\"optimization_objective\\\": @@ -10308,7 +10308,7 @@ deploymentSpec: \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", - "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625", "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", @@ -10363,7 +10363,7 @@ deploymentSpec: \ f'{directory}/prediction.results-*',\n ],\n 'coder':\ \ 'PROTO_VALUE',\n },\n }\n with open(result, 'w') as f:\n f.write(json.dumps(data_source))\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 exec-write-bp-result-path-2: container: args: @@ -10393,7 +10393,7 @@ deploymentSpec: \ f'{directory}/prediction.results-*',\n ],\n 'coder':\ \ 'PROTO_VALUE',\n },\n }\n with open(result, 'w') as f:\n f.write(json.dumps(data_source))\n\ \n" - image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625 + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625 pipelineInfo: description: 'Complete AutoML Tables pipeline. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/cv_trainer.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/cv_trainer.py index 3be82605357..57ec51e7645 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/cv_trainer.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/cv_trainer.py @@ -99,11 +99,11 @@ def automl_tabular_cv_trainer( ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' ' "container_spec": {"image_uri":"' ), - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625', '", "args": ["l2l_cv_tuner", "--transform_output_path=', transform_output.uri, '", "--training_docker_uri=', - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625', ( f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' ' "--training_base_dir=' diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/ensemble.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/ensemble.py index 090d218222b..85293dca107 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/ensemble.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/ensemble.py @@ -106,7 +106,7 @@ def automl_tabular_ensemble( ' 1, "machine_spec": {"machine_type": "n1-highmem-8"},' ' "container_spec": {"image_uri":"' ), - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625', '", "args": ["ensemble", "--transform_output_path=', transform_output.uri, '", "--model_output_path=', @@ -137,7 +137,7 @@ def automl_tabular_ensemble( '", "--warmup_data=', warmup_data.uri, '", "--prediction_docker_uri=', - 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625', '", "--model_path=', model.uri, '", "--custom_model_path=', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/finalizer.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/finalizer.py index 68d9009b759..86f51dbffcd 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/finalizer.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/finalizer.py @@ -72,7 +72,7 @@ def automl_tabular_finalizer( ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' ' "container_spec": {"image_uri":"' ), - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625', '", "args": ["cancel_l2l_tuner", "--error_file_path=', root_dir, ( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/infra_validator.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/infra_validator.py index e78c5b76970..705b6dffd63 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/infra_validator.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/infra_validator.py @@ -32,7 +32,7 @@ def automl_tabular_infra_validator( # fmt: on return dsl.ContainerSpec( - image='us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625', + image='us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625', command=[], args=['--executor_input', '{{$}}'], ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/split_materialized_data.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/split_materialized_data.py index c68f7da4a54..656a5440574 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/split_materialized_data.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/split_materialized_data.py @@ -52,7 +52,7 @@ def split_materialized_data( # fmt: on return dsl.ContainerSpec( - image='us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625', + image='us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625', command=[ 'sh', '-ec', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stage_1_tuner.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stage_1_tuner.py index b6b1777029c..36508453505 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stage_1_tuner.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stage_1_tuner.py @@ -109,11 +109,11 @@ def automl_tabular_stage_1_tuner( ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' ' "container_spec": {"image_uri":"' ), - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625', '", "args": ["l2l_stage_1_tuner", "--transform_output_path=', transform_output.uri, '", "--training_docker_uri=', - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625', '", "--feature_selection_result_path=', feature_ranking.uri, '", "--disable_early_stopping=', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stats_and_example_gen.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stats_and_example_gen.py index 5c58a4ab761..ea28e7d1b6e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stats_and_example_gen.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stats_and_example_gen.py @@ -136,7 +136,7 @@ def tabular_stats_and_example_gen( ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' ' "container_spec": {"image_uri":"' ), - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625', '", "args": ["stats_generator",', '"--train_spec={\\"prediction_type\\": \\"', prediction_type, @@ -215,7 +215,7 @@ def tabular_stats_and_example_gen( ), dataflow_max_num_workers, '", "--dataflow_worker_container_image=', - 'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625', '", "--dataflow_machine_type=', dataflow_machine_type, '", "--dataflow_disk_size_gb=', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/training_configurator_and_validator.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/training_configurator_and_validator.py index ba9b4c647d3..79d5edbd0b7 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/training_configurator_and_validator.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/training_configurator_and_validator.py @@ -95,7 +95,7 @@ def training_configurator_and_validator( # fmt: on return dsl.ContainerSpec( - image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625', + image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625', command=[], args=[ 'training_configurator_and_validator', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/transform.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/transform.py index 7a4d23c61d9..a97c5fce619 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/transform.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/transform.py @@ -108,7 +108,7 @@ def automl_tabular_transform( ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' ' "container_spec": {"image_uri":"' ), - 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625', ( '", "args": ["transform", "--is_mp=true",' ' "--transform_output_artifact_path=' @@ -167,7 +167,7 @@ def automl_tabular_transform( '", "--dataflow_machine_type=', dataflow_machine_type, '", "--dataflow_worker_container_image=', - 'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625', '", "--dataflow_disk_size_gb=', dataflow_disk_size_gb, '", "--dataflow_subnetwork_fully_qualified=', From b632f1dff0fffb55a2d3d7e7baca419597f8ea66 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 12 Aug 2024 15:11:08 -0700 Subject: [PATCH 08/54] feat(components): Support dynamic values for boot_disk_type, boot_disk_size in preview.custom_job.utils.create_custom_training_job_from_component Signed-off-by: Googler PiperOrigin-RevId: 662242688 Signed-off-by: KevinGrantLee --- components/google-cloud/RELEASE.md | 1 + .../preview/custom_job/remote_runner.py | 39 ++++++++++++------- .../preview/custom_job/utils.py | 38 +++++++++++------- 3 files changed, 49 insertions(+), 29 deletions(-) diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md index af292c0ab47..3dd1e6cd211 100644 --- a/components/google-cloud/RELEASE.md +++ b/components/google-cloud/RELEASE.md @@ -1,5 +1,6 @@ ## Upcoming release * Fix to model batch explanation component for Structured Data pipelines; image bump. +* Add dynamic support for boot_disk_type, boot_disk_size in `preview.custom_job.utils.create_custom_training_job_from_component`. ## Release 2.16.0 * Updated the Starry Net pipeline's template gallery description, and added dataprep_nan_threshold and dataprep_zero_threshold args to the Starry Net pipeline. diff --git a/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/remote_runner.py index e56548c002f..bd490895251 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/remote_runner.py @@ -32,23 +32,32 @@ def insert_system_labels_into_payload(payload): return json.dumps(job_spec) -def cast_accelerator_count_to_int(payload): - """Casts accelerator_count from string to an int.""" +def is_json(test_string: str) -> bool: + try: + json.loads(test_string) + except ValueError: + return False + return True + + +def parse_nested_json_strings(payload): + """Parse nested json strings in the payload.""" job_spec = json.loads(payload) - # TODO(b/353577594): accelerator_count placeholder is not resolved to int. - # Need to typecast to int to avoid type mismatch error. Can remove when fix - # placeholder resolution. - if ( - 'accelerator_count' - in job_spec['job_spec']['worker_pool_specs'][0]['machine_spec'] + # TODO(b/353577594): Nested placeholder fields inside worker_pool_specs are + # not parsed correctly in backend. Can remove when fix backend logic. + worker_pool_spec = job_spec['job_spec']['worker_pool_specs'][0] + if is_json( + worker_pool_spec.get('machine_spec', {}).get('accelerator_count', '') + ): + worker_pool_spec['machine_spec']['accelerator_count'] = json.loads( + worker_pool_spec['machine_spec']['accelerator_count'] + ) + if is_json( + worker_pool_spec.get('disk_spec', {}).get('boot_disk_size_gb', '') ): - job_spec['job_spec']['worker_pool_specs'][0]['machine_spec'][ - 'accelerator_count' - ] = int( - job_spec['job_spec']['worker_pool_specs'][0]['machine_spec'][ - 'accelerator_count' - ] + worker_pool_spec['disk_spec']['boot_disk_size_gb'] = json.loads( + worker_pool_spec['disk_spec']['boot_disk_size_gb'] ) return json.dumps(job_spec) @@ -107,7 +116,7 @@ def create_custom_job( # Create custom job if it does not exist job_name = remote_runner.check_if_job_exists() if job_name is None: - payload = cast_accelerator_count_to_int(payload) + payload = parse_nested_json_strings(payload) job_name = remote_runner.create_job( create_custom_job_with_client, insert_system_labels_into_payload(payload), diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/utils.py b/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/utils.py index ed35a559f55..ec5bdaa7826 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/utils.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/utils.py @@ -84,8 +84,8 @@ def create_custom_training_job_from_component( machine_type: The type of the machine to run the CustomJob. The default value is "n1-standard-4". See [more information](https://cloud.google.com/vertex-ai/docs/training/configure-compute#machine-types). accelerator_type: The type of accelerator(s) that may be attached to the machine per `accelerator_count`. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec#acceleratortype). accelerator_count: The number of accelerators to attach to the machine. Defaults to 1 if `accelerator_type` is set statically. - boot_disk_type: Type of the boot disk (default is "pd-ssd"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive). boot_disk_type is set as a static value and cannot be changed as a pipeline parameter. - boot_disk_size_gb: Size in GB of the boot disk (default is 100GB). `boot_disk_size_gb` is set as a static value and cannot be changed as a pipeline parameter. + boot_disk_type: Type of the boot disk (default is "pd-ssd"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive). + boot_disk_size_gb: Size in GB of the boot disk (default is 100GB). timeout: The maximum job running time. The default is 7 days. A duration in seconds with up to nine fractional digits, terminated by 's', for example: "3.5s". restart_job_on_worker_restart: Restarts the entire CustomJob if a worker gets restarted. This feature can be used by distributed training jobs that are not resilient to workers leaving and joining a job. service_account: Sets the default service account for workload run-as account. The [service account](https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) running the pipeline submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code [Service Agent](https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project. @@ -94,11 +94,11 @@ def create_custom_training_job_from_component( tensorboard: The name of a Vertex AI TensorBoard resource to which this CustomJob will upload TensorBoard logs. enable_web_access: Whether you want Vertex AI to enable [interactive shell access](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell) to training containers. If `True`, you can access interactive shells at the URIs given by [CustomJob.web_access_uris][]. reserved_ip_ranges: A list of names for the reserved IP ranges under the VPC network that can be used for this job. If set, we will deploy the job within the provided IP ranges. Otherwise, the job will be deployed to any IP ranges under the provided VPC network. - nfs_mounts: A list of [NfsMount](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/CustomJobSpec#NfsMount) resource specs in Json dict format. For more details about mounting NFS for CustomJob, see [Mount an NFS share for custom training](https://cloud.google.com/vertex-ai/docs/training/train-nfs-share). + nfs_mounts: A list of [NfsMount](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/CustomJobSpec#NfsMount) resource specs in Json dict format. For more details about mounting NFS for CustomJob, see [Mount an NFS share for custom training](https://cloud.google.com/vertex-ai/docs/training/train-nfs-share). `nfs_mounts` is set as a static value and cannot be changed as a pipeline parameter. base_output_directory: The Cloud Storage location to store the output of this CustomJob or HyperparameterTuningJob. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/GcsDestination). labels: The labels with user-defined metadata to organize the CustomJob. See [more information](https://goo.gl/xmQnxf). persistent_resource_id: The ID of the PersistentResource in the same Project and Location which to run. The default value is a placeholder that will be resolved to the PipelineJob [RuntimeConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.pipelineJobs#PipelineJob.RuntimeConfig)'s persistent resource id at runtime. However, if the PipelineJob doesn't set Persistent Resource as the job level runtime, the placedholder will be resolved to an empty string and the custom job will be run on demand. If the value is set explicitly, the custom job will runs in the specified persistent resource, in this case, please note the network and CMEK configs on the job should be consistent with those on the PersistentResource, otherwise, the job will be rejected. (This is a Preview feature not yet recommended for production workloads.) - env: Environment variables to be passed to the container. Takes the form `[{'name': '...', 'value': '...'}]`. Maximum limit is 100. + env: Environment variables to be passed to the container. Takes the form `[{'name': '...', 'value': '...'}]`. Maximum limit is 100. `env` is set as a static value and cannot be changed as a pipeline parameter. Returns: A KFP component with CustomJob specification applied. @@ -164,12 +164,11 @@ def create_custom_training_job_from_component( ), 'env': env or [], }, + 'disk_spec': { + 'boot_disk_type': "{{$.inputs.parameters['boot_disk_type']}}", + 'boot_disk_size_gb': "{{$.inputs.parameters['boot_disk_size_gb']}}", + }, } - if boot_disk_type: - worker_pool_spec['disk_spec'] = { - 'boot_disk_type': boot_disk_type, - 'boot_disk_size_gb': boot_disk_size_gb, - } if nfs_mounts: worker_pool_spec['nfs_mounts'] = nfs_mounts @@ -211,10 +210,7 @@ def create_custom_training_job_from_component( 'defaultValue' ] = default_value - # add machine parameters into the customjob component - if accelerator_type == 'ACCELERATOR_TYPE_UNSPECIFIED': - accelerator_count = 0 - + # add workerPoolSpec parameters into the customjob component cj_component_spec['inputDefinitions']['parameters']['machine_type'] = { 'parameterType': 'STRING', 'defaultValue': machine_type, @@ -227,7 +223,21 @@ def create_custom_training_job_from_component( } cj_component_spec['inputDefinitions']['parameters']['accelerator_count'] = { 'parameterType': 'NUMBER_INTEGER', - 'defaultValue': accelerator_count, + 'defaultValue': ( + accelerator_count + if accelerator_type != 'ACCELERATOR_TYPE_UNSPECIFIED' + else 0 + ), + 'isOptional': True, + } + cj_component_spec['inputDefinitions']['parameters']['boot_disk_type'] = { + 'parameterType': 'STRING', + 'defaultValue': boot_disk_type, + 'isOptional': True, + } + cj_component_spec['inputDefinitions']['parameters']['boot_disk_size_gb'] = { + 'parameterType': 'NUMBER_INTEGER', + 'defaultValue': boot_disk_size_gb, 'isOptional': True, } From a188e545a0515bd8035e43146247a9c6bb3857dc Mon Sep 17 00:00:00 2001 From: Giulio Frasca Date: Wed, 14 Aug 2024 13:48:08 -0400 Subject: [PATCH 09/54] chore: Upgrade Argo to v3.4.17 (#10978) Signed-off-by: Giulio Frasca Signed-off-by: KevinGrantLee --- .cloudbuild.yaml | 4 +- .release.cloudbuild.yaml | 20 +-- backend/Dockerfile | 2 +- backend/src/common/types.go | 2 +- backend/third_party_licenses/apiserver.csv | 57 +++++---- backend/third_party_licenses/cache_server.csv | 42 +++---- backend/third_party_licenses/driver.csv | 51 ++++---- backend/third_party_licenses/launcher.csv | 51 ++++---- .../persistence_agent.csv | 42 +++---- backend/third_party_licenses/swf.csv | 42 +++---- backend/third_party_licenses/viewer.csv | 24 ++-- go.mod | 61 +++++---- go.sum | 104 +++++++++++----- .../kubeflow-pipelines/templates/argo.yaml | 6 +- .../workflow-controller-configmap-patch.yaml | 6 +- .../workflow-controller-deployment-patch.yaml | 4 +- .../argo/upstream/manifests/Kptfile | 6 +- .../namespace-install/kustomization.yaml | 12 +- .../quick-start/base/kustomization.yaml | 6 +- .../quick-start/minimal/kustomization.yaml | 4 +- .../quick-start/mysql/kustomization.yaml | 4 +- .../quick-start/postgres/kustomization.yaml | 4 +- .../quick-start/sso/kustomization.yaml | 6 +- test/install-argo-cli.sh | 2 +- test/tag_for_hosted.sh | 8 +- third_party/argo/Dockerfile.argoexec | 2 +- .../argo/Dockerfile.workflow-controller | 2 +- third_party/argo/README.md | 2 +- third_party/argo/VERSION | 2 +- third_party/argo/licenses-argoexec.csv | 91 +++++++------- .../argo/licenses-workflow-controller.csv | 116 +++++++++--------- 31 files changed, 429 insertions(+), 356 deletions(-) diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml index 7a4d777ef89..c532b6fd217 100644 --- a/.cloudbuild.yaml +++ b/.cloudbuild.yaml @@ -170,10 +170,10 @@ steps: args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.25.0'] id: 'pullCloudsqlProxy' - name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance'] + args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.4.17-license-compliance'] id: 'pullArgoExecutor' - name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance'] + args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.4.17-license-compliance'] id: 'pullArgoWorkflowController' # V2 related images diff --git a/.release.cloudbuild.yaml b/.release.cloudbuild.yaml index 80a93d91fc3..c616635afa8 100644 --- a/.release.cloudbuild.yaml +++ b/.release.cloudbuild.yaml @@ -478,14 +478,14 @@ steps: docker push gcr.io/ml-pipeline/google/pipelines-test/cloudsqlproxy:$(cat /workspace/mm.ver) - name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance'] + args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.4.17-license-compliance'] id: 'pullArgoExecutor' - name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME'] + args: ['tag', 'gcr.io/ml-pipeline/argoexec:v3.4.17-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME'] id: 'tagArgoExecutorForMarketplace' waitFor: ['pullArgoExecutor'] - name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME'] + args: ['tag', 'gcr.io/ml-pipeline/argoexec:v3.4.17-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME'] id: 'tagArgoExecutorForMarketplaceTest' waitFor: ['pullArgoExecutor'] - id: 'tagArgoExecutorForMarketplaceMajorMinor' @@ -495,20 +495,20 @@ steps: args: - -ceux - | - docker tag gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver) - docker tag gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver) + docker tag gcr.io/ml-pipeline/argoexec:v3.4.17-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver) + docker tag gcr.io/ml-pipeline/argoexec:v3.4.17-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver) docker push gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver) docker push gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver) - name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance'] + args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.4.17-license-compliance'] id: 'pullArgoWorkflowController' - name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME'] + args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v3.4.17-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME'] id: 'tagArgoWorkflowControllerForMarketplace' waitFor: ['pullArgoWorkflowController'] - name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME'] + args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v3.4.17-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME'] id: 'tagArgoWorkflowControllerForMarketplaceTest' waitFor: ['pullArgoWorkflowController'] - id: 'tagArgoWorkflowControllerForMarketplaceMajorMinor' @@ -518,8 +518,8 @@ steps: args: - -ceux - | - docker tag gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver) - docker tag gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver) + docker tag gcr.io/ml-pipeline/workflow-controller:v3.4.17-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver) + docker tag gcr.io/ml-pipeline/workflow-controller:v3.4.17-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver) docker push gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver) docker push gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver) diff --git a/backend/Dockerfile b/backend/Dockerfile index 0caa127a3e4..559e3655bf7 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -34,7 +34,7 @@ COPY backend/requirements.txt . RUN python3 -m pip install -r requirements.txt --no-cache-dir # Downloading Argo CLI so that the samples are validated -ENV ARGO_VERSION v3.4.16 +ENV ARGO_VERSION v3.4.17 RUN curl -sLO https://github.com/argoproj/argo-workflows/releases/download/${ARGO_VERSION}/argo-linux-amd64.gz && \ gunzip argo-linux-amd64.gz && \ chmod +x argo-linux-amd64 && \ diff --git a/backend/src/common/types.go b/backend/src/common/types.go index 3f341ca55a2..0816546187c 100644 --- a/backend/src/common/types.go +++ b/backend/src/common/types.go @@ -21,7 +21,7 @@ package common type ExecutionPhase string // borrow from Workflow.Status.Phase: -// https://pkg.go.dev/github.com/argoproj/argo-workflows/v3@v3.4.16/pkg/apis/workflow/v1alpha1#WorkflowPhase +// https://pkg.go.dev/github.com/argoproj/argo-workflows/v3@v3.4.17/pkg/apis/workflow/v1alpha1#WorkflowPhase const ( ExecutionUnknown ExecutionPhase = "" ExecutionPending ExecutionPhase = "Pending" // pending some set-up - rarely used diff --git a/backend/third_party_licenses/apiserver.csv b/backend/third_party_licenses/apiserver.csv index 33c16a0f64d..b5dd1e1f6ed 100644 --- a/backend/third_party_licenses/apiserver.csv +++ b/backend/third_party_licenses/apiserver.csv @@ -1,6 +1,6 @@ cloud.google.com/go/compute/metadata,https://github.com/googleapis/google-cloud-go/blob/compute/metadata/v0.2.3/compute/metadata/LICENSE,Apache-2.0 -cloud.google.com/go/iam,https://github.com/googleapis/google-cloud-go/blob/iam/v1.1.2/iam/LICENSE,Apache-2.0 -cloud.google.com/go/internal,https://github.com/googleapis/google-cloud-go/blob/v0.110.8/LICENSE,Apache-2.0 +cloud.google.com/go/iam,https://github.com/googleapis/google-cloud-go/blob/iam/v1.1.5/iam/LICENSE,Apache-2.0 +cloud.google.com/go/internal,https://github.com/googleapis/google-cloud-go/blob/v0.111.0/LICENSE,Apache-2.0 cloud.google.com/go/storage,https://github.com/googleapis/google-cloud-go/blob/storage/v1.30.1/storage/LICENSE,Apache-2.0 contrib.go.opencensus.io/exporter/ocagent,https://github.com/census-ecosystem/opencensus-go-exporter-ocagent/blob/05415f1de66d/LICENSE,Apache-2.0 contrib.go.opencensus.io/exporter/prometheus,https://github.com/census-ecosystem/opencensus-go-exporter-prometheus/blob/v0.4.0/LICENSE,Apache-2.0 @@ -10,8 +10,7 @@ github.com/Masterminds/sprig/v3,https://github.com/Masterminds/sprig/blob/v3.2.3 github.com/Masterminds/squirrel,https://github.com/Masterminds/squirrel/blob/fa735ea14f09/LICENSE.txt,MIT github.com/VividCortex/mysqlerr,https://github.com/VividCortex/mysqlerr/blob/6c6b55f8796f/LICENSE,MIT github.com/antlr/antlr4/runtime/Go/antlr,https://github.com/antlr/antlr4/blob/runtime/Go/antlr/v1.4.10/runtime/Go/antlr/LICENSE,BSD-3-Clause -github.com/antonmedv/expr,https://github.com/antonmedv/expr/blob/v1.12.6/LICENSE,MIT -github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.4.16/LICENSE,Apache-2.0 +github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.4.17/LICENSE,Apache-2.0 github.com/argoproj/pkg,https://github.com/argoproj/pkg/blob/v0.13.6/LICENSE,Apache-2.0 github.com/asaskevich/govalidator,https://github.com/asaskevich/govalidator/blob/7a23bdc65eef/LICENSE,MIT github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.45.25/LICENSE.txt,Apache-2.0 @@ -26,17 +25,20 @@ github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/L github.com/doublerebel/bellows,https://github.com/doublerebel/bellows/blob/f177d92a03d3/LICENSE,MIT github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.10.2/LICENSE,MIT github.com/evanphx/json-patch/v5,https://github.com/evanphx/json-patch/blob/v5.6.0/v5/LICENSE,BSD-3-Clause +github.com/expr-lang/expr,https://github.com/expr-lang/expr/blob/v1.16.0/LICENSE,MIT +github.com/felixge/httpsnoop,https://github.com/felixge/httpsnoop/blob/v1.0.4/LICENSE.txt,MIT github.com/fsnotify/fsnotify,https://github.com/fsnotify/fsnotify/blob/v1.6.0/LICENSE,BSD-3-Clause github.com/ghodss/yaml,https://github.com/ghodss/yaml/blob/25d852aebe32/LICENSE,MIT github.com/go-kit/log,https://github.com/go-kit/log/blob/v0.2.1/LICENSE,MIT github.com/go-logfmt/logfmt,https://github.com/go-logfmt/logfmt/blob/v0.5.1/LICENSE,MIT -github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.4/LICENSE,Apache-2.0 +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.3.0/LICENSE,Apache-2.0 +github.com/go-logr/stdr,https://github.com/go-logr/stdr/blob/v1.2.2/LICENSE,Apache-2.0 github.com/go-openapi/errors,https://github.com/go-openapi/errors/blob/v0.20.2/LICENSE,Apache-2.0 -github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 -github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.4/LICENSE,Apache-2.0 github.com/go-openapi/runtime,https://github.com/go-openapi/runtime/blob/v0.21.1/LICENSE,Apache-2.0 github.com/go-openapi/strfmt,https://github.com/go-openapi/strfmt/blob/v0.21.1/LICENSE,Apache-2.0 -github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.6/LICENSE,Apache-2.0 github.com/go-sql-driver/mysql,https://github.com/go-sql-driver/mysql/blob/v1.7.1/LICENSE,MPL-2.0 github.com/go-stack/stack,https://github.com/go-stack/stack/blob/v1.8.0/LICENSE.md,MIT github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause @@ -49,9 +51,9 @@ github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.6.0/LICENS github.com/google/go-containerregistry/pkg/name,https://github.com/google/go-containerregistry/blob/v0.16.1/LICENSE,Apache-2.0 github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 github.com/google/s2a-go,https://github.com/google/s2a-go/blob/v0.1.7/LICENSE.md,Apache-2.0 -github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.1/LICENSE,BSD-3-Clause +github.com/google/uuid,https://github.com/google/uuid/blob/v1.5.0/LICENSE,BSD-3-Clause github.com/google/wire,https://github.com/google/wire/blob/v0.4.0/LICENSE,Apache-2.0 -github.com/googleapis/enterprise-certificate-proxy/client,https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.1/LICENSE,Apache-2.0 +github.com/googleapis/enterprise-certificate-proxy/client,https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.2/LICENSE,Apache-2.0 github.com/googleapis/gax-go/v2,https://github.com/googleapis/gax-go/blob/v2.12.0/v2/LICENSE,BSD-3-Clause github.com/gorilla/mux,https://github.com/gorilla/mux/blob/v1.8.0/LICENSE,BSD-3-Clause github.com/gorilla/websocket,https://github.com/gorilla/websocket/blob/v1.5.0/LICENSE,BSD-2-Clause @@ -130,28 +132,33 @@ github.com/valyala/bytebufferpool,https://github.com/valyala/bytebufferpool/blob github.com/valyala/fasttemplate,https://github.com/valyala/fasttemplate/blob/v1.2.2/LICENSE,MIT go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.7.5/LICENSE,Apache-2.0 go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0.24.0/LICENSE,Apache-2.0 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/google.golang.org/grpc/otelgrpc/v0.46.1/instrumentation/google.golang.org/grpc/otelgrpc/LICENSE,Apache-2.0 +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/net/http/otelhttp/v0.46.1/instrumentation/net/http/otelhttp/LICENSE,Apache-2.0 +go.opentelemetry.io/otel,https://github.com/open-telemetry/opentelemetry-go/blob/v1.21.0/LICENSE,Apache-2.0 +go.opentelemetry.io/otel/metric,https://github.com/open-telemetry/opentelemetry-go/blob/metric/v1.21.0/metric/LICENSE,Apache-2.0 +go.opentelemetry.io/otel/trace,https://github.com/open-telemetry/opentelemetry-go/blob/trace/v1.21.0/trace/LICENSE,Apache-2.0 go.uber.org/atomic,https://github.com/uber-go/atomic/blob/v1.10.0/LICENSE.txt,MIT go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.10.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.26.0/LICENSE.txt,MIT gocloud.dev,https://github.com/google/go-cloud/blob/v0.22.0/LICENSE,Apache-2.0 -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.16.0:LICENSE,BSD-3-Clause -golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.19.0:LICENSE,BSD-3-Clause -golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.13.0:LICENSE,BSD-3-Clause -golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.4.0:LICENSE,BSD-3-Clause -golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.22.0:LICENSE,BSD-3-Clause +golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/92128663:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.21.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.16.0:LICENSE,BSD-3-Clause +golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.6.0:LICENSE,BSD-3-Clause +golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.19.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.19.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause -golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause golang.org/x/xerrors,https://cs.opensource.google/go/x/xerrors/+/04be3eba:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.4.0/v2/LICENSE,Apache-2.0 -google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.147.0/LICENSE,BSD-3-Clause -google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.147.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause -google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/d307bd883b97/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/d307bd883b97/googleapis/api/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/8bfb1ae86b6c/googleapis/rpc/LICENSE,Apache-2.0 -google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.58.3/LICENSE,Apache-2.0 -google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE,BSD-3-Clause +google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/LICENSE,BSD-3-Clause +google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause +google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/995d672761c0/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/995d672761c0/googleapis/api/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/50ed04b92917/googleapis/rpc/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.60.1/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/ini.v1,https://github.com/go-ini/ini/blob/v1.67.0/LICENSE,Apache-2.0 gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 diff --git a/backend/third_party_licenses/cache_server.csv b/backend/third_party_licenses/cache_server.csv index ac5da669688..ae3168d927f 100644 --- a/backend/third_party_licenses/cache_server.csv +++ b/backend/third_party_licenses/cache_server.csv @@ -4,8 +4,7 @@ github.com/Masterminds/goutils,https://github.com/Masterminds/goutils/blob/v1.1. github.com/Masterminds/semver/v3,https://github.com/Masterminds/semver/blob/v3.2.0/LICENSE.txt,MIT github.com/Masterminds/sprig/v3,https://github.com/Masterminds/sprig/blob/v3.2.3/LICENSE.txt,MIT github.com/antlr/antlr4/runtime/Go/antlr,https://github.com/antlr/antlr4/blob/runtime/Go/antlr/v1.4.10/runtime/Go/antlr/LICENSE,BSD-3-Clause -github.com/antonmedv/expr,https://github.com/antonmedv/expr/blob/v1.12.6/LICENSE,MIT -github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.4.16/LICENSE,Apache-2.0 +github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.4.17/LICENSE,Apache-2.0 github.com/argoproj/pkg,https://github.com/argoproj/pkg/blob/v0.13.6/LICENSE,Apache-2.0 github.com/asaskevich/govalidator,https://github.com/asaskevich/govalidator/blob/7a23bdc65eef/LICENSE,MIT github.com/beorn7/perks/quantile,https://github.com/beorn7/perks/blob/v1.0.1/LICENSE,MIT @@ -18,16 +17,17 @@ github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/L github.com/doublerebel/bellows,https://github.com/doublerebel/bellows/blob/f177d92a03d3/LICENSE,MIT github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.10.2/LICENSE,MIT github.com/evanphx/json-patch/v5,https://github.com/evanphx/json-patch/blob/v5.6.0/v5/LICENSE,BSD-3-Clause +github.com/expr-lang/expr,https://github.com/expr-lang/expr/blob/v1.16.0/LICENSE,MIT github.com/ghodss/yaml,https://github.com/ghodss/yaml/blob/25d852aebe32/LICENSE,MIT github.com/go-kit/log,https://github.com/go-kit/log/blob/v0.2.1/LICENSE,MIT github.com/go-logfmt/logfmt,https://github.com/go-logfmt/logfmt/blob/v0.5.1/LICENSE,MIT -github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.4/LICENSE,Apache-2.0 +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.3.0/LICENSE,Apache-2.0 github.com/go-openapi/errors,https://github.com/go-openapi/errors/blob/v0.20.2/LICENSE,Apache-2.0 -github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 -github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.4/LICENSE,Apache-2.0 github.com/go-openapi/runtime,https://github.com/go-openapi/runtime/blob/v0.21.1/LICENSE,Apache-2.0 github.com/go-openapi/strfmt,https://github.com/go-openapi/strfmt/blob/v0.21.1/LICENSE,Apache-2.0 -github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.6/LICENSE,Apache-2.0 github.com/go-sql-driver/mysql,https://github.com/go-sql-driver/mysql/blob/v1.7.1/LICENSE,MPL-2.0 github.com/go-stack/stack,https://github.com/go-stack/stack/blob/v1.8.0/LICENSE.md,MIT github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause @@ -39,7 +39,7 @@ github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.6.9/LICENSE, github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.6.0/LICENSE,BSD-3-Clause github.com/google/go-containerregistry/pkg/name,https://github.com/google/go-containerregistry/blob/v0.16.1/LICENSE,Apache-2.0 github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 -github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.1/LICENSE,BSD-3-Clause +github.com/google/uuid,https://github.com/google/uuid/blob/v1.5.0/LICENSE,BSD-3-Clause github.com/gorilla/websocket,https://github.com/gorilla/websocket/blob/v1.5.0/LICENSE,BSD-2-Clause github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause github.com/grpc-ecosystem/grpc-gateway/v2,https://github.com/grpc-ecosystem/grpc-gateway/blob/v2.11.3/LICENSE.txt,BSD-3-Clause @@ -97,22 +97,22 @@ go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0 go.uber.org/atomic,https://github.com/uber-go/atomic/blob/v1.10.0/LICENSE.txt,MIT go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.10.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.26.0/LICENSE.txt,MIT -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.16.0:LICENSE,BSD-3-Clause -golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.19.0:LICENSE,BSD-3-Clause -golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.13.0:LICENSE,BSD-3-Clause -golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.4.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.22.0:LICENSE,BSD-3-Clause +golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/92128663:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.21.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.16.0:LICENSE,BSD-3-Clause +golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.6.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.19.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.19.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause -golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.4.0/v2/LICENSE,Apache-2.0 -google.golang.org/api/support/bundler,https://github.com/googleapis/google-api-go-client/blob/v0.147.0/LICENSE,BSD-3-Clause -google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/d307bd883b97/googleapis/api/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/rpc/status,https://github.com/googleapis/go-genproto/blob/8bfb1ae86b6c/googleapis/rpc/LICENSE,Apache-2.0 -google.golang.org/genproto/protobuf/field_mask,https://github.com/googleapis/go-genproto/blob/d307bd883b97/LICENSE,Apache-2.0 -google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.58.3/LICENSE,Apache-2.0 -google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE,BSD-3-Clause +google.golang.org/api/support/bundler,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/LICENSE,BSD-3-Clause +google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/995d672761c0/googleapis/api/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/rpc/status,https://github.com/googleapis/go-genproto/blob/50ed04b92917/googleapis/rpc/LICENSE,Apache-2.0 +google.golang.org/genproto/protobuf/field_mask,https://github.com/googleapis/go-genproto/blob/995d672761c0/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.60.1/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT diff --git a/backend/third_party_licenses/driver.csv b/backend/third_party_licenses/driver.csv index a65e0dae45d..a2cf7c59854 100644 --- a/backend/third_party_licenses/driver.csv +++ b/backend/third_party_licenses/driver.csv @@ -1,16 +1,18 @@ cloud.google.com/go/compute/metadata,https://github.com/googleapis/google-cloud-go/blob/compute/metadata/v0.2.3/compute/metadata/LICENSE,Apache-2.0 -cloud.google.com/go/iam,https://github.com/googleapis/google-cloud-go/blob/iam/v1.1.2/iam/LICENSE,Apache-2.0 -cloud.google.com/go/internal,https://github.com/googleapis/google-cloud-go/blob/v0.110.8/LICENSE,Apache-2.0 +cloud.google.com/go/iam,https://github.com/googleapis/google-cloud-go/blob/iam/v1.1.5/iam/LICENSE,Apache-2.0 +cloud.google.com/go/internal,https://github.com/googleapis/google-cloud-go/blob/v0.111.0/LICENSE,Apache-2.0 cloud.google.com/go/storage,https://github.com/googleapis/google-cloud-go/blob/storage/v1.30.1/storage/LICENSE,Apache-2.0 github.com/antlr/antlr4/runtime/Go/antlr,https://github.com/antlr/antlr4/blob/runtime/Go/antlr/v1.4.10/runtime/Go/antlr/LICENSE,BSD-3-Clause github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.45.25/LICENSE.txt,Apache-2.0 github.com/aws/aws-sdk-go/internal/sync/singleflight,https://github.com/aws/aws-sdk-go/blob/v1.45.25/internal/sync/singleflight/LICENSE,BSD-3-Clause github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.10.2/LICENSE,MIT -github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.4/LICENSE,Apache-2.0 -github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 -github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 -github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/felixge/httpsnoop,https://github.com/felixge/httpsnoop/blob/v1.0.4/LICENSE.txt,MIT +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.3.0/LICENSE,Apache-2.0 +github.com/go-logr/stdr,https://github.com/go-logr/stdr/blob/v1.2.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.4/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.6/LICENSE,Apache-2.0 github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause github.com/golang/glog,https://github.com/golang/glog/blob/v1.2.0/LICENSE,Apache-2.0 github.com/golang/groupcache/lru,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0 @@ -20,9 +22,9 @@ github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.6.9/LICENSE, github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.6.0/LICENSE,BSD-3-Clause github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 github.com/google/s2a-go,https://github.com/google/s2a-go/blob/v0.1.7/LICENSE.md,Apache-2.0 -github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.1/LICENSE,BSD-3-Clause +github.com/google/uuid,https://github.com/google/uuid/blob/v1.5.0/LICENSE,BSD-3-Clause github.com/google/wire,https://github.com/google/wire/blob/v0.4.0/LICENSE,Apache-2.0 -github.com/googleapis/enterprise-certificate-proxy/client,https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.1/LICENSE,Apache-2.0 +github.com/googleapis/enterprise-certificate-proxy/client,https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.2/LICENSE,Apache-2.0 github.com/googleapis/gax-go/v2,https://github.com/googleapis/gax-go/blob/v2.12.0/v2/LICENSE,BSD-3-Clause github.com/grpc-ecosystem/go-grpc-middleware,https://github.com/grpc-ecosystem/go-grpc-middleware/blob/v1.3.0/LICENSE,Apache-2.0 github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause @@ -39,23 +41,28 @@ github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.2/ github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE,BSD-3-Clause github.com/stoewer/go-strcase,https://github.com/stoewer/go-strcase/blob/v1.2.0/LICENSE,MIT go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0.24.0/LICENSE,Apache-2.0 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/google.golang.org/grpc/otelgrpc/v0.46.1/instrumentation/google.golang.org/grpc/otelgrpc/LICENSE,Apache-2.0 +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/net/http/otelhttp/v0.46.1/instrumentation/net/http/otelhttp/LICENSE,Apache-2.0 +go.opentelemetry.io/otel,https://github.com/open-telemetry/opentelemetry-go/blob/v1.21.0/LICENSE,Apache-2.0 +go.opentelemetry.io/otel/metric,https://github.com/open-telemetry/opentelemetry-go/blob/metric/v1.21.0/metric/LICENSE,Apache-2.0 +go.opentelemetry.io/otel/trace,https://github.com/open-telemetry/opentelemetry-go/blob/trace/v1.21.0/trace/LICENSE,Apache-2.0 gocloud.dev,https://github.com/google/go-cloud/blob/v0.22.0/LICENSE,Apache-2.0 -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.16.0:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.19.0:LICENSE,BSD-3-Clause -golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.13.0:LICENSE,BSD-3-Clause -golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.4.0:LICENSE,BSD-3-Clause -golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.22.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.21.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.16.0:LICENSE,BSD-3-Clause +golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.6.0:LICENSE,BSD-3-Clause +golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.19.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.19.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause -golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause golang.org/x/xerrors,https://cs.opensource.google/go/x/xerrors/+/04be3eba:LICENSE,BSD-3-Clause -google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.147.0/LICENSE,BSD-3-Clause -google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.147.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause -google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/d307bd883b97/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/d307bd883b97/googleapis/api/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/8bfb1ae86b6c/googleapis/rpc/LICENSE,Apache-2.0 -google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.58.3/LICENSE,Apache-2.0 -google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE,BSD-3-Clause +google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/LICENSE,BSD-3-Clause +google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause +google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/995d672761c0/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/995d672761c0/googleapis/api/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/50ed04b92917/googleapis/rpc/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.60.1/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT diff --git a/backend/third_party_licenses/launcher.csv b/backend/third_party_licenses/launcher.csv index 9a7bebc5ca9..ef271ec8e09 100644 --- a/backend/third_party_licenses/launcher.csv +++ b/backend/third_party_licenses/launcher.csv @@ -1,15 +1,17 @@ cloud.google.com/go/compute/metadata,https://github.com/googleapis/google-cloud-go/blob/compute/metadata/v0.2.3/compute/metadata/LICENSE,Apache-2.0 -cloud.google.com/go/iam,https://github.com/googleapis/google-cloud-go/blob/iam/v1.1.2/iam/LICENSE,Apache-2.0 -cloud.google.com/go/internal,https://github.com/googleapis/google-cloud-go/blob/v0.110.8/LICENSE,Apache-2.0 +cloud.google.com/go/iam,https://github.com/googleapis/google-cloud-go/blob/iam/v1.1.5/iam/LICENSE,Apache-2.0 +cloud.google.com/go/internal,https://github.com/googleapis/google-cloud-go/blob/v0.111.0/LICENSE,Apache-2.0 cloud.google.com/go/storage,https://github.com/googleapis/google-cloud-go/blob/storage/v1.30.1/storage/LICENSE,Apache-2.0 github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.45.25/LICENSE.txt,Apache-2.0 github.com/aws/aws-sdk-go/internal/sync/singleflight,https://github.com/aws/aws-sdk-go/blob/v1.45.25/internal/sync/singleflight/LICENSE,BSD-3-Clause github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.10.2/LICENSE,MIT -github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.4/LICENSE,Apache-2.0 -github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 -github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 -github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/felixge/httpsnoop,https://github.com/felixge/httpsnoop/blob/v1.0.4/LICENSE.txt,MIT +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.3.0/LICENSE,Apache-2.0 +github.com/go-logr/stdr,https://github.com/go-logr/stdr/blob/v1.2.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.4/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.6/LICENSE,Apache-2.0 github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause github.com/golang/glog,https://github.com/golang/glog/blob/v1.2.0/LICENSE,Apache-2.0 github.com/golang/groupcache/lru,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0 @@ -18,9 +20,9 @@ github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.6.9/LICENSE, github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.6.0/LICENSE,BSD-3-Clause github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 github.com/google/s2a-go,https://github.com/google/s2a-go/blob/v0.1.7/LICENSE.md,Apache-2.0 -github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.1/LICENSE,BSD-3-Clause +github.com/google/uuid,https://github.com/google/uuid/blob/v1.5.0/LICENSE,BSD-3-Clause github.com/google/wire,https://github.com/google/wire/blob/v0.4.0/LICENSE,Apache-2.0 -github.com/googleapis/enterprise-certificate-proxy/client,https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.1/LICENSE,Apache-2.0 +github.com/googleapis/enterprise-certificate-proxy/client,https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.2/LICENSE,Apache-2.0 github.com/googleapis/gax-go/v2,https://github.com/googleapis/gax-go/blob/v2.12.0/v2/LICENSE,BSD-3-Clause github.com/grpc-ecosystem/go-grpc-middleware,https://github.com/grpc-ecosystem/go-grpc-middleware/blob/v1.3.0/LICENSE,Apache-2.0 github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause @@ -35,23 +37,28 @@ github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bac github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE,Apache-2.0 github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE,BSD-3-Clause go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0.24.0/LICENSE,Apache-2.0 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/google.golang.org/grpc/otelgrpc/v0.46.1/instrumentation/google.golang.org/grpc/otelgrpc/LICENSE,Apache-2.0 +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/net/http/otelhttp/v0.46.1/instrumentation/net/http/otelhttp/LICENSE,Apache-2.0 +go.opentelemetry.io/otel,https://github.com/open-telemetry/opentelemetry-go/blob/v1.21.0/LICENSE,Apache-2.0 +go.opentelemetry.io/otel/metric,https://github.com/open-telemetry/opentelemetry-go/blob/metric/v1.21.0/metric/LICENSE,Apache-2.0 +go.opentelemetry.io/otel/trace,https://github.com/open-telemetry/opentelemetry-go/blob/trace/v1.21.0/trace/LICENSE,Apache-2.0 gocloud.dev,https://github.com/google/go-cloud/blob/v0.22.0/LICENSE,Apache-2.0 -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.16.0:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.19.0:LICENSE,BSD-3-Clause -golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.13.0:LICENSE,BSD-3-Clause -golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.4.0:LICENSE,BSD-3-Clause -golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.22.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.21.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.16.0:LICENSE,BSD-3-Clause +golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.6.0:LICENSE,BSD-3-Clause +golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.19.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.19.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause -golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause golang.org/x/xerrors,https://cs.opensource.google/go/x/xerrors/+/04be3eba:LICENSE,BSD-3-Clause -google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.147.0/LICENSE,BSD-3-Clause -google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.147.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause -google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/d307bd883b97/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/d307bd883b97/googleapis/api/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/8bfb1ae86b6c/googleapis/rpc/LICENSE,Apache-2.0 -google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.58.3/LICENSE,Apache-2.0 -google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE,BSD-3-Clause +google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/LICENSE,BSD-3-Clause +google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause +google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/995d672761c0/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/995d672761c0/googleapis/api/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/50ed04b92917/googleapis/rpc/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.60.1/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT diff --git a/backend/third_party_licenses/persistence_agent.csv b/backend/third_party_licenses/persistence_agent.csv index a656ba005fc..700f00f65b5 100644 --- a/backend/third_party_licenses/persistence_agent.csv +++ b/backend/third_party_licenses/persistence_agent.csv @@ -5,8 +5,7 @@ github.com/Masterminds/goutils,https://github.com/Masterminds/goutils/blob/v1.1. github.com/Masterminds/semver/v3,https://github.com/Masterminds/semver/blob/v3.2.0/LICENSE.txt,MIT github.com/Masterminds/sprig/v3,https://github.com/Masterminds/sprig/blob/v3.2.3/LICENSE.txt,MIT github.com/antlr/antlr4/runtime/Go/antlr,https://github.com/antlr/antlr4/blob/runtime/Go/antlr/v1.4.10/runtime/Go/antlr/LICENSE,BSD-3-Clause -github.com/antonmedv/expr,https://github.com/antonmedv/expr/blob/v1.12.6/LICENSE,MIT -github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.4.16/LICENSE,Apache-2.0 +github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.4.17/LICENSE,Apache-2.0 github.com/argoproj/pkg,https://github.com/argoproj/pkg/blob/v0.13.6/LICENSE,Apache-2.0 github.com/asaskevich/govalidator,https://github.com/asaskevich/govalidator/blob/7a23bdc65eef/LICENSE,MIT github.com/beorn7/perks/quantile,https://github.com/beorn7/perks/blob/v1.0.1/LICENSE,MIT @@ -19,16 +18,17 @@ github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/L github.com/doublerebel/bellows,https://github.com/doublerebel/bellows/blob/f177d92a03d3/LICENSE,MIT github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.10.2/LICENSE,MIT github.com/evanphx/json-patch/v5,https://github.com/evanphx/json-patch/blob/v5.6.0/v5/LICENSE,BSD-3-Clause +github.com/expr-lang/expr,https://github.com/expr-lang/expr/blob/v1.16.0/LICENSE,MIT github.com/ghodss/yaml,https://github.com/ghodss/yaml/blob/25d852aebe32/LICENSE,MIT github.com/go-kit/log,https://github.com/go-kit/log/blob/v0.2.1/LICENSE,MIT github.com/go-logfmt/logfmt,https://github.com/go-logfmt/logfmt/blob/v0.5.1/LICENSE,MIT -github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.4/LICENSE,Apache-2.0 +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.3.0/LICENSE,Apache-2.0 github.com/go-openapi/errors,https://github.com/go-openapi/errors/blob/v0.20.2/LICENSE,Apache-2.0 -github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 -github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.4/LICENSE,Apache-2.0 github.com/go-openapi/runtime,https://github.com/go-openapi/runtime/blob/v0.21.1/LICENSE,Apache-2.0 github.com/go-openapi/strfmt,https://github.com/go-openapi/strfmt/blob/v0.21.1/LICENSE,Apache-2.0 -github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.6/LICENSE,Apache-2.0 github.com/go-stack/stack,https://github.com/go-stack/stack/blob/v1.8.0/LICENSE.md,MIT github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause github.com/golang/glog,https://github.com/golang/glog/blob/v1.2.0/LICENSE,Apache-2.0 @@ -39,7 +39,7 @@ github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.6.9/LICENSE, github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.6.0/LICENSE,BSD-3-Clause github.com/google/go-containerregistry/pkg/name,https://github.com/google/go-containerregistry/blob/v0.16.1/LICENSE,Apache-2.0 github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 -github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.1/LICENSE,BSD-3-Clause +github.com/google/uuid,https://github.com/google/uuid/blob/v1.5.0/LICENSE,BSD-3-Clause github.com/gorilla/websocket,https://github.com/gorilla/websocket/blob/v1.5.0/LICENSE,BSD-2-Clause github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause github.com/grpc-ecosystem/grpc-gateway/v2,https://github.com/grpc-ecosystem/grpc-gateway/blob/v2.11.3/LICENSE.txt,BSD-3-Clause @@ -93,22 +93,22 @@ go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0 go.uber.org/atomic,https://github.com/uber-go/atomic/blob/v1.10.0/LICENSE.txt,MIT go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.10.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.26.0/LICENSE.txt,MIT -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.16.0:LICENSE,BSD-3-Clause -golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.19.0:LICENSE,BSD-3-Clause -golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.13.0:LICENSE,BSD-3-Clause -golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.4.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.22.0:LICENSE,BSD-3-Clause +golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/92128663:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.21.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.16.0:LICENSE,BSD-3-Clause +golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.6.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.19.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.19.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause -golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.4.0/v2/LICENSE,Apache-2.0 -google.golang.org/api/support/bundler,https://github.com/googleapis/google-api-go-client/blob/v0.147.0/LICENSE,BSD-3-Clause -google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/d307bd883b97/googleapis/api/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/rpc/status,https://github.com/googleapis/go-genproto/blob/8bfb1ae86b6c/googleapis/rpc/LICENSE,Apache-2.0 -google.golang.org/genproto/protobuf/field_mask,https://github.com/googleapis/go-genproto/blob/d307bd883b97/LICENSE,Apache-2.0 -google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.58.3/LICENSE,Apache-2.0 -google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE,BSD-3-Clause +google.golang.org/api/support/bundler,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/LICENSE,BSD-3-Clause +google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/995d672761c0/googleapis/api/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/rpc/status,https://github.com/googleapis/go-genproto/blob/50ed04b92917/googleapis/rpc/LICENSE,Apache-2.0 +google.golang.org/genproto/protobuf/field_mask,https://github.com/googleapis/go-genproto/blob/995d672761c0/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.60.1/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT diff --git a/backend/third_party_licenses/swf.csv b/backend/third_party_licenses/swf.csv index 630e8491a4f..a2b7f46ed43 100644 --- a/backend/third_party_licenses/swf.csv +++ b/backend/third_party_licenses/swf.csv @@ -5,8 +5,7 @@ github.com/Masterminds/goutils,https://github.com/Masterminds/goutils/blob/v1.1. github.com/Masterminds/semver/v3,https://github.com/Masterminds/semver/blob/v3.2.0/LICENSE.txt,MIT github.com/Masterminds/sprig/v3,https://github.com/Masterminds/sprig/blob/v3.2.3/LICENSE.txt,MIT github.com/antlr/antlr4/runtime/Go/antlr,https://github.com/antlr/antlr4/blob/runtime/Go/antlr/v1.4.10/runtime/Go/antlr/LICENSE,BSD-3-Clause -github.com/antonmedv/expr,https://github.com/antonmedv/expr/blob/v1.12.6/LICENSE,MIT -github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.4.16/LICENSE,Apache-2.0 +github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.4.17/LICENSE,Apache-2.0 github.com/argoproj/pkg,https://github.com/argoproj/pkg/blob/v0.13.6/LICENSE,Apache-2.0 github.com/asaskevich/govalidator,https://github.com/asaskevich/govalidator/blob/7a23bdc65eef/LICENSE,MIT github.com/beorn7/perks/quantile,https://github.com/beorn7/perks/blob/v1.0.1/LICENSE,MIT @@ -19,17 +18,18 @@ github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/L github.com/doublerebel/bellows,https://github.com/doublerebel/bellows/blob/f177d92a03d3/LICENSE,MIT github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.10.2/LICENSE,MIT github.com/evanphx/json-patch/v5,https://github.com/evanphx/json-patch/blob/v5.6.0/v5/LICENSE,BSD-3-Clause +github.com/expr-lang/expr,https://github.com/expr-lang/expr/blob/v1.16.0/LICENSE,MIT github.com/fsnotify/fsnotify,https://github.com/fsnotify/fsnotify/blob/v1.6.0/LICENSE,BSD-3-Clause github.com/ghodss/yaml,https://github.com/ghodss/yaml/blob/25d852aebe32/LICENSE,MIT github.com/go-kit/log,https://github.com/go-kit/log/blob/v0.2.1/LICENSE,MIT github.com/go-logfmt/logfmt,https://github.com/go-logfmt/logfmt/blob/v0.5.1/LICENSE,MIT -github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.4/LICENSE,Apache-2.0 +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.3.0/LICENSE,Apache-2.0 github.com/go-openapi/errors,https://github.com/go-openapi/errors/blob/v0.20.2/LICENSE,Apache-2.0 -github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 -github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.4/LICENSE,Apache-2.0 github.com/go-openapi/runtime,https://github.com/go-openapi/runtime/blob/v0.21.1/LICENSE,Apache-2.0 github.com/go-openapi/strfmt,https://github.com/go-openapi/strfmt/blob/v0.21.1/LICENSE,Apache-2.0 -github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.6/LICENSE,Apache-2.0 github.com/go-stack/stack,https://github.com/go-stack/stack/blob/v1.8.0/LICENSE.md,MIT github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause github.com/golang/glog,https://github.com/golang/glog/blob/v1.2.0/LICENSE,Apache-2.0 @@ -40,7 +40,7 @@ github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.6.9/LICENSE, github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.6.0/LICENSE,BSD-3-Clause github.com/google/go-containerregistry/pkg/name,https://github.com/google/go-containerregistry/blob/v0.16.1/LICENSE,Apache-2.0 github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 -github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.1/LICENSE,BSD-3-Clause +github.com/google/uuid,https://github.com/google/uuid/blob/v1.5.0/LICENSE,BSD-3-Clause github.com/gorilla/websocket,https://github.com/gorilla/websocket/blob/v1.5.0/LICENSE,BSD-2-Clause github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause github.com/grpc-ecosystem/grpc-gateway/v2,https://github.com/grpc-ecosystem/grpc-gateway/blob/v2.11.3/LICENSE.txt,BSD-3-Clause @@ -102,22 +102,22 @@ go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0 go.uber.org/atomic,https://github.com/uber-go/atomic/blob/v1.10.0/LICENSE.txt,MIT go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.10.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.26.0/LICENSE.txt,MIT -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.16.0:LICENSE,BSD-3-Clause -golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.19.0:LICENSE,BSD-3-Clause -golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.13.0:LICENSE,BSD-3-Clause -golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.4.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.22.0:LICENSE,BSD-3-Clause +golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/92128663:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.21.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.16.0:LICENSE,BSD-3-Clause +golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.6.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.19.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.19.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause -golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.4.0/v2/LICENSE,Apache-2.0 -google.golang.org/api/support/bundler,https://github.com/googleapis/google-api-go-client/blob/v0.147.0/LICENSE,BSD-3-Clause -google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/d307bd883b97/googleapis/api/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/rpc/status,https://github.com/googleapis/go-genproto/blob/8bfb1ae86b6c/googleapis/rpc/LICENSE,Apache-2.0 -google.golang.org/genproto/protobuf/field_mask,https://github.com/googleapis/go-genproto/blob/d307bd883b97/LICENSE,Apache-2.0 -google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.58.3/LICENSE,Apache-2.0 -google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE,BSD-3-Clause +google.golang.org/api/support/bundler,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/LICENSE,BSD-3-Clause +google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/995d672761c0/googleapis/api/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/rpc/status,https://github.com/googleapis/go-genproto/blob/50ed04b92917/googleapis/rpc/LICENSE,Apache-2.0 +google.golang.org/genproto/protobuf/field_mask,https://github.com/googleapis/go-genproto/blob/995d672761c0/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.60.1/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/ini.v1,https://github.com/go-ini/ini/blob/v1.67.0/LICENSE,Apache-2.0 gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 diff --git a/backend/third_party_licenses/viewer.csv b/backend/third_party_licenses/viewer.csv index 862391f98ec..0c13c0f6a41 100644 --- a/backend/third_party_licenses/viewer.csv +++ b/backend/third_party_licenses/viewer.csv @@ -3,12 +3,12 @@ github.com/beorn7/perks/quantile,https://github.com/beorn7/perks/blob/v1.0.1/LIC github.com/cespare/xxhash/v2,https://github.com/cespare/xxhash/blob/v2.2.0/LICENSE.txt,MIT github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.10.2/LICENSE,MIT -github.com/evanphx/json-patch,https://github.com/evanphx/json-patch/blob/v5.6.0/LICENSE,BSD-3-Clause +github.com/evanphx/json-patch,https://github.com/evanphx/json-patch/blob/v5.8.0/LICENSE,BSD-3-Clause github.com/fsnotify/fsnotify,https://github.com/fsnotify/fsnotify/blob/v1.6.0/LICENSE,BSD-3-Clause -github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.4/LICENSE,Apache-2.0 -github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 -github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 -github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.3.0/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.4/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.6/LICENSE,Apache-2.0 github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause github.com/golang/glog,https://github.com/golang/glog/blob/v1.2.0/LICENSE,Apache-2.0 github.com/golang/groupcache/lru,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0 @@ -16,7 +16,7 @@ github.com/golang/protobuf,https://github.com/golang/protobuf/blob/v1.5.3/LICENS github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.6.9/LICENSE,Apache-2.0 github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.6.0/LICENSE,BSD-3-Clause github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 -github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.1/LICENSE,BSD-3-Clause +github.com/google/uuid,https://github.com/google/uuid/blob/v1.5.0/LICENSE,BSD-3-Clause github.com/imdario/mergo,https://github.com/imdario/mergo/blob/v0.3.13/LICENSE,BSD-3-Clause github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT @@ -33,14 +33,14 @@ github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.42.0/L github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.42.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.10.1/LICENSE,Apache-2.0 github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.19.0:LICENSE,BSD-3-Clause -golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.13.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.21.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.16.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.19.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.19.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause -golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.4.0/v2/LICENSE,Apache-2.0 -google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE,BSD-3-Clause +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT diff --git a/go.mod b/go.mod index 43ac2889ea8..44e9ae0b7f4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/kubeflow/pipelines require ( github.com/Masterminds/squirrel v0.0.0-20190107164353-fa735ea14f09 github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f - github.com/argoproj/argo-workflows/v3 v3.4.16 + github.com/argoproj/argo-workflows/v3 v3.4.17 github.com/aws/aws-sdk-go v1.45.25 github.com/cenkalti/backoff v2.2.1+incompatible github.com/eapache/go-resiliency v1.2.0 @@ -13,7 +13,7 @@ require ( github.com/go-openapi/errors v0.20.2 github.com/go-openapi/runtime v0.21.1 github.com/go-openapi/strfmt v0.21.1 - github.com/go-openapi/swag v0.22.3 + github.com/go-openapi/swag v0.22.6 github.com/go-openapi/validate v0.20.3 github.com/go-sql-driver/mysql v1.7.1 github.com/golang/glog v1.2.0 @@ -21,7 +21,7 @@ require ( github.com/google/addlicense v0.0.0-20200906110928-a0294312aa76 github.com/google/cel-go v0.12.6 github.com/google/go-cmp v0.6.0 - github.com/google/uuid v1.3.1 + github.com/google/uuid v1.5.0 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 @@ -47,11 +47,11 @@ require ( github.com/tektoncd/pipeline v0.53.2 go.uber.org/zap v1.26.0 // indirect gocloud.dev v0.22.0 - golang.org/x/net v0.19.0 - google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/grpc v1.58.3 + golang.org/x/net v0.21.0 + google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 // indirect + google.golang.org/grpc v1.60.1 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 - google.golang.org/protobuf v1.31.0 + google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.27.2 k8s.io/apimachinery v0.27.3 @@ -66,16 +66,16 @@ require ( require ( github.com/prometheus/client_golang v1.16.0 - golang.org/x/oauth2 v0.13.0 - google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 - google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c + golang.org/x/oauth2 v0.16.0 + google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 ) require ( - cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go v0.111.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.2 // indirect + cloud.google.com/go/iam v1.1.5 // indirect cloud.google.com/go/storage v1.30.1 // indirect contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect @@ -83,7 +83,6 @@ require ( github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect - github.com/antonmedv/expr v1.12.6 // indirect github.com/argoproj/pkg v0.13.6 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -94,14 +93,17 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/doublerebel/bellows v0.0.0-20160303004610-f177d92a03d3 // indirect github.com/emicklei/go-restful/v3 v3.10.2 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/evanphx/json-patch v5.8.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/expr-lang/expr v1.16.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.20.1 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/jsonpointer v0.20.2 // indirect + github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/loads v0.21.0 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-stack/stack v1.8.0 // indirect @@ -112,7 +114,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/wire v0.4.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect @@ -172,21 +174,26 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect go.mongodb.org/mongo-driver v1.7.5 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.16.0 // indirect - golang.org/x/exp v0.0.0-20230307190834-24139beb5833 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/term v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/api v0.147.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + google.golang.org/api v0.156.0 // indirect + google.golang.org/appengine v1.6.8 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 1b88be0d91f..013220bd44d 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,9 @@ cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVqux cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= -cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= +cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM= +cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -227,8 +228,9 @@ cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IK cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= @@ -427,8 +429,9 @@ cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCta cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= @@ -977,14 +980,12 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/antonmedv/expr v1.12.6 h1:qtgMHOFissxhePwokx0xB9eqS6PUy0SbhDRPD67PInA= -github.com/antonmedv/expr v1.12.6/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= -github.com/argoproj/argo-workflows/v3 v3.4.16 h1:e1wOjKQ69gqnN1S9CIIaUVM7Sm6z49cWO/dXQLTmVxA= -github.com/argoproj/argo-workflows/v3 v3.4.16/go.mod h1:jJF/JgTZazmsw2ThKgwHACKwc61fO4avhxp9vfu3Lzc= +github.com/argoproj/argo-workflows/v3 v3.4.17 h1:ns3kikZL8QZip0rGChAFrIU6gx2nyGC+VWIwZ9JCEb8= +github.com/argoproj/argo-workflows/v3 v3.4.17/go.mod h1:9eunmGsSLQU1+1CGzpNIJi5tPk/dR6eWPLfQIgwmCt0= github.com/argoproj/pkg v0.13.6 h1:36WPD9MNYECHcO1/R1pj6teYspiK7uMQLCgLGft2abM= github.com/argoproj/pkg v0.13.6/go.mod h1:I698DoJBKuvNFaixh4vFl2C88cNIT1WS7KCbz5ewyF8= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -1152,6 +1153,7 @@ github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -1435,6 +1437,7 @@ github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0+ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= @@ -1442,10 +1445,12 @@ github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMi github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.8.0+incompatible h1:1Av9pn2FyxPdvrWNQszj1g6D6YthSmvCfcN6SYclTJg= +github.com/evanphx/json-patch v5.8.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/expr-lang/expr v1.16.0 h1:BQabx+PbjsL2PEQwkJ4GIn3CcuUh8flduHhJ0lHjWwE= +github.com/expr-lang/expr v1.16.0/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01/go.mod h1:ypD5nozFk9vcGw1ATYefw6jHe/jZP++Z15/+VTMcWhc= @@ -1458,6 +1463,8 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/flynn/go-docopt v0.0.0-20140912013429-f6dd2ebbb31e/go.mod h1:HyVoz1Mz5Co8TFO8EupIdlcpwShBmY98dkT2xeHkvEI= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -1533,9 +1540,11 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= @@ -1566,8 +1575,9 @@ github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwds github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= +github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= @@ -1576,8 +1586,9 @@ github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= +github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -1633,8 +1644,9 @@ github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5H github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.6 h1:dnqg1XfHXL9aBxSbktBqFR5CxVyVI+7fYWhAf1JOeTw= +github.com/go-openapi/swag v0.22.6/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= @@ -1889,8 +1901,9 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE= github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= @@ -1899,8 +1912,9 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/ github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -2919,9 +2933,13 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.2 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.40.0/go.mod h1:UMklln0+MRhZC4e3PwmN3pCtq4DyIadWw4yikh6bNrw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.32.0/go.mod h1:5eCOqeGphOyz6TsY3ZDNjE33SM/TFAK3RGuCL2naTgY= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= @@ -2931,6 +2949,8 @@ go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9 go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= go.opentelemetry.io/otel v1.17.0/go.mod h1:I2vmBGtFaODIVMBSTPVDlJSzBDNf93k60E6Ft0nyjo0= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= @@ -2955,6 +2975,8 @@ go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOa go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s= go.opentelemetry.io/otel/metric v1.17.0/go.mod h1:h4skoxdZI17AxwITdmdZjjYJQH5nzijUUjm+wtPph5o= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= @@ -2963,6 +2985,7 @@ go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe3 go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= @@ -2975,6 +2998,8 @@ go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/A go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7aHeaz0ZBLWjY= go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= @@ -3083,8 +3108,8 @@ golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIi golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -3100,8 +3125,9 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20230307190834-24139beb5833 h1:SChBja7BCQewoTAU7IgvucQKMIXrEpFxNMs0spT3/5s= golang.org/x/exp v0.0.0-20230307190834-24139beb5833/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -3253,8 +3279,8 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -3291,8 +3317,9 @@ golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4 golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= -golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -3312,8 +3339,9 @@ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -3482,8 +3510,8 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -3502,8 +3530,8 @@ golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -3539,8 +3567,9 @@ golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -3743,8 +3772,9 @@ google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2 google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= -google.golang.org/api v0.147.0 h1:Can3FaQo9LlVqxJCodNmeZW/ib3/qKAY3rFeXiHo5gc= google.golang.org/api v0.147.0/go.mod h1:pQ/9j83DcmPd/5C9e2nFOdjjNkDZ1G+zkbK2uvdkJMs= +google.golang.org/api v0.156.0 h1:yloYcGbBtVYjLKQe4enCunxvwn3s2w/XPrrhVf6MsvQ= +google.golang.org/api v0.156.0/go.mod h1:bUSmn4KFO0Q+69zo9CNIDp4Psi6BqM0np0CbzKRSiSY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -3752,8 +3782,9 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -3918,8 +3949,9 @@ google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02 google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:CCviP9RmpZ1mxVr8MUjCnSiY09IbAXZxhLE6EhHIdPU= -google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= +google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 h1:YJ5pD9rF8o9Qtta0Cmy9rdBwkSjrTCT6XTiUQVOtIos= +google.golang.org/genproto v0.0.0-20231212172506-995d672761c0/go.mod h1:l/k7rMz0vFTBPy+tFSGvXEd3z+BcoG1k7EHbqm+YBsY= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= @@ -3931,8 +3963,9 @@ google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go. google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:RdyHbowztCGQySiCvQPgWQWgWhGnouTdCflKoDBt32U= -google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU= google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= +google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 h1:s1w3X6gQxwrLEpxnLd/qXTVLgQE2yXwaOaoa6IlY/+o= +google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0/go.mod h1:CAny0tYF+0/9rmDB9fahA9YLzX3+AEVl1qXbv5hhj6c= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/bytestream v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:itlFWGBbEyD32PUeJsTG8h8Wz7iJXfVK4gt1EJ+pAG0= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= @@ -3947,8 +3980,9 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/genproto/googleapis/rpc v0.0.0-20230920183334-c177e329c48b/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -4003,8 +4037,9 @@ google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpX google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20230224211313-3775f633ce20/go.mod h1:Nr5H8+MlGWr5+xX/STzdoEqJrO+YteqFbMyCsrb6mH0= @@ -4026,8 +4061,9 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml index f8f12b83b82..79277c2a41c 100644 --- a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml +++ b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml @@ -828,9 +828,9 @@ subjects: apiVersion: v1 data: # References: - # * https://github.com/argoproj/argo-workflows/blob/v3.4.16/config/config.go - # * https://github.com/argoproj/argo-workflows/blob/v3.4.16/docs/workflow-controller-configmap.md - # * https://github.com/argoproj/argo-workflows/blob/v3.4.16/docs/workflow-controller-configmap.yaml + # * https://github.com/argoproj/argo-workflows/blob/v3.4.17/config/config.go + # * https://github.com/argoproj/argo-workflows/blob/v3.4.17/docs/workflow-controller-configmap.md + # * https://github.com/argoproj/argo-workflows/blob/v3.4.17/docs/workflow-controller-configmap.yaml # Choice of an executor is deprecated in favor of emissary executor executor: | diff --git a/manifests/kustomize/third-party/argo/base/workflow-controller-configmap-patch.yaml b/manifests/kustomize/third-party/argo/base/workflow-controller-configmap-patch.yaml index 21a08fcb64c..2b9b0773180 100644 --- a/manifests/kustomize/third-party/argo/base/workflow-controller-configmap-patch.yaml +++ b/manifests/kustomize/third-party/argo/base/workflow-controller-configmap-patch.yaml @@ -4,9 +4,9 @@ metadata: name: workflow-controller-configmap data: # References: - # * https://github.com/argoproj/argo-workflows/blob/v3.4.16/config/config.go - # * https://github.com/argoproj/argo-workflows/blob/v3.4.16/docs/workflow-controller-configmap.md - # * https://github.com/argoproj/argo-workflows/blob/v3.4.16/docs/workflow-controller-configmap.yaml + # * https://github.com/argoproj/argo-workflows/blob/v3.4.17/config/config.go + # * https://github.com/argoproj/argo-workflows/blob/v3.4.17/docs/workflow-controller-configmap.md + # * https://github.com/argoproj/argo-workflows/blob/v3.4.17/docs/workflow-controller-configmap.yaml # In artifactRepository.s3.endpoint, $(kfp-namespace) is needed, because in multi-user mode, pipelines may run in other namespaces. artifactRepository: | diff --git a/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml b/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml index 7b0b5db7601..c221a642023 100644 --- a/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml +++ b/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml @@ -7,12 +7,12 @@ spec: spec: containers: - name: workflow-controller - image: gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance + image: gcr.io/ml-pipeline/workflow-controller:v3.4.17-license-compliance args: - --configmap - workflow-controller-configmap - --executor-image - - gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance + - gcr.io/ml-pipeline/argoexec:v3.4.17-license-compliance resources: requests: cpu: 100m diff --git a/manifests/kustomize/third-party/argo/upstream/manifests/Kptfile b/manifests/kustomize/third-party/argo/upstream/manifests/Kptfile index d7222e19d5c..237910f8223 100644 --- a/manifests/kustomize/third-party/argo/upstream/manifests/Kptfile +++ b/manifests/kustomize/third-party/argo/upstream/manifests/Kptfile @@ -7,12 +7,12 @@ upstream: git: repo: https://github.com/argoproj/argo-workflows directory: /manifests - ref: v3.4.16 + ref: v3.4.17 updateStrategy: resource-merge upstreamLock: type: git git: repo: https://github.com/argoproj/argo-workflows directory: /manifests - ref: v3.4.16 - commit: 910a9aabce5de6568b54350c181a431f8263605a + ref: v3.4.17 + commit: 89cbdb53361cbe59fbe81b887ee82722cce5de54 diff --git a/manifests/kustomize/third-party/argo/upstream/manifests/namespace-install/kustomization.yaml b/manifests/kustomize/third-party/argo/upstream/manifests/namespace-install/kustomization.yaml index 4692dbd32d2..a3c7fe6fbcf 100644 --- a/manifests/kustomize/third-party/argo/upstream/manifests/namespace-install/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/upstream/manifests/namespace-install/kustomization.yaml @@ -4,16 +4,16 @@ resources: - ../base - ./argo-server-rbac - ./workflow-controller-rbac -patches: - - path: ./overlays/workflow-controller-deployment.yaml - target: +patchesJson6902: + - target: + version: v1 group: apps kind: Deployment name: workflow-controller + path: ./overlays/workflow-controller-deployment.yaml + - target: version: v1 - - path: ./overlays/argo-server-deployment.yaml - target: group: apps kind: Deployment name: argo-server - version: v1 + path: ./overlays/argo-server-deployment.yaml diff --git a/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/base/kustomization.yaml b/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/base/kustomization.yaml index 23f16528426..b03beac7164 100644 --- a/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/base/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/base/kustomization.yaml @@ -19,6 +19,6 @@ resources: - artifactgc-default-rolebinding.yaml - cluster-workflow-template-rbac.yaml - artifact-repositories-configmap.yaml -patches: - - path: overlays/workflow-controller-configmap.yaml - - path: overlays/argo-server-deployment.yaml +patchesStrategicMerge: + - overlays/workflow-controller-configmap.yaml + - overlays/argo-server-deployment.yaml diff --git a/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/minimal/kustomization.yaml b/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/minimal/kustomization.yaml index 00b4d98f3cf..b376c091eba 100644 --- a/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/minimal/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/minimal/kustomization.yaml @@ -2,5 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../base -patches: - - path: overlays/workflow-controller-configmap.yaml +patchesStrategicMerge: + - overlays/workflow-controller-configmap.yaml diff --git a/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/mysql/kustomization.yaml b/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/mysql/kustomization.yaml index cf0cdb12f47..edacf51ff4a 100644 --- a/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/mysql/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/mysql/kustomization.yaml @@ -5,5 +5,5 @@ resources: - argo-mysql-config-secret.yaml - mysql-deployment.yaml - mysql-service.yaml -patches: - - path: overlays/workflow-controller-configmap.yaml +patchesStrategicMerge: + - overlays/workflow-controller-configmap.yaml diff --git a/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/postgres/kustomization.yaml b/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/postgres/kustomization.yaml index 531c0291dae..a70a0cc26b3 100644 --- a/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/postgres/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/postgres/kustomization.yaml @@ -5,5 +5,5 @@ resources: - argo-postgres-config-secret.yaml - postgres-deployment.yaml - postgres-service.yaml -patches: - - path: overlays/workflow-controller-configmap.yaml +patchesStrategicMerge: + - overlays/workflow-controller-configmap.yaml diff --git a/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/sso/kustomization.yaml b/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/sso/kustomization.yaml index ce3d3aa8e85..70aafea6549 100644 --- a/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/sso/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/upstream/manifests/quick-start/sso/kustomization.yaml @@ -3,6 +3,6 @@ kind: Kustomization resources: - ../base - dex -patches: - - path: overlays/workflow-controller-configmap.yaml - - path: overlays/argo-server-sa.yaml +patchesStrategicMerge: + - overlays/workflow-controller-configmap.yaml + - overlays/argo-server-sa.yaml diff --git a/test/install-argo-cli.sh b/test/install-argo-cli.sh index 8188e4f9ea0..fab820e8970 100755 --- a/test/install-argo-cli.sh +++ b/test/install-argo-cli.sh @@ -19,7 +19,7 @@ set -ex DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" REPO_ROOT="${DIR}/.." ARGO_VERSION="$(cat ${REPO_ROOT}/third_party/argo/VERSION)" -# ARGO_VERSION=v3.4.16 +# ARGO_VERSION=v3.4.17 OS=${OS:-"linux-amd64"} # if argo is not installed diff --git a/test/tag_for_hosted.sh b/test/tag_for_hosted.sh index 47f256848f4..b919c58c82b 100755 --- a/test/tag_for_hosted.sh +++ b/test/tag_for_hosted.sh @@ -120,12 +120,12 @@ docker tag gcr.io/cloudsql-docker/gce-proxy:1.25.0 gcr.io/$PROJECT_ID/hosted/$CO docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cloudsqlproxy:$SEM_VER docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cloudsqlproxy:$MM_VER -docker tag gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER -docker tag gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER +docker tag gcr.io/ml-pipeline/argoexec:v3.4.17-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER +docker tag gcr.io/ml-pipeline/argoexec:v3.4.17-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER -docker tag gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER -docker tag gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER +docker tag gcr.io/ml-pipeline/workflow-controller:v3.4.17-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER +docker tag gcr.io/ml-pipeline/workflow-controller:v3.4.17-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER diff --git a/third_party/argo/Dockerfile.argoexec b/third_party/argo/Dockerfile.argoexec index ce0d0c02d84..d764072d1fa 100644 --- a/third_party/argo/Dockerfile.argoexec +++ b/third_party/argo/Dockerfile.argoexec @@ -15,7 +15,7 @@ ARG TAG FROM docker.io/argoproj/argoexec:${TAG} # Use the following path when we need to fork temporarily. -# FROM gcr.io/ml-pipeline-test/argoexec:v3.4.16 +# FROM gcr.io/ml-pipeline-test/argoexec:v3.4.17 # Copy notices, licenses and source code. COPY NOTICES/argoexec /NOTICES diff --git a/third_party/argo/Dockerfile.workflow-controller b/third_party/argo/Dockerfile.workflow-controller index 400ed9161fb..8aef971cfc8 100644 --- a/third_party/argo/Dockerfile.workflow-controller +++ b/third_party/argo/Dockerfile.workflow-controller @@ -15,7 +15,7 @@ ARG TAG FROM docker.io/argoproj/workflow-controller:${TAG} # Use the following path when we need to fork temporarily. -# FROM gcr.io/ml-pipeline-test/workflow-controller:v3.4.16 +# FROM gcr.io/ml-pipeline-test/workflow-controller:v3.4.17 # Copy notices, licenses and source code. COPY NOTICES/workflow-controller /NOTICES diff --git a/third_party/argo/README.md b/third_party/argo/README.md index 2b0649da5a6..702c892e4bd 100644 --- a/third_party/argo/README.md +++ b/third_party/argo/README.md @@ -21,7 +21,7 @@ Instructions: 1. Set version of argo you want to upgrade to, for example: ```bash - ARGO_TAG=v3.4.16 + ARGO_TAG=v3.4.17 ``` 1. ```bash diff --git a/third_party/argo/VERSION b/third_party/argo/VERSION index b2db0a1b893..5b2e1e73407 100644 --- a/third_party/argo/VERSION +++ b/third_party/argo/VERSION @@ -1 +1 @@ -v3.4.16 +v3.4.17 diff --git a/third_party/argo/licenses-argoexec.csv b/third_party/argo/licenses-argoexec.csv index aa2701f55da..0841e4a3b30 100644 --- a/third_party/argo/licenses-argoexec.csv +++ b/third_party/argo/licenses-argoexec.csv @@ -1,14 +1,14 @@ cloud.google.com/go/compute/metadata,https://github.com/googleapis/google-cloud-go/blob/compute/metadata/v0.2.3/compute/metadata/LICENSE,Apache-2.0 -cloud.google.com/go/iam,https://github.com/googleapis/google-cloud-go/blob/iam/v1.1.1/iam/LICENSE,Apache-2.0 -cloud.google.com/go/internal,https://github.com/googleapis/google-cloud-go/blob/v0.110.7/LICENSE,Apache-2.0 +cloud.google.com/go/iam,https://github.com/googleapis/google-cloud-go/blob/iam/v1.1.5/iam/LICENSE,Apache-2.0 +cloud.google.com/go/internal,https://github.com/googleapis/google-cloud-go/blob/v0.111.0/LICENSE,Apache-2.0 cloud.google.com/go/storage,https://github.com/googleapis/google-cloud-go/blob/storage/v1.30.1/storage/LICENSE,Apache-2.0 dario.cat/mergo,https://github.com/imdario/mergo/blob/v1.0.0/LICENSE,BSD-3-Clause -github.com/Azure/azure-sdk-for-go/sdk/azcore,https://github.com/Azure/azure-sdk-for-go/blob/sdk/azcore/v1.6.0/sdk/azcore/LICENSE.txt,MIT +github.com/Azure/azure-sdk-for-go/sdk/azcore,https://github.com/Azure/azure-sdk-for-go/blob/sdk/azcore/v1.7.1/sdk/azcore/LICENSE.txt,MIT github.com/Azure/azure-sdk-for-go/sdk/azidentity,https://github.com/Azure/azure-sdk-for-go/blob/sdk/azidentity/v1.2.2/sdk/azidentity/LICENSE.txt,MIT github.com/Azure/azure-sdk-for-go/sdk/internal,https://github.com/Azure/azure-sdk-for-go/blob/sdk/internal/v1.3.0/sdk/internal/LICENSE.txt,MIT github.com/Azure/azure-sdk-for-go/sdk/storage/azblob,https://github.com/Azure/azure-sdk-for-go/blob/sdk/storage/azblob/v0.4.1/sdk/storage/azblob/LICENSE.txt,MIT -github.com/Azure/go-autorest/autorest,https://github.com/Azure/go-autorest/blob/autorest/v0.11.24/autorest/LICENSE,Apache-2.0 -github.com/Azure/go-autorest/autorest/adal,https://github.com/Azure/go-autorest/blob/autorest/adal/v0.9.18/autorest/adal/LICENSE,Apache-2.0 +github.com/Azure/go-autorest/autorest,https://github.com/Azure/go-autorest/blob/autorest/v0.11.27/autorest/LICENSE,Apache-2.0 +github.com/Azure/go-autorest/autorest/adal,https://github.com/Azure/go-autorest/blob/autorest/adal/v0.9.20/autorest/adal/LICENSE,Apache-2.0 github.com/Azure/go-autorest/autorest/date,https://github.com/Azure/go-autorest/blob/autorest/date/v0.3.0/autorest/date/LICENSE,Apache-2.0 github.com/Azure/go-autorest/logger,https://github.com/Azure/go-autorest/blob/logger/v0.2.1/logger/LICENSE,Apache-2.0 github.com/Azure/go-autorest/tracing,https://github.com/Azure/go-autorest/blob/tracing/v0.6.0/tracing/LICENSE,Apache-2.0 @@ -18,17 +18,16 @@ github.com/Masterminds/goutils,https://github.com/Masterminds/goutils/blob/v1.1. github.com/Masterminds/semver/v3,https://github.com/Masterminds/semver/blob/v3.2.0/LICENSE.txt,MIT github.com/Masterminds/sprig/v3,https://github.com/Masterminds/sprig/blob/v3.2.3/LICENSE.txt,MIT github.com/ProtonMail/go-crypto,https://github.com/ProtonMail/go-crypto/blob/3c4c8a2d2371/LICENSE,BSD-3-Clause -github.com/aliyun/aliyun-oss-go-sdk/oss,https://github.com/aliyun/aliyun-oss-go-sdk/blob/v2.2.7/LICENSE,MIT -github.com/antonmedv/expr,https://github.com/antonmedv/expr/blob/v1.12.6/LICENSE,MIT +github.com/aliyun/aliyun-oss-go-sdk/oss,https://github.com/aliyun/aliyun-oss-go-sdk/blob/v3.0.2/LICENSE,MIT github.com/argoproj/argo-events/pkg,https://github.com/argoproj/argo-events/blob/v1.7.3/LICENSE,Apache-2.0 github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/HEAD/LICENSE,Apache-2.0 github.com/argoproj/pkg,https://github.com/argoproj/pkg/blob/v0.13.6/LICENSE,Apache-2.0 -github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.44.105/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go/internal/sync/singleflight,https://github.com/aws/aws-sdk-go/blob/v1.44.105/internal/sync/singleflight/LICENSE,BSD-3-Clause +github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.44.322/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go/internal/sync/singleflight,https://github.com/aws/aws-sdk-go/blob/v1.44.322/internal/sync/singleflight/LICENSE,BSD-3-Clause github.com/beorn7/perks/quantile,https://github.com/beorn7/perks/blob/v1.0.1/LICENSE,MIT github.com/cespare/xxhash/v2,https://github.com/cespare/xxhash/blob/v2.2.0/LICENSE.txt,MIT github.com/chai2010/gettext-go/gettext,https://github.com/chai2010/gettext-go/blob/c6fed771bfd5/LICENSE,BSD-3-Clause -github.com/cloudflare/circl,https://github.com/cloudflare/circl/blob/v1.3.3/LICENSE,BSD-3-Clause +github.com/cloudflare/circl,https://github.com/cloudflare/circl/blob/v1.3.7/LICENSE,BSD-3-Clause github.com/colinmarc/hdfs/v2,https://github.com/colinmarc/hdfs/blob/v2.4.0/LICENSE.txt,MIT github.com/coreos/go-oidc/v3/oidc,https://github.com/coreos/go-oidc/blob/v3.5.0/LICENSE,Apache-2.0 github.com/creack/pty,https://github.com/creack/pty/blob/v1.1.18/LICENSE,MIT @@ -39,34 +38,36 @@ github.com/docker/distribution,https://github.com/docker/distribution/blob/v2.8. github.com/doublerebel/bellows,https://github.com/doublerebel/bellows/blob/f177d92a03d3/LICENSE,MIT github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.10.0/LICENSE,MIT github.com/emirpasic/gods,https://github.com/emirpasic/gods/blob/v1.18.1/LICENSE,BSD-2-Clause -github.com/evanphx/json-patch,https://github.com/evanphx/json-patch/blob/v5.6.0/LICENSE,BSD-3-Clause +github.com/evanphx/json-patch,https://github.com/evanphx/json-patch/blob/v5.8.0/LICENSE,BSD-3-Clause github.com/exponent-io/jsonpath,https://github.com/exponent-io/jsonpath/blob/d6023ce2651d/LICENSE,MIT +github.com/expr-lang/expr,https://github.com/expr-lang/expr/blob/v1.16.0/LICENSE,MIT github.com/fatih/camelcase,https://github.com/fatih/camelcase/blob/v1.0.0/LICENSE.md,MIT -github.com/felixge/httpsnoop,https://github.com/felixge/httpsnoop/blob/v1.0.3/LICENSE.txt,MIT +github.com/felixge/httpsnoop,https://github.com/felixge/httpsnoop/blob/v1.0.4/LICENSE.txt,MIT github.com/fvbommel/sortorder,https://github.com/fvbommel/sortorder/blob/v1.0.1/LICENSE,MIT github.com/go-errors/errors,https://github.com/go-errors/errors/blob/v1.0.1/LICENSE.MIT,MIT github.com/go-git/gcfg,https://github.com/go-git/gcfg/blob/3a3c6141e376/LICENSE,BSD-3-Clause github.com/go-git/go-billy/v5,https://github.com/go-git/go-billy/blob/v5.5.0/LICENSE,Apache-2.0 github.com/go-git/go-git/v5,https://github.com/go-git/go-git/blob/v5.11.0/LICENSE,Apache-2.0 -github.com/go-jose/go-jose/v3,https://github.com/go-jose/go-jose/blob/v3.0.1/LICENSE,Apache-2.0 -github.com/go-jose/go-jose/v3/json,https://github.com/go-jose/go-jose/blob/v3.0.1/json/LICENSE,BSD-3-Clause -github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.3/LICENSE,Apache-2.0 -github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 -github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 -github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/go-jose/go-jose/v3,https://github.com/go-jose/go-jose/blob/v3.0.3/LICENSE,Apache-2.0 +github.com/go-jose/go-jose/v3/json,https://github.com/go-jose/go-jose/blob/v3.0.3/json/LICENSE,BSD-3-Clause +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.3.0/LICENSE,Apache-2.0 +github.com/go-logr/stdr,https://github.com/go-logr/stdr/blob/v1.2.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.4/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.6/LICENSE,Apache-2.0 github.com/go-sql-driver/mysql,https://github.com/go-sql-driver/mysql/blob/v1.7.1/LICENSE,MPL-2.0 github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause github.com/golang-jwt/jwt/v4,https://github.com/golang-jwt/jwt/blob/v4.5.0/LICENSE,MIT github.com/golang/groupcache/lru,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0 github.com/golang/protobuf,https://github.com/golang/protobuf/blob/v1.5.3/LICENSE,BSD-3-Clause github.com/google/btree,https://github.com/google/btree/blob/v1.0.1/LICENSE,Apache-2.0 -github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.5.7-v3refs/LICENSE,Apache-2.0 +github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.6.9/LICENSE,Apache-2.0 github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.6.0/LICENSE,BSD-3-Clause github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 github.com/google/s2a-go,https://github.com/google/s2a-go/blob/v0.1.7/LICENSE.md,Apache-2.0 github.com/google/shlex,https://github.com/google/shlex/blob/e7afc7fbc510/COPYING,Apache-2.0 -github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.1/LICENSE,BSD-3-Clause -github.com/googleapis/enterprise-certificate-proxy/client,https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.1/LICENSE,Apache-2.0 +github.com/google/uuid,https://github.com/google/uuid/blob/v1.5.0/LICENSE,BSD-3-Clause +github.com/googleapis/enterprise-certificate-proxy/client,https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.2/LICENSE,Apache-2.0 github.com/googleapis/gax-go/v2,https://github.com/googleapis/gax-go/blob/v2.12.0/v2/LICENSE,BSD-3-Clause github.com/gorilla/websocket,https://github.com/gorilla/websocket/blob/v1.5.0/LICENSE,BSD-2-Clause github.com/gregjones/httpcache,https://github.com/gregjones/httpcache/blob/901d90724c79/LICENSE.txt,MIT @@ -105,7 +106,7 @@ github.com/mitchellh/copystructure,https://github.com/mitchellh/copystructure/bl github.com/mitchellh/go-wordwrap,https://github.com/mitchellh/go-wordwrap/blob/v1.0.1/LICENSE.md,MIT github.com/mitchellh/reflectwalk,https://github.com/mitchellh/reflectwalk/blob/v1.0.2/LICENSE,MIT github.com/moby/spdystream,https://github.com/moby/spdystream/blob/v0.2.0/LICENSE,Apache-2.0 -github.com/moby/term,https://github.com/moby/term/blob/3f7ff695adc6/LICENSE,Apache-2.0 +github.com/moby/term,https://github.com/moby/term/blob/1aeaba878587/LICENSE,Apache-2.0 github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE,Apache-2.0 github.com/monochromegane/go-gitignore,https://github.com/monochromegane/go-gitignore/blob/205db1a8cc00/LICENSE,MIT @@ -132,7 +133,7 @@ github.com/shopspring/decimal,https://github.com/shopspring/decimal/blob/v1.2.0/ github.com/sirupsen/logrus,https://github.com/sirupsen/logrus/blob/v1.9.3/LICENSE,MIT github.com/skeema/knownhosts,https://github.com/skeema/knownhosts/blob/v1.2.1/LICENSE,Apache-2.0 github.com/spf13/cast,https://github.com/spf13/cast/blob/v1.5.0/LICENSE,MIT -github.com/spf13/cobra,https://github.com/spf13/cobra/blob/v1.5.0/LICENSE.txt,Apache-2.0 +github.com/spf13/cobra,https://github.com/spf13/cobra/blob/v1.7.0/LICENSE.txt,Apache-2.0 github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause github.com/stretchr/testify,https://github.com/stretchr/testify/blob/v1.8.4/LICENSE,MIT github.com/tidwall/gjson,https://github.com/tidwall/gjson/blob/v1.15.0/LICENSE,MIT @@ -143,24 +144,28 @@ github.com/valyala/fasttemplate,https://github.com/valyala/fasttemplate/blob/v1. github.com/xanzy/ssh-agent,https://github.com/xanzy/ssh-agent/blob/v0.3.3/LICENSE,Apache-2.0 github.com/xlab/treeprint,https://github.com/xlab/treeprint/blob/a009c3971eca/LICENSE,MIT go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0.24.0/LICENSE,Apache-2.0 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/google.golang.org/grpc/otelgrpc/v0.46.1/instrumentation/google.golang.org/grpc/otelgrpc/LICENSE,Apache-2.0 +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/net/http/otelhttp/v0.46.1/instrumentation/net/http/otelhttp/LICENSE,Apache-2.0 +go.opentelemetry.io/otel,https://github.com/open-telemetry/opentelemetry-go/blob/v1.21.0/LICENSE,Apache-2.0 +go.opentelemetry.io/otel/metric,https://github.com/open-telemetry/opentelemetry-go/blob/metric/v1.21.0/metric/LICENSE,Apache-2.0 +go.opentelemetry.io/otel/trace,https://github.com/open-telemetry/opentelemetry-go/blob/trace/v1.21.0/trace/LICENSE,Apache-2.0 go.starlark.net,https://github.com/google/starlark-go/blob/8dd3e2ee1dd5/LICENSE,BSD-3-Clause -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.16.0:LICENSE,BSD-3-Clause -golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/4a0574d9:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.19.0:LICENSE,BSD-3-Clause -golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.13.0:LICENSE,BSD-3-Clause -golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.3.0:LICENSE,BSD-3-Clause -golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.22.0:LICENSE,BSD-3-Clause +golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/92128663:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.23.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.16.0:LICENSE,BSD-3-Clause +golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.6.0:LICENSE,BSD-3-Clause +golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.19.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.19.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause -golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause -golang.org/x/xerrors,https://cs.opensource.google/go/x/xerrors/+/04be3eba:LICENSE,BSD-3-Clause -google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.143.0/LICENSE,BSD-3-Clause -google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.143.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause -google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/007df8e322eb/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/007df8e322eb/googleapis/api/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/e6e6cdab5c13/googleapis/rpc/LICENSE,Apache-2.0 -google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.58.3/LICENSE,Apache-2.0 -google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause +google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/LICENSE,BSD-3-Clause +google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.156.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause +google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/995d672761c0/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/995d672761c0/googleapis/api/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/50ed04b92917/googleapis/rpc/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.60.1/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/ini.v1,https://github.com/go-ini/ini/blob/v1.67.0/LICENSE,Apache-2.0 gopkg.in/warnings.v0,https://github.com/go-warnings/warnings/blob/v0.1.2/LICENSE,BSD-2-Clause @@ -175,14 +180,14 @@ k8s.io/client-go/third_party/forked/golang/template,https://github.com/kubernete k8s.io/component-base,https://github.com/kubernetes/component-base/blob/v0.24.3/LICENSE,Apache-2.0 k8s.io/component-helpers/auth/rbac,https://github.com/kubernetes/component-helpers/blob/v0.24.3/LICENSE,Apache-2.0 k8s.io/gengo,https://github.com/kubernetes/gengo/blob/397b4ae3bce7/LICENSE,Apache-2.0 -k8s.io/klog/v2,https://github.com/kubernetes/klog/blob/v2.60.1/LICENSE,Apache-2.0 +k8s.io/klog/v2,https://github.com/kubernetes/klog/blob/v2.70.1/LICENSE,Apache-2.0 k8s.io/kube-openapi/pkg,https://github.com/kubernetes/kube-openapi/blob/011e075b9cb8/LICENSE,Apache-2.0 k8s.io/kube-openapi/pkg/validation/spec,https://github.com/kubernetes/kube-openapi/blob/011e075b9cb8/pkg/validation/spec/LICENSE,Apache-2.0 k8s.io/kubectl/pkg,https://github.com/kubernetes/kubectl/blob/v0.24.3/LICENSE,Apache-2.0 k8s.io/metrics/pkg,https://github.com/kubernetes/metrics/blob/v0.24.3/LICENSE,Apache-2.0 -k8s.io/utils,https://github.com/kubernetes/utils/blob/3a6ce19ff2f9/LICENSE,Apache-2.0 -k8s.io/utils/internal/third_party/forked/golang,https://github.com/kubernetes/utils/blob/3a6ce19ff2f9/internal/third_party/forked/golang/LICENSE,BSD-3-Clause -sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/9f7c6b3444d2/LICENSE,Apache-2.0 +k8s.io/utils,https://github.com/kubernetes/utils/blob/56c0de1e6f5e/LICENSE,Apache-2.0 +k8s.io/utils/internal/third_party/forked/golang,https://github.com/kubernetes/utils/blob/56c0de1e6f5e/internal/third_party/forked/golang/LICENSE,BSD-3-Clause +sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/f223a00ba0e2/LICENSE,Apache-2.0 sigs.k8s.io/kustomize/api,https://github.com/kubernetes-sigs/kustomize/blob/api/v0.11.4/api/LICENSE,Apache-2.0 sigs.k8s.io/kustomize/kustomize/v4/commands/build,https://github.com/kubernetes-sigs/kustomize/blob/kustomize/v4.5.4/kustomize/LICENSE,Apache-2.0 sigs.k8s.io/kustomize/kyaml,https://github.com/kubernetes-sigs/kustomize/blob/kyaml/v0.13.6/kyaml/LICENSE,Apache-2.0 diff --git a/third_party/argo/licenses-workflow-controller.csv b/third_party/argo/licenses-workflow-controller.csv index 00ecad44c82..926b63e0528 100644 --- a/third_party/argo/licenses-workflow-controller.csv +++ b/third_party/argo/licenses-workflow-controller.csv @@ -1,7 +1,7 @@ cloud.google.com/go/compute/metadata,https://github.com/googleapis/google-cloud-go/blob/compute/metadata/v0.2.3/compute/metadata/LICENSE,Apache-2.0 -github.com/Azure/azure-sdk-for-go,https://github.com/Azure/azure-sdk-for-go/blob/v62.0.0/LICENSE.txt,MIT -github.com/Azure/go-autorest/autorest,https://github.com/Azure/go-autorest/blob/autorest/v0.11.24/autorest/LICENSE,Apache-2.0 -github.com/Azure/go-autorest/autorest/adal,https://github.com/Azure/go-autorest/blob/autorest/adal/v0.9.18/autorest/adal/LICENSE,Apache-2.0 +github.com/Azure/azure-sdk-for-go,https://github.com/Azure/azure-sdk-for-go/blob/v66.0.0/LICENSE.txt,MIT +github.com/Azure/go-autorest/autorest,https://github.com/Azure/go-autorest/blob/autorest/v0.11.27/autorest/LICENSE,Apache-2.0 +github.com/Azure/go-autorest/autorest/adal,https://github.com/Azure/go-autorest/blob/autorest/adal/v0.9.20/autorest/adal/LICENSE,Apache-2.0 github.com/Azure/go-autorest/autorest/azure/auth,https://github.com/Azure/go-autorest/blob/autorest/azure/auth/v0.5.11/autorest/azure/auth/LICENSE,Apache-2.0 github.com/Azure/go-autorest/autorest/azure/cli,https://github.com/Azure/go-autorest/blob/autorest/azure/cli/v0.4.5/autorest/azure/cli/LICENSE,Apache-2.0 github.com/Azure/go-autorest/autorest/date,https://github.com/Azure/go-autorest/blob/autorest/date/v0.3.0/autorest/date/LICENSE,Apache-2.0 @@ -11,60 +11,61 @@ github.com/Knetic/govaluate,https://github.com/Knetic/govaluate/blob/9aa49832a73 github.com/Masterminds/goutils,https://github.com/Masterminds/goutils/blob/v1.1.1/LICENSE.txt,Apache-2.0 github.com/Masterminds/semver/v3,https://github.com/Masterminds/semver/blob/v3.2.0/LICENSE.txt,MIT github.com/Masterminds/sprig/v3,https://github.com/Masterminds/sprig/blob/v3.2.3/LICENSE.txt,MIT -github.com/antonmedv/expr,https://github.com/antonmedv/expr/blob/v1.12.6/LICENSE,MIT github.com/argoproj/argo-events/pkg,https://github.com/argoproj/argo-events/blob/v1.7.3/LICENSE,Apache-2.0 github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/HEAD/LICENSE,Apache-2.0 github.com/argoproj/pkg,https://github.com/argoproj/pkg/blob/v0.13.6/LICENSE,Apache-2.0 -github.com/aws/aws-sdk-go-v2,https://github.com/aws/aws-sdk-go-v2/blob/v1.16.2/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/config,https://github.com/aws/aws-sdk-go-v2/blob/config/v1.15.3/config/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/credentials,https://github.com/aws/aws-sdk-go-v2/blob/credentials/v1.11.2/credentials/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/feature/ec2/imds,https://github.com/aws/aws-sdk-go-v2/blob/feature/ec2/imds/v1.12.3/feature/ec2/imds/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/internal/configsources,https://github.com/aws/aws-sdk-go-v2/blob/internal/configsources/v1.1.9/internal/configsources/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2,https://github.com/aws/aws-sdk-go-v2/blob/internal/endpoints/v2.4.3/internal/endpoints/v2/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/internal/ini,https://github.com/aws/aws-sdk-go-v2/blob/internal/ini/v1.3.10/internal/ini/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/internal/sync/singleflight,https://github.com/aws/aws-sdk-go-v2/blob/v1.16.2/internal/sync/singleflight/LICENSE,BSD-3-Clause -github.com/aws/aws-sdk-go-v2/service/ecr,https://github.com/aws/aws-sdk-go-v2/blob/service/ecr/v1.15.0/service/ecr/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/service/ecrpublic,https://github.com/aws/aws-sdk-go-v2/blob/service/ecrpublic/v1.12.0/service/ecrpublic/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url,https://github.com/aws/aws-sdk-go-v2/blob/service/internal/presigned-url/v1.9.3/service/internal/presigned-url/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/service/sso,https://github.com/aws/aws-sdk-go-v2/blob/service/sso/v1.11.3/service/sso/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go-v2/service/sts,https://github.com/aws/aws-sdk-go-v2/blob/service/sts/v1.16.3/service/sts/LICENSE.txt,Apache-2.0 -github.com/aws/smithy-go,https://github.com/aws/smithy-go/blob/v1.11.2/LICENSE,Apache-2.0 -github.com/awslabs/amazon-ecr-credential-helper/ecr-login,https://github.com/awslabs/amazon-ecr-credential-helper/blob/396b2034c795/ecr-login/LICENSE,Apache-2.0 +github.com/aws/aws-sdk-go-v2,https://github.com/aws/aws-sdk-go-v2/blob/v1.16.7/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/config,https://github.com/aws/aws-sdk-go-v2/blob/config/v1.15.14/config/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/credentials,https://github.com/aws/aws-sdk-go-v2/blob/credentials/v1.12.9/credentials/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/feature/ec2/imds,https://github.com/aws/aws-sdk-go-v2/blob/feature/ec2/imds/v1.12.8/feature/ec2/imds/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/internal/configsources,https://github.com/aws/aws-sdk-go-v2/blob/internal/configsources/v1.1.14/internal/configsources/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2,https://github.com/aws/aws-sdk-go-v2/blob/internal/endpoints/v2.4.8/internal/endpoints/v2/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/internal/ini,https://github.com/aws/aws-sdk-go-v2/blob/internal/ini/v1.3.15/internal/ini/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/internal/sync/singleflight,https://github.com/aws/aws-sdk-go-v2/blob/v1.16.7/internal/sync/singleflight/LICENSE,BSD-3-Clause +github.com/aws/aws-sdk-go-v2/service/ecr,https://github.com/aws/aws-sdk-go-v2/blob/service/ecr/v1.17.8/service/ecr/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/service/ecrpublic,https://github.com/aws/aws-sdk-go-v2/blob/service/ecrpublic/v1.13.8/service/ecrpublic/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url,https://github.com/aws/aws-sdk-go-v2/blob/service/internal/presigned-url/v1.9.8/service/internal/presigned-url/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/service/sso,https://github.com/aws/aws-sdk-go-v2/blob/service/sso/v1.11.12/service/sso/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go-v2/service/sts,https://github.com/aws/aws-sdk-go-v2/blob/service/sts/v1.16.9/service/sts/LICENSE.txt,Apache-2.0 +github.com/aws/smithy-go,https://github.com/aws/smithy-go/blob/v1.12.0/LICENSE,Apache-2.0 +github.com/awslabs/amazon-ecr-credential-helper/ecr-login,https://github.com/awslabs/amazon-ecr-credential-helper/blob/ce46abcd012b/ecr-login/LICENSE,Apache-2.0 github.com/beorn7/perks/quantile,https://github.com/beorn7/perks/blob/v1.0.1/LICENSE,MIT github.com/cespare/xxhash/v2,https://github.com/cespare/xxhash/blob/v2.2.0/LICENSE.txt,MIT -github.com/chrismellard/docker-credential-acr-env/pkg,https://github.com/chrismellard/docker-credential-acr-env/blob/fe33c00cee21/LICENSE,Apache-2.0 +github.com/chrismellard/docker-credential-acr-env/pkg,https://github.com/chrismellard/docker-credential-acr-env/blob/c57b701bfc08/LICENSE,Apache-2.0 github.com/colinmarc/hdfs/v2,https://github.com/colinmarc/hdfs/blob/v2.4.0/LICENSE.txt,MIT +github.com/containerd/stargz-snapshotter/estargz,https://github.com/containerd/stargz-snapshotter/blob/estargz/v0.14.3/estargz/LICENSE,Apache-2.0 github.com/coreos/go-oidc/v3/oidc,https://github.com/coreos/go-oidc/blob/v3.5.0/LICENSE,Apache-2.0 github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC github.com/dimchansky/utfbom,https://github.com/dimchansky/utfbom/blob/v1.1.1/LICENSE,Apache-2.0 -github.com/docker/cli/cli/config,https://github.com/docker/cli/blob/v20.10.17/LICENSE,Apache-2.0 +github.com/docker/cli/cli/config,https://github.com/docker/cli/blob/v24.0.7/LICENSE,Apache-2.0 github.com/docker/distribution/registry/client/auth/challenge,https://github.com/docker/distribution/blob/v2.8.2/LICENSE,Apache-2.0 -github.com/docker/docker-credential-helpers,https://github.com/docker/docker-credential-helpers/blob/v0.6.4/LICENSE,MIT -github.com/docker/docker/pkg/homedir,https://github.com/docker/docker/blob/v20.10.24/LICENSE,Apache-2.0 +github.com/docker/docker-credential-helpers,https://github.com/docker/docker-credential-helpers/blob/v0.7.0/LICENSE,MIT +github.com/docker/docker/pkg/homedir,https://github.com/docker/docker/blob/v24.0.9/LICENSE,Apache-2.0 github.com/doublerebel/bellows,https://github.com/doublerebel/bellows/blob/f177d92a03d3/LICENSE,MIT github.com/dustin/go-humanize,https://github.com/dustin/go-humanize/blob/v1.0.1/LICENSE,MIT github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.10.0/LICENSE,MIT -github.com/evanphx/json-patch,https://github.com/evanphx/json-patch/blob/v5.6.0/LICENSE,BSD-3-Clause -github.com/felixge/httpsnoop,https://github.com/felixge/httpsnoop/blob/v1.0.3/LICENSE.txt,MIT +github.com/evanphx/json-patch,https://github.com/evanphx/json-patch/blob/v5.8.0/LICENSE,BSD-3-Clause +github.com/expr-lang/expr,https://github.com/expr-lang/expr/blob/v1.16.0/LICENSE,MIT +github.com/felixge/httpsnoop,https://github.com/felixge/httpsnoop/blob/v1.0.4/LICENSE.txt,MIT github.com/fsnotify/fsnotify,https://github.com/fsnotify/fsnotify/blob/v1.6.0/LICENSE,BSD-3-Clause -github.com/go-jose/go-jose/v3,https://github.com/go-jose/go-jose/blob/v3.0.1/LICENSE,Apache-2.0 -github.com/go-jose/go-jose/v3/json,https://github.com/go-jose/go-jose/blob/v3.0.1/json/LICENSE,BSD-3-Clause -github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.3/LICENSE,Apache-2.0 -github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 -github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 -github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/go-jose/go-jose/v3,https://github.com/go-jose/go-jose/blob/v3.0.3/LICENSE,Apache-2.0 +github.com/go-jose/go-jose/v3/json,https://github.com/go-jose/go-jose/blob/v3.0.3/json/LICENSE,BSD-3-Clause +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.3.0/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.4/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.6/LICENSE,Apache-2.0 github.com/go-sql-driver/mysql,https://github.com/go-sql-driver/mysql/blob/v1.7.1/LICENSE,MPL-2.0 github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause github.com/golang-jwt/jwt/v4,https://github.com/golang-jwt/jwt/blob/v4.5.0/LICENSE,MIT github.com/golang/groupcache/lru,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0 github.com/golang/protobuf,https://github.com/golang/protobuf/blob/v1.5.3/LICENSE,BSD-3-Clause -github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.5.7-v3refs/LICENSE,Apache-2.0 +github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.6.9/LICENSE,Apache-2.0 github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.6.0/LICENSE,BSD-3-Clause -github.com/google/go-containerregistry,https://github.com/google/go-containerregistry/blob/v0.11.0/LICENSE,Apache-2.0 -github.com/google/go-containerregistry/pkg/authn/k8schain,https://github.com/google/go-containerregistry/blob/2042cc9d6401/pkg/authn/k8schain/LICENSE,Apache-2.0 -github.com/google/go-containerregistry/pkg/authn/kubernetes,https://github.com/google/go-containerregistry/blob/bfe2ffc6b6bd/pkg/authn/kubernetes/LICENSE,Apache-2.0 +github.com/google/go-containerregistry,https://github.com/google/go-containerregistry/blob/v0.16.1/LICENSE,Apache-2.0 +github.com/google/go-containerregistry/pkg/authn/k8schain,https://github.com/google/go-containerregistry/blob/31786c6cbb82/pkg/authn/k8schain/LICENSE,Apache-2.0 +github.com/google/go-containerregistry/pkg/authn/kubernetes,https://github.com/google/go-containerregistry/blob/f79ec2192282/pkg/authn/kubernetes/LICENSE,Apache-2.0 github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 -github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.1/LICENSE,BSD-3-Clause +github.com/google/uuid,https://github.com/google/uuid/blob/v1.5.0/LICENSE,BSD-3-Clause github.com/gorilla/websocket,https://github.com/gorilla/websocket/blob/v1.5.0/LICENSE,BSD-2-Clause github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause github.com/hashicorp/go-uuid,https://github.com/hashicorp/go-uuid/blob/v1.0.3/LICENSE,MPL-2.0 @@ -80,7 +81,9 @@ github.com/jcmturner/rpc/v2,https://github.com/jcmturner/rpc/blob/v2.0.3/v2/LICE github.com/jmespath/go-jmespath,https://github.com/jmespath/go-jmespath/blob/v0.4.0/LICENSE,Apache-2.0 github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT -github.com/klauspost/compress/flate,https://github.com/klauspost/compress/blob/v1.16.7/LICENSE,Apache-2.0 +github.com/klauspost/compress,https://github.com/klauspost/compress/blob/v1.16.7/LICENSE,Apache-2.0 +github.com/klauspost/compress/internal/snapref,https://github.com/klauspost/compress/blob/v1.16.7/internal/snapref/LICENSE,BSD-3-Clause +github.com/klauspost/compress/zstd/internal/xxhash,https://github.com/klauspost/compress/blob/v1.16.7/zstd/internal/xxhash/LICENSE.txt,MIT github.com/klauspost/pgzip,https://github.com/klauspost/pgzip/blob/v1.2.6/LICENSE,MIT github.com/lib/pq,https://github.com/lib/pq/blob/v1.10.4/LICENSE.md,MIT github.com/magiconair/properties,https://github.com/magiconair/properties/blob/v1.8.7/LICENSE.md,BSD-2-Clause @@ -96,7 +99,7 @@ github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.2/ github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE,BSD-3-Clause github.com/oliveagle/jsonpath,https://github.com/oliveagle/jsonpath/blob/2e52cf6e6852/LICENSE,MIT github.com/opencontainers/go-digest,https://github.com/opencontainers/go-digest/blob/v1.0.0/LICENSE,Apache-2.0 -github.com/opencontainers/image-spec/specs-go,https://github.com/opencontainers/image-spec/blob/8b9d41f48198/LICENSE,Apache-2.0 +github.com/opencontainers/image-spec/specs-go,https://github.com/opencontainers/image-spec/blob/v1.1.0-rc3/LICENSE,Apache-2.0 github.com/pelletier/go-toml/v2,https://github.com/pelletier/go-toml/blob/v2.0.6/LICENSE,MIT github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.16.0/LICENSE,Apache-2.0 @@ -109,27 +112,28 @@ github.com/shopspring/decimal,https://github.com/shopspring/decimal/blob/v1.2.0/ github.com/sirupsen/logrus,https://github.com/sirupsen/logrus/blob/v1.9.3/LICENSE,MIT github.com/spf13/afero,https://github.com/spf13/afero/blob/v1.9.3/LICENSE.txt,Apache-2.0 github.com/spf13/cast,https://github.com/spf13/cast/blob/v1.5.0/LICENSE,MIT -github.com/spf13/cobra,https://github.com/spf13/cobra/blob/v1.5.0/LICENSE.txt,Apache-2.0 +github.com/spf13/cobra,https://github.com/spf13/cobra/blob/v1.7.0/LICENSE.txt,Apache-2.0 github.com/spf13/jwalterweatherman,https://github.com/spf13/jwalterweatherman/blob/v1.1.0/LICENSE,MIT github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause github.com/spf13/viper,https://github.com/spf13/viper/blob/v1.15.0/LICENSE,MIT github.com/subosito/gotenv,https://github.com/subosito/gotenv/blob/v1.4.2/LICENSE,MIT github.com/valyala/bytebufferpool,https://github.com/valyala/bytebufferpool/blob/v1.0.0/LICENSE,MIT github.com/valyala/fasttemplate,https://github.com/valyala/fasttemplate/blob/v1.2.2/LICENSE,MIT -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.16.0:LICENSE,BSD-3-Clause -golang.org/x/exp,https://cs.opensource.google/go/x/exp/+/4a0574d9:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.19.0:LICENSE,BSD-3-Clause -golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.13.0:LICENSE,BSD-3-Clause -golang.org/x/sync,https://cs.opensource.google/go/x/sync/+/v0.3.0:LICENSE,BSD-3-Clause -golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause +github.com/vbatts/tar-split/archive/tar,https://github.com/vbatts/tar-split/blob/v0.11.3/LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.22.0:LICENSE,BSD-3-Clause +golang.org/x/exp,https://cs.opensource.google/go/x/exp/+/92128663:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.23.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.16.0:LICENSE,BSD-3-Clause +golang.org/x/sync,https://cs.opensource.google/go/x/sync/+/v0.6.0:LICENSE,BSD-3-Clause +golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.19.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.19.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause -golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause -google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/007df8e322eb/googleapis/api/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/rpc/status,https://github.com/googleapis/go-genproto/blob/e6e6cdab5c13/googleapis/rpc/LICENSE,Apache-2.0 -google.golang.org/genproto/protobuf/field_mask,https://github.com/googleapis/go-genproto/blob/007df8e322eb/LICENSE,Apache-2.0 -google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.58.3/LICENSE,Apache-2.0 -google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause +google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/995d672761c0/googleapis/api/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/rpc/status,https://github.com/googleapis/go-genproto/blob/50ed04b92917/googleapis/rpc/LICENSE,Apache-2.0 +google.golang.org/genproto/protobuf/field_mask,https://github.com/googleapis/go-genproto/blob/995d672761c0/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.60.1/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/ini.v1,https://github.com/go-ini/ini/blob/v1.67.0/LICENSE,Apache-2.0 gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 @@ -139,12 +143,12 @@ k8s.io/apimachinery/pkg,https://github.com/kubernetes/apimachinery/blob/v0.24.3/ k8s.io/apimachinery/third_party/forked/golang,https://github.com/kubernetes/apimachinery/blob/v0.24.3/third_party/forked/golang/LICENSE,BSD-3-Clause k8s.io/client-go,https://github.com/kubernetes/client-go/blob/v0.24.3/LICENSE,Apache-2.0 k8s.io/client-go/third_party/forked/golang/template,https://github.com/kubernetes/client-go/blob/v0.24.3/third_party/forked/golang/LICENSE,BSD-3-Clause -k8s.io/klog/v2,https://github.com/kubernetes/klog/blob/v2.60.1/LICENSE,Apache-2.0 +k8s.io/klog/v2,https://github.com/kubernetes/klog/blob/v2.70.1/LICENSE,Apache-2.0 k8s.io/kube-openapi/pkg,https://github.com/kubernetes/kube-openapi/blob/011e075b9cb8/LICENSE,Apache-2.0 k8s.io/kube-openapi/pkg/validation/spec,https://github.com/kubernetes/kube-openapi/blob/011e075b9cb8/pkg/validation/spec/LICENSE,Apache-2.0 -k8s.io/utils,https://github.com/kubernetes/utils/blob/3a6ce19ff2f9/LICENSE,Apache-2.0 -k8s.io/utils/internal/third_party/forked/golang,https://github.com/kubernetes/utils/blob/3a6ce19ff2f9/internal/third_party/forked/golang/LICENSE,BSD-3-Clause -sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/9f7c6b3444d2/LICENSE,Apache-2.0 +k8s.io/utils,https://github.com/kubernetes/utils/blob/56c0de1e6f5e/LICENSE,Apache-2.0 +k8s.io/utils/internal/third_party/forked/golang,https://github.com/kubernetes/utils/blob/56c0de1e6f5e/internal/third_party/forked/golang/LICENSE,BSD-3-Clause +sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/f223a00ba0e2/LICENSE,Apache-2.0 sigs.k8s.io/structured-merge-diff/v4,https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.2.1/LICENSE,Apache-2.0 sigs.k8s.io/yaml,https://github.com/kubernetes-sigs/yaml/blob/v1.3.0/LICENSE,MIT upper.io/db.v3,https://github.com/upper/db/blob/v3.8.0/LICENSE,MIT From 90df1c467ee0cbd66536d84cbbfbe437efad3a36 Mon Sep 17 00:00:00 2001 From: Vani Haripriya Mudadla Date: Wed, 14 Aug 2024 13:26:11 -0500 Subject: [PATCH 10/54] test: Moved kubeflow-pipelines-manifests to GitHub Actions (#11066) Signed-off-by: vmudadla Signed-off-by: KevinGrantLee --- .../kubeflow-pipelines-manifests.yml | 19 +++++++++++++ manifests/kustomize/hack/presubmit.sh | 28 ++++++++++++------- 2 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/kubeflow-pipelines-manifests.yml diff --git a/.github/workflows/kubeflow-pipelines-manifests.yml b/.github/workflows/kubeflow-pipelines-manifests.yml new file mode 100644 index 00000000000..d494e2e0d0a --- /dev/null +++ b/.github/workflows/kubeflow-pipelines-manifests.yml @@ -0,0 +1,19 @@ +name: KFP Manifests + +on: + push: + branches: [master] + pull_request: + paths: + - '.github/workflows/kubeflow-pipelines-manifests.yml' + - 'manifests/kustomize/**' + +jobs: + kubeflow-pipelines-manifests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run tests + run: ./manifests/kustomize/hack/presubmit.sh diff --git a/manifests/kustomize/hack/presubmit.sh b/manifests/kustomize/hack/presubmit.sh index e4334f980b1..ccdba703e3a 100755 --- a/manifests/kustomize/hack/presubmit.sh +++ b/manifests/kustomize/hack/presubmit.sh @@ -21,24 +21,32 @@ set -ex DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" TMP="$(mktemp -d)" +# Add TMP to PATH +PATH="$TMP:$PATH" + pushd "${TMP}" -# Install Kustomize + +# Install kustomize KUSTOMIZE_VERSION=5.2.1 -# Reference: https://kubectl.docs.kubernetes.io/installation/kustomize/binaries/ -curl -s -O "https://raw.githubusercontent.com/\ -kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" -chmod +x install_kustomize.sh -./install_kustomize.sh "${KUSTOMIZE_VERSION}" /usr/local/bin/ +# Reference: https://github.com/kubernetes-sigs/kustomize/releases/tag/kustomize%2Fv5.2.1 +curl -s -LO "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" +tar -xzf kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz +chmod +x kustomize +# Install yq # Reference: https://github.com/mikefarah/yq/releases/tag/3.4.1 curl -s -LO "https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64" chmod +x yq_linux_amd64 -mv yq_linux_amd64 /usr/local/bin/yq +mv yq_linux_amd64 yq + +# Install kpt +KPT_VERSION=1.0.0-beta.54 +# Reference: https://github.com/kptdev/kpt/releases/tag/v1.0.0-beta.54 +curl -s -LO "https://github.com/kptdev/kpt/releases/download/v${KPT_VERSION}/kpt_linux_amd64" +chmod +x kpt_linux_amd64 +mv kpt_linux_amd64 kpt popd -# kpt and kubectl should already be installed in gcr.io/google.com/cloudsdktool/cloud-sdk:latest -# so we do not need to install them here - # trigger real unit tests ${DIR}/test.sh # verify release script runs properly From 79becefe2e4c17f39910cbb322296dbb77579a86 Mon Sep 17 00:00:00 2001 From: Liav Weiss <74174727+liavweiss@users.noreply.github.com> Date: Thu, 15 Aug 2024 14:39:27 +0300 Subject: [PATCH 11/54] fix: re-enable exit hanler test. (#11100) Signed-off-by: Liav Weiss (EXT-Nokia) Co-authored-by: Liav Weiss (EXT-Nokia) Signed-off-by: KevinGrantLee --- .github/workflows/e2e-test.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 6b5afffb337..daef2ae49a7 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -153,9 +153,8 @@ jobs: - name: Basic sample tests - sequential run: pip3 install -r ./test/sample-test/requirements.txt && pip3 install kfp~=2.0 && python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name sequential --results-gcs-dir output -# Disabled while https://github.com/kubeflow/pipelines/issues/10885 is not resolved -# - name: Basic sample tests - exit_handler -# run: pip3 install -r ./test/sample-test/requirements.txt && pip3 install kfp~=2.0 && python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name exit_handler --results-gcs-dir output + - name: Basic sample tests - exit_handler + run: pip3 install -r ./test/sample-test/requirements.txt && pip3 install kfp~=2.0 && python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name exit_handler --results-gcs-dir output - name: Collect test results if: always() From 12aae02d7490975e37c701c8e08afc9ae26fc3a5 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 15 Aug 2024 15:39:27 -0400 Subject: [PATCH 12/54] fix(frontend): retrieve archived logs from correct location (#11010) * fix(frontend): retrieve archived logs from correct location Signed-off-by: droctothorpe Co-authored-by: andreafehrman Co-authored-by: owmasch * Add namespace tag handling and validation Signed-off-by: droctothorpe Co-authored-by: andreafehrman Co-authored-by: owmasch * Remove whitespace from keyFormat Signed-off-by: droctothorpe Co-authored-by: andreafehrman Co-authored-by: owmasch * Update frontend unit tests Signed-off-by: droctothorpe * Remove superfluous log statements Signed-off-by: droctothorpe Co-authored-by: quinnovator * Add link to keyFormat in manifests Signed-off-by: droctothorpe * Fix workflow parsing for log artifact Signed-off-by: droctothorpe Co-authored-by: quinnovator * Fix unit test Signed-off-by: droctothorpe --------- Signed-off-by: droctothorpe Co-authored-by: andreafehrman Co-authored-by: owmasch Co-authored-by: quinnovator Signed-off-by: KevinGrantLee --- frontend/server/configs.ts | 11 +- frontend/server/handlers/pod-logs.ts | 13 +- frontend/server/workflow-helper.test.ts | 93 ++++++----- frontend/server/workflow-helper.ts | 152 ++++++++++++------ .../components/tabs/RuntimeNodeDetailsV2.tsx | 3 +- frontend/src/lib/Apis.test.ts | 19 ++- frontend/src/lib/Apis.ts | 10 +- frontend/src/pages/RunDetails.test.tsx | 2 + frontend/src/pages/RunDetails.tsx | 2 +- 9 files changed, 204 insertions(+), 101 deletions(-) diff --git a/frontend/server/configs.ts b/frontend/server/configs.ts index c2d3ef30c15..7108db23df9 100644 --- a/frontend/server/configs.ts +++ b/frontend/server/configs.ts @@ -91,8 +91,11 @@ export function loadConfigs(argv: string[], env: ProcessEnv): UIConfigs { ARGO_ARCHIVE_ARTIFACTORY = 'minio', /** Bucket to retrive logs from */ ARGO_ARCHIVE_BUCKETNAME = 'mlpipeline', - /** Prefix to logs. */ - ARGO_ARCHIVE_PREFIX = 'logs', + /** This should match the keyFormat specified in the Argo workflow-controller-configmap. + * It's set here in the manifests: + * https://github.com/kubeflow/pipelines/blob/7b7918ebf8c30e6ceec99283ef20dbc02fdf6a42/manifests/kustomize/third-party/argo/base/workflow-controller-configmap-patch.yaml#L28 + */ + ARGO_KEYFORMAT = 'artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}', /** Should use server API for log streaming? */ STREAM_LOGS_FROM_SERVER_API = 'false', /** The main container name of a pod where logs are retrieved */ @@ -127,7 +130,7 @@ export function loadConfigs(argv: string[], env: ProcessEnv): UIConfigs { archiveArtifactory: ARGO_ARCHIVE_ARTIFACTORY, archiveBucketName: ARGO_ARCHIVE_BUCKETNAME, archiveLogs: asBool(ARGO_ARCHIVE_LOGS), - archivePrefix: ARGO_ARCHIVE_PREFIX, + keyFormat: ARGO_KEYFORMAT, }, pod: { logContainerName: POD_LOG_CONTAINER_NAME, @@ -253,7 +256,7 @@ export interface ArgoConfigs { archiveLogs: boolean; archiveArtifactory: string; archiveBucketName: string; - archivePrefix: string; + keyFormat: string; } export interface ServerConfigs { basePath: string; diff --git a/frontend/server/handlers/pod-logs.ts b/frontend/server/handlers/pod-logs.ts index 7b8a683a122..6d2d7ca864d 100644 --- a/frontend/server/handlers/pod-logs.ts +++ b/frontend/server/handlers/pod-logs.ts @@ -39,21 +39,21 @@ export function getPodLogsHandler( }, podLogContainerName: string, ): Handler { - const { archiveLogs, archiveArtifactory, archiveBucketName, archivePrefix = '' } = argoOptions; + const { archiveLogs, archiveArtifactory, archiveBucketName, keyFormat } = argoOptions; - // get pod log from the provided bucket and prefix. + // get pod log from the provided bucket and keyFormat. const getPodLogsStreamFromArchive = toGetPodLogsStream( createPodLogsMinioRequestConfig( archiveArtifactory === 'minio' ? artifactsOptions.minio : artifactsOptions.aws, archiveBucketName, - archivePrefix, + keyFormat, ), ); // get the pod log stream (with fallbacks). const getPodLogsStream = composePodLogsStreamHandler( - (podName: string, namespace?: string) => { - return getPodLogsStreamFromK8s(podName, namespace, podLogContainerName); + (podName: string, createdAt: string, namespace?: string) => { + return getPodLogsStreamFromK8s(podName, createdAt, namespace, podLogContainerName); }, // if archive logs flag is set, then final attempt will try to retrieve the artifacts // from the bucket and prefix provided in the config. Otherwise, only attempts @@ -69,13 +69,14 @@ export function getPodLogsHandler( return; } const podName = decodeURIComponent(req.query.podname); + const createdAt = decodeURIComponent(req.query.createdat); // This is optional. // Note decodeURIComponent(undefined) === 'undefined', so I cannot pass the argument directly. const podNamespace = decodeURIComponent(req.query.podnamespace || '') || undefined; try { - const stream = await getPodLogsStream(podName, podNamespace); + const stream = await getPodLogsStream(podName, createdAt, podNamespace); stream.on('error', err => { if ( err?.message && diff --git a/frontend/server/workflow-helper.test.ts b/frontend/server/workflow-helper.test.ts index 4ee23878c22..61ee127113f 100644 --- a/frontend/server/workflow-helper.test.ts +++ b/frontend/server/workflow-helper.test.ts @@ -39,40 +39,49 @@ describe('workflow-helper', () => { describe('composePodLogsStreamHandler', () => { it('returns the stream from the default handler if there is no errors.', async () => { const defaultStream = new PassThrough(); - const defaultHandler = jest.fn((_podName: string, _namespace?: string) => + const defaultHandler = jest.fn((_podName: string, _createdAt: string, _namespace?: string) => Promise.resolve(defaultStream), ); - const stream = await composePodLogsStreamHandler(defaultHandler)('podName', 'namespace'); - expect(defaultHandler).toBeCalledWith('podName', 'namespace'); + const stream = await composePodLogsStreamHandler(defaultHandler)( + 'podName', + '2024-08-13', + 'namespace', + ); + expect(defaultHandler).toBeCalledWith('podName', '2024-08-13', 'namespace'); expect(stream).toBe(defaultStream); }); it('returns the stream from the fallback handler if there is any error.', async () => { const fallbackStream = new PassThrough(); - const defaultHandler = jest.fn((_podName: string, _namespace?: string) => + const defaultHandler = jest.fn((_podName: string, _createdAt: string, _namespace?: string) => Promise.reject('unknown error'), ); - const fallbackHandler = jest.fn((_podName: string, _namespace?: string) => + const fallbackHandler = jest.fn((_podName: string, _createdAt: string, _namespace?: string) => Promise.resolve(fallbackStream), ); const stream = await composePodLogsStreamHandler(defaultHandler, fallbackHandler)( 'podName', + '2024-08-13', 'namespace', ); - expect(defaultHandler).toBeCalledWith('podName', 'namespace'); - expect(fallbackHandler).toBeCalledWith('podName', 'namespace'); + expect(defaultHandler).toBeCalledWith('podName', '2024-08-13', 'namespace'); + expect(fallbackHandler).toBeCalledWith('podName', '2024-08-13', 'namespace'); expect(stream).toBe(fallbackStream); }); it('throws error if both handler and fallback fails.', async () => { - const defaultHandler = jest.fn((_podName: string, _namespace?: string) => + const defaultHandler = jest.fn((_podName: string, _createdAt: string, _namespace?: string) => Promise.reject('unknown error for default'), ); - const fallbackHandler = jest.fn((_podName: string, _namespace?: string) => + const fallbackHandler = jest.fn((_podName: string, _createdAt: string, _namespace?: string) => Promise.reject('unknown error for fallback'), ); await expect( - composePodLogsStreamHandler(defaultHandler, fallbackHandler)('podName', 'namespace'), + composePodLogsStreamHandler(defaultHandler, fallbackHandler)( + 'podName', + '2024-08-13', + 'namespace', + ), ).rejects.toEqual('unknown error for fallback'); }); }); @@ -82,7 +91,7 @@ describe('workflow-helper', () => { const mockedGetPodLogs: jest.Mock = getPodLogs as any; mockedGetPodLogs.mockResolvedValueOnce('pod logs'); - const stream = await getPodLogsStreamFromK8s('podName', 'namespace'); + const stream = await getPodLogsStreamFromK8s('podName', '', 'namespace'); expect(mockedGetPodLogs).toBeCalledWith('podName', 'namespace', 'main'); expect(stream.read().toString()).toBe('pod logs'); }); @@ -101,10 +110,10 @@ describe('workflow-helper', () => { client, key: 'folder/key', }; - const createRequest = jest.fn((_podName: string, _namespace?: string) => + const createRequest = jest.fn((_podName: string, _createdAt: string, _namespace?: string) => Promise.resolve(configs), ); - const stream = await toGetPodLogsStream(createRequest)('podName', 'namespace'); + const stream = await toGetPodLogsStream(createRequest)('podName', '2024-08-13', 'namespace'); expect(mockedClientGetObject).toBeCalledWith('bucket', 'folder/key'); }); }); @@ -112,13 +121,23 @@ describe('workflow-helper', () => { describe('createPodLogsMinioRequestConfig', () => { it('returns a MinioRequestConfig factory with the provided minioClientOptions, bucket, and prefix.', async () => { const mockedClient: jest.Mock = MinioClient as any; - const requestFunc = await createPodLogsMinioRequestConfig(minioConfig, 'bucket', 'prefix'); - const request = await requestFunc('workflow-name-abc', 'namespace'); + const requestFunc = await createPodLogsMinioRequestConfig( + minioConfig, + 'bucket', + 'artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}', + ); + const request = await requestFunc( + 'workflow-name-system-container-impl-foo', + '2024-08-13', + 'namespace', + ); expect(mockedClient).toBeCalledWith(minioConfig); expect(request.client).toBeInstanceOf(MinioClient); expect(request.bucket).toBe('bucket'); - expect(request.key).toBe('prefix/workflow-name/workflow-name-abc/main.log'); + expect(request.key).toBe( + 'artifacts/workflow-name/2024/08/13/workflow-name-system-container-impl-foo/main.log', + ); }); }); @@ -128,31 +147,28 @@ describe('workflow-helper', () => { apiVersion: 'argoproj.io/v1alpha1', kind: 'Workflow', status: { + artifactRepositoryRef: { + artifactRepository: { + archiveLogs: true, + s3: { + accessKeySecret: { key: 'accessKey', name: 'accessKeyName' }, + bucket: 'bucket', + endpoint: 'minio-service.kubeflow', + insecure: true, + key: + 'prefix/workflow-name/workflow-name-system-container-impl-abc/some-artifact.csv', + secretKeySecret: { key: 'secretKey', name: 'secretKeyName' }, + }, + }, + }, nodes: { 'workflow-name-abc': { outputs: { artifacts: [ { - name: 'some-artifact.csv', - s3: { - accessKeySecret: { key: 'accessKey', name: 'accessKeyName' }, - bucket: 'bucket', - endpoint: 'minio-service.kubeflow', - insecure: true, - key: 'prefix/workflow-name/workflow-name-abc/some-artifact.csv', - secretKeySecret: { key: 'secretKey', name: 'secretKeyName' }, - }, - }, - { - archiveLogs: true, - name: 'main.log', + name: 'main-logs', s3: { - accessKeySecret: { key: 'accessKey', name: 'accessKeyName' }, - bucket: 'bucket', - endpoint: 'minio-service.kubeflow', - insecure: true, - key: 'prefix/workflow-name/workflow-name-abc/main.log', - secretKeySecret: { key: 'secretKey', name: 'secretKeyName' }, + key: 'prefix/workflow-name/workflow-name-system-container-impl-abc/main.log', }, }, ], @@ -174,7 +190,10 @@ describe('workflow-helper', () => { mockedClientGetObject.mockResolvedValueOnce(objStream); objStream.end('some fake logs.'); - const stream = await getPodLogsStreamFromWorkflow('workflow-name-abc'); + const stream = await getPodLogsStreamFromWorkflow( + 'workflow-name-system-container-impl-abc', + '2024-07-09', + ); expect(mockedGetArgoWorkflow).toBeCalledWith('workflow-name'); @@ -193,7 +212,7 @@ describe('workflow-helper', () => { expect(mockedClientGetObject).toBeCalledTimes(1); expect(mockedClientGetObject).toBeCalledWith( 'bucket', - 'prefix/workflow-name/workflow-name-abc/main.log', + 'prefix/workflow-name/workflow-name-system-container-impl-abc/main.log', ); }); }); diff --git a/frontend/server/workflow-helper.ts b/frontend/server/workflow-helper.ts index d6ad124684b..e0f9796594f 100644 --- a/frontend/server/workflow-helper.ts +++ b/frontend/server/workflow-helper.ts @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -import path from 'path'; import { PassThrough, Stream } from 'stream'; import { ClientOptions as MinioClientOptions } from 'minio'; import { getK8sSecret, getArgoWorkflow, getPodLogs } from './k8s-helper'; @@ -19,10 +18,20 @@ import { createMinioClient, MinioRequestConfig, getObjectStream } from './minio- export interface PartialArgoWorkflow { status: { + artifactRepositoryRef?: ArtifactRepositoryRef; nodes?: ArgoWorkflowStatusNode; }; } +export interface ArtifactRepositoryRef { + artifactRepository?: ArtifactRepository; +} + +export interface ArtifactRepository { + archiveLogs?: boolean; + s3?: S3Artifact; +} + export interface ArgoWorkflowStatusNode { [key: string]: ArgoWorkflowStatusNodeInfo; } @@ -34,9 +43,12 @@ export interface ArgoWorkflowStatusNodeInfo { } export interface ArtifactRecord { - archiveLogs?: boolean; - name: string; - s3?: S3Artifact; + name?: string; + s3: S3Key; +} + +export interface S3Key { + key: string; } export interface S3Artifact { @@ -61,15 +73,15 @@ export interface SecretSelector { * fails. */ export function composePodLogsStreamHandler( - handler: (podName: string, namespace?: string) => Promise, - fallback?: (podName: string, namespace?: string) => Promise, + handler: (podName: string, createdAt: string, namespace?: string) => Promise, + fallback?: (podName: string, createdAt: string, namespace?: string) => Promise, ) { - return async (podName: string, namespace?: string) => { + return async (podName: string, createdAt: string, namespace?: string) => { try { - return await handler(podName, namespace); + return await handler(podName, createdAt, namespace); } catch (err) { if (fallback) { - return await fallback(podName, namespace); + return await fallback(podName, createdAt, namespace); } console.warn(err); throw err; @@ -80,17 +92,21 @@ export function composePodLogsStreamHandler( /** * Returns a stream containing the pod logs using kubernetes api. * @param podName name of the pod. + * @param createdAt YYYY-MM-DD run was created. Not used. * @param namespace namespace of the pod (uses the same namespace as the server if not provided). * @param containerName container's name of the pod, the default value is 'main'. */ export async function getPodLogsStreamFromK8s( podName: string, + createdAt: string, namespace?: string, containerName: string = 'main', ) { const stream = new PassThrough(); stream.end(await getPodLogs(podName, namespace, containerName)); - console.log(`Getting logs for pod:${podName} in namespace ${namespace}.`); + console.log( + `Getting logs for pod, ${podName}, in namespace, ${namespace}, by calling the Kubernetes API.`, + ); return stream; } @@ -98,6 +114,7 @@ export async function getPodLogsStreamFromK8s( * Returns a stream containing the pod logs using the information provided in the * workflow status (uses k8s api to retrieve the workflow and secrets). * @param podName name of the pod. + * @param createdAt YYYY-MM-DD run was created. Not used. * @param namespace namespace of the pod (uses the same namespace as the server if not provided). */ export const getPodLogsStreamFromWorkflow = toGetPodLogsStream( @@ -112,11 +129,15 @@ export const getPodLogsStreamFromWorkflow = toGetPodLogsStream( * on the provided pod name and namespace (optional). */ export function toGetPodLogsStream( - getMinioRequestConfig: (podName: string, namespace?: string) => Promise, + getMinioRequestConfig: ( + podName: string, + createdAt: string, + namespace?: string, + ) => Promise, ) { - return async (podName: string, namespace?: string) => { - const request = await getMinioRequestConfig(podName, namespace); - console.log(`Getting logs for pod:${podName} from ${request.bucket}/${request.key}.`); + return async (podName: string, createdAt: string, namespace?: string) => { + const request = await getMinioRequestConfig(podName, createdAt, namespace); + console.log(`Getting logs for pod, ${podName}, from ${request.bucket}/${request.key}.`); return await getObjectStream(request); }; } @@ -127,24 +148,54 @@ export function toGetPodLogsStream( * client). * @param minioOptions Minio options to create a minio client. * @param bucket bucket containing the pod logs artifacts. - * @param prefix prefix for pod logs artifacts stored in the bucket. + * @param keyFormat the keyFormat for pod logs artifacts stored in the bucket. */ export function createPodLogsMinioRequestConfig( minioOptions: MinioClientOptions, bucket: string, - prefix: string, + keyFormat: string, ) { - // TODO: support pod log artifacts for diff namespace. - // different bucket/prefix for diff namespace? - return async (podName: string, _namespace?: string): Promise => { + return async ( + podName: string, + createdAt: string, + namespace: string = '', + ): Promise => { // create a new client each time to ensure session token has not expired const client = await createMinioClient(minioOptions, 's3'); - const workflowName = workflowNameFromPodName(podName); - return { - bucket, - client, - key: path.join(prefix, workflowName, podName, 'main.log'), - }; + const createdAtArray = createdAt.split('-'); + + let key: string = keyFormat + .replace(/\s+/g, '') // Remove all whitespace. + .replace('{{workflow.name}}', podName.replace(/-system-container-impl-.*/, '')) + .replace('{{workflow.creationTimestamp.Y}}', createdAtArray[0]) + .replace('{{workflow.creationTimestamp.m}}', createdAtArray[1]) + .replace('{{workflow.creationTimestamp.d}}', createdAtArray[2]) + .replace('{{pod.name}}', podName) + .replace('{{workflow.namespace}}', namespace); + + if (!key.endsWith('/')) { + key = key + '/'; + } + key = key + 'main.log'; + + // If there are unresolved template tags in the keyFormat, throw an error + // that surfaces in the frontend's console log. + if (key.includes('{') || key.includes('}')) { + throw new Error( + `keyFormat, which is defined in config.ts or through the ARGO_KEYFORMAT env var, appears to include template tags that are not supported. ` + + `The resulting log key, ${key}, includes unresolved template tags and is therefore invalid.`, + ); + } + + const regex = /^[a-zA-Z0-9\-._/]+$/; // Allow letters, numbers, -, ., _, / + if (!regex.test(key)) { + throw new Error( + `The log key, ${key}, which is derived from keyFormat in config.ts or through the ARGO_KEYFORMAT env var, is an invalid path. ` + + `Supported characters include: letters, numbers, -, ., _, and /.`, + ); + } + + return { bucket, client, key }; }; } @@ -158,33 +209,42 @@ export async function getPodLogsMinioRequestConfigfromWorkflow( podName: string, ): Promise { let workflow: PartialArgoWorkflow; + // We should probably parameterize this replace statement. It's brittle to + // changes in implementation. But brittle is better than completely broken. + let workflowName = podName.replace(/-system-container-impl-.*/, ''); try { - workflow = await getArgoWorkflow(workflowNameFromPodName(podName)); + workflow = await getArgoWorkflow(workflowName); } catch (err) { throw new Error(`Unable to retrieve workflow status: ${err}.`); } + // archiveLogs can be set globally for the workflow as a whole and / or for + // each individual task. The compiler sets it globally so we look for it in + // the global field, which is documented here: + // https://argo-workflows.readthedocs.io/en/release-3.4/fields/#workflow + if (!workflow.status.artifactRepositoryRef?.artifactRepository?.archiveLogs) { + throw new Error('Unable to retrieve logs from artifact store; archiveLogs is disabled.'); + } + let artifacts: ArtifactRecord[] | undefined; - // check if required fields are available if (workflow.status && workflow.status.nodes) { - const node = workflow.status.nodes[podName]; - if (node && node.outputs && node.outputs.artifacts) { - artifacts = node.outputs.artifacts; - } + const nodeName = podName.replace('-system-container-impl', ''); + const node = workflow.status.nodes[nodeName]; + artifacts = node?.outputs?.artifacts || undefined; } if (!artifacts) { - throw new Error('Unable to find pod info in workflow status to retrieve logs.'); + throw new Error('Unable to find corresponding log artifact in node.'); } - const archiveLogs: ArtifactRecord[] = artifacts.filter((artifact: any) => artifact.archiveLogs); - - if (archiveLogs.length === 0) { - throw new Error('Unable to find pod log archive information from workflow status.'); + const logKey = + artifacts.find((artifact: ArtifactRecord) => artifact.name === 'main-logs')?.s3.key || false; + if (!logKey) { + throw new Error('No artifact named "main-logs" for node.'); } - const s3Artifact = archiveLogs[0].s3; + const s3Artifact = workflow.status.artifactRepositoryRef.artifactRepository.s3 || false; if (!s3Artifact) { - throw new Error('Unable to find s3 artifact info from workflow status.'); + throw new Error('Unable to find artifact repository information from workflow status.'); } const { host, port } = urlSplit(s3Artifact.endpoint, s3Artifact.insecure); @@ -193,6 +253,10 @@ export async function getPodLogsMinioRequestConfigfromWorkflow( const client = await createMinioClient( { accessKey, + // TODO: endPoint needs to be set to 'localhost' for local development. + // start-proxy-and-server.sh sets MINIO_HOST=localhost, but it doesn't + // seem to be respected when running the server in development mode. + // Investigate and fix this. endPoint: host, port, secretKey, @@ -203,7 +267,7 @@ export async function getPodLogsMinioRequestConfigfromWorkflow( return { bucket: s3Artifact.bucket, client, - key: s3Artifact.key, + key: logKey, }; } @@ -233,13 +297,3 @@ function urlSplit(uri: string, insecure: boolean) { } return { host: chunks[0], port: parseInt(chunks[1], 10) }; } - -/** - * Infers workflow name from pod name. - * @param podName name of the pod. - */ -function workflowNameFromPodName(podName: string) { - const chunks = podName.split('-'); - chunks.pop(); - return chunks.join('-'); -} diff --git a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx index 8a1cac7faa4..6c22ecc9933 100644 --- a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx +++ b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx @@ -306,6 +306,7 @@ async function getLogsInfo(execution: Execution, runId?: string): Promise { it('getPodLogs', async () => { const spy = fetchSpy('http://some/address'); - expect(await Apis.getPodLogs('a-run-id', 'some-pod-name', 'ns')).toEqual('http://some/address'); + expect(await Apis.getPodLogs('a-run-id', 'some-pod-name', 'ns', '')).toEqual( + 'http://some/address', + ); expect(spy).toHaveBeenCalledWith( 'k8s/pod/logs?podname=some-pod-name&runid=a-run-id&podnamespace=ns', { @@ -71,7 +73,7 @@ describe('Apis', () => { it('getPodLogs in a specific namespace', async () => { const spy = fetchSpy('http://some/address'); - expect(await Apis.getPodLogs('a-run-id', 'some-pod-name', 'some-namespace-name')).toEqual( + expect(await Apis.getPodLogs('a-run-id', 'some-pod-name', 'some-namespace-name', '')).toEqual( 'http://some/address', ); expect(spy).toHaveBeenCalledWith( @@ -82,6 +84,19 @@ describe('Apis', () => { ); }); + it('getPodLogs with createdat specified', async () => { + const spy = fetchSpy('http://some/address'); + expect(await Apis.getPodLogs('a-run-id', 'some-pod-name', 'ns', '2024-08-13')).toEqual( + 'http://some/address', + ); + expect(spy).toHaveBeenCalledWith( + 'k8s/pod/logs?podname=some-pod-name&runid=a-run-id&podnamespace=ns&createdat=2024-08-13', + { + credentials: 'same-origin', + }, + ); + }); + it('getPodLogs error', async () => { jest.spyOn(console, 'error').mockImplementation(() => null); window.fetch = jest.fn(() => diff --git a/frontend/src/lib/Apis.ts b/frontend/src/lib/Apis.ts index c87fa6d5078..83ade5db2e0 100644 --- a/frontend/src/lib/Apis.ts +++ b/frontend/src/lib/Apis.ts @@ -108,13 +108,21 @@ export class Apis { /** * Get pod logs */ - public static getPodLogs(runId: string, podName: string, podNamespace: string): Promise { + public static getPodLogs( + runId: string, + podName: string, + podNamespace: string, + createdAt: string, + ): Promise { let query = `k8s/pod/logs?podname=${encodeURIComponent(podName)}&runid=${encodeURIComponent( runId, )}`; if (podNamespace) { query += `&podnamespace=${encodeURIComponent(podNamespace)}`; } + if (createdAt) { + query += `&createdat=${encodeURIComponent(createdAt)}`; + } return this._fetch(query); } diff --git a/frontend/src/pages/RunDetails.test.tsx b/frontend/src/pages/RunDetails.test.tsx index fa70e8917c0..8e5a24a1ae5 100644 --- a/frontend/src/pages/RunDetails.test.tsx +++ b/frontend/src/pages/RunDetails.test.tsx @@ -1165,6 +1165,7 @@ describe('RunDetails', () => { 'test-run-id', 'workflow1-template1-node1', 'ns', + '', ); expect(tree).toMatchSnapshot(); }); @@ -1255,6 +1256,7 @@ describe('RunDetails', () => { 'test-run-id', 'workflow1-template1-node1', 'username', + '', ); }); diff --git a/frontend/src/pages/RunDetails.tsx b/frontend/src/pages/RunDetails.tsx index efff01f7588..afca9e5f0a4 100644 --- a/frontend/src/pages/RunDetails.tsx +++ b/frontend/src/pages/RunDetails.tsx @@ -1062,7 +1062,7 @@ class RunDetails extends Page { try { const nodeName = getNodeNameFromNodeId(this.state.workflow!, selectedNodeDetails.id); - selectedNodeDetails.logs = await Apis.getPodLogs(runId, nodeName, namespace); + selectedNodeDetails.logs = await Apis.getPodLogs(runId, nodeName, namespace, ''); } catch (err) { let errMsg = await errorToMessage(err); logsBannerMessage = 'Failed to retrieve pod logs.'; From ae3e33c96e48fb842ab61407e8fcaa815b96213f Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 16 Aug 2024 10:27:35 -0700 Subject: [PATCH 13/54] feat(component): internal Signed-off-by: Googler PiperOrigin-RevId: 663774557 Signed-off-by: KevinGrantLee --- components/google-cloud/RELEASE.md | 1 + .../evaluation_llm_text_generation_pipeline.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md index 3dd1e6cd211..5536f883dc5 100644 --- a/components/google-cloud/RELEASE.md +++ b/components/google-cloud/RELEASE.md @@ -11,6 +11,7 @@ * Bump image for Structured Data pipelines. * Add check that component in preview.custom_job.utils.create_custom_training_job_from_component doesn't have any parameters that share names with any custom job fields * Add dynamic machine spec support for `preview.custom_job.utils.create_custom_training_job_from_component`. +* Add preflight validations for LLM text generation pipeline. * Apply latest GCPC image vulnerability resolutions (base OS and software updates). ## Release 2.15.0 diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py index 9b613ee8eb3..d00fb809852 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py @@ -15,10 +15,12 @@ from typing import Dict, List, NamedTuple +from google_cloud_pipeline_components import google_template_metadata from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationPreprocessorOp from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationTextGenerationOp from google_cloud_pipeline_components._implementation.model_evaluation import ModelNamePreprocessorOp from google_cloud_pipeline_components.preview.model_evaluation.model_evaluation_import_component import model_evaluation_import as ModelImportEvaluationOp +from google_cloud_pipeline_components.proto import template_metadata_pb2 from google_cloud_pipeline_components.types.artifact_types import VertexModel from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp from kfp import dsl @@ -29,6 +31,21 @@ _PIPELINE_NAME = 'evaluation-llm-text-generation-pipeline' +output_gcs_validation = template_metadata_pb2.GoogleCloudStorageValidation( + gcs_uri='{{$.parameter.batch_predict_gcs_destination_output_uri}}', + is_input=False, + default_service_account='{{$.pipeline_google_cloud_project_number}}-compute@developer.gserviceaccount.com', + override_placeholder='{{$.parameter.service_account}}', +) + + +@google_template_metadata.set_template_metadata( + template_metadata=template_metadata_pb2.TemplateMetadata( + preflight_validations=template_metadata_pb2.ValidationItems( + gcs_validations=[output_gcs_validation] + ) + ) +) @dsl.pipeline(name=_PIPELINE_NAME) def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-default-value project: str, From 27595c301091f6508ac0b8c06e662a21a8b92621 Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 16 Aug 2024 14:39:55 -0700 Subject: [PATCH 14/54] feat(component): internal Signed-off-by: Googler PiperOrigin-RevId: 663872006 Signed-off-by: KevinGrantLee --- components/google-cloud/RELEASE.md | 1 + .../evaluation_llm_text_generation_pipeline.py | 17 ----------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md index 5536f883dc5..acdc613185b 100644 --- a/components/google-cloud/RELEASE.md +++ b/components/google-cloud/RELEASE.md @@ -1,6 +1,7 @@ ## Upcoming release * Fix to model batch explanation component for Structured Data pipelines; image bump. * Add dynamic support for boot_disk_type, boot_disk_size in `preview.custom_job.utils.create_custom_training_job_from_component`. +* Remove preflight validations temporarily. ## Release 2.16.0 * Updated the Starry Net pipeline's template gallery description, and added dataprep_nan_threshold and dataprep_zero_threshold args to the Starry Net pipeline. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py index d00fb809852..9b613ee8eb3 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py @@ -15,12 +15,10 @@ from typing import Dict, List, NamedTuple -from google_cloud_pipeline_components import google_template_metadata from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationPreprocessorOp from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationTextGenerationOp from google_cloud_pipeline_components._implementation.model_evaluation import ModelNamePreprocessorOp from google_cloud_pipeline_components.preview.model_evaluation.model_evaluation_import_component import model_evaluation_import as ModelImportEvaluationOp -from google_cloud_pipeline_components.proto import template_metadata_pb2 from google_cloud_pipeline_components.types.artifact_types import VertexModel from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp from kfp import dsl @@ -31,21 +29,6 @@ _PIPELINE_NAME = 'evaluation-llm-text-generation-pipeline' -output_gcs_validation = template_metadata_pb2.GoogleCloudStorageValidation( - gcs_uri='{{$.parameter.batch_predict_gcs_destination_output_uri}}', - is_input=False, - default_service_account='{{$.pipeline_google_cloud_project_number}}-compute@developer.gserviceaccount.com', - override_placeholder='{{$.parameter.service_account}}', -) - - -@google_template_metadata.set_template_metadata( - template_metadata=template_metadata_pb2.TemplateMetadata( - preflight_validations=template_metadata_pb2.ValidationItems( - gcs_validations=[output_gcs_validation] - ) - ) -) @dsl.pipeline(name=_PIPELINE_NAME) def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-default-value project: str, From 48cb7ddfff78af0131d3b0e5183c060e4ab8b92c Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 16 Aug 2024 15:09:23 -0700 Subject: [PATCH 15/54] chore(components): GCPC 2.16.1 Release Signed-off-by: Googler PiperOrigin-RevId: 663883139 Signed-off-by: KevinGrantLee --- components/google-cloud/Dockerfile | 2 +- components/google-cloud/RELEASE.md | 2 ++ components/google-cloud/docs/source/versions.json | 5 +++++ .../google-cloud/google_cloud_pipeline_components/version.py | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/components/google-cloud/Dockerfile b/components/google-cloud/Dockerfile index c8b8cccff7d..bb00098b464 100644 --- a/components/google-cloud/Dockerfile +++ b/components/google-cloud/Dockerfile @@ -44,7 +44,7 @@ RUN pip3 install -U "fsspec>=0.7.4" "gcsfs>=0.6.0" "pandas<=1.3.5" "scikit-learn RUN pip3 install -U google-cloud-notebooks # Install main package -RUN pip3 install "git+https://github.com/kubeflow/pipelines.git@google-cloud-pipeline-components-2.16.0#egg=google-cloud-pipeline-components&subdirectory=components/google-cloud" +RUN pip3 install "git+https://github.com/kubeflow/pipelines.git@google-cloud-pipeline-components-2.16.1#egg=google-cloud-pipeline-components&subdirectory=components/google-cloud" # Note that components can override the container entry ponint. ENTRYPOINT ["python3","-m","google_cloud_pipeline_components.container.v1.aiplatform.remote_runner"] diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md index acdc613185b..aa691855efc 100644 --- a/components/google-cloud/RELEASE.md +++ b/components/google-cloud/RELEASE.md @@ -1,7 +1,9 @@ ## Upcoming release +## Release 2.16.1 * Fix to model batch explanation component for Structured Data pipelines; image bump. * Add dynamic support for boot_disk_type, boot_disk_size in `preview.custom_job.utils.create_custom_training_job_from_component`. * Remove preflight validations temporarily. +* Apply latest GCPC image vulnerability resolutions (base OS and software updates). ## Release 2.16.0 * Updated the Starry Net pipeline's template gallery description, and added dataprep_nan_threshold and dataprep_zero_threshold args to the Starry Net pipeline. diff --git a/components/google-cloud/docs/source/versions.json b/components/google-cloud/docs/source/versions.json index 8f41139b439..a6058ba9c5a 100644 --- a/components/google-cloud/docs/source/versions.json +++ b/components/google-cloud/docs/source/versions.json @@ -1,4 +1,9 @@ [ + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.16.1", + "title": "2.16.1", + "aliases": [] + }, { "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.16.0", "title": "2.16.0", diff --git a/components/google-cloud/google_cloud_pipeline_components/version.py b/components/google-cloud/google_cloud_pipeline_components/version.py index d4f5998c150..545b00629d2 100644 --- a/components/google-cloud/google_cloud_pipeline_components/version.py +++ b/components/google-cloud/google_cloud_pipeline_components/version.py @@ -13,4 +13,4 @@ # limitations under the License. """Google Cloud Pipeline Components version.""" -__version__ = "2.16.0" +__version__ = "2.16.1" From c73637110fd4a4210b25f4b1692a9beba55dc1df Mon Sep 17 00:00:00 2001 From: ElayAharoni <62550608+ElayAharoni@users.noreply.github.com> Date: Tue, 20 Aug 2024 20:54:56 +0300 Subject: [PATCH 16/54] test: Fail fast when image build fails on tests #11102 (#11115) * Fail fast when image build fails on tests #11102 Signed-off-by: Elay Aharoni (EXT-Nokia) * Fail fast when image build fails on tests #11102 Signed-off-by: Elay Aharoni (EXT-Nokia) --------- Signed-off-by: Elay Aharoni (EXT-Nokia) Co-authored-by: Elay Aharoni (EXT-Nokia) Signed-off-by: KevinGrantLee --- scripts/deploy/github/build-images.sh | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/scripts/deploy/github/build-images.sh b/scripts/deploy/github/build-images.sh index ffa6db2e216..1b90d9e7bad 100755 --- a/scripts/deploy/github/build-images.sh +++ b/scripts/deploy/github/build-images.sh @@ -20,14 +20,31 @@ set -e REGISTRY="${REGISTRY:-kind-registry:5000}" TAG="${TAG:-latest}" +EXIT_CODE=0 docker system prune -a -f -docker build -q -t "${REGISTRY}/apiserver:${TAG}" -f backend/Dockerfile . && docker push "${REGISTRY}/apiserver:${TAG}" & -docker build -q -t "${REGISTRY}/persistenceagent:${TAG}" -f backend/Dockerfile.persistenceagent . && docker push "${REGISTRY}/persistenceagent:${TAG}" & -docker build -q -t "${REGISTRY}/scheduledworkflow:${TAG}" -f backend/Dockerfile.scheduledworkflow . && docker push "${REGISTRY}/scheduledworkflow:${TAG}" & +docker build -q -t "${REGISTRY}/apiserver:${TAG}" -f backend/Dockerfile . && docker push "${REGISTRY}/apiserver:${TAG}" || EXIT_CODE=$? +if [[ $EXIT_CODE -ne 0 ]] +then + echo "Failed to build apiserver image." + exit $EXIT_CODE +fi + +docker build -q -t "${REGISTRY}/persistenceagent:${TAG}" -f backend/Dockerfile.persistenceagent . && docker push "${REGISTRY}/persistenceagent:${TAG}" || EXIT_CODE=$? +if [[ $EXIT_CODE -ne 0 ]] +then + echo "Failed to build persistenceagent image." + exit $EXIT_CODE +fi + +docker build -q -t "${REGISTRY}/scheduledworkflow:${TAG}" -f backend/Dockerfile.scheduledworkflow . && docker push "${REGISTRY}/scheduledworkflow:${TAG}" || EXIT_CODE=$? +if [[ $EXIT_CODE -ne 0 ]] +then + echo "Failed to build scheduledworkflow image." + exit $EXIT_CODE +fi -wait # clean up intermittent build caches to free up disk space docker system prune -a -f From 370b449e0a63ec56a4d51a475c5a0ce923bd90b0 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 20 Aug 2024 19:33:12 -0700 Subject: [PATCH 17/54] fix(components): Use instance.target_field_name format for text-bison models only, use target_field_name for gemini models Signed-off-by: Googler PiperOrigin-RevId: 665638487 Signed-off-by: KevinGrantLee --- .../llm_evaluation_preprocessor/component.py | 6 ++---- .../evaluation_llm_text_generation_pipeline.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/component.py index f102fe541ac..f31f28823d7 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/component.py @@ -101,10 +101,8 @@ def evaluation_dataset_preprocessor_internal( f'--gcs_source_uris={gcs_source_uris}', f'--input_field_name={input_field_name}', f'--role_field_name={role_field_name}', - ( - f'--target_field_name={target_field_name}' - f'--model_name={model_name}' - ), + f'--target_field_name={target_field_name}', + f'--model_name={model_name}', f'--output_dirs={output_dirs}', '--executor_input={{$.json_escape[1]}}', ], diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py index 9b613ee8eb3..e9022932463 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py @@ -152,7 +152,7 @@ def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-defaul project=project, location=location, evaluation_task=evaluation_task, - target_field_name=f'instance.{target_field_name}', + target_field_name=target_field_name, predictions_format=batch_predict_predictions_format, enable_row_based_metrics=enable_row_based_metrics, joined_predictions_gcs_source=batch_predict_task.outputs[ From 65fa56a0ee4f6395aafb7c79578e4929e1fde974 Mon Sep 17 00:00:00 2001 From: Helber Belmiro Date: Wed, 21 Aug 2024 11:10:57 -0300 Subject: [PATCH 18/54] chore: Renamed GitHub workflows from *.yaml to *.yml for consistency (#11126) Signed-off-by: hbelmiro Signed-off-by: KevinGrantLee --- .github/workflows/{e2e-test.yaml => e2e-test.yml} | 2 +- ...rnetes-library-test.yaml => kfp-kubernetes-library-test.yml} | 2 +- .github/workflows/{rerun-pr-jobs.yaml => rerun-pr-jobs.yml} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{e2e-test.yaml => e2e-test.yml} (99%) rename .github/workflows/{kfp-kubernetes-library-test.yaml => kfp-kubernetes-library-test.yml} (93%) rename .github/workflows/{rerun-pr-jobs.yaml => rerun-pr-jobs.yml} (100%) diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yml similarity index 99% rename from .github/workflows/e2e-test.yaml rename to .github/workflows/e2e-test.yml index daef2ae49a7..73f4ac36a65 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yml @@ -6,7 +6,7 @@ on: pull_request: paths: - - '.github/workflows/e2e-test.yaml' + - '.github/workflows/e2e-test.yml' - 'scripts/deploy/github/**' - 'go.mod' - 'go.sum' diff --git a/.github/workflows/kfp-kubernetes-library-test.yaml b/.github/workflows/kfp-kubernetes-library-test.yml similarity index 93% rename from .github/workflows/kfp-kubernetes-library-test.yaml rename to .github/workflows/kfp-kubernetes-library-test.yml index 4249d2e5736..f1ac56e1eea 100644 --- a/.github/workflows/kfp-kubernetes-library-test.yaml +++ b/.github/workflows/kfp-kubernetes-library-test.yml @@ -5,7 +5,7 @@ on: branches: [master] pull_request: paths: - - '.github/workflows/kfp-kubernetes-library-test.yaml' + - '.github/workflows/kfp-kubernetes-library-test.yml' - 'sdk/python/**' - 'api/v2alpha1/**' - 'kubernetes_platform/**' diff --git a/.github/workflows/rerun-pr-jobs.yaml b/.github/workflows/rerun-pr-jobs.yml similarity index 100% rename from .github/workflows/rerun-pr-jobs.yaml rename to .github/workflows/rerun-pr-jobs.yml From b0f4dfef4e8b753e4dafa0d74b4518761424f496 Mon Sep 17 00:00:00 2001 From: Oswaldo Gomez Date: Wed, 21 Aug 2024 21:04:57 +0200 Subject: [PATCH 19/54] Fix view edit cluster roles (#11067) * Fixing incorrect typing in loop_parallism example Signed-off-by: Oswaldo Gomez * Fixing samples/core/loop_parameter example Signed-off-by: Oswaldo Gomez * Fixing aggregate-to-kubeflow-pipelines-edit Signed-off-by: Oswaldo Gomez * keeping MRs separate. Signed-off-by: Oswaldo Gomez * Adding blank line Signed-off-by: Oswaldo Gomez --------- Signed-off-by: Oswaldo Gomez Co-authored-by: Oswaldo Gomez Signed-off-by: KevinGrantLee --- .../base/installs/multi-user/view-edit-cluster-roles.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/kustomize/base/installs/multi-user/view-edit-cluster-roles.yaml b/manifests/kustomize/base/installs/multi-user/view-edit-cluster-roles.yaml index abb531ee5a0..ed2e2ba0e44 100644 --- a/manifests/kustomize/base/installs/multi-user/view-edit-cluster-roles.yaml +++ b/manifests/kustomize/base/installs/multi-user/view-edit-cluster-roles.yaml @@ -97,6 +97,7 @@ rules: - workflows/finalizers - workfloweventbindings - workflowtemplates + - workflowtaskresults --- From e32f9953babff41ffa2d85edbef7d1ebb9980cd9 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 21 Aug 2024 12:06:44 -0700 Subject: [PATCH 20/54] fix(components): Pass moddel name to eval_runner to process batch prediction's output as per the output schema of model used Signed-off-by: Googler PiperOrigin-RevId: 665977093 Signed-off-by: KevinGrantLee --- .../model_evaluation/llm_evaluation/component.py | 3 +++ .../evaluation_llm_text_generation_pipeline.py | 1 + 2 files changed, 4 insertions(+) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py index e0d118bcb26..fe362b230e9 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py @@ -32,6 +32,7 @@ def model_evaluation_text_generation( row_based_metrics: Output[Metrics], project: str, location: str, + model_name: str, evaluation_task: str = 'text-generation', target_field_name: str = 'instance.output_text', prediction_field_name: str = 'predictions.content', @@ -55,6 +56,7 @@ def model_evaluation_text_generation( Args: project: The GCP project that runs the pipeline component. location: The GCP region that runs the pipeline component. + model_name: The name of the model to be evaluated. evaluation_task: The task that the large language model will be evaluated on. The evaluation component computes a set of metrics relevant to that specific task. Currently supported tasks are: `summarization`, @@ -124,6 +126,7 @@ def model_evaluation_text_generation( machine_type=machine_type, image_uri=version.LLM_EVAL_IMAGE_TAG, args=[ + f'--model_name={model_name}', f'--evaluation_task={evaluation_task}', f'--target_field_name={target_field_name}', f'--prediction_field_name={prediction_field_name}', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py index e9022932463..5a2f75e11b5 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py @@ -151,6 +151,7 @@ def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-defaul eval_task = LLMEvaluationTextGenerationOp( project=project, location=location, + model_name=model_name, evaluation_task=evaluation_task, target_field_name=target_field_name, predictions_format=batch_predict_predictions_format, From a34803960f6f2b901d232a15fb6b742ca64309a1 Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Wed, 21 Aug 2024 17:34:38 -0700 Subject: [PATCH 21/54] feat(components): release LLM Model Evaluation image version v0.7 Signed-off-by: Jason Dai PiperOrigin-RevId: 666102687 Signed-off-by: KevinGrantLee --- .../_implementation/model_evaluation/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/version.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/version.py index 8ce2c98a96c..95c5199c74c 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/version.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/version.py @@ -14,7 +14,7 @@ """Version constants for model evaluation components.""" _EVAL_VERSION = 'v0.9.4' -_LLM_EVAL_VERSION = 'v0.6' +_LLM_EVAL_VERSION = 'v0.7' _EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/model-evaluation' _LLM_EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/llm-model-evaluation' From b75b2105545e69301d041361ddbc582d96c055ac Mon Sep 17 00:00:00 2001 From: Dharmit Dalvi Date: Fri, 23 Aug 2024 03:29:58 +0530 Subject: [PATCH 22/54] chore: Adding @DharmitD to SDK reviewers (#11131) Signed-off-by: ddalvi Signed-off-by: KevinGrantLee --- kubernetes_platform/OWNERS | 1 + sdk/OWNERS | 1 + 2 files changed, 2 insertions(+) diff --git a/kubernetes_platform/OWNERS b/kubernetes_platform/OWNERS index 5e02df9d9c4..d1ba856cd99 100644 --- a/kubernetes_platform/OWNERS +++ b/kubernetes_platform/OWNERS @@ -3,5 +3,6 @@ approvers: - hbelmiro reviewers: - chensun + - DharmitD - hbelmiro - gregsheremeta diff --git a/sdk/OWNERS b/sdk/OWNERS index 00313ff0eff..167a029e4ae 100644 --- a/sdk/OWNERS +++ b/sdk/OWNERS @@ -3,4 +3,5 @@ approvers: - connor-mccarthy reviewers: - chensun + - DharmitD - gregsheremeta From 1e18f45864e2088278013f96b8bcfb38a9f536ea Mon Sep 17 00:00:00 2001 From: Diego Lovison Date: Fri, 23 Aug 2024 16:46:00 -0300 Subject: [PATCH 23/54] test: Kubeflow Pipelines V2 integration Tests (#11125) Signed-off-by: Diego Lovison Signed-off-by: KevinGrantLee --- .../kubeflow-pipelines-integration-v2.yml | 38 +++++++++++++++++++ backend/src/v2/test/Makefile | 14 +------ backend/src/v2/test/requirements.txt | 6 --- .../src/v2/test/scripts/upload_gcs_blob.py | 20 ---------- 4 files changed, 40 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/kubeflow-pipelines-integration-v2.yml delete mode 100644 backend/src/v2/test/scripts/upload_gcs_blob.py diff --git a/.github/workflows/kubeflow-pipelines-integration-v2.yml b/.github/workflows/kubeflow-pipelines-integration-v2.yml new file mode 100644 index 00000000000..5de0b55b937 --- /dev/null +++ b/.github/workflows/kubeflow-pipelines-integration-v2.yml @@ -0,0 +1,38 @@ +name: Kubeflow Pipelines V2 integration Tests + +on: + push: + branches: + - master + pull_request: + paths: + - '.github/workflows/kubeflow-pipelines-integration-v2.yml' + - 'samples' + - 'core' + - 'backend' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: Create KFP cluster + uses: ./.github/actions/kfp-cluster + + - name: Forward API port + run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + + - name: Run the Integration Tests + run: | + ./backend/src/v2/test/integration-test.sh diff --git a/backend/src/v2/test/Makefile b/backend/src/v2/test/Makefile index caa945b05fc..7a6a7db2498 100644 --- a/backend/src/v2/test/Makefile +++ b/backend/src/v2/test/Makefile @@ -9,19 +9,9 @@ include $(ENV_PATH) SHELL = /bin/bash .PHONY: integration-test -integration-test: upload +integration-test: export KF_PIPELINES_ENDPOINT=$(HOST) \ - && python -u sample_test.py \ - --samples_config samples/test/config-integration.yaml \ - --timeout_mins 60 \ - --context $(GCS_ROOT)/src/context.tar.gz \ - --gcs_root $(GCS_ROOT)/data \ - --gcr_root $(GCR_ROOT) \ - --kfp_package_path "$(KFP_PACKAGE_PATH)" - -.PHONY: upload -upload: context - python -u scripts/upload_gcs_blob.py tmp/context.tar.gz $(GCS_ROOT)/src/context.tar.gz + && python -u ../../../../samples/v2/sample_test.py .PHONY: context context: diff --git a/backend/src/v2/test/requirements.txt b/backend/src/v2/test/requirements.txt index 6eece4be3a9..9e690ab4125 100644 --- a/backend/src/v2/test/requirements.txt +++ b/backend/src/v2/test/requirements.txt @@ -1,9 +1,3 @@ # install kfp sdk from local path -e ../../../../sdk/python -# TODO(chensun): remove the deprecated dependencies once migrated tests. --r ../../../../sdk/python/requirements-deprecated.txt -ml-metadata==1.14.0 -minio==7.0.4 -google-cloud-storage fire --e ../../../../samples/test/utils diff --git a/backend/src/v2/test/scripts/upload_gcs_blob.py b/backend/src/v2/test/scripts/upload_gcs_blob.py deleted file mode 100644 index 1c1f577a656..00000000000 --- a/backend/src/v2/test/scripts/upload_gcs_blob.py +++ /dev/null @@ -1,20 +0,0 @@ -from google.cloud import storage - - -# This function is mainly written for environments without gsutil. -def upload_blob(source: str, destination: str): - """Uploads a file to the bucket.""" - # source = "local/path/to/file" - # destination = "gs://your-bucket-name/storage-object-name" - - storage_client = storage.Client() - blob = storage.Blob.from_string(destination, storage_client) - - blob.upload_from_filename(source) - - print(f"File {source} uploaded to destination {destination}") - - -if __name__ == '__main__': - import fire - fire.Fire(upload_blob) From f71cecca5d71773898ea35f5e4accd8b0b1e0bb2 Mon Sep 17 00:00:00 2001 From: Giulio Frasca Date: Fri, 23 Aug 2024 15:51:59 -0400 Subject: [PATCH 24/54] chore: Add make targets for building driver and launcher images (#11103) Signed-off-by: Giulio Frasca Signed-off-by: KevinGrantLee --- backend/Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/Makefile b/backend/Makefile index 9884d44b38e..583a0d6674e 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -10,6 +10,8 @@ IMG_TAG_CACHESERVER ?= cache-server IMG_TAG_SCHEDULEDWORKFLOW ?= scheduledworkflow IMG_TAG_VIEWERCONTROLLER ?= viewercontroller IMG_TAG_VISUALIZATION ?= visualization +IMG_TAG_DRIVER ?= kfp-driver +IMG_TAG_LAUNCHER ?= kfp-launcher # Whenever build command for any of the binaries change, we should update them both here and in backend/Dockerfiles. @@ -60,7 +62,7 @@ license_launcher: $(BUILD)/launcher cd $(MOD_ROOT) && go-licenses csv ./backend/src/v2/cmd/launcher-v2 > $(CSV_PATH)/launcher.csv .PHONY: image_all -image_all: image_apiserver image_persistence_agent image_cache image_swf image_viewer image_visualization +image_all: image_apiserver image_persistence_agent image_cache image_swf image_viewer image_visualization image_driver image_launcher .PHONY: image_apiserver image_apiserver: @@ -80,3 +82,9 @@ image_viewer: .PHONY: image_visualization image_visualization: cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build -t ${IMG_TAG_VISUALIZATION} -f backend/Dockerfile.visualization . +.PHONY: image_driver +image_driver: + cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build -t ${IMG_TAG_DRIVER} -f backend/Dockerfile.driver . +.PHONY: image_launcher +image_launcher: + cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build -t ${IMG_TAG_LAUNCHER} -f backend/Dockerfile.launcher . From c8c3eddc2d6df3df12d177e14a140dadc42aecbd Mon Sep 17 00:00:00 2001 From: Greg Sheremeta Date: Fri, 23 Aug 2024 15:52:59 -0400 Subject: [PATCH 25/54] feat(Backend + SDK): Update kfp backend and kubernetes sdk to support EmptyDir (#10913) Update kfp backend and kubernetes sdk to support mounting EmptyDir volumes to task pods. Inspired by #10427 Fixes: #10656 Signed-off-by: Greg Sheremeta Signed-off-by: KevinGrantLee --- backend/src/v2/driver/driver.go | 27 +++ backend/src/v2/driver/driver_test.go | 166 +++++++++++++++++- backend/third_party_licenses/apiserver.csv | 2 +- backend/third_party_licenses/driver.csv | 2 +- go.mod | 2 +- go.sum | 4 +- .../python/kfp/kubernetes/__init__.py | 2 + .../python/kfp/kubernetes/empty_dir.py | 56 ++++++ .../test/snapshot/data/empty_dir_mounts.py | 36 ++++ .../test/snapshot/data/empty_dir_mounts.yaml | 60 +++++++ .../python/test/unit/test_empty_dir_mounts.py | 136 ++++++++++++++ 11 files changed, 485 insertions(+), 8 deletions(-) create mode 100644 kubernetes_platform/python/kfp/kubernetes/empty_dir.py create mode 100644 kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.py create mode 100644 kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.yaml create mode 100644 kubernetes_platform/python/test/unit/test_empty_dir_mounts.py diff --git a/backend/src/v2/driver/driver.go b/backend/src/v2/driver/driver.go index ebb194f646e..56fb45b96f4 100644 --- a/backend/src/v2/driver/driver.go +++ b/backend/src/v2/driver/driver.go @@ -665,6 +665,33 @@ func extendPodSpecPatch( podSpec.Volumes = append(podSpec.Volumes, ephemeralVolume) podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, ephemeralVolumeMount) } + + // EmptyDirMounts + for _, emptyDirVolumeSpec := range kubernetesExecutorConfig.GetEmptyDirMounts() { + var sizeLimitResource *k8sres.Quantity + if emptyDirVolumeSpec.GetSizeLimit() != "" { + r := k8sres.MustParse(emptyDirVolumeSpec.GetSizeLimit()) + sizeLimitResource = &r + } + + emptyDirVolume := k8score.Volume{ + Name: emptyDirVolumeSpec.GetVolumeName(), + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{ + Medium: k8score.StorageMedium(emptyDirVolumeSpec.GetMedium()), + SizeLimit: sizeLimitResource, + }, + }, + } + emptyDirVolumeMount := k8score.VolumeMount{ + Name: emptyDirVolumeSpec.GetVolumeName(), + MountPath: emptyDirVolumeSpec.GetMountPath(), + } + + podSpec.Volumes = append(podSpec.Volumes, emptyDirVolume) + podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, emptyDirVolumeMount) + } + return nil } diff --git a/backend/src/v2/driver/driver_test.go b/backend/src/v2/driver/driver_test.go index bea24890033..be64723ccfb 100644 --- a/backend/src/v2/driver/driver_test.go +++ b/backend/src/v2/driver/driver_test.go @@ -15,9 +15,10 @@ package driver import ( "encoding/json" + "testing" + k8sres "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "testing" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/v2/metadata" @@ -532,7 +533,7 @@ func Test_extendPodSpecPatch_Secret(t *testing.T) { { Name: "secret1", VolumeSource: k8score.VolumeSource{ - Secret: &k8score.SecretVolumeSource{SecretName: "secret1", Optional: &[]bool{false}[0],}, + Secret: &k8score.SecretVolumeSource{SecretName: "secret1", Optional: &[]bool{false}[0]}, }, }, }, @@ -730,7 +731,7 @@ func Test_extendPodSpecPatch_ConfigMap(t *testing.T) { VolumeSource: k8score.VolumeSource{ ConfigMap: &k8score.ConfigMapVolumeSource{ LocalObjectReference: k8score.LocalObjectReference{Name: "cm1"}, - Optional: &[]bool{false}[0],}, + Optional: &[]bool{false}[0]}, }, }, }, @@ -890,6 +891,165 @@ func Test_extendPodSpecPatch_ConfigMap(t *testing.T) { } } +func Test_extendPodSpecPatch_EmptyVolumeMount(t *testing.T) { + medium := "Memory" + sizeLimit := "1Gi" + var sizeLimitResource *k8sres.Quantity + r := k8sres.MustParse(sizeLimit) + sizeLimitResource = &r + + tests := []struct { + name string + k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig + podSpec *k8score.PodSpec + expected *k8score.PodSpec + }{ + { + "Valid - emptydir mount with no medium or size limit", + &kubernetesplatform.KubernetesExecutorConfig{ + EmptyDirMounts: []*kubernetesplatform.EmptyDirMount{ + { + VolumeName: "emptydir1", + MountPath: "/data/path", + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + VolumeMounts: []k8score.VolumeMount{ + { + Name: "emptydir1", + MountPath: "/data/path", + }, + }, + }, + }, + Volumes: []k8score.Volume{ + { + Name: "emptydir1", + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + { + "Valid - emptydir mount with medium and size limit", + &kubernetesplatform.KubernetesExecutorConfig{ + EmptyDirMounts: []*kubernetesplatform.EmptyDirMount{ + { + VolumeName: "emptydir1", + MountPath: "/data/path", + Medium: &medium, + SizeLimit: &sizeLimit, + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + VolumeMounts: []k8score.VolumeMount{ + { + Name: "emptydir1", + MountPath: "/data/path", + }, + }, + }, + }, + Volumes: []k8score.Volume{ + { + Name: "emptydir1", + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{ + Medium: k8score.StorageMedium(medium), + SizeLimit: sizeLimitResource, + }, + }, + }, + }, + }, + }, + { + "Valid - multiple emptydir mounts", + &kubernetesplatform.KubernetesExecutorConfig{ + EmptyDirMounts: []*kubernetesplatform.EmptyDirMount{ + { + VolumeName: "emptydir1", + MountPath: "/data/path", + }, + { + VolumeName: "emptydir2", + MountPath: "/data/path2", + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + VolumeMounts: []k8score.VolumeMount{ + { + Name: "emptydir1", + MountPath: "/data/path", + }, + { + Name: "emptydir2", + MountPath: "/data/path2", + }, + }, + }, + }, + Volumes: []k8score.Volume{ + { + Name: "emptydir1", + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + { + Name: "emptydir2", + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := extendPodSpecPatch(tt.podSpec, tt.k8sExecCfg, nil, nil) + assert.Nil(t, err) + assert.Equal(t, tt.expected, tt.podSpec) + }) + } +} + func Test_extendPodSpecPatch_ImagePullSecrets(t *testing.T) { tests := []struct { name string diff --git a/backend/third_party_licenses/apiserver.csv b/backend/third_party_licenses/apiserver.csv index b5dd1e1f6ed..ed15297596f 100644 --- a/backend/third_party_licenses/apiserver.csv +++ b/backend/third_party_licenses/apiserver.csv @@ -85,7 +85,7 @@ github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-exithandler/pkg/apis/exitha github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-kfptask/pkg/apis/kfptask,https://github.com/kubeflow/kfp-tekton/blob/a75d4b3711ff/tekton-catalog/tekton-kfptask/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/api/v2alpha1/go,https://github.com/kubeflow/pipelines/blob/58ce09e07d03/api/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 -github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/8b2a099e8c9f/kubernetes_platform/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/d911c8b73b49/kubernetes_platform/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e1f0c010f800/third_party/ml-metadata/LICENSE,Apache-2.0 github.com/lann/builder,https://github.com/lann/builder/blob/47ae307949d0/LICENSE,MIT github.com/lann/ps,https://github.com/lann/ps/blob/62de8c46ede0/LICENSE,MIT diff --git a/backend/third_party_licenses/driver.csv b/backend/third_party_licenses/driver.csv index a2cf7c59854..fb514d83c70 100644 --- a/backend/third_party_licenses/driver.csv +++ b/backend/third_party_licenses/driver.csv @@ -33,7 +33,7 @@ github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/lice github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT github.com/kubeflow/pipelines/api/v2alpha1/go,https://github.com/kubeflow/pipelines/blob/58ce09e07d03/api/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 -github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/8b2a099e8c9f/kubernetes_platform/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/d911c8b73b49/kubernetes_platform/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e1f0c010f800/third_party/ml-metadata/LICENSE,Apache-2.0 github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.7/LICENSE,MIT github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 diff --git a/go.mod b/go.mod index 44e9ae0b7f4..1337ebb719b 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-exithandler v0.0.0-20231127195001-a75d4b3711ff github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-kfptask v0.0.0-20231127195001-a75d4b3711ff github.com/kubeflow/pipelines/api v0.0.0-20231027040853-58ce09e07d03 - github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240403164522-8b2a099e8c9f + github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240725205754-d911c8b73b49 github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800 github.com/lestrrat-go/strftime v1.0.4 github.com/mattn/go-sqlite3 v1.14.19 diff --git a/go.sum b/go.sum index 013220bd44d..01d5672e4ad 100644 --- a/go.sum +++ b/go.sum @@ -2212,8 +2212,8 @@ github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-kfptask v0.0.0-202311271950 github.com/kubeflow/kfp-tekton/tekton-catalog/tekton-kfptask v0.0.0-20231127195001-a75d4b3711ff/go.mod h1:lAFdPugzj3bcAXyN3+8y0NByidZ88zwGxMc+gdc8cHw= github.com/kubeflow/pipelines/api v0.0.0-20231027040853-58ce09e07d03 h1:reL3LbkRIozBkKSUYjtQFV2kVC1R4WHG9FrTClRT1FY= github.com/kubeflow/pipelines/api v0.0.0-20231027040853-58ce09e07d03/go.mod h1:T7TOQB36gGe97yUdfVAnYK5uuT0+uQbLNHDUHxYkmE4= -github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240403164522-8b2a099e8c9f h1:O5GmJN8tALpiqL0dUo4uhOkqHG8xOkNCgT7QI9q9GnE= -github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240403164522-8b2a099e8c9f/go.mod h1:CJkKr356RlpZP/gQRuHf3Myrn1qJtoUVe4EMCmtwarg= +github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240725205754-d911c8b73b49 h1:Xf1qun8x4ZJj/nLZpUSIaphDK04NKeV7WME31qIC8Xo= +github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240725205754-d911c8b73b49/go.mod h1:UhJrmpVSWawsAsSr1OzZfsEZTvQce+GvGwcZ58ULEhM= github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800 h1:YAW+X9xCW8Yq5tQaBBQaLTNU9CJj8Nr7lx1+k66ZHJ0= github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800/go.mod h1:chIDffBaVQ/asNl1pTTdbAymYcuBKf8BR3YtSP+3FEU= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= diff --git a/kubernetes_platform/python/kfp/kubernetes/__init__.py b/kubernetes_platform/python/kfp/kubernetes/__init__.py index fa149c31c09..91fb119212a 100644 --- a/kubernetes_platform/python/kfp/kubernetes/__init__.py +++ b/kubernetes_platform/python/kfp/kubernetes/__init__.py @@ -22,6 +22,7 @@ 'add_toleration', 'CreatePVC', 'DeletePVC', + 'empty_dir_mount', 'mount_pvc', 'set_image_pull_policy', 'use_field_path_as_env', @@ -49,3 +50,4 @@ from kfp.kubernetes.volume import CreatePVC from kfp.kubernetes.volume import DeletePVC from kfp.kubernetes.volume import mount_pvc +from kfp.kubernetes.empty_dir import empty_dir_mount diff --git a/kubernetes_platform/python/kfp/kubernetes/empty_dir.py b/kubernetes_platform/python/kfp/kubernetes/empty_dir.py new file mode 100644 index 00000000000..a8873d872ce --- /dev/null +++ b/kubernetes_platform/python/kfp/kubernetes/empty_dir.py @@ -0,0 +1,56 @@ +# Copyright 2024 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from google.protobuf import json_format +from kfp.dsl import PipelineTask +from kfp.kubernetes import common +from kfp.kubernetes import kubernetes_executor_config_pb2 as pb + + +def empty_dir_mount( + task: PipelineTask, + volume_name: str, + mount_path: str, + medium: Optional[str] = None, + size_limit: Optional[str] = None, +) -> PipelineTask: + """Mount an EmptyDir volume to the task's container. + + Args: + task: Pipeline task. + volume_name: Name of the EmptyDir volume. + mount_path: Path within the container at which the EmptyDir should be mounted. + medium: Storage medium to back the EmptyDir. Must be one of `Memory` or `HugePages`. Defaults to `None`. + size_limit: Maximum size of the EmptyDir. For example, `5Gi`. Defaults to `None`. + + Returns: + Task object with updated EmptyDir mount configuration. + """ + + msg = common.get_existing_kubernetes_config_as_message(task) + + empty_dir_mount = pb.EmptyDirMount( + volume_name=volume_name, + mount_path=mount_path, + medium=medium, + size_limit=size_limit, + ) + + msg.empty_dir_mounts.append(empty_dir_mount) + + task.platform_config['kubernetes'] = json_format.MessageToDict(msg) + + return task diff --git a/kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.py b/kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.py new file mode 100644 index 00000000000..a86c93d3cb7 --- /dev/null +++ b/kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.py @@ -0,0 +1,36 @@ +# Copyright 2024 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from kfp import dsl +from kfp import kubernetes + + +@dsl.component +def comp(): + pass + +@dsl.pipeline +def my_pipeline(): + task = comp() + kubernetes.empty_dir_mount( + task, + volume_name='emptydir-vol-1', + mount_path='/mnt/my_vol_1', + medium='Memory', + size_limit='1Gi' + ) + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile(my_pipeline, __file__.replace('.py', '.yaml')) diff --git a/kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.yaml b/kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.yaml new file mode 100644 index 00000000000..177fbd2a7cf --- /dev/null +++ b/kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.yaml @@ -0,0 +1,60 @@ +# PIPELINE DEFINITION +# Name: my-pipeline +components: + comp-comp: + executorLabel: exec-comp +deploymentSpec: + executors: + exec-comp: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - comp + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef comp():\n pass\n\n" + image: python:3.9 +pipelineInfo: + name: my-pipeline +root: + dag: + tasks: + comp: + cachingOptions: + enableCache: true + componentRef: + name: comp-comp + taskInfo: + name: comp +schemaVersion: 2.1.0 +sdkVersion: kfp-2.7.0 +--- +platforms: + kubernetes: + deploymentSpec: + executors: + exec-comp: + emptyDirMounts: + - medium: Memory + mountPath: /mnt/my_vol_1 + sizeLimit: 1Gi + volumeName: emptydir-vol-1 diff --git a/kubernetes_platform/python/test/unit/test_empty_dir_mounts.py b/kubernetes_platform/python/test/unit/test_empty_dir_mounts.py new file mode 100644 index 00000000000..dcde2d140c4 --- /dev/null +++ b/kubernetes_platform/python/test/unit/test_empty_dir_mounts.py @@ -0,0 +1,136 @@ +# Copyright 2024 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.protobuf import json_format +from kfp import dsl +from kfp import kubernetes + + +class TestEmptyDirMounts: + + def test_add_one(self): + + @dsl.pipeline + def my_pipeline(): + task = comp() + kubernetes.empty_dir_mount( + task, + volume_name='emptydir-vol-1', + mount_path='/mnt/my_vol_1', + medium='Memory', + size_limit='1Gi' + ) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'emptyDirMounts': [{ + 'medium': 'Memory', + 'mountPath': '/mnt/my_vol_1', + 'sizeLimit': '1Gi', + 'volumeName': 'emptydir-vol-1' + }] + } + } + } + } + } + } + + def test_add_two(self): + + @dsl.pipeline + def my_pipeline(): + task = comp() + kubernetes.empty_dir_mount( + task, + volume_name='emptydir-vol-1', + mount_path='/mnt/my_vol_1', + medium='Memory', + size_limit='1Gi' + ) + kubernetes.empty_dir_mount( + task, + volume_name='emptydir-vol-2', + mount_path='/mnt/my_vol_2' + ) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'emptyDirMounts': [{ + 'medium': 'Memory', + 'mountPath': '/mnt/my_vol_1', + 'sizeLimit': '1Gi', + 'volumeName': 'emptydir-vol-1' + }, + { + 'mountPath': '/mnt/my_vol_2', + 'volumeName': 'emptydir-vol-2' + }] + } + } + } + } + } + } + + def test_respects_other_configuration(self): + + @dsl.pipeline + def my_pipeline(): + task = comp() + + kubernetes.empty_dir_mount( + task, + volume_name='emptydir-vol-1', + mount_path='/mnt/my_vol_1', + medium='Memory', + size_limit='1Gi' + ) + + # this should exist too + kubernetes.set_image_pull_secrets(task, ['secret-name']) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'emptyDirMounts': [{ + 'medium': 'Memory', + 'mountPath': '/mnt/my_vol_1', + 'sizeLimit': '1Gi', + 'volumeName': 'emptydir-vol-1' + }], + 'imagePullSecret': [{ + 'secretName': 'secret-name' + }] + } + } + } + } + } + } + +@dsl.component +def comp(): + pass From c1cdc1ba436902f22e73918384ba7229404eed6d Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 23 Aug 2024 22:41:58 +0100 Subject: [PATCH 26/54] docs:fixing broken links in readme (#11108) Signed-off-by: Fiona Waters Signed-off-by: KevinGrantLee --- README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0262729048a..99bfadb9b81 100644 --- a/README.md +++ b/README.md @@ -17,22 +17,21 @@ The Kubeflow pipelines service has the following goals: ## Installation -* Install Kubeflow Pipelines from choices described in [Installation Options for Kubeflow Pipelines](https://www.kubeflow.org/docs/pipelines/installation/overview/). +* Kubeflow Pipelines can be installed as part of the [Kubeflow Platform](https://www.kubeflow.org/docs/started/installing-kubeflow/#kubeflow-platform). Alternatively you can deploy [Kubeflow Pipelines](https://www.kubeflow.org/docs/components/pipelines/operator-guides/installation/) as a standalone service. -* The Docker container runtime has been deprecated on Kubernetes 1.20+. Kubeflow Pipelines has switched to use [Emissary Executor](https://www.kubeflow.org/docs/components/pipelines/installation/choose-executor/#emissary-executor) by default from Kubeflow Pipelines 1.8. Emissary executor is Container runtime agnostic, meaning you are able to run Kubeflow Pipelines on Kubernetes cluster with any [Container runtimes](https://kubernetes.io/docs/setup/production-environment/container-runtimes/). + +* The Docker container runtime has been deprecated on Kubernetes 1.20+. Kubeflow Pipelines has switched to use [Emissary Executor](https://www.kubeflow.org/docs/components/pipelines/legacy-v1/installation/choose-executor/#emissary-executor) by default from Kubeflow Pipelines 1.8. Emissary executor is Container runtime agnostic, meaning you are able to run Kubeflow Pipelines on Kubernetes cluster with any [Container runtimes](https://kubernetes.io/docs/setup/production-environment/container-runtimes/). ## Documentation -Get started with your first pipeline and read further information in the [Kubeflow Pipelines overview](https://www.kubeflow.org/docs/components/pipelines/introduction/). +Get started with your first pipeline and read further information in the [Kubeflow Pipelines overview](https://www.kubeflow.org/docs/components/pipelines/overview/). -See the various ways you can [use the Kubeflow Pipelines SDK](https://www.kubeflow.org/docs/pipelines/sdk/sdk-overview/). +See the various ways you can [use the Kubeflow Pipelines SDK](https://kubeflow-pipelines.readthedocs.io/en/stable/). -See the Kubeflow [Pipelines API doc](https://www.kubeflow.org/docs/pipelines/reference/api/kubeflow-pipeline-api-spec/) for API specification. +See the Kubeflow [Pipelines API doc](https://www.kubeflow.org/docs/components/pipelines/reference/api/kubeflow-pipeline-api-spec/) for API specification. Consult the [Python SDK reference docs](https://kubeflow-pipelines.readthedocs.io/en/stable/) when writing pipelines using the Python SDK. -Refer to the [versioning policy](./docs/release/versioning-policy.md) and [feature stages](./docs/release/feature-stages.md) documentation for more information about how we manage versions and feature stages (such as Alpha, Beta, and Stable). - ## Contributing to Kubeflow Pipelines Before you start contributing to Kubeflow Pipelines, read the guidelines in [How to Contribute](./CONTRIBUTING.md). To learn how to build and deploy Kubeflow Pipelines from source code, read the [developer guide](./developer_guide.md). From 4a6044979723bdc6abfb85c42bc6752c4bdcfa27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 19:41:02 +0000 Subject: [PATCH 27/54] chore(deps): bump micromatch from 4.0.5 to 4.0.8 in /test/frontend-integration-test (#11132) Bumps [micromatch](https://github.com/micromatch/micromatch) from 4.0.5 to 4.0.8. - [Release notes](https://github.com/micromatch/micromatch/releases) - [Changelog](https://github.com/micromatch/micromatch/blob/4.0.8/CHANGELOG.md) - [Commits](https://github.com/micromatch/micromatch/compare/4.0.5...4.0.8) --- updated-dependencies: - dependency-name: micromatch dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: KevinGrantLee --- .../package-lock.json | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/frontend-integration-test/package-lock.json b/test/frontend-integration-test/package-lock.json index 337f97dbadc..9e4fcd1661e 100644 --- a/test/frontend-integration-test/package-lock.json +++ b/test/frontend-integration-test/package-lock.json @@ -989,11 +989,11 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -1992,9 +1992,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -3418,11 +3418,11 @@ "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" }, "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dependencies": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { From d982b0cbc277b7e898fe3b1c67ea93281f1904e9 Mon Sep 17 00:00:00 2001 From: Diego Lovison Date: Tue, 27 Aug 2024 15:25:02 -0300 Subject: [PATCH 28/54] Fix: Basic sample tests - sequential is flaky (#11138) Signed-off-by: Diego Lovison Signed-off-by: KevinGrantLee --- .github/workflows/e2e-test.yml | 7 +- scripts/deploy/github/build-images.sh | 1 + scripts/deploy/github/forward-port.sh | 1 + test/sample-test/requirements.in | 10 +- test/sample-test/requirements.txt | 220 ++++++++++++----------- test/sample-test/run_sample_test.py | 41 ++--- test/sample-test/sample_test_launcher.py | 15 +- 7 files changed, 154 insertions(+), 141 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 73f4ac36a65..9bb0ae250a7 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -150,11 +150,14 @@ jobs: - name: Forward API port run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + - name: Install prerequisites + run: pip3 install -r ./test/sample-test/requirements.txt + - name: Basic sample tests - sequential - run: pip3 install -r ./test/sample-test/requirements.txt && pip3 install kfp~=2.0 && python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name sequential --results-gcs-dir output + run: python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name sequential --results-gcs-dir output - name: Basic sample tests - exit_handler - run: pip3 install -r ./test/sample-test/requirements.txt && pip3 install kfp~=2.0 && python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name exit_handler --results-gcs-dir output + run: python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name exit_handler --results-gcs-dir output - name: Collect test results if: always() diff --git a/scripts/deploy/github/build-images.sh b/scripts/deploy/github/build-images.sh index 1b90d9e7bad..1f4e7318e8e 100755 --- a/scripts/deploy/github/build-images.sh +++ b/scripts/deploy/github/build-images.sh @@ -19,6 +19,7 @@ set -e REGISTRY="${REGISTRY:-kind-registry:5000}" +echo "REGISTRY=$REGISTRY" TAG="${TAG:-latest}" EXIT_CODE=0 diff --git a/scripts/deploy/github/forward-port.sh b/scripts/deploy/github/forward-port.sh index 416958c764c..5423439f339 100755 --- a/scripts/deploy/github/forward-port.sh +++ b/scripts/deploy/github/forward-port.sh @@ -24,6 +24,7 @@ LOCAL_PORT=$3 REMOTE_PORT=$4 POD_NAME=$(kubectl get pods -n "$KUBEFLOW_NS" -l "app=$APP_NAME" -o jsonpath='{.items[0].metadata.name}') +echo "POD_NAME=$POD_NAME" if [ $QUIET -eq 1 ]; then kubectl port-forward -n "$KUBEFLOW_NS" "$POD_NAME" "$LOCAL_PORT:$REMOTE_PORT" > /dev/null 2>&1 & diff --git a/test/sample-test/requirements.in b/test/sample-test/requirements.in index b98dc7e3771..2d48979e5b6 100644 --- a/test/sample-test/requirements.in +++ b/test/sample-test/requirements.in @@ -1,12 +1,8 @@ +kfp==2.8.0 junit-xml minio -black==21.7b0 -papermill~=2.2 +black +papermill fire yamale kubernetes - -google-cloud-bigquery<3 -google-cloud-storage<2 -# TODO: remove deprecated dependency --r sdk/python/requirements-deprecated.txt diff --git a/test/sample-test/requirements.txt b/test/sample-test/requirements.txt index 42322bcaa95..c52407539db 100644 --- a/test/sample-test/requirements.txt +++ b/test/sample-test/requirements.txt @@ -1,194 +1,210 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # -# pip-compile --output-file=- - +# pip-compile # -ansiwrap==0.8.4 +ansicolors==1.1.8 # via papermill -appdirs==1.4.4 - # via black -attrs==20.3.0 - # via jsonschema -black==21.7b0 +argon2-cffi==23.1.0 + # via minio +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +attrs==24.2.0 # via - # -r requirements.in - # papermill -cachetools==4.2.4 + # jsonschema + # referencing +black==24.8.0 + # via -r requirements.in +cachetools==5.5.0 # via google-auth -certifi==2021.10.8 +certifi==2024.7.4 # via + # kfp-server-api # kubernetes # minio # requests -charset-normalizer==2.0.9 +cffi==1.17.0 + # via argon2-cffi-bindings +charset-normalizer==3.3.2 # via requests -click==7.1.2 +click==8.1.7 # via # black + # kfp # papermill -entrypoints==0.3 - # via - # jupyter-client - # papermill -fire==0.4.0 +docstring-parser==0.16 + # via kfp +entrypoints==0.4 + # via papermill +fastjsonschema==2.20.0 + # via nbformat +fire==0.6.0 # via -r requirements.in -google-api-core[grpc]==1.31.5 +google-api-core==2.19.1 # via - # google-cloud-bigquery # google-cloud-core # google-cloud-storage -google-auth==1.35.0 + # kfp +google-auth==2.34.0 # via # google-api-core # google-cloud-core # google-cloud-storage + # kfp # kubernetes -google-cloud-bigquery==2.31.0 - # via -r requirements.in -google-cloud-core==1.7.2 +google-cloud-core==2.4.1 + # via google-cloud-storage +google-cloud-storage==2.18.2 + # via kfp +google-crc32c==1.5.0 # via - # google-cloud-bigquery # google-cloud-storage -google-cloud-storage==1.43.0 - # via -r requirements.in -google-crc32c==1.3.0 - # via google-resumable-media -google-resumable-media==2.1.0 - # via - # google-cloud-bigquery - # google-cloud-storage -googleapis-common-protos==1.54.0 + # google-resumable-media +google-resumable-media==2.7.2 + # via google-cloud-storage +googleapis-common-protos==1.64.0 # via google-api-core -grpcio==1.43.0 - # via - # google-api-core - # google-cloud-bigquery -idna==3.3 +idna==3.8 # via requests -ipython-genutils==0.2.0 - # via nbformat -jsonschema==3.2.0 +jsonschema==4.23.0 # via nbformat +jsonschema-specifications==2023.12.1 + # via jsonschema junit-xml==1.9 # via -r requirements.in -jupyter-client==7.1.0 +jupyter-client==8.6.2 # via nbclient -jupyter-core==4.11.2 +jupyter-core==5.7.2 # via # jupyter-client + # nbclient # nbformat -kubernetes==12.0.1 +kfp==2.8.0 # via -r requirements.in -minio==7.1.2 +kfp-pipeline-spec==0.3.0 + # via kfp +kfp-server-api==2.0.5 + # via kfp +kubernetes==26.1.0 + # via + # -r requirements.in + # kfp +minio==7.2.8 # via -r requirements.in -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via black -nbclient==0.5.9 +nbclient==0.10.0 # via papermill -nbformat==5.1.3 +nbformat==5.10.4 # via # nbclient # papermill -nest-asyncio==1.5.4 - # via - # jupyter-client - # nbclient -oauthlib==3.1.1 +oauthlib==3.2.2 # via requests-oauthlib -packaging==20.9 - # via - # google-api-core - # google-cloud-bigquery -papermill==2.3.3 +packaging==24.1 + # via black +papermill==2.6.0 # via -r requirements.in -pathspec==0.9.0 +pathspec==0.12.1 # via black -proto-plus==1.19.8 - # via google-cloud-bigquery -protobuf==3.19.1 +platformdirs==4.2.2 + # via + # black + # jupyter-core +proto-plus==1.24.0 + # via google-api-core +protobuf==4.25.4 # via # google-api-core - # google-cloud-bigquery - # google-cloud-storage # googleapis-common-protos + # kfp + # kfp-pipeline-spec # proto-plus -pyasn1==0.4.8 +pyasn1==0.6.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 +pyasn1-modules==0.4.0 # via google-auth -pyparsing==2.4.7 - # via packaging -pyrsistent==0.18.0 - # via jsonschema -python-dateutil==2.8.2 +pycparser==2.22 + # via cffi +pycryptodome==3.20.0 + # via minio +python-dateutil==2.9.0.post0 # via - # google-cloud-bigquery # jupyter-client + # kfp-server-api # kubernetes -pytz==2021.3 - # via google-api-core -pyyaml==5.4.1 +pyyaml==6.0.2 # via + # kfp # kubernetes # papermill # yamale -pyzmq==22.3.0 +pyzmq==26.2.0 # via jupyter-client -regex==2021.11.10 - # via black -requests==2.26.0 +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 # via # google-api-core - # google-cloud-bigquery # google-cloud-storage # kubernetes # papermill # requests-oauthlib -requests-oauthlib==1.3.0 + # requests-toolbelt +requests-oauthlib==2.0.0 # via kubernetes -rsa==4.8 +requests-toolbelt==0.10.1 + # via kfp +rpds-py==0.20.0 + # via + # jsonschema + # referencing +rsa==4.9 # via google-auth six==1.15.0 # via # fire - # google-api-core - # google-auth - # google-cloud-core - # google-cloud-storage - # grpcio - # jsonschema # junit-xml + # kfp-server-api # kubernetes # python-dateutil -tenacity==8.0.1 +tabulate==0.9.0 + # via kfp +tenacity==9.0.0 # via papermill -termcolor==1.1.0 +termcolor==2.4.0 # via fire -textwrap3==0.9.2 - # via ansiwrap -tomli==1.2.3 +tomli==2.0.1 # via black -tornado==6.1 +tornado==6.4.1 # via jupyter-client -tqdm==4.62.3 +tqdm==4.66.5 # via papermill -traitlets==5.1.1 +traitlets==5.14.3 # via # jupyter-client # jupyter-core # nbclient # nbformat -urllib3==1.26.7 +typing-extensions==4.12.2 + # via + # black + # minio +urllib3==1.26.19 # via + # kfp + # kfp-server-api # kubernetes # minio # requests -websocket-client==1.2.3 +websocket-client==1.8.0 # via kubernetes -yamale==4.0.2 +yamale==5.2.1 # via -r requirements.in # The following packages are considered to be unsafe in a requirements file: diff --git a/test/sample-test/run_sample_test.py b/test/sample-test/run_sample_test.py index 1a77d2fd050..2c9410fd148 100644 --- a/test/sample-test/run_sample_test.py +++ b/test/sample-test/run_sample_test.py @@ -121,8 +121,7 @@ def run(self): yamlerr)) except OSError as ose: print( - 'Config file with the same name not found, use default args:{}' - .format(ose)) + f'Config file "{config_file}" not found, using default args: {raw_args}') else: if 'arguments' in raw_args.keys() and raw_args['arguments']: self._test_args.update(raw_args['arguments']) @@ -153,27 +152,17 @@ def check(self): """Check pipeline run results.""" if self._run_pipeline: ###### Monitor Job ###### - try: - start_time = datetime.now() - response = self._client.wait_for_run_completion( - self._run_id, self._test_timeout) - succ = (response.state.lower() == 'succeeded') - end_time = datetime.now() - elapsed_time = (end_time - start_time).seconds - utils.add_junit_test(self._test_cases, 'job completion', succ, - 'waiting for job completion failure', - elapsed_time) - finally: - # TODO(chensun): print log for debugging - pass - - if not succ: - utils.write_junit_xml(self._test_name, self._result, - self._test_cases) - exit(1) - - ###### Delete Job ###### - #TODO: add deletion when the backend API offers the interface. - - ###### Write out the test result in junit xml ###### - utils.write_junit_xml(self._test_name, self._result, self._test_cases) + start_time = datetime.now() + response = self._client.wait_for_run_completion(self._run_id, self._test_timeout) + succ = (response.state.lower() == 'succeeded') + end_time = datetime.now() + elapsed_time = (end_time - start_time).seconds + utils.add_junit_test(self._test_cases, 'job completion', succ, + 'waiting for job completion failure', + elapsed_time) + print(f'Pipeline {"worked" if succ else "Failed"}. Elapsed time: {elapsed_time}s') + + ###### Delete Job ###### + #TODO: add deletion when the backend API offers the interface. + + assert succ diff --git a/test/sample-test/sample_test_launcher.py b/test/sample-test/sample_test_launcher.py index f503194acd4..3de1c06d7be 100644 --- a/test/sample-test/sample_test_launcher.py +++ b/test/sample-test/sample_test_launcher.py @@ -71,7 +71,9 @@ def __init__(self, raise RuntimeError( 'Failed to get inverse proxy hostname') from err - print('KFP API host is %s' % self._host) + # With the healthz API in place, when the developer clicks the link, + # it will lead to a functional URL instead of a 404 error. + print(f'KFP API healthz endpoint is: {self._host}/apis/v1beta1/healthz') self._is_notebook = None self._work_dir = os.path.join(BASE_DIR, 'samples/core/', @@ -88,7 +90,9 @@ def _compile(self): # Looking for the entry point of the test. list_of_files = os.listdir('.') for file in list_of_files: - m = re.match(self._test_name + '\.[a-zA-Z]+', file) + # matching by .py or .ipynb, there will be yaml ( compiled ) files in the folder. + # if you rerun the test suite twice, the test suite will fail + m = re.match(self._test_name + '\.(py|ipynb)$', file) if m: file_name, ext_name = os.path.splitext(file) if self._is_notebook is not None: @@ -156,13 +160,16 @@ def _compile(self): parameters=nb_params, prepare_only=True) # Convert to python script. - subprocess.call([ + return_code = subprocess.call([ 'jupyter', 'nbconvert', '--to', 'python', '%s.ipynb' % self._test_name ]) else: - subprocess.call(['python3', '%s.py' % self._test_name]) + return_code = subprocess.call(['python3', '%s.py' % self._test_name]) + + # Command executed successfully! + assert return_code == 0 def _injection(self): """Inject images for pipeline components. From 2c563d0ea8479dc8123e6e5b7f177f22a89440ed Mon Sep 17 00:00:00 2001 From: Helber Belmiro Date: Wed, 28 Aug 2024 11:16:03 -0300 Subject: [PATCH 29/54] chore: Wrapped "Failed GetContextByTypeAndName" error for better troubleshooting (#11098) Signed-off-by: hbelmiro Signed-off-by: KevinGrantLee --- backend/src/v2/metadata/client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/v2/metadata/client.go b/backend/src/v2/metadata/client.go index 16d0c47dc58..a292c1fe643 100644 --- a/backend/src/v2/metadata/client.go +++ b/backend/src/v2/metadata/client.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/objectstore" "path" "strconv" @@ -1073,7 +1074,7 @@ func (c *Client) getOrInsertContext(ctx context.Context, name string, contextTyp getCtxRes, err := c.svc.GetContextByTypeAndName(ctx, &pb.GetContextByTypeAndNameRequest{TypeName: contextType.Name, ContextName: proto.String(name)}) if err != nil { - return nil, fmt.Errorf("Failed GetContextByTypeAndName(type=%q, name=%q)", contextType.GetName(), name) + return nil, util.Wrap(err, fmt.Sprintf("Failed GetContextByTypeAndName(type=%q, name=%q)", contextType.GetName(), name)) } // Bug in MLMD GetContextsByTypeAndName? It doesn't return error even when no // context was found. From 60b1d5d0efe476e868a3cc1eec6838aa9909a195 Mon Sep 17 00:00:00 2001 From: Michael Hu Date: Wed, 28 Aug 2024 11:28:01 -0700 Subject: [PATCH 30/54] chore(components): Update AutoSxS and RLHF image tags Signed-off-by: Michael Hu PiperOrigin-RevId: 668536503 Signed-off-by: KevinGrantLee --- components/google-cloud/RELEASE.md | 2 ++ .../_implementation/llm/generated/refined_image_versions.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md index aa691855efc..0c3a04d13af 100644 --- a/components/google-cloud/RELEASE.md +++ b/components/google-cloud/RELEASE.md @@ -1,4 +1,6 @@ ## Upcoming release +* Fix Gemini batch prediction support to `v1.model_evaluation.autosxs_pipeline` after output schema change. + ## Release 2.16.1 * Fix to model batch explanation component for Structured Data pipelines; image bump. * Add dynamic support for boot_disk_type, boot_disk_size in `preview.custom_job.utils.create_custom_training_job_from_component`. diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/generated/refined_image_versions.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/generated/refined_image_versions.py index 7c74a59c603..9fb8dc56ff9 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/generated/refined_image_versions.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/generated/refined_image_versions.py @@ -17,4 +17,4 @@ DO NOT EDIT - This file is generated, manual changes will be overridden. """ -IMAGE_TAG = '20240623_1707' +IMAGE_TAG = '20240818_1707' From 245bafeba8b33b73460dff3e5f19224467ceae80 Mon Sep 17 00:00:00 2001 From: Helber Belmiro Date: Tue, 3 Sep 2024 15:42:14 -0300 Subject: [PATCH 31/54] test: Improvements to wait_for_pods function (#11162) Signed-off-by: hbelmiro Signed-off-by: KevinGrantLee --- .github/workflows/backend.yml | 8 +- .github/workflows/e2e-test.yml | 30 ++++++++ .../kfp-kubernetes-execution-tests.yml | 11 +-- .github/workflows/kfp-samples.yml | 3 +- .../kubeflow-pipelines-integration-v2.yml | 1 + .github/workflows/periodic.yml | 4 + .github/workflows/sdk-execution.yml | 11 +-- .github/workflows/upgrade-test.yml | 6 ++ scripts/deploy/github/deploy-kfp-tekton.sh | 6 +- scripts/deploy/github/deploy-kfp.sh | 6 +- scripts/deploy/github/helper-functions.sh | 54 +------------ .../github/kfp-readiness/wait_for_pods.py | 76 +++++++++++++++++++ 12 files changed, 142 insertions(+), 74 deletions(-) create mode 100644 scripts/deploy/github/kfp-readiness/wait_for_pods.py diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index e11c177026d..faef43a0042 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -34,17 +34,17 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - - name: Create KFP cluster - uses: ./.github/actions/kfp-tekton-cluster - - name: Set up Python 3.10 + - name: Set up Python 3.9 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.9' - name: Install sdk run: | python3 -m venv .venv . .venv/bin/activate pip install -e sdk/python + - name: Create KFP cluster + uses: ./.github/actions/kfp-tekton-cluster - name: "flip coin test" run: | . .venv/bin/activate diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 9bb0ae250a7..bc7783bafda 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -23,6 +23,11 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Create KFP cluster uses: ./.github/actions/kfp-cluster @@ -46,6 +51,11 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Create KFP cluster uses: ./.github/actions/kfp-cluster @@ -69,6 +79,11 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Create KFP cluster uses: ./.github/actions/kfp-cluster @@ -92,6 +107,11 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Create KFP cluster uses: ./.github/actions/kfp-cluster @@ -115,6 +135,11 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Create KFP cluster uses: ./.github/actions/kfp-cluster @@ -144,6 +169,11 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Create KFP cluster uses: ./.github/actions/kfp-cluster diff --git a/.github/workflows/kfp-kubernetes-execution-tests.yml b/.github/workflows/kfp-kubernetes-execution-tests.yml index f382c1d8324..92c60ccf43d 100644 --- a/.github/workflows/kfp-kubernetes-execution-tests.yml +++ b/.github/workflows/kfp-kubernetes-execution-tests.yml @@ -7,6 +7,7 @@ on: pull_request: paths: - '.github/workflows/kfp-kubernetes-execution-tests.yml' + - 'scripts/deploy/github/**' - 'sdk/python/**' - 'api/v2alpha1/**' - 'kubernetes_platform/**' @@ -18,17 +19,17 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + - name: Create KFP cluster uses: ./.github/actions/kfp-cluster - name: Forward API port run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: apt-get update run: sudo apt-get update diff --git a/.github/workflows/kfp-samples.yml b/.github/workflows/kfp-samples.yml index 08cbed5d14a..e63619f2574 100644 --- a/.github/workflows/kfp-samples.yml +++ b/.github/workflows/kfp-samples.yml @@ -6,6 +6,7 @@ on: - master pull_request: paths: + - 'scripts/deploy/github/**' - 'samples/**' - 'backend/src/v2/**' - '.github/workflows/kfp-samples.yml' @@ -21,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Create KFP cluster uses: ./.github/actions/kfp-cluster diff --git a/.github/workflows/kubeflow-pipelines-integration-v2.yml b/.github/workflows/kubeflow-pipelines-integration-v2.yml index 5de0b55b937..0afa4161701 100644 --- a/.github/workflows/kubeflow-pipelines-integration-v2.yml +++ b/.github/workflows/kubeflow-pipelines-integration-v2.yml @@ -7,6 +7,7 @@ on: pull_request: paths: - '.github/workflows/kubeflow-pipelines-integration-v2.yml' + - 'scripts/deploy/github/**' - 'samples' - 'core' - 'backend' diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml index 2f81dde8347..be95ad737e5 100644 --- a/.github/workflows/periodic.yml +++ b/.github/workflows/periodic.yml @@ -10,6 +10,10 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 - name: Create KFP cluster uses: ./.github/actions/kfp-cluster - name: Port forward kfp apiserver diff --git a/.github/workflows/sdk-execution.yml b/.github/workflows/sdk-execution.yml index 22f605fca9f..b646534fa68 100644 --- a/.github/workflows/sdk-execution.yml +++ b/.github/workflows/sdk-execution.yml @@ -7,6 +7,7 @@ on: pull_request: paths: - '.github/workflows/sdk-execution.yml' + - 'scripts/deploy/github/**' - 'sdk/python/**' - 'api/v2alpha1/**' @@ -17,17 +18,17 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Create KFP cluster uses: ./.github/actions/kfp-cluster - name: Forward API port run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: apt-get update run: sudo apt-get update diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml index 83b13c9af5c..bd40b868549 100644 --- a/.github/workflows/upgrade-test.yml +++ b/.github/workflows/upgrade-test.yml @@ -7,6 +7,7 @@ on: pull_request: paths: - '.github/workflows/upgrade-test.yml' + - 'scripts/deploy/github/**' - 'backend/**' - 'manifests/kustomize/**' @@ -17,6 +18,11 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Create KFP cluster uses: ./.github/actions/kfp-cluster diff --git a/scripts/deploy/github/deploy-kfp-tekton.sh b/scripts/deploy/github/deploy-kfp-tekton.sh index c4e39633394..17df6278df4 100755 --- a/scripts/deploy/github/deploy-kfp-tekton.sh +++ b/scripts/deploy/github/deploy-kfp-tekton.sh @@ -40,16 +40,14 @@ then exit 1 fi -# Check if all pods are running - allow 20 retries (10 minutes) -wait_for_pods kubeflow 40 30 || EXIT_CODE=$? +# Check if all pods are running - (10 minutes) +wait_for_pods || EXIT_CODE=$? if [[ $EXIT_CODE -ne 0 ]] then echo "Deploy unsuccessful. Not all pods running." exit 1 fi -echo "List Kubeflow: " -kubectl get pod -n kubeflow collect_artifacts kubeflow echo "List Tekton control plane: " diff --git a/scripts/deploy/github/deploy-kfp.sh b/scripts/deploy/github/deploy-kfp.sh index 5da00d75219..6acd46293ba 100755 --- a/scripts/deploy/github/deploy-kfp.sh +++ b/scripts/deploy/github/deploy-kfp.sh @@ -41,16 +41,14 @@ then exit 1 fi -# Check if all pods are running - allow 20 retries (10 minutes) -wait_for_pods kubeflow 40 30 || EXIT_CODE=$? +# Check if all pods are running - (10 minutes) +wait_for_pods || EXIT_CODE=$? if [[ $EXIT_CODE -ne 0 ]] then echo "Deploy unsuccessful. Not all pods running." exit 1 fi -echo "List Kubeflow: " -kubectl get pod -n kubeflow collect_artifacts kubeflow echo "Finished KFP deployment." diff --git a/scripts/deploy/github/helper-functions.sh b/scripts/deploy/github/helper-functions.sh index 8c42a923f46..f7780feca3a 100644 --- a/scripts/deploy/github/helper-functions.sh +++ b/scripts/deploy/github/helper-functions.sh @@ -56,57 +56,9 @@ wait_for_namespace () { } wait_for_pods () { - if [[ $# -ne 3 ]] - then - echo "Usage: wait_for_pods namespace max_retries sleep_time" - return 1 - fi - - local namespace=$1 - local max_retries=$2 - local sleep_time=$3 - - local i=0 - - while [[ $i -lt $max_retries ]] - do - local pods - local statuses - local num_pods - local num_running - pods=$(kubectl get pod -n "$namespace") - # echo "$pods" - # kubectl get pvc -n "$namespace" - - if [[ -z $pods ]] - then - echo "no pod is up yet" - else - # Using quotations around variables to keep column format in echo - # Remove 1st line (header line) -> trim whitespace -> cut statuses column (3rd column) - # Might be overkill to parse down to specific columns :). - statuses=$(echo "$pods" | tail -n +2 | tr -s ' ' | cut -d ' ' -f 3) - num_pods=$(echo "$statuses" | wc -l | xargs) - num_running=$(echo "$statuses" | grep -ow "Running\|Completed" | wc -l | xargs) - - local msg="${num_running}/${num_pods} pods running in \"${namespace}\"." - - if [[ $num_running -ne $num_pods ]] - then - # for debugging - # kubectl get pod -n "$namespace" | grep '0/1' | awk '{print $1}' | xargs kubectl describe pod -n "$namespace" - echo "$msg Checking again in ${sleep_time}s." - else - echo "$msg" - return 0 - fi - fi - - sleep "$sleep_time" - i=$((i+1)) - done - - return 1 + C_DIR="${BASH_SOURCE%/*}" + pip install -r "${C_DIR}"/../../../sdk/python/requirements.txt + python "${C_DIR}"/kfp-readiness/wait_for_pods.py } deploy_with_retries () { diff --git a/scripts/deploy/github/kfp-readiness/wait_for_pods.py b/scripts/deploy/github/kfp-readiness/wait_for_pods.py new file mode 100644 index 00000000000..3a61086afbb --- /dev/null +++ b/scripts/deploy/github/kfp-readiness/wait_for_pods.py @@ -0,0 +1,76 @@ +import logging +import time +import urllib3 +import sys +from kubernetes import client, config + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + +namespace = 'kubeflow' + +config.load_kube_config() +v1 = client.CoreV1Api() + + +def get_pod_statuses(): + pods = v1.list_namespaced_pod(namespace=namespace) + statuses = {} + for pod in pods.items: + pod_name = pod.metadata.name + pod_status = pod.status.phase + container_statuses = pod.status.container_statuses or [] + ready_containers = sum(1 for status in container_statuses if status.ready) + total_containers = len(container_statuses) + statuses[pod_name] = (pod_status, ready_containers, total_containers) + return statuses + + +def all_pods_ready(statuses): + return all(pod_status == 'Running' and ready == total + for pod_status, ready, total in statuses.values()) + + +def check_pods(calm_time=10, timeout=600, retries_after_ready=5): + start_time = time.time() + stable_count = 0 + previous_statuses = {} + + while time.time() - start_time < timeout: + current_statuses = get_pod_statuses() + + logging.info("Checking pod statuses...") + for pod_name, (pod_status, ready, total) in current_statuses.items(): + logging.info(f"Pod {pod_name} - Status: {pod_status}, Ready: {ready}/{total}") + + if current_statuses == previous_statuses: + if all_pods_ready(current_statuses): + stable_count += 1 + if stable_count >= retries_after_ready: + logging.info("All pods are calm and fully ready.") + break + else: + logging.info( + f"Pods are calm but have only been stable for {stable_count}/{retries_after_ready} retries.") + else: + stable_count = 0 + else: + stable_count = 0 + + previous_statuses = current_statuses + logging.info(f"Pods are still stabilizing. Retrying in {calm_time} seconds...") + time.sleep(calm_time) + else: + raise Exception("Pods did not stabilize within the timeout period.") + + logging.info("Final pod statuses:") + for pod_name, (pod_status, ready, total) in previous_statuses.items(): + if pod_status == 'Running' and ready == total: + logging.info(f"Pod {pod_name} is fully ready ({ready}/{total})") + else: + logging.info(f"Pod {pod_name} is not ready (Status: {pod_status}, Ready: {ready}/{total})") + + +if __name__ == "__main__": + check_pods() From 4caba7bb837e6e3d427c8e320c534ce1dac7cae4 Mon Sep 17 00:00:00 2001 From: ElayAharoni <62550608+ElayAharoni@users.noreply.github.com> Date: Wed, 4 Sep 2024 02:21:15 +0300 Subject: [PATCH 32/54] fix(frontend): fixes filter pipeline text box shows error when typing anything in it. Fixes #10241 (#11096) * Filter pipeline text box shows error when typing anything in it #10241 Signed-off-by: Elay Aharoni (EXT-Nokia) * Filter pipeline text box shows error when typing anything in it #10241 Signed-off-by: Elay Aharoni (EXT-Nokia) --------- Signed-off-by: Elay Aharoni (EXT-Nokia) Co-authored-by: Elay Aharoni (EXT-Nokia) Signed-off-by: KevinGrantLee --- frontend/src/pages/ResourceSelector.tsx | 2 +- frontend/src/pages/__snapshots__/ResourceSelector.test.tsx.snap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/ResourceSelector.tsx b/frontend/src/pages/ResourceSelector.tsx index be81359edee..44bd8973d88 100644 --- a/frontend/src/pages/ResourceSelector.tsx +++ b/frontend/src/pages/ResourceSelector.tsx @@ -78,7 +78,7 @@ class ResourceSelector extends React.Component} Date: Tue, 3 Sep 2024 19:24:14 -0400 Subject: [PATCH 33/54] correct artifact preview behavior in UI (#11059) This change allows KFP UI to fallback to UI host namespace when no namespaces are provided when referencing the artifact object store provider secret, in default kubeflow deployments this namespace is simply "kubeflow", however the user can customize this behavior by providing the environment variable "SERVER_NAMESPACE" to the KFP UI deployment. Further more, this change addresses a bug that caused URL parse to fail when parsing endpoints without a protocol, this will support such endpoint types as : for object store endpoints, as is the case in the default kfp deployment manifests. Signed-off-by: Humair Khan Signed-off-by: KevinGrantLee --- frontend/server/app.ts | 2 + frontend/server/configs.ts | 4 + frontend/server/handlers/artifacts.ts | 12 ++- .../integration-tests/artifact-get.test.ts | 95 +++++++++++++++++-- frontend/server/minio-helper.ts | 8 +- .../pipeline/ml-pipeline-ui-deployment.yaml | 4 + 6 files changed, 109 insertions(+), 16 deletions(-) diff --git a/frontend/server/app.ts b/frontend/server/app.ts index f6ae1988bf4..50d8565772c 100644 --- a/frontend/server/app.ts +++ b/frontend/server/app.ts @@ -143,6 +143,7 @@ function createUIServer(options: UIConfigs) { artifactsConfigs: options.artifacts, useParameter: false, tryExtract: true, + options: options, }), ); // /artifacts/ endpoint downloads the artifact as is, it does not try to unzip or untar. @@ -158,6 +159,7 @@ function createUIServer(options: UIConfigs) { artifactsConfigs: options.artifacts, useParameter: true, tryExtract: false, + options: options, }), ); diff --git a/frontend/server/configs.ts b/frontend/server/configs.ts index 7108db23df9..4188338498f 100644 --- a/frontend/server/configs.ts +++ b/frontend/server/configs.ts @@ -123,6 +123,7 @@ export function loadConfigs(argv: string[], env: ProcessEnv): UIConfigs { * e.g. a valid header value for default values can be like `accounts.google.com:user@gmail.com`. */ KUBEFLOW_USERID_PREFIX = 'accounts.google.com:', + FRONTEND_SERVER_NAMESPACE = 'kubeflow', } = env; return { @@ -190,6 +191,7 @@ export function loadConfigs(argv: string[], env: ProcessEnv): UIConfigs { : asBool(HIDE_SIDENAV), port, staticDir, + serverNamespace: FRONTEND_SERVER_NAMESPACE, }, viewer: { tensorboard: { @@ -266,6 +268,8 @@ export interface ServerConfigs { apiVersion2Prefix: string; deployment: Deployments; hideSideNav: boolean; + // Namespace where the server is deployed + serverNamespace: string; } export interface GkeMetadataConfigs { disabled: boolean; diff --git a/frontend/server/handlers/artifacts.ts b/frontend/server/handlers/artifacts.ts index d5bea2cbf6a..2f04e232b66 100644 --- a/frontend/server/handlers/artifacts.ts +++ b/frontend/server/handlers/artifacts.ts @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. import fetch from 'node-fetch'; -import { AWSConfigs, HttpConfigs, MinioConfigs, ProcessEnv } from '../configs'; +import { AWSConfigs, HttpConfigs, MinioConfigs, ProcessEnv, UIConfigs } from '../configs'; import { Client as MinioClient } from 'minio'; import { PreviewStream, findFileOnPodVolume, parseJSONString } from '../utils'; import { createMinioClient, getObjectStream } from '../minio-helper'; @@ -80,6 +80,7 @@ export function getArtifactsHandler({ artifactsConfigs, useParameter, tryExtract, + options, }: { artifactsConfigs: { aws: AWSConfigs; @@ -89,15 +90,18 @@ export function getArtifactsHandler({ }; tryExtract: boolean; useParameter: boolean; + options: UIConfigs; }): Handler { const { aws, http, minio, allowedDomain } = artifactsConfigs; return async (req, res) => { const source = useParameter ? req.params.source : req.query.source; const bucket = useParameter ? req.params.bucket : req.query.bucket; const key = useParameter ? req.params[0] : req.query.key; - const { peek = 0, providerInfo = '', namespace = '' } = req.query as Partial< - ArtifactsQueryStrings - >; + const { + peek = 0, + providerInfo = '', + namespace = options.server.serverNamespace, + } = req.query as Partial; if (!source) { res.status(500).send('Storage source is missing from artifact request'); return; diff --git a/frontend/server/integration-tests/artifact-get.test.ts b/frontend/server/integration-tests/artifact-get.test.ts index d104e1a8e63..bbe37f849be 100644 --- a/frontend/server/integration-tests/artifact-get.test.ts +++ b/frontend/server/integration-tests/artifact-get.test.ts @@ -184,8 +184,11 @@ describe('/artifacts', () => { }); }); - it('responds error when source is s3, and creds are sourced from Provider Configs, but no namespace is provided', done => { + it('responds with artifact if source is AWS S3, and creds are sourced from Provider Configs, and uses default kubeflow namespace when no namespace is provided', done => { const mockedGetK8sSecret: jest.Mock = getK8sSecret as any; + mockedGetK8sSecret.mockResolvedValue('somevalue'); + const mockedMinioClient: jest.Mock = minio.Client as any; + const namespace = 'kubeflow'; const configs = loadConfigs(argv, {}); app = new UIServer(configs); const request = requests(app.start()); @@ -203,18 +206,90 @@ describe('/artifacts', () => { }; request .get( - `/artifacts/get?source=s3&bucket=ml-pipeline&key=hello%2Fworld.txt$&providerInfo=${JSON.stringify( + `/artifacts/get?source=s3&bucket=ml-pipeline&key=hello%2Fworld.txt&providerInfo=${JSON.stringify( + providerInfo, + )}`, + ) + .expect(200, artifactContent, err => { + expect(mockedMinioClient).toBeCalledWith({ + accessKey: 'somevalue', + endPoint: 's3.amazonaws.com', + port: undefined, + region: 'us-east-2', + secretKey: 'somevalue', + useSSL: undefined, + }); + expect(mockedMinioClient).toBeCalledTimes(1); + expect(mockedGetK8sSecret).toBeCalledTimes(2); + expect(mockedGetK8sSecret).toHaveBeenNthCalledWith( + 1, + 'aws-s3-creds', + 'AWS_ACCESS_KEY_ID', + `${namespace}`, + ); + expect(mockedGetK8sSecret).toHaveBeenNthCalledWith( + 2, + 'aws-s3-creds', + 'AWS_SECRET_ACCESS_KEY', + `${namespace}`, + ); + expect(mockedGetK8sSecret).toBeCalledTimes(2); + done(err); + }); + }); + + it('responds with artifact if source is AWS S3, and creds are sourced from Provider Configs, and uses default namespace when no namespace is provided, as specified in ENV', done => { + const mockedGetK8sSecret: jest.Mock = getK8sSecret as any; + mockedGetK8sSecret.mockResolvedValue('somevalue'); + const mockedMinioClient: jest.Mock = minio.Client as any; + const namespace = 'notkubeflow'; + const configs = loadConfigs(argv, { FRONTEND_SERVER_NAMESPACE: namespace }); + app = new UIServer(configs); + const request = requests(app.start()); + const providerInfo = { + Params: { + accessKeyKey: 'AWS_ACCESS_KEY_ID', + disableSSL: 'false', + endpoint: 's3.amazonaws.com', + fromEnv: 'false', + region: 'us-east-2', + secretKeyKey: 'AWS_SECRET_ACCESS_KEY', + secretName: 'aws-s3-creds', + }, + Provider: 's3', + }; + request + .get( + `/artifacts/get?source=s3&bucket=ml-pipeline&key=hello%2Fworld.txt&providerInfo=${JSON.stringify( providerInfo, )}`, ) - .expect( - 500, - 'Failed to initialize Minio Client for S3 Provider: Error: Artifact Store provider given, but no namespace provided.', - err => { - expect(mockedGetK8sSecret).toBeCalledTimes(0); - done(err); - }, - ); + .expect(200, artifactContent, err => { + expect(mockedMinioClient).toBeCalledWith({ + accessKey: 'somevalue', + endPoint: 's3.amazonaws.com', + port: undefined, + region: 'us-east-2', + secretKey: 'somevalue', + useSSL: undefined, + }); + expect(mockedMinioClient).toBeCalledTimes(1); + expect(mockedGetK8sSecret).toBeCalledTimes(2); + expect(mockedGetK8sSecret).toHaveBeenNthCalledWith( + 1, + 'aws-s3-creds', + 'AWS_ACCESS_KEY_ID', + `${namespace}`, + ); + expect(mockedGetK8sSecret).toHaveBeenNthCalledWith( + 2, + 'aws-s3-creds', + 'AWS_SECRET_ACCESS_KEY', + `${namespace}`, + ); + expect(mockedGetK8sSecret).toBeCalledTimes(2); + done(err); + }); }); it('responds with artifact if source is s3-compatible, and creds are sourced from Provider Configs', done => { diff --git a/frontend/server/minio-helper.ts b/frontend/server/minio-helper.ts index 5f244800f2a..9295f6cbcf6 100644 --- a/frontend/server/minio-helper.ts +++ b/frontend/server/minio-helper.ts @@ -78,7 +78,7 @@ export async function createMinioClient( } // If using s3 and sourcing credentials from environment (currently only aws is supported) - if (providerType === 's3' && (!config.accessKey || !config.secretKey)) { + if (providerType === 's3' && !(config.accessKey && config.secretKey)) { // AWS S3 with credentials from provider chain if (isAWSS3Endpoint(config.endPoint)) { try { @@ -168,7 +168,11 @@ async function parseS3ProviderInfo( config.useSSL = undefined; } else { if (providerInfo.Params.endpoint) { - const parseEndpoint = new URL(providerInfo.Params.endpoint); + const url = providerInfo.Params.endpoint; + // this is a bit of a hack to add support for endpoints without a protocol (required by WHATWG URL standard) + // example: : format. In general should expect most endpoints to provide a protocol as serviced + // by the backend + const parseEndpoint = new URL(url.startsWith('http') ? url : `https://${url}`); const host = parseEndpoint.hostname; const port = parseEndpoint.port; config.endPoint = host; diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml index 8e8923a3659..adfcfc9f928 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml @@ -48,6 +48,10 @@ spec: key: secretkey - name: ALLOW_CUSTOM_VISUALIZATIONS value: "true" + - name: FRONTEND_SERVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace readinessProbe: exec: command: From 18e0695512f21994c34960dc5e905eb6668616b6 Mon Sep 17 00:00:00 2001 From: Helber Belmiro Date: Fri, 6 Sep 2024 09:49:16 -0300 Subject: [PATCH 34/54] chore: Added DCO link to PR template (#11176) Signed-off-by: Helber Belmiro Signed-off-by: KevinGrantLee --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ddf0e2deeca..d6525e272fd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -2,6 +2,7 @@ **Checklist:** +- [ ] You have [signed off your commits](https://www.kubeflow.org/docs/about/contributing/#sign-off-your-commits) - [ ] The title for your pull request (PR) should follow our title convention. [Learn more about the pull request title convention used in this repository](https://github.com/kubeflow/pipelines/blob/master/CONTRIBUTING.md#pull-request-title-convention).