diff --git a/.github/workflows/kfp-kubernetes-execution-tests.yml b/.github/workflows/kfp-kubernetes-execution-tests.yml new file mode 100644 index 00000000000..f382c1d8324 --- /dev/null +++ b/.github/workflows/kfp-kubernetes-execution-tests.yml @@ -0,0 +1,72 @@ +name: k8s execution tests + +on: + push: + branches: [master] + + pull_request: + paths: + - '.github/workflows/kfp-kubernetes-execution-tests.yml' + - 'sdk/python/**' + - 'api/v2alpha1/**' + - 'kubernetes_platform/**' + +jobs: + kfp-kubernetes-execution-tests: + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create KFP cluster + uses: ./.github/actions/kfp-cluster + + - name: Forward API port + run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: apt-get update + run: sudo apt-get update + + - name: Install protobuf-compiler + run: sudo apt-get install protobuf-compiler -y + + - name: Install setuptools + run: | + pip3 install setuptools + pip3 freeze + + - name: Install Wheel + run: pip3 install wheel==0.42.0 + + - name: Install protobuf + run: pip3 install protobuf==4.25.3 + + - name: Generate API proto files + working-directory: ./api + run: make clean python + + - name: Install kfp-pipeline-spec from source + run: | + python3 -m pip install api/v2alpha1/python + + - name: Generate kfp-kubernetes proto files from source + working-directory: ./kubernetes_platform + run: make clean python + + - name: Install kfp-kubernetes from source + run: | + pip install -e ./kubernetes_platform/python[dev] + + - name: Install requirements + run: pip install -r ./test/kfp-kubernetes-execution-tests/requirements.txt + + - name: Run tests + run: | + export KFP_ENDPOINT="http://localhost:8888" + export TIMEOUT_SECONDS=2700 + pytest ./test/kfp-kubernetes-execution-tests/sdk_execution_tests.py --asyncio-task-timeout $TIMEOUT_SECONDS diff --git a/kubernetes_platform/python/generate_proto.py b/kubernetes_platform/python/generate_proto.py index 5d7aa52f5f6..4349ef8f71e 100644 --- a/kubernetes_platform/python/generate_proto.py +++ b/kubernetes_platform/python/generate_proto.py @@ -65,6 +65,7 @@ def generate_proto(source: str) -> None: protoc_command = [ PROTOC, f'-I={PROTO_DIR}', + f'--experimental_allow_proto3_optional', f'--python_out={PKG_DIR}', source, ] diff --git a/kubernetes_platform/python/test/snapshot/test_data_config.yaml b/kubernetes_platform/python/test/snapshot/test_data_config.yaml index 1246d71e5e2..4ff7b681c1b 100644 --- a/kubernetes_platform/python/test/snapshot/test_data_config.yaml +++ b/kubernetes_platform/python/test/snapshot/test_data_config.yaml @@ -14,15 +14,23 @@ test_cases: - - module: create_mount_delete_dynamic_pvc - name: my_pipeline +# Disabled while https://github.com/kubeflow/pipelines/issues/10918 is failing +# - module: create_mount_delete_dynamic_pvc +# name: my_pipeline + - module: create_mount_delete_existing_pvc name: my_pipeline - module: create_mount_delete_existing_pvc_from_task_output name: my_pipeline - - module: secret_as_env - name: my_pipeline - - module: secret_as_vol - name: my_pipeline - - module: node_selector - name: my_pipeline + +# Disabled while https://github.com/kubeflow/pipelines/issues/10918 is failing +# - module: secret_as_env +# name: my_pipeline + +# Disabled while https://github.com/kubeflow/pipelines/issues/10918 is failing +# - module: secret_as_vol +# name: my_pipeline + +# Disabled while https://github.com/kubeflow/pipelines/issues/10918 is failing +# - module: node_selector +# name: my_pipeline diff --git a/test/presubmit-kfp-kubernetes-execution-tests.sh b/test/presubmit-kfp-kubernetes-execution-tests.sh deleted file mode 100755 index 0fa2327fbef..00000000000 --- a/test/presubmit-kfp-kubernetes-execution-tests.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -ex -# Copyright 2023 Kubeflow Pipelines contributors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source_root=$(pwd) - -python3 -m pip install --upgrade pip -python3 -m pip install $source_root/sdk/python -apt-get update && apt-get install -y protobuf-compiler -# install kfp-pipeline-spec from source -pushd api -make clean python -popd -python3 -m pip install api/v2alpha1/python - -# generate kfp-kubernetes proto files from source -pushd "$source_root/kubernetes_platform" -make clean python -popd - -# install kfp-kubernetes from source -# rust needed for transitive deps in dev extras on Python:3.12 -apt-get install rustc -y -pip install -e $source_root/kubernetes_platform/python[dev] - -pip install -r $source_root/test/kfp-kubernetes-execution-tests/requirements.txt - -export KFP_ENDPOINT="https://$(curl https://raw.githubusercontent.com/kubeflow/testing/master/test-infra/kfp/endpoint)" -export TIMEOUT_SECONDS=2700 -pytest $source_root/test/kfp-kubernetes-execution-tests/sdk_execution_tests.py --asyncio-task-timeout $TIMEOUT_SECONDS