From c643f72b6955b17e9256ec0426e7cf81ddfc1eb5 Mon Sep 17 00:00:00 2001 From: Amanpreet Singh Bedi Date: Sat, 31 Aug 2024 05:26:45 +0530 Subject: [PATCH 1/9] add gcpc modules tests to gha Signed-off-by: Amanpreet Singh Bedi --- .github/workflows/gcpc-modules-tests.yml | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/gcpc-modules-tests.yml diff --git a/.github/workflows/gcpc-modules-tests.yml b/.github/workflows/gcpc-modules-tests.yml new file mode 100644 index 00000000000..7907bf2edc6 --- /dev/null +++ b/.github/workflows/gcpc-modules-tests.yml @@ -0,0 +1,56 @@ +name: GCPC modules test + +on: + push: + branches: [master] + + pull_request: + paths: + - '.github/workflows/run-all-gcpc-modules.yml' + - 'sdk/python/**' + +jobs: + all-gcpc-tests: + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: apt-get update + run: sudo apt-get update + + - name: Install protobuf-compiler + run: sudo apt-get install protobuf-compiler -y + + - name: Install setuptools + run: | + pip3 install setuptools + pip3 freeze + + - name: Install Wheel + run: pip3 install wheel==0.42.0 + + - name: Install python sdk + run: pip install sdk/python + + - name: Generate API proto files + working-directory: ./api + run: make clean python + + - name: Install kfp-pipeline-spec from source + run: | + python3 -m pip install api/v2alpha1/python + + - name: Install google-cloud component + run: pip install components/google-cloud + + - name: Install Pytest + run: pip install $(grep 'pytest==' sdk/python/requirements-dev.txt) + + - name: Run test + run: pytest ./test/gcpc-tests/run_all_gcpc_modules.py From 03fdd31a7f1b007d9a4650a9152c1b27dcd55a0b Mon Sep 17 00:00:00 2001 From: Amanpreet Singh Bedi Date: Sat, 31 Aug 2024 05:27:18 +0530 Subject: [PATCH 2/9] remove run-all-gcpc-modules test driver script Signed-off-by: Amanpreet Singh Bedi --- test/presubmit-test-run-all-gcpc-modules.sh | 28 --------------------- 1 file changed, 28 deletions(-) delete mode 100755 test/presubmit-test-run-all-gcpc-modules.sh diff --git a/test/presubmit-test-run-all-gcpc-modules.sh b/test/presubmit-test-run-all-gcpc-modules.sh deleted file mode 100755 index c47d92bc4cd..00000000000 --- a/test/presubmit-test-run-all-gcpc-modules.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -ex -# Copyright 2023 Kubeflow Pipelines contributors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source_root=$(pwd) - -pip install --upgrade pip -pip install $source_root/sdk/python -apt-get update && apt-get install -y protobuf-compiler -pushd api -make clean python -popd -python3 -m pip install api/v2alpha1/python -pip install components/google-cloud -pip install $(grep 'pytest==' sdk/python/requirements-dev.txt) - -pytest test/gcpc-tests/run_all_gcpc_modules.py From 32f22e186428134a9c8dd62a0fb2aaca19c764f5 Mon Sep 17 00:00:00 2001 From: Amanpreet Singh Bedi Date: Sat, 31 Aug 2024 05:41:44 +0530 Subject: [PATCH 3/9] fix path under gcpc modules tests github action Signed-off-by: Amanpreet Singh Bedi --- .github/workflows/gcpc-modules-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gcpc-modules-tests.yml b/.github/workflows/gcpc-modules-tests.yml index 7907bf2edc6..e70df4064bf 100644 --- a/.github/workflows/gcpc-modules-tests.yml +++ b/.github/workflows/gcpc-modules-tests.yml @@ -6,7 +6,7 @@ on: pull_request: paths: - - '.github/workflows/run-all-gcpc-modules.yml' + - '.github/workflows/gcpc-modules-tests.yml' - 'sdk/python/**' jobs: From 97b7acc204fc827d1c9303e66972d3855cc15da4 Mon Sep 17 00:00:00 2001 From: Amanpreet Singh Bedi Date: Sun, 1 Sep 2024 00:28:53 +0530 Subject: [PATCH 4/9] some test changes Signed-off-by: Amanpreet Singh Bedi --- .github/workflows/backend-visualization.yml | 26 --- .github/workflows/backend.yml | 73 -------- .github/workflows/codeql.yml | 72 -------- .github/workflows/e2e-test.yml | 167 ------------------ .github/workflows/frontend.yml | 35 ---- .github/workflows/gcpc-modules-tests.yml | 5 +- .../kfp-kubernetes-execution-tests.yml | 72 -------- .../workflows/kfp-kubernetes-library-test.yml | 36 ---- .github/workflows/kfp-samples.yml | 34 ---- .github/workflows/kfp-sdk-runtime-tests.yml | 31 ---- .github/workflows/kfp-sdk-tests.yml | 30 ---- .../kubeflow-pipelines-integration-v2.yml | 38 ---- .../kubeflow-pipelines-manifests.yml | 19 -- .github/workflows/periodic.yml | 27 --- .github/workflows/presubmit-backend.yml | 28 --- .github/workflows/rerun-pr-jobs.yml | 16 -- .github/workflows/sdk-component-yaml.yml | 56 ------ .github/workflows/sdk-docformatter.yml | 26 --- .github/workflows/sdk-execution.yml | 63 ------- .github/workflows/sdk-isort.yml | 26 --- .github/workflows/sdk-upgrade.yml | 26 --- .github/workflows/sdk-yapf.yml | 36 ---- .github/workflows/stale.yml | 37 ---- .github/workflows/upgrade-test.yml | 37 ---- 24 files changed, 4 insertions(+), 1012 deletions(-) delete mode 100644 .github/workflows/backend-visualization.yml delete mode 100644 .github/workflows/backend.yml delete mode 100644 .github/workflows/codeql.yml delete mode 100644 .github/workflows/e2e-test.yml delete mode 100644 .github/workflows/frontend.yml delete mode 100644 .github/workflows/kfp-kubernetes-execution-tests.yml delete mode 100644 .github/workflows/kfp-kubernetes-library-test.yml delete mode 100644 .github/workflows/kfp-samples.yml delete mode 100644 .github/workflows/kfp-sdk-runtime-tests.yml delete mode 100644 .github/workflows/kfp-sdk-tests.yml delete mode 100644 .github/workflows/kubeflow-pipelines-integration-v2.yml delete mode 100644 .github/workflows/kubeflow-pipelines-manifests.yml delete mode 100644 .github/workflows/periodic.yml delete mode 100644 .github/workflows/presubmit-backend.yml delete mode 100644 .github/workflows/rerun-pr-jobs.yml delete mode 100644 .github/workflows/sdk-component-yaml.yml delete mode 100644 .github/workflows/sdk-docformatter.yml delete mode 100644 .github/workflows/sdk-execution.yml delete mode 100644 .github/workflows/sdk-isort.yml delete mode 100644 .github/workflows/sdk-upgrade.yml delete mode 100644 .github/workflows/sdk-yapf.yml delete mode 100644 .github/workflows/stale.yml delete mode 100644 .github/workflows/upgrade-test.yml diff --git a/.github/workflows/backend-visualization.yml b/.github/workflows/backend-visualization.yml deleted file mode 100644 index 8df4ff0c7a6..00000000000 --- a/.github/workflows/backend-visualization.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: KFP backend visualization tests - -on: - push: - branches: [ master ] - - pull_request: - paths: - - '.github/workflows/backend-visualization.yml' - - 'backend/src/apiserver/visualization/**' - - 'test/presubmit-backend-visualization.sh' - -jobs: - backend-visualization-test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Run tests - run: ./test/presubmit-backend-visualization.sh diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml deleted file mode 100644 index e11c177026d..00000000000 --- a/.github/workflows/backend.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: KFP Tekton backend unit tests - -on: - push: - branches: [master] - - # Run tests for any PRs which change the backend code - pull_request: - paths: - - '.github/workflows/backend.yml' - - 'go.mod' - - 'backend/**' - - 'scripts/deploy/github/**' - - 'manifests/kustomize/**' - -env: - GITHUB_ACTION: "true" - SETUPTOOLS_USE_DISTUTILS: "stdlib" - -jobs: - run-go-unittests: - runs-on: ubuntu-latest - steps: - - name: Install Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - name: Checkout code - uses: actions/checkout@v4 - - name: "run go unit tests" - run: go test -v -cover ./backend/... - backend-integration: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Create KFP cluster - uses: ./.github/actions/kfp-tekton-cluster - - name: Set up Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Install sdk - run: | - python3 -m venv .venv - . .venv/bin/activate - pip install -e sdk/python - - name: "flip coin test" - run: | - . .venv/bin/activate - TEST_SCRIPT="test-flip-coin.sh" ./scripts/deploy/github/e2e-test.sh - - name: "static loop test" - run: | - . .venv/bin/activate - TEST_SCRIPT="test-static-loop.sh" ./scripts/deploy/github/e2e-test.sh - - name: "dynamic loop test" - run: | - . .venv/bin/activate - TEST_SCRIPT="test-dynamic-loop.sh" ./scripts/deploy/github/e2e-test.sh - - name: "use env" - run: | - . .venv/bin/activate - TEST_SCRIPT="test-env.sh" ./scripts/deploy/github/e2e-test.sh - - name: "use volume" - run: | - . .venv/bin/activate - TEST_SCRIPT="test-volume.sh" ./scripts/deploy/github/e2e-test.sh - - name: Collect test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: kfp-tekton-backend-artifacts - path: /tmp/tmp.*/* diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 2f0b0b472e0..00000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,72 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - schedule: - # Every Friday at 19:39 - - cron: '39 19 * * 5' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'go', 'javascript', 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Use only 'java' to analyze code written in Java, Kotlin or both - # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:${{matrix.language}}" diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml deleted file mode 100644 index 9bb0ae250a7..00000000000 --- a/.github/workflows/e2e-test.yml +++ /dev/null @@ -1,167 +0,0 @@ -name: KFP e2e tests - -on: - push: - branches: [master] - - pull_request: - paths: - - '.github/workflows/e2e-test.yml' - - 'scripts/deploy/github/**' - - 'go.mod' - - 'go.sum' - - 'backend/**' - - 'frontend/**' - - 'proxy/**' - - 'manifests/kustomize/**' - - 'test/**' - -jobs: - initialization-tests-v1: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - - name: Initialization tests v1 - working-directory: ./backend/test/initialization - run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true - - - name: Collect test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: kfp-initialization-tests-v1-artifacts - path: /tmp/tmp.*/* - - initialization-tests-v2: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - - name: Initialization tests v2 - working-directory: ./backend/test/v2/initialization - run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true - - - name: Collect test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: kfp-initialization-tests-v2-artifacts - path: /tmp/tmp.*/* - - api-integration-tests-v1: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - - name: API integration tests v1 - working-directory: ./backend/test/integration - run: go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true - - - name: Collect test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: kfp-api-integration-tests-v1-artifacts - path: /tmp/tmp.*/* - - api-integration-tests-v2: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - - name: API integration tests v2 - working-directory: ./backend/test/v2/integration - run: go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true - - - name: Collect test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: kfp-api-integration-tests-v2-artifacts - path: /tmp/tmp.*/* - - frontend-integration-test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - - name: Forward Frontend port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline-ui" 3000 3000 - - - name: Build frontend integration tests image - working-directory: ./test/frontend-integration-test - run: docker build . -t kfp-frontend-integration-test:local - - - name: Frontend integration tests - run: docker run --net=host kfp-frontend-integration-test:local --remote-run true - - - name: Collect test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: kfp-frontend-integration-test-artifacts - path: /tmp/tmp.*/* - - basic-sample-tests: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - - name: Install prerequisites - run: pip3 install -r ./test/sample-test/requirements.txt - - - name: Basic sample tests - sequential - run: python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name sequential --results-gcs-dir output - - - name: Basic sample tests - exit_handler - run: python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name exit_handler --results-gcs-dir output - - - name: Collect test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: kfp-basic-sample-tests-artifacts - path: /tmp/tmp.*/* diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml deleted file mode 100644 index 3e4d8eebe01..00000000000 --- a/.github/workflows/frontend.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Frontend Tests - -on: - push: - branches: - - master - pull_request: - branches: - - master - paths: - - 'frontend/**' - - '.github/workflows/frontend.yml' - -jobs: - frontend-tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: '14' - - - name: Clean npm cache - run: npm cache clean --force - - - name: Install dependencies - run: cd ./frontend && npm ci - - - name: Run Frontend Tests - run: cd ./frontend && npm run test:ci - diff --git a/.github/workflows/gcpc-modules-tests.yml b/.github/workflows/gcpc-modules-tests.yml index e70df4064bf..168f33ee181 100644 --- a/.github/workflows/gcpc-modules-tests.yml +++ b/.github/workflows/gcpc-modules-tests.yml @@ -33,7 +33,10 @@ jobs: pip3 freeze - name: Install Wheel - run: pip3 install wheel==0.42.0 + run: pip3 install wheel==0.42.0 + + - name: Install protobuf + run: pip3 install protobuf==4.25.3 - name: Install python sdk run: pip install sdk/python diff --git a/.github/workflows/kfp-kubernetes-execution-tests.yml b/.github/workflows/kfp-kubernetes-execution-tests.yml deleted file mode 100644 index f382c1d8324..00000000000 --- a/.github/workflows/kfp-kubernetes-execution-tests.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: k8s execution tests - -on: - push: - branches: [master] - - pull_request: - paths: - - '.github/workflows/kfp-kubernetes-execution-tests.yml' - - 'sdk/python/**' - - 'api/v2alpha1/**' - - 'kubernetes_platform/**' - -jobs: - kfp-kubernetes-execution-tests: - runs-on: ubuntu-24.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: apt-get update - run: sudo apt-get update - - - name: Install protobuf-compiler - run: sudo apt-get install protobuf-compiler -y - - - name: Install setuptools - run: | - pip3 install setuptools - pip3 freeze - - - name: Install Wheel - run: pip3 install wheel==0.42.0 - - - name: Install protobuf - run: pip3 install protobuf==4.25.3 - - - name: Generate API proto files - working-directory: ./api - run: make clean python - - - name: Install kfp-pipeline-spec from source - run: | - python3 -m pip install api/v2alpha1/python - - - name: Generate kfp-kubernetes proto files from source - working-directory: ./kubernetes_platform - run: make clean python - - - name: Install kfp-kubernetes from source - run: | - pip install -e ./kubernetes_platform/python[dev] - - - name: Install requirements - run: pip install -r ./test/kfp-kubernetes-execution-tests/requirements.txt - - - name: Run tests - run: | - export KFP_ENDPOINT="http://localhost:8888" - export TIMEOUT_SECONDS=2700 - pytest ./test/kfp-kubernetes-execution-tests/sdk_execution_tests.py --asyncio-task-timeout $TIMEOUT_SECONDS diff --git a/.github/workflows/kfp-kubernetes-library-test.yml b/.github/workflows/kfp-kubernetes-library-test.yml deleted file mode 100644 index f1ac56e1eea..00000000000 --- a/.github/workflows/kfp-kubernetes-library-test.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: kfp-kubernetes library tests - -on: - push: - branches: [master] - pull_request: - paths: - - '.github/workflows/kfp-kubernetes-library-test.yml' - - 'sdk/python/**' - - 'api/v2alpha1/**' - - 'kubernetes_platform/**' - - 'test/presubmit-test-kfp-kubernetes-library.sh' - -jobs: - kfp-kubernetes-library-test: - runs-on: ubuntu-24.04 - strategy: - matrix: - python: [ - { 'version': '3.8' }, - { 'version': '3.9' }, - { 'version': '3.10' }, - { 'version': '3.11' }, - { 'version': '3.12' } - ] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{matrix.python.version}} - - - name: Run tests - run: ./test/presubmit-test-kfp-kubernetes-library.sh diff --git a/.github/workflows/kfp-samples.yml b/.github/workflows/kfp-samples.yml deleted file mode 100644 index 08cbed5d14a..00000000000 --- a/.github/workflows/kfp-samples.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: KFP Samples - -on: - push: - branches: - - master - pull_request: - paths: - - 'samples/**' - - 'backend/src/v2/**' - - '.github/workflows/kfp-samples.yml' - -jobs: - samples: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - - name: Run Samples Tests - run: | - ./backend/src/v2/test/sample-test.sh diff --git a/.github/workflows/kfp-sdk-runtime-tests.yml b/.github/workflows/kfp-sdk-runtime-tests.yml deleted file mode 100644 index 14a419a780f..00000000000 --- a/.github/workflows/kfp-sdk-runtime-tests.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: KFP Runtime Code Tests - -on: - push: - branches: [master] - - pull_request: - paths: - - '.github/workflows/kfp-sdk-runtime-tests.yml' - - 'sdk/python/**' - - 'test/presubmit-test-kfp-runtime-code.sh' - -jobs: - kfp-runtime-tests: - runs-on: ubuntu-24.04 - strategy: - matrix: - python: ['3.8', '3.9', '3.10', '3.11', '3.12'] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Run KFP Runtime Code Tests - run: | - export PULL_NUMBER="${{ github.event.inputs.pull_number || github.event.pull_request.number }}" - ./test/presubmit-test-kfp-runtime-code.sh diff --git a/.github/workflows/kfp-sdk-tests.yml b/.github/workflows/kfp-sdk-tests.yml deleted file mode 100644 index 49ff474bd25..00000000000 --- a/.github/workflows/kfp-sdk-tests.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: KFP SDK Tests - -on: - push: - branches: - - master - pull_request: - paths: - - 'sdk/**' - - '.github/workflows/kfp-sdk-tests.yml' - -jobs: - sdk-tests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8, 3.9, 3.10, 3.11, 3.12] - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} - - - name: Run SDK Tests - run: | - ./test/presubmit-tests-sdk.sh diff --git a/.github/workflows/kubeflow-pipelines-integration-v2.yml b/.github/workflows/kubeflow-pipelines-integration-v2.yml deleted file mode 100644 index 5de0b55b937..00000000000 --- a/.github/workflows/kubeflow-pipelines-integration-v2.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Kubeflow Pipelines V2 integration Tests - -on: - push: - branches: - - master - pull_request: - paths: - - '.github/workflows/kubeflow-pipelines-integration-v2.yml' - - 'samples' - - 'core' - - 'backend' - -jobs: - integration-tests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8] - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - - name: Run the Integration Tests - run: | - ./backend/src/v2/test/integration-test.sh diff --git a/.github/workflows/kubeflow-pipelines-manifests.yml b/.github/workflows/kubeflow-pipelines-manifests.yml deleted file mode 100644 index d494e2e0d0a..00000000000 --- a/.github/workflows/kubeflow-pipelines-manifests.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: KFP Manifests - -on: - push: - branches: [master] - pull_request: - paths: - - '.github/workflows/kubeflow-pipelines-manifests.yml' - - 'manifests/kustomize/**' - -jobs: - kubeflow-pipelines-manifests: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Run tests - run: ./manifests/kustomize/hack/presubmit.sh diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml deleted file mode 100644 index 2f81dde8347..00000000000 --- a/.github/workflows/periodic.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Periodic Functional Tests - -on: - schedule: - - cron: '0 0 * * *' # Run every day at midnight - -jobs: - run_tests: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - name: Port forward kfp apiserver - run: | - nohup kubectl port-forward --namespace kubeflow svc/ml-pipeline 8888:8888 & - - name: Run Functional Tests - run: | - log_dir=$(mktemp -d) - ./test/kfp-functional-test/kfp-functional-test.sh > $log_dir/periodic_tests.txt - - name: Collect test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: periodic-functional-artifacts - path: /tmp/tmp.*/* diff --git a/.github/workflows/presubmit-backend.yml b/.github/workflows/presubmit-backend.yml deleted file mode 100644 index 23adb2aa059..00000000000 --- a/.github/workflows/presubmit-backend.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: KFP Backend Tests - -on: - push: - branches: - - master - pull_request: - branches: - - master - paths: - - 'backend/**' - - 'test/presubmit-backend-test.sh' - -jobs: - backend-tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: '1.21' - - - name: Run Backend Tests - run: ./test/presubmit-backend-test.sh diff --git a/.github/workflows/rerun-pr-jobs.yml b/.github/workflows/rerun-pr-jobs.yml deleted file mode 100644 index 2c4c30eb402..00000000000 --- a/.github/workflows/rerun-pr-jobs.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Re-Run PR tests - -on: - issue_comment: - types: [created] - -jobs: - rerun_pr_tests: - name: rerun_pr_tests - if: ${{ github.event.issue.pull_request }} - runs-on: ubuntu-20.04 - steps: - - uses: estroz/rerun-actions@main - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - comment_id: ${{ github.event.comment.id }} \ No newline at end of file diff --git a/.github/workflows/sdk-component-yaml.yml b/.github/workflows/sdk-component-yaml.yml deleted file mode 100644 index 316ce85caca..00000000000 --- a/.github/workflows/sdk-component-yaml.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: KFP Component YAML Test - -on: - push: - branches: [master] - - pull_request: - paths: - - 'components/**/*.yaml' - - 'test/presubmit-component-yaml.sh' - - 'sdk/python/**' - - 'api/v2alpha1/**' - - '.github/workflows/sdk-component-yaml.yml' - -jobs: - test-component-yaml-kfp: - runs-on: ubuntu-24.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: apt-get update - run: sudo apt-get update - - - name: Install protobuf-compiler - run: sudo apt-get install protobuf-compiler -y - - - name: Install setuptools - run: | - pip3 install setuptools - pip3 freeze - - - name: Install Wheel - run: pip3 install wheel==0.42.0 - - - name: Install protobuf - run: pip3 install protobuf==4.25.3 - - - name: Generate API proto files - working-directory: ./api - run: make clean python - - - name: Install kfp-pipeline-spec from source - run: | - python3 -m pip install api/v2alpha1/python - - - name: Install requirements - run: pip install -r ./test/sdk-execution-tests/requirements.txt - - - name: Run component YAML tests - run: ./test/presubmit-component-yaml.sh diff --git a/.github/workflows/sdk-docformatter.yml b/.github/workflows/sdk-docformatter.yml deleted file mode 100644 index 83b583ca3aa..00000000000 --- a/.github/workflows/sdk-docformatter.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: KFP SDK Docformatter Test - -on: - push: - branches: [master] - - pull_request: - paths: - - 'sdk/python/**' - - 'test/presubmit-docformatter-sdk.sh' - - '.github/workflows/sdk-docformatter.yml' - -jobs: - test-docformatter-kfp-sdk: - runs-on: ubuntu-24.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Run docformatter tests - run: ./test/presubmit-docformatter-sdk.sh diff --git a/.github/workflows/sdk-execution.yml b/.github/workflows/sdk-execution.yml deleted file mode 100644 index 22f605fca9f..00000000000 --- a/.github/workflows/sdk-execution.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: KFP SDK execution tests - -on: - push: - branches: [master] - - pull_request: - paths: - - '.github/workflows/sdk-execution.yml' - - 'sdk/python/**' - - 'api/v2alpha1/**' - -jobs: - sdk-execution-tests: - runs-on: ubuntu-24.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: apt-get update - run: sudo apt-get update - - - name: Install protobuf-compiler - run: sudo apt-get install protobuf-compiler -y - - - name: Install setuptools - run: | - pip3 install setuptools - pip3 freeze - - - name: Install Wheel - run: pip3 install wheel==0.42.0 - - - name: Install protobuf - run: pip3 install protobuf==4.25.3 - - - name: Generate API proto files - working-directory: ./api - run: make clean python - - - name: Install kfp-pipeline-spec from source - run: | - python3 -m pip install api/v2alpha1/python - - - name: Install requirements - run: pip install -r ./test/sdk-execution-tests/requirements.txt - - - name: Run tests - run: | - export KFP_ENDPOINT="http://localhost:8888" - export TIMEOUT_SECONDS=2700 - pytest ./test/sdk-execution-tests/sdk_execution_tests.py --asyncio-task-timeout $TIMEOUT_SECONDS diff --git a/.github/workflows/sdk-isort.yml b/.github/workflows/sdk-isort.yml deleted file mode 100644 index 05124d19378..00000000000 --- a/.github/workflows/sdk-isort.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: KFP SDK Isort Test - -on: - push: - branches: [master] - - pull_request: - paths: - - 'sdk/python/**' - - 'test/presubmit-isort-sdk.sh' - - '.github/workflows/sdk-isort.yml' - -jobs: - test-isort-kfp-sdk: - runs-on: ubuntu-24.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Run isort tests - run: ./test/presubmit-isort-sdk.sh diff --git a/.github/workflows/sdk-upgrade.yml b/.github/workflows/sdk-upgrade.yml deleted file mode 100644 index 9d8a7714c46..00000000000 --- a/.github/workflows/sdk-upgrade.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: KFP SDK Upgrade Test - -on: - push: - branches: [master] - - pull_request: - paths: - - 'sdk/python/**' - - 'test/presubmit-test-sdk-upgrade.sh' - - '.github/workflows/sdk-upgrade.yml' - -jobs: - test-upgrade-kfp-sdk: - runs-on: ubuntu-24.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Run SDK upgrade tests - run: ./test/presubmit-test-sdk-upgrade.sh diff --git a/.github/workflows/sdk-yapf.yml b/.github/workflows/sdk-yapf.yml deleted file mode 100644 index 24a6782fbe7..00000000000 --- a/.github/workflows/sdk-yapf.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: KFP SDK YAPF Tests - -on: - push: - branches: - - master - paths: - - 'sdk/python/**' - - 'test/presubmit-yapf-sdk.sh' - - '.github/workflows/sdk-yapf.yml' - pull_request: - branches: - - master - paths: - - 'sdk/python/**' - - 'test/presubmit-yapf-sdk.sh' - - '.github/workflows/sdk-yapf.yml' - -jobs: - yapf-sdk: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install dependencies - run: pip install yapf - - - name: Run YAPF SDK Tests - run: ./test/presubmit-yapf-sdk.sh diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 53cf010a8c1..00000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,37 +0,0 @@ -# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. -# -# You can adjust the behavior by modifying this file. -# For more information, see: -# https://github.com/actions/stale -name: Mark stale issues and pull requests - -on: - schedule: - - cron: '39 7 * * *' - -jobs: - stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v5 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - days-before-stale: 60 - days-before-close: 21 - stale-issue-message: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. - close-issue-message: > - This issue has been automatically closed because it has not had recent - activity. Please comment "/reopen" to reopen it. - stale-issue-label: 'lifecycle/stale' - exempt-issue-labels: lifecycle/frozen - exempt-pr-labels: lifecycle/frozen - # Learn more about operations: https://github.com/actions/stale#operations-per-run. - operations-per-run: 200 \ No newline at end of file diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml deleted file mode 100644 index 83b13c9af5c..00000000000 --- a/.github/workflows/upgrade-test.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: KFP upgrade tests - -on: - push: - branches: [master] - - pull_request: - paths: - - '.github/workflows/upgrade-test.yml' - - 'backend/**' - - 'manifests/kustomize/**' - -jobs: - upgrade-test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create KFP cluster - uses: ./.github/actions/kfp-cluster - - - name: Prepare upgrade tests - working-directory: backend/test/integration - run: go test -v ./... -namespace ${NAMESPACE} -args -runUpgradeTests=true -testify.m=Prepare - - - name: Prepare verification tests - working-directory: backend/test/integration - run: go test -v ./... -namespace ${NAMESPACE} -args -runUpgradeTests=true -testify.m=Verify - - - name: Prepare upgrade tests v2 - working-directory: backend/test/v2/integration/ - run: go test -v ./... -namespace ${NAMESPACE} -args -runUpgradeTests=true -testify.m=Prepare - - - name: Prepare verification tests v2 - working-directory: backend/test/v2/integration - run: go test -v ./... -namespace ${NAMESPACE} -args -runUpgradeTests=true -testify.m=Verify From 9dd339bdbb227fd1ce06c8dd94784ec0c877b358 Mon Sep 17 00:00:00 2001 From: Amanpreet Singh Bedi Date: Sun, 1 Sep 2024 00:31:20 +0530 Subject: [PATCH 5/9] some test changes Signed-off-by: Amanpreet Singh Bedi --- .github/workflows/gcpc-modules-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gcpc-modules-tests.yml b/.github/workflows/gcpc-modules-tests.yml index 168f33ee181..c8d8e9b0712 100644 --- a/.github/workflows/gcpc-modules-tests.yml +++ b/.github/workflows/gcpc-modules-tests.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.9 - name: apt-get update run: sudo apt-get update From 0051259162c38466559bea06f791e501c0a8d51f Mon Sep 17 00:00:00 2001 From: Amanpreet Singh Bedi Date: Sun, 1 Sep 2024 00:35:40 +0530 Subject: [PATCH 6/9] upgrade python version Signed-off-by: Amanpreet Singh Bedi --- .github/workflows/gcpc-modules-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gcpc-modules-tests.yml b/.github/workflows/gcpc-modules-tests.yml index c8d8e9b0712..162a407c5bc 100644 --- a/.github/workflows/gcpc-modules-tests.yml +++ b/.github/workflows/gcpc-modules-tests.yml @@ -11,7 +11,7 @@ on: jobs: all-gcpc-tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@v4 From 38ebb06fe7aa93af1bfe52c303c4101a9078eb42 Mon Sep 17 00:00:00 2001 From: Amanpreet Singh Bedi Date: Sun, 1 Sep 2024 00:44:33 +0530 Subject: [PATCH 7/9] some changes Signed-off-by: Amanpreet Singh Bedi --- .github/workflows/gcpc-modules-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gcpc-modules-tests.yml b/.github/workflows/gcpc-modules-tests.yml index 162a407c5bc..7f4da3deebb 100644 --- a/.github/workflows/gcpc-modules-tests.yml +++ b/.github/workflows/gcpc-modules-tests.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.8 - name: apt-get update run: sudo apt-get update From 6cd1d373f07fc0f953c8b4253a36cf84c087a6fc Mon Sep 17 00:00:00 2001 From: Amanpreet Singh Bedi Date: Sun, 1 Sep 2024 00:52:33 +0530 Subject: [PATCH 8/9] downgrade ubuntu base image Signed-off-by: Amanpreet Singh Bedi --- .github/workflows/gcpc-modules-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gcpc-modules-tests.yml b/.github/workflows/gcpc-modules-tests.yml index 7f4da3deebb..7dc167da2b9 100644 --- a/.github/workflows/gcpc-modules-tests.yml +++ b/.github/workflows/gcpc-modules-tests.yml @@ -11,7 +11,7 @@ on: jobs: all-gcpc-tests: - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - name: Checkout code uses: actions/checkout@v4 From 3d1cb289f38ae362d9f874f0805372bb31072408 Mon Sep 17 00:00:00 2001 From: Amanpreet Singh Bedi Date: Mon, 2 Sep 2024 21:48:12 +0530 Subject: [PATCH 9/9] upgrade python: Signed-off-by: Amanpreet Singh Bedi --- .github/workflows/gcpc-modules-tests.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gcpc-modules-tests.yml b/.github/workflows/gcpc-modules-tests.yml index 7dc167da2b9..7ce9f4145e3 100644 --- a/.github/workflows/gcpc-modules-tests.yml +++ b/.github/workflows/gcpc-modules-tests.yml @@ -11,7 +11,7 @@ on: jobs: all-gcpc-tests: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@v4 @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.9 - name: apt-get update run: sudo apt-get update @@ -33,10 +33,7 @@ jobs: pip3 freeze - name: Install Wheel - run: pip3 install wheel==0.42.0 - - - name: Install protobuf - run: pip3 install protobuf==4.25.3 + run: pip3 install wheel==0.42.0 - name: Install python sdk run: pip install sdk/python