diff --git a/README.md b/README.md index 5065ed5e..219502a0 100644 --- a/README.md +++ b/README.md @@ -41,4 +41,12 @@ To deploy a signed certificate in cluster follow [trusted cluster cert](signed-c ### Object Storage This solution requires object storage to be in place either through S3 or using Noobaa. -If you are using Noobaa apply the following [tuning paramters](noobaa/README.md) \ No newline at end of file +If you are using Noobaa apply the following [tuning paramters](noobaa/README.md) + +## How to run 🏃🏼 + +1. Create K8s config map and K8s secret based on the target Model Server Info. Use [kfp-model-server.yaml](./sdg/kfp-model-server.yaml). + +2. Use pipeline.py file to generate the pipeline.yaml which will create RHOAI pipeline. + +3. Create a run in RHOAI by providing required input parameter values. \ No newline at end of file diff --git a/pipeline.py b/pipeline.py index 4b463ebb..bfb56233 100644 --- a/pipeline.py +++ b/pipeline.py @@ -20,9 +20,9 @@ def pipeline_wrapper(mock: List[Literal[MOCKED_STAGES]]): # Imports for SDG stage if "sdg" in mock: - from sdg.faked import git_clone_op, sdg_op + from sdg.faked import preflight_check_op, git_clone_op, sdg_op else: - from sdg import git_clone_op, sdg_op + from sdg import preflight_check_op, git_clone_op, sdg_op # Imports for Training stage if "train" in mock: @@ -70,11 +70,18 @@ def pipeline( device: str = None, ): - # SDG stage + preflight_check_task = preflight_check_op( + repo_branch=repo_branch, repo_pr=repo_pr + ) + use_config_map_as_env(preflight_check_task, K8S_NAME, dict(endpoint="endpoint", model="model")) + use_secret_as_env(preflight_check_task, K8S_NAME, {"api_key": "api_key"}) + git_clone_task = git_clone_op( repo_branch=repo_branch, repo_pr=repo_pr, repo_url=repo_url - ) + ).after(preflight_check_task) + # SDG stage + sdg_task = sdg_op( num_instructions_to_generate=num_instructions_to_generate, taxonomy=git_clone_task.outputs["taxonomy"], diff --git a/pipeline.yaml b/pipeline.yaml index ca5a77e1..452c6eee 100644 --- a/pipeline.yaml +++ b/pipeline.yaml @@ -1049,6 +1049,8 @@ root: enableCache: true componentRef: name: comp-git-clone-op + dependentTasks: + - preflight-check-op inputs: parameters: repo_branch: diff --git a/sdg/__init__.py b/sdg/__init__.py index 095fcb91..fac20054 100644 --- a/sdg/__init__.py +++ b/sdg/__init__.py @@ -1,4 +1,4 @@ -from .components import git_clone_op, sdg_op +from .components import preflight_check_op, git_clone_op, sdg_op from . import faked -__all__ = ["git_clone_op", "sdg_op", "faked"] +__all__ = ["preflight_check_op", "git_clone_op", "sdg_op", "faked"] diff --git a/sdg/components.py b/sdg/components.py index 2f716ec1..460cf8d1 100644 --- a/sdg/components.py +++ b/sdg/components.py @@ -5,6 +5,26 @@ IMAGE = "quay.io/tcoufal/ilab-sdg:latest" +@dsl.component(base_image=IMAGE) +def preflight_check_op( + repo_branch: str, + repo_pr: Optional[int], +): + from os import getenv + + if (not repo_branch) and (repo_pr is None or repo_pr <= 0 ): + raise Exception("Both taxonomy repo branch and taxonomy pull request number cannot be empty") + api_key = getenv("api_key") + model = getenv("model") + endpoint = getenv("endpoint") + + if not api_key: + raise Exception("Model Server Auth Key is missing in kfp-model-server secret") + if not model: + raise Exception("Model name is missing in kfp-model-server configMap") + if not endpoint: + raise Exception("Model Server endpoint URL is missing in kfp-model-server configMap") + @dsl.container_component def git_clone_op( taxonomy: dsl.Output[dsl.Dataset], diff --git a/sdg/faked/__init__.py b/sdg/faked/__init__.py index 7559c2de..c1db1698 100644 --- a/sdg/faked/__init__.py +++ b/sdg/faked/__init__.py @@ -1,3 +1,3 @@ -from .components import git_clone_op, sdg_op +from .components import preflight_check_op, git_clone_op, sdg_op -__all__ = ["git_clone_op", "sdg_op"] +__all__ = ["preflight_check_op", "git_clone_op", "sdg_op"] diff --git a/sdg/faked/components.py b/sdg/faked/components.py index 33193619..a6cc8950 100644 --- a/sdg/faked/components.py +++ b/sdg/faked/components.py @@ -4,7 +4,16 @@ from kfp import dsl from utils.consts import PYTHON_IMAGE -@dsl.component(base_image=PYTHON_IMAGE) +IMAGE = "registry.access.redhat.com/ubi9/python-311:latest" + +@dsl.component(base_image=IMAGE) +def preflight_check_op( + repo_branch: str, + repo_pr: Optional[int], +): + pass + +@dsl.component(base_image=IMAGE) def git_clone_op( taxonomy: dsl.Output[dsl.Dataset], repo_branch: str,