Add basic preflight checks as the first component in the pipeline

README.md

@@ -41,4 +41,12 @@ To deploy a signed certificate in cluster follow [trusted cluster cert](signed-c
 ### Object Storage
 This solution requires object storage to be in place either through S3 or using Noobaa.
 
-If you are using Noobaa apply the following [tuning paramters](noobaa/README.md)
+If you are using Noobaa apply the following [tuning paramters](noobaa/README.md)
+
+## How to run 🏃🏼
+
+1. Create K8s config map and K8s secret based on the target Model Server Info. Use [kfp-model-server.yaml](./sdg/kfp-model-server.yaml).
+
+2. Use pipeline.py file to generate the pipeline.yaml which will create RHOAI pipeline.
+
+3. Create a run in RHOAI by providing required input parameter values.

pipeline.py

@@ -20,9 +20,9 @@ def pipeline_wrapper(mock: List[Literal[MOCKED_STAGES]]):
 
     # Imports for SDG stage
     if "sdg" in mock:
-        from sdg.faked import git_clone_op, sdg_op
+        from sdg.faked import preflight_check_op, git_clone_op, sdg_op
     else:
-        from sdg import git_clone_op, sdg_op
+        from sdg import preflight_check_op, git_clone_op, sdg_op
 
     # Imports for Training stage
     if "train" in mock:
@@ -70,11 +70,18 @@ def pipeline(
         device: str = None,
     ):
 
-        # SDG stage
+        preflight_check_task = preflight_check_op(
+            repo_branch=repo_branch, repo_pr=repo_pr
+        )
+        use_config_map_as_env(preflight_check_task, K8S_NAME, dict(endpoint="endpoint", model="model"))
+        use_secret_as_env(preflight_check_task, K8S_NAME, {"api_key": "api_key"})
+
         git_clone_task = git_clone_op(
             repo_branch=repo_branch, repo_pr=repo_pr, repo_url=repo_url
-        )
+        ).after(preflight_check_task)
 
+        # SDG stage
+
         sdg_task = sdg_op(
             num_instructions_to_generate=num_instructions_to_generate,
             taxonomy=git_clone_task.outputs["taxonomy"],

pipeline.yaml

@@ -1049,6 +1049,8 @@ root:
           enableCache: true
         componentRef:
           name: comp-git-clone-op
+        dependentTasks:
+        - preflight-check-op
         inputs:
           parameters:
             repo_branch:

sdg/__init__.py

@@ -1,4 +1,4 @@
-from .components import git_clone_op, sdg_op
+from .components import preflight_check_op, git_clone_op, sdg_op
 from . import faked
 
-__all__ = ["git_clone_op", "sdg_op", "faked"]
+__all__ = ["preflight_check_op", "git_clone_op", "sdg_op", "faked"]

sdg/components.py

@@ -5,6 +5,26 @@
 
 IMAGE = "quay.io/tcoufal/ilab-sdg:latest"
 
+@dsl.component(base_image=IMAGE)
+def preflight_check_op(
+    repo_branch: str,
+    repo_pr: Optional[int],
+):
+    from os import getenv
+
+    if (not repo_branch) and (repo_pr is None or repo_pr <= 0 ):
+        raise Exception("Both taxonomy repo branch and taxonomy pull request number cannot be empty")
+    api_key = getenv("api_key")
+    model = getenv("model")
+    endpoint = getenv("endpoint")
+
+    if not api_key:
+        raise Exception("Model Server Auth Key is missing in kfp-model-server secret")
+    if not model:
+        raise Exception("Model name is missing in kfp-model-server configMap")
+    if not endpoint:
+        raise Exception("Model Server endpoint URL is missing in kfp-model-server configMap")
+
 @dsl.container_component
 def git_clone_op(
     taxonomy: dsl.Output[dsl.Dataset],

sdg/faked/__init__.py

@@ -1,3 +1,3 @@
-from .components import git_clone_op, sdg_op
+from .components import preflight_check_op, git_clone_op, sdg_op
 
-__all__ = ["git_clone_op", "sdg_op"]
+__all__ = ["preflight_check_op", "git_clone_op", "sdg_op"]

sdg/faked/components.py

@@ -4,7 +4,16 @@
 from kfp import dsl
 from utils.consts import PYTHON_IMAGE
 
-@dsl.component(base_image=PYTHON_IMAGE)
+IMAGE = "registry.access.redhat.com/ubi9/python-311:latest"
+
+@dsl.component(base_image=IMAGE)
+def preflight_check_op(
+    repo_branch: str,
+    repo_pr: Optional[int],
+):
+    pass
+
+@dsl.component(base_image=IMAGE)
 def git_clone_op(
     taxonomy: dsl.Output[dsl.Dataset],
     repo_branch: str,