feat(Backend + SDK): Update kfp backend and kubernetes sdk to support…

… ConfigMaps as volumes and as env variables (kubeflow#10483) * Update kfp backend and kubernetes sdk to support ConfigMaps as volumes and as env Signed-off-by: Alexey Roytman <roytman@il.ibm.com> * update go.mod, apiserver.csv and driver.csv Signed-off-by: Alexey Roytman <roytman@il.ibm.com> * add test/snapshot/data files Signed-off-by: Alexey Roytman <roytman@il.ibm.com> * fix tests Signed-off-by: Alexey Roytman <roytman@il.ibm.com> * go mod tidy Signed-off-by: Alexey Roytman <roytman@il.ibm.com> * update backend/third_party_licenses/apiserver.csv Signed-off-by: Alexey Roytman <roytman@il.ibm.com> * update backend/third_party_licenses/apiserver.csv Signed-off-by: Alexey Roytman <roytman@il.ibm.com> * fix comments Signed-off-by: Alexey Roytman <roytman@il.ibm.com> * fix comments Signed-off-by: Alexey Roytman <roytman@il.ibm.com> * update go.mod, apiserver.csv and driver.csv Signed-off-by: Alexey Roytman <roytman@il.ibm.com> --------- Signed-off-by: Alexey Roytman <roytman@il.ibm.com>
VaniHaripriya · Sep 23, 2024 · ddb121b · ddb121b
1 parent 56741bb
commit ddb121b
Show file tree

Hide file tree

Showing 7 changed files with 48 additions and 119 deletions.
diff --git a/backend/src/v2/driver/driver.go b/backend/src/v2/driver/driver.go
@@ -551,6 +551,39 @@ func extendPodSpecPatch(
 		}
 	}
 
+	// Get config map mount information
+	for _, configMapAsVolume := range kubernetesExecutorConfig.GetConfigMapAsVolume() {
+		configMapVolume := k8score.Volume{
+			Name: configMapAsVolume.GetConfigMapName(),
+			VolumeSource: k8score.VolumeSource{
+				ConfigMap: &k8score.ConfigMapVolumeSource{
+					LocalObjectReference: k8score.LocalObjectReference{Name: configMapAsVolume.GetConfigMapName()}},
+			},
+		}
+		configMapVolumeMount := k8score.VolumeMount{
+			Name:      configMapAsVolume.GetConfigMapName(),
+			MountPath: configMapAsVolume.GetMountPath(),
+		}
+		podSpec.Volumes = append(podSpec.Volumes, configMapVolume)
+		podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, configMapVolumeMount)
+	}
+
+	// Get config map env information
+	for _, configMapAsEnv := range kubernetesExecutorConfig.GetConfigMapAsEnv() {
+		for _, keyToEnv := range configMapAsEnv.GetKeyToEnv() {
+			configMapEnvVar := k8score.EnvVar{
+				Name: keyToEnv.GetEnvVar(),
+				ValueFrom: &k8score.EnvVarSource{
+					ConfigMapKeyRef: &k8score.ConfigMapKeySelector{
+						Key: keyToEnv.GetConfigMapKey(),
+					},
+				},
+			}
+			configMapEnvVar.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name = configMapAsEnv.GetConfigMapName()
+			podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, configMapEnvVar)
+		}
+	}
+
 	// Get image pull secret information
 	for _, imagePullSecret := range kubernetesExecutorConfig.GetImagePullSecret() {
 		podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, k8score.LocalObjectReference{Name: imagePullSecret.GetSecretName()})

diff --git a/kubernetes_platform/python/README.md b/kubernetes_platform/python/README.md
@@ -63,25 +63,6 @@ def pipeline():
                                     mount_path='/mnt/my_vol')
 ```
 
-### Secret: As optional source for a mounted volume
-```python
-from kfp import dsl
-from kfp import kubernetes
-
-@dsl.component
-def print_secret():
-    with open('/mnt/my_vol') as f:
-        print(f.read())
-
-@dsl.pipeline
-def pipeline():
-    task = print_secret()
-    kubernetes.use_secret_as_volume(task,
-                                    secret_name='my-secret',
-                                    mount_path='/mnt/my_vol'
-                                    optional=True)
-```
-
 ### ConfigMap: As environment variable
 ```python
 from kfp import dsl
@@ -90,14 +71,14 @@ from kfp import kubernetes
 @dsl.component
 def print_config_map():
     import os
-    print(os.environ['CM_VAR'])
+    print(os.environ['my-cm'])
 
 @dsl.pipeline
 def pipeline():
     task = print_config_map()
     kubernetes.use_config_map_as_env(task,
                                  config_map_name='my-cm',
-                                 config_map_key_to_env={'foo': 'CM_VAR'})
+                                 secret_key_to_env={'foo': 'CM_VAR'})
 ```
 
 ### ConfigMap: As mounted volume
@@ -113,28 +94,9 @@ def print_config_map():
 @dsl.pipeline
 def pipeline():
     task = print_config_map()
-    kubernetes.use_config_map_as_volume(task,
-                                       config_map_name='my-cm',
-                                       mount_path='/mnt/my_vol')
-```
-
-### ConfigMap: As optional source for a mounted volume
-```python
-from kfp import dsl
-from kfp import kubernetes
-
-@dsl.component
-def print_config_map():
-    with open('/mnt/my_vol') as f:
-        print(f.read())
-
-@dsl.pipeline
-def pipeline():
-    task = print_config_map()
-    kubernetes.use_config_map_as_volume(task,
-                                       config_map_name='my-cm',
-                                       mount_path='/mnt/my_vol',
-				       optional=True)
+    kubernetes.use_secret_as_volume(task,
+                                    config_map_name='my-cm',
+                                    mount_path='/mnt/my_vol')
 ```
 
 

diff --git a/kubernetes_platform/python/kfp/kubernetes/__init__.py b/kubernetes_platform/python/kfp/kubernetes/__init__.py
@@ -41,6 +41,8 @@
 from kfp.kubernetes.field import use_field_path_as_env
 from kfp.kubernetes.image import set_image_pull_policy
 from kfp.kubernetes.image import set_image_pull_secrets
+from kfp.kubernetes.config_map import use_config_map_as_volume
+from kfp.kubernetes.config_map import use_config_map_as_env
 from kfp.kubernetes.node_selector import add_node_selector
 from kfp.kubernetes.pod_metadata import add_pod_annotation
 from kfp.kubernetes.pod_metadata import add_pod_label

diff --git a/kubernetes_platform/python/kfp/kubernetes/config_map.py b/kubernetes_platform/python/kfp/kubernetes/config_map.py
@@ -61,7 +61,6 @@ def use_config_map_as_volume(
     task: PipelineTask,
     config_map_name: str,
     mount_path: str,
-    optional: bool = False,
 ) -> PipelineTask:
     """Use a Kubernetes ConfigMap by mounting its data to the task's container as
     described by the `Kubernetes documentation <https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#add-configmap-data-to-a-volume>`_.
@@ -70,7 +69,6 @@ def use_config_map_as_volume(
         task: Pipeline task.
         config_map_name: Name of the ConfigMap.
         mount_path: Path to which to mount the ConfigMap data.
-        optional: Optional field specifying whether the ConfigMap must be defined.
 
     Returns:
         Task object with updated ConfigMap configuration.
@@ -81,7 +79,6 @@ def use_config_map_as_volume(
     config_map_as_vol = pb.ConfigMapAsVolume(
         config_map_name=config_map_name,
         mount_path=mount_path,
-        optional=optional,
     )
     msg.config_map_as_volume.append(config_map_as_vol)
 

diff --git a/kubernetes_platform/python/test/snapshot/data/config_map_as_env.yaml b/kubernetes_platform/python/test/snapshot/data/config_map_as_env.yaml
@@ -32,7 +32,7 @@ deploymentSpec:
           '
         - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
           \ *\n\ndef comp():\n    pass\n\n"
-        image: python:3.9
+        image: python:3.7
 pipelineInfo:
   name: my-pipeline
 root:

diff --git a/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml b/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml
@@ -32,7 +32,7 @@ deploymentSpec:
           '
         - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
           \ *\n\ndef comp():\n    pass\n\n"
-        image: python:3.9
+        image: python:3.7
 pipelineInfo:
   name: my-pipeline
 root:
@@ -56,4 +56,3 @@ platforms:
           configMapAsVolume:
           - mountPath: /mnt/my_vol
             configMapName: my-cm
-            optional: False
diff --git a/kubernetes_platform/python/test/unit/test_config_map.py b/kubernetes_platform/python/test/unit/test_config_map.py
@@ -38,66 +38,7 @@ def my_pipeline():
                             'exec-comp': {
                                 'configMapAsVolume': [{
                                     'configMapName': 'cm-name',
-                                    'mountPath': 'cmpath',
-                                    'optional': False
-                                }]
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-    def test_use_one_optional_true(self):
-
-        @dsl.pipeline
-        def my_pipeline():
-            task = comp()
-            kubernetes.use_config_map_as_volume(
-                task,
-                config_map_name='cm-name',
-                mount_path='cmpath',
-                optional=True)
-
-        assert json_format.MessageToDict(my_pipeline.platform_spec) == {
-            'platforms': {
-                'kubernetes': {
-                    'deploymentSpec': {
-                        'executors': {
-                            'exec-comp': {
-                                'configMapAsVolume': [{
-                                    'configMapName': 'cm-name',
-                                    'mountPath': 'cmpath',
-                                    'optional': True
-                                }]
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-    def test_use_one_optional_false(self):
-
-        @dsl.pipeline
-        def my_pipeline():
-            task = comp()
-            kubernetes.use_config_map_as_volume(
-                task,
-                config_map_name='cm-name',
-                mount_path='cmpath',
-                optional=False)
-
-        assert json_format.MessageToDict(my_pipeline.platform_spec) == {
-            'platforms': {
-                'kubernetes': {
-                    'deploymentSpec': {
-                        'executors': {
-                            'exec-comp': {
-                                'configMapAsVolume': [{
-                                    'configMapName': 'cm-name',
-                                    'mountPath': 'cmpath',
-                                    'optional': False
+                                    'mountPath': 'cmpath'
                                 }]
                             }
                         }
@@ -131,13 +72,11 @@ def my_pipeline():
                                 'configMapAsVolume': [
                                     {
                                         'configMapName': 'cm-name1',
-                                        'mountPath': 'cmpath1',
-                                        'optional': False
+                                        'mountPath': 'cmpath1'
                                     },
                                     {
                                         'configMapName': 'cm-name2',
-                                        'mountPath': 'cmpath2',
-                                        'optional': False
+                                        'mountPath': 'cmpath2'
                                     },
                                 ]
                             }
@@ -180,8 +119,7 @@ def my_pipeline():
                                 }],
                                 'configMapAsVolume': [{
                                     'configMapName': 'cm-name2',
-                                    'mountPath': 'cmpath2',
-                                    'optional': False
+                                    'mountPath': 'cmpath2'
                                 },]
                             }
                         }
@@ -218,8 +156,7 @@ def my_pipeline():
                                 }],
                                 'configMapAsVolume': [{
                                     'configMapName': 'cm-name',
-                                    'mountPath': 'cmpath',
-                                    'optional': False
+                                    'mountPath': 'cmpath'
                                 }]
                             }
                         }
@@ -352,8 +289,7 @@ def my_pipeline():
                                 }],
                                 'configMapAsVolume': [{
                                     'configMapName': 'cm-name2',
-                                    'mountPath': 'cmpath2',
-                                    'optional': False
+                                    'mountPath': 'cmpath2'
                                 },]
                             }
                         }