feat(backend): mount EmptyDir volumes for launcher write locations (#…

…10857) Launcher writes input artifacts to root paths /gcs, /minio, and /s3. These paths are not accessible by non-root users by default, which is problematic in locked-down Kubernetes installations and/or OpenShift. /gcs is currently a contract for KFP v2 python component wrappers, so the path cannot be changed. Mount an EmptyDir scratch volume to these paths to work around this. Additionally, /.local and /.cache are written to by pip, so add EmptyDir mounts for those too. Fixes: #5673 Fixes: #7345 Signed-off-by: Humair Khan <HumairAK@users.noreply.github.com> Co-authored-by: Greg Sheremeta <gshereme@redhat.com>
kubeflow · Jun 6, 2024 · 65839ce · 65839ce
1 parent 8bcfb3d
commit 65839ce
Show file tree

Hide file tree

Showing 4 changed files with 172 additions and 22 deletions.
diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go
@@ -26,11 +26,23 @@ import (
 )
 
 const (
-	volumeNameKFPLauncher = "kfp-launcher"
-	DefaultLauncherImage  = "gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f"
-	LauncherImageEnvVar   = "V2_LAUNCHER_IMAGE"
-	DefaultDriverImage    = "gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec"
-	DriverImageEnvVar     = "V2_DRIVER_IMAGE"
+	volumeNameKFPLauncher    = "kfp-launcher"
+	DefaultLauncherImage     = "gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f"
+	LauncherImageEnvVar      = "V2_LAUNCHER_IMAGE"
+	DefaultDriverImage       = "gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec"
+	DriverImageEnvVar        = "V2_DRIVER_IMAGE"
+	gcsScratchLocation       = "/gcs"
+	gcsScratchName           = "gcs-scratch"
+	s3ScratchLocation        = "/s3"
+	s3ScratchName            = "s3-scratch"
+	minioScratchLocation     = "/minio"
+	minioScratchName         = "minio-scratch"
+	dotLocalScratchLocation  = "/.local"
+	dotLocalScratchName      = "dot-local-scratch"
+	dotCacheScratchLocation  = "/.cache"
+	dotCacheScratchName      = "dot-cache-scratch"
+	dotConfigScratchLocation = "/.config"
+	dotConfigScratchName     = "dot-config-scratch"
 )
 
 func (c *workflowCompiler) Container(name string, component *pipelinespec.ComponentSpec, container *pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec) error {
@@ -241,21 +253,61 @@ func (c *workflowCompiler) addContainerExecutorTemplate(refName string) string {
 		// args come from. It is treated as a strategic merge patch on
 		// top of the Pod spec.
 		PodSpecPatch: inputValue(paramPodSpecPatch),
-		Volumes: []k8score.Volume{{
-			Name: volumeNameKFPLauncher,
-			VolumeSource: k8score.VolumeSource{
-				EmptyDir: &k8score.EmptyDirVolumeSource{},
+		Volumes: []k8score.Volume{
+			{
+				Name: volumeNameKFPLauncher,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
 			},
-		}},
+			{
+				Name: gcsScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: s3ScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: minioScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: dotLocalScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: dotCacheScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: dotConfigScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+		},
 		InitContainers: []wfapi.UserContainer{{
 			Container: k8score.Container{
 				Name:    "kfp-launcher",
 				Image:   GetLauncherImage(),
 				Command: []string{"launcher-v2", "--copy", component.KFPLauncherPath},
-				VolumeMounts: []k8score.VolumeMount{{
-					Name:      volumeNameKFPLauncher,
-					MountPath: component.VolumePathKFPLauncher,
-				}},
+				VolumeMounts: []k8score.VolumeMount{
+					{
+						Name:      volumeNameKFPLauncher,
+						MountPath: component.VolumePathKFPLauncher,
+					},
+				},
 				Resources: launcherResources,
 			},
 		}},
@@ -268,10 +320,36 @@ func (c *workflowCompiler) addContainerExecutorTemplate(refName string) string {
 			// These are added to pass argo workflows linting.
 			Image:   "gcr.io/ml-pipeline/should-be-overridden-during-runtime",
 			Command: []string{"should-be-overridden-during-runtime"},
-			VolumeMounts: []k8score.VolumeMount{{
-				Name:      volumeNameKFPLauncher,
-				MountPath: component.VolumePathKFPLauncher,
-			}},
+			VolumeMounts: []k8score.VolumeMount{
+				{
+					Name:      volumeNameKFPLauncher,
+					MountPath: component.VolumePathKFPLauncher,
+				},
+				{
+					Name:      gcsScratchName,
+					MountPath: gcsScratchLocation,
+				},
+				{
+					Name:      s3ScratchName,
+					MountPath: s3ScratchLocation,
+				},
+				{
+					Name:      minioScratchName,
+					MountPath: minioScratchLocation,
+				},
+				{
+					Name:      dotLocalScratchName,
+					MountPath: dotLocalScratchLocation,
+				},
+				{
+					Name:      dotCacheScratchName,
+					MountPath: dotCacheScratchLocation,
+				},
+				{
+					Name:      dotConfigScratchName,
+					MountPath: dotConfigScratchLocation,
+				},
+			},
 			EnvFrom: []k8score.EnvFromSource{metadataEnvFrom},
 			Env:     commonEnvs,
 		},

diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml
@@ -142,6 +142,18 @@ spec:
       volumeMounts:
       - mountPath: /kfp-launcher
         name: kfp-launcher
+      - mountPath: /gcs
+        name: gcs-scratch
+      - mountPath: /s3
+        name: s3-scratch
+      - mountPath: /minio
+        name: minio-scratch
+      - mountPath: /.local
+        name: dot-local-scratch
+      - mountPath: /.cache
+        name: dot-cache-scratch
+      - mountPath: /.config
+        name: dot-config-scratch
     initContainers:
     - command:
       - launcher-v2
@@ -168,6 +180,18 @@ spec:
     volumes:
     - emptyDir: {}
       name: kfp-launcher
+    - emptyDir: { }
+      name: gcs-scratch
+    - emptyDir: { }
+      name: s3-scratch
+    - emptyDir: { }
+      name: minio-scratch
+    - emptyDir: { }
+      name: dot-local-scratch
+    - emptyDir: { }
+      name: dot-cache-scratch
+    - emptyDir: { }
+      name: dot-config-scratch
   - dag:
       tasks:
       - arguments:

diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml
@@ -124,8 +124,20 @@ spec:
       name: ""
       resources: {}
       volumeMounts:
-      - mountPath: /kfp-launcher
-        name: kfp-launcher
+        - mountPath: /kfp-launcher
+          name: kfp-launcher
+        - mountPath: /gcs
+          name: gcs-scratch
+        - mountPath: /s3
+          name: s3-scratch
+        - mountPath: /minio
+          name: minio-scratch
+        - mountPath: /.local
+          name: dot-local-scratch
+        - mountPath: /.cache
+          name: dot-cache-scratch
+        - mountPath: /.config
+          name: dot-config-scratch
     initContainers:
     - command:
       - launcher-v2
@@ -156,8 +168,20 @@ spec:
     outputs: {}
     podSpecPatch: '{{inputs.parameters.pod-spec-patch}}'
     volumes:
-    - emptyDir: {}
-      name: kfp-launcher
+      - emptyDir: {}
+        name: kfp-launcher
+      - emptyDir: { }
+        name: gcs-scratch
+      - emptyDir: { }
+        name: s3-scratch
+      - emptyDir: { }
+        name: minio-scratch
+      - emptyDir: { }
+        name: dot-local-scratch
+      - emptyDir: { }
+        name: dot-cache-scratch
+      - emptyDir: { }
+        name: dot-config-scratch
   - dag:
       tasks:
       - arguments:

diff --git a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml
@@ -125,6 +125,18 @@ spec:
       volumeMounts:
       - mountPath: /kfp-launcher
         name: kfp-launcher
+      - mountPath: /gcs
+        name: gcs-scratch
+      - mountPath: /s3
+        name: s3-scratch
+      - mountPath: /minio
+        name: minio-scratch
+      - mountPath: /.local
+        name: dot-local-scratch
+      - mountPath: /.cache
+        name: dot-cache-scratch
+      - mountPath: /.config
+        name: dot-config-scratch
     initContainers:
     - command:
       - launcher-v2
@@ -151,6 +163,18 @@ spec:
     volumes:
     - emptyDir: {}
       name: kfp-launcher
+    - emptyDir: {}
+      name: gcs-scratch
+    - emptyDir: {}
+      name: s3-scratch
+    - emptyDir: {}
+      name: minio-scratch
+    - emptyDir: {}
+      name: dot-local-scratch
+    - emptyDir: {}
+      name: dot-cache-scratch
+    - emptyDir: {}
+      name: dot-config-scratch
   - dag:
       tasks:
       - arguments: