increase 'termination_grace_period_seconds' in podspec

Signed-off-by: helenxie-bit <helenxiehz@gmail.com>
kubeflow · Sep 29, 2024 · e468b27 · e468b27
1 parent b088815
commit e468b27
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 7 deletions.
diff --git a/.github/workflows/e2e-test-tune-api.yaml b/.github/workflows/e2e-test-tune-api.yaml
@@ -30,6 +30,8 @@ jobs:
       # Step 2: Check Disk Space Before Test
       - name: Check Disk Space Before Test
         run: |
+          docker system prune -a
+          docker volume prune
           echo "Checking disk space usage before e2e test..."
           df -h  # Run 'df' to check free disk space
       

diff --git a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py
@@ -656,15 +656,27 @@ class name in this argument.
                 ),
             )
 
-            worker_pod_template_spec = training_utils.get_pod_template_spec(
-                containers=[container_spec],
-                volumes=[storage_initializer_volume],
+            worker_pod_template_spec = models.V1PodTemplateSpec(
+                metadata=models.V1ObjectMeta(
+                    annotations={constants.ISTIO_SIDECAR_INJECTION: "false"}
+                ),
+                spec=models.V1PodSpec(
+                    containers=[container_spec],
+                    volumes=[storage_initializer_volume],
+                    termination_grace_period_seconds=60,
+                ),
             )
 
-            master_pod_template_spec = training_utils.get_pod_template_spec(
-                containers=[container_spec],
-                init_containers=[init_container_spec],
-                volumes=[storage_initializer_volume],
+            master_pod_template_spec = models.V1PodTemplateSpec(
+                metadata=models.V1ObjectMeta(
+                    annotations={constants.ISTIO_SIDECAR_INJECTION: "false"}
+                ),
+                spec=models.V1PodSpec(
+                    init_containers=[init_container_spec],
+                    containers=[container_spec],
+                    volumes=[storage_initializer_volume],
+                    termination_grace_period_seconds=60,
+                ),
             )
 
             # Create PyTorchJob.