From 3791fdef32294391e34b55a27ed8c89f43fec177 Mon Sep 17 00:00:00 2001 From: yetone Date: Thu, 18 Jul 2024 13:10:45 +0800 Subject: [PATCH] fix: do not raise deployment pods not found error --- src/bentoml/_internal/cloud/client.py | 2 +- src/bentoml/_internal/cloud/deployment.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/bentoml/_internal/cloud/client.py b/src/bentoml/_internal/cloud/client.py index c418d3ee0ef..49f0a62d8d1 100644 --- a/src/bentoml/_internal/cloud/client.py +++ b/src/bentoml/_internal/cloud/client.py @@ -693,7 +693,7 @@ def get_deployment_image_builder_pod( ) -> KubePodSchema | None: pods = self.list_deployment_pods(name, cluster=cluster) if not pods: - raise NotFound(f"Deployment {name} pods is not found") + return None for pod in pods: if pod.labels.get("yatai.ai/is-bento-image-builder") == "true": return pod diff --git a/src/bentoml/_internal/cloud/deployment.py b/src/bentoml/_internal/cloud/deployment.py index 15318979d41..35bb251d25b 100644 --- a/src/bentoml/_internal/cloud/deployment.py +++ b/src/bentoml/_internal/cloud/deployment.py @@ -537,17 +537,29 @@ def tail_image_builder_logs(stop_spinner: t.Callable[[], None]): nonlocal ws_session cloud_rest_client = get_rest_api_client(self._context) + start_time = time.time() + wait_pod_timeout = 60 * 10 pod: KubePodSchema | None = None while True: pod = cloud_rest_client.v2.get_deployment_image_builder_pod( self.name, self.cluster ) if pod is None: + if time.time() - start_time > timeout: + console.print( + "🚨 [bold red]Time out waiting for image builder pod created[/bold red]" + ) + return if stop_tail_event.wait(check_interval): return continue if pod.pod_status.status == "Running": break + if time.time() - start_time > wait_pod_timeout: + console.print( + "🚨 [bold red]Time out waiting for image builder pod running[/bold red]" + ) + return if stop_tail_event.wait(check_interval): return if pod is None: