Merge pull request #21 from beckermr/lsf-timing

ENH make it a bit more responsive
beckermr · Sep 7, 2022 · 33d2b63 · 33d2b63
2 parents e40b224 + 01f12f3
commit 33d2b63
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/mattspy/lsf_yield.py b/mattspy/lsf_yield.py
@@ -311,11 +311,21 @@ def __call__(self, jobs):
             if len(cjobs) == 0 and done:
                 return
 
+            status_time = time.time()
             statuses = self._get_all_job_statuses(cjobs)
+            status_time = time.time() - status_time
+            status_time *= 10
 
+            n_yield = 0
+            yield_result_time = time.time()
             for cjob, status_code in statuses.items():
+                if not done and (
+                    time.time() - yield_result_time > status_time or n_yield >= 100
+                ):
+                    break
                 didit, res, _index = self._attempt_result(cjob, status_code)
                 if didit:
+                    n_yield += 1
                     yield ParallelResult(res, _index)
 
     def _attempt_result(self, cjob, status_code):