Merge pull request #20 from beckermr/timing

PERF ensure condor queue can be fed well
beckermr · Sep 6, 2022 · e40b224 · e40b224
2 parents 87d9141 + f6e55cc
commit e40b224
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 504 deletions.
diff --git a/mattspy/condor_yield.py b/mattspy/condor_yield.py
@@ -375,11 +375,22 @@ def __call__(self, jobs):
             if len(cjobs) == 0 and done:
                 return
 
+            status_time = time.time()
             statuses = _get_all_job_statuses(cjobs)
+            status_time = time.time() - status_time
+            status_time *= 10
 
+            n_yield = 0
+            yield_result_time = time.time()
             for cjob, status_code in statuses.items():
+                if not done and (
+                    time.time() - yield_result_time > status_time or n_yield >= 100
+                ):
+                    break
+
                 didit, res, _index = self._attempt_result(cjob, status_code)
                 if didit:
+                    n_yield += 1
                     yield ParallelResult(res, _index)
 
     def _attempt_result(self, cjob, status_code):

diff --git a/mattspy/matcher.py b/mattspy/matcher.py