revert wait_until_cmd_or_err; address comments

roles/openshift_health_checker/openshift_checks/logging/logging.py

@@ -94,3 +94,13 @@ def exec_oc(execute_module=None, namespace="logging", cmd_str="", extra_args=Non
             raise OpenShiftCheckException(msg)
 
         return result.get("result", "")
+
+    def oc_cmd(self, cmd_str, extra_args, task_vars):
+        """Wrap parent exec_oc method. Allows for testing without actually invoking other modules."""
+        return self.exec_oc(
+            self.execute_module,
+            self.logging_namespace,
+            cmd_str,
+            extra_args,
+            task_vars
+        )

roles/openshift_health_checker/openshift_checks/logging/logging_index_time.py

@@ -63,19 +63,19 @@ def run(self, tmp, task_vars):
     def wait_until_cmd_or_err(self, es_pod, uuid, timeout_secs, task_vars):
         """Retry an Elasticsearch query every second until query success, or a defined
         length of time has passed."""
-        interval = 1
-        max_tries = int(timeout_secs / interval)
+        now = time.time()
+        time_start = now
+        time_end = time_start + timeout_secs
+        interval = 1  # seconds to wait between retries
 
-        # for test cases where timeout_secs == a decimal less than 1.
-        if timeout_secs < 1:
-            max_tries = 1
-            interval = timeout_secs
-
-        for i in range(max_tries):
+        while now < time_end:
             if self.query_es_from_es(es_pod, uuid, task_vars):
                 return
-            if i < max_tries - 1:
-                time.sleep(interval)  # note: we actually don't need to sleep after the last try...
+            time.sleep(interval)
+            now = int(time.time())
+            if now < time_start:  # saw a large clock reset: start the timer again
+                time_start = now
+                time_end = now + timeout_secs
 
         msg = "expecting match in Elasticsearch for message with uuid {}, but no matches were found after {}s."
         raise OpenShiftCheckException(msg.format(uuid, timeout_secs))
@@ -140,12 +140,3 @@ def generate_uuid():
         """Wrap uuid generator. Allows for testing with expected values."""
         return str(uuid4())
 
-    def oc_cmd(self, cmd_str, extra_args, task_vars):
-        """Wrap parent exec_oc method. Allows for testing without actually invoking other modules."""
-        return self.exec_oc(
-            self.execute_module,
-            self.logging_namespace,
-            cmd_str,
-            extra_args,
-            task_vars
-        )