Add check_log_for_errors to detect and handle multiple errors

easybuild/tools/run.py

@@ -539,7 +539,7 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp):
     if use_regexp or regexp:
         res = parse_log_for_error(stdouterr, regexp, msg="Command used: %s" % cmd)
         if len(res) > 0:
-            message = "Found %s errors in command output (output: %s)" % (len(res), ", ".join([r[0] for r in res]))
+            message = "Found %s errors in command output (output: %s)" % (len(res), "\n\t".join([r[0] for r in res]))
             if use_regexp:
                 raise EasyBuildError(message)
             else:
@@ -589,3 +589,70 @@ def parse_log_for_error(txt, regExp=None, stdout=True, msg=None):
                   (regExp, '\n'.join([x[0] for x in res])))
 
     return res
+
+
+def extract_errors_from_log(log_txt, reg_exps):
+    """
+    Check provided string (command output) for messages matching specified regular expressions,
+    and return 2-tuple with list of warnings and errors.
+    :param log_txt: String containing the log, will be split into individual lines
+    :param reg_exps: List of: regular expressions (as strings) to error on,
+                    or tuple of regular expression and action (any of [IGNORE, WARN, ERROR])
+    :return (warnings, errors) as lists of lines containing a match
+    """
+    actions = (IGNORE, WARN, ERROR)
+
+    # promote single string value to list, since code below expects a list
+    if isinstance(reg_exps, string_type):
+        reg_exps = [reg_exps]
+
+    re_tuples = []
+    for cur in reg_exps:
+        try:
+            if isinstance(cur, str):
+                # use ERROR as default action if only regexp pattern is specified
+                reg_exp, action = cur, ERROR
+            elif isinstance(cur, tuple) and len(cur) == 2:
+                reg_exp, action = cur
+            else:
+                raise TypeError("Incorrect type of value, expected string or 2-tuple")
+
+            if not isinstance(reg_exp, str):
+                raise TypeError("Regular expressions must be passed as string, got %s" % type(reg_exp))
+            if action not in actions:
+                raise TypeError("action must be one of %s, got %s" % (actions, action))
+
+            re_tuples.append((re.compile(reg_exp), action))
+        except Exception as err:
+            raise EasyBuildError("Invalid input: No regexp or tuple of regexp and action '%s': %s", str(cur), err)
+
+    warnings = []
+    errors = []
+    for line in log_txt.split('\n'):
+        for reg_exp, action in re_tuples:
+            if reg_exp.search(line):
+                if action == ERROR:
+                    errors.append(line)
+                elif action == WARN:
+                    warnings.append(line)
+                break
+    return warnings, errors
+
+
+def check_log_for_errors(log_txt, reg_exps):
+    """
+    Check log_txt for messages matching regExps in order and do appropriate action
+    :param log_txt: String containing the log, will be split into individual lines
+    :param reg_exps: List of: regular expressions (as strings) to error on,
+                    or tuple of regular expression and action (any of [IGNORE, WARN, ERROR])
+    """
+    global errors_found_in_log
+    warnings, errors = extract_errors_from_log(log_txt, reg_exps)
+
+    errors_found_in_log += len(warnings) + len(errors)
+    if warnings:
+        _log.warning("Found %s potential error(s) in command output (output: %s)",
+                     len(warnings), "\n\t".join(warnings))
+    if errors:
+        raise EasyBuildError("Found %s error(s) in command output (output: %s)",
+                             len(errors), "\n\t".join(errors))

test/framework/run.py

@@ -46,7 +46,14 @@
 import easybuild.tools.utilities
 from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging
 from easybuild.tools.filetools import adjust_permissions, read_file, write_file
-from easybuild.tools.run import get_output_from_process, run_cmd, run_cmd_qa, parse_log_for_error
+from easybuild.tools.run import (
+    check_log_for_errors,
+    get_output_from_process,
+    run_cmd,
+    run_cmd_qa,
+    parse_log_for_error,
+)
+from easybuild.tools.config import ERROR, IGNORE, WARN
 
 
 class RunTest(EnhancedTestCase):
@@ -520,6 +527,56 @@ def test_run_cmd_stream(self):
         ])
         self.assertEqual(stdout, expected)
 
+    def test_check_log_for_errors(self):
+        fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
+        os.close(fd)
+
+        self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [42])
+        self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [(42, IGNORE)])
+        self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", "invalid-mode")])
+        self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", IGNORE, "")])
+
+        input_text = "\n".join([
+            "OK",
+            "error found",
+            "test failed",
+            "msg: allowed-test failed",
+            "enabling -Werror",
+            "the process crashed with 0"
+        ])
+        expected_msg = r"Found 2 error\(s\) in command output "\
+                       r"\(output: error found\n\tthe process crashed with 0\)"
+
+        # String promoted to list
+        self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
+                              r"\b(error|crashed)\b")
+        # List of string(s)
+        self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
+                              [r"\b(error|crashed)\b"])
+        # List of tuple(s)
+        self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
+                              [(r"\b(error|crashed)\b", ERROR)])
+
+        expected_msg = "Found 2 potential error(s) in command output " \
+                       "(output: error found\n\tthe process crashed with 0)"
+        init_logging(logfile, silent=True)
+        check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)])
+        stop_logging(logfile)
+        self.assertTrue(expected_msg in read_file(logfile))
+
+        expected_msg = r"Found 2 error\(s\) in command output \(output: error found\n\ttest failed\)"
+        write_file(logfile, '')
+        init_logging(logfile, silent=True)
+        self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [
+            r"\berror\b",
+            (r"\ballowed-test failed\b", IGNORE),
+            (r"(?i)\bCRASHED\b", WARN),
+            "fail"
+        ])
+        stop_logging(logfile)
+        expected_msg = "Found 1 potential error(s) in command output (output: the process crashed with 0)"
+        self.assertTrue(expected_msg in read_file(logfile))
+
 
 def suite():
     """ returns all the testcases in this module """