[Test/SCons] Add fast failure option for tests

Add fast_fail_tests option. If True, the first test failure aborts
running the test suite. Useful when debugging tests and multiple tests
are failing, this option allows them to be easily handled one-by-one.

SConstruct

@@ -651,7 +651,12 @@ config_options = [
            files in the subdirectory defined by 'prefix'. This layout is best
            with a prefix like '/opt/cantera'. 'debian' installs to the stage
            directory in a layout used for generating Debian packages.""",
-        defaults.fsLayout, ('standard','compact','debian'))
+        defaults.fsLayout, ('standard','compact','debian')),
+    BoolVariable(
+        "fast_fail_tests",
+        """If enabled, tests will exit at the first failure.""",
+        False,
+    )
 ]
 
 opts.AddVariables(*config_options)

site_scons/buildutils.py

@@ -172,9 +172,12 @@ def regression_test(target, source, env):
             os.path.unlink(target[0].abspath)
 
         testResults.failed[env['active_test_name']] = 1
+        if env["fast_fail_tests"]:
+            sys.exit(1)
     else:
         print('PASSED')
-        open(target[0].path, 'w').write(time.asctime()+'\n')
+        with open(target[0].path, 'w') as passed_file:
+            passed_file.write(time.asctime()+'\n')
         testResults.passed[env['active_test_name']] = 1
 
 

test/SConscript

@@ -24,7 +24,7 @@ if env['googletest'] == 'submodule':
 if env['googletest'] != 'none':
     localenv.Append(LIBS=['gtest', 'gmock'])
 
-# Turn of optimization to speed up compilation
+# Turn off optimization to speed up compilation
 ccflags = localenv['CCFLAGS']
 for optimize_flag in ('-O3', '-O2', '/O2'):
     if optimize_flag in ccflags:
@@ -58,13 +58,18 @@ def addTestProgram(subdir, progName, env_vars={}):
             os.mkdir(workDir)
         cmd = [program.abspath, '--gtest_output=xml:'+resultsFile]
         cmd.extend(env['gtest_flags'].split())
+        if env["fast_fail_tests"]:
+            env["ENV"]["GTEST_BREAK_ON_FAILURE"] = "1"
         code = subprocess.call(cmd, env=env['ENV'], cwd=workDir)
 
         if code:
             print("FAILED: Test '{0}' exited with code {1}".format(progName, code))
+            if env["fast_fail_tests"]:
+                sys.exit(1)
         else:
             # Test was successful
-            open(passedFile.path, 'w').write(time.asctime()+'\n')
+            with open(passedFile.path, 'w') as passed_file:
+                passed_file.write(time.asctime()+'\n')
 
         if os.path.exists(resultsFile):
             # Determine individual test status
@@ -123,11 +128,21 @@ def addPythonTest(testname, subdir, script, interpreter, outfile,
         for k,v in env_vars.items():
             print(k,v)
             environ[k] = v
-        code = subprocess.call([env.subst(interpreter), source[0].abspath] + args.split(),
-                               env=environ)
+
+        cmdargs = args.split()
+        if env["fast_fail_tests"]:
+            cmdargs.insert(0, "fast_fail")
+
+        code = subprocess.call(
+            [env.subst(interpreter), source[0].abspath] + cmdargs,
+            env=environ
+        )
         if not code:
             # Test was successful
-            open(target[0].path, 'w').write(time.asctime()+'\n')
+            with open(target[0].path, 'w') as passed_file:
+                passed_file.write(time.asctime()+'\n')
+        elif env["fast_fail_tests"]:
+            sys.exit(1)
 
         failures = 0
         if os.path.exists(outfile):
@@ -187,8 +202,11 @@ def addMatlabTest(script, testName, dependencies=None, env_vars=()):
         code = subprocess.call([pjoin(env['matlab_path'], 'bin', 'matlab')] +
                                matlabOptions + ['-r', runCommand],
                                env=environ, cwd=Dir('#test/matlab').abspath)
+        if code and env["fast_fail_tests"]:
+            sys.exit(1)
         try:
-            results = open(outfile).read()
+            with open(outfile, "r") as output_file:
+                results = output_file.read()
         except EnvironmentError: # TODO: replace with 'FileNotFoundError' after end of Python 2.7 support
             testResults.failed['Matlab' +
                 ' ***no results for entire test suite***'] = 100

test/python/runCythonTests.py

@@ -75,11 +75,19 @@ def addError(self, test, err):
     print('* INFO: Git commit:', cantera.__git_commit__, '\n')
     sys.stdout.flush()
 
+    if len(sys.argv) > 1 and sys.argv[1] == "fast_fail":
+        fast_fail = True
+        subset_start = 2
+    else:
+        fast_fail = False
+        subset_start = 1
     loader = unittest.TestLoader()
-    runner = unittest.TextTestRunner(verbosity=2, resultclass=TestResult)
+    runner = unittest.TextTestRunner(
+        verbosity=2, resultclass=TestResult, failfast=fast_fail
+    )
     suite = unittest.TestSuite()
     subsets = []
-    for name in sys.argv[1:]:
+    for name in sys.argv[subset_start:]:
         subsets.append('cantera.test.test_' + name)
 
     if not subsets: