[MERGE #5039 @Cellule] RL: Stop on first error seen

Merge pull request #5039 from Cellule:rl_stoponerror

Add an option to rl to stop testing upon seeing an error.
This is great when working on a feature and you just want to find a failing unit test quick to iterate faster.

bin/rl/rl.cpp

@@ -322,6 +322,8 @@ BOOL FVerbose;
 BOOL FQuiet;
 BOOL FNoWarn;
 BOOL FTest;
+BOOL FStopOnError = FALSE;
+BOOL GStopDueToError = FALSE;
 BOOL FLow;
 BOOL FNoDelete;
 BOOL FCopyOnFail;
@@ -1155,6 +1157,7 @@ Usage(
          "\n"
          "    -nosummary disables output of summary diff/failure info (implies -dirname)\n"
          "    -dirname enables output of '*** dirname ***'\n"
+         "    -stoponerror stop on first test failure\n"
          "    -nomovediffs to not place assembly diffs in DIFF_DIR\n"
          "    -nodelete to not delete objs and exes when doing -exe\n"
          "         (only compatible with a single EXEC_TESTS_FLAGS flags)\n"
@@ -2848,6 +2851,11 @@ ParseArg(
             FTest = TRUE;
             break;
          }
+         if (!_stricmp(&arg[1], "stoponerror"))
+         {
+             FStopOnError = TRUE;
+             break;
+         }
          if (!_stricmp(&arg[1], "exeflags")) {
             EXEC_TESTS_FLAGS = ComplainIfNoArg(arg, s);
             break;
@@ -3583,7 +3591,7 @@ GetTestInfoFromNode
             {
                // Validate the timeout string now to fail early so we don't run any tests when there is an error.
                if (!IsTimeoutStringValid(testInfo->data[i])) {
-                  CFG_ERROR_EX(fileName, node->LineNumber, 
+                  CFG_ERROR_EX(fileName, node->LineNumber,
                      "Invalid timeout specified. Cannot parse or too large.\n", NULL);
                   childNode->Dump();
                   return FALSE;
@@ -4621,7 +4629,7 @@ PerformSingleRegression(
     // Before executing, make sure directory is started.
     pDir->TryBeginDirectory();
 
-    for (TestVariant * pTestVariant = pTest->variants; pTestVariant != NULL; pTestVariant = pTestVariant->next)
+    for (TestVariant * pTestVariant = pTest->variants; pTestVariant != NULL && !GStopDueToError; pTestVariant = pTestVariant->next)
     {
         ThreadInfo[ThreadId].SetCurrentTest(pDir->GetDirectoryName(), pTest->name, pDir->IsBaseline());
 
@@ -4791,7 +4799,7 @@ RegressDirectory(
    // variation. We do not need to write any directory summary here, the CDirectory
    // object itself will take care of that when status is updated during regression
    // execution.
-   for (pTest = pTestList->first; pTest; pTest = pTest->next)
+   for (pTest = pTestList->first; pTest && !GStopDueToError; pTest = pTest->next)
    {
        PerformSingleRegression(pDir, pTest);
    }
@@ -4859,7 +4867,7 @@ ThreadWorker( void *arg )
    }
 
    while(TRUE) {
-       if(bThreadStop)
+       if(bThreadStop || GStopDueToError)
            break;
 
        // Poll for new stuff. If the thread is set to stop, then go away.
@@ -4887,7 +4895,7 @@ ThreadWorker( void *arg )
    }
 
    while(TRUE) {
-       if(bThreadStop)
+       if(bThreadStop || GStopDueToError)
            break;
 
        // Poll for new stuff. If the thread is set to stop, then go away.
@@ -5075,7 +5083,7 @@ main(int argc, char *argv[])
       if (status == PCS_ERROR)
       {
           exit(1);
-      } 
+      }
       else
       {
 

bin/rl/rl.h

@@ -871,6 +871,7 @@ extern BOOL FParallel;
 extern BOOL FSyncEnumDirs;
 extern BOOL FNogpfnt;
 extern BOOL FTest;
+extern BOOL FStopOnError;
 extern BOOL FAppendTestNameToExtraCCFlags;
 extern BOOL FNoProgramOutput;
 extern BOOL FOnlyAssertOutput;
@@ -882,6 +883,7 @@ extern const char *OptFlags[MAXOPTIONS + 1], *PogoOptFlags[MAXOPTIONS + 1];
 extern BOOL FDebug;
 #endif
 
+extern BOOL GStopDueToError;
 extern BOOL bThreadStop;
 
 extern BOOL FSyncImmediate;

bin/rl/rlrun.cpp

@@ -42,8 +42,6 @@ __declspec(thread) char EnvFlags[MAX_ENV_LEN];
 // either be protected by synchronization or use thread-local storage.
 //
 
-// currently, none
-
 
 LOCAL void __cdecl
     RunCleanUp()
@@ -526,7 +524,7 @@ int
         }
 
         //
-        // If the test is a JS test and we've passed in a custom config file, 
+        // If the test is a JS test and we've passed in a custom config file,
         // ignore all of the other flags and just pass the config file in
         //
         if (kind == TK_JSCRIPT && pTestVariant->testInfo.data[TIK_CUSTOM_CONFIG_FILE] != nullptr)
@@ -600,6 +598,10 @@ int
         if (fDumpOutputFile) {
             DumpFileToLog(full);
         }
+        if (FStopOnError)
+        {
+            GStopDueToError = TRUE;
+        }
     }
 
 SkipLogFailure:
@@ -1333,7 +1335,7 @@ int
 
     // Check to see if all of the files exist.
 
-    for (StringList * pFile = pTest->files; pFile != NULL; pFile = pFile->next)
+    for (StringList * pFile = pTest->files; pFile != NULL && !GStopDueToError; pFile = pFile->next)
     {
         // Get a pointer to the filename sans path, if present.
 

test/runtests.cmd

@@ -58,6 +58,8 @@ goto :main
   echo Options:
   echo.
   echo   -dirs dirname  Run only the specified directory
+  echo   -rebase        Create .rebase file on baseline comparision failure
+  echo   -stoponerror   Stop testing after first failure (will finish current execution)
   :: TODO Add more usage help
 
   goto :eof
@@ -177,6 +179,7 @@ goto :main
   :: TODO Consider removing -drt and exclude_drt in some reasonable manner
   if /i "%1" == "-drt"              set _drt=1& set _NOTTAGS=%_NOTTAGS% -nottags:exclude_drt&   goto :ArgOk
   if /i "%1" == "-rebase"           set _rebase=-rebase&                                        goto :ArgOk
+  if /i "%1" == "-stoponerror"      set _stoponerror=-stoponerror&                              goto :ArgOk
   if /i "%1" == "-rundebug"         set _RUNDEBUG=1&                                            goto :ArgOk
   :: TODO Figure out best way to specify build arch for tests that are excluded to specific archs
   if /i "%1" == "-platform"         set _buildArch=%2&                                          goto :ArgOkShift2
@@ -273,6 +276,7 @@ goto :main
   set _DIRTAGS=
   set _drt=
   set _rebase=
+  set _stoponerror=
   set _ExtraVariants=
   set _dynamicprofilecache=-dynamicprofilecache:profile.dpl
   set _dynamicprofileinput=-dynamicprofileinput:profile.dpl
@@ -381,6 +385,15 @@ goto :main
 :: Run one variant
 :: ============================================================================
 :RunOneVariant
+  if exist %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG% (
+    rd /q /s %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG%
+  )
+
+  if %_HadFailures% NEQ 0 (
+    if "%_stoponerror%" NEQ "" (
+      goto :eof
+    )
+  )
 
   if "%_BuildType%" == "test" (
     rem bytecode layout switches not available in test build
@@ -521,6 +534,7 @@ goto :main
   set _rlArgs=%_rlArgs% -exe
   set _rlArgs=%_rlArgs% %EXTRA_RL_FLAGS%
   set _rlArgs=%_rlArgs% %_rebase%
+  set _rlArgs=%_rlArgs% %_stoponerror%
 
   set REGRESS=%CD%