Make FailFast blocks cold (GS cookies) (#93429)

src/coreclr/jit/codegenarmarch.cpp

@@ -631,11 +631,8 @@ void CodeGen::genEmitGSCookieCheck(bool pushReg)
     // Compare with the GC cookie constant
     GetEmitter()->emitIns_R_R(INS_cmp, EA_PTRSIZE, regGSConst, regGSValue);
 
-    BasicBlock* gsCheckBlk = genCreateTempLabel();
-    inst_JMP(EJ_eq, gsCheckBlk);
-    // regGSConst and regGSValue aren't needed anymore, we can use them for helper call
-    genEmitHelperCall(CORINFO_HELP_FAIL_FAST, 0, EA_UNKNOWN, regGSConst);
-    genDefineTempLabel(gsCheckBlk);
+    Compiler::AddCodeDsc* codeDsc = compiler->fgFindExcptnTarget(SpecialCodeKind::SCK_FAIL_FAST, 0);
+    inst_JMP(EJ_ne, codeDsc->acdDstBlk);
 }
 
 //---------------------------------------------------------------------

src/coreclr/jit/codegenloongarch64.cpp

@@ -5192,12 +5192,8 @@ void CodeGen::genEmitGSCookieCheck(bool pushReg)
     GetEmitter()->emitIns_R_S(INS_ld_d, EA_PTRSIZE, regGSValue, compiler->lvaGSSecurityCookie, 0);
 
     // Compare with the GC cookie constant
-    BasicBlock* gsCheckBlk = genCreateTempLabel();
-    GetEmitter()->emitIns_J_cond_la(INS_beq, gsCheckBlk, regGSConst, regGSValue);
-
-    // regGSConst and regGSValue aren't needed anymore, we can use them for helper call
-    genEmitHelperCall(CORINFO_HELP_FAIL_FAST, 0, EA_UNKNOWN, regGSConst);
-    genDefineTempLabel(gsCheckBlk);
+    Compiler::AddCodeDsc* codeDsc = compiler->fgFindExcptnTarget(SpecialCodeKind::SCK_FAIL_FAST, 0);
+    GetEmitter()->emitIns_J_cond_la(INS_bne, codeDsc->acdDstBlk, regGSConst, regGSValue);
 }
 
 //---------------------------------------------------------------------

src/coreclr/jit/codegenriscv64.cpp

@@ -4958,12 +4958,8 @@ void CodeGen::genEmitGSCookieCheck(bool pushReg)
     GetEmitter()->emitIns_R_S(INS_ld, EA_PTRSIZE, regGSValue, compiler->lvaGSSecurityCookie, 0);
 
     // Compare with the GC cookie constant
-    BasicBlock* gsCheckBlk = genCreateTempLabel();
-    GetEmitter()->emitIns_J_cond_la(INS_beq, gsCheckBlk, regGSConst, regGSValue);
-
-    // regGSConst and regGSValue aren't needed anymore, we can use them for helper call
-    genEmitHelperCall(CORINFO_HELP_FAIL_FAST, 0, EA_UNKNOWN, regGSConst);
-    genDefineTempLabel(gsCheckBlk);
+    Compiler::AddCodeDsc* codeDsc = compiler->fgFindExcptnTarget(SpecialCodeKind::SCK_FAIL_FAST, 0);
+    GetEmitter()->emitIns_J_cond_la(INS_bne, codeDsc->acdDstBlk, regGSConst, regGSValue);
 }
 
 //---------------------------------------------------------------------

src/coreclr/jit/codegenxarch.cpp

@@ -195,11 +195,8 @@ void CodeGen::genEmitGSCookieCheck(bool pushReg)
         GetEmitter()->emitIns_S_R(INS_cmp, EA_PTRSIZE, regGSCheck, compiler->lvaGSSecurityCookie, 0);
     }
 
-    BasicBlock* gsCheckBlk = genCreateTempLabel();
-    inst_JMP(EJ_je, gsCheckBlk);
-    genEmitHelperCall(CORINFO_HELP_FAIL_FAST, 0, EA_UNKNOWN);
-    genDefineTempLabel(gsCheckBlk);
-
+    Compiler::AddCodeDsc* codeDsc = compiler->fgFindExcptnTarget(SpecialCodeKind::SCK_FAIL_FAST, 0);
+    inst_JMP(EJ_jne, codeDsc->acdDstBlk);
     genPopRegs(pushedRegs, byrefPushedRegs, norefPushedRegs);
 }
 

src/coreclr/jit/compiler.h

@@ -6117,7 +6117,6 @@ class Compiler
 
     AddCodeDsc* fgAddCodeList;
     bool        fgAddCodeModf;
-    bool        fgRngChkThrowAdded;
     AddCodeDsc* fgExcptnTargetCache[SCK_COUNT];
 
     BasicBlock* fgRngChkTarget(BasicBlock* block, SpecialCodeKind kind);

src/coreclr/jit/compiler.hpp

@@ -3095,6 +3095,7 @@ inline bool Compiler::fgIsThrowHlpBlk(BasicBlock* block)
 
     if (!((call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_RNGCHKFAIL)) ||
           (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROWDIVZERO)) ||
+          (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_FAIL_FAST)) ||
           (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROW_ARGUMENTEXCEPTION)) ||
           (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROW_ARGUMENTOUTOFRANGEEXCEPTION)) ||
           (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_OVERFLOW))))
@@ -3111,7 +3112,7 @@ inline bool Compiler::fgIsThrowHlpBlk(BasicBlock* block)
         if (block == add->acdDstBlk)
         {
             return add->acdKind == SCK_RNGCHK_FAIL || add->acdKind == SCK_DIV_BY_ZERO || add->acdKind == SCK_OVERFLOW ||
-                   add->acdKind == SCK_ARG_EXCPN || add->acdKind == SCK_ARG_RNG_EXCPN;
+                   add->acdKind == SCK_ARG_EXCPN || add->acdKind == SCK_ARG_RNG_EXCPN || add->acdKind == SCK_FAIL_FAST;
         }
     }
 
@@ -3136,7 +3137,7 @@ inline unsigned Compiler::fgThrowHlpBlkStkLevel(BasicBlock* block)
             // Compute assert cond separately as assert macro cannot have conditional compilation directives.
             bool cond =
                 (add->acdKind == SCK_RNGCHK_FAIL || add->acdKind == SCK_DIV_BY_ZERO || add->acdKind == SCK_OVERFLOW ||
-                 add->acdKind == SCK_ARG_EXCPN || add->acdKind == SCK_ARG_RNG_EXCPN);
+                 add->acdKind == SCK_ARG_EXCPN || add->acdKind == SCK_ARG_RNG_EXCPN || add->acdKind == SCK_FAIL_FAST);
             assert(cond);
 
             // TODO: bbTgtStkDepth is DEBUG-only.

src/coreclr/jit/fgbasic.cpp

@@ -109,9 +109,6 @@ void Compiler::fgInit()
     /* This global flag is set whenever we remove a statement */
     fgStmtRemoved = false;
 
-    /* This global flag is set whenever we add a throw block for a RngChk */
-    fgRngChkThrowAdded = false; /* reset flag for fgIsCodeAdded() */
-
     /* Keep track of whether or not EH statements have been optimized */
     fgOptimizedFinally = false;
 

src/coreclr/jit/flowgraph.cpp

@@ -3584,6 +3584,8 @@ unsigned Compiler::acdHelper(SpecialCodeKind codeKind)
             return CORINFO_HELP_THROWDIVZERO;
         case SCK_ARITH_EXCPN:
             return CORINFO_HELP_OVERFLOW;
+        case SCK_FAIL_FAST:
+            return CORINFO_HELP_FAIL_FAST;
         default:
             assert(!"Bad codeKind");
             return 0;
@@ -3608,8 +3610,10 @@ BasicBlock* Compiler::fgAddCodeRef(BasicBlock* srcBlk, unsigned refData, Special
     // arg slots on the stack frame if there are no other calls.
     compUsesThrowHelper = true;
 
-    if (!fgUseThrowHelperBlocks())
+    if (!fgUseThrowHelperBlocks() && (kind != SCK_FAIL_FAST))
     {
+        // We'll create a throw block in-place then (for better debugging)
+        // It's not needed for fail fast, since it's not recoverable anyway.
         return nullptr;
     }
 
@@ -3620,6 +3624,7 @@ BasicBlock* Compiler::fgAddCodeRef(BasicBlock* srcBlk, unsigned refData, Special
         BBJ_THROW, // SCK_ARITH_EXCP, SCK_OVERFLOW
         BBJ_THROW, // SCK_ARG_EXCPN
         BBJ_THROW, // SCK_ARG_RNG_EXCPN
+        BBJ_THROW, // SCK_FAIL_FAST
     };
 
     noway_assert(sizeof(jumpKinds) == SCK_COUNT); // sanity check
@@ -3695,6 +3700,9 @@ BasicBlock* Compiler::fgAddCodeRef(BasicBlock* srcBlk, unsigned refData, Special
             case SCK_ARG_RNG_EXCPN:
                 msg = " for ARG_RNG_EXCPN";
                 break;
+            case SCK_FAIL_FAST:
+                msg = " for FAIL_FAST";
+                break;
             default:
                 msg = " for ??";
                 break;
@@ -3712,8 +3720,7 @@ BasicBlock* Compiler::fgAddCodeRef(BasicBlock* srcBlk, unsigned refData, Special
 
     /* Remember that we're adding a new basic block */
 
-    fgAddCodeModf      = true;
-    fgRngChkThrowAdded = true;
+    fgAddCodeModf = true;
 
     /* Now figure out what code to insert */
 
@@ -3743,6 +3750,10 @@ BasicBlock* Compiler::fgAddCodeRef(BasicBlock* srcBlk, unsigned refData, Special
             helper = CORINFO_HELP_THROW_ARGUMENTOUTOFRANGEEXCEPTION;
             break;
 
+        case SCK_FAIL_FAST:
+            helper = CORINFO_HELP_FAIL_FAST;
+            break;
+
         default:
             noway_assert(!"unexpected code addition kind");
             return nullptr;
@@ -3759,7 +3770,6 @@ BasicBlock* Compiler::fgAddCodeRef(BasicBlock* srcBlk, unsigned refData, Special
     tree = fgMorphArgs(tree);
 
     // Store the tree in the new basic block.
-    assert(!srcBlk->isEmpty());
     if (!srcBlk->IsLIR())
     {
         fgInsertStmtAtEnd(newBlk, fgNewStmtFromTree(tree));
@@ -3781,7 +3791,7 @@ BasicBlock* Compiler::fgAddCodeRef(BasicBlock* srcBlk, unsigned refData, Special
 
 Compiler::AddCodeDsc* Compiler::fgFindExcptnTarget(SpecialCodeKind kind, unsigned refData)
 {
-    assert(fgUseThrowHelperBlocks());
+    assert(fgUseThrowHelperBlocks() || (kind == SCK_FAIL_FAST));
     if (!(fgExcptnTargetCache[kind] && // Try the cached value first
           fgExcptnTargetCache[kind]->acdData == refData))
     {

src/coreclr/jit/gentree.h

@@ -60,6 +60,7 @@ enum SpecialCodeKind
     SCK_OVERFLOW = SCK_ARITH_EXCPN, // target on overflow
     SCK_ARG_EXCPN,                  // target on ArgumentException (currently used only for SIMD intrinsics)
     SCK_ARG_RNG_EXCPN,              // target on ArgumentOutOfRangeException (currently used only for SIMD intrinsics)
+    SCK_FAIL_FAST,                  // target for fail fast exception
     SCK_COUNT
 };
 

src/coreclr/jit/gschecks.cpp

@@ -30,6 +30,8 @@ PhaseStatus Compiler::gsPhase()
         unsigned const prevBBCount = fgBBcount;
         gsGSChecksInitCookie();
 
+        fgAddCodeRef(fgLastBB, 0, SCK_FAIL_FAST);
+
         if (compGSReorderStackLayout)
         {
             gsCopyShadowParams();