From 4ff821510a60c6af8bb379668fdba834ab9f41c9 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Thu, 15 Jun 2023 14:20:22 -0700 Subject: [PATCH 01/35] Optimize if range check to bit operation by changing if tree to switch tree. --- src/coreclr/jit/compiler.cpp | 2 + src/coreclr/jit/compiler.h | 1 + src/coreclr/jit/compphases.h | 1 + src/coreclr/jit/optimizebools.cpp | 441 ++++++++++++++++++++++++++++++ 4 files changed, 445 insertions(+) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index a04bd56057abb0..e9f6aebda231df 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5074,6 +5074,8 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // DoPhase(this, PHASE_OPTIMIZE_BOOLS, &Compiler::optOptimizeBools); + DoPhase(this, PHASE_Find_Specific_Pattern, &Compiler::optFindSpecificPattern); + // If conversion // DoPhase(this, PHASE_IF_CONVERSION, &Compiler::optIfConversion); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 3214a8a581c82e..45dc53e85c7ce8 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6316,6 +6316,7 @@ class Compiler public: PhaseStatus optOptimizeBools(); + PhaseStatus optFindSpecificPattern(); public: PhaseStatus optInvertLoops(); // Invert loops so they're entered at top and tested at bottom. diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h index 9fe31ca846410e..5814a68823266b 100644 --- a/src/coreclr/jit/compphases.h +++ b/src/coreclr/jit/compphases.h @@ -72,6 +72,7 @@ CompPhaseNameMacro(PHASE_MORPH_MDARR, "Morph array ops", CompPhaseNameMacro(PHASE_HOIST_LOOP_CODE, "Hoist loop code", false, -1, false) CompPhaseNameMacro(PHASE_MARK_LOCAL_VARS, "Mark local vars", false, -1, false) CompPhaseNameMacro(PHASE_OPTIMIZE_BOOLS, "Optimize bools", false, -1, false) +CompPhaseNameMacro(PHASE_Find_Specific_Pattern, "Find Specific pattern", false, -1, false) CompPhaseNameMacro(PHASE_FIND_OPER_ORDER, "Find oper order", false, -1, false) CompPhaseNameMacro(PHASE_SET_BLOCK_ORDER, "Set block order", false, -1, true) CompPhaseNameMacro(PHASE_BUILD_SSA, "Build SSA representation", true, -1, false) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 043d4acb476ebf..238b6dda3c2fa7 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1458,6 +1458,12 @@ GenTree* OptBoolsDsc::optIsBoolComp(OptTestInfo* pOptTest) // PhaseStatus Compiler::optOptimizeBools() { + + if (!ISMETHOD("Test")) + { + return PhaseStatus::MODIFIED_NOTHING; + } + #ifdef DEBUG if (verbose) { @@ -1570,3 +1576,438 @@ PhaseStatus Compiler::optOptimizeBools() const bool modified = stress || ((numCond + numReturn) > 0); return modified ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; } + +//----------------------------------------------------------------------------- +// OptRangePatternDsc: Descriptor used for `if` range pattern optimization +// +class OptRangePatternDsc +{ +public: + OptRangePatternDsc(Compiler* comp) + { + m_comp = comp; + } + + static const int m_sizePatterns = 64; // The size of the patterns array + int m_patterns[m_sizePatterns] = {}; // Reserved patterns + int m_patternsLength = 0; // The number of patterns in the array + int m_numFoundPatterns = 0; // The number of patterns found + + int m_minPattern = 0; // The minimum pattern + int m_maxPattern = 0; // The maximum pattern + int m_rangePattern = 0; // The range of values in patterns[] + + unsigned int m_firstPatternBBNum = 0; // BB that the first pattern is found + BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern + BasicBlock* m_trueJmpBB = nullptr; // The BB to jump to in case of true condition + BasicBlock* m_falseJmpBB = nullptr; // The BB to jump to in case of false condition + + unsigned m_bbCodeOffs = 0; + unsigned m_bbCodeOffsEnd = 0; + BasicBlock* switchBBdesc = nullptr; + +private: + Compiler* m_comp = nullptr; // The pointer to the Compiler instance + +public: + bool optMakeSwitchBBdesc(); + bool optUpdateBlocks(); +}; + +//----------------------------------------------------------------------------- +// optUpdateBlocks: Remove pattern blocks and update predecessors of jump target blocks +// +bool OptRangePatternDsc::optUpdateBlocks() +{ + if (ISMETHOD("Foo")) + { + printf("optUpdateBlocks()\n"); + } + + // Special args to fgAddRefPred + FlowEdge* const oldEdge = nullptr; + + // Skip the Swithc block and start from the next block + BasicBlock* currBb = m_optFirstBB->bbNext; + + for (int idxRng = 1; idxRng < m_rangePattern && currBb != nullptr; idxRng++) + { + auto condition1 = currBb->lastStmt()->GetRootNode()->gtGetOp1(); + if ((idxRng + m_minPattern) == (int)condition1->gtGetOp2()->AsIntCon()->IconValue()) + { + // Reference count updates of true block or flase block + if (condition1->OperGet() == GT_EQ) + { + m_comp->fgAddRefPred(m_trueJmpBB, switchBBdesc, oldEdge); + } + else if (condition1->OperGet() == GT_NE) + { + m_comp->fgAddRefPred(m_falseJmpBB, switchBBdesc, oldEdge); + } + else + { + return false; + } + + // Remove the current block + BasicBlock* nextBlock = currBb->bbNext; + BasicBlock* prevBlock = currBb->bbPrev; + + m_comp->fgRemoveRefPred(currBb, prevBlock); + m_comp->fgAddRefPred(nextBlock, prevBlock, oldEdge); + m_comp->fgRemoveBlock(currBb, /* unreachable */ true); + + prevBlock->bbNext = nextBlock; + currBb = nextBlock; + } + else if ((idxRng + m_minPattern) < (int)condition1->gtGetOp2()->AsIntCon()->IconValue()) + { + // False patterns + m_comp->fgAddRefPred(m_falseJmpBB, switchBBdesc, oldEdge); + } + else + { + return false; + } + } + + return true; +} + +//----------------------------------------------------------------------------- +// optMakeSwitchBBdesc: Create a switch block descriptor for the range pattern optimization +// +bool OptRangePatternDsc::optMakeSwitchBBdesc() +{ + + if (ISMETHOD("Foo")) + { + printf("\nStart optMakeSwitchBlock()\n"); + } + + if (m_numFoundPatterns == 0) + { + return false; + } + + ////bool printed = false; + ////bool foundPattern = false; + BasicBlock* prevBb = nullptr; + int patternIndex = 0; + BBswtDesc* swtDsc = nullptr; + BBjumpKinds jmpKind = BBJ_NONE; + + //BasicBlockFlags bbFlags = BBF_EMPTY; + unsigned curBBoffs = 0; + unsigned nxtBBoffs = 0; + unsigned jmpCnt = 0; // # of switch cases (excluding default) + + BasicBlock** jmpTab = nullptr; + BasicBlock** jmpPtr = nullptr; + BasicBlock* currBb = m_optFirstBB; + unsigned sz = 0; + bool tailCall = false; + + // Special args to fgAddRefPred so it will use the initialization fast path + FlowEdge* const oldEdge = nullptr; + bool const initializingPreds = true; + + // Make a jump table for the range of patterns + for (int idxRng = 0; idxRng < m_rangePattern && currBb != nullptr; idxRng++) + { + assert(m_numFoundPatterns <= m_rangePattern && m_numFoundPatterns <= m_sizePatterns); + + // First pattern + if (idxRng == 0) + { + assert(currBb->bbNum == m_firstPatternBBNum && currBb->lastStmt()->GetRootNode()->gtGetOp1()->OperIs(GT_EQ)); + + // Allocate the switch descriptor + swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; + + // Allocate the jump table + jmpCnt = m_rangePattern; + jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt + 1]; + + // Change if basic block to a Switch basic block + // curBBdesc = m_comp->bbNewBasicBlock(jmpKind); + switchBBdesc = currBb; + } + + // Fill in the jump table with the jump code offset + // If the pattern is found, jump to true BB. If not, to false BB. + auto condition1 = currBb->lastStmt()->GetRootNode()->gtGetOp1(); + if ((idxRng + m_minPattern) == (int)condition1->gtGetOp2()->AsIntCon()->IconValue()) + { + *jmpPtr = (BasicBlock*)(size_t)m_trueJmpBB->bbCodeOffs; + *(jmpPtr) = m_trueJmpBB; + jmpPtr++; + + // Update the code offset range + if (currBb->bbCodeOffs <= m_bbCodeOffs) + m_bbCodeOffs = currBb->bbCodeOffs; + if (currBb->bbCodeOffsEnd >= m_bbCodeOffsEnd) + m_bbCodeOffsEnd = currBb->bbCodeOffsEnd; + + currBb = currBb->bbNext; + } + else if ((idxRng + m_minPattern) < (int)condition1->gtGetOp2()->AsIntCon()->IconValue()) + { + *jmpPtr = (BasicBlock*)(size_t)m_falseJmpBB->bbCodeOffs; + *(jmpPtr) = m_falseJmpBB; + jmpPtr++; + } + else + { + assert(false && "Unexpected pattern"); + } + } + + // Append the default label to the target table + *jmpPtr = (BasicBlock*)(size_t)m_falseJmpBB->bbCodeOffs; + *(jmpPtr) = m_falseJmpBB; + jmpPtr++; + + // Make sure we found the right number of labels + noway_assert(jmpPtr == jmpTab + jmpCnt + 1); + + // Compute the size of the switch opcode operands + sz = sizeof(DWORD) + jmpCnt * sizeof(DWORD); + + // Fill in the remaining fields of the switch descriptor + swtDsc->bbsCount = jmpCnt + 1; + swtDsc->bbsDstTab = jmpTab; + jmpKind = BBJ_SWITCH; + + m_comp->fgHasSwitch = true; + + if (m_comp->opts.compProcedureSplitting) + { + // TODO-CQ: We might need to create a switch table; we won't know for sure until much later. + // However, switch tables don't work with hot/cold splitting, currently. The switch table data needs + // a relocation such that if the base (the first block after the prolog) and target of the switch + // branch are put in different sections, the difference stored in the table is updated. However, our + // relocation implementation doesn't support three different pointers (relocation address, base, and + // target). So, we need to change our switch table implementation to be more like + // JIT64: put the table in the code section, in the same hot/cold section as the switch jump itself + // (maybe immediately after the switch jump), and make the "base" address be also in that section, + // probably the address after the switch jump. + m_comp->opts.compProcedureSplitting = false; + JITDUMP("Turning off procedure splitting for this method, as it might need switch tables; " + "implementation limitation.\n"); + } + + tailCall = false; + + // Fill in the rest of switch basic block fields + switchBBdesc->bbJumpKind = BBJ_SWITCH; + //switchBBdesc->bbFlags |= bbFlags; + if (m_comp->compRationalIRForm) + { + switchBBdesc->bbFlags |= BBF_IS_LIR; + } + // switchBBdesc->bbRefs = 0; + switchBBdesc->bbCodeOffs = m_bbCodeOffs; + switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; + switchBBdesc->bbJumpSwt = swtDsc; + + // Print bbNum of each jmpTab + for (unsigned i = 0; i < swtDsc->bbsCount; i++) + { + printf("%c\n" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); + } + + // Change from GT_JTRUE to GT_SWITCH + switchBBdesc->lastStmt()->GetRootNode()->SetOper(GT_SWITCH, GenTree::PRESERVE_VN); + + // Change from GT_EQ or GT_NE to SUB + // tree: SUB + // op1: LCL_VAR + // op2: GT_CNS_INT + GenTree* tree = switchBBdesc->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node + tree->SetOper(GT_SUB, GenTree::PRESERVE_VN); + + // get LCL_VAR node from GT_EQ or GT_NE node in case of COMMA node + if (tree->gtGetOp1()->OperIs(GT_COMMA)) + { + GenTree* op1 = tree->gtGetOp1()->gtGetOp1(); // op1 of COMMA node + GenTree* op2 = tree->gtGetOp1()->gtGetOp2(); // LCL_VAR node + tree->AsOp()->gtOp1 = op2; // Set LCL_VAR node to op1 of tree + + DEBUG_DESTROY_NODE(op1); + } + + return true; +} + +//----------------------------------------------------------------------------- +// optFindSpecificPattern: Optimize range check for if (A || B || C || D) pattern. +// +// Returns: +// MODIFIED_NOTHING if no optimization is performed. +// MODIFIED_EVERYTHING otherwise. +// +// Notes: +// Detect if (a == val1 || a == val2 || a == val3 || a == val4) pattern and change it to switch tree to reduce +// jumps and perform bit operation instead. +// +PhaseStatus Compiler::optFindSpecificPattern() +{ + if (!ISMETHOD("Foo")) // TODO clean up code + { + return PhaseStatus::MODIFIED_NOTHING; + } + + OptRangePatternDsc optRngPattern(this); + + bool printed = false; + int patternIndex = 0; // The index of the pattern in the array + bool foundPattern = false; + BasicBlock* prevBb = nullptr; + + for (BasicBlock* currBb = fgFirstBB; currBb != nullptr; currBb = currBb->bbNext) + { + if ((currBb->KindIs(BBJ_COND) || currBb->KindIs(BBJ_RETURN)) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) + { + if (!printed) + { + printf("Display Basic Blocks for optFindSpecificPattern:\n"); + fgDispBasicBlocks(true); + printed = true; + } + + if (currBb->lastStmt() == currBb->firstStmt() && prevBb->lastStmt() == prevBb->firstStmt()) + { + auto condition1 = currBb->lastStmt()->GetRootNode()->gtGetOp1(); + auto condition2 = prevBb->lastStmt()->GetRootNode()->gtGetOp1(); + if (condition1->OperIsCompare() && condition2->OperIsCompare()) + { + if (currBb->bbJumpDest == prevBb->bbJumpDest || + (condition1->OperIs(GT_NE) && condition2->OperIs(GT_EQ) && currBb->bbNext == prevBb->bbJumpDest)) + { + // Check both conditions to have constant on the right side + if (condition1->gtGetOp2()->IsIntegralConst() && condition2->gtGetOp2()->IsIntegralConst()) + { + auto leftCondition1 = condition1->gtGetOp1(); // op1 of condition1 from currBb + auto leftCondition2 = condition2->gtGetOp1(); // op1 of condition2 from prevBb + if (leftCondition1->IsLocal() && + ((leftCondition2->IsLocal() && leftCondition1->AsLclVarCommon()->GetLclNum() == + leftCondition2->AsLclVarCommon()->GetLclNum()) || + (leftCondition2->OperIs(GT_COMMA) && + leftCondition1->AsLclVarCommon()->GetLclNum() == + leftCondition2->gtEffectiveVal(/* commaOnly */ true) + ->AsLclVarCommon() + ->GetLclNum()))) + { + printf("\nFound pattern (Curr vs Prev):\n"); + gtDispTree(condition1); + printf("\n"); + gtDispTree(condition2); + printf("\n\n\n"); + + // Store the found pattern to the patterns + if (patternIndex >= optRngPattern.m_sizePatterns) + { + printf("Too many patterns found (> 64), no optimization done.\n"); + return PhaseStatus::MODIFIED_NOTHING; + } + + // Previous pattern + if (!foundPattern) // First pattern found + { + optRngPattern.m_patterns[patternIndex] = + (int)condition2->gtGetOp2()->AsIntCon()->IconValue(); + optRngPattern.m_firstPatternBBNum = prevBb->bbNum; + optRngPattern.m_optFirstBB = prevBb; + + assert(condition2->OperIs(GT_EQ)); + optRngPattern.m_trueJmpBB = prevBb->bbJumpDest; + + // min and max patterns + optRngPattern.m_minPattern = optRngPattern.m_patterns[patternIndex]; + optRngPattern.m_maxPattern = optRngPattern.m_patterns[patternIndex++]; + } + + // Current pattern + optRngPattern.m_patterns[patternIndex] = + (int)condition1->gtGetOp2()->AsIntCon()->IconValue(); + + // False jump + if (condition1->OperIs(GT_NE)) + { + optRngPattern.m_falseJmpBB = currBb->bbJumpDest; + } + + // Update min and max patterns + if (optRngPattern.m_patterns[patternIndex] < optRngPattern.m_minPattern) + { + optRngPattern.m_minPattern = optRngPattern.m_patterns[patternIndex]; + } + else if (optRngPattern.m_patterns[patternIndex] > optRngPattern.m_maxPattern) + { + optRngPattern.m_maxPattern = optRngPattern.m_patterns[patternIndex]; + } + patternIndex++; + + foundPattern = true; + } + } + } + } + } + } + + // Stop searching if pattern(s) has been found in previous BBs, but the current BB does not have a pattern + if (foundPattern && patternIndex < (int)(currBb->bbNum - optRngPattern.m_firstPatternBBNum + 1)) + { + break; + } + + prevBb = currBb; + } + + if (foundPattern) + { + optRngPattern.m_numFoundPatterns = patternIndex; + + printf("Reserved values:\n"); + for (int idx = 0; idx < optRngPattern.m_numFoundPatterns; idx++) + { + printf("%d ", optRngPattern.m_patterns[idx]); + } + printf("\n\n"); + + // Find range of pattern values + printf("Min pattern value: %d\n", optRngPattern.m_minPattern); + printf("Max pattern value: %d\n", optRngPattern.m_maxPattern); + optRngPattern.m_rangePattern = + optRngPattern.m_maxPattern - optRngPattern.m_minPattern + 1; // min 0 max 2 -> range 3 + printf("Range of pattern values: %d\n", optRngPattern.m_rangePattern); + if (optRngPattern.m_rangePattern > optRngPattern.m_sizePatterns) // TODO: item 0 is 1 and item 1 is 100, rnage + // is 64 but only 2 items + { + printf("Range of pattern values is too big (> %d): %d\n", optRngPattern.m_sizePatterns, + optRngPattern.m_rangePattern); + return PhaseStatus::MODIFIED_NOTHING; + } + + //auto bitmapPatterns = 0; + //for (int idxPattern = 0; idxPattern < optRngPattern.m_numFoundPatterns; idxPattern++) + //{ + // bitmapPatterns |= (1 << (optRngPattern.m_patterns[idxPattern] - optRngPattern.m_minPattern)); + //} + //printf("Bitmap of pattern values: %d\n", bitmapPatterns); + + // Replace BBs with a Switch tree + if (!optRngPattern.optMakeSwitchBBdesc()) + { + return PhaseStatus::MODIFIED_NOTHING; + } + if (optRngPattern.optUpdateBlocks()) + { + return PhaseStatus::MODIFIED_EVERYTHING; + } + } + + return PhaseStatus::MODIFIED_NOTHING; +} From dd7a34ebbe2114f868165945086cc4a463ed0dae Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Fri, 16 Jun 2023 15:30:53 -0700 Subject: [PATCH 02/35] [if range check opt] handle unsorted patterns. --- src/coreclr/jit/optimizebools.cpp | 218 ++++++++++++++++-------------- 1 file changed, 117 insertions(+), 101 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 238b6dda3c2fa7..6fec7a96c97392 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1590,12 +1590,13 @@ class OptRangePatternDsc static const int m_sizePatterns = 64; // The size of the patterns array int m_patterns[m_sizePatterns] = {}; // Reserved patterns - int m_patternsLength = 0; // The number of patterns in the array int m_numFoundPatterns = 0; // The number of patterns found int m_minPattern = 0; // The minimum pattern int m_maxPattern = 0; // The maximum pattern int m_rangePattern = 0; // The range of values in patterns[] + unsigned int bitmapPatterns = 0; // The bitmap of patterns found + GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern unsigned int m_firstPatternBBNum = 0; // BB that the first pattern is found BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern @@ -1619,56 +1620,64 @@ class OptRangePatternDsc // bool OptRangePatternDsc::optUpdateBlocks() { - if (ISMETHOD("Foo")) + if (ISMETHOD("FooNum2")) // TODO clean up code { printf("optUpdateBlocks()\n"); } + if (m_rangePattern == 0 || m_numFoundPatterns > m_rangePattern || m_rangePattern > m_sizePatterns) + { + return false; + } + // Special args to fgAddRefPred FlowEdge* const oldEdge = nullptr; + BasicBlock* currBb = m_optFirstBB; - // Skip the Swithc block and start from the next block - BasicBlock* currBb = m_optFirstBB->bbNext; - - for (int idxRng = 1; idxRng < m_rangePattern && currBb != nullptr; idxRng++) + // Update reference count of jump target blocks and remove pattern blocks + for (int idxPattern = 0; idxPattern < m_numFoundPatterns && currBb != nullptr; idxPattern++) { - auto condition1 = currBb->lastStmt()->GetRootNode()->gtGetOp1(); - if ((idxRng + m_minPattern) == (int)condition1->gtGetOp2()->AsIntCon()->IconValue()) + // Skip to add reference count for the Swithc block because it already has a link to jump block + if (idxPattern == 0) { - // Reference count updates of true block or flase block - if (condition1->OperGet() == GT_EQ) - { - m_comp->fgAddRefPred(m_trueJmpBB, switchBBdesc, oldEdge); - } - else if (condition1->OperGet() == GT_NE) - { - m_comp->fgAddRefPred(m_falseJmpBB, switchBBdesc, oldEdge); - } - else - { - return false; - } - - // Remove the current block - BasicBlock* nextBlock = currBb->bbNext; - BasicBlock* prevBlock = currBb->bbPrev; - - m_comp->fgRemoveRefPred(currBb, prevBlock); - m_comp->fgAddRefPred(nextBlock, prevBlock, oldEdge); - m_comp->fgRemoveBlock(currBb, /* unreachable */ true); + currBb = currBb->bbNext; + continue; + } - prevBlock->bbNext = nextBlock; - currBb = nextBlock; + // Reference count updates of the jump target blocks + auto operTree = currBb->lastStmt()->GetRootNode()->gtGetOp1(); + if (operTree->OperGet() == GT_EQ) + { + m_comp->fgAddRefPred(m_trueJmpBB, switchBBdesc, oldEdge); } - else if ((idxRng + m_minPattern) < (int)condition1->gtGetOp2()->AsIntCon()->IconValue()) + else if (operTree->OperGet() == GT_NE) { - // False patterns m_comp->fgAddRefPred(m_falseJmpBB, switchBBdesc, oldEdge); } else { return false; } + + // Remove the current block + BasicBlock* prevBlock = currBb->bbPrev; + BasicBlock* nextBlock = currBb->bbNext; + + // Unlink to the previous block and link the next block to the previous block + m_comp->fgRemoveRefPred(currBb, prevBlock); + m_comp->fgAddRefPred(nextBlock, prevBlock, oldEdge); + + m_comp->fgRemoveBlock(currBb, /* unreachable */ true); + + prevBlock->bbNext = nextBlock; + currBb = nextBlock; + } + + // Update the reference count of false jump block + int numNotFound = m_rangePattern - m_numFoundPatterns; + for (int idxFalse = 0; idxFalse < numNotFound; idxFalse++) + { + m_comp->fgAddRefPred(m_falseJmpBB, switchBBdesc, oldEdge); } return true; @@ -1680,24 +1689,22 @@ bool OptRangePatternDsc::optUpdateBlocks() bool OptRangePatternDsc::optMakeSwitchBBdesc() { - if (ISMETHOD("Foo")) + if (ISMETHOD("FooNum2")) // TODO clean up code { printf("\nStart optMakeSwitchBlock()\n"); } - if (m_numFoundPatterns == 0) + if (m_rangePattern == 0 || m_numFoundPatterns > m_rangePattern || m_rangePattern > m_sizePatterns) { return false; } - ////bool printed = false; - ////bool foundPattern = false; BasicBlock* prevBb = nullptr; int patternIndex = 0; BBswtDesc* swtDsc = nullptr; BBjumpKinds jmpKind = BBJ_NONE; - //BasicBlockFlags bbFlags = BBF_EMPTY; + BasicBlockFlags bbFlags = BBF_EMPTY; unsigned curBBoffs = 0; unsigned nxtBBoffs = 0; unsigned jmpCnt = 0; // # of switch cases (excluding default) @@ -1705,75 +1712,48 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() BasicBlock** jmpTab = nullptr; BasicBlock** jmpPtr = nullptr; BasicBlock* currBb = m_optFirstBB; - unsigned sz = 0; bool tailCall = false; // Special args to fgAddRefPred so it will use the initialization fast path FlowEdge* const oldEdge = nullptr; bool const initializingPreds = true; - // Make a jump table for the range of patterns - for (int idxRng = 0; idxRng < m_rangePattern && currBb != nullptr; idxRng++) + // Make a Switch descriptor and jump table for the range of patterns + for (int idxRng = 0; idxRng < m_rangePattern; idxRng++) { - assert(m_numFoundPatterns <= m_rangePattern && m_numFoundPatterns <= m_sizePatterns); - - // First pattern + // Create switch descriptor and its jump table if (idxRng == 0) { - assert(currBb->bbNum == m_firstPatternBBNum && currBb->lastStmt()->GetRootNode()->gtGetOp1()->OperIs(GT_EQ)); - // Allocate the switch descriptor swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; // Allocate the jump table jmpCnt = m_rangePattern; jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt + 1]; - - // Change if basic block to a Switch basic block - // curBBdesc = m_comp->bbNewBasicBlock(jmpKind); - switchBBdesc = currBb; } // Fill in the jump table with the jump code offset - // If the pattern is found, jump to true BB. If not, to false BB. - auto condition1 = currBb->lastStmt()->GetRootNode()->gtGetOp1(); - if ((idxRng + m_minPattern) == (int)condition1->gtGetOp2()->AsIntCon()->IconValue()) + // If the pattern is found, jump to true target BB. If not, to false target BB. + bool reservedPattern = bitmapPatterns & (1 << idxRng); + if (reservedPattern) { - *jmpPtr = (BasicBlock*)(size_t)m_trueJmpBB->bbCodeOffs; - *(jmpPtr) = m_trueJmpBB; - jmpPtr++; - - // Update the code offset range - if (currBb->bbCodeOffs <= m_bbCodeOffs) - m_bbCodeOffs = currBb->bbCodeOffs; - if (currBb->bbCodeOffsEnd >= m_bbCodeOffsEnd) - m_bbCodeOffsEnd = currBb->bbCodeOffsEnd; - - currBb = currBb->bbNext; - } - else if ((idxRng + m_minPattern) < (int)condition1->gtGetOp2()->AsIntCon()->IconValue()) - { - *jmpPtr = (BasicBlock*)(size_t)m_falseJmpBB->bbCodeOffs; - *(jmpPtr) = m_falseJmpBB; - jmpPtr++; + *jmpPtr = (BasicBlock*)(size_t)m_trueJmpBB->bbCodeOffs; + *(jmpPtr++) = m_trueJmpBB; } else { - assert(false && "Unexpected pattern"); + *jmpPtr = (BasicBlock*)(size_t)m_falseJmpBB->bbCodeOffs; + *(jmpPtr++) = m_falseJmpBB; } } - // Append the default label to the target table + // Append the default label to the jump table *jmpPtr = (BasicBlock*)(size_t)m_falseJmpBB->bbCodeOffs; - *(jmpPtr) = m_falseJmpBB; - jmpPtr++; + *(jmpPtr++) = m_falseJmpBB; // Make sure we found the right number of labels noway_assert(jmpPtr == jmpTab + jmpCnt + 1); - // Compute the size of the switch opcode operands - sz = sizeof(DWORD) + jmpCnt * sizeof(DWORD); - // Fill in the remaining fields of the switch descriptor swtDsc->bbsCount = jmpCnt + 1; swtDsc->bbsDstTab = jmpTab; @@ -1799,42 +1779,64 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() tailCall = false; - // Fill in the rest of switch basic block fields + // Change `if` basic block to `Switch` basic block + switchBBdesc = m_optFirstBB; switchBBdesc->bbJumpKind = BBJ_SWITCH; - //switchBBdesc->bbFlags |= bbFlags; + + switchBBdesc->bbCodeOffs = m_bbCodeOffs; + switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; + switchBBdesc->bbJumpSwt = swtDsc; + switchBBdesc->bbFlags |= bbFlags; if (m_comp->compRationalIRForm) { switchBBdesc->bbFlags |= BBF_IS_LIR; } - // switchBBdesc->bbRefs = 0; - switchBBdesc->bbCodeOffs = m_bbCodeOffs; - switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; - switchBBdesc->bbJumpSwt = swtDsc; // Print bbNum of each jmpTab for (unsigned i = 0; i < swtDsc->bbsCount; i++) { - printf("%c\n" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); + printf("%c" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); } + printf("\n"); // Change from GT_JTRUE to GT_SWITCH - switchBBdesc->lastStmt()->GetRootNode()->SetOper(GT_SWITCH, GenTree::PRESERVE_VN); + switchBBdesc->lastStmt()->GetRootNode()->ChangeOper(GT_SWITCH, GenTree::PRESERVE_VN); + switchBBdesc->lastStmt()->GetRootNode()->gtFlags &= ~GTF_ASG; // TODO check the right value to set // Change from GT_EQ or GT_NE to SUB // tree: SUB // op1: LCL_VAR // op2: GT_CNS_INT GenTree* tree = switchBBdesc->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node - tree->SetOper(GT_SUB, GenTree::PRESERVE_VN); + tree->ChangeOper(GT_SUB, GenTree::PRESERVE_VN); + //tree->gtFlags &= ~GTF_ASG; // TODO check the right value to set - // get LCL_VAR node from GT_EQ or GT_NE node in case of COMMA node + // get LCL_VAR node in case of COMMA node if (tree->gtGetOp1()->OperIs(GT_COMMA)) { - GenTree* op1 = tree->gtGetOp1()->gtGetOp1(); // op1 of COMMA node - GenTree* op2 = tree->gtGetOp1()->gtGetOp2(); // LCL_VAR node - tree->AsOp()->gtOp1 = op2; // Set LCL_VAR node to op1 of tree + GenTree* commaNode = tree->gtGetOp1(); // COMMA node to be removed + GenTree* op1 = commaNode->gtGetOp1(); // op1 of COMMA node to be removed + GenTree* op2 = commaNode->gtGetOp2(); // LCL_VAR node + + tree->AsOp()->gtOp1 = op2; // Set LCL_VAR node to op1 of GT_SUB tree + + // TODO: Update the statement m_tree list. It causes assert during fgDebugCheckStmtsList > + // fgDebugCheckNodeLinks() DEBUG_DESTROY_NODE(op1); + DEBUG_DESTROY_NODE(commaNode); // Destroy COMMA node + } + + // Change CNS_INT node value if siwtch tree does not have the mininum pattern + + GenTree* operTree = switchBBdesc->lastStmt()->GetRootNode()->gtGetOp1(); // GT_SUB node + assert(operTree->gtGetOp2() != nullptr); + if (operTree->gtGetOp2()->AsIntCon()->IconValue() != m_minPattern) + { + operTree->AsOp()->gtOp2 = m_minOp; + + // TODO: Update the statement m_tree list. It causes assert during fgDebugCheckStmtsList > + // fgDebugCheckNodeLinks() } return true; @@ -1853,7 +1855,7 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() // PhaseStatus Compiler::optFindSpecificPattern() { - if (!ISMETHOD("Foo")) // TODO clean up code + if (!ISMETHOD("FooNum2")) // TODO clean up code { return PhaseStatus::MODIFIED_NOTHING; } @@ -1863,9 +1865,14 @@ PhaseStatus Compiler::optFindSpecificPattern() bool printed = false; int patternIndex = 0; // The index of the pattern in the array bool foundPattern = false; - BasicBlock* prevBb = nullptr; + BasicBlock* prevBb = fgFirstBB; - for (BasicBlock* currBb = fgFirstBB; currBb != nullptr; currBb = currBb->bbNext) + if (fgFirstBB->bbNext == nullptr) + { + return PhaseStatus::MODIFIED_NOTHING; + } + + for (BasicBlock* currBb = fgFirstBB->bbNext; currBb != nullptr; currBb = currBb->bbNext) { if ((currBb->KindIs(BBJ_COND) || currBb->KindIs(BBJ_RETURN)) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) { @@ -1899,10 +1906,10 @@ PhaseStatus Compiler::optFindSpecificPattern() ->AsLclVarCommon() ->GetLclNum()))) { - printf("\nFound pattern (Curr vs Prev):\n"); - gtDispTree(condition1); - printf("\n"); + printf("\nFound pattern (Prev vs Curr):\n"); gtDispTree(condition2); + printf("\n"); + gtDispTree(condition1); printf("\n\n\n"); // Store the found pattern to the patterns @@ -1926,6 +1933,10 @@ PhaseStatus Compiler::optFindSpecificPattern() // min and max patterns optRngPattern.m_minPattern = optRngPattern.m_patterns[patternIndex]; optRngPattern.m_maxPattern = optRngPattern.m_patterns[patternIndex++]; + + // Update the code offset range + optRngPattern.m_bbCodeOffs = prevBb->bbCodeOffs; + optRngPattern.m_bbCodeOffsEnd = prevBb->bbCodeOffsEnd; } // Current pattern @@ -1942,6 +1953,7 @@ PhaseStatus Compiler::optFindSpecificPattern() if (optRngPattern.m_patterns[patternIndex] < optRngPattern.m_minPattern) { optRngPattern.m_minPattern = optRngPattern.m_patterns[patternIndex]; + optRngPattern.m_minOp = condition1->gtGetOp2(); } else if (optRngPattern.m_patterns[patternIndex] > optRngPattern.m_maxPattern) { @@ -1949,6 +1961,9 @@ PhaseStatus Compiler::optFindSpecificPattern() } patternIndex++; + // Update the code offset range + optRngPattern.m_bbCodeOffsEnd = currBb->bbCodeOffsEnd; + foundPattern = true; } } @@ -1992,11 +2007,11 @@ PhaseStatus Compiler::optFindSpecificPattern() } //auto bitmapPatterns = 0; - //for (int idxPattern = 0; idxPattern < optRngPattern.m_numFoundPatterns; idxPattern++) - //{ - // bitmapPatterns |= (1 << (optRngPattern.m_patterns[idxPattern] - optRngPattern.m_minPattern)); - //} - //printf("Bitmap of pattern values: %d\n", bitmapPatterns); + for (int idxPattern = 0; idxPattern < optRngPattern.m_numFoundPatterns; idxPattern++) + { + optRngPattern.bitmapPatterns |= (1 << (optRngPattern.m_patterns[idxPattern] - optRngPattern.m_minPattern)); + } + printf("Bitmap of pattern values: %d\n", optRngPattern.bitmapPatterns); // Replace BBs with a Switch tree if (!optRngPattern.optMakeSwitchBBdesc()) @@ -2005,6 +2020,7 @@ PhaseStatus Compiler::optFindSpecificPattern() } if (optRngPattern.optUpdateBlocks()) { + fgDispBasicBlocks(true); return PhaseStatus::MODIFIED_EVERYTHING; } } From 6defaf2fd90ffbf56a31af786ebe32b0a2450938 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Mon, 19 Jun 2023 19:13:18 -0700 Subject: [PATCH 03/35] [if range check opt] Handle appending a statement for side effects. --- src/coreclr/jit/optimizebools.cpp | 79 +++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 6fec7a96c97392..56cdc4f5c40adf 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1458,12 +1458,6 @@ GenTree* OptBoolsDsc::optIsBoolComp(OptTestInfo* pOptTest) // PhaseStatus Compiler::optOptimizeBools() { - - if (!ISMETHOD("Test")) - { - return PhaseStatus::MODIFIED_NOTHING; - } - #ifdef DEBUG if (verbose) { @@ -1694,6 +1688,8 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() printf("\nStart optMakeSwitchBlock()\n"); } + assert(m_optFirstBB->bbJumpKind == BBJ_COND); + if (m_rangePattern == 0 || m_numFoundPatterns > m_rangePattern || m_rangePattern > m_sizePatterns) { return false; @@ -1781,7 +1777,10 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() // Change `if` basic block to `Switch` basic block switchBBdesc = m_optFirstBB; + + // Change the BBJ_COND to BBJ_SWITCH switchBBdesc->bbJumpKind = BBJ_SWITCH; + switchBBdesc->bbJumpDest = nullptr; switchBBdesc->bbCodeOffs = m_bbCodeOffs; switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; @@ -1799,17 +1798,53 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() } printf("\n"); + // Transform Statement and GenTree of the switch basic block + + Statement* stmt = switchBBdesc->lastStmt(); + GenTree* rootTree = stmt->GetRootNode(); // JTRUE node + assert(rootTree->OperIs(GT_JTRUE)); + + // Extract side effects if there are any and + // append them before the current statement as a new statement + GenTree* sideEffList = nullptr; + if (rootTree->gtFlags & GTF_SIDE_EFFECT) + { + m_comp->gtExtractSideEffList(rootTree, &sideEffList); + if (sideEffList != nullptr) + { + noway_assert(sideEffList->gtFlags & GTF_SIDE_EFFECT); +#ifdef DEBUG + if (m_comp->verbose) + { + printf("Extracted side effects list...\n"); + m_comp->gtDispTree(sideEffList); + printf("\n"); + } +#endif // DEBUG + + Statement* sideEffStmt = m_comp->fgNewStmtFromTree(sideEffList); + m_comp->fgInsertStmtBefore(switchBBdesc, stmt, sideEffStmt); + m_comp->gtSetStmtInfo(sideEffStmt); + m_comp->fgSetStmtSeq(sideEffStmt); + + // TODO set stmt info in STMT00006 ( ??? ... ??? ) + + m_comp->fgDispBasicBlocks(true); + printf("\n"); + } + } + // Change from GT_JTRUE to GT_SWITCH - switchBBdesc->lastStmt()->GetRootNode()->ChangeOper(GT_SWITCH, GenTree::PRESERVE_VN); - switchBBdesc->lastStmt()->GetRootNode()->gtFlags &= ~GTF_ASG; // TODO check the right value to set + rootTree->ChangeOper(GT_SWITCH, GenTree::PRESERVE_VN); + rootTree->gtFlags &= ~GTF_ASG; // TODO check the right value to set - // Change from GT_EQ or GT_NE to SUB + // Change from GT_EQ or GT_NE to GT_SUB // tree: SUB // op1: LCL_VAR // op2: GT_CNS_INT - GenTree* tree = switchBBdesc->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node + GenTree* tree = rootTree->gtGetOp1(); // GT_EQ or GT_NE node to chnage to GT_SUB tree->ChangeOper(GT_SUB, GenTree::PRESERVE_VN); - //tree->gtFlags &= ~GTF_ASG; // TODO check the right value to set + tree->gtFlags &= ~GTF_ASG; // TODO check the right value to set // get LCL_VAR node in case of COMMA node if (tree->gtGetOp1()->OperIs(GT_COMMA)) @@ -1820,23 +1855,25 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() tree->AsOp()->gtOp1 = op2; // Set LCL_VAR node to op1 of GT_SUB tree - // TODO: Update the statement m_tree list. It causes assert during fgDebugCheckStmtsList > - // fgDebugCheckNodeLinks() + m_comp->gtSetStmtInfo(stmt); + m_comp->fgSetStmtSeq(stmt); - DEBUG_DESTROY_NODE(op1); + //DEBUG_DESTROY_NODE(op1); DEBUG_DESTROY_NODE(commaNode); // Destroy COMMA node } - // Change CNS_INT node value if siwtch tree does not have the mininum pattern + // Change CNS_INT node if siwtch tree does not have the mininum pattern - GenTree* operTree = switchBBdesc->lastStmt()->GetRootNode()->gtGetOp1(); // GT_SUB node - assert(operTree->gtGetOp2() != nullptr); - if (operTree->gtGetOp2()->AsIntCon()->IconValue() != m_minPattern) + assert(tree->gtGetOp2() != nullptr); + if (tree->gtGetOp2()->AsIntCon()->IconValue() != m_minPattern) { - operTree->AsOp()->gtOp2 = m_minOp; + GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT node + tree->AsOp()->gtOp2 = m_minOp; + + m_comp->gtSetStmtInfo(stmt); + m_comp->fgSetStmtSeq(stmt); - // TODO: Update the statement m_tree list. It causes assert during fgDebugCheckStmtsList > - // fgDebugCheckNodeLinks() + DEBUG_DESTROY_NODE(op2); } return true; From b9dd06803d81d3db3dec1a022578ee9e192117bc Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Tue, 20 Jun 2023 17:48:41 -0700 Subject: [PATCH 04/35] [if range check opt] Code clean-up. --- src/coreclr/jit/optimizebools.cpp | 405 ++++++++++++++++++++++-------- 1 file changed, 303 insertions(+), 102 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 56cdc4f5c40adf..96617f5909b663 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1579,45 +1579,170 @@ class OptRangePatternDsc public: OptRangePatternDsc(Compiler* comp) { - m_comp = comp; + m_comp = comp; // Set the Compiler instance } - static const int m_sizePatterns = 64; // The size of the patterns array - int m_patterns[m_sizePatterns] = {}; // Reserved patterns - int m_numFoundPatterns = 0; // The number of patterns found +public: + static const int m_sizePatterns = 64; // The size of the patterns array - int m_minPattern = 0; // The minimum pattern - int m_maxPattern = 0; // The maximum pattern - int m_rangePattern = 0; // The range of values in patterns[] - unsigned int bitmapPatterns = 0; // The bitmap of patterns found - GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern +private: + Compiler* m_comp = nullptr; // The pointer to the Compiler instance + BasicBlock* switchBBdesc = nullptr; // The switch basic block descriptor - unsigned int m_firstPatternBBNum = 0; // BB that the first pattern is found - BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern - BasicBlock* m_trueJmpBB = nullptr; // The BB to jump to in case of true condition - BasicBlock* m_falseJmpBB = nullptr; // The BB to jump to in case of false condition + int m_patterns[m_sizePatterns] = {}; // Reserved patterns + int m_numFoundPatterns = 0; // The number of patterns found + int m_minPattern = 0; // The minimum pattern + int m_maxPattern = 0; // The maximum pattern + int m_rangePattern = 0; // The range of values in patterns[] + unsigned int bitmapPatterns = 0; // The bitmap of patterns found - unsigned m_bbCodeOffs = 0; - unsigned m_bbCodeOffsEnd = 0; - BasicBlock* switchBBdesc = nullptr; + GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern + BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern + BasicBlock* m_trueJmpBB = nullptr; // The basic block to jump to in case of true condition + BasicBlock* m_falseJmpBB = nullptr; // The basic block to jump to in case of false condition -private: - Compiler* m_comp = nullptr; // The pointer to the Compiler instance + unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block + unsigned m_bbCodeOffsEnd = 0; // IL code offset end of the switch basic block public: + int optGetPattern(int idxPattern); + bool optSetPattern(int idxPattern, int patternVal); + int optGetMinPattern(); + void optSetMinPattern(int patternVal); + int optGetMaxPattern(); + void optSetMaxPattern(int patternVal); + void optSetRangePattern(int rangeVal); + void optSetPatternCount(int patternCounts); + void optSetBitmapPatterns(unsigned int bitmapVal); + void optSetMinOp(GenTree* minOpNode); + void optSetFirstPatternBB(BasicBlock* firstBB); + void optSetTrueJmpBB(BasicBlock* trueJmpBB); + void optSetFalseJmpBB(BasicBlock* falseJmpBB); + void optSetBbCodeOffs(IL_OFFSET bbCodeOffs); + void optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd); + bool optMakeSwitchBBdesc(); bool optUpdateBlocks(); }; +// Methods to set and get the values of the OptRangePatternDsc class members + +int OptRangePatternDsc::optGetPattern(int idxPattern) +{ + assert(idxPattern >= 0 && idxPattern < m_sizePatterns); + return m_patterns[idxPattern]; +} + +bool OptRangePatternDsc::optSetPattern(int idxPattern, int patternVal) +{ + if (idxPattern < 0 || idxPattern >= m_sizePatterns) + { + assert(false && "idxPattern out of range"); + return false; + } + if (patternVal < 0 || patternVal > INT_MAX) + { + assert(false && "patternVal out of range"); + return false; + } + m_patterns[idxPattern] = patternVal; + return true; +} + +int OptRangePatternDsc::optGetMinPattern() +{ + return m_minPattern; +} + +void OptRangePatternDsc::optSetMinPattern(int patternVal) +{ + assert(patternVal >= 0 && patternVal <= INT_MAX); + m_minPattern = patternVal; +} + +int OptRangePatternDsc::optGetMaxPattern() +{ + return m_maxPattern; +} + +void OptRangePatternDsc::optSetMaxPattern(int patternVal) +{ + assert(patternVal >= 0 && patternVal <= INT_MAX); + m_maxPattern = patternVal; +} + +void OptRangePatternDsc::optSetRangePattern(int rangeVal) +{ + assert(rangeVal >= 0 && rangeVal <= m_sizePatterns); + m_rangePattern = rangeVal; +} + +void OptRangePatternDsc::optSetPatternCount(int patternCounts) +{ + assert(patternCounts >= 0 && patternCounts <= m_sizePatterns); + m_numFoundPatterns = patternCounts; +} + +void OptRangePatternDsc::optSetBitmapPatterns(unsigned int bitmapVal) +{ + assert(bitmapVal >= 0 && bitmapVal <= UINT_MAX); + bitmapPatterns = bitmapVal; +} + +void OptRangePatternDsc::optSetMinOp(GenTree* minOpNode) +{ + assert(minOpNode != nullptr && minOpNode->OperIs(GT_CNS_INT)); + m_minOp = minOpNode; +} + +void OptRangePatternDsc::optSetFirstPatternBB(BasicBlock* firstBB) +{ + assert(firstBB != nullptr && firstBB->bbJumpKind == BBJ_COND); + m_optFirstBB = firstBB; +} + +void OptRangePatternDsc::optSetTrueJmpBB(BasicBlock* trueJmpBB) +{ + assert(trueJmpBB != nullptr); + m_trueJmpBB = trueJmpBB; +} + +void OptRangePatternDsc::optSetFalseJmpBB(BasicBlock* FalseJmpBB) +{ + assert(FalseJmpBB != nullptr); + m_falseJmpBB = FalseJmpBB; +} + +void OptRangePatternDsc::optSetBbCodeOffs(IL_OFFSET bbCodeOffs) +{ + assert(bbCodeOffs >= 0 && bbCodeOffs <= UINT_MAX); + m_bbCodeOffs = bbCodeOffs; +} + +void OptRangePatternDsc::optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd) +{ + assert(bbCodeOffsEnd >= 0 && bbCodeOffsEnd <= UINT_MAX); + m_bbCodeOffsEnd = bbCodeOffsEnd; +} + //----------------------------------------------------------------------------- -// optUpdateBlocks: Remove pattern blocks and update predecessors of jump target blocks +// optUpdateBlocks: Remove non-first pattern blocks and update predecessors of jump target blocks. +// +// Return Value: +// true if the blocks were successfully updated, false otherwise. +// +// Notes: +// Leave Switch basic block only and remove all other blocks in the range pattern. +// Update reference count of jump target blocks. // bool OptRangePatternDsc::optUpdateBlocks() { - if (ISMETHOD("FooNum2")) // TODO clean up code +#ifdef DEBUG + if (m_comp->verbose) { - printf("optUpdateBlocks()\n"); + printf("\n*************** In optUpdateBlocks()\n"); } +#endif // DEBUG if (m_rangePattern == 0 || m_numFoundPatterns > m_rangePattern || m_rangePattern > m_sizePatterns) { @@ -1678,15 +1803,45 @@ bool OptRangePatternDsc::optUpdateBlocks() } //----------------------------------------------------------------------------- -// optMakeSwitchBBdesc: Create a switch block descriptor for the range pattern optimization +// optMakeSwitchBBdesc: Replace `if` basic block with a `switch` block for the range pattern optimization +// +// Returns: +// true if the switch block is created successfully with the switch jump tables +// +// Notes: +// bbJumpSwt of the Switch basic block is created with a jump table for each range pattern and default case. +// If the value within the range is part of the reserved patterns, the switch jump table is set to the jump target +// for true case. +// Otherwise, the switch jump table is set to the jump target for false case. +// +// If the tree has COMMA node, a new statement is created with the side effect list from the COMMA and inserted +// before the current statement. +// If the CNS_INT node does not have the min pattern, replace it with the CNS_INT node with the min value. +// +// Tree before the optimization: +// ``` +// JTRUE +// \EQ +// \LCL_VAR +// \CNS_INT +// ``` +// +// Tree after the optimization: +// ``` +// SWITCH +// \SUB +// \LCL_VAR +// \CNS_INT (min pattern) +// ``` // bool OptRangePatternDsc::optMakeSwitchBBdesc() { - - if (ISMETHOD("FooNum2")) // TODO clean up code +#ifdef DEBUG + if (m_comp->verbose) { - printf("\nStart optMakeSwitchBlock()\n"); + printf("\n*************** In optMakeSwitchBBdesc()\n"); } +#endif // DEBUG assert(m_optFirstBB->bbJumpKind == BBJ_COND); @@ -1705,9 +1860,9 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() unsigned nxtBBoffs = 0; unsigned jmpCnt = 0; // # of switch cases (excluding default) - BasicBlock** jmpTab = nullptr; - BasicBlock** jmpPtr = nullptr; - BasicBlock* currBb = m_optFirstBB; + BasicBlock** jmpTab = nullptr; + BasicBlock** jmpPtr = nullptr; + BasicBlock* currBb = m_optFirstBB; bool tailCall = false; // Special args to fgAddRefPred so it will use the initialization fast path @@ -1759,15 +1914,6 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() if (m_comp->opts.compProcedureSplitting) { - // TODO-CQ: We might need to create a switch table; we won't know for sure until much later. - // However, switch tables don't work with hot/cold splitting, currently. The switch table data needs - // a relocation such that if the base (the first block after the prolog) and target of the switch - // branch are put in different sections, the difference stored in the table is updated. However, our - // relocation implementation doesn't support three different pointers (relocation address, base, and - // target). So, we need to change our switch table implementation to be more like - // JIT64: put the table in the code section, in the same hot/cold section as the switch jump itself - // (maybe immediately after the switch jump), and make the "base" address be also in that section, - // probably the address after the switch jump. m_comp->opts.compProcedureSplitting = false; JITDUMP("Turning off procedure splitting for this method, as it might need switch tables; " "implementation limitation.\n"); @@ -1791,12 +1937,17 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() switchBBdesc->bbFlags |= BBF_IS_LIR; } - // Print bbNum of each jmpTab - for (unsigned i = 0; i < swtDsc->bbsCount; i++) +#ifdef DEBUG + if (m_comp->verbose) { - printf("%c" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); + // Print bbNum of each jmpTab + for (unsigned i = 0; i < swtDsc->bbsCount; i++) + { + printf("%c" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); + } + printf("\n"); } - printf("\n"); +#endif // DEBUG // Transform Statement and GenTree of the switch basic block @@ -1827,10 +1978,15 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() m_comp->gtSetStmtInfo(sideEffStmt); m_comp->fgSetStmtSeq(sideEffStmt); - // TODO set stmt info in STMT00006 ( ??? ... ??? ) + // TODO do we need to set the IL values in STMT0000X ( ??? ... ??? ) for the new statement? - m_comp->fgDispBasicBlocks(true); - printf("\n"); +#ifdef DEBUG + if (m_comp->verbose) + { + m_comp->fgDispBasicBlocks(true); + printf("\n"); + } +#endif // DEBUG } } @@ -1865,7 +2021,7 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() // Change CNS_INT node if siwtch tree does not have the mininum pattern assert(tree->gtGetOp2() != nullptr); - if (tree->gtGetOp2()->AsIntCon()->IconValue() != m_minPattern) + if (tree->gtGetOp2()->AsIntCon()->IconValue() != optGetMinPattern()) { GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT node tree->AsOp()->gtOp2 = m_minOp; @@ -1892,17 +2048,25 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() // PhaseStatus Compiler::optFindSpecificPattern() { - if (!ISMETHOD("FooNum2")) // TODO clean up code +#ifdef DEBUG + if (this->verbose) + { + printf("\n*************** In optFindSpecificPattern()\n"); + } +#endif // DEBUG + + if (!opts.OptimizationEnabled()) { return PhaseStatus::MODIFIED_NOTHING; } OptRangePatternDsc optRngPattern(this); - bool printed = false; - int patternIndex = 0; // The index of the pattern in the array - bool foundPattern = false; - BasicBlock* prevBb = fgFirstBB; + bool printed = false; + int patternIndex = 0; // The index of the pattern in the array + bool foundPattern = false; + unsigned int firstPatternBBNum = 0; // Basic block number of the first pattern found + BasicBlock* prevBb = fgFirstBB; if (fgFirstBB->bbNext == nullptr) { @@ -1911,14 +2075,20 @@ PhaseStatus Compiler::optFindSpecificPattern() for (BasicBlock* currBb = fgFirstBB->bbNext; currBb != nullptr; currBb = currBb->bbNext) { - if ((currBb->KindIs(BBJ_COND) || currBb->KindIs(BBJ_RETURN)) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) + //if ((currBb->KindIs(BBJ_COND) || currBb->KindIs(BBJ_RETURN)) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) + if (currBb->KindIs(BBJ_COND) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) { - if (!printed) +#ifdef DEBUG + if (this->verbose) { - printf("Display Basic Blocks for optFindSpecificPattern:\n"); - fgDispBasicBlocks(true); - printed = true; + if (!printed) + { + printf("Display Basic Blocks for optFindSpecificPattern:\n"); + fgDispBasicBlocks(true); + printed = true; + } } +#endif // DEBUG if (currBb->lastStmt() == currBb->firstStmt() && prevBb->lastStmt() == prevBb->firstStmt()) { @@ -1943,12 +2113,16 @@ PhaseStatus Compiler::optFindSpecificPattern() ->AsLclVarCommon() ->GetLclNum()))) { - printf("\nFound pattern (Prev vs Curr):\n"); - gtDispTree(condition2); - printf("\n"); - gtDispTree(condition1); - printf("\n\n\n"); - +#ifdef DEBUG + if (this->verbose) + { + printf("\nFound pattern (Prev vs Curr):\n"); + gtDispTree(condition2); + printf("\n"); + gtDispTree(condition1); + printf("\n\n"); + } +#endif // DEBUG // Store the found pattern to the patterns if (patternIndex >= optRngPattern.m_sizePatterns) { @@ -1959,47 +2133,55 @@ PhaseStatus Compiler::optFindSpecificPattern() // Previous pattern if (!foundPattern) // First pattern found { - optRngPattern.m_patterns[patternIndex] = - (int)condition2->gtGetOp2()->AsIntCon()->IconValue(); - optRngPattern.m_firstPatternBBNum = prevBb->bbNum; - optRngPattern.m_optFirstBB = prevBb; + int firstPatternVal = (int)condition2->gtGetOp2()->AsIntCon()->IconValue(); + assert(patternIndex == 0); + optRngPattern.optSetPattern(patternIndex, firstPatternVal); + optRngPattern.optSetFirstPatternBB(prevBb); + firstPatternBBNum = prevBb->bbNum; assert(condition2->OperIs(GT_EQ)); - optRngPattern.m_trueJmpBB = prevBb->bbJumpDest; + optRngPattern.optSetTrueJmpBB(prevBb->bbJumpDest); - // min and max patterns - optRngPattern.m_minPattern = optRngPattern.m_patterns[patternIndex]; - optRngPattern.m_maxPattern = optRngPattern.m_patterns[patternIndex++]; + // min and max patterns. Both are set to the first pattern + optRngPattern.optSetMinPattern(firstPatternVal); + optRngPattern.optSetMaxPattern(firstPatternVal); // Update the code offset range - optRngPattern.m_bbCodeOffs = prevBb->bbCodeOffs; - optRngPattern.m_bbCodeOffsEnd = prevBb->bbCodeOffsEnd; + optRngPattern.optSetBbCodeOffs(prevBb->bbCodeOffs); + optRngPattern.optSetBbCodeOffsEnd(prevBb->bbCodeOffsEnd); + + patternIndex++; } // Current pattern - optRngPattern.m_patterns[patternIndex] = - (int)condition1->gtGetOp2()->AsIntCon()->IconValue(); + int currentPattern = (int)condition1->gtGetOp2()->AsIntCon()->IconValue(); + if (currentPattern <0 || !optRngPattern.optSetPattern(patternIndex, currentPattern)) + { + return PhaseStatus::MODIFIED_NOTHING; + } // False jump if (condition1->OperIs(GT_NE)) { - optRngPattern.m_falseJmpBB = currBb->bbJumpDest; + optRngPattern.optSetFalseJmpBB(currBb->bbJumpDest); } // Update min and max patterns - if (optRngPattern.m_patterns[patternIndex] < optRngPattern.m_minPattern) + if (currentPattern < optRngPattern.optGetMinPattern()) { - optRngPattern.m_minPattern = optRngPattern.m_patterns[patternIndex]; - optRngPattern.m_minOp = condition1->gtGetOp2(); + // Update the min pattern and the minOp to the CNS_INT tree with the min pattern + optRngPattern.optSetMinPattern(currentPattern); + optRngPattern.optSetMinOp(condition1->gtGetOp2()); } - else if (optRngPattern.m_patterns[patternIndex] > optRngPattern.m_maxPattern) + else if (currentPattern > optRngPattern.optGetMaxPattern()) { - optRngPattern.m_maxPattern = optRngPattern.m_patterns[patternIndex]; + // Update the max pattern + optRngPattern.optSetMaxPattern(currentPattern); } patternIndex++; // Update the code offset range - optRngPattern.m_bbCodeOffsEnd = currBb->bbCodeOffsEnd; + optRngPattern.optSetBbCodeOffsEnd(currBb->bbCodeOffsEnd); foundPattern = true; } @@ -2009,8 +2191,8 @@ PhaseStatus Compiler::optFindSpecificPattern() } } - // Stop searching if pattern(s) has been found in previous BBs, but the current BB does not have a pattern - if (foundPattern && patternIndex < (int)(currBb->bbNum - optRngPattern.m_firstPatternBBNum + 1)) + // Stop searching if patterns have been found in previous BBs, but the current BB does not have a pattern + if (foundPattern && patternIndex < (int)(currBb->bbNum - firstPatternBBNum + 1)) { break; } @@ -2020,37 +2202,56 @@ PhaseStatus Compiler::optFindSpecificPattern() if (foundPattern) { - optRngPattern.m_numFoundPatterns = patternIndex; + int patternCount = patternIndex; + optRngPattern.optSetPatternCount(patternCount); - printf("Reserved values:\n"); - for (int idx = 0; idx < optRngPattern.m_numFoundPatterns; idx++) +#ifdef DEBUG + if (this->verbose) { - printf("%d ", optRngPattern.m_patterns[idx]); + printf("Reserved patterns:\n"); + for (int idx = 0; idx < patternCount; idx++) + { + printf("%d ", optRngPattern.optGetPattern(idx)); + } + printf("\n\n"); } - printf("\n\n"); - +#endif // Find range of pattern values - printf("Min pattern value: %d\n", optRngPattern.m_minPattern); - printf("Max pattern value: %d\n", optRngPattern.m_maxPattern); - optRngPattern.m_rangePattern = - optRngPattern.m_maxPattern - optRngPattern.m_minPattern + 1; // min 0 max 2 -> range 3 - printf("Range of pattern values: %d\n", optRngPattern.m_rangePattern); - if (optRngPattern.m_rangePattern > optRngPattern.m_sizePatterns) // TODO: item 0 is 1 and item 1 is 100, rnage - // is 64 but only 2 items + int minPattern = optRngPattern.optGetMinPattern(); + int maxPattern = optRngPattern.optGetMaxPattern(); +#ifdef DEBUG + if (this->verbose) + { + printf("Min pattern value: %d\n", minPattern); + printf("Max pattern value: %d\n", maxPattern); + } +#endif // DEBUG + + int rangePattern = maxPattern - minPattern + 1; +#ifdef DEBUG + if (this->verbose) + { + printf("Range of pattern values: %d\n", rangePattern); + } +#endif // DEBUG + + if (rangePattern > optRngPattern.m_sizePatterns) { - printf("Range of pattern values is too big (> %d): %d\n", optRngPattern.m_sizePatterns, - optRngPattern.m_rangePattern); + printf("Range of pattern values is too big (> %d): %d\n", optRngPattern.m_sizePatterns, rangePattern); return PhaseStatus::MODIFIED_NOTHING; } + optRngPattern.optSetRangePattern(rangePattern); - //auto bitmapPatterns = 0; - for (int idxPattern = 0; idxPattern < optRngPattern.m_numFoundPatterns; idxPattern++) + // Find bitmap of pattern values + unsigned int btPatterns = 0; + for (int idxPattern = 0; idxPattern < patternCount; idxPattern++) { - optRngPattern.bitmapPatterns |= (1 << (optRngPattern.m_patterns[idxPattern] - optRngPattern.m_minPattern)); + btPatterns |= (1 << (optRngPattern.optGetPattern(idxPattern) - minPattern)); } - printf("Bitmap of pattern values: %d\n", optRngPattern.bitmapPatterns); + optRngPattern.optSetBitmapPatterns(btPatterns); + printf("Bitmap of pattern values: %d\n", btPatterns); - // Replace BBs with a Switch tree + // Replace "if" BB with a "Switch" BB if (!optRngPattern.optMakeSwitchBBdesc()) { return PhaseStatus::MODIFIED_NOTHING; From 2740251b799d8a8d086576b307379dfaed83e3d3 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Wed, 21 Jun 2023 10:33:34 -0700 Subject: [PATCH 05/35] [if range check opt] Use gtUpdateStmtSideEffects to update flags after changing statement and trees. --- src/coreclr/jit/optimizebools.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 96617f5909b663..c4d3ed6cb3c863 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1991,16 +1991,14 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() } // Change from GT_JTRUE to GT_SWITCH - rootTree->ChangeOper(GT_SWITCH, GenTree::PRESERVE_VN); - rootTree->gtFlags &= ~GTF_ASG; // TODO check the right value to set + rootTree->ChangeOper(GT_SWITCH); // Change from GT_EQ or GT_NE to GT_SUB // tree: SUB // op1: LCL_VAR // op2: GT_CNS_INT GenTree* tree = rootTree->gtGetOp1(); // GT_EQ or GT_NE node to chnage to GT_SUB - tree->ChangeOper(GT_SUB, GenTree::PRESERVE_VN); - tree->gtFlags &= ~GTF_ASG; // TODO check the right value to set + tree->ChangeOper(GT_SUB); // get LCL_VAR node in case of COMMA node if (tree->gtGetOp1()->OperIs(GT_COMMA)) @@ -2019,7 +2017,6 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() } // Change CNS_INT node if siwtch tree does not have the mininum pattern - assert(tree->gtGetOp2() != nullptr); if (tree->gtGetOp2()->AsIntCon()->IconValue() != optGetMinPattern()) { @@ -2032,6 +2029,8 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() DEBUG_DESTROY_NODE(op2); } + m_comp->gtUpdateStmtSideEffects(stmt); + return true; } @@ -2075,7 +2074,6 @@ PhaseStatus Compiler::optFindSpecificPattern() for (BasicBlock* currBb = fgFirstBB->bbNext; currBb != nullptr; currBb = currBb->bbNext) { - //if ((currBb->KindIs(BBJ_COND) || currBb->KindIs(BBJ_RETURN)) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) if (currBb->KindIs(BBJ_COND) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) { #ifdef DEBUG @@ -2104,8 +2102,8 @@ PhaseStatus Compiler::optFindSpecificPattern() { auto leftCondition1 = condition1->gtGetOp1(); // op1 of condition1 from currBb auto leftCondition2 = condition2->gtGetOp1(); // op1 of condition2 from prevBb - if (leftCondition1->IsLocal() && - ((leftCondition2->IsLocal() && leftCondition1->AsLclVarCommon()->GetLclNum() == + if (leftCondition1->OperIs(GT_LCL_VAR) && + ((leftCondition2->OperIs(GT_LCL_VAR) && leftCondition1->AsLclVarCommon()->GetLclNum() == leftCondition2->AsLclVarCommon()->GetLclNum()) || (leftCondition2->OperIs(GT_COMMA) && leftCondition1->AsLclVarCommon()->GetLclNum() == From f6de9c7b03c4c8231e0c5f0c6d19a619e1c1ca22 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Wed, 21 Jun 2023 17:34:58 -0700 Subject: [PATCH 06/35] [if range check opt] Handle MIN and MAX limits for pattern values. --- src/coreclr/jit/optimizebools.cpp | 158 ++++++++++++++++++++++-------- 1 file changed, 119 insertions(+), 39 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index c4d3ed6cb3c863..456304e9cc5863 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1589,10 +1589,10 @@ class OptRangePatternDsc Compiler* m_comp = nullptr; // The pointer to the Compiler instance BasicBlock* switchBBdesc = nullptr; // The switch basic block descriptor - int m_patterns[m_sizePatterns] = {}; // Reserved patterns + ssize_t m_patterns[m_sizePatterns] = {}; // Reserved patterns int m_numFoundPatterns = 0; // The number of patterns found - int m_minPattern = 0; // The minimum pattern - int m_maxPattern = 0; // The maximum pattern + ssize_t m_minPattern = 0; // The minimum pattern + ssize_t m_maxPattern = 0; // The maximum pattern int m_rangePattern = 0; // The range of values in patterns[] unsigned int bitmapPatterns = 0; // The bitmap of patterns found @@ -1604,13 +1604,15 @@ class OptRangePatternDsc unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block unsigned m_bbCodeOffsEnd = 0; // IL code offset end of the switch basic block + bool optIsInIntegralRange(ssize_t val, GenTree* node); // Checks if the value is in INT32 or INT64 range + public: - int optGetPattern(int idxPattern); - bool optSetPattern(int idxPattern, int patternVal); - int optGetMinPattern(); - void optSetMinPattern(int patternVal); - int optGetMaxPattern(); - void optSetMaxPattern(int patternVal); + ssize_t optGetPattern(int idxPattern); + bool optSetPattern(int idxPattern, ssize_t patternVal, GenTree* node); + ssize_t optGetMinPattern(); + bool optSetMinPattern(ssize_t patternVal, GenTree* node); + ssize_t optGetMaxPattern(); + bool optSetMaxPattern(ssize_t patternVal, GenTree* node); void optSetRangePattern(int rangeVal); void optSetPatternCount(int patternCounts); void optSetBitmapPatterns(unsigned int bitmapVal); @@ -1627,48 +1629,92 @@ class OptRangePatternDsc // Methods to set and get the values of the OptRangePatternDsc class members -int OptRangePatternDsc::optGetPattern(int idxPattern) +//----------------------------------------------------------------------------- +// optIsInINTRange: Checks if the value is within min and max range of INT32 or INT64 +// +// Return Value: +// true if the value is within min and max range of INT32 or INT64 +// false otherwise. +// +// Arguments: +// val - the value to check +// node - the node to check the integral range +// +bool OptRangePatternDsc::optIsInIntegralRange(ssize_t val, GenTree* node) +{ +#ifdef TARGET_64BIT + if (val < _I64_MIN || val > _I64_MAX) + { + printf("val out of range"); + return false; + } +#else // !TARGET_64BIT + IntegralRange integralRange = IntegralRange::ForNode(node, m_comp); + int64_t integralMin = IntegralRange::SymbolicToRealValue(integralRange.GetLowerBound()); + int64_t integralMax = IntegralRange::SymbolicToRealValue(integralRange.GetUpperBound()); + + if (val < integralMin || val > integralMax) + { + printf("val out of range"); + return false; + } +#endif // !TARGET_64BIT + return true; +} + +ssize_t OptRangePatternDsc::optGetPattern(int idxPattern) { assert(idxPattern >= 0 && idxPattern < m_sizePatterns); return m_patterns[idxPattern]; } -bool OptRangePatternDsc::optSetPattern(int idxPattern, int patternVal) +bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, GenTree* node) { if (idxPattern < 0 || idxPattern >= m_sizePatterns) { - assert(false && "idxPattern out of range"); + printf("idxPattern out of range"); return false; } - if (patternVal < 0 || patternVal > INT_MAX) + + if (!optIsInIntegralRange(patternVal, node)) { - assert(false && "patternVal out of range"); return false; } + m_patterns[idxPattern] = patternVal; return true; } -int OptRangePatternDsc::optGetMinPattern() +ssize_t OptRangePatternDsc::optGetMinPattern() { return m_minPattern; } -void OptRangePatternDsc::optSetMinPattern(int patternVal) +bool OptRangePatternDsc::optSetMinPattern(ssize_t patternVal, GenTree* node) { - assert(patternVal >= 0 && patternVal <= INT_MAX); + if (!optIsInIntegralRange(patternVal, node)) + { + return false; + } + m_minPattern = patternVal; + return true; } -int OptRangePatternDsc::optGetMaxPattern() +ssize_t OptRangePatternDsc::optGetMaxPattern() { return m_maxPattern; } -void OptRangePatternDsc::optSetMaxPattern(int patternVal) +bool OptRangePatternDsc::optSetMaxPattern(ssize_t patternVal, GenTree* node) { - assert(patternVal >= 0 && patternVal <= INT_MAX); + if (!optIsInIntegralRange(patternVal, node)) + { + return false; + } + m_maxPattern = patternVal; + return true; } void OptRangePatternDsc::optSetRangePattern(int rangeVal) @@ -1691,7 +1737,7 @@ void OptRangePatternDsc::optSetBitmapPatterns(unsigned int bitmapVal) void OptRangePatternDsc::optSetMinOp(GenTree* minOpNode) { - assert(minOpNode != nullptr && minOpNode->OperIs(GT_CNS_INT)); + assert(minOpNode != nullptr && minOpNode->IsIntegralConst()); m_minOp = minOpNode; } @@ -1996,7 +2042,7 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() // Change from GT_EQ or GT_NE to GT_SUB // tree: SUB // op1: LCL_VAR - // op2: GT_CNS_INT + // op2: GT_CNS_INT or GT_CNS_LNG GenTree* tree = rootTree->gtGetOp1(); // GT_EQ or GT_NE node to chnage to GT_SUB tree->ChangeOper(GT_SUB); @@ -2016,11 +2062,11 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() DEBUG_DESTROY_NODE(commaNode); // Destroy COMMA node } - // Change CNS_INT node if siwtch tree does not have the mininum pattern + // Change constant node if siwtch tree does not have the mininum pattern assert(tree->gtGetOp2() != nullptr); if (tree->gtGetOp2()->AsIntCon()->IconValue() != optGetMinPattern()) { - GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT node + GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT or GT_CNS_LNG node tree->AsOp()->gtOp2 = m_minOp; m_comp->gtSetStmtInfo(stmt); @@ -2059,6 +2105,16 @@ PhaseStatus Compiler::optFindSpecificPattern() return PhaseStatus::MODIFIED_NOTHING; } +//#ifdef DEBUG // TODO cleanup code +// if (this->verbose) +// { +// if (!ISMETHOD("FooNum2")) +// { +// return PhaseStatus::MODIFIED_NOTHING; +// } +// } +//#endif // DEBUG + OptRangePatternDsc optRngPattern(this); bool printed = false; @@ -2100,6 +2156,7 @@ PhaseStatus Compiler::optFindSpecificPattern() // Check both conditions to have constant on the right side if (condition1->gtGetOp2()->IsIntegralConst() && condition2->gtGetOp2()->IsIntegralConst()) { + // Check both conditions to have the same local variable number auto leftCondition1 = condition1->gtGetOp1(); // op1 of condition1 from currBb auto leftCondition2 = condition2->gtGetOp1(); // op1 of condition2 from prevBb if (leftCondition1->OperIs(GT_LCL_VAR) && @@ -2128,12 +2185,17 @@ PhaseStatus Compiler::optFindSpecificPattern() return PhaseStatus::MODIFIED_NOTHING; } - // Previous pattern - if (!foundPattern) // First pattern found + // First pattern found + if (!foundPattern) { - int firstPatternVal = (int)condition2->gtGetOp2()->AsIntCon()->IconValue(); + ssize_t firstPatternVal = condition2->gtGetOp2()->AsIntCon()->IconValue(); assert(patternIndex == 0); - optRngPattern.optSetPattern(patternIndex, firstPatternVal); + if (!optRngPattern.optSetPattern(patternIndex, firstPatternVal, + condition2->gtGetOp2())) + { + return PhaseStatus::MODIFIED_NOTHING; + } + optRngPattern.optSetFirstPatternBB(prevBb); firstPatternBBNum = prevBb->bbNum; @@ -2141,8 +2203,8 @@ PhaseStatus Compiler::optFindSpecificPattern() optRngPattern.optSetTrueJmpBB(prevBb->bbJumpDest); // min and max patterns. Both are set to the first pattern - optRngPattern.optSetMinPattern(firstPatternVal); - optRngPattern.optSetMaxPattern(firstPatternVal); + optRngPattern.optSetMinPattern(firstPatternVal, condition2->gtGetOp2()); + optRngPattern.optSetMaxPattern(firstPatternVal, condition2->gtGetOp2()); // Update the code offset range optRngPattern.optSetBbCodeOffs(prevBb->bbCodeOffs); @@ -2152,29 +2214,38 @@ PhaseStatus Compiler::optFindSpecificPattern() } // Current pattern - int currentPattern = (int)condition1->gtGetOp2()->AsIntCon()->IconValue(); - if (currentPattern <0 || !optRngPattern.optSetPattern(patternIndex, currentPattern)) + ssize_t currentPattern = condition1->gtGetOp2()->AsIntCon()->IconValue(); + if (!optRngPattern.optSetPattern(patternIndex, currentPattern, condition1->gtGetOp2())) { return PhaseStatus::MODIFIED_NOTHING; } // False jump - if (condition1->OperIs(GT_NE)) + if (condition1->OperIs(GT_EQ)) + { + optRngPattern.optSetFalseJmpBB(currBb->bbNext); + } + else if (condition1->OperIs(GT_NE)) { optRngPattern.optSetFalseJmpBB(currBb->bbJumpDest); } + else + { + printf("Unexpected condition1->gtOper\n"); + return PhaseStatus::MODIFIED_NOTHING; + } // Update min and max patterns if (currentPattern < optRngPattern.optGetMinPattern()) { - // Update the min pattern and the minOp to the CNS_INT tree with the min pattern - optRngPattern.optSetMinPattern(currentPattern); + // Update the min pattern and the minOp to the CNS_INT or CNS_LNG tree with the min pattern + optRngPattern.optSetMinPattern(currentPattern, condition1->gtGetOp2()); optRngPattern.optSetMinOp(condition1->gtGetOp2()); } else if (currentPattern > optRngPattern.optGetMaxPattern()) { // Update the max pattern - optRngPattern.optSetMaxPattern(currentPattern); + optRngPattern.optSetMaxPattern(currentPattern, condition1->gtGetOp2()); } patternIndex++; @@ -2209,23 +2280,32 @@ PhaseStatus Compiler::optFindSpecificPattern() printf("Reserved patterns:\n"); for (int idx = 0; idx < patternCount; idx++) { +#ifdef TARGET_64BIT + printf("%lld ", optRngPattern.optGetPattern(idx)); +#else // !TARGET_64BIT printf("%d ", optRngPattern.optGetPattern(idx)); +#endif // !TARGET_64BIT } printf("\n\n"); } #endif // Find range of pattern values - int minPattern = optRngPattern.optGetMinPattern(); - int maxPattern = optRngPattern.optGetMaxPattern(); + ssize_t minPattern = optRngPattern.optGetMinPattern(); + ssize_t maxPattern = optRngPattern.optGetMaxPattern(); #ifdef DEBUG if (this->verbose) { +#ifdef TARGET_64BIT + printf("Min pattern value: %lld\n", minPattern); + printf("Max pattern value: %lld\n", maxPattern); +#else // !TARGET_64BIT printf("Min pattern value: %d\n", minPattern); printf("Max pattern value: %d\n", maxPattern); +#endif // !TARGET_64BIT } #endif // DEBUG - int rangePattern = maxPattern - minPattern + 1; + int rangePattern = (int)(maxPattern - minPattern + 1); #ifdef DEBUG if (this->verbose) { From 63a2d5a567d024f82430b9a8a5506bda3c515005 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Wed, 28 Jun 2023 20:15:11 -0700 Subject: [PATCH 07/35] [if range check opt] Skip optimization for a tree with side effects. --- src/coreclr/jit/optimizebools.cpp | 1025 ++++++++++++++++------------- 1 file changed, 581 insertions(+), 444 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 456304e9cc5863..63c5d71e7dcc72 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1576,61 +1576,82 @@ PhaseStatus Compiler::optOptimizeBools() // class OptRangePatternDsc { -public: - OptRangePatternDsc(Compiler* comp) - { - m_comp = comp; // Set the Compiler instance - } - public: static const int m_sizePatterns = 64; // The size of the patterns array private: - Compiler* m_comp = nullptr; // The pointer to the Compiler instance + Compiler* m_comp = nullptr; // The pointer to the Compiler instance + CompAllocator m_allocator; // The memory allocator + typedef JitHashTable, BasicBlock*> PatternToBlockMap; + PatternToBlockMap m_patternToBlockMap; + BasicBlock* switchBBdesc = nullptr; // The switch basic block descriptor + BBswtDesc* swtDsc = nullptr; // The switch descriptor - ssize_t m_patterns[m_sizePatterns] = {}; // Reserved patterns - int m_numFoundPatterns = 0; // The number of patterns found - ssize_t m_minPattern = 0; // The minimum pattern - ssize_t m_maxPattern = 0; // The maximum pattern - int m_rangePattern = 0; // The range of values in patterns[] - unsigned int bitmapPatterns = 0; // The bitmap of patterns found + ssize_t m_minPattern = 0; // The minimum pattern + ssize_t m_maxPattern = 0; // The maximum pattern + int m_rangePattern = 0; // The range of values in patterns[] - GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern - BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern - BasicBlock* m_trueJmpBB = nullptr; // The basic block to jump to in case of true condition - BasicBlock* m_falseJmpBB = nullptr; // The basic block to jump to in case of false condition + GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern + BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern + BasicBlock* m_defaultJmpBB = nullptr; // The Switch default jump target unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block unsigned m_bbCodeOffsEnd = 0; // IL code offset end of the switch basic block - bool optIsInIntegralRange(ssize_t val, GenTree* node); // Checks if the value is in INT32 or INT64 range + bool optIsInIntegralRange(ssize_t val, GenTree* node); // Checks if the value is in INT32 or INT64 range + BasicBlock* optGetDefaultJmpBB(); + void optSetDefaultJmpBB(BasicBlock* defaultJmpBB); + void optSetMinOp(GenTree* minOpNode); + void optSetMinPattern(ssize_t patternVal, GenTree* node); + void optSetMaxPattern(ssize_t patternVal); + void optSetBbCodeOffs(IL_OFFSET bbCodeOffs); + void optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd); + bool optBlockIsPred(BasicBlock* block, BasicBlock* blockPred); + BasicBlock* optGetJumpTargetBB(BasicBlock* block); + bool optMakeSwitchDesc(); public: - ssize_t optGetPattern(int idxPattern); - bool optSetPattern(int idxPattern, ssize_t patternVal, GenTree* node); + OptRangePatternDsc(Compiler* comp) + : m_comp(comp), m_allocator(comp->getAllocator(CMK_Generic)), m_patternToBlockMap(m_allocator) + { + } + + bool optInitializeRngPattern(BasicBlock* firstBB, ssize_t patternVal); + bool optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block); + int optGetPatternCount(); + void optPrintPatterns(); + bool optJumpsToPatternBlock(); ssize_t optGetMinPattern(); - bool optSetMinPattern(ssize_t patternVal, GenTree* node); ssize_t optGetMaxPattern(); - bool optSetMaxPattern(ssize_t patternVal, GenTree* node); - void optSetRangePattern(int rangeVal); - void optSetPatternCount(int patternCounts); - void optSetBitmapPatterns(unsigned int bitmapVal); - void optSetMinOp(GenTree* minOpNode); - void optSetFirstPatternBB(BasicBlock* firstBB); - void optSetTrueJmpBB(BasicBlock* trueJmpBB); - void optSetFalseJmpBB(BasicBlock* falseJmpBB); - void optSetBbCodeOffs(IL_OFFSET bbCodeOffs); - void optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd); - - bool optMakeSwitchBBdesc(); - bool optUpdateBlocks(); + void optSetRangePattern(int rangeVal); + bool optChangeToSwitch(); }; -// Methods to set and get the values of the OptRangePatternDsc class members +//----------------------------------------------------------------------------- +// optBlockIsPred: Check if blockPred is the predecessor of block +// +// Return Value: +// True if blockPred is the predecessor of block. False otherwise. +// +// Arguments: +// block - the block to check. +// blockPred - the block to check if it is the predecessor of block. +// +bool OptRangePatternDsc::optBlockIsPred(BasicBlock* block, BasicBlock* blockPred) +{ + // Check if blockPred is the predecessor of block + assert(block != nullptr && blockPred != nullptr); + FlowEdge** ptrToPred; + FlowEdge* predBb = m_comp->fgGetPredForBlock(block, blockPred, &ptrToPred); + if (predBb != nullptr) + return true; + else + return false; +} //----------------------------------------------------------------------------- -// optIsInINTRange: Checks if the value is within min and max range of INT32 or INT64 +// optIsInIntegralRange: Checks if the value is within min and max range of INT32 or INT64 // // Return Value: // true if the value is within min and max range of INT32 or INT64 @@ -1662,109 +1683,129 @@ bool OptRangePatternDsc::optIsInIntegralRange(ssize_t val, GenTree* node) return true; } -ssize_t OptRangePatternDsc::optGetPattern(int idxPattern) -{ - assert(idxPattern >= 0 && idxPattern < m_sizePatterns); - return m_patterns[idxPattern]; -} - -bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, GenTree* node) +//----------------------------------------------------------------------------- +// optSetMinPattern: Sets the min pattern value +// +// Arguments: +// patternVal - the pattern value to set +// node - the node that contains the patternVal +// +void OptRangePatternDsc::optSetMinPattern(ssize_t patternVal, GenTree* node) { - if (idxPattern < 0 || idxPattern >= m_sizePatterns) + if (patternVal < m_minPattern) { - printf("idxPattern out of range"); - return false; - } + m_minPattern = patternVal; - if (!optIsInIntegralRange(patternVal, node)) - { - return false; + // Update minOp to the tree with the min pattern + optSetMinOp(node); } - - m_patterns[idxPattern] = patternVal; - return true; -} - -ssize_t OptRangePatternDsc::optGetMinPattern() -{ - return m_minPattern; } -bool OptRangePatternDsc::optSetMinPattern(ssize_t patternVal, GenTree* node) +void OptRangePatternDsc::optSetMaxPattern(ssize_t patternVal) { - if (!optIsInIntegralRange(patternVal, node)) + if (patternVal > m_maxPattern) { - return false; + m_maxPattern = patternVal; } - - m_minPattern = patternVal; - return true; } -ssize_t OptRangePatternDsc::optGetMaxPattern() +void OptRangePatternDsc::optSetMinOp(GenTree* minOpNode) { - return m_maxPattern; + assert(minOpNode != nullptr && minOpNode->IsIntegralConst()); + m_minOp = minOpNode; } -bool OptRangePatternDsc::optSetMaxPattern(ssize_t patternVal, GenTree* node) +//----------------------------------------------------------------------------- +// optGetJumpTargetBB: Get jumpTargetBB for the pattern +// +// Arguments: +// block - the basic block to get its jump target. +// +// Return Value: +// The jump target BB for the pattern +// +// Notes: +// If compare operator is GT_EQ, the switch jump target (true case) is bbJumpDest. +// If compare operator is GT_NE,it is bbNext. +// +BasicBlock* OptRangePatternDsc::optGetJumpTargetBB(BasicBlock* block) { - if (!optIsInIntegralRange(patternVal, node)) - { - return false; - } + assert(block != nullptr && (block->bbJumpKind == BBJ_SWITCH || block->bbJumpKind == BBJ_COND)); + assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); - m_maxPattern = patternVal; - return true; -} - -void OptRangePatternDsc::optSetRangePattern(int rangeVal) -{ - assert(rangeVal >= 0 && rangeVal <= m_sizePatterns); - m_rangePattern = rangeVal; -} + GenTree* op1 = block->lastStmt()->GetRootNode()->gtGetOp1(); + auto oper = op1->OperGet(); + assert(oper == GT_EQ || oper == GT_NE); -void OptRangePatternDsc::optSetPatternCount(int patternCounts) -{ - assert(patternCounts >= 0 && patternCounts <= m_sizePatterns); - m_numFoundPatterns = patternCounts; -} + if (oper == GT_EQ) + { + return block->bbJumpDest; + } + else if (oper == GT_NE) + { + return block->bbNext; + } -void OptRangePatternDsc::optSetBitmapPatterns(unsigned int bitmapVal) -{ - assert(bitmapVal >= 0 && bitmapVal <= UINT_MAX); - bitmapPatterns = bitmapVal; + return nullptr; } -void OptRangePatternDsc::optSetMinOp(GenTree* minOpNode) +//----------------------------------------------------------------------------- +// optSetDefaultJmpBB: Get Switch Default jump target +// +// Arguments: +// defaultJmpBB - the basic block to set its jump target. +// +// Return Value: +// true if the m_defaultJmpBB is set, false otherwise. +// +// Notes: +// For GT_EQ, Swithc Default jump target (for false case) is set to bbNext. +// For GT_NE, Swithc Default jump target (for false case) is set to bbJumpDest. +// +void OptRangePatternDsc::optSetDefaultJmpBB(BasicBlock* block) { - assert(minOpNode != nullptr && minOpNode->IsIntegralConst()); - m_minOp = minOpNode; -} + assert(block != nullptr && block->bbJumpKind == BBJ_SWITCH || block->bbJumpKind == BBJ_COND); + assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); -void OptRangePatternDsc::optSetFirstPatternBB(BasicBlock* firstBB) -{ - assert(firstBB != nullptr && firstBB->bbJumpKind == BBJ_COND); - m_optFirstBB = firstBB; -} + auto oper = block->lastStmt()->GetRootNode()->gtGetOp1()->OperGet(); + assert(oper == GT_EQ || oper == GT_NE); -void OptRangePatternDsc::optSetTrueJmpBB(BasicBlock* trueJmpBB) -{ - assert(trueJmpBB != nullptr); - m_trueJmpBB = trueJmpBB; + if (oper == GT_EQ) + { + m_defaultJmpBB = block->bbNext; + } + else if (oper == GT_NE) + { + m_defaultJmpBB = block->bbJumpDest; + } } -void OptRangePatternDsc::optSetFalseJmpBB(BasicBlock* FalseJmpBB) +//----------------------------------------------------------------------------- +// optGetDefaultJmpBB: Get Switch Default jump target +// +// Return Value: +// The default jump target BB of SWITCH block +// +BasicBlock* OptRangePatternDsc::optGetDefaultJmpBB() { - assert(FalseJmpBB != nullptr); - m_falseJmpBB = FalseJmpBB; + return m_defaultJmpBB; } +//----------------------------------------------------------------------------- +// optSetBbCodeOffs: Set the code offset of the basic block +// void OptRangePatternDsc::optSetBbCodeOffs(IL_OFFSET bbCodeOffs) { assert(bbCodeOffs >= 0 && bbCodeOffs <= UINT_MAX); - m_bbCodeOffs = bbCodeOffs; + if (optGetPatternCount() == 0) + { + m_bbCodeOffs = bbCodeOffs; + } } +//----------------------------------------------------------------------------- +// optSetBbCodeOffsEnd: Set the code offset of the basic block end +// void OptRangePatternDsc::optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd) { assert(bbCodeOffsEnd >= 0 && bbCodeOffsEnd <= UINT_MAX); @@ -1772,189 +1813,258 @@ void OptRangePatternDsc::optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd) } //----------------------------------------------------------------------------- -// optUpdateBlocks: Remove non-first pattern blocks and update predecessors of jump target blocks. +// optInitializeRngPattern: Initializes the range pattern descriptor +// +// Arguments: +// firstBB - The first basic block of the range pattern +// firstVal - The first value of the range pattern // // Return Value: -// true if the blocks were successfully updated, false otherwise. +// True if the range pattern is initialized successfully +// False otherwise. // -// Notes: -// Leave Switch basic block only and remove all other blocks in the range pattern. -// Update reference count of jump target blocks. +bool OptRangePatternDsc::optInitializeRngPattern(BasicBlock* firstBB, ssize_t firstVal) +{ + assert(firstBB != nullptr && firstBB->bbJumpKind == BBJ_COND && + firstBB->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); + + // Check if the value is within min and max range of INT32 or INT64 + GenTree* rootTree = firstBB->lastStmt()->GetRootNode(); + if (rootTree->gtGetOp1()->gtGetOp2() != nullptr && + !optIsInIntegralRange(firstVal, rootTree->gtGetOp1()->gtGetOp2())) + { + return false; + } + + m_optFirstBB = firstBB; + + // Initialize min pattern and max pattern to the first pattern value + m_minPattern = firstVal; + m_maxPattern = firstVal; + + // Initialize the code offset range from the first block + optSetBbCodeOffs(firstBB->bbCodeOffs); + + return true; +} + +//----------------------------------------------------------------------------- +// optSetPattern: Save pattern value and basic block // -bool OptRangePatternDsc::optUpdateBlocks() +// Arguments: +// idxPattern - the index of the pattern to set +// patternVal - the value of the pattern +// block - the basic block to set its jump target. +// +// Return Value: +// true if the pattern is set, false otherwise. +// +bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block) { + if (idxPattern < 0 || idxPattern >= m_sizePatterns) + { #ifdef DEBUG - if (m_comp->verbose) + if (m_comp->verbose) + { + printf("idxPattern out of range"); + } +#endif // DEBUG + + return false; + } + + // Check if the value is within min and max range of INT32 or INT64 + if (!optIsInIntegralRange(patternVal, block->lastStmt()->GetRootNode()->gtGetOp1()->gtGetOp2())) { - printf("\n*************** In optUpdateBlocks()\n"); + return false; } -#endif // DEBUG - if (m_rangePattern == 0 || m_numFoundPatterns > m_rangePattern || m_rangePattern > m_sizePatterns) + // Set the pattern value if it does not already exists in the map + BasicBlock* mappedBlock = nullptr; + if (m_patternToBlockMap.Lookup(patternVal, &mappedBlock)) // Pattern already exists { return false; } - // Special args to fgAddRefPred - FlowEdge* const oldEdge = nullptr; - BasicBlock* currBb = m_optFirstBB; + m_patternToBlockMap.Set(patternVal, block); + + // Set min and max pattern + optSetMinPattern(patternVal, block->lastStmt()->GetRootNode()->gtGetOp1()->gtGetOp2()); + optSetMaxPattern(patternVal); + + // Set default jump target of the basic block + optSetDefaultJmpBB(block); + + // Update the code offset end range + optSetBbCodeOffsEnd(block->bbCodeOffsEnd); + + return true; +} + +//----------------------------------------------------------------------------- +// optGetPatternCount: Get the number of patterns +// +// Return Value: +// The number of reserved patterns +// +int OptRangePatternDsc::optGetPatternCount() +{ + return m_patternToBlockMap.GetCount(); +} - // Update reference count of jump target blocks and remove pattern blocks - for (int idxPattern = 0; idxPattern < m_numFoundPatterns && currBb != nullptr; idxPattern++) +//----------------------------------------------------------------------------- +// optPrintPatterns: Prints the patterns from m_patternToBlockMap +// +void OptRangePatternDsc::optPrintPatterns() +{ + // print patterns from m_patternToBlockMap using key iterator + PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; + for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) { - // Skip to add reference count for the Swithc block because it already has a link to jump block - if (idxPattern == 0) + BasicBlock* mappedBlock; + if (patternToBlockMap->Lookup(patternVal, &mappedBlock)) { - currBb = currBb->bbNext; - continue; + printf("patternVal = %d, block = %d\n", patternVal, mappedBlock->bbNum); } + } +} - // Reference count updates of the jump target blocks - auto operTree = currBb->lastStmt()->GetRootNode()->gtGetOp1(); - if (operTree->OperGet() == GT_EQ) - { - m_comp->fgAddRefPred(m_trueJmpBB, switchBBdesc, oldEdge); - } - else if (operTree->OperGet() == GT_NE) - { - m_comp->fgAddRefPred(m_falseJmpBB, switchBBdesc, oldEdge); - } - else +//----------------------------------------------------------------------------- +// optJumpsToPatternBlock: Checks if any pattern block jumps to one of the blocks +// within m_patternToBlockMap +// +// Arguments: +// None +// +// Return Value: +// True if any of m_patternToBlockMap block jumps to one of the blocks within m_patternToBlockMap +// False otherwise. +// +bool OptRangePatternDsc::optJumpsToPatternBlock() +{ + PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; + const unsigned int maxPatternBbNum = m_optFirstBB->bbNum + optGetPatternCount() - 1; + + for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) + { + BasicBlock* sourceBlock; + if (patternToBlockMap->Lookup(patternVal, &sourceBlock)) { - return false; - } + BasicBlock* destBlock = sourceBlock->bbJumpDest; + assert(destBlock != nullptr); - // Remove the current block - BasicBlock* prevBlock = currBb->bbPrev; - BasicBlock* nextBlock = currBb->bbNext; + if (destBlock->bbNum >= m_optFirstBB->bbNum && destBlock->bbNum <= maxPatternBbNum) + { + return true; + } + } + } - // Unlink to the previous block and link the next block to the previous block - m_comp->fgRemoveRefPred(currBb, prevBlock); - m_comp->fgAddRefPred(nextBlock, prevBlock, oldEdge); - - m_comp->fgRemoveBlock(currBb, /* unreachable */ true); + return false; +} - prevBlock->bbNext = nextBlock; - currBb = nextBlock; - } +//----------------------------------------------------------------------------- +// optGetMinPattern: Gets the min pattern value +// +// Return Value: +// The min pattern value +// +ssize_t OptRangePatternDsc::optGetMinPattern() +{ + return m_minPattern; +} - // Update the reference count of false jump block - int numNotFound = m_rangePattern - m_numFoundPatterns; - for (int idxFalse = 0; idxFalse < numNotFound; idxFalse++) - { - m_comp->fgAddRefPred(m_falseJmpBB, switchBBdesc, oldEdge); - } +ssize_t OptRangePatternDsc::optGetMaxPattern() +{ + return m_maxPattern; +} - return true; +void OptRangePatternDsc::optSetRangePattern(int rangeVal) +{ + assert(rangeVal >= 0 && rangeVal <= m_sizePatterns); + m_rangePattern = rangeVal; } //----------------------------------------------------------------------------- -// optMakeSwitchBBdesc: Replace `if` basic block with a `switch` block for the range pattern optimization +// optMakeSwitchDesc: Make a Switch descriptor with a switch jump table // // Returns: -// true if the switch block is created successfully with the switch jump tables +// true if the switch descriptor is created successfully with the switch jump tables. False otherwise. // // Notes: -// bbJumpSwt of the Switch basic block is created with a jump table for each range pattern and default case. -// If the value within the range is part of the reserved patterns, the switch jump table is set to the jump target -// for true case. -// Otherwise, the switch jump table is set to the jump target for false case. -// -// If the tree has COMMA node, a new statement is created with the side effect list from the COMMA and inserted -// before the current statement. -// If the CNS_INT node does not have the min pattern, replace it with the CNS_INT node with the min value. -// -// Tree before the optimization: -// ``` -// JTRUE -// \EQ -// \LCL_VAR -// \CNS_INT -// ``` -// -// Tree after the optimization: -// ``` -// SWITCH -// \SUB -// \LCL_VAR -// \CNS_INT (min pattern) -// ``` -// -bool OptRangePatternDsc::optMakeSwitchBBdesc() +// It creates switch descriptor with a jump table for each range pattern and default case. +// If the value is part of the reserved patterns, the switch jump table is set to the jump target +// for true case +// If GT_EQ, jump target is block's bbJumpDest. If GT_NE, jump target is block's bbNext. +// Otherwise, the switch jump table is set to the default jump target. +// No tree is updated in this method. +// +bool OptRangePatternDsc::optMakeSwitchDesc() { #ifdef DEBUG if (m_comp->verbose) { - printf("\n*************** In optMakeSwitchBBdesc()\n"); + printf("\n*************** In optMakeSwitchDesc()\n"); } #endif // DEBUG - assert(m_optFirstBB->bbJumpKind == BBJ_COND); - - if (m_rangePattern == 0 || m_numFoundPatterns > m_rangePattern || m_rangePattern > m_sizePatterns) + if (optGetPatternCount() > m_rangePattern || m_rangePattern < 2 || optGetPatternCount() > m_rangePattern || + m_rangePattern > m_sizePatterns) { return false; } - BasicBlock* prevBb = nullptr; - int patternIndex = 0; - BBswtDesc* swtDsc = nullptr; - BBjumpKinds jmpKind = BBJ_NONE; + BasicBlock* prevBb = nullptr; + int patternIndex = 0; - BasicBlockFlags bbFlags = BBF_EMPTY; + BasicBlockFlags bbFlags = BBF_EMPTY; unsigned curBBoffs = 0; unsigned nxtBBoffs = 0; - unsigned jmpCnt = 0; // # of switch cases (excluding default) + unsigned jmpCnt = 0; // # of switch cases (excluding default) BasicBlock** jmpTab = nullptr; BasicBlock** jmpPtr = nullptr; - BasicBlock* currBb = m_optFirstBB; bool tailCall = false; + ssize_t minVal = optGetMinPattern(); - // Special args to fgAddRefPred so it will use the initialization fast path - FlowEdge* const oldEdge = nullptr; - bool const initializingPreds = true; + // Allocate the jump table + jmpCnt = m_rangePattern; + jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt + 1]; - // Make a Switch descriptor and jump table for the range of patterns + // Make a jump table for the range of patterns + // If reserved pattern, jump to jump target of source block. If not, jump to default target. for (int idxRng = 0; idxRng < m_rangePattern; idxRng++) { - // Create switch descriptor and its jump table - if (idxRng == 0) + // Find a mapped block from a pattern + ssize_t key = minVal + idxRng; + BasicBlock* mappedBlock = nullptr; + if (m_patternToBlockMap.Lookup(key, &mappedBlock)) // A mapped block is found { - // Allocate the switch descriptor - swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; - - // Allocate the jump table - jmpCnt = m_rangePattern; - jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt + 1]; - } - - // Fill in the jump table with the jump code offset - // If the pattern is found, jump to true target BB. If not, to false target BB. - bool reservedPattern = bitmapPatterns & (1 << idxRng); - if (reservedPattern) - { - *jmpPtr = (BasicBlock*)(size_t)m_trueJmpBB->bbCodeOffs; - *(jmpPtr++) = m_trueJmpBB; + BasicBlock* jumpTargetBb = optGetJumpTargetBB(mappedBlock); + *jmpPtr = (BasicBlock*)(size_t)(jumpTargetBb->bbCodeOffs); + *(jmpPtr++) = jumpTargetBb; } else { - *jmpPtr = (BasicBlock*)(size_t)m_falseJmpBB->bbCodeOffs; - *(jmpPtr++) = m_falseJmpBB; + BasicBlock* defaultJmpBb = optGetDefaultJmpBB(); + *jmpPtr = (BasicBlock*)(size_t)(defaultJmpBb->bbCodeOffs); + *(jmpPtr++) = defaultJmpBb; } } // Append the default label to the jump table - *jmpPtr = (BasicBlock*)(size_t)m_falseJmpBB->bbCodeOffs; - *(jmpPtr++) = m_falseJmpBB; + *jmpPtr = (BasicBlock*)(size_t)(optGetDefaultJmpBB()->bbCodeOffs); + *(jmpPtr++) = optGetDefaultJmpBB(); // Make sure we found the right number of labels noway_assert(jmpPtr == jmpTab + jmpCnt + 1); + // Allocate the switch descriptor + swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; + // Fill in the remaining fields of the switch descriptor swtDsc->bbsCount = jmpCnt + 1; swtDsc->bbsDstTab = jmpTab; - jmpKind = BBJ_SWITCH; m_comp->fgHasSwitch = true; @@ -1967,77 +2077,132 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() tailCall = false; - // Change `if` basic block to `Switch` basic block - switchBBdesc = m_optFirstBB; - - // Change the BBJ_COND to BBJ_SWITCH - switchBBdesc->bbJumpKind = BBJ_SWITCH; - switchBBdesc->bbJumpDest = nullptr; - - switchBBdesc->bbCodeOffs = m_bbCodeOffs; - switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; - switchBBdesc->bbJumpSwt = swtDsc; - switchBBdesc->bbFlags |= bbFlags; - if (m_comp->compRationalIRForm) - { - switchBBdesc->bbFlags |= BBF_IS_LIR; - } - #ifdef DEBUG if (m_comp->verbose) { - // Print bbNum of each jmpTab + // Print bbNum of each jmpTab + printf("------Switch " FMT_BB " jumps to: ", m_optFirstBB->bbNum); for (unsigned i = 0; i < swtDsc->bbsCount; i++) { printf("%c" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); } - printf("\n"); + printf("\n\n"); } #endif // DEBUG - // Transform Statement and GenTree of the switch basic block + return true; +} - Statement* stmt = switchBBdesc->lastStmt(); - GenTree* rootTree = stmt->GetRootNode(); // JTRUE node +//----------------------------------------------------------------------------- +// optChangeToSwitch: Change the first pattern block to SWITCH block and remove other pattern blocks +// +// Return Value: +// true if the blocks were successfully updated, false otherwise. +// +// Notes: +// Leave Switch basic block only and remove all other blocks in the range pattern. +// Update reference count of jump target blocks. +// If the CNS_INT node does not have a min pattern, replace it with the CNS_INT node with the min value. +// +// Tree before the optimization: +// ``` +// JTRUE +// \EQ +// \LCL_VAR +// \CNS_INT +// ``` +// +// Tree after the optimization: +// ``` +// SWITCH +// \SUB +// \LCL_VAR +// \CNS_INT (min pattern) +// ``` +bool OptRangePatternDsc::optChangeToSwitch() +{ +#ifdef DEBUG + if (m_comp->verbose) + { + printf("\n*************** In optChangeToSwitch()\n"); + } +#endif // DEBUG + + assert(m_optFirstBB != nullptr && m_optFirstBB->KindIs(BBJ_COND) && m_optFirstBB->bbNext != nullptr); + assert(optGetPatternCount() <= m_rangePattern && m_rangePattern >= 2 && optGetPatternCount() <= m_rangePattern && + m_rangePattern <= m_sizePatterns); + + // Make Switch descriptor with a jump table + if (!optMakeSwitchDesc()) + { + return false; + } + + bool updatedBlocks = false; + switchBBdesc = m_optFirstBB; + int patternCount = optGetPatternCount(); + + // Update the Switch basic block + + // Change `JTRUE` basic block to `Switch` basic block + switchBBdesc = m_optFirstBB; + + // Change BBJ_COND to BBJ_SWITCH + switchBBdesc->bbJumpKind = BBJ_SWITCH; + switchBBdesc->bbJumpDest = nullptr; + + switchBBdesc->bbCodeOffs = m_bbCodeOffs; + switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; + switchBBdesc->bbJumpSwt = swtDsc; + GenTree* rootTree = switchBBdesc->lastStmt()->GetRootNode(); assert(rootTree->OperIs(GT_JTRUE)); + assert(!(rootTree->gtFlags & GTF_SIDE_EFFECT)); // JTRUE node should not have side effects + + // Change from GT_JTRUE to GT_SWITCH + rootTree->ChangeOper(GT_SWITCH); - // Extract side effects if there are any and - // append them before the current statement as a new statement - GenTree* sideEffList = nullptr; - if (rootTree->gtFlags & GTF_SIDE_EFFECT) + // Special args to fgAddRefPred + FlowEdge* const oldEdge = nullptr; + + // Remove non-switch pattern blocks. Update the reference count of jump target block from the removed block. + BasicBlock* currBb = switchBBdesc->bbNext; + for (int idxPattern = 1; idxPattern < patternCount && currBb != nullptr; idxPattern++) { - m_comp->gtExtractSideEffList(rootTree, &sideEffList); - if (sideEffList != nullptr) - { - noway_assert(sideEffList->gtFlags & GTF_SIDE_EFFECT); -#ifdef DEBUG - if (m_comp->verbose) - { - printf("Extracted side effects list...\n"); - m_comp->gtDispTree(sideEffList); - printf("\n"); - } -#endif // DEBUG + BasicBlock* prevBlock = currBb->bbPrev; + BasicBlock* nextBlock = currBb->bbNext; + BasicBlock* jumpBlock = optGetJumpTargetBB(currBb); - Statement* sideEffStmt = m_comp->fgNewStmtFromTree(sideEffList); - m_comp->fgInsertStmtBefore(switchBBdesc, stmt, sideEffStmt); - m_comp->gtSetStmtInfo(sideEffStmt); - m_comp->fgSetStmtSeq(sideEffStmt); + // Unlink the current block and its pred block + assert(optBlockIsPred(currBb, prevBlock)); + m_comp->fgRemoveRefPred(currBb, prevBlock); - // TODO do we need to set the IL values in STMT0000X ( ??? ... ??? ) for the new statement? + // Link Switch block and current block's jump target block + m_comp->fgAddRefPred(jumpBlock, switchBBdesc, oldEdge); -#ifdef DEBUG - if (m_comp->verbose) - { - m_comp->fgDispBasicBlocks(true); - printf("\n"); - } -#endif // DEBUG + // Link Switch block and the next block: + // if GT_EQ and currBb is the last pattern block, skip because bbNext is already linked as default jump + // target if GT_NE, it is already linked when linking its jump target block to Switch block + if (currBb->lastStmt()->GetRootNode()->gtGetOp1()->OperIs(GT_EQ) && idxPattern != (patternCount - 1)) + { + m_comp->fgAddRefPred(nextBlock, switchBBdesc, oldEdge); } + + m_comp->fgRemoveBlock(currBb, /* unreachable */ true); + + updatedBlocks = true; + currBb = nextBlock; } - // Change from GT_JTRUE to GT_SWITCH - rootTree->ChangeOper(GT_SWITCH); + // Update the reference count of the default jump block + int numNotFound = m_rangePattern - patternCount + 1; // +1 for switch default case + for (int idxFalse = 0; idxFalse < numNotFound; idxFalse++) + { + m_comp->fgAddRefPred(optGetDefaultJmpBB(), switchBBdesc, oldEdge); + } + + // Continue to transform Switch node + + Statement* stmt = switchBBdesc->lastStmt(); // Change from GT_EQ or GT_NE to GT_SUB // tree: SUB @@ -2045,28 +2210,12 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() // op2: GT_CNS_INT or GT_CNS_LNG GenTree* tree = rootTree->gtGetOp1(); // GT_EQ or GT_NE node to chnage to GT_SUB tree->ChangeOper(GT_SUB); - - // get LCL_VAR node in case of COMMA node - if (tree->gtGetOp1()->OperIs(GT_COMMA)) - { - GenTree* commaNode = tree->gtGetOp1(); // COMMA node to be removed - GenTree* op1 = commaNode->gtGetOp1(); // op1 of COMMA node to be removed - GenTree* op2 = commaNode->gtGetOp2(); // LCL_VAR node - - tree->AsOp()->gtOp1 = op2; // Set LCL_VAR node to op1 of GT_SUB tree - - m_comp->gtSetStmtInfo(stmt); - m_comp->fgSetStmtSeq(stmt); - - //DEBUG_DESTROY_NODE(op1); - DEBUG_DESTROY_NODE(commaNode); // Destroy COMMA node - } + assert(tree->gtGetOp1() != nullptr && tree->gtGetOp1()->OperIs(GT_LCL_VAR)); // Change constant node if siwtch tree does not have the mininum pattern - assert(tree->gtGetOp2() != nullptr); - if (tree->gtGetOp2()->AsIntCon()->IconValue() != optGetMinPattern()) + if (tree->gtGetOp2() != nullptr && tree->gtGetOp2()->AsIntCon()->IconValue() != optGetMinPattern()) { - GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT or GT_CNS_LNG node + GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT or GT_CNS_LNG node tree->AsOp()->gtOp2 = m_minOp; m_comp->gtSetStmtInfo(stmt); @@ -2077,7 +2226,7 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() m_comp->gtUpdateStmtSideEffects(stmt); - return true; + return updatedBlocks; } //----------------------------------------------------------------------------- @@ -2088,8 +2237,8 @@ bool OptRangePatternDsc::optMakeSwitchBBdesc() // MODIFIED_EVERYTHING otherwise. // // Notes: -// Detect if (a == val1 || a == val2 || a == val3 || a == val4) pattern and change it to switch tree to reduce -// jumps and perform bit operation instead. +// Detect if (a == val1 || a == val2 || a == val3 || ...) pattern and change it to switch tree +// to reduce jumps and perform bit operation instead. // PhaseStatus Compiler::optFindSpecificPattern() { @@ -2100,27 +2249,12 @@ PhaseStatus Compiler::optFindSpecificPattern() } #endif // DEBUG - if (!opts.OptimizationEnabled()) - { - return PhaseStatus::MODIFIED_NOTHING; - } - -//#ifdef DEBUG // TODO cleanup code -// if (this->verbose) -// { -// if (!ISMETHOD("FooNum2")) -// { -// return PhaseStatus::MODIFIED_NOTHING; -// } -// } -//#endif // DEBUG - OptRangePatternDsc optRngPattern(this); bool printed = false; - int patternIndex = 0; // The index of the pattern in the array + int patternIndex = 0; // The index of the pattern in the array bool foundPattern = false; - unsigned int firstPatternBBNum = 0; // Basic block number of the first pattern found + unsigned int firstPatternBBNum = 0; // Basic block number of the first pattern found BasicBlock* prevBb = fgFirstBB; if (fgFirstBB->bbNext == nullptr) @@ -2144,115 +2278,117 @@ PhaseStatus Compiler::optFindSpecificPattern() } #endif // DEBUG + // Check if prevBb is the predecessor of currBb and currBb has only one predecessor + FlowEdge** ptrToPred; + FlowEdge* pred = fgGetPredForBlock(currBb, prevBb, &ptrToPred); + if (pred == nullptr || currBb->bbRefs != 1) + { + if (foundPattern) + break; // Stop searching if patterns are not from consecutive basic blocks + else + continue; + } + + // Basic block must have only one statement if (currBb->lastStmt() == currBb->firstStmt() && prevBb->lastStmt() == prevBb->firstStmt()) { - auto condition1 = currBb->lastStmt()->GetRootNode()->gtGetOp1(); - auto condition2 = prevBb->lastStmt()->GetRootNode()->gtGetOp1(); - if (condition1->OperIsCompare() && condition2->OperIsCompare()) + // Skip if there is any side effect + assert(currBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); // JTRUE node + if (currBb->lastStmt()->GetRootNode()->gtFlags & GTF_SIDE_EFFECT) { - if (currBb->bbJumpDest == prevBb->bbJumpDest || - (condition1->OperIs(GT_NE) && condition2->OperIs(GT_EQ) && currBb->bbNext == prevBb->bbJumpDest)) + // Stop searching if patterns are not from consecutive basic blocks + if (foundPattern) + break; + else + continue; + } + + auto currCmpOp = currBb->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node + auto prevCmpOp = prevBb->lastStmt()->GetRootNode()->gtGetOp1(); + assert(currCmpOp != nullptr && prevCmpOp != nullptr); + + // Compare operator is GT_EQ. If it is GT_NE, it is the end of the pattern check. + if ((prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_EQ)) || + (prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_NE))) + { + if (prevBb->bbJumpDest == currBb) + { + if (foundPattern) + break; + else + continue; + } + + // Check both conditions to have constant on the right side + if (currCmpOp->gtGetOp2()->IsIntegralConst() && prevCmpOp->gtGetOp2()->IsIntegralConst() && + currCmpOp->gtGetOp2()->OperGet() == prevCmpOp->gtGetOp2()->OperGet()) { - // Check both conditions to have constant on the right side - if (condition1->gtGetOp2()->IsIntegralConst() && condition2->gtGetOp2()->IsIntegralConst()) + // Check both conditions to have the same local variable number + if (prevCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && currCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && + prevCmpOp->gtGetOp1()->AsLclVar()->GetLclNum() == + currCmpOp->gtGetOp1()->AsLclVar()->GetLclNum()) { - // Check both conditions to have the same local variable number - auto leftCondition1 = condition1->gtGetOp1(); // op1 of condition1 from currBb - auto leftCondition2 = condition2->gtGetOp1(); // op1 of condition2 from prevBb - if (leftCondition1->OperIs(GT_LCL_VAR) && - ((leftCondition2->OperIs(GT_LCL_VAR) && leftCondition1->AsLclVarCommon()->GetLclNum() == - leftCondition2->AsLclVarCommon()->GetLclNum()) || - (leftCondition2->OperIs(GT_COMMA) && - leftCondition1->AsLclVarCommon()->GetLclNum() == - leftCondition2->gtEffectiveVal(/* commaOnly */ true) - ->AsLclVarCommon() - ->GetLclNum()))) + // Found pattern +#ifdef DEBUG + if (this->verbose) + { + printf("\nFound pattern (Prev vs Curr):\n"); + gtDispTree(prevCmpOp); + printf("\n"); + gtDispTree(currCmpOp); + printf("\n\n"); + } +#endif // DEBUG + // No optimization if the number of patterns is greater than 64. + if (patternIndex >= optRngPattern.m_sizePatterns) { #ifdef DEBUG if (this->verbose) - { - printf("\nFound pattern (Prev vs Curr):\n"); - gtDispTree(condition2); - printf("\n"); - gtDispTree(condition1); - printf("\n\n"); - } -#endif // DEBUG - // Store the found pattern to the patterns - if (patternIndex >= optRngPattern.m_sizePatterns) { printf("Too many patterns found (> 64), no optimization done.\n"); - return PhaseStatus::MODIFIED_NOTHING; } +#endif // DEBUG + return PhaseStatus::MODIFIED_NOTHING; + } - // First pattern found - if (!foundPattern) - { - ssize_t firstPatternVal = condition2->gtGetOp2()->AsIntCon()->IconValue(); - assert(patternIndex == 0); - if (!optRngPattern.optSetPattern(patternIndex, firstPatternVal, - condition2->gtGetOp2())) - { - return PhaseStatus::MODIFIED_NOTHING; - } - - optRngPattern.optSetFirstPatternBB(prevBb); - firstPatternBBNum = prevBb->bbNum; - - assert(condition2->OperIs(GT_EQ)); - optRngPattern.optSetTrueJmpBB(prevBb->bbJumpDest); - - // min and max patterns. Both are set to the first pattern - optRngPattern.optSetMinPattern(firstPatternVal, condition2->gtGetOp2()); - optRngPattern.optSetMaxPattern(firstPatternVal, condition2->gtGetOp2()); - - // Update the code offset range - optRngPattern.optSetBbCodeOffs(prevBb->bbCodeOffs); - optRngPattern.optSetBbCodeOffsEnd(prevBb->bbCodeOffsEnd); - - patternIndex++; - } + // First pattern found + if (!foundPattern) + { + assert(patternIndex == 0 && prevCmpOp->OperIs(GT_EQ)); + ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntCon()->IconValue(); - // Current pattern - ssize_t currentPattern = condition1->gtGetOp2()->AsIntCon()->IconValue(); - if (!optRngPattern.optSetPattern(patternIndex, currentPattern, condition1->gtGetOp2())) + // Initialize the pattern range + if (!optRngPattern.optInitializeRngPattern(prevBb, firstPatternVal)) { - return PhaseStatus::MODIFIED_NOTHING; + break; } - // False jump - if (condition1->OperIs(GT_EQ)) + // Save the first pattern + if (!optRngPattern.optSetPattern(patternIndex, firstPatternVal, prevBb)) { - optRngPattern.optSetFalseJmpBB(currBb->bbNext); - } - else if (condition1->OperIs(GT_NE)) - { - optRngPattern.optSetFalseJmpBB(currBb->bbJumpDest); - } - else - { - printf("Unexpected condition1->gtOper\n"); - return PhaseStatus::MODIFIED_NOTHING; + break; } - // Update min and max patterns - if (currentPattern < optRngPattern.optGetMinPattern()) - { - // Update the min pattern and the minOp to the CNS_INT or CNS_LNG tree with the min pattern - optRngPattern.optSetMinPattern(currentPattern, condition1->gtGetOp2()); - optRngPattern.optSetMinOp(condition1->gtGetOp2()); - } - else if (currentPattern > optRngPattern.optGetMaxPattern()) - { - // Update the max pattern - optRngPattern.optSetMaxPattern(currentPattern, condition1->gtGetOp2()); - } + firstPatternBBNum = prevBb->bbNum; patternIndex++; + } + + // Current pattern + + // Save the pattern and Switch default jump target for the pattern (false case) + ssize_t currentPatternVal = currCmpOp->gtGetOp2()->AsIntCon()->IconValue(); + if (!optRngPattern.optSetPattern(patternIndex, currentPatternVal, currBb)) + { + break; + } - // Update the code offset range - optRngPattern.optSetBbCodeOffsEnd(currBb->bbCodeOffsEnd); + patternIndex++; + foundPattern = true; - foundPattern = true; + // Stop searching if the current BB is GT_NE. It is the last pattern. + if (currCmpOp->OperIs(GT_NE)) + { + break; } } } @@ -2260,6 +2396,7 @@ PhaseStatus Compiler::optFindSpecificPattern() } } + // Optimize only when patterns are found in consecutive BBs. // Stop searching if patterns have been found in previous BBs, but the current BB does not have a pattern if (foundPattern && patternIndex < (int)(currBb->bbNum - firstPatternBBNum + 1)) { @@ -2271,72 +2408,72 @@ PhaseStatus Compiler::optFindSpecificPattern() if (foundPattern) { - int patternCount = patternIndex; - optRngPattern.optSetPatternCount(patternCount); + int patternCount = optRngPattern.optGetPatternCount(); + if (patternCount <= 1 || patternCount > optRngPattern.m_sizePatterns) + { + return PhaseStatus::MODIFIED_NOTHING; + } #ifdef DEBUG - if (this->verbose) + if (verbose) { printf("Reserved patterns:\n"); - for (int idx = 0; idx < patternCount; idx++) + optRngPattern.optPrintPatterns(); + } +#endif // DEBUG + + // Check if blocks jump to any of the pattern block + if (optRngPattern.optJumpsToPatternBlock()) + { +#ifdef DEBUG + if (verbose) { -#ifdef TARGET_64BIT - printf("%lld ", optRngPattern.optGetPattern(idx)); -#else // !TARGET_64BIT - printf("%d ", optRngPattern.optGetPattern(idx)); -#endif // !TARGET_64BIT + printf("A pattern block jumps to another pattern block, no optimization done.\n"); } - printf("\n\n"); +#endif // DEBUG + + return PhaseStatus::MODIFIED_NOTHING; } -#endif + // Find range of pattern values - ssize_t minPattern = optRngPattern.optGetMinPattern(); - ssize_t maxPattern = optRngPattern.optGetMaxPattern(); -#ifdef DEBUG - if (this->verbose) + ssize_t minPattern = optRngPattern.optGetMinPattern(); + ssize_t maxPattern = optRngPattern.optGetMaxPattern(); + int rangePattern = (int)(maxPattern - minPattern + 1); + if (patternCount > rangePattern || rangePattern < 2 || rangePattern > optRngPattern.m_sizePatterns) { -#ifdef TARGET_64BIT - printf("Min pattern value: %lld\n", minPattern); - printf("Max pattern value: %lld\n", maxPattern); -#else // !TARGET_64BIT - printf("Min pattern value: %d\n", minPattern); - printf("Max pattern value: %d\n", maxPattern); -#endif // !TARGET_64BIT - } -#endif // DEBUG - - int rangePattern = (int)(maxPattern - minPattern + 1); #ifdef DEBUG - if (this->verbose) - { - printf("Range of pattern values: %d\n", rangePattern); - } + if (verbose) + { + printf("Range of pattern values is too small (< 2) or too big (> %d): %d\n", + optRngPattern.m_sizePatterns, rangePattern); + } #endif // DEBUG - if (rangePattern > optRngPattern.m_sizePatterns) - { - printf("Range of pattern values is too big (> %d): %d\n", optRngPattern.m_sizePatterns, rangePattern); return PhaseStatus::MODIFIED_NOTHING; } optRngPattern.optSetRangePattern(rangePattern); - // Find bitmap of pattern values - unsigned int btPatterns = 0; - for (int idxPattern = 0; idxPattern < patternCount; idxPattern++) +#ifdef DEBUG + if (verbose) { - btPatterns |= (1 << (optRngPattern.optGetPattern(idxPattern) - minPattern)); +#ifdef TARGET_64BIT + printf("Min Max Range: %lld, %lld, %d\n", minPattern, maxPattern, rangePattern); +#else // !TARGET_64BIT + printf("Min Max Range Bitmap: %d, %d, %d\n", minPattern, maxPattern, rangePattern); +#endif // !TARGET_64BIT } - optRngPattern.optSetBitmapPatterns(btPatterns); - printf("Bitmap of pattern values: %d\n", btPatterns); +#endif // DEBUG - // Replace "if" BB with a "Switch" BB - if (!optRngPattern.optMakeSwitchBBdesc()) - { - return PhaseStatus::MODIFIED_NOTHING; - } - if (optRngPattern.optUpdateBlocks()) + // Replace "JTRUE" block with a "Switch" block and remove other pattern blocks + if (optRngPattern.optChangeToSwitch()) { - fgDispBasicBlocks(true); +#ifdef DEBUG + if (verbose) + { + fgDispBasicBlocks(true); + } +#endif // DEBUG + return PhaseStatus::MODIFIED_EVERYTHING; } } From 0fb697f89e64d7203b4b6836d25abaee5511fd22 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Thu, 29 Jun 2023 11:12:37 -0700 Subject: [PATCH 08/35] [if range check opt] Don't optimize if block is marked with BBF_DONT_REMOVE. --- src/coreclr/jit/optimizebools.cpp | 44 ++++++++++++++++--------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 63c5d71e7dcc72..698d0ecb5a28dc 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1583,7 +1583,7 @@ class OptRangePatternDsc Compiler* m_comp = nullptr; // The pointer to the Compiler instance CompAllocator m_allocator; // The memory allocator typedef JitHashTable, BasicBlock*> PatternToBlockMap; - PatternToBlockMap m_patternToBlockMap; + PatternToBlockMap m_patternToBlockMap; BasicBlock* switchBBdesc = nullptr; // The switch basic block descriptor BBswtDesc* swtDsc = nullptr; // The switch descriptor @@ -1599,17 +1599,17 @@ class OptRangePatternDsc unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block unsigned m_bbCodeOffsEnd = 0; // IL code offset end of the switch basic block - bool optIsInIntegralRange(ssize_t val, GenTree* node); // Checks if the value is in INT32 or INT64 range + bool optIsInIntegralRange(ssize_t val, GenTree* node); // Checks if the value is in INT32 or INT64 range BasicBlock* optGetDefaultJmpBB(); - void optSetDefaultJmpBB(BasicBlock* defaultJmpBB); - void optSetMinOp(GenTree* minOpNode); - void optSetMinPattern(ssize_t patternVal, GenTree* node); - void optSetMaxPattern(ssize_t patternVal); - void optSetBbCodeOffs(IL_OFFSET bbCodeOffs); - void optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd); - bool optBlockIsPred(BasicBlock* block, BasicBlock* blockPred); + void optSetDefaultJmpBB(BasicBlock* defaultJmpBB); + void optSetMinOp(GenTree* minOpNode); + void optSetMinPattern(ssize_t patternVal, GenTree* node); + void optSetMaxPattern(ssize_t patternVal); + void optSetBbCodeOffs(IL_OFFSET bbCodeOffs); + void optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd); + bool optBlockIsPred(BasicBlock* block, BasicBlock* blockPred); BasicBlock* optGetJumpTargetBB(BasicBlock* block); - bool optMakeSwitchDesc(); + bool optMakeSwitchDesc(); public: OptRangePatternDsc(Compiler* comp) @@ -1617,15 +1617,15 @@ class OptRangePatternDsc { } - bool optInitializeRngPattern(BasicBlock* firstBB, ssize_t patternVal); - bool optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block); + bool optInitializeRngPattern(BasicBlock* firstBB, ssize_t patternVal); + bool optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block); int optGetPatternCount(); void optPrintPatterns(); bool optJumpsToPatternBlock(); ssize_t optGetMinPattern(); ssize_t optGetMaxPattern(); - void optSetRangePattern(int rangeVal); - bool optChangeToSwitch(); + void optSetRangePattern(int rangeVal); + bool optChangeToSwitch(); }; //----------------------------------------------------------------------------- @@ -1764,7 +1764,7 @@ BasicBlock* OptRangePatternDsc::optGetJumpTargetBB(BasicBlock* block) // void OptRangePatternDsc::optSetDefaultJmpBB(BasicBlock* block) { - assert(block != nullptr && block->bbJumpKind == BBJ_SWITCH || block->bbJumpKind == BBJ_COND); + assert(block != nullptr && block->bbJumpKind == BBJ_COND); assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); auto oper = block->lastStmt()->GetRootNode()->gtGetOp1()->OperGet(); @@ -2168,6 +2168,8 @@ bool OptRangePatternDsc::optChangeToSwitch() BasicBlock* currBb = switchBBdesc->bbNext; for (int idxPattern = 1; idxPattern < patternCount && currBb != nullptr; idxPattern++) { + assert(!(currBb->bbFlags & BBF_DONT_REMOVE)); + BasicBlock* prevBlock = currBb->bbPrev; BasicBlock* nextBlock = currBb->bbNext; BasicBlock* jumpBlock = optGetJumpTargetBB(currBb); @@ -2213,7 +2215,7 @@ bool OptRangePatternDsc::optChangeToSwitch() assert(tree->gtGetOp1() != nullptr && tree->gtGetOp1()->OperIs(GT_LCL_VAR)); // Change constant node if siwtch tree does not have the mininum pattern - if (tree->gtGetOp2() != nullptr && tree->gtGetOp2()->AsIntCon()->IconValue() != optGetMinPattern()) + if (tree->gtGetOp2() != nullptr && tree->gtGetOp2()->AsIntConCommon()->IntegralValue() != optGetMinPattern()) { GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT or GT_CNS_LNG node tree->AsOp()->gtOp2 = m_minOp; @@ -2281,7 +2283,7 @@ PhaseStatus Compiler::optFindSpecificPattern() // Check if prevBb is the predecessor of currBb and currBb has only one predecessor FlowEdge** ptrToPred; FlowEdge* pred = fgGetPredForBlock(currBb, prevBb, &ptrToPred); - if (pred == nullptr || currBb->bbRefs != 1) + if (pred == nullptr || currBb->bbRefs != 1 || currBb->bbFlags & BBF_DONT_REMOVE) { if (foundPattern) break; // Stop searching if patterns are not from consecutive basic blocks @@ -2328,7 +2330,6 @@ PhaseStatus Compiler::optFindSpecificPattern() prevCmpOp->gtGetOp1()->AsLclVar()->GetLclNum() == currCmpOp->gtGetOp1()->AsLclVar()->GetLclNum()) { - // Found pattern #ifdef DEBUG if (this->verbose) { @@ -2339,7 +2340,8 @@ PhaseStatus Compiler::optFindSpecificPattern() printf("\n\n"); } #endif // DEBUG - // No optimization if the number of patterns is greater than 64. + + // No optimization if the number of patterns is greater than 64. if (patternIndex >= optRngPattern.m_sizePatterns) { #ifdef DEBUG @@ -2355,7 +2357,7 @@ PhaseStatus Compiler::optFindSpecificPattern() if (!foundPattern) { assert(patternIndex == 0 && prevCmpOp->OperIs(GT_EQ)); - ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntCon()->IconValue(); + ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntConCommon()->IntegralValue(); // Initialize the pattern range if (!optRngPattern.optInitializeRngPattern(prevBb, firstPatternVal)) @@ -2376,7 +2378,7 @@ PhaseStatus Compiler::optFindSpecificPattern() // Current pattern // Save the pattern and Switch default jump target for the pattern (false case) - ssize_t currentPatternVal = currCmpOp->gtGetOp2()->AsIntCon()->IconValue(); + ssize_t currentPatternVal = currCmpOp->gtGetOp2()->AsIntConCommon()->IntegralValue(); if (!optRngPattern.optSetPattern(patternIndex, currentPatternVal, currBb)) { break; From abad4b4d33ee8068385fef60dfacebed3d3531f4 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Fri, 30 Jun 2023 11:17:19 -0700 Subject: [PATCH 09/35] [if range check opt] Optimize GT_CNS_INT patterns only. --- src/coreclr/jit/optimizebools.cpp | 74 ++++--------------------------- 1 file changed, 8 insertions(+), 66 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 698d0ecb5a28dc..5aed0e25086b99 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1599,7 +1599,6 @@ class OptRangePatternDsc unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block unsigned m_bbCodeOffsEnd = 0; // IL code offset end of the switch basic block - bool optIsInIntegralRange(ssize_t val, GenTree* node); // Checks if the value is in INT32 or INT64 range BasicBlock* optGetDefaultJmpBB(); void optSetDefaultJmpBB(BasicBlock* defaultJmpBB); void optSetMinOp(GenTree* minOpNode); @@ -1617,7 +1616,7 @@ class OptRangePatternDsc { } - bool optInitializeRngPattern(BasicBlock* firstBB, ssize_t patternVal); + void optInitializeRngPattern(BasicBlock* firstBB, ssize_t patternVal); bool optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block); int optGetPatternCount(); void optPrintPatterns(); @@ -1650,39 +1649,6 @@ bool OptRangePatternDsc::optBlockIsPred(BasicBlock* block, BasicBlock* blockPred return false; } -//----------------------------------------------------------------------------- -// optIsInIntegralRange: Checks if the value is within min and max range of INT32 or INT64 -// -// Return Value: -// true if the value is within min and max range of INT32 or INT64 -// false otherwise. -// -// Arguments: -// val - the value to check -// node - the node to check the integral range -// -bool OptRangePatternDsc::optIsInIntegralRange(ssize_t val, GenTree* node) -{ -#ifdef TARGET_64BIT - if (val < _I64_MIN || val > _I64_MAX) - { - printf("val out of range"); - return false; - } -#else // !TARGET_64BIT - IntegralRange integralRange = IntegralRange::ForNode(node, m_comp); - int64_t integralMin = IntegralRange::SymbolicToRealValue(integralRange.GetLowerBound()); - int64_t integralMax = IntegralRange::SymbolicToRealValue(integralRange.GetUpperBound()); - - if (val < integralMin || val > integralMax) - { - printf("val out of range"); - return false; - } -#endif // !TARGET_64BIT - return true; -} - //----------------------------------------------------------------------------- // optSetMinPattern: Sets the min pattern value // @@ -1819,23 +1785,11 @@ void OptRangePatternDsc::optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd) // firstBB - The first basic block of the range pattern // firstVal - The first value of the range pattern // -// Return Value: -// True if the range pattern is initialized successfully -// False otherwise. -// -bool OptRangePatternDsc::optInitializeRngPattern(BasicBlock* firstBB, ssize_t firstVal) +void OptRangePatternDsc::optInitializeRngPattern(BasicBlock* firstBB, ssize_t firstVal) { assert(firstBB != nullptr && firstBB->bbJumpKind == BBJ_COND && firstBB->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); - // Check if the value is within min and max range of INT32 or INT64 - GenTree* rootTree = firstBB->lastStmt()->GetRootNode(); - if (rootTree->gtGetOp1()->gtGetOp2() != nullptr && - !optIsInIntegralRange(firstVal, rootTree->gtGetOp1()->gtGetOp2())) - { - return false; - } - m_optFirstBB = firstBB; // Initialize min pattern and max pattern to the first pattern value @@ -1844,8 +1798,6 @@ bool OptRangePatternDsc::optInitializeRngPattern(BasicBlock* firstBB, ssize_t fi // Initialize the code offset range from the first block optSetBbCodeOffs(firstBB->bbCodeOffs); - - return true; } //----------------------------------------------------------------------------- @@ -1873,12 +1825,6 @@ bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, Basic return false; } - // Check if the value is within min and max range of INT32 or INT64 - if (!optIsInIntegralRange(patternVal, block->lastStmt()->GetRootNode()->gtGetOp1()->gtGetOp2())) - { - return false; - } - // Set the pattern value if it does not already exists in the map BasicBlock* mappedBlock = nullptr; if (m_patternToBlockMap.Lookup(patternVal, &mappedBlock)) // Pattern already exists @@ -2215,7 +2161,7 @@ bool OptRangePatternDsc::optChangeToSwitch() assert(tree->gtGetOp1() != nullptr && tree->gtGetOp1()->OperIs(GT_LCL_VAR)); // Change constant node if siwtch tree does not have the mininum pattern - if (tree->gtGetOp2() != nullptr && tree->gtGetOp2()->AsIntConCommon()->IntegralValue() != optGetMinPattern()) + if (tree->gtGetOp2() != nullptr && tree->gtGetOp2()->AsIntCon()->IconValue() != optGetMinPattern()) { GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT or GT_CNS_LNG node tree->AsOp()->gtOp2 = m_minOp; @@ -2321,9 +2267,8 @@ PhaseStatus Compiler::optFindSpecificPattern() continue; } - // Check both conditions to have constant on the right side - if (currCmpOp->gtGetOp2()->IsIntegralConst() && prevCmpOp->gtGetOp2()->IsIntegralConst() && - currCmpOp->gtGetOp2()->OperGet() == prevCmpOp->gtGetOp2()->OperGet()) + // Check both conditions to have constant on the right side (optimize GT_CNS_INT only) + if (currCmpOp->gtGetOp2()->IsCnsIntOrI() && prevCmpOp->gtGetOp2()->IsCnsIntOrI()) { // Check both conditions to have the same local variable number if (prevCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && currCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && @@ -2357,13 +2302,10 @@ PhaseStatus Compiler::optFindSpecificPattern() if (!foundPattern) { assert(patternIndex == 0 && prevCmpOp->OperIs(GT_EQ)); - ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntConCommon()->IntegralValue(); + ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntCon()->IconValue(); // Initialize the pattern range - if (!optRngPattern.optInitializeRngPattern(prevBb, firstPatternVal)) - { - break; - } + optRngPattern.optInitializeRngPattern(prevBb, firstPatternVal); // Save the first pattern if (!optRngPattern.optSetPattern(patternIndex, firstPatternVal, prevBb)) @@ -2378,7 +2320,7 @@ PhaseStatus Compiler::optFindSpecificPattern() // Current pattern // Save the pattern and Switch default jump target for the pattern (false case) - ssize_t currentPatternVal = currCmpOp->gtGetOp2()->AsIntConCommon()->IntegralValue(); + ssize_t currentPatternVal = currCmpOp->gtGetOp2()->AsIntCon()->IconValue(); if (!optRngPattern.optSetPattern(patternIndex, currentPatternVal, currBb)) { break; From cbe75c93be85ff3c639ee73453e225ffdf407b1e Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Mon, 3 Jul 2023 16:03:38 -0700 Subject: [PATCH 10/35] [if range check opt] Skip cases that Lowering does not convert to a bit test. --- src/coreclr/jit/optimizebools.cpp | 132 ++++++++++++++++++++++++++---- 1 file changed, 116 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 5aed0e25086b99..6aeb81ff4d1563 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1585,6 +1585,10 @@ class OptRangePatternDsc typedef JitHashTable, BasicBlock*> PatternToBlockMap; PatternToBlockMap m_patternToBlockMap; + CompAllocator m_allocatorTarget; // The memory allocator + typedef JitHashTable, int> BlockToIntMap; + BlockToIntMap m_blockToIntMap; + BasicBlock* switchBBdesc = nullptr; // The switch basic block descriptor BBswtDesc* swtDsc = nullptr; // The switch descriptor @@ -1608,11 +1612,16 @@ class OptRangePatternDsc void optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd); bool optBlockIsPred(BasicBlock* block, BasicBlock* blockPred); BasicBlock* optGetJumpTargetBB(BasicBlock* block); + void optSetBlockToIntMap(BasicBlock* targetBb); bool optMakeSwitchDesc(); public: OptRangePatternDsc(Compiler* comp) - : m_comp(comp), m_allocator(comp->getAllocator(CMK_Generic)), m_patternToBlockMap(m_allocator) + : m_comp(comp) + , m_allocator(comp->getAllocator(CMK_Generic)) + , m_patternToBlockMap(m_allocator) + , m_allocatorTarget(comp->getAllocator(CMK_Generic)) + , m_blockToIntMap(m_allocatorTarget) { } @@ -1826,8 +1835,7 @@ bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, Basic } // Set the pattern value if it does not already exists in the map - BasicBlock* mappedBlock = nullptr; - if (m_patternToBlockMap.Lookup(patternVal, &mappedBlock)) // Pattern already exists + if (m_patternToBlockMap.Lookup(patternVal, nullptr)) // Pattern already exists { return false; } @@ -1931,6 +1939,20 @@ void OptRangePatternDsc::optSetRangePattern(int rangeVal) m_rangePattern = rangeVal; } +//----------------------------------------------------------------------------- +// optSetBlockToIntMap: Set the map for unique target blocks +// +// Arguments: +// targetBb - the target basic block +// +void OptRangePatternDsc::optSetBlockToIntMap(BasicBlock* targetBb) +{ + if (!m_blockToIntMap.Lookup(targetBb, nullptr)) + { + m_blockToIntMap.Set(targetBb, 1); + } +} + //----------------------------------------------------------------------------- // optMakeSwitchDesc: Make a Switch descriptor with a switch jump table // @@ -1966,16 +1988,17 @@ bool OptRangePatternDsc::optMakeSwitchDesc() BasicBlockFlags bbFlags = BBF_EMPTY; unsigned curBBoffs = 0; unsigned nxtBBoffs = 0; - unsigned jmpCnt = 0; // # of switch cases (excluding default) + unsigned jmpCnt = 0; // # of switch cases (including default case) - BasicBlock** jmpTab = nullptr; - BasicBlock** jmpPtr = nullptr; - bool tailCall = false; - ssize_t minVal = optGetMinPattern(); + BasicBlock** jmpTab = nullptr; + BasicBlock** jmpPtr = nullptr; + unsigned uniqueTargetCnt = 0; // # of unique jump targets + bool tailCall = false; + ssize_t minVal = optGetMinPattern(); // Allocate the jump table - jmpCnt = m_rangePattern; - jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt + 1]; + jmpCnt = m_rangePattern + 1; + jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt]; // Make a jump table for the range of patterns // If reserved pattern, jump to jump target of source block. If not, jump to default target. @@ -1988,28 +2011,105 @@ bool OptRangePatternDsc::optMakeSwitchDesc() { BasicBlock* jumpTargetBb = optGetJumpTargetBB(mappedBlock); *jmpPtr = (BasicBlock*)(size_t)(jumpTargetBb->bbCodeOffs); - *(jmpPtr++) = jumpTargetBb; + *(jmpPtr) = jumpTargetBb; + // Update the target basic block to count map + optSetBlockToIntMap(jumpTargetBb); } else { BasicBlock* defaultJmpBb = optGetDefaultJmpBB(); *jmpPtr = (BasicBlock*)(size_t)(defaultJmpBb->bbCodeOffs); - *(jmpPtr++) = defaultJmpBb; + *(jmpPtr) = defaultJmpBb; + // Update the target basic block to count map + optSetBlockToIntMap(defaultJmpBb); } + jmpPtr++; } // Append the default label to the jump table - *jmpPtr = (BasicBlock*)(size_t)(optGetDefaultJmpBB()->bbCodeOffs); - *(jmpPtr++) = optGetDefaultJmpBB(); + *jmpPtr = (BasicBlock*)(size_t)(optGetDefaultJmpBB()->bbCodeOffs); + *(jmpPtr) = optGetDefaultJmpBB(); + optSetBlockToIntMap(optGetDefaultJmpBB()); + jmpPtr++; // Make sure we found the right number of labels - noway_assert(jmpPtr == jmpTab + jmpCnt + 1); + noway_assert(jmpPtr == jmpTab + jmpCnt); + + // + // Check if it is profitable to use a switch instead of a series of conditional branches + // + + // If the number of unique target counts is 1, it has only default case. Not profitable. + // If it is > 3, it it not converted to a bit test in Lowering. So, skip it. + uniqueTargetCnt = m_blockToIntMap.GetCount(); + if (uniqueTargetCnt == 1 || uniqueTargetCnt > 3) + { + return false; + } + + noway_assert(jmpCnt >= 2); + + // If all jumps to the same target block, BBJ_NONE or BBJ_ALWAYS is better. + BasicBlock* uniqueSucc = nullptr; + if (uniqueTargetCnt == 2) + { + uniqueSucc = jmpTab[0]; + noway_assert(jmpCnt >= 2); + for (unsigned i = 1; i < jmpCnt - 1; i++) + { + if (jmpTab[i] != uniqueSucc) + { + uniqueSucc = nullptr; + break; + } + } + } + if (uniqueSucc != nullptr) + { + return false; + } + + // Check if it is better to use Jmp instead of Switch + BasicBlock* defaultBB = jmpTab[jmpCnt - 1]; + BasicBlock* followingBB = m_optFirstBB->bbNext; + + // Is the number of cases right for a jump switch? + const bool firstCaseFollows = (followingBB == jmpTab[0]); + const bool defaultFollows = (followingBB == defaultBB); + + unsigned minSwitchTabJumpCnt = 2; // table is better than just 2 cmp/jcc + + // This means really just a single cmp/jcc (aka a simple if/else) + if (firstCaseFollows || defaultFollows) + { + minSwitchTabJumpCnt++; + } + +#if defined(TARGET_ARM) + // On ARM for small switch tables we will + // generate a sequence of compare and branch instructions + // because the code to load the base of the switch + // table is huge and hideous due to the relocation... :( + minSwitchTabJumpCnt += 2; +#endif // TARGET_ARM + + bool useJumpSequence = jmpCnt < minSwitchTabJumpCnt; + + if (TargetOS::IsUnix && TargetArchitecture::IsArm32) + { + useJumpSequence = useJumpSequence || m_comp->IsTargetAbi(CORINFO_NATIVEAOT_ABI); + } + + if (useJumpSequence) // It is better to use a series of compare and branch IR trees. + { + return false; + } // Allocate the switch descriptor swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; // Fill in the remaining fields of the switch descriptor - swtDsc->bbsCount = jmpCnt + 1; + swtDsc->bbsCount = jmpCnt; swtDsc->bbsDstTab = jmpTab; m_comp->fgHasSwitch = true; From eb0bed6cc91b746c3872cef8cb0dc6df89232331 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Mon, 3 Jul 2023 18:37:34 -0700 Subject: [PATCH 11/35] [if range check opt] Skip if switch cases or default don't jump to switch->bbNext. --- src/coreclr/jit/optimizebools.cpp | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 6aeb81ff4d1563..32838c61e2ff29 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1598,6 +1598,7 @@ class OptRangePatternDsc GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern + BasicBlock* m_optLastBB = nullptr; // The last BB of the range pattern BasicBlock* m_defaultJmpBB = nullptr; // The Switch default jump target unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block @@ -1800,6 +1801,7 @@ void OptRangePatternDsc::optInitializeRngPattern(BasicBlock* firstBB, ssize_t fi firstBB->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); m_optFirstBB = firstBB; + m_optLastBB = firstBB; // Initialize min pattern and max pattern to the first pattern value m_minPattern = firstVal; @@ -1852,6 +1854,9 @@ bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, Basic // Update the code offset end range optSetBbCodeOffsEnd(block->bbCodeOffsEnd); + // Set the last basic block of the range pattern + m_optLastBB = block; + return true; } @@ -2040,9 +2045,9 @@ bool OptRangePatternDsc::optMakeSwitchDesc() // // If the number of unique target counts is 1, it has only default case. Not profitable. - // If it is > 3, it it not converted to a bit test in Lowering. So, skip it. + // If it is >= 3, it it not converted to a bit test in Lowering. So, skip it. uniqueTargetCnt = m_blockToIntMap.GetCount(); - if (uniqueTargetCnt == 1 || uniqueTargetCnt > 3) + if (uniqueTargetCnt != 2) { return false; } @@ -2071,7 +2076,7 @@ bool OptRangePatternDsc::optMakeSwitchDesc() // Check if it is better to use Jmp instead of Switch BasicBlock* defaultBB = jmpTab[jmpCnt - 1]; - BasicBlock* followingBB = m_optFirstBB->bbNext; + BasicBlock* followingBB = m_optLastBB->bbNext; // Is the number of cases right for a jump switch? const bool firstCaseFollows = (followingBB == jmpTab[0]); @@ -2105,6 +2110,25 @@ bool OptRangePatternDsc::optMakeSwitchDesc() return false; } + // One of the case blocks has to follow the switch block. All the pattern blocks will be removed and the first + // pattern block will be converted to Switch, so it has to follow the last pattern block. + BasicBlock* bbCase0 = nullptr; + BasicBlock* bbCase1 = jmpTab[0]; + BasicBlock* nextBbAfterSwitch = m_optLastBB->bbNext; + + for (unsigned tabIdx = 1; tabIdx < (jmpCnt - 1); tabIdx++) + { + if (jmpTab[tabIdx] != bbCase1 && bbCase0 == nullptr) + { + bbCase0 = jmpTab[tabIdx]; + break; + } + } + if ((nextBbAfterSwitch != bbCase0) && (nextBbAfterSwitch != bbCase1)) + { + return false; + } + // Allocate the switch descriptor swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; From aa779d5c4a57bc0bb9bacfd4f910402a75a01750 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Wed, 12 Jul 2023 15:43:47 -0700 Subject: [PATCH 12/35] Change Switch conversion to happen after Optimize Layout to avoid Switch and bbNext blocks get separated. --- src/coreclr/jit/compiler.cpp | 6 ++++-- src/coreclr/jit/optimizebools.cpp | 31 +++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index e9f6aebda231df..ca44dfe6c2d540 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5074,8 +5074,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // DoPhase(this, PHASE_OPTIMIZE_BOOLS, &Compiler::optOptimizeBools); - DoPhase(this, PHASE_Find_Specific_Pattern, &Compiler::optFindSpecificPattern); - // If conversion // DoPhase(this, PHASE_IF_CONVERSION, &Compiler::optIfConversion); @@ -5083,6 +5081,10 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // Optimize block order // DoPhase(this, PHASE_OPTIMIZE_LAYOUT, &Compiler::optOptimizeLayout); + + // Conditional to Switch conversion + // + DoPhase(this, PHASE_Find_Specific_Pattern, &Compiler::optFindSpecificPattern); } // Determine start of cold region if we are hot/cold splitting diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 32838c61e2ff29..97a89f26d5477c 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -2110,8 +2110,8 @@ bool OptRangePatternDsc::optMakeSwitchDesc() return false; } - // One of the case blocks has to follow the switch block. All the pattern blocks will be removed and the first - // pattern block will be converted to Switch, so it has to follow the last pattern block. + // One of the case blocks has to follow the switch block. All the pattern blocks except for the first pattern block + // will be removed from Switch conversion. So, we need to check if jump target follows the last pattern block. BasicBlock* bbCase0 = nullptr; BasicBlock* bbCase1 = jmpTab[0]; BasicBlock* nextBbAfterSwitch = m_optLastBB->bbNext; @@ -2129,6 +2129,21 @@ bool OptRangePatternDsc::optMakeSwitchDesc() return false; } + // If the next basic block after Switch is an empty block with an unconditional jump, skip it. + if (nextBbAfterSwitch->isEmpty() && (nextBbAfterSwitch->bbJumpKind == BBJ_ALWAYS) && + (nextBbAfterSwitch != nextBbAfterSwitch->bbJumpDest)) + { +#ifdef DEBUG + if (m_comp->verbose) + { + printf("\nSkip converting to Switch block if Switch jumps to an empty block with an unconditional " + "jump (" FMT_BB " -> " FMT_BB ")\n", + m_optFirstBB->bbNum, nextBbAfterSwitch->bbNum); + } +#endif // DEBUG + return false; + } + // Allocate the switch descriptor swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; @@ -2426,6 +2441,18 @@ PhaseStatus Compiler::optFindSpecificPattern() if (!foundPattern) { assert(patternIndex == 0 && prevCmpOp->OperIs(GT_EQ)); + // If the first pattern block is rarely run, skip it. + if (prevBb->isRunRarely()) + { +#ifdef DEBUG + if (this->verbose) + { + printf(FMT_BB " is run rarely. Skip optimizing this block.\n", prevBb->bbNum); + } +#endif // DEBUG + prevBb = currBb; + continue; + } ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntCon()->IconValue(); // Initialize the pattern range From c4272faf4ccddc5ce5f49ad56055df13fac3c5a1 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Mon, 17 Jul 2023 17:03:23 -0700 Subject: [PATCH 13/35] Refactored the code for Switch Recognition. Match pattern for >=3 conditionals. --- src/coreclr/jit/compiler.cpp | 2 +- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/compphases.h | 2 +- src/coreclr/jit/optimizebools.cpp | 1005 ----------------------------- 4 files changed, 3 insertions(+), 1008 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index ca44dfe6c2d540..255d030ae4096a 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5084,7 +5084,7 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // Conditional to Switch conversion // - DoPhase(this, PHASE_Find_Specific_Pattern, &Compiler::optFindSpecificPattern); + DoPhase(this, PHASE_SWITCH_RECOGNITION, &Compiler::optSwitchRecognition); } // Determine start of cold region if we are hot/cold splitting diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 45dc53e85c7ce8..6c18223d15323c 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6316,7 +6316,7 @@ class Compiler public: PhaseStatus optOptimizeBools(); - PhaseStatus optFindSpecificPattern(); + PhaseStatus optSwitchRecognition(); public: PhaseStatus optInvertLoops(); // Invert loops so they're entered at top and tested at bottom. diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h index 5814a68823266b..5320d37eba070a 100644 --- a/src/coreclr/jit/compphases.h +++ b/src/coreclr/jit/compphases.h @@ -72,7 +72,7 @@ CompPhaseNameMacro(PHASE_MORPH_MDARR, "Morph array ops", CompPhaseNameMacro(PHASE_HOIST_LOOP_CODE, "Hoist loop code", false, -1, false) CompPhaseNameMacro(PHASE_MARK_LOCAL_VARS, "Mark local vars", false, -1, false) CompPhaseNameMacro(PHASE_OPTIMIZE_BOOLS, "Optimize bools", false, -1, false) -CompPhaseNameMacro(PHASE_Find_Specific_Pattern, "Find Specific pattern", false, -1, false) +CompPhaseNameMacro(PHASE_SWITCH_RECOGNITION, "Find Specific pattern", false, -1, false) CompPhaseNameMacro(PHASE_FIND_OPER_ORDER, "Find oper order", false, -1, false) CompPhaseNameMacro(PHASE_SET_BLOCK_ORDER, "Set block order", false, -1, true) CompPhaseNameMacro(PHASE_BUILD_SSA, "Build SSA representation", true, -1, false) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 97a89f26d5477c..043d4acb476ebf 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1570,1008 +1570,3 @@ PhaseStatus Compiler::optOptimizeBools() const bool modified = stress || ((numCond + numReturn) > 0); return modified ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; } - -//----------------------------------------------------------------------------- -// OptRangePatternDsc: Descriptor used for `if` range pattern optimization -// -class OptRangePatternDsc -{ -public: - static const int m_sizePatterns = 64; // The size of the patterns array - -private: - Compiler* m_comp = nullptr; // The pointer to the Compiler instance - CompAllocator m_allocator; // The memory allocator - typedef JitHashTable, BasicBlock*> PatternToBlockMap; - PatternToBlockMap m_patternToBlockMap; - - CompAllocator m_allocatorTarget; // The memory allocator - typedef JitHashTable, int> BlockToIntMap; - BlockToIntMap m_blockToIntMap; - - BasicBlock* switchBBdesc = nullptr; // The switch basic block descriptor - BBswtDesc* swtDsc = nullptr; // The switch descriptor - - ssize_t m_minPattern = 0; // The minimum pattern - ssize_t m_maxPattern = 0; // The maximum pattern - int m_rangePattern = 0; // The range of values in patterns[] - - GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern - BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern - BasicBlock* m_optLastBB = nullptr; // The last BB of the range pattern - BasicBlock* m_defaultJmpBB = nullptr; // The Switch default jump target - - unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block - unsigned m_bbCodeOffsEnd = 0; // IL code offset end of the switch basic block - - BasicBlock* optGetDefaultJmpBB(); - void optSetDefaultJmpBB(BasicBlock* defaultJmpBB); - void optSetMinOp(GenTree* minOpNode); - void optSetMinPattern(ssize_t patternVal, GenTree* node); - void optSetMaxPattern(ssize_t patternVal); - void optSetBbCodeOffs(IL_OFFSET bbCodeOffs); - void optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd); - bool optBlockIsPred(BasicBlock* block, BasicBlock* blockPred); - BasicBlock* optGetJumpTargetBB(BasicBlock* block); - void optSetBlockToIntMap(BasicBlock* targetBb); - bool optMakeSwitchDesc(); - -public: - OptRangePatternDsc(Compiler* comp) - : m_comp(comp) - , m_allocator(comp->getAllocator(CMK_Generic)) - , m_patternToBlockMap(m_allocator) - , m_allocatorTarget(comp->getAllocator(CMK_Generic)) - , m_blockToIntMap(m_allocatorTarget) - { - } - - void optInitializeRngPattern(BasicBlock* firstBB, ssize_t patternVal); - bool optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block); - int optGetPatternCount(); - void optPrintPatterns(); - bool optJumpsToPatternBlock(); - ssize_t optGetMinPattern(); - ssize_t optGetMaxPattern(); - void optSetRangePattern(int rangeVal); - bool optChangeToSwitch(); -}; - -//----------------------------------------------------------------------------- -// optBlockIsPred: Check if blockPred is the predecessor of block -// -// Return Value: -// True if blockPred is the predecessor of block. False otherwise. -// -// Arguments: -// block - the block to check. -// blockPred - the block to check if it is the predecessor of block. -// -bool OptRangePatternDsc::optBlockIsPred(BasicBlock* block, BasicBlock* blockPred) -{ - // Check if blockPred is the predecessor of block - assert(block != nullptr && blockPred != nullptr); - FlowEdge** ptrToPred; - FlowEdge* predBb = m_comp->fgGetPredForBlock(block, blockPred, &ptrToPred); - if (predBb != nullptr) - return true; - else - return false; -} - -//----------------------------------------------------------------------------- -// optSetMinPattern: Sets the min pattern value -// -// Arguments: -// patternVal - the pattern value to set -// node - the node that contains the patternVal -// -void OptRangePatternDsc::optSetMinPattern(ssize_t patternVal, GenTree* node) -{ - if (patternVal < m_minPattern) - { - m_minPattern = patternVal; - - // Update minOp to the tree with the min pattern - optSetMinOp(node); - } -} - -void OptRangePatternDsc::optSetMaxPattern(ssize_t patternVal) -{ - if (patternVal > m_maxPattern) - { - m_maxPattern = patternVal; - } -} - -void OptRangePatternDsc::optSetMinOp(GenTree* minOpNode) -{ - assert(minOpNode != nullptr && minOpNode->IsIntegralConst()); - m_minOp = minOpNode; -} - -//----------------------------------------------------------------------------- -// optGetJumpTargetBB: Get jumpTargetBB for the pattern -// -// Arguments: -// block - the basic block to get its jump target. -// -// Return Value: -// The jump target BB for the pattern -// -// Notes: -// If compare operator is GT_EQ, the switch jump target (true case) is bbJumpDest. -// If compare operator is GT_NE,it is bbNext. -// -BasicBlock* OptRangePatternDsc::optGetJumpTargetBB(BasicBlock* block) -{ - assert(block != nullptr && (block->bbJumpKind == BBJ_SWITCH || block->bbJumpKind == BBJ_COND)); - assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); - - GenTree* op1 = block->lastStmt()->GetRootNode()->gtGetOp1(); - auto oper = op1->OperGet(); - assert(oper == GT_EQ || oper == GT_NE); - - if (oper == GT_EQ) - { - return block->bbJumpDest; - } - else if (oper == GT_NE) - { - return block->bbNext; - } - - return nullptr; -} - -//----------------------------------------------------------------------------- -// optSetDefaultJmpBB: Get Switch Default jump target -// -// Arguments: -// defaultJmpBB - the basic block to set its jump target. -// -// Return Value: -// true if the m_defaultJmpBB is set, false otherwise. -// -// Notes: -// For GT_EQ, Swithc Default jump target (for false case) is set to bbNext. -// For GT_NE, Swithc Default jump target (for false case) is set to bbJumpDest. -// -void OptRangePatternDsc::optSetDefaultJmpBB(BasicBlock* block) -{ - assert(block != nullptr && block->bbJumpKind == BBJ_COND); - assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); - - auto oper = block->lastStmt()->GetRootNode()->gtGetOp1()->OperGet(); - assert(oper == GT_EQ || oper == GT_NE); - - if (oper == GT_EQ) - { - m_defaultJmpBB = block->bbNext; - } - else if (oper == GT_NE) - { - m_defaultJmpBB = block->bbJumpDest; - } -} - -//----------------------------------------------------------------------------- -// optGetDefaultJmpBB: Get Switch Default jump target -// -// Return Value: -// The default jump target BB of SWITCH block -// -BasicBlock* OptRangePatternDsc::optGetDefaultJmpBB() -{ - return m_defaultJmpBB; -} - -//----------------------------------------------------------------------------- -// optSetBbCodeOffs: Set the code offset of the basic block -// -void OptRangePatternDsc::optSetBbCodeOffs(IL_OFFSET bbCodeOffs) -{ - assert(bbCodeOffs >= 0 && bbCodeOffs <= UINT_MAX); - if (optGetPatternCount() == 0) - { - m_bbCodeOffs = bbCodeOffs; - } -} - -//----------------------------------------------------------------------------- -// optSetBbCodeOffsEnd: Set the code offset of the basic block end -// -void OptRangePatternDsc::optSetBbCodeOffsEnd(IL_OFFSET bbCodeOffsEnd) -{ - assert(bbCodeOffsEnd >= 0 && bbCodeOffsEnd <= UINT_MAX); - m_bbCodeOffsEnd = bbCodeOffsEnd; -} - -//----------------------------------------------------------------------------- -// optInitializeRngPattern: Initializes the range pattern descriptor -// -// Arguments: -// firstBB - The first basic block of the range pattern -// firstVal - The first value of the range pattern -// -void OptRangePatternDsc::optInitializeRngPattern(BasicBlock* firstBB, ssize_t firstVal) -{ - assert(firstBB != nullptr && firstBB->bbJumpKind == BBJ_COND && - firstBB->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); - - m_optFirstBB = firstBB; - m_optLastBB = firstBB; - - // Initialize min pattern and max pattern to the first pattern value - m_minPattern = firstVal; - m_maxPattern = firstVal; - - // Initialize the code offset range from the first block - optSetBbCodeOffs(firstBB->bbCodeOffs); -} - -//----------------------------------------------------------------------------- -// optSetPattern: Save pattern value and basic block -// -// Arguments: -// idxPattern - the index of the pattern to set -// patternVal - the value of the pattern -// block - the basic block to set its jump target. -// -// Return Value: -// true if the pattern is set, false otherwise. -// -bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block) -{ - if (idxPattern < 0 || idxPattern >= m_sizePatterns) - { -#ifdef DEBUG - if (m_comp->verbose) - { - printf("idxPattern out of range"); - } -#endif // DEBUG - - return false; - } - - // Set the pattern value if it does not already exists in the map - if (m_patternToBlockMap.Lookup(patternVal, nullptr)) // Pattern already exists - { - return false; - } - - m_patternToBlockMap.Set(patternVal, block); - - // Set min and max pattern - optSetMinPattern(patternVal, block->lastStmt()->GetRootNode()->gtGetOp1()->gtGetOp2()); - optSetMaxPattern(patternVal); - - // Set default jump target of the basic block - optSetDefaultJmpBB(block); - - // Update the code offset end range - optSetBbCodeOffsEnd(block->bbCodeOffsEnd); - - // Set the last basic block of the range pattern - m_optLastBB = block; - - return true; -} - -//----------------------------------------------------------------------------- -// optGetPatternCount: Get the number of patterns -// -// Return Value: -// The number of reserved patterns -// -int OptRangePatternDsc::optGetPatternCount() -{ - return m_patternToBlockMap.GetCount(); -} - -//----------------------------------------------------------------------------- -// optPrintPatterns: Prints the patterns from m_patternToBlockMap -// -void OptRangePatternDsc::optPrintPatterns() -{ - // print patterns from m_patternToBlockMap using key iterator - PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; - for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) - { - BasicBlock* mappedBlock; - if (patternToBlockMap->Lookup(patternVal, &mappedBlock)) - { - printf("patternVal = %d, block = %d\n", patternVal, mappedBlock->bbNum); - } - } -} - -//----------------------------------------------------------------------------- -// optJumpsToPatternBlock: Checks if any pattern block jumps to one of the blocks -// within m_patternToBlockMap -// -// Arguments: -// None -// -// Return Value: -// True if any of m_patternToBlockMap block jumps to one of the blocks within m_patternToBlockMap -// False otherwise. -// -bool OptRangePatternDsc::optJumpsToPatternBlock() -{ - PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; - const unsigned int maxPatternBbNum = m_optFirstBB->bbNum + optGetPatternCount() - 1; - - for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) - { - BasicBlock* sourceBlock; - if (patternToBlockMap->Lookup(patternVal, &sourceBlock)) - { - BasicBlock* destBlock = sourceBlock->bbJumpDest; - assert(destBlock != nullptr); - - if (destBlock->bbNum >= m_optFirstBB->bbNum && destBlock->bbNum <= maxPatternBbNum) - { - return true; - } - } - } - - return false; -} - -//----------------------------------------------------------------------------- -// optGetMinPattern: Gets the min pattern value -// -// Return Value: -// The min pattern value -// -ssize_t OptRangePatternDsc::optGetMinPattern() -{ - return m_minPattern; -} - -ssize_t OptRangePatternDsc::optGetMaxPattern() -{ - return m_maxPattern; -} - -void OptRangePatternDsc::optSetRangePattern(int rangeVal) -{ - assert(rangeVal >= 0 && rangeVal <= m_sizePatterns); - m_rangePattern = rangeVal; -} - -//----------------------------------------------------------------------------- -// optSetBlockToIntMap: Set the map for unique target blocks -// -// Arguments: -// targetBb - the target basic block -// -void OptRangePatternDsc::optSetBlockToIntMap(BasicBlock* targetBb) -{ - if (!m_blockToIntMap.Lookup(targetBb, nullptr)) - { - m_blockToIntMap.Set(targetBb, 1); - } -} - -//----------------------------------------------------------------------------- -// optMakeSwitchDesc: Make a Switch descriptor with a switch jump table -// -// Returns: -// true if the switch descriptor is created successfully with the switch jump tables. False otherwise. -// -// Notes: -// It creates switch descriptor with a jump table for each range pattern and default case. -// If the value is part of the reserved patterns, the switch jump table is set to the jump target -// for true case -// If GT_EQ, jump target is block's bbJumpDest. If GT_NE, jump target is block's bbNext. -// Otherwise, the switch jump table is set to the default jump target. -// No tree is updated in this method. -// -bool OptRangePatternDsc::optMakeSwitchDesc() -{ -#ifdef DEBUG - if (m_comp->verbose) - { - printf("\n*************** In optMakeSwitchDesc()\n"); - } -#endif // DEBUG - - if (optGetPatternCount() > m_rangePattern || m_rangePattern < 2 || optGetPatternCount() > m_rangePattern || - m_rangePattern > m_sizePatterns) - { - return false; - } - - BasicBlock* prevBb = nullptr; - int patternIndex = 0; - - BasicBlockFlags bbFlags = BBF_EMPTY; - unsigned curBBoffs = 0; - unsigned nxtBBoffs = 0; - unsigned jmpCnt = 0; // # of switch cases (including default case) - - BasicBlock** jmpTab = nullptr; - BasicBlock** jmpPtr = nullptr; - unsigned uniqueTargetCnt = 0; // # of unique jump targets - bool tailCall = false; - ssize_t minVal = optGetMinPattern(); - - // Allocate the jump table - jmpCnt = m_rangePattern + 1; - jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt]; - - // Make a jump table for the range of patterns - // If reserved pattern, jump to jump target of source block. If not, jump to default target. - for (int idxRng = 0; idxRng < m_rangePattern; idxRng++) - { - // Find a mapped block from a pattern - ssize_t key = minVal + idxRng; - BasicBlock* mappedBlock = nullptr; - if (m_patternToBlockMap.Lookup(key, &mappedBlock)) // A mapped block is found - { - BasicBlock* jumpTargetBb = optGetJumpTargetBB(mappedBlock); - *jmpPtr = (BasicBlock*)(size_t)(jumpTargetBb->bbCodeOffs); - *(jmpPtr) = jumpTargetBb; - // Update the target basic block to count map - optSetBlockToIntMap(jumpTargetBb); - } - else - { - BasicBlock* defaultJmpBb = optGetDefaultJmpBB(); - *jmpPtr = (BasicBlock*)(size_t)(defaultJmpBb->bbCodeOffs); - *(jmpPtr) = defaultJmpBb; - // Update the target basic block to count map - optSetBlockToIntMap(defaultJmpBb); - } - jmpPtr++; - } - - // Append the default label to the jump table - *jmpPtr = (BasicBlock*)(size_t)(optGetDefaultJmpBB()->bbCodeOffs); - *(jmpPtr) = optGetDefaultJmpBB(); - optSetBlockToIntMap(optGetDefaultJmpBB()); - jmpPtr++; - - // Make sure we found the right number of labels - noway_assert(jmpPtr == jmpTab + jmpCnt); - - // - // Check if it is profitable to use a switch instead of a series of conditional branches - // - - // If the number of unique target counts is 1, it has only default case. Not profitable. - // If it is >= 3, it it not converted to a bit test in Lowering. So, skip it. - uniqueTargetCnt = m_blockToIntMap.GetCount(); - if (uniqueTargetCnt != 2) - { - return false; - } - - noway_assert(jmpCnt >= 2); - - // If all jumps to the same target block, BBJ_NONE or BBJ_ALWAYS is better. - BasicBlock* uniqueSucc = nullptr; - if (uniqueTargetCnt == 2) - { - uniqueSucc = jmpTab[0]; - noway_assert(jmpCnt >= 2); - for (unsigned i = 1; i < jmpCnt - 1; i++) - { - if (jmpTab[i] != uniqueSucc) - { - uniqueSucc = nullptr; - break; - } - } - } - if (uniqueSucc != nullptr) - { - return false; - } - - // Check if it is better to use Jmp instead of Switch - BasicBlock* defaultBB = jmpTab[jmpCnt - 1]; - BasicBlock* followingBB = m_optLastBB->bbNext; - - // Is the number of cases right for a jump switch? - const bool firstCaseFollows = (followingBB == jmpTab[0]); - const bool defaultFollows = (followingBB == defaultBB); - - unsigned minSwitchTabJumpCnt = 2; // table is better than just 2 cmp/jcc - - // This means really just a single cmp/jcc (aka a simple if/else) - if (firstCaseFollows || defaultFollows) - { - minSwitchTabJumpCnt++; - } - -#if defined(TARGET_ARM) - // On ARM for small switch tables we will - // generate a sequence of compare and branch instructions - // because the code to load the base of the switch - // table is huge and hideous due to the relocation... :( - minSwitchTabJumpCnt += 2; -#endif // TARGET_ARM - - bool useJumpSequence = jmpCnt < minSwitchTabJumpCnt; - - if (TargetOS::IsUnix && TargetArchitecture::IsArm32) - { - useJumpSequence = useJumpSequence || m_comp->IsTargetAbi(CORINFO_NATIVEAOT_ABI); - } - - if (useJumpSequence) // It is better to use a series of compare and branch IR trees. - { - return false; - } - - // One of the case blocks has to follow the switch block. All the pattern blocks except for the first pattern block - // will be removed from Switch conversion. So, we need to check if jump target follows the last pattern block. - BasicBlock* bbCase0 = nullptr; - BasicBlock* bbCase1 = jmpTab[0]; - BasicBlock* nextBbAfterSwitch = m_optLastBB->bbNext; - - for (unsigned tabIdx = 1; tabIdx < (jmpCnt - 1); tabIdx++) - { - if (jmpTab[tabIdx] != bbCase1 && bbCase0 == nullptr) - { - bbCase0 = jmpTab[tabIdx]; - break; - } - } - if ((nextBbAfterSwitch != bbCase0) && (nextBbAfterSwitch != bbCase1)) - { - return false; - } - - // If the next basic block after Switch is an empty block with an unconditional jump, skip it. - if (nextBbAfterSwitch->isEmpty() && (nextBbAfterSwitch->bbJumpKind == BBJ_ALWAYS) && - (nextBbAfterSwitch != nextBbAfterSwitch->bbJumpDest)) - { -#ifdef DEBUG - if (m_comp->verbose) - { - printf("\nSkip converting to Switch block if Switch jumps to an empty block with an unconditional " - "jump (" FMT_BB " -> " FMT_BB ")\n", - m_optFirstBB->bbNum, nextBbAfterSwitch->bbNum); - } -#endif // DEBUG - return false; - } - - // Allocate the switch descriptor - swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; - - // Fill in the remaining fields of the switch descriptor - swtDsc->bbsCount = jmpCnt; - swtDsc->bbsDstTab = jmpTab; - - m_comp->fgHasSwitch = true; - - if (m_comp->opts.compProcedureSplitting) - { - m_comp->opts.compProcedureSplitting = false; - JITDUMP("Turning off procedure splitting for this method, as it might need switch tables; " - "implementation limitation.\n"); - } - - tailCall = false; - -#ifdef DEBUG - if (m_comp->verbose) - { - // Print bbNum of each jmpTab - printf("------Switch " FMT_BB " jumps to: ", m_optFirstBB->bbNum); - for (unsigned i = 0; i < swtDsc->bbsCount; i++) - { - printf("%c" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); - } - printf("\n\n"); - } -#endif // DEBUG - - return true; -} - -//----------------------------------------------------------------------------- -// optChangeToSwitch: Change the first pattern block to SWITCH block and remove other pattern blocks -// -// Return Value: -// true if the blocks were successfully updated, false otherwise. -// -// Notes: -// Leave Switch basic block only and remove all other blocks in the range pattern. -// Update reference count of jump target blocks. -// If the CNS_INT node does not have a min pattern, replace it with the CNS_INT node with the min value. -// -// Tree before the optimization: -// ``` -// JTRUE -// \EQ -// \LCL_VAR -// \CNS_INT -// ``` -// -// Tree after the optimization: -// ``` -// SWITCH -// \SUB -// \LCL_VAR -// \CNS_INT (min pattern) -// ``` -bool OptRangePatternDsc::optChangeToSwitch() -{ -#ifdef DEBUG - if (m_comp->verbose) - { - printf("\n*************** In optChangeToSwitch()\n"); - } -#endif // DEBUG - - assert(m_optFirstBB != nullptr && m_optFirstBB->KindIs(BBJ_COND) && m_optFirstBB->bbNext != nullptr); - assert(optGetPatternCount() <= m_rangePattern && m_rangePattern >= 2 && optGetPatternCount() <= m_rangePattern && - m_rangePattern <= m_sizePatterns); - - // Make Switch descriptor with a jump table - if (!optMakeSwitchDesc()) - { - return false; - } - - bool updatedBlocks = false; - switchBBdesc = m_optFirstBB; - int patternCount = optGetPatternCount(); - - // Update the Switch basic block - - // Change `JTRUE` basic block to `Switch` basic block - switchBBdesc = m_optFirstBB; - - // Change BBJ_COND to BBJ_SWITCH - switchBBdesc->bbJumpKind = BBJ_SWITCH; - switchBBdesc->bbJumpDest = nullptr; - - switchBBdesc->bbCodeOffs = m_bbCodeOffs; - switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; - switchBBdesc->bbJumpSwt = swtDsc; - GenTree* rootTree = switchBBdesc->lastStmt()->GetRootNode(); - assert(rootTree->OperIs(GT_JTRUE)); - assert(!(rootTree->gtFlags & GTF_SIDE_EFFECT)); // JTRUE node should not have side effects - - // Change from GT_JTRUE to GT_SWITCH - rootTree->ChangeOper(GT_SWITCH); - - // Special args to fgAddRefPred - FlowEdge* const oldEdge = nullptr; - - // Remove non-switch pattern blocks. Update the reference count of jump target block from the removed block. - BasicBlock* currBb = switchBBdesc->bbNext; - for (int idxPattern = 1; idxPattern < patternCount && currBb != nullptr; idxPattern++) - { - assert(!(currBb->bbFlags & BBF_DONT_REMOVE)); - - BasicBlock* prevBlock = currBb->bbPrev; - BasicBlock* nextBlock = currBb->bbNext; - BasicBlock* jumpBlock = optGetJumpTargetBB(currBb); - - // Unlink the current block and its pred block - assert(optBlockIsPred(currBb, prevBlock)); - m_comp->fgRemoveRefPred(currBb, prevBlock); - - // Link Switch block and current block's jump target block - m_comp->fgAddRefPred(jumpBlock, switchBBdesc, oldEdge); - - // Link Switch block and the next block: - // if GT_EQ and currBb is the last pattern block, skip because bbNext is already linked as default jump - // target if GT_NE, it is already linked when linking its jump target block to Switch block - if (currBb->lastStmt()->GetRootNode()->gtGetOp1()->OperIs(GT_EQ) && idxPattern != (patternCount - 1)) - { - m_comp->fgAddRefPred(nextBlock, switchBBdesc, oldEdge); - } - - m_comp->fgRemoveBlock(currBb, /* unreachable */ true); - - updatedBlocks = true; - currBb = nextBlock; - } - - // Update the reference count of the default jump block - int numNotFound = m_rangePattern - patternCount + 1; // +1 for switch default case - for (int idxFalse = 0; idxFalse < numNotFound; idxFalse++) - { - m_comp->fgAddRefPred(optGetDefaultJmpBB(), switchBBdesc, oldEdge); - } - - // Continue to transform Switch node - - Statement* stmt = switchBBdesc->lastStmt(); - - // Change from GT_EQ or GT_NE to GT_SUB - // tree: SUB - // op1: LCL_VAR - // op2: GT_CNS_INT or GT_CNS_LNG - GenTree* tree = rootTree->gtGetOp1(); // GT_EQ or GT_NE node to chnage to GT_SUB - tree->ChangeOper(GT_SUB); - assert(tree->gtGetOp1() != nullptr && tree->gtGetOp1()->OperIs(GT_LCL_VAR)); - - // Change constant node if siwtch tree does not have the mininum pattern - if (tree->gtGetOp2() != nullptr && tree->gtGetOp2()->AsIntCon()->IconValue() != optGetMinPattern()) - { - GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT or GT_CNS_LNG node - tree->AsOp()->gtOp2 = m_minOp; - - m_comp->gtSetStmtInfo(stmt); - m_comp->fgSetStmtSeq(stmt); - - DEBUG_DESTROY_NODE(op2); - } - - m_comp->gtUpdateStmtSideEffects(stmt); - - return updatedBlocks; -} - -//----------------------------------------------------------------------------- -// optFindSpecificPattern: Optimize range check for if (A || B || C || D) pattern. -// -// Returns: -// MODIFIED_NOTHING if no optimization is performed. -// MODIFIED_EVERYTHING otherwise. -// -// Notes: -// Detect if (a == val1 || a == val2 || a == val3 || ...) pattern and change it to switch tree -// to reduce jumps and perform bit operation instead. -// -PhaseStatus Compiler::optFindSpecificPattern() -{ -#ifdef DEBUG - if (this->verbose) - { - printf("\n*************** In optFindSpecificPattern()\n"); - } -#endif // DEBUG - - OptRangePatternDsc optRngPattern(this); - - bool printed = false; - int patternIndex = 0; // The index of the pattern in the array - bool foundPattern = false; - unsigned int firstPatternBBNum = 0; // Basic block number of the first pattern found - BasicBlock* prevBb = fgFirstBB; - - if (fgFirstBB->bbNext == nullptr) - { - return PhaseStatus::MODIFIED_NOTHING; - } - - for (BasicBlock* currBb = fgFirstBB->bbNext; currBb != nullptr; currBb = currBb->bbNext) - { - if (currBb->KindIs(BBJ_COND) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) - { -#ifdef DEBUG - if (this->verbose) - { - if (!printed) - { - printf("Display Basic Blocks for optFindSpecificPattern:\n"); - fgDispBasicBlocks(true); - printed = true; - } - } -#endif // DEBUG - - // Check if prevBb is the predecessor of currBb and currBb has only one predecessor - FlowEdge** ptrToPred; - FlowEdge* pred = fgGetPredForBlock(currBb, prevBb, &ptrToPred); - if (pred == nullptr || currBb->bbRefs != 1 || currBb->bbFlags & BBF_DONT_REMOVE) - { - if (foundPattern) - break; // Stop searching if patterns are not from consecutive basic blocks - else - continue; - } - - // Basic block must have only one statement - if (currBb->lastStmt() == currBb->firstStmt() && prevBb->lastStmt() == prevBb->firstStmt()) - { - // Skip if there is any side effect - assert(currBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); // JTRUE node - if (currBb->lastStmt()->GetRootNode()->gtFlags & GTF_SIDE_EFFECT) - { - // Stop searching if patterns are not from consecutive basic blocks - if (foundPattern) - break; - else - continue; - } - - auto currCmpOp = currBb->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node - auto prevCmpOp = prevBb->lastStmt()->GetRootNode()->gtGetOp1(); - assert(currCmpOp != nullptr && prevCmpOp != nullptr); - - // Compare operator is GT_EQ. If it is GT_NE, it is the end of the pattern check. - if ((prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_EQ)) || - (prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_NE))) - { - if (prevBb->bbJumpDest == currBb) - { - if (foundPattern) - break; - else - continue; - } - - // Check both conditions to have constant on the right side (optimize GT_CNS_INT only) - if (currCmpOp->gtGetOp2()->IsCnsIntOrI() && prevCmpOp->gtGetOp2()->IsCnsIntOrI()) - { - // Check both conditions to have the same local variable number - if (prevCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && currCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && - prevCmpOp->gtGetOp1()->AsLclVar()->GetLclNum() == - currCmpOp->gtGetOp1()->AsLclVar()->GetLclNum()) - { -#ifdef DEBUG - if (this->verbose) - { - printf("\nFound pattern (Prev vs Curr):\n"); - gtDispTree(prevCmpOp); - printf("\n"); - gtDispTree(currCmpOp); - printf("\n\n"); - } -#endif // DEBUG - - // No optimization if the number of patterns is greater than 64. - if (patternIndex >= optRngPattern.m_sizePatterns) - { -#ifdef DEBUG - if (this->verbose) - { - printf("Too many patterns found (> 64), no optimization done.\n"); - } -#endif // DEBUG - return PhaseStatus::MODIFIED_NOTHING; - } - - // First pattern found - if (!foundPattern) - { - assert(patternIndex == 0 && prevCmpOp->OperIs(GT_EQ)); - // If the first pattern block is rarely run, skip it. - if (prevBb->isRunRarely()) - { -#ifdef DEBUG - if (this->verbose) - { - printf(FMT_BB " is run rarely. Skip optimizing this block.\n", prevBb->bbNum); - } -#endif // DEBUG - prevBb = currBb; - continue; - } - ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntCon()->IconValue(); - - // Initialize the pattern range - optRngPattern.optInitializeRngPattern(prevBb, firstPatternVal); - - // Save the first pattern - if (!optRngPattern.optSetPattern(patternIndex, firstPatternVal, prevBb)) - { - break; - } - - firstPatternBBNum = prevBb->bbNum; - patternIndex++; - } - - // Current pattern - - // Save the pattern and Switch default jump target for the pattern (false case) - ssize_t currentPatternVal = currCmpOp->gtGetOp2()->AsIntCon()->IconValue(); - if (!optRngPattern.optSetPattern(patternIndex, currentPatternVal, currBb)) - { - break; - } - - patternIndex++; - foundPattern = true; - - // Stop searching if the current BB is GT_NE. It is the last pattern. - if (currCmpOp->OperIs(GT_NE)) - { - break; - } - } - } - } - } - } - - // Optimize only when patterns are found in consecutive BBs. - // Stop searching if patterns have been found in previous BBs, but the current BB does not have a pattern - if (foundPattern && patternIndex < (int)(currBb->bbNum - firstPatternBBNum + 1)) - { - break; - } - - prevBb = currBb; - } - - if (foundPattern) - { - int patternCount = optRngPattern.optGetPatternCount(); - if (patternCount <= 1 || patternCount > optRngPattern.m_sizePatterns) - { - return PhaseStatus::MODIFIED_NOTHING; - } - -#ifdef DEBUG - if (verbose) - { - printf("Reserved patterns:\n"); - optRngPattern.optPrintPatterns(); - } -#endif // DEBUG - - // Check if blocks jump to any of the pattern block - if (optRngPattern.optJumpsToPatternBlock()) - { -#ifdef DEBUG - if (verbose) - { - printf("A pattern block jumps to another pattern block, no optimization done.\n"); - } -#endif // DEBUG - - return PhaseStatus::MODIFIED_NOTHING; - } - - // Find range of pattern values - ssize_t minPattern = optRngPattern.optGetMinPattern(); - ssize_t maxPattern = optRngPattern.optGetMaxPattern(); - int rangePattern = (int)(maxPattern - minPattern + 1); - if (patternCount > rangePattern || rangePattern < 2 || rangePattern > optRngPattern.m_sizePatterns) - { -#ifdef DEBUG - if (verbose) - { - printf("Range of pattern values is too small (< 2) or too big (> %d): %d\n", - optRngPattern.m_sizePatterns, rangePattern); - } -#endif // DEBUG - - return PhaseStatus::MODIFIED_NOTHING; - } - optRngPattern.optSetRangePattern(rangePattern); - -#ifdef DEBUG - if (verbose) - { -#ifdef TARGET_64BIT - printf("Min Max Range: %lld, %lld, %d\n", minPattern, maxPattern, rangePattern); -#else // !TARGET_64BIT - printf("Min Max Range Bitmap: %d, %d, %d\n", minPattern, maxPattern, rangePattern); -#endif // !TARGET_64BIT - } -#endif // DEBUG - - // Replace "JTRUE" block with a "Switch" block and remove other pattern blocks - if (optRngPattern.optChangeToSwitch()) - { -#ifdef DEBUG - if (verbose) - { - fgDispBasicBlocks(true); - } -#endif // DEBUG - - return PhaseStatus::MODIFIED_EVERYTHING; - } - } - - return PhaseStatus::MODIFIED_NOTHING; -} From fbf975ed8166868bc792177dac2d6684fc838016 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Mon, 17 Jul 2023 17:35:08 -0700 Subject: [PATCH 14/35] Refactored Switch recognition. --- src/coreclr/jit/optimizebools.cpp | 876 ++++++++++++++++++++++++++++++ 1 file changed, 876 insertions(+) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 043d4acb476ebf..fbd82bb6b86a00 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1570,3 +1570,879 @@ PhaseStatus Compiler::optOptimizeBools() const bool modified = stress || ((numCond + numReturn) > 0); return modified ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; } + +//----------------------------------------------------------------------------- +// OptRangePatternDsc: Descriptor used for `if` range pattern optimization +// +class OptRangePatternDsc +{ +public: + static const int m_sizePatterns = 64; // The size of the patterns array + +private: + Compiler* m_comp = nullptr; // The pointer to the Compiler instance + CompAllocator m_allocator; // The memory allocator + typedef JitHashTable, BasicBlock*> PatternToBlockMap; + PatternToBlockMap m_patternToBlockMap; + + CompAllocator m_allocatorTarget; // The memory allocator + typedef JitHashTable, int> BlockToIntMap; + BlockToIntMap m_blockToIntMap; + + BasicBlock* switchBBdesc = nullptr; // The switch basic block descriptor + BBswtDesc* swtDsc = nullptr; // The switch descriptor + GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern + BasicBlock* m_defaultJmpBB = nullptr; // The Switch default jump target + BasicBlock* optGetJumpTargetBB(BasicBlock* block); + bool optMakeSwitchDesc(); + +public: + OptRangePatternDsc(Compiler* comp) + : m_comp(comp) + , m_allocator(comp->getAllocator(CMK_Generic)) + , m_patternToBlockMap(m_allocator) + , m_allocatorTarget(comp->getAllocator(CMK_Generic)) + , m_blockToIntMap(m_allocatorTarget) + { + } + + BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern + BasicBlock* m_optLastBB = nullptr; // The last BB of the range pattern + ssize_t m_minPattern = 0; // The minimum pattern + ssize_t m_maxPattern = 0; // The maximum pattern + int m_rangePattern = 0; // The range of values in patterns[] + unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block + unsigned m_bbCodeOffsEnd = 0; // IL code offset end of the switch basic block + + bool optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block); + int optGetPatternCount(); + void optPrintPatterns(); + bool optJumpsToPatternBlock(); + bool optChangeToSwitch(); +}; + +//----------------------------------------------------------------------------- +// optGetJumpTargetBB: Get jumpTargetBB for the pattern +// +// Arguments: +// block - the basic block to get its jump target. +// +// Return Value: +// The jump target BB for the pattern +// +// Notes: +// If compare operator is GT_EQ, the switch jump target (true case) is bbJumpDest. +// If compare operator is GT_NE,it is bbNext. +// +BasicBlock* OptRangePatternDsc::optGetJumpTargetBB(BasicBlock* block) +{ + assert(block != nullptr && (block->bbJumpKind == BBJ_SWITCH || block->bbJumpKind == BBJ_COND)); + assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); + + GenTree* op1 = block->lastStmt()->GetRootNode()->gtGetOp1(); + auto oper = op1->OperGet(); + assert(oper == GT_EQ || oper == GT_NE); + + if (oper == GT_EQ) + { + return block->bbJumpDest; + } + else if (oper == GT_NE) + { + return block->bbNext; + } + + return nullptr; +} + +//----------------------------------------------------------------------------- +// optSetPattern: Save pattern value and basic block +// +// Arguments: +// idxPattern - the index of the pattern to set +// patternVal - the value of the pattern +// block - the basic block to set its jump target. +// +// Return Value: +// true if the pattern is set, false otherwise. +// +bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block) +{ + if (idxPattern < 0 || idxPattern >= m_sizePatterns) + { +#ifdef DEBUG + if (m_comp->verbose) + { + printf("idxPattern out of range"); + } +#endif // DEBUG + + return false; + } + + // Set the pattern value if it does not already exists in the map + if (m_patternToBlockMap.Lookup(patternVal, nullptr)) // Pattern already exists + { + return false; + } + + m_patternToBlockMap.Set(patternVal, block); + + // Set min pattern + if (patternVal < m_minPattern) + { + m_minPattern = patternVal; + + // Update minOp to the tree with the min pattern + GenTree* minOpNode = block->lastStmt()->GetRootNode()->gtGetOp1()->gtGetOp2(); + assert(minOpNode != nullptr && minOpNode->IsIntegralConst()); + m_minOp = minOpNode; + } + // Set max pattern + if (patternVal > m_maxPattern) + { + m_maxPattern = patternVal; + } + + // Set default jump target of the Switch basic block: + // For GT_EQ, Swithc Default jump target (for false case) is set to bbNext. + // For GT_NE, Swithc Default jump target (for false case) is set to bbJumpDest. + assert(block != nullptr && block->bbJumpKind == BBJ_COND); + assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); + + auto oper = block->lastStmt()->GetRootNode()->gtGetOp1()->OperGet(); + assert(oper == GT_EQ || oper == GT_NE); + if (oper == GT_EQ) + { + m_defaultJmpBB = block->bbNext; + } + else if (oper == GT_NE) + { + m_defaultJmpBB = block->bbJumpDest; + } + + // Update the code offset end range + assert(block->bbCodeOffsEnd >= 0 && block->bbCodeOffsEnd <= UINT_MAX); + m_bbCodeOffsEnd = block->bbCodeOffsEnd; + + // Set the last basic block of the range pattern + m_optLastBB = block; + + return true; +} + +//----------------------------------------------------------------------------- +// optGetPatternCount: Get the number of patterns +// +// Return Value: +// The number of reserved patterns +// +int OptRangePatternDsc::optGetPatternCount() +{ + return m_patternToBlockMap.GetCount(); +} + +//----------------------------------------------------------------------------- +// optPrintPatterns: Prints the patterns from m_patternToBlockMap +// +void OptRangePatternDsc::optPrintPatterns() +{ + // print patterns from m_patternToBlockMap using key iterator + PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; + for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) + { + BasicBlock* mappedBlock; + if (patternToBlockMap->Lookup(patternVal, &mappedBlock)) + { + printf("patternVal = %d, block = %d\n", patternVal, mappedBlock->bbNum); + } + } +} + +//----------------------------------------------------------------------------- +// optJumpsToPatternBlock: Checks if any pattern block jumps to one of the blocks +// within m_patternToBlockMap +// +// Arguments: +// None +// +// Return Value: +// True if any of m_patternToBlockMap block jumps to one of the blocks within m_patternToBlockMap +// False otherwise. +// +bool OptRangePatternDsc::optJumpsToPatternBlock() +{ + PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; + const unsigned int maxPatternBbNum = m_optFirstBB->bbNum + optGetPatternCount() - 1; + + for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) + { + BasicBlock* sourceBlock; + if (patternToBlockMap->Lookup(patternVal, &sourceBlock)) + { + BasicBlock* destBlock = sourceBlock->bbJumpDest; + assert(destBlock != nullptr); + + if (destBlock->bbNum >= m_optFirstBB->bbNum && destBlock->bbNum <= maxPatternBbNum) + { + return true; + } + } + } + + return false; +} + +//----------------------------------------------------------------------------- +// optMakeSwitchDesc: Make a Switch descriptor with a switch jump table +// +// Returns: +// true if the switch descriptor is created successfully with the switch jump tables. False otherwise. +// +// Notes: +// It creates switch descriptor with a jump table for each range pattern and default case. +// If the value is part of the reserved patterns, the switch jump table is set to the jump target +// for true case +// If GT_EQ, jump target is block's bbJumpDest. If GT_NE, jump target is block's bbNext. +// Otherwise, the switch jump table is set to the default jump target. +// No tree is updated in this method. +// +bool OptRangePatternDsc::optMakeSwitchDesc() +{ +#ifdef DEBUG + if (m_comp->verbose) + { + printf("\n*************** In optMakeSwitchDesc()\n"); + } +#endif // DEBUG + + if (optGetPatternCount() > m_rangePattern || m_rangePattern < 2 || optGetPatternCount() > m_rangePattern || + m_rangePattern > m_sizePatterns) + { + return false; + } + + BasicBlock* prevBb = nullptr; + int patternIndex = 0; + + BasicBlockFlags bbFlags = BBF_EMPTY; + unsigned curBBoffs = 0; + unsigned nxtBBoffs = 0; + unsigned jmpCnt = 0; // # of switch cases (including default case) + + BasicBlock** jmpTab = nullptr; + BasicBlock** jmpPtr = nullptr; + unsigned uniqueTargetCnt = 0; // # of unique jump targets + bool tailCall = false; + ssize_t minVal = m_minPattern; + + // Allocate the jump table + jmpCnt = m_rangePattern + 1; + jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt]; + + // Make a jump table for the range of patterns + // If reserved pattern, jump to jump target of source block. If not, jump to default target. + for (int idxRng = 0; idxRng < m_rangePattern; idxRng++) + { + // Find a mapped block from a pattern + ssize_t key = minVal + idxRng; + BasicBlock* mappedBlock = nullptr; + if (m_patternToBlockMap.Lookup(key, &mappedBlock)) // A mapped block is found + { + BasicBlock* jumpTargetBb = optGetJumpTargetBB(mappedBlock); + *jmpPtr = (BasicBlock*)(size_t)(jumpTargetBb->bbCodeOffs); + *(jmpPtr) = jumpTargetBb; + // Update the unique target basic block to count map + if (!m_blockToIntMap.Lookup(jumpTargetBb, nullptr)) + { + m_blockToIntMap.Set(jumpTargetBb, 1); + } + } + else + { + BasicBlock* defaultJmpBb = m_defaultJmpBB; + *jmpPtr = (BasicBlock*)(size_t)(defaultJmpBb->bbCodeOffs); + *(jmpPtr) = defaultJmpBb; + // Update the unique target basic block to count map + if (!m_blockToIntMap.Lookup(defaultJmpBb, nullptr)) + { + m_blockToIntMap.Set(defaultJmpBb, 1); + } + } + jmpPtr++; + } + + // Append the default label to the jump table + *jmpPtr = (BasicBlock*)(size_t)(m_defaultJmpBB->bbCodeOffs); + *(jmpPtr) = m_defaultJmpBB; + // Update the unique target basic block to count map + if (!m_blockToIntMap.Lookup(m_defaultJmpBB, nullptr)) + { + m_blockToIntMap.Set(m_defaultJmpBB, 1); + } + jmpPtr++; + + // Make sure we found the right number of labels + noway_assert(jmpPtr == jmpTab + jmpCnt); + + // + // Check if it is profitable to use a switch instead of a series of conditional branches + // + + // If the number of unique target counts is 1, it has only default case. Not profitable. + // If it is >= 3, it it not converted to a bit test in Lowering. So, skip it. + uniqueTargetCnt = m_blockToIntMap.GetCount(); + if (uniqueTargetCnt != 2) + { + return false; + } + + noway_assert(jmpCnt >= 2); + + // If all jumps to the same target block, BBJ_NONE or BBJ_ALWAYS is better. + BasicBlock* uniqueSucc = nullptr; + if (uniqueTargetCnt == 2) + { + uniqueSucc = jmpTab[0]; + noway_assert(jmpCnt >= 2); + for (unsigned i = 1; i < jmpCnt - 1; i++) + { + if (jmpTab[i] != uniqueSucc) + { + uniqueSucc = nullptr; + break; + } + } + } + if (uniqueSucc != nullptr) + { + return false; + } + + // Check if it is better to use Jmp instead of Switch + BasicBlock* defaultBB = jmpTab[jmpCnt - 1]; + BasicBlock* followingBB = m_optLastBB->bbNext; + + // Is the number of cases right for a jump switch? + const bool firstCaseFollows = (followingBB == jmpTab[0]); + const bool defaultFollows = (followingBB == defaultBB); + + unsigned minSwitchTabJumpCnt = 2; // table is better than just 2 cmp/jcc + + // This means really just a single cmp/jcc (aka a simple if/else) + if (firstCaseFollows || defaultFollows) + { + minSwitchTabJumpCnt++; + } + +#if defined(TARGET_ARM) + // On ARM for small switch tables we will + // generate a sequence of compare and branch instructions + // because the code to load the base of the switch + // table is huge and hideous due to the relocation... :( + minSwitchTabJumpCnt += 2; +#endif // TARGET_ARM + + bool useJumpSequence = jmpCnt < minSwitchTabJumpCnt; + + if (TargetOS::IsUnix && TargetArchitecture::IsArm32) + { + useJumpSequence = useJumpSequence || m_comp->IsTargetAbi(CORINFO_NATIVEAOT_ABI); + } + + if (useJumpSequence) // It is better to use a series of compare and branch IR trees. + { + return false; + } + + // One of the case blocks has to follow the switch block. All the pattern blocks except for the first pattern block + // will be removed from Switch conversion. So, we need to check if jump target follows the last pattern block. + BasicBlock* bbCase0 = nullptr; + BasicBlock* bbCase1 = jmpTab[0]; + BasicBlock* nextBbAfterSwitch = m_optLastBB->bbNext; + + for (unsigned tabIdx = 1; tabIdx < (jmpCnt - 1); tabIdx++) + { + if (jmpTab[tabIdx] != bbCase1 && bbCase0 == nullptr) + { + bbCase0 = jmpTab[tabIdx]; + break; + } + } + if ((nextBbAfterSwitch != bbCase0) && (nextBbAfterSwitch != bbCase1)) + { + return false; + } + + // If the next basic block after Switch is an empty block with an unconditional jump, skip it. + if (nextBbAfterSwitch->isEmpty() && (nextBbAfterSwitch->bbJumpKind == BBJ_ALWAYS) && + (nextBbAfterSwitch != nextBbAfterSwitch->bbJumpDest)) + { +#ifdef DEBUG + if (m_comp->verbose) + { + printf("\nSkip converting to Switch block if Switch jumps to an empty block with an unconditional " + "jump (" FMT_BB " -> " FMT_BB ")\n", + m_optFirstBB->bbNum, nextBbAfterSwitch->bbNum); + } +#endif // DEBUG + return false; + } + + // Allocate the switch descriptor + swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; + + // Fill in the remaining fields of the switch descriptor + swtDsc->bbsCount = jmpCnt; + swtDsc->bbsDstTab = jmpTab; + + m_comp->fgHasSwitch = true; + + if (m_comp->opts.compProcedureSplitting) + { + m_comp->opts.compProcedureSplitting = false; + JITDUMP("Turning off procedure splitting for this method, as it might need switch tables; " + "implementation limitation.\n"); + } + + tailCall = false; + +#ifdef DEBUG + if (m_comp->verbose) + { + // Print bbNum of each jmpTab + printf("------Switch " FMT_BB " jumps to: ", m_optFirstBB->bbNum); + for (unsigned i = 0; i < swtDsc->bbsCount; i++) + { + printf("%c" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); + } + printf("\n\n"); + } +#endif // DEBUG + + return true; +} + +//----------------------------------------------------------------------------- +// optChangeToSwitch: Change the first pattern block to SWITCH block and remove other pattern blocks +// +// Return Value: +// true if the blocks were successfully updated, false otherwise. +// +// Notes: +// Leave Switch basic block only and remove all other blocks in the range pattern. +// Update reference count of jump target blocks. +// If the CNS_INT node does not have a min pattern, replace it with the CNS_INT node with the min value. +// +// Tree before the optimization: +// ``` +// JTRUE +// \EQ +// \LCL_VAR +// \CNS_INT +// ``` +// +// Tree after the optimization: +// ``` +// SWITCH +// \SUB +// \LCL_VAR +// \CNS_INT (min pattern) +// ``` +bool OptRangePatternDsc::optChangeToSwitch() +{ +#ifdef DEBUG + if (m_comp->verbose) + { + printf("\n*************** In optChangeToSwitch()\n"); + } +#endif // DEBUG + + assert(m_optFirstBB != nullptr && m_optFirstBB->KindIs(BBJ_COND) && m_optFirstBB->bbNext != nullptr); + assert(optGetPatternCount() <= m_rangePattern && m_rangePattern >= 2 && optGetPatternCount() <= m_rangePattern && + m_rangePattern <= m_sizePatterns); + + // Make Switch descriptor with a jump table + if (!optMakeSwitchDesc()) + { + return false; + } + + bool updatedBlocks = false; + switchBBdesc = m_optFirstBB; + int patternCount = optGetPatternCount(); + + // Update the Switch basic block + + // Change `JTRUE` basic block to `Switch` basic block + switchBBdesc = m_optFirstBB; + + // Change BBJ_COND to BBJ_SWITCH + switchBBdesc->bbJumpKind = BBJ_SWITCH; + switchBBdesc->bbJumpDest = nullptr; + + switchBBdesc->bbCodeOffs = m_bbCodeOffs; + switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; + switchBBdesc->bbJumpSwt = swtDsc; + GenTree* rootTree = switchBBdesc->lastStmt()->GetRootNode(); + assert(rootTree->OperIs(GT_JTRUE)); + assert(!(rootTree->gtFlags & GTF_SIDE_EFFECT)); // JTRUE node should not have side effects + + // Change from GT_JTRUE to GT_SWITCH + rootTree->ChangeOper(GT_SWITCH); + + // Special args to fgAddRefPred + FlowEdge* const oldEdge = nullptr; + + // Remove non-switch pattern blocks. Update the reference count of jump target block from the removed block. + BasicBlock* currBb = switchBBdesc->bbNext; + for (int idxPattern = 1; idxPattern < patternCount && currBb != nullptr; idxPattern++) + { + assert(!(currBb->bbFlags & BBF_DONT_REMOVE)); + + BasicBlock* prevBlock = currBb->bbPrev; + BasicBlock* nextBlock = currBb->bbNext; + BasicBlock* jumpBlock = optGetJumpTargetBB(currBb); + + // Unlink the current block and its pred block + + // Check if prevBlock is the predecessor of currBb + assert(currBb != nullptr && prevBlock != nullptr); + FlowEdge** ptrToPred; + assert(m_comp->fgGetPredForBlock(currBb, prevBlock, &ptrToPred) != nullptr); + + m_comp->fgRemoveRefPred(currBb, prevBlock); + + // Link Switch block and current block's jump target block + m_comp->fgAddRefPred(jumpBlock, switchBBdesc, oldEdge); + + // Link Switch block and the next block: + // if GT_EQ and currBb is the last pattern block, skip because bbNext is already linked as default jump + // target if GT_NE, it is already linked when linking its jump target block to Switch block + if (currBb->lastStmt()->GetRootNode()->gtGetOp1()->OperIs(GT_EQ) && idxPattern != (patternCount - 1)) + { + m_comp->fgAddRefPred(nextBlock, switchBBdesc, oldEdge); + } + + m_comp->fgRemoveBlock(currBb, /* unreachable */ true); + + updatedBlocks = true; + currBb = nextBlock; + } + + // Update the reference count of the default jump block + int numNotFound = m_rangePattern - patternCount + 1; // +1 for switch default case + for (int idxFalse = 0; idxFalse < numNotFound; idxFalse++) + { + m_comp->fgAddRefPred(m_defaultJmpBB, switchBBdesc, oldEdge); + } + + // Continue to transform Switch node + + Statement* stmt = switchBBdesc->lastStmt(); + + // Change from GT_EQ or GT_NE to GT_SUB + // tree: SUB + // op1: LCL_VAR + // op2: GT_CNS_INT or GT_CNS_LNG + GenTree* tree = rootTree->gtGetOp1(); // GT_EQ or GT_NE node to chnage to GT_SUB + tree->ChangeOper(GT_SUB); + assert(tree->gtGetOp1() != nullptr && tree->gtGetOp1()->OperIs(GT_LCL_VAR)); + + // Change constant node if siwtch tree does not have the mininum pattern + if (tree->gtGetOp2() != nullptr && tree->gtGetOp2()->AsIntCon()->IconValue() != m_minPattern) + { + GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT or GT_CNS_LNG node + tree->AsOp()->gtOp2 = m_minOp; + + m_comp->gtSetStmtInfo(stmt); + m_comp->fgSetStmtSeq(stmt); + + DEBUG_DESTROY_NODE(op2); + } + + m_comp->gtUpdateStmtSideEffects(stmt); + + return updatedBlocks; +} + +//----------------------------------------------------------------------------- +// optSwitchRecognition: Optimize range check for if (A || B || C || D) pattern and convert it to Switch block +// +// Returns: +// MODIFIED_NOTHING if no optimization is performed. +// MODIFIED_EVERYTHING otherwise. +// +// Notes: +// Detect if (a == val1 || a == val2 || a == val3 || ...) pattern and change it to switch tree +// to reduce compares and jumps, and perform bit operation instead in Lowering phase. +// This optimization is performed only for integer types. +// Run this phase after "Optimize Layout" phase to avoid Switch block being reordered and separated from its bbNext +// target block. If they are not adjacent, Lowering does not convert them to a bit test. Limit the Switch +// conversion to only for 3 or more conditional patterns to reduce code size regressions. +// +PhaseStatus Compiler::optSwitchRecognition() +{ +#ifdef DEBUG + if (this->verbose) + { + printf("\n*************** In optSwitchRecognition()\n"); + } +#endif // DEBUG + + OptRangePatternDsc optRngPattern(this); + + bool printed = false; + int patternIndex = 0; // The index of the pattern in the array + bool foundPattern = false; + unsigned int firstPatternBBNum = 0; // Basic block number of the first pattern found + BasicBlock* prevBb = fgFirstBB; + + if (fgFirstBB->bbNext == nullptr) + { + return PhaseStatus::MODIFIED_NOTHING; + } + + for (BasicBlock* currBb = fgFirstBB->bbNext; currBb != nullptr; currBb = currBb->bbNext) + { + if (currBb->KindIs(BBJ_COND) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) + { +#ifdef DEBUG + if (this->verbose) + { + if (!printed) + { + fgDispBasicBlocks(true); + printed = true; + } + } +#endif // DEBUG + + // Check if prevBb is the predecessor of currBb and currBb has only one predecessor + FlowEdge** ptrToPred; + FlowEdge* pred = fgGetPredForBlock(currBb, prevBb, &ptrToPred); + if (pred == nullptr || currBb->bbRefs != 1 || currBb->bbFlags & BBF_DONT_REMOVE) + { + if (foundPattern) + break; // Stop searching if patterns are not from consecutive basic blocks + else + continue; + } + + // Basic block must have only one statement + if (currBb->lastStmt() == currBb->firstStmt() && prevBb->lastStmt() == prevBb->firstStmt()) + { + // Skip if there is any side effect + assert(currBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); // JTRUE node + if (currBb->lastStmt()->GetRootNode()->gtFlags & GTF_SIDE_EFFECT) + { + // Stop searching if patterns are not from consecutive basic blocks + if (foundPattern) + break; + else + continue; + } + + auto currCmpOp = currBb->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node + auto prevCmpOp = prevBb->lastStmt()->GetRootNode()->gtGetOp1(); + assert(currCmpOp != nullptr && prevCmpOp != nullptr); + + // Compare operator is GT_EQ. If it is GT_NE, it is the end of the pattern check. + if ((prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_EQ)) || + (prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_NE))) + { + if (prevBb->bbJumpDest == currBb) + { + if (foundPattern) + break; + else + continue; + } + + // Check both conditions to have constant on the right side (optimize GT_CNS_INT only) + if (currCmpOp->gtGetOp2()->IsCnsIntOrI() && prevCmpOp->gtGetOp2()->IsCnsIntOrI()) + { + // Check both conditions to have the same local variable number + if (prevCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && currCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && + prevCmpOp->gtGetOp1()->AsLclVar()->GetLclNum() == + currCmpOp->gtGetOp1()->AsLclVar()->GetLclNum()) + { +#ifdef DEBUG + if (this->verbose) + { + printf("\nFound pattern (Prev vs Curr):\n"); + gtDispTree(prevCmpOp); + printf("\n"); + gtDispTree(currCmpOp); + printf("\n\n"); + } +#endif // DEBUG + + // No optimization if the number of patterns is greater than 64. + if (patternIndex >= optRngPattern.m_sizePatterns) + { +#ifdef DEBUG + if (this->verbose) + { + printf("Too many patterns found (> 64), no optimization done.\n"); + } +#endif // DEBUG + return PhaseStatus::MODIFIED_NOTHING; + } + + // First pattern found + if (!foundPattern) + { + assert(patternIndex == 0 && prevCmpOp->OperIs(GT_EQ)); + // If the first pattern block is rarely run, skip it. + if (prevBb->isRunRarely()) + { +#ifdef DEBUG + if (this->verbose) + { + printf(FMT_BB " is run rarely. Skip optimizing this block.\n", prevBb->bbNum); + } +#endif // DEBUG + prevBb = currBb; + continue; + } + ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntCon()->IconValue(); + + // Initialize the pattern range + // + assert(prevBb != nullptr && prevBb->bbJumpKind == BBJ_COND && + prevBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); + + optRngPattern.m_optFirstBB = prevBb; + optRngPattern.m_optLastBB = prevBb; + + // Initialize min pattern and max pattern to the first pattern value + optRngPattern.m_minPattern = firstPatternVal; + optRngPattern.m_maxPattern = firstPatternVal; + + // Initialize the code offset range from the first block + assert(prevBb->bbCodeOffs >= 0 && prevBb->bbCodeOffs <= UINT_MAX); + if (optRngPattern.optGetPatternCount() == 0) + { + optRngPattern.m_bbCodeOffs = prevBb->bbCodeOffs; + } + + // Save the first pattern + if (!optRngPattern.optSetPattern(patternIndex, firstPatternVal, prevBb)) + { + break; + } + + firstPatternBBNum = prevBb->bbNum; + patternIndex++; + } + + // Current pattern + + // Save the pattern and Switch default jump target for the pattern (false case) + ssize_t currentPatternVal = currCmpOp->gtGetOp2()->AsIntCon()->IconValue(); + if (!optRngPattern.optSetPattern(patternIndex, currentPatternVal, currBb)) + { + break; + } + + patternIndex++; + foundPattern = true; + + // Stop searching if the current BB is GT_NE. It is the last pattern. + if (currCmpOp->OperIs(GT_NE)) + { + break; + } + } + } + } + } + } + + // Optimize only when patterns are found in consecutive BBs. + // Stop searching if patterns have been found in previous BBs, but the current BB does not have a pattern + if (foundPattern && patternIndex < (int)(currBb->bbNum - firstPatternBBNum + 1)) + { + break; + } + + prevBb = currBb; + } + + if (foundPattern) + { + int patternCount = optRngPattern.optGetPatternCount(); + // If there are less than 3 patterns, no optimization is done. It is not profitable. + if (patternCount <= 2 || patternCount > optRngPattern.m_sizePatterns) + { + return PhaseStatus::MODIFIED_NOTHING; + } + +#ifdef DEBUG + if (verbose) + { + printf("Reserved patterns:\n"); + optRngPattern.optPrintPatterns(); + } +#endif // DEBUG + + // Check if blocks jump to any of the pattern block + if (optRngPattern.optJumpsToPatternBlock()) + { +#ifdef DEBUG + if (verbose) + { + printf("A pattern block jumps to another pattern block, no optimization done.\n"); + } +#endif // DEBUG + + return PhaseStatus::MODIFIED_NOTHING; + } + + // Find range of pattern values + ssize_t minPattern = optRngPattern.m_minPattern; + ssize_t maxPattern = optRngPattern.m_maxPattern; + int rangePattern = (int)(maxPattern - minPattern + 1); + if (patternCount > rangePattern || rangePattern < 2 || rangePattern > optRngPattern.m_sizePatterns) + { +#ifdef DEBUG + if (verbose) + { + printf("Range of pattern values is too small (< 2) or too big (> %d): %d\n", + optRngPattern.m_sizePatterns, rangePattern); + } +#endif // DEBUG + + return PhaseStatus::MODIFIED_NOTHING; + } + assert(rangePattern >= 0 && rangePattern <= optRngPattern.m_sizePatterns); + optRngPattern.m_rangePattern = rangePattern; + +#ifdef DEBUG + if (verbose) + { +#ifdef TARGET_64BIT + printf("Min Max Range: %lld, %lld, %d\n", minPattern, maxPattern, rangePattern); +#else // !TARGET_64BIT + printf("Min Max Range Bitmap: %d, %d, %d\n", minPattern, maxPattern, rangePattern); +#endif // !TARGET_64BIT + } +#endif // DEBUG + + // Replace "JTRUE" block with a "Switch" block and remove other pattern blocks + if (optRngPattern.optChangeToSwitch()) + { +#ifdef DEBUG + if (verbose) + { + fgDispBasicBlocks(true); + } +#endif // DEBUG + + return PhaseStatus::MODIFIED_EVERYTHING; + } + } + + return PhaseStatus::MODIFIED_NOTHING; +} From eb3f68e4948e409f35f4a6a8be9067e5166eaeca Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Tue, 18 Jul 2023 12:06:32 -0700 Subject: [PATCH 15/35] Switch conversion build error fix. --- src/coreclr/jit/optimizebools.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index fbd82bb6b86a00..63a0656c96b550 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -2109,7 +2109,8 @@ bool OptRangePatternDsc::optChangeToSwitch() // Check if prevBlock is the predecessor of currBb assert(currBb != nullptr && prevBlock != nullptr); FlowEdge** ptrToPred; - assert(m_comp->fgGetPredForBlock(currBb, prevBlock, &ptrToPred) != nullptr); + FlowEdge* pred = m_comp->fgGetPredForBlock(currBb, prevBlock, &ptrToPred); + assert(pred != nullptr); m_comp->fgRemoveRefPred(currBb, prevBlock); From f2db6fc87d9c6d546ae79e4a05e64bdaeb2eb5f1 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Tue, 18 Jul 2023 12:16:10 -0700 Subject: [PATCH 16/35] Switch conversion optimization: build error fix for an unused variable. --- src/coreclr/jit/optimizebools.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 63a0656c96b550..50b1a625aa3c68 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -2108,8 +2108,7 @@ bool OptRangePatternDsc::optChangeToSwitch() // Check if prevBlock is the predecessor of currBb assert(currBb != nullptr && prevBlock != nullptr); - FlowEdge** ptrToPred; - FlowEdge* pred = m_comp->fgGetPredForBlock(currBb, prevBlock, &ptrToPred); + FlowEdge* pred = m_comp->fgGetPredForBlock(currBb, prevBlock); assert(pred != nullptr); m_comp->fgRemoveRefPred(currBb, prevBlock); @@ -2220,8 +2219,7 @@ PhaseStatus Compiler::optSwitchRecognition() #endif // DEBUG // Check if prevBb is the predecessor of currBb and currBb has only one predecessor - FlowEdge** ptrToPred; - FlowEdge* pred = fgGetPredForBlock(currBb, prevBb, &ptrToPred); + FlowEdge* pred = fgGetPredForBlock(currBb, prevBb); if (pred == nullptr || currBb->bbRefs != 1 || currBb->bbFlags & BBF_DONT_REMOVE) { if (foundPattern) From 55b86e1380aa03dbd186d5e46d4cfb55764a1ea2 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Tue, 18 Jul 2023 14:00:50 -0700 Subject: [PATCH 17/35] Switch conversion optimization: fortmat fix. --- src/coreclr/jit/optimizebools.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 50b1a625aa3c68..40bbffa2c46342 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -2108,7 +2108,7 @@ bool OptRangePatternDsc::optChangeToSwitch() // Check if prevBlock is the predecessor of currBb assert(currBb != nullptr && prevBlock != nullptr); - FlowEdge* pred = m_comp->fgGetPredForBlock(currBb, prevBlock); + FlowEdge* pred = m_comp->fgGetPredForBlock(currBb, prevBlock); assert(pred != nullptr); m_comp->fgRemoveRefPred(currBb, prevBlock); @@ -2219,7 +2219,7 @@ PhaseStatus Compiler::optSwitchRecognition() #endif // DEBUG // Check if prevBb is the predecessor of currBb and currBb has only one predecessor - FlowEdge* pred = fgGetPredForBlock(currBb, prevBb); + FlowEdge* pred = fgGetPredForBlock(currBb, prevBb); if (pred == nullptr || currBb->bbRefs != 1 || currBb->bbFlags & BBF_DONT_REMOVE) { if (foundPattern) From 8f5cb4a3009b7c6a5a31103c5a6d83e260f7f805 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Fri, 21 Jul 2023 11:56:52 -0700 Subject: [PATCH 18/35] Switch recognition: change phase text name. --- src/coreclr/jit/compphases.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h index 5320d37eba070a..0c081e4d158a21 100644 --- a/src/coreclr/jit/compphases.h +++ b/src/coreclr/jit/compphases.h @@ -72,7 +72,7 @@ CompPhaseNameMacro(PHASE_MORPH_MDARR, "Morph array ops", CompPhaseNameMacro(PHASE_HOIST_LOOP_CODE, "Hoist loop code", false, -1, false) CompPhaseNameMacro(PHASE_MARK_LOCAL_VARS, "Mark local vars", false, -1, false) CompPhaseNameMacro(PHASE_OPTIMIZE_BOOLS, "Optimize bools", false, -1, false) -CompPhaseNameMacro(PHASE_SWITCH_RECOGNITION, "Find Specific pattern", false, -1, false) +CompPhaseNameMacro(PHASE_SWITCH_RECOGNITION, "Recognize Switch", false, -1, false) CompPhaseNameMacro(PHASE_FIND_OPER_ORDER, "Find oper order", false, -1, false) CompPhaseNameMacro(PHASE_SET_BLOCK_ORDER, "Set block order", false, -1, true) CompPhaseNameMacro(PHASE_BUILD_SSA, "Build SSA representation", true, -1, false) From 459137ad5e1f781d001554b8b627c63f6cf71ba4 Mon Sep 17 00:00:00 2001 From: Julie Lee Date: Wed, 2 Aug 2023 15:44:23 -0700 Subject: [PATCH 19/35] Add test cases for Switch recognition. --- .../optSwitchRecognition.cs | 173 ++++++++++++++++++ .../optSwitchRecognition.csproj | 9 + 2 files changed, 182 insertions(+) create mode 100644 src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.cs create mode 100644 src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.csproj diff --git a/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.cs b/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.cs new file mode 100644 index 00000000000000..58000d7b02d41a --- /dev/null +++ b/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.cs @@ -0,0 +1,173 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// unit test for Switch recognition optimization + +using System; +using System.Runtime.CompilerServices; +using Xunit; + +public class CSwitchRecognitionTest +{ + // Test sorted char cases + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool RecSwitchSortedChar(char c) + { + return (c == 'a' || c == 'b' || c == 'd' || c == 'f'); + } + + [Theory] + [InlineData('a', true)] + [InlineData('b', true)] + [InlineData('c', false)] + [InlineData('d', true)] + [InlineData('e', false)] + [InlineData('f', true)] + [InlineData('z', false)] + [InlineData('A', false)] + [InlineData('Z', false)] + [InlineData('?', false)] + public static void TestRecSwitchSortedChar(char arg1,bool expected) => Assert.Equal(RecSwitchSortedChar(arg1), expected); + + // Test unsorted char cases + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool RecSwitchUnsortedChar(char c) + { + return (c == 'd' || c == 'f' || c == 'a' || c == 'b'); + } + + [Theory] + [InlineData('a', true)] + [InlineData('b', true)] + [InlineData('c', false)] + [InlineData('d', true)] + [InlineData('e', false)] + [InlineData('f', true)] + [InlineData('z', false)] + [InlineData('A', false)] + [InlineData('Z', false)] + [InlineData('?', false)] + public static void TestRecSwitchUnsortedChar(char arg1, bool expected) => Assert.Equal(RecSwitchUnsortedChar(arg1), expected); + + // Test sorted int cases + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool RecSwitchSortedInt(int i) + { + return (i == -10 || i == -20 || i == 30 || i == 40); + } + + [Theory] + [InlineData(-100, false)] + [InlineData(-10, true)] + [InlineData(-20, true)] + [InlineData(0, false)] + [InlineData(30, true)] + [InlineData(35, false)] + [InlineData(40, true)] + [InlineData(70, false)] + [InlineData(100, false)] + public static void TestRecSwitchSortedInt(int arg1, bool expected) => Assert.Equal(RecSwitchSortedInt(arg1), expected); + + // Test unsorted int cases + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool RecSwitchUnsortedInt(int i) + { + return (i == 30 || i == 40 || i == -10 || i == -20); + } + + [Theory] + [InlineData(-100, false)] + [InlineData(-10, true)] + [InlineData(-20, true)] + [InlineData(0, false)] + [InlineData(30, true)] + [InlineData(35, false)] + [InlineData(40, true)] + [InlineData(70, false)] + [InlineData(100, false)] + public static void TestRecSwitchUnsortedInt(int arg1, bool expected) => Assert.Equal(RecSwitchUnsortedInt(arg1), expected); + + // Test min limits + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool RecSwitchMinLimits(int i) + { + return (i == -9223372036854775807 || i == -9223372036854775806 + || i == -9223372036854775804 || i == -9223372036854775802); + } + + [Theory] + [InlineData(-9223372036854775807, true)] + [InlineData(-9223372036854775806, true)] + [InlineData(-9223372036854775805, false)] + [InlineData(-9223372036854775804, true)] + [InlineData(-9223372036854775802, true)] + public static void TestRecSwitchMinLimits(int arg1, bool expected) => Assert.Equal(RecSwitchMinLimits(arg1), expected); + + // Test max limits + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool RecSwitchMaxLimits(int i) + { + return (i == 9223372036854775807 || i == 9223372036854775806 + || i == 9223372036854775804 || i == 9223372036854775802); + } + + [Theory] + [InlineData(9223372036854775807, true)] + [InlineData(9223372036854775806, true)] + [InlineData(9223372036854775805, false)] + [InlineData(9223372036854775804, true)] + [InlineData(9223372036854775802, true)] + public static void TestRecSwitchMaxLimits(int arg1, bool expected) => Assert.Equal(RecSwitchMaxLimits(arg1), expected); + + // Test <= 64 switch cases + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool RecSwitch64JumpTables(int i) + { + return (i == 0 || i == 4 || i == 6 || i == 63); + } + + [Theory] + [InlineData(-63, false)] + [InlineData(60, false)] + [InlineData(63, true)] + [InlineData(64, false)] + public static void TestRecSwitch64JumpTables(int arg1, bool expected) => Assert.Equal(RecSwitch64JumpTables(arg1), expected); + + // + // Skip optimization + // + + // Test > 64 Switch cases (should skip Switch Recognition optimization) + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool RecSwitch128JumpTables(int i) + { + return (i == 0 || i == 4 || i == 6 || i == 127); + } + + [Theory] + [InlineData(-127, false)] + [InlineData(6, true)] + [InlineData(127, true)] + [InlineData(128, false)] + public static void TestRecSwitch128JumpTables(int arg1, bool expected) => Assert.Equal(RecSwitch128JumpTables(arg1), expected); + + // Skips `bit test` conversion because Switch jump targets are > 2 (should skip Switch Recognition optimization) + [MethodImpl(MethodImplOptions.NoInlining)] + public static int RecSwitchSkipBitTest(int arch) + { + if (arch == 1) + return 2; + else if (arch == 2 || arch == 6) + return 4; + else + return 1; + } + + [Theory] + [InlineData(0, 1)] + [InlineData(1, 2)] + [InlineData(2, 4)] + [InlineData(6, 4)] + [InlineData(10, 1)] + public static void TestRecSwitchSkipBitTest(int arg1, int expected) => Assert.Equal(RecSwitchSkipBitTest(arg1), expected); +} diff --git a/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.csproj b/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.csproj new file mode 100644 index 00000000000000..b47c3e8e8d9f55 --- /dev/null +++ b/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.csproj @@ -0,0 +1,9 @@ + + + PdbOnly + True + + + + + From 0b4e65230e376adf577e09d2174f1f9840549b22 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 6 Sep 2023 13:44:10 +0200 Subject: [PATCH 20/35] Move to a separate file --- src/coreclr/jit/CMakeLists.txt | 1 + src/coreclr/jit/optimizebools.cpp | 875 ------------------------- src/coreclr/jit/switchrecognition.cpp | 882 ++++++++++++++++++++++++++ 3 files changed, 883 insertions(+), 875 deletions(-) create mode 100644 src/coreclr/jit/switchrecognition.cpp diff --git a/src/coreclr/jit/CMakeLists.txt b/src/coreclr/jit/CMakeLists.txt index d00b5b27fe59a8..1981a3842cac79 100644 --- a/src/coreclr/jit/CMakeLists.txt +++ b/src/coreclr/jit/CMakeLists.txt @@ -152,6 +152,7 @@ set( JIT_SOURCES objectalloc.cpp optcse.cpp optimizebools.cpp + switchrecognition.cpp optimizer.cpp patchpoint.cpp phase.cpp diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index bfb2a69217d3fc..2efbf40b6d5357 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1557,878 +1557,3 @@ PhaseStatus Compiler::optOptimizeBools() const bool modified = stress || ((numCond + numReturn) > 0); return modified ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; } - -//----------------------------------------------------------------------------- -// OptRangePatternDsc: Descriptor used for `if` range pattern optimization -// -class OptRangePatternDsc -{ -public: - static const int m_sizePatterns = 64; // The size of the patterns array - -private: - Compiler* m_comp = nullptr; // The pointer to the Compiler instance - CompAllocator m_allocator; // The memory allocator - typedef JitHashTable, BasicBlock*> PatternToBlockMap; - PatternToBlockMap m_patternToBlockMap; - - CompAllocator m_allocatorTarget; // The memory allocator - typedef JitHashTable, int> BlockToIntMap; - BlockToIntMap m_blockToIntMap; - - BasicBlock* switchBBdesc = nullptr; // The switch basic block descriptor - BBswtDesc* swtDsc = nullptr; // The switch descriptor - GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern - BasicBlock* m_defaultJmpBB = nullptr; // The Switch default jump target - BasicBlock* optGetJumpTargetBB(BasicBlock* block); - bool optMakeSwitchDesc(); - -public: - OptRangePatternDsc(Compiler* comp) - : m_comp(comp) - , m_allocator(comp->getAllocator(CMK_Generic)) - , m_patternToBlockMap(m_allocator) - , m_allocatorTarget(comp->getAllocator(CMK_Generic)) - , m_blockToIntMap(m_allocatorTarget) - { - } - - BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern - BasicBlock* m_optLastBB = nullptr; // The last BB of the range pattern - ssize_t m_minPattern = 0; // The minimum pattern - ssize_t m_maxPattern = 0; // The maximum pattern - int m_rangePattern = 0; // The range of values in patterns[] - unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block - unsigned m_bbCodeOffsEnd = 0; // IL code offset end of the switch basic block - - bool optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block); - int optGetPatternCount(); - void optPrintPatterns(); - bool optJumpsToPatternBlock(); - bool optChangeToSwitch(); -}; - -//----------------------------------------------------------------------------- -// optGetJumpTargetBB: Get jumpTargetBB for the pattern -// -// Arguments: -// block - the basic block to get its jump target. -// -// Return Value: -// The jump target BB for the pattern -// -// Notes: -// If compare operator is GT_EQ, the switch jump target (true case) is bbJumpDest. -// If compare operator is GT_NE,it is bbNext. -// -BasicBlock* OptRangePatternDsc::optGetJumpTargetBB(BasicBlock* block) -{ - assert(block != nullptr && (block->bbJumpKind == BBJ_SWITCH || block->bbJumpKind == BBJ_COND)); - assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); - - GenTree* op1 = block->lastStmt()->GetRootNode()->gtGetOp1(); - auto oper = op1->OperGet(); - assert(oper == GT_EQ || oper == GT_NE); - - if (oper == GT_EQ) - { - return block->bbJumpDest; - } - else if (oper == GT_NE) - { - return block->bbNext; - } - - return nullptr; -} - -//----------------------------------------------------------------------------- -// optSetPattern: Save pattern value and basic block -// -// Arguments: -// idxPattern - the index of the pattern to set -// patternVal - the value of the pattern -// block - the basic block to set its jump target. -// -// Return Value: -// true if the pattern is set, false otherwise. -// -bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block) -{ - if (idxPattern < 0 || idxPattern >= m_sizePatterns) - { -#ifdef DEBUG - if (m_comp->verbose) - { - printf("idxPattern out of range"); - } -#endif // DEBUG - - return false; - } - - // Set the pattern value if it does not already exists in the map - if (m_patternToBlockMap.Lookup(patternVal, nullptr)) // Pattern already exists - { - return false; - } - - m_patternToBlockMap.Set(patternVal, block); - - // Set min pattern - if (patternVal < m_minPattern) - { - m_minPattern = patternVal; - - // Update minOp to the tree with the min pattern - GenTree* minOpNode = block->lastStmt()->GetRootNode()->gtGetOp1()->gtGetOp2(); - assert(minOpNode != nullptr && minOpNode->IsIntegralConst()); - m_minOp = minOpNode; - } - // Set max pattern - if (patternVal > m_maxPattern) - { - m_maxPattern = patternVal; - } - - // Set default jump target of the Switch basic block: - // For GT_EQ, Swithc Default jump target (for false case) is set to bbNext. - // For GT_NE, Swithc Default jump target (for false case) is set to bbJumpDest. - assert(block != nullptr && block->bbJumpKind == BBJ_COND); - assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); - - auto oper = block->lastStmt()->GetRootNode()->gtGetOp1()->OperGet(); - assert(oper == GT_EQ || oper == GT_NE); - if (oper == GT_EQ) - { - m_defaultJmpBB = block->bbNext; - } - else if (oper == GT_NE) - { - m_defaultJmpBB = block->bbJumpDest; - } - - // Update the code offset end range - assert(block->bbCodeOffsEnd >= 0 && block->bbCodeOffsEnd <= UINT_MAX); - m_bbCodeOffsEnd = block->bbCodeOffsEnd; - - // Set the last basic block of the range pattern - m_optLastBB = block; - - return true; -} - -//----------------------------------------------------------------------------- -// optGetPatternCount: Get the number of patterns -// -// Return Value: -// The number of reserved patterns -// -int OptRangePatternDsc::optGetPatternCount() -{ - return m_patternToBlockMap.GetCount(); -} - -//----------------------------------------------------------------------------- -// optPrintPatterns: Prints the patterns from m_patternToBlockMap -// -void OptRangePatternDsc::optPrintPatterns() -{ - // print patterns from m_patternToBlockMap using key iterator - PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; - for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) - { - BasicBlock* mappedBlock; - if (patternToBlockMap->Lookup(patternVal, &mappedBlock)) - { - printf("patternVal = %d, block = %d\n", patternVal, mappedBlock->bbNum); - } - } -} - -//----------------------------------------------------------------------------- -// optJumpsToPatternBlock: Checks if any pattern block jumps to one of the blocks -// within m_patternToBlockMap -// -// Arguments: -// None -// -// Return Value: -// True if any of m_patternToBlockMap block jumps to one of the blocks within m_patternToBlockMap -// False otherwise. -// -bool OptRangePatternDsc::optJumpsToPatternBlock() -{ - PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; - const unsigned int maxPatternBbNum = m_optFirstBB->bbNum + optGetPatternCount() - 1; - - for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) - { - BasicBlock* sourceBlock; - if (patternToBlockMap->Lookup(patternVal, &sourceBlock)) - { - BasicBlock* destBlock = sourceBlock->bbJumpDest; - assert(destBlock != nullptr); - - if (destBlock->bbNum >= m_optFirstBB->bbNum && destBlock->bbNum <= maxPatternBbNum) - { - return true; - } - } - } - - return false; -} - -//----------------------------------------------------------------------------- -// optMakeSwitchDesc: Make a Switch descriptor with a switch jump table -// -// Returns: -// true if the switch descriptor is created successfully with the switch jump tables. False otherwise. -// -// Notes: -// It creates switch descriptor with a jump table for each range pattern and default case. -// If the value is part of the reserved patterns, the switch jump table is set to the jump target -// for true case -// If GT_EQ, jump target is block's bbJumpDest. If GT_NE, jump target is block's bbNext. -// Otherwise, the switch jump table is set to the default jump target. -// No tree is updated in this method. -// -bool OptRangePatternDsc::optMakeSwitchDesc() -{ -#ifdef DEBUG - if (m_comp->verbose) - { - printf("\n*************** In optMakeSwitchDesc()\n"); - } -#endif // DEBUG - - if (optGetPatternCount() > m_rangePattern || m_rangePattern < 2 || optGetPatternCount() > m_rangePattern || - m_rangePattern > m_sizePatterns) - { - return false; - } - - BasicBlock* prevBb = nullptr; - int patternIndex = 0; - - BasicBlockFlags bbFlags = BBF_EMPTY; - unsigned curBBoffs = 0; - unsigned nxtBBoffs = 0; - unsigned jmpCnt = 0; // # of switch cases (including default case) - - BasicBlock** jmpTab = nullptr; - BasicBlock** jmpPtr = nullptr; - unsigned uniqueTargetCnt = 0; // # of unique jump targets - bool tailCall = false; - ssize_t minVal = m_minPattern; - - // Allocate the jump table - jmpCnt = m_rangePattern + 1; - jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt]; - - // Make a jump table for the range of patterns - // If reserved pattern, jump to jump target of source block. If not, jump to default target. - for (int idxRng = 0; idxRng < m_rangePattern; idxRng++) - { - // Find a mapped block from a pattern - ssize_t key = minVal + idxRng; - BasicBlock* mappedBlock = nullptr; - if (m_patternToBlockMap.Lookup(key, &mappedBlock)) // A mapped block is found - { - BasicBlock* jumpTargetBb = optGetJumpTargetBB(mappedBlock); - *jmpPtr = (BasicBlock*)(size_t)(jumpTargetBb->bbCodeOffs); - *(jmpPtr) = jumpTargetBb; - // Update the unique target basic block to count map - if (!m_blockToIntMap.Lookup(jumpTargetBb, nullptr)) - { - m_blockToIntMap.Set(jumpTargetBb, 1); - } - } - else - { - BasicBlock* defaultJmpBb = m_defaultJmpBB; - *jmpPtr = (BasicBlock*)(size_t)(defaultJmpBb->bbCodeOffs); - *(jmpPtr) = defaultJmpBb; - // Update the unique target basic block to count map - if (!m_blockToIntMap.Lookup(defaultJmpBb, nullptr)) - { - m_blockToIntMap.Set(defaultJmpBb, 1); - } - } - jmpPtr++; - } - - // Append the default label to the jump table - *jmpPtr = (BasicBlock*)(size_t)(m_defaultJmpBB->bbCodeOffs); - *(jmpPtr) = m_defaultJmpBB; - // Update the unique target basic block to count map - if (!m_blockToIntMap.Lookup(m_defaultJmpBB, nullptr)) - { - m_blockToIntMap.Set(m_defaultJmpBB, 1); - } - jmpPtr++; - - // Make sure we found the right number of labels - noway_assert(jmpPtr == jmpTab + jmpCnt); - - // - // Check if it is profitable to use a switch instead of a series of conditional branches - // - - // If the number of unique target counts is 1, it has only default case. Not profitable. - // If it is >= 3, it it not converted to a bit test in Lowering. So, skip it. - uniqueTargetCnt = m_blockToIntMap.GetCount(); - if (uniqueTargetCnt != 2) - { - return false; - } - - noway_assert(jmpCnt >= 2); - - // If all jumps to the same target block, BBJ_NONE or BBJ_ALWAYS is better. - BasicBlock* uniqueSucc = nullptr; - if (uniqueTargetCnt == 2) - { - uniqueSucc = jmpTab[0]; - noway_assert(jmpCnt >= 2); - for (unsigned i = 1; i < jmpCnt - 1; i++) - { - if (jmpTab[i] != uniqueSucc) - { - uniqueSucc = nullptr; - break; - } - } - } - if (uniqueSucc != nullptr) - { - return false; - } - - // Check if it is better to use Jmp instead of Switch - BasicBlock* defaultBB = jmpTab[jmpCnt - 1]; - BasicBlock* followingBB = m_optLastBB->bbNext; - - // Is the number of cases right for a jump switch? - const bool firstCaseFollows = (followingBB == jmpTab[0]); - const bool defaultFollows = (followingBB == defaultBB); - - unsigned minSwitchTabJumpCnt = 2; // table is better than just 2 cmp/jcc - - // This means really just a single cmp/jcc (aka a simple if/else) - if (firstCaseFollows || defaultFollows) - { - minSwitchTabJumpCnt++; - } - -#if defined(TARGET_ARM) - // On ARM for small switch tables we will - // generate a sequence of compare and branch instructions - // because the code to load the base of the switch - // table is huge and hideous due to the relocation... :( - minSwitchTabJumpCnt += 2; -#endif // TARGET_ARM - - bool useJumpSequence = jmpCnt < minSwitchTabJumpCnt; - - if (TargetOS::IsUnix && TargetArchitecture::IsArm32) - { - useJumpSequence = useJumpSequence || m_comp->IsTargetAbi(CORINFO_NATIVEAOT_ABI); - } - - if (useJumpSequence) // It is better to use a series of compare and branch IR trees. - { - return false; - } - - // One of the case blocks has to follow the switch block. All the pattern blocks except for the first pattern block - // will be removed from Switch conversion. So, we need to check if jump target follows the last pattern block. - BasicBlock* bbCase0 = nullptr; - BasicBlock* bbCase1 = jmpTab[0]; - BasicBlock* nextBbAfterSwitch = m_optLastBB->bbNext; - - for (unsigned tabIdx = 1; tabIdx < (jmpCnt - 1); tabIdx++) - { - if (jmpTab[tabIdx] != bbCase1 && bbCase0 == nullptr) - { - bbCase0 = jmpTab[tabIdx]; - break; - } - } - if ((nextBbAfterSwitch != bbCase0) && (nextBbAfterSwitch != bbCase1)) - { - return false; - } - - // If the next basic block after Switch is an empty block with an unconditional jump, skip it. - if (nextBbAfterSwitch->isEmpty() && (nextBbAfterSwitch->bbJumpKind == BBJ_ALWAYS) && - (nextBbAfterSwitch != nextBbAfterSwitch->bbJumpDest)) - { -#ifdef DEBUG - if (m_comp->verbose) - { - printf("\nSkip converting to Switch block if Switch jumps to an empty block with an unconditional " - "jump (" FMT_BB " -> " FMT_BB ")\n", - m_optFirstBB->bbNum, nextBbAfterSwitch->bbNum); - } -#endif // DEBUG - return false; - } - - // Allocate the switch descriptor - swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; - - // Fill in the remaining fields of the switch descriptor - swtDsc->bbsCount = jmpCnt; - swtDsc->bbsDstTab = jmpTab; - - m_comp->fgHasSwitch = true; - - if (m_comp->opts.compProcedureSplitting) - { - m_comp->opts.compProcedureSplitting = false; - JITDUMP("Turning off procedure splitting for this method, as it might need switch tables; " - "implementation limitation.\n"); - } - - tailCall = false; - -#ifdef DEBUG - if (m_comp->verbose) - { - // Print bbNum of each jmpTab - printf("------Switch " FMT_BB " jumps to: ", m_optFirstBB->bbNum); - for (unsigned i = 0; i < swtDsc->bbsCount; i++) - { - printf("%c" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); - } - printf("\n\n"); - } -#endif // DEBUG - - return true; -} - -//----------------------------------------------------------------------------- -// optChangeToSwitch: Change the first pattern block to SWITCH block and remove other pattern blocks -// -// Return Value: -// true if the blocks were successfully updated, false otherwise. -// -// Notes: -// Leave Switch basic block only and remove all other blocks in the range pattern. -// Update reference count of jump target blocks. -// If the CNS_INT node does not have a min pattern, replace it with the CNS_INT node with the min value. -// -// Tree before the optimization: -// ``` -// JTRUE -// \EQ -// \LCL_VAR -// \CNS_INT -// ``` -// -// Tree after the optimization: -// ``` -// SWITCH -// \SUB -// \LCL_VAR -// \CNS_INT (min pattern) -// ``` -bool OptRangePatternDsc::optChangeToSwitch() -{ -#ifdef DEBUG - if (m_comp->verbose) - { - printf("\n*************** In optChangeToSwitch()\n"); - } -#endif // DEBUG - - assert(m_optFirstBB != nullptr && m_optFirstBB->KindIs(BBJ_COND) && m_optFirstBB->bbNext != nullptr); - assert(optGetPatternCount() <= m_rangePattern && m_rangePattern >= 2 && optGetPatternCount() <= m_rangePattern && - m_rangePattern <= m_sizePatterns); - - // Make Switch descriptor with a jump table - if (!optMakeSwitchDesc()) - { - return false; - } - - bool updatedBlocks = false; - switchBBdesc = m_optFirstBB; - int patternCount = optGetPatternCount(); - - // Update the Switch basic block - - // Change `JTRUE` basic block to `Switch` basic block - switchBBdesc = m_optFirstBB; - - // Change BBJ_COND to BBJ_SWITCH - switchBBdesc->bbJumpKind = BBJ_SWITCH; - switchBBdesc->bbJumpDest = nullptr; - - switchBBdesc->bbCodeOffs = m_bbCodeOffs; - switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; - switchBBdesc->bbJumpSwt = swtDsc; - GenTree* rootTree = switchBBdesc->lastStmt()->GetRootNode(); - assert(rootTree->OperIs(GT_JTRUE)); - assert(!(rootTree->gtFlags & GTF_SIDE_EFFECT)); // JTRUE node should not have side effects - - // Change from GT_JTRUE to GT_SWITCH - rootTree->ChangeOper(GT_SWITCH); - - // Special args to fgAddRefPred - FlowEdge* const oldEdge = nullptr; - - // Remove non-switch pattern blocks. Update the reference count of jump target block from the removed block. - BasicBlock* currBb = switchBBdesc->bbNext; - for (int idxPattern = 1; idxPattern < patternCount && currBb != nullptr; idxPattern++) - { - assert(!(currBb->bbFlags & BBF_DONT_REMOVE)); - - BasicBlock* prevBlock = currBb->bbPrev; - BasicBlock* nextBlock = currBb->bbNext; - BasicBlock* jumpBlock = optGetJumpTargetBB(currBb); - - // Unlink the current block and its pred block - - // Check if prevBlock is the predecessor of currBb - assert(currBb != nullptr && prevBlock != nullptr); - FlowEdge* pred = m_comp->fgGetPredForBlock(currBb, prevBlock); - assert(pred != nullptr); - - m_comp->fgRemoveRefPred(currBb, prevBlock); - - // Link Switch block and current block's jump target block - m_comp->fgAddRefPred(jumpBlock, switchBBdesc, oldEdge); - - // Link Switch block and the next block: - // if GT_EQ and currBb is the last pattern block, skip because bbNext is already linked as default jump - // target if GT_NE, it is already linked when linking its jump target block to Switch block - if (currBb->lastStmt()->GetRootNode()->gtGetOp1()->OperIs(GT_EQ) && idxPattern != (patternCount - 1)) - { - m_comp->fgAddRefPred(nextBlock, switchBBdesc, oldEdge); - } - - m_comp->fgRemoveBlock(currBb, /* unreachable */ true); - - updatedBlocks = true; - currBb = nextBlock; - } - - // Update the reference count of the default jump block - int numNotFound = m_rangePattern - patternCount + 1; // +1 for switch default case - for (int idxFalse = 0; idxFalse < numNotFound; idxFalse++) - { - m_comp->fgAddRefPred(m_defaultJmpBB, switchBBdesc, oldEdge); - } - - // Continue to transform Switch node - - Statement* stmt = switchBBdesc->lastStmt(); - - // Change from GT_EQ or GT_NE to GT_SUB - // tree: SUB - // op1: LCL_VAR - // op2: GT_CNS_INT or GT_CNS_LNG - GenTree* tree = rootTree->gtGetOp1(); // GT_EQ or GT_NE node to chnage to GT_SUB - tree->ChangeOper(GT_SUB); - assert(tree->gtGetOp1() != nullptr && tree->gtGetOp1()->OperIs(GT_LCL_VAR)); - - // Change constant node if siwtch tree does not have the mininum pattern - if (tree->gtGetOp2() != nullptr && tree->gtGetOp2()->AsIntCon()->IconValue() != m_minPattern) - { - GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT or GT_CNS_LNG node - tree->AsOp()->gtOp2 = m_minOp; - - m_comp->gtSetStmtInfo(stmt); - m_comp->fgSetStmtSeq(stmt); - - DEBUG_DESTROY_NODE(op2); - } - - m_comp->gtUpdateStmtSideEffects(stmt); - - return updatedBlocks; -} - -//----------------------------------------------------------------------------- -// optSwitchRecognition: Optimize range check for if (A || B || C || D) pattern and convert it to Switch block -// -// Returns: -// MODIFIED_NOTHING if no optimization is performed. -// MODIFIED_EVERYTHING otherwise. -// -// Notes: -// Detect if (a == val1 || a == val2 || a == val3 || ...) pattern and change it to switch tree -// to reduce compares and jumps, and perform bit operation instead in Lowering phase. -// This optimization is performed only for integer types. -// Run this phase after "Optimize Layout" phase to avoid Switch block being reordered and separated from its bbNext -// target block. If they are not adjacent, Lowering does not convert them to a bit test. Limit the Switch -// conversion to only for 3 or more conditional patterns to reduce code size regressions. -// -PhaseStatus Compiler::optSwitchRecognition() -{ -#ifdef DEBUG - if (this->verbose) - { - printf("\n*************** In optSwitchRecognition()\n"); - } -#endif // DEBUG - - OptRangePatternDsc optRngPattern(this); - - bool printed = false; - int patternIndex = 0; // The index of the pattern in the array - bool foundPattern = false; - unsigned int firstPatternBBNum = 0; // Basic block number of the first pattern found - BasicBlock* prevBb = fgFirstBB; - - if (fgFirstBB->bbNext == nullptr) - { - return PhaseStatus::MODIFIED_NOTHING; - } - - for (BasicBlock* currBb = fgFirstBB->bbNext; currBb != nullptr; currBb = currBb->bbNext) - { - if (currBb->KindIs(BBJ_COND) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) - { -#ifdef DEBUG - if (this->verbose) - { - if (!printed) - { - fgDispBasicBlocks(true); - printed = true; - } - } -#endif // DEBUG - - // Check if prevBb is the predecessor of currBb and currBb has only one predecessor - FlowEdge* pred = fgGetPredForBlock(currBb, prevBb); - if (pred == nullptr || currBb->bbRefs != 1 || currBb->bbFlags & BBF_DONT_REMOVE) - { - if (foundPattern) - break; // Stop searching if patterns are not from consecutive basic blocks - else - continue; - } - - // Basic block must have only one statement - if (currBb->lastStmt() == currBb->firstStmt() && prevBb->lastStmt() == prevBb->firstStmt()) - { - // Skip if there is any side effect - assert(currBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); // JTRUE node - if (currBb->lastStmt()->GetRootNode()->gtFlags & GTF_SIDE_EFFECT) - { - // Stop searching if patterns are not from consecutive basic blocks - if (foundPattern) - break; - else - continue; - } - - auto currCmpOp = currBb->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node - auto prevCmpOp = prevBb->lastStmt()->GetRootNode()->gtGetOp1(); - assert(currCmpOp != nullptr && prevCmpOp != nullptr); - - // Compare operator is GT_EQ. If it is GT_NE, it is the end of the pattern check. - if ((prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_EQ)) || - (prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_NE))) - { - if (prevBb->bbJumpDest == currBb) - { - if (foundPattern) - break; - else - continue; - } - - // Check both conditions to have constant on the right side (optimize GT_CNS_INT only) - if (currCmpOp->gtGetOp2()->IsCnsIntOrI() && prevCmpOp->gtGetOp2()->IsCnsIntOrI()) - { - // Check both conditions to have the same local variable number - if (prevCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && currCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && - prevCmpOp->gtGetOp1()->AsLclVar()->GetLclNum() == - currCmpOp->gtGetOp1()->AsLclVar()->GetLclNum()) - { -#ifdef DEBUG - if (this->verbose) - { - printf("\nFound pattern (Prev vs Curr):\n"); - gtDispTree(prevCmpOp); - printf("\n"); - gtDispTree(currCmpOp); - printf("\n\n"); - } -#endif // DEBUG - - // No optimization if the number of patterns is greater than 64. - if (patternIndex >= optRngPattern.m_sizePatterns) - { -#ifdef DEBUG - if (this->verbose) - { - printf("Too many patterns found (> 64), no optimization done.\n"); - } -#endif // DEBUG - return PhaseStatus::MODIFIED_NOTHING; - } - - // First pattern found - if (!foundPattern) - { - assert(patternIndex == 0 && prevCmpOp->OperIs(GT_EQ)); - // If the first pattern block is rarely run, skip it. - if (prevBb->isRunRarely()) - { -#ifdef DEBUG - if (this->verbose) - { - printf(FMT_BB " is run rarely. Skip optimizing this block.\n", prevBb->bbNum); - } -#endif // DEBUG - prevBb = currBb; - continue; - } - ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntCon()->IconValue(); - - // Initialize the pattern range - // - assert(prevBb != nullptr && prevBb->bbJumpKind == BBJ_COND && - prevBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); - - optRngPattern.m_optFirstBB = prevBb; - optRngPattern.m_optLastBB = prevBb; - - // Initialize min pattern and max pattern to the first pattern value - optRngPattern.m_minPattern = firstPatternVal; - optRngPattern.m_maxPattern = firstPatternVal; - - // Initialize the code offset range from the first block - assert(prevBb->bbCodeOffs >= 0 && prevBb->bbCodeOffs <= UINT_MAX); - if (optRngPattern.optGetPatternCount() == 0) - { - optRngPattern.m_bbCodeOffs = prevBb->bbCodeOffs; - } - - // Save the first pattern - if (!optRngPattern.optSetPattern(patternIndex, firstPatternVal, prevBb)) - { - break; - } - - firstPatternBBNum = prevBb->bbNum; - patternIndex++; - } - - // Current pattern - - // Save the pattern and Switch default jump target for the pattern (false case) - ssize_t currentPatternVal = currCmpOp->gtGetOp2()->AsIntCon()->IconValue(); - if (!optRngPattern.optSetPattern(patternIndex, currentPatternVal, currBb)) - { - break; - } - - patternIndex++; - foundPattern = true; - - // Stop searching if the current BB is GT_NE. It is the last pattern. - if (currCmpOp->OperIs(GT_NE)) - { - break; - } - } - } - } - } - } - - // Optimize only when patterns are found in consecutive BBs. - // Stop searching if patterns have been found in previous BBs, but the current BB does not have a pattern - if (foundPattern && patternIndex < (int)(currBb->bbNum - firstPatternBBNum + 1)) - { - break; - } - - prevBb = currBb; - } - - if (foundPattern) - { - int patternCount = optRngPattern.optGetPatternCount(); - // If there are less than 3 patterns, no optimization is done. It is not profitable. - if (patternCount <= 2 || patternCount > optRngPattern.m_sizePatterns) - { - return PhaseStatus::MODIFIED_NOTHING; - } - -#ifdef DEBUG - if (verbose) - { - printf("Reserved patterns:\n"); - optRngPattern.optPrintPatterns(); - } -#endif // DEBUG - - // Check if blocks jump to any of the pattern block - if (optRngPattern.optJumpsToPatternBlock()) - { -#ifdef DEBUG - if (verbose) - { - printf("A pattern block jumps to another pattern block, no optimization done.\n"); - } -#endif // DEBUG - - return PhaseStatus::MODIFIED_NOTHING; - } - - // Find range of pattern values - ssize_t minPattern = optRngPattern.m_minPattern; - ssize_t maxPattern = optRngPattern.m_maxPattern; - int rangePattern = (int)(maxPattern - minPattern + 1); - if (patternCount > rangePattern || rangePattern < 2 || rangePattern > optRngPattern.m_sizePatterns) - { -#ifdef DEBUG - if (verbose) - { - printf("Range of pattern values is too small (< 2) or too big (> %d): %d\n", - optRngPattern.m_sizePatterns, rangePattern); - } -#endif // DEBUG - - return PhaseStatus::MODIFIED_NOTHING; - } - assert(rangePattern >= 0 && rangePattern <= optRngPattern.m_sizePatterns); - optRngPattern.m_rangePattern = rangePattern; - -#ifdef DEBUG - if (verbose) - { -#ifdef TARGET_64BIT - printf("Min Max Range: %lld, %lld, %d\n", minPattern, maxPattern, rangePattern); -#else // !TARGET_64BIT - printf("Min Max Range Bitmap: %d, %d, %d\n", minPattern, maxPattern, rangePattern); -#endif // !TARGET_64BIT - } -#endif // DEBUG - - // Replace "JTRUE" block with a "Switch" block and remove other pattern blocks - if (optRngPattern.optChangeToSwitch()) - { -#ifdef DEBUG - if (verbose) - { - fgDispBasicBlocks(true); - } -#endif // DEBUG - - return PhaseStatus::MODIFIED_EVERYTHING; - } - } - - return PhaseStatus::MODIFIED_NOTHING; -} diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp new file mode 100644 index 00000000000000..136b07e40d0ca2 --- /dev/null +++ b/src/coreclr/jit/switchrecognition.cpp @@ -0,0 +1,882 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "jitpch.h" +#ifdef _MSC_VER +#pragma hdrstop +#endif + +//----------------------------------------------------------------------------- +// OptRangePatternDsc: Descriptor used for `if` range pattern optimization +// +class OptRangePatternDsc +{ +public: + static const int m_sizePatterns = 64; // The size of the patterns array + +private: + Compiler* m_comp = nullptr; // The pointer to the Compiler instance + CompAllocator m_allocator; // The memory allocator + typedef JitHashTable, BasicBlock*> PatternToBlockMap; + PatternToBlockMap m_patternToBlockMap; + + CompAllocator m_allocatorTarget; // The memory allocator + typedef JitHashTable, int> BlockToIntMap; + BlockToIntMap m_blockToIntMap; + + BasicBlock* switchBBdesc = nullptr; // The switch basic block descriptor + BBswtDesc* swtDsc = nullptr; // The switch descriptor + GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern + BasicBlock* m_defaultJmpBB = nullptr; // The Switch default jump target + BasicBlock* optGetJumpTargetBB(BasicBlock* block); + bool optMakeSwitchDesc(); + +public: + OptRangePatternDsc(Compiler* comp) + : m_comp(comp) + , m_allocator(comp->getAllocator(CMK_Generic)) + , m_patternToBlockMap(m_allocator) + , m_allocatorTarget(comp->getAllocator(CMK_Generic)) + , m_blockToIntMap(m_allocatorTarget) + { + } + + BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern + BasicBlock* m_optLastBB = nullptr; // The last BB of the range pattern + ssize_t m_minPattern = 0; // The minimum pattern + ssize_t m_maxPattern = 0; // The maximum pattern + int m_rangePattern = 0; // The range of values in patterns[] + unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block + unsigned m_bbCodeOffsEnd = 0; // IL code offset end of the switch basic block + + bool optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block); + int optGetPatternCount(); + void optPrintPatterns(); + bool optJumpsToPatternBlock(); + bool optChangeToSwitch(); +}; + +//----------------------------------------------------------------------------- +// optGetJumpTargetBB: Get jumpTargetBB for the pattern +// +// Arguments: +// block - the basic block to get its jump target. +// +// Return Value: +// The jump target BB for the pattern +// +// Notes: +// If compare operator is GT_EQ, the switch jump target (true case) is bbJumpDest. +// If compare operator is GT_NE,it is bbNext. +// +BasicBlock* OptRangePatternDsc::optGetJumpTargetBB(BasicBlock* block) +{ + assert(block != nullptr && (block->bbJumpKind == BBJ_SWITCH || block->bbJumpKind == BBJ_COND)); + assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); + + GenTree* op1 = block->lastStmt()->GetRootNode()->gtGetOp1(); + auto oper = op1->OperGet(); + assert(oper == GT_EQ || oper == GT_NE); + + if (oper == GT_EQ) + { + return block->bbJumpDest; + } + else if (oper == GT_NE) + { + return block->bbNext; + } + + return nullptr; +} + +//----------------------------------------------------------------------------- +// optSetPattern: Save pattern value and basic block +// +// Arguments: +// idxPattern - the index of the pattern to set +// patternVal - the value of the pattern +// block - the basic block to set its jump target. +// +// Return Value: +// true if the pattern is set, false otherwise. +// +bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block) +{ + if (idxPattern < 0 || idxPattern >= m_sizePatterns) + { +#ifdef DEBUG + if (m_comp->verbose) + { + printf("idxPattern out of range"); + } +#endif // DEBUG + + return false; + } + + // Set the pattern value if it does not already exists in the map + if (m_patternToBlockMap.Lookup(patternVal, nullptr)) // Pattern already exists + { + return false; + } + + m_patternToBlockMap.Set(patternVal, block); + + // Set min pattern + if (patternVal < m_minPattern) + { + m_minPattern = patternVal; + + // Update minOp to the tree with the min pattern + GenTree* minOpNode = block->lastStmt()->GetRootNode()->gtGetOp1()->gtGetOp2(); + assert(minOpNode != nullptr && minOpNode->IsIntegralConst()); + m_minOp = minOpNode; + } + // Set max pattern + if (patternVal > m_maxPattern) + { + m_maxPattern = patternVal; + } + + // Set default jump target of the Switch basic block: + // For GT_EQ, Swithc Default jump target (for false case) is set to bbNext. + // For GT_NE, Swithc Default jump target (for false case) is set to bbJumpDest. + assert(block != nullptr && block->bbJumpKind == BBJ_COND); + assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); + + auto oper = block->lastStmt()->GetRootNode()->gtGetOp1()->OperGet(); + assert(oper == GT_EQ || oper == GT_NE); + if (oper == GT_EQ) + { + m_defaultJmpBB = block->bbNext; + } + else if (oper == GT_NE) + { + m_defaultJmpBB = block->bbJumpDest; + } + + // Update the code offset end range + assert(block->bbCodeOffsEnd >= 0 && block->bbCodeOffsEnd <= UINT_MAX); + m_bbCodeOffsEnd = block->bbCodeOffsEnd; + + // Set the last basic block of the range pattern + m_optLastBB = block; + + return true; +} + +//----------------------------------------------------------------------------- +// optGetPatternCount: Get the number of patterns +// +// Return Value: +// The number of reserved patterns +// +int OptRangePatternDsc::optGetPatternCount() +{ + return m_patternToBlockMap.GetCount(); +} + +//----------------------------------------------------------------------------- +// optPrintPatterns: Prints the patterns from m_patternToBlockMap +// +void OptRangePatternDsc::optPrintPatterns() +{ + // print patterns from m_patternToBlockMap using key iterator + PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; + for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) + { + BasicBlock* mappedBlock; + if (patternToBlockMap->Lookup(patternVal, &mappedBlock)) + { + printf("patternVal = %d, block = %d\n", patternVal, mappedBlock->bbNum); + } + } +} + +//----------------------------------------------------------------------------- +// optJumpsToPatternBlock: Checks if any pattern block jumps to one of the blocks +// within m_patternToBlockMap +// +// Arguments: +// None +// +// Return Value: +// True if any of m_patternToBlockMap block jumps to one of the blocks within m_patternToBlockMap +// False otherwise. +// +bool OptRangePatternDsc::optJumpsToPatternBlock() +{ + PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; + const unsigned int maxPatternBbNum = m_optFirstBB->bbNum + optGetPatternCount() - 1; + + for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) + { + BasicBlock* sourceBlock; + if (patternToBlockMap->Lookup(patternVal, &sourceBlock)) + { + BasicBlock* destBlock = sourceBlock->bbJumpDest; + assert(destBlock != nullptr); + + if (destBlock->bbNum >= m_optFirstBB->bbNum && destBlock->bbNum <= maxPatternBbNum) + { + return true; + } + } + } + + return false; +} + +//----------------------------------------------------------------------------- +// optMakeSwitchDesc: Make a Switch descriptor with a switch jump table +// +// Returns: +// true if the switch descriptor is created successfully with the switch jump tables. False otherwise. +// +// Notes: +// It creates switch descriptor with a jump table for each range pattern and default case. +// If the value is part of the reserved patterns, the switch jump table is set to the jump target +// for true case +// If GT_EQ, jump target is block's bbJumpDest. If GT_NE, jump target is block's bbNext. +// Otherwise, the switch jump table is set to the default jump target. +// No tree is updated in this method. +// +bool OptRangePatternDsc::optMakeSwitchDesc() +{ +#ifdef DEBUG + if (m_comp->verbose) + { + printf("\n*************** In optMakeSwitchDesc()\n"); + } +#endif // DEBUG + + if (optGetPatternCount() > m_rangePattern || m_rangePattern < 2 || optGetPatternCount() > m_rangePattern || + m_rangePattern > m_sizePatterns) + { + return false; + } + + BasicBlock* prevBb = nullptr; + int patternIndex = 0; + + BasicBlockFlags bbFlags = BBF_EMPTY; + unsigned curBBoffs = 0; + unsigned nxtBBoffs = 0; + unsigned jmpCnt = 0; // # of switch cases (including default case) + + BasicBlock** jmpTab = nullptr; + BasicBlock** jmpPtr = nullptr; + unsigned uniqueTargetCnt = 0; // # of unique jump targets + bool tailCall = false; + ssize_t minVal = m_minPattern; + + // Allocate the jump table + jmpCnt = m_rangePattern + 1; + jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt]; + + // Make a jump table for the range of patterns + // If reserved pattern, jump to jump target of source block. If not, jump to default target. + for (int idxRng = 0; idxRng < m_rangePattern; idxRng++) + { + // Find a mapped block from a pattern + ssize_t key = minVal + idxRng; + BasicBlock* mappedBlock = nullptr; + if (m_patternToBlockMap.Lookup(key, &mappedBlock)) // A mapped block is found + { + BasicBlock* jumpTargetBb = optGetJumpTargetBB(mappedBlock); + *jmpPtr = (BasicBlock*)(size_t)(jumpTargetBb->bbCodeOffs); + *(jmpPtr) = jumpTargetBb; + // Update the unique target basic block to count map + if (!m_blockToIntMap.Lookup(jumpTargetBb, nullptr)) + { + m_blockToIntMap.Set(jumpTargetBb, 1); + } + } + else + { + BasicBlock* defaultJmpBb = m_defaultJmpBB; + *jmpPtr = (BasicBlock*)(size_t)(defaultJmpBb->bbCodeOffs); + *(jmpPtr) = defaultJmpBb; + // Update the unique target basic block to count map + if (!m_blockToIntMap.Lookup(defaultJmpBb, nullptr)) + { + m_blockToIntMap.Set(defaultJmpBb, 1); + } + } + jmpPtr++; + } + + // Append the default label to the jump table + *jmpPtr = (BasicBlock*)(size_t)(m_defaultJmpBB->bbCodeOffs); + *(jmpPtr) = m_defaultJmpBB; + // Update the unique target basic block to count map + if (!m_blockToIntMap.Lookup(m_defaultJmpBB, nullptr)) + { + m_blockToIntMap.Set(m_defaultJmpBB, 1); + } + jmpPtr++; + + // Make sure we found the right number of labels + noway_assert(jmpPtr == jmpTab + jmpCnt); + + // + // Check if it is profitable to use a switch instead of a series of conditional branches + // + + // If the number of unique target counts is 1, it has only default case. Not profitable. + // If it is >= 3, it it not converted to a bit test in Lowering. So, skip it. + uniqueTargetCnt = m_blockToIntMap.GetCount(); + if (uniqueTargetCnt != 2) + { + return false; + } + + noway_assert(jmpCnt >= 2); + + // If all jumps to the same target block, BBJ_NONE or BBJ_ALWAYS is better. + BasicBlock* uniqueSucc = nullptr; + if (uniqueTargetCnt == 2) + { + uniqueSucc = jmpTab[0]; + noway_assert(jmpCnt >= 2); + for (unsigned i = 1; i < jmpCnt - 1; i++) + { + if (jmpTab[i] != uniqueSucc) + { + uniqueSucc = nullptr; + break; + } + } + } + if (uniqueSucc != nullptr) + { + return false; + } + + // Check if it is better to use Jmp instead of Switch + BasicBlock* defaultBB = jmpTab[jmpCnt - 1]; + BasicBlock* followingBB = m_optLastBB->bbNext; + + // Is the number of cases right for a jump switch? + const bool firstCaseFollows = (followingBB == jmpTab[0]); + const bool defaultFollows = (followingBB == defaultBB); + + unsigned minSwitchTabJumpCnt = 2; // table is better than just 2 cmp/jcc + + // This means really just a single cmp/jcc (aka a simple if/else) + if (firstCaseFollows || defaultFollows) + { + minSwitchTabJumpCnt++; + } + +#if defined(TARGET_ARM) + // On ARM for small switch tables we will + // generate a sequence of compare and branch instructions + // because the code to load the base of the switch + // table is huge and hideous due to the relocation... :( + minSwitchTabJumpCnt += 2; +#endif // TARGET_ARM + + bool useJumpSequence = jmpCnt < minSwitchTabJumpCnt; + + if (TargetOS::IsUnix && TargetArchitecture::IsArm32) + { + useJumpSequence = useJumpSequence || m_comp->IsTargetAbi(CORINFO_NATIVEAOT_ABI); + } + + if (useJumpSequence) // It is better to use a series of compare and branch IR trees. + { + return false; + } + + // One of the case blocks has to follow the switch block. All the pattern blocks except for the first pattern block + // will be removed from Switch conversion. So, we need to check if jump target follows the last pattern block. + BasicBlock* bbCase0 = nullptr; + BasicBlock* bbCase1 = jmpTab[0]; + BasicBlock* nextBbAfterSwitch = m_optLastBB->bbNext; + + for (unsigned tabIdx = 1; tabIdx < (jmpCnt - 1); tabIdx++) + { + if (jmpTab[tabIdx] != bbCase1 && bbCase0 == nullptr) + { + bbCase0 = jmpTab[tabIdx]; + break; + } + } + if ((nextBbAfterSwitch != bbCase0) && (nextBbAfterSwitch != bbCase1)) + { + return false; + } + + // If the next basic block after Switch is an empty block with an unconditional jump, skip it. + if (nextBbAfterSwitch->isEmpty() && (nextBbAfterSwitch->bbJumpKind == BBJ_ALWAYS) && + (nextBbAfterSwitch != nextBbAfterSwitch->bbJumpDest)) + { +#ifdef DEBUG + if (m_comp->verbose) + { + printf("\nSkip converting to Switch block if Switch jumps to an empty block with an unconditional " + "jump (" FMT_BB " -> " FMT_BB ")\n", + m_optFirstBB->bbNum, nextBbAfterSwitch->bbNum); + } +#endif // DEBUG + return false; + } + + // Allocate the switch descriptor + swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; + + // Fill in the remaining fields of the switch descriptor + swtDsc->bbsCount = jmpCnt; + swtDsc->bbsDstTab = jmpTab; + + m_comp->fgHasSwitch = true; + + if (m_comp->opts.compProcedureSplitting) + { + m_comp->opts.compProcedureSplitting = false; + JITDUMP("Turning off procedure splitting for this method, as it might need switch tables; " + "implementation limitation.\n"); + } + + tailCall = false; + +#ifdef DEBUG + if (m_comp->verbose) + { + // Print bbNum of each jmpTab + printf("------Switch " FMT_BB " jumps to: ", m_optFirstBB->bbNum); + for (unsigned i = 0; i < swtDsc->bbsCount; i++) + { + printf("%c" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); + } + printf("\n\n"); + } +#endif // DEBUG + + return true; +} + +//----------------------------------------------------------------------------- +// optChangeToSwitch: Change the first pattern block to SWITCH block and remove other pattern blocks +// +// Return Value: +// true if the blocks were successfully updated, false otherwise. +// +// Notes: +// Leave Switch basic block only and remove all other blocks in the range pattern. +// Update reference count of jump target blocks. +// If the CNS_INT node does not have a min pattern, replace it with the CNS_INT node with the min value. +// +// Tree before the optimization: +// ``` +// JTRUE +// \EQ +// \LCL_VAR +// \CNS_INT +// ``` +// +// Tree after the optimization: +// ``` +// SWITCH +// \SUB +// \LCL_VAR +// \CNS_INT (min pattern) +// ``` +bool OptRangePatternDsc::optChangeToSwitch() +{ +#ifdef DEBUG + if (m_comp->verbose) + { + printf("\n*************** In optChangeToSwitch()\n"); + } +#endif // DEBUG + + assert(m_optFirstBB != nullptr && m_optFirstBB->KindIs(BBJ_COND) && m_optFirstBB->bbNext != nullptr); + assert(optGetPatternCount() <= m_rangePattern && m_rangePattern >= 2 && optGetPatternCount() <= m_rangePattern && + m_rangePattern <= m_sizePatterns); + + // Make Switch descriptor with a jump table + if (!optMakeSwitchDesc()) + { + return false; + } + + bool updatedBlocks = false; + switchBBdesc = m_optFirstBB; + int patternCount = optGetPatternCount(); + + // Update the Switch basic block + + // Change `JTRUE` basic block to `Switch` basic block + switchBBdesc = m_optFirstBB; + + // Change BBJ_COND to BBJ_SWITCH + switchBBdesc->bbJumpKind = BBJ_SWITCH; + switchBBdesc->bbJumpDest = nullptr; + + switchBBdesc->bbCodeOffs = m_bbCodeOffs; + switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; + switchBBdesc->bbJumpSwt = swtDsc; + GenTree* rootTree = switchBBdesc->lastStmt()->GetRootNode(); + assert(rootTree->OperIs(GT_JTRUE)); + assert(!(rootTree->gtFlags & GTF_SIDE_EFFECT)); // JTRUE node should not have side effects + + // Change from GT_JTRUE to GT_SWITCH + rootTree->ChangeOper(GT_SWITCH); + + // Special args to fgAddRefPred + FlowEdge* const oldEdge = nullptr; + + // Remove non-switch pattern blocks. Update the reference count of jump target block from the removed block. + BasicBlock* currBb = switchBBdesc->bbNext; + for (int idxPattern = 1; idxPattern < patternCount && currBb != nullptr; idxPattern++) + { + assert(!(currBb->bbFlags & BBF_DONT_REMOVE)); + + BasicBlock* prevBlock = currBb->bbPrev; + BasicBlock* nextBlock = currBb->bbNext; + BasicBlock* jumpBlock = optGetJumpTargetBB(currBb); + + // Unlink the current block and its pred block + + // Check if prevBlock is the predecessor of currBb + assert(currBb != nullptr && prevBlock != nullptr); + FlowEdge* pred = m_comp->fgGetPredForBlock(currBb, prevBlock); + assert(pred != nullptr); + + m_comp->fgRemoveRefPred(currBb, prevBlock); + + // Link Switch block and current block's jump target block + m_comp->fgAddRefPred(jumpBlock, switchBBdesc, oldEdge); + + // Link Switch block and the next block: + // if GT_EQ and currBb is the last pattern block, skip because bbNext is already linked as default jump + // target if GT_NE, it is already linked when linking its jump target block to Switch block + if (currBb->lastStmt()->GetRootNode()->gtGetOp1()->OperIs(GT_EQ) && idxPattern != (patternCount - 1)) + { + m_comp->fgAddRefPred(nextBlock, switchBBdesc, oldEdge); + } + + m_comp->fgRemoveBlock(currBb, /* unreachable */ true); + + updatedBlocks = true; + currBb = nextBlock; + } + + // Update the reference count of the default jump block + int numNotFound = m_rangePattern - patternCount + 1; // +1 for switch default case + for (int idxFalse = 0; idxFalse < numNotFound; idxFalse++) + { + m_comp->fgAddRefPred(m_defaultJmpBB, switchBBdesc, oldEdge); + } + + // Continue to transform Switch node + + Statement* stmt = switchBBdesc->lastStmt(); + + // Change from GT_EQ or GT_NE to GT_SUB + // tree: SUB + // op1: LCL_VAR + // op2: GT_CNS_INT or GT_CNS_LNG + GenTree* tree = rootTree->gtGetOp1(); // GT_EQ or GT_NE node to chnage to GT_SUB + tree->ChangeOper(GT_SUB); + assert(tree->gtGetOp1() != nullptr && tree->gtGetOp1()->OperIs(GT_LCL_VAR)); + + // Change constant node if siwtch tree does not have the mininum pattern + if (tree->gtGetOp2() != nullptr && tree->gtGetOp2()->AsIntCon()->IconValue() != m_minPattern) + { + GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT or GT_CNS_LNG node + tree->AsOp()->gtOp2 = m_minOp; + + m_comp->gtSetStmtInfo(stmt); + m_comp->fgSetStmtSeq(stmt); + + DEBUG_DESTROY_NODE(op2); + } + + m_comp->gtUpdateStmtSideEffects(stmt); + + return updatedBlocks; +} + +//----------------------------------------------------------------------------- +// optSwitchRecognition: Optimize range check for if (A || B || C || D) pattern and convert it to Switch block +// +// Returns: +// MODIFIED_NOTHING if no optimization is performed. +// MODIFIED_EVERYTHING otherwise. +// +// Notes: +// Detect if (a == val1 || a == val2 || a == val3 || ...) pattern and change it to switch tree +// to reduce compares and jumps, and perform bit operation instead in Lowering phase. +// This optimization is performed only for integer types. +// Run this phase after "Optimize Layout" phase to avoid Switch block being reordered and separated from its bbNext +// target block. If they are not adjacent, Lowering does not convert them to a bit test. Limit the Switch +// conversion to only for 3 or more conditional patterns to reduce code size regressions. +// +PhaseStatus Compiler::optSwitchRecognition() +{ +#ifdef DEBUG + if (this->verbose) + { + printf("\n*************** In optSwitchRecognition()\n"); + } +#endif // DEBUG + + OptRangePatternDsc optRngPattern(this); + + bool printed = false; + int patternIndex = 0; // The index of the pattern in the array + bool foundPattern = false; + unsigned int firstPatternBBNum = 0; // Basic block number of the first pattern found + BasicBlock* prevBb = fgFirstBB; + + if (fgFirstBB->bbNext == nullptr) + { + return PhaseStatus::MODIFIED_NOTHING; + } + + for (BasicBlock* currBb = fgFirstBB->bbNext; currBb != nullptr; currBb = currBb->bbNext) + { + if (currBb->KindIs(BBJ_COND) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) + { +#ifdef DEBUG + if (this->verbose) + { + if (!printed) + { + fgDispBasicBlocks(true); + printed = true; + } + } +#endif // DEBUG + + // Check if prevBb is the predecessor of currBb and currBb has only one predecessor + FlowEdge* pred = fgGetPredForBlock(currBb, prevBb); + if (pred == nullptr || currBb->bbRefs != 1 || currBb->bbFlags & BBF_DONT_REMOVE) + { + if (foundPattern) + break; // Stop searching if patterns are not from consecutive basic blocks + else + continue; + } + + // Basic block must have only one statement + if (currBb->lastStmt() == currBb->firstStmt() && prevBb->lastStmt() == prevBb->firstStmt()) + { + // Skip if there is any side effect + assert(currBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); // JTRUE node + if (currBb->lastStmt()->GetRootNode()->gtFlags & GTF_SIDE_EFFECT) + { + // Stop searching if patterns are not from consecutive basic blocks + if (foundPattern) + break; + else + continue; + } + + auto currCmpOp = currBb->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node + auto prevCmpOp = prevBb->lastStmt()->GetRootNode()->gtGetOp1(); + assert(currCmpOp != nullptr && prevCmpOp != nullptr); + + // Compare operator is GT_EQ. If it is GT_NE, it is the end of the pattern check. + if ((prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_EQ)) || + (prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_NE))) + { + if (prevBb->bbJumpDest == currBb) + { + if (foundPattern) + break; + else + continue; + } + + // Check both conditions to have constant on the right side (optimize GT_CNS_INT only) + if (currCmpOp->gtGetOp2()->IsCnsIntOrI() && prevCmpOp->gtGetOp2()->IsCnsIntOrI()) + { + // Check both conditions to have the same local variable number + if (prevCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && currCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && + prevCmpOp->gtGetOp1()->AsLclVar()->GetLclNum() == + currCmpOp->gtGetOp1()->AsLclVar()->GetLclNum()) + { +#ifdef DEBUG + if (this->verbose) + { + printf("\nFound pattern (Prev vs Curr):\n"); + gtDispTree(prevCmpOp); + printf("\n"); + gtDispTree(currCmpOp); + printf("\n\n"); + } +#endif // DEBUG + + // No optimization if the number of patterns is greater than 64. + if (patternIndex >= optRngPattern.m_sizePatterns) + { +#ifdef DEBUG + if (this->verbose) + { + printf("Too many patterns found (> 64), no optimization done.\n"); + } +#endif // DEBUG + return PhaseStatus::MODIFIED_NOTHING; + } + + // First pattern found + if (!foundPattern) + { + assert(patternIndex == 0 && prevCmpOp->OperIs(GT_EQ)); + // If the first pattern block is rarely run, skip it. + if (prevBb->isRunRarely()) + { +#ifdef DEBUG + if (this->verbose) + { + printf(FMT_BB " is run rarely. Skip optimizing this block.\n", prevBb->bbNum); + } +#endif // DEBUG + prevBb = currBb; + continue; + } + ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntCon()->IconValue(); + + // Initialize the pattern range + // + assert(prevBb != nullptr && prevBb->bbJumpKind == BBJ_COND && + prevBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); + + optRngPattern.m_optFirstBB = prevBb; + optRngPattern.m_optLastBB = prevBb; + + // Initialize min pattern and max pattern to the first pattern value + optRngPattern.m_minPattern = firstPatternVal; + optRngPattern.m_maxPattern = firstPatternVal; + + // Initialize the code offset range from the first block + assert(prevBb->bbCodeOffs >= 0 && prevBb->bbCodeOffs <= UINT_MAX); + if (optRngPattern.optGetPatternCount() == 0) + { + optRngPattern.m_bbCodeOffs = prevBb->bbCodeOffs; + } + + // Save the first pattern + if (!optRngPattern.optSetPattern(patternIndex, firstPatternVal, prevBb)) + { + break; + } + + firstPatternBBNum = prevBb->bbNum; + patternIndex++; + } + + // Current pattern + + // Save the pattern and Switch default jump target for the pattern (false case) + ssize_t currentPatternVal = currCmpOp->gtGetOp2()->AsIntCon()->IconValue(); + if (!optRngPattern.optSetPattern(patternIndex, currentPatternVal, currBb)) + { + break; + } + + patternIndex++; + foundPattern = true; + + // Stop searching if the current BB is GT_NE. It is the last pattern. + if (currCmpOp->OperIs(GT_NE)) + { + break; + } + } + } + } + } + } + + // Optimize only when patterns are found in consecutive BBs. + // Stop searching if patterns have been found in previous BBs, but the current BB does not have a pattern + if (foundPattern && patternIndex < (int)(currBb->bbNum - firstPatternBBNum + 1)) + { + break; + } + + prevBb = currBb; + } + + if (foundPattern) + { + int patternCount = optRngPattern.optGetPatternCount(); + // If there are less than 3 patterns, no optimization is done. It is not profitable. + if (patternCount <= 2 || patternCount > optRngPattern.m_sizePatterns) + { + return PhaseStatus::MODIFIED_NOTHING; + } + +#ifdef DEBUG + if (verbose) + { + printf("Reserved patterns:\n"); + optRngPattern.optPrintPatterns(); + } +#endif // DEBUG + + // Check if blocks jump to any of the pattern block + if (optRngPattern.optJumpsToPatternBlock()) + { +#ifdef DEBUG + if (verbose) + { + printf("A pattern block jumps to another pattern block, no optimization done.\n"); + } +#endif // DEBUG + + return PhaseStatus::MODIFIED_NOTHING; + } + + // Find range of pattern values + ssize_t minPattern = optRngPattern.m_minPattern; + ssize_t maxPattern = optRngPattern.m_maxPattern; + int rangePattern = (int)(maxPattern - minPattern + 1); + if (patternCount > rangePattern || rangePattern < 2 || rangePattern > optRngPattern.m_sizePatterns) + { +#ifdef DEBUG + if (verbose) + { + printf("Range of pattern values is too small (< 2) or too big (> %d): %d\n", + optRngPattern.m_sizePatterns, rangePattern); + } +#endif // DEBUG + + return PhaseStatus::MODIFIED_NOTHING; + } + assert(rangePattern >= 0 && rangePattern <= optRngPattern.m_sizePatterns); + optRngPattern.m_rangePattern = rangePattern; + +#ifdef DEBUG + if (verbose) + { +#ifdef TARGET_64BIT + printf("Min Max Range: %lld, %lld, %d\n", minPattern, maxPattern, rangePattern); +#else // !TARGET_64BIT + printf("Min Max Range Bitmap: %d, %d, %d\n", minPattern, maxPattern, rangePattern); +#endif // !TARGET_64BIT + } +#endif // DEBUG + + // Replace "JTRUE" block with a "Switch" block and remove other pattern blocks + if (optRngPattern.optChangeToSwitch()) + { +#ifdef DEBUG + if (verbose) + { + fgDispBasicBlocks(true); + } +#endif // DEBUG + + return PhaseStatus::MODIFIED_EVERYTHING; + } + } + + return PhaseStatus::MODIFIED_NOTHING; +} From cc00c06e4d345af7d2536feee5cf378c7765c6f0 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 6 Sep 2023 14:04:32 +0200 Subject: [PATCH 21/35] Code clean up --- src/coreclr/jit/switchrecognition.cpp | 151 ++++++-------------------- 1 file changed, 35 insertions(+), 116 deletions(-) diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index 136b07e40d0ca2..ffe6ebc58d7628 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -28,7 +28,7 @@ class OptRangePatternDsc BBswtDesc* swtDsc = nullptr; // The switch descriptor GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern BasicBlock* m_defaultJmpBB = nullptr; // The Switch default jump target - BasicBlock* optGetJumpTargetBB(BasicBlock* block); + BasicBlock* optGetJumpTargetBB(BasicBlock* block, bool reversed = false); bool optMakeSwitchDesc(); public: @@ -69,25 +69,18 @@ class OptRangePatternDsc // If compare operator is GT_EQ, the switch jump target (true case) is bbJumpDest. // If compare operator is GT_NE,it is bbNext. // -BasicBlock* OptRangePatternDsc::optGetJumpTargetBB(BasicBlock* block) +BasicBlock* OptRangePatternDsc::optGetJumpTargetBB(BasicBlock* block, bool reversed) { - assert(block != nullptr && (block->bbJumpKind == BBJ_SWITCH || block->bbJumpKind == BBJ_COND)); + assert(block != nullptr && (block->KindIs(BBJ_SWITCH, BBJ_COND))); assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); - GenTree* op1 = block->lastStmt()->GetRootNode()->gtGetOp1(); - auto oper = op1->OperGet(); - assert(oper == GT_EQ || oper == GT_NE); - - if (oper == GT_EQ) - { - return block->bbJumpDest; - } - else if (oper == GT_NE) + const GenTree* rootNode = block->lastStmt()->GetRootNode()->gtGetOp1(); + assert(rootNode->OperIs(GT_EQ, GT_NE)); + if (reversed) { - return block->bbNext; + return rootNode->OperIs(GT_EQ) ? block->bbNext : block->bbJumpDest; } - - return nullptr; + return rootNode->OperIs(GT_EQ) ? block->bbJumpDest : block->bbNext; } //----------------------------------------------------------------------------- @@ -105,13 +98,7 @@ bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, Basic { if (idxPattern < 0 || idxPattern >= m_sizePatterns) { -#ifdef DEBUG - if (m_comp->verbose) - { - printf("idxPattern out of range"); - } -#endif // DEBUG - + JITDUMP("idxPattern out of range") return false; } @@ -140,24 +127,15 @@ bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, Basic } // Set default jump target of the Switch basic block: - // For GT_EQ, Swithc Default jump target (for false case) is set to bbNext. - // For GT_NE, Swithc Default jump target (for false case) is set to bbJumpDest. - assert(block != nullptr && block->bbJumpKind == BBJ_COND); + // For GT_EQ, Switch Default jump target (for false case) is set to bbNext. + // For GT_NE, Switch Default jump target (for false case) is set to bbJumpDest. + assert((block != nullptr) && block->KindIs(BBJ_COND)); assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); - auto oper = block->lastStmt()->GetRootNode()->gtGetOp1()->OperGet(); - assert(oper == GT_EQ || oper == GT_NE); - if (oper == GT_EQ) - { - m_defaultJmpBB = block->bbNext; - } - else if (oper == GT_NE) - { - m_defaultJmpBB = block->bbJumpDest; - } + m_defaultJmpBB = optGetJumpTargetBB(block, true); // Update the code offset end range - assert(block->bbCodeOffsEnd >= 0 && block->bbCodeOffsEnd <= UINT_MAX); + assert(block->bbCodeOffsEnd <= UINT_MAX); m_bbCodeOffsEnd = block->bbCodeOffsEnd; // Set the last basic block of the range pattern @@ -413,14 +391,10 @@ bool OptRangePatternDsc::optMakeSwitchDesc() if (nextBbAfterSwitch->isEmpty() && (nextBbAfterSwitch->bbJumpKind == BBJ_ALWAYS) && (nextBbAfterSwitch != nextBbAfterSwitch->bbJumpDest)) { -#ifdef DEBUG - if (m_comp->verbose) - { - printf("\nSkip converting to Switch block if Switch jumps to an empty block with an unconditional " - "jump (" FMT_BB " -> " FMT_BB ")\n", - m_optFirstBB->bbNum, nextBbAfterSwitch->bbNum); - } -#endif // DEBUG + JITDUMP("\nSkip converting to Switch block if Switch jumps to an empty block with an unconditional " + "jump (" FMT_BB " -> " FMT_BB ")\n", + m_optFirstBB->bbNum, nextBbAfterSwitch->bbNum) + return false; } @@ -486,12 +460,7 @@ bool OptRangePatternDsc::optMakeSwitchDesc() // ``` bool OptRangePatternDsc::optChangeToSwitch() { -#ifdef DEBUG - if (m_comp->verbose) - { - printf("\n*************** In optChangeToSwitch()\n"); - } -#endif // DEBUG + JITDUMP("\n*************** In optChangeToSwitch()\n") assert(m_optFirstBB != nullptr && m_optFirstBB->KindIs(BBJ_COND) && m_optFirstBB->bbNext != nullptr); assert(optGetPatternCount() <= m_rangePattern && m_rangePattern >= 2 && optGetPatternCount() <= m_rangePattern && @@ -618,12 +587,7 @@ bool OptRangePatternDsc::optChangeToSwitch() // PhaseStatus Compiler::optSwitchRecognition() { -#ifdef DEBUG - if (this->verbose) - { - printf("\n*************** In optSwitchRecognition()\n"); - } -#endif // DEBUG + JITDUMP("\n*************** In optSwitchRecognition()\n"); OptRangePatternDsc optRngPattern(this); @@ -642,25 +606,15 @@ PhaseStatus Compiler::optSwitchRecognition() { if (currBb->KindIs(BBJ_COND) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) { -#ifdef DEBUG - if (this->verbose) - { - if (!printed) - { - fgDispBasicBlocks(true); - printed = true; - } - } -#endif // DEBUG - // Check if prevBb is the predecessor of currBb and currBb has only one predecessor FlowEdge* pred = fgGetPredForBlock(currBb, prevBb); if (pred == nullptr || currBb->bbRefs != 1 || currBb->bbFlags & BBF_DONT_REMOVE) { if (foundPattern) + { break; // Stop searching if patterns are not from consecutive basic blocks - else - continue; + } + continue; } // Basic block must have only one statement @@ -672,9 +626,10 @@ PhaseStatus Compiler::optSwitchRecognition() { // Stop searching if patterns are not from consecutive basic blocks if (foundPattern) + { break; - else - continue; + } + continue; } auto currCmpOp = currBb->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node @@ -688,9 +643,10 @@ PhaseStatus Compiler::optSwitchRecognition() if (prevBb->bbJumpDest == currBb) { if (foundPattern) + { break; - else - continue; + } + continue; } // Check both conditions to have constant on the right side (optimize GT_CNS_INT only) @@ -715,12 +671,7 @@ PhaseStatus Compiler::optSwitchRecognition() // No optimization if the number of patterns is greater than 64. if (patternIndex >= optRngPattern.m_sizePatterns) { -#ifdef DEBUG - if (this->verbose) - { - printf("Too many patterns found (> 64), no optimization done.\n"); - } -#endif // DEBUG + JITDUMP("Too many patterns found (> 64), no optimization done.\n") return PhaseStatus::MODIFIED_NOTHING; } @@ -731,12 +682,7 @@ PhaseStatus Compiler::optSwitchRecognition() // If the first pattern block is rarely run, skip it. if (prevBb->isRunRarely()) { -#ifdef DEBUG - if (this->verbose) - { - printf(FMT_BB " is run rarely. Skip optimizing this block.\n", prevBb->bbNum); - } -#endif // DEBUG + JITDUMP(FMT_BB " is run rarely. Skip optimizing this block.\n", prevBb->bbNum) prevBb = currBb; continue; } @@ -824,13 +770,7 @@ PhaseStatus Compiler::optSwitchRecognition() // Check if blocks jump to any of the pattern block if (optRngPattern.optJumpsToPatternBlock()) { -#ifdef DEBUG - if (verbose) - { - printf("A pattern block jumps to another pattern block, no optimization done.\n"); - } -#endif // DEBUG - + JITDUMP("A pattern block jumps to another pattern block, no optimization done.\n"); return PhaseStatus::MODIFIED_NOTHING; } @@ -840,40 +780,19 @@ PhaseStatus Compiler::optSwitchRecognition() int rangePattern = (int)(maxPattern - minPattern + 1); if (patternCount > rangePattern || rangePattern < 2 || rangePattern > optRngPattern.m_sizePatterns) { -#ifdef DEBUG - if (verbose) - { - printf("Range of pattern values is too small (< 2) or too big (> %d): %d\n", - optRngPattern.m_sizePatterns, rangePattern); - } -#endif // DEBUG + JITDUMP("Range of pattern values is too small (< 2) or too big (> %d): %d\n", optRngPattern.m_sizePatterns, + rangePattern); return PhaseStatus::MODIFIED_NOTHING; } assert(rangePattern >= 0 && rangePattern <= optRngPattern.m_sizePatterns); optRngPattern.m_rangePattern = rangePattern; -#ifdef DEBUG - if (verbose) - { -#ifdef TARGET_64BIT - printf("Min Max Range: %lld, %lld, %d\n", minPattern, maxPattern, rangePattern); -#else // !TARGET_64BIT - printf("Min Max Range Bitmap: %d, %d, %d\n", minPattern, maxPattern, rangePattern); -#endif // !TARGET_64BIT - } -#endif // DEBUG + JITDUMP("Min Max Range: %lld, %lld, %d\n", (int64_t)minPattern, (int64_t)maxPattern, rangePattern); // Replace "JTRUE" block with a "Switch" block and remove other pattern blocks if (optRngPattern.optChangeToSwitch()) { -#ifdef DEBUG - if (verbose) - { - fgDispBasicBlocks(true); - } -#endif // DEBUG - return PhaseStatus::MODIFIED_EVERYTHING; } } From a034b9d9bc12e3cabaea15e8920d70cf1ceda4ac Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 6 Sep 2023 15:53:21 +0200 Subject: [PATCH 22/35] More clean up --- src/coreclr/jit/block.cpp | 17 +++++++++++------ src/coreclr/jit/block.h | 1 + src/coreclr/jit/switchrecognition.cpp | 19 ++++++++----------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/block.cpp b/src/coreclr/jit/block.cpp index d7eabe832ac98a..8b5cef28a71a82 100644 --- a/src/coreclr/jit/block.cpp +++ b/src/coreclr/jit/block.cpp @@ -792,9 +792,6 @@ bool BasicBlock::IsLIR() const //------------------------------------------------------------------------ // firstStmt: Returns the first statement in the block // -// Arguments: -// None. -// // Return Value: // The first statement in the block's bbStmtList. // @@ -804,10 +801,18 @@ Statement* BasicBlock::firstStmt() const } //------------------------------------------------------------------------ -// lastStmt: Returns the last statement in the block +// hasSingleStmt: Returns true if block has a single statement // -// Arguments: -// None. +// Return Value: +// true if block has a single statement, false otherwise +// +bool BasicBlock::hasSingleStmt() const +{ + return (firstStmt() != nullptr) && (firstStmt() == lastStmt()); +} + +//------------------------------------------------------------------------ +// lastStmt: Returns the last statement in the block // // Return Value: // The last statement in the block's bbStmtList. diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index 676efcfc9485c0..99323e69001ae2 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -1127,6 +1127,7 @@ struct BasicBlock : private LIR::Range Statement* firstStmt() const; Statement* lastStmt() const; + bool hasSingleStmt() const; // Statements: convenience method for enabling range-based `for` iteration over the statement list, e.g.: // for (Statement* const stmt : block->Statements()) diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index ffe6ebc58d7628..869678803a6d28 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -392,8 +392,7 @@ bool OptRangePatternDsc::optMakeSwitchDesc() (nextBbAfterSwitch != nextBbAfterSwitch->bbJumpDest)) { JITDUMP("\nSkip converting to Switch block if Switch jumps to an empty block with an unconditional " - "jump (" FMT_BB " -> " FMT_BB ")\n", - m_optFirstBB->bbNum, nextBbAfterSwitch->bbNum) + "jump (" FMT_BB " -> " FMT_BB ")\n", m_optFirstBB->bbNum, nextBbAfterSwitch->bbNum) return false; } @@ -591,7 +590,6 @@ PhaseStatus Compiler::optSwitchRecognition() OptRangePatternDsc optRngPattern(this); - bool printed = false; int patternIndex = 0; // The index of the pattern in the array bool foundPattern = false; unsigned int firstPatternBBNum = 0; // Basic block number of the first pattern found @@ -602,13 +600,12 @@ PhaseStatus Compiler::optSwitchRecognition() return PhaseStatus::MODIFIED_NOTHING; } - for (BasicBlock* currBb = fgFirstBB->bbNext; currBb != nullptr; currBb = currBb->bbNext) + for (BasicBlock* const currBb : Blocks()) { if (currBb->KindIs(BBJ_COND) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) { // Check if prevBb is the predecessor of currBb and currBb has only one predecessor - FlowEdge* pred = fgGetPredForBlock(currBb, prevBb); - if (pred == nullptr || currBb->bbRefs != 1 || currBb->bbFlags & BBF_DONT_REMOVE) + if ((currBb->GetUniquePred(this) != prevBb) || ((currBb->bbFlags & BBF_DONT_REMOVE) != 0)) { if (foundPattern) { @@ -618,7 +615,7 @@ PhaseStatus Compiler::optSwitchRecognition() } // Basic block must have only one statement - if (currBb->lastStmt() == currBb->firstStmt() && prevBb->lastStmt() == prevBb->firstStmt()) + if (currBb->hasSingleStmt() && prevBb->hasSingleStmt()) { // Skip if there is any side effect assert(currBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); // JTRUE node @@ -632,8 +629,8 @@ PhaseStatus Compiler::optSwitchRecognition() continue; } - auto currCmpOp = currBb->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node - auto prevCmpOp = prevBb->lastStmt()->GetRootNode()->gtGetOp1(); + GenTree* currCmpOp = currBb->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node + GenTree* prevCmpOp = prevBb->lastStmt()->GetRootNode()->gtGetOp1(); assert(currCmpOp != nullptr && prevCmpOp != nullptr); // Compare operator is GT_EQ. If it is GT_NE, it is the end of the pattern check. @@ -780,8 +777,8 @@ PhaseStatus Compiler::optSwitchRecognition() int rangePattern = (int)(maxPattern - minPattern + 1); if (patternCount > rangePattern || rangePattern < 2 || rangePattern > optRngPattern.m_sizePatterns) { - JITDUMP("Range of pattern values is too small (< 2) or too big (> %d): %d\n", optRngPattern.m_sizePatterns, - rangePattern); + JITDUMP("Range of pattern values is too small (< 2) or too big (> %d): %d\n", + optRngPattern.m_sizePatterns, rangePattern); return PhaseStatus::MODIFIED_NOTHING; } From 7fec66335d16e2694c1cc19ea265d23075594b53 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 6 Sep 2023 23:10:56 +0200 Subject: [PATCH 23/35] More clean up --- src/coreclr/jit/block.h | 2 +- src/coreclr/jit/compiler.cpp | 8 +- src/coreclr/jit/compiler.h | 3 +- src/coreclr/jit/switchrecognition.cpp | 913 +++++++------------------- 4 files changed, 229 insertions(+), 697 deletions(-) diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index 99323e69001ae2..9c7953a12b9e56 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -1127,7 +1127,7 @@ struct BasicBlock : private LIR::Range Statement* firstStmt() const; Statement* lastStmt() const; - bool hasSingleStmt() const; + bool hasSingleStmt() const; // Statements: convenience method for enabling range-based `for` iteration over the statement list, e.g.: // for (Statement* const stmt : block->Statements()) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 2e33c948f8baea..d7471f2ee161b5 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5072,13 +5072,13 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // DoPhase(this, PHASE_IF_CONVERSION, &Compiler::optIfConversion); - // Optimize block order - // - DoPhase(this, PHASE_OPTIMIZE_LAYOUT, &Compiler::optOptimizeLayout); - // Conditional to Switch conversion // DoPhase(this, PHASE_SWITCH_RECOGNITION, &Compiler::optSwitchRecognition); + + // Optimize block order + // + DoPhase(this, PHASE_OPTIMIZE_LAYOUT, &Compiler::optOptimizeLayout); } // Determine start of cold region if we are hot/cold splitting diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 36439e201dde81..01f9be6fd57257 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6325,8 +6325,9 @@ class Compiler public: PhaseStatus optOptimizeBools(); PhaseStatus optSwitchRecognition(); + bool optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* testValues, GenTree* nodeToTest); + bool optSwitchDetectAndConvert(Compiler* comp, BasicBlock* block); -public: PhaseStatus optInvertLoops(); // Invert loops so they're entered at top and tested at bottom. PhaseStatus optOptimizeFlow(); // Simplify flow graph and do tail duplication PhaseStatus optOptimizeLayout(); // Optimize the BasicBlock layout of the method diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index 869678803a6d28..9b655c5d3c9f2f 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -6,567 +6,291 @@ #pragma hdrstop #endif -//----------------------------------------------------------------------------- -// OptRangePatternDsc: Descriptor used for `if` range pattern optimization -// -class OptRangePatternDsc -{ -public: - static const int m_sizePatterns = 64; // The size of the patterns array - -private: - Compiler* m_comp = nullptr; // The pointer to the Compiler instance - CompAllocator m_allocator; // The memory allocator - typedef JitHashTable, BasicBlock*> PatternToBlockMap; - PatternToBlockMap m_patternToBlockMap; - - CompAllocator m_allocatorTarget; // The memory allocator - typedef JitHashTable, int> BlockToIntMap; - BlockToIntMap m_blockToIntMap; - - BasicBlock* switchBBdesc = nullptr; // The switch basic block descriptor - BBswtDesc* swtDsc = nullptr; // The switch descriptor - GenTree* m_minOp = nullptr; // The CNS_INT node with the minimum pattern - BasicBlock* m_defaultJmpBB = nullptr; // The Switch default jump target - BasicBlock* optGetJumpTargetBB(BasicBlock* block, bool reversed = false); - bool optMakeSwitchDesc(); - -public: - OptRangePatternDsc(Compiler* comp) - : m_comp(comp) - , m_allocator(comp->getAllocator(CMK_Generic)) - , m_patternToBlockMap(m_allocator) - , m_allocatorTarget(comp->getAllocator(CMK_Generic)) - , m_blockToIntMap(m_allocatorTarget) - { - } - - BasicBlock* m_optFirstBB = nullptr; // The first BB of the range pattern - BasicBlock* m_optLastBB = nullptr; // The last BB of the range pattern - ssize_t m_minPattern = 0; // The minimum pattern - ssize_t m_maxPattern = 0; // The maximum pattern - int m_rangePattern = 0; // The range of values in patterns[] - unsigned m_bbCodeOffs = 0; // IL code offset of the switch basic block - unsigned m_bbCodeOffsEnd = 0; // IL code offset end of the switch basic block - - bool optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block); - int optGetPatternCount(); - void optPrintPatterns(); - bool optJumpsToPatternBlock(); - bool optChangeToSwitch(); -}; - -//----------------------------------------------------------------------------- -// optGetJumpTargetBB: Get jumpTargetBB for the pattern -// -// Arguments: -// block - the basic block to get its jump target. -// -// Return Value: -// The jump target BB for the pattern -// -// Notes: -// If compare operator is GT_EQ, the switch jump target (true case) is bbJumpDest. -// If compare operator is GT_NE,it is bbNext. -// -BasicBlock* OptRangePatternDsc::optGetJumpTargetBB(BasicBlock* block, bool reversed) -{ - assert(block != nullptr && (block->KindIs(BBJ_SWITCH, BBJ_COND))); - assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); - - const GenTree* rootNode = block->lastStmt()->GetRootNode()->gtGetOp1(); - assert(rootNode->OperIs(GT_EQ, GT_NE)); - if (reversed) - { - return rootNode->OperIs(GT_EQ) ? block->bbNext : block->bbJumpDest; - } - return rootNode->OperIs(GT_EQ) ? block->bbJumpDest : block->bbNext; -} - -//----------------------------------------------------------------------------- -// optSetPattern: Save pattern value and basic block -// -// Arguments: -// idxPattern - the index of the pattern to set -// patternVal - the value of the pattern -// block - the basic block to set its jump target. -// -// Return Value: -// true if the pattern is set, false otherwise. -// -bool OptRangePatternDsc::optSetPattern(int idxPattern, ssize_t patternVal, BasicBlock* block) +#define SWITCH_MAX_DISTANCE (TARGET_POINTER_SIZE * BITS_IN_BYTE - 1) +#define SWITCH_MIN_TESTS 3 + +bool IsConstantTestBlock(const BasicBlock* block, + BasicBlock** blockIfTrue, + BasicBlock** blockIfFalse, + bool* isReversed, + GenTree** variableNode = nullptr, + ssize_t* cns = nullptr) { - if (idxPattern < 0 || idxPattern >= m_sizePatterns) + if (block->KindIs(BBJ_COND) && block->hasSingleStmt() && ((block->bbFlags & BBF_DONT_REMOVE) == 0)) { - JITDUMP("idxPattern out of range") - return false; - } + const GenTree* rootNode = block->firstStmt()->GetRootNode(); + assert(rootNode->OperIs(GT_JTRUE)); - // Set the pattern value if it does not already exists in the map - if (m_patternToBlockMap.Lookup(patternVal, nullptr)) // Pattern already exists - { - return false; - } - - m_patternToBlockMap.Set(patternVal, block); - - // Set min pattern - if (patternVal < m_minPattern) - { - m_minPattern = patternVal; - - // Update minOp to the tree with the min pattern - GenTree* minOpNode = block->lastStmt()->GetRootNode()->gtGetOp1()->gtGetOp2(); - assert(minOpNode != nullptr && minOpNode->IsIntegralConst()); - m_minOp = minOpNode; - } - // Set max pattern - if (patternVal > m_maxPattern) - { - m_maxPattern = patternVal; - } - - // Set default jump target of the Switch basic block: - // For GT_EQ, Switch Default jump target (for false case) is set to bbNext. - // For GT_NE, Switch Default jump target (for false case) is set to bbJumpDest. - assert((block != nullptr) && block->KindIs(BBJ_COND)); - assert(block->lastStmt()->GetRootNode()->gtGetOp1() != nullptr); - - m_defaultJmpBB = optGetJumpTargetBB(block, true); - - // Update the code offset end range - assert(block->bbCodeOffsEnd <= UINT_MAX); - m_bbCodeOffsEnd = block->bbCodeOffsEnd; - - // Set the last basic block of the range pattern - m_optLastBB = block; - - return true; -} + // It has to be JTRUE(GT_EQ or GT_NE) + if (rootNode->gtGetOp1()->OperIs(GT_EQ, GT_NE)) + { + // Let's rely on constants to be always on the right side for simplicity + GenTree* op1 = rootNode->gtGetOp1()->gtGetOp1(); + GenTree* op2 = rootNode->gtGetOp1()->gtGetOp2(); -//----------------------------------------------------------------------------- -// optGetPatternCount: Get the number of patterns -// -// Return Value: -// The number of reserved patterns -// -int OptRangePatternDsc::optGetPatternCount() -{ - return m_patternToBlockMap.GetCount(); -} + if (!varTypeIsIntegral(op1) || !varTypeIsIntegral(op2)) + { + // Only integral types are supported + return false; + } -//----------------------------------------------------------------------------- -// optPrintPatterns: Prints the patterns from m_patternToBlockMap -// -void OptRangePatternDsc::optPrintPatterns() -{ - // print patterns from m_patternToBlockMap using key iterator - PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; - for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) - { - BasicBlock* mappedBlock; - if (patternToBlockMap->Lookup(patternVal, &mappedBlock)) - { - printf("patternVal = %d, block = %d\n", patternVal, mappedBlock->bbNum); - } - } -} + // We're looking for "X EQ/NE CNS" or "CNS EQ/NE X" pattern + if (op1->IsCnsIntOrI() ^ op2->IsCnsIntOrI()) + { + if (!op1->OperIs(GT_LCL_VAR) && !op2->OperIs(GT_LCL_VAR)) + { + // We can slightly relax this in future, for now we only expect "LCL_VAR EQ/NE CNS" (or reversed) + return false; + } -//----------------------------------------------------------------------------- -// optJumpsToPatternBlock: Checks if any pattern block jumps to one of the blocks -// within m_patternToBlockMap -// -// Arguments: -// None -// -// Return Value: -// True if any of m_patternToBlockMap block jumps to one of the blocks within m_patternToBlockMap -// False otherwise. -// -bool OptRangePatternDsc::optJumpsToPatternBlock() -{ - PatternToBlockMap* patternToBlockMap = &m_patternToBlockMap; - const unsigned int maxPatternBbNum = m_optFirstBB->bbNum + optGetPatternCount() - 1; + *isReversed = rootNode->gtGetOp1()->OperIs(GT_NE); + *blockIfTrue = *isReversed ? block->bbNext : block->bbJumpDest; + *blockIfFalse = *isReversed ? block->bbJumpDest : block->bbNext; - for (ssize_t patternVal : PatternToBlockMap::KeyIteration(patternToBlockMap)) - { - BasicBlock* sourceBlock; - if (patternToBlockMap->Lookup(patternVal, &sourceBlock)) - { - BasicBlock* destBlock = sourceBlock->bbJumpDest; - assert(destBlock != nullptr); + if ((block->bbNext == block->bbJumpDest) || (block->bbJumpDest == block)) + { + // Ignoring weird cases like a condition jumping to itself + return false; + } - if (destBlock->bbNum >= m_optFirstBB->bbNum && destBlock->bbNum <= maxPatternBbNum) - { + if ((variableNode != nullptr) && (cns != nullptr)) + { + if (op1->IsCnsIntOrI()) + { + *cns = op1->AsIntCon()->IconValue(); + *variableNode = op2; + } + else + { + *cns = op2->AsIntCon()->IconValue(); + *variableNode = op1; + } + } return true; } } } - return false; } -//----------------------------------------------------------------------------- -// optMakeSwitchDesc: Make a Switch descriptor with a switch jump table -// -// Returns: -// true if the switch descriptor is created successfully with the switch jump tables. False otherwise. -// -// Notes: -// It creates switch descriptor with a jump table for each range pattern and default case. -// If the value is part of the reserved patterns, the switch jump table is set to the jump target -// for true case -// If GT_EQ, jump target is block's bbJumpDest. If GT_NE, jump target is block's bbNext. -// Otherwise, the switch jump table is set to the default jump target. -// No tree is updated in this method. -// -bool OptRangePatternDsc::optMakeSwitchDesc() +bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* testValues, GenTree* nodeToTest) { -#ifdef DEBUG - if (m_comp->verbose) - { - printf("\n*************** In optMakeSwitchDesc()\n"); - } -#endif // DEBUG + assert(firstBlock->KindIs(BBJ_COND)); - if (optGetPatternCount() > m_rangePattern || m_rangePattern < 2 || optGetPatternCount() > m_rangePattern || - m_rangePattern > m_sizePatterns) + if (testsCount < SWITCH_MIN_TESTS) { + // Early out - short chains. return false; } - BasicBlock* prevBb = nullptr; - int patternIndex = 0; - - BasicBlockFlags bbFlags = BBF_EMPTY; - unsigned curBBoffs = 0; - unsigned nxtBBoffs = 0; - unsigned jmpCnt = 0; // # of switch cases (including default case) - - BasicBlock** jmpTab = nullptr; - BasicBlock** jmpPtr = nullptr; - unsigned uniqueTargetCnt = 0; // # of unique jump targets - bool tailCall = false; - ssize_t minVal = m_minPattern; + // Find max and min values in the testValues array + ssize_t minValue = testValues[0]; + ssize_t maxValue = testValues[0]; - // Allocate the jump table - jmpCnt = m_rangePattern + 1; - jmpPtr = jmpTab = new (m_comp, CMK_BasicBlock) BasicBlock*[jmpCnt]; - - // Make a jump table for the range of patterns - // If reserved pattern, jump to jump target of source block. If not, jump to default target. - for (int idxRng = 0; idxRng < m_rangePattern; idxRng++) + int testIdx = 0; + for (; testIdx < testsCount; testIdx++) { - // Find a mapped block from a pattern - ssize_t key = minVal + idxRng; - BasicBlock* mappedBlock = nullptr; - if (m_patternToBlockMap.Lookup(key, &mappedBlock)) // A mapped block is found + ssize_t testValue = testValues[testIdx]; + if (testValue < 0) { - BasicBlock* jumpTargetBb = optGetJumpTargetBB(mappedBlock); - *jmpPtr = (BasicBlock*)(size_t)(jumpTargetBb->bbCodeOffs); - *(jmpPtr) = jumpTargetBb; - // Update the unique target basic block to count map - if (!m_blockToIntMap.Lookup(jumpTargetBb, nullptr)) - { - m_blockToIntMap.Set(jumpTargetBb, 1); - } + // We don't support negative values + break; } - else + + const ssize_t newMinValue = min(minValue, testValue); + const ssize_t newMaxValue = max(maxValue, testValue); + if ((newMaxValue - newMinValue) > SWITCH_MAX_DISTANCE) { - BasicBlock* defaultJmpBb = m_defaultJmpBB; - *jmpPtr = (BasicBlock*)(size_t)(defaultJmpBb->bbCodeOffs); - *(jmpPtr) = defaultJmpBb; - // Update the unique target basic block to count map - if (!m_blockToIntMap.Lookup(defaultJmpBb, nullptr)) - { - m_blockToIntMap.Set(defaultJmpBb, 1); - } + // Stop here, the distance between min and max is too big + break; } - jmpPtr++; - } - // Append the default label to the jump table - *jmpPtr = (BasicBlock*)(size_t)(m_defaultJmpBB->bbCodeOffs); - *(jmpPtr) = m_defaultJmpBB; - // Update the unique target basic block to count map - if (!m_blockToIntMap.Lookup(m_defaultJmpBB, nullptr)) - { - m_blockToIntMap.Set(m_defaultJmpBB, 1); + minValue = newMinValue; + maxValue = newMaxValue; } - jmpPtr++; - // Make sure we found the right number of labels - noway_assert(jmpPtr == jmpTab + jmpCnt); - - // - // Check if it is profitable to use a switch instead of a series of conditional branches - // - - // If the number of unique target counts is 1, it has only default case. Not profitable. - // If it is >= 3, it it not converted to a bit test in Lowering. So, skip it. - uniqueTargetCnt = m_blockToIntMap.GetCount(); - if (uniqueTargetCnt != 2) + if (testIdx < SWITCH_MIN_TESTS) { + // Make sure we still have at least SWITCH_MIN_TESTS values after we filtered out some of them return false; } - noway_assert(jmpCnt >= 2); - - // If all jumps to the same target block, BBJ_NONE or BBJ_ALWAYS is better. - BasicBlock* uniqueSucc = nullptr; - if (uniqueTargetCnt == 2) + // Find the last block in the chain + BasicBlock* lastBlock = firstBlock; + for (int i = 0; i < testIdx - 1; i++) { - uniqueSucc = jmpTab[0]; - noway_assert(jmpCnt >= 2); - for (unsigned i = 1; i < jmpCnt - 1; i++) - { - if (jmpTab[i] != uniqueSucc) - { - uniqueSucc = nullptr; - break; - } - } + lastBlock = lastBlock->bbNext; } - if (uniqueSucc != nullptr) - { - return false; - } - - // Check if it is better to use Jmp instead of Switch - BasicBlock* defaultBB = jmpTab[jmpCnt - 1]; - BasicBlock* followingBB = m_optLastBB->bbNext; - // Is the number of cases right for a jump switch? - const bool firstCaseFollows = (followingBB == jmpTab[0]); - const bool defaultFollows = (followingBB == defaultBB); - - unsigned minSwitchTabJumpCnt = 2; // table is better than just 2 cmp/jcc - - // This means really just a single cmp/jcc (aka a simple if/else) - if (firstCaseFollows || defaultFollows) + BasicBlock* blockIfTrue = nullptr; + BasicBlock* blockIfFalse = nullptr; + bool isReversed = false; + const bool isTest = IsConstantTestBlock(lastBlock, &blockIfTrue, &blockIfFalse, &isReversed); + assert(isTest); + + // Convert firstBlock to a switch block + firstBlock->bbJumpKind = BBJ_SWITCH; + firstBlock->bbJumpDest = nullptr; + firstBlock->bbCodeOffsEnd = lastBlock->bbCodeOffsEnd; + firstBlock->lastStmt()->GetRootNode()->ChangeOper(GT_SWITCH); + // The root node is now SUB(nodeToTest, minValue) + firstBlock->lastStmt()->GetRootNode()->AsOp()->gtOp1 = + gtNewOperNode(GT_SUB, nodeToTest->TypeGet(), gtCloneExpr(nodeToTest), + gtNewIconNode(maxValue - minValue, nodeToTest->TypeGet())); + gtSetStmtInfo(firstBlock->lastStmt()); + fgSetStmtSeq(firstBlock->lastStmt()); + gtUpdateStmtSideEffects(firstBlock->lastStmt()); + + // Unlink and remove the whole chain of conditional blocks + BasicBlock* blockToRemove = firstBlock->bbNext; + fgRemoveRefPred(blockToRemove, firstBlock); + while (blockToRemove != lastBlock->bbNext) { - minSwitchTabJumpCnt++; + BasicBlock* nextBlock = blockToRemove->bbNext; + fgRemoveBlock(blockToRemove, true); + blockToRemove = nextBlock; } -#if defined(TARGET_ARM) - // On ARM for small switch tables we will - // generate a sequence of compare and branch instructions - // because the code to load the base of the switch - // table is huge and hideous due to the relocation... :( - minSwitchTabJumpCnt += 2; -#endif // TARGET_ARM - - bool useJumpSequence = jmpCnt < minSwitchTabJumpCnt; - - if (TargetOS::IsUnix && TargetArchitecture::IsArm32) + const auto jumpCount = static_cast(maxValue - minValue); + assert(jumpCount > 0 && jumpCount <= SWITCH_MAX_DISTANCE); + const auto jmpTab = new (this, CMK_BasicBlock) BasicBlock*[jumpCount + 1 /*default case*/]; + + firstBlock->bbJumpSwt = new (this, CMK_BasicBlock) BBswtDesc; + firstBlock->bbJumpSwt->bbsCount = jumpCount + 1; + firstBlock->bbJumpSwt->bbsHasDefault = true; + firstBlock->bbJumpSwt->bbsDstTab = jmpTab; + firstBlock->bbNext = isReversed ? blockIfTrue : blockIfFalse; + fgHasSwitch = true; + + // Compose a bit vector of all the values we have in the testValues array + // to quickly check if a value is in the array + ssize_t bitVector = 0; + for (testIdx = 0; testIdx < testsCount; testIdx++) { - useJumpSequence = useJumpSequence || m_comp->IsTargetAbi(CORINFO_NATIVEAOT_ABI); + bitVector |= (1ULL << static_cast((testValues[testIdx] - minValue))); } - if (useJumpSequence) // It is better to use a series of compare and branch IR trees. + for (unsigned i = 0; i < jumpCount; i++) { - return false; - } - - // One of the case blocks has to follow the switch block. All the pattern blocks except for the first pattern block - // will be removed from Switch conversion. So, we need to check if jump target follows the last pattern block. - BasicBlock* bbCase0 = nullptr; - BasicBlock* bbCase1 = jmpTab[0]; - BasicBlock* nextBbAfterSwitch = m_optLastBB->bbNext; + // value exists in the testValues array (via bitVector) - 'true' case. + const bool isTrue = (bitVector & static_cast(1 << i)); + jmpTab[i] = isTrue ? blockIfTrue : blockIfFalse; - for (unsigned tabIdx = 1; tabIdx < (jmpCnt - 1); tabIdx++) - { - if (jmpTab[tabIdx] != bbCase1 && bbCase0 == nullptr) + // firstBlock already has a link to blockIfTrue so skip the first iteration + if (i > 0) { - bbCase0 = jmpTab[tabIdx]; - break; + fgAddRefPred(jmpTab[i], firstBlock); } - } - if ((nextBbAfterSwitch != bbCase0) && (nextBbAfterSwitch != bbCase1)) - { - return false; - } - - // If the next basic block after Switch is an empty block with an unconditional jump, skip it. - if (nextBbAfterSwitch->isEmpty() && (nextBbAfterSwitch->bbJumpKind == BBJ_ALWAYS) && - (nextBbAfterSwitch != nextBbAfterSwitch->bbJumpDest)) - { - JITDUMP("\nSkip converting to Switch block if Switch jumps to an empty block with an unconditional " - "jump (" FMT_BB " -> " FMT_BB ")\n", m_optFirstBB->bbNum, nextBbAfterSwitch->bbNum) - - return false; - } - - // Allocate the switch descriptor - swtDsc = new (m_comp, CMK_BasicBlock) BBswtDesc; - - // Fill in the remaining fields of the switch descriptor - swtDsc->bbsCount = jmpCnt; - swtDsc->bbsDstTab = jmpTab; - - m_comp->fgHasSwitch = true; - - if (m_comp->opts.compProcedureSplitting) - { - m_comp->opts.compProcedureSplitting = false; - JITDUMP("Turning off procedure splitting for this method, as it might need switch tables; " - "implementation limitation.\n"); - } - - tailCall = false; - -#ifdef DEBUG - if (m_comp->verbose) - { - // Print bbNum of each jmpTab - printf("------Switch " FMT_BB " jumps to: ", m_optFirstBB->bbNum); - for (unsigned i = 0; i < swtDsc->bbsCount; i++) + else { - printf("%c" FMT_BB, (i == 0) ? ' ' : ',', jmpTab[i]->bbNum); + assert(isTrue); } - printf("\n\n"); } -#endif // DEBUG + + // Link the 'default' case + jmpTab[jumpCount] = blockIfFalse; + fgAddRefPred(blockIfFalse, firstBlock); return true; } -//----------------------------------------------------------------------------- -// optChangeToSwitch: Change the first pattern block to SWITCH block and remove other pattern blocks -// -// Return Value: -// true if the blocks were successfully updated, false otherwise. -// -// Notes: -// Leave Switch basic block only and remove all other blocks in the range pattern. -// Update reference count of jump target blocks. -// If the CNS_INT node does not have a min pattern, replace it with the CNS_INT node with the min value. -// -// Tree before the optimization: -// ``` -// JTRUE -// \EQ -// \LCL_VAR -// \CNS_INT -// ``` -// -// Tree after the optimization: -// ``` -// SWITCH -// \SUB -// \LCL_VAR -// \CNS_INT (min pattern) -// ``` -bool OptRangePatternDsc::optChangeToSwitch() +bool Compiler::optSwitchDetectAndConvert(Compiler* comp, BasicBlock* block) { - JITDUMP("\n*************** In optChangeToSwitch()\n") - - assert(m_optFirstBB != nullptr && m_optFirstBB->KindIs(BBJ_COND) && m_optFirstBB->bbNext != nullptr); - assert(optGetPatternCount() <= m_rangePattern && m_rangePattern >= 2 && optGetPatternCount() <= m_rangePattern && - m_rangePattern <= m_sizePatterns); + GenTree* variableNode = nullptr; + ssize_t cns = 0; + BasicBlock* blockIfTrue = nullptr; + BasicBlock* blockIfFalse = nullptr; - // Make Switch descriptor with a jump table - if (!optMakeSwitchDesc()) + bool isReversed = false; + if (IsConstantTestBlock(block, &blockIfTrue, &blockIfFalse, &isReversed, &variableNode, &cns)) { - return false; - } - - bool updatedBlocks = false; - switchBBdesc = m_optFirstBB; - int patternCount = optGetPatternCount(); - - // Update the Switch basic block - - // Change `JTRUE` basic block to `Switch` basic block - switchBBdesc = m_optFirstBB; - - // Change BBJ_COND to BBJ_SWITCH - switchBBdesc->bbJumpKind = BBJ_SWITCH; - switchBBdesc->bbJumpDest = nullptr; - - switchBBdesc->bbCodeOffs = m_bbCodeOffs; - switchBBdesc->bbCodeOffsEnd = m_bbCodeOffsEnd; - switchBBdesc->bbJumpSwt = swtDsc; - GenTree* rootTree = switchBBdesc->lastStmt()->GetRootNode(); - assert(rootTree->OperIs(GT_JTRUE)); - assert(!(rootTree->gtFlags & GTF_SIDE_EFFECT)); // JTRUE node should not have side effects - - // Change from GT_JTRUE to GT_SWITCH - rootTree->ChangeOper(GT_SWITCH); - - // Special args to fgAddRefPred - FlowEdge* const oldEdge = nullptr; - - // Remove non-switch pattern blocks. Update the reference count of jump target block from the removed block. - BasicBlock* currBb = switchBBdesc->bbNext; - for (int idxPattern = 1; idxPattern < patternCount && currBb != nullptr; idxPattern++) - { - assert(!(currBb->bbFlags & BBF_DONT_REMOVE)); - - BasicBlock* prevBlock = currBb->bbPrev; - BasicBlock* nextBlock = currBb->bbNext; - BasicBlock* jumpBlock = optGetJumpTargetBB(currBb); - - // Unlink the current block and its pred block + if (isReversed) + { + // First block uses NE - we don't support this yet. We currently expect all blocks to use EQ + // and allow NE for the last one (because it's what Roslyn usually emits) + return false; + } - // Check if prevBlock is the predecessor of currBb - assert(currBb != nullptr && prevBlock != nullptr); - FlowEdge* pred = m_comp->fgGetPredForBlock(currBb, prevBlock); - assert(pred != nullptr); + if (block->isRunRarely()) + { + // We don't want to convert rarely executed blocks to switch + return false; + } - m_comp->fgRemoveRefPred(currBb, prevBlock); + int testValueIndex = 0; + ssize_t testValues[SWITCH_MAX_DISTANCE] = {}; + testValues[testValueIndex++] = cns; - // Link Switch block and current block's jump target block - m_comp->fgAddRefPred(jumpBlock, switchBBdesc, oldEdge); + const BasicBlock* prevBlock = block; - // Link Switch block and the next block: - // if GT_EQ and currBb is the last pattern block, skip because bbNext is already linked as default jump - // target if GT_NE, it is already linked when linking its jump target block to Switch block - if (currBb->lastStmt()->GetRootNode()->gtGetOp1()->OperIs(GT_EQ) && idxPattern != (patternCount - 1)) + // Now walk the next blocks and see if they are basically the same type of test + for (const BasicBlock* currBb = block->bbNext; currBb != nullptr; currBb = currBb->bbNext) { - m_comp->fgAddRefPred(nextBlock, switchBBdesc, oldEdge); - } + GenTree* currVariableNode = nullptr; + ssize_t currCns = 0; + BasicBlock* currBlockIfTrue = nullptr; + BasicBlock* currBlockIfFalse = nullptr; - m_comp->fgRemoveBlock(currBb, /* unreachable */ true); + if (IsConstantTestBlock(currBb, &currBlockIfTrue, &currBlockIfFalse, &isReversed, &currVariableNode, + &currCns)) + { + if (currBb->isRunRarely()) + { + // Target blocks don't match, stop searching and process what we already have + return optSwitchConvert(block, testValueIndex, testValues, variableNode); + } - updatedBlocks = true; - currBb = nextBlock; - } + if (currBlockIfTrue != blockIfTrue) + { + // Target blocks don't match, stop searching and process what we already have + return optSwitchConvert(block, testValueIndex, testValues, variableNode); + } - // Update the reference count of the default jump block - int numNotFound = m_rangePattern - patternCount + 1; // +1 for switch default case - for (int idxFalse = 0; idxFalse < numNotFound; idxFalse++) - { - m_comp->fgAddRefPred(m_defaultJmpBB, switchBBdesc, oldEdge); - } + if (!GenTree::Compare(currVariableNode, variableNode)) + { + // Variable nodes don't match, stop searching and process what we already have + return optSwitchConvert(block, testValueIndex, testValues, variableNode); + } - // Continue to transform Switch node + if (currBb->GetUniquePred(comp) != prevBlock) + { + // Current block has multiple preds, stop searching and process what we already have + return optSwitchConvert(block, testValueIndex, testValues, variableNode); + } - Statement* stmt = switchBBdesc->lastStmt(); + if (!BasicBlock::sameEHRegion(prevBlock, currBb)) + { + // Current block is in a different EH region, stop searching and process what we already have + return optSwitchConvert(block, testValueIndex, testValues, variableNode); + } - // Change from GT_EQ or GT_NE to GT_SUB - // tree: SUB - // op1: LCL_VAR - // op2: GT_CNS_INT or GT_CNS_LNG - GenTree* tree = rootTree->gtGetOp1(); // GT_EQ or GT_NE node to chnage to GT_SUB - tree->ChangeOper(GT_SUB); - assert(tree->gtGetOp1() != nullptr && tree->gtGetOp1()->OperIs(GT_LCL_VAR)); + // It's currently limited to LCL_VAR + assert(variableNode->OperIs(GT_LCL_VAR)); - // Change constant node if siwtch tree does not have the mininum pattern - if (tree->gtGetOp2() != nullptr && tree->gtGetOp2()->AsIntCon()->IconValue() != m_minPattern) - { - GenTree* op2 = tree->gtGetOp2(); // GT_CNS_INT or GT_CNS_LNG node - tree->AsOp()->gtOp2 = m_minOp; + testValues[testValueIndex++] = currCns; + if (testValueIndex == SWITCH_MAX_DISTANCE) + { + // Too many suitable tests found - stop and check what we already have + return optSwitchConvert(block, testValueIndex, testValues, variableNode); + } - m_comp->gtSetStmtInfo(stmt); - m_comp->fgSetStmtSeq(stmt); + if (isReversed) + { + // We only support reversed test (GT_NE) for the last block - stop and check what we already have + return optSwitchConvert(block, testValueIndex, testValues, variableNode); + } - DEBUG_DESTROY_NODE(op2); + prevBlock = currBb; + } + else + { + // Current block is not a suitable test, stop searching and process what we already have + return optSwitchConvert(block, testValueIndex, testValues, variableNode); + } + } } - m_comp->gtUpdateStmtSideEffects(stmt); - - return updatedBlocks; + return false; } //----------------------------------------------------------------------------- @@ -579,219 +303,26 @@ bool OptRangePatternDsc::optChangeToSwitch() // Notes: // Detect if (a == val1 || a == val2 || a == val3 || ...) pattern and change it to switch tree // to reduce compares and jumps, and perform bit operation instead in Lowering phase. -// This optimization is performed only for integer types. -// Run this phase after "Optimize Layout" phase to avoid Switch block being reordered and separated from its bbNext -// target block. If they are not adjacent, Lowering does not convert them to a bit test. Limit the Switch +// This optimization is performed only for integer types. Limit the Switch // conversion to only for 3 or more conditional patterns to reduce code size regressions. // PhaseStatus Compiler::optSwitchRecognition() { - JITDUMP("\n*************** In optSwitchRecognition()\n"); - - OptRangePatternDsc optRngPattern(this); - - int patternIndex = 0; // The index of the pattern in the array - bool foundPattern = false; - unsigned int firstPatternBBNum = 0; // Basic block number of the first pattern found - BasicBlock* prevBb = fgFirstBB; - - if (fgFirstBB->bbNext == nullptr) + bool modified = false; + for (BasicBlock* block = fgFirstBB; block != nullptr; block = block->bbNext) { - return PhaseStatus::MODIFIED_NOTHING; - } - - for (BasicBlock* const currBb : Blocks()) - { - if (currBb->KindIs(BBJ_COND) && prevBb != nullptr && prevBb->KindIs(BBJ_COND)) - { - // Check if prevBb is the predecessor of currBb and currBb has only one predecessor - if ((currBb->GetUniquePred(this) != prevBb) || ((currBb->bbFlags & BBF_DONT_REMOVE) != 0)) - { - if (foundPattern) - { - break; // Stop searching if patterns are not from consecutive basic blocks - } - continue; - } - - // Basic block must have only one statement - if (currBb->hasSingleStmt() && prevBb->hasSingleStmt()) - { - // Skip if there is any side effect - assert(currBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); // JTRUE node - if (currBb->lastStmt()->GetRootNode()->gtFlags & GTF_SIDE_EFFECT) - { - // Stop searching if patterns are not from consecutive basic blocks - if (foundPattern) - { - break; - } - continue; - } - - GenTree* currCmpOp = currBb->lastStmt()->GetRootNode()->gtGetOp1(); // GT_EQ or GT_NE node - GenTree* prevCmpOp = prevBb->lastStmt()->GetRootNode()->gtGetOp1(); - assert(currCmpOp != nullptr && prevCmpOp != nullptr); - - // Compare operator is GT_EQ. If it is GT_NE, it is the end of the pattern check. - if ((prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_EQ)) || - (prevCmpOp->OperIs(GT_EQ) && currCmpOp->OperIs(GT_NE))) - { - if (prevBb->bbJumpDest == currBb) - { - if (foundPattern) - { - break; - } - continue; - } - - // Check both conditions to have constant on the right side (optimize GT_CNS_INT only) - if (currCmpOp->gtGetOp2()->IsCnsIntOrI() && prevCmpOp->gtGetOp2()->IsCnsIntOrI()) - { - // Check both conditions to have the same local variable number - if (prevCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && currCmpOp->gtGetOp1()->OperIs(GT_LCL_VAR) && - prevCmpOp->gtGetOp1()->AsLclVar()->GetLclNum() == - currCmpOp->gtGetOp1()->AsLclVar()->GetLclNum()) - { -#ifdef DEBUG - if (this->verbose) - { - printf("\nFound pattern (Prev vs Curr):\n"); - gtDispTree(prevCmpOp); - printf("\n"); - gtDispTree(currCmpOp); - printf("\n\n"); - } -#endif // DEBUG - - // No optimization if the number of patterns is greater than 64. - if (patternIndex >= optRngPattern.m_sizePatterns) - { - JITDUMP("Too many patterns found (> 64), no optimization done.\n") - return PhaseStatus::MODIFIED_NOTHING; - } - - // First pattern found - if (!foundPattern) - { - assert(patternIndex == 0 && prevCmpOp->OperIs(GT_EQ)); - // If the first pattern block is rarely run, skip it. - if (prevBb->isRunRarely()) - { - JITDUMP(FMT_BB " is run rarely. Skip optimizing this block.\n", prevBb->bbNum) - prevBb = currBb; - continue; - } - ssize_t firstPatternVal = prevCmpOp->gtGetOp2()->AsIntCon()->IconValue(); - - // Initialize the pattern range - // - assert(prevBb != nullptr && prevBb->bbJumpKind == BBJ_COND && - prevBb->lastStmt()->GetRootNode()->OperIs(GT_JTRUE)); - - optRngPattern.m_optFirstBB = prevBb; - optRngPattern.m_optLastBB = prevBb; - - // Initialize min pattern and max pattern to the first pattern value - optRngPattern.m_minPattern = firstPatternVal; - optRngPattern.m_maxPattern = firstPatternVal; - - // Initialize the code offset range from the first block - assert(prevBb->bbCodeOffs >= 0 && prevBb->bbCodeOffs <= UINT_MAX); - if (optRngPattern.optGetPatternCount() == 0) - { - optRngPattern.m_bbCodeOffs = prevBb->bbCodeOffs; - } - - // Save the first pattern - if (!optRngPattern.optSetPattern(patternIndex, firstPatternVal, prevBb)) - { - break; - } - - firstPatternBBNum = prevBb->bbNum; - patternIndex++; - } - - // Current pattern - - // Save the pattern and Switch default jump target for the pattern (false case) - ssize_t currentPatternVal = currCmpOp->gtGetOp2()->AsIntCon()->IconValue(); - if (!optRngPattern.optSetPattern(patternIndex, currentPatternVal, currBb)) - { - break; - } - - patternIndex++; - foundPattern = true; - - // Stop searching if the current BB is GT_NE. It is the last pattern. - if (currCmpOp->OperIs(GT_NE)) - { - break; - } - } - } - } - } - } - - // Optimize only when patterns are found in consecutive BBs. - // Stop searching if patterns have been found in previous BBs, but the current BB does not have a pattern - if (foundPattern && patternIndex < (int)(currBb->bbNum - firstPatternBBNum + 1)) + if (optSwitchDetectAndConvert(this, block)) { - break; + JITDUMP("Converted block " FMT_BB " to switch\n", block->bbNum) + modified = true; } - - prevBb = currBb; } - if (foundPattern) + if (modified) { - int patternCount = optRngPattern.optGetPatternCount(); - // If there are less than 3 patterns, no optimization is done. It is not profitable. - if (patternCount <= 2 || patternCount > optRngPattern.m_sizePatterns) - { - return PhaseStatus::MODIFIED_NOTHING; - } - -#ifdef DEBUG - if (verbose) - { - printf("Reserved patterns:\n"); - optRngPattern.optPrintPatterns(); - } -#endif // DEBUG - - // Check if blocks jump to any of the pattern block - if (optRngPattern.optJumpsToPatternBlock()) - { - JITDUMP("A pattern block jumps to another pattern block, no optimization done.\n"); - return PhaseStatus::MODIFIED_NOTHING; - } - - // Find range of pattern values - ssize_t minPattern = optRngPattern.m_minPattern; - ssize_t maxPattern = optRngPattern.m_maxPattern; - int rangePattern = (int)(maxPattern - minPattern + 1); - if (patternCount > rangePattern || rangePattern < 2 || rangePattern > optRngPattern.m_sizePatterns) - { - JITDUMP("Range of pattern values is too small (< 2) or too big (> %d): %d\n", - optRngPattern.m_sizePatterns, rangePattern); - - return PhaseStatus::MODIFIED_NOTHING; - } - assert(rangePattern >= 0 && rangePattern <= optRngPattern.m_sizePatterns); - optRngPattern.m_rangePattern = rangePattern; - - JITDUMP("Min Max Range: %lld, %lld, %d\n", (int64_t)minPattern, (int64_t)maxPattern, rangePattern); - - // Replace "JTRUE" block with a "Switch" block and remove other pattern blocks - if (optRngPattern.optChangeToSwitch()) - { - return PhaseStatus::MODIFIED_EVERYTHING; - } + fgReorderBlocks(/* useProfileData */ false); + fgUpdateChangedFlowGraph(FlowGraphUpdates::COMPUTE_BASICS); + return PhaseStatus::MODIFIED_NOTHING; } return PhaseStatus::MODIFIED_NOTHING; From b9e8d4fd7107d7cf3389c531b5acc05e0b932638 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 6 Sep 2023 23:47:00 +0200 Subject: [PATCH 24/35] Better version --- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/switchrecognition.cpp | 53 ++++++++++++++++++--------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 01f9be6fd57257..54cdd7f8e6d691 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6326,7 +6326,7 @@ class Compiler PhaseStatus optOptimizeBools(); PhaseStatus optSwitchRecognition(); bool optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* testValues, GenTree* nodeToTest); - bool optSwitchDetectAndConvert(Compiler* comp, BasicBlock* block); + bool optSwitchDetectAndConvert(BasicBlock* block); PhaseStatus optInvertLoops(); // Invert loops so they're entered at top and tested at bottom. PhaseStatus optOptimizeFlow(); // Simplify flow graph and do tail duplication diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index 9b655c5d3c9f2f..f0a441acb3fd34 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -9,6 +9,7 @@ #define SWITCH_MAX_DISTANCE (TARGET_POINTER_SIZE * BITS_IN_BYTE - 1) #define SWITCH_MIN_TESTS 3 +// Does given block represent JTRUE(X ==/!= CNS) construct? bool IsConstantTestBlock(const BasicBlock* block, BasicBlock** blockIfTrue, BasicBlock** blockIfFalse, @@ -73,6 +74,21 @@ bool IsConstantTestBlock(const BasicBlock* block, return false; } +//------------------------------------------------------------------------------ +// optSwitchConvert : Convert a series of conditional blocks into a switch block +// conditional blocks are blocks that have a single statement that is a GT_EQ +// or GT_NE node. The blocks are expected jump into the same target and test +// the same variable against different constants +// +// Arguments: +// firstBlock - First conditional block in the chain +// testsCount - Number of conditional blocks in the chain +// testValues - Array of constants that are tested against the variable +// nodeToTest - Variable node that is tested against the constants +// +// Return Value: +// True if the conversion was successful, false otherwise +// bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* testValues, GenTree* nodeToTest) { assert(firstBlock->KindIs(BBJ_COND)); @@ -105,6 +121,9 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* break; } + // TODO: currently we mainly focus on creating the shape that is then expanded into a bit test + // Ideally we should create jump tables for other cases as well and we need some cost-benefit analysis. + minValue = newMinValue; maxValue = newMaxValue; } @@ -116,7 +135,7 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* } // Find the last block in the chain - BasicBlock* lastBlock = firstBlock; + const BasicBlock* lastBlock = firstBlock; for (int i = 0; i < testIdx - 1; i++) { lastBlock = lastBlock->bbNext; @@ -194,7 +213,18 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* return true; } -bool Compiler::optSwitchDetectAndConvert(Compiler* comp, BasicBlock* block) +//------------------------------------------------------------------------------ +// optSwitchDetectAndConvert : Try to detect a series of conditional blocks which +// can be converted into a switch (jump-table) construct. See optSwitchConvert +// for more details. +// +// Arguments: +// firstBlock - A block to start the search from +// +// Return Value: +// True if the conversion was successful, false otherwise +// +bool Compiler::optSwitchDetectAndConvert(BasicBlock* block) { GenTree* variableNode = nullptr; ssize_t cns = 0; @@ -211,12 +241,6 @@ bool Compiler::optSwitchDetectAndConvert(Compiler* comp, BasicBlock* block) return false; } - if (block->isRunRarely()) - { - // We don't want to convert rarely executed blocks to switch - return false; - } - int testValueIndex = 0; ssize_t testValues[SWITCH_MAX_DISTANCE] = {}; testValues[testValueIndex++] = cns; @@ -234,12 +258,6 @@ bool Compiler::optSwitchDetectAndConvert(Compiler* comp, BasicBlock* block) if (IsConstantTestBlock(currBb, &currBlockIfTrue, &currBlockIfFalse, &isReversed, &currVariableNode, &currCns)) { - if (currBb->isRunRarely()) - { - // Target blocks don't match, stop searching and process what we already have - return optSwitchConvert(block, testValueIndex, testValues, variableNode); - } - if (currBlockIfTrue != blockIfTrue) { // Target blocks don't match, stop searching and process what we already have @@ -252,7 +270,7 @@ bool Compiler::optSwitchDetectAndConvert(Compiler* comp, BasicBlock* block) return optSwitchConvert(block, testValueIndex, testValues, variableNode); } - if (currBb->GetUniquePred(comp) != prevBlock) + if (currBb->GetUniquePred(this) != prevBlock) { // Current block has multiple preds, stop searching and process what we already have return optSwitchConvert(block, testValueIndex, testValues, variableNode); @@ -311,7 +329,8 @@ PhaseStatus Compiler::optSwitchRecognition() bool modified = false; for (BasicBlock* block = fgFirstBB; block != nullptr; block = block->bbNext) { - if (optSwitchDetectAndConvert(this, block)) + // block->KindIs(BBJ_COND) check is for better throughput. + if (block->KindIs(BBJ_COND) && optSwitchDetectAndConvert(block)) { JITDUMP("Converted block " FMT_BB " to switch\n", block->bbNum) modified = true; @@ -322,7 +341,7 @@ PhaseStatus Compiler::optSwitchRecognition() { fgReorderBlocks(/* useProfileData */ false); fgUpdateChangedFlowGraph(FlowGraphUpdates::COMPUTE_BASICS); - return PhaseStatus::MODIFIED_NOTHING; + return PhaseStatus::MODIFIED_EVERYTHING; } return PhaseStatus::MODIFIED_NOTHING; From 82d5b7359a2fd1ab10a9aaa26b205fdf79a43fe8 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 6 Sep 2023 23:56:06 +0200 Subject: [PATCH 25/35] Fix bug --- src/coreclr/jit/switchrecognition.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index f0a441acb3fd34..c1b64efd3a5ffd 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -155,7 +155,7 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* // The root node is now SUB(nodeToTest, minValue) firstBlock->lastStmt()->GetRootNode()->AsOp()->gtOp1 = gtNewOperNode(GT_SUB, nodeToTest->TypeGet(), gtCloneExpr(nodeToTest), - gtNewIconNode(maxValue - minValue, nodeToTest->TypeGet())); + gtNewIconNode(minValue, nodeToTest->TypeGet())); gtSetStmtInfo(firstBlock->lastStmt()); fgSetStmtSeq(firstBlock->lastStmt()); gtUpdateStmtSideEffects(firstBlock->lastStmt()); From 04e77cbbdc072b1587856b841405b67a8f2b7e16 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 7 Sep 2023 01:38:39 +0200 Subject: [PATCH 26/35] More conservative --- src/coreclr/jit/compiler.cpp | 8 ++++---- src/coreclr/jit/switchrecognition.cpp | 8 ++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index d7471f2ee161b5..2e33c948f8baea 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5072,13 +5072,13 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // DoPhase(this, PHASE_IF_CONVERSION, &Compiler::optIfConversion); - // Conditional to Switch conversion - // - DoPhase(this, PHASE_SWITCH_RECOGNITION, &Compiler::optSwitchRecognition); - // Optimize block order // DoPhase(this, PHASE_OPTIMIZE_LAYOUT, &Compiler::optOptimizeLayout); + + // Conditional to Switch conversion + // + DoPhase(this, PHASE_SWITCH_RECOGNITION, &Compiler::optSwitchRecognition); } // Determine start of cold region if we are hot/cold splitting diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index c1b64efd3a5ffd..bb1edb7bd3fc61 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -7,7 +7,7 @@ #endif #define SWITCH_MAX_DISTANCE (TARGET_POINTER_SIZE * BITS_IN_BYTE - 1) -#define SWITCH_MIN_TESTS 3 +#define SWITCH_MIN_TESTS 4 // Does given block represent JTRUE(X ==/!= CNS) construct? bool IsConstantTestBlock(const BasicBlock* block, @@ -19,13 +19,12 @@ bool IsConstantTestBlock(const BasicBlock* block, { if (block->KindIs(BBJ_COND) && block->hasSingleStmt() && ((block->bbFlags & BBF_DONT_REMOVE) == 0)) { - const GenTree* rootNode = block->firstStmt()->GetRootNode(); + const GenTree* rootNode = block->lastStmt()->GetRootNode(); assert(rootNode->OperIs(GT_JTRUE)); // It has to be JTRUE(GT_EQ or GT_NE) if (rootNode->gtGetOp1()->OperIs(GT_EQ, GT_NE)) { - // Let's rely on constants to be always on the right side for simplicity GenTree* op1 = rootNode->gtGetOp1()->gtGetOp1(); GenTree* op2 = rootNode->gtGetOp1()->gtGetOp2(); @@ -282,9 +281,6 @@ bool Compiler::optSwitchDetectAndConvert(BasicBlock* block) return optSwitchConvert(block, testValueIndex, testValues, variableNode); } - // It's currently limited to LCL_VAR - assert(variableNode->OperIs(GT_LCL_VAR)); - testValues[testValueIndex++] = currCns; if (testValueIndex == SWITCH_MAX_DISTANCE) { From 146ff1fb19d82e344d0e5711829911ed65f26bae Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 7 Sep 2023 15:26:37 +0200 Subject: [PATCH 27/35] fix bug --- src/coreclr/jit/switchrecognition.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index bb1edb7bd3fc61..f0b8ee30a76568 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -151,10 +151,16 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* firstBlock->bbJumpDest = nullptr; firstBlock->bbCodeOffsEnd = lastBlock->bbCodeOffsEnd; firstBlock->lastStmt()->GetRootNode()->ChangeOper(GT_SWITCH); - // The root node is now SUB(nodeToTest, minValue) - firstBlock->lastStmt()->GetRootNode()->AsOp()->gtOp1 = - gtNewOperNode(GT_SUB, nodeToTest->TypeGet(), gtCloneExpr(nodeToTest), - gtNewIconNode(minValue, nodeToTest->TypeGet())); + + // The root node is now SUB(nodeToTest, minValue) if minValue != 0 + GenTree* switchValue = gtCloneExpr(nodeToTest); + if (minValue != 0) + { + switchValue = + gtNewOperNode(GT_SUB, nodeToTest->TypeGet(), switchValue, gtNewIconNode(minValue, nodeToTest->TypeGet())); + } + + firstBlock->lastStmt()->GetRootNode()->AsOp()->gtOp1 = switchValue; gtSetStmtInfo(firstBlock->lastStmt()); fgSetStmtSeq(firstBlock->lastStmt()); gtUpdateStmtSideEffects(firstBlock->lastStmt()); From b6b814d9cfa6e514ad669b87c4c73482516ef086 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 7 Sep 2023 20:18:12 +0200 Subject: [PATCH 28/35] Disable compProcedureSplitting for switches --- src/coreclr/jit/switchrecognition.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index f0b8ee30a76568..cd052bb52281bb 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -186,6 +186,9 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* firstBlock->bbNext = isReversed ? blockIfTrue : blockIfFalse; fgHasSwitch = true; + // Disable splitting, see fgMakeBasicBlocks which does the same for switch blocks + opts.compProcedureSplitting = false; + // Compose a bit vector of all the values we have in the testValues array // to quickly check if a value is in the array ssize_t bitVector = 0; From 942d7d9c2611297100abc82feef51ada443b8d04 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 7 Sep 2023 23:59:17 +0200 Subject: [PATCH 29/35] Fix bug --- src/coreclr/jit/switchrecognition.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index cd052bb52281bb..dab9fc19a8b2c4 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -175,7 +175,7 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* blockToRemove = nextBlock; } - const auto jumpCount = static_cast(maxValue - minValue); + const auto jumpCount = static_cast(maxValue - minValue + 1); assert(jumpCount > 0 && jumpCount <= SWITCH_MAX_DISTANCE); const auto jmpTab = new (this, CMK_BasicBlock) BasicBlock*[jumpCount + 1 /*default case*/]; From b4e9fed51386fa417d6e4845cd548863803a71c5 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 8 Sep 2023 14:49:36 +0200 Subject: [PATCH 30/35] Fix another bug --- src/coreclr/jit/switchrecognition.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index dab9fc19a8b2c4..5481e7b52df783 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -6,7 +6,7 @@ #pragma hdrstop #endif -#define SWITCH_MAX_DISTANCE (TARGET_POINTER_SIZE * BITS_IN_BYTE - 1) +#define SWITCH_MAX_DISTANCE ((TARGET_POINTER_SIZE * BITS_IN_BYTE) - 1) #define SWITCH_MIN_TESTS 4 // Does given block represent JTRUE(X ==/!= CNS) construct? @@ -194,13 +194,14 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* ssize_t bitVector = 0; for (testIdx = 0; testIdx < testsCount; testIdx++) { - bitVector |= (1ULL << static_cast((testValues[testIdx] - minValue))); + assert(testIdx <= (int)((sizeof(ssize_t) * BITS_PER_BYTE) - 1)); + bitVector |= (ssize_t)(1ULL << static_cast((testValues[testIdx] - minValue))); } for (unsigned i = 0; i < jumpCount; i++) { // value exists in the testValues array (via bitVector) - 'true' case. - const bool isTrue = (bitVector & static_cast(1 << i)); + const bool isTrue = (bitVector & static_cast(1ULL << i)) != 0; jmpTab[i] = isTrue ? blockIfTrue : blockIfFalse; // firstBlock already has a link to blockIfTrue so skip the first iteration From c01adf2dd4f16a4c6a621293fd913c96febb078d Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 8 Sep 2023 18:39:08 +0200 Subject: [PATCH 31/35] Fix assert --- src/coreclr/jit/switchrecognition.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index 5481e7b52df783..9a63b6497df3d9 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -176,7 +176,7 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* } const auto jumpCount = static_cast(maxValue - minValue + 1); - assert(jumpCount > 0 && jumpCount <= SWITCH_MAX_DISTANCE); + assert((jumpCount > 0) && (jumpCount <= SWITCH_MAX_DISTANCE + 1)); const auto jmpTab = new (this, CMK_BasicBlock) BasicBlock*[jumpCount + 1 /*default case*/]; firstBlock->bbJumpSwt = new (this, CMK_BasicBlock) BBswtDesc; From 963c62876f4ca096b4dbc3eebcef2e0742eafbe9 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 8 Sep 2023 18:57:35 +0200 Subject: [PATCH 32/35] It's less efficient on non-xarch --- src/coreclr/jit/switchrecognition.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index 9a63b6497df3d9..7080d9d25d8584 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -120,6 +120,16 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* break; } +#ifndef TARGET_XARCH + if ((newMaxValue - newMinValue) > 16) + { + // Currently, we rely on TryLowerAndOpToExtractLowestSetBit to lower the switch + // and it only supports XARCH targets. For non-XARCH targets, we only support + // low deltas to avoid emitting huge jump tables. + break; + } +#endif + // TODO: currently we mainly focus on creating the shape that is then expanded into a bit test // Ideally we should create jump tables for other cases as well and we need some cost-benefit analysis. From 56b680d5eab8bd2fffafcca794aac5ab433d156c Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 8 Sep 2023 20:24:55 +0200 Subject: [PATCH 33/35] code formatting --- src/coreclr/jit/switchrecognition.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index 7080d9d25d8584..3db6d2baa3804d 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -121,13 +121,13 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* } #ifndef TARGET_XARCH - if ((newMaxValue - newMinValue) > 16) - { - // Currently, we rely on TryLowerAndOpToExtractLowestSetBit to lower the switch - // and it only supports XARCH targets. For non-XARCH targets, we only support - // low deltas to avoid emitting huge jump tables. - break; - } + if ((newMaxValue - newMinValue) > 16) + { + // Currently, we rely on TryLowerAndOpToExtractLowestSetBit to lower the switch + // and it only supports XARCH targets. For non-XARCH targets, we only support + // low deltas to avoid emitting huge jump tables. + break; + } #endif // TODO: currently we mainly focus on creating the shape that is then expanded into a bit test From 40cf1e5a0251a324673d890a521baafdd319fe51 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 10 Sep 2023 00:01:05 +0200 Subject: [PATCH 34/35] Clean up --- src/coreclr/jit/lower.cpp | 2 +- src/coreclr/jit/switchrecognition.cpp | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 7983f4aad47e5d..62c6481b797da0 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -1161,7 +1161,7 @@ bool Lowering::TryLowerSwitchToBitTest( // We'll ensure that there are only 2 targets when building the bit table. // - if (targetCount > 3) + if (targetCount > 0) // test { return false; } diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index 3db6d2baa3804d..9a63b6497df3d9 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -120,16 +120,6 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* break; } -#ifndef TARGET_XARCH - if ((newMaxValue - newMinValue) > 16) - { - // Currently, we rely on TryLowerAndOpToExtractLowestSetBit to lower the switch - // and it only supports XARCH targets. For non-XARCH targets, we only support - // low deltas to avoid emitting huge jump tables. - break; - } -#endif - // TODO: currently we mainly focus on creating the shape that is then expanded into a bit test // Ideally we should create jump tables for other cases as well and we need some cost-benefit analysis. From 5f176f7831844b54672ada6b79cffcba89f42782 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 10 Sep 2023 20:14:42 +0200 Subject: [PATCH 35/35] Clean up --- src/coreclr/jit/lower.cpp | 2 +- src/coreclr/jit/switchrecognition.cpp | 4 +- .../optSwitchRecognition.cs | 73 ++++++------------- .../optSwitchRecognition.csproj | 1 - 4 files changed, 25 insertions(+), 55 deletions(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 62c6481b797da0..7983f4aad47e5d 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -1161,7 +1161,7 @@ bool Lowering::TryLowerSwitchToBitTest( // We'll ensure that there are only 2 targets when building the bit table. // - if (targetCount > 0) // test + if (targetCount > 3) { return false; } diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index 9a63b6497df3d9..702cc0dbf0207f 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. #include "jitpch.h" @@ -127,6 +127,8 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* maxValue = newMaxValue; } + assert(testIdx <= testsCount); + if (testIdx < SWITCH_MIN_TESTS) { // Make sure we still have at least SWITCH_MIN_TESTS values after we filtered out some of them diff --git a/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.cs b/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.cs index 58000d7b02d41a..1292ad4088e5bb 100644 --- a/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.cs +++ b/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.cs @@ -3,17 +3,18 @@ // unit test for Switch recognition optimization -using System; using System.Runtime.CompilerServices; using Xunit; +namespace optSwitchRecognition; + public class CSwitchRecognitionTest { // Test sorted char cases [MethodImpl(MethodImplOptions.NoInlining)] - public static bool RecSwitchSortedChar(char c) + private static bool RecSwitchSortedChar(char c) { - return (c == 'a' || c == 'b' || c == 'd' || c == 'f'); + return (c == 'a' || c == 'b' || c == 'd' || c == 'f') ? true : false; } [Theory] @@ -27,13 +28,13 @@ public static bool RecSwitchSortedChar(char c) [InlineData('A', false)] [InlineData('Z', false)] [InlineData('?', false)] - public static void TestRecSwitchSortedChar(char arg1,bool expected) => Assert.Equal(RecSwitchSortedChar(arg1), expected); + public static void TestRecSwitchSortedChar(char arg1, bool expected) => Assert.Equal(expected, RecSwitchSortedChar(arg1)); // Test unsorted char cases [MethodImpl(MethodImplOptions.NoInlining)] - public static bool RecSwitchUnsortedChar(char c) + private static bool RecSwitchUnsortedChar(char c) { - return (c == 'd' || c == 'f' || c == 'a' || c == 'b'); + return (c == 'd' || c == 'f' || c == 'a' || c == 'b') ? true : false; } [Theory] @@ -47,13 +48,13 @@ public static bool RecSwitchUnsortedChar(char c) [InlineData('A', false)] [InlineData('Z', false)] [InlineData('?', false)] - public static void TestRecSwitchUnsortedChar(char arg1, bool expected) => Assert.Equal(RecSwitchUnsortedChar(arg1), expected); + public static void TestRecSwitchUnsortedChar(char arg1, bool expected) => Assert.Equal(expected, RecSwitchUnsortedChar(arg1)); // Test sorted int cases [MethodImpl(MethodImplOptions.NoInlining)] - public static bool RecSwitchSortedInt(int i) + private static bool RecSwitchSortedInt(int i) { - return (i == -10 || i == -20 || i == 30 || i == 40); + return (i == -10 || i == -20 || i == 30 || i == 40) ? true : false; } [Theory] @@ -66,13 +67,13 @@ public static bool RecSwitchSortedInt(int i) [InlineData(40, true)] [InlineData(70, false)] [InlineData(100, false)] - public static void TestRecSwitchSortedInt(int arg1, bool expected) => Assert.Equal(RecSwitchSortedInt(arg1), expected); + public static void TestRecSwitchSortedInt(int arg1, bool expected) => Assert.Equal(expected, RecSwitchSortedInt(arg1)); // Test unsorted int cases [MethodImpl(MethodImplOptions.NoInlining)] - public static bool RecSwitchUnsortedInt(int i) + private static bool RecSwitchUnsortedInt(int i) { - return (i == 30 || i == 40 || i == -10 || i == -20); + return (i == 30 || i == 40 || i == -10 || i == -20) ? true : false; } [Theory] @@ -85,45 +86,13 @@ public static bool RecSwitchUnsortedInt(int i) [InlineData(40, true)] [InlineData(70, false)] [InlineData(100, false)] - public static void TestRecSwitchUnsortedInt(int arg1, bool expected) => Assert.Equal(RecSwitchUnsortedInt(arg1), expected); - - // Test min limits - [MethodImpl(MethodImplOptions.NoInlining)] - public static bool RecSwitchMinLimits(int i) - { - return (i == -9223372036854775807 || i == -9223372036854775806 - || i == -9223372036854775804 || i == -9223372036854775802); - } - - [Theory] - [InlineData(-9223372036854775807, true)] - [InlineData(-9223372036854775806, true)] - [InlineData(-9223372036854775805, false)] - [InlineData(-9223372036854775804, true)] - [InlineData(-9223372036854775802, true)] - public static void TestRecSwitchMinLimits(int arg1, bool expected) => Assert.Equal(RecSwitchMinLimits(arg1), expected); - - // Test max limits - [MethodImpl(MethodImplOptions.NoInlining)] - public static bool RecSwitchMaxLimits(int i) - { - return (i == 9223372036854775807 || i == 9223372036854775806 - || i == 9223372036854775804 || i == 9223372036854775802); - } - - [Theory] - [InlineData(9223372036854775807, true)] - [InlineData(9223372036854775806, true)] - [InlineData(9223372036854775805, false)] - [InlineData(9223372036854775804, true)] - [InlineData(9223372036854775802, true)] - public static void TestRecSwitchMaxLimits(int arg1, bool expected) => Assert.Equal(RecSwitchMaxLimits(arg1), expected); + public static void TestRecSwitchUnsortedInt(int arg1, bool expected) => Assert.Equal(expected, RecSwitchUnsortedInt(arg1)); // Test <= 64 switch cases [MethodImpl(MethodImplOptions.NoInlining)] - public static bool RecSwitch64JumpTables(int i) + private static bool RecSwitch64JumpTables(int i) { - return (i == 0 || i == 4 || i == 6 || i == 63); + return (i == 0 || i == 4 || i == 6 || i == 63) ? true : false; } [Theory] @@ -131,7 +100,7 @@ public static bool RecSwitch64JumpTables(int i) [InlineData(60, false)] [InlineData(63, true)] [InlineData(64, false)] - public static void TestRecSwitch64JumpTables(int arg1, bool expected) => Assert.Equal(RecSwitch64JumpTables(arg1), expected); + public static void TestRecSwitch64JumpTables(int arg1, bool expected) => Assert.Equal(expected, RecSwitch64JumpTables(arg1)); // // Skip optimization @@ -139,7 +108,7 @@ public static bool RecSwitch64JumpTables(int i) // Test > 64 Switch cases (should skip Switch Recognition optimization) [MethodImpl(MethodImplOptions.NoInlining)] - public static bool RecSwitch128JumpTables(int i) + private static bool RecSwitch128JumpTables(int i) { return (i == 0 || i == 4 || i == 6 || i == 127); } @@ -149,11 +118,11 @@ public static bool RecSwitch128JumpTables(int i) [InlineData(6, true)] [InlineData(127, true)] [InlineData(128, false)] - public static void TestRecSwitch128JumpTables(int arg1, bool expected) => Assert.Equal(RecSwitch128JumpTables(arg1), expected); + public static void TestRecSwitch128JumpTables(int arg1, bool expected) => Assert.Equal(expected, RecSwitch128JumpTables(arg1)); // Skips `bit test` conversion because Switch jump targets are > 2 (should skip Switch Recognition optimization) [MethodImpl(MethodImplOptions.NoInlining)] - public static int RecSwitchSkipBitTest(int arch) + private static int RecSwitchSkipBitTest(int arch) { if (arch == 1) return 2; @@ -169,5 +138,5 @@ public static int RecSwitchSkipBitTest(int arch) [InlineData(2, 4)] [InlineData(6, 4)] [InlineData(10, 1)] - public static void TestRecSwitchSkipBitTest(int arg1, int expected) => Assert.Equal(RecSwitchSkipBitTest(arg1), expected); + public static void TestRecSwitchSkipBitTest(int arg1, int expected) => Assert.Equal(expected, RecSwitchSkipBitTest(arg1)); } diff --git a/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.csproj b/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.csproj index b47c3e8e8d9f55..de6d5e08882e86 100644 --- a/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.csproj +++ b/src/tests/JIT/opt/OptSwitchRecognition/optSwitchRecognition.csproj @@ -1,6 +1,5 @@ - PdbOnly True