Implement DivRem intrinsic for X86

src/coreclr/jit/compiler.h

@@ -4497,6 +4497,10 @@ class Compiler
                               CorInfoType           simdBaseJitType,
                               var_types             retType,
                               unsigned              simdSize);
+    GenTree* impX86BaseIntrinsic(NamedIntrinsic        intrinsic,
+                                 CORINFO_METHOD_HANDLE method,
+                                 CORINFO_SIG_INFO*     sig,
+                                 CorInfoType           simdBaseJitType);
     GenTree* impSSEIntrinsic(NamedIntrinsic intrinsic, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig);
     GenTree* impSSE2Intrinsic(NamedIntrinsic intrinsic, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig);
     GenTree* impAvxOrAvx2Intrinsic(NamedIntrinsic intrinsic, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig);

src/coreclr/jit/gentree.h

@@ -8146,7 +8146,7 @@ inline var_types GenTree::GetRegTypeByIndex(int regIndex) const
 #elif defined(TARGET_XARCH)
         // At this time, the only multi-reg HW intrinsics all return the type of their
         // arguments. If this changes, we will need a way to record or determine this.
-        return gtGetOp1()->TypeGet();
+        return AsHWIntrinsic()->Op(1)->TypeGet();
 #endif
     }
 

src/coreclr/jit/hwintrinsic.h

@@ -744,6 +744,12 @@ struct HWIntrinsicInfo
                 return 2;
 #endif
 
+#ifdef TARGET_XARCH
+            case NI_X86Base_DivRem:
+            case NI_X86Base_X64_DivRem:
+                return 2;
+#endif // TARGET_XARCH
+
             default:
                 unreached();
         }

src/coreclr/jit/hwintrinsiccodegenxarch.cpp

@@ -1175,6 +1175,39 @@ void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node)
             break;
         }
 
+        case NI_X86Base_DivRem:
+        case NI_X86Base_X64_DivRem:
+        {
+            assert(node->GetOperandCount() == 3);
+
+            // SIMD base type is from signature and can distinguish signed and unsigned
+            var_types   targetType = node->GetSimdBaseType();
+            GenTree*    op1        = node->Op(1);
+            GenTree*    op2        = node->Op(2);
+            GenTree*    op3        = node->Op(3);
+            instruction ins        = HWIntrinsicInfo::lookupIns(intrinsicId, targetType);
+
+            regNumber op1Reg = op1->GetRegNum();
+            regNumber op2Reg = op2->GetRegNum();
+            regNumber op3Reg = op3->GetRegNum();
+
+            emitAttr attr = emitTypeSize(targetType);
+            emitter* emit = GetEmitter();
+
+            // op1: EAX, op2: EDX, op3: free
+            assert(op1Reg != REG_EDX);
+            assert(op2Reg != REG_EAX);
+            assert(op3Reg != REG_EDX);
+            assert(op3Reg != REG_EAX);
+            emit->emitIns_Mov(INS_mov, attr, REG_EAX, op1Reg, /* canSkip */ true);
+            emit->emitIns_Mov(INS_mov, attr, REG_EDX, op2Reg, /* canSkip */ true);
+
+            // emit the DIV/IDIV instruction
+            emit->emitInsBinary(ins, attr, node, op3);
+
+            break;
+        }
+
         default:
             unreached();
             break;

src/coreclr/jit/hwintrinsiclistxarch.h

@@ -224,6 +224,7 @@ HARDWARE_INTRINSIC(Vector256,       Xor,
 HARDWARE_INTRINSIC(X86Base,         BitScanForward,                             0,              1,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_bsf,                INS_bsf,                INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics)
 HARDWARE_INTRINSIC(X86Base,         BitScanReverse,                             0,              1,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_bsr,                INS_bsr,                INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics)
 HARDWARE_INTRINSIC(X86Base,         Pause,                                      0,              0,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid},           HW_Category_Special,                HW_Flag_NoContainment|HW_Flag_NoRMWSemantics)
+HARDWARE_INTRINSIC(X86Base,         DivRem,                                     0,              3,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_idiv,               INS_div,                INS_idiv,               INS_div,                INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
 
 // ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
 //                 ISA              Function name                               SIMD size       NumArg                                                                                                         Instructions                                                                                                                             Category                            Flags
@@ -232,6 +233,7 @@ HARDWARE_INTRINSIC(X86Base,         Pause,
 //  X86Base 64-bit-only Intrinsics
 HARDWARE_INTRINSIC(X86Base_X64,     BitScanForward,                             0,              1,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_bsf,                INS_bsf,                INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics)
 HARDWARE_INTRINSIC(X86Base_X64,     BitScanReverse,                             0,              1,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_bsr,                INS_bsr,                INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics)
+HARDWARE_INTRINSIC(X86Base_X64,     DivRem,                                     0,              3,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_idiv,               INS_div,                INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
 
 // ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
 //                 ISA              Function name                               SIMD size       NumArg                                                                                                         Instructions                                                                                                                             Category                            Flags

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -491,8 +491,10 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic        intrinsic,
     {
         case InstructionSet_Vector256:
         case InstructionSet_Vector128:
-        case InstructionSet_X86Base:
             return impBaseIntrinsic(intrinsic, clsHnd, method, sig, simdBaseJitType, retType, simdSize);
+        case InstructionSet_X86Base:
+        case InstructionSet_X86Base_X64:
+            return impX86BaseIntrinsic(intrinsic, method, sig, simdBaseJitType);
         case InstructionSet_SSE:
             return impSSEIntrinsic(intrinsic, method, sig);
         case InstructionSet_SSE2:
@@ -550,13 +552,8 @@ GenTree* Compiler::impBaseIntrinsic(NamedIntrinsic        intrinsic,
         return nullptr;
     }
 
-    var_types simdBaseType = TYP_UNKNOWN;
-
-    if (intrinsic != NI_X86Base_Pause)
-    {
-        simdBaseType = JitType2PreciseVarType(simdBaseJitType);
-        assert(varTypeIsArithmetic(simdBaseType));
-    }
+    var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
+    assert(varTypeIsArithmetic(simdBaseType));
 
     switch (intrinsic)
     {
@@ -2194,16 +2191,74 @@ GenTree* Compiler::impBaseIntrinsic(NamedIntrinsic        intrinsic,
             break;
         }
 
+        default:
+        {
+            return nullptr;
+        }
+    }
+
+    return retNode;
+}
+
+
+GenTree* Compiler::impX86BaseIntrinsic(NamedIntrinsic        intrinsic,
+                                       CORINFO_METHOD_HANDLE method,
+                                       CORINFO_SIG_INFO*     sig,
+                                       CorInfoType           simdBaseJitType)
+{
+    GenTree* retNode = nullptr;
+    GenTree* op1     = nullptr;
+    GenTree* op2     = nullptr;
+    GenTree* op3     = nullptr;
+    GenTree* op4     = nullptr;
+
+    var_types retType = JITtype2varType(sig->retType);
+
+    switch (intrinsic)
+    {
+
         case NI_X86Base_Pause:
         {
             assert(sig->numArgs == 0);
-            assert(JITtype2varType(sig->retType) == TYP_VOID);
-            assert(simdSize == 0);
+            assert(retType == TYP_VOID);
+            assert(simdBaseJitType == CORINFO_TYPE_UNDEF);
 
             retNode = gtNewScalarHWIntrinsicNode(TYP_VOID, intrinsic);
             break;
         }
 
+        case NI_X86Base_DivRem:
+        case NI_X86Base_X64_DivRem:
+        {
+            assert(sig->numArgs == 3);
+            assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
+            assert(retType == TYP_STRUCT);
+            assert(simdBaseJitType != CORINFO_TYPE_UNDEF);
+
+            op3 = impPopStack().val;
+            op2 = impPopStack().val;
+            op1 = impPopStack().val;
+
+            GenTreeHWIntrinsic* divRemIntrinsic = gtNewScalarHWIntrinsicNode(retType, op1, op2, op3, intrinsic);
+
+            // Store the type from signature into SIMD base type for convenience
+            divRemIntrinsic->SetSimdBaseJitType(simdBaseJitType);
+
+            const unsigned lclNum = lvaGrabTemp(true DEBUGARG("Return value temp for multireg intrinsic"));
+            impAssignTempGen(lclNum, divRemIntrinsic, sig->retTypeSigClass, (unsigned)CHECK_SPILL_ALL);
+
+            LclVarDsc* varDsc = lvaGetDesc(lclNum);
+            // The following is to exclude the fields of the local to have SSA.
+            varDsc->lvIsMultiRegRet = true;
+
+            GenTreeLclVar* lclVar = gtNewLclvNode(lclNum, varDsc->lvType);
+            lclVar->SetDoNotCSE();
+            lclVar->SetMultiReg();
+
+            retNode = lclVar;
+            break;
+        }
+
         default:
         {
             return nullptr;

src/coreclr/jit/lower.cpp

@@ -6749,7 +6749,7 @@ bool Lowering::NodesAreEquivalentLeaves(GenTree* tree1, GenTree* tree2)
 bool Lowering::CheckMultiRegLclVar(GenTreeLclVar* lclNode, const ReturnTypeDesc* retTypeDesc)
 {
     bool canEnregister = false;
-#if FEATURE_MULTIREG_RET
+#if FEATURE_MULTIREG_RET || FEATURE_HW_INTRINSICS
     LclVarDsc* varDsc = comp->lvaGetDesc(lclNode->GetLclNum());
     if ((comp->lvaEnregMultiRegVars) && varDsc->lvPromoted)
     {
@@ -6776,10 +6776,10 @@ bool Lowering::CheckMultiRegLclVar(GenTreeLclVar* lclNode, const ReturnTypeDesc*
         }
     }
 #ifdef TARGET_XARCH
-    // For local stores on XARCH we only handle mismatched src/dest register count for
-    // calls of SIMD type. If the source was another lclVar similarly promoted, we would
+    // For local stores on XARCH we can't handle another lclVar source.
+    // If the source was another lclVar similarly promoted, we would
     // have broken it into multiple stores.
-    if (lclNode->OperIs(GT_STORE_LCL_VAR) && !lclNode->gtGetOp1()->OperIs(GT_CALL))
+    if (lclNode->OperIs(GT_STORE_LCL_VAR) && lclNode->gtGetOp1()->OperIs(GT_LCL_VAR))
     {
         canEnregister = false;
     }

src/coreclr/jit/lowerxarch.cpp

@@ -384,6 +384,16 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode)
 
             ContainBlockStoreAddress(blkNode, size, dstAddr);
         }
+        else if (src->OperIsHWIntrinsic())
+        {
+            assert(!blkNode->AsObj()->GetLayout()->HasGCPtr());
+            if (blkNode->OperIs(GT_STORE_OBJ))
+            {
+                blkNode->SetOper(GT_STORE_BLK);
+            }
+            blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll;
+            ContainBlockStoreAddress(blkNode, size, dstAddr);
+        }
         else
         {
             assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK));
@@ -6478,6 +6488,20 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
                                 }
                                 break;
                             }
+                            case NI_X86Base_DivRem:
+                            case NI_X86Base_X64_DivRem:
+                            {
+                                // DIV only allows divisor (op3) in memory
+                                if (TryGetContainableHWIntrinsicOp(node, &op3, &supportsRegOptional))
+                                {
+                                    MakeSrcContained(node, op3);
+                                }
+                                else if (supportsRegOptional)
+                                {
+                                    op3->SetRegOptional();
+                                }
+                                break;
+                            }
 
                             default:
                             {

src/coreclr/jit/lsraxarch.cpp

@@ -2015,7 +2015,23 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
     }
 
     int srcCount = 0;
-    int dstCount = intrinsicTree->IsValue() ? 1 : 0;
+    int dstCount;
+
+    if (intrinsicTree->IsValue())
+    {
+        if (HWIntrinsicInfo::IsMultiReg(intrinsicId))
+        {
+            dstCount = HWIntrinsicInfo::GetMultiRegCount(intrinsicId);
+        }
+        else
+        {
+            dstCount = 1;
+        }
+    }
+    else
+    {
+        dstCount = 0;
+    }
 
     regMaskTP dstCandidates = RBM_NONE;
 
@@ -2193,6 +2209,26 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
             }
 #endif // TARGET_X86
 
+            case NI_X86Base_DivRem:
+            case NI_X86Base_X64_DivRem:
+            {
+                assert(numArgs == 3);
+                assert(dstCount == 2);
+                assert(!isRMW);
+
+                // DIV implicitly put op1(lower) to EAX and op2(upper) to EDX
+                srcCount += BuildOperandUses(op1, RBM_EAX);
+                srcCount += BuildOperandUses(op2, RBM_EDX);
+                srcCount += BuildOperandUses(op3);
+
+                // result put in EAX and EDX
+                BuildDef(intrinsicTree, RBM_EAX, 0);
+                BuildDef(intrinsicTree, RBM_EDX, 1);
+
+                buildUses = false;
+                break;
+            }
+
             case NI_BMI2_MultiplyNoFlags:
             case NI_BMI2_X64_MultiplyNoFlags:
             {
@@ -2463,7 +2499,8 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
     }
     else
     {
-        assert(dstCount == 0);
+        // Currently dstCount = 2 is only used for DivRem, which has special constriants and handled above
+        assert((dstCount == 0) || (dstCount == 2));
     }
 
     *pDstCount = dstCount;

src/coreclr/jit/morph.cpp

@@ -10485,7 +10485,7 @@ GenTree* Compiler::fgMorphBlockOperand(GenTree* tree, var_types asgType, unsigne
             assert(blockWidth == info.compCompHnd->getClassSize(call->gtRetClsHnd));
 #endif
         }
-#ifdef TARGET_ARM64
+#ifdef FEATURE_HW_INTRINSICS
         else if (effectiveVal->OperIsHWIntrinsic())
         {
             needsIndirection = false;
@@ -10495,7 +10495,7 @@ GenTree* Compiler::fgMorphBlockOperand(GenTree* tree, var_types asgType, unsigne
             assert(HWIntrinsicInfo::IsMultiReg(intrinsic->GetHWIntrinsicId()));
 #endif
         }
-#endif // TARGET_ARM64
+#endif // FEATURE_HW_INTRINSICS
 
         if (lclNode != nullptr)
         {