diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 1e734193850ee..b3dd5beaa89c6 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -98,8 +98,8 @@ class CodeGen final : public CodeGenInterface static bool genShouldRoundFP(); - GenTreeIndir indirForm(var_types type, GenTree* base); - GenTreeStoreInd storeIndirForm(var_types type, GenTree* base, GenTree* data); + static GenTreeIndir indirForm(var_types type, GenTree* base); + static GenTreeStoreInd storeIndirForm(var_types type, GenTree* base, GenTree* data); GenTreeIntCon intForm(var_types type, ssize_t value); @@ -1410,6 +1410,139 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void inst_RV_SH(instruction ins, emitAttr size, regNumber reg, unsigned val, insFlags flags = INS_FLAGS_DONT_CARE); #if defined(TARGET_XARCH) + + enum class OperandKind{ + ClsVar, // [CLS_VAR_ADDR] - "C" in the emitter. + Local, // [Local or spill temp + offset] - "S" in the emitter. + Indir, // [base+index*scale+disp] - "A" in the emitter. + Imm, // immediate - "I" in the emitter. + Reg // reg - "R" in the emitter. + }; + + class OperandDesc + { + OperandKind m_kind; + union { + struct + { + CORINFO_FIELD_HANDLE m_fieldHnd; + }; + struct + { + int m_varNum; + uint16_t m_offset; + }; + struct + { + GenTree* m_addr; + GenTreeIndir* m_indir; + var_types m_indirType; + }; + struct + { + ssize_t m_immediate; + bool m_immediateNeedsReloc; + }; + struct + { + regNumber m_reg; + }; + }; + + public: + OperandDesc(CORINFO_FIELD_HANDLE fieldHnd) : m_kind(OperandKind::ClsVar), m_fieldHnd(fieldHnd) + { + } + + OperandDesc(int varNum, uint16_t offset) : m_kind(OperandKind::Local), m_varNum(varNum), m_offset(offset) + { + } + + OperandDesc(GenTreeIndir* indir) + : m_kind(OperandKind::Indir), m_addr(indir->Addr()), m_indir(indir), m_indirType(indir->TypeGet()) + { + } + + OperandDesc(var_types indirType, GenTree* addr) + : m_kind(OperandKind::Indir), m_addr(addr), m_indir(nullptr), m_indirType(indirType) + { + } + + OperandDesc(ssize_t immediate, bool immediateNeedsReloc) + : m_kind(OperandKind::Imm), m_immediate(immediate), m_immediateNeedsReloc(immediateNeedsReloc) + { + } + + OperandDesc(regNumber reg) : m_kind(OperandKind::Reg), m_reg(reg) + { + } + + OperandKind GetKind() const + { + return m_kind; + } + + CORINFO_FIELD_HANDLE GetFieldHnd() const + { + assert(m_kind == OperandKind::ClsVar); + return m_fieldHnd; + } + + int GetVarNum() const + { + assert(m_kind == OperandKind::Local); + return m_varNum; + } + + int GetLclOffset() const + { + assert(m_kind == OperandKind::Local); + return m_offset; + } + + // TODO-Cleanup: instead of this rather unsightly workaround with + // "indirForm", create a new abstraction for address modes to pass + // to the emitter (or at least just use "addr"...). + GenTreeIndir* GetIndirForm(GenTreeIndir* pIndirForm) + { + if (m_indir == nullptr) + { + GenTreeIndir indirForm = CodeGen::indirForm(m_indirType, m_addr); + memcpy(pIndirForm, &indirForm, sizeof(GenTreeIndir)); + } + else + { + pIndirForm = m_indir; + } + + return pIndirForm; + } + + ssize_t GetImmediate() const + { + assert(m_kind == OperandKind::Imm); + return m_immediate; + } + + bool ImmediateNeedsReloc() const + { + assert(m_kind == OperandKind::Imm); + return m_immediateNeedsReloc; + } + + regNumber GetReg() const + { + return m_reg; + } + + bool IsContained() const + { + return m_kind != OperandKind::Reg; + } + }; + + OperandDesc genOperandDesc(GenTree* op); + void inst_RV_RV_IV(instruction ins, emitAttr size, regNumber reg1, regNumber reg2, unsigned ival); void inst_RV_TT_IV(instruction ins, emitAttr attr, regNumber reg1, GenTree* rmOp, int ival); void inst_RV_RV_TT(instruction ins, emitAttr size, regNumber targetReg, regNumber op1Reg, GenTree* op2, bool isRMW); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 81e2dddff69e8..d56d3e86d70ed 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -10924,13 +10924,12 @@ const char* CodeGen::siStackVarName(size_t offs, size_t size, unsigned reg, unsi /*****************************************************************************/ #endif // !defined(DEBUG) #endif // defined(LATE_DISASM) -/*****************************************************************************/ //------------------------------------------------------------------------ // indirForm: Make a temporary indir we can feed to pattern matching routines // in cases where we don't want to instantiate all the indirs that happen. // -GenTreeIndir CodeGen::indirForm(var_types type, GenTree* base) +/* static */ GenTreeIndir CodeGen::indirForm(var_types type, GenTree* base) { GenTreeIndir i(GT_IND, type, base, nullptr); i.SetRegNum(REG_NA); @@ -10942,7 +10941,7 @@ GenTreeIndir CodeGen::indirForm(var_types type, GenTree* base) // indirForm: Make a temporary indir we can feed to pattern matching routines // in cases where we don't want to instantiate all the indirs that happen. // -GenTreeStoreInd CodeGen::storeIndirForm(var_types type, GenTree* base, GenTree* data) +/* static */ GenTreeStoreInd CodeGen::storeIndirForm(var_types type, GenTree* base, GenTree* data) { GenTreeStoreInd i(type, base, data); i.SetRegNum(REG_NA); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 5d37fb3656265..990229d5b7d20 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -6086,12 +6086,15 @@ struct GenTreeIndir : public GenTreeOp } #if DEBUGGABLE_GENTREE -protected: - friend GenTree; // Used only for GenTree::GetVtableForOper() GenTreeIndir() : GenTreeOp() { } +#else + // Used by XARCH codegen to construct temporary trees to pass to the emitter. + GenTreeIndir() : GenTreeOp(GT_NOP, TYP_UNDEF) + { + } #endif }; diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index c2c453362f02f..34229d6fff61b 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -447,127 +447,48 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) void CodeGen::genHWIntrinsic_R_RM( GenTreeHWIntrinsic* node, instruction ins, emitAttr attr, regNumber reg, GenTree* rmOp) { - emitter* emit = GetEmitter(); - - assert(reg != REG_NA); + emitter* emit = GetEmitter(); + OperandDesc rmOpDesc = genOperandDesc(rmOp); - if (rmOp->isContained() || rmOp->isUsedFromSpillTemp()) + if (rmOpDesc.IsContained()) { assert(HWIntrinsicInfo::SupportsContainment(node->GetHWIntrinsicId())); assertIsContainableHWIntrinsicOp(compiler->m_pLowering, node, rmOp); + } - TempDsc* tmpDsc = nullptr; - unsigned varNum = BAD_VAR_NUM; - unsigned offset = (unsigned)-1; - - if (rmOp->isUsedFromSpillTemp()) - { - assert(rmOp->IsRegOptional()); + switch (rmOpDesc.GetKind()) + { + case OperandKind::ClsVar: + emit->emitIns_R_C(ins, attr, reg, rmOpDesc.GetFieldHnd(), 0); + break; - tmpDsc = getSpillTempDsc(rmOp); - varNum = tmpDsc->tdTempNum(); - offset = 0; + case OperandKind::Local: + emit->emitIns_R_S(ins, attr, reg, rmOpDesc.GetVarNum(), rmOpDesc.GetLclOffset()); + break; - regSet.tmpRlsTemp(tmpDsc); - } - else if (rmOp->isIndir() || rmOp->OperIsHWIntrinsic()) + case OperandKind::Indir: { - GenTree* addr; - GenTreeIndir* memIndir = nullptr; + // Until we improve the handling of addressing modes in the emitter, we'll create a + // temporary GT_IND to generate code with. + GenTreeIndir indirForm; + GenTreeIndir* indir = rmOpDesc.GetIndirForm(&indirForm); + emit->emitIns_R_A(ins, attr, reg, indir); + } + break; - if (rmOp->isIndir()) + case OperandKind::Reg: + if (emit->IsMovInstruction(ins)) { - memIndir = rmOp->AsIndir(); - addr = memIndir->Addr(); + emit->emitIns_Mov(ins, attr, reg, rmOp->GetRegNum(), /* canSkip */ false); } else { - assert(rmOp->AsHWIntrinsic()->OperIsMemoryLoad()); - assert(rmOp->AsHWIntrinsic()->GetOperandCount() == 1); - addr = rmOp->AsHWIntrinsic()->Op(1); - } - - switch (addr->OperGet()) - { - case GT_LCL_VAR_ADDR: - case GT_LCL_FLD_ADDR: - { - assert(addr->isContained()); - varNum = addr->AsLclVarCommon()->GetLclNum(); - offset = addr->AsLclVarCommon()->GetLclOffs(); - break; - } - - case GT_CLS_VAR_ADDR: - { - emit->emitIns_R_C(ins, attr, reg, addr->AsClsVar()->gtClsVarHnd, 0); - return; - } - - default: - { - GenTreeIndir load = indirForm(rmOp->TypeGet(), addr); - - if (memIndir == nullptr) - { - // This is the HW intrinsic load case. - // Until we improve the handling of addressing modes in the emitter, we'll create a - // temporary GT_IND to generate code with. - memIndir = &load; - } - emit->emitIns_R_A(ins, attr, reg, memIndir); - return; - } - } - } - else - { - switch (rmOp->OperGet()) - { - case GT_LCL_FLD: - varNum = rmOp->AsLclFld()->GetLclNum(); - offset = rmOp->AsLclFld()->GetLclOffs(); - break; - - case GT_LCL_VAR: - { - assert(rmOp->IsRegOptional() || !compiler->lvaGetDesc(rmOp->AsLclVar())->lvIsRegCandidate()); - varNum = rmOp->AsLclVar()->GetLclNum(); - offset = 0; - break; - } - - case GT_CNS_DBL: - { - GenTreeDblCon* cns = rmOp->AsDblCon(); - CORINFO_FIELD_HANDLE hnd = emit->emitFltOrDblConst(cns->gtDconVal, emitTypeSize(cns)); - emit->emitIns_R_C(ins, attr, reg, hnd, 0); - return; - } - - default: - { - unreached(); - break; - } + emit->emitIns_R_R(ins, attr, reg, rmOp->GetRegNum()); } - } - - // Ensure we got a good varNum and offset. - // We also need to check for `tmpDsc != nullptr` since spill temp numbers - // are negative and start with -1, which also happens to be BAD_VAR_NUM. - assert((varNum != BAD_VAR_NUM) || (tmpDsc != nullptr)); - assert(offset != (unsigned)-1); + break; - emit->emitIns_R_S(ins, attr, reg, varNum, offset); - } - else if (emit->IsMovInstruction(ins)) - { - emit->emitIns_Mov(ins, attr, reg, rmOp->GetRegNum(), /* canSkip */ false); - } - else - { - emit->emitIns_R_R(ins, attr, reg, rmOp->GetRegNum()); + default: + unreached(); } } @@ -673,135 +594,59 @@ void CodeGen::genHWIntrinsic_R_R_RM_I(GenTreeHWIntrinsic* node, instruction ins, assert(targetReg != REG_NA); assert(op1Reg != REG_NA); - if (op2->isContained() || op2->isUsedFromSpillTemp()) + OperandDesc op2Desc = genOperandDesc(op2); + + if (op2Desc.IsContained()) { assert(HWIntrinsicInfo::SupportsContainment(node->GetHWIntrinsicId())); assertIsContainableHWIntrinsicOp(compiler->m_pLowering, node, op2); + } - TempDsc* tmpDsc = nullptr; - unsigned varNum = BAD_VAR_NUM; - unsigned offset = (unsigned)-1; - - if (op2->isUsedFromSpillTemp()) - { - assert(op2->IsRegOptional()); + switch (op2Desc.GetKind()) + { + case OperandKind::ClsVar: + emit->emitIns_SIMD_R_R_C_I(ins, simdSize, targetReg, op1Reg, op2Desc.GetFieldHnd(), 0, ival); + break; - tmpDsc = getSpillTempDsc(op2); - varNum = tmpDsc->tdTempNum(); - offset = 0; + case OperandKind::Local: + emit->emitIns_SIMD_R_R_S_I(ins, simdSize, targetReg, op1Reg, op2Desc.GetVarNum(), op2Desc.GetLclOffset(), + ival); + break; - regSet.tmpRlsTemp(tmpDsc); - } - else if (op2->isIndir() || op2->OperIsHWIntrinsic()) + case OperandKind::Indir: { - GenTree* addr; - GenTreeIndir* memIndir = nullptr; - - if (op2->isIndir()) - { - memIndir = op2->AsIndir(); - addr = memIndir->Addr(); - } - else - { - assert(op2->AsHWIntrinsic()->OperIsMemoryLoad()); - assert(op2->AsHWIntrinsic()->GetOperandCount() == 1); - addr = op2->AsHWIntrinsic()->Op(1); - } - - switch (addr->OperGet()) - { - case GT_LCL_VAR_ADDR: - case GT_LCL_FLD_ADDR: - { - assert(addr->isContained()); - varNum = addr->AsLclVarCommon()->GetLclNum(); - offset = addr->AsLclVarCommon()->GetLclOffs(); - break; - } - - case GT_CLS_VAR_ADDR: - { - emit->emitIns_SIMD_R_R_C_I(ins, simdSize, targetReg, op1Reg, addr->AsClsVar()->gtClsVarHnd, 0, - ival); - return; - } - - default: - { - GenTreeIndir load = indirForm(op2->TypeGet(), addr); - - if (memIndir == nullptr) - { - // This is the HW intrinsic load case. - // Until we improve the handling of addressing modes in the emitter, we'll create a - // temporary GT_IND to generate code with. - memIndir = &load; - } - emit->emitIns_SIMD_R_R_A_I(ins, simdSize, targetReg, op1Reg, memIndir, ival); - return; - } - } + // Until we improve the handling of addressing modes in the emitter, we'll create a + // temporary GT_IND to generate code with. + GenTreeIndir indirForm; + GenTreeIndir* indir = op2Desc.GetIndirForm(&indirForm); + emit->emitIns_SIMD_R_R_A_I(ins, simdSize, targetReg, op1Reg, indir, ival); } - else - { - switch (op2->OperGet()) - { - case GT_LCL_FLD: - varNum = op2->AsLclFld()->GetLclNum(); - offset = op2->AsLclFld()->GetLclOffs(); - break; - - case GT_LCL_VAR: - { - assert(op2->IsRegOptional() || - !compiler->lvaTable[op2->AsLclVar()->GetLclNum()].lvIsRegCandidate()); - varNum = op2->AsLclVar()->GetLclNum(); - offset = 0; - break; - } + break; - case GT_CNS_DBL: - { - GenTreeDblCon* cns = op2->AsDblCon(); - CORINFO_FIELD_HANDLE hnd = emit->emitFltOrDblConst(cns->gtDconVal, emitTypeSize(cns)); - emit->emitIns_SIMD_R_R_C_I(ins, simdSize, targetReg, op1Reg, hnd, 0, ival); - return; - } + case OperandKind::Reg: + { + regNumber op2Reg = op2->GetRegNum(); - default: - unreached(); - break; + if ((op1Reg != targetReg) && (op2Reg == targetReg) && node->isRMWHWIntrinsic(compiler)) + { + // We have "reg2 = reg1 op reg2" where "reg1 != reg2" on a RMW intrinsic. + // + // For non-commutative intrinsics, we should have ensured that op2 was marked + // delay free in order to prevent it from getting assigned the same register + // as target. However, for commutative intrinsics, we can just swap the operands + // in order to have "reg2 = reg2 op reg1" which will end up producing the right code. + + noway_assert(node->OperIsCommutative()); + op2Reg = op1Reg; + op1Reg = targetReg; } - } - - // Ensure we got a good varNum and offset. - // We also need to check for `tmpDsc != nullptr` since spill temp numbers - // are negative and start with -1, which also happens to be BAD_VAR_NUM. - assert((varNum != BAD_VAR_NUM) || (tmpDsc != nullptr)); - assert(offset != (unsigned)-1); - - emit->emitIns_SIMD_R_R_S_I(ins, simdSize, targetReg, op1Reg, varNum, offset, ival); - } - else - { - regNumber op2Reg = op2->GetRegNum(); - if ((op1Reg != targetReg) && (op2Reg == targetReg) && node->isRMWHWIntrinsic(compiler)) - { - // We have "reg2 = reg1 op reg2" where "reg1 != reg2" on a RMW intrinsic. - // - // For non-commutative intrinsics, we should have ensured that op2 was marked - // delay free in order to prevent it from getting assigned the same register - // as target. However, for commutative intrinsics, we can just swap the operands - // in order to have "reg2 = reg2 op reg1" which will end up producing the right code. - - noway_assert(node->OperIsCommutative()); - op2Reg = op1Reg; - op1Reg = targetReg; + emit->emitIns_SIMD_R_R_R_I(ins, simdSize, targetReg, op1Reg, op2Reg, ival); } + break; - emit->emitIns_SIMD_R_R_R_I(ins, simdSize, targetReg, op1Reg, op2Reg, ival); + default: + unreached(); } } @@ -828,121 +673,41 @@ void CodeGen::genHWIntrinsic_R_R_RM_R(GenTreeHWIntrinsic* node, instruction ins, assert(op1Reg != REG_NA); assert(op3Reg != REG_NA); - if (op2->isContained() || op2->isUsedFromSpillTemp()) + OperandDesc op2Desc = genOperandDesc(op2); + + if (op2Desc.IsContained()) { assert(HWIntrinsicInfo::SupportsContainment(node->GetHWIntrinsicId())); assertIsContainableHWIntrinsicOp(compiler->m_pLowering, node, op2); + } - TempDsc* tmpDsc = nullptr; - unsigned varNum = BAD_VAR_NUM; - unsigned offset = (unsigned)-1; - - if (op2->isUsedFromSpillTemp()) - { - assert(op2->IsRegOptional()); - - // TODO-XArch-Cleanup: The getSpillTempDsc...tempRlsTemp code is a fairly common - // pattern. It could probably be extracted to its own method. - tmpDsc = getSpillTempDsc(op2); - varNum = tmpDsc->tdTempNum(); - offset = 0; - - regSet.tmpRlsTemp(tmpDsc); - } - else if (op2->isIndir() || op2->OperIsHWIntrinsic()) - { - GenTree* addr; - GenTreeIndir* memIndir = nullptr; - - if (op2->isIndir()) - { - memIndir = op2->AsIndir(); - addr = memIndir->Addr(); - } - else - { - assert(op2->AsHWIntrinsic()->OperIsMemoryLoad()); - assert(op2->AsHWIntrinsic()->GetOperandCount() == 1); - addr = op2->AsHWIntrinsic()->Op(1); - } - - switch (addr->OperGet()) - { - case GT_LCL_VAR_ADDR: - case GT_LCL_FLD_ADDR: - { - assert(addr->isContained()); - varNum = addr->AsLclVarCommon()->GetLclNum(); - offset = addr->AsLclVarCommon()->GetLclOffs(); - break; - } - - case GT_CLS_VAR_ADDR: - { - emit->emitIns_SIMD_R_R_C_R(ins, simdSize, targetReg, op1Reg, op3Reg, addr->AsClsVar()->gtClsVarHnd, - 0); - return; - } + switch (op2Desc.GetKind()) + { + case OperandKind::ClsVar: + emit->emitIns_SIMD_R_R_C_R(ins, simdSize, targetReg, op1Reg, op3Reg, op2Desc.GetFieldHnd(), 0); + break; - default: - { - GenTreeIndir load = indirForm(op2->TypeGet(), addr); + case OperandKind::Local: + emit->emitIns_SIMD_R_R_S_R(ins, simdSize, targetReg, op1Reg, op3Reg, op2Desc.GetVarNum(), + op2Desc.GetLclOffset()); + break; - if (memIndir == nullptr) - { - // This is the HW intrinsic load case. - // Until we improve the handling of addressing modes in the emitter, we'll create a - // temporary GT_IND to generate code with. - memIndir = &load; - } - emit->emitIns_SIMD_R_R_A_R(ins, simdSize, targetReg, op1Reg, op3Reg, memIndir); - return; - } - } - } - else + case OperandKind::Indir: { - switch (op2->OperGet()) - { - case GT_LCL_FLD: - varNum = op2->AsLclFld()->GetLclNum(); - offset = op2->AsLclFld()->GetLclOffs(); - break; - - case GT_LCL_VAR: - { - assert(op2->IsRegOptional() || - !compiler->lvaTable[op2->AsLclVar()->GetLclNum()].lvIsRegCandidate()); - varNum = op2->AsLclVar()->GetLclNum(); - offset = 0; - break; - } - - case GT_CNS_DBL: - { - GenTreeDblCon* cns = op2->AsDblCon(); - CORINFO_FIELD_HANDLE hnd = emit->emitFltOrDblConst(cns->gtDconVal, emitTypeSize(cns)); - emit->emitIns_SIMD_R_R_C_R(ins, simdSize, targetReg, op1Reg, op3Reg, hnd, 0); - return; - } - - default: - unreached(); - break; - } + // Until we improve the handling of addressing modes in the emitter, we'll create a + // temporary GT_IND to generate code with. + GenTreeIndir indirForm; + GenTreeIndir* indir = op2Desc.GetIndirForm(&indirForm); + emit->emitIns_SIMD_R_R_A_R(ins, simdSize, targetReg, op1Reg, op3Reg, indir); } + break; - // Ensure we got a good varNum and offset. - // We also need to check for `tmpDsc != nullptr` since spill temp numbers - // are negative and start with -1, which also happens to be BAD_VAR_NUM. - assert((varNum != BAD_VAR_NUM) || (tmpDsc != nullptr)); - assert(offset != (unsigned)-1); + case OperandKind::Reg: + emit->emitIns_SIMD_R_R_R_R(ins, simdSize, targetReg, op1Reg, op2->GetRegNum(), op3Reg); + break; - emit->emitIns_SIMD_R_R_S_R(ins, simdSize, targetReg, op1Reg, op3Reg, varNum, offset); - } - else - { - emit->emitIns_SIMD_R_R_R_R(ins, simdSize, targetReg, op1Reg, op2->GetRegNum(), op3Reg); + default: + unreached(); } } @@ -965,118 +730,36 @@ void CodeGen::genHWIntrinsic_R_R_R_RM( assert(op1Reg != REG_NA); assert(op2Reg != REG_NA); - emitter* emit = GetEmitter(); + emitter* emit = GetEmitter(); + OperandDesc op3Desc = genOperandDesc(op3); - if (op3->isContained() || op3->isUsedFromSpillTemp()) + switch (op3Desc.GetKind()) { - TempDsc* tmpDsc = nullptr; - unsigned varNum = BAD_VAR_NUM; - unsigned offset = (unsigned)-1; - - if (op3->isUsedFromSpillTemp()) - { - assert(op3->IsRegOptional()); - - // TODO-XArch-Cleanup: The getSpillTempDsc...tempRlsTemp code is a fairly common - // pattern. It could probably be extracted to its own method. - tmpDsc = getSpillTempDsc(op3); - varNum = tmpDsc->tdTempNum(); - offset = 0; - - regSet.tmpRlsTemp(tmpDsc); - } - else if (op3->isIndir() || op3->OperIsHWIntrinsic()) - { - GenTree* addr; - GenTreeIndir* memIndir = nullptr; - if (op3->isIndir()) - { - memIndir = op3->AsIndir(); - addr = memIndir->Addr(); - } - else - { - assert(op3->AsHWIntrinsic()->OperIsMemoryLoad()); - assert(op3->AsHWIntrinsic()->GetOperandCount() == 1); - addr = op3->AsHWIntrinsic()->Op(1); - } - - switch (addr->OperGet()) - { - case GT_LCL_VAR_ADDR: - case GT_LCL_FLD_ADDR: - { - assert(addr->isContained()); - varNum = addr->AsLclVarCommon()->GetLclNum(); - offset = addr->AsLclVarCommon()->GetLclOffs(); - break; - } - - case GT_CLS_VAR_ADDR: - { - emit->emitIns_SIMD_R_R_R_C(ins, attr, targetReg, op1Reg, op2Reg, addr->AsClsVar()->gtClsVarHnd, 0); - return; - } + case OperandKind::ClsVar: + emit->emitIns_SIMD_R_R_R_C(ins, attr, targetReg, op1Reg, op2Reg, op3Desc.GetFieldHnd(), 0); + break; - default: - { - GenTreeIndir load = indirForm(op3->TypeGet(), addr); + case OperandKind::Local: + emit->emitIns_SIMD_R_R_R_S(ins, attr, targetReg, op1Reg, op2Reg, op3Desc.GetVarNum(), + op3Desc.GetLclOffset()); + break; - if (memIndir == nullptr) - { - // This is the HW intrinsic load case. - // Until we improve the handling of addressing modes in the emitter, we'll create a - // temporary GT_IND to generate code with. - memIndir = &load; - } - emit->emitIns_SIMD_R_R_R_A(ins, attr, targetReg, op1Reg, op2Reg, memIndir); - return; - } - } - } - else + case OperandKind::Indir: { - switch (op3->OperGet()) - { - case GT_LCL_FLD: - varNum = op3->AsLclFld()->GetLclNum(); - offset = op3->AsLclFld()->GetLclOffs(); - break; - - case GT_LCL_VAR: - { - assert(op3->IsRegOptional() || - !compiler->lvaTable[op3->AsLclVar()->GetLclNum()].lvIsRegCandidate()); - varNum = op3->AsLclVar()->GetLclNum(); - offset = 0; - break; - } - - case GT_CNS_DBL: - { - GenTreeDblCon* cns = op3->AsDblCon(); - CORINFO_FIELD_HANDLE hnd = emit->emitFltOrDblConst(cns->gtDconVal, emitTypeSize(cns)); - emit->emitIns_SIMD_R_R_R_C(ins, attr, targetReg, op1Reg, op2Reg, hnd, 0); - return; - } - - default: - unreached(); - break; - } + // Until we improve the handling of addressing modes in the emitter, we'll create a + // temporary GT_IND to generate code with. + GenTreeIndir indirForm; + GenTreeIndir* indir = op3Desc.GetIndirForm(&indirForm); + emit->emitIns_SIMD_R_R_R_A(ins, attr, targetReg, op1Reg, op2Reg, indir); } + break; - // Ensure we got a good varNum and offset. - // We also need to check for `tmpDsc != nullptr` since spill temp numbers - // are negative and start with -1, which also happens to be BAD_VAR_NUM. - assert((varNum != BAD_VAR_NUM) || (tmpDsc != nullptr)); - assert(offset != (unsigned)-1); + case OperandKind::Reg: + emit->emitIns_SIMD_R_R_R_R(ins, attr, targetReg, op1Reg, op2Reg, op3->GetRegNum()); + break; - emit->emitIns_SIMD_R_R_R_S(ins, attr, targetReg, op1Reg, op2Reg, varNum, offset); - } - else - { - emit->emitIns_SIMD_R_R_R_R(ins, attr, targetReg, op1Reg, op2Reg, op3->GetRegNum()); + default: + unreached(); } } diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index e0ecfeab9f084..fc33d9f06a0ee 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -988,12 +988,128 @@ void CodeGen::inst_RV_SH( #endif // TARGET* } +#if defined(TARGET_XARCH) +//------------------------------------------------------------------------ +// genOperandDesc: Create an operand descriptor for the given operand node. +// +// The XARCH emitter requires codegen to use different methods for different +// kinds of operands. However, the logic for determining which ones, in +// general, is not simple (due to the fact that "memory" in the emitter can +// be represented in more than one way). This helper method encapsulated the +// logic for determining what "kind" of operand "op" is. +// +// Arguments: +// op - The operand node for which to obtain the descriptor +// +// Return Value: +// The operand descriptor for "op". +// +// Notes: +// This method is not idempotent - it can only be called once for a +// given node. +// +CodeGen::OperandDesc CodeGen::genOperandDesc(GenTree* op) +{ + if (!op->isContained() && !op->isUsedFromSpillTemp()) + { + return OperandDesc(op->GetRegNum()); + } + + emitter* emit = GetEmitter(); + TempDsc* tmpDsc = nullptr; + unsigned varNum = BAD_VAR_NUM; + uint16_t offset = UINT16_MAX; + + if (op->isUsedFromSpillTemp()) + { + assert(op->IsRegOptional()); + + tmpDsc = getSpillTempDsc(op); + varNum = tmpDsc->tdTempNum(); + offset = 0; + + regSet.tmpRlsTemp(tmpDsc); + } + else if (op->isIndir() || op->OperIsHWIntrinsic()) + { + GenTree* addr; + GenTreeIndir* memIndir = nullptr; + + if (op->isIndir()) + { + memIndir = op->AsIndir(); + addr = memIndir->Addr(); + } + else + { +#if defined(FEATURE_HW_INTRINSICS) + assert(op->AsHWIntrinsic()->OperIsMemoryLoad()); + assert(op->AsHWIntrinsic()->GetOperandCount() == 1); + addr = op->AsHWIntrinsic()->Op(1); +#else + unreached(); +#endif // FEATURE_HW_INTRINSICS + } + + switch (addr->OperGet()) + { + case GT_LCL_VAR_ADDR: + case GT_LCL_FLD_ADDR: + { + assert(addr->isContained()); + varNum = addr->AsLclVarCommon()->GetLclNum(); + offset = addr->AsLclVarCommon()->GetLclOffs(); + break; + } + + case GT_CLS_VAR_ADDR: + return OperandDesc(addr->AsClsVar()->gtClsVarHnd); + + default: + return (memIndir != nullptr) ? OperandDesc(memIndir) : OperandDesc(op->TypeGet(), addr); + } + } + else + { + switch (op->OperGet()) + { + case GT_LCL_FLD: + varNum = op->AsLclFld()->GetLclNum(); + offset = op->AsLclFld()->GetLclOffs(); + break; + + case GT_LCL_VAR: + assert(op->IsRegOptional() || !compiler->lvaGetDesc(op->AsLclVar())->lvIsRegCandidate()); + varNum = op->AsLclVar()->GetLclNum(); + offset = 0; + break; + + case GT_CNS_DBL: + return OperandDesc(emit->emitFltOrDblConst(op->AsDblCon()->gtDconVal, emitTypeSize(op))); + + case GT_CNS_INT: + assert(op->isContainedIntOrIImmed()); + return OperandDesc(op->AsIntCon()->IconValue(), op->AsIntCon()->ImmedValNeedsReloc(compiler)); + + default: + unreached(); + } + } + + // Ensure we got a good varNum and offset. + // We also need to check for `tmpDsc != nullptr` since spill temp numbers + // are negative and start with -1, which also happens to be BAD_VAR_NUM. + assert((varNum != BAD_VAR_NUM) || (tmpDsc != nullptr)); + assert(offset != UINT16_MAX); + + return OperandDesc(varNum, offset); +} + /***************************************************************************** - * - * Generate an instruction of the form "op reg1, reg2, icon". - */ +* +* Generate an instruction of the form "op reg1, reg2, icon". +*/ -#if defined(TARGET_XARCH) void CodeGen::inst_RV_RV_IV(instruction ins, emitAttr size, regNumber reg1, regNumber reg2, unsigned ival) { assert(ins == INS_shld || ins == INS_shrd || ins == INS_shufps || ins == INS_shufpd || ins == INS_pshufd || @@ -1020,118 +1136,34 @@ void CodeGen::inst_RV_TT_IV(instruction ins, emitAttr attr, regNumber reg1, GenT emitter* emit = GetEmitter(); noway_assert(emit->emitVerifyEncodable(ins, EA_SIZE(attr), reg1)); - if (rmOp->isContained() || rmOp->isUsedFromSpillTemp()) - { - TempDsc* tmpDsc = nullptr; - unsigned varNum = BAD_VAR_NUM; - unsigned offset = (unsigned)-1; - - if (rmOp->isUsedFromSpillTemp()) - { - assert(rmOp->IsRegOptional()); - - tmpDsc = getSpillTempDsc(rmOp); - varNum = tmpDsc->tdTempNum(); - offset = 0; - - regSet.tmpRlsTemp(tmpDsc); - } - else if (rmOp->isIndir() || rmOp->OperIsHWIntrinsic()) - { - GenTree* addr; - GenTreeIndir* memIndir = nullptr; - - if (rmOp->isIndir()) - { - memIndir = rmOp->AsIndir(); - addr = memIndir->Addr(); - } - else - { -#if defined(FEATURE_HW_INTRINSICS) - assert(rmOp->AsHWIntrinsic()->OperIsMemoryLoad()); - assert(rmOp->AsHWIntrinsic()->GetOperandCount() == 1); - addr = rmOp->AsHWIntrinsic()->Op(1); -#else - unreached(); -#endif // FEATURE_HW_INTRINSICS - } + OperandDesc rmOpDesc = genOperandDesc(rmOp); - switch (addr->OperGet()) - { - case GT_LCL_VAR_ADDR: - case GT_LCL_FLD_ADDR: - { - assert(addr->isContained()); - varNum = addr->AsLclVarCommon()->GetLclNum(); - offset = addr->AsLclVarCommon()->GetLclOffs(); - break; - } + switch (rmOpDesc.GetKind()) + { + case OperandKind::ClsVar: + emit->emitIns_R_C_I(ins, attr, reg1, rmOpDesc.GetFieldHnd(), 0, ival); + break; - case GT_CLS_VAR_ADDR: - { - emit->emitIns_R_C_I(ins, attr, reg1, addr->AsClsVar()->gtClsVarHnd, 0, ival); - return; - } + case OperandKind::Local: + emit->emitIns_R_S_I(ins, attr, reg1, rmOpDesc.GetVarNum(), rmOpDesc.GetLclOffset(), ival); + break; - default: - { - GenTreeIndir load = indirForm(rmOp->TypeGet(), addr); - - if (memIndir == nullptr) - { - // This is the HW intrinsic load case. - // Until we improve the handling of addressing modes in the emitter, we'll create a - // temporary GT_IND to generate code with. - memIndir = &load; - } - emit->emitIns_R_A_I(ins, attr, reg1, memIndir, ival); - return; - } - } - } - else + case OperandKind::Indir: { - switch (rmOp->OperGet()) - { - case GT_LCL_FLD: - varNum = rmOp->AsLclFld()->GetLclNum(); - offset = rmOp->AsLclFld()->GetLclOffs(); - break; - - case GT_LCL_VAR: - { - assert(rmOp->IsRegOptional() || !compiler->lvaGetDesc(rmOp->AsLclVar())->lvIsRegCandidate()); - varNum = rmOp->AsLclVar()->GetLclNum(); - offset = 0; - break; - } - - case GT_CNS_DBL: - { - GenTreeDblCon* cns = rmOp->AsDblCon(); - CORINFO_FIELD_HANDLE hnd = emit->emitFltOrDblConst(cns->gtDconVal, emitTypeSize(cns)); - emit->emitIns_R_C_I(ins, attr, reg1, hnd, 0, ival); - return; - } - - default: - unreached(); - } + // Until we improve the handling of addressing modes in the emitter, we'll create a + // temporary GT_IND to generate code with. + GenTreeIndir indirForm; + GenTreeIndir* indir = rmOpDesc.GetIndirForm(&indirForm); + emit->emitIns_R_A_I(ins, attr, reg1, indir, ival); } + break; - // Ensure we got a good varNum and offset. - // We also need to check for `tmpDsc != nullptr` since spill temp numbers - // are negative and start with -1, which also happens to be BAD_VAR_NUM. - assert((varNum != BAD_VAR_NUM) || (tmpDsc != nullptr)); - assert(offset != (unsigned)-1); + case OperandKind::Reg: + emit->emitIns_SIMD_R_R_I(ins, attr, reg1, rmOp->GetRegNum(), ival); + break; - emit->emitIns_R_S_I(ins, attr, reg1, varNum, offset, ival); - } - else - { - regNumber rmOpReg = rmOp->GetRegNum(); - emit->emitIns_SIMD_R_R_I(ins, attr, reg1, rmOpReg, ival); + default: + unreached(); } } @@ -1157,136 +1189,50 @@ void CodeGen::inst_RV_RV_TT( // TODO-XArch-CQ: Commutative operations can have op1 be contained // TODO-XArch-CQ: Non-VEX encoded instructions can have both ops contained - if (op2->isContained() || op2->isUsedFromSpillTemp()) + OperandDesc op2Desc = genOperandDesc(op2); + switch (op2Desc.GetKind()) { - TempDsc* tmpDsc = nullptr; - unsigned varNum = BAD_VAR_NUM; - unsigned offset = (unsigned)-1; - - if (op2->isUsedFromSpillTemp()) - { - assert(op2->IsRegOptional()); + case OperandKind::ClsVar: + emit->emitIns_SIMD_R_R_C(ins, size, targetReg, op1Reg, op2Desc.GetFieldHnd(), 0); + break; - tmpDsc = getSpillTempDsc(op2); - varNum = tmpDsc->tdTempNum(); - offset = 0; + case OperandKind::Local: + emit->emitIns_SIMD_R_R_S(ins, size, targetReg, op1Reg, op2Desc.GetVarNum(), op2Desc.GetLclOffset()); + break; - regSet.tmpRlsTemp(tmpDsc); - } - else if (op2->isIndir() || op2->OperIsHWIntrinsic()) + case OperandKind::Indir: { - GenTree* addr; - GenTreeIndir* memIndir = nullptr; - - if (op2->isIndir()) - { - memIndir = op2->AsIndir(); - addr = memIndir->Addr(); - } - else - { -#if defined(FEATURE_HW_INTRINSICS) - assert(op2->AsHWIntrinsic()->OperIsMemoryLoad()); - assert(op2->AsHWIntrinsic()->GetOperandCount() == 1); - addr = op2->AsHWIntrinsic()->Op(1); -#else - unreached(); -#endif // FEATURE_HW_INTRINSICS - } - - switch (addr->OperGet()) - { - case GT_LCL_VAR_ADDR: - case GT_LCL_FLD_ADDR: - { - assert(addr->isContained()); - varNum = addr->AsLclVarCommon()->GetLclNum(); - offset = addr->AsLclVarCommon()->GetLclOffs(); - break; - } - - case GT_CLS_VAR_ADDR: - { - emit->emitIns_SIMD_R_R_C(ins, size, targetReg, op1Reg, addr->AsClsVar()->gtClsVarHnd, 0); - return; - } - - default: - { - GenTreeIndir load = indirForm(op2->TypeGet(), addr); - - if (memIndir == nullptr) - { - // This is the HW intrinsic load case. - // Until we improve the handling of addressing modes in the emitter, we'll create a - // temporary GT_IND to generate code with. - memIndir = &load; - } - emit->emitIns_SIMD_R_R_A(ins, size, targetReg, op1Reg, memIndir); - return; - } - } + // Until we improve the handling of addressing modes in the emitter, we'll create a + // temporary GT_IND to generate code with. + GenTreeIndir indirForm; + GenTreeIndir* indir = op2Desc.GetIndirForm(&indirForm); + emit->emitIns_SIMD_R_R_A(ins, size, targetReg, op1Reg, indir); } - else - { - switch (op2->OperGet()) - { - case GT_LCL_FLD: - { - varNum = op2->AsLclFld()->GetLclNum(); - offset = op2->AsLclFld()->GetLclOffs(); - break; - } - - case GT_LCL_VAR: - { - assert(op2->IsRegOptional() || !compiler->lvaGetDesc(op2->AsLclVar())->lvIsRegCandidate()); - varNum = op2->AsLclVar()->GetLclNum(); - offset = 0; - break; - } + break; - case GT_CNS_DBL: - { - GenTreeDblCon* cns = op2->AsDblCon(); - CORINFO_FIELD_HANDLE hnd = emit->emitFltOrDblConst(cns->gtDconVal, emitTypeSize(cns)); - emit->emitIns_SIMD_R_R_C(ins, size, targetReg, op1Reg, hnd, 0); - return; - } + case OperandKind::Reg: + { + regNumber op2Reg = op2->GetRegNum(); - default: - { - unreached(); - } + if ((op1Reg != targetReg) && (op2Reg == targetReg) && isRMW) + { + // We have "reg2 = reg1 op reg2" where "reg1 != reg2" on a RMW instruction. + // + // For non-commutative instructions, we should have ensured that op2 was marked + // delay free in order to prevent it from getting assigned the same register + // as target. However, for commutative instructions, we can just swap the operands + // in order to have "reg2 = reg2 op reg1" which will end up producing the right code. + + op2Reg = op1Reg; + op1Reg = targetReg; } - } - - // Ensure we got a good varNum and offset. - // We also need to check for `tmpDsc != nullptr` since spill temp numbers - // are negative and start with -1, which also happens to be BAD_VAR_NUM. - assert((varNum != BAD_VAR_NUM) || (tmpDsc != nullptr)); - assert(offset != (unsigned)-1); - emit->emitIns_SIMD_R_R_S(ins, size, targetReg, op1Reg, varNum, offset); - } - else - { - regNumber op2Reg = op2->GetRegNum(); - - if ((op1Reg != targetReg) && (op2Reg == targetReg) && isRMW) - { - // We have "reg2 = reg1 op reg2" where "reg1 != reg2" on a RMW instruction. - // - // For non-commutative instructions, we should have ensured that op2 was marked - // delay free in order to prevent it from getting assigned the same register - // as target. However, for commutative instructions, we can just swap the operands - // in order to have "reg2 = reg2 op reg1" which will end up producing the right code. - - op2Reg = op1Reg; - op1Reg = targetReg; + emit->emitIns_SIMD_R_R_R(ins, size, targetReg, op1Reg, op2Reg); } + break; - emit->emitIns_SIMD_R_R_R(ins, size, targetReg, op1Reg, op2Reg); + default: + unreached(); } } #endif // TARGET_XARCH