Skip to content

Commit

Permalink
Optimize indirection cell call sequences more generally (dotnet#59602)
Browse files Browse the repository at this point in the history 
Currently we apply an optimization for ARM architectures where we make
sure we do not duplicate instructions to compute the target address for
calls that involve indirection cells, instead loading it from the
indirection cell directly. We can apply this optimization for x64 VSD
and tailcalls that also use indirection cells. This decreases the size
of these calls.

I have also included a bug fix for ARM/ARM64: the optimization was only
enabled under FEATURE_READYTORUN which is not always defined (e.g. in
single-file scenarios).
  • Loading branch information
@jakobbotsch
jakobbotsch committed Oct 15, 2021
1 parent 33c0b57 commit 8076775
Show file tree
Hide file tree
Showing 9 changed files with 266 additions and 187 deletions.
1 change: 1 addition & 0 deletions src/coreclr/jit/codegen.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1259,6 +1259,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
instruction genGetInsForOper(genTreeOps oper, var_types type);
bool genEmitOptimizedGCWriteBarrier(GCInfo::WriteBarrierForm writeBarrierForm, GenTree* addr, GenTree* data);
GenTree* getCallTarget(const GenTreeCall* call, CORINFO_METHOD_HANDLE* methHnd);
regNumber getCallIndirectionCellReg(const GenTreeCall* call);
void genCall(GenTreeCall* call);
void genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackArgBytes));
void genJmpMethod(GenTree* jmp);
Expand Down
176 changes: 88 additions & 88 deletions src/coreclr/jit/codegenarmarch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2565,109 +2565,109 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
call->IsFastTailCall());
// clang-format on
}
else if (call->IsR2ROrVirtualStubRelativeIndir())
{
// Generate a indirect call to a virtual user defined function or helper method
assert(call->gtCallType == CT_HELPER || call->gtCallType == CT_USER_FUNC);
#ifdef FEATURE_READYTORUN
assert(((call->IsR2RRelativeIndir()) && (call->gtEntryPoint.accessType == IAT_PVALUE)) ||
((call->IsVirtualStubRelativeIndir()) && (call->gtEntryPoint.accessType == IAT_VALUE)));
#endif // FEATURE_READYTORUN
assert(call->gtControlExpr == nullptr);

regNumber tmpReg = call->GetSingleTempReg();
// For fast tailcalls we have already loaded the call target when processing the call node.
if (!call->IsFastTailCall())
{
regNumber callAddrReg =
call->IsVirtualStubRelativeIndir() ? compiler->virtualStubParamInfo->GetReg() : REG_R2R_INDIRECT_PARAM;
GetEmitter()->emitIns_R_R(ins_Load(TYP_I_IMPL), emitActualTypeSize(TYP_I_IMPL), tmpReg, callAddrReg);
}
else
{
// Register where we save call address in should not be overridden by epilog.
assert((tmpReg & (RBM_INT_CALLEE_TRASH & ~RBM_LR)) == tmpReg);
}

// We have now generated code for gtControlExpr evaluating it into `tmpReg`.
// We just need to emit "call tmpReg" in this case.
//
assert(genIsValidIntReg(tmpReg));

// clang-format off
genEmitCall(emitter::EC_INDIR_R,
methHnd,
INDEBUG_LDISASM_COMMA(sigInfo)
nullptr, // addr
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
ilOffset,
tmpReg,
call->IsFastTailCall());
// clang-format on
}
else
{
// Generate a direct call to a non-virtual user defined or helper method
assert(call->gtCallType == CT_HELPER || call->gtCallType == CT_USER_FUNC);

void* addr = nullptr;
#ifdef FEATURE_READYTORUN
if (call->gtEntryPoint.addr != NULL)
// If we have no target and this is a call with indirection cell
// then we do an optimization where we load the call address directly
// from the indirection cell instead of duplicating the tree.
// In BuildCall we ensure that get an extra register for the purpose.
regNumber indirCellReg = getCallIndirectionCellReg(call);
if (indirCellReg != REG_NA)
{
assert(call->gtEntryPoint.accessType == IAT_VALUE);
addr = call->gtEntryPoint.addr;
}
else
#endif // FEATURE_READYTORUN
if (call->gtCallType == CT_HELPER)
{
CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd);
noway_assert(helperNum != CORINFO_HELP_UNDEF);

void* pAddr = nullptr;
addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr);
assert(pAddr == nullptr);
}
else
{
// Direct call to a non-virtual user function.
addr = call->gtDirectCallAddress;
}
assert(call->IsR2ROrVirtualStubRelativeIndir());
regNumber targetAddrReg = call->GetSingleTempReg();
// For fast tailcalls we have already loaded the call target when processing the call node.
if (!call->IsFastTailCall())
{
GetEmitter()->emitIns_R_R(ins_Load(TYP_I_IMPL), emitActualTypeSize(TYP_I_IMPL), targetAddrReg,
indirCellReg);
}
else
{
// Register where we save call address in should not be overridden by epilog.
assert((targetAddrReg & (RBM_INT_CALLEE_TRASH & ~RBM_LR)) == targetAddrReg);
}

assert(addr != nullptr);
// We have now generated code loading the target address from the indirection cell into `targetAddrReg`.
// We just need to emit "bl targetAddrReg" in this case.
//
assert(genIsValidIntReg(targetAddrReg));

// Non-virtual direct call to known addresses
#ifdef TARGET_ARM
if (!arm_Valid_Imm_For_BL((ssize_t)addr))
{
regNumber tmpReg = call->GetSingleTempReg();
instGen_Set_Reg_To_Imm(EA_HANDLE_CNS_RELOC, tmpReg, (ssize_t)addr);
// clang-format off
genEmitCall(emitter::EC_INDIR_R,
methHnd,
INDEBUG_LDISASM_COMMA(sigInfo)
NULL,
retSize,
nullptr, // addr
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
ilOffset,
tmpReg,
targetAddrReg,
call->IsFastTailCall());
// clang-format on
}
else
#endif // TARGET_ARM
{
// clang-format off
genEmitCall(emitter::EC_FUNC_TOKEN,
methHnd,
INDEBUG_LDISASM_COMMA(sigInfo)
addr,
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
ilOffset,
REG_NA,
call->IsFastTailCall());
// clang-format on
// Generate a direct call to a non-virtual user defined or helper method
assert(call->gtCallType == CT_HELPER || call->gtCallType == CT_USER_FUNC);

void* addr = nullptr;
#ifdef FEATURE_READYTORUN
if (call->gtEntryPoint.addr != NULL)
{
assert(call->gtEntryPoint.accessType == IAT_VALUE);
addr = call->gtEntryPoint.addr;
}
else
#endif // FEATURE_READYTORUN
if (call->gtCallType == CT_HELPER)
{
CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd);
noway_assert(helperNum != CORINFO_HELP_UNDEF);

void* pAddr = nullptr;
addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr);
assert(pAddr == nullptr);
}
else
{
// Direct call to a non-virtual user function.
addr = call->gtDirectCallAddress;
}

assert(addr != nullptr);

// Non-virtual direct call to known addresses
#ifdef TARGET_ARM
if (!arm_Valid_Imm_For_BL((ssize_t)addr))
{
regNumber tmpReg = call->GetSingleTempReg();
instGen_Set_Reg_To_Imm(EA_HANDLE_CNS_RELOC, tmpReg, (ssize_t)addr);
// clang-format off
genEmitCall(emitter::EC_INDIR_R,
methHnd,
INDEBUG_LDISASM_COMMA(sigInfo)
NULL,
retSize,
ilOffset,
tmpReg,
call->IsFastTailCall());
// clang-format on
}
else
#endif // TARGET_ARM
{
// clang-format off
genEmitCall(emitter::EC_FUNC_TOKEN,
methHnd,
INDEBUG_LDISASM_COMMA(sigInfo)
addr,
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
ilOffset,
REG_NA,
call->IsFastTailCall());
// clang-format on
}
}
}
}
Expand Down
49 changes: 49 additions & 0 deletions src/coreclr/jit/codegencommon.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7802,6 +7802,55 @@ GenTree* CodeGen::getCallTarget(const GenTreeCall* call, CORINFO_METHOD_HANDLE*
return call->gtControlExpr;
}

//------------------------------------------------------------------------
// getCallIndirectionCellReg - Get the register containing the indirection cell for a call
//
// Arguments:
// call - the node
//
// Returns:
// The register containing the indirection cell, or REG_NA if this call does not use an indirection cell argument.
//
// Notes:
// We currently use indirection cells for VSD on all platforms and for R2R calls on ARM architectures.
//
regNumber CodeGen::getCallIndirectionCellReg(const GenTreeCall* call)
{
regNumber result = REG_NA;
switch (call->GetIndirectionCellArgKind())
{
case NonStandardArgKind::None:
break;
case NonStandardArgKind::R2RIndirectionCell:
result = REG_R2R_INDIRECT_PARAM;
break;
case NonStandardArgKind::VirtualStubCell:
result = compiler->virtualStubParamInfo->GetReg();
break;
default:
unreached();
}

#ifdef DEBUG
regNumber foundReg = REG_NA;
unsigned argCount = call->fgArgInfo->ArgCount();
fgArgTabEntry** argTable = call->fgArgInfo->ArgTable();
for (unsigned i = 0; i < argCount; i++)
{
NonStandardArgKind kind = argTable[i]->nonStandardArgKind;
if ((kind == NonStandardArgKind::R2RIndirectionCell) || (kind == NonStandardArgKind::VirtualStubCell))
{
foundReg = argTable[i]->GetRegNum();
break;
}
}

assert(foundReg == result);
#endif

return result;
}

/*****************************************************************************
*
* Generates code for a function epilog.
Expand Down
93 changes: 50 additions & 43 deletions src/coreclr/jit/codegenxarch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5696,15 +5696,18 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
// clang-format on
}
}
#ifdef FEATURE_READYTORUN
else if (call->gtEntryPoint.addr != nullptr)
else
{
emitter::EmitCallType type =
(call->gtEntryPoint.accessType == IAT_VALUE) ? emitter::EC_FUNC_TOKEN : emitter::EC_FUNC_TOKEN_INDIR;
if (call->IsFastTailCall() && (type == emitter::EC_FUNC_TOKEN_INDIR))
// If we have no target and this is a call with indirection cell
// then emit call through that indir cell. This means we generate e.g.
// lea r11, [addr of cell]
// call [r11]
// which is more efficent than
// lea r11, [addr of cell]
// call [addr of cell]
regNumber indirCellReg = getCallIndirectionCellReg(call);
if (indirCellReg != REG_NA)
{
// For fast tailcall with func token indir we already have the indirection cell in REG_R2R_INDIRECT_PARAM,
// so get it from there.
// clang-format off
GetEmitter()->emitIns_Call(
emitter::EC_INDIR_ARD,
Expand All @@ -5717,11 +5720,15 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
gcInfo.gcVarPtrSetCur,
gcInfo.gcRegGCrefSetCur,
gcInfo.gcRegByrefSetCur,
ilOffset, REG_R2R_INDIRECT_PARAM, REG_NA, 0, 0, true);
ilOffset, indirCellReg, REG_NA, 0, 0,
call->IsFastTailCall());
// clang-format on
}
else
#ifdef FEATURE_READYTORUN
else if (call->gtEntryPoint.addr != nullptr)
{
emitter::EmitCallType type =
(call->gtEntryPoint.accessType == IAT_VALUE) ? emitter::EC_FUNC_TOKEN : emitter::EC_FUNC_TOKEN_INDIR;
// clang-format off
genEmitCall(type,
methHnd,
Expand All @@ -5735,46 +5742,46 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
call->IsFastTailCall());
// clang-format on
}
}
#endif
else
{
// Generate a direct call to a non-virtual user defined or helper method
assert(call->gtCallType == CT_HELPER || call->gtCallType == CT_USER_FUNC);

void* addr = nullptr;
if (call->gtCallType == CT_HELPER)
{
// Direct call to a helper method.
CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd);
noway_assert(helperNum != CORINFO_HELP_UNDEF);

void* pAddr = nullptr;
addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr);
assert(pAddr == nullptr);
}
else
{
// Direct call to a non-virtual user function.
addr = call->gtDirectCallAddress;
}
// Generate a direct call to a non-virtual user defined or helper method
assert(call->gtCallType == CT_HELPER || call->gtCallType == CT_USER_FUNC);

void* addr = nullptr;
if (call->gtCallType == CT_HELPER)
{
// Direct call to a helper method.
CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd);
noway_assert(helperNum != CORINFO_HELP_UNDEF);

void* pAddr = nullptr;
addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr);
assert(pAddr == nullptr);
}
else
{
// Direct call to a non-virtual user function.
addr = call->gtDirectCallAddress;
}

assert(addr != nullptr);
assert(addr != nullptr);

// Non-virtual direct calls to known addresses
// Non-virtual direct calls to known addresses

// clang-format off
genEmitCall(emitter::EC_FUNC_TOKEN,
methHnd,
INDEBUG_LDISASM_COMMA(sigInfo)
addr
X86_ARG(argSizeForEmitter),
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
ilOffset,
REG_NA,
call->IsFastTailCall());
// clang-format on
// clang-format off
genEmitCall(emitter::EC_FUNC_TOKEN,
methHnd,
INDEBUG_LDISASM_COMMA(sigInfo)
addr
X86_ARG(argSizeForEmitter),
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
ilOffset,
REG_NA,
call->IsFastTailCall());
// clang-format on
}
}
}

Expand Down
Loading

0 comments on commit 8076775

Please sign in to comment.