Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JIT: Always track the context for late devirt #112396

Merged
merged 15 commits into from
Feb 19, 2025
Merged

JIT: Always track the context for late devirt #112396

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
8563a18
Track InlinersContext in LateDevirtualizationInfo
hez2010 Feb 11, 2025
3678070
Rework fix
hez2010 Feb 11, 2025
1173d97
Update inline context for statement as well
hez2010 Feb 11, 2025
dc917f2
Use IL offset from call
hez2010 Feb 11, 2025
57427a4
Always set DebugInfo
hez2010 Feb 11, 2025
e0e7344
Track InlineContext
hez2010 Feb 11, 2025
4b30625
Cleanup
hez2010 Feb 11, 2025
d01f255
Always record late devirt info
hez2010 Feb 11, 2025
d746d95
Always record late devirt info
hez2010 Feb 12, 2025
6a2dcd6
Copy gtLateDevirtualizationInfo
hez2010 Feb 12, 2025
125600a
Refactor to take InlineContext explicitly
hez2010 Feb 12, 2025
1c711a3
Set inlinersContext to nullptr
hez2010 Feb 13, 2025
7f7c0f4
Late devirt info always available
hez2010 Feb 13, 2025
55585bd
Oops
hez2010 Feb 13, 2025
4fa6b92
Remove the reduandant flag
hez2010 Feb 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5244,6 +5244,7 @@ class Compiler
CORINFO_METHOD_HANDLE fncHandle,
unsigned methAttr,
CORINFO_CONTEXT_HANDLE exactContextHnd,
InlineContext* inlinersContext,
InlineCandidateInfo** ppInlineCandidateInfo,
InlineResult* inlineResult);

Expand All @@ -5265,13 +5266,15 @@ class Compiler
void impMarkInlineCandidate(GenTree* call,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo);
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext);

void impMarkInlineCandidateHelper(GenTreeCall* call,
uint8_t candidateIndex,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext,
InlineResult* inlineResult);

bool impTailCallRetTypeCompatible(bool allowWidening,
Expand Down
10 changes: 5 additions & 5 deletions src/coreclr/jit/fginline.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,14 +590,16 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
#endif // DEBUG

CORINFO_CONTEXT_HANDLE context = nullptr;
InlineContext* inlinersContext = m_compiler->compInlineContext;
CORINFO_METHOD_HANDLE method = call->gtCallMethHnd;
unsigned methodFlags = 0;
const bool isLateDevirtualization = true;
const bool explicitTailCall = call->IsTailPrefixedCall();

if ((call->gtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0)
{
context = call->gtLateDevirtualizationInfo->exactContextHnd;
context = call->gtLateDevirtualizationInfo->exactContextHnd;
inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext;
// Note: we might call this multiple times for the same trees.
// If the devirtualization below succeeds, the call becomes
// non-virtual and we won't get here again. If it does not
Expand All @@ -613,10 +615,11 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
if (!call->IsVirtual())
{
assert(context != nullptr);
assert(inlinersContext != nullptr);
CORINFO_CALL_INFO callInfo = {};
callInfo.hMethod = method;
callInfo.methodFlags = methodFlags;
m_compiler->impMarkInlineCandidate(call, context, false, &callInfo);
m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, inlinersContext);

if (call->IsInlineCandidate())
{
Expand Down Expand Up @@ -652,9 +655,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
*pTree = retExpr;
}

call->GetSingleInlineCandidateInfo()->exactContextHandle = context;
INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext);

JITDUMP("New inline candidate due to late devirtualization:\n");
DISPTREE(call);
}
Expand Down
2 changes: 2 additions & 0 deletions src/coreclr/jit/gentree.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10013,6 +10013,8 @@ GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree)
copy->gtInlineInfoCount = tree->gtInlineInfoCount;
}

copy->gtLateDevirtualizationInfo = tree->gtLateDevirtualizationInfo;

copy->gtCallType = tree->gtCallType;
copy->gtReturnType = tree->gtReturnType;

Expand Down
4 changes: 3 additions & 1 deletion src/coreclr/jit/gentree.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5758,11 +5758,13 @@ struct GenTreeCall final : public GenTree
jitstd::vector<InlineCandidateInfo*>* gtInlineCandidateInfoList;

HandleHistogramProfileCandidateInfo* gtHandleHistogramProfileCandidateInfo;
LateDevirtualizationInfo* gtLateDevirtualizationInfo;

CORINFO_GENERIC_HANDLE compileTimeHelperArgumentHandle; // Used to track type handle argument of dynamic helpers
void* gtDirectCallAddress; // Used to pass direct call address between lower and codegen
};

LateDevirtualizationInfo* gtLateDevirtualizationInfo;

// expression evaluated after args are placed which determines the control target
GenTree* gtControlExpr;

Expand Down
55 changes: 32 additions & 23 deletions src/coreclr/jit/importercalls.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1030,7 +1030,8 @@ var_types Compiler::impImportCall(OPCODE opcode,
INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset);

// Is it an inline candidate?
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo);
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
compInlineContext);
}

// append the call node.
Expand Down Expand Up @@ -1240,7 +1241,21 @@ var_types Compiler::impImportCall(OPCODE opcode,
INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset);

// Is it an inline candidate?
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo);
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, compInlineContext);

// If the call is virtual, record the inliner's context for possible use during late devirt inlining.
// Also record the generics context if there is any.
//
if (call->AsCall()->IsVirtual() && (call->AsCall()->gtCallType != CT_INDIRECT))
{
JITDUMP("\nSaving generic context %p and inline context %p for call [%06u]\n", dspPtr(exactContextHnd),
dspPtr(compInlineContext), dspTreeID(call->AsCall()));
call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO;
LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo;
info->exactContextHnd = exactContextHnd;
info->inlinersContext = compInlineContext;
call->AsCall()->gtLateDevirtualizationInfo = info;
}
}

// Extra checks for tail calls and tail recursion.
Expand Down Expand Up @@ -1431,22 +1446,6 @@ var_types Compiler::impImportCall(OPCODE opcode,
}
else
{
// If the call is virtual, and has a generics context, and is not going to have a class probe,
// record the context for possible use during late devirt.
//
// If we ever want to devirt at Tier0, and/or see issues where OSR methods under PGO lose
// important devirtualizations, we'll want to allow both a class probe and a captured context.
//
if (origCall->IsVirtual() && (origCall->gtCallType != CT_INDIRECT) && (exactContextHnd != nullptr) &&
(origCall->gtHandleHistogramProfileCandidateInfo == nullptr))
{
JITDUMP("\nSaving context %p for call [%06u]\n", dspPtr(exactContextHnd), dspTreeID(origCall));
origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO;
LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo;
info->exactContextHnd = exactContextHnd;
origCall->gtLateDevirtualizationInfo = info;
}

if (isFatPointerCandidate)
{
// fatPointer candidates should be in statements of the form call() or var = call().
Expand Down Expand Up @@ -7450,6 +7449,7 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call,
// exactContextHnd -- context handle for inlining
// exactContextNeedsRuntimeLookup -- true if context required runtime lookup
// callInfo -- call info from VM
// inlinersContext -- the inliner's context
//
// Notes:
// Mostly a wrapper for impMarkInlineCandidateHelper that also undoes
Expand All @@ -7459,7 +7459,8 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call,
void Compiler::impMarkInlineCandidate(GenTree* callNode,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo)
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext)
{
if (!opts.OptEnabled(CLFLG_INLINING))
{
Expand All @@ -7482,7 +7483,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,

// Do the actual evaluation
impMarkInlineCandidateHelper(call, candidateId, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
&inlineResult);
inlinersContext, &inlineResult);
// Ignore non-inlineable candidates
// TODO: Consider keeping them to just devirtualize without inlining, at least for interface
// calls on NativeAOT, but that requires more changes elsewhere too.
Expand All @@ -7505,7 +7506,8 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,
const uint8_t candidatesCount = call->GetInlineCandidatesCount();
assert(candidatesCount <= 1);
InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate");
impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, &inlineResult);
impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
inlinersContext, &inlineResult);
}

// If this call is an inline candidate or is not a guarded devirtualization
Expand Down Expand Up @@ -7538,6 +7540,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,
// exactContextHnd -- context handle for inlining
// exactContextNeedsRuntimeLookup -- true if context required runtime lookup
// callInfo -- call info from VM
// inlinersContext -- the inliner's context
//
// Notes:
// If callNode is an inline candidate, this method sets the flag
Expand All @@ -7554,6 +7557,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext,
InlineResult* inlineResult)
{
// Let the strategy know there's another call
Expand Down Expand Up @@ -7748,7 +7752,8 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call,
}

InlineCandidateInfo* inlineCandidateInfo = nullptr;
impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, inlineResult);
impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, inlinersContext, &inlineCandidateInfo,
inlineResult);

if (inlineResult->IsFailure())
{
Expand Down Expand Up @@ -9056,6 +9061,7 @@ bool Compiler::impTailCallRetTypeCompatible(bool allowWideni
// fncHandle - method that will be called
// methAttr - attributes for the method
// exactContextHnd - exact context for the method
// inlinersContext - the inliner's context
// ppInlineCandidateInfo [out] - information needed later for inlining
// inlineResult - result of ongoing inline evaluation
//
Expand All @@ -9068,6 +9074,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
CORINFO_METHOD_HANDLE fncHandle,
unsigned methAttr,
CORINFO_CONTEXT_HANDLE exactContextHnd,
InlineContext* inlinersContext,
InlineCandidateInfo** ppInlineCandidateInfo,
InlineResult* inlineResult)
{
Expand All @@ -9082,6 +9089,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
CORINFO_METHOD_HANDLE fncHandle;
unsigned methAttr;
CORINFO_CONTEXT_HANDLE exactContextHnd;
InlineContext* inlinersContext;
InlineResult* result;
InlineCandidateInfo** ppInlineCandidateInfo;
} param;
Expand All @@ -9093,6 +9101,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
param.fncHandle = fncHandle;
param.methAttr = methAttr;
param.exactContextHnd = (exactContextHnd != nullptr) ? exactContextHnd : MAKE_METHODCONTEXT(fncHandle);
param.inlinersContext = inlinersContext;
param.result = inlineResult;
param.ppInlineCandidateInfo = ppInlineCandidateInfo;

Expand Down Expand Up @@ -9243,7 +9252,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
pInfo->methAttr = pParam->methAttr;
pInfo->initClassResult = initClassResult;
pInfo->exactContextNeedsRuntimeLookup = false;
pInfo->inlinersContext = pParam->pThis->compInlineContext;
pInfo->inlinersContext = pParam->inlinersContext;

// Note exactContextNeedsRuntimeLookup is reset later on,
// over in impMarkInlineCandidate.
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/inline.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,7 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo
struct LateDevirtualizationInfo
{
CORINFO_CONTEXT_HANDLE exactContextHnd;
InlineContext* inlinersContext;
};

// InlArgInfo describes inline candidate argument properties.
Expand Down
Loading