Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JIT: Always track the context for late devirt #112396

Merged
merged 15 commits into from
Feb 19, 2025
+49 −46
Merged

JIT: Always track the context for late devirt #112396

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
8563a18
Track InlinersContext in LateDevirtualizationInfo
hez2010 Feb 11, 2025
3678070
Rework fix
hez2010 Feb 11, 2025
1173d97
Update inline context for statement as well
hez2010 Feb 11, 2025
dc917f2
Use IL offset from call
hez2010 Feb 11, 2025
57427a4
Always set DebugInfo
hez2010 Feb 11, 2025
e0e7344
Track InlineContext
hez2010 Feb 11, 2025
4b30625
Cleanup
hez2010 Feb 11, 2025
d01f255
Always record late devirt info
hez2010 Feb 11, 2025
d746d95
Always record late devirt info
hez2010 Feb 12, 2025
6a2dcd6
Copy gtLateDevirtualizationInfo
hez2010 Feb 12, 2025
125600a
Refactor to take InlineContext explicitly
hez2010 Feb 12, 2025
1c711a3
Set inlinersContext to nullptr
hez2010 Feb 13, 2025
7f7c0f4
Late devirt info always available
hez2010 Feb 13, 2025
55585bd
Oops
hez2010 Feb 13, 2025
4fa6b92
Remove the reduandant flag
hez2010 Feb 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5244,6 +5244,7 @@ class Compiler
CORINFO_METHOD_HANDLE fncHandle,
unsigned methAttr,
CORINFO_CONTEXT_HANDLE exactContextHnd,
InlineContext* inlinersContext,
InlineCandidateInfo** ppInlineCandidateInfo,
InlineResult* inlineResult);

Expand All @@ -5265,13 +5266,15 @@ class Compiler
void impMarkInlineCandidate(GenTree* call,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo);
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext);

void impMarkInlineCandidateHelper(GenTreeCall* call,
uint8_t candidateIndex,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext,
InlineResult* inlineResult);

bool impTailCallRetTypeCompatible(bool allowWidening,
Expand Down
19 changes: 4 additions & 15 deletions src/coreclr/jit/fginline.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -589,22 +589,13 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
}
#endif // DEBUG

CORINFO_CONTEXT_HANDLE context = nullptr;
CORINFO_CONTEXT_HANDLE context = call->gtLateDevirtualizationInfo->exactContextHnd;
InlineContext* inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext;
CORINFO_METHOD_HANDLE method = call->gtCallMethHnd;
unsigned methodFlags = 0;
const bool isLateDevirtualization = true;
const bool explicitTailCall = call->IsTailPrefixedCall();

if ((call->gtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0)
{
context = call->gtLateDevirtualizationInfo->exactContextHnd;
// Note: we might call this multiple times for the same trees.
// If the devirtualization below succeeds, the call becomes
// non-virtual and we won't get here again. If it does not
// succeed we might get here again so we keep the late devirt
// info.
}

CORINFO_CONTEXT_HANDLE contextInput = context;
context = nullptr;
m_compiler->impDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context,
Expand All @@ -613,10 +604,11 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
if (!call->IsVirtual())
{
assert(context != nullptr);
assert(inlinersContext != nullptr);
CORINFO_CALL_INFO callInfo = {};
callInfo.hMethod = method;
callInfo.methodFlags = methodFlags;
m_compiler->impMarkInlineCandidate(call, context, false, &callInfo);
m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, inlinersContext);

if (call->IsInlineCandidate())
{
Expand Down Expand Up @@ -652,9 +644,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
*pTree = retExpr;
}

call->GetSingleInlineCandidateInfo()->exactContextHandle = context;
INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext);

JITDUMP("New inline candidate due to late devirtualization:\n");
DISPTREE(call);
}
Expand Down
2 changes: 2 additions & 0 deletions src/coreclr/jit/gentree.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10013,6 +10013,8 @@ GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree)
copy->gtInlineInfoCount = tree->gtInlineInfoCount;
}

copy->gtLateDevirtualizationInfo = tree->gtLateDevirtualizationInfo;

copy->gtCallType = tree->gtCallType;
copy->gtReturnType = tree->gtReturnType;

Expand Down
13 changes: 7 additions & 6 deletions src/coreclr/jit/gentree.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4222,12 +4222,11 @@ enum GenTreeCallFlags : unsigned int
GTF_CALL_M_ALLOC_SIDE_EFFECTS = 0x00100000, // this is a call to an allocator with side effects
GTF_CALL_M_SUPPRESS_GC_TRANSITION = 0x00200000, // suppress the GC transition (i.e. during a pinvoke) but a separate GC safe point is required.
GTF_CALL_M_EXPANDED_EARLY = 0x00800000, // the Virtual Call target address is expanded and placed in gtControlExpr in Morph rather than in Lower
GTF_CALL_M_HAS_LATE_DEVIRT_INFO = 0x01000000, // this call has late devirtualzation info
GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x02000000, // ldvirtftn on an interface type
GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x04000000, // this cast (helper call) can be expanded if it's profitable. To be removed.
GTF_CALL_M_CAST_OBJ_NONNULL = 0x08000000, // if we expand this specific cast we don't need to check the input object for null
GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x01000000, // ldvirtftn on an interface type
GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x02000000, // this cast (helper call) can be expanded if it's profitable. To be removed.
GTF_CALL_M_CAST_OBJ_NONNULL = 0x04000000, // if we expand this specific cast we don't need to check the input object for null
// NOTE: if needed, this flag can be removed, and we can introduce new _NONNUL cast helpers
GTF_CALL_M_STACK_ARRAY = 0x10000000, // this call is a new array helper for a stack allocated array.
GTF_CALL_M_STACK_ARRAY = 0x08000000, // this call is a new array helper for a stack allocated array.
};

inline constexpr GenTreeCallFlags operator ~(GenTreeCallFlags a)
Expand Down Expand Up @@ -5758,11 +5757,13 @@ struct GenTreeCall final : public GenTree
jitstd::vector<InlineCandidateInfo*>* gtInlineCandidateInfoList;

HandleHistogramProfileCandidateInfo* gtHandleHistogramProfileCandidateInfo;
LateDevirtualizationInfo* gtLateDevirtualizationInfo;

CORINFO_GENERIC_HANDLE compileTimeHelperArgumentHandle; // Used to track type handle argument of dynamic helpers
void* gtDirectCallAddress; // Used to pass direct call address between lower and codegen
};

LateDevirtualizationInfo* gtLateDevirtualizationInfo; // Always available for user virtual calls
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have you checked whether this can be merged with some other information of GenTreeCall to avoid growing the size of all large nodes?

Copy link
Contributor Author

@hez2010 hez2010 Feb 15, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Late devirtualization can happen even without generic context, so we need to always track the InlineContext now regardless of whether it has a generic context or not.
We used to union the gtLateDevirtualizationInfo with gtHandleHistogramProfileCandidateInfo together because we didn't save generic context for methods that have class probe, but now we will have to save an InlineContext for them anyway so we can also save the generic context for those methods as well.
This happens to enable late devirtualization for tier-1 methods under PGO and this is why it has diffs for PGO cases only.
I was trying to merge it into the union but it would require class probing to "overwite" the gtLateDevirtualizationInfo and unset the flag for it else where, which may be hard to get right... see https://github.com/dotnet/runtime/runs/37064711496, we have cases where late devit inlining happens while we have class probing.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead we can remove the member class InlineContext* gtInlineContext from GenTreeCall now, though it's debug only.

Copy link
Contributor Author

@hez2010 hez2010 Feb 15, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or alternatively, we will have to block all inlining for devirted methods that don't have a generic context in late devirt.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For debug it's ok to use extra memory, we are more concerned with release. I suppose we cannot move this into the union above this one because of gtStubCallStubAddr?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, unfortunately. Maybe we can merge it with gtControlExpr, do we ever morph a call before inlining?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's ok to leave it as a separate field for now. Maybe we should consider some better approaches in the future to reducing the size of GenTreeCall (e.g. storing all rarely used fields in a separate object that gets allocated on demand).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay. I just pushed a commit to remove the redundant flag.


// expression evaluated after args are placed which determines the control target
GenTree* gtControlExpr;

Expand Down
55 changes: 31 additions & 24 deletions src/coreclr/jit/importercalls.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1030,7 +1030,8 @@ var_types Compiler::impImportCall(OPCODE opcode,
INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset);

// Is it an inline candidate?
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo);
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
compInlineContext);
}

// append the call node.
Expand Down Expand Up @@ -1240,7 +1241,20 @@ var_types Compiler::impImportCall(OPCODE opcode,
INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset);

// Is it an inline candidate?
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo);
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, compInlineContext);

// If the call is virtual, record the inliner's context for possible use during late devirt inlining.
// Also record the generics context if there is any.
//
if (call->AsCall()->IsVirtual() && (call->AsCall()->gtCallType != CT_INDIRECT))
{
JITDUMP("\nSaving generic context %p and inline context %p for call [%06u]\n", dspPtr(exactContextHnd),
dspPtr(compInlineContext), dspTreeID(call->AsCall()));
LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo;
info->exactContextHnd = exactContextHnd;
info->inlinersContext = compInlineContext;
call->AsCall()->gtLateDevirtualizationInfo = info;
}
}

// Extra checks for tail calls and tail recursion.
Expand Down Expand Up @@ -1431,22 +1445,6 @@ var_types Compiler::impImportCall(OPCODE opcode,
}
else
{
// If the call is virtual, and has a generics context, and is not going to have a class probe,
// record the context for possible use during late devirt.
//
// If we ever want to devirt at Tier0, and/or see issues where OSR methods under PGO lose
// important devirtualizations, we'll want to allow both a class probe and a captured context.
//
if (origCall->IsVirtual() && (origCall->gtCallType != CT_INDIRECT) && (exactContextHnd != nullptr) &&
(origCall->gtHandleHistogramProfileCandidateInfo == nullptr))
{
JITDUMP("\nSaving context %p for call [%06u]\n", dspPtr(exactContextHnd), dspTreeID(origCall));
origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO;
LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo;
info->exactContextHnd = exactContextHnd;
origCall->gtLateDevirtualizationInfo = info;
}

if (isFatPointerCandidate)
{
// fatPointer candidates should be in statements of the form call() or var = call().
Expand Down Expand Up @@ -7450,6 +7448,7 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call,
// exactContextHnd -- context handle for inlining
// exactContextNeedsRuntimeLookup -- true if context required runtime lookup
// callInfo -- call info from VM
// inlinersContext -- the inliner's context
//
// Notes:
// Mostly a wrapper for impMarkInlineCandidateHelper that also undoes
Expand All @@ -7459,7 +7458,8 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call,
void Compiler::impMarkInlineCandidate(GenTree* callNode,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo)
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext)
{
if (!opts.OptEnabled(CLFLG_INLINING))
{
Expand All @@ -7482,7 +7482,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,

// Do the actual evaluation
impMarkInlineCandidateHelper(call, candidateId, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
&inlineResult);
inlinersContext, &inlineResult);
// Ignore non-inlineable candidates
// TODO: Consider keeping them to just devirtualize without inlining, at least for interface
// calls on NativeAOT, but that requires more changes elsewhere too.
Expand All @@ -7505,7 +7505,8 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,
const uint8_t candidatesCount = call->GetInlineCandidatesCount();
assert(candidatesCount <= 1);
InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate");
impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, &inlineResult);
impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
inlinersContext, &inlineResult);
}

// If this call is an inline candidate or is not a guarded devirtualization
Expand Down Expand Up @@ -7538,6 +7539,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,
// exactContextHnd -- context handle for inlining
// exactContextNeedsRuntimeLookup -- true if context required runtime lookup
// callInfo -- call info from VM
// inlinersContext -- the inliner's context
//
// Notes:
// If callNode is an inline candidate, this method sets the flag
Expand All @@ -7554,6 +7556,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext,
InlineResult* inlineResult)
{
// Let the strategy know there's another call
Expand Down Expand Up @@ -7748,7 +7751,8 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call,
}

InlineCandidateInfo* inlineCandidateInfo = nullptr;
impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, inlineResult);
impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, inlinersContext, &inlineCandidateInfo,
inlineResult);

if (inlineResult->IsFailure())
{
Expand Down Expand Up @@ -8377,7 +8381,6 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call,
// it's a union field used for other things by virtual
// stubs)
call->ClearInlineInfo();
call->gtCallMoreFlags &= ~GTF_CALL_M_HAS_LATE_DEVIRT_INFO;

#if defined(DEBUG)
if (verbose)
Expand Down Expand Up @@ -9056,6 +9059,7 @@ bool Compiler::impTailCallRetTypeCompatible(bool allowWideni
// fncHandle - method that will be called
// methAttr - attributes for the method
// exactContextHnd - exact context for the method
// inlinersContext - the inliner's context
// ppInlineCandidateInfo [out] - information needed later for inlining
// inlineResult - result of ongoing inline evaluation
//
Expand All @@ -9068,6 +9072,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
CORINFO_METHOD_HANDLE fncHandle,
unsigned methAttr,
CORINFO_CONTEXT_HANDLE exactContextHnd,
InlineContext* inlinersContext,
InlineCandidateInfo** ppInlineCandidateInfo,
InlineResult* inlineResult)
{
Expand All @@ -9082,6 +9087,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
CORINFO_METHOD_HANDLE fncHandle;
unsigned methAttr;
CORINFO_CONTEXT_HANDLE exactContextHnd;
InlineContext* inlinersContext;
InlineResult* result;
InlineCandidateInfo** ppInlineCandidateInfo;
} param;
Expand All @@ -9093,6 +9099,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
param.fncHandle = fncHandle;
param.methAttr = methAttr;
param.exactContextHnd = (exactContextHnd != nullptr) ? exactContextHnd : MAKE_METHODCONTEXT(fncHandle);
param.inlinersContext = inlinersContext;
param.result = inlineResult;
param.ppInlineCandidateInfo = ppInlineCandidateInfo;

Expand Down Expand Up @@ -9243,7 +9250,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
pInfo->methAttr = pParam->methAttr;
pInfo->initClassResult = initClassResult;
pInfo->exactContextNeedsRuntimeLookup = false;
pInfo->inlinersContext = pParam->pThis->compInlineContext;
pInfo->inlinersContext = pParam->inlinersContext;

// Note exactContextNeedsRuntimeLookup is reset later on,
// over in impMarkInlineCandidate.
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/inline.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,7 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo
struct LateDevirtualizationInfo
{
CORINFO_CONTEXT_HANDLE exactContextHnd;
InlineContext* inlinersContext;
};

// InlArgInfo describes inline candidate argument properties.
Expand Down
Loading