From fae05be4f67a05ce024363ec4f393670d967c532 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 5 Feb 2025 22:28:15 +0100 Subject: [PATCH] JIT: Support custom `ClassLayout` instances with GC pointers in them (#112064) - Add a `ClassLayoutBuilder` that can be used to build new `ClassLayout` instances with arbitrary GC pointers - Add support for GC types to `LCL_FLD` stress to test some of this new support Subsumes #111942 Fixes #103362 --- src/coreclr/jit/compiler.cpp | 2 +- src/coreclr/jit/compiler.h | 12 +- src/coreclr/jit/layout.cpp | 392 ++++++++++++++++++++++++++++------- src/coreclr/jit/layout.h | 90 +++++--- src/coreclr/jit/lclvars.cpp | 55 +++-- src/coreclr/jit/morph.cpp | 2 +- 6 files changed, 424 insertions(+), 129 deletions(-) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 8cc80f91b48b09..b8482f4592ae9a 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -8292,7 +8292,7 @@ const StructSegments& Compiler::GetSignificantSegments(ClassLayout* layout) StructSegments* newSegments = new (this, CMK_Promotion) StructSegments(getAllocator(CMK_Promotion)); - if (layout->IsBlockLayout()) + if (layout->IsCustomLayout()) { newSegments->Add(StructSegments::Segment(0, layout->GetSize())); } diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 6a7298cf3d2373..c3bf108a19a92b 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -9246,7 +9246,7 @@ class Compiler bool isOpaqueSIMDType(ClassLayout* layout) const { - if (layout->IsBlockLayout()) + if (layout->IsCustomLayout()) { return true; } @@ -11104,14 +11104,16 @@ class Compiler ClassLayout* typGetLayoutByNum(unsigned layoutNum); // Get the layout number of the specified layout. unsigned typGetLayoutNum(ClassLayout* layout); + // Get the layout for the specified class handle. + ClassLayout* typGetObjLayout(CORINFO_CLASS_HANDLE classHandle); + // Get the number of a layout for the specified class handle. + unsigned typGetObjLayoutNum(CORINFO_CLASS_HANDLE classHandle); + ClassLayout* typGetCustomLayout(const ClassLayoutBuilder& builder); + unsigned typGetCustomLayoutNum(const ClassLayoutBuilder& builder); // Get the layout having the specified size but no class handle. ClassLayout* typGetBlkLayout(unsigned blockSize); // Get the number of a layout having the specified size but no class handle. unsigned typGetBlkLayoutNum(unsigned blockSize); - // Get the layout for the specified class handle. - ClassLayout* typGetObjLayout(CORINFO_CLASS_HANDLE classHandle); - // Get the number of a layout for the specified class handle. - unsigned typGetObjLayoutNum(CORINFO_CLASS_HANDLE classHandle); var_types TypeHandleToVarType(CORINFO_CLASS_HANDLE handle, ClassLayout** pLayout = nullptr); var_types TypeHandleToVarType(CorInfoType jitType, CORINFO_CLASS_HANDLE handle, ClassLayout** pLayout = nullptr); diff --git a/src/coreclr/jit/layout.cpp b/src/coreclr/jit/layout.cpp index 655750718bc734..5c9105b04072d6 100644 --- a/src/coreclr/jit/layout.cpp +++ b/src/coreclr/jit/layout.cpp @@ -5,6 +5,65 @@ #include "layout.h" #include "compiler.h" +// Key used in ClassLayoutTable's hash table for custom layouts. +struct CustomLayoutKey +{ + unsigned Size; + const BYTE* GCPtrTypes; + + CustomLayoutKey(ClassLayout* layout) + : Size(layout->GetSize()) + , GCPtrTypes(layout->m_gcPtrCount > 0 ? layout->GetGCPtrs() : nullptr) + { + assert(layout->IsCustomLayout()); + } + + CustomLayoutKey(const ClassLayoutBuilder& builder) + : Size(builder.m_size) + , GCPtrTypes(builder.m_gcPtrCount > 0 ? builder.m_gcPtrs : nullptr) + { + } + + static bool Equals(const CustomLayoutKey& l, const CustomLayoutKey& r) + { + if (l.Size != r.Size) + { + return false; + } + + if ((l.GCPtrTypes == nullptr) != (r.GCPtrTypes == nullptr)) + { + return false; + } + + if ((l.GCPtrTypes != nullptr) && (memcmp(l.GCPtrTypes, r.GCPtrTypes, l.Size / TARGET_POINTER_SIZE) != 0)) + { + return false; + } + + return true; + } + + static unsigned GetHashCode(const CustomLayoutKey& key) + { + unsigned hash = key.Size; + if (key.GCPtrTypes != nullptr) + { + hash ^= 0xc4cfbb2a + (hash << 19) + (hash >> 13); + for (unsigned i = 0; i < key.Size / TARGET_POINTER_SIZE; i++) + { + hash ^= key.GCPtrTypes[i] + 0x9e3779b9 + (hash << 19) + (hash >> 13); + } + } + else + { + hash ^= 0x324ba6da + (hash << 19) + (hash >> 13); + } + + return hash; + } +}; + // Keeps track of layout objects associated to class handles or block sizes. A layout is usually // referenced by a pointer (ClassLayout*) but can also be referenced by a number (unsigned, // FirstLayoutNum-based), when space constraints or other needs make numbers more appealing. @@ -18,7 +77,7 @@ class ClassLayoutTable static constexpr unsigned ZeroSizedBlockLayoutNum = TYP_UNKNOWN + 1; static constexpr unsigned FirstLayoutNum = TYP_UNKNOWN + 2; - typedef JitHashTable, unsigned> BlkLayoutIndexMap; + typedef JitHashTable CustomLayoutIndexMap; typedef JitHashTable, unsigned> ObjLayoutIndexMap; union @@ -29,15 +88,15 @@ class ClassLayoutTable // Otherwise a dynamic array is allocated and hashtables are used to map from handle/size to layout array index. struct { - ClassLayout** m_layoutLargeArray; - BlkLayoutIndexMap* m_blkLayoutMap; - ObjLayoutIndexMap* m_objLayoutMap; + ClassLayout** m_layoutLargeArray; + CustomLayoutIndexMap* m_customLayoutMap; + ObjLayoutIndexMap* m_objLayoutMap; }; }; // The number of layout objects stored in this table. - unsigned m_layoutCount; + unsigned m_layoutCount = 0; // The capacity of m_layoutLargeArray (when more than 3 layouts are stored). - unsigned m_layoutLargeCapacity; + unsigned m_layoutLargeCapacity = 0; // We furthermore fast-path the 0-sized block layout which is used for // block locals that may grow (e.g. the outgoing arg area in every non-x86 // compilation). @@ -45,9 +104,7 @@ class ClassLayoutTable public: ClassLayoutTable() - : m_layoutCount(0) - , m_layoutLargeCapacity(0) - , m_zeroSizedBlockLayout(0) + : m_zeroSizedBlockLayout(0) { } @@ -76,25 +133,25 @@ class ClassLayoutTable } // Get the layout having the specified size but no class handle. - ClassLayout* GetBlkLayout(Compiler* compiler, unsigned blockSize) + ClassLayout* GetCustomLayout(Compiler* compiler, const ClassLayoutBuilder& builder) { - if (blockSize == 0) + if (builder.m_size == 0) { return &m_zeroSizedBlockLayout; } - return GetLayoutByIndex(GetBlkLayoutIndex(compiler, blockSize)); + return GetLayoutByIndex(GetCustomLayoutIndex(compiler, builder)); } // Get a number that uniquely identifies a layout having the specified size but no class handle. - unsigned GetBlkLayoutNum(Compiler* compiler, unsigned blockSize) + unsigned GetCustomLayoutNum(Compiler* compiler, const ClassLayoutBuilder& builder) { - if (blockSize == 0) + if (builder.m_size == 0) { return ZeroSizedBlockLayoutNum; } - return GetBlkLayoutIndex(compiler, blockSize) + FirstLayoutNum; + return GetCustomLayoutIndex(compiler, builder) + FirstLayoutNum; } // Get the layout for the specified class handle. @@ -147,8 +204,8 @@ class ClassLayoutTable else { unsigned index = 0; - if ((layout->IsBlockLayout() && m_blkLayoutMap->Lookup(layout->GetSize(), &index)) || - m_objLayoutMap->Lookup(layout->GetClassHandle(), &index)) + if (layout->IsCustomLayout() ? m_customLayoutMap->Lookup(CustomLayoutKey(layout), &index) + : m_objLayoutMap->Lookup(layout->GetClassHandle(), &index)) { return index; } @@ -157,16 +214,19 @@ class ClassLayoutTable unreached(); } - unsigned GetBlkLayoutIndex(Compiler* compiler, unsigned blockSize) + unsigned GetCustomLayoutIndex(Compiler* compiler, const ClassLayoutBuilder& builder) { - // The 0-sized block layout has its own fast path. - assert(blockSize != 0); + // The 0-sized layout has its own fast path. + assert(builder.m_size != 0); + + CustomLayoutKey key(builder); if (HasSmallCapacity()) { for (unsigned i = 0; i < m_layoutCount; i++) { - if (m_layoutArray[i]->IsBlockLayout() && (m_layoutArray[i]->GetSize() == blockSize)) + if (m_layoutArray[i]->IsCustomLayout() && + CustomLayoutKey::Equals(key, CustomLayoutKey(m_layoutArray[i]))) { return i; } @@ -175,21 +235,16 @@ class ClassLayoutTable else { unsigned index; - if (m_blkLayoutMap->Lookup(blockSize, &index)) + if (m_customLayoutMap->Lookup(key, &index)) { return index; } } - return AddBlkLayout(compiler, CreateBlkLayout(compiler, blockSize)); + return AddCustomLayout(compiler, ClassLayout::Create(compiler, builder)); } - ClassLayout* CreateBlkLayout(Compiler* compiler, unsigned blockSize) - { - return new (compiler, CMK_ClassLayout) ClassLayout(blockSize); - } - - unsigned AddBlkLayout(Compiler* compiler, ClassLayout* layout) + unsigned AddCustomLayout(Compiler* compiler, ClassLayout* layout) { if (m_layoutCount < ArrLen(m_layoutArray)) { @@ -198,7 +253,7 @@ class ClassLayoutTable } unsigned index = AddLayoutLarge(compiler, layout); - m_blkLayoutMap->Set(layout->GetSize(), index); + m_customLayoutMap->Set(CustomLayoutKey(layout), index); return index; } @@ -225,12 +280,7 @@ class ClassLayoutTable } } - return AddObjLayout(compiler, CreateObjLayout(compiler, classHandle)); - } - - ClassLayout* CreateObjLayout(Compiler* compiler, CORINFO_CLASS_HANDLE classHandle) - { - return ClassLayout::Create(compiler, classHandle); + return AddObjLayout(compiler, ClassLayout::Create(compiler, classHandle)); } unsigned AddObjLayout(Compiler* compiler, ClassLayout* layout) @@ -256,17 +306,17 @@ class ClassLayoutTable if (m_layoutCount <= ArrLen(m_layoutArray)) { - BlkLayoutIndexMap* blkLayoutMap = new (alloc) BlkLayoutIndexMap(alloc); - ObjLayoutIndexMap* objLayoutMap = new (alloc) ObjLayoutIndexMap(alloc); + CustomLayoutIndexMap* customLayoutMap = new (alloc) CustomLayoutIndexMap(alloc); + ObjLayoutIndexMap* objLayoutMap = new (alloc) ObjLayoutIndexMap(alloc); for (unsigned i = 0; i < m_layoutCount; i++) { ClassLayout* l = m_layoutArray[i]; newArray[i] = l; - if (l->IsBlockLayout()) + if (l->IsCustomLayout()) { - blkLayoutMap->Set(l->GetSize(), i); + customLayoutMap->Set(CustomLayoutKey(l), i); } else { @@ -274,8 +324,8 @@ class ClassLayoutTable } } - m_blkLayoutMap = blkLayoutMap; - m_objLayoutMap = objLayoutMap; + m_customLayoutMap = customLayoutMap; + m_objLayoutMap = objLayoutMap; } else { @@ -334,26 +384,46 @@ unsigned Compiler::typGetLayoutNum(ClassLayout* layout) return typGetClassLayoutTable()->GetLayoutNum(layout); } -unsigned Compiler::typGetBlkLayoutNum(unsigned blockSize) +unsigned Compiler::typGetObjLayoutNum(CORINFO_CLASS_HANDLE classHandle) { - return typGetClassLayoutTable()->GetBlkLayoutNum(this, blockSize); + return typGetClassLayoutTable()->GetObjLayoutNum(this, classHandle); } -ClassLayout* Compiler::typGetBlkLayout(unsigned blockSize) +ClassLayout* Compiler::typGetObjLayout(CORINFO_CLASS_HANDLE classHandle) { - return typGetClassLayoutTable()->GetBlkLayout(this, blockSize); + return typGetClassLayoutTable()->GetObjLayout(this, classHandle); } -unsigned Compiler::typGetObjLayoutNum(CORINFO_CLASS_HANDLE classHandle) +unsigned Compiler::typGetCustomLayoutNum(const ClassLayoutBuilder& builder) { - return typGetClassLayoutTable()->GetObjLayoutNum(this, classHandle); + return typGetClassLayoutTable()->GetCustomLayoutNum(this, builder); } -ClassLayout* Compiler::typGetObjLayout(CORINFO_CLASS_HANDLE classHandle) +ClassLayout* Compiler::typGetCustomLayout(const ClassLayoutBuilder& builder) { - return typGetClassLayoutTable()->GetObjLayout(this, classHandle); + return typGetClassLayoutTable()->GetCustomLayout(this, builder); } +unsigned Compiler::typGetBlkLayoutNum(unsigned blockSize) +{ + return typGetCustomLayoutNum(ClassLayoutBuilder(this, blockSize)); +} + +ClassLayout* Compiler::typGetBlkLayout(unsigned blockSize) +{ + return typGetCustomLayout(ClassLayoutBuilder(this, blockSize)); +} + +//------------------------------------------------------------------------ +// Create: Create a ClassLayout from an EE side class handle. +// +// Parameters: +// compiler - The Compiler object +// classHandle - The class handle +// +// Return value: +// New layout representing an EE side class. +// ClassLayout* ClassLayout::Create(Compiler* compiler, CORINFO_CLASS_HANDLE classHandle) { bool isValueClass = compiler->eeIsValueClass(classHandle); @@ -375,49 +445,82 @@ ClassLayout* ClassLayout::Create(Compiler* compiler, CORINFO_CLASS_HANDLE classH ClassLayout* layout = new (compiler, CMK_ClassLayout) ClassLayout(classHandle, isValueClass, size, type DEBUGARG(className) DEBUGARG(shortClassName)); - layout->InitializeGCPtrs(compiler); - return layout; -} - -void ClassLayout::InitializeGCPtrs(Compiler* compiler) -{ - assert(!m_gcPtrsInitialized); - assert(!IsBlockLayout()); - - if (m_size < TARGET_POINTER_SIZE) + if (layout->m_size < TARGET_POINTER_SIZE) { - assert(GetSlotCount() == 1); - assert(m_gcPtrCount == 0); + assert(layout->GetSlotCount() == 1); + assert(layout->m_gcPtrCount == 0); - m_gcPtrsArray[0] = TYPE_GC_NONE; + layout->m_gcPtrsArray[0] = TYPE_GC_NONE; } else { BYTE* gcPtrs; - - if (GetSlotCount() > sizeof(m_gcPtrsArray)) + if (layout->GetSlotCount() <= sizeof(m_gcPtrsArray)) { - gcPtrs = m_gcPtrs = new (compiler, CMK_ClassLayout) BYTE[GetSlotCount()]; + gcPtrs = layout->m_gcPtrsArray; } else { - gcPtrs = m_gcPtrsArray; + layout->m_gcPtrs = gcPtrs = new (compiler, CMK_ClassLayout) BYTE[layout->GetSlotCount()]; } - unsigned gcPtrCount = compiler->info.compCompHnd->getClassGClayout(m_classHandle, gcPtrs); + unsigned gcPtrCount = compiler->info.compCompHnd->getClassGClayout(classHandle, gcPtrs); - assert((gcPtrCount == 0) || ((compiler->info.compCompHnd->getClassAttribs(m_classHandle) & + assert((gcPtrCount == 0) || ((compiler->info.compCompHnd->getClassAttribs(classHandle) & (CORINFO_FLG_CONTAINS_GC_PTR | CORINFO_FLG_BYREF_LIKE)) != 0)); // Since class size is unsigned there's no way we could have more than 2^30 slots // so it should be safe to fit this into a 30 bits bit field. assert(gcPtrCount < (1 << 30)); - m_gcPtrCount = gcPtrCount; + layout->m_gcPtrCount = gcPtrCount; } - INDEBUG(m_gcPtrsInitialized = true;) + return layout; +} + +//------------------------------------------------------------------------ +// Create: Create a ClassLayout from a ClassLayoutBuilder. +// +// Parameters: +// compiler - The Compiler object +// builder - Builder representing the layout +// +// Return value: +// New layout representing a custom (JIT internal) class layout. +// +ClassLayout* ClassLayout::Create(Compiler* compiler, const ClassLayoutBuilder& builder) +{ + ClassLayout* newLayout = new (compiler, CMK_ClassLayout) ClassLayout(builder.m_size); + newLayout->m_gcPtrCount = builder.m_gcPtrCount; + +#ifdef DEBUG + newLayout->m_name = builder.m_name; + newLayout->m_shortName = builder.m_shortName; +#endif + + if (newLayout->GetSlotCount() <= sizeof(newLayout->m_gcPtrsArray)) + { + if (builder.m_gcPtrCount > 0) + { + memcpy(newLayout->m_gcPtrsArray, builder.m_gcPtrs, newLayout->GetSlotCount()); + } + else + { + memset(newLayout->m_gcPtrsArray, TYPE_GC_NONE, newLayout->GetSlotCount()); + } + } + else if (builder.m_gcPtrCount > 0) + { + newLayout->m_gcPtrs = builder.m_gcPtrs; + } + else + { + newLayout->m_gcPtrs = new (compiler, CMK_ClassLayout) BYTE[newLayout->GetSlotCount()]{}; + } + + return newLayout; } //------------------------------------------------------------------------ @@ -497,9 +600,23 @@ bool ClassLayout::AreCompatible(const ClassLayout* layout1, const ClassLayout* l CORINFO_CLASS_HANDLE clsHnd1 = layout1->GetClassHandle(); CORINFO_CLASS_HANDLE clsHnd2 = layout2->GetClassHandle(); - if ((clsHnd1 != NO_CLASS_HANDLE) && (clsHnd1 == clsHnd2)) + if ((clsHnd1 != NO_CLASS_HANDLE) == (clsHnd2 != NO_CLASS_HANDLE)) { - return true; + // Either both are class-based layout or both are custom layouts. + // Custom layouts only match each other if they are the same pointer. + if (clsHnd1 == NO_CLASS_HANDLE) + { + return layout1 == layout2; + } + + // For class-based layouts they are definitely compatible for the same + // handle + if (clsHnd1 == clsHnd2) + { + return true; + } + + // But they may still be compatible for different handles. } if (layout1->GetSize() != layout2->GetSize()) @@ -543,3 +660,132 @@ bool ClassLayout::AreCompatible(const ClassLayout* layout1, const ClassLayout* l } return true; } + +//------------------------------------------------------------------------ +// ClassLayoutbuilder: Construct a new builder for a class layout of the +// specified size. +// +// Arguments: +// compiler - Compiler instance +// size - Size of the layout +// +ClassLayoutBuilder::ClassLayoutBuilder(Compiler* compiler, unsigned size) + : m_compiler(compiler) + , m_size(size) +{ +} + +//------------------------------------------------------------------------ +// GetOrCreateGCPtrs: Get or create the array indicating GC pointer types. +// +// Returns: +// The array of CorInfoGCType. +// +BYTE* ClassLayoutBuilder::GetOrCreateGCPtrs() +{ + assert(m_size % TARGET_POINTER_SIZE == 0); + + if (m_gcPtrs == nullptr) + { + m_gcPtrs = new (m_compiler, CMK_ClassLayout) BYTE[m_size / TARGET_POINTER_SIZE]{}; + } + + return m_gcPtrs; +} + +//------------------------------------------------------------------------ +// SetGCPtr: Set a slot to have specified GC pointer type. +// +// Arguments: +// slot - The GC pointer slot. The slot number corresponds to offset slot * TARGET_POINTER_SIZE. +// type - Type of GC pointer that this slot contains. +// +// Remarks: +// GC pointer information can only be set in layouts of size divisible by +// TARGET_POINTER_SIZE. +// +void ClassLayoutBuilder::SetGCPtr(unsigned slot, CorInfoGCType type) +{ + BYTE* ptrs = GetOrCreateGCPtrs(); + + assert(slot * TARGET_POINTER_SIZE < m_size); + + if (ptrs[slot] != TYPE_GC_NONE) + { + m_gcPtrCount--; + } + + ptrs[slot] = static_cast(type); + + if (type != TYPE_GC_NONE) + { + m_gcPtrCount++; + } +} + +//------------------------------------------------------------------------ +// SetGCPtrType: Set a slot to have specified type. +// +// Arguments: +// slot - The GC pointer slot. The slot number corresponds to offset slot * TARGET_POINTER_SIZE. +// type - Type that this slot contains. Must be TYP_REF, TYP_BYREF or TYP_I_IMPL. +// +// Remarks: +// GC pointer information can only be set in layouts of size divisible by +// TARGET_POINTER_SIZE. +// +void ClassLayoutBuilder::SetGCPtrType(unsigned slot, var_types type) +{ + switch (type) + { + case TYP_REF: + SetGCPtr(slot, TYPE_GC_REF); + break; + case TYP_BYREF: + SetGCPtr(slot, TYPE_GC_BYREF); + break; + case TYP_I_IMPL: + SetGCPtr(slot, TYPE_GC_NONE); + break; + default: + assert(!"Invalid type passed to ClassLayoutBuilder::SetGCPtrType"); + break; + } +} + +//------------------------------------------------------------------------ +// CopyInfoFrom: Copy GC pointers from another layout. +// +// Arguments: +// offset - Offset in this builder to start copy information into. +// layout - Layout to get information from. +// +void ClassLayoutBuilder::CopyInfoFrom(unsigned offset, ClassLayout* layout) +{ + assert(offset + layout->GetSize() <= m_size); + + if (layout->GetGCPtrCount() > 0) + { + assert(offset % TARGET_POINTER_SIZE == 0); + unsigned startSlot = offset / TARGET_POINTER_SIZE; + for (unsigned slot = 0; slot < layout->GetSlotCount(); slot++) + { + SetGCPtr(startSlot + slot, layout->GetGCPtr(slot)); + } + } +} + +#ifdef DEBUG +//------------------------------------------------------------------------ +// SetName: Set the long and short name of the layout. +// +// Arguments: +// name - The long name +// shortName - The short name +// +void ClassLayoutBuilder::SetName(const char* name, const char* shortName) +{ + m_name = name; + m_shortName = shortName; +} +#endif diff --git a/src/coreclr/jit/layout.h b/src/coreclr/jit/layout.h index c2c901d1f33c9a..97122ced8d5476 100644 --- a/src/coreclr/jit/layout.h +++ b/src/coreclr/jit/layout.h @@ -6,13 +6,46 @@ #include "jit.h" +// Builder for class layouts +// +class ClassLayoutBuilder +{ + friend class ClassLayout; + friend class ClassLayoutTable; + friend struct CustomLayoutKey; + + Compiler* m_compiler; + BYTE* m_gcPtrs = nullptr; + unsigned m_size; + unsigned m_gcPtrCount = 0; +#ifdef DEBUG + const char* m_name = "UNNAMED"; + const char* m_shortName = "UNNAMED"; +#endif + + BYTE* GetOrCreateGCPtrs(); + void SetGCPtr(unsigned slot, CorInfoGCType type); +public: + // Create a class layout builder. + // + ClassLayoutBuilder(Compiler* compiler, unsigned size); + + void SetGCPtrType(unsigned slot, var_types type); + void CopyInfoFrom(unsigned offset, ClassLayout* layout); + +#ifdef DEBUG + void SetName(const char* name, const char* shortName); +#endif +}; + // Encapsulates layout information about a class (typically a value class but this can also be // be used for reference classes when they are stack allocated). The class handle is optional, -// allowing the creation of "block" layout objects having a specific size but lacking any other -// layout information. The JIT uses such layout objects in cases where a class handle is not -// available (cpblk/initblk operations) or not necessary (classes that do not contain GC pointers). +// allowing the creation of custom layout objects having a specific size where the offsets of +// GC fields can be specified during creation. class ClassLayout { +private: + // Class handle or NO_CLASS_HANDLE for "block" layouts. const CORINFO_CLASS_HANDLE m_classHandle; @@ -22,7 +55,6 @@ class ClassLayout const unsigned m_size; const unsigned m_isValueClass : 1; - INDEBUG(unsigned m_gcPtrsInitialized : 1;) // The number of GC pointers in this layout. Since the maximum size is 2^32-1 the count // can fit in at most 30 bits. unsigned m_gcPtrCount : 30; @@ -39,33 +71,33 @@ class ClassLayout // The normalized type to use in IR for block nodes with this layout. const var_types m_type; - // Class name as reported by ICorJitInfo::getClassName - INDEBUG(const char* m_className;) + // Name of the layout + INDEBUG(const char* m_name;) - // Shortened class name as constructed by Compiler::eeGetShortClassName() - INDEBUG(const char* m_shortClassName;) + // Short name of the layout + INDEBUG(const char* m_shortName;) // ClassLayout instances should only be obtained via ClassLayoutTable. friend class ClassLayoutTable; + friend class ClassLayoutBuilder; + friend struct CustomLayoutKey; ClassLayout(unsigned size) : m_classHandle(NO_CLASS_HANDLE) , m_size(size) , m_isValueClass(false) -#ifdef DEBUG - , m_gcPtrsInitialized(true) -#endif , m_gcPtrCount(0) , m_gcPtrs(nullptr) , m_type(TYP_STRUCT) #ifdef DEBUG - , m_className("block") - , m_shortClassName("block") + , m_name(size == 0 ? "Empty" : "Custom") + , m_shortName(size == 0 ? "Empty" : "Custom") #endif { } static ClassLayout* Create(Compiler* compiler, CORINFO_CLASS_HANDLE classHandle); + static ClassLayout* Create(Compiler* compiler, const ClassLayoutBuilder& builder); ClassLayout(CORINFO_CLASS_HANDLE classHandle, bool isValueClass, @@ -74,43 +106,43 @@ class ClassLayout : m_classHandle(classHandle) , m_size(size) , m_isValueClass(isValueClass) -#ifdef DEBUG - , m_gcPtrsInitialized(false) -#endif , m_gcPtrCount(0) , m_gcPtrs(nullptr) , m_type(type) #ifdef DEBUG - , m_className(className) - , m_shortClassName(shortClassName) + , m_name(className) + , m_shortName(shortClassName) #endif { assert(size != 0); } - - void InitializeGCPtrs(Compiler* compiler); - public: + CORINFO_CLASS_HANDLE GetClassHandle() const { return m_classHandle; } - bool IsBlockLayout() const + bool IsCustomLayout() const { return m_classHandle == NO_CLASS_HANDLE; } + bool IsBlockLayout() const + { + return IsCustomLayout() && !HasGCPtr(); + } + #ifdef DEBUG const char* GetClassName() const { - return m_className; + return m_name; } const char* GetShortClassName() const { - return m_shortClassName; + return m_shortName; } #endif // DEBUG @@ -134,7 +166,7 @@ class ClassLayout // GetRegisterType: Determine register type for the layout. // // Return Value: - // TYP_UNDEF if the layout is enregistrable, register type otherwise. + // TYP_UNDEF if the layout is not enregistrable, register type otherwise. // var_types GetRegisterType() const { @@ -173,15 +205,11 @@ class ClassLayout unsigned GetGCPtrCount() const { - assert(m_gcPtrsInitialized); - return m_gcPtrCount; } bool HasGCPtr() const { - assert(m_gcPtrsInitialized); - return m_gcPtrCount != 0; } @@ -224,15 +252,11 @@ class ClassLayout private: const BYTE* GetGCPtrs() const { - assert(m_gcPtrsInitialized); - assert(!IsBlockLayout()); - return (GetSlotCount() > sizeof(m_gcPtrsArray)) ? m_gcPtrs : m_gcPtrsArray; } CorInfoGCType GetGCPtr(unsigned slot) const { - assert(m_gcPtrsInitialized); assert(slot < GetSlotCount()); if (m_gcPtrCount == 0) diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index dc5144b0431659..f9d84a12bc9a28 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -2497,9 +2497,9 @@ bool Compiler::StructPromotionHelper::CanPromoteStructVar(unsigned lclNum) return false; } - if (varDsc->GetLayout()->IsBlockLayout()) + if (varDsc->GetLayout()->IsCustomLayout()) { - JITDUMP(" struct promotion of V%02u is disabled because it has block layout\n", lclNum); + JITDUMP(" struct promotion of V%02u is disabled because it has custom layout\n", lclNum); return false; } @@ -2764,8 +2764,7 @@ void Compiler::StructPromotionHelper::PromoteStructVar(unsigned lclNum) #ifdef DEBUG if (compiler->verbose) { - printf("\nPromoting struct local V%02u (%s):", lclNum, - compiler->eeGetClassName(varDsc->GetLayout()->GetClassHandle())); + printf("\nPromoting struct local V%02u (%s):", lclNum, varDsc->GetLayout()->GetClassName()); } #endif @@ -3368,10 +3367,10 @@ void Compiler::lvaSetStruct(unsigned varNum, ClassLayout* layout, bool unsafeVal assert(ClassLayout::AreCompatible(varDsc->GetLayout(), layout)); // Inlining could replace a canon struct type with an exact one. varDsc->SetLayout(layout); - assert(layout->IsBlockLayout() || (layout->GetSize() != 0)); + assert(layout->IsCustomLayout() || (layout->GetSize() != 0)); } - if (!layout->IsBlockLayout()) + if (!layout->IsCustomLayout()) { #ifndef TARGET_64BIT bool fDoubleAlignHint = false; @@ -7643,7 +7642,7 @@ void Compiler::lvaDumpEntry(unsigned lclNum, FrameLayoutState curState, size_t r else if (varTypeIsStruct(varDsc->TypeGet())) { ClassLayout* layout = varDsc->GetLayout(); - if (layout != nullptr && !layout->IsBlockLayout()) + if (layout != nullptr) { printf(" <%s>", layout->GetClassName()); } @@ -8097,8 +8096,8 @@ Compiler::fgWalkResult Compiler::lvaStressLclFldCB(GenTree** pTree, fgWalkData* return WALK_CONTINUE; } - // Can't have GC ptrs in block layouts. - if (!varTypeIsArithmetic(lclType)) + // Structs are not currently supported + if (varTypeIsStruct(lclType)) { varDsc->lvNoLclFldStress = true; return WALK_CONTINUE; @@ -8112,6 +8111,13 @@ Compiler::fgWalkResult Compiler::lvaStressLclFldCB(GenTree** pTree, fgWalkData* return WALK_CONTINUE; } + // Pinned locals would not remain pinned if we did this transformation. + if (varDsc->lvPinned) + { + varDsc->lvNoLclFldStress = true; + return WALK_CONTINUE; + } + // Weed out "small" types like TYP_BYTE as we don't mark the GT_LCL_VAR // node with the accurate small type. If we bash lvaTable[].lvType, // then there will be no indication that it was ever a small type. @@ -8134,7 +8140,7 @@ Compiler::fgWalkResult Compiler::lvaStressLclFldCB(GenTree** pTree, fgWalkData* else { // Do the morphing - noway_assert((varType == lclType) || ((varType == TYP_STRUCT) && varDsc->GetLayout()->IsBlockLayout())); + noway_assert((varType == lclType) || ((varType == TYP_STRUCT) && varDsc->GetLayout()->IsCustomLayout())); // Calculate padding unsigned padding = pComp->lvaStressLclFldPadding(lclNum); @@ -8145,17 +8151,34 @@ Compiler::fgWalkResult Compiler::lvaStressLclFldCB(GenTree** pTree, fgWalkData* padding = roundUp(padding, genTypeSize(TYP_DOUBLE)); #endif // defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) + // Also for GC types we need to round up + if (varTypeIsGC(varType) || ((varType == TYP_STRUCT) && varDsc->GetLayout()->HasGCPtr())) + { + padding = roundUp(padding, TARGET_POINTER_SIZE); + } + if (varType != TYP_STRUCT) { - // Change the variable to a block struct - ClassLayout* layout = - pComp->typGetBlkLayout(roundUp(padding + pComp->lvaLclSize(lclNum), TARGET_POINTER_SIZE)); - varDsc->lvType = TYP_STRUCT; + // Change the variable to a custom layout struct + unsigned size = roundUp(padding + pComp->lvaLclSize(lclNum), TARGET_POINTER_SIZE); + ClassLayoutBuilder builder(pComp, size); +#ifdef DEBUG + builder.SetName(pComp->printfAlloc("%s_%u_Stress", varTypeName(varType), size), + pComp->printfAlloc("%s_%u", varTypeName(varType), size)); +#endif + + if (varTypeIsGC(varType)) + { + builder.SetGCPtrType(padding / TARGET_POINTER_SIZE, varType); + } + + ClassLayout* layout = pComp->typGetCustomLayout(builder); + varDsc->lvType = TYP_STRUCT; varDsc->SetLayout(layout); pComp->lvaSetVarAddrExposed(lclNum DEBUGARG(AddressExposedReason::STRESS_LCL_FLD)); - JITDUMP("Converting V%02u to %u sized block with LCL_FLD at offset (padding %u)\n", lclNum, - layout->GetSize(), padding); + JITDUMP("Converting V%02u of type %s to %u sized block with LCL_FLD at offset (padding %u)\n", lclNum, + varTypeName(varType), layout->GetSize(), padding); } tree->gtFlags |= GTF_GLOB_REF; diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 4c0fc233196482..fb362d51b381b9 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -3296,7 +3296,7 @@ void Compiler::fgMakeOutgoingStructArgCopy(GenTreeCall* call, CallArg* arg) found = ForEachHbvBitSet(*fgAvailableOutgoingArgTemps, [&](indexType lclNum) { LclVarDsc* varDsc = lvaGetDesc((unsigned)lclNum); ClassLayout* layout = varDsc->GetLayout(); - if (!layout->IsBlockLayout() && (layout->GetClassHandle() == copyBlkClass)) + if (!layout->IsCustomLayout() && (layout->GetClassHandle() == copyBlkClass)) { tmp = (unsigned)lclNum; JITDUMP("reusing outgoing struct arg V%02u\n", tmp);