Move return buffer handling from interop to the JIT

src/coreclr/src/inc/corinfo.h

@@ -734,15 +734,21 @@ inline bool IsCallerPop(CorInfoCallConv callConv)
 }
 #endif // UNIX_X86_ABI
 
+// Represents the calling conventions supported with the extensible calling convention syntax
+// as well as the metadata-encoded calling conventions and the managed calling convention.
 enum CorInfoUnmanagedCallConv
 {
     // These correspond to CorUnmanagedCallingConvention
-
     CORINFO_UNMANAGED_CALLCONV_UNKNOWN,
     CORINFO_UNMANAGED_CALLCONV_C,
     CORINFO_UNMANAGED_CALLCONV_STDCALL,
     CORINFO_UNMANAGED_CALLCONV_THISCALL,
-    CORINFO_UNMANAGED_CALLCONV_FASTCALL
+    CORINFO_UNMANAGED_CALLCONV_FASTCALL,
+    // This represents the managed calling convention. We need some way to represent the
+    // managed calling convention here since we use this type within the JIT for choosing
+    // how to home return values and arguments.
+    CORINFO_UNMANAGED_CALLCONV_MANAGED
+    // New calling conventions supported with the extensible calling convention encoding go here.
 };
 
 // These are returned from getMethodOptions

src/coreclr/src/jit/codegencommon.cpp

@@ -10065,15 +10065,15 @@ void CodeGen::genVzeroupperIfNeeded(bool check256bitOnly /* = true*/)
 // Return Value:
 //     true if type is returned in multiple registers, false otherwise.
 //
-bool Compiler::IsMultiRegReturnedType(CORINFO_CLASS_HANDLE hClass)
+bool Compiler::IsMultiRegReturnedType(CORINFO_CLASS_HANDLE hClass, CorInfoUnmanagedCallConv callConv)
 {
     if (hClass == NO_CLASS_HANDLE)
     {
         return false;
     }
 
     structPassingKind howToReturnStruct;
-    var_types         returnType = getReturnTypeForStruct(hClass, &howToReturnStruct);
+    var_types         returnType = getReturnTypeForStruct(hClass, callConv, &howToReturnStruct);
 
 #ifdef TARGET_ARM64
     return (varTypeIsStruct(returnType) && (howToReturnStruct != SPK_PrimitiveType));
@@ -11529,7 +11529,11 @@ void CodeGen::genReturn(GenTree* treeNode)
             }
             else // we must have a struct return type
             {
-                retTypeDesc.InitializeStructReturnType(compiler, compiler->info.compMethodInfo->args.retTypeClass);
+                CorInfoUnmanagedCallConv callConv =
+                    compiler->compMethodInfoGetUnmanagedCallConv(compiler->info.compMethodInfo);
+
+                retTypeDesc.InitializeStructReturnType(compiler, compiler->info.compMethodInfo->args.retTypeClass,
+                                                       callConv);
             }
             regCount = retTypeDesc.GetReturnRegCount();
         }
@@ -11654,7 +11658,9 @@ void CodeGen::genStructReturn(GenTree* treeNode)
     if (actualOp1->OperIs(GT_LCL_VAR))
     {
         varDsc = compiler->lvaGetDesc(actualOp1->AsLclVar()->GetLclNum());
-        retTypeDesc.InitializeStructReturnType(compiler, varDsc->GetStructHnd());
+        retTypeDesc.InitializeStructReturnType(compiler, varDsc->GetStructHnd(),
+                                               compiler->compMethodInfoGetUnmanagedCallConv(
+                                                   compiler->info.compMethodInfo));
         assert(varDsc->lvIsMultiRegRet);
     }
     else
@@ -11834,7 +11840,8 @@ void CodeGen::genMultiRegStoreToLocal(GenTreeLclVar* lclNode)
                 hasRegs = true;
                 if (varReg != reg)
                 {
-                    inst_RV_RV(ins_Copy(type), varReg, reg, type);
+                    // We may need a cross register-file copy here.
+                    inst_RV_RV(ins_Copy(reg, type), varReg, reg, type);
                 }
                 fieldVarDsc->SetRegNum(varReg);
             }

src/coreclr/src/jit/codegenxarch.cpp

@@ -140,7 +140,9 @@ void CodeGen::genEmitGSCookieCheck(bool pushReg)
             }
             else // we must have a struct return type
             {
-                retTypeDesc.InitializeStructReturnType(compiler, compiler->info.compMethodInfo->args.retTypeClass);
+                retTypeDesc.InitializeStructReturnType(compiler, compiler->info.compMethodInfo->args.retTypeClass,
+                                                       compiler->compMethodInfoGetUnmanagedCallConv(
+                                                           compiler->info.compMethodInfo));
             }
 
             const unsigned regCount = retTypeDesc.GetReturnRegCount();
@@ -5709,12 +5711,30 @@ void CodeGen::genJmpMethod(GenTree* jmp)
 #endif // !defined(UNIX_AMD64_ABI)
         {
             // Register argument
+            CLANG_FORMAT_COMMENT_ANCHOR;
+#ifdef TARGET_X86
+            noway_assert(
+                isRegParamType(genActualType(varDsc->TypeGet())) ||
+                (varTypeIsStruct(varDsc->TypeGet()) && compiler->isTrivialPointerSizedStruct(varDsc->GetStructHnd())));
+#else
             noway_assert(isRegParamType(genActualType(varDsc->TypeGet())));
+#endif // TARGET_X86
 
             // Is register argument already in the right register?
             // If not load it from its stack location.
             var_types loadType = varDsc->lvaArgType();
-            regNumber argReg   = varDsc->GetArgReg(); // incoming arg register
+
+#ifdef TARGET_X86
+            if (varTypeIsStruct(varDsc->TypeGet()))
+            {
+                assert(compiler->isTrivialPointerSizedStruct(varDsc->GetStructHnd()));
+            }
+            // Treat trivial pointer-sized structs as a pointer sized primitive
+            // for the purposes of registers.
+            loadType = TYP_I_IMPL;
+#endif
+
+            regNumber argReg = varDsc->GetArgReg(); // incoming arg register
 
             if (varDsc->GetRegNum() != argReg)
             {

src/coreclr/src/jit/compiler.cpp

@@ -512,6 +512,55 @@ bool Compiler::isSingleFloat32Struct(CORINFO_CLASS_HANDLE clsHnd)
 }
 #endif // ARM_SOFTFP
 
+#ifdef TARGET_X86
+//---------------------------------------------------------------------------
+// isTrivialPointerSizedStruct:
+//    Check if the given struct type contains only one pointer-sized integer value type
+//
+// Arguments:
+//    clsHnd - the handle for the struct type.
+//
+// Return Value:
+//    true if the given struct type contains only one pointer-sized integer value type,
+//    false otherwise.
+//
+bool Compiler::isTrivialPointerSizedStruct(CORINFO_CLASS_HANDLE clsHnd) const
+{
+    assert(info.compCompHnd->isValueClass(clsHnd));
+    if (info.compCompHnd->getClassSize(clsHnd) != TARGET_POINTER_SIZE)
+    {
+        return false;
+    }
+    for (;;)
+    {
+        // all of class chain must be of value type and must have only one field
+        if (!info.compCompHnd->isValueClass(clsHnd) || info.compCompHnd->getClassNumInstanceFields(clsHnd) != 1)
+        {
+            return false;
+        }
+
+        CORINFO_CLASS_HANDLE* pClsHnd   = &clsHnd;
+        CORINFO_FIELD_HANDLE  fldHnd    = info.compCompHnd->getFieldInClass(clsHnd, 0);
+        CorInfoType           fieldType = info.compCompHnd->getFieldType(fldHnd, pClsHnd);
+
+        var_types vt = JITtype2varType(fieldType);
+
+        if (fieldType == CORINFO_TYPE_VALUECLASS)
+        {
+            clsHnd = *pClsHnd;
+        }
+        else if (varTypeIsI(vt) && !varTypeIsGC(vt))
+        {
+            return true;
+        }
+        else
+        {
+            return false;
+        }
+    }
+}
+#endif // TARGET_X86
+
 //-----------------------------------------------------------------------------
 // getPrimitiveTypeForStruct:
 //     Get the "primitive" type that is is used for a struct
@@ -693,7 +742,7 @@ var_types Compiler::getArgTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
     assert(structSize != 0);
 
 // Determine if we can pass the struct as a primitive type.
-// Note that on x86 we never pass structs as primitive types (unless the VM unwraps them for us).
+// Note that on x86 we only pass specific pointer-sized structs as primitives that the VM used to unwrap.
 #ifndef TARGET_X86
 #ifdef UNIX_AMD64_ABI
 
@@ -728,7 +777,11 @@ var_types Compiler::getArgTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
         // and also examine the clsHnd to see if it is an HFA of count one
         useType = getPrimitiveTypeForStruct(structSize, clsHnd, isVarArg);
     }
-
+#else
+    if (isTrivialPointerSizedStruct(clsHnd))
+    {
+        useType = TYP_I_IMPL;
+    }
 #endif // !TARGET_X86
 
     // Did we change this struct type into a simple "primitive" type?
@@ -876,6 +929,8 @@ var_types Compiler::getArgTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
 //
 // Arguments:
 //    clsHnd         - the handle for the struct type
+//    callConv       - the calling convention of the function
+//                     that returns this struct.
 //    wbReturnStruct - An "out" argument with information about how
 //                     the struct is to be returned
 //    structSize     - the size of the struct type,
@@ -910,9 +965,10 @@ var_types Compiler::getArgTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
 //        Whenever this method's return value is TYP_STRUCT it always means
 //         that multiple registers are used to return this struct.
 //
-var_types Compiler::getReturnTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
-                                           structPassingKind*   wbReturnStruct /* = nullptr */,
-                                           unsigned             structSize /* = 0 */)
+var_types Compiler::getReturnTypeForStruct(CORINFO_CLASS_HANDLE     clsHnd,
+                                           CorInfoUnmanagedCallConv callConv,
+                                           structPassingKind*       wbReturnStruct /* = nullptr */,
+                                           unsigned                 structSize /* = 0 */)
 {
     var_types         useType             = TYP_UNKNOWN;
     structPassingKind howToReturnStruct   = SPK_Unknown; // We must change this before we return
@@ -950,10 +1006,32 @@ var_types Compiler::getReturnTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
     {
         // Return classification is not always size based...
         canReturnInRegister = structDesc.passedInRegisters;
+        if (!canReturnInRegister)
+        {
+            assert(structDesc.eightByteCount == 0);
+            howToReturnStruct = SPK_ByReference;
+            useType           = TYP_UNKNOWN;
+        }
     }
 
 #endif // UNIX_AMD64_ABI
 
+#ifdef UNIX_X86_ABI
+    if (callConv != CORINFO_UNMANAGED_CALLCONV_MANAGED)
+    {
+        canReturnInRegister = false;
+        howToReturnStruct   = SPK_ByReference;
+        useType             = TYP_UNKNOWN;
+    }
+#elif defined(TARGET_WINDOWS) && !defined(TARGET_ARM)
+    if (callConv == CORINFO_UNMANAGED_CALLCONV_THISCALL)
+    {
+        canReturnInRegister = false;
+        howToReturnStruct   = SPK_ByReference;
+        useType             = TYP_UNKNOWN;
+    }
+#endif
+
     // Check for cases where a small struct is returned in a register
     // via a primitive type.
     //
@@ -1008,7 +1086,7 @@ var_types Compiler::getReturnTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
         // If so, we should have already set howToReturnStruct, too.
         assert(howToReturnStruct != SPK_Unknown);
     }
-    else // We can't replace the struct with a "primitive" type
+    else if (canReturnInRegister) // We can't replace the struct with a "primitive" type
     {
         // See if we can return this struct by value, possibly in multiple registers
         // or if we should return it using a return buffer register
@@ -1031,24 +1109,13 @@ var_types Compiler::getReturnTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
 
 #ifdef UNIX_AMD64_ABI
 
-                // The case of (structDesc.eightByteCount == 1) should have already been handled
-                if (structDesc.eightByteCount > 1)
-                {
-                    // setup wbPassType and useType indicate that this is returned by value in multiple registers
-                    howToReturnStruct = SPK_ByValue;
-                    useType           = TYP_STRUCT;
-                    assert(structDesc.passedInRegisters == true);
-                }
-                else
-                {
-                    assert(structDesc.eightByteCount == 0);
-                    // Otherwise we return this struct using a return buffer
-                    // setup wbPassType and useType indicate that this is return using a return buffer register
-                    //  (reference to a return buffer)
-                    howToReturnStruct = SPK_ByReference;
-                    useType           = TYP_UNKNOWN;
-                    assert(structDesc.passedInRegisters == false);
-                }
+                // The cases of (structDesc.eightByteCount == 1) and (structDesc.eightByteCount == 0)
+                // should have already been handled
+                assert(structDesc.eightByteCount > 1);
+                // setup wbPassType and useType indicate that this is returned by value in multiple registers
+                howToReturnStruct = SPK_ByValue;
+                useType           = TYP_STRUCT;
+                assert(structDesc.passedInRegisters == true);
 
 #elif defined(TARGET_ARM64)
 
@@ -1071,8 +1138,26 @@ var_types Compiler::getReturnTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
                     howToReturnStruct = SPK_ByReference;
                     useType           = TYP_UNKNOWN;
                 }
+#elif defined(TARGET_X86)
 
-#elif defined(TARGET_ARM) || defined(TARGET_X86)
+                // Only 8-byte structs are return in multiple registers.
+                // We also only support multireg struct returns on x86 to match the native calling convention.
+                // So return 8-byte structs only when the calling convention is a native calling convention.
+                if (structSize == MAX_RET_MULTIREG_BYTES && callConv != CORINFO_UNMANAGED_CALLCONV_MANAGED)
+                {
+                    // setup wbPassType and useType indicate that this is return by value in multiple registers
+                    howToReturnStruct = SPK_ByValue;
+                    useType           = TYP_STRUCT;
+                }
+                else
+                {
+                    // Otherwise we return this struct using a return buffer
+                    // setup wbPassType and useType indicate that this is returned using a return buffer register
+                    //  (reference to a return buffer)
+                    howToReturnStruct = SPK_ByReference;
+                    useType           = TYP_UNKNOWN;
+                }
+#elif defined(TARGET_ARM)
 
                 // Otherwise we return this struct using a return buffer
                 // setup wbPassType and useType indicate that this is returned using a return buffer register
@@ -1975,6 +2060,22 @@ unsigned Compiler::compGetTypeSize(CorInfoType cit, CORINFO_CLASS_HANDLE clsHnd)
     return sigSize;
 }
 
+bool Compiler::callConvIsInstanceMethodCallConv(CorInfoUnmanagedCallConv callConv)
+{
+    return callConv == CORINFO_UNMANAGED_CALLCONV_THISCALL;
+}
+
+CorInfoUnmanagedCallConv Compiler::compMethodInfoGetUnmanagedCallConv(CORINFO_METHOD_INFO* mthInfo)
+{
+    CorInfoCallConv callConv = mthInfo->args.getCallConv();
+    if (callConv == CORINFO_CALLCONV_DEFAULT || callConv == CORINFO_CALLCONV_VARARG)
+    {
+        // Both the default and the varargs calling conventions represent a managed callconv.
+        return CORINFO_UNMANAGED_CALLCONV_MANAGED;
+    }
+    return (CorInfoUnmanagedCallConv)callConv;
+}
+
 #ifdef DEBUG
 static bool DidComponentUnitTests = false;
 
@@ -2209,7 +2310,7 @@ void Compiler::compSetProcessor()
 #elif defined(TARGET_ARM64)
     info.genCPU      = CPU_ARM64;
 #elif defined(TARGET_AMD64)
-    info.genCPU                   = CPU_X64;
+    info.genCPU = CPU_X64;
 #elif defined(TARGET_X86)
     if (jitFlags.IsSet(JitFlags::JIT_FLAG_TARGET_P4))
         info.genCPU = CPU_X86_PENTIUM_4;
@@ -5976,6 +6077,9 @@ int Compiler::compCompileHelper(CORINFO_MODULE_HANDLE classPtr,
         case CORINFO_CALLCONV_NATIVEVARARG:
             info.compIsVarArgs = true;
             break;
+        case CORINFO_CALLCONV_C:
+        case CORINFO_CALLCONV_STDCALL:
+        case CORINFO_CALLCONV_THISCALL:
         case CORINFO_CALLCONV_DEFAULT:
             info.compIsVarArgs = false;
             break;