ComInterfaceGenerator generates code with pointer type in generic

[jtschuster]

In custom collections generated code, the code generated for the following snippet contains byte* as a generic argument, causing a compiler error.

Input Code:

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.InteropServices.Marshalling;
[assembly:DisableRuntimeMarshalling]

[global::System.Runtime.InteropServices.Marshalling.GeneratedComInterfaceAttribute]
partial interface INativeAPI : IUnmanagedInterfaceType
{
    
    [return:MarshalUsing(ConstantElementCount=10)]
    System.Byte[] Method(
        System.Byte[] p,
        in System.Byte[] pIn,
        int pRefSize,
        [MarshalUsing(CountElementName = "pRefSize")] ref System.Byte[] pRef,
        [MarshalUsing(CountElementName = "pOutSize")] out System.Byte[] pOut,
        out int pOutSize);
}
// Try using the generated native interface
sealed class NativeAPI : IUnmanagedVirtualMethodTableProvider, INativeAPI.Native
{
    public VirtualMethodTableInfo GetVirtualMethodTableInfoForKey(System.Type type) => throw null;
}
partial interface INativeAPI
{
        static unsafe void* IUnmanagedInterfaceType.VirtualMethodTableManagedImplementation => null;
}

Generated Code:

// <auto-generated/>
unsafe partial interface INativeAPI
{
    internal unsafe partial interface Native
    {
        [System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute]
        internal static ABI_Method(void* __this_native, byte* __p_native, byte** __pIn_native__param, int pRefSize, byte** __pRef_native__param, byte** __pOut_native__param, int* __pOutSize_native__param)
        {
            ...
            try
            {
                ...
            }
            catch (System.Exception __exception)
            {
                // error CS0306: The type 'byte*' may not be used as a type argument
                __retVal_native = System.Runtime.InteropServices.Marshalling.ExceptionDefaultMarshaller<byte*>.ConvertToUnmanaged(__exception);
            }
            finally
            {
                // Cleanup - Perform required cleanup.
               ...
            }

            return __retVal_native;
        }
    }
}

[ghost]

Tagging subscribers to this area: @dotnet/interop-contrib
See info in area-owners.md if you want to be subscribed.

Issue Details

---

In custom collections generated code, the code generated for the following snippet contains byte* as a generic argument, causing a compiler error.

Input Code:

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.InteropServices.Marshalling;
[assembly:DisableRuntimeMarshalling]

[global::System.Runtime.InteropServices.Marshalling.GeneratedComInterfaceAttribute]
partial interface INativeAPI : IUnmanagedInterfaceType
{
    
    [return:MarshalUsing(ConstantElementCount=10)]
    System.Byte[] Method(
        System.Byte[] p,
        in System.Byte[] pIn,
        int pRefSize,
        [MarshalUsing(CountElementName = "pRefSize")] ref System.Byte[] pRef,
        [MarshalUsing(CountElementName = "pOutSize")] out System.Byte[] pOut,
        out int pOutSize);
}
// Try using the generated native interface
sealed class NativeAPI : IUnmanagedVirtualMethodTableProvider, INativeAPI.Native
{
    public VirtualMethodTableInfo GetVirtualMethodTableInfoForKey(System.Type type) => throw null;
}
partial interface INativeAPI
{
        static unsafe void* IUnmanagedInterfaceType.VirtualMethodTableManagedImplementation => null;
}

Generated Code:

// <auto-generated/>
unsafe partial interface INativeAPI
{
    internal unsafe partial interface Native
    {
        [System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute]
        internal static ABI_Method(void* __this_native, byte* __p_native, byte** __pIn_native__param, int pRefSize, byte** __pRef_native__param, byte** __pOut_native__param, int* __pOutSize_native__param)
        {
            ...
            try
            {
                ...
            }
            catch (System.Exception __exception)
            {
                // error CS0306: The type 'byte*' may not be used as a type argument
                __retVal_native = System.Runtime.InteropServices.Marshalling.ExceptionDefaultMarshaller<byte*>.ConvertToUnmanaged(__exception);
            }
            finally
            {
                // Cleanup - Perform required cleanup.
               ...
            }

            return __retVal_native;
        }
    }
}

Author:	jtschuster
Assignees:	jtschuster
Labels:	area-System.Runtime.InteropServices
Milestone:	-

[elinor-fung]

We created a helper for handling this when we were doing LibraryImportGenerator:

runtime/src/libraries/System.Runtime.InteropServices/gen/Microsoft.Interop.SourceGeneration/Marshalling/MarshallerHelpers.cs

Lines 84 to 93 in 07f5fca

	public static TypeSyntax GetCompatibleGenericTypeParameterSyntax(this TypeSyntax type)
	{
	TypeSyntax spanElementTypeSyntax = type;
	if (spanElementTypeSyntax is PointerTypeSyntax)
	{
	// Pointers cannot be passed to generics, so use IntPtr for this case.
	spanElementTypeSyntax = SystemIntPtrType;
	}
	return spanElementTypeSyntax;
	}

[jkoritzinsky]

It looks like I missed/forgot to use this for the COM exception marshalling logic. Should be pretty easy to fix.

[jtschuster]

fixed in #82582

[jtschuster]

Not quite fixed, the value returned by the generic methods needs to be cast as the variable type.

byte* __retVal_native;
catch (System.Exception __exception)
{
    // Cannot implicity convert nint to byte*
    __retVal_native = System.Runtime.InteropServices.Marshalling.ExceptionDefaultMarshaller<System.IntPtr>.ConvertToUnmanaged(__exception);
}