Runtime incorrectly interprets methodimpl entries for class overrides

[gafter]

The runtime incorrectly interprets methodimpl table entries for overrides of class methods. Since they are used for generating code for covariant returns, we will need the runtime to obey them in order to correctly execute code using covariant returns. However, the problem can be illustrated without covariant returns using a compiler from Roslyn's covariant return feature branch.

The following program produces the IL shown below it. Running the program produces the (incorrect) output shown afterwards. The first two lines in each group of output lines are wrong.

using System;
using System.Collections.Generic;
abstract class Base<T, U>
{
    public virtual void Method(out List<U> y, ref List<T> x) { y = null; Console.WriteLine("Base<T, U>.Method(out List<U> y, ref List<T> x)"); }
    public virtual void Method(ref List<T> x, out List<U> y) { y = null; Console.WriteLine("Base<T, U>.Method(ref List<T> x, out List<U> y)"); }
    public virtual void Method(ref List<U> x) { Console.WriteLine("Base<T, U>.Method(ref List<U> x)"); }
}
class Base2<A, B> : Base<A, B>
{
    public virtual void Method(out List<A> x)
    {
        x = null;
        Console.WriteLine("Base2<A, B>.Method(out List<A> x)");
    }
}
class Derived : Base2<int, int>
{
    public override void Method(ref List<int> a, out List<int> b) // No warning - the compiler produces a methodimpl record to disambiguate
    {
        b = null;
        Console.WriteLine("Derived.Method(ref List<int> a, out List<int> b)");
    }
    public override void Method(ref List<int> a) // No warning when ambiguous signatures are spread across multiple base types
    {
        Console.WriteLine("Derived.Method(ref List<int> a)");
    }
}
class Program
{
    static void Main()
    {
        List<int> t = null;
        Derived d = new Derived();
        d.Method(ref t, out t);
        d.Method(out t, ref t);
        d.Method(ref t);
        d.Method(out t);
        Console.WriteLine();
        Base2<int, int> b2 = d;
        b2.Method(ref t, out t);
        b2.Method(out t, ref t);
        b2.Method(ref t);
        b2.Method(out t);
        Console.WriteLine();
        Base<int, int> b1 = d;
        b1.Method(ref t, out t);
        b1.Method(out t, ref t);
        b1.Method(ref t);
    }
}

//  Microsoft (R) .NET Framework IL Disassembler.  Version 4.8.3928.0
//  Copyright (c) Microsoft Corporation.  All rights reserved.

// Metadata version: v4.0.30319
.assembly extern mscorlib
{
  .publickeytoken = (B7 7A 5C 56 19 34 E0 89 )                         // .z\V.4..
  .ver 4:0:0:0
}
.assembly Program
{
  .custom instance void [mscorlib]System.Runtime.CompilerServices.CompilationRelaxationsAttribute::.ctor(int32) = ( 01 00 08 00 00 00 00 00 ) 
  .custom instance void [mscorlib]System.Runtime.CompilerServices.RuntimeCompatibilityAttribute::.ctor() = ( 01 00 01 00 54 02 16 57 72 61 70 4E 6F 6E 45 78   // ....T..WrapNonEx
                                                                                                             63 65 70 74 69 6F 6E 54 68 72 6F 77 73 01 )       // ceptionThrows.

  // --- The following custom attribute is added automatically, do not uncomment -------
  //  .custom instance void [mscorlib]System.Diagnostics.DebuggableAttribute::.ctor(valuetype [mscorlib]System.Diagnostics.DebuggableAttribute/DebuggingModes) = ( 01 00 07 01 00 00 00 00 ) 

  .hash algorithm 0x00008004
  .ver 0:0:0:0
}
.module Program.exe
// MVID: {C6C266E9-7949-4242-B429-F9011402D996}
.imagebase 0x00400000
.file alignment 0x00000200
.stackreserve 0x00100000
.subsystem 0x0003       // WINDOWS_CUI
.corflags 0x00000001    //  ILONLY
// Image base: 0x04F20000

// =============== CLASS MEMBERS DECLARATION ===================

.class private abstract auto ansi beforefieldinit Base`2<T,U>
       extends [mscorlib]System.Object
{
  .method public hidebysig newslot virtual 
          instance void  Method([out] class [mscorlib]System.Collections.Generic.List`1<!U>& y,
                                class [mscorlib]System.Collections.Generic.List`1<!T>& x) cil managed
  {
    // Code size       16 (0x10)
    .maxstack  8
    IL_0000:  nop
    IL_0001:  ldarg.1
    IL_0002:  ldnull
    IL_0003:  stind.ref
    IL_0004:  ldstr      "Base<T, U>.Method(out List<U> y, ref List<T> x)"
    IL_0009:  call       void [mscorlib]System.Console::WriteLine(string)
    IL_000e:  nop
    IL_000f:  ret
  } // end of method Base`2::Method

  .method public hidebysig newslot virtual 
          instance void  Method(class [mscorlib]System.Collections.Generic.List`1<!T>& x,
                                [out] class [mscorlib]System.Collections.Generic.List`1<!U>& y) cil managed
  {
    // Code size       16 (0x10)
    .maxstack  8
    IL_0000:  nop
    IL_0001:  ldarg.2
    IL_0002:  ldnull
    IL_0003:  stind.ref
    IL_0004:  ldstr      "Base<T, U>.Method(ref List<T> x, out List<U> y)"
    IL_0009:  call       void [mscorlib]System.Console::WriteLine(string)
    IL_000e:  nop
    IL_000f:  ret
  } // end of method Base`2::Method

  .method public hidebysig newslot virtual 
          instance void  Method(class [mscorlib]System.Collections.Generic.List`1<!U>& x) cil managed
  {
    // Code size       13 (0xd)
    .maxstack  8
    IL_0000:  nop
    IL_0001:  ldstr      "Base<T, U>.Method(ref List<U> x)"
    IL_0006:  call       void [mscorlib]System.Console::WriteLine(string)
    IL_000b:  nop
    IL_000c:  ret
  } // end of method Base`2::Method

  .method family hidebysig specialname rtspecialname 
          instance void  .ctor() cil managed
  {
    // Code size       8 (0x8)
    .maxstack  8
    IL_0000:  ldarg.0
    IL_0001:  call       instance void [mscorlib]System.Object::.ctor()
    IL_0006:  nop
    IL_0007:  ret
  } // end of method Base`2::.ctor

} // end of class Base`2

.class private auto ansi beforefieldinit Base2`2<A,B>
       extends class Base`2<!A,!B>
{
  .method public hidebysig newslot virtual 
          instance void  Method([out] class [mscorlib]System.Collections.Generic.List`1<!A>& x) cil managed
  {
    // Code size       16 (0x10)
    .maxstack  8
    IL_0000:  nop
    IL_0001:  ldarg.1
    IL_0002:  ldnull
    IL_0003:  stind.ref
    IL_0004:  ldstr      "Base2<A, B>.Method(out List<A> x)"
    IL_0009:  call       void [mscorlib]System.Console::WriteLine(string)
    IL_000e:  nop
    IL_000f:  ret
  } // end of method Base2`2::Method

  .method public hidebysig specialname rtspecialname 
          instance void  .ctor() cil managed
  {
    // Code size       8 (0x8)
    .maxstack  8
    IL_0000:  ldarg.0
    IL_0001:  call       instance void class Base`2<!A,!B>::.ctor()
    IL_0006:  nop
    IL_0007:  ret
  } // end of method Base2`2::.ctor

} // end of class Base2`2

.class private auto ansi beforefieldinit Derived
       extends class Base2`2<int32,int32>
{
  .method public hidebysig newslot virtual 
          instance void  Method(class [mscorlib]System.Collections.Generic.List`1<int32>& a,
                                [out] class [mscorlib]System.Collections.Generic.List`1<int32>& b) cil managed
  {
    .override  method instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!0>&,
                                                                      class [mscorlib]System.Collections.Generic.List`1<!1>&)
    // Code size       16 (0x10)
    .maxstack  8
    IL_0000:  nop
    IL_0001:  ldarg.2
    IL_0002:  ldnull
    IL_0003:  stind.ref
    IL_0004:  ldstr      "Derived.Method(ref List<int> a, out List<int> b)"
    IL_0009:  call       void [mscorlib]System.Console::WriteLine(string)
    IL_000e:  nop
    IL_000f:  ret
  } // end of method Derived::Method

  .method public hidebysig newslot virtual 
          instance void  Method(class [mscorlib]System.Collections.Generic.List`1<int32>& a) cil managed
  {
    .override  method instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!1>&)
    // Code size       13 (0xd)
    .maxstack  8
    IL_0000:  nop
    IL_0001:  ldstr      "Derived.Method(ref List<int> a)"
    IL_0006:  call       void [mscorlib]System.Console::WriteLine(string)
    IL_000b:  nop
    IL_000c:  ret
  } // end of method Derived::Method

  .method public hidebysig specialname rtspecialname 
          instance void  .ctor() cil managed
  {
    // Code size       8 (0x8)
    .maxstack  8
    IL_0000:  ldarg.0
    IL_0001:  call       instance void class Base2`2<int32,int32>::.ctor()
    IL_0006:  nop
    IL_0007:  ret
  } // end of method Derived::.ctor

} // end of class Derived

.class private auto ansi beforefieldinit Program
       extends [mscorlib]System.Object
{
  .method private hidebysig static void  Main() cil managed
  {
    .entrypoint
    // Code size       137 (0x89)
    .maxstack  3
    .locals init ([0] class [mscorlib]System.Collections.Generic.List`1<int32> t,
             [1] class Derived d,
             [2] class Base2`2<int32,int32> b2,
             [3] class Base`2<int32,int32> b1)
    IL_0000:  nop
    IL_0001:  ldnull
    IL_0002:  stloc.0
    IL_0003:  newobj     instance void Derived::.ctor()
    IL_0008:  stloc.1
    IL_0009:  ldloc.1
    IL_000a:  ldloca.s   t
    IL_000c:  ldloca.s   t
    IL_000e:  callvirt   instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!0>&,
                                                                         class [mscorlib]System.Collections.Generic.List`1<!1>&)
    IL_0013:  nop
    IL_0014:  ldloc.1
    IL_0015:  ldloca.s   t
    IL_0017:  ldloca.s   t
    IL_0019:  callvirt   instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!1>&,
                                                                         class [mscorlib]System.Collections.Generic.List`1<!0>&)
    IL_001e:  nop
    IL_001f:  ldloc.1
    IL_0020:  ldloca.s   t
    IL_0022:  callvirt   instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!1>&)
    IL_0027:  nop
    IL_0028:  ldloc.1
    IL_0029:  ldloca.s   t
    IL_002b:  callvirt   instance void class Base2`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!0>&)
    IL_0030:  nop
    IL_0031:  call       void [mscorlib]System.Console::WriteLine()
    IL_0036:  nop
    IL_0037:  ldloc.1
    IL_0038:  stloc.2
    IL_0039:  ldloc.2
    IL_003a:  ldloca.s   t
    IL_003c:  ldloca.s   t
    IL_003e:  callvirt   instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!0>&,
                                                                         class [mscorlib]System.Collections.Generic.List`1<!1>&)
    IL_0043:  nop
    IL_0044:  ldloc.2
    IL_0045:  ldloca.s   t
    IL_0047:  ldloca.s   t
    IL_0049:  callvirt   instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!1>&,
                                                                         class [mscorlib]System.Collections.Generic.List`1<!0>&)
    IL_004e:  nop
    IL_004f:  ldloc.2
    IL_0050:  ldloca.s   t
    IL_0052:  callvirt   instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!1>&)
    IL_0057:  nop
    IL_0058:  ldloc.2
    IL_0059:  ldloca.s   t
    IL_005b:  callvirt   instance void class Base2`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!0>&)
    IL_0060:  nop
    IL_0061:  call       void [mscorlib]System.Console::WriteLine()
    IL_0066:  nop
    IL_0067:  ldloc.1
    IL_0068:  stloc.3
    IL_0069:  ldloc.3
    IL_006a:  ldloca.s   t
    IL_006c:  ldloca.s   t
    IL_006e:  callvirt   instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!0>&,
                                                                         class [mscorlib]System.Collections.Generic.List`1<!1>&)
    IL_0073:  nop
    IL_0074:  ldloc.3
    IL_0075:  ldloca.s   t
    IL_0077:  ldloca.s   t
    IL_0079:  callvirt   instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!1>&,
                                                                         class [mscorlib]System.Collections.Generic.List`1<!0>&)
    IL_007e:  nop
    IL_007f:  ldloc.3
    IL_0080:  ldloca.s   t
    IL_0082:  callvirt   instance void class Base`2<int32,int32>::Method(class [mscorlib]System.Collections.Generic.List`1<!1>&)
    IL_0087:  nop
    IL_0088:  ret
  } // end of method Program::Main

  .method public hidebysig specialname rtspecialname 
          instance void  .ctor() cil managed
  {
    // Code size       8 (0x8)
    .maxstack  8
    IL_0000:  ldarg.0
    IL_0001:  call       instance void [mscorlib]System.Object::.ctor()
    IL_0006:  nop
    IL_0007:  ret
  } // end of method Program::.ctor

} // end of class Program

// =============================================================

// *********** DISASSEMBLY COMPLETE ***********************
// WARNING: Created Win32 resource file Program.res

Output produced by .net 3.1:

Base<T, U>.Method(ref List<T> x, out List<U> y)
Derived.Method(ref List<int> a, out List<int> b)
Derived.Method(ref List<int> a)
Base2<A, B>.Method(out List<A> x)

Base<T, U>.Method(ref List<T> x, out List<U> y)
Derived.Method(ref List<int> a, out List<int> b)
Derived.Method(ref List<int> a)
Base2<A, B>.Method(out List<A> x)

Base<T, U>.Method(ref List<T> x, out List<U> y)
Derived.Method(ref List<int> a, out List<int> b)
Derived.Method(ref List<int> a)

The correct output should be:

Derived.Method(ref List<int> a, out List<int> b)
Base<T, U>.Method(out List<U> y, ref List<T> x)
Derived.Method(ref List<int> a)
Base2<A, B>.Method(out List<A> x)

Derived.Method(ref List<int> a, out List<int> b)
Base<T, U>.Method(out List<U> y, ref List<T> x)
Derived.Method(ref List<int> a)
Base2<A, B>.Method(out List<A> x)

Derived.Method(ref List<int> a, out List<int> b)
Base<T, U>.Method(out List<U> y, ref List<T> x)
Derived.Method(ref List<int> a)

[Dotnet-GitSync-Bot]

I couldn't figure out the best area label to add to this issue. Please help me learn by adding exactly one area label.

[gafter]

@lambdageek @jaredpar @janvorli This runtime bug will affect covariant returns.

[janvorli]

cc: @davidwrighton

[gafter]

It is possible that methodimpl entries for class overrides are not actually supported before netcore 5. If that is the case, this is not a bug (but would be a bug if these symptoms were exhibited in netcore 5). However, note that existing compilers do produce methodimpl entries for overrides of the object destructor.

[gafter]

From ECMA-335 4th edition (June 2006):

9.10 Explicit method overrides

A type, be it generic or non-generic, can implement particular virtual methods (whether the method was introduced in an interface or base class) using an explicit override. (See §10.3.2 and §15.1.4.)

10.3.2 The .override directive

The .override directive specifies that a virtual method shall be implemented (overridden), in this type, by a virtual method with a different name, but with the same signature. This directive can be used to provide an implementation for a virtual method inherited from a base class, or a virtual method specified in an interface implemented by this type. The .override directive specifies a Method Implementation (MethodImpl) in the metadata (§15.1.4).

15.1.4 Method implementations

A MethodImpl, or method implementation, supplies the executable body for an existing virtual method. It associates a Method (representing the body) with a MethodDecl or Method (representing the virtual method). A MethodImpl is used to provide an implementation for an inherited virtual method or a virtual method from an interface when the default mechanism (matching by name and signature) would not provide the correct result. See §22.27.

22.27 MethodImpl : 0x19

MethodImpl tables let a compiler override the default inheritance rules provided by the CLI. Their original use was to allow a class C, that inherited method M from both interfaces I and J, to provide implementations for both methods (rather than have only one slot for M in its vtable). However, MethodImpls can be used for other reasons too, limited only by the compiler writer’s ingenuity within the constraints defined in the Validation rules below.

So this is indeed a bug.

[janvorli]

I am no expert on the type system intricacies, but I wonder, looking at the override:

.override  method instance void class Base`2<int32,int32>::Method(
class [mscorlib]System.Collections.Generic.List`1<!0>&, 
class [mscorlib]System.Collections.Generic.List`1<!1>&)

There is no [Out] at any of these parameters in the IL code. So how should it choose whether to override the (ref, out) or (out, ref) flavor?

[gafter]

The first parameter is of type

List`1<!0>&

and the second of type

List`1<!1>&

indicating a list of the first type parameter or the second type parameter in the original type declaration where the method was introduced. These are different type tokens. They would have to be reversed in the parameter list for the other method.

[janvorli]

I guess I am missing something. The !0 and !1 are the generic arguments of the Base type (!0 is int32, !1 is int32) in the override above, it is not clear to me how would they be related to whether the method arguments that become List, List<32> were out or ref.

[gafter]

It has nothing to do with whether they are out or ref. The two methods are distinguished by their signature (type) where originally declared, not as substituted. The method tokens do not substitute generics in the arguments or return part of the signature (otherwise you would see int32 instead of !0). You see the same thing in the call instructions.

[janvorli]

It seems we are each talking about something different. I guess my question was not clear. Let me try to rephrase it. The Base<T, U> class has the following two methods:

 .method public hidebysig newslot virtual instance void  Method(
[out] class [mscorlib]System.Collections.Generic.List`1<!U>& y,
class [mscorlib]System.Collections.Generic.List`1<!T>& x) cil managed

.method public hidebysig newslot virtual instance void  Method(
class [mscorlib]System.Collections.Generic.List`1<!T>& x,
[out] class [mscorlib]System.Collections.Generic.List`1<!U>& y) cil managed

Then the Derived class is derived from the instantiation with both T and U being int32

And then we have this override:

.override  method instance void class Base`2<int32,int32>::Method(
class [mscorlib]System.Collections.Generic.List`1<!0>&, 
class [mscorlib]System.Collections.Generic.List`1<!1>&)

In the C#, it is clear which one we are overriding, as the overriding method signature has ref at the first parameter and out at the second.

But in the IL, the [Out] is not on either of those. So which one of those should it override? If you omit the [Out], those two are the same.

[gafter]

The point is that

Method(
class [mscorlib]System.Collections.Generic.List`1<!0>&, 
class [mscorlib]System.Collections.Generic.List`1<!1>&)

is not identical to

Method(
class [mscorlib]System.Collections.Generic.List`1<!1>&, 
class [mscorlib]System.Collections.Generic.List`1<!0>&)

so the runtime can distinguish them. The type parameter references are different. The former clearly only matches the second method declared in source, and the latter clearly only matches the first method declared in source.

[janvorli]

Ah, now I got it, thanks for bearing with me!

[lambdageek]

mono/mono produces the correct output

$ mono --version
Mono JIT compiler version 6.12.0.74 (2020-02/e9d3af508e4 Fri May 15 10:10:32 EDT 2020)
...
$ ilasm foo.il
Assembling 'foo.il' , no listing file, to exe --> 'foo.exe'

Operation completed successfully
$ mono foo.exe
Derived.Method(ref List<int> a, out List<int> b)
Base<T, U>.Method(out List<U> y, ref List<T> x)
Derived.Method(ref List<int> a)
Base2<A, B>.Method(out List<A> x)

Derived.Method(ref List<int> a, out List<int> b)
Base<T, U>.Method(out List<U> y, ref List<T> x)
Derived.Method(ref List<int> a)
Base2<A, B>.Method(out List<A> x)

Derived.Method(ref List<int> a, out List<int> b)
Base<T, U>.Method(out List<U> y, ref List<T> x)
Derived.Method(ref List<int> a)

I will check whether my WIP PR #37516 does the right thing too.

---

Update based on the comment below also tried the same thing with the order of Method definitions in Base reordered. Same (correct) output.

[gafter]

Please also check whether mono produces the correct result when the order of the methods in Base are changed.

[davidwrighton]

Let me look into this. I believe the rules that Neal is using for handling of MethodImpl's aren't quite what is specified in the latest version of the Ecma 335 spec. Those rules were determined to be incorrect some years ago and fixed in more recent editions of Ecma 335. See II.10.3.4 in https://www.ecma-international.org/publications/files/ECMA-ST/ECMA-335.pdf and the other MethodImpl spec contents. I will take a deeper look here and attempt to determine if the runtime is behaving incorrectly here, or if your reading of the spec in this area is incorrect.

[davidwrighton]

Ah, I misunderstood the issue here. This isn't about the multi-level override problem which is deeply problematic with MethodImpls, and for which we've added a new attribute that will need to be used by these covariant overrides, but instead, this is a signature comparison problem.

I agree that this looks like a runtime bug, although one of frightful long-standing. The issue here is that the runtime is applying the rules for substitutions in a way which is incompatible with specifying the exact override in this case. I believe this bug is in the substitutions setup for the comparisons. (Substitutions are used and non-exact signatures are considered to match in order to handle the scenario where a base type's virtual method is moved to its base type.)

[davidwrighton]

Fixed with PR #38310