Leaking internal type parameter reference in GADT casting

Compiler version

3.2.0-RC1

Minimized code

trait Tag { val data: Int }

enum EQ[A, B]:
  case Refl[C]() extends EQ[C, C]

def foo[T, B <: Tag](ev: EQ[T, B], x: T) = ev match
  case EQ.Refl() =>
    val i: Int = x.data

Output (click arrow to expand)

```scala error when pickling type B(param)2 error when pickling tree B(param)2 error when pickling tree x.$asInstanceOf[B(param)2] error when pickling tree x.$asInstanceOf[B(param)2].data error when pickling tree val i: Int = x.$asInstanceOf[B(param)2].data error when pickling tree { val i: Int = x.$asInstanceOf[B(param)2].data () } error when pickling tree case EQ.Refl.unapply[T]():EQ.Refl[T] => val i: Int = x.$asInstanceOf[B(param)2].data () error when pickling tree ev match { case EQ.Refl.unapply[T]():EQ.Refl[T] => val i: Int = x.$asInstanceOf[B(param)2].data () } error when pickling tree def foo[T >: Nothing <: Any, B >: Nothing <: Tag](ev: EQ[T, B], x: T): Unit = ev match { case EQ.Refl.unapply[T]():EQ.Refl[T] => val i: Int = x.$asInstanceOf[B(param)2].data () } error when pickling tree () extends Object() { this: gadt-selection$package.type => private def writeReplace(): AnyRef = new scala.runtime.ModuleSerializationProxy(classOf[gadt-selection$package.type]) def foo[T >: Nothing <: Any, B >: Nothing <: Tag](ev: EQ[T, B], x: T): Unit = ev match { case EQ.Refl.unapply[T]():EQ.Refl[T] => val i: Int = x.$asInstanceOf[B(param)2].data () } } error when pickling tree @SourceFile("issues/gadt-selection.scala") final module class gadt-selection$package() extends Object() { this: gadt-selection$package.type => private def writeReplace(): AnyRef = new scala.runtime.ModuleSerializationProxy(classOf[gadt-selection$package.type]) def foo[T >: Nothing <: Any, B >: Nothing <: Tag](ev: EQ[T, B], x: T): Unit = ev match { case EQ.Refl.unapply[T]():EQ.Refl[T] => val i: Int = x.$asInstanceOf[B(param)2].data () } } error when pickling tree package { final lazy module val gadt-selection$package: gadt-selection$package = new gadt-selection$package() @SourceFile("issues/gadt-selection.scala") final module class gadt-selection$package() extends Object() { this: gadt-selection$package.type => private def writeReplace(): AnyRef = new scala.runtime.ModuleSerializationProxy(classOf[gadt-selection$package.type]) def foo[T >: Nothing <: Any, B >: Nothing <: Tag](ev: EQ[T, B], x: T): Unit = ev match { case EQ.Refl.unapply[T]():EQ.Refl[T] => val i: Int = x.$asInstanceOf[B(param)2].data () } } } exception occurred while compiling issues/gadt-selection.scala java.lang.AssertionError: assertion failed: orphan parameter reference: TypeParamRef(B(param)2) while compiling issues/gadt-selection.scala Exception in thread "main" java.lang.AssertionError: assertion failed: orphan parameter reference: TypeParamRef(B(param)2) at scala.runtime.Scala3RunTime$.assertFailed(Scala3RunTime.scala:8) at dotty.tools.dotc.core.tasty.TreePickler.pickleNewType(TreePickler.scala:291) at dotty.tools.dotc.core.tasty.TreePickler.pickleType(TreePickler.scala:160) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:609) at dotty.tools.dotc.core.tasty.TreePickler.pickleTpt(TreePickler.scala:318) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$4$$anonfun$1(TreePickler.scala:440) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10) at scala.collection.immutable.List.foreach(List.scala:333) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$4(TreePickler.scala:440) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$4(TreePickler.scala:441) at scala.Function0.apply$mcV$sp(Function0.scala:39) at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:441) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:415) at dotty.tools.dotc.core.tasty.TreePickler.pickleTreeUnlessEmpty(TreePickler.scala:321) at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$1(TreePickler.scala:338) at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$adapted$1(TreePickler.scala:340) at scala.Function0.apply$mcV$sp(Function0.scala:39) at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58) at dotty.tools.dotc.core.tasty.TreePickler.pickleDef(TreePickler.scala:340) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:550) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$9$$anonfun$1(TreePickler.scala:474) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10) at scala.collection.immutable.List.foreach(List.scala:333) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$9(TreePickler.scala:474) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$8(TreePickler.scala:474) at scala.Function0.apply$mcV$sp(Function0.scala:39) at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:474) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$13(TreePickler.scala:501) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$12(TreePickler.scala:501) at scala.Function0.apply$mcV$sp(Function0.scala:39) at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:501) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$12$$anonfun$1(TreePickler.scala:497) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10) at scala.collection.immutable.List.foreach(List.scala:333) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$12(TreePickler.scala:497) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$11(TreePickler.scala:498) at scala.Function0.apply$mcV$sp(Function0.scala:39) at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:498) at dotty.tools.dotc.core.tasty.TreePickler.pickleTreeUnlessEmpty(TreePickler.scala:321) at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$1(TreePickler.scala:338) at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$adapted$1(TreePickler.scala:340) at scala.Function0.apply$mcV$sp(Function0.scala:39) at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58) at dotty.tools.dotc.core.tasty.TreePickler.pickleDef(TreePickler.scala:340) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:565) at dotty.tools.dotc.core.tasty.TreePickler.pickleStats$$anonfun$2(TreePickler.scala:365) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10) at scala.collection.immutable.List.foreach(List.scala:333) at dotty.tools.dotc.core.tasty.TreePickler.pickleStats(TreePickler.scala:365) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$26(TreePickler.scala:591) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$24(TreePickler.scala:592) at scala.Function0.apply$mcV$sp(Function0.scala:39) at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:592) at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$1(TreePickler.scala:335) at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$adapted$1(TreePickler.scala:340) at scala.Function0.apply$mcV$sp(Function0.scala:39) at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58) at dotty.tools.dotc.core.tasty.TreePickler.pickleDef(TreePickler.scala:340) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:567) at dotty.tools.dotc.core.tasty.TreePickler.pickleStats$$anonfun$2(TreePickler.scala:365) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10) at scala.collection.immutable.List.foreach(List.scala:333) at dotty.tools.dotc.core.tasty.TreePickler.pickleStats(TreePickler.scala:365) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$29(TreePickler.scala:607) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$27(TreePickler.scala:607) at scala.Function0.apply$mcV$sp(Function0.scala:39) at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58) at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:607) at dotty.tools.dotc.core.tasty.TreePickler.pickle$$anonfun$1(TreePickler.scala:779) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10) at scala.collection.immutable.List.foreach(List.scala:333) at dotty.tools.dotc.core.tasty.TreePickler.pickle(TreePickler.scala:779) at dotty.tools.dotc.transform.Pickler.run$$anonfun$1$$anonfun$1(Pickler.scala:72) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10) at scala.collection.immutable.List.foreach(List.scala:333) at dotty.tools.dotc.transform.Pickler.run$$anonfun$1(Pickler.scala:110) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10) at scala.collection.immutable.List.foreach(List.scala:333) at dotty.tools.dotc.transform.Pickler.run(Pickler.scala:110) at dotty.tools.dotc.core.Phases$Phase.runOn$$anonfun$1(Phases.scala:311) at scala.collection.immutable.List.map(List.scala:246) at dotty.tools.dotc.core.Phases$Phase.runOn(Phases.scala:312) at dotty.tools.dotc.transform.Pickler.runOn(Pickler.scala:115) at dotty.tools.dotc.Run.runPhases$1$$anonfun$1(Run.scala:234) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10) at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1328) at dotty.tools.dotc.Run.runPhases$1(Run.scala:245) at dotty.tools.dotc.Run.compileUnits$$anonfun$1(Run.scala:253) at dotty.tools.dotc.Run.compileUnits$$anonfun$adapted$1(Run.scala:262) at dotty.tools.dotc.util.Stats$.maybeMonitored(Stats.scala:68) at dotty.tools.dotc.Run.compileUnits(Run.scala:262) at dotty.tools.dotc.Run.compileSources(Run.scala:186) at dotty.tools.dotc.Run.compile(Run.scala:170) at dotty.tools.dotc.Driver.doCompile(Driver.scala:35) at dotty.tools.dotc.Driver.process(Driver.scala:195) at dotty.tools.dotc.Driver.process(Driver.scala:163) at dotty.tools.dotc.Driver.process(Driver.scala:175) at dotty.tools.dotc.Driver.main(Driver.scala:205) at dotty.tools.dotc.Main.main(Main.scala) [error] Nonzero exit code returned from runner: 1 [error] (scala3-compiler / Compile / runMain) Nonzero exit code returned from runner: 1 [error] Total time: 1 s, completed Jun 27, 2022, 7:43:30 PM ```

As seen in the following typed tree, the internal type of GADT constraint solver B(param)2 is leaked into the GADT casting, crashing the pickler.

def foo[T >: Nothing <: Any, B >: Nothing <: Tag](ev: EQ[T, B], x: T): Unit = 
  ev match 
    {
      case EQ.Refl.unapply[T]():EQ.Refl[T] => 
        val i: Int = x.$asInstanceOf[B(param)2].data
        ()
    }

The cause of this behavior:

When typing the selection x.data we try to GADT-approximate x: T and check whether it matches the SelectionProto.
During this, we will call GADTConstraint.approximation, which will call ConstraintHandling.approximation here.
As seen here, if the type variable has been instantiated to another type parameter, the internal representation is returned directly and back in GADTConstraint we do not externalize it.
This leaks the internal type and breaks the pickler.

This is a bug discovered when I am trying to solve #14776. I am working on a PR fixing both issues.

scala / scala3