Two fixes to constraint solving (#16353)

## 1. Fix replace operation

In OrderingConstraint#replace we moved the actual replacement of a
parameter with a type from the start of replace to its end, since that
was more convenient for dependency adjustments. It turns out that doing
so causes infinite recursion in instantiations in some cases,
specifically if a parameter contains itself as an indirect lower bound
that goes through an alias. Here is a situation that arises in
i16311.scala:
```scala
  type WithTag[T, U] = T & Tagged[U]

  T1 >: WithTag[T2, Int]
  T2 >: T1 & Tagged[Int]
```
The correct instantiation for T1 and T2 is Nothing. But we ran into a
cycle instead.

The fix is to move the parameter replacement back to the start of
`replace`, and to account for that in the dependency adjustment logic.

Fixes #16311 (with failing Ycheck)

## 2. See through aliases before decomposing And/Or in isSubType

There seem to be two missing cases in TypeComparer where we
have a TypeParamRef on one side and an And/Or type under an alias on the
other.
Examples:

    type AND = A & B
    type OR  = A | B
    p <:< AND
    OR <:< p

In this case we missed the decomposition into smaller types that would
happen
otherwise. This broke i16311.scala in Ycheck and broke i15365.scala with
an
infinite recursion in avoidance.

I verified that having an AndType as super or subtype of an abstract
type works
as expected. So if in the example above

    type AND >: A & B

or

    type AND <: A & B

it worked before. It was just aliases that were the problem (I assume
it's the same for OrTypes
as lower bounds).

This fixes #16311 completely and also
Fixes #15365

Author: smarter

compiler/src/dotty/tools/dotc/core/OrderingConstraint.scala

@@ -297,27 +297,36 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
    */
   private trait ConstraintAwareTraversal[T] extends TypeAccumulator[T]:
 
+    /** Does `param` have bounds in the current constraint? */
+    protected def hasBounds(param: TypeParamRef): Boolean = entry(param).isInstanceOf[TypeBounds]
+
     override def tyconTypeParams(tp: AppliedType)(using Context): List[ParamInfo] =
       def tparams(tycon: Type): List[ParamInfo] = tycon match
         case tycon: TypeVar if !tycon.inst.exists => tparams(tycon.origin)
-        case tycon: TypeParamRef =>
-          entry(tycon) match
-            case _: TypeBounds => tp.tyconTypeParams
-            case tycon1 if tycon1.typeParams.nonEmpty => tycon1.typeParams
-            case _ => tp.tyconTypeParams
+        case tycon: TypeParamRef if !hasBounds(tycon) =>
+          val entryParams = entry(tycon).typeParams
+          if entryParams.nonEmpty then entryParams
+          else tp.tyconTypeParams
         case _ => tp.tyconTypeParams
       tparams(tp.tycon)
 
     override def applyToPrefix(x: T, tp: NamedType): T =
       this(x, tp.prefix)
   end ConstraintAwareTraversal
 
-  private class Adjuster(srcParam: TypeParamRef)(using Context)
+  /** A type traverser that adjust dependencies originating from a given type
+   *  @param ignoreBinding  if not null, a parameter that is assumed to be still uninstantiated.
+   *                        This is necessary to handle replacements.
+   */
+  private class Adjuster(srcParam: TypeParamRef, ignoreBinding: TypeParamRef | Null)(using Context)
   extends TypeTraverser, ConstraintAwareTraversal[Unit]:
 
     var add: Boolean = compiletime.uninitialized
     val seen = util.HashSet[LazyRef]()
 
+    override protected def hasBounds(param: TypeParamRef) =
+      (param eq ignoreBinding) || super.hasBounds(param)
+
     def update(deps: ReverseDeps, referenced: TypeParamRef): ReverseDeps =
       val prev = deps.at(referenced)
       val newSet = if add then prev + srcParam else prev - srcParam
@@ -326,12 +335,11 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
 
     def traverse(t: Type) = t match
       case param: TypeParamRef =>
-        entry(param) match
-          case _: TypeBounds =>
-            if variance >= 0 then coDeps = update(coDeps, param)
-            if variance <= 0 then contraDeps = update(contraDeps, param)
-          case tp =>
-            traverse(tp)
+        if hasBounds(param) then
+          if variance >= 0 then coDeps = update(coDeps, param)
+          if variance <= 0 then contraDeps = update(contraDeps, param)
+        else
+          traverse(entry(param))
       case tp: LazyRef =>
         if !seen.contains(tp) then
           seen += tp
@@ -342,8 +350,8 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
   /** Adjust dependencies to account for the delta of previous entry `prevEntry`
    *  and the new bound `entry` for the type parameter `srcParam`.
    */
-  def adjustDeps(entry: Type | Null, prevEntry: Type | Null, srcParam: TypeParamRef)(using Context): this.type =
-    val adjuster = new Adjuster(srcParam)
+  def adjustDeps(entry: Type | Null, prevEntry: Type | Null, srcParam: TypeParamRef, ignoreBinding: TypeParamRef | Null = null)(using Context): this.type =
+    val adjuster = new Adjuster(srcParam, ignoreBinding)
 
     /** Adjust reverse dependencies of all type parameters referenced by `bound`
      *  @param  isLower `bound` is a lower bound
@@ -676,7 +684,14 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
         override def apply(t: Type): Type =
           if (t eq replacedTypeVar) && t.exists then to else mapOver(t)
 
-      var current = this
+      val coDepsOfParam = coDeps.at(param)
+      val contraDepsOfParam = contraDeps.at(param)
+
+      var current = updateEntry(this, param, replacement)
+        // Need to update param early to avoid infinite recursion on instantiation.
+        // See i16311.scala for a test case. On the other hand, for the purposes of
+        // dependency adjustment, we need to pretend that `param` is still unbound.
+        // We achieve that by passing a `ignoreBinding = param` to `adjustDeps` below.
 
       def removeParamFrom(ps: List[TypeParamRef]) =
         ps.filterConserve(param ne _)
@@ -710,22 +725,15 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
               newDepEntry = mapReplacedTypeVarTo(replacement)(newDepEntry)
           case _ =>
         if oldDepEntry ne newDepEntry then
-          if current eq this then
-            // We can end up here if oldEntry eq newEntry, so posssibly no new constraint
-            // was created, but oldDepEntry ne newDepEntry. In that case we must make
-            // sure we have a new constraint before updating dependencies.
-            current = newConstraint()
-          current.adjustDeps(newDepEntry, oldDepEntry, other)
+          current.adjustDeps(newDepEntry, oldDepEntry, other, ignoreBinding = param)
       end replaceParamIn
 
       if optimizeReplace then
-        val co = current.coDeps.at(param)
-        val contra = current.contraDeps.at(param)
         current.foreachParam { (p, i) =>
           val other = p.paramRefs(i)
           entry(other) match
             case _: TypeBounds =>
-              if co.contains(other) || contra.contains(other) then
+              if coDepsOfParam.contains(other) || contraDepsOfParam.contains(other) then
                 replaceParamIn(other)
             case _ => replaceParamIn(other)
         }
@@ -734,10 +742,7 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
           val other = p.paramRefs(i)
           if other != param then replaceParamIn(other)
         }
-
-      current =
-        if isRemovable(param.binder) then current.remove(param.binder)
-        else updateEntry(current, param, replacement)
+      if isRemovable(param.binder) then current = current.remove(param.binder)
       current.dropDeps(param)
       current.checkWellFormed()
   end replace

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -418,16 +418,16 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
           true
         }
         def compareTypeParamRef =
-          assumedTrue(tp1) ||
-          tp2.match {
-            case tp2: TypeParamRef => constraint.isLess(tp1, tp2)
-            case _ => false
-          } ||
-          isSubTypeWhenFrozen(bounds(tp1).hi.boxed, tp2) || {
-            if (canConstrain(tp1) && !approx.high)
-              addConstraint(tp1, tp2, fromBelow = false) && flagNothingBound
-            else thirdTry
-          }
+          assumedTrue(tp1)
+          || tp2.dealias.match
+              case tp2a: TypeParamRef => constraint.isLess(tp1, tp2a)
+              case tp2a: AndType => recur(tp1, tp2a)
+              case _ => false
+          || isSubTypeWhenFrozen(bounds(tp1).hi.boxed, tp2)
+          || (if canConstrain(tp1) && !approx.high then
+                addConstraint(tp1, tp2, fromBelow = false) && flagNothingBound
+              else thirdTry)
+
         compareTypeParamRef
       case tp1: ThisType =>
         val cls1 = tp1.cls
@@ -585,7 +585,8 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
     }
 
     def compareTypeParamRef(tp2: TypeParamRef): Boolean =
-      assumedTrue(tp2) || {
+      assumedTrue(tp2)
+      || {
         val alwaysTrue =
           // The following condition is carefully formulated to catch all cases
           // where the subtype relation is true without needing to add a constraint
@@ -596,11 +597,13 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
           // widening in `fourthTry` before adding to the constraint.
           if (frozenConstraint) recur(tp1, bounds(tp2).lo.boxed)
           else isSubTypeWhenFrozen(tp1, tp2)
-        alwaysTrue || {
-          if (canConstrain(tp2) && !approx.low)
-            addConstraint(tp2, tp1.widenExpr, fromBelow = true)
-          else fourthTry
-        }
+        alwaysTrue
+        || tp1.dealias.match
+            case tp1a: OrType => recur(tp1a, tp2)
+            case _ => false
+        || (if canConstrain(tp2) && !approx.low then
+              addConstraint(tp2, tp1.widenExpr, fromBelow = true)
+            else fourthTry)
       }
 
     def thirdTry: Boolean = tp2 match {

tests/pending/pos/i15365.scala renamed to tests/pos/i15365.scala

@@ -0,0 +1,19 @@
+object test:
+
+  trait Tagged[U]
+  type WithTag[+T, U] = T & Tagged[U]
+
+  trait FromInput[Val]
+  implicit def coercedScalaInput[T]: FromInput[WithTag[T, Int]] = ???
+  implicit def optionInput[T](implicit ev: FromInput[T]): FromInput[Option[T]] = ???
+
+  trait WithoutInputTypeTags[T]
+  implicit def coercedOptArgTpe[T]: WithoutInputTypeTags[Option[T & Tagged[Int]]] = ???
+
+  trait InputType[+T]
+  class OptionInputType[T](ofType: InputType[T]) extends InputType[Option[T]]
+
+  type Argument[T]
+  def argument[T](argumentType: InputType[T])(implicit fromInput: FromInput[T], res: WithoutInputTypeTags[T]): Argument[Option[T]] = ???
+
+  def test = argument(OptionInputType(??? : InputType[Boolean & Tagged[Int]]))

tests/pending/pos/i16311.scala renamed to tests/pos/i16311.scala

@@ -0,0 +1,17 @@
+trait Tagged[U]
+type WithTag[+T, U] = T & Tagged[U]
+
+trait FromInput[Val]
+implicit def coercedScalaInput[T]: FromInput[T & Tagged[Int]] = ???
+implicit def optionInput[T](implicit ev: FromInput[T]): FromInput[Option[T]] = ???
+
+trait WithoutInputTypeTags[T]
+implicit def coercedOptArgTpe[T]: WithoutInputTypeTags[Option[T & Tagged[Int]]] = ???
+
+trait InputType[+T]
+class OptionInputType[T](ofType: InputType[T]) extends InputType[Option[T]]
+
+type Argument[T]
+def argument[T](argumentType: InputType[T])(implicit fromInput: FromInput[T], res: WithoutInputTypeTags[T]): Argument[Option[T]] = ???
+
+def test = argument(OptionInputType(??? : InputType[Boolean & Tagged[Int]]))

tests/pos/i16311a.scala

@@ -0,0 +1,18 @@
+class C:
+  trait Tagged[U]
+  type WithTag[+T, U] >: T & Tagged[U]
+
+  trait FromInput[Val]
+  implicit def coercedScalaInput[T]: FromInput[WithTag[T, Int]] = ???
+  implicit def optionInput[T](implicit ev: FromInput[T]): FromInput[Option[T]] = ???
+
+  trait WithoutInputTypeTags[T]
+  implicit def coercedOptArgTpe[T]: WithoutInputTypeTags[Option[WithTag[T, Int]]] = ???
+
+  trait InputType[+T]
+  class OptionInputType[T](ofType: InputType[T]) extends InputType[Option[T]]
+
+  type Argument[T]
+  def argument[T](argumentType: InputType[T])(implicit fromInput: FromInput[T], res: WithoutInputTypeTags[T]): Argument[Option[T]] = ???
+
+  def test = argument(OptionInputType(??? : InputType[WithTag[Boolean, Int]]))

tests/pos/i16311b.scala

@@ -0,0 +1,18 @@
+class C:
+  trait Tagged[U]
+  type WithTag[+T, U] <: T & Tagged[U]
+
+  trait FromInput[Val]
+  implicit def coercedScalaInput[T]: FromInput[WithTag[T, Int]] = ???
+  implicit def optionInput[T](implicit ev: FromInput[T]): FromInput[Option[T]] = ???
+
+  trait WithoutInputTypeTags[T]
+  implicit def coercedOptArgTpe[T]: WithoutInputTypeTags[Option[WithTag[T, Int]]] = ???
+
+  trait InputType[+T]
+  class OptionInputType[T](ofType: InputType[T]) extends InputType[Option[T]]
+
+  type Argument[T]
+  def argument[T](argumentType: InputType[T])(implicit fromInput: FromInput[T], res: WithoutInputTypeTags[T]): Argument[Option[T]] = ???
+
+  def test = argument(OptionInputType(??? : InputType[WithTag[Boolean, Int]]))