[sbt-dotty] use sbt loader as parent of scala instance loader

project/Build.scala

@@ -337,7 +337,8 @@ object Build {
         scalaLibrary,
         dottyLibrary,
         dottyCompiler,
-        allJars
+        allJars,
+        appConfiguration.value
       )
     },
     // sbt-dotty defines `scalaInstance in doc` so we need to override it manually

sbt-dotty/src/dotty/tools/sbtplugin/DottyPlugin.scala

@@ -8,11 +8,16 @@ import sbt.librarymanagement.{
   VersionNumber
 }
 import sbt.internal.inc.ScalaInstance
+import sbt.internal.inc.classpath.ClassLoaderCache
 import xsbti.compile._
+import xsbti.AppConfiguration
 import java.net.URLClassLoader
 import java.util.Optional
+import java.util.{Enumeration, Collections}
+import java.net.URL
 import scala.util.Properties.isJavaAtLeast
 
+
 object DottyPlugin extends AutoPlugin {
   object autoImport {
     val isDotty = settingKey[Boolean]("Is this project compiled with Dotty?")
@@ -524,15 +529,34 @@ object DottyPlugin extends AutoPlugin {
       scalaLibraryJar,
       dottyLibraryJar,
       compilerJar,
-      allJars
+      allJars,
+      appConfiguration.value
     )
   }
 
   // Adapted from private mkScalaInstance in sbt
   def makeScalaInstance(
-    state: State, dottyVersion: String, scalaLibrary: File, dottyLibrary: File, compiler: File, all: Seq[File]
+    state: State, dottyVersion: String, scalaLibrary: File, dottyLibrary: File, compiler: File, all: Seq[File], appConfiguration: AppConfiguration
   ): ScalaInstance = {
-    val libraryLoader = state.classLoaderCache(List(dottyLibrary, scalaLibrary))
+    /**
+      * The compiler bridge must load the xsbti classes from the sbt
+      * classloader, and similarly the Scala repl must load the sbt provided
+      * jline terminal. To do so we add the `appConfiguration` loader in
+      * the parent hierarchy of the scala 3 instance loader.
+      *
+      * The [[TopClassLoader]] ensures that the xsbti and jline classes
+      * only are loaded from the sbt loader. That is necessary because
+      * the sbt class loader contains the Scala 2.12 library and compiler
+      * bridge.
+      */
+    val topLoader = new TopClassLoader(appConfiguration.provider.loader)
+
+    val libraryJars = Array(dottyLibrary, scalaLibrary)
+    val libraryLoader = state.classLoaderCache.cachedCustomClassloader(
+      libraryJars.toList,
+      () => new URLClassLoader(libraryJars.map(_.toURI.toURL), topLoader)
+    )
+
     class DottyLoader
         extends URLClassLoader(all.map(_.toURI.toURL).toArray, libraryLoader)
     val fullLoader = state.classLoaderCache.cachedCustomClassloader(
@@ -543,10 +567,53 @@ object DottyPlugin extends AutoPlugin {
       dottyVersion,
       fullLoader,
       libraryLoader,
-      Array(dottyLibrary, scalaLibrary),
+      libraryJars,
       compiler,
       all.toArray,
       None)
+  }
+}
 
+/**
+ * The parent classloader of the Scala compiler.
+ *
+ * A TopClassLoader is constructed from the sbt classloader.
+ *
+ * To understand why a custom parent classloader is needed for the compiler,
+ * let us describe some alternatives that wouldn't work.
+ *
+ * - `new URLClassLoader(urls)`:
+ *   The compiler contains sbt phases that callback to sbt using the `xsbti.*`
+ *   interfaces. If `urls` does not contain the sbt interfaces we'll get a
+ *   `ClassNotFoundException` in the compiler when we try to use them, if
+ *   `urls` does contain the interfaces we'll get a `ClassCastException` or a
+ *   `LinkageError` because if the same class is loaded by two different
+ *   classloaders, they are considered distinct by the JVM.
+ *
+ * - `new URLClassLoader(urls, sbtLoader)`:
+ *    Because of the JVM delegation model, this means that we will only load
+ *    a class from `urls` if it's not present in the parent `sbtLoader`, but
+ *    sbt uses its own version of the scala compiler and scala library which
+ *    is not the one we need to run the compiler.
+ *
+ * Our solution is to implement an URLClassLoader whose parent is
+ * `new TopClassLoader(sbtLoader)`. We override `loadClass` to load the
+ * `xsbti.*` interfaces from `sbtLoader`.
+ *
+ * The parent loader of the TopClassLoader is set to `null` so that the JDK
+ * classes and only the JDK classes are loade from it.
+ */
+private class TopClassLoader(sbtLoader: ClassLoader) extends ClassLoader(null) {
+  // We can't use the loadClass overload with two arguments because it's
+  // protected, but we can do the same by hand (the classloader instance
+  // from which we call resolveClass does not matter).
+  // The one argument overload of loadClass delegates to this one.
+  override protected def loadClass(name: String, resolve: Boolean): Class[_] = {
+    if (name.startsWith("xsbti.") || name.startsWith("org.jline.")) {
+      val c = sbtLoader.loadClass(name)
+      if (resolve) resolveClass(c)
+      c
+    }
+    else super.loadClass(name, resolve)
   }
 }