Skip to content

Commit

Permalink
Compile Scala library with Dotty and test its TASTy
Browse files Browse the repository at this point in the history
  • Loading branch information
nicolasstucki committed Oct 20, 2020
1 parent ce67fd1 commit b02abfc
Show file tree
Hide file tree
Showing 6 changed files with 298 additions and 3 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ jobs:

- name: Test
run: |
./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test ;sbt-dotty/scripted scala2-compat/* ;configureIDE"
./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test ;sbt-dotty/scripted scala2-compat/* ;configureIDE ;stdlib-bootstrapped/test:run ;stdlib-bootstrapped-tasty-tests/test"
./project/scripts/bootstrapCmdTests
## Only run bootstrapped tests for Windows since that's a superset of the
Expand Down
2 changes: 2 additions & 0 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ val `scala3-tasty-inspector` = Build.`scala3-tasty-inspector`
val `scala3-language-server` = Build.`scala3-language-server`
val `scala3-bench` = Build.`scala3-bench`
val `scala3-bench-bootstrapped` = Build.`scala3-bench-bootstrapped`
val `stdlib-bootstrapped` = Build.`stdlib-bootstrapped`
val `stdlib-bootstrapped-tasty-tests` = Build.`stdlib-bootstrapped-tasty-tests`
val `tasty-core` = Build.`tasty-core`
val `tasty-core-bootstrapped` = Build.`tasty-core-bootstrapped`
val `tasty-core-scala2` = Build.`tasty-core-scala2`
Expand Down
69 changes: 69 additions & 0 deletions project/Build.scala
Original file line number Diff line number Diff line change
Expand Up @@ -805,6 +805,75 @@ object Build {
javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value
)

/** Scala library compiled by dotty using the latest published sources of the library */
lazy val `stdlib-bootstrapped` = project.in(file("stdlib-bootstrapped")).
withCommonSettings(Bootstrapped).
dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
dependsOn(`scala3-tasty-inspector` % "test->test").
settings(commonBootstrappedSettings).
settings(
moduleName := "scala-library",
javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value,
scalacOptions -= "-Xfatal-warnings",
ivyConfigurations += SourceDeps.hide,
transitiveClassifiers := Seq("sources"),
libraryDependencies +=
("org.scala-lang" % "scala-library" % stdlibVersion(Bootstrapped) % "sourcedeps"),
sourceGenerators in Compile += Def.task {
val s = streams.value
val cacheDir = s.cacheDirectory
val trgDir = (sourceManaged in Compile).value / "scala-library-src"

val report = updateClassifiers.value
val scalaLibrarySourcesJar = report.select(
configuration = configurationFilter("sourcedeps"),
module = (_: ModuleID).name == "scala-library",
artifact = artifactFilter(`type` = "src")).headOption.getOrElse {
sys.error(s"Could not fetch scala-library sources")
}

FileFunction.cached(cacheDir / s"fetchScalaLibrarySrc",
FilesInfo.lastModified, FilesInfo.exists) { dependencies =>
s.log.info(s"Unpacking scala-library sources to $trgDir...")
if (trgDir.exists)
IO.delete(trgDir)
IO.createDirectory(trgDir)
IO.unzip(scalaLibrarySourcesJar, trgDir)

((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet
} (Set(scalaLibrarySourcesJar)).toSeq
}.taskValue,
sources in Compile ~= (_.filterNot(file =>
// sources from https://github.com/scala/scala/tree/2.13.x/src/library-aux
file.getPath.endsWith("scala-library-src/scala/Any.scala") ||
file.getPath.endsWith("scala-library-src/scala/AnyVal.scala") ||
file.getPath.endsWith("scala-library-src/scala/AnyRef.scala") ||
file.getPath.endsWith("scala-library-src/scala/Nothing.scala") ||
file.getPath.endsWith("scala-library-src/scala/Null.scala") ||
file.getPath.endsWith("scala-library-src/scala/Singleton.scala"))),
managedClasspath in Test ~= {
_.filterNot(file => file.data.getName == s"scala-library-${stdlibVersion(Bootstrapped)}.jar")
},
)

/** Test the tasty generated by `stdlib-bootstrapped`
*
* The tests are run with the bootstrapped compiler and the tasty inpector on the classpath.
* The classpath has the default `scala-library` and not `stdlib-bootstrapped`.
*
* The jar of `stdlib-bootstrapped` is provided for to the tests.
* - inspector: test that we can load the contents of the jar using the tasty inspector
* - from-tasty: test that we can recompile the contents of the jar using `dotc -from-tasty`
*/
lazy val `stdlib-bootstrapped-tasty-tests` = project.in(file("stdlib-bootstrapped-tasty-tests")).
withCommonSettings(Bootstrapped).
dependsOn(`scala3-tasty-inspector` % "test->test").
settings(commonBootstrappedSettings).
settings(
javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value,
javaOptions += "-Ddotty.scala.library=" + packageBin.in(`stdlib-bootstrapped`, Compile).value.getAbsolutePath
)

lazy val `scala3-sbt-bridge` = project.in(file("sbt-bridge/src")).
// We cannot depend on any bootstrapped project to compile the bridge, since the
// bridge is needed to compile these projects.
Expand Down
209 changes: 209 additions & 0 deletions stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
package dotty.tools.dotc

import org.junit.Test
import org.junit.Ignore
import org.junit.Assert._

import dotty.tools.io._
import dotty.tools.dotc.util.ClasspathFromClassloader

import scala.quoted._

import java.io.File.pathSeparator

class BootstrappedStdLibTASYyTest:

import BootstrappedStdLibTASYyTest._

/** Test that we can load trees from TASTy */
@Test def testTastyInspector: Unit =
loadWithTastyInspector(loadBlacklisted)

/** Test that we can load and compile trees from TASTy */
@Test def testFromTasty: Unit =
compileFromTasty(loadBlacklisted.union(compileBlacklisted))

@Ignore
@Test def testWhiteListFromTasty: Unit =
val whitelist = Set(
"scala.collection.mutable.StringBuilder"
)
compileFromTasty(x => !whitelist(x))

@Test def blacklistNoDuplicates =
def testDup(name: String, list: List[String], set: Set[String]) =
assert(list.size == set.size,
list.diff(set.toSeq).mkString(s"`$name` has duplicate entries:\n ", "\n ", "\n\n"))
testDup("loadBlacklist", loadBlacklist, loadBlacklisted)
testDup("compileBlacklist", compileBlacklist, compileBlacklisted)

@Test def blacklistsNoIntersection =
val intersection = loadBlacklisted & compileBlacklisted
assert(intersection.isEmpty,
intersection.mkString(
"`compileBlacklist` contains names that are already in `loadBlacklist`: \n ", "\n ", "\n\n"))

@Test def blacklistsOnlyContainsClassesThatExist =
val scalaLibJarTastyClassNamesSet = scalaLibJarTastyClassNames.toSet
val intersection = loadBlacklisted & compileBlacklisted
assert(loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty,
loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString(
"`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n"))
assert(compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty,
compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString(
"`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n"))

@Ignore
@Test def testLoadBacklistIsMinimal =
var shouldBeWhitelisted = List.empty[String]
val size = loadBlacklisted.size
for (notBlacklisted, i) <- loadBlacklist.zipWithIndex do
val blacklist = loadBlacklisted - notBlacklisted
println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)")
try {
loadWithTastyInspector(blacklist)
shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted
}
catch {
case ex: Throwable => // ok
}
assert(shouldBeWhitelisted.isEmpty,
shouldBeWhitelisted.mkString("Some classes do not need to be blacklisted in `loadBlacklisted`\n ", "\n ", "\n\n"))

@Ignore
@Test def testCompileBlacklistIsMinimal =
var shouldBeWhitelisted = List.empty[String]
val size = compileBlacklisted.size
val blacklist0 = loadBlacklisted.union(compileBlacklisted)
for (notBlacklisted, i) <- compileBlacklist.zipWithIndex do
val blacklist = blacklist0 - notBlacklisted
println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)")
try {
compileFromTasty(blacklist)
shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted
}
catch {
case ex: Throwable => // ok
}
assert(shouldBeWhitelisted.isEmpty,
shouldBeWhitelisted.mkString("Some classes do not need to be blacklisted in `compileBlacklisted`\n ", "\n ", "\n\n"))

end BootstrappedStdLibTASYyTest

object BootstrappedStdLibTASYyTest:

val scalaLibJarPath = System.getProperty("dotty.scala.library")

val scalaLibJarTastyClassNames = {
val scalaLibJar = Jar(new File(java.nio.file.Paths.get(scalaLibJarPath)))
scalaLibJar.toList.map(_.toString).filter(_.endsWith(".tasty"))
.map(_.stripSuffix(".tasty").replace("/", "."))
.sorted
}

def loadWithTastyInspector(blacklisted: String => Boolean): Unit =
val inspector = new scala.tasty.inspector.TastyInspector {
def processCompilationUnit(using QuoteContext)(root: qctx.reflect.Tree): Unit =
root.showExtractors // Check that we can traverse the full tree
()
}
val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted)
val hasErrors = inspector.inspect(scalaLibJarPath, classNames)
assert(!hasErrors, "Errors reported while loading from TASTy")

def compileFromTasty(blacklisted: String => Boolean): Unit = {
val driver = new dotty.tools.dotc.Driver
val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader)
val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted)
val args = Array(
"-classpath", s"$scalaLibJarPath$pathSeparator$currentClasspath",
"-from-tasty",
"-nowarn"
) ++ classNames
val reporter = driver.process(args)
assert(reporter.errorCount == 0, "Errors while re-compiling")
}

/** List of classes that cannot be loaded from TASTy */
def loadBlacklist = List[String](
// No issues :)
)

/** List of classes that cannot be recompilied from TASTy */
def compileBlacklist = List[String](
// See #10048
// failed: java.lang.AssertionError: assertion failed: class Boolean
// at dotty.DottyPredef$.assertFail(DottyPredef.scala:17)
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.assertClassNotArrayNotPrimitive(BCodeHelpers.scala:247)
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass(BCodeHelpers.scala:265)
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass$(BCodeHelpers.scala:210)
// at dotty.tools.backend.jvm.BCodeSkelBuilder$PlainSkelBuilder.getClassBTypeAndRegisterInnerClass(BCodeSkelBuilder.scala:62)
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.internalName(BCodeHelpers.scala:237)
"scala.Array",
"scala.Boolean",
"scala.Byte",
"scala.Char",
"scala.Double",
"scala.Float",
"scala.Int",
"scala.Long",
"scala.Short",
"scala.Unit",

// See #9994
// -- Error:
// | def addOne(kv: (K, V)) = {
// | ^
// |error overriding method addOne in trait Growable of type (elem: (K, V)): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]);
// | method addOne of type (kv: (K, V)): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]) has incompatible type
// -- Error:
// | def subtractOne(k: K) = {
// | ^
// |error overriding method subtractOne in trait Shrinkable of type (elem: K): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]);
// | method subtractOne of type (k: K): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]) has incompatible type
"scala.collection.concurrent.TrieMap",
"scala.collection.immutable.HashMapBuilder",
"scala.collection.immutable.HashSetBuilder",
"scala.collection.immutable.LazyList",
"scala.collection.immutable.ListMapBuilder",
"scala.collection.immutable.MapBuilderImpl",
"scala.collection.immutable.SetBuilderImpl",
"scala.collection.immutable.TreeSeqMap",
"scala.collection.immutable.VectorBuilder",
"scala.collection.immutable.VectorMapBuilder",
"scala.collection.mutable.AnyRefMap",
"scala.collection.mutable.ArrayBuilder",
"scala.collection.mutable.CollisionProofHashMap",
"scala.collection.mutable.LongMap",
"scala.collection.mutable.SortedMap",
"scala.collection.mutable.StringBuilder",
"scala.jdk.AnyAccumulator",
"scala.jdk.DoubleAccumulator",
"scala.jdk.IntAccumulator",
"scala.jdk.LongAccumulator",

// See #9994
// -- Error:
// | override def filterInPlace(p: A => Boolean): this.type = {
// | ^
// |error overriding method filterInPlace in trait SetOps of type (p: A => Boolean): (HashSet.this : scala.collection.mutable.HashSet[A]);
// | method filterInPlace of type (p: A => Boolean): (HashSet.this : scala.collection.mutable.HashSet[A]) has incompatible type
"scala.collection.mutable.HashSet",

// See #9994
// -- Error:
// | def force: this.type = {
// | ^
// |error overriding method force in class Stream of type => (Cons.this : scala.collection.immutable.Stream.Cons[A]);
// | method force of type => (Cons.this : scala.collection.immutable.Stream.Cons[A]) has incompatible type
"scala.collection.immutable.Stream",

)

/** Set of classes that cannot be loaded from TASTy */
def loadBlacklisted = loadBlacklist.toSet

/** Set of classes that cannot be recompilied from TASTy */
def compileBlacklisted = compileBlacklist.toSet

end BootstrappedStdLibTASYyTest
12 changes: 12 additions & 0 deletions stdlib-bootstrapped/test/Main.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package hello

enum Color:
case Red, Green, Blue

object HelloWorld:
def main(args: Array[String]): Unit = {
println("hello dotty.superbootstrapped!")
println(Color.Red)
println(Color.Green)
println(Color.Blue)
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ trait TastyInspector:
*
* @param classpath Classpath where the classes are located
* @param classes classes to be inspected
* @return if an error was reported
*/
def inspect(classpath: String, classes: List[String]): Unit =
def inspect(classpath: String, classes: List[String]): Boolean =
if (classes.isEmpty)
throw new IllegalArgumentException("Parameter classes should no be empty")

Expand Down Expand Up @@ -64,7 +65,9 @@ trait TastyInspector:

val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader)
val args = "-from-tasty" :: "-Yretain-trees" :: "-classpath" :: s"$classpath$pathSeparator$currentClasspath" :: classes
(new InspectorDriver).process(args.toArray)
val reporter = (new InspectorDriver).process(args.toArray)
reporter.hasErrors

end inspect


Expand Down

0 comments on commit b02abfc

Please sign in to comment.