Skip to content

Commit

Permalink
community build: add automatic dependency override handling
Browse files Browse the repository at this point in the history
Introduce a new sbt plugin: `sbt-community-build`, which tracks the
modules published during a community build run, and injects dependency
overrides to force version alignment.
  • Loading branch information
griggt committed Feb 23, 2021
1 parent 5eb3258 commit 5175255
Show file tree
Hide file tree
Showing 26 changed files with 494 additions and 79 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ jobs:
run: cp -vf .github/workflows/repositories /root/.sbt/ ; true

- name: Test
run: ./project/scripts/sbt sbt-dotty/scripted
run: ./project/scripts/sbt "sbt-dotty/scripted; sbt-community-build/scripted"

test_java8:
runs-on: [self-hosted, Linux]
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ vscode-dotty/.vscode-test
community-build/scala3-bootstrapped.version
community-build/sbt-dotty-sbt
community-build/sbt-scalajs-sbt
community-build/dotty-community-build-deps

# Vulpix output files
*.check.out
Expand Down
1 change: 1 addition & 0 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ val `scaladoc-js` = Build.`scaladoc-js`
val `scala3-bench-run` = Build.`scala3-bench-run`
val dist = Build.dist
val `community-build` = Build.`community-build`
val `sbt-community-build` = Build.`sbt-community-build`

val sjsSandbox = Build.sjsSandbox
val sjsJUnitTests = Build.sjsJUnitTests
Expand Down
86 changes: 8 additions & 78 deletions community-build/src/scala/dotty/communitybuild/projects.scala
Original file line number Diff line number Diff line change
Expand Up @@ -27,27 +27,6 @@ def exec(projectDir: Path, binary: String, arguments: String*): Int =
exitCode


/** Versions of published projects, needs to be updated when a project in the build is updated.
*
* TODO: instead of harcoding these numbers, we could get them from the
* projects themselves. This likely requires injecting a custom task in the
* projects to output the version number to a file.
*/
object Versions:
val cats = "2.3.1-SNAPSHOT"
val catsMtl = "1.1+DOTTY-SNAPSHOT"
val coop = "1.0+DOTTY-SNAPSHOT"
val discipline = "1.1.3-SNAPSHOT"
val disciplineMunit = "1.0.3+DOTTY-SNAPSHOT"
val disciplineSpecs2 = "1.1.3-SNAPSHOT"
val izumiReflect = "1.0.0-SNAPSHOT"
val scalacheck = "1.15.2-SNAPSHOT"
val scalatest = "3.2.3"
val munit = "0.7.19+DOTTY-SNAPSHOT"
val scodecBits = "1.1+DOTTY-SNAPSHOT"
val simulacrumScalafix = "0.5.1-SNAPSHOT"
val scalaCollectionCompat = "2.3.0+DOTTY-SNAPSHOT"

sealed trait CommunityProject:
private var published = false

Expand Down Expand Up @@ -115,56 +94,8 @@ final case class SbtCommunityProject(
) extends CommunityProject:
override val binaryName: String = "sbt"

// A project in the community build can depend on an arbitrary version of
// another project in the build, so we force the use of the version that is
// actually in the community build.
val dependencyOverrides = List(
// dependencyOverrides doesn't seem to understand `%%%`
s""""org.scalacheck" %% "scalacheck" % "${Versions.scalacheck}"""",
s""""org.scalacheck" %% "scalacheck_sjs1" % "${Versions.scalacheck}"""",
s""""org.scalatest" %% "scalatest" % "${Versions.scalatest}"""",
s""""org.scalatest" %% "scalatest_sjs1" % "${Versions.scalatest}"""",
s""""org.scalatestplus" %% "junit-4-13" % "${Versions.scalatest}.0"""",
s""""org.scalameta" %% "munit" % "${Versions.munit}"""",
s""""org.scalameta" %% "munit_sjs1" % "${Versions.munit}"""",
s""""org.scalameta" %% "munit-scalacheck" % "${Versions.munit}"""",
s""""org.scalameta" %% "munit-scalacheck_sjs1" % "${Versions.munit}"""",
s""""org.scalameta" %% "junit-interface" % "${Versions.munit}"""",
s""""org.scodec" %% "scodec-bits" % "${Versions.scodecBits}"""",
s""""org.scodec" %% "scodec-bits_sjs1" % "${Versions.scodecBits}"""",
s""""org.typelevel" %% "discipline-core" % "${Versions.discipline}"""",
s""""org.typelevel" %% "discipline-core_sjs1" % "${Versions.discipline}"""",
s""""org.typelevel" %% "discipline-munit" % "${Versions.disciplineMunit}"""",
s""""org.typelevel" %% "discipline-munit_sjs1" % "${Versions.disciplineMunit}"""",
s""""org.typelevel" %% "discipline-specs2" % "${Versions.disciplineSpecs2}"""",
s""""org.typelevel" %% "discipline-specs2_sjs1" % "${Versions.disciplineSpecs2}"""",
s""""org.typelevel" %% "simulacrum-scalafix-annotations" % "${Versions.simulacrumScalafix}"""",
s""""org.typelevel" %% "simulacrum-scalafix-annotations_sjs1" % "${Versions.simulacrumScalafix}"""",
s""""org.typelevel" %% "cats-core" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-core_sjs1" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-free" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-free_sjs1" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-kernel" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-kernel_sjs1" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-kernel-laws" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-kernel-laws_sjs1" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-laws" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-laws_sjs1" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-testkit" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-testkit_sjs1" % "${Versions.cats}"""",
s""""org.typelevel" %% "cats-mtl" % "${Versions.catsMtl}"""",
s""""org.typelevel" %% "cats-mtl_sjs1" % "${Versions.catsMtl}"""",
s""""org.typelevel" %% "cats-mtl-laws" % "${Versions.catsMtl}"""",
s""""org.typelevel" %% "cats-mtl-laws_sjs1" % "${Versions.catsMtl}"""",
s""""org.typelevel" %% "coop" % "${Versions.coop}"""",
s""""org.typelevel" %% "coop_sjs1" % "${Versions.coop}"""",
s""""dev.zio" %% "izumi-reflect" % "${Versions.izumiReflect}"""",
s""""org.scala-lang.modules" %% "scala-collection-compat" % "${Versions.scalaCollectionCompat}"""",
)

private val baseCommand =
"clean; set logLevel in Global := Level.Error; set updateOptions in Global ~= (_.withLatestSnapshots(false)); "
++ s"""set dependencyOverrides in ThisBuild ++= ${dependencyOverrides.mkString("Seq(", ", ", ")")}; """
++ s"++$compilerVersion!; "

override val testCommand =
Expand All @@ -186,7 +117,8 @@ final case class SbtCommunityProject(
case _ => Nil
extraSbtArgs ++ sbtProps ++ List(
"-sbt-version", "1.4.7",
"-Dsbt.supershell=false",
"-Dsbt.supershell=false",
s"-Ddotty.communitybuild.dir=$communitybuildDir",
s"--addPluginSbtFile=$sbtPluginFilePath"
)

Expand Down Expand Up @@ -423,17 +355,15 @@ object projects:
lazy val munit = SbtCommunityProject(
project = "munit",
sbtTestCommand = "testsJVM/test;testsJS/test;",
// Hardcode the version to avoid having to deal with something set by sbt-dynver
sbtPublishCommand = s"""set every version := "${Versions.munit}"; munitJVM/publishLocal; munitJS/publishLocal; munitScalacheckJVM/publishLocal; munitScalacheckJS/publishLocal; junit/publishLocal""",
sbtPublishCommand = "munitJVM/publishLocal; munitJS/publishLocal; munitScalacheckJVM/publishLocal; munitScalacheckJS/publishLocal; junit/publishLocal",
sbtDocCommand = "junit/doc; munitJVM/doc",
dependencies = List(scalacheck)
)

lazy val scodecBits = SbtCommunityProject(
project = "scodec-bits",
sbtTestCommand = "coreJVM/test;coreJS/test",
// Hardcode the version to avoid having to deal with something set by sbt-git
sbtPublishCommand = s"""set every version := "${Versions.scodecBits}"; coreJVM/publishLocal;coreJS/publishLocal""",
sbtPublishCommand = "coreJVM/publishLocal;coreJS/publishLocal",
sbtDocCommand = "coreJVM/doc",
dependencies = List(munit)
)
Expand Down Expand Up @@ -509,7 +439,7 @@ object projects:
lazy val scalaCollectionCompat = SbtCommunityProject(
project = "scala-collection-compat",
sbtTestCommand = "compat30/test",
sbtPublishCommand = s"""set every version := "${Versions.scalaCollectionCompat}"; compat30/publishLocal""",
sbtPublishCommand = "compat30/publishLocal",
)

lazy val verify = SbtCommunityProject(
Expand All @@ -528,7 +458,7 @@ object projects:
lazy val disciplineMunit = SbtCommunityProject(
project = "discipline-munit",
sbtTestCommand = "test",
sbtPublishCommand = s"""set every version := "${Versions.disciplineMunit}";coreJVM/publishLocal;coreJS/publishLocal""",
sbtPublishCommand = "coreJVM/publishLocal;coreJS/publishLocal",
dependencies = List(discipline, munit)
)

Expand All @@ -555,14 +485,14 @@ object projects:
lazy val catsMtl = SbtCommunityProject(
project = "cats-mtl",
sbtTestCommand = "testsJVM/test;testsJS/test",
sbtPublishCommand = s"""set every version := "${Versions.catsMtl}";coreJVM/publishLocal;coreJS/publishLocal;lawsJVM/publishLocal;lawsJS/publishLocal""",
sbtPublishCommand = "coreJVM/publishLocal;coreJS/publishLocal;lawsJVM/publishLocal;lawsJS/publishLocal",
dependencies = List(cats, disciplineMunit)
)

lazy val coop = SbtCommunityProject(
project = "coop",
sbtTestCommand = "test",
sbtPublishCommand = s"""set every version := "${Versions.coop}";coreJVM/publishLocal;coreJS/publishLocal""",
sbtPublishCommand = "coreJVM/publishLocal;coreJS/publishLocal",
dependencies = List(cats, catsMtl)
)

Expand Down
44 changes: 44 additions & 0 deletions project/Build.scala
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ object Build {
if (isRelease) baseSbtDottyVersion else baseSbtDottyVersion + "-SNAPSHOT"
}

val sbtCommunityBuildVersion = "0.1.0-SNAPSHOT"

val agentOptions = List(
// "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
// "-agentpath:/home/dark/opt/yjp-2013-build-13072/bin/linux-x86-64/libyjpagent.so"
Expand Down Expand Up @@ -1272,6 +1274,45 @@ object Build {
}.dependsOn(compile in Compile).evaluated
)

lazy val `sbt-community-build` = project.in(file("sbt-community-build")).
enablePlugins(SbtPlugin).
settings(commonSettings).
settings(
name := "sbt-community-build",
version := sbtCommunityBuildVersion,
organization := "ch.epfl.lamp",
sbtTestDirectory := baseDirectory.value / "sbt-test",
scriptedLaunchOpts ++= Seq(
"-Dplugin.version=" + version.value,
"-Dplugin.scalaVersion=" + dottyVersion,
"-Dplugin.scalaJSVersion=" + scalaJSVersion,
"-Dplugin.sbtDottyVersion=" + sbtDottyVersion,
"-Ddotty.communitybuild.dir=" + baseDirectory.value / "target",
"-Dsbt.boot.directory=" + ((baseDirectory in ThisBuild).value / ".sbt-scripted").getAbsolutePath // Workaround sbt/sbt#3469
),
// Pass along ivy home and repositories settings to sbt instances run from the tests
scriptedLaunchOpts ++= {
val repositoryPath = (io.Path.userHome / ".sbt" / "repositories").absolutePath
s"-Dsbt.repository.config=$repositoryPath" ::
ivyPaths.value.ivyHome.map("-Dsbt.ivy.home=" + _.getAbsolutePath).toList
},
scriptedBufferLog := true,
scriptedBatchExecution := true,
scripted := scripted.dependsOn(
publishLocal in `scala3-sbt-bridge`,
publishLocal in `scala3-interfaces`,
publishLocal in `scala3-compiler-bootstrapped`,
publishLocal in `scala3-library-bootstrapped`,
publishLocal in `scala3-library-bootstrappedJS`,
publishLocal in `tasty-core-bootstrapped`,
publishLocal in `scala3-staging`,
publishLocal in `scala3-tasty-inspector`,
publishLocal in `scaladoc`,
publishLocal in `scala3-bootstrapped`,
publishLocal in `sbt-dotty`,
).evaluated
)

val prepareCommunityBuild = taskKey[Unit]("Publish local the compiler and the sbt plugin. Also store the versions of the published local artefacts in two files, community-build/{scala3-bootstrapped.version,sbt-dotty-sbt}.")

lazy val `community-build` = project.in(file("community-build")).
Expand All @@ -1289,13 +1330,16 @@ object Build {
(publishLocal in `sbt-dotty`).value
(publishLocal in `scala3-bootstrapped`).value
(publishLocal in `scala3-library-bootstrappedJS`).value
(publishLocal in `sbt-community-build`).value
// (publishLocal in `scala3-staging`).value
val pluginText =
s"""updateOptions in Global ~= (_.withLatestSnapshots(false))
|addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "$sbtDottyVersion")
|addSbtPlugin("ch.epfl.lamp" % "sbt-community-build" % "$sbtCommunityBuildVersion")
|addSbtPlugin("org.scala-js" % "sbt-scalajs" % "$scalaJSVersion")""".stripMargin
IO.write(baseDirectory.value / "sbt-dotty-sbt", pluginText)
IO.write(baseDirectory.value / "scala3-bootstrapped.version", dottyVersion)
IO.delete(baseDirectory.value / "dotty-community-build-deps") // delete any stale deps file
},
testOptions in Test += Tests.Argument(
TestFrameworks.JUnit,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
ThisBuild / scalaVersion := sys.props("plugin.scalaVersion")
ThisBuild / organization := "org.example"

lazy val a = project
.settings(
name := "a",
version := "0.4.1-SNAPSHOT",
libraryDependencies := Seq(), // don't depend on scala-library
)

lazy val b = project
.settings(onlyThisTestResolverSettings)
.settings(
name := "b",
libraryDependencies := Seq(organization.value %% "a" % "0.4.0-SNAPSHOT"),
)

lazy val c = project
.settings(onlyThisTestResolverSettings)
.settings(
name := "c",
libraryDependencies := Seq(), // don't depend on scala-library
).dependsOn(b)
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import sbt._
import Keys._

object ThisTestPlugin extends AutoPlugin {
override def requires = plugins.IvyPlugin
override def trigger = allRequirements

val thisTestIvyHome = settingKey[File]("Ivy home directory for artifacts published by this test")
val thisTestResolver = settingKey[Resolver]("Resolver for artifacts published by this test")
val deleteDepsFile = taskKey[Unit]("Deletes the dotty-community-build-deps dependency tracking file")

override val projectSettings = Seq(
publishLocalConfiguration := publishLocalConfiguration.value.withResolverName("this-test")
)

override val buildSettings = defaultThisTestSettings ++ Seq(
resolvers += thisTestResolver.value
)

def defaultThisTestSettings: Seq[Setting[_]] = {
Seq(
thisTestIvyHome := (LocalRootProject / target).value / "ivy-cache",
thisTestResolver := Resolver.file("this-test", thisTestIvyHome.value / "local")(Resolver.ivyStylePatterns),
deleteDepsFile := IO.delete(file(sys.props("dotty.communitybuild.dir")) / "dotty-community-build-deps"),
)
}

object autoImport {
def onlyThisTestResolverSettings: Seq[Setting[_]] = Seq(
externalResolvers := thisTestResolver.value :: Nil
)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
addSbtPlugin("ch.epfl.lamp" % "sbt-community-build" % sys.props("plugin.version"))
addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % sys.props("plugin.sbtDottyVersion"))
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
> deleteDepsFile
> reload

> a/publishLocal

-> c/update

> reload

> c/update
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
ThisBuild / scalaVersion := sys.props("plugin.scalaVersion")
ThisBuild / organization := "org.example"

lazy val a = project
.settings(
name := "a",
version := "0.2.1-SNAPSHOT",
libraryDependencies := Seq(), // don't depend on scala-library
)

lazy val b = project
.settings(
name := "b",
version := "1.2.1-SNAPSHOT",
libraryDependencies := Seq(), // don't depend on scala-library
)

lazy val c = project
.settings(onlyThisTestResolverSettings)
.settings(
name := "c",
libraryDependencies := Seq(
organization.value %% "a" % "0.2.0-SNAPSHOT",
organization.value %% "b" % "1.2.0-SNAPSHOT",
),
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import sbt._
import Keys._

object ThisTestPlugin extends AutoPlugin {
override def requires = plugins.IvyPlugin
override def trigger = allRequirements

val thisTestIvyHome = settingKey[File]("Ivy home directory for artifacts published by this test")
val thisTestResolver = settingKey[Resolver]("Resolver for artifacts published by this test")
val deleteDepsFile = taskKey[Unit]("Deletes the dotty-community-build-deps dependency tracking file")

override val projectSettings = Seq(
publishLocalConfiguration := publishLocalConfiguration.value.withResolverName("this-test")
)

override val buildSettings = defaultThisTestSettings ++ Seq(
resolvers += thisTestResolver.value
)

def defaultThisTestSettings: Seq[Setting[_]] = {
Seq(
thisTestIvyHome := (LocalRootProject / target).value / "ivy-cache",
thisTestResolver := Resolver.file("this-test", thisTestIvyHome.value / "local")(Resolver.ivyStylePatterns),
deleteDepsFile := IO.delete(file(sys.props("dotty.communitybuild.dir")) / "dotty-community-build-deps"),
)
}

object autoImport {
def onlyThisTestResolverSettings: Seq[Setting[_]] = Seq(
externalResolvers := thisTestResolver.value :: Nil
)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
addSbtPlugin("ch.epfl.lamp" % "sbt-community-build" % sys.props("plugin.version"))
addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % sys.props("plugin.sbtDottyVersion"))
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
> deleteDepsFile
> reload

> a/publishLocal
> b/publishLocal

-> c/update

> reload

> c/update
Loading

0 comments on commit 5175255

Please sign in to comment.