Support explicit dependencies with scala_artifact (pantsbuild#19187)

This PR prevents scala_artifacts to be matched by the JvmArtifactFieldSet and provides with an implementation of ClasspathEntryRequest

src/python/pants/backend/scala/compile/cats.test.lock

@@ -0,0 +1,85 @@
+# This lockfile was autogenerated by Pants. To regenerate, run:
+#
+#    pants internal-generate-test-lockfile-fixtures ::
+#
+# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
+# {
+#   "version": 1,
+#   "generated_with_requirements": [
+#     "org.scala-lang:scala-library:2.13.8,url=not_provided,jar=not_provided",
+#     "org.typelevel:cats-core_2.13:2.9.0,url=not_provided,jar=not_provided"
+#   ]
+# }
+# --- END PANTS LOCKFILE METADATA ---
+
+[[entries]]
+directDependencies = []
+dependencies = []
+file_name = "org.scala-lang_scala-library_2.13.10.jar"
+
+[entries.coord]
+group = "org.scala-lang"
+artifact = "scala-library"
+version = "2.13.10"
+packaging = "jar"
+[entries.file_digest]
+fingerprint = "e6ca607c3fce03e8fa38af3374ce1f8bb098e316e8bf6f6d27331360feddb1c1"
+serialized_bytes_length = 5949244
+[[entries]]
+file_name = "org.typelevel_cats-core_2.13_2.9.0.jar"
+[[entries.directDependencies]]
+group = "org.scala-lang"
+artifact = "scala-library"
+version = "2.13.10"
+packaging = "jar"
+
+[[entries.directDependencies]]
+group = "org.typelevel"
+artifact = "cats-kernel_2.13"
+version = "2.9.0"
+packaging = "jar"
+
+[[entries.dependencies]]
+group = "org.scala-lang"
+artifact = "scala-library"
+version = "2.13.10"
+packaging = "jar"
+
+[[entries.dependencies]]
+group = "org.typelevel"
+artifact = "cats-kernel_2.13"
+version = "2.9.0"
+packaging = "jar"
+
+
+[entries.coord]
+group = "org.typelevel"
+artifact = "cats-core_2.13"
+version = "2.9.0"
+packaging = "jar"
+[entries.file_digest]
+fingerprint = "1a4beb49987dea5c5028b2950a1fbc23c43978f247b43b4f8f21d1613870565a"
+serialized_bytes_length = 6055599
+[[entries]]
+file_name = "org.typelevel_cats-kernel_2.13_2.9.0.jar"
+[[entries.directDependencies]]
+group = "org.scala-lang"
+artifact = "scala-library"
+version = "2.13.10"
+packaging = "jar"
+
+[[entries.dependencies]]
+group = "org.scala-lang"
+artifact = "scala-library"
+version = "2.13.10"
+packaging = "jar"
+
+
+[entries.coord]
+group = "org.typelevel"
+artifact = "cats-kernel_2.13"
+version = "2.9.0"
+packaging = "jar"
+[entries.file_digest]
+fingerprint = "e43384e88928788c7eb6aadf6999e94d4f25d8a9068055636228af0ebbaec0c9"
+serialized_bytes_length = 3578309

src/python/pants/backend/scala/compile/scalac_test.py

@@ -17,7 +17,12 @@
 from pants.backend.scala.dependency_inference.rules import rules as scala_dep_inf_rules
 from pants.backend.scala.goals.check import ScalacCheckRequest
 from pants.backend.scala.goals.check import rules as scalac_check_rules
-from pants.backend.scala.target_types import ScalacPluginTarget, ScalaSourcesGeneratorTarget
+from pants.backend.scala.resolve.artifact import rules as scala_artifact_rules
+from pants.backend.scala.target_types import (
+    ScalaArtifactTarget,
+    ScalacPluginTarget,
+    ScalaSourcesGeneratorTarget,
+)
 from pants.backend.scala.target_types import rules as target_types_rules
 from pants.build_graph.address import Address
 from pants.core.goals.check import CheckResults
@@ -54,13 +59,19 @@ def rule_runner() -> RuleRunner:
             *testutil.rules(),
             *util_rules(),
             *scala_dep_inf_rules(),
+            *scala_artifact_rules(),
             QueryRule(CheckResults, (ScalacCheckRequest,)),
             QueryRule(CoarsenedTargets, (Addresses,)),
             QueryRule(FallibleClasspathEntry, (CompileScalaSourceRequest,)),
             QueryRule(RenderedClasspath, (CompileScalaSourceRequest,)),
             QueryRule(ClasspathEntry, (CompileScalaSourceRequest,)),
         ],
-        target_types=[JvmArtifactTarget, ScalaSourcesGeneratorTarget, ScalacPluginTarget],
+        target_types=[
+            JvmArtifactTarget,
+            ScalaArtifactTarget,
+            ScalaSourcesGeneratorTarget,
+            ScalacPluginTarget,
+        ],
     )
     rule_runner.set_options(
         args=["--scala-version-for-resolve={'jvm-default':'2.13.8'}"],
@@ -872,3 +883,86 @@ def test_compile_no_deps_scala3(
     assert len(check_results.results) == 1
     check_result = check_results.results[0]
     assert check_result.exit_code == 0
+
+
+@pytest.fixture
+def cats_jvm_lockfile_def() -> JVMLockfileFixtureDefinition:
+    return JVMLockfileFixtureDefinition(
+        "cats.test.lock",
+        [
+            "org.typelevel:cats-core_2.13:2.9.0",
+            "org.scala-lang:scala-library:2.13.8",
+        ],
+    )
+
+
+@pytest.fixture
+def cats_jvm_lockfile(
+    cats_jvm_lockfile_def: JVMLockfileFixtureDefinition, request
+) -> JVMLockfileFixture:
+    return cats_jvm_lockfile_def.load(request)
+
+
+@maybe_skip_jdk_test
+def test_compile_dep_on_scala_artifact(
+    rule_runner: RuleRunner,
+    scala_stdlib_jvm_lockfile: JVMLockfileFixture,
+    cats_jvm_lockfile: JVMLockfileFixture,
+) -> None:
+    third_party_build_file = scala_stdlib_jvm_lockfile.requirements_as_jvm_artifact_targets() + dedent(
+        """\
+        scala_artifact(
+            name = "cats",
+            group = "org.typelevel",
+            artifact = "cats-core",
+            version = "2.9.0"
+        )
+        """
+    )
+    rule_runner.write_files(
+        {
+            "BUILD": dedent(
+                """\
+                scala_sources(
+                    name = 'lib',
+                    dependencies = ["3rdparty/jvm:cats"]
+                )
+                """
+            ),
+            "3rdparty/jvm/BUILD": third_party_build_file,
+            "3rdparty/jvm/default.lock": cats_jvm_lockfile.serialized_lockfile,
+            "ExampleLib.scala": dedent(
+                """
+                import cats._
+                import cats.implicits._
+
+                object ExampleLib {
+                    val values = Functor[List].map(List(1, 2, 3, 4))(_.toString)
+                }
+                """
+            ),
+        }
+    )
+
+    rule_runner.set_options(
+        args=[
+            "--scala-version-for-resolve={'jvm-default': '2.13.8'}",
+        ],
+        env_inherit=PYTHON_BOOTSTRAP_ENV,
+    )
+
+    coarsened_target = expect_single_expanded_coarsened_target(
+        rule_runner, Address(spec_path="", target_name="lib")
+    )
+
+    classpath = rule_runner.request(
+        RenderedClasspath,
+        [CompileScalaSourceRequest(component=coarsened_target, resolve=make_resolve(rule_runner))],
+    )
+    assert classpath.content == {
+        ".ExampleLib.scala.lib.scalac.jar": {
+            "ExampleLib$.class",
+            "ExampleLib.class",
+            "META-INF/MANIFEST.MF",
+        }
+    }

src/python/pants/backend/scala/resolve/artifact.py

@@ -0,0 +1,49 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from pants.backend.scala.target_types import ScalaArtifactFieldSet
+from pants.engine.fs import EMPTY_DIGEST
+from pants.engine.rules import Get, collect_rules, rule
+from pants.engine.unions import UnionRule
+from pants.jvm.compile import (
+    ClasspathDependenciesRequest,
+    ClasspathEntry,
+    ClasspathEntryRequest,
+    CompileResult,
+    FallibleClasspathEntries,
+    FallibleClasspathEntry,
+)
+
+
+class ScalaArtifactClasspathEntryRequest(ClasspathEntryRequest):
+    field_sets = (ScalaArtifactFieldSet,)
+
+
+@rule
+async def scala_artifact_classpath(
+    request: ScalaArtifactClasspathEntryRequest,
+) -> FallibleClasspathEntry:
+    fallible_entries = await Get(FallibleClasspathEntries, ClasspathDependenciesRequest(request))
+    classpath_entries = fallible_entries.if_all_succeeded()
+    if classpath_entries is None:
+        return FallibleClasspathEntry(
+            description=str(request.component),
+            result=CompileResult.DEPENDENCY_FAILED,
+            output=None,
+            exit_code=1,
+        )
+    return FallibleClasspathEntry(
+        description=str(request.component),
+        result=CompileResult.SUCCEEDED,
+        output=ClasspathEntry(EMPTY_DIGEST, dependencies=classpath_entries),
+        exit_code=0,
+    )
+
+
+def rules():
+    return [
+        *collect_rules(),
+        UnionRule(ClasspathEntryRequest, ScalaArtifactClasspathEntryRequest),
+    ]

src/python/pants/backend/scala/target_types.py

@@ -383,6 +383,26 @@ class ScalacPluginTarget(Target):
 # -----------------------------------------------------------------------------------------------
 
 
+# Defining this field and making it required in the `ScalaArtifactFieldSet`
+# prevents the `JvmArtifactFieldSet` matching against `scala_artifact` targets
+# and raising an error when resolving a classpath in which such targets have been
+# used as explicit dependencies of other targets.
+#
+# This way classpath entries for `scala_artifact` targets will be resolved using
+# their own rules, bringing the actual JAR dependency as a transitive one.
+class ScalaArtifactArtifactField(StringField):
+    alias = "artifact"
+    required = True
+    value: str
+    help = help_text(
+        """
+        The 'artifact' part of a Maven-compatible Scala-versioned coordinate to a third-party JAR artifact.
+
+        For the JAR coordinate `org.typelevel:cats-core_2.13:2.9.0`, the artifact is `cats-core`.
+        """
+    )
+
+
 class ScalaCrossVersion(Enum):
     PARTIAL = "partial"
     FULL = "full"
@@ -441,15 +461,15 @@ class ScalaArtifactExclusionsField(JvmArtifactExclusionsField):
 @dataclass(frozen=True)
 class ScalaArtifactFieldSet(FieldSet):
     group: JvmArtifactGroupField
-    artifact: JvmArtifactArtifactField
+    artifact: ScalaArtifactArtifactField
     version: JvmArtifactVersionField
     packages: JvmArtifactPackagesField
     exclusions: ScalaArtifactExclusionsField
     crossversion: ScalaArtifactCrossversionField
 
     required_fields = (
         JvmArtifactGroupField,
-        JvmArtifactArtifactField,
+        ScalaArtifactArtifactField,
         JvmArtifactVersionField,
         JvmArtifactPackagesField,
         ScalaArtifactCrossversionField,