Skip to content

Commit

Permalink
[jvm] Finish enabling thirdparty dependency inference for Scala (pant…
Browse files Browse the repository at this point in the history 
…sbuild#13583)

Adds the final piece of the puzzle to enable thirdparty dependency inference for Scala... which after everyone's great prepwork just involved adding usage of the `ThirdPartyPackageToArtifactMapping` to the Scala inference rules.

Fixes pantsbuild#13365.
  • Loading branch information
@stuhood
stuhood authored Nov 11, 2021
1 parent c810f37 commit 2457d5b
Show file tree
Hide file tree
Showing 5 changed files with 277 additions and 216 deletions.
215 changes: 0 additions & 215 deletions src/python/pants/backend/java/dependency_inference/rules_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,6 @@
import pytest

from pants.backend.java.compile.javac import rules as javac_rules
from pants.backend.java.dependency_inference.java_parser import rules as java_parser_rules
from pants.backend.java.dependency_inference.java_parser_launcher import (
rules as java_parser_launcher_rules,
)
from pants.backend.java.dependency_inference.rules import InferJavaSourceDependencies
from pants.backend.java.dependency_inference.rules import rules as dep_inference_rules
from pants.backend.java.target_types import (
Expand All @@ -29,11 +25,6 @@
InferredDependencies,
Targets,
)
from pants.jvm.dependency_inference.artifact_mapper import (
FrozenTrieNode,
ThirdPartyPackageToArtifactMapping,
UnversionedCoordinate,
)
from pants.jvm.jdk_rules import rules as java_util_rules
from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules
from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules
Expand All @@ -56,8 +47,6 @@ def rule_runner() -> RuleRunner:
*coursier_setup_rules(),
*dep_inference_rules(),
*external_tool_rules(),
*java_parser_launcher_rules(),
*java_parser_rules(),
*java_target_rules(),
*java_util_rules(),
*javac_rules(),
Expand All @@ -68,7 +57,6 @@ def rule_runner() -> RuleRunner:
QueryRule(ExplicitlyProvidedDependencies, [DependenciesRequest]),
QueryRule(InferredDependencies, [InferJavaSourceDependencies]),
QueryRule(Targets, [UnparsedAddressInputs]),
QueryRule(ThirdPartyPackageToArtifactMapping, []),
],
target_types=[JavaSourcesGeneratorTarget, JunitTestsGeneratorTarget, JvmArtifact],
)
Expand Down Expand Up @@ -558,206 +546,3 @@ class C {}
# B.java does NOT have a dependency on A.java, as it would if we just had subtargets without
# inferred dependencies.
assert rule_runner.request(Addresses, [DependenciesRequest(lib[Dependencies])]) == Addresses()


@maybe_skip_jdk_test
def test_third_party_mapping_parsing(rule_runner: RuleRunner) -> None:
rule_runner.set_options(
[
"--java-infer-third-party-import-mapping={'io.github.frenchtoast.savory.**': 'github-frenchtoast:savory'}"
],
env_inherit=PYTHON_BOOTSTRAP_ENV,
)
mapping = rule_runner.request(ThirdPartyPackageToArtifactMapping, [])
root_node = mapping.mapping_root

# Handy trie traversal function to placate mypy
def traverse(*children) -> FrozenTrieNode:
node = root_node
for child in children:
new_node = node.find_child(child)
if not new_node:
coord = ".".join(children)
raise Exception(f"Could not find the package specifed by {coord}.")
node = new_node
return node

# Verify some provided by `JVM_ARTFACT_MAPPINGS`
assert set(traverse("org", "junit").coordinates) == {
UnversionedCoordinate(group="junit", artifact="junit")
}
assert set(traverse("org", "aopalliance").coordinates) == {
UnversionedCoordinate(group="aopalliance", artifact="aopalliance")
}

# Verify the one that we provided in the options is there too.
assert set(traverse("io", "github", "frenchtoast", "savory").coordinates) == {
UnversionedCoordinate(group="github-frenchtoast", artifact="savory")
}


@maybe_skip_jdk_test
def test_third_party_dep_inference(rule_runner: RuleRunner) -> None:
rule_runner.set_options(
["--java-infer-third-party-import-mapping={'org.joda.time.**': 'joda-time:joda-time'}"],
env_inherit=PYTHON_BOOTSTRAP_ENV,
)
rule_runner.write_files(
{
"BUILD": dedent(
"""\
jvm_artifact(
name = "joda-time_joda-time",
group = "joda-time",
artifact = "joda-time",
version = "2.10.10",
)
java_sources(name = 'lib')
"""
),
"PrintDate.java": dedent(
"""\
package org.pantsbuild.example;
import org.joda.time.DateTime;
public class PrintDate {
public static void main(String[] args) {
DateTime dt = new DateTime();
System.out.println(dt.toString());
}
}
"""
),
}
)

lib = rule_runner.get_target(
Address("", target_name="lib", relative_file_path="PrintDate.java")
)
assert rule_runner.request(Addresses, [DependenciesRequest(lib[Dependencies])]) == Addresses(
[Address("", target_name="joda-time_joda-time")]
)


@maybe_skip_jdk_test
def test_third_party_dep_inference_fqtn(rule_runner: RuleRunner) -> None:
rule_runner.set_options(
["--java-infer-third-party-import-mapping={'org.joda.time.**': 'joda-time:joda-time'}"],
env_inherit=PYTHON_BOOTSTRAP_ENV,
)
rule_runner.write_files(
{
"BUILD": dedent(
"""\
jvm_artifact(
name = "joda-time_joda-time",
group = "joda-time",
artifact = "joda-time",
version = "2.10.10",
)
java_sources(name = 'lib')
"""
),
"PrintDate.java": dedent(
"""\
package org.pantsbuild.example;
public class PrintDate {
public static void main(String[] args) {
org.joda.time.DateTime dt = new DateTime();
System.out.println(dt.toString());
}
}
"""
),
}
)

lib = rule_runner.get_target(
Address("", target_name="lib", relative_file_path="PrintDate.java")
)
assert rule_runner.request(Addresses, [DependenciesRequest(lib[Dependencies])]) == Addresses(
[Address("", target_name="joda-time_joda-time")]
)


@maybe_skip_jdk_test
def test_third_party_dep_inference_nonrecursive(rule_runner: RuleRunner) -> None:
rule_runner.set_options(
[
"--java-infer-third-party-import-mapping={'org.joda.time.**':'joda-time:joda-time', 'org.joda.time.DateTime':'joda-time:joda-time-2'}"
],
env_inherit=PYTHON_BOOTSTRAP_ENV,
)
rule_runner.write_files(
{
"BUILD": dedent(
"""\
jvm_artifact(
name = "joda-time_joda-time",
group = "joda-time",
artifact = "joda-time",
version = "2.10.10",
)
jvm_artifact(
name = "joda-time_joda-time-2",
group = "joda-time",
artifact = "joda-time-2", # doesn't really exist, but useful for this test
version = "2.10.10",
)
java_sources(name = 'lib')
"""
),
"PrintDate.java": dedent(
"""\
package org.pantsbuild.example;
import org.joda.time.DateTime;
public class PrintDate {
public static void main(String[] args) {
DateTime dt = new DateTime();
System.out.println(dt.toString());
}
}
"""
),
"PrintDate2.java": dedent(
"""\
package org.pantsbuild.example;
import org.joda.time.LocalDateTime;
public class PrintDate {
public static void main(String[] args) {
DateTime dt = new LocalDateTime();
System.out.println(dt.toString());
}
}
"""
),
}
)

# First test whether the specific import mapping for org.joda.time.DateTime takes effect over the recursive
# mapping.
lib1 = rule_runner.get_target(
Address("", target_name="lib", relative_file_path="PrintDate.java")
)
assert rule_runner.request(Addresses, [DependenciesRequest(lib1[Dependencies])]) == Addresses(
[Address("", target_name="joda-time_joda-time-2")]
)

# Then try a file which should not match the specific import mapping and which will then match the
# recursive mapping.
lib2 = rule_runner.get_target(
Address("", target_name="lib", relative_file_path="PrintDate2.java")
)
assert rule_runner.request(Addresses, [DependenciesRequest(lib2[Dependencies])]) == Addresses(
[Address("", target_name="joda-time_joda-time")]
)
1 change: 1 addition & 0 deletions src/python/pants/backend/java/subsystems/java_infer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ def register_options(cls, register):
_default_package_mapping_url = git_url(
"src/python/pants/backend/java/dependency_inference/jvm_artifact_mappings.py"
)
# TODO: Move to `coursier` or a generic `jvm` subsystem.
register(
"--third-party-import-mapping",
type=dict,
Expand Down
15 changes: 14 additions & 1 deletion src/python/pants/backend/scala/dependency_inference/rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@
WrappedTarget,
)
from pants.engine.unions import UnionRule
from pants.jvm.dependency_inference import artifact_mapper
from pants.jvm.dependency_inference.artifact_mapper import (
AvailableThirdPartyArtifacts,
ThirdPartyPackageToArtifactMapping,
find_artifact_mapping,
)
from pants.jvm.dependency_inference.symbol_mapper import FirstPartySymbolMapping
from pants.util.ordered_set import OrderedSet

Expand All @@ -36,6 +42,8 @@ async def infer_scala_dependencies_via_source_analysis(
request: InferScalaSourceDependencies,
scala_infer_subsystem: ScalaInferSubsystem,
first_party_symbol_map: FirstPartySymbolMapping,
third_party_artifact_mapping: ThirdPartyPackageToArtifactMapping,
available_artifacts: AvailableThirdPartyArtifacts,
) -> InferredDependencies:
if not scala_infer_subsystem.imports:
return InferredDependencies([])
Expand All @@ -53,7 +61,11 @@ async def infer_scala_dependencies_via_source_analysis(

dependencies: OrderedSet[Address] = OrderedSet()
for symbol in symbols:
matches = first_party_symbol_map.symbols.addresses_for_symbol(symbol)
first_party_matches = first_party_symbol_map.symbols.addresses_for_symbol(symbol)
third_party_matches = find_artifact_mapping(
symbol, third_party_artifact_mapping, available_artifacts
)
matches = first_party_matches.union(third_party_matches)
if not matches:
continue

Expand All @@ -74,6 +86,7 @@ async def infer_scala_dependencies_via_source_analysis(
def rules():
return [
*collect_rules(),
*artifact_mapper.rules(),
*scala_parser.rules(),
*symbol_mapper.rules(),
UnionRule(InferDependenciesRequest, InferScalaSourceDependencies),
Expand Down
4 changes: 4 additions & 0 deletions src/python/pants/jvm/dependency_inference/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,7 @@
# Licensed under the Apache License, Version 2.0 (see LICENSE).

python_sources()

python_tests(
name="tests",
)
Loading

0 comments on commit 2457d5b

Please sign in to comment.