diff --git a/build-support/bin/generate_docs.py b/build-support/bin/generate_docs.py index 99cbbe1076c..2b55df673a7 100644 --- a/build-support/bin/generate_docs.py +++ b/build-support/bin/generate_docs.py @@ -203,6 +203,8 @@ def run_pants_help_all() -> dict[str, Any]: "internal_plugins.releases", "pants.backend.experimental.java", "pants.backend.experimental.java.debug_goals", + "pants.backend.experimental.scala", + "pants.backend.experimental.scala.debug_goals", ] activated_backends = [ "pants.backend.codegen.protobuf.python", diff --git a/build-support/githooks/pre-commit b/build-support/githooks/pre-commit index a73e0056f29..98848e2b051 100755 --- a/build-support/githooks/pre-commit +++ b/build-support/githooks/pre-commit @@ -20,7 +20,7 @@ if git diff "${MERGE_BASE}" --name-only | grep '\.rs$' > /dev/null; then ./build-support/bin/check_rust_pre_commit.sh fi -echo "* Checking MyPy" +echo "* Typechecking" ./pants --changed-since="${MERGE_BASE}" --changed-dependees=transitive check echo "* Checking linters, formatters, and headers" diff --git a/src/python/pants/backend/experimental/java/register.py b/src/python/pants/backend/experimental/java/register.py index 7fa9a5c1fe4..d6764b44ed2 100644 --- a/src/python/pants/backend/experimental/java/register.py +++ b/src/python/pants/backend/experimental/java/register.py @@ -2,12 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pants.backend.java.compile import javac -from pants.backend.java.dependency_inference import ( - import_parser, - java_parser, - java_parser_launcher, - package_mapper, -) +from pants.backend.java.dependency_inference import import_parser, java_parser, java_parser_launcher from pants.backend.java.dependency_inference import rules as dependency_inference_rules from pants.backend.java.goals import check, tailor from pants.backend.java.package import deploy_jar @@ -22,6 +17,7 @@ from pants.backend.java.test import junit from pants.jvm import classpath, jdk_rules from pants.jvm import util_rules as jvm_util_rules +from pants.jvm.dependency_inference import symbol_mapper from pants.jvm.goals import coursier from pants.jvm.resolve import coursier_fetch, coursier_setup from pants.jvm.target_types import JvmArtifact, JvmDependencyLockfile @@ -52,7 +48,7 @@ def rules(): *import_parser.rules(), *java_parser.rules(), *java_parser_launcher.rules(), - *package_mapper.rules(), + *symbol_mapper.rules(), *dependency_inference_rules.rules(), *tailor.rules(), *jvm_util_rules.rules(), diff --git a/src/python/pants/backend/java/dependency_inference/package_mapper.py b/src/python/pants/backend/java/dependency_inference/package_mapper.py deleted file mode 100644 index 0d45eb4d0c4..00000000000 --- a/src/python/pants/backend/java/dependency_inference/package_mapper.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import annotations - -import logging -from dataclasses import dataclass - -from pants.backend.java.dependency_inference.package_prefix_tree import PackageRootedDependencyMap -from pants.backend.java.dependency_inference.types import JavaSourceDependencyAnalysis -from pants.backend.java.target_types import JavaSourceField -from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest -from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.engine.target import AllTargets, Targets -from pants.engine.unions import UnionMembership, UnionRule, union -from pants.util.logging import LogLevel - -logger = logging.getLogger(__name__) - - -# ----------------------------------------------------------------------------------------------- -# First-party package mapping -# ----------------------------------------------------------------------------------------------- - - -# TODO: add third-party targets here? That would allow us to avoid iterating over AllTargets twice. -# See `backend/python/dependency_inference/module_mapper.py` for an example. -class AllJavaTargets(Targets): - pass - - -@rule(desc="Find all Java targets in project", level=LogLevel.DEBUG) -def find_all_java_targets(tgts: AllTargets) -> AllJavaTargets: - return AllJavaTargets(tgt for tgt in tgts if tgt.has_field(JavaSourceField)) - - -@dataclass(frozen=True) -class FirstPartyJavaMappingImpl: - """A mapping of package names to owning addresses that a specific implementation adds for Java - import dependency inference.""" - - package_rooted_dependency_map: PackageRootedDependencyMap - - -@union -class FirstPartyJavaMappingImplMarker: - """An entry point for a specific implementation of mapping package names to owning targets for - Java import dependency inference. - - All implementations will be merged together. - - The addresses should all be file addresses, rather than BUILD addresses. - """ - - -@dataclass(frozen=True) -class FirstPartyJavaPackageMapping: - """A merged mapping of package names to owning addresses.""" - - package_rooted_dependency_map: PackageRootedDependencyMap - - -@rule(level=LogLevel.DEBUG) -async def merge_first_party_module_mappings( - union_membership: UnionMembership, -) -> FirstPartyJavaPackageMapping: - all_mappings = await MultiGet( - Get( - FirstPartyJavaMappingImpl, - FirstPartyJavaMappingImplMarker, - marker_cls(), - ) - for marker_cls in union_membership.get(FirstPartyJavaMappingImplMarker) - ) - - merged_dep_map = PackageRootedDependencyMap() - for dep_map in all_mappings: - merged_dep_map.merge(dep_map.package_rooted_dependency_map) - - return FirstPartyJavaPackageMapping(package_rooted_dependency_map=merged_dep_map) - - -# This is only used to register our implementation with the plugin hook via unions. Note that we -# implement this like any other plugin implementation so that we can run them all in parallel. -class FirstPartyJavaTargetsMappingMarker(FirstPartyJavaMappingImplMarker): - pass - - -@rule(desc="Map all first party Java targets to their packages", level=LogLevel.DEBUG) -async def map_first_party_java_targets_to_symbols( - _: FirstPartyJavaTargetsMappingMarker, java_targets: AllJavaTargets -) -> FirstPartyJavaMappingImpl: - source_files = await MultiGet( - Get(SourceFiles, SourceFilesRequest([target[JavaSourceField]])) for target in java_targets - ) - source_analysis = await MultiGet( - Get(JavaSourceDependencyAnalysis, SourceFiles, source_files) - for source_files in source_files - ) - address_and_analysis = zip([t.address for t in java_targets], source_analysis) - - dep_map = PackageRootedDependencyMap() - for address, analysis in address_and_analysis: - for top_level_type in analysis.top_level_types: - dep_map.add_top_level_type(top_level_type, address=address) - - return FirstPartyJavaMappingImpl(package_rooted_dependency_map=dep_map) - - -def rules(): - return ( - *collect_rules(), - UnionRule(FirstPartyJavaMappingImplMarker, FirstPartyJavaTargetsMappingMarker), - ) diff --git a/src/python/pants/backend/java/dependency_inference/package_prefix_tree.py b/src/python/pants/backend/java/dependency_inference/package_prefix_tree.py deleted file mode 100644 index c6d682247b3..00000000000 --- a/src/python/pants/backend/java/dependency_inference/package_prefix_tree.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import annotations - -from collections import defaultdict - -from pants.engine.addresses import Address - - -class PackageRootedDependencyMap: - """A utility class for mapping Java FQTs to owning source Addresses. - - Keep tracks of which Address provides a fully qualified symbol. - """ - - def __init__(self): - self._type_map: dict[str, set[Address]] = defaultdict(set) - - def add_top_level_type(self, type_: str, address: Address): - """Declare a single Address as a provider of a top level type.""" - self._type_map[type_].add(address) - - def addresses_for_type(self, symbol: str) -> frozenset[Address]: - """Returns the set of addresses that provide the passed type. - - `symbol` should be a fully qualified Java type (FQT) (e.g. `foo.bar.Thing`). - """ - return frozenset(self._type_map[symbol]) - - def merge(self, other: PackageRootedDependencyMap): - """Merge 'other' into this dependency map.""" - - for type_, addresses in other._type_map.items(): - self._type_map[type_] |= addresses - - def to_json_dict(self): - return { - "type_map": {ty: [str(addr) for addr in addrs] for ty, addrs in self._type_map.items()}, - } - - def __repr__(self) -> str: - type_map = ", ".join( - f"{ty}:{', '.join(str(addr) for addr in addrs)}" for ty, addrs in self._type_map.items() - ) - return f"PackageRootedDependencyMap(type_map={type_map})" diff --git a/src/python/pants/backend/java/dependency_inference/package_prefix_tree_test.py b/src/python/pants/backend/java/dependency_inference/package_prefix_tree_test.py deleted file mode 100644 index 831c6d096b8..00000000000 --- a/src/python/pants/backend/java/dependency_inference/package_prefix_tree_test.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import annotations - -from pants.backend.java.dependency_inference.package_prefix_tree import PackageRootedDependencyMap -from pants.engine.addresses import Address - - -def test_package_rooted_dependency_map() -> None: - dep_map = PackageRootedDependencyMap() - - a = Address("a") - dep_map.add_top_level_type(type_="org.pantsbuild.Foo", address=a) - # An exact match yields the exact matching address - assert dep_map.addresses_for_type("org.pantsbuild.Foo") == frozenset([a]) - # A miss returns nothing - assert dep_map.addresses_for_type("org.Foo") == frozenset() - assert dep_map.addresses_for_type("org.other.Foo") == frozenset() - - b = Address("b") - dep_map.add_top_level_type(type_="org.pantsbuild.Baz", address=b) - # Again, exact matches yield exact providers - assert dep_map.addresses_for_type("org.pantsbuild.Foo") == frozenset([a]) - assert dep_map.addresses_for_type("org.pantsbuild.Baz") == frozenset([b]) - # Misses result in nothing. - assert dep_map.addresses_for_type("org.Foo") == frozenset() - assert dep_map.addresses_for_type("org.other.Foo") == frozenset() diff --git a/src/python/pants/backend/java/dependency_inference/rules.py b/src/python/pants/backend/java/dependency_inference/rules.py index ded6e29152a..213d384d82f 100644 --- a/src/python/pants/backend/java/dependency_inference/rules.py +++ b/src/python/pants/backend/java/dependency_inference/rules.py @@ -4,18 +4,7 @@ import logging -from pants.backend.java.dependency_inference import ( - artifact_mapper, - import_parser, - java_parser, - package_mapper, -) -from pants.backend.java.dependency_inference.artifact_mapper import ( - AvailableThirdPartyArtifacts, - ThirdPartyJavaPackageToArtifactMapping, - find_artifact_mapping, -) -from pants.backend.java.dependency_inference.package_mapper import FirstPartyJavaPackageMapping +from pants.backend.java.dependency_inference import import_parser, java_parser, symbol_mapper from pants.backend.java.dependency_inference.types import JavaSourceDependencyAnalysis from pants.backend.java.subsystems.java_infer import JavaInferSubsystem from pants.backend.java.target_types import JavaSourceField @@ -32,6 +21,13 @@ WrappedTarget, ) from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference import artifact_mapper +from pants.jvm.dependency_inference.artifact_mapper import ( + AvailableThirdPartyArtifacts, + ThirdPartyPackageToArtifactMapping, + find_artifact_mapping, +) +from pants.jvm.dependency_inference.symbol_mapper import FirstPartySymbolMapping from pants.util.ordered_set import FrozenOrderedSet, OrderedSet logger = logging.getLogger(__name__) @@ -45,8 +41,8 @@ class InferJavaSourceDependencies(InferDependenciesRequest): async def infer_java_dependencies_via_imports( request: InferJavaSourceDependencies, java_infer_subsystem: JavaInferSubsystem, - first_party_dep_map: FirstPartyJavaPackageMapping, - third_party_artifact_mapping: ThirdPartyJavaPackageToArtifactMapping, + first_party_dep_map: FirstPartySymbolMapping, + third_party_artifact_mapping: ThirdPartyPackageToArtifactMapping, available_artifacts: AvailableThirdPartyArtifacts, ) -> InferredDependencies: if ( @@ -79,12 +75,12 @@ async def infer_java_dependencies_via_imports( types.update(maybe_qualify_types) - dep_map = first_party_dep_map.package_rooted_dependency_map + dep_map = first_party_dep_map.symbols dependencies: OrderedSet[Address] = OrderedSet() for typ in types: - first_party_matches = dep_map.addresses_for_type(typ) + first_party_matches = dep_map.addresses_for_symbol(typ) third_party_matches: FrozenOrderedSet[Address] = FrozenOrderedSet() if java_infer_subsystem.third_party_imports: third_party_matches = find_artifact_mapping( @@ -120,7 +116,7 @@ def rules(): *artifact_mapper.rules(), *java_parser.rules(), *import_parser.rules(), - *package_mapper.rules(), + *symbol_mapper.rules(), *source_files_rules(), UnionRule(InferDependenciesRequest, InferJavaSourceDependencies), ] diff --git a/src/python/pants/backend/java/dependency_inference/rules_test.py b/src/python/pants/backend/java/dependency_inference/rules_test.py index 3c498cf415c..556df613fac 100644 --- a/src/python/pants/backend/java/dependency_inference/rules_test.py +++ b/src/python/pants/backend/java/dependency_inference/rules_test.py @@ -6,11 +6,6 @@ import pytest from pants.backend.java.compile.javac import rules as javac_rules -from pants.backend.java.dependency_inference.artifact_mapper import ( - FrozenTrieNode, - ThirdPartyJavaPackageToArtifactMapping, - UnversionedCoordinate, -) from pants.backend.java.dependency_inference.java_parser import rules as java_parser_rules from pants.backend.java.dependency_inference.java_parser_launcher import ( rules as java_parser_launcher_rules, @@ -34,6 +29,11 @@ InferredDependencies, Targets, ) +from pants.jvm.dependency_inference.artifact_mapper import ( + FrozenTrieNode, + ThirdPartyPackageToArtifactMapping, + UnversionedCoordinate, +) from pants.jvm.jdk_rules import rules as java_util_rules from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules @@ -68,7 +68,7 @@ def rule_runner() -> RuleRunner: QueryRule(ExplicitlyProvidedDependencies, [DependenciesRequest]), QueryRule(InferredDependencies, [InferJavaSourceDependencies]), QueryRule(Targets, [UnparsedAddressInputs]), - QueryRule(ThirdPartyJavaPackageToArtifactMapping, []), + QueryRule(ThirdPartyPackageToArtifactMapping, []), ], target_types=[JavaSourcesGeneratorTarget, JunitTestsGeneratorTarget, JvmArtifact], ) @@ -568,7 +568,7 @@ def test_third_party_mapping_parsing(rule_runner: RuleRunner) -> None: ], env_inherit=PYTHON_BOOTSTRAP_ENV, ) - mapping = rule_runner.request(ThirdPartyJavaPackageToArtifactMapping, []) + mapping = rule_runner.request(ThirdPartyPackageToArtifactMapping, []) root_node = mapping.mapping_root # Handy trie traversal function to placate mypy diff --git a/src/python/pants/backend/java/dependency_inference/symbol_mapper.py b/src/python/pants/backend/java/dependency_inference/symbol_mapper.py new file mode 100644 index 00000000000..3c7c81a7f7d --- /dev/null +++ b/src/python/pants/backend/java/dependency_inference/symbol_mapper.py @@ -0,0 +1,57 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import logging + +from pants.backend.java.dependency_inference.types import JavaSourceDependencyAnalysis +from pants.backend.java.target_types import JavaSourceField +from pants.core.util_rules.source_files import SourceFilesRequest +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import AllTargets, Targets +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference import symbol_mapper +from pants.jvm.dependency_inference.symbol_mapper import FirstPartyMappingRequest, SymbolMap +from pants.util.logging import LogLevel + +logger = logging.getLogger(__name__) + + +class AllJavaTargets(Targets): + pass + + +@rule(desc="Find all Java targets in project", level=LogLevel.DEBUG) +def find_all_java_targets(tgts: AllTargets) -> AllJavaTargets: + return AllJavaTargets(tgt for tgt in tgts if tgt.has_field(JavaSourceField)) + + +class FirstPartyJavaTargetsMappingRequest(FirstPartyMappingRequest): + pass + + +@rule(desc="Map all first party Java targets to their packages", level=LogLevel.DEBUG) +async def map_first_party_java_targets_to_symbols( + _: FirstPartyJavaTargetsMappingRequest, java_targets: AllJavaTargets +) -> SymbolMap: + source_analysis = await MultiGet( + Get(JavaSourceDependencyAnalysis, SourceFilesRequest([target[JavaSourceField]])) + for target in java_targets + ) + address_and_analysis = zip([t.address for t in java_targets], source_analysis) + + dep_map = SymbolMap() + for address, analysis in address_and_analysis: + for top_level_type in analysis.top_level_types: + dep_map.add_symbol(top_level_type, address=address) + + return dep_map + + +def rules(): + return ( + *collect_rules(), + *symbol_mapper.rules(), + UnionRule(FirstPartyMappingRequest, FirstPartyJavaTargetsMappingRequest), + ) diff --git a/src/python/pants/backend/java/goals/debug_goals.py b/src/python/pants/backend/java/goals/debug_goals.py index 5ba680970a5..04a332015b5 100644 --- a/src/python/pants/backend/java/goals/debug_goals.py +++ b/src/python/pants/backend/java/goals/debug_goals.py @@ -4,7 +4,6 @@ import json from pants.backend.experimental.java.register import rules as java_rules -from pants.backend.java.dependency_inference.package_mapper import FirstPartyJavaPackageMapping from pants.backend.java.dependency_inference.types import JavaSourceDependencyAnalysis from pants.backend.java.target_types import JavaFieldSet from pants.core.util_rules.source_files import SourceFilesRequest @@ -13,6 +12,7 @@ from pants.engine.internals.selectors import Get, MultiGet from pants.engine.rules import collect_rules, goal_rule from pants.engine.target import Targets +from pants.jvm.dependency_inference.symbol_mapper import FirstPartySymbolMapping class DumpFirstPartyDepMapSubsystem(GoalSubsystem): @@ -26,11 +26,9 @@ class DumpFirstPartyDepMap(Goal): @goal_rule async def dump_dep_inference_data( - console: Console, first_party_dep_map: FirstPartyJavaPackageMapping + console: Console, first_party_dep_map: FirstPartySymbolMapping ) -> DumpFirstPartyDepMap: - console.write_stdout( - json.dumps(first_party_dep_map.package_rooted_dependency_map.to_json_dict()) - ) + console.write_stdout(json.dumps(first_party_dep_map.symbols.to_json_dict())) return DumpFirstPartyDepMap(exit_code=0) diff --git a/src/python/pants/backend/scala/dependency_inference/rules.py b/src/python/pants/backend/scala/dependency_inference/rules.py index 63a1d8ff4f1..22fb4c3125a 100644 --- a/src/python/pants/backend/scala/dependency_inference/rules.py +++ b/src/python/pants/backend/scala/dependency_inference/rules.py @@ -6,7 +6,6 @@ from pants.backend.scala.dependency_inference import scala_parser, symbol_mapper from pants.backend.scala.dependency_inference.scala_parser import ScalaSourceDependencyAnalysis -from pants.backend.scala.dependency_inference.symbol_mapper import FirstPartyScalaSymbolMapping from pants.backend.scala.subsystems.scala_infer import ScalaInferSubsystem from pants.backend.scala.target_types import ScalaSourceField from pants.build_graph.address import Address @@ -21,6 +20,8 @@ InferredDependencies, WrappedTarget, ) +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference.symbol_mapper import FirstPartySymbolMapping from pants.util.ordered_set import OrderedSet logger = logging.getLogger(__name__) @@ -34,7 +35,7 @@ class InferScalaSourceDependencies(InferDependenciesRequest): async def infer_scala_dependencies_via_source_analysis( request: InferScalaSourceDependencies, scala_infer_subsystem: ScalaInferSubsystem, - first_party_symbol_map: FirstPartyScalaSymbolMapping, + first_party_symbol_map: FirstPartySymbolMapping, ) -> InferredDependencies: if not scala_infer_subsystem.imports: return InferredDependencies([]) @@ -75,4 +76,5 @@ def rules(): *collect_rules(), *scala_parser.rules(), *symbol_mapper.rules(), + UnionRule(InferDependenciesRequest, InferScalaSourceDependencies), ] diff --git a/src/python/pants/backend/scala/dependency_inference/scala_parser.py b/src/python/pants/backend/scala/dependency_inference/scala_parser.py index bc363ba206c..c818508ba2d 100644 --- a/src/python/pants/backend/scala/dependency_inference/scala_parser.py +++ b/src/python/pants/backend/scala/dependency_inference/scala_parser.py @@ -135,7 +135,7 @@ def from_json_dict(cls, d: dict) -> ScalaSourceDependencyAnalysis: def to_debug_json_dict(self) -> dict[str, Any]: return { - "provided_names": self.provided_names, + "provided_names": list(self.provided_names), "imports_by_scope": { key: [v.to_debug_json_dict() for v in values] for key, values in self.imports_by_scope.items() diff --git a/src/python/pants/backend/scala/dependency_inference/symbol_mapper.py b/src/python/pants/backend/scala/dependency_inference/symbol_mapper.py index 69bc6c3d277..56af34dbef5 100644 --- a/src/python/pants/backend/scala/dependency_inference/symbol_mapper.py +++ b/src/python/pants/backend/scala/dependency_inference/symbol_mapper.py @@ -2,67 +2,28 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations -from collections import defaultdict -from dataclasses import dataclass - from pants.backend.scala.dependency_inference.scala_parser import ScalaSourceDependencyAnalysis from pants.backend.scala.target_types import ScalaSourceField -from pants.build_graph.address import Address -from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest +from pants.core.util_rules.source_files import SourceFilesRequest from pants.engine.internals.selectors import Get, MultiGet from pants.engine.rules import collect_rules, rule from pants.engine.target import AllTargets, Targets +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference import symbol_mapper +from pants.jvm.dependency_inference.symbol_mapper import FirstPartyMappingRequest, SymbolMap from pants.util.logging import LogLevel -class SymbolMap: - def __init__(self): - self._symbol_map: dict[str, set[Address]] = defaultdict(set) - - def add_symbol(self, symbol: str, address: Address): - """Declare a single Address as a provider of a symbol.""" - self._symbol_map[symbol].add(address) - - def addresses_for_symbol(self, symbol: str) -> frozenset[Address]: - """Returns the set of addresses that provide the passed symbol. - - :param symbol: a fully-qualified Scala symbol (e.g. `foo.bar.Thing`). - """ - return frozenset(self._symbol_map[symbol]) - - def merge(self, other: SymbolMap) -> None: - """Merge 'other' into this dependency map.""" - for symbol, addresses in other._symbol_map.items(): - self._symbol_map[symbol] |= addresses - - def to_json_dict(self): - return { - "symbol_map": { - sym: [str(addr) for addr in addrs] for sym, addrs in self._symbol_map.items() - }, - } - - def __repr__(self) -> str: - symbol_map = ", ".join( - f"{ty}:{', '.join(str(addr) for addr in addrs)}" - for ty, addrs in self._symbol_map.items() - ) - return f"SymbolMap(symbol_map={symbol_map})" - - -@dataclass(frozen=True) -class FirstPartyScalaSymbolMapping: - """A merged mapping of symbol names to owning addresses for Scala code.""" - - symbols: SymbolMap - - # TODO: add third-party targets here? That would allow us to avoid iterating over AllTargets twice. # See `backend/python/dependency_inference/module_mapper.py` for an example. class AllScalaTargets(Targets): pass +class FirstPartyScalaTargetsMappingRequest(FirstPartyMappingRequest): + pass + + @rule(desc="Find all Scala targets in project", level=LogLevel.DEBUG) def find_all_java_targets(targets: AllTargets) -> AllScalaTargets: return AllScalaTargets(tgt for tgt in targets if tgt.has_field(ScalaSourceField)) @@ -70,14 +31,12 @@ def find_all_java_targets(targets: AllTargets) -> AllScalaTargets: @rule(desc="Map all first party Scala targets to their symbols", level=LogLevel.DEBUG) async def map_first_party_scala_targets_to_symbols( + _: FirstPartyScalaTargetsMappingRequest, scala_targets: AllScalaTargets, -) -> FirstPartyScalaSymbolMapping: - source_files = await MultiGet( - Get(SourceFiles, SourceFilesRequest([target[ScalaSourceField]])) for target in scala_targets - ) +) -> SymbolMap: source_analysis = await MultiGet( - Get(ScalaSourceDependencyAnalysis, SourceFiles, source_files) - for source_files in source_files + Get(ScalaSourceDependencyAnalysis, SourceFilesRequest([target[ScalaSourceField]])) + for target in scala_targets ) address_and_analysis = zip([t.address for t in scala_targets], source_analysis) @@ -86,8 +45,12 @@ async def map_first_party_scala_targets_to_symbols( for symbol in analysis.provided_names: symbol_map.add_symbol(symbol, address) - return FirstPartyScalaSymbolMapping(symbol_map) + return symbol_map def rules(): - return collect_rules() + return ( + *collect_rules(), + *symbol_mapper.rules(), + UnionRule(FirstPartyMappingRequest, FirstPartyScalaTargetsMappingRequest), + ) diff --git a/src/python/pants/jvm/dependency_inference/BUILD b/src/python/pants/jvm/dependency_inference/BUILD new file mode 100644 index 00000000000..760486c9dd3 --- /dev/null +++ b/src/python/pants/jvm/dependency_inference/BUILD @@ -0,0 +1,4 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_sources() diff --git a/src/python/pants/backend/java/dependency_inference/artifact_mapper.py b/src/python/pants/jvm/dependency_inference/artifact_mapper.py similarity index 95% rename from src/python/pants/backend/java/dependency_inference/artifact_mapper.py rename to src/python/pants/jvm/dependency_inference/artifact_mapper.py index 687f4c1c3d8..f167bf69a5f 100644 --- a/src/python/pants/backend/java/dependency_inference/artifact_mapper.py +++ b/src/python/pants/jvm/dependency_inference/artifact_mapper.py @@ -6,11 +6,11 @@ from dataclasses import dataclass from typing import Any, Iterable, Set -from pants.backend.java.dependency_inference.jvm_artifact_mappings import JVM_ARTIFACT_MAPPINGS from pants.backend.java.subsystems.java_infer import JavaInferSubsystem from pants.build_graph.address import Address from pants.engine.rules import collect_rules, rule from pants.engine.target import AllTargets, Targets +from pants.jvm.dependency_inference.jvm_artifact_mappings import JVM_ARTIFACT_MAPPINGS from pants.jvm.target_types import JvmArtifactArtifactField, JvmArtifactGroupField from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel @@ -113,7 +113,7 @@ def find_all_jvm_artifact_targets(targets: AllTargets) -> AllJvmArtifactTargets: @dataclass(frozen=True) -class ThirdPartyJavaPackageToArtifactMapping: +class ThirdPartyPackageToArtifactMapping: mapping_root: FrozenTrieNode @@ -146,7 +146,7 @@ async def find_available_third_party_artifacts( @rule async def compute_java_third_party_artifact_mapping( java_infer_subsystem: JavaInferSubsystem, -) -> ThirdPartyJavaPackageToArtifactMapping: +) -> ThirdPartyPackageToArtifactMapping: def insert(mapping: MutableTrieNode, imp: str, coordinate: UnversionedCoordinate) -> None: imp_parts = imp.split(".") recursive = False @@ -170,12 +170,12 @@ def insert(mapping: MutableTrieNode, imp: str, coordinate: UnversionedCoordinate value = UnversionedCoordinate.from_coord_str(imp_action) insert(mapping, imp_name, value) - return ThirdPartyJavaPackageToArtifactMapping(FrozenTrieNode(mapping)) + return ThirdPartyPackageToArtifactMapping(FrozenTrieNode(mapping)) def find_artifact_mapping( import_name: str, - mapping: ThirdPartyJavaPackageToArtifactMapping, + mapping: ThirdPartyPackageToArtifactMapping, available_artifacts: AvailableThirdPartyArtifacts, ) -> FrozenOrderedSet[Address]: imp_parts = import_name.split(".") diff --git a/src/python/pants/backend/java/dependency_inference/jvm_artifact_mappings.py b/src/python/pants/jvm/dependency_inference/jvm_artifact_mappings.py similarity index 100% rename from src/python/pants/backend/java/dependency_inference/jvm_artifact_mappings.py rename to src/python/pants/jvm/dependency_inference/jvm_artifact_mappings.py diff --git a/src/python/pants/jvm/dependency_inference/symbol_mapper.py b/src/python/pants/jvm/dependency_inference/symbol_mapper.py new file mode 100644 index 00000000000..73372ce7923 --- /dev/null +++ b/src/python/pants/jvm/dependency_inference/symbol_mapper.py @@ -0,0 +1,99 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import logging +from collections import defaultdict +from dataclasses import dataclass + +from pants.build_graph.address import Address +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.unions import UnionMembership, union +from pants.util.logging import LogLevel + +logger = logging.getLogger(__name__) + + +# ----------------------------------------------------------------------------------------------- +# First-party package mapping +# ----------------------------------------------------------------------------------------------- + + +class SymbolMap: + """A mapping of JVM package names to owning addresses.""" + + def __init__(self): + self._symbol_map: dict[str, set[Address]] = defaultdict(set) + + def add_symbol(self, symbol: str, address: Address): + """Declare a single Address as a provider of a symbol.""" + self._symbol_map[symbol].add(address) + + def addresses_for_symbol(self, symbol: str) -> frozenset[Address]: + """Returns the set of addresses that provide the passed symbol. + + :param symbol: a fully-qualified JVM symbol (e.g. `foo.bar.Thing`). + """ + return frozenset(self._symbol_map[symbol]) + + def merge(self, other: SymbolMap) -> None: + """Merge 'other' into this dependency map.""" + for symbol, addresses in other._symbol_map.items(): + self._symbol_map[symbol] |= addresses + + def to_json_dict(self): + return { + "symbol_map": { + sym: [str(addr) for addr in addrs] for sym, addrs in self._symbol_map.items() + }, + } + + def __repr__(self) -> str: + symbol_map = ", ".join( + f"{ty}:{', '.join(str(addr) for addr in addrs)}" + for ty, addrs in self._symbol_map.items() + ) + return f"SymbolMap(symbol_map={symbol_map})" + + +@union +@dataclass(frozen=True) +class FirstPartyMappingRequest: + """An entry point for a specific implementation of mapping JVM package names to owning targets. + + All implementations will be merged together. + + The addresses should all be file addresses, rather than BUILD addresses. + """ + + +@dataclass(frozen=True) +class FirstPartySymbolMapping: + """A merged mapping of package names to owning addresses.""" + + symbols: SymbolMap + + +@rule(level=LogLevel.DEBUG) +async def merge_first_party_module_mappings( + union_membership: UnionMembership, +) -> FirstPartySymbolMapping: + all_mappings = await MultiGet( + Get( + SymbolMap, + FirstPartyMappingRequest, + marker_cls(), + ) + for marker_cls in union_membership.get(FirstPartyMappingRequest) + ) + + merged_dep_map = SymbolMap() + for dep_map in all_mappings: + merged_dep_map.merge(dep_map) + + return FirstPartySymbolMapping(merged_dep_map) + + +def rules(): + return collect_rules() diff --git a/testprojects/src/jvm/org/pantsbuild/example/app/BUILD b/testprojects/src/jvm/org/pantsbuild/example/app/BUILD index f5a773f5984..1132fb30b19 100644 --- a/testprojects/src/jvm/org/pantsbuild/example/app/BUILD +++ b/testprojects/src/jvm/org/pantsbuild/example/app/BUILD @@ -9,8 +9,4 @@ deploy_jar( scala_sources( compatible_resolves=["exampleapp"], - dependencies=[ - # TODO: Remove via https://github.com/pantsbuild/pants/issues/13365. - "testprojects/src/jvm/org/pantsbuild/example/lib/ExampleLib.java", - ], )