Skip to content

Commit

Permalink
Extend dependencies goal with output format to support JSON (pantsbui…
Browse files Browse the repository at this point in the history 
…ld#20443)

Add `--format` option to the `dependencies` goal to be able to emit
dependencies in JSON format (think adjacency list). See [Exporting
project's dependency
graph](pantsbuild#20242) to learn
about the use cases.
  • Loading branch information
@AlexTereshenkov
AlexTereshenkov authored Jan 23, 2024
1 parent fb615a4 commit 98a1f73
Show file tree
Hide file tree
Showing 2 changed files with 266 additions and 20 deletions.
103 changes: 96 additions & 7 deletions src/python/pants/backend/project_info/dependencies.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import itertools
import json
from enum import Enum

from pants.engine.addresses import Addresses
from pants.engine.console import Console
Expand All @@ -16,7 +17,18 @@
TransitiveTargetsRequest,
UnexpandedTargets,
)
from pants.option.option_types import BoolOption
from pants.option.option_types import BoolOption, EnumOption


class DependenciesOutputFormat(Enum):
"""Output format for listing dependencies.
text: List all dependencies as a single list of targets in plain text.
json: List all dependencies as a mapping `{target: [dependencies]}`.
"""

text = "text"
json = "json"


class DependenciesSubsystem(LineOriented, GoalSubsystem):
Expand All @@ -29,7 +41,12 @@ class DependenciesSubsystem(LineOriented, GoalSubsystem):
)
closed = BoolOption(
default=False,
help="Include the input targets in the output, along with the dependencies.",
help="Include the input targets in the output, along with the dependencies. This option "
"only applies when using the `text` format.",
)
format = EnumOption(
default=DependenciesOutputFormat.text,
help="Output format for listing dependencies.",
)


Expand All @@ -38,10 +55,63 @@ class Dependencies(Goal):
environment_behavior = Goal.EnvironmentBehavior.LOCAL_ONLY


@goal_rule
async def dependencies(
console: Console, addresses: Addresses, dependencies_subsystem: DependenciesSubsystem
) -> Dependencies:
async def list_dependencies_as_json(
addresses: Addresses, dependencies_subsystem: DependenciesSubsystem, console: Console
) -> None:
"""Get dependencies for given addresses and list them in the console in JSON.
Note that `--closed` option is ignored as it doesn't make sense to duplicate source address in
the list of its dependencies.
"""
# NB: We must preserve target generators for the roots, i.e. not replace with their
# generated targets.
target_roots = await Get(UnexpandedTargets, Addresses, addresses)
# NB: When determining dependencies, though, we replace target generators with their
# generated targets.
if dependencies_subsystem.transitive:
transitive_targets_group = await MultiGet(
Get(
TransitiveTargets,
TransitiveTargetsRequest(
(address,), should_traverse_deps_predicate=AlwaysTraverseDeps()
),
)
for address in addresses
)

iterated_targets = []
for transitive_targets in transitive_targets_group:
iterated_targets.append(
sorted([str(tgt.address) for tgt in transitive_targets.dependencies])
)

else:
dependencies_per_target_root = await MultiGet(
Get(
Targets,
DependenciesRequest(
tgt.get(DependenciesField),
should_traverse_deps_predicate=AlwaysTraverseDeps(),
),
)
for tgt in target_roots
)

iterated_targets = []
for targets in dependencies_per_target_root:
iterated_targets.append(sorted([str(tgt.address) for tgt in targets]))

# The assumption is that when iterating the targets and sending dependency requests
# for them, the lists of dependencies are returned in the very same order.
mapping = dict(zip([str(tgt.address) for tgt in target_roots], iterated_targets))
output = json.dumps(mapping, indent=4)
console.print_stdout(output)


async def list_dependencies_as_plain_text(
addresses: Addresses, dependencies_subsystem: DependenciesSubsystem, console: Console
) -> None:
"""Get dependencies for given addresses and list them in the console as a single list."""
if dependencies_subsystem.transitive:
transitive_targets = await Get(
TransitiveTargets,
Expand Down Expand Up @@ -76,6 +146,25 @@ async def dependencies(
for address in sorted(address_strings):
print_stdout(address)


@goal_rule
async def dependencies(
console: Console, addresses: Addresses, dependencies_subsystem: DependenciesSubsystem
) -> Dependencies:
if DependenciesOutputFormat.text == dependencies_subsystem.format:
await list_dependencies_as_plain_text(
addresses=addresses,
dependencies_subsystem=dependencies_subsystem,
console=console,
)

elif DependenciesOutputFormat.json == dependencies_subsystem.format:
await list_dependencies_as_json(
addresses=addresses,
dependencies_subsystem=dependencies_subsystem,
console=console,
)

return Dependencies(exit_code=0)


Expand Down
183 changes: 170 additions & 13 deletions src/python/pants/backend/project_info/dependencies_test.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import json
from functools import partial
from textwrap import dedent
from typing import List, Optional

import pytest

from pants.backend.project_info.dependencies import Dependencies, rules
from pants.backend.project_info.dependencies import Dependencies, DependenciesOutputFormat, rules
from pants.backend.python import target_types_rules
from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget
from pants.engine.target import SpecialCasedDependencies, Target
Expand Down Expand Up @@ -65,23 +65,43 @@ def create_python_requirement_tgts(rule_runner: PythonRuleRunner, *names: str) -
)


def create_targets(rule_runner: PythonRuleRunner) -> None:
"""Create necessary targets used in tests before querying the graph for dependencies."""
create_python_requirement_tgts(rule_runner, "req1", "req2")
create_python_sources(rule_runner, "dep/target")
create_python_sources(
rule_runner, "some/target", dependencies=["dep/target", "3rdparty/python:req1"]
)
create_python_sources(
rule_runner,
"some/other/target",
dependencies=["some/target", "3rdparty/python:req2"],
)


def assert_dependencies(
rule_runner: PythonRuleRunner,
*,
specs: List[str],
expected: List[str],
transitive: bool = False,
closed: bool = False,
output_format: DependenciesOutputFormat = DependenciesOutputFormat.text,
) -> None:
args = []
if transitive:
args.append("--transitive")
if closed:
args.append("--closed")
args.append(f"--format={output_format.value}")

result = rule_runner.run_goal_rule(
Dependencies, args=[*args, *specs], env_inherit={"PATH", "PYENV_ROOT", "HOME"}
)
assert result.stdout.splitlines() == expected
if output_format == DependenciesOutputFormat.text:
assert result.stdout.splitlines() == expected
elif output_format == DependenciesOutputFormat.json:
assert json.loads(result.stdout) == expected


def test_no_target(rule_runner: PythonRuleRunner) -> None:
Expand Down Expand Up @@ -122,19 +142,13 @@ def test_special_cased_dependencies(rule_runner: PythonRuleRunner) -> None:


def test_python_dependencies(rule_runner: PythonRuleRunner) -> None:
create_python_requirement_tgts(rule_runner, "req1", "req2")
create_python_sources(rule_runner, "dep/target")
create_python_sources(
rule_runner, "some/target", dependencies=["dep/target", "3rdparty/python:req1"]
)
create_python_sources(
create_targets(rule_runner)
assert_deps = partial(
assert_dependencies,
rule_runner,
"some/other/target",
dependencies=["some/target", "3rdparty/python:req2"],
output_format=DependenciesOutputFormat.text,
)

assert_deps = partial(assert_dependencies, rule_runner)

assert_deps(
specs=["some/other/target:target"],
transitive=False,
Expand Down Expand Up @@ -209,3 +223,146 @@ def test_python_dependencies(rule_runner: PythonRuleRunner) -> None:
],
closed=True,
)


def test_python_dependencies_output_format_json_direct_deps(rule_runner: PythonRuleRunner) -> None:
create_targets(rule_runner)
assert_deps = partial(
assert_dependencies,
rule_runner,
output_format=DependenciesOutputFormat.json,
)

# input: single module
assert_deps(
specs=["some/target/a.py"],
transitive=False,
expected={
"some/target/a.py": [
"3rdparty/python:req1",
"dep/target/a.py",
]
},
)

# input: multiple modules
assert_deps(
specs=["some/target/a.py", "some/other/target/a.py"],
transitive=False,
expected={
"some/target/a.py": [
"3rdparty/python:req1",
"dep/target/a.py",
],
"some/other/target/a.py": [
"3rdparty/python:req2",
"some/target/a.py",
],
},
)

# input: directory, recursively
assert_deps(
specs=["some::"],
transitive=False,
expected={
"some/target:target": [
"some/target/a.py",
],
"some/target/a.py": [
"3rdparty/python:req1",
"dep/target/a.py",
],
"some/other/target:target": [
"some/other/target/a.py",
],
"some/other/target/a.py": [
"3rdparty/python:req2",
"some/target/a.py",
],
},
)
assert_deps(
specs=["some/other/target:target"],
transitive=True,
expected={
"some/other/target:target": [
"3rdparty/python:req1",
"3rdparty/python:req2",
"dep/target/a.py",
"some/other/target/a.py",
"some/target/a.py",
]
},
)


def test_python_dependencies_output_format_json_transitive_deps(
rule_runner: PythonRuleRunner,
) -> None:
create_targets(rule_runner)
assert_deps = partial(
assert_dependencies,
rule_runner,
output_format=DependenciesOutputFormat.json,
)

# input: single module
assert_deps(
specs=["some/target/a.py"],
transitive=True,
expected={
"some/target/a.py": [
"3rdparty/python:req1",
"dep/target/a.py",
]
},
)

# input: multiple modules
assert_deps(
specs=["some/target/a.py", "some/other/target/a.py"],
transitive=True,
expected={
"some/target/a.py": [
"3rdparty/python:req1",
"dep/target/a.py",
],
"some/other/target/a.py": [
"3rdparty/python:req1",
"3rdparty/python:req2",
"dep/target/a.py",
"some/target/a.py",
],
},
)

# input: directory, recursively
assert_deps(
specs=["some::"],
transitive=True,
expected={
"some/target:target": [
"3rdparty/python:req1",
"dep/target/a.py",
"some/target/a.py",
],
"some/target/a.py": [
"3rdparty/python:req1",
"dep/target/a.py",
],
"some/other/target:target": [
"3rdparty/python:req1",
"3rdparty/python:req2",
"dep/target/a.py",
"some/other/target/a.py",
"some/target/a.py",
],
"some/other/target/a.py": [
"3rdparty/python:req1",
"3rdparty/python:req2",
"dep/target/a.py",
"some/target/a.py",
],
},
)

0 comments on commit 98a1f73

Please sign in to comment.