Skip to content

Commit

Permalink
Deprecate not implementing TargetGenerator in `GenerateTargetsReque…
Browse files Browse the repository at this point in the history 
…st` implementors. (pantsbuild#14959)

As described in pantsbuild#14844, we currently fall back gracefully if a `GenerateTargetsRequest.generate_from` field is not a subclass of `TargetGenerator`, but consuming the template which is automatically created via the `TargetGenerator.{copied_fields,moved_fields}` fields is important to allow `parametrize` to be used with a generator (since only `moved_fields` may be `parametrized`).

Deprecate not subclassing `TargetGenerator` for a generator target, and adjust implementations to consume the template. Fixes pantsbuild#14844.

[ci skip-build-wheels]
  • Loading branch information
@stuhood
stuhood authored Mar 30, 2022
1 parent ed83668 commit d45bbfa
Show file tree
Hide file tree
Showing 13 changed files with 146 additions and 139 deletions.
14 changes: 11 additions & 3 deletions src/python/pants/backend/docker/util_rules/dockerfile_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,12 @@
from pants.engine.fs import DigestContents, FileContent
from pants.engine.internals.scheduler import ExecutionError
from pants.engine.rules import rule
from pants.engine.target import GeneratedTargets, GenerateTargetsRequest, SourcesField, Target
from pants.engine.target import (
GeneratedTargets,
GenerateTargetsRequest,
SourcesField,
TargetGenerator,
)
from pants.engine.unions import UnionMembership, UnionRule
from pants.testutil.rule_runner import QueryRule, RuleRunner

Expand Down Expand Up @@ -102,9 +107,12 @@ def test_generate_dockerfile(rule_runner: RuleRunner) -> None:


def setup_target_generator(rule_runner_args: dict) -> None:
class GenerateOriginTarget(Target):
class GenerateOriginTarget(TargetGenerator):
alias = "docker_image_generator"
generated_target_cls = DockerImageTarget
core_fields = ()
copied_fields = ()
moved_fields = ()

class GenerateDockerImageTargetFromOrigin(GenerateTargetsRequest):
generate_from = GenerateOriginTarget
Expand All @@ -120,7 +128,7 @@ async def generate_docker_image_rule(
{
"instructions": DOCKERFILE.strip().split("\n"),
},
request.generator.address.create_generated("generated-image"),
request.template_address.create_generated("generated-image"),
union_membership,
)
],
Expand Down
2 changes: 1 addition & 1 deletion src/python/pants/backend/go/target_type_rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ async def generate_targets_from_go_mod(

def create_tgt(pkg_info: ThirdPartyPkgAnalysis) -> GoThirdPartyPackageTarget:
return GoThirdPartyPackageTarget(
{GoImportPathField.alias: pkg_info.import_path},
{**request.template, GoImportPathField.alias: pkg_info.import_path},
# E.g. `src/go:mod#github.com/google/uuid`.
generator_addr.create_generated(pkg_info.import_path),
union_membership,
Expand Down
98 changes: 51 additions & 47 deletions src/python/pants/backend/go/target_types.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,51 @@
MultipleSourcesField,
StringField,
Target,
TargetGenerator,
ValidNumbers,
)

# -----------------------------------------------------------------------------------------------
# `go_third_party_package` target
# -----------------------------------------------------------------------------------------------


class GoImportPathField(StringField):
alias = "import_path"
help = (
"Import path in Go code to import this package.\n\n"
"This field should not be overridden; use the value from target generation."
)
required = True
value: str


class GoThirdPartyPackageDependenciesField(Dependencies):
pass


class GoThirdPartyPackageTarget(Target):
alias = "go_third_party_package"
core_fields = (*COMMON_TARGET_FIELDS, GoThirdPartyPackageDependenciesField, GoImportPathField)
help = (
"A package from a third-party Go module.\n\n"
"You should not explicitly create this target in BUILD files. Instead, add a `go_mod` "
"target where you have your `go.mod` file, which will generate "
"`go_third_party_package` targets for you.\n\n"
"Make sure that your `go.mod` and `go.sum` files include this package's module."
)

def validate(self) -> None:
if not self.address.is_generated_target:
raise InvalidTargetException(
f"The `{self.alias}` target type should not be manually created in BUILD "
f"files, but it was created for {self.address}.\n\n"
"Instead, add a `go_mod` target where you have your `go.mod` file, which will "
f"generate `{self.alias}` targets for you based on the `require` directives in "
f"your `go.mod`."
)


# -----------------------------------------------------------------------------------------------
# `go_mod` target generator
# -----------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -69,20 +111,23 @@ class GoModDependenciesField(Dependencies):
alias = "_dependencies"


class GoModTarget(Target):
class GoModTarget(TargetGenerator):
alias = "go_mod"
core_fields = (
*COMMON_TARGET_FIELDS,
GoModDependenciesField,
GoModSourcesField,
)
help = (
"A first-party Go module (corresponding to a `go.mod` file).\n\n"
"Generates `go_third_party_package` targets based on the `require` directives in your "
"`go.mod`.\n\n"
"If you have third-party packages, make sure you have an up-to-date `go.sum`. Run "
"`go mod tidy` directly to update your `go.mod` and `go.sum`."
)
generated_target_cls = GoThirdPartyPackageTarget
core_fields = (
*COMMON_TARGET_FIELDS,
GoModDependenciesField,
GoModSourcesField,
)
copied_fields = COMMON_TARGET_FIELDS
moved_fields = ()


# -----------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -154,47 +199,6 @@ class GoPackageTarget(Target):
)


# -----------------------------------------------------------------------------------------------
# `go_third_party_package` target
# -----------------------------------------------------------------------------------------------


class GoImportPathField(StringField):
alias = "import_path"
help = (
"Import path in Go code to import this package.\n\n"
"This field should not be overridden; use the value from target generation."
)
required = True
value: str


class GoThirdPartyPackageDependenciesField(Dependencies):
pass


class GoThirdPartyPackageTarget(Target):
alias = "go_third_party_package"
core_fields = (*COMMON_TARGET_FIELDS, GoThirdPartyPackageDependenciesField, GoImportPathField)
help = (
"A package from a third-party Go module.\n\n"
"You should not explicitly create this target in BUILD files. Instead, add a `go_mod` "
"target where you have your `go.mod` file, which will generate "
"`go_third_party_package` targets for you.\n\n"
"Make sure that your `go.mod` and `go.sum` files include this package's module."
)

def validate(self) -> None:
if not self.address.is_generated_target:
raise InvalidTargetException(
f"The `{self.alias}` target type should not be manually created in BUILD "
f"files, but it was created for {self.address}.\n\n"
"Instead, add a `go_mod` target where you have your `go.mod` file, which will "
f"generate `{self.alias}` targets for you based on the `require` directives in "
f"your `go.mod`."
)


# -----------------------------------------------------------------------------------------------
# `go_binary` target
# -----------------------------------------------------------------------------------------------
Expand Down
12 changes: 7 additions & 5 deletions src/python/pants/backend/plugin_development/pants_requirements.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
BoolField,
GeneratedTargets,
GenerateTargetsRequest,
Target,
TargetGenerator,
)
from pants.engine.unions import UnionRule
from pants.version import MAJOR_MINOR, PANTS_SEMVER
Expand All @@ -25,7 +25,7 @@ class PantsRequirementsTestutilField(BoolField):
help = "If true, include `pantsbuild.pants.testutil` to write tests for your plugin."


class PantsRequirementsTargetGenerator(Target):
class PantsRequirementsTargetGenerator(TargetGenerator):
alias = "pants_requirements"
help = (
"Generate `python_requirement` targets for Pants itself to use with Pants plugins.\n\n"
Expand All @@ -42,11 +42,13 @@ class PantsRequirementsTargetGenerator(Target):
"also invite you to share your ideas at "
"https://github.com/pantsbuild/pants/issues/new/choose)"
)
generated_target_cls = PythonRequirementTarget
core_fields = (
*COMMON_TARGET_FIELDS,
PantsRequirementsTestutilField,
PythonRequirementResolveField,
)
copied_fields = COMMON_TARGET_FIELDS
moved_fields = (PythonRequirementResolveField,)


class GenerateFromPantsRequirementsRequest(GenerateTargetsRequest):
Expand Down Expand Up @@ -88,9 +90,9 @@ def create_tgt(dist: str, module: str) -> PythonRequirementTarget:
{
PythonRequirementsField.alias: (f"{dist}{version}",),
PythonRequirementModulesField.alias: (module,),
PythonRequirementResolveField.alias: generator[PythonRequirementResolveField].value,
**request.template,
},
generator.address.create_generated(dist),
request.template_address.create_generated(dist),
)

result = [create_tgt("pantsbuild.pants", "pants")]
Expand Down
26 changes: 10 additions & 16 deletions src/python/pants/backend/python/macros/pipenv_requirements.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
GenerateTargetsRequest,
InvalidFieldException,
SingleSourceField,
Target,
TargetGenerator,
)
from pants.engine.unions import UnionRule
from pants.util.logging import LogLevel
Expand All @@ -46,18 +46,20 @@ class PipenvSourceField(SingleSourceField):
required = False


class PipenvRequirementsTargetGenerator(Target):
class PipenvRequirementsTargetGenerator(TargetGenerator):
alias = "pipenv_requirements"
help = "Generate a `python_requirement` for each entry in `Pipenv.lock`."
generated_target_cls = PythonRequirementTarget
# Note that this does not have a `dependencies` field.
core_fields = (
*COMMON_TARGET_FIELDS,
ModuleMappingField,
TypeStubsModuleMappingField,
PipenvSourceField,
RequirementsOverrideField,
PythonRequirementResolveField,
)
copied_fields = COMMON_TARGET_FIELDS
moved_fields = (PythonRequirementResolveField,)


class GenerateFromPipenvRequirementsRequest(GenerateTargetsRequest):
Expand All @@ -81,8 +83,8 @@ async def generate_from_pipenv_requirement(
file_tgt = TargetGeneratorSourcesHelperTarget(
{TargetGeneratorSourcesHelperSourcesField.alias: [lock_rel_path]},
Address(
generator.address.spec_path,
target_name=generator.address.target_name,
request.template_address.spec_path,
target_name=request.template_address.target_name,
relative_file_path=lock_rel_path,
),
)
Expand All @@ -97,16 +99,8 @@ async def generate_from_pipenv_requirement(
)
lock_info = json.loads(digest_contents[0].content)

# Validate the resolve is legal.
generator[PythonRequirementResolveField].normalized_value(python_setup)

module_mapping = generator[ModuleMappingField].value
stubs_mapping = generator[TypeStubsModuleMappingField].value
inherited_fields = {
field.alias: field.value
for field in request.generator.field_values.values()
if isinstance(field, (*COMMON_TARGET_FIELDS, PythonRequirementResolveField))
}

def generate_tgt(raw_req: str, info: dict) -> PythonRequirementTarget:
if info.get("extras"):
Expand All @@ -125,7 +119,7 @@ def generate_tgt(raw_req: str, info: dict) -> PythonRequirementTarget:

return PythonRequirementTarget(
{
**inherited_fields,
**request.template,
PythonRequirementsField.alias: [parsed_req],
PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
Expand All @@ -136,7 +130,7 @@ def generate_tgt(raw_req: str, info: dict) -> PythonRequirementTarget:
Dependencies.alias: [file_tgt.address.spec],
**tgt_overrides,
},
generator.address.create_generated(parsed_req.project_name),
request.template_address.create_generated(parsed_req.project_name),
)

result = tuple(
Expand All @@ -146,7 +140,7 @@ def generate_tgt(raw_req: str, info: dict) -> PythonRequirementTarget:

if overrides:
raise InvalidFieldException(
f"Unused key in the `overrides` field for {request.generator.address}: "
f"Unused key in the `overrides` field for {request.template_address}: "
f"{sorted(overrides)}"
)

Expand Down
26 changes: 10 additions & 16 deletions src/python/pants/backend/python/macros/poetry_requirements.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
GenerateTargetsRequest,
InvalidFieldException,
SingleSourceField,
Target,
TargetGenerator,
)
from pants.engine.unions import UnionRule
from pants.util.logging import LogLevel
Expand Down Expand Up @@ -385,18 +385,20 @@ class PoetryRequirementsSourceField(SingleSourceField):
required = False


class PoetryRequirementsTargetGenerator(Target):
class PoetryRequirementsTargetGenerator(TargetGenerator):
alias = "poetry_requirements"
help = "Generate a `python_requirement` for each entry in a Poetry pyproject.toml."
generated_target_cls = PythonRequirementTarget
# Note that this does not have a `dependencies` field.
core_fields = (
*COMMON_TARGET_FIELDS,
ModuleMappingField,
TypeStubsModuleMappingField,
PoetryRequirementsSourceField,
RequirementsOverrideField,
PythonRequirementResolveField,
)
copied_fields = COMMON_TARGET_FIELDS
moved_fields = (PythonRequirementResolveField,)


class GenerateFromPoetryRequirementsRequest(GenerateTargetsRequest):
Expand All @@ -418,8 +420,8 @@ async def generate_from_python_requirement(
file_tgt = TargetGeneratorSourcesHelperTarget(
{TargetGeneratorSourcesHelperSourcesField.alias: [pyproject_rel_path]},
Address(
generator.address.spec_path,
target_name=generator.address.target_name,
request.template_address.spec_path,
target_name=request.template_address.target_name,
relative_file_path=pyproject_rel_path,
),
)
Expand All @@ -441,16 +443,8 @@ async def generate_from_python_requirement(
)
)

# Validate the resolve is legal.
generator[PythonRequirementResolveField].normalized_value(python_setup)

module_mapping = generator[ModuleMappingField].value
stubs_mapping = generator[TypeStubsModuleMappingField].value
inherited_fields = {
field.alias: field.value
for field in request.generator.field_values.values()
if isinstance(field, (*COMMON_TARGET_FIELDS, PythonRequirementResolveField))
}

def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget:
normalized_proj_name = canonicalize_project_name(parsed_req.project_name)
Expand All @@ -462,7 +456,7 @@ def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget:

return PythonRequirementTarget(
{
**inherited_fields,
**request.template,
PythonRequirementsField.alias: [parsed_req],
PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
Expand All @@ -473,14 +467,14 @@ def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget:
Dependencies.alias: [file_tgt.address.spec],
**tgt_overrides,
},
generator.address.create_generated(parsed_req.project_name),
request.template_address.create_generated(parsed_req.project_name),
)

result = tuple(generate_tgt(requirement) for requirement in requirements) + (file_tgt,)

if overrides:
raise InvalidFieldException(
f"Unused key in the `overrides` field for {request.generator.address}: "
f"Unused key in the `overrides` field for {request.template_address}: "
f"{sorted(overrides)}"
)

Expand Down
Loading

0 comments on commit d45bbfa

Please sign in to comment.