Infer FaaS runtime from interpreter constraints, when unambiguous (pa…

…ntsbuild#19314)

This allows the `runtime` argument to `python_aws_lambda_function`,
`python_aws_lambda_layer` and `python_google_cloud_function` to be
inferred from the relevant interpreter constraints, when they cover only
one major/minor version. For instance, having `==3.9.*` will infer
`runtime="python3.9"` for AWS Lambda.

The inference is powered by checking for two patterns of interpreter
constraints that limit to a single major version: equality `==3.9.*`
(implies 3.9) and range `>=3.10,<3.11` (implies 3.10). This inference
doesn't always work: when it doesn't work, the user gets an error
message to clarify by providing the `runtime` field explicitly. Failure
cases:

- if the interpreter constraints are too wide (for instance,
`>=3.7,<3.9` covering 2 versions, or `>=3.11` that'll eventually include
many versions), we can't be sure which is meant

- if the interpreter constraints limit the patch versions (for instance,
`==3.8.9` matching a specific version, or `==3.9.*,!=3.9.10` excluding
one), we can't be sure the cloud environment runs that version, so
inferring the runtime would be misleading

- if the interpreter constraints are non-obvious (for instance,
`>=3.7,<3.10,!=3.9.*` is technically 3.8 only), we don't try _too_ hard
to handle it. We can expand the inference if required in future.

For instance, if one has set `[python].interpreter_constraints =
["==3.9.*"]` in `pants.toml`, one can build a lambda artefact like (and
similarly for a GCF artifact):

```python
python_sources()
python_aws_lambda_function(name="func", entry_point="./foo.py:handler")
```

This is the final piece* of my work to improve the FaaS backends in
Pants 2.18:

- using the simpler "zip" layout as recommended by AWS and GCF,
deprecating Lambdex (pantsbuild#18879)
- support for AWS Lambda layers (pantsbuild#18880)
- Pants-provided complete platforms JSON files* when specifying a known
`runtime` (pantsbuild#18195)
- this PR, inferring the `runtime` from ICs, when unambiguous (including
using the new Pants-provided complete platform when available) (pantsbuild#19304)

(* The fixed complete platform files are currently only provided for AWS
Lambda, not GCF. pantsbuild#18195.)

The commits are individually reviewable.

Fixes pantsbuild#19304

src/python/pants/backend/awslambda/python/rules_test.py

@@ -281,7 +281,16 @@ def handler(event, context):
     assert "assets:resources" not in caplog.text
 
 
-def test_create_hello_world_lambda(rule_runner: PythonRuleRunner) -> None:
+@pytest.mark.parametrize(
+    ("ics", "runtime"),
+    [
+        pytest.param(["==3.7.*"], None, id="runtime inferred from ICs"),
+        pytest.param(None, "python3.7", id="runtime explicitly set"),
+    ],
+)
+def test_create_hello_world_lambda(
+    ics: list[str] | None, runtime: None | str, rule_runner: PythonRuleRunner
+) -> None:
     rule_runner.write_files(
         {
             "src/python/foo/bar/hello_world.py": dedent(
@@ -293,20 +302,20 @@ def handler(event, context):
                 """
             ),
             "src/python/foo/bar/BUILD": dedent(
-                """
+                f"""
                 python_requirement(name="mureq", requirements=["mureq==0.2"])
-                python_sources()
+                python_sources(interpreter_constraints={ics!r})
 
                 python_aws_lambda_function(
                     name='lambda',
                     handler='foo.bar.hello_world:handler',
-                    runtime="python3.7",
+                    runtime={runtime!r},
                 )
                 python_aws_lambda_function(
                     name='slimlambda',
                     include_requirements=False,
                     handler='foo.bar.hello_world:handler',
-                    runtime="python3.7",
+                    runtime={runtime!r},
                 )
                 """
             ),
@@ -316,7 +325,10 @@ def handler(event, context):
     zip_file_relpath, content = create_python_awslambda(
         rule_runner,
         Address("src/python/foo/bar", target_name="lambda"),
-        expected_extra_log_lines=("    Handler: lambda_function.handler",),
+        expected_extra_log_lines=(
+            "    Runtime: python3.7",
+            "    Handler: lambda_function.handler",
+        ),
     )
     assert "src.python.foo.bar/lambda.zip" == zip_file_relpath
 
@@ -331,7 +343,10 @@ def handler(event, context):
     zip_file_relpath, content = create_python_awslambda(
         rule_runner,
         Address("src/python/foo/bar", target_name="slimlambda"),
-        expected_extra_log_lines=("    Handler: lambda_function.handler",),
+        expected_extra_log_lines=(
+            "    Runtime: python3.7",
+            "    Handler: lambda_function.handler",
+        ),
     )
     assert "src.python.foo.bar/slimlambda.zip" == zip_file_relpath
 
@@ -379,7 +394,7 @@ def handler(event, context):
     zip_file_relpath, content = create_python_awslambda(
         rule_runner,
         Address("src/python/foo/bar", target_name="lambda"),
-        expected_extra_log_lines=(),
+        expected_extra_log_lines=("    Runtime: python3.7",),
         layer=True,
     )
     assert "src.python.foo.bar/lambda.zip" == zip_file_relpath
@@ -394,7 +409,7 @@ def handler(event, context):
     zip_file_relpath, content = create_python_awslambda(
         rule_runner,
         Address("src/python/foo/bar", target_name="slimlambda"),
-        expected_extra_log_lines=(),
+        expected_extra_log_lines=("    Runtime: python3.7",),
         layer=True,
     )
     assert "src.python.foo.bar/slimlambda.zip" == zip_file_relpath
@@ -418,6 +433,6 @@ def test_layer_must_have_dependencies(rule_runner: PythonRuleRunner) -> None:
         create_python_awslambda(
             rule_runner,
             Address("", target_name="lambda"),
-            expected_extra_log_lines=(),
+            expected_extra_log_lines=("    Runtime: python3.7",),
             layer=True,
         )

src/python/pants/backend/awslambda/python/target_types.py

@@ -26,7 +26,6 @@
     BoolField,
     Field,
     InvalidFieldException,
-    InvalidTargetException,
     Target,
 )
 from pants.util.docutil import doc_url
@@ -135,6 +134,10 @@ def to_interpreter_version(self) -> Optional[Tuple[int, int]]:
         mo = cast(Match, re.match(self.PYTHON_RUNTIME_REGEX, self.value))
         return int(mo.group("major")), int(mo.group("minor"))
 
+    @classmethod
+    def from_interpreter_version(cls, py_major: int, py_minor: int) -> str:
+        return f"python{py_major}.{py_minor}"
+
 
 class PythonAwsLambdaLayerDependenciesField(PythonFaaSDependencies):
     required = True
@@ -158,16 +161,6 @@ def validate(self) -> None:
         runtime_alias = self[PythonAwsLambdaRuntime].alias
         complete_platforms_alias = self[PexCompletePlatformsField].alias
 
-        if not (has_runtime or has_complete_platforms):
-            raise InvalidTargetException(
-                softwrap(
-                    f"""
-                    The `{self.alias}` target {self.address} must specify either a
-                    `{runtime_alias}` or `{complete_platforms_alias}`.
-                    """
-                )
-            )
-
         if has_runtime and has_complete_platforms:
             warn_or_error(
                 "2.19.0.dev0",

src/python/pants/backend/awslambda/python/target_types_test.py

@@ -1,21 +1,16 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
-import re
-from textwrap import dedent
 
 import pytest
 
 from pants.backend.awslambda.python.target_types import PythonAWSLambda, PythonAwsLambdaRuntime
 from pants.backend.awslambda.python.target_types import rules as target_type_rules
 from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget
 from pants.backend.python.target_types_rules import rules as python_target_types_rules
-from pants.backend.python.util_rules.faas import PythonFaaSCompletePlatforms
 from pants.build_graph.address import Address
 from pants.core.target_types import FileTarget
-from pants.engine.internals.scheduler import ExecutionError
 from pants.engine.target import InvalidFieldException
 from pants.testutil.rule_runner import RuleRunner
-from pants.util.strutil import softwrap
 
 
 @pytest.fixture
@@ -55,66 +50,3 @@ def test_to_interpreter_version(runtime: str, expected_major: int, expected_mino
 def test_runtime_validation(invalid_runtime: str) -> None:
     with pytest.raises(InvalidFieldException):
         PythonAwsLambdaRuntime(invalid_runtime, Address("", target_name="t"))
-
-
-def test_at_least_one_target_platform(rule_runner: RuleRunner) -> None:
-    rule_runner.write_files(
-        {
-            "project/app.py": "",
-            "project/platform-py37.json": "",
-            "project/BUILD": dedent(
-                """\
-                python_aws_lambda_function(
-                    name='runtime',
-                    handler='project.app:func',
-                    runtime='python3.7',
-                )
-                file(name="python37", source="platform-py37.json")
-                python_aws_lambda_function(
-                    name='complete_platforms',
-                    handler='project.app:func',
-                    complete_platforms=[':python37'],
-                )
-                python_aws_lambda_function(
-                    name='both',
-                    handler='project.app:func',
-                    runtime='python3.7',
-                    complete_platforms=[':python37'],
-                )
-                python_aws_lambda_function(
-                    name='neither',
-                    handler='project.app:func',
-                )
-                """
-            ),
-        }
-    )
-
-    runtime = rule_runner.get_target(Address("project", target_name="runtime"))
-    assert "python3.7" == runtime[PythonAwsLambdaRuntime].value
-    assert runtime[PythonFaaSCompletePlatforms].value is None
-
-    complete_platforms = rule_runner.get_target(
-        Address("project", target_name="complete_platforms")
-    )
-    assert complete_platforms[PythonAwsLambdaRuntime].value is None
-    assert (":python37",) == complete_platforms[PythonFaaSCompletePlatforms].value
-
-    both = rule_runner.get_target(Address("project", target_name="both"))
-    assert "python3.7" == both[PythonAwsLambdaRuntime].value
-    assert (":python37",) == both[PythonFaaSCompletePlatforms].value
-
-    with pytest.raises(
-        ExecutionError,
-        match=r".*{}.*".format(
-            re.escape(
-                softwrap(
-                    """
-                    InvalidTargetException: The `python_aws_lambda_function` target project:neither must
-                    specify either a `runtime` or `complete_platforms`.
-                    """
-                )
-            )
-        ),
-    ):
-        rule_runner.get_target(Address("project", target_name="neither"))

src/python/pants/backend/google_cloud_function/python/rules_test.py

@@ -249,7 +249,16 @@ def handler(event, context):
     assert "assets:resources" not in caplog.text
 
 
-def test_create_hello_world_gcf(rule_runner: PythonRuleRunner) -> None:
+@pytest.mark.parametrize(
+    ("ics", "runtime"),
+    [
+        pytest.param(["==3.7.*"], None, id="runtime inferred from ICs"),
+        pytest.param(None, "python37", id="runtime explicitly set"),
+    ],
+)
+def test_create_hello_world_gcf(
+    ics: list[str] | None, runtime: None | str, rule_runner: PythonRuleRunner
+) -> None:
     rule_runner.write_files(
         {
             "src/python/foo/bar/hello_world.py": dedent(
@@ -261,14 +270,14 @@ def handler(event, context):
                 """
             ),
             "src/python/foo/bar/BUILD": dedent(
-                """
+                f"""
                 python_requirement(name="mureq", requirements=["mureq==0.2"])
-                python_sources()
+                python_sources(interpreter_constraints={ics!r})
 
                 python_google_cloud_function(
                     name='gcf',
                     handler='foo.bar.hello_world:handler',
-                    runtime="python37",
+                    runtime={runtime!r},
                     type='event',
                 )
                 """
@@ -279,7 +288,10 @@ def handler(event, context):
     zip_file_relpath, content = create_python_google_cloud_function(
         rule_runner,
         Address("src/python/foo/bar", target_name="gcf"),
-        expected_extra_log_lines=("    Handler: handler",),
+        expected_extra_log_lines=(
+            "    Runtime: python37",
+            "    Handler: handler",
+        ),
     )
     assert "src.python.foo.bar/gcf.zip" == zip_file_relpath
 

src/python/pants/backend/google_cloud_function/python/target_types.py

@@ -18,13 +18,7 @@
 from pants.core.util_rules.environments import EnvironmentField
 from pants.engine.addresses import Address
 from pants.engine.rules import collect_rules
-from pants.engine.target import (
-    COMMON_TARGET_FIELDS,
-    InvalidFieldException,
-    InvalidTargetException,
-    StringField,
-    Target,
-)
+from pants.engine.target import COMMON_TARGET_FIELDS, InvalidFieldException, StringField, Target
 from pants.util.docutil import doc_url
 from pants.util.strutil import help_text, softwrap
 
@@ -90,6 +84,10 @@ def to_interpreter_version(self) -> Optional[Tuple[int, int]]:
         mo = cast(Match, re.match(self.PYTHON_RUNTIME_REGEX, self.value))
         return int(mo.group("major")), int(mo.group("minor"))
 
+    @classmethod
+    def from_interpreter_version(cls, py_major: int, py_minor) -> str:
+        return f"python{py_major}{py_minor}"
+
 
 class GoogleCloudFunctionTypes(Enum):
     EVENT = "event"
@@ -137,16 +135,6 @@ def validate(self) -> None:
         runtime_alias = self[PythonGoogleCloudFunctionRuntime].alias
         complete_platforms_alias = self[PexCompletePlatformsField].alias
 
-        if not (has_runtime or has_complete_platforms):
-            raise InvalidTargetException(
-                softwrap(
-                    f"""
-                    The `{self.alias}` target {self.address} must specify either a
-                    `{runtime_alias}` or `{complete_platforms_alias}`.
-                    """
-                )
-            )
-
         if has_runtime and has_complete_platforms:
             warn_or_error(
                 "2.19.0.dev0",

src/python/pants/backend/google_cloud_function/python/target_types_test.py

@@ -1,7 +1,5 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
-import re
-from textwrap import dedent
 
 import pytest
 
@@ -12,10 +10,8 @@
 from pants.backend.google_cloud_function.python.target_types import rules as target_type_rules
 from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget
 from pants.backend.python.target_types_rules import rules as python_target_types_rules
-from pants.backend.python.util_rules.faas import PythonFaaSCompletePlatforms
 from pants.build_graph.address import Address
 from pants.core.target_types import FileTarget
-from pants.engine.internals.scheduler import ExecutionError
 from pants.engine.target import InvalidFieldException
 from pants.testutil.rule_runner import RuleRunner
 
@@ -58,66 +54,3 @@ def test_to_interpreter_version(runtime: str, expected_major: int, expected_mino
 def test_runtime_validation(invalid_runtime: str) -> None:
     with pytest.raises(InvalidFieldException):
         PythonGoogleCloudFunctionRuntime(invalid_runtime, Address("", target_name="t"))
-
-
-def test_at_least_one_target_platform(rule_runner: RuleRunner) -> None:
-    rule_runner.write_files(
-        {
-            "project/app.py": "",
-            "project/platform-py37.json": "",
-            "project/BUILD": dedent(
-                """\
-                python_google_cloud_function(
-                    name='runtime',
-                    handler='project.app:func',
-                    runtime='python37',
-                    type='event',
-                )
-                file(name="python37", source="platform-py37.json")
-                python_google_cloud_function(
-                    name='complete_platforms',
-                    handler='project.app:func',
-                    complete_platforms=[':python37'],
-                    type='event',
-                )
-                python_google_cloud_function(
-                    name='both',
-                    handler='project.app:func',
-                    runtime='python37',
-                    complete_platforms=[':python37'],
-                    type='event',
-                )
-                python_google_cloud_function(
-                    name='neither',
-                    handler='project.app:func',
-                    type='event',
-                )
-                """
-            ),
-        }
-    )
-
-    runtime = rule_runner.get_target(Address("project", target_name="runtime"))
-    assert "python37" == runtime[PythonGoogleCloudFunctionRuntime].value
-    assert runtime[PythonFaaSCompletePlatforms].value is None
-
-    complete_platforms = rule_runner.get_target(
-        Address("project", target_name="complete_platforms")
-    )
-    assert complete_platforms[PythonGoogleCloudFunctionRuntime].value is None
-    assert (":python37",) == complete_platforms[PythonFaaSCompletePlatforms].value
-
-    both = rule_runner.get_target(Address("project", target_name="both"))
-    assert "python37" == both[PythonGoogleCloudFunctionRuntime].value
-    assert (":python37",) == both[PythonFaaSCompletePlatforms].value
-
-    with pytest.raises(
-        ExecutionError,
-        match=r".*{}.*".format(
-            re.escape(
-                "InvalidTargetException: The `python_google_cloud_function` target project:neither "
-                "must specify either a `runtime` or `complete_platforms`."
-            )
-        ),
-    ):
-        rule_runner.get_target(Address("project", target_name="neither"))