[Internal] Refactoring how we integrate with dockerfile (pantsbuild#1…

…3027)

Rather than having it as a direct dependency, package it into a PEX tool that we run out of process on demand.

Closes pantsbuild#13026.

3rdparty/python/lockfiles/user_reqs.txt

@@ -11,7 +11,6 @@
 #   "generated_with_requirements": [
 #     "PyYAML<5.5,>=5.4",
 #     "ansicolors==1.1.8",
-#     "dockerfile==3.2.0",
 #     "fasteners==0.16",
 #     "freezegun==1.1.0",
 #     "humbug==0.2.6",
@@ -53,10 +52,6 @@ charset-normalizer==2.0.5; python_full_version >= "3.6.0" and python_version >=
 colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.5.0" \
     --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 \
     --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b
-dockerfile==3.2.0; python_full_version >= "3.6.1" \
-    --hash=sha256:e6e00b82b82042fb4df569ae00bd2648ac6c8823f51c406da31ab01c728926c2 \
-    --hash=sha256:e6bd64408386b7ba2259d85820e0fe90de1b6b8269f560f18aba100c6aa40b7d \
-    --hash=sha256:e13fd3768216a788189e0667521e1435a273a4129119a8453085d897fc34aac8
 fasteners==0.16 \
     --hash=sha256:74b6847e0a6bb3b56c8511af8e24c40e4cf7a774dfff5b251c260ed338096a4b \
     --hash=sha256:c995d8c26b017c5d6a6de9ad29a0f9cdd57de61ae1113d28fac26622b06a0933

3rdparty/python/requirements.txt

@@ -1,5 +1,4 @@
 ansicolors==1.1.8
-dockerfile==3.2.0
 fasteners==0.16
 freezegun==1.1.0
 

build-support/bin/_generate_all_lockfiles_helper.py

@@ -12,6 +12,7 @@
 import subprocess
 
 from pants.backend.codegen.protobuf.python.python_protobuf_subsystem import PythonProtobufMypyPlugin
+from pants.backend.docker.dockerfile_parser import DockerfileParser
 from pants.backend.python.goals.coverage_py import CoverageSubsystem
 from pants.backend.python.lint.autoflake.subsystem import Autoflake
 from pants.backend.python.lint.bandit.subsystem import Bandit
@@ -141,6 +142,12 @@ def main() -> None:
             f"--terraform-hcl2-parser-extra-requirements={repr(TerraformHcl2Parser.default_extra_requirements)}",
             f"--terraform-hcl2-parser-interpreter-constraints={repr(TerraformHcl2Parser.default_interpreter_constraints)}",
             f"--terraform-hcl2-parser-lockfile={TerraformHcl2Parser.default_lockfile_path}",
+            # Dockerfile parser for Docker dependency inference
+            "--backend-packages=+['pants.backend.experimental.docker']",
+            f"--dockerfile-parser-version={DockerfileParser.default_version}",
+            f"--dockerfile-parser-extra-requirements={repr(DockerfileParser.default_extra_requirements)}",
+            f"--dockerfile-parser-interpreter-constraints={repr(DockerfileParser.default_interpreter_constraints)}",
+            f"--dockerfile-parser-lockfile={DockerfileParser.default_lockfile_path}",
             # Run the goal.
             "generate-lockfiles",
         ],

src/python/pants/backend/docker/BUILD

@@ -1,7 +1,27 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-python_library()
+python_library(
+  sources=[
+      "*.py",
+      "!*_test.py",
+      "!dockerfile_parser.py",
+      "!dockerfile_wrapper_script.py",
+  ],
+)
+
+python_library(
+  name="dockerfile_parser",
+  sources=["dockerfile_parser.py"],
+  dependencies=[":dockerfile_tool", ":lockfile"],
+)
+
+resources(name="lockfile", sources=["dockerfile_lockfile.txt"])
+
+python_library(
+  name="dockerfile_tool",
+  sources=["dockerfile_wrapper_script.py"],
+)
 
 python_tests(
     name="tests",

src/python/pants/backend/docker/dependencies.py

@@ -2,7 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 
-from pants.backend.docker.parser import DockerfileInfo, DockerfileParseRequest
+from pants.backend.docker.dockerfile_parser import DockerfileInfo
 from pants.backend.docker.target_types import DockerDependencies, DockerImageSources
 from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet
 from pants.engine.addresses import Address, Addresses, UnparsedAddressInputs
@@ -28,7 +28,7 @@ async def inject_docker_dependencies(request: InjectDockerDependencies) -> Injec
     if not sources.value:
         return InjectedDependencies()
 
-    dockerfile = await Get(DockerfileInfo, DockerfileParseRequest(sources))
+    dockerfile = await Get(DockerfileInfo, DockerImageSources, sources)
     targets = await Get(
         Targets,
         UnparsedAddressInputs(

src/python/pants/backend/docker/dependencies_test.py

@@ -6,24 +6,31 @@
 import pytest
 
 from pants.backend.docker.dependencies import InjectDockerDependencies, inject_docker_dependencies
-from pants.backend.docker.parser import parse_dockerfile
+from pants.backend.docker.dockerfile_parser import rules as parser_rules
 from pants.backend.docker.target_types import DockerDependencies, DockerImage
 from pants.backend.python.target_types import PexBinary
+from pants.backend.python.util_rules.pex import rules as pex_rules
 from pants.engine.addresses import Address
 from pants.engine.target import InjectedDependencies
 from pants.testutil.rule_runner import QueryRule, RuleRunner
 
 
 @pytest.fixture
 def rule_runner() -> RuleRunner:
-    return RuleRunner(
+    rule_runner = RuleRunner(
         rules=[
+            *parser_rules(),
+            *pex_rules(),
             inject_docker_dependencies,
-            parse_dockerfile,
             QueryRule(InjectedDependencies, (InjectDockerDependencies,)),
         ],
         target_types=[DockerImage, PexBinary],
     )
+    rule_runner.set_options(
+        [],
+        env_inherit={"PATH", "PYENV_ROOT", "HOME"},
+    )
+    return rule_runner
 
 
 def test_inject_docker_dependencies(rule_runner: RuleRunner) -> None:

src/python/pants/backend/docker/docker_build_integration_test.py

@@ -8,6 +8,7 @@
 from pants.backend.docker.docker_build import DockerFieldSet
 from pants.backend.docker.rules import rules as docker_rules
 from pants.backend.docker.target_types import DockerImage
+from pants.backend.python.util_rules import pex
 from pants.core.goals.package import BuiltPackage
 from pants.core.util_rules.source_files import rules as source_files_rules
 from pants.engine.addresses import Address
@@ -20,6 +21,7 @@ def rule_runner() -> RuleRunner:
     return RuleRunner(
         rules=[
             *docker_rules(),
+            *pex.rules(),
             *source_files_rules(),
             QueryRule(BuiltPackage, [DockerFieldSet]),
         ],
@@ -33,7 +35,10 @@ def run_docker(
     *,
     extra_args: list[str] | None = None,
 ) -> BuiltPackage:
-    rule_runner.set_options(extra_args or ())
+    rule_runner.set_options(
+        extra_args or (),
+        env_inherit={"PATH", "PYENV_ROOT", "HOME"},
+    )
     result = rule_runner.request(
         BuiltPackage,
         [DockerFieldSet.create(target)],

src/python/pants/backend/docker/dockerfile_lockfile.txt

@@ -0,0 +1,20 @@
+# This lockfile was autogenerated by Pants. To regenerate, run:
+#
+#    build-support/bin/generate_all_lockfiles.sh
+#
+# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
+# {
+#   "version": 2,
+#   "valid_for_interpreter_constraints": [
+#     "CPython>=3.7"
+#   ],
+#   "generated_with_requirements": [
+#     "dockerfile==3.2.0"
+#   ]
+# }
+# --- END PANTS LOCKFILE METADATA ---
+
+dockerfile==3.2.0; python_full_version >= "3.6.1" \
+    --hash=sha256:e6e00b82b82042fb4df569ae00bd2648ac6c8823f51c406da31ab01c728926c2 \
+    --hash=sha256:e6bd64408386b7ba2259d85820e0fe90de1b6b8269f560f18aba100c6aa40b7d \
+    --hash=sha256:e13fd3768216a788189e0667521e1435a273a4129119a8453085d897fc34aac8

src/python/pants/backend/docker/dockerfile_parser.py

@@ -0,0 +1,133 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import pkgutil
+from dataclasses import dataclass
+from pathlib import PurePath
+
+from pants.backend.docker.target_types import DockerImageSources
+from pants.backend.python.goals.lockfile import PythonLockfileRequest, PythonToolLockfileSentinel
+from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase
+from pants.backend.python.target_types import EntryPoint
+from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess
+from pants.engine.fs import CreateDigest, Digest, FileContent
+from pants.engine.process import Process, ProcessResult
+from pants.engine.rules import Get, collect_rules, rule
+from pants.engine.target import HydratedSources, HydrateSourcesRequest
+from pants.engine.unions import UnionRule
+from pants.util.docutil import git_url
+
+_DOCKERFILE_SANDBOX_TOOL = "dockerfile_wrapper_script.py"
+
+
+class DockerfileParser(PythonToolRequirementsBase):
+    options_scope = "dockerfile-parser"
+    help = "Used to parse Dockerfile build specs to infer their dependencies."
+
+    default_version = "dockerfile==3.2.0"
+
+    register_interpreter_constraints = True
+    default_interpreter_constraints = ["CPython>=3.7"]
+
+    register_lockfile = True
+    default_lockfile_resource = ("pants.backend.docker", "dockerfile_lockfile.txt")
+    default_lockfile_path = "src/python/pants/backend/docker/dockerfile_lockfile.txt"
+    default_lockfile_url = git_url(default_lockfile_path)
+
+
+class DockerfileParserLockfileSentinel(PythonToolLockfileSentinel):
+    options_scope = DockerfileParser.options_scope
+
+
+@rule
+def setup_lockfile_request(
+    _: DockerfileParserLockfileSentinel, dockerfile_parser: DockerfileParser
+) -> PythonLockfileRequest:
+    return PythonLockfileRequest.from_tool(dockerfile_parser)
+
+
+@dataclass(frozen=True)
+class ParserSetup:
+    pex: VenvPex
+
+
+@rule
+async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup:
+    parser_script_content = pkgutil.get_data("pants.backend.docker", _DOCKERFILE_SANDBOX_TOOL)
+    if not parser_script_content:
+        raise ValueError(
+            "Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in pants.backend.docker."
+        )
+
+    parser_content = FileContent(
+        path="__pants_df_parser.py",
+        content=parser_script_content,
+        is_executable=True,
+    )
+
+    parser_digest = await Get(Digest, CreateDigest([parser_content]))
+
+    parser_pex = await Get(
+        VenvPex,
+        PexRequest(
+            output_filename="dockerfile_parser.pex",
+            internal_only=True,
+            requirements=dockerfile_parser.pex_requirements(),
+            interpreter_constraints=dockerfile_parser.interpreter_constraints,
+            main=EntryPoint(PurePath(parser_content.path).stem),
+            sources=parser_digest,
+        ),
+    )
+
+    return ParserSetup(parser_pex)
+
+
+@dataclass(frozen=True)
+class DockerfileParseRequest:
+    sources_digest: Digest
+    paths: tuple[str, ...]
+
+
+@rule
+async def setup_process_for_parse_dockerfile(
+    request: DockerfileParseRequest, parser: ParserSetup
+) -> Process:
+    process = await Get(
+        Process,
+        VenvPexProcess(
+            parser.pex,
+            argv=request.paths,
+            input_digest=request.sources_digest,
+            description="Parse Dockerfile.",
+        ),
+    )
+    return process
+
+
+@dataclass(frozen=True)
+class DockerfileInfo:
+    putative_target_addresses: tuple[str, ...] = ()
+
+
+@rule
+async def parse_dockerfile(sources: DockerImageSources) -> DockerfileInfo:
+    hydrated_sources = await Get(HydratedSources, HydrateSourcesRequest(sources))
+    result = await Get(
+        ProcessResult,
+        DockerfileParseRequest(hydrated_sources.snapshot.digest, hydrated_sources.snapshot.files),
+    )
+
+    putative_target_addresses = [line for line in result.stdout.decode("utf-8").split("\n") if line]
+
+    return DockerfileInfo(
+        putative_target_addresses=tuple(putative_target_addresses),
+    )
+
+
+def rules():
+    return (
+        *collect_rules(),
+        UnionRule(PythonToolLockfileSentinel, DockerfileParserLockfileSentinel),
+    )

src/python/pants/backend/docker/dockerfile_parser_test.py

@@ -0,0 +1,55 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+
+from textwrap import dedent
+
+import pytest
+
+from pants.backend.docker.dockerfile_parser import DockerfileInfo
+from pants.backend.docker.dockerfile_parser import rules as parser_rules
+from pants.backend.docker.target_types import DockerImage, DockerImageSources
+from pants.backend.python.target_types import PexBinary
+from pants.backend.python.util_rules.pex import rules as pex_rules
+from pants.engine.addresses import Address
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    rule_runner = RuleRunner(
+        rules=[
+            *parser_rules(),
+            *pex_rules(),
+            QueryRule(DockerfileInfo, (DockerImageSources,)),
+        ],
+        target_types=[DockerImage, PexBinary],
+    )
+    rule_runner.set_options(
+        [],
+        env_inherit={"PATH", "PYENV_ROOT", "HOME"},
+    )
+    return rule_runner
+
+
+def test_putative_target_addresses(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "test/BUILD": "docker_image()",
+            "test/Dockerfile": dedent(
+                """\
+            FROM base
+            COPY some.target/binary.pex some.target/tool.pex /bin
+            COPY --from=scratch this.is/ignored.pex /opt
+            COPY binary another/cli.pex tool /bin
+            """
+            ),
+        }
+    )
+    tgt = rule_runner.get_target(Address("test"))
+    info = rule_runner.request(DockerfileInfo, [tgt[DockerImageSources]])
+    assert info.putative_target_addresses == (
+        "some/target:binary",
+        "some/target:tool",
+        "another:cli",
+    )