Auto-detect Helm chart source roots (pantsbuild#16531)

docs/markdown/Helm/helm-deployments.md

@@ -69,10 +69,6 @@ There are quite a few things to notice in the previous example:
 * One of those value files (`common-values.yaml`) provides with default values that are common to all deployments.
 * Each deployment uses an additional `xxx-override.yaml` file with values that are specific to the given deployment.
 
-> 📘 Source roots
-> 
-> Don't forget to configure your source roots such that each of the shown files in the previous example sit at their respective source root level.
-
 The `helm_deployment` target has many additional fields including the target kubernetes namespace, adding inline override values (similar to using helm's `--set` arg) and many others. Please run `./pants help helm_deployment` to see all the posibilities.
 
 Dependencies with `docker_image` targets

docs/markdown/Helm/helm-overview.md

@@ -25,7 +25,7 @@ backend_packages = [
 ]
 ```
 
-In the case in which you may have more than one chart in the same repository, it is important that you configure your Pants' source roots in a way that Pants recognises each of your chart folders as a source root. In the following example `foo` and `bar` are Helm charts, so we give Pants a source root pattern to consider `src/helm/foo` and `src/helm/bar` as source roots.
+If you have more than one Helm chart in the same repository, organise them such that each of them lives in a separate folder with the chart definition file (`Chart.yaml`) at their root. The Helm backend is capable of auto-detecting the root folder of your Helm charts taking the chart definition file `Chart.yaml` as the reference for that root. 
 
 ```yaml src/helm/foo/Chart.yaml
 apiVersion: v2
@@ -39,14 +39,6 @@ description: Bar Helm chart
 name: bar
 version: 0.1.0
 ```
-```toml pants.toml
-[source]
-root_patterns = [
-  ...
-  "src/helm/*",
-  ...
-]
-```
 
 Adding `helm_chart` targets
 ---------------------------

src/python/pants/backend/helm/dependency_inference/chart.py

@@ -17,6 +17,7 @@
     HelmChartTarget,
 )
 from pants.backend.helm.target_types import rules as helm_target_types_rules
+from pants.backend.helm.util_rules import chart_metadata
 from pants.backend.helm.util_rules.chart_metadata import HelmChartDependency, HelmChartMetadata
 from pants.engine.addresses import Address
 from pants.engine.internals.selectors import Get, MultiGet
@@ -162,5 +163,6 @@ def rules():
         *collect_rules(),
         *artifacts.rules(),
         *helm_target_types_rules(),
+        *chart_metadata.rules(),
         UnionRule(InferDependenciesRequest, InferHelmChartDependenciesRequest),
     ]

src/python/pants/backend/helm/dependency_inference/chart_test.py

@@ -25,7 +25,7 @@
 
 @pytest.fixture
 def rule_runner() -> RuleRunner:
-    return RuleRunner(
+    rule_runner = RuleRunner(
         target_types=[HelmArtifactTarget, HelmChartTarget],
         rules=[
             *artifacts.rules(),
@@ -36,13 +36,14 @@ def rule_runner() -> RuleRunner:
             QueryRule(InferredDependencies, (InferHelmChartDependenciesRequest,)),
         ],
     )
+    return rule_runner
 
 
 def test_build_first_party_mapping(rule_runner: RuleRunner) -> None:
     rule_runner.write_files(
         {
-            "src/BUILD": "helm_chart(name='foo')",
-            "src/Chart.yaml": dedent(
+            "src/foo/BUILD": "helm_chart(name='foo')",
+            "src/foo/Chart.yaml": dedent(
                 """\
                 apiVersion: v2
                 name: chart-name
@@ -52,7 +53,7 @@ def test_build_first_party_mapping(rule_runner: RuleRunner) -> None:
         }
     )
 
-    tgt = rule_runner.get_target(Address("src", target_name="foo"))
+    tgt = rule_runner.get_target(Address("src/foo", target_name="foo"))
     mapping = rule_runner.request(FirstPartyHelmChartMapping, [])
     assert mapping["chart-name"] == tgt.address
 
@@ -135,13 +136,6 @@ def test_infer_chart_dependencies(rule_runner: RuleRunner) -> None:
         }
     )
 
-    source_root_patterns = ("/src/*",)
-    rule_runner.set_options(
-        [
-            f"--source-root-patterns={repr(source_root_patterns)}",
-        ]
-    )
-
     tgt = rule_runner.get_target(Address("src/foo", target_name="foo"))
     inferred_deps = rule_runner.request(
         InferredDependencies,
@@ -184,9 +178,6 @@ def test_disambiguate_chart_dependencies(rule_runner: RuleRunner) -> None:
         }
     )
 
-    source_root_patterns = ("/src/*",)
-    rule_runner.set_options([f"--source-root-patterns={repr(source_root_patterns)}"])
-
     tgt = rule_runner.get_target(Address("src/foo", target_name="foo"))
     inferred_deps = rule_runner.request(
         InferredDependencies,
@@ -215,9 +206,6 @@ def test_raise_error_when_unknown_dependency_is_found(rule_runner: RuleRunner) -
         }
     )
 
-    source_root_patterns = ("/src/*",)
-    rule_runner.set_options([f"--source-root-patterns={repr(source_root_patterns)}"])
-
     tgt = rule_runner.get_target(Address("src/foo", target_name="foo"))
 
     with pytest.raises(

src/python/pants/backend/helm/dependency_inference/unittest_test.py

@@ -92,9 +92,6 @@ def test_injects_parent_chart(rule_runner: RuleRunner) -> None:
         }
     )
 
-    source_roots = ["src/*"]
-    rule_runner.set_options([f"--source-roots-patterns={repr(source_roots)}"])
-
     chart1_tgt = rule_runner.get_target(Address("src/chart1", target_name="chart1"))
     chart1_unittest_tgt = rule_runner.get_target(Address("src/chart1/tests", target_name="tests"))
 

src/python/pants/backend/helm/goals/deploy_test.py

@@ -19,6 +19,7 @@
 from pants.backend.helm.testutil import HELM_CHART_FILE
 from pants.backend.helm.util_rules.tool import HelmBinary
 from pants.core.goals.deploy import DeployProcess
+from pants.core.util_rules import source_files
 from pants.engine.addresses import Address
 from pants.engine.internals.scheduler import ExecutionError
 from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, QueryRule, RuleRunner
@@ -30,6 +31,7 @@ def rule_runner() -> RuleRunner:
         target_types=[HelmArtifactTarget, HelmChartTarget, HelmDeploymentTarget, DockerImageTarget],
         rules=[
             *helm_deploy_rules(),
+            *source_files.rules(),
             QueryRule(HelmBinary, ()),
             QueryRule(DeployProcess, (DeployHelmDeploymentFieldSet,)),
         ],
@@ -94,20 +96,29 @@ def test_run_helm_deploy(rule_runner: RuleRunner) -> None:
         }
     )
 
-    source_root_patterns = ["/src/*"]
     deploy_args = ["--kubeconfig", "./kubeconfig"]
     deploy_process = _run_deployment(
         rule_runner,
         "src/deployment",
         "foo",
         args=[
-            f"--source-root-patterns={repr(source_root_patterns)}",
             f"--helm-args={repr(deploy_args)}",
         ],
     )
 
     helm = rule_runner.request(HelmBinary, [])
 
+    expected_value_files_order = [
+        "common.yaml",
+        "bar.yaml",
+        "foo.yaml",
+        "bar-override.yaml",
+        "subdir/bar.yaml",
+        "subdir/foo.yaml",
+        "subdir/foo-override.yaml",
+        "subdir/last.yaml",
+    ]
+
     assert deploy_process.process
     assert deploy_process.process.process.argv == (
         helm.path,
@@ -124,7 +135,9 @@ def test_run_helm_deploy(rule_runner: RuleRunner) -> None:
         "--post-renderer",
         "./post_renderer_wrapper.sh",
         "--values",
-        "common.yaml,bar.yaml,foo.yaml,bar-override.yaml,subdir/bar.yaml,subdir/foo.yaml,subdir/foo-override.yaml,subdir/last.yaml",
+        ",".join(
+            [f"__values/src/deployment/{filename}" for filename in expected_value_files_order]
+        ),
         "--set",
         "key=foo",
         "--set",

src/python/pants/backend/helm/goals/lint.py

@@ -46,15 +46,15 @@ async def run_helm_lint(request: HelmLintRequest, helm_subsystem: HelmSubsystem)
     logger.debug(f"Linting {pluralize(len(charts), 'chart')}...")
 
     def create_process(chart: HelmChart, field_set: HelmLintFieldSet) -> HelmProcess:
-        argv = ["lint", chart.path]
+        argv = ["lint", chart.name]
 
         strict: bool = field_set.lint_strict.value or helm_subsystem.lint_strict
         if strict:
             argv.append("--strict")
 
         return HelmProcess(
             argv,
-            input_digest=chart.snapshot.digest,
+            extra_immutable_input_digests=chart.immutable_input_digests,
             description=f"Linting chart: {chart.info.name}",
         )
 

src/python/pants/backend/helm/goals/lint_test.py

@@ -3,7 +3,7 @@
 
 from __future__ import annotations
 
-from typing import Iterable, Sequence
+from typing import Iterable
 
 import pytest
 
@@ -22,7 +22,7 @@
 from pants.backend.helm.util_rules import chart, sources
 from pants.build_graph.address import Address
 from pants.core.goals.lint import LintResult, LintResults
-from pants.core.util_rules import config_files, stripped_source_files
+from pants.core.util_rules import config_files, source_files
 from pants.engine.rules import QueryRule, SubsystemRule
 from pants.engine.target import Target
 from pants.source.source_root import rules as source_root_rules
@@ -37,7 +37,7 @@ def rule_runner() -> RuleRunner:
             *config_files.rules(),
             *chart.rules(),
             *helm_lint_rules(),
-            *stripped_source_files.rules(),
+            *source_files.rules(),
             *source_root_rules(),
             *sources.rules(),
             *target_types_rules(),
@@ -52,14 +52,11 @@ def run_helm_lint(
     rule_runner: RuleRunner,
     targets: list[Target],
     *,
-    source_root_patterns: Sequence[str] = ("/",),
     extra_options: Iterable[str] = [],
 ) -> tuple[LintResult, ...]:
     field_sets = [HelmLintFieldSet.create(tgt) for tgt in targets]
 
-    opts = [f"--source-root-patterns={repr(source_root_patterns)}"]
-    opts.extend(extra_options)
-    rule_runner.set_options(opts)
+    rule_runner.set_options(extra_options)
 
     lint_results = rule_runner.request(LintResults, [HelmLintRequest(field_sets)])
     return lint_results.results
@@ -172,13 +169,13 @@ def test_one_lint_result_per_chart(rule_runner: RuleRunner) -> None:
             "src/chart2/templates/service.yaml": K8S_SERVICE_TEMPLATE,
         }
     )
-    source_root_patterns = ("src/*",)
 
     chart1_target = rule_runner.get_target(Address("src/chart1", target_name="chart1"))
     chart2_target = rule_runner.get_target(Address("src/chart2", target_name="chart2"))
 
     lint_results = run_helm_lint(
-        rule_runner, [chart1_target, chart2_target], source_root_patterns=source_root_patterns
+        rule_runner,
+        [chart1_target, chart2_target],
     )
     assert len(lint_results) == 2
     assert lint_results[0].exit_code == 0
@@ -198,10 +195,9 @@ def test_skip_lint(rule_runner: RuleRunner) -> None:
         }
     )
 
-    source_root_patterns = ("src/*",)
-
     chart_target = rule_runner.get_target(Address("src/chart", target_name="chart"))
     lint_results = run_helm_lint(
-        rule_runner, [chart_target], source_root_patterns=source_root_patterns
+        rule_runner,
+        [chart_target],
     )
     assert len(lint_results) == 0

src/python/pants/backend/helm/goals/package.py

@@ -12,15 +12,7 @@
 from pants.backend.helm.util_rules.chart_metadata import HelmChartMetadata
 from pants.backend.helm.util_rules.tool import HelmProcess
 from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact, PackageFieldSet
-from pants.engine.fs import (
-    AddPrefix,
-    CreateDigest,
-    Digest,
-    Directory,
-    MergeDigests,
-    RemovePrefix,
-    Snapshot,
-)
+from pants.engine.fs import AddPrefix, CreateDigest, Digest, Directory, RemovePrefix, Snapshot
 from pants.engine.process import ProcessResult
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.unions import UnionRule
@@ -56,14 +48,14 @@ async def run_helm_package(field_set: HelmPackageFieldSet) -> BuiltPackage:
         Get(Digest, CreateDigest([Directory(result_dir)])),
     )
 
-    input_digest = await Get(Digest, MergeDigests([chart.snapshot.digest, result_digest]))
     process_output_file = os.path.join(result_dir, f"{chart.info.artifact_name}.tgz")
 
     process_result = await Get(
         ProcessResult,
         HelmProcess(
-            argv=["package", chart.path, "-d", result_dir],
-            input_digest=input_digest,
+            argv=["package", chart.name, "-d", result_dir],
+            input_digest=result_digest,
+            extra_immutable_input_digests=chart.immutable_input_digests,
             output_files=(process_output_file,),
             description=f"Packaging Helm chart: {field_set.address.spec_path}",
         ),

src/python/pants/backend/helm/goals/package_test.py

@@ -21,7 +21,7 @@
 from pants.backend.helm.util_rules import chart, sources, tool
 from pants.build_graph.address import Address
 from pants.core.goals.package import BuiltPackage
-from pants.core.util_rules import config_files, external_tool, stripped_source_files
+from pants.core.util_rules import config_files, external_tool, source_files
 from pants.engine.rules import QueryRule, SubsystemRule
 from pants.source.source_root import rules as source_root_rules
 from pants.testutil.rule_runner import RuleRunner
@@ -37,7 +37,7 @@ def rule_runner() -> RuleRunner:
             *tool.rules(),
             *chart.rules(),
             *package.rules(),
-            *stripped_source_files.rules(),
+            *source_files.rules(),
             *source_root_rules(),
             *sources.rules(),
             *target_types_rules(),
@@ -48,8 +48,6 @@ def rule_runner() -> RuleRunner:
 
 
 def _assert_build_package(rule_runner: RuleRunner, *, chart_name: str, chart_version: str) -> None:
-    rule_runner.set_options(["--source-root-patterns=['src/*']"])
-
     target = rule_runner.get_target(Address(f"src/{chart_name}", target_name=chart_name))
     field_set = HelmPackageFieldSet.create(target)
 

src/python/pants/backend/helm/resolve/fetch.py

@@ -78,6 +78,20 @@ def debug_hint(self) -> str | None:
         return f"{self.description_of_origin}: fetch {pluralize(len(self.field_sets), 'artifact')}"
 
 
+@dataclass(frozen=True)
+class _StripHelmArtifactPrefixRequest:
+    artifact: ResolvedHelmArtifact
+    digest: Digest
+
+
+@rule
+async def _strip_prefix_from_fetched_artifact(request: _StripHelmArtifactPrefixRequest) -> Digest:
+    subset_digest = await Get(
+        Digest, DigestSubset(request.digest, PathGlobs([os.path.join(request.artifact.name, "**")]))
+    )
+    return await Get(Digest, RemovePrefix(subset_digest, request.artifact.name))
+
+
 @rule(desc="Fetch third party Helm Chart artifacts", level=LogLevel.DEBUG)
 async def fetch_helm_artifacts(request: FetchHelmArfifactsRequest) -> FetchedHelmArtifacts:
     download_prefix = "__downloads"
@@ -122,7 +136,7 @@ def create_fetch_process(artifact: ResolvedHelmArtifact) -> HelmProcess:
 
     # Avoid capturing the tarball that has been downloaded by Helm during the pull.
     artifact_snapshots = await MultiGet(
-        Get(Snapshot, DigestSubset(digest, PathGlobs([os.path.join(artifact.name, "**")])))
+        Get(Snapshot, _StripHelmArtifactPrefixRequest(artifact, digest))
         for artifact, digest in zip(artifacts, stripped_artifact_digests)
     )
 

src/python/pants/backend/helm/resolve/fetch_test.py

@@ -78,4 +78,4 @@ def test_download_artifacts(rule_runner: RuleRunner) -> None:
     assert len(fetched_artifacts) == len(expected_artifacts)
     for fetched, expected in zip(fetched_artifacts, expected_artifacts):
         assert fetched.artifact == expected
-        assert f"{expected.name}/Chart.yaml" in fetched.snapshot.files
+        assert "Chart.yaml" in fetched.snapshot.files