Add suport for string interpolation in Helm deployments (pantsbuild#1…

…6611)

Generalises the DockerInterpolationContext implementation making it available to other backends and implements interpolated values based on environment variables in helm_deployment targets.

docs/markdown/Helm/helm-deployments.md

@@ -176,8 +176,53 @@ src/deployment/last.yaml
 
 We believe that this approach gives a very consistent and predictable ordering while at the same time total flexibility to the end user to organise their files as they best fit each particular case of a deployment.
 
+Inline values
+-------------
+
 In addition to value files, you can also use inline values in your `helm_deployment` targets by means of the `values` field. All inlines values that are set this way will override any entry that may come from value files.
 
+Inline values are defined as a key-value dictionary, like in the following example:
+
+```python src/deployment/BUILD
+helm_deployment(
+  name="dev",
+  dependencies=["//src/chart"],
+  values={
+    "nameOverride": "my_custom_name",
+    "image.pullPolicy": "Always",
+  },
+)
+```
+
+### Using dynamic values
+
+Inline values also support interpolation of environment variables. Since Pants runs all processes in a hermetic sandbox, to be able to use environment variables you must first tell Pants what variables to make available to the Helm process using the `[helm].extra_env_vars` option. Consider the following example:
+
+```python src/deployment/BUILD
+helm_deployment(
+  name="dev",
+  dependencies=["//src/chart"],
+  values={
+    "configmap.deployedAt": "{env.DEPLOY_TIME}",
+  },
+)
+```
+```toml pants.toml
+[helm]
+extra_env_vars = ["DEPLOY_TIME"]
+```
+
+Now you can launch a deployment using the following command:
+
+```
+DEPLOY_TIME=$(date) ./pants experimental-deploy src/deployment:dev
+```
+
+> 🚧 Ensuring repeatable deployments
+> 
+> You should always favor using static values (or value files) VS dynamic values in your deployments. Using interpolated environment variables in your deployments can render your deployments non-repetable anymore if those values can affect the behaviour of the system deployed, or what gets deployed (i.e. Docker image addresses).
+> Dynamic values are supported to give the option of passing some info or metadata to the software being deployed (i.e. deploy time, commit hash, etc) or some less harmful settings of a deployment (i.e. replica count. etc). Be careful when chossing the values that are going to be calculated dynamically.
+
 Third party chart artifacts
 ---------------------------
 

src/python/pants/backend/docker/goals/package_image.py

@@ -34,10 +34,6 @@
     DockerBuildContextRequest,
 )
 from pants.backend.docker.utils import format_rename_suggestion
-from pants.backend.docker.value_interpolation import (
-    DockerInterpolationContext,
-    DockerInterpolationError,
-)
 from pants.core.goals.package import BuiltPackage, PackageFieldSet
 from pants.core.goals.run import RunFieldSet
 from pants.engine.addresses import Address
@@ -47,15 +43,16 @@
 from pants.engine.unions import UnionMembership, UnionRule
 from pants.option.global_options import GlobalOptions, KeepSandboxes
 from pants.util.strutil import bullet_list
+from pants.util.value_interpolation import InterpolationContext, InterpolationError
 
 logger = logging.getLogger(__name__)
 
 
-class DockerImageTagValueError(DockerInterpolationError):
+class DockerImageTagValueError(InterpolationError):
     pass
 
 
-class DockerRepositoryNameError(DockerInterpolationError):
+class DockerRepositoryNameError(InterpolationError):
     pass
 
 
@@ -78,19 +75,19 @@ class DockerFieldSet(PackageFieldSet, RunFieldSet):
     tags: DockerImageTagsField
     target_stage: DockerImageTargetStageField
 
-    def format_tag(self, tag: str, interpolation_context: DockerInterpolationContext) -> str:
-        source = DockerInterpolationContext.TextSource(
+    def format_tag(self, tag: str, interpolation_context: InterpolationContext) -> str:
+        source = InterpolationContext.TextSource(
             address=self.address, target_alias="docker_image", field_alias=self.tags.alias
         )
         return interpolation_context.format(tag, source=source, error_cls=DockerImageTagValueError)
 
     def format_repository(
         self,
         default_repository: str,
-        interpolation_context: DockerInterpolationContext,
+        interpolation_context: InterpolationContext,
         registry: DockerRegistryOptions | None = None,
     ) -> str:
-        repository_context = DockerInterpolationContext.from_dict(
+        repository_context = InterpolationContext.from_dict(
             {
                 "directory": os.path.basename(self.address.spec_path),
                 "name": self.address.target_name,
@@ -102,19 +99,17 @@ def format_repository(
         )
         if registry and registry.repository:
             repository_text = registry.repository
-            source = DockerInterpolationContext.TextSource(
+            source = InterpolationContext.TextSource(
                 options_scope=f"[docker.registries.{registry.alias or registry.address}].repository"
             )
         elif self.repository.value:
             repository_text = self.repository.value
-            source = DockerInterpolationContext.TextSource(
+            source = InterpolationContext.TextSource(
                 address=self.address, target_alias="docker_image", field_alias=self.repository.alias
             )
         else:
             repository_text = default_repository
-            source = DockerInterpolationContext.TextSource(
-                options_scope="[docker].default_repository"
-            )
+            source = InterpolationContext.TextSource(options_scope="[docker].default_repository")
         return repository_context.format(
             repository_text, source=source, error_cls=DockerRepositoryNameError
         ).lower()
@@ -123,7 +118,7 @@ def format_names(
         self,
         repository: str,
         tags: tuple[str, ...],
-        interpolation_context: DockerInterpolationContext,
+        interpolation_context: InterpolationContext,
     ) -> Iterator[str]:
         for tag in tags:
             yield ":".join(
@@ -134,7 +129,7 @@ def image_refs(
         self,
         default_repository: str,
         registries: DockerRegistries,
-        interpolation_context: DockerInterpolationContext,
+        interpolation_context: InterpolationContext,
         additional_tags: tuple[str, ...] = (),
     ) -> tuple[str, ...]:
         """The image refs are the full image name, including any registry and version tag.
@@ -201,7 +196,7 @@ def get_build_options(
     # Build options from target fields inheriting from DockerBuildOptionFieldMixin
     for field_type in target.field_types:
         if issubclass(field_type, DockerBuildOptionFieldMixin):
-            source = DockerInterpolationContext.TextSource(
+            source = InterpolationContext.TextSource(
                 address=target.address, target_alias=target.alias, field_alias=field_type.alias
             )
             format = partial(

src/python/pants/backend/docker/goals/package_image_test.py

@@ -43,10 +43,6 @@
     DockerBuildEnvironmentRequest,
 )
 from pants.backend.docker.util_rules.docker_build_env import rules as build_env_rules
-from pants.backend.docker.value_interpolation import (
-    DockerInterpolationContext,
-    DockerInterpolationError,
-)
 from pants.engine.addresses import Address
 from pants.engine.fs import EMPTY_DIGEST, EMPTY_FILE_DIGEST, EMPTY_SNAPSHOT, Snapshot
 from pants.engine.platform import Platform
@@ -63,6 +59,7 @@
 from pants.testutil.pytest_util import assert_logged, no_exception
 from pants.testutil.rule_runner import MockGet, QueryRule, RuleRunner, run_rule_with_mocks
 from pants.util.frozendict import FrozenDict
+from pants.util.value_interpolation import InterpolationContext, InterpolationError
 
 
 @pytest.fixture
@@ -380,7 +377,7 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None:
 
 
 def test_dynamic_image_version(rule_runner: RuleRunner) -> None:
-    interpolation_context = DockerInterpolationContext.from_dict(
+    interpolation_context = InterpolationContext.from_dict(
         {
             "baseimage": {"tag": "3.8"},
             "stage0": {"tag": "3.8"},
@@ -1122,7 +1119,7 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std
             docker_image=dict(repository="{default_repository}/a"),
             default_repository="{target_repository}/b",
             expect_error=pytest.raises(
-                DockerInterpolationError,
+                InterpolationError,
                 match=(
                     r"Invalid value for the `repository` field of the `docker_image` target at "
                     r"src/test/docker:image: '\{default_repository\}/a'\.\n\n"
@@ -1139,7 +1136,7 @@ def test_image_ref_formatting(test: ImageRefTest) -> None:
     tgt = DockerImageTarget(test.docker_image, address)
     field_set = DockerFieldSet.create(tgt)
     registries = DockerRegistries.from_dict(test.registries)
-    interpolation_context = DockerInterpolationContext.from_dict({})
+    interpolation_context = InterpolationContext.from_dict({})
     with test.expect_error or no_exception():
         assert (
             field_set.image_refs(test.default_repository, registries, interpolation_context)

src/python/pants/backend/docker/goals/publish_test.py

@@ -17,7 +17,6 @@
 from pants.backend.docker.target_types import DockerImageTarget
 from pants.backend.docker.util_rules import docker_binary
 from pants.backend.docker.util_rules.docker_binary import DockerBinary
-from pants.backend.docker.value_interpolation import DockerInterpolationContext
 from pants.core.goals.package import BuiltPackage
 from pants.core.goals.publish import PublishPackages, PublishProcesses
 from pants.engine.addresses import Address
@@ -27,6 +26,7 @@
 from pants.testutil.process_util import process_assertion
 from pants.testutil.rule_runner import QueryRule, RuleRunner
 from pants.util.frozendict import FrozenDict
+from pants.util.value_interpolation import InterpolationContext
 
 
 @pytest.fixture
@@ -65,7 +65,7 @@ def build(tgt: DockerImageTarget, options: DockerOptions):
                     fs.image_refs(
                         options.default_repository,
                         options.registries(),
-                        DockerInterpolationContext(),
+                        InterpolationContext(),
                     ),
                 ),
             ),

src/python/pants/backend/docker/util_rules/docker_build_context.py

@@ -22,11 +22,7 @@
     DockerBuildEnvironmentRequest,
 )
 from pants.backend.docker.utils import get_hash, suggest_renames
-from pants.backend.docker.value_interpolation import (
-    DockerBuildArgsInterpolationValue,
-    DockerInterpolationContext,
-    DockerInterpolationValue,
-)
+from pants.backend.docker.value_interpolation import DockerBuildArgsInterpolationValue
 from pants.backend.shell.target_types import ShellSourceField
 from pants.core.goals.package import BuiltPackage, PackageFieldSet
 from pants.core.target_types import FileSourceField
@@ -49,6 +45,7 @@
 from pants.engine.unions import UnionRule
 from pants.util.meta import classproperty
 from pants.util.strutil import softwrap
+from pants.util.value_interpolation import InterpolationContext, InterpolationValue
 
 logger = logging.getLogger(__name__)
 
@@ -105,7 +102,7 @@ class DockerBuildContext:
     build_env: DockerBuildEnvironment
     upstream_image_ids: tuple[str, ...]
     dockerfile: str
-    interpolation_context: DockerInterpolationContext
+    interpolation_context: InterpolationContext
     copy_source_vs_context_source: tuple[tuple[str, str], ...]
     stages: tuple[str, ...]
 
@@ -118,7 +115,7 @@ def create(
         upstream_image_ids: Iterable[str],
         dockerfile_info: DockerfileInfo,
     ) -> DockerBuildContext:
-        interpolation_context: dict[str, dict[str, str] | DockerInterpolationValue] = {}
+        interpolation_context: dict[str, dict[str, str] | InterpolationValue] = {}
 
         if build_args:
             interpolation_context["build_args"] = cls._merge_build_args(
@@ -148,7 +145,7 @@ def create(
             dockerfile=dockerfile_info.source,
             build_env=build_env,
             upstream_image_ids=tuple(sorted(upstream_image_ids)),
-            interpolation_context=DockerInterpolationContext.from_dict(interpolation_context),
+            interpolation_context=InterpolationContext.from_dict(interpolation_context),
             copy_source_vs_context_source=tuple(
                 suggest_renames(
                     tentative_paths=(

src/python/pants/backend/docker/util_rules/docker_build_context_test.py

@@ -26,11 +26,7 @@
     DockerBuildContextRequest,
 )
 from pants.backend.docker.util_rules.docker_build_env import DockerBuildEnvironment
-from pants.backend.docker.value_interpolation import (
-    DockerBuildArgsInterpolationValue,
-    DockerInterpolationContext,
-    DockerInterpolationValue,
-)
+from pants.backend.docker.value_interpolation import DockerBuildArgsInterpolationValue
 from pants.backend.python import target_types_rules
 from pants.backend.python.goals import package_pex_binary
 from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet
@@ -47,6 +43,7 @@
 from pants.engine.internals.scheduler import ExecutionError
 from pants.testutil.pytest_util import no_exception
 from pants.testutil.rule_runner import QueryRule, RuleRunner
+from pants.util.value_interpolation import InterpolationContext, InterpolationValue
 
 
 def create_rule_runner() -> RuleRunner:
@@ -91,7 +88,7 @@ def assert_build_context(
     *,
     build_upstream_images: bool = False,
     expected_files: list[str],
-    expected_interpolation_context: dict[str, str | dict[str, str] | DockerInterpolationValue]
+    expected_interpolation_context: dict[str, str | dict[str, str] | InterpolationValue]
     | None = None,
     expected_num_upstream_images: int = 0,
     pants_args: list[str] | None = None,
@@ -125,7 +122,7 @@ def assert_build_context(
 
         # Converting to `dict` to avoid the fact that FrozenDict is sensitive to the order of the keys.
         assert dict(context.interpolation_context) == dict(
-            DockerInterpolationContext.from_dict(expected_interpolation_context)
+            InterpolationContext.from_dict(expected_interpolation_context)
         )
 
     if build_upstream_images: