Add codec for ExternalTool known version field (pantsbuild#15950)

ExternalTool has known_versions as packed strings in the form `"1.2.3|linux_arm64 |feed6789feed6789feed6789feed6789feed6789feed6789feed6789feed6789|112233"`. This MR adds an internal dataclass to handle encoding and decoding this field. Having a separate codec (which includes encoding) will help with tooling to generate these. For example, this MR is prework for scraping all the versions of Terraform.

Backwards compatibility note: This is fully backwards compatible. This MR only extracts the codec, and uses it as a container for returning values. Packed strings are still used in positions where they were before, so any user configs will still work. This MR does not add the ability to supply either a string or an ExternalToolVersion; although used in the Terraform tool, the dataclasses are encoded to strings. (I wasn't sure if there was an easy way to have a field that is a `Union[ExternalToolVersion, str]`). This MR also preserves the signature of the original `split_known_version_str` method, in case it is used in plugins; it modifies the body to forward to the codec. (I guess we could deprecate this if we wanted.)

[ci skip-rust]

pants-plugins/internal_plugins/releases/register.py

@@ -167,15 +167,15 @@ async def check_default_tools(
             tool_cls = si.subsystem_cls
             console.print_stdout(f"Checking {console.cyan(tool_cls.name)}:")
             for known_version in tool_cls.default_known_versions:
-                ver, plat_val, sha256, length = tool_cls.split_known_version_str(known_version)
+                version = tool_cls.decode_known_version(known_version)
                 # Note that we don't want to use the real option values here - we want to
                 # verify that the *defaults* aren't broken. However the get_request_for() method
                 # requires an instance (since it can consult option values, including custom
                 # options for specific tools, that we don't know about), so we construct a
                 # default one, but we force the --version to the one we're checking (which will
                 # typically be the same as the default version, but doesn't have to be, if the
                 # tool provides default_known_versions for versions other than default_version).
-                args = ("./pants", f"--{scope}-version={ver}")
+                args = ("./pants", f"--{scope}-version={version.version}")
                 blank_opts = await Get(
                     _Options,
                     SessionValues(
@@ -187,8 +187,8 @@ async def check_default_tools(
                     ),
                 )
                 instance = tool_cls(blank_opts.options.for_scope(scope))
-                req = instance.get_request_for(plat_val, sha256, length)
-                console.write_stdout(f"  version {ver} for {plat_val}... ")
+                req = instance.get_request_for(version.platform, version.sha256, version.filesize)
+                console.write_stdout(f"  version {version.version} for {version.platform}... ")
                 # TODO: We'd like to run all the requests concurrently, but since we can't catch
                 #  engine exceptions, we wouldn't have an easy way to output which one failed.
                 await Get(DownloadedExternalTool, ExternalToolRequest, req)

src/python/pants/backend/terraform/tool.py

@@ -7,6 +7,7 @@
 from pants.core.util_rules.external_tool import (
     DownloadedExternalTool,
     ExternalToolRequest,
+    ExternalToolVersion,
     TemplatedExternalTool,
 )
 from pants.engine.fs import EMPTY_DIGEST, Digest, MergeDigests
@@ -37,9 +38,27 @@ class TerraformTool(TemplatedExternalTool):
     @classproperty
     def default_known_versions(cls):
         return [
-            "1.0.7|macos_arm64 |cbab9aca5bc4e604565697355eed185bb699733811374761b92000cc188a7725|32071346",
-            "1.0.7|macos_x86_64|80ae021d6143c7f7cbf4571f65595d154561a2a25fd934b7a8ccc1ebf3014b9b|33020029",
-            "1.0.7|linux_x86_64|bc79e47649e2529049a356f9e60e06b47462bf6743534a10a4c16594f443be7b|32671441",
+            v.encode()
+            for v in [
+                ExternalToolVersion(
+                    "1.0.7",
+                    "macos_arm64",
+                    "cbab9aca5bc4e604565697355eed185bb699733811374761b92000cc188a7725",
+                    32071346,
+                ),
+                ExternalToolVersion(
+                    "1.0.7",
+                    "macos_x86_64",
+                    "80ae021d6143c7f7cbf4571f65595d154561a2a25fd934b7a8ccc1ebf3014b9b",
+                    33020029,
+                ),
+                ExternalToolVersion(
+                    "1.0.7",
+                    "linux_x86_64",
+                    "bc79e47649e2529049a356f9e60e06b47462bf6743534a10a4c16594f443be7b",
+                    32671441,
+                ),
+            ]
         ]
 
     tailor = BoolOption(

src/python/pants/core/util_rules/external_tool.py

@@ -59,6 +59,22 @@ class DownloadedExternalTool:
     exe: str
 
 
+@dataclass(frozen=True)
+class ExternalToolVersion:
+    version: str
+    platform: str
+    sha256: str
+    filesize: int
+
+    def encode(self) -> str:
+        return "|".join([self.version, self.platform, self.sha256, str(self.filesize)])
+
+    @classmethod
+    def decode(cls, version_str: str) -> ExternalToolVersion:
+        version, platform, sha256, filesize = [x.strip() for x in version_str.split("|")]
+        return cls(version, platform, sha256, int(filesize))
+
+
 class ExternalTool(Subsystem, metaclass=ABCMeta):
     """Configuration for an invocable tool that we download from an external source.
 
@@ -185,23 +201,27 @@ def generate_exe(self, plat: Platform) -> str:
     def get_request(self, plat: Platform) -> ExternalToolRequest:
         """Generate a request for this tool."""
         for known_version in self.known_versions:
-            ver, plat_val, sha256, length = self.split_known_version_str(known_version)
-            if plat.value == plat_val and ver == self.version:
-                return self.get_request_for(plat_val, sha256, length)
+            version = self.decode_known_version(known_version)
+            if plat.value == version.platform and version.version == self.version:
+                return self.get_request_for(version.platform, version.sha256, version.filesize)
         raise UnknownVersion(
             f"No known version of {self.name} {self.version} for {plat.value} found in "
             f"{self.known_versions}"
         )
 
     @classmethod
-    def split_known_version_str(cls, known_version: str) -> tuple[str, str, str, int]:
+    def decode_known_version(cls, known_version: str) -> ExternalToolVersion:
         try:
-            ver, plat_val, sha256, length = (x.strip() for x in known_version.split("|"))
+            return ExternalToolVersion.decode(known_version)
         except ValueError:
             raise ExternalToolError(
                 f"Bad value for [{cls.options_scope}].known_versions: {known_version}"
             )
-        return ver, plat_val, sha256, int(length)
+
+    @classmethod
+    def split_known_version_str(cls, known_version: str) -> tuple[str, str, str, int]:
+        version = cls.decode_known_version(known_version)
+        return version.version, version.platform, version.sha256, version.filesize
 
     def get_request_for(self, plat_val: str, sha256: str, length: int) -> ExternalToolRequest:
         """Generate a request for this tool from the given info."""