Support linux arm64 binary tools. (pantsbuild#13249)

Adds the ability to differentiate the architecture, and adds
defaults for all binary tools that have an arm64 release, or that
I was able to build on that architecture.

The only tool that does not currently build on linux arm64 is Hadolint.

[ci skip-rust]

[ci skip-build-wheels]

src/python/pants/backend/codegen/protobuf/protoc.py

@@ -13,6 +13,7 @@ class Protoc(TemplatedExternalTool):
 
     default_version = "3.11.4"
     default_known_versions = [
+        "3.11.4|linux_arm64 |f24c9fa1fc4a7770b8a5da66e515cb8a638d086ad2afa633abb97137c5f029a8|1481946",
         "3.11.4|linux_x86_64|6d0f18cd84b918c7b3edd0203e75569e0c8caecb1367bbbe409b45e28514f5be|1591191",
         "3.11.4|macos_arm64 |8c6af11e1058efe953830ecb38324c0e0fd2fb67df3891896d138c535932e7db|2482119",
         "3.11.4|macos_x86_64|8c6af11e1058efe953830ecb38324c0e0fd2fb67df3891896d138c535932e7db|2482119",
@@ -22,6 +23,7 @@ class Protoc(TemplatedExternalTool):
         "v{version}/protoc-{version}-{platform}.zip"
     )
     default_url_platform_mapping = {
+        "linux_arm64": "linux-aarch_64",
         "linux_x86_64": "linux-x86_64",
         "macos_arm64": "osx-x86_64",  # May require rosetta, but output is arch-independent
         "macos_x86_64": "osx-x86_64",

src/python/pants/backend/codegen/protobuf/python/grpc_python_plugin.py

@@ -12,14 +12,16 @@ class GrpcPythonPlugin(TemplatedExternalTool):
     default_known_versions = [
         "1.32.0|macos_arm64 |b2db586656463841aa2fd4aab34fb6bd3ef887b522d80e4f2f292146c357f533|6215304",
         "1.32.0|macos_x86_64|b2db586656463841aa2fd4aab34fb6bd3ef887b522d80e4f2f292146c357f533|6215304",
+        "1.32.0|linux_arm64 |9365e728c603d64735963074340994245d324712344f63557ef3630864dd9f52|5233664",
         "1.32.0|linux_x86_64|1af99df9bf733c17a75cbe379f3f9d9ff1627d8a8035ea057c3c78575afe1687|4965728",
     ]
     default_url_template = (
         "https://binaries.pantsbuild.org/bin/grpc_python_plugin/{version}/"
-        "{platform}/x86_64/grpc_python_plugin"
+        "{platform}/grpc_python_plugin"
     )
     default_url_platform_mapping = {
         "macos_arm64": "macos",
         "macos_x86_64": "macos",
-        "linux_x86_64": "linux",
+        "linux_arm64": "linux/arm64",
+        "linux_x86_64": "linux/x86_64",
     }

src/python/pants/backend/docker/lint/hadolint/subsystem.py

@@ -16,10 +16,12 @@ class Hadolint(TemplatedExternalTool):
     help = "A linter for Dockerfiles."
 
     default_version = "v2.6.0"
+    # TODO: https://github.com/hadolint/hadolint/issues/411 tracks building and releasing
+    #  hadolint for Linux ARM64.
     default_known_versions = [
-        f"{default_version}|macos_arm64 |7d41496bf591f2b9c7daa76d4aa1db04ea97b9e11b44a24a4e404a10aab33686|2392080",
-        f"{default_version}|macos_x86_64|7d41496bf591f2b9c7daa76d4aa1db04ea97b9e11b44a24a4e404a10aab33686|2392080",
-        f"{default_version}|linux_x86_64|152e3c3375f26711650d4e11f9e382cf1bdf3f912d7379823e8fac4b1bce88d6|5812840",
+        "v2.6.0|macos_arm64 |7d41496bf591f2b9c7daa76d4aa1db04ea97b9e11b44a24a4e404a10aab33686|2392080",
+        "v2.6.0|macos_x86_64|7d41496bf591f2b9c7daa76d4aa1db04ea97b9e11b44a24a4e404a10aab33686|2392080",
+        "v2.6.0|linux_x86_64|152e3c3375f26711650d4e11f9e382cf1bdf3f912d7379823e8fac4b1bce88d6|5812840",
     ]
     default_url_template = (
         "https://github.com/hadolint/hadolint/releases/download/{version}/hadolint-{platform}"

src/python/pants/backend/project_info/count_loc.py

@@ -23,18 +23,20 @@ class SuccinctCodeCounter(TemplatedExternalTool):
     options_scope = "scc"
     help = "The Succinct Code Counter, aka `scc` (https://github.com/boyter/scc)."
 
-    default_version = "2.12.0"
+    default_version = "3.0.0"
     default_known_versions = [
-        "2.12.0|macos_arm64 |70b7002cd1e4541cb37b7b9cbc0eeedd13ceacb49628e82ab46332bb2e65a5a6|1842530",
-        "2.12.0|macos_x86_64|70b7002cd1e4541cb37b7b9cbc0eeedd13ceacb49628e82ab46332bb2e65a5a6|1842530",
-        "2.12.0|linux_x86_64|8eca3e98fe8a78d417d3779a51724515ac4459760d3ec256295f80954a0da044|1753059",
+        "3.0.0|macos_arm64 |846cb1b25025a0794d455719bc17cfb3f588576a58af1d95036f6c654e294f98|2006145",
+        "3.0.0|macos_x86_64|9c3064e477ab36e16204ad34f649372034bca4df669615eff5de4aa05b2ddf1a|2048134",
+        "3.0.0|linux_arm64 |04f9e797b70a678833e49df5e744f95080dfb7f963c0cd34f5b5d4712d290f33|1768037",
+        "3.0.0|linux_x86_64|13ca47ce00b5bd032f97f3af7aa8eb3c717b8972b404b155a378b09110e4aa0c|1948341",
     ]
     default_url_template = (
-        "https://github.com/boyter/scc/releases/download/v{version}/scc-{version}-" "{platform}.zip"
+        "https://github.com/boyter/scc/releases/download/v{version}/scc-{version}-{platform}.zip"
     )
     default_url_platform_mapping = {
-        "macos_arm64": "x86_64-apple-darwin",  # TODO: use native arm64 binary for version 3.0
+        "macos_arm64": "arm64-apple-darwin",
         "macos_x86_64": "x86_64-apple-darwin",
+        "linux_arm64": "arm64-unknown-linux",
         "linux_x86_64": "x86_64-unknown-linux",
     }
 

src/python/pants/backend/shell/lint/shellcheck/subsystem.py

@@ -18,9 +18,10 @@ class Shellcheck(TemplatedExternalTool):
 
     default_version = "v0.7.1"
     default_known_versions = [
-        f"{default_version}|macos_arm64 |b080c3b659f7286e27004aa33759664d91e15ef2498ac709a452445d47e3ac23|1348272",
-        f"{default_version}|macos_x86_64|b080c3b659f7286e27004aa33759664d91e15ef2498ac709a452445d47e3ac23|1348272",
-        f"{default_version}|linux_x86_64|64f17152d96d7ec261ad3086ed42d18232fcb65148b44571b564d688269d36c8|1443836",
+        "v0.7.1|macos_arm64 |b080c3b659f7286e27004aa33759664d91e15ef2498ac709a452445d47e3ac23|1348272",
+        "v0.7.1|macos_x86_64|b080c3b659f7286e27004aa33759664d91e15ef2498ac709a452445d47e3ac23|1348272",
+        "v0.7.1|linux_arm64 |b50cc31509b354ab5bbfc160bc0967567ed98cd9308fd43f38551b36cccc4446|1432492",
+        "v0.7.1|linux_x86_64|64f17152d96d7ec261ad3086ed42d18232fcb65148b44571b564d688269d36c8|1443836",
     ]
 
     default_url_template = (
@@ -30,6 +31,7 @@ class Shellcheck(TemplatedExternalTool):
     default_url_platform_mapping = {
         "macos_arm64": "darwin.x86_64",
         "macos_x86_64": "darwin.x86_64",
+        "linux_arm64": "linux.aarch64",
         "linux_x86_64": "linux.x86_64",
     }
 

src/python/pants/backend/shell/lint/shfmt/subsystem.py

@@ -18,9 +18,10 @@ class Shfmt(TemplatedExternalTool):
 
     default_version = "v3.2.4"
     default_known_versions = [
-        f"{default_version}|macos_arm64 |e70fc42e69debe3e400347d4f918630cdf4bf2537277d672bbc43490387508ec|2998546",
-        f"{default_version}|macos_x86_64|43a0461a1b54070ddc04fbbf1b78f7861ee39a65a61f5466d15a39c4aba4f917|2980208",
-        f"{default_version}|linux_x86_64|3f5a47f8fec27fae3e06d611559a2063f5d27e4b9501171dde9959b8c60a3538|2797568",
+        "v3.2.4|macos_arm64 |e70fc42e69debe3e400347d4f918630cdf4bf2537277d672bbc43490387508ec|2998546",
+        "v3.2.4|macos_x86_64|43a0461a1b54070ddc04fbbf1b78f7861ee39a65a61f5466d15a39c4aba4f917|2980208",
+        "v3.2.4|linux_arm64 |6474d9cc08a1c9fe2ef4be7a004951998e3067d46cf55a011ddd5ff7bfab3de6|2752512",
+        "v3.2.4|linux_x86_64|3f5a47f8fec27fae3e06d611559a2063f5d27e4b9501171dde9959b8c60a3538|2797568",
     ]
 
     default_url_template = (
@@ -29,6 +30,7 @@ class Shfmt(TemplatedExternalTool):
     default_url_platform_mapping = {
         "macos_arm64": "darwin_arm64",
         "macos_x86_64": "darwin_amd64",
+        "linux_arm64": "linux_arm64",
         "linux_x86_64": "linux_amd64",
     }
 

src/python/pants/base/exception_sink_test.py

@@ -55,6 +55,10 @@ def test_set_invalid_log_location():
             "The provided log location path at '/' is not writable or could not be created: "
             "[Errno 21] Is a directory: '/'."
         ),
+        Platform.linux_arm64: (
+            "Error opening fatal error log streams for log location '/': [Errno 13] Permission "
+            "denied: '/.pids'"
+        ),
         Platform.linux_x86_64: (
             "Error opening fatal error log streams for log location '/': [Errno 13] Permission "
             "denied: '/.pids'"

src/python/pants/core/util_rules/external_tool.py

@@ -68,6 +68,7 @@ class MyExternalTool(ExternalTool):
         options_scope = "my-external-tool"
         default_version = "1.2.3"
         default_known_versions = [
+          "1.2.3|linux_arm64 |feed6789feed6789feed6789feed6789feed6789feed6789feed6789feed6789|112233",
           "1.2.3|linux_x86_64|cafebabacafebabacafebabacafebabacafebabacafebabacafebabacafebaba|878986",
           "1.2.3|macos_arm64 |deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef|222222",
           "1.2.3|macos_x86_64|1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd|333333",

src/python/pants/engine/platform.py

@@ -12,6 +12,7 @@
 
 
 class Platform(Enum):
+    linux_arm64 = "linux_arm64"
     linux_x86_64 = "linux_x86_64"
     macos_arm64 = "macos_arm64"
     macos_x86_64 = "macos_x86_64"

src/python/pants/jvm/resolve/coursier_setup.py

@@ -60,6 +60,7 @@ class CoursierBinary(TemplatedExternalTool):
 
     default_version = "v2.0.13"
     default_known_versions = [
+        "v2.0.13|linux_arm64 |8d428bede2d9d0e48ffad8360d49de48bd0c2c3b0e54e82e3a7665019b65e4d0|58622664",
         "v2.0.13|linux_x86_64|1ae089789cc4b0a4d296d6852b760d7f8bf72805267a6b7571e99b681d5e13b4|59652208",
         "v2.0.13|macos_arm64 |d74b8fe4ffc2f4e9011d7151722fc8b5ffca8a72b3bc4188c61df3326228c4ef|57625024",
         "v2.0.13|macos_x86_64|d74b8fe4ffc2f4e9011d7151722fc8b5ffca8a72b3bc4188c61df3326228c4ef|57625024",
@@ -70,6 +71,7 @@ class CoursierBinary(TemplatedExternalTool):
     default_url_platform_mapping = {
         "macos_arm64": "x86_64-apple-darwin",
         "macos_x86_64": "x86_64-apple-darwin",
+        "linux_arm64": "aarch64-pc-linux",
         "linux_x86_64": "x86_64-pc-linux",
     }