Bare minimum "Pants compiles its engine" (pantsbuild#19817)

This change introduces the necessary targets to allow for Pants as it exists right now, to compile the engine. It doesn't introduce any mechanism for actually compiling the engine when doing `pants`/`./pants` runs in the repo, as that'll come later. After this change, you should be able to do `pants export-codegen src/rust/engine:engine-and-client` and get the engine/client in your distdir (as well as again but with `MODE=debug`). In addition to the targets required for wiring this up, `rust-toolchain` was also moved under `src/rust/engine`. I also tested that: - editing `rust-toolchain` re-forces a new build - editing `cargo_build_shim.sh` does not force a new build --------- Co-authored-by: Huon Wilson <[email protected]>
joshainglis · Sep 18, 2023 · 0afc51c · 0afc51c
1 parent a064a6b
commit 0afc51c
Show file tree

Hide file tree

Showing 11 changed files with 115 additions and 17 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -188,7 +188,8 @@ jobs:
     - name: Cache Rust toolchain
       uses: actions/cache@v3
       with:
-        key: macOS10-15-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2
+        key: macOS10-15-x86_64-rustup-${{ hashFiles('src/rust/engine/rust-toolchain')
+          }}-v2
         path: '~/.rustup/toolchains/1.72.0-*
 
           ~/.rustup/update-hashes
@@ -269,7 +270,8 @@ jobs:
     - name: Cache Rust toolchain
       uses: actions/cache@v3
       with:
-        key: macOS11-ARM64-rustup-${{ hashFiles('rust-toolchain') }}-v2
+        key: macOS11-ARM64-rustup-${{ hashFiles('src/rust/engine/rust-toolchain')
+          }}-v2
         path: '~/.rustup/toolchains/1.72.0-*
 
           ~/.rustup/update-hashes

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -35,7 +35,7 @@ jobs:
     - name: Cache Rust toolchain
       uses: actions/cache@v3
       with:
-        key: Linux-ARM64-rustup-${{ hashFiles('rust-toolchain') }}-v2
+        key: Linux-ARM64-rustup-${{ hashFiles('src/rust/engine/rust-toolchain') }}-v2
         path: '~/.rustup/toolchains/1.72.0-*
 
           ~/.rustup/update-hashes
@@ -125,7 +125,7 @@ jobs:
     - name: Cache Rust toolchain
       uses: actions/cache@v3
       with:
-        key: Linux-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2
+        key: Linux-x86_64-rustup-${{ hashFiles('src/rust/engine/rust-toolchain') }}-v2
         path: '~/.rustup/toolchains/1.72.0-*
 
           ~/.rustup/update-hashes
@@ -227,7 +227,7 @@ jobs:
     - name: Cache Rust toolchain
       uses: actions/cache@v3
       with:
-        key: macOS11-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2
+        key: macOS11-x86_64-rustup-${{ hashFiles('src/rust/engine/rust-toolchain') }}-v2
         path: '~/.rustup/toolchains/1.72.0-*
 
           ~/.rustup/update-hashes
@@ -434,7 +434,7 @@ jobs:
     - name: Cache Rust toolchain
       uses: actions/cache@v3
       with:
-        key: macOS10-15-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2
+        key: macOS10-15-x86_64-rustup-${{ hashFiles('src/rust/engine/rust-toolchain') }}-v2
         path: '~/.rustup/toolchains/1.72.0-*
 
           ~/.rustup/update-hashes
@@ -499,7 +499,7 @@ jobs:
     - name: Cache Rust toolchain
       uses: actions/cache@v3
       with:
-        key: macOS11-ARM64-rustup-${{ hashFiles('rust-toolchain') }}-v2
+        key: macOS11-ARM64-rustup-${{ hashFiles('src/rust/engine/rust-toolchain') }}-v2
         path: '~/.rustup/toolchains/1.72.0-*
 
           ~/.rustup/update-hashes

diff --git a/3rdparty/tools/python3/BUILD b/3rdparty/tools/python3/BUILD
@@ -0,0 +1,40 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+file(
+    name="compressed-python-build-standalone-3.9",
+    source=per_platform(
+        linux_arm64=http_source(
+            url="https://github.com/indygreg/python-build-standalone/releases/download/20230507/cpython-3.9.16+20230507-aarch64-unknown-linux-gnu-install_only.tar.gz",
+            len=23583066,
+            sha256="f629b75ebfcafe9ceee2e796b7e4df5cf8dbd14f3c021afca078d159ab797acf",
+            filename="python-build-standalone.tar.gz",
+        ),
+        linux_x86_64=http_source(
+            url="https://github.com/indygreg/python-build-standalone/releases/download/20230507/cpython-3.9.16+20230507-x86_64-unknown-linux-gnu-install_only.tar.gz",
+            len=25738357,
+            sha256="2b6e146234a4ef2a8946081fc3fbfffe0765b80b690425a49ebe40b47c33445b",
+            filename="python-build-standalone.tar.gz",
+        ),
+        macos_arm64=http_source(
+            url="https://github.com/indygreg/python-build-standalone/releases/download/20230507/cpython-3.9.16+20230507-aarch64-apple-darwin-install_only.tar.gz",
+            len=16634170,
+            sha256="c1de1d854717a6245f45262ef1bb17b09e2c587590e7e3f406593c143ff875bd",
+            filename="python-build-standalone.tar.gz",
+        ),
+        macos_x86_64=http_source(
+            url="https://github.com/indygreg/python-build-standalone/releases/download/20230507/cpython-3.9.16+20230507-x86_64-apple-darwin-install_only.tar.gz",
+            len=16908516,
+            sha256="3abc4d5fbbc80f5f848f280927ac5d13de8dc03aabb6ae65d8247cbb68e6f6bf",
+            filename="python-build-standalone.tar.gz",
+        ),
+    ),
+)
+
+shell_command(
+    name="python3",
+    command="tar -xzf python-build-standalone.tar.gz",
+    tools=["tar", "gzip"],
+    execution_dependencies=[":compressed-python-build-standalone-3.9"],
+    output_directories=["python"],
+)
diff --git a/build-support/bin/classify_changed_files.py b/build-support/bin/classify_changed_files.py
@@ -28,7 +28,6 @@ class Affected(enum.Enum):
 ]
 _rust_globs = [
     "src/rust/engine/*",
-    "rust-toolchain",
     "build-support/bin/rust/*",
 ]
 _release_globs = [

diff --git a/build-support/bin/classify_changed_files_test.py b/build-support/bin/classify_changed_files_test.py
@@ -21,9 +21,16 @@
         [["src/python/pants/VERSION"], {Affected.release}],
         [["src/python/pants_release/generate_github_workflows.py"], {Affected.ci_config}],
         [["src/python/pants/whatever.py"], {Affected.other}],
-        [["docs/path/to/some/doc", "rust-toolchain"], {Affected.docs, Affected.rust}],
         [
-            ["docs/path/to/some/doc", "rust-toolchain", "src/python/pants/whatever.py"],
+            ["docs/path/to/some/doc", "src/rust/engine/rust-toolchain"],
+            {Affected.docs, Affected.rust},
+        ],
+        [
+            [
+                "docs/path/to/some/doc",
+                "src/rust/engine/rust-toolchain",
+                "src/python/pants/whatever.py",
+            ],
             {Affected.docs, Affected.rust, Affected.other},
         ],
     ),

diff --git a/build-support/bin/rust/calculate_engine_hash.sh b/build-support/bin/rust/calculate_engine_hash.sh
@@ -19,8 +19,6 @@ case "$MODE" in
   *) MODE_FLAG="--release" ;;
 esac
 
-RUST_TOOLCHAIN_CONTENTS="$(cat "${REPO_ROOT}/rust-toolchain")"
-
 function calculate_current_hash() {
   # Cached and unstaged files, with ignored files excluded.
   # NB: We fork a subshell because one or both of `ls-files`/`hash-object` are
@@ -33,15 +31,13 @@ function calculate_current_hash() {
     cd "${REPO_ROOT}" || exit 1
     (
       echo "${MODE_FLAG}"
-      echo "${RUST_TOOLCHAIN_CONTENTS}"
       uname -mps
       # the engine only depends on the implementation and major.minor version, not the patch
       "${PY}" -c 'import sys; print(sys.implementation.name, sys.version_info.major, sys.version_info.minor)'
       git ls-files --cached --others --exclude-standard \
         "${NATIVE_ROOT}" \
-        "${REPO_ROOT}/rust-toolchain" \
         "${REPO_ROOT}/build-support/bin/rust" |
-        grep -v -E -e "/BUILD$" -e "/[^/]*\.md$" |
+        grep -v -E -e "/BUILD$" -e "/[^/]*\.md$" -e "src/rust/engine/cargo_build_shim\.sh$" |
         git hash-object --stdin-paths
     ) | fingerprint_data
   )

diff --git a/pants.toml b/pants.toml
@@ -68,6 +68,7 @@ pants_ignore.add = [
   "/docs/node_modules",
   # We have Pants stuff in here
   "!.github/",
+  "!/src/rust/engine/.cargo",
 ]
 
 build_ignore.add = [

diff --git a/src/python/pants_release/generate_github_workflows.py b/src/python/pants_release/generate_github_workflows.py
@@ -264,7 +264,7 @@ def global_env() -> Env:
 
 
 def rust_channel() -> str:
-    with open("rust-toolchain") as fp:
+    with open("src/rust/engine/rust-toolchain") as fp:
         rust_toolchain = toml.load(fp)
     return cast(str, rust_toolchain["toolchain"]["channel"])
 
@@ -446,7 +446,7 @@ def rust_caches(self) -> Sequence[Step]:
                 "uses": "actions/cache@v3",
                 "with": {
                     "path": f"~/.rustup/toolchains/{rust_channel()}-*\n~/.rustup/update-hashes\n~/.rustup/settings.toml\n",
-                    "key": f"{self.platform_name()}-rustup-{hash_files('rust-toolchain')}-v2",
+                    "key": f"{self.platform_name()}-rustup-{hash_files('src/rust/engine/rust-toolchain')}-v2",
                 },
             },
             {

diff --git a/src/rust/engine/BUILD b/src/rust/engine/BUILD
@@ -0,0 +1,39 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+files(
+    name="rust_sources",
+    sources=[
+        "Cargo.lock",
+        "build.rs",
+        "**/Cargo.toml",
+        "**/*.rs",
+        "!**/*tests.rs",
+        "**/*.proto",
+        ".cargo/config",
+        "rust-toolchain",
+    ],
+)
+
+shell_sources(name="shell")
+
+shell_command(
+    name="engine-and-client",
+    command=f"./cargo_build_shim.sh --features=extension-module -p engine -p client",
+    execution_dependencies=[
+        ":rust_sources",
+        "./cargo_build_shim.sh:shell",
+        "3rdparty/tools/python3:python3",
+    ],
+    extra_env_vars=["CHROOT={chroot}", "MODE"],
+    tools=["bash", "cc", "ld", "as", "ar", "cargo", "protoc", "python3.9"],
+    output_files=[
+        f"target/debug/libengine.so",
+        f"target/debug/libengine.dylib",
+        f"target/debug/pants",
+        f"target/release/libengine.so",
+        f"target/release/libengine.dylib",
+        f"target/release/pants",
+    ],
+    timeout=600,
+)
diff --git a/src/rust/engine/cargo_build_shim.sh b/src/rust/engine/cargo_build_shim.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# This simply exists because `shell_command` at the time of writing doesn't know how to merge `PATH`
+# from the rule's env and the `extra_env_vars`.
+
+PYTHON_PATH="${CHROOT}/3rdparty/tools/python3/python/bin"
+export PATH="$PATH:$PYTHON_PATH"
+
+RELTYPE_FLAG=""
+[ "$MODE" == "debug" ] || RELTYPE_FLAG="--release"
+
+cargo build $RELTYPE_FLAG "$@"
diff --git a/rust-toolchain → src/rust/engine/rust-toolchain b/rust-toolchain → src/rust/engine/rust-toolchain