go: gather and link transitive prebuilt object files for cgo mode (pa…

…ntsbuild#17971)

The `go` tool will gather and link transitive prebuilt object files for a package (`.syso` files) into the cgo output object file. This is necessary to avoid linking errors due to missing libraries in some cases with packages using cgo. (I actually encountered just such a linker error when developing pantsbuild#17914.)

See `go` tool source: https://github.com/golang/go/blob/6ad27161f8d1b9c5e03fb3415977e1d3c3b11323/src/cmd/go/internal/work/exec.go#L3291-L3311 

This is part of fixing pantsbuild#17950 and was extracted from pantsbuild#17914.

src/python/pants/backend/go/util_rules/build_pkg.py

@@ -5,6 +5,7 @@
 import dataclasses
 import hashlib
 import os.path
+from collections import deque
 from dataclasses import dataclass
 from pathlib import PurePath
 from typing import Iterable
@@ -426,6 +427,37 @@ async def _maybe_copy_headers_to_platform_independent_names(
         return None
 
 
+# Gather transitive prebuilt object files for Cgo. Traverse the provided dependencies and lifts `.syso`
+# object files into a single `Digest`.
+@rule_helper
+async def _gather_transitive_prebuilt_object_files(
+    build_request: BuildGoPackageRequest,
+) -> tuple[Digest, frozenset[str]]:
+    prebuilt_objects: list[tuple[Digest, list[str]]] = []
+
+    queue: deque[BuildGoPackageRequest] = deque([build_request])
+    while queue:
+        pkg = queue.popleft()
+        queue.extend(pkg.direct_dependencies)
+        if pkg.prebuilt_object_files:
+            prebuilt_objects.append(
+                (
+                    pkg.digest,
+                    [
+                        os.path.join(pkg.dir_path, obj_file)
+                        for obj_file in pkg.prebuilt_object_files
+                    ],
+                )
+            )
+
+    object_digest = await Get(Digest, MergeDigests([digest for digest, _ in prebuilt_objects]))
+    object_files = set()
+    for _, files in prebuilt_objects:
+        object_files.update(files)
+
+    return object_digest, frozenset(object_files)
+
+
 # NB: We must have a description for the streaming of this rule to work properly
 # (triggered by `FallibleBuiltGoPackage` subclassing `EngineAwareReturnType`).
 @rule(desc="Compile with Go", level=LogLevel.DEBUG)
@@ -501,7 +533,7 @@ async def build_go_package(
     # Add any prebuilt object files (".syso" extension) to the list of objects to link into the package.
     if request.prebuilt_object_files:
         objects.extend(
-            (f"./{request.dir_path}/{prebuilt_object_file}", request.digest)
+            (os.path.join(request.dir_path, prebuilt_object_file), request.digest)
             for prebuilt_object_file in request.prebuilt_object_files
         )
 
@@ -517,6 +549,11 @@ async def build_go_package(
                 f"Package {request.import_path} is a cgo package but contains Go assembly files."
             )
 
+        # Gather all prebuilt object files transitively and pass them to the Cgo rule for linking into the
+        # Cgo object output. This is necessary to avoid linking errors.
+        # See https://github.com/golang/go/blob/6ad27161f8d1b9c5e03fb3415977e1d3c3b11323/src/cmd/go/internal/work/exec.go#L3291-L3311.
+        transitive_prebuilt_object_files = await _gather_transitive_prebuilt_object_files(request)
+
         assert request.cgo_flags is not None
         cgo_compile_result = await Get(
             CGoCompileResult,
@@ -533,6 +570,7 @@ async def build_go_package(
                 cxx_files=request.cxx_files,
                 objc_files=request.objc_files,
                 fortran_files=request.fortran_files,
+                transitive_prebuilt_object_files=transitive_prebuilt_object_files,
             ),
         )
         assert cgo_compile_result is not None
@@ -543,7 +581,6 @@ async def build_go_package(
                 for obj_file in cgo_compile_result.output_obj_files
             ]
         )
-        s_files = []  # Clear s_files since assembly has already been handled in cgo rules.
 
     # Copy header files with platform-specific values in their name to platform independent names.
     # For example, defs_linux_amd64.h becomes defs_GOOS_GOARCH.h.

src/python/pants/backend/go/util_rules/cgo.py

@@ -64,6 +64,7 @@ class CGoCompileRequest(EngineAwareParameter):
     objc_files: tuple[str, ...] = ()
     fortran_files: tuple[str, ...] = ()
     s_files: tuple[str, ...] = ()
+    transitive_prebuilt_object_files: tuple[Digest, frozenset[str]] | None = None
 
     def debug_hint(self) -> str | None:
         return self.import_path
@@ -455,6 +456,8 @@ async def _dynimport(
     import_go_path: str,
     golang_env_aware: GolangSubsystem.EnvironmentAware,
     use_cxx_linker: bool,
+    transitive_prebuilt_objects_digest: Digest,
+    transitive_prebuilt_objects: frozenset[str],
 ) -> _DynImportResult:
     cgo_main_compile_process = await _cc(
         binary_name=golang_env_aware.cgo_gcc_binary_name,
@@ -468,12 +471,16 @@ async def _dynimport(
     )
     cgo_main_compile_result = await Get(ProcessResult, Process, cgo_main_compile_process)
     obj_digest = await Get(
-        Digest, MergeDigests([input_digest, cgo_main_compile_result.output_digest])
+        Digest,
+        MergeDigests(
+            [
+                input_digest,
+                cgo_main_compile_result.output_digest,
+                transitive_prebuilt_objects_digest,
+            ]
+        ),
     )
 
-    # TODO(#16827): Gather .syso files from this package and all (transitive) dependencies. Cgo support requires
-    # linking all object files with `.syso` extension into the package archive.
-
     dynobj = os.path.join(dir_path, "_cgo_.o")
     ldflags = list(ldflags)
     if (goroot.goarch == "arm" and goroot.goos == "linux") or goroot.goos == "android":
@@ -498,7 +505,11 @@ async def _dynimport(
         dir_path=dir_path,
         outfile=dynobj,
         flags=ldflags,
-        objs=[*obj_files, os.path.join(dir_path, "_cgo_main.o")],
+        objs=[
+            *obj_files,
+            os.path.join(dir_path, "_cgo_main.o"),
+            *sorted(transitive_prebuilt_objects),
+        ],
         description=f"Link _cgo_.o ({import_path})",
     )
     if cgo_binary_link_result.exit_code != 0:
@@ -897,6 +908,12 @@ async def cgo_compile_request(
             ]
         ),
     )
+    transitive_prebuilt_objects_digest: Digest = EMPTY_DIGEST
+    transitive_prebuilt_objects: frozenset[str] = frozenset()
+    if request.transitive_prebuilt_object_files:
+        transitive_prebuilt_objects_digest = request.transitive_prebuilt_object_files[0]
+        transitive_prebuilt_objects = request.transitive_prebuilt_object_files[1]
+
     dynimport_result = await _dynimport(
         import_path=request.import_path,
         input_digest=dynimport_input_digest,
@@ -909,6 +926,8 @@ async def cgo_compile_request(
         import_go_path=os.path.join(dir_path, "_cgo_import.go"),
         golang_env_aware=golang_env_aware,
         use_cxx_linker=bool(request.cxx_files),
+        transitive_prebuilt_objects_digest=transitive_prebuilt_objects_digest,
+        transitive_prebuilt_objects=transitive_prebuilt_objects,
     )
     if dynimport_result.dyn_out_go:
         go_files.append(dynimport_result.dyn_out_go)