Pass derivation details through from Rust. (pantsbuild#21631)

This allows us to specify which config file each value came from.

We intern the details strings, as they are very repetitive. This
requires
tying the lifetime of various data structures to that of the GIL.

src/python/pants/engine/internals/native_engine.pyi

@@ -662,10 +662,8 @@ class PyConfigSource:
 # See src/rust/engine/src/externs/options.rs for the Rust-side versions of these types.
 T = TypeVar("T")
 
-# List of pairs of (value, ranks of the sources of the value).
-# A scalar value will always have a single source. A list/dict value
-# may come from multiple sources, if its elements were appended.
-OptionValueDerivation = list[Tuple[T, list[int]]]
+# List of tuples of (value, rank, details string).
+OptionValueDerivation = list[Tuple[T, int, str]]
 
 # A tuple (value, rank of value, optional derivation of value).
 OptionValue = Tuple[Optional[T], int, Optional[OptionValueDerivation]]

src/python/pants/help/help_info_extracter.py

@@ -1031,19 +1031,8 @@ def get_option_scope_help_info(
             empty_details = "" if (is_list or is_dict) else None
             ranked_values.append(RankedValue(Rank.NONE, empty_val, empty_details))
 
-            for value, ranks in derivation:
-                if len(ranks) == 0:
-                    rank = Rank.NONE
-                    details = None
-                else:
-                    rank = ranks[-1]
-                    # Again, distinguishing between '' vs None in the details field
-                    # does not matter, but we want to be consistent with the idiosyncratic
-                    # behavior of the legacy parser, until we get rid of it.
-                    details = (
-                        ", ".join(filter(None, [r.description() for r in ranks])) or empty_details
-                    )
-                ranked_values.append(RankedValue(rank, value, details))
+            for value, rank, details in derivation:
+                ranked_values.append(RankedValue(rank, value, details or empty_details))
             history = OptionValueHistory(tuple(ranked_values))
             ohi = self.get_option_help_info(args, kwargs)
             ohi = dataclasses.replace(ohi, value_history=history)

src/python/pants/help/help_info_extracter_test.py

@@ -324,7 +324,7 @@ def fake_consumed_scopes_mapper(scope: str) -> Tuple[str, ...]:
                                 {
                                     "rank": Rank.ENVIRONMENT,
                                     "value": [42, 99, 88],
-                                    "details": "from config, from an env var",
+                                    "details": "pants.test.toml, env var",
                                 },
                             ),
                         },
@@ -388,7 +388,7 @@ def fake_consumed_scopes_mapper(scope: str) -> Tuple[str, ...]:
                                 {"details": "", "rank": Rank.NONE, "value": []},
                                 {"details": "", "rank": Rank.HARDCODED, "value": []},
                                 {
-                                    "details": "from command-line flag",
+                                    "details": "command-line flag",
                                     "rank": Rank.FLAG,
                                     "value": ["internal_plugins.releases"],
                                 },
@@ -647,7 +647,7 @@ def fake_consumed_scopes_mapper(scope: str) -> Tuple[str, ...]:
                         {
                             "rank": Rank.ENVIRONMENT,
                             "value": [42, 99, 88],
-                            "details": "from config, from an env var",
+                            "details": "pants.test.toml, env var",
                         },
                     ),
                 },
@@ -711,7 +711,7 @@ def fake_consumed_scopes_mapper(scope: str) -> Tuple[str, ...]:
                         {"details": "", "rank": Rank.NONE, "value": []},
                         {"details": "", "rank": Rank.HARDCODED, "value": []},
                         {
-                            "details": "from command-line flag",
+                            "details": "command-line flag",
                             "rank": Rank.FLAG,
                             "value": ["internal_plugins.releases"],
                         },

src/python/pants/option/native_options.py

@@ -116,15 +116,15 @@ def get_value(self, *, scope, registration_args, registration_kwargs) -> Tuple[A
 
     def get_derivation(
         self, *, scope, registration_args, registration_kwargs
-    ) -> list[Tuple[Any, list[Rank]]]:
+    ) -> list[Tuple[Any, Rank, Optional[str]]]:
         _, _, derivation = self._get_value_and_derivation(
             scope, registration_args, registration_kwargs
         )
         return derivation
 
     def _get_value_and_derivation(
         self, scope, registration_args, registration_kwargs
-    ) -> Tuple[Any, Rank, list[Tuple[Any, list[Rank]]]]:
+    ) -> Tuple[Any, Rank, list[Tuple[Any, Rank, Optional[str]]]]:
         return self._get(
             scope=scope,
             dest=parse_dest(*registration_args, **registration_kwargs),
@@ -147,7 +147,7 @@ def _get(
         member_type=None,
         choices=None,
         passthrough=False,
-    ) -> Tuple[Any, Rank, list[Tuple[Any, list[Rank]]]]:
+    ) -> Tuple[Any, Rank, list[Tuple[Any, Rank, Optional[str]]]]:
         def scope_str() -> str:
             return "global scope" if scope == GLOBAL_SCOPE else f"scope '{scope}'"
 
@@ -235,9 +235,7 @@ def process_value(v):
             return v
 
         if derivation:
-            derivation = [
-                (process_value(v), [self.int_to_rank[r] for r in rs]) for (v, rs) in derivation
-            ]
+            derivation = [(process_value(v), self.int_to_rank[r], d) for (v, r, d) in derivation]
 
         if val is not None:
             val = process_value(val)