javascript: jsconfig (tsconfig) parsing for dependency inference (pan…

…tsbuild#21176)

This adds support for a lot of "extensions" that bundlers allow for
javascript that regular node does not.

In particular, inference can now find:
1. "index files" by assuming that a matching "directory import" attempts
to target said file
2. "extensionless files", essentially imports that omit the
file-extensions. This has the potential to be extended to any kind of
source you want to, however this PR does not add support for additional
extensions.
3. Files that are mapped via the "paths" mapping. This had enough
analogies with subpath imports that it reuses the latter wholesale, but
is applied after the first one.

I expect there to be cases I have missed [(as microsofts extended
definition of node resolution algo is mega
complex),](https://www.typescriptlang.org/docs/handbook/modules/reference.html).
My hope is this implementation is _permissive_ and mostly includes false
positive errors w.r.t runtime (meaning too many dependents are
inferred).

This PR also introduces the `UnknownDependencyBehaviour` for the
`nodejs-infer` subsystem, so that users can opt into and out of warnings
and errors using this new more permissive mode.

---------

Co-authored-by: Huon Wilson <[email protected]>

build-support/cherry_pick/package.json

@@ -7,6 +7,7 @@
     "test": "jest"
   },
   "devDependencies": {
-    "jest": "^29.5.0"
+    "jest": "^29.5.0",
+    "semver": "^6.0.0"
   }
 }

docs/docs/javascript/overview/enabling-javascript-support.mdx

@@ -38,6 +38,11 @@ Created project/BUILD:
   - Add javascript_tests target tests
 ```
 
+:::tip Improved inference and introspection for bundled projects
+For [dependency inference](../../introduction/how-does-pants-work.mdx#dependency-inference), Pants reads both your
+projects' `package.json` sections and additionally
+supports [`jsconfig.json`](https://code.visualstudio.com/docs/languages/jsconfig), if one is present.
+:::
 
 ### Setting up node
 Pants will by default download a distribution of `node` according to the

docs/notes/2.23.x.md

@@ -166,6 +166,10 @@ including `node_modules`. This can be used to inspect the installation, or to en
 
 Pants will output a more helpful error message if there is no `name` field defined in the `package.json` file, or if the `name` field is empty.
 
+Pants now applies dependency inference according to the most permissive "bundler" setting according to 
+[jsconfig.json](https://code.visualstudio.com/docs/languages/jsconfig), when a jsconfig.json is
+part of your javascript workspace.
+
 #### Shell
 
 The `tailor` goal now has independent options for tailoring `shell_sources` and `shunit2_tests` targets. The option was split from `tailor` into [`tailor_sources`](https://www.pantsbuild.org/2.23/reference/subsystems/shell-setup#tailor_sources) and [`tailor_shunit2_tests`](https://www.pantsbuild.org/2.23/reference/subsystems/shell-setup#tailor_shunit2_tests).

src/python/pants/backend/javascript/dependency_inference/rules.py

@@ -3,8 +3,10 @@
 from __future__ import annotations
 
 import itertools
+import logging
 import os.path
 from dataclasses import dataclass
+from pathlib import PurePath
 from typing import Iterable
 
 from pants.backend.javascript import package_json
@@ -17,16 +19,30 @@
     PackageJsonEntryPoints,
     PackageJsonImports,
     PackageJsonSourceField,
+    find_owning_package,
+    subpath_imports_for_source,
 )
 from pants.backend.javascript.subsystems.nodejs_infer import NodeJSInfer
-from pants.backend.javascript.target_types import JSDependenciesField, JSSourceField
+from pants.backend.javascript.target_types import (
+    JS_FILE_EXTENSIONS,
+    JSDependenciesField,
+    JSSourceField,
+)
+from pants.backend.typescript import tsconfig
+from pants.backend.typescript.tsconfig import ParentTSConfigRequest, TSConfig, find_parent_ts_config
 from pants.build_graph.address import Address
+from pants.core.util_rules.unowned_dependency_behavior import (
+    UnownedDependencyError,
+    UnownedDependencyUsage,
+)
 from pants.engine.addresses import Addresses
+from pants.engine.fs import PathGlobs
 from pants.engine.internals.graph import Owners, OwnersRequest
-from pants.engine.internals.native_dep_inference import NativeParsedJavascriptDependencies
+from pants.engine.internals.native_dep_inference import ParsedJavascriptDependencyCandidate
 from pants.engine.internals.native_engine import InferenceMetadata, NativeDependenciesRequest
-from pants.engine.internals.selectors import Get
-from pants.engine.rules import Rule, collect_rules, rule
+from pants.engine.internals.selectors import Get, concurrently
+from pants.engine.intrinsics import parse_javascript_deps, path_globs_to_paths
+from pants.engine.rules import Rule, collect_rules, implicitly, rule
 from pants.engine.target import (
     FieldSet,
     HydratedSources,
@@ -36,8 +52,12 @@
     Targets,
 )
 from pants.engine.unions import UnionRule
+from pants.util.docutil import doc_url
 from pants.util.frozendict import FrozenDict
 from pants.util.ordered_set import FrozenOrderedSet
+from pants.util.strutil import bullet_list, softwrap
+
+logger = logging.getLogger(__name__)
 
 
 @dataclass(frozen=True)
@@ -101,23 +121,107 @@ async def map_candidate_node_packages(
     )
 
 
-@rule
-async def prepare_inference_metadata(imports: PackageJsonImports) -> InferenceMetadata:
+def _create_inference_metadata(
+    imports: PackageJsonImports, config: TSConfig | None
+) -> InferenceMetadata:
     return InferenceMetadata.javascript(
         imports.root_dir,
-        {pattern: list(replacements) for pattern, replacements in imports.imports.items()},
+        dict(imports.imports),
+        config.resolution_root_dir if config else None,
+        dict(config.paths or {}) if config else {},
     )
 
 
-async def _prepare_inference_metadata(address: Address) -> InferenceMetadata:
-    owning_pkg = await Get(OwningNodePackage, OwningNodePackageRequest(address))
+async def _prepare_inference_metadata(address: Address, file_path: str) -> InferenceMetadata:
+    owning_pkg, maybe_config = await concurrently(
+        find_owning_package(OwningNodePackageRequest(address)),
+        find_parent_ts_config(ParentTSConfigRequest(file_path, "jsconfig.json"), **implicitly()),
+    )
     if not owning_pkg.target:
-        return InferenceMetadata.javascript(address.spec_path, {})
-    return await Get(
-        InferenceMetadata, PackageJsonSourceField, owning_pkg.target[PackageJsonSourceField]
+        return InferenceMetadata.javascript(
+            (
+                os.path.dirname(maybe_config.ts_config.path)
+                if maybe_config.ts_config
+                else address.spec_path
+            ),
+            {},
+            maybe_config.ts_config.resolution_root_dir if maybe_config.ts_config else None,
+            dict(maybe_config.ts_config.paths or {}) if maybe_config.ts_config else {},
+        )
+    return _create_inference_metadata(
+        await subpath_imports_for_source(owning_pkg.target[PackageJsonSourceField]),
+        maybe_config.ts_config,
     )
 
 
+def _add_extensions(file_imports: frozenset[str]) -> PathGlobs:
+    extensions = JS_FILE_EXTENSIONS + tuple(f"/index{ext}" for ext in JS_FILE_EXTENSIONS)
+    return PathGlobs(
+        string
+        for file_import in file_imports
+        for string in (
+            [file_import]
+            if PurePath(file_import).suffix
+            else [f"{file_import}{ext}" for ext in extensions]
+        )
+    )
+
+
+async def _determine_import_from_candidates(
+    candidates: ParsedJavascriptDependencyCandidate,
+    package_candidate_map: NodePackageCandidateMap,
+) -> Addresses:
+    paths = await path_globs_to_paths(_add_extensions(candidates.file_imports))
+    local_owners = await Get(Owners, OwnersRequest(paths.files))
+
+    owning_targets = await Get(Targets, Addresses(local_owners))
+
+    addresses = Addresses(tgt.address for tgt in owning_targets)
+    if not local_owners:
+        non_path_string_bases = FrozenOrderedSet(
+            non_path_string.partition(os.path.sep)[0]
+            for non_path_string in candidates.package_imports
+        )
+        addresses = Addresses(
+            package_candidate_map[pkg_name]
+            for pkg_name in non_path_string_bases
+            if pkg_name in package_candidate_map
+        )
+    return addresses
+
+
+def _handle_unowned_imports(
+    address: Address,
+    unowned_dependency_behavior: UnownedDependencyUsage,
+    unowned_imports: frozenset[str],
+) -> None:
+    if not unowned_imports or unowned_dependency_behavior is UnownedDependencyUsage.DoNothing:
+        return
+
+    url = doc_url(
+        "docs/using-pants/troubleshooting-common-issues#import-errors-and-missing-dependencies"
+    )
+    msg = softwrap(
+        f"""
+        Pants cannot infer owners for the following imports in the target {address}:
+
+        {bullet_list(sorted(unowned_imports))}
+
+        If you do not expect an import to be inferrable, add `// pants: no-infer-dep` to the
+        import line. Otherwise, see {url} for common problems.
+        """
+    )
+    if unowned_dependency_behavior is UnownedDependencyUsage.LogWarning:
+        logger.warning(msg)
+    else:
+        raise UnownedDependencyError(msg)
+
+
+def _is_node_builtin_module(import_string: str) -> bool:
+    """https://nodejs.org/api/modules.html#built-in-modules."""
+    return import_string.startswith("node:")
+
+
 @rule
 async def infer_js_source_dependencies(
     request: InferJSDependenciesRequest,
@@ -130,41 +234,41 @@ async def infer_js_source_dependencies(
     sources = await Get(
         HydratedSources, HydrateSourcesRequest(source, for_sources_types=[JSSourceField])
     )
-    metadata = await _prepare_inference_metadata(request.field_set.address)
+    metadata = await _prepare_inference_metadata(request.field_set.address, source.file_path)
 
-    import_strings = await Get(
-        NativeParsedJavascriptDependencies,
-        NativeDependenciesRequest(sources.snapshot.digest, metadata),
+    import_strings, candidate_pkgs = await concurrently(
+        parse_javascript_deps(NativeDependenciesRequest(sources.snapshot.digest, metadata)),
+        map_candidate_node_packages(
+            RequestNodePackagesCandidateMap(request.field_set.address), **implicitly()
+        ),
     )
-
-    owners = await Get(Owners, OwnersRequest(tuple(import_strings.file_imports)))
-    owning_targets = await Get(Targets, Addresses(owners))
-
-    non_path_string_bases = FrozenOrderedSet(
-        non_path_string.partition(os.path.sep)[0]
-        for non_path_string in import_strings.package_imports
-    )
-
-    candidate_pkgs = await Get(
-        NodePackageCandidateMap, RequestNodePackagesCandidateMap(request.field_set.address)
+    imports = dict(
+        zip(
+            import_strings.imports,
+            await concurrently(
+                _determine_import_from_candidates(candidates, candidate_pkgs)
+                for string, candidates in import_strings.imports.items()
+            ),
+        )
     )
-
-    pkg_addresses = (
-        candidate_pkgs[pkg_name] for pkg_name in non_path_string_bases if pkg_name in candidate_pkgs
+    _handle_unowned_imports(
+        request.field_set.address,
+        nodejs_infer.unowned_dependency_behavior,
+        frozenset(
+            string
+            for string, addresses in imports.items()
+            if not addresses and not _is_node_builtin_module(string)
+        ),
     )
 
-    return InferredDependencies(
-        itertools.chain(
-            pkg_addresses,
-            (tgt.address for tgt in owning_targets if tgt.has_field(JSSourceField)),
-        )
-    )
+    return InferredDependencies(itertools.chain.from_iterable(imports.values()))
 
 
 def rules() -> Iterable[Rule | UnionRule]:
     return [
         *collect_rules(),
         *package_json.rules(),
+        *tsconfig.rules(),
         UnionRule(InferDependenciesRequest, InferNodePackageDependenciesRequest),
         UnionRule(InferDependenciesRequest, InferJSDependenciesRequest),
     ]