From 952b7733d2f18fc8de7eeaa7616ca33899258739 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Tue, 7 Mar 2023 21:22:25 -0500 Subject: [PATCH] Support running `pants` from anywhere in the project. (#18412) Allows the use of `pants` from anywhere in a project, not just the project root, while also respecting specs on the command line relative to `CWD` rather than the build root. Closes #6750 --- src/python/pants/base/specs_parser.py | 17 +++++++++- src/python/pants/base/specs_parser_test.py | 32 +++++++++++++++++-- src/python/pants/bin/daemon_pants_runner.py | 9 +++++- src/python/pants/bin/local_pants_runner.py | 6 +++- src/python/pants/bin/pants_loader.py | 1 - src/python/pants/bin/pants_runner.py | 4 ++- .../pants/engine/internals/graph_test.py | 2 +- .../pants/engine/internals/native_engine.pyi | 1 + .../engine/internals/specs_rules_test.py | 2 +- src/python/pants/init/specs_calculator.py | 3 +- src/python/pants/testutil/rule_runner.py | 2 +- src/rust/engine/src/externs/interface.rs | 1 + 12 files changed, 69 insertions(+), 11 deletions(-) diff --git a/src/python/pants/base/specs_parser.py b/src/python/pants/base/specs_parser.py index 2f719cf8277..507014878a8 100644 --- a/src/python/pants/base/specs_parser.py +++ b/src/python/pants/base/specs_parser.py @@ -44,8 +44,17 @@ class SpecsParser: class BadSpecError(Exception): """Indicates an unparseable command line selector.""" - def __init__(self, root_dir: str | None = None) -> None: + def __init__(self, *, root_dir: str | None = None, working_dir: str | None = None) -> None: self._root_dir = os.path.realpath(root_dir or get_buildroot()) + self._working_dir = ( + os.path.relpath(os.path.join(self._root_dir, working_dir), self._root_dir) + if working_dir + else "" + ) + if self._working_dir.startswith(".."): + raise self.BadSpecError( + f"Work directory {self._working_dir} escapes build root {self._root_dir}" + ) def _normalize_spec_path(self, path: str) -> str: is_abs = not path.startswith("//") and os.path.isabs(path) @@ -58,9 +67,15 @@ def _normalize_spec_path(self, path: str) -> str: else: if path.startswith("//"): path = path[2:] + elif self._working_dir: + path = os.path.join(self._working_dir, path) path = os.path.join(self._root_dir, path) normalized = os.path.relpath(path, self._root_dir) + if normalized.startswith(".."): + raise self.BadSpecError( + f"Relative spec path {path} escapes build root {self._root_dir}" + ) if normalized == ".": normalized = "" return normalized diff --git a/src/python/pants/base/specs_parser_test.py b/src/python/pants/base/specs_parser_test.py index 90a9f04ebce..89b989a2df1 100644 --- a/src/python/pants/base/specs_parser_test.py +++ b/src/python/pants/base/specs_parser_test.py @@ -52,8 +52,10 @@ def file_glob(val: str) -> FileGlobSpec: return FileGlobSpec(val) -def assert_spec_parsed(build_root: Path, spec_str: str, expected_spec: Spec) -> None: - parser = SpecsParser(str(build_root)) +def assert_spec_parsed( + build_root: Path, spec_str: str, expected_spec: Spec, working_dir: str | None = None +) -> None: + parser = SpecsParser(root_dir=str(build_root), working_dir=working_dir) spec, is_ignore = parser.parse_spec(spec_str) assert isinstance(spec, type(expected_spec)) assert spec == expected_spec @@ -241,3 +243,29 @@ def test_cmd_line_affordances_absolute_path( ) -> None: spec = os.path.join(tmp_path, spec_suffix) assert_spec_parsed(tmp_path, spec, expected) + + +@pytest.mark.parametrize( + "working_dir, spec, expected", + [ + ("a/b", "c", dir_literal("a/b/c")), + ("a/b", "../d", dir_literal("a/d")), + ("a/b", "//e", dir_literal("e")), + ], +) +def test_working_dir(tmp_path: Path, working_dir: str, spec: str, expected: Spec) -> None: + assert_spec_parsed(tmp_path, spec, expected, working_dir) + assert_spec_parsed(tmp_path, spec, expected, str(tmp_path / working_dir)) + + +def test_invalid_working_dir(tmp_path: Path) -> None: + with pytest.raises(SpecsParser.BadSpecError): + assert_spec_parsed(tmp_path, "../../foo", dir_literal("foo"), "a") + with pytest.raises(SpecsParser.BadSpecError): + assert_spec_parsed(tmp_path, "../../foo", dir_literal("foo"), str(tmp_path / "a")) + with pytest.raises(SpecsParser.BadSpecError): + assert_spec_parsed(tmp_path, "test_invalid_working_dir0/foo", dir_literal("foo"), "..") + with pytest.raises(SpecsParser.BadSpecError): + assert_spec_parsed( + tmp_path, "test_invalid_working_dir0/foo", dir_literal("foo"), str(tmp_path / "..") + ) diff --git a/src/python/pants/bin/daemon_pants_runner.py b/src/python/pants/bin/daemon_pants_runner.py index 0f65c99f516..815b2d03b6a 100644 --- a/src/python/pants/bin/daemon_pants_runner.py +++ b/src/python/pants/bin/daemon_pants_runner.py @@ -99,6 +99,7 @@ def single_daemonized_run( self, args: Tuple[str, ...], env: Dict[str, str], + working_dir: str, cancellation_latch: PySessionCancellationLatch, ) -> ExitCode: """Run a single daemonized run of Pants. @@ -110,6 +111,8 @@ def single_daemonized_run( try: logger.debug("Connected to pantsd") + logger.debug(f"work dir: {working_dir}") + # Capture the client's start time, which we propagate here in order to get an accurate # view of total time. env_start_time = env.get("PANTSD_RUNTRACKER_CLIENT_START_TIME", None) @@ -130,6 +133,7 @@ def single_daemonized_run( scheduler, options_initializer = self._core.prepare(options_bootstrapper, complete_env) runner = LocalPantsRunner.create( complete_env, + working_dir, options_bootstrapper, scheduler=scheduler, options_initializer=options_initializer, @@ -148,6 +152,7 @@ def __call__( command: str, args: Tuple[str, ...], env: Dict[str, str], + working_dir: str, cancellation_latch: PySessionCancellationLatch, stdin_fileno: int, stdout_fileno: int, @@ -169,6 +174,8 @@ def __call__( stdout_fileno=stdout_fileno, stderr_fileno=stderr_fileno, ): - return self.single_daemonized_run(((command,) + args), env, cancellation_latch) + return self.single_daemonized_run( + ((command,) + args), env, working_dir, cancellation_latch + ) finally: logger.info(f"request completed: `{' '.join(args)}`") diff --git a/src/python/pants/bin/local_pants_runner.py b/src/python/pants/bin/local_pants_runner.py index 8cc8e752b25..f732a01c071 100644 --- a/src/python/pants/bin/local_pants_runner.py +++ b/src/python/pants/bin/local_pants_runner.py @@ -58,11 +58,13 @@ class LocalPantsRunner: executor: PyExecutor union_membership: UnionMembership is_pantsd_run: bool + working_dir: str @classmethod def create( cls, env: CompleteEnvironmentVars, + working_dir: str, options_bootstrapper: OptionsBootstrapper, options_initializer: OptionsInitializer | None = None, scheduler: GraphScheduler | None = None, @@ -150,6 +152,7 @@ def create( options_bootstrapper=options_bootstrapper, options=options, session=graph_session.scheduler_session, + working_dir=working_dir, ) return cls( @@ -163,6 +166,7 @@ def create( executor=executor, union_membership=union_membership, is_pantsd_run=is_pantsd_run, + working_dir=working_dir, ) def _perform_run(self, goals: tuple[str, ...]) -> ExitCode: @@ -237,7 +241,7 @@ def _run_inner(self) -> ExitCode: return PANTS_FAILED_EXIT_CODE def run(self, start_time: float) -> ExitCode: - spec_parser = SpecsParser() + spec_parser = SpecsParser(working_dir=self.working_dir) specs = [] for spec_str in self.options.specs: spec, is_ignore = spec_parser.parse_spec(spec_str) diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index b493485a6c2..4925d78e6ac 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -103,7 +103,6 @@ def main(cls) -> None: sys.setrecursionlimit(int(os.environ.get(RECURSION_LIMIT, "10000"))) entrypoint = os.environ.pop(DAEMON_ENTRYPOINT, None) - if entrypoint: cls.run_alternate_entrypoint(entrypoint) else: diff --git a/src/python/pants/bin/pants_runner.py b/src/python/pants/bin/pants_runner.py index f343114348f..b2f1688f2e4 100644 --- a/src/python/pants/bin/pants_runner.py +++ b/src/python/pants/bin/pants_runner.py @@ -97,6 +97,8 @@ def run(self, start_time: float) -> ExitCode: log_location=init_workdir(global_bootstrap_options), pantsd_instance=False ) runner = LocalPantsRunner.create( - env=CompleteEnvironmentVars(self.env), options_bootstrapper=options_bootstrapper + env=CompleteEnvironmentVars(self.env), + working_dir=os.getcwd(), + options_bootstrapper=options_bootstrapper, ) return runner.run(start_time) diff --git a/src/python/pants/engine/internals/graph_test.py b/src/python/pants/engine/internals/graph_test.py index e0a02d0058b..dbabb1ea0a8 100644 --- a/src/python/pants/engine/internals/graph_test.py +++ b/src/python/pants/engine/internals/graph_test.py @@ -919,7 +919,7 @@ def assert_generated( if expected_dependencies is not None: # TODO: Adjust the `TransitiveTargets` API to expose the complete mapping. # see https://github.com/pantsbuild/pants/issues/11270 - specs = SpecsParser(rule_runner.build_root).parse_specs( + specs = SpecsParser(root_dir=rule_runner.build_root).parse_specs( ["::"], description_of_origin="tests" ) addresses = rule_runner.request(Addresses, [specs]) diff --git a/src/python/pants/engine/internals/native_engine.pyi b/src/python/pants/engine/internals/native_engine.pyi index cc237df586d..3c846a04a41 100644 --- a/src/python/pants/engine/internals/native_engine.pyi +++ b/src/python/pants/engine/internals/native_engine.pyi @@ -246,6 +246,7 @@ class RawFdRunner(Protocol): command: str, args: tuple[str, ...], env: dict[str, str], + working_dir: str, cancellation_latch: PySessionCancellationLatch, stdin_fileno: int, stdout_fileno: int, diff --git a/src/python/pants/engine/internals/specs_rules_test.py b/src/python/pants/engine/internals/specs_rules_test.py index bbf48d39171..d02d7745e39 100644 --- a/src/python/pants/engine/internals/specs_rules_test.py +++ b/src/python/pants/engine/internals/specs_rules_test.py @@ -633,7 +633,7 @@ def test_resolve_addresses_from_raw_specs(rule_runner: RuleRunner) -> None: "address_spec:nonfile#gen", ] multiple_files_specs = ["multiple_files/f2.txt", "multiple_files:multiple_files"] - specs = SpecsParser(rule_runner.build_root).parse_specs( + specs = SpecsParser(root_dir=rule_runner.build_root).parse_specs( [*no_interaction_specs, *multiple_files_specs], description_of_origin="tests", ) diff --git a/src/python/pants/init/specs_calculator.py b/src/python/pants/init/specs_calculator.py index 561bfac8694..35d1f4ddb9e 100644 --- a/src/python/pants/init/specs_calculator.py +++ b/src/python/pants/init/specs_calculator.py @@ -29,11 +29,12 @@ def calculate_specs( options_bootstrapper: OptionsBootstrapper, options: Options, session: SchedulerSession, + working_dir: str, ) -> Specs: """Determine the specs for a given Pants run.""" global_options = options.for_global_scope() unmatched_cli_globs = global_options.unmatched_cli_globs - specs = SpecsParser().parse_specs( + specs = SpecsParser(working_dir=working_dir).parse_specs( options.specs, description_of_origin="CLI arguments", unmatched_glob_behavior=unmatched_cli_globs, diff --git a/src/python/pants/testutil/rule_runner.py b/src/python/pants/testutil/rule_runner.py index 0ac24b79096..d2fd57f5096 100644 --- a/src/python/pants/testutil/rule_runner.py +++ b/src/python/pants/testutil/rule_runner.py @@ -416,7 +416,7 @@ def run_goal_rule( [GlobalOptions.get_scope_info(), goal.subsystem_cls.get_scope_info()], self.union_membership, ).specs - specs = SpecsParser(self.build_root).parse_specs( + specs = SpecsParser(root_dir=self.build_root).parse_specs( raw_specs, description_of_origin="RuleRunner.run_goal_rule()" ) diff --git a/src/rust/engine/src/externs/interface.rs b/src/rust/engine/src/externs/interface.rs index 7f1c262d992..d83de654c14 100644 --- a/src/rust/engine/src/externs/interface.rs +++ b/src/rust/engine/src/externs/interface.rs @@ -559,6 +559,7 @@ fn nailgun_server_create( exe.cmd.command, PyTuple::new(py, exe.cmd.args), exe.cmd.env.into_iter().collect::>(), + exe.cmd.working_dir, PySessionCancellationLatch(exe.cancelled), exe.stdin_fd as i64, exe.stdout_fd as i64,