lint: add pydocstyle python backend (pantsbuild#17596)

build-support/bin/_generate_all_lockfiles_helper.py

@@ -28,6 +28,7 @@
 from pants.backend.python.lint.docformatter.subsystem import Docformatter
 from pants.backend.python.lint.flake8.subsystem import Flake8
 from pants.backend.python.lint.isort.subsystem import Isort
+from pants.backend.python.lint.pydocstyle.subsystem import Pydocstyle
 from pants.backend.python.lint.pylint.subsystem import Pylint
 from pants.backend.python.lint.pyupgrade.subsystem import PyUpgrade
 from pants.backend.python.lint.yapf.subsystem import Yapf
@@ -115,6 +116,7 @@ def jvm(cls, tool: type[JvmToolBase], *, backend: str | None = None) -> DefaultT
     DefaultTool.python(Isort),
     DefaultTool.python(Lambdex, backend="pants.backend.awslambda.python"),
     DefaultTool.python(MyPy, source_plugins=True),
+    DefaultTool.python(Pydocstyle, backend="pants.backend.python.lint.pydocstyle"),
     DefaultTool.python(PyTest),
     DefaultTool.python(PyUpgrade, backend="pants.backend.experimental.python.lint.pyupgrade"),
     DefaultTool.python(Pylint, backend="pants.backend.python.lint.pylint", source_plugins=True),

src/python/pants/backend/python/lint/pydocstyle/BUILD

@@ -0,0 +1,18 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+resource(name="lockfile", source="pydocstyle.lock")
+
+python_sources(
+    overrides={"subsystem.py": {"dependencies": [":lockfile"]}},
+)
+
+python_tests(
+    name="tests",
+    overrides={
+        "rules_integration_test.py": {
+            "timeout": 240,
+            "tags": ["platform_specific_behavior"],
+        }
+    },
+)

src/python/pants/backend/python/lint/pydocstyle/pydocstyle.lock

@@ -0,0 +1,109 @@
+// This lockfile was autogenerated by Pants. To regenerate, run:
+//
+//    build-support/bin/generate_all_lockfiles.sh
+//
+// --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
+// {
+//   "version": 3,
+//   "valid_for_interpreter_constraints": [
+//     "CPython<4,>=3.7"
+//   ],
+//   "generated_with_requirements": [
+//     "pydocstyle[toml]<7.0,>=6.1.1"
+//   ],
+//   "manylinux": "manylinux2014",
+//   "requirement_constraints": [],
+//   "only_binary": [],
+//   "no_binary": []
+// }
+// --- END PANTS LOCKFILE METADATA ---
+
+{
+  "allow_builds": true,
+  "allow_prereleases": false,
+  "allow_wheels": true,
+  "build_isolation": true,
+  "constraints": [],
+  "locked_resolves": [
+    {
+      "locked_requirements": [
+        {
+          "artifacts": [
+            {
+              "algorithm": "sha256",
+              "hash": "6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4",
+              "url": "https://files.pythonhosted.org/packages/87/67/4df10786068766000518c6ad9c4a614e77585a12ab8f0654c776757ac9dc/pydocstyle-6.1.1-py3-none-any.whl"
+            },
+            {
+              "algorithm": "sha256",
+              "hash": "1d41b7c459ba0ee6c345f2eb9ae827cab14a7533a88c5c6f7e94923f72df92dc",
+              "url": "https://files.pythonhosted.org/packages/4c/30/4cdea3c8342ad343d41603afc1372167c224a04dc5dc0bf4193ccb39b370/pydocstyle-6.1.1.tar.gz"
+            }
+          ],
+          "project_name": "pydocstyle",
+          "requires_dists": [
+            "snowballstemmer",
+            "toml; extra == \"toml\""
+          ],
+          "requires_python": ">=3.6",
+          "version": "6.1.1"
+        },
+        {
+          "artifacts": [
+            {
+              "algorithm": "sha256",
+              "hash": "c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a",
+              "url": "https://files.pythonhosted.org/packages/ed/dc/c02e01294f7265e63a7315fe086dd1df7dacb9f840a804da846b96d01b96/snowballstemmer-2.2.0-py2.py3-none-any.whl"
+            },
+            {
+              "algorithm": "sha256",
+              "hash": "09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1",
+              "url": "https://files.pythonhosted.org/packages/44/7b/af302bebf22c749c56c9c3e8ae13190b5b5db37a33d9068652e8f73b7089/snowballstemmer-2.2.0.tar.gz"
+            }
+          ],
+          "project_name": "snowballstemmer",
+          "requires_dists": [],
+          "requires_python": null,
+          "version": "2.2.0"
+        },
+        {
+          "artifacts": [
+            {
+              "algorithm": "sha256",
+              "hash": "806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+              "url": "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl"
+            },
+            {
+              "algorithm": "sha256",
+              "hash": "b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f",
+              "url": "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz"
+            }
+          ],
+          "project_name": "toml",
+          "requires_dists": [],
+          "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,>=2.6",
+          "version": "0.10.2"
+        }
+      ],
+      "platform_tag": null
+    }
+  ],
+  "path_mappings": {},
+  "pex_version": "2.1.113",
+  "pip_version": "20.3.4-patched",
+  "prefer_older_binary": false,
+  "requirements": [
+    "pydocstyle[toml]<7.0,>=6.1.1"
+  ],
+  "requires_python": [
+    "<4,>=3.7"
+  ],
+  "resolver_version": "pip-2020-resolver",
+  "style": "universal",
+  "target_systems": [
+    "linux",
+    "mac"
+  ],
+  "transitive": true,
+  "use_pep517": null
+}

src/python/pants/backend/python/lint/pydocstyle/register.py

@@ -0,0 +1,15 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+"""Static analysis tool for checking compliance with Python docstring conventions.
+
+See https://www.pantsbuild.org/docs/python-linters-and-formatters and
+http://www.pydocstyle.org/en/stable/.
+"""
+
+from pants.backend.python.lint.pydocstyle import rules as pydocstyle_rules
+from pants.backend.python.lint.pydocstyle import skip_field, subsystem
+
+
+def rules():
+    return (*pydocstyle_rules.rules(), *skip_field.rules(), *subsystem.rules())

src/python/pants/backend/python/lint/pydocstyle/rules.py

@@ -0,0 +1,72 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from typing import Tuple
+
+from pants.backend.python.lint.pydocstyle.subsystem import Pydocstyle, PydocstyleFieldSet
+from pants.backend.python.util_rules import pex
+from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess
+from pants.core.goals.lint import LintResult, LintTargetsRequest
+from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest
+from pants.core.util_rules.partitions import PartitionerType
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
+from pants.engine.fs import Digest, MergeDigests
+from pants.engine.process import FallibleProcessResult
+from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.util.logging import LogLevel
+from pants.util.strutil import pluralize
+
+
+class PydocstyleRequest(LintTargetsRequest):
+    field_set_type = PydocstyleFieldSet
+    tool_subsystem = Pydocstyle
+    partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION
+
+
+def generate_argv(source_files: SourceFiles, pydocstyle: Pydocstyle) -> Tuple[str, ...]:
+    args = []
+    if pydocstyle.config is not None:
+        args.append(f"--config={pydocstyle.config}")
+    args.extend(pydocstyle.args)
+    args.extend(source_files.files)
+    return tuple(args)
+
+
+@rule(desc="Lint with Pydocstyle", level=LogLevel.DEBUG)
+async def pydocstyle_lint(
+    request: PydocstyleRequest.Batch,
+    pydocstyle: Pydocstyle,
+) -> LintResult:
+    pydocstyle_pex_get = Get(VenvPex, PexRequest, pydocstyle.to_pex_request())
+
+    config_files_get = Get(ConfigFiles, ConfigFilesRequest, pydocstyle.config_request)
+    source_files_get = Get(
+        SourceFiles, SourceFilesRequest(field_set.source for field_set in request.elements)
+    )
+
+    pydocstyle_pex, config_files, source_files = await MultiGet(
+        pydocstyle_pex_get, config_files_get, source_files_get
+    )
+
+    input_digest = await Get(
+        Digest,
+        MergeDigests((source_files.snapshot.digest, config_files.snapshot.digest)),
+    )
+
+    result = await Get(
+        FallibleProcessResult,
+        VenvPexProcess(
+            pydocstyle_pex,
+            argv=generate_argv(source_files, pydocstyle),
+            input_digest=input_digest,
+            description=f"Run Pydocstyle on {pluralize(len(request.elements), 'file')}.",
+            level=LogLevel.DEBUG,
+        ),
+    )
+    return LintResult.create(request, result)
+
+
+def rules():
+    return [*collect_rules(), *PydocstyleRequest.rules(), *pex.rules()]

src/python/pants/backend/python/lint/pydocstyle/rules_integration_test.py

@@ -0,0 +1,154 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from typing import Sequence
+
+import pytest
+
+from pants.backend.python import target_types_rules
+from pants.backend.python.lint.pydocstyle.rules import PydocstyleRequest
+from pants.backend.python.lint.pydocstyle.rules import rules as pydocstyle_rules
+from pants.backend.python.lint.pydocstyle.subsystem import PydocstyleFieldSet
+from pants.backend.python.lint.pydocstyle.subsystem import rules as pydocstyle_subsystem_rules
+from pants.backend.python.subsystems.setup import PythonSetup
+from pants.backend.python.target_types import PythonSourcesGeneratorTarget
+from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
+from pants.core.goals.lint import LintResult, Partitions
+from pants.core.util_rules import config_files, source_files
+from pants.engine.addresses import Address
+from pants.engine.target import Target
+from pants.testutil.python_interpreter_selection import all_major_minor_python_versions
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    return RuleRunner(
+        rules=[
+            *pydocstyle_rules(),
+            *pydocstyle_subsystem_rules(),
+            *source_files.rules(),
+            *config_files.rules(),
+            *target_types_rules.rules(),
+            QueryRule(Partitions, [PydocstyleRequest.PartitionRequest]),
+            QueryRule(LintResult, [PydocstyleRequest.Batch]),
+        ],
+        target_types=[PythonSourcesGeneratorTarget],
+    )
+
+
+GOOD_FILE = '''
+"""Public module docstring is present."""
+def fun():
+  """Pretty docstring is present."""
+  pass
+'''
+BAD_FILE = """
+def fun():
+  '''ugly docstring!'''
+  pass
+"""
+
+
+def run_pydocstyle(
+    rule_runner: RuleRunner, targets: list[Target], *, extra_args: list[str] | None = None
+) -> Sequence[LintResult]:
+    rule_runner.set_options(
+        [
+            "--backend-packages=pants.backend.python.lint.pydocstyle",
+            *(extra_args or ()),
+        ],
+        env_inherit={"PATH", "PYENV_ROOT", "HOME"},
+    )
+    partitions = rule_runner.request(
+        Partitions[PydocstyleFieldSet, InterpreterConstraints],
+        [
+            PydocstyleRequest.PartitionRequest(
+                tuple(PydocstyleFieldSet.create(tgt) for tgt in targets)
+            )
+        ],
+    )
+    results = []
+    for partition in partitions:
+        result = rule_runner.request(
+            LintResult,
+            [PydocstyleRequest.Batch("", partition.elements, partition.metadata)],
+        )
+        results.append(result)
+    return tuple(results)
+
+
+def assert_success(
+    rule_runner: RuleRunner, target: Target, *, extra_args: list[str] | None = None
+) -> None:
+    result = run_pydocstyle(rule_runner, [target], extra_args=extra_args)
+    assert len(result) == 1
+    assert result[0].exit_code == 0
+
+
+@pytest.mark.platform_specific_behavior
+@pytest.mark.parametrize(
+    "major_minor_interpreter",
+    all_major_minor_python_versions(PythonSetup.default_interpreter_constraints),
+)
+def test_passing(rule_runner: RuleRunner, major_minor_interpreter: str) -> None:
+    rule_runner.write_files({"f.py": GOOD_FILE, "BUILD": "python_sources(name='t')"})
+    tgt = rule_runner.get_target(Address("", target_name="t", relative_file_path="f.py"))
+    assert_success(
+        rule_runner,
+        tgt,
+        extra_args=[f"--python-interpreter-constraints=['=={major_minor_interpreter}.*']"],
+    )
+
+
+def test_failing(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files({"f.py": BAD_FILE, "BUILD": "python_sources(name='t')"})
+    tgt = rule_runner.get_target(Address("", target_name="t", relative_file_path="f.py"))
+    result = run_pydocstyle(rule_runner, [tgt])
+    assert len(result) == 1
+    assert result[0].exit_code == 1
+    assert "D100: Missing docstring in public module" in result[0].stdout
+
+
+def test_multiple_targets(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {"good.py": GOOD_FILE, "bad.py": BAD_FILE, "BUILD": "python_sources(name='t')"}
+    )
+    tgts = [
+        rule_runner.get_target(Address("", target_name="t", relative_file_path="good.py")),
+        rule_runner.get_target(Address("", target_name="t", relative_file_path="bad.py")),
+    ]
+    result = run_pydocstyle(rule_runner, tgts)
+    assert len(result) == 1
+    assert result[0].exit_code == 1
+    assert "good.py" not in result[0].stdout
+    assert "D400: First line should end with a period (not '!')" in result[0].stdout
+
+
+def test_respects_config_file(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "f.py": BAD_FILE,
+            "BUILD": "python_sources(name='t')",
+            ".pydocstyle.ini": "[pydocstyle]\nignore = D100,D300,D400,D403",
+        }
+    )
+    tgt = rule_runner.get_target(Address("", target_name="t", relative_file_path="f.py"))
+    assert_success(rule_runner, tgt, extra_args=["--pydocstyle-config=.pydocstyle.ini"])
+
+
+def test_respects_passthrough_args(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files({"f.py": BAD_FILE, "BUILD": "python_sources(name='t')"})
+    tgt = rule_runner.get_target(Address("", target_name="t", relative_file_path="f.py"))
+    assert_success(
+        rule_runner, tgt, extra_args=["--pydocstyle-args='--ignore=D100,D300,D400,D403'"]
+    )
+
+
+def test_skip(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files({"f.py": BAD_FILE, "BUILD": "python_sources(name='t')"})
+    tgt = rule_runner.get_target(Address("", target_name="t", relative_file_path="f.py"))
+    result = run_pydocstyle(rule_runner, [tgt], extra_args=["--pydocstyle-skip"])
+    assert not result