Internally stop ignoring missing imports with MyPy (pantsbuild#10828)

Now we're explicit about every dependency that is missing requirements. This is generally a best practice, and it better dogfoods that MyPy requirements are indeed working.

We add `pytest` to our `requirements.txt` so that Pants can properly infer a dep on Pytest. This is an awkward edge though that it duplicates with `--pytest-version`. This is a known issue.

[ci skip-build-wheels]
[ci skip-rust]

3rdparty/python/constraints.txt

@@ -1,25 +1,32 @@
-# Generated by build-support/bin/generate_lockfile.sh on Thu Sep 17 15:16:52 PDT 2020
+# Generated by build-support/bin/generate_lockfile.sh on Mon Sep 21 16:22:12 MST 2020
 ansicolors==1.1.8
+attrs==20.2.0
 beautifulsoup4==4.6.3
 certifi==2020.6.20
-cffi==1.14.2
+cffi==1.14.3
 chardet==3.0.4
 cryptography==3.1
 dataclasses==0.6
 fasteners==0.15
 idna==2.10
+importlib-metadata==1.7.0
+iniconfig==1.0.1
 monotonic==1.5
+more-itertools==8.5.0
 mypy==0.782
 mypy-extensions==0.4.3
 packaging==20.4
 pathspec==0.8.0
 pex==2.1.16
-pip==20.1.1
+pip==18.1
+pluggy==0.13.1
 psutil==5.7.0
+py==1.9.0
 pycparser==2.20
 pyOpenSSL==19.1.0
 pyparsing==2.4.7
 pystache==0.5.4
+pytest==6.0.2
 python-Levenshtein==0.12.0
 PyYAML==5.3.1
 requests==2.24.0
@@ -31,3 +38,4 @@ typed-ast==1.4.1
 typing-extensions==3.7.4.2
 urllib3==1.25.10
 www-authenticate==0.9.2
+zipp==3.1.0

3rdparty/python/requirements.txt

@@ -20,6 +20,8 @@ pathspec==0.8.0
 pex==2.1.16
 psutil==5.7.0
 pystache==0.5.4
+# This should be kept in sync with `pytest.py`.
+pytest>=6.0.1,<6.1
 python-Levenshtein==0.12.0
 PyYAML>=5.3.1,<5.4
 requests[security]>=2.20.1

build-support/mypy/mypy.ini

@@ -52,7 +52,32 @@ pretty = True
 color_output = True
 error_summary = True
 
-# Imports. See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports.
+[mypy-bs4]
+ignore_missing_imports = True
+
+[mypy-colors]
+ignore_missing_imports = True
+
+[mypy-fasteners]
+ignore_missing_imports = True
+
+[mypy-Levenshtein]
+ignore_missing_imports = True
+
+[mypy-pathspec.*]
+ignore_missing_imports = True
+
+[mypy-pex.*]
+ignore_missing_imports = True
+
+[mypy-psutil]
+ignore_missing_imports = True
+
+[mypy-pystache]
+ignore_missing_imports = True
+
+[mypy-setproctitle]
+ignore_missing_imports = True
+
+[mypy-www_authenticate]
 ignore_missing_imports = True
-follow_imports = normal
-follow_imports_for_stubs = False

src/python/pants/backend/python/subsystems/pytest.py

@@ -24,6 +24,9 @@ def register_options(cls, register):
         )
         register(
             "--version",
+            # This should be kept in sync with `requirements.txt`.
+            # TODO: To fix this, we should allow using a `target_option` referring to a
+            #  `python_requirement_library` to override the version.
             default="pytest>=6.0.1,<6.1",
             advanced=True,
             help="Requirement string for Pytest.",

src/python/pants/engine/internals/native.py

@@ -10,7 +10,7 @@
 from pants.base.exiter import ExitCode
 from pants.engine.fs import PathGlobs
 from pants.engine.internals import native_engine
-from pants.engine.internals.native_engine import (
+from pants.engine.internals.native_engine import (  # type: ignore[import]
     PyExecutionRequest,
     PyExecutionStrategyOptions,
     PyExecutor,

src/python/pants/engine/internals/scheduler.py

@@ -29,7 +29,7 @@
     RemovePrefix,
     Snapshot,
 )
-from pants.engine.internals.native_engine import PyTypes
+from pants.engine.internals.native_engine import PyTypes  # type: ignore[import]
 from pants.engine.internals.nodes import Return, Throw
 from pants.engine.internals.selectors import Params
 from pants.engine.platform import Platform

testprojects/src/python/python_targets/test_binary.py

@@ -1,6 +1,6 @@
 # Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from python_targets.test_library import say_test
+from python_targets.test_library import say_test  # type: ignore[import]
 
 say_test()

tests/python/pants_test/init/test_plugin_resolver.py

@@ -240,7 +240,7 @@ def _do_test_exact_requirements_interpreter_change(self, sdist: bool) -> None:
 
             # But for a compatible interpreter the exact resolve results should be re-used and load
             # directly from the still in-tact cache.
-            with self.plugin_resolution(
+            with self.plugin_resolution(  # type: ignore[unreachable]
                 interpreter=python36, chroot=chroot, plugins=[("jake", "1.2.3"), ("jane", "3.4.5")]
             ) as results2: