split out file discovery and test it

docs/source/internal/utils.rst

@@ -60,24 +60,6 @@ Another helpful function that is named only to be explicit given it is a very
 trivial check, this checks if the user specified ``-`` in their arguments to
 |Flake8| to indicate we should read from stdin.
 
-.. autofunction:: flake8.utils.filenames_from
-
-When provided an argument to |Flake8|, we need to be able to traverse
-directories in a convenient manner. For example, if someone runs
-
-.. code::
-
-    $ flake8 flake8/
-
-Then they want us to check all of the files in the directory ``flake8/``. This
-function will handle that while also handling the case where they specify a
-file like:
-
-.. code::
-
-    $ flake8 flake8/__init__.py
-
-
 .. autofunction:: flake8.utils.fnmatch
 
 The standard library's :func:`fnmatch.fnmatch` is excellent at deciding if a

setup.cfg

@@ -111,6 +111,8 @@ warn_unused_ignores = true
 disallow_untyped_defs = true
 [mypy-flake8.defaults]
 disallow_untyped_defs = true
+[mypy-flake8.discover_files]
+disallow_untyped_defs = true
 [mypy-flake8.exceptions]
 disallow_untyped_defs = true
 [mypy-flake8.formatting.*]

src/flake8/checker.py

@@ -15,6 +15,7 @@
 from flake8 import exceptions
 from flake8 import processor
 from flake8 import utils
+from flake8.discover_files import expand_paths
 
 Results = List[Tuple[str, int, int, str, Optional[str]]]
 
@@ -155,70 +156,21 @@ def _handle_results(self, filename, results):
             )
         return reported_results_count
 
-    def is_path_excluded(self, path: str) -> bool:
-        """Check if a path is excluded.
-
-        :param str path:
-            Path to check against the exclude patterns.
-        :returns:
-            True if there are exclude patterns and the path matches,
-            otherwise False.
-        :rtype:
-            bool
-        """
-        if path == "-":
-            if self.options.stdin_display_name == "stdin":
-                return False
-            path = self.options.stdin_display_name
-
-        return utils.matches_filename(
-            path,
-            patterns=self.exclude,
-            log_message='"%(path)s" has %(whether)sbeen excluded',
-            logger=LOG,
-        )
-
     def make_checkers(self, paths: Optional[List[str]] = None) -> None:
         """Create checkers for each file."""
         if paths is None:
             paths = self.arguments
 
-        if not paths:
-            paths = ["."]
-
-        filename_patterns = self.options.filename
-        running_from_diff = self.options.diff
-
-        # NOTE(sigmavirus24): Yes this is a little unsightly, but it's our
-        # best solution right now.
-        def should_create_file_checker(filename, argument):
-            """Determine if we should create a file checker."""
-            matches_filename_patterns = utils.fnmatch(
-                filename, filename_patterns
-            )
-            is_stdin = filename == "-"
-            # NOTE(sigmavirus24): If a user explicitly specifies something,
-            # e.g, ``flake8 bin/script`` then we should run Flake8 against
-            # that. Since should_create_file_checker looks to see if the
-            # filename patterns match the filename, we want to skip that in
-            # the event that the argument and the filename are identical.
-            # If it was specified explicitly, the user intended for it to be
-            # checked.
-            explicitly_provided = not running_from_diff and (
-                argument == filename
-            )
-            return (
-                explicitly_provided or matches_filename_patterns
-            ) or is_stdin
-
         checks = self.checks.to_dictionary()
         self._all_checkers = [
             FileChecker(filename, checks, self.options)
-            for argument in paths
-            for filename in utils.filenames_from(
-                argument, self.is_path_excluded
+            for filename in expand_paths(
+                paths=paths,
+                stdin_display_name=self.options.stdin_display_name,
+                filename_patterns=self.options.filename,
+                exclude=self.exclude,
+                is_running_from_diff=self.options.diff,
             )
-            if should_create_file_checker(filename, argument)
         ]
         self.checkers = [c for c in self._all_checkers if c.should_process]
         LOG.info("Checking %d files", len(self.checkers))

src/flake8/discover_files.py

@@ -0,0 +1,96 @@
+"""Functions related to discovering paths."""
+import logging
+import os.path
+from typing import Callable
+from typing import Generator
+from typing import Sequence
+
+from flake8 import utils
+
+LOG = logging.getLogger(__name__)
+
+
+def _filenames_from(
+    arg: str,
+    *,
+    predicate: Callable[[str], bool],
+) -> Generator[str, None, None]:
+    """Generate filenames from an argument.
+
+    :param str arg:
+        Parameter from the command-line.
+    :param callable predicate:
+        Predicate to use to filter out filenames. If the predicate
+        returns ``True`` we will exclude the filename, otherwise we
+        will yield it. By default, we include every filename
+        generated.
+    :returns:
+        Generator of paths
+    """
+    if predicate(arg):
+        return
+
+    if os.path.isdir(arg):
+        for root, sub_directories, files in os.walk(arg):
+            # NOTE(sigmavirus24): os.walk() will skip a directory if you
+            # remove it from the list of sub-directories.
+            for directory in tuple(sub_directories):
+                joined = os.path.join(root, directory)
+                if predicate(joined):
+                    sub_directories.remove(directory)
+
+            for filename in files:
+                joined = os.path.join(root, filename)
+                if not predicate(joined):
+                    yield joined
+    else:
+        yield arg
+
+
+def expand_paths(
+    *,
+    paths: Sequence[str],
+    stdin_display_name: str,
+    filename_patterns: Sequence[str],
+    exclude: Sequence[str],
+    is_running_from_diff: bool,
+) -> Generator[str, None, None]:
+    """Expand out ``paths`` from commandline to the lintable files."""
+    if not paths:
+        paths = ["."]
+
+    def is_excluded(arg: str) -> bool:
+        if arg == "-":
+            # if the stdin_display_name is the default, always include it
+            if stdin_display_name == "stdin":
+                return False
+            arg = stdin_display_name
+
+        return utils.matches_filename(
+            arg,
+            patterns=exclude,
+            log_message='"%(path)s" has %(whether)sbeen excluded',
+            logger=LOG,
+        )
+
+    def is_included(arg: str, fname: str) -> bool:
+        # while running from a diff, the arguments aren't _explicitly_
+        # listed so we still filter them
+        if is_running_from_diff:
+            return utils.fnmatch(fname, filename_patterns)
+        else:
+            return (
+                # always lint `-`
+                fname == "-"
+                # always lint explicitly passed (even if not matching filter)
+                or arg == fname
+                # otherwise, check the file against filtered patterns
+                or utils.fnmatch(fname, filename_patterns)
+            )
+
+    return (
+        filename
+        for path in paths
+        for filename in _filenames_from(path, predicate=is_excluded)
+        if is_included(path, filename)
+    )

src/flake8/utils.py

@@ -11,9 +11,7 @@
 import sys
 import textwrap
 import tokenize
-from typing import Callable
 from typing import Dict
-from typing import Generator
 from typing import List
 from typing import Optional
 from typing import Pattern
@@ -294,52 +292,6 @@ def is_using_stdin(paths: List[str]) -> bool:
     return "-" in paths
 
 
-def _default_predicate(*args: str) -> bool:
-    return False
-
-
-def filenames_from(
-    arg: str, predicate: Optional[Callable[[str], bool]] = None
-) -> Generator[str, None, None]:
-    """Generate filenames from an argument.
-
-    :param str arg:
-        Parameter from the command-line.
-    :param callable predicate:
-        Predicate to use to filter out filenames. If the predicate
-        returns ``True`` we will exclude the filename, otherwise we
-        will yield it. By default, we include every filename
-        generated.
-    :returns:
-        Generator of paths
-    """
-    if predicate is None:
-        predicate = _default_predicate
-
-    if predicate(arg):
-        return
-
-    if os.path.isdir(arg):
-        for root, sub_directories, files in os.walk(arg):
-            if predicate(root):
-                sub_directories[:] = []
-                continue
-
-            # NOTE(sigmavirus24): os.walk() will skip a directory if you
-            # remove it from the list of sub-directories.
-            for directory in sub_directories:
-                joined = os.path.join(root, directory)
-                if predicate(joined):
-                    sub_directories.remove(directory)
-
-            for filename in files:
-                joined = os.path.join(root, filename)
-                if not predicate(joined):
-                    yield joined
-    else:
-        yield arg
-
-
 def fnmatch(filename: str, patterns: Sequence[str]) -> bool:
     """Wrap :func:`fnmatch.fnmatch` to add some functionality.
 
@@ -351,7 +303,7 @@ def fnmatch(filename: str, patterns: Sequence[str]) -> bool:
         The default value if patterns is empty
     :returns:
         True if a pattern matches the filename, False if it doesn't.
-        ``default`` if patterns is empty.
+        ``True`` if patterns is empty.
     """
     if not patterns:
         return True

tests/unit/test_checker_manager.py

@@ -85,11 +85,9 @@ def test_make_checkers(_):
     }
     manager = checker.Manager(style_guide, files, checkplugins)
 
-    with mock.patch("flake8.utils.filenames_from") as filenames_from:
-        filenames_from.side_effect = [["file1"], ["file2"]]
-        with mock.patch("flake8.utils.fnmatch", return_value=True):
-            with mock.patch("flake8.processor.FileProcessor"):
-                manager.make_checkers()
+    with mock.patch("flake8.utils.fnmatch", return_value=True):
+        with mock.patch("flake8.processor.FileProcessor"):
+            manager.make_checkers(["file1", "file2"])
 
     assert manager._all_checkers
     for file_checker in manager._all_checkers: