diff --git a/ci/lint/format.sh b/ci/lint/format.sh index 026d7d3dda466..2f243b3b56f9e 100755 --- a/ci/lint/format.sh +++ b/ci/lint/format.sh @@ -115,11 +115,11 @@ MYPY_FILES=( ) BLACK_EXCLUDES=( - '--extend-exclude' 'python/ray/cloudpickle/*' - '--extend-exclude' 'python/build/*' - '--extend-exclude' 'python/ray/core/src/ray/gcs/*' - '--extend-exclude' 'python/ray/thirdparty_files/*' - '--extend-exclude' 'python/ray/_private/thirdparty/*' + '--force-exclude' 'python/ray/cloudpickle/*' + '--force-exclude' 'python/build/*' + '--force-exclude' 'python/ray/core/src/ray/gcs/*' + '--force-exclude' 'python/ray/thirdparty_files/*' + '--force-exclude' 'python/ray/_private/thirdparty/*' ) GIT_LS_EXCLUDES=( diff --git a/python/ray/_private/thirdparty/pathspec/__init__.py b/python/ray/_private/thirdparty/pathspec/__init__.py index bda8556abb3a6..423759fd5bcc5 100644 --- a/python/ray/_private/thirdparty/pathspec/__init__.py +++ b/python/ray/_private/thirdparty/pathspec/__init__.py @@ -27,28 +27,28 @@ __copyright__ = "Copyright © 2013-2020 Caleb P. Burns" __created__ = "2013-10-12" __credits__ = [ - "dahlia ", - "highb ", - "029xue ", - "mikexstudios ", - "nhumrich ", - "davidfraser ", - "demurgos ", - "ghickman ", - "nvie ", - "adrienverge ", - "AndersBlomdell ", - "highb ", - "thmxv ", - "wimglenn ", - "hugovk ", - "dcecile ", - "mroutis ", - "jdufresne ", - "groodt ", - "ftrofin ", - "pykong ", - "nhhollander ", + "dahlia ", + "highb ", + "029xue ", + "mikexstudios ", + "nhumrich ", + "davidfraser ", + "demurgos ", + "ghickman ", + "nvie ", + "adrienverge ", + "AndersBlomdell ", + "highb ", + "thmxv ", + "wimglenn ", + "hugovk ", + "dcecile ", + "mroutis ", + "jdufresne ", + "groodt ", + "ftrofin ", + "pykong ", + "nhhollander ", ] __email__ = "cpburnz@gmail.com" __license__ = "MPL 2.0" diff --git a/python/ray/_private/thirdparty/pathspec/compat.py b/python/ray/_private/thirdparty/pathspec/compat.py index f22c6187a35a2..37c6480510f49 100644 --- a/python/ray/_private/thirdparty/pathspec/compat.py +++ b/python/ray/_private/thirdparty/pathspec/compat.py @@ -9,31 +9,30 @@ import sys if sys.version_info[0] < 3: - # Python 2. - unicode = unicode - string_types = (basestring,) + # Python 2. + unicode = unicode + string_types = (basestring,) - from collections import Iterable - from itertools import izip_longest + from collections import Iterable + from itertools import izip_longest - def iterkeys(mapping): - return mapping.iterkeys() + def iterkeys(mapping): + return mapping.iterkeys() else: - # Python 3. - unicode = str - string_types = (unicode,) + # Python 3. + unicode = str + string_types = (unicode,) - from collections.abc import Iterable - from itertools import zip_longest as izip_longest - - def iterkeys(mapping): - return mapping.keys() + from collections.abc import Iterable + from itertools import zip_longest as izip_longest + def iterkeys(mapping): + return mapping.keys() try: - # Python 3.6+. - from collections.abc import Collection + # Python 3.6+. + from collections.abc import Collection except ImportError: - # Python 2.7 - 3.5. - from collections import Container as Collection + # Python 2.7 - 3.5. + from collections import Container as Collection diff --git a/python/ray/_private/thirdparty/pathspec/pathspec.py b/python/ray/_private/thirdparty/pathspec/pathspec.py index 0e8f57bf8e200..73250efeb4da2 100644 --- a/python/ray/_private/thirdparty/pathspec/pathspec.py +++ b/python/ray/_private/thirdparty/pathspec/pathspec.py @@ -9,202 +9,198 @@ class PathSpec(object): - """ - The :class:`PathSpec` class is a wrapper around a list of compiled - :class:`.Pattern` instances. - """ + """ + The :class:`PathSpec` class is a wrapper around a list of compiled + :class:`.Pattern` instances. + """ - def __init__(self, patterns): - """ - Initializes the :class:`PathSpec` instance. + def __init__(self, patterns): + """ + Initializes the :class:`PathSpec` instance. - *patterns* (:class:`~collections.abc.Collection` or :class:`~collections.abc.Iterable`) - yields each compiled pattern (:class:`.Pattern`). - """ + *patterns* (:class:`~collections.abc.Collection` or :class:`~collections.abc.Iterable`) + yields each compiled pattern (:class:`.Pattern`). + """ - self.patterns = patterns if isinstance(patterns, Collection) else list(patterns) - """ + self.patterns = patterns if isinstance(patterns, Collection) else list(patterns) + """ *patterns* (:class:`~collections.abc.Collection` of :class:`.Pattern`) contains the compiled patterns. """ - def __eq__(self, other): - """ - Tests the equality of this path-spec with *other* (:class:`PathSpec`) - by comparing their :attr:`~PathSpec.patterns` attributes. - """ - if isinstance(other, PathSpec): - paired_patterns = izip_longest(self.patterns, other.patterns) - return all(a == b for a, b in paired_patterns) - else: - return NotImplemented - - def __len__(self): - """ - Returns the number of compiled patterns this path-spec contains - (:class:`int`). - """ - return len(self.patterns) - - def __add__(self, other): - """ - Combines the :attr:`Pathspec.patterns` patterns from two - :class:`PathSpec` instances. - """ - if isinstance(other, PathSpec): - return PathSpec(self.patterns + other.patterns) - else: - return NotImplemented - - def __iadd__(self, other): - """ - Adds the :attr:`Pathspec.patterns` patterns from one :class:`PathSpec` - instance to this instance. - """ - if isinstance(other, PathSpec): - self.patterns += other.patterns - return self - else: - return NotImplemented - - @classmethod - def from_lines(cls, pattern_factory, lines): - """ - Compiles the pattern lines. - - *pattern_factory* can be either the name of a registered pattern - factory (:class:`str`), or a :class:`~collections.abc.Callable` used - to compile patterns. It must accept an uncompiled pattern (:class:`str`) - and return the compiled pattern (:class:`.Pattern`). - - *lines* (:class:`~collections.abc.Iterable`) yields each uncompiled - pattern (:class:`str`). This simply has to yield each line so it can - be a :class:`file` (e.g., from :func:`open` or :class:`io.StringIO`) - or the result from :meth:`str.splitlines`. - - Returns the :class:`PathSpec` instance. - """ - if isinstance(pattern_factory, string_types): - pattern_factory = util.lookup_pattern(pattern_factory) - if not callable(pattern_factory): - raise TypeError( - "pattern_factory:{!r} is not callable.".format(pattern_factory) - ) - - if not util._is_iterable(lines): - raise TypeError("lines:{!r} is not an iterable.".format(lines)) - - lines = [pattern_factory(line) for line in lines if line] - return cls(lines) - - def match_file(self, file, separators=None): - """ - Matches the file to this path-spec. - - *file* (:class:`str` or :class:`~pathlib.PurePath`) is the file path - to be matched against :attr:`self.patterns `. - - *separators* (:class:`~collections.abc.Collection` of :class:`str`) - optionally contains the path separators to normalize. See - :func:`~pathspec.util.normalize_file` for more information. - - Returns :data:`True` if *file* matched; otherwise, :data:`False`. - """ - norm_file = util.normalize_file(file, separators=separators) - return util.match_file(self.patterns, norm_file) - - def match_entries(self, entries, separators=None): - """ - Matches the entries to this path-spec. - - *entries* (:class:`~collections.abc.Iterable` of :class:`~util.TreeEntry`) - contains the entries to be matched against :attr:`self.patterns `. - - *separators* (:class:`~collections.abc.Collection` of :class:`str`; - or :data:`None`) optionally contains the path separators to - normalize. See :func:`~pathspec.util.normalize_file` for more - information. - - Returns the matched entries (:class:`~collections.abc.Iterable` of - :class:`~util.TreeEntry`). - """ - if not util._is_iterable(entries): - raise TypeError("entries:{!r} is not an iterable.".format(entries)) - - entry_map = util._normalize_entries(entries, separators=separators) - match_paths = util.match_files(self.patterns, iterkeys(entry_map)) - for path in match_paths: - yield entry_map[path] - - def match_files(self, files, separators=None): - """ - Matches the files to this path-spec. - - *files* (:class:`~collections.abc.Iterable` of :class:`str; or - :class:`pathlib.PurePath`) contains the file paths to be matched - against :attr:`self.patterns `. - - *separators* (:class:`~collections.abc.Collection` of :class:`str`; - or :data:`None`) optionally contains the path separators to - normalize. See :func:`~pathspec.util.normalize_file` for more - information. - - Returns the matched files (:class:`~collections.abc.Iterable` of - :class:`str`). - """ - if not util._is_iterable(files): - raise TypeError("files:{!r} is not an iterable.".format(files)) - - file_map = util.normalize_files(files, separators=separators) - matched_files = util.match_files(self.patterns, iterkeys(file_map)) - for path in matched_files: - yield file_map[path] - - def match_tree_entries(self, root, on_error=None, follow_links=None): - """ - Walks the specified root path for all files and matches them to this - path-spec. - - *root* (:class:`str`; or :class:`pathlib.PurePath`) is the root - directory to search. - - *on_error* (:class:`~collections.abc.Callable` or :data:`None`) - optionally is the error handler for file-system exceptions. See - :func:`~pathspec.util.iter_tree_entries` for more information. - - *follow_links* (:class:`bool` or :data:`None`) optionally is whether - to walk symbolic links that resolve to directories. See - :func:`~pathspec.util.iter_tree_files` for more information. - - Returns the matched files (:class:`~collections.abc.Iterable` of - :class:`str`). - """ - entries = util.iter_tree_entries( - root, on_error=on_error, follow_links=follow_links - ) - return self.match_entries(entries) - - def match_tree_files(self, root, on_error=None, follow_links=None): - """ - Walks the specified root path for all files and matches them to this - path-spec. - - *root* (:class:`str`; or :class:`pathlib.PurePath`) is the root - directory to search for files. - - *on_error* (:class:`~collections.abc.Callable` or :data:`None`) - optionally is the error handler for file-system exceptions. See - :func:`~pathspec.util.iter_tree_files` for more information. - - *follow_links* (:class:`bool` or :data:`None`) optionally is whether - to walk symbolic links that resolve to directories. See - :func:`~pathspec.util.iter_tree_files` for more information. - - Returns the matched files (:class:`~collections.abc.Iterable` of - :class:`str`). - """ - files = util.iter_tree_files(root, on_error=on_error, follow_links=follow_links) - return self.match_files(files) - - # Alias `match_tree_files()` as `match_tree()`. - match_tree = match_tree_files + def __eq__(self, other): + """ + Tests the equality of this path-spec with *other* (:class:`PathSpec`) + by comparing their :attr:`~PathSpec.patterns` attributes. + """ + if isinstance(other, PathSpec): + paired_patterns = izip_longest(self.patterns, other.patterns) + return all(a == b for a, b in paired_patterns) + else: + return NotImplemented + + def __len__(self): + """ + Returns the number of compiled patterns this path-spec contains + (:class:`int`). + """ + return len(self.patterns) + + def __add__(self, other): + """ + Combines the :attr:`Pathspec.patterns` patterns from two + :class:`PathSpec` instances. + """ + if isinstance(other, PathSpec): + return PathSpec(self.patterns + other.patterns) + else: + return NotImplemented + + def __iadd__(self, other): + """ + Adds the :attr:`Pathspec.patterns` patterns from one :class:`PathSpec` + instance to this instance. + """ + if isinstance(other, PathSpec): + self.patterns += other.patterns + return self + else: + return NotImplemented + + @classmethod + def from_lines(cls, pattern_factory, lines): + """ + Compiles the pattern lines. + + *pattern_factory* can be either the name of a registered pattern + factory (:class:`str`), or a :class:`~collections.abc.Callable` used + to compile patterns. It must accept an uncompiled pattern (:class:`str`) + and return the compiled pattern (:class:`.Pattern`). + + *lines* (:class:`~collections.abc.Iterable`) yields each uncompiled + pattern (:class:`str`). This simply has to yield each line so it can + be a :class:`file` (e.g., from :func:`open` or :class:`io.StringIO`) + or the result from :meth:`str.splitlines`. + + Returns the :class:`PathSpec` instance. + """ + if isinstance(pattern_factory, string_types): + pattern_factory = util.lookup_pattern(pattern_factory) + if not callable(pattern_factory): + raise TypeError("pattern_factory:{!r} is not callable.".format(pattern_factory)) + + if not util._is_iterable(lines): + raise TypeError("lines:{!r} is not an iterable.".format(lines)) + + lines = [pattern_factory(line) for line in lines if line] + return cls(lines) + + def match_file(self, file, separators=None): + """ + Matches the file to this path-spec. + + *file* (:class:`str` or :class:`~pathlib.PurePath`) is the file path + to be matched against :attr:`self.patterns `. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`) + optionally contains the path separators to normalize. See + :func:`~pathspec.util.normalize_file` for more information. + + Returns :data:`True` if *file* matched; otherwise, :data:`False`. + """ + norm_file = util.normalize_file(file, separators=separators) + return util.match_file(self.patterns, norm_file) + + def match_entries(self, entries, separators=None): + """ + Matches the entries to this path-spec. + + *entries* (:class:`~collections.abc.Iterable` of :class:`~util.TreeEntry`) + contains the entries to be matched against :attr:`self.patterns `. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`; + or :data:`None`) optionally contains the path separators to + normalize. See :func:`~pathspec.util.normalize_file` for more + information. + + Returns the matched entries (:class:`~collections.abc.Iterable` of + :class:`~util.TreeEntry`). + """ + if not util._is_iterable(entries): + raise TypeError("entries:{!r} is not an iterable.".format(entries)) + + entry_map = util._normalize_entries(entries, separators=separators) + match_paths = util.match_files(self.patterns, iterkeys(entry_map)) + for path in match_paths: + yield entry_map[path] + + def match_files(self, files, separators=None): + """ + Matches the files to this path-spec. + + *files* (:class:`~collections.abc.Iterable` of :class:`str; or + :class:`pathlib.PurePath`) contains the file paths to be matched + against :attr:`self.patterns `. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`; + or :data:`None`) optionally contains the path separators to + normalize. See :func:`~pathspec.util.normalize_file` for more + information. + + Returns the matched files (:class:`~collections.abc.Iterable` of + :class:`str`). + """ + if not util._is_iterable(files): + raise TypeError("files:{!r} is not an iterable.".format(files)) + + file_map = util.normalize_files(files, separators=separators) + matched_files = util.match_files(self.patterns, iterkeys(file_map)) + for path in matched_files: + yield file_map[path] + + def match_tree_entries(self, root, on_error=None, follow_links=None): + """ + Walks the specified root path for all files and matches them to this + path-spec. + + *root* (:class:`str`; or :class:`pathlib.PurePath`) is the root + directory to search. + + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) + optionally is the error handler for file-system exceptions. See + :func:`~pathspec.util.iter_tree_entries` for more information. + + *follow_links* (:class:`bool` or :data:`None`) optionally is whether + to walk symbolic links that resolve to directories. See + :func:`~pathspec.util.iter_tree_files` for more information. + + Returns the matched files (:class:`~collections.abc.Iterable` of + :class:`str`). + """ + entries = util.iter_tree_entries(root, on_error=on_error, follow_links=follow_links) + return self.match_entries(entries) + + def match_tree_files(self, root, on_error=None, follow_links=None): + """ + Walks the specified root path for all files and matches them to this + path-spec. + + *root* (:class:`str`; or :class:`pathlib.PurePath`) is the root + directory to search for files. + + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) + optionally is the error handler for file-system exceptions. See + :func:`~pathspec.util.iter_tree_files` for more information. + + *follow_links* (:class:`bool` or :data:`None`) optionally is whether + to walk symbolic links that resolve to directories. See + :func:`~pathspec.util.iter_tree_files` for more information. + + Returns the matched files (:class:`~collections.abc.Iterable` of + :class:`str`). + """ + files = util.iter_tree_files(root, on_error=on_error, follow_links=follow_links) + return self.match_files(files) + + # Alias `match_tree_files()` as `match_tree()`. + match_tree = match_tree_files diff --git a/python/ray/_private/thirdparty/pathspec/pattern.py b/python/ray/_private/thirdparty/pathspec/pattern.py index 83a17ffb05a7f..4ba4edf790c84 100644 --- a/python/ray/_private/thirdparty/pathspec/pattern.py +++ b/python/ray/_private/thirdparty/pathspec/pattern.py @@ -9,152 +9,138 @@ class Pattern(object): - """ - The :class:`Pattern` class is the abstract definition of a pattern. - """ + """ + The :class:`Pattern` class is the abstract definition of a pattern. + """ - # Make the class dict-less. - __slots__ = ("include",) + # Make the class dict-less. + __slots__ = ('include',) - def __init__(self, include): - """ - Initializes the :class:`Pattern` instance. + def __init__(self, include): + """ + Initializes the :class:`Pattern` instance. - *include* (:class:`bool` or :data:`None`) is whether the matched - files should be included (:data:`True`), excluded (:data:`False`), - or is a null-operation (:data:`None`). - """ + *include* (:class:`bool` or :data:`None`) is whether the matched + files should be included (:data:`True`), excluded (:data:`False`), + or is a null-operation (:data:`None`). + """ - self.include = include - """ + self.include = include + """ *include* (:class:`bool` or :data:`None`) is whether the matched files should be included (:data:`True`), excluded (:data:`False`), or is a null-operation (:data:`None`). """ - def match(self, files): - """ - Matches this pattern against the specified files. + def match(self, files): + """ + Matches this pattern against the specified files. - *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains - each file relative to the root directory (e.g., ``"relative/path/to/file"``). + *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains + each file relative to the root directory (e.g., ``"relative/path/to/file"``). - Returns an :class:`~collections.abc.Iterable` yielding each matched - file path (:class:`str`). - """ - raise NotImplementedError( - "{}.{} must override match().".format( - self.__class__.__module__, self.__class__.__name__ - ) - ) + Returns an :class:`~collections.abc.Iterable` yielding each matched + file path (:class:`str`). + """ + raise NotImplementedError("{}.{} must override match().".format(self.__class__.__module__, self.__class__.__name__)) class RegexPattern(Pattern): - """ - The :class:`RegexPattern` class is an implementation of a pattern - using regular expressions. - """ - - # Make the class dict-less. - __slots__ = ("regex",) - - def __init__(self, pattern, include=None): - """ - Initializes the :class:`RegexPattern` instance. - - *pattern* (:class:`unicode`, :class:`bytes`, :class:`re.RegexObject`, - or :data:`None`) is the pattern to compile into a regular - expression. - - *include* (:class:`bool` or :data:`None`) must be :data:`None` - unless *pattern* is a precompiled regular expression (:class:`re.RegexObject`) - in which case it is whether matched files should be included - (:data:`True`), excluded (:data:`False`), or is a null operation - (:data:`None`). - - .. NOTE:: Subclasses do not need to support the *include* - parameter. - """ - - self.regex = None - """ + """ + The :class:`RegexPattern` class is an implementation of a pattern + using regular expressions. + """ + + # Make the class dict-less. + __slots__ = ('regex',) + + def __init__(self, pattern, include=None): + """ + Initializes the :class:`RegexPattern` instance. + + *pattern* (:class:`unicode`, :class:`bytes`, :class:`re.RegexObject`, + or :data:`None`) is the pattern to compile into a regular + expression. + + *include* (:class:`bool` or :data:`None`) must be :data:`None` + unless *pattern* is a precompiled regular expression (:class:`re.RegexObject`) + in which case it is whether matched files should be included + (:data:`True`), excluded (:data:`False`), or is a null operation + (:data:`None`). + + .. NOTE:: Subclasses do not need to support the *include* + parameter. + """ + + self.regex = None + """ *regex* (:class:`re.RegexObject`) is the regular expression for the pattern. """ - if isinstance(pattern, (unicode, bytes)): - assert ( - include is None - ), "include:{!r} must be null when pattern:{!r} is a string.".format( - include, pattern - ) - regex, include = self.pattern_to_regex(pattern) - # NOTE: Make sure to allow a null regular expression to be - # returned for a null-operation. - if include is not None: - regex = re.compile(regex) - - elif pattern is not None and hasattr(pattern, "match"): - # Assume pattern is a precompiled regular expression. - # - NOTE: Used specified *include*. - regex = pattern - - elif pattern is None: - # NOTE: Make sure to allow a null pattern to be passed for a - # null-operation. - assert ( - include is None - ), "include:{!r} must be null when pattern:{!r} is null.".format( - include, pattern - ) - - else: - raise TypeError( - "pattern:{!r} is not a string, RegexObject, or None.".format(pattern) - ) - - super(RegexPattern, self).__init__(include) - self.regex = regex - - def __eq__(self, other): - """ - Tests the equality of this regex pattern with *other* (:class:`RegexPattern`) - by comparing their :attr:`~Pattern.include` and :attr:`~RegexPattern.regex` - attributes. - """ - if isinstance(other, RegexPattern): - return self.include == other.include and self.regex == other.regex - else: - return NotImplemented - - def match(self, files): - """ - Matches this pattern against the specified files. - - *files* (:class:`~collections.abc.Iterable` of :class:`str`) - contains each file relative to the root directory (e.g., "relative/path/to/file"). - - Returns an :class:`~collections.abc.Iterable` yielding each matched - file path (:class:`str`). - """ - if self.include is not None: - for path in files: - if self.regex.match(path) is not None: - yield path - - @classmethod - def pattern_to_regex(cls, pattern): - """ - Convert the pattern into an uncompiled regular expression. - - *pattern* (:class:`str`) is the pattern to convert into a regular - expression. - - Returns the uncompiled regular expression (:class:`str` or :data:`None`), - and whether matched files should be included (:data:`True`), - excluded (:data:`False`), or is a null-operation (:data:`None`). - - .. NOTE:: The default implementation simply returns *pattern* and - :data:`True`. - """ - return pattern, True + if isinstance(pattern, (unicode, bytes)): + assert include is None, "include:{!r} must be null when pattern:{!r} is a string.".format(include, pattern) + regex, include = self.pattern_to_regex(pattern) + # NOTE: Make sure to allow a null regular expression to be + # returned for a null-operation. + if include is not None: + regex = re.compile(regex) + + elif pattern is not None and hasattr(pattern, 'match'): + # Assume pattern is a precompiled regular expression. + # - NOTE: Used specified *include*. + regex = pattern + + elif pattern is None: + # NOTE: Make sure to allow a null pattern to be passed for a + # null-operation. + assert include is None, "include:{!r} must be null when pattern:{!r} is null.".format(include, pattern) + + else: + raise TypeError("pattern:{!r} is not a string, RegexObject, or None.".format(pattern)) + + super(RegexPattern, self).__init__(include) + self.regex = regex + + def __eq__(self, other): + """ + Tests the equality of this regex pattern with *other* (:class:`RegexPattern`) + by comparing their :attr:`~Pattern.include` and :attr:`~RegexPattern.regex` + attributes. + """ + if isinstance(other, RegexPattern): + return self.include == other.include and self.regex == other.regex + else: + return NotImplemented + + def match(self, files): + """ + Matches this pattern against the specified files. + + *files* (:class:`~collections.abc.Iterable` of :class:`str`) + contains each file relative to the root directory (e.g., "relative/path/to/file"). + + Returns an :class:`~collections.abc.Iterable` yielding each matched + file path (:class:`str`). + """ + if self.include is not None: + for path in files: + if self.regex.match(path) is not None: + yield path + + @classmethod + def pattern_to_regex(cls, pattern): + """ + Convert the pattern into an uncompiled regular expression. + + *pattern* (:class:`str`) is the pattern to convert into a regular + expression. + + Returns the uncompiled regular expression (:class:`str` or :data:`None`), + and whether matched files should be included (:data:`True`), + excluded (:data:`False`), or is a null-operation (:data:`None`). + + .. NOTE:: The default implementation simply returns *pattern* and + :data:`True`. + """ + return pattern, True diff --git a/python/ray/_private/thirdparty/pathspec/patterns/gitwildmatch.py b/python/ray/_private/thirdparty/pathspec/patterns/gitwildmatch.py index d78ce01a781f6..07fd03880a9ad 100644 --- a/python/ray/_private/thirdparty/pathspec/patterns/gitwildmatch.py +++ b/python/ray/_private/thirdparty/pathspec/patterns/gitwildmatch.py @@ -14,327 +14,317 @@ from ..pattern import RegexPattern #: The encoding to use when parsing a byte string pattern. -_BYTES_ENCODING = "latin1" +_BYTES_ENCODING = 'latin1' class GitWildMatchPattern(RegexPattern): - """ - The :class:`GitWildMatchPattern` class represents a compiled Git - wildmatch pattern. - """ - - # Keep the dict-less class hierarchy. - __slots__ = () - - @classmethod - def pattern_to_regex(cls, pattern): - """ - Convert the pattern into a regular expression. - - *pattern* (:class:`unicode` or :class:`bytes`) is the pattern to - convert into a regular expression. - - Returns the uncompiled regular expression (:class:`unicode`, :class:`bytes`, - or :data:`None`), and whether matched files should be included - (:data:`True`), excluded (:data:`False`), or if it is a - null-operation (:data:`None`). - """ - if isinstance(pattern, unicode): - return_type = unicode - elif isinstance(pattern, bytes): - return_type = bytes - pattern = pattern.decode(_BYTES_ENCODING) - else: - raise TypeError( - "pattern:{!r} is not a unicode or byte string.".format(pattern) - ) - - pattern = pattern.strip() - - if pattern.startswith("#"): - # A pattern starting with a hash ('#') serves as a comment - # (neither includes nor excludes files). Escape the hash with a - # back-slash to match a literal hash (i.e., '\#'). - regex = None - include = None - - elif pattern == "/": - # EDGE CASE: According to `git check-ignore` (v2.4.1), a single - # '/' does not match any file. - regex = None - include = None - - elif pattern: - - if pattern.startswith("!"): - # A pattern starting with an exclamation mark ('!') negates the - # pattern (exclude instead of include). Escape the exclamation - # mark with a back-slash to match a literal exclamation mark - # (i.e., '\!'). - include = False - # Remove leading exclamation mark. - pattern = pattern[1:] - else: - include = True - - if pattern.startswith("\\"): - # Remove leading back-slash escape for escaped hash ('#') or - # exclamation mark ('!'). - pattern = pattern[1:] - - # Split pattern into segments. - pattern_segs = pattern.split("/") - - # Normalize pattern to make processing easier. - - if not pattern_segs[0]: - # A pattern beginning with a slash ('/') will only match paths - # directly on the root directory instead of any descendant - # paths. So, remove empty first segment to make pattern relative - # to root. - del pattern_segs[0] - - elif len(pattern_segs) == 1 or ( - len(pattern_segs) == 2 and not pattern_segs[1] - ): - # A single pattern without a beginning slash ('/') will match - # any descendant path. This is equivalent to "**/{pattern}". So, - # prepend with double-asterisks to make pattern relative to - # root. - # EDGE CASE: This also holds for a single pattern with a - # trailing slash (e.g. dir/). - if pattern_segs[0] != "**": - pattern_segs.insert(0, "**") - - else: - # EDGE CASE: A pattern without a beginning slash ('/') but - # contains at least one prepended directory (e.g. - # "dir/{pattern}") should not match "**/dir/{pattern}", - # according to `git check-ignore` (v2.4.1). - pass - - if not pattern_segs[-1] and len(pattern_segs) > 1: - # A pattern ending with a slash ('/') will match all descendant - # paths if it is a directory but not if it is a regular file. - # This is equivilent to "{pattern}/**". So, set last segment to - # double asterisks to include all descendants. - pattern_segs[-1] = "**" - - # Build regular expression from pattern. - output = ["^"] - need_slash = False - end = len(pattern_segs) - 1 - for i, seg in enumerate(pattern_segs): - if seg == "**": - if i == 0 and i == end: - # A pattern consisting solely of double-asterisks ('**') - # will match every path. - output.append(".+") - elif i == 0: - # A normalized pattern beginning with double-asterisks - # ('**') will match any leading path segments. - output.append("(?:.+/)?") - need_slash = False - elif i == end: - # A normalized pattern ending with double-asterisks ('**') - # will match any trailing path segments. - output.append("/.*") - else: - # A pattern with inner double-asterisks ('**') will match - # multiple (or zero) inner path segments. - output.append("(?:/.+)?") - need_slash = True - elif seg == "*": - # Match single path segment. - if need_slash: - output.append("/") - output.append("[^/]+") - need_slash = True - else: - # Match segment glob pattern. - if need_slash: - output.append("/") - output.append(cls._translate_segment_glob(seg)) - if i == end and include is True: - # A pattern ending without a slash ('/') will match a file - # or a directory (with paths underneath it). E.g., "foo" - # matches "foo", "foo/bar", "foo/bar/baz", etc. - # EDGE CASE: However, this does not hold for exclusion cases - # according to `git check-ignore` (v2.4.1). - output.append("(?:/.*)?") - need_slash = True - output.append("$") - regex = "".join(output) - - else: - # A blank pattern is a null-operation (neither includes nor - # excludes files). - regex = None - include = None - - if regex is not None and return_type is bytes: - regex = regex.encode(_BYTES_ENCODING) - - return regex, include - - @staticmethod - def _translate_segment_glob(pattern): - """ - Translates the glob pattern to a regular expression. This is used in - the constructor to translate a path segment glob pattern to its - corresponding regular expression. - - *pattern* (:class:`str`) is the glob pattern. - - Returns the regular expression (:class:`str`). - """ - # NOTE: This is derived from `fnmatch.translate()` and is similar to - # the POSIX function `fnmatch()` with the `FNM_PATHNAME` flag set. - - escape = False - regex = "" - i, end = 0, len(pattern) - while i < end: - # Get next character. - char = pattern[i] - i += 1 - - if escape: - # Escape the character. - escape = False - regex += re.escape(char) - - elif char == "\\": - # Escape character, escape next character. - escape = True - - elif char == "*": - # Multi-character wildcard. Match any string (except slashes), - # including an empty string. - regex += "[^/]*" - - elif char == "?": - # Single-character wildcard. Match any single character (except - # a slash). - regex += "[^/]" - - elif char == "[": - # Braket expression wildcard. Except for the beginning - # exclamation mark, the whole braket expression can be used - # directly as regex but we have to find where the expression - # ends. - # - "[][!]" matchs ']', '[' and '!'. - # - "[]-]" matchs ']' and '-'. - # - "[!]a-]" matchs any character except ']', 'a' and '-'. - j = i - # Pass brack expression negation. - if j < end and pattern[j] == "!": - j += 1 - # Pass first closing braket if it is at the beginning of the - # expression. - if j < end and pattern[j] == "]": - j += 1 - # Find closing braket. Stop once we reach the end or find it. - while j < end and pattern[j] != "]": - j += 1 - - if j < end: - # Found end of braket expression. Increment j to be one past - # the closing braket: - # - # [...] - # ^ ^ - # i j - # - j += 1 - expr = "[" - - if pattern[i] == "!": - # Braket expression needs to be negated. - expr += "^" - i += 1 - elif pattern[i] == "^": - # POSIX declares that the regex braket expression negation - # "[^...]" is undefined in a glob pattern. Python's - # `fnmatch.translate()` escapes the caret ('^') as a - # literal. To maintain consistency with undefined behavior, - # I am escaping the '^' as well. - expr += "\\^" - i += 1 - - # Build regex braket expression. Escape slashes so they are - # treated as literal slashes by regex as defined by POSIX. - expr += pattern[i:j].replace("\\", "\\\\") - - # Add regex braket expression to regex result. - regex += expr - - # Set i to one past the closing braket. - i = j - - else: - # Failed to find closing braket, treat opening braket as a - # braket literal instead of as an expression. - regex += "\\[" - - else: - # Regular character, escape it for regex. - regex += re.escape(char) - - return regex - - @staticmethod - def escape(s): - """ - Escape special characters in the given string. - - *s* (:class:`unicode` or :class:`bytes`) a filename or a string - that you want to escape, usually before adding it to a `.gitignore` - - Returns the escaped string (:class:`unicode`, :class:`bytes`) - """ - # Reference: https://git-scm.com/docs/gitignore#_pattern_format - meta_characters = r"[]!*#?" - - return "".join("\\" + x if x in meta_characters else x for x in s) - - -util.register_pattern("gitwildmatch", GitWildMatchPattern) + """ + The :class:`GitWildMatchPattern` class represents a compiled Git + wildmatch pattern. + """ + + # Keep the dict-less class hierarchy. + __slots__ = () + + @classmethod + def pattern_to_regex(cls, pattern): + """ + Convert the pattern into a regular expression. + + *pattern* (:class:`unicode` or :class:`bytes`) is the pattern to + convert into a regular expression. + + Returns the uncompiled regular expression (:class:`unicode`, :class:`bytes`, + or :data:`None`), and whether matched files should be included + (:data:`True`), excluded (:data:`False`), or if it is a + null-operation (:data:`None`). + """ + if isinstance(pattern, unicode): + return_type = unicode + elif isinstance(pattern, bytes): + return_type = bytes + pattern = pattern.decode(_BYTES_ENCODING) + else: + raise TypeError("pattern:{!r} is not a unicode or byte string.".format(pattern)) + + pattern = pattern.strip() + + if pattern.startswith('#'): + # A pattern starting with a hash ('#') serves as a comment + # (neither includes nor excludes files). Escape the hash with a + # back-slash to match a literal hash (i.e., '\#'). + regex = None + include = None + + elif pattern == '/': + # EDGE CASE: According to `git check-ignore` (v2.4.1), a single + # '/' does not match any file. + regex = None + include = None + + elif pattern: + + if pattern.startswith('!'): + # A pattern starting with an exclamation mark ('!') negates the + # pattern (exclude instead of include). Escape the exclamation + # mark with a back-slash to match a literal exclamation mark + # (i.e., '\!'). + include = False + # Remove leading exclamation mark. + pattern = pattern[1:] + else: + include = True + + if pattern.startswith('\\'): + # Remove leading back-slash escape for escaped hash ('#') or + # exclamation mark ('!'). + pattern = pattern[1:] + + # Split pattern into segments. + pattern_segs = pattern.split('/') + + # Normalize pattern to make processing easier. + + if not pattern_segs[0]: + # A pattern beginning with a slash ('/') will only match paths + # directly on the root directory instead of any descendant + # paths. So, remove empty first segment to make pattern relative + # to root. + del pattern_segs[0] + + elif len(pattern_segs) == 1 or (len(pattern_segs) == 2 and not pattern_segs[1]): + # A single pattern without a beginning slash ('/') will match + # any descendant path. This is equivalent to "**/{pattern}". So, + # prepend with double-asterisks to make pattern relative to + # root. + # EDGE CASE: This also holds for a single pattern with a + # trailing slash (e.g. dir/). + if pattern_segs[0] != '**': + pattern_segs.insert(0, '**') + + else: + # EDGE CASE: A pattern without a beginning slash ('/') but + # contains at least one prepended directory (e.g. + # "dir/{pattern}") should not match "**/dir/{pattern}", + # according to `git check-ignore` (v2.4.1). + pass + + if not pattern_segs[-1] and len(pattern_segs) > 1: + # A pattern ending with a slash ('/') will match all descendant + # paths if it is a directory but not if it is a regular file. + # This is equivilent to "{pattern}/**". So, set last segment to + # double asterisks to include all descendants. + pattern_segs[-1] = '**' + + # Build regular expression from pattern. + output = ['^'] + need_slash = False + end = len(pattern_segs) - 1 + for i, seg in enumerate(pattern_segs): + if seg == '**': + if i == 0 and i == end: + # A pattern consisting solely of double-asterisks ('**') + # will match every path. + output.append('.+') + elif i == 0: + # A normalized pattern beginning with double-asterisks + # ('**') will match any leading path segments. + output.append('(?:.+/)?') + need_slash = False + elif i == end: + # A normalized pattern ending with double-asterisks ('**') + # will match any trailing path segments. + output.append('/.*') + else: + # A pattern with inner double-asterisks ('**') will match + # multiple (or zero) inner path segments. + output.append('(?:/.+)?') + need_slash = True + elif seg == '*': + # Match single path segment. + if need_slash: + output.append('/') + output.append('[^/]+') + need_slash = True + else: + # Match segment glob pattern. + if need_slash: + output.append('/') + output.append(cls._translate_segment_glob(seg)) + if i == end and include is True: + # A pattern ending without a slash ('/') will match a file + # or a directory (with paths underneath it). E.g., "foo" + # matches "foo", "foo/bar", "foo/bar/baz", etc. + # EDGE CASE: However, this does not hold for exclusion cases + # according to `git check-ignore` (v2.4.1). + output.append('(?:/.*)?') + need_slash = True + output.append('$') + regex = ''.join(output) + + else: + # A blank pattern is a null-operation (neither includes nor + # excludes files). + regex = None + include = None + + if regex is not None and return_type is bytes: + regex = regex.encode(_BYTES_ENCODING) + + return regex, include + + @staticmethod + def _translate_segment_glob(pattern): + """ + Translates the glob pattern to a regular expression. This is used in + the constructor to translate a path segment glob pattern to its + corresponding regular expression. + + *pattern* (:class:`str`) is the glob pattern. + + Returns the regular expression (:class:`str`). + """ + # NOTE: This is derived from `fnmatch.translate()` and is similar to + # the POSIX function `fnmatch()` with the `FNM_PATHNAME` flag set. + + escape = False + regex = '' + i, end = 0, len(pattern) + while i < end: + # Get next character. + char = pattern[i] + i += 1 + + if escape: + # Escape the character. + escape = False + regex += re.escape(char) + + elif char == '\\': + # Escape character, escape next character. + escape = True + + elif char == '*': + # Multi-character wildcard. Match any string (except slashes), + # including an empty string. + regex += '[^/]*' + + elif char == '?': + # Single-character wildcard. Match any single character (except + # a slash). + regex += '[^/]' + + elif char == '[': + # Braket expression wildcard. Except for the beginning + # exclamation mark, the whole braket expression can be used + # directly as regex but we have to find where the expression + # ends. + # - "[][!]" matchs ']', '[' and '!'. + # - "[]-]" matchs ']' and '-'. + # - "[!]a-]" matchs any character except ']', 'a' and '-'. + j = i + # Pass brack expression negation. + if j < end and pattern[j] == '!': + j += 1 + # Pass first closing braket if it is at the beginning of the + # expression. + if j < end and pattern[j] == ']': + j += 1 + # Find closing braket. Stop once we reach the end or find it. + while j < end and pattern[j] != ']': + j += 1 + + if j < end: + # Found end of braket expression. Increment j to be one past + # the closing braket: + # + # [...] + # ^ ^ + # i j + # + j += 1 + expr = '[' + + if pattern[i] == '!': + # Braket expression needs to be negated. + expr += '^' + i += 1 + elif pattern[i] == '^': + # POSIX declares that the regex braket expression negation + # "[^...]" is undefined in a glob pattern. Python's + # `fnmatch.translate()` escapes the caret ('^') as a + # literal. To maintain consistency with undefined behavior, + # I am escaping the '^' as well. + expr += '\\^' + i += 1 + + # Build regex braket expression. Escape slashes so they are + # treated as literal slashes by regex as defined by POSIX. + expr += pattern[i:j].replace('\\', '\\\\') + + # Add regex braket expression to regex result. + regex += expr + + # Set i to one past the closing braket. + i = j + + else: + # Failed to find closing braket, treat opening braket as a + # braket literal instead of as an expression. + regex += '\\[' + + else: + # Regular character, escape it for regex. + regex += re.escape(char) + + return regex + + @staticmethod + def escape(s): + """ + Escape special characters in the given string. + + *s* (:class:`unicode` or :class:`bytes`) a filename or a string + that you want to escape, usually before adding it to a `.gitignore` + + Returns the escaped string (:class:`unicode`, :class:`bytes`) + """ + # Reference: https://git-scm.com/docs/gitignore#_pattern_format + meta_characters = r"[]!*#?" + + return "".join("\\" + x if x in meta_characters else x for x in s) + +util.register_pattern('gitwildmatch', GitWildMatchPattern) class GitIgnorePattern(GitWildMatchPattern): - """ - The :class:`GitIgnorePattern` class is deprecated by :class:`GitWildMatchPattern`. - This class only exists to maintain compatibility with v0.4. - """ - - def __init__(self, *args, **kw): - """ - Warn about deprecation. - """ - self._deprecated() - return super(GitIgnorePattern, self).__init__(*args, **kw) - - @staticmethod - def _deprecated(): - """ - Warn about deprecation. - """ - warnings.warn( - "GitIgnorePattern ('gitignore') is deprecated. Use GitWildMatchPattern ('gitwildmatch') instead.", - DeprecationWarning, - stacklevel=3, - ) - - @classmethod - def pattern_to_regex(cls, *args, **kw): - """ - Warn about deprecation. - """ - cls._deprecated() - return super(GitIgnorePattern, cls).pattern_to_regex(*args, **kw) - + """ + The :class:`GitIgnorePattern` class is deprecated by :class:`GitWildMatchPattern`. + This class only exists to maintain compatibility with v0.4. + """ + + def __init__(self, *args, **kw): + """ + Warn about deprecation. + """ + self._deprecated() + return super(GitIgnorePattern, self).__init__(*args, **kw) + + @staticmethod + def _deprecated(): + """ + Warn about deprecation. + """ + warnings.warn("GitIgnorePattern ('gitignore') is deprecated. Use GitWildMatchPattern ('gitwildmatch') instead.", DeprecationWarning, stacklevel=3) + + @classmethod + def pattern_to_regex(cls, *args, **kw): + """ + Warn about deprecation. + """ + cls._deprecated() + return super(GitIgnorePattern, cls).pattern_to_regex(*args, **kw) # Register `GitIgnorePattern` as "gitignore" for backward compatibility # with v0.4. -util.register_pattern("gitignore", GitIgnorePattern) +util.register_pattern('gitignore', GitIgnorePattern) diff --git a/python/ray/_private/thirdparty/pathspec/util.py b/python/ray/_private/thirdparty/pathspec/util.py index 5b7e408c64fa7..bcba8783b61c9 100644 --- a/python/ray/_private/thirdparty/pathspec/util.py +++ b/python/ray/_private/thirdparty/pathspec/util.py @@ -10,9 +10,7 @@ from .compat import Collection, Iterable, string_types, unicode -NORMALIZE_PATH_SEPS = [ - sep for sep in [os.sep, os.altsep] if sep and sep != posixpath.sep -] +NORMALIZE_PATH_SEPS = [sep for sep in [os.sep, os.altsep] if sep and sep != posixpath.sep] """ *NORMALIZE_PATH_SEPS* (:class:`list` of :class:`str`) contains the path separators that need to be normalized to the POSIX separator for the @@ -28,123 +26,119 @@ def detailed_match_files(patterns, files, all_matches=None): - """ - Matches the files to the patterns, and returns which patterns matched - the files. - - *patterns* (:class:`~collections.abc.Iterable` of :class:`~pathspec.pattern.Pattern`) - contains the patterns to use. - - *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains - the normalized file paths to be matched against *patterns*. - - *all_matches* (:class:`boot` or :data:`None`) is whether to return all - matches patterns (:data:`True`), or only the last matched pattern - (:data:`False`). Default is :data:`None` for :data:`False`. - - Returns the matched files (:class:`dict`) which maps each matched file - (:class:`str`) to the patterns that matched in order (:class:`.MatchDetail`). - """ - all_files = files if isinstance(files, Collection) else list(files) - return_files = {} - for pattern in patterns: - if pattern.include is not None: - result_files = pattern.match(all_files) - if pattern.include: - # Add files and record pattern. - for result_file in result_files: - if result_file in return_files: - if all_matches: - return_files[result_file].patterns.append(pattern) - else: - return_files[result_file].patterns[0] = pattern - else: - return_files[result_file] = MatchDetail([pattern]) - - else: - # Remove files. - for file in result_files: - del return_files[file] - - return return_files + """ + Matches the files to the patterns, and returns which patterns matched + the files. + + *patterns* (:class:`~collections.abc.Iterable` of :class:`~pathspec.pattern.Pattern`) + contains the patterns to use. + + *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains + the normalized file paths to be matched against *patterns*. + + *all_matches* (:class:`boot` or :data:`None`) is whether to return all + matches patterns (:data:`True`), or only the last matched pattern + (:data:`False`). Default is :data:`None` for :data:`False`. + + Returns the matched files (:class:`dict`) which maps each matched file + (:class:`str`) to the patterns that matched in order (:class:`.MatchDetail`). + """ + all_files = files if isinstance(files, Collection) else list(files) + return_files = {} + for pattern in patterns: + if pattern.include is not None: + result_files = pattern.match(all_files) + if pattern.include: + # Add files and record pattern. + for result_file in result_files: + if result_file in return_files: + if all_matches: + return_files[result_file].patterns.append(pattern) + else: + return_files[result_file].patterns[0] = pattern + else: + return_files[result_file] = MatchDetail([pattern]) + + else: + # Remove files. + for file in result_files: + del return_files[file] + + return return_files def _is_iterable(value): - """ - Check whether the value is an iterable (excludes strings). + """ + Check whether the value is an iterable (excludes strings). - *value* is the value to check, + *value* is the value to check, - Returns whether *value* is a iterable (:class:`bool`). - """ - return isinstance(value, Iterable) and not isinstance(value, (unicode, bytes)) + Returns whether *value* is a iterable (:class:`bool`). + """ + return isinstance(value, Iterable) and not isinstance(value, (unicode, bytes)) def iter_tree_entries(root, on_error=None, follow_links=None): - """ - Walks the specified directory for all files and directories. + """ + Walks the specified directory for all files and directories. - *root* (:class:`str`) is the root directory to search. + *root* (:class:`str`) is the root directory to search. - *on_error* (:class:`~collections.abc.Callable` or :data:`None`) - optionally is the error handler for file-system exceptions. It will be - called with the exception (:exc:`OSError`). Reraise the exception to - abort the walk. Default is :data:`None` to ignore file-system - exceptions. + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) + optionally is the error handler for file-system exceptions. It will be + called with the exception (:exc:`OSError`). Reraise the exception to + abort the walk. Default is :data:`None` to ignore file-system + exceptions. - *follow_links* (:class:`bool` or :data:`None`) optionally is whether - to walk symbolic links that resolve to directories. Default is - :data:`None` for :data:`True`. + *follow_links* (:class:`bool` or :data:`None`) optionally is whether + to walk symbolic links that resolve to directories. Default is + :data:`None` for :data:`True`. - Raises :exc:`RecursionError` if recursion is detected. + Raises :exc:`RecursionError` if recursion is detected. - Returns an :class:`~collections.abc.Iterable` yielding each file or - directory entry (:class:`.TreeEntry`) relative to *root*. - """ - if on_error is not None and not callable(on_error): - raise TypeError("on_error:{!r} is not callable.".format(on_error)) + Returns an :class:`~collections.abc.Iterable` yielding each file or + directory entry (:class:`.TreeEntry`) relative to *root*. + """ + if on_error is not None and not callable(on_error): + raise TypeError("on_error:{!r} is not callable.".format(on_error)) - if follow_links is None: - follow_links = True + if follow_links is None: + follow_links = True - for entry in _iter_tree_entries_next( - os.path.abspath(root), "", {}, on_error, follow_links - ): - yield entry + for entry in _iter_tree_entries_next(os.path.abspath(root), '', {}, on_error, follow_links): + yield entry def iter_tree_files(root, on_error=None, follow_links=None): - """ - Walks the specified directory for all files. + """ + Walks the specified directory for all files. - *root* (:class:`str`) is the root directory to search for files. + *root* (:class:`str`) is the root directory to search for files. - *on_error* (:class:`~collections.abc.Callable` or :data:`None`) - optionally is the error handler for file-system exceptions. It will be - called with the exception (:exc:`OSError`). Reraise the exception to - abort the walk. Default is :data:`None` to ignore file-system - exceptions. + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) + optionally is the error handler for file-system exceptions. It will be + called with the exception (:exc:`OSError`). Reraise the exception to + abort the walk. Default is :data:`None` to ignore file-system + exceptions. - *follow_links* (:class:`bool` or :data:`None`) optionally is whether - to walk symbolic links that resolve to directories. Default is - :data:`None` for :data:`True`. + *follow_links* (:class:`bool` or :data:`None`) optionally is whether + to walk symbolic links that resolve to directories. Default is + :data:`None` for :data:`True`. - Raises :exc:`RecursionError` if recursion is detected. + Raises :exc:`RecursionError` if recursion is detected. - Returns an :class:`~collections.abc.Iterable` yielding the path to - each file (:class:`str`) relative to *root*. - """ - if on_error is not None and not callable(on_error): - raise TypeError("on_error:{!r} is not callable.".format(on_error)) + Returns an :class:`~collections.abc.Iterable` yielding the path to + each file (:class:`str`) relative to *root*. + """ + if on_error is not None and not callable(on_error): + raise TypeError("on_error:{!r} is not callable.".format(on_error)) - if follow_links is None: - follow_links = True + if follow_links is None: + follow_links = True - for entry in _iter_tree_entries_next( - os.path.abspath(root), "", {}, on_error, follow_links - ): - if not entry.is_dir(follow_links): - yield entry.path + for entry in _iter_tree_entries_next(os.path.abspath(root), '', {}, on_error, follow_links): + if not entry.is_dir(follow_links): + yield entry.path # Alias `iter_tree_files()` as `iter_tree()`. @@ -152,358 +146,354 @@ def iter_tree_files(root, on_error=None, follow_links=None): def _iter_tree_entries_next(root_full, dir_rel, memo, on_error, follow_links): - """ - Scan the directory for all descendant files. - - *root_full* (:class:`str`) the absolute path to the root directory. - - *dir_rel* (:class:`str`) the path to the directory to scan relative to - *root_full*. - - *memo* (:class:`dict`) keeps track of ancestor directories - encountered. Maps each ancestor real path (:class:`str`) to relative - path (:class:`str`). - - *on_error* (:class:`~collections.abc.Callable` or :data:`None`) - optionally is the error handler for file-system exceptions. - - *follow_links* (:class:`bool`) is whether to walk symbolic links that - resolve to directories. - - Yields each entry (:class:`.TreeEntry`). - """ - dir_full = os.path.join(root_full, dir_rel) - dir_real = os.path.realpath(dir_full) - - # Remember each encountered ancestor directory and its canonical - # (real) path. If a canonical path is encountered more than once, - # recursion has occurred. - if dir_real not in memo: - memo[dir_real] = dir_rel - else: - raise RecursionError( - real_path=dir_real, first_path=memo[dir_real], second_path=dir_rel - ) - - for node_name in os.listdir(dir_full): - node_rel = os.path.join(dir_rel, node_name) - node_full = os.path.join(root_full, node_rel) - - # Inspect child node. - try: - node_lstat = os.lstat(node_full) - except OSError as e: - if on_error is not None: - on_error(e) - continue - - if stat.S_ISLNK(node_lstat.st_mode): - # Child node is a link, inspect the target node. - is_link = True - try: - node_stat = os.stat(node_full) - except OSError as e: - if on_error is not None: - on_error(e) - continue - else: - is_link = False - node_stat = node_lstat - - if stat.S_ISDIR(node_stat.st_mode) and (follow_links or not is_link): - # Child node is a directory, recurse into it and yield its - # descendant files. - yield TreeEntry(node_name, node_rel, node_lstat, node_stat) - - for entry in _iter_tree_entries_next( - root_full, node_rel, memo, on_error, follow_links - ): - yield entry - - elif stat.S_ISREG(node_stat.st_mode) or is_link: - # Child node is either a file or an unfollowed link, yield it. - yield TreeEntry(node_name, node_rel, node_lstat, node_stat) - - # NOTE: Make sure to remove the canonical (real) path of the directory - # from the ancestors memo once we are done with it. This allows the - # same directory to appear multiple times. If this is not done, the - # second occurrence of the directory will be incorrectly interpreted - # as a recursion. See . - del memo[dir_real] + """ + Scan the directory for all descendant files. + + *root_full* (:class:`str`) the absolute path to the root directory. + + *dir_rel* (:class:`str`) the path to the directory to scan relative to + *root_full*. + + *memo* (:class:`dict`) keeps track of ancestor directories + encountered. Maps each ancestor real path (:class:`str`) to relative + path (:class:`str`). + + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) + optionally is the error handler for file-system exceptions. + + *follow_links* (:class:`bool`) is whether to walk symbolic links that + resolve to directories. + + Yields each entry (:class:`.TreeEntry`). + """ + dir_full = os.path.join(root_full, dir_rel) + dir_real = os.path.realpath(dir_full) + + # Remember each encountered ancestor directory and its canonical + # (real) path. If a canonical path is encountered more than once, + # recursion has occurred. + if dir_real not in memo: + memo[dir_real] = dir_rel + else: + raise RecursionError(real_path=dir_real, first_path=memo[dir_real], second_path=dir_rel) + + for node_name in os.listdir(dir_full): + node_rel = os.path.join(dir_rel, node_name) + node_full = os.path.join(root_full, node_rel) + + # Inspect child node. + try: + node_lstat = os.lstat(node_full) + except OSError as e: + if on_error is not None: + on_error(e) + continue + + if stat.S_ISLNK(node_lstat.st_mode): + # Child node is a link, inspect the target node. + is_link = True + try: + node_stat = os.stat(node_full) + except OSError as e: + if on_error is not None: + on_error(e) + continue + else: + is_link = False + node_stat = node_lstat + + if stat.S_ISDIR(node_stat.st_mode) and (follow_links or not is_link): + # Child node is a directory, recurse into it and yield its + # descendant files. + yield TreeEntry(node_name, node_rel, node_lstat, node_stat) + + for entry in _iter_tree_entries_next(root_full, node_rel, memo, on_error, follow_links): + yield entry + + elif stat.S_ISREG(node_stat.st_mode) or is_link: + # Child node is either a file or an unfollowed link, yield it. + yield TreeEntry(node_name, node_rel, node_lstat, node_stat) + + # NOTE: Make sure to remove the canonical (real) path of the directory + # from the ancestors memo once we are done with it. This allows the + # same directory to appear multiple times. If this is not done, the + # second occurrence of the directory will be incorrectly interpreted + # as a recursion. See . + del memo[dir_real] def lookup_pattern(name): - """ - Lookups a registered pattern factory by name. + """ + Lookups a registered pattern factory by name. - *name* (:class:`str`) is the name of the pattern factory. + *name* (:class:`str`) is the name of the pattern factory. - Returns the registered pattern factory (:class:`~collections.abc.Callable`). - If no pattern factory is registered, raises :exc:`KeyError`. - """ - return _registered_patterns[name] + Returns the registered pattern factory (:class:`~collections.abc.Callable`). + If no pattern factory is registered, raises :exc:`KeyError`. + """ + return _registered_patterns[name] def match_file(patterns, file): - """ - Matches the file to the patterns. + """ + Matches the file to the patterns. - *patterns* (:class:`~collections.abc.Iterable` of :class:`~pathspec.pattern.Pattern`) - contains the patterns to use. + *patterns* (:class:`~collections.abc.Iterable` of :class:`~pathspec.pattern.Pattern`) + contains the patterns to use. - *file* (:class:`str`) is the normalized file path to be matched - against *patterns*. + *file* (:class:`str`) is the normalized file path to be matched + against *patterns*. - Returns :data:`True` if *file* matched; otherwise, :data:`False`. - """ - matched = False - for pattern in patterns: - if pattern.include is not None: - if file in pattern.match((file,)): - matched = pattern.include - return matched + Returns :data:`True` if *file* matched; otherwise, :data:`False`. + """ + matched = False + for pattern in patterns: + if pattern.include is not None: + if file in pattern.match((file,)): + matched = pattern.include + return matched def match_files(patterns, files): - """ - Matches the files to the patterns. + """ + Matches the files to the patterns. - *patterns* (:class:`~collections.abc.Iterable` of :class:`~pathspec.pattern.Pattern`) - contains the patterns to use. + *patterns* (:class:`~collections.abc.Iterable` of :class:`~pathspec.pattern.Pattern`) + contains the patterns to use. - *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains - the normalized file paths to be matched against *patterns*. + *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains + the normalized file paths to be matched against *patterns*. - Returns the matched files (:class:`set` of :class:`str`). - """ - all_files = files if isinstance(files, Collection) else list(files) - return_files = set() - for pattern in patterns: - if pattern.include is not None: - result_files = pattern.match(all_files) - if pattern.include: - return_files.update(result_files) - else: - return_files.difference_update(result_files) - return return_files + Returns the matched files (:class:`set` of :class:`str`). + """ + all_files = files if isinstance(files, Collection) else list(files) + return_files = set() + for pattern in patterns: + if pattern.include is not None: + result_files = pattern.match(all_files) + if pattern.include: + return_files.update(result_files) + else: + return_files.difference_update(result_files) + return return_files def _normalize_entries(entries, separators=None): - """ - Normalizes the entry paths to use the POSIX path separator. + """ + Normalizes the entry paths to use the POSIX path separator. - *entries* (:class:`~collections.abc.Iterable` of :class:`.TreeEntry`) - contains the entries to be normalized. + *entries* (:class:`~collections.abc.Iterable` of :class:`.TreeEntry`) + contains the entries to be normalized. - *separators* (:class:`~collections.abc.Collection` of :class:`str`; or - :data:`None`) optionally contains the path separators to normalize. - See :func:`normalize_file` for more information. + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. + See :func:`normalize_file` for more information. - Returns a :class:`dict` mapping the each normalized file path (:class:`str`) - to the entry (:class:`.TreeEntry`) - """ - norm_files = {} - for entry in entries: - norm_files[normalize_file(entry.path, separators=separators)] = entry - return norm_files + Returns a :class:`dict` mapping the each normalized file path (:class:`str`) + to the entry (:class:`.TreeEntry`) + """ + norm_files = {} + for entry in entries: + norm_files[normalize_file(entry.path, separators=separators)] = entry + return norm_files def normalize_file(file, separators=None): - """ - Normalizes the file path to use the POSIX path separator (i.e., ``'/'``). + """ + Normalizes the file path to use the POSIX path separator (i.e., ``'/'``). - *file* (:class:`str` or :class:`pathlib.PurePath`) is the file path. + *file* (:class:`str` or :class:`pathlib.PurePath`) is the file path. - *separators* (:class:`~collections.abc.Collection` of :class:`str`; or - :data:`None`) optionally contains the path separators to normalize. - This does not need to include the POSIX path separator (``'/'``), but - including it will not affect the results. Default is :data:`None` for - :data:`NORMALIZE_PATH_SEPS`. To prevent normalization, pass an empty - container (e.g., an empty tuple ``()``). + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. + This does not need to include the POSIX path separator (``'/'``), but + including it will not affect the results. Default is :data:`None` for + :data:`NORMALIZE_PATH_SEPS`. To prevent normalization, pass an empty + container (e.g., an empty tuple ``()``). - Returns the normalized file path (:class:`str`). - """ - # Normalize path separators. - if separators is None: - separators = NORMALIZE_PATH_SEPS + Returns the normalized file path (:class:`str`). + """ + # Normalize path separators. + if separators is None: + separators = NORMALIZE_PATH_SEPS - # Convert path object to string. - norm_file = str(file) + # Convert path object to string. + norm_file = str(file) - for sep in separators: - norm_file = norm_file.replace(sep, posixpath.sep) + for sep in separators: + norm_file = norm_file.replace(sep, posixpath.sep) - # Remove current directory prefix. - if norm_file.startswith("./"): - norm_file = norm_file[2:] + # Remove current directory prefix. + if norm_file.startswith('./'): + norm_file = norm_file[2:] - return norm_file + return norm_file def normalize_files(files, separators=None): - """ - Normalizes the file paths to use the POSIX path separator. + """ + Normalizes the file paths to use the POSIX path separator. - *files* (:class:`~collections.abc.Iterable` of :class:`str` or - :class:`pathlib.PurePath`) contains the file paths to be normalized. + *files* (:class:`~collections.abc.Iterable` of :class:`str` or + :class:`pathlib.PurePath`) contains the file paths to be normalized. - *separators* (:class:`~collections.abc.Collection` of :class:`str`; or - :data:`None`) optionally contains the path separators to normalize. - See :func:`normalize_file` for more information. + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. + See :func:`normalize_file` for more information. - Returns a :class:`dict` mapping the each normalized file path (:class:`str`) - to the original file path (:class:`str`) - """ - norm_files = {} - for path in files: - norm_files[normalize_file(path, separators=separators)] = path - return norm_files + Returns a :class:`dict` mapping the each normalized file path (:class:`str`) + to the original file path (:class:`str`) + """ + norm_files = {} + for path in files: + norm_files[normalize_file(path, separators=separators)] = path + return norm_files def register_pattern(name, pattern_factory, override=None): - """ - Registers the specified pattern factory. - - *name* (:class:`str`) is the name to register the pattern factory - under. - - *pattern_factory* (:class:`~collections.abc.Callable`) is used to - compile patterns. It must accept an uncompiled pattern (:class:`str`) - and return the compiled pattern (:class:`.Pattern`). - - *override* (:class:`bool` or :data:`None`) optionally is whether to - allow overriding an already registered pattern under the same name - (:data:`True`), instead of raising an :exc:`AlreadyRegisteredError` - (:data:`False`). Default is :data:`None` for :data:`False`. - """ - if not isinstance(name, string_types): - raise TypeError("name:{!r} is not a string.".format(name)) - if not callable(pattern_factory): - raise TypeError("pattern_factory:{!r} is not callable.".format(pattern_factory)) - if name in _registered_patterns and not override: - raise AlreadyRegisteredError(name, _registered_patterns[name]) - _registered_patterns[name] = pattern_factory + """ + Registers the specified pattern factory. + + *name* (:class:`str`) is the name to register the pattern factory + under. + + *pattern_factory* (:class:`~collections.abc.Callable`) is used to + compile patterns. It must accept an uncompiled pattern (:class:`str`) + and return the compiled pattern (:class:`.Pattern`). + + *override* (:class:`bool` or :data:`None`) optionally is whether to + allow overriding an already registered pattern under the same name + (:data:`True`), instead of raising an :exc:`AlreadyRegisteredError` + (:data:`False`). Default is :data:`None` for :data:`False`. + """ + if not isinstance(name, string_types): + raise TypeError("name:{!r} is not a string.".format(name)) + if not callable(pattern_factory): + raise TypeError("pattern_factory:{!r} is not callable.".format(pattern_factory)) + if name in _registered_patterns and not override: + raise AlreadyRegisteredError(name, _registered_patterns[name]) + _registered_patterns[name] = pattern_factory class AlreadyRegisteredError(Exception): - """ - The :exc:`AlreadyRegisteredError` exception is raised when a pattern - factory is registered under a name already in use. - """ - - def __init__(self, name, pattern_factory): - """ - Initializes the :exc:`AlreadyRegisteredError` instance. - - *name* (:class:`str`) is the name of the registered pattern. - - *pattern_factory* (:class:`~collections.abc.Callable`) is the - registered pattern factory. - """ - super(AlreadyRegisteredError, self).__init__(name, pattern_factory) - - @property - def message(self): - """ - *message* (:class:`str`) is the error message. - """ - return "{name!r} is already registered for pattern factory:{pattern_factory!r}.".format( - name=self.name, - pattern_factory=self.pattern_factory, - ) - - @property - def name(self): - """ - *name* (:class:`str`) is the name of the registered pattern. - """ - return self.args[0] - - @property - def pattern_factory(self): - """ - *pattern_factory* (:class:`~collections.abc.Callable`) is the - registered pattern factory. - """ - return self.args[1] + """ + The :exc:`AlreadyRegisteredError` exception is raised when a pattern + factory is registered under a name already in use. + """ + + def __init__(self, name, pattern_factory): + """ + Initializes the :exc:`AlreadyRegisteredError` instance. + + *name* (:class:`str`) is the name of the registered pattern. + + *pattern_factory* (:class:`~collections.abc.Callable`) is the + registered pattern factory. + """ + super(AlreadyRegisteredError, self).__init__(name, pattern_factory) + + @property + def message(self): + """ + *message* (:class:`str`) is the error message. + """ + return "{name!r} is already registered for pattern factory:{pattern_factory!r}.".format( + name=self.name, + pattern_factory=self.pattern_factory, + ) + + @property + def name(self): + """ + *name* (:class:`str`) is the name of the registered pattern. + """ + return self.args[0] + + @property + def pattern_factory(self): + """ + *pattern_factory* (:class:`~collections.abc.Callable`) is the + registered pattern factory. + """ + return self.args[1] class RecursionError(Exception): - """ - The :exc:`RecursionError` exception is raised when recursion is - detected. - """ - - def __init__(self, real_path, first_path, second_path): - """ - Initializes the :exc:`RecursionError` instance. - - *real_path* (:class:`str`) is the real path that recursion was - encountered on. - - *first_path* (:class:`str`) is the first path encountered for - *real_path*. - - *second_path* (:class:`str`) is the second path encountered for - *real_path*. - """ - super(RecursionError, self).__init__(real_path, first_path, second_path) - - @property - def first_path(self): - """ - *first_path* (:class:`str`) is the first path encountered for - :attr:`self.real_path `. - """ - return self.args[1] - - @property - def message(self): - """ - *message* (:class:`str`) is the error message. - """ - return "Real path {real!r} was encountered at {first!r} and then {second!r}.".format( - real=self.real_path, - first=self.first_path, - second=self.second_path, - ) - - @property - def real_path(self): - """ - *real_path* (:class:`str`) is the real path that recursion was - encountered on. - """ - return self.args[0] - - @property - def second_path(self): - """ - *second_path* (:class:`str`) is the second path encountered for - :attr:`self.real_path `. - """ - return self.args[2] + """ + The :exc:`RecursionError` exception is raised when recursion is + detected. + """ + + def __init__(self, real_path, first_path, second_path): + """ + Initializes the :exc:`RecursionError` instance. + + *real_path* (:class:`str`) is the real path that recursion was + encountered on. + + *first_path* (:class:`str`) is the first path encountered for + *real_path*. + + *second_path* (:class:`str`) is the second path encountered for + *real_path*. + """ + super(RecursionError, self).__init__(real_path, first_path, second_path) + + @property + def first_path(self): + """ + *first_path* (:class:`str`) is the first path encountered for + :attr:`self.real_path `. + """ + return self.args[1] + + @property + def message(self): + """ + *message* (:class:`str`) is the error message. + """ + return "Real path {real!r} was encountered at {first!r} and then {second!r}.".format( + real=self.real_path, + first=self.first_path, + second=self.second_path, + ) + + @property + def real_path(self): + """ + *real_path* (:class:`str`) is the real path that recursion was + encountered on. + """ + return self.args[0] + + @property + def second_path(self): + """ + *second_path* (:class:`str`) is the second path encountered for + :attr:`self.real_path `. + """ + return self.args[2] class MatchDetail(object): - """ - The :class:`.MatchDetail` class contains information about - """ + """ + The :class:`.MatchDetail` class contains information about + """ - #: Make the class dict-less. - __slots__ = ("patterns",) + #: Make the class dict-less. + __slots__ = ('patterns',) - def __init__(self, patterns): - """ - Initialize the :class:`.MatchDetail` instance. + def __init__(self, patterns): + """ + Initialize the :class:`.MatchDetail` instance. - *patterns* (:class:`~collections.abc.Sequence` of :class:`~pathspec.pattern.Pattern`) - contains the patterns that matched the file in the order they were - encountered. - """ + *patterns* (:class:`~collections.abc.Sequence` of :class:`~pathspec.pattern.Pattern`) + contains the patterns that matched the file in the order they were + encountered. + """ - self.patterns = patterns - """ + self.patterns = patterns + """ *patterns* (:class:`~collections.abc.Sequence` of :class:`~pathspec.pattern.Pattern`) contains the patterns that matched the file in the order they were encountered. @@ -511,100 +501,100 @@ def __init__(self, patterns): class TreeEntry(object): - """ - The :class:`.TreeEntry` class contains information about a file-system - entry. - """ + """ + The :class:`.TreeEntry` class contains information about a file-system + entry. + """ - #: Make the class dict-less. - __slots__ = ("_lstat", "name", "path", "_stat") + #: Make the class dict-less. + __slots__ = ('_lstat', 'name', 'path', '_stat') - def __init__(self, name, path, lstat, stat): - """ - Initialize the :class:`.TreeEntry` instance. + def __init__(self, name, path, lstat, stat): + """ + Initialize the :class:`.TreeEntry` instance. - *name* (:class:`str`) is the base name of the entry. + *name* (:class:`str`) is the base name of the entry. - *path* (:class:`str`) is the relative path of the entry. + *path* (:class:`str`) is the relative path of the entry. - *lstat* (:class:`~os.stat_result`) is the stat result of the direct - entry. + *lstat* (:class:`~os.stat_result`) is the stat result of the direct + entry. - *stat* (:class:`~os.stat_result`) is the stat result of the entry, - potentially linked. - """ + *stat* (:class:`~os.stat_result`) is the stat result of the entry, + potentially linked. + """ - self._lstat = lstat - """ + self._lstat = lstat + """ *_lstat* (:class:`~os.stat_result`) is the stat result of the direct entry. """ - self.name = name - """ + self.name = name + """ *name* (:class:`str`) is the base name of the entry. """ - self.path = path - """ + self.path = path + """ *path* (:class:`str`) is the path of the entry. """ - self._stat = stat - """ + self._stat = stat + """ *_stat* (:class:`~os.stat_result`) is the stat result of the linked entry. """ - def is_dir(self, follow_links=None): - """ - Get whether the entry is a directory. + def is_dir(self, follow_links=None): + """ + Get whether the entry is a directory. - *follow_links* (:class:`bool` or :data:`None`) is whether to follow - symbolic links. If this is :data:`True`, a symlink to a directory - will result in :data:`True`. Default is :data:`None` for :data:`True`. + *follow_links* (:class:`bool` or :data:`None`) is whether to follow + symbolic links. If this is :data:`True`, a symlink to a directory + will result in :data:`True`. Default is :data:`None` for :data:`True`. - Returns whether the entry is a directory (:class:`bool`). - """ - if follow_links is None: - follow_links = True + Returns whether the entry is a directory (:class:`bool`). + """ + if follow_links is None: + follow_links = True - node_stat = self._stat if follow_links else self._lstat - return stat.S_ISDIR(node_stat.st_mode) + node_stat = self._stat if follow_links else self._lstat + return stat.S_ISDIR(node_stat.st_mode) - def is_file(self, follow_links=None): - """ - Get whether the entry is a regular file. + def is_file(self, follow_links=None): + """ + Get whether the entry is a regular file. - *follow_links* (:class:`bool` or :data:`None`) is whether to follow - symbolic links. If this is :data:`True`, a symlink to a regular file - will result in :data:`True`. Default is :data:`None` for :data:`True`. + *follow_links* (:class:`bool` or :data:`None`) is whether to follow + symbolic links. If this is :data:`True`, a symlink to a regular file + will result in :data:`True`. Default is :data:`None` for :data:`True`. - Returns whether the entry is a regular file (:class:`bool`). - """ - if follow_links is None: - follow_links = True + Returns whether the entry is a regular file (:class:`bool`). + """ + if follow_links is None: + follow_links = True - node_stat = self._stat if follow_links else self._lstat - return stat.S_ISREG(node_stat.st_mode) + node_stat = self._stat if follow_links else self._lstat + return stat.S_ISREG(node_stat.st_mode) - def is_symlink(self): - """ - Returns whether the entry is a symbolic link (:class:`bool`). - """ - return stat.S_ISLNK(self._lstat.st_mode) + def is_symlink(self): + """ + Returns whether the entry is a symbolic link (:class:`bool`). + """ + return stat.S_ISLNK(self._lstat.st_mode) - def stat(self, follow_links=None): - """ - Get the cached stat result for the entry. + def stat(self, follow_links=None): + """ + Get the cached stat result for the entry. - *follow_links* (:class:`bool` or :data:`None`) is whether to follow - symbolic links. If this is :data:`True`, the stat result of the - linked file will be returned. Default is :data:`None` for :data:`True`. + *follow_links* (:class:`bool` or :data:`None`) is whether to follow + symbolic links. If this is :data:`True`, the stat result of the + linked file will be returned. Default is :data:`None` for :data:`True`. - Returns that stat result (:class:`~os.stat_result`). - """ - if follow_links is None: - follow_links = True + Returns that stat result (:class:`~os.stat_result`). + """ + if follow_links is None: + follow_links = True - return self._stat if follow_links else self._lstat + return self._stat if follow_links else self._lstat